diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc index 8877ebde2466cbddd4446852f83d8a5765c70fd2..4a8b0a18800e16321b1c3d69294ce17fd827aaf7 100644 --- a/gcc/tree-vect-loop.cc +++ b/gcc/tree-vect-loop.cc @@ -11361,8 +11361,12 @@ update_epilogue_loop_vinfo (class loop *epilogue, tree advance) /* Data references for gather loads and scatter stores do not use the updated offset we set using ADVANCE. Instead we have to make sure the reference in the data references point to the corresponding copy of - the original in the epilogue. */ - if (STMT_VINFO_GATHER_SCATTER_P (vect_stmt_to_vectorize (stmt_vinfo))) + the original in the epilogue. Make sure to update both + gather/scatters recognized by dataref analysis and also other + refs that get_load_store_type classified as VMAT_GATHER_SCATTER. */ + auto vstmt_vinfo = vect_stmt_to_vectorize (stmt_vinfo); + if (STMT_VINFO_MEMORY_ACCESS_TYPE (vstmt_vinfo) == VMAT_GATHER_SCATTER + || STMT_VINFO_GATHER_SCATTER_P (vstmt_vinfo)) { DR_REF (dr) = simplify_replace_tree (DR_REF (dr), NULL_TREE, NULL_TREE, @@ -11371,9 +11375,6 @@ update_epilogue_loop_vinfo (class loop *epilogue, tree advance) = simplify_replace_tree (DR_BASE_ADDRESS (dr), NULL_TREE, NULL_TREE, &find_in_mapping, &mapping); } - else - gcc_assert (STMT_VINFO_MEMORY_ACCESS_TYPE (vect_stmt_to_vectorize (stmt_vinfo)) - != VMAT_GATHER_SCATTER); DR_STMT (dr) = STMT_VINFO_STMT (stmt_vinfo); stmt_vinfo->dr_aux.stmt = stmt_vinfo; /* The vector size of the epilogue is smaller than that of the main loop