diff --git a/gcc/testsuite/gcc.dg/vect/pr106081.c b/gcc/testsuite/gcc.dg/vect/pr106081.c
new file mode 100644
index 0000000000000000000000000000000000000000..8f97af2d642b78773524073856023ac53cad2b7c
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr106081.c
@@ -0,0 +1,33 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-ffast-math -fdump-tree-optimized" } */
+/* { dg-additional-options "-mavx2" { target x86_64-*-* i?86-*-* } } */
+/* { dg-require-effective-target vect_double } */
+/* { dg-require-effective-target vect_unpack } */
+/* { dg-require-effective-target vect_intdouble_cvt } */
+/* { dg-require-effective-target vect_perm } */
+
+struct pixels
+{
+  short a,b,c,d;
+} *pixels;
+struct dpixels
+{
+  double a,b,c,d;
+};
+
+double
+test(double *k)
+{
+  struct dpixels results={};
+  for (int u=0; u<1000*16;u++,k--)
+    {
+      results.a += *k*pixels[u].a;
+      results.b += *k*pixels[u].b;
+      results.c += *k*pixels[u].c;
+      results.d += *k*pixels[u].d;
+    }
+  return results.a+results.b*2+results.c*3+results.d*4;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" } } */
+/* { dg-final { scan-tree-dump-times "VEC_PERM" 4 "optimized" { target x86_64-*-* i?86-*-* } } } */
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index e4430248ab51b926c65078ce8b4dbde5041a58a6..a1b153035e100867814d6c9ef1fbfce6ccdcab2c 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -4605,7 +4605,10 @@ vect_optimize_slp_pass::start_choosing_layouts ()
 	     IFN_MASK_LOADs).  */
 	  gcc_assert (partition.layout == 0 && !m_slpg->vertices[node_i].succ);
 	  if (!STMT_VINFO_GROUPED_ACCESS (dr_stmt))
-	    continue;
+	    {
+	      partition.layout = -1;
+	      continue;
+	    }
 	  dr_stmt = DR_GROUP_FIRST_ELEMENT (dr_stmt);
 	  imin = DR_GROUP_SIZE (dr_stmt) + 1;
 	  tmp_perm.safe_splice (SLP_TREE_LOAD_PERMUTATION (node));