diff --git a/gcc/cfgloop.h b/gcc/cfgloop.h
index 22293e1c23741d0af9f9ce53e154410f3e0932c3..c4622d4b8538a822eb5bbbc4b1befcae58f73400 100644
--- a/gcc/cfgloop.h
+++ b/gcc/cfgloop.h
@@ -921,6 +921,7 @@ extern bool get_estimated_loop_iterations (class loop *loop, widest_int *nit);
extern bool get_max_loop_iterations (const class loop *loop, widest_int *nit);
extern bool get_likely_max_loop_iterations (class loop *loop, widest_int *nit);
extern int bb_loop_depth (const_basic_block);
+extern edge single_dom_exit (class loop *);
/* Converts VAL to widest_int. */
diff --git a/gcc/cfgloopmanip.h b/gcc/cfgloopmanip.h
index af6a29f70c42e42747364ce385255aefd15dcc63..dab7b31c1e77e91e7f1138101c759900768870d5 100644
--- a/gcc/cfgloopmanip.h
+++ b/gcc/cfgloopmanip.h
@@ -68,5 +68,6 @@ class loop * loop_version (class loop *, void *,
void adjust_loop_info_after_peeling (class loop *loop, int npeel, bool precise);
void scale_dominated_blocks_in_loop (class loop *loop, basic_block bb,
profile_count num, profile_count den);
+void update_exit_probability_after_unrolling (class loop *loop, edge new_exit);
#endif /* GCC_CFGLOOPMANIP_H */
diff --git a/gcc/cfgrtl.cc b/gcc/cfgrtl.cc
index 36e43d0d7373cdd2c2227d8a8ba4f493ac1b5d2a..abcb472e2a2d750befbd0be9a44fe39f21d25293 100644
--- a/gcc/cfgrtl.cc
+++ b/gcc/cfgrtl.cc
@@ -5409,7 +5409,7 @@ struct cfg_hooks cfg_layout_rtl_cfg_hooks = {
rtl_flow_call_edges_add,
NULL, /* execute_on_growing_pred */
NULL, /* execute_on_shrinking_pred */
- duplicate_loop_body_to_header_edge, /* duplicate loop for trees */
+ duplicate_loop_body_to_header_edge, /* duplicate loop for rtl */
rtl_lv_add_condition_to_bb, /* lv_add_condition_to_bb */
NULL, /* lv_adjust_loop_header_phi*/
rtl_extract_cond_bb_edges, /* extract_cond_bb_edges */
diff --git a/gcc/loop-unroll.cc b/gcc/loop-unroll.cc
index 93333d8ba11658761e91838296d13fbf069d9134..bbfa6ccc77004d37b0eb9624dc33f0024c5306a2 100644
--- a/gcc/loop-unroll.cc
+++ b/gcc/loop-unroll.cc
@@ -487,6 +487,7 @@ unroll_loop_constant_iterations (class loop *loop)
bool exit_at_end = loop_exit_at_end_p (loop);
struct opt_info *opt_info = NULL;
bool ok;
+ bool flat = maybe_flat_loop_profile (loop);
niter = desc->niter;
@@ -603,9 +604,14 @@ unroll_loop_constant_iterations (class loop *loop)
ok = duplicate_loop_body_to_header_edge (
loop, loop_latch_edge (loop), max_unroll, wont_exit, desc->out_edge,
&remove_edges,
- DLTHE_FLAG_UPDATE_FREQ | (opt_info ? DLTHE_RECORD_COPY_NUMBER : 0));
+ DLTHE_FLAG_UPDATE_FREQ | (opt_info ? DLTHE_RECORD_COPY_NUMBER : 0)
+ | (flat ? DLTHE_FLAG_FLAT_PROFILE : 0));
gcc_assert (ok);
+ edge new_exit = single_dom_exit (loop);
+ if (new_exit)
+ update_exit_probability_after_unrolling (loop, new_exit);
+
if (opt_info)
{
apply_opt_in_copies (opt_info, max_unroll, true, true);
diff --git a/gcc/testsuite/gcc.dg/tree-prof/peel-1.c b/gcc/testsuite/gcc.dg/tree-prof/peel-1.c
index 7245b68c1ee90468cc520653b528d68327944ee0..32ecccb16da2d2152d0da7e0c94bcf30c44e75d4 100644
--- a/gcc/testsuite/gcc.dg/tree-prof/peel-1.c
+++ b/gcc/testsuite/gcc.dg/tree-prof/peel-1.c
@@ -1,4 +1,4 @@
-/* { dg-options "-O3 -fdump-tree-cunroll-details -fno-unroll-loops -fpeel-loops" } */
+/* { dg-options "-O3 -fdump-tree-cunroll-details-blocks -fdump-tree-optimized-details-blocks -fno-unroll-loops -fpeel-loops" } */
void abort();
int a[1000];
@@ -21,3 +21,5 @@ main()
return 0;
}
/* { dg-final-use { scan-tree-dump "Peeled loop ., 1 times" "cunroll" } } */
+/* { dg-final-use-not-autofdo { scan-tree-dump-not "Invalid sum" "cunroll" } } */
+/* { dg-final-use-not-autofdo { scan-tree-dump-not "Invalid sum" "optimized" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-prof/unroll-1.c b/gcc/testsuite/gcc.dg/tree-prof/unroll-1.c
index 3ad0cf019b3458c445151ff5f1db1f76aa0a3760..0b25c1f2f1cfb11642125998cf431a7786b04155 100644
--- a/gcc/testsuite/gcc.dg/tree-prof/unroll-1.c
+++ b/gcc/testsuite/gcc.dg/tree-prof/unroll-1.c
@@ -1,4 +1,4 @@
-/* { dg-options "-O3 -fdump-rtl-loop2_unroll-details -funroll-loops -fno-peel-loops" } */
+/* { dg-options "-O3 -fdump-rtl-loop2_unroll-details-blocks -funroll-loops -fno-peel-loops" } */
void abort ();
int a[1000];
@@ -20,4 +20,5 @@ main()
t();
return 0;
}
-/* { dg-final-use { scan-rtl-dump "considering unrolling loop with constant number of iterations" "loop2_unroll" } } */
+/* { dg-final-use-not-autofdo { scan-rtl-dump "considering unrolling loop with constant number of iterations" "loop2_unroll" } } */
+/* { dg-final-use-not-autofdo { scan-rtl-dump-not "Invalid sum" "loop2_unroll" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/peel1.c b/gcc/testsuite/gcc.dg/tree-ssa/peel1.c
index dc5848cb5c5142b5d3eb5273e2a04b3a5dcbe27a..bc136605e94c321f2372d57a35ad25fc0a7e4c19 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/peel1.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/peel1.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O3 -fno-tree-vectorize -fdump-tree-cunroll-details" } */
+/* { dg-options "-O3 -fno-tree-vectorize -fdump-tree-cunroll-details-blocks" } */
struct foo {int b; int a[3];} foo;
void add(struct foo *a,int l)
{
@@ -9,3 +9,4 @@ void add(struct foo *a,int l)
}
/* { dg-final { scan-tree-dump "Loop 1 likely iterates at most 2 times." "cunroll"} } */
/* { dg-final { scan-tree-dump "Peeled loop 1, 3 times." "cunroll"} } */
+/* { dg-final { scan-tree-dump-not "Invalid sum" "cunroll" } } */
diff --git a/gcc/testsuite/gcc.dg/unroll-1.c b/gcc/testsuite/gcc.dg/unroll-1.c
index e7032891823f9e55c36bb82eca52da37b5bfa2f5..ff2cbb07b222285056c5730c0910b798e8c22387 100644
--- a/gcc/testsuite/gcc.dg/unroll-1.c
+++ b/gcc/testsuite/gcc.dg/unroll-1.c
@@ -1,7 +1,7 @@
/* PR optimization/8599 */
/* { dg-do run } */
/* { dg-options "-O2 -funroll-loops" } */
-/* { dg-options "-mtune=k6 -O2 -funroll-loops" { target { { i?86-*-* x86_64-*-* } && ia32 } } } */
+/* { dg-options "-mtune=k6 -O2 -funroll-loops -fdump-rtl-loop2_unroll-details-blocks" { target { { i?86-*-* x86_64-*-* } && ia32 } } } */
extern void abort (void);
@@ -25,3 +25,5 @@ int main()
abort ();
return 0;
}
+/* { dg-final { scan-rtl-dump-not "Invalid sum" "loop2_unroll" } } */
+/* { dg-final { scan-rtl-dump-not "Invalid sum" "loop2_unroll" } } */
diff --git a/gcc/testsuite/gcc.dg/unroll-3.c b/gcc/testsuite/gcc.dg/unroll-3.c
index 10bf59b9a2e72ea879d1a8fb3eee02f2d85f6811..fbc8378c73d99cd741e0ee8fd4941557e6fc7e06 100644
--- a/gcc/testsuite/gcc.dg/unroll-3.c
+++ b/gcc/testsuite/gcc.dg/unroll-3.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-cunrolli-details -fno-peel-loops -fno-tree-vrp -fdisable-tree-cunrolli=foo -fenable-tree-cunrolli=foo" } */
+/* { dg-options "-O2 -fdump-tree-cunrolli-details-blocks -fno-peel-loops -fno-tree-vrp -fdisable-tree-cunrolli=foo -fenable-tree-cunrolli=foo" } */
unsigned a[100], b[100];
inline void bar()
@@ -29,3 +29,4 @@ int foo2(void)
}
/* { dg-final { scan-tree-dump-times "loop with 2 iterations completely unrolled" 1 "cunrolli" } } */
+/* { dg-final { scan-tree-dump-not "Invalid sum" "cunrolli" } } */
diff --git a/gcc/testsuite/gcc.dg/unroll-4.c b/gcc/testsuite/gcc.dg/unroll-4.c
index 17f194212279560b4c2883b19c00aa271d970d99..055ef3f3545585e8958cd43aedde0bf03eb13459 100644
--- a/gcc/testsuite/gcc.dg/unroll-4.c
+++ b/gcc/testsuite/gcc.dg/unroll-4.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-cunrolli-details -fno-peel-loops -fno-tree-vrp -fdisable-tree-cunroll -fenable-tree-cunrolli=foo -fdisable-tree-cunrolli=foo2" } */
+/* { dg-options "-O2 -fdump-tree-cunrolli-details-blocks -fno-peel-loops -fno-tree-vrp -fdisable-tree-cunroll -fenable-tree-cunrolli=foo -fdisable-tree-cunrolli=foo2" } */
unsigned a[100], b[100];
inline void bar()
@@ -29,3 +29,4 @@ int foo2(void)
}
/* { dg-final { scan-tree-dump-times "loop with 2 iterations completely unrolled" 1 "cunrolli" } } */
+/* { dg-final { scan-tree-dump-not "Invalid sum" "cunrolli" } } */
diff --git a/gcc/testsuite/gcc.dg/unroll-5.c b/gcc/testsuite/gcc.dg/unroll-5.c
index f3bdebe9882f2c42149b9472c0a8b299b8da7af3..1f22b1fa5d6bffbc483e47b4d2e81b9c92966b07 100644
--- a/gcc/testsuite/gcc.dg/unroll-5.c
+++ b/gcc/testsuite/gcc.dg/unroll-5.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-cunrolli-details -fno-peel-loops -fno-tree-vrp -fdisable-tree-cunroll -fenable-tree-cunrolli=foo2 -fdisable-tree-cunrolli=foo" } */
+/* { dg-options "-O2 -fdump-tree-cunrolli-details-blocks -fno-peel-loops -fno-tree-vrp -fdisable-tree-cunroll -fenable-tree-cunrolli=foo2 -fdisable-tree-cunrolli=foo" } */
unsigned a[100], b[100];
inline void bar()
@@ -29,3 +29,4 @@ int foo2(void)
}
/* { dg-final { scan-tree-dump-times "loop with 2 iterations completely unrolled" 1 "cunrolli" } } */
+/* { dg-final { scan-tree-dump-not "Invalid sum" "cunrolli" } } */
diff --git a/gcc/testsuite/gcc.dg/unroll-6.c b/gcc/testsuite/gcc.dg/unroll-6.c
index e4c231ea79f6d6d4b3d74082a65f18c6976d8d9d..7664bbff109f9d79fef623cab8e60368cb1487c7 100644
--- a/gcc/testsuite/gcc.dg/unroll-6.c
+++ b/gcc/testsuite/gcc.dg/unroll-6.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O3 -fdump-rtl-loop2_unroll -funroll-loops" } */
+/* { dg-options "-O3 -fdump-rtl-loop2_unroll-details-blocks -funroll-loops" } */
/* { dg-require-effective-target int32plus } */
void abort (void);
@@ -32,3 +32,4 @@ int t2()
/* { dg-final { scan-rtl-dump-not "realistic bound: 999999" "loop2_unroll" } } */
/* { dg-final { scan-rtl-dump-times " upper bound: 2999999" 1 "loop2_unroll" } } */
/* { dg-final { scan-rtl-dump-times "realistic bound: 2999999" 1 "loop2_unroll" } } */
+/* { dg-final { scan-rtl-dump-not "Invalid sum" "loop2_unroll" { xfail *-*-* } } } */
diff --git a/gcc/tree-ssa-loop-ivopts.h b/gcc/tree-ssa-loop-ivopts.h
index 7a53ce47f1099a547f89a3e93c293ce708d24921..31ec893b9cb05bd2c69a9b88520e3165f6511b54 100644
--- a/gcc/tree-ssa-loop-ivopts.h
+++ b/gcc/tree-ssa-loop-ivopts.h
@@ -20,7 +20,6 @@ along with GCC; see the file COPYING3. If not see
#ifndef GCC_TREE_SSA_LOOP_IVOPTS_H
#define GCC_TREE_SSA_LOOP_IVOPTS_H
-extern edge single_dom_exit (class loop *);
extern void dump_iv (FILE *, struct iv *);
extern void dump_use (FILE *, struct iv_use *);
extern void dump_uses (FILE *, struct ivopts_data *);
diff --git a/gcc/tree-ssa-loop-manip.cc b/gcc/tree-ssa-loop-manip.cc
index 8e3b1057b6ff67e870d69e44de94f8a19cef3de5..e58892e235ca7aea2cf6bdfb624abe4b36ec8801 100644
--- a/gcc/tree-ssa-loop-manip.cc
+++ b/gcc/tree-ssa-loop-manip.cc
@@ -1040,6 +1040,29 @@ determine_exit_conditions (class loop *loop, class tree_niter_desc *desc,
*exit_bound = bound;
}
+/* Updat NEW_EXIT probability after loop has been unrolled. */
+
+void
+update_exit_probability_after_unrolling (class loop *loop, edge new_exit)
+{
+ /* gimple_duplicate_loop_body_to_header_edge depending on
+ DLTHE_FLAG_UPDATE_FREQ either keeps original frequency of the loop header
+ or scales it down accordingly.
+ However exit edge probability is kept as original. Fix it if needed
+ and compensate. */
+ profile_probability new_prob
+ = loop_preheader_edge
+ (loop)->count ().probability_in (new_exit->src->count);
+ if (!(new_prob == new_exit->probability))
+ {
+ profile_count old_count = new_exit->src->count - new_exit->count ();
+ set_edge_probability_and_rescale_others (new_exit, new_prob);
+ profile_count new_count = new_exit->src->count - new_exit->count ();
+ scale_dominated_blocks_in_loop (loop, new_exit->src,
+ new_count, old_count);
+ }
+}
+
/* Unroll LOOP FACTOR times. LOOP is known to have a single exit edge
whose source block dominates the latch. DESC describes the number of
iterations of LOOP.
@@ -1266,23 +1289,7 @@ tree_transform_and_unroll_loop (class loop *loop, unsigned factor,
update_ssa (TODO_update_ssa);
new_exit = single_dom_exit (loop);
-
- /* gimple_duplicate_loop_body_to_header_edge depending on
- DLTHE_FLAG_UPDATE_FREQ either keeps original frequency of the loop header
- or scales it down accordingly.
- However exit edge probability is kept as original. Fix it if needed
- and compensate. */
- profile_probability new_prob
- = loop_preheader_edge
- (loop)->count ().probability_in (new_exit->src->count);
- if (!(new_prob == new_exit->probability))
- {
- profile_count old_count = new_exit->src->count - new_exit->count ();
- set_edge_probability_and_rescale_others (new_exit, new_prob);
- profile_count new_count = new_exit->src->count - new_exit->count ();
- scale_dominated_blocks_in_loop (loop, new_exit->src,
- new_count, old_count);
- }
+ update_exit_probability_after_unrolling (loop, new_exit);
if (!single_loop_p)
{
/* Finally create the new counter for number of iterations and add