===================================================================
@@ -1516,7 +1516,7 @@ vect_generate_tmps_on_preheader (loop_ve
edge pe;
basic_block new_bb;
gimple_seq stmts;
- tree ni_name;
+ tree ni_name, ni_minus_gap_name;
tree var;
tree ratio_name;
tree ratio_mult_vf_name;
@@ -1533,9 +1533,39 @@ vect_generate_tmps_on_preheader (loop_ve
ni_name = vect_build_loop_niters (loop_vinfo, cond_expr_stmt_list);
log_vf = build_int_cst (TREE_TYPE (ni), exact_log2 (vf));
+ /* If epilogue loop is required because of data accesses with gaps, we
+ subtract one iteration from the total number of iterations here for
+ correct calculation of RATIO. */
+ if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
+ {
+ ni_minus_gap_name = fold_build2 (MINUS_EXPR, TREE_TYPE (ni_name),
+ ni_name,
+ build_one_cst (TREE_TYPE (ni_name)));
+ if (!is_gimple_val (ni_minus_gap_name))
+ {
+ var = create_tmp_var (TREE_TYPE (ni), "ni_gap");
+ add_referenced_var (var);
+
+ stmts = NULL;
+ ni_minus_gap_name = force_gimple_operand (ni_minus_gap_name, &stmts,
+ true, var);
+ if (cond_expr_stmt_list)
+ gimple_seq_add_seq (&cond_expr_stmt_list, stmts);
+ else
+ {
+ pe = loop_preheader_edge (loop);
+ new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
+ gcc_assert (!new_bb);
+ }
+ }
+ }
+ else
+ ni_minus_gap_name = ni_name;
+
/* Create: ratio = ni >> log2(vf) */
- ratio_name = fold_build2 (RSHIFT_EXPR, TREE_TYPE (ni_name), ni_name, log_vf);
+ ratio_name = fold_build2 (RSHIFT_EXPR, TREE_TYPE (ni_minus_gap_name),
+ ni_minus_gap_name, log_vf);
if (!is_gimple_val (ratio_name))
{
var = create_tmp_var (TREE_TYPE (ni), "bnd");
===================================================================
@@ -0,0 +1,116 @@
+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include <stdio.h>
+#include "tree-vect.h"
+
+#define N 160
+
+typedef struct {
+ unsigned char a;
+ unsigned char b;
+ unsigned char c;
+ unsigned char d;
+ unsigned char e;
+ unsigned char f;
+ unsigned char g;
+ unsigned char h;
+} s;
+
+__attribute__ ((noinline)) int
+main1 (s *arr, int n)
+{
+ int i;
+ s *ptr = arr;
+ s res[N];
+ unsigned char x;
+
+ for (i = 0; i < N; i++)
+ {
+ res[i].a = 0;
+ res[i].b = 0;
+ res[i].c = 0;
+ res[i].d = 0;
+ res[i].e = 0;
+ res[i].f = 0;
+ res[i].g = 0;
+ res[i].h = 0;
+ __asm__ volatile ("");
+ }
+
+ /* Check peeling for gaps for unknown loop bound. */
+ for (i = 0; i < n; i++)
+ {
+ res[i].c = ptr->b + ptr->c;
+ x = ptr->c + ptr->f;
+ res[i].a = x + ptr->b;
+ res[i].d = ptr->b + ptr->c;
+ res[i].b = ptr->c;
+ res[i].f = ptr->f + ptr->e;
+ res[i].e = ptr->b + ptr->e;
+ res[i].h = ptr->c;
+ res[i].g = ptr->b + ptr->c;
+ ptr++;
+ }
+
+ /* check results: */
+ for (i = 0; i < n; i++)
+ {
+ if (res[i].c != arr[i].b + arr[i].c
+ || res[i].a != arr[i].c + arr[i].f + arr[i].b
+ || res[i].d != arr[i].b + arr[i].c
+ || res[i].b != arr[i].c
+ || res[i].f != arr[i].f + arr[i].e
+ || res[i].e != arr[i].b + arr[i].e
+ || res[i].h != arr[i].c
+ || res[i].g != arr[i].b + arr[i].c)
+ abort ();
+ }
+
+ /* Check also that we don't do more iterations than needed. */
+ for (i = n; i < N; i++)
+ {
+ if (res[i].c == arr[i].b + arr[i].c
+ || res[i].a == arr[i].c + arr[i].f + arr[i].b
+ || res[i].d == arr[i].b + arr[i].c
+ || res[i].b == arr[i].c
+ || res[i].f == arr[i].f + arr[i].e
+ || res[i].e == arr[i].b + arr[i].e
+ || res[i].h == arr[i].c
+ || res[i].g == arr[i].b + arr[i].c)
+ abort ();
+ }
+
+ return 0;
+}
+
+
+int main (void)
+{
+ int i;
+ s arr[N];
+
+ check_vect ();
+
+ for (i = 0; i < N; i++)
+ {
+ arr[i].a = 5;
+ arr[i].b = 6;
+ arr[i].c = 17;
+ arr[i].d = 3;
+ arr[i].e = 16;
+ arr[i].f = 16;
+ arr[i].g = 3;
+ arr[i].h = 56;
+ if (arr[i].a == 178)
+ abort();
+ }
+
+ main1 (arr, N-2);
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { target { vect_interleave && vect_extract_even_odd } } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
+
===================================================================
@@ -0,0 +1,42 @@
+#include <sys/mman.h>
+#include <stdio.h>
+
+#define COUNT 320
+#define MMAP_SIZE 0x10000
+#define ADDRESS 0x1122000000
+#define TYPE unsigned short
+
+#ifndef MAP_ANONYMOUS
+#define MAP_ANONYMOUS MAP_ANON
+#endif
+
+void __attribute__((noinline))
+foo (TYPE *__restrict a, TYPE *__restrict b)
+{
+ int n;
+
+ for (n = 0; n < COUNT; n++)
+ a[n] = b[n * 2];
+}
+
+int
+main (void)
+{
+ void *x;
+ size_t b_offset;
+
+ x = mmap ((void *) ADDRESS, MMAP_SIZE, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (x == MAP_FAILED)
+ {
+ perror ("mmap");
+ return 1;
+ }
+
+ b_offset = MMAP_SIZE - (2 * COUNT - 1) * sizeof (TYPE);
+ foo ((unsigned short *) x,
+ (unsigned short *) ((char *) x + b_offset));
+ return 0;
+}
+
+/* { dg-final { cleanup-tree-dump "vect" } } */
===================================================================
@@ -242,6 +242,12 @@ typedef struct _loop_vec_info {
/* The unrolling factor needed to SLP the loop. In case of that pure SLP is
applied to the loop, i.e., no unrolling is needed, this is 1. */
unsigned slp_unrolling_factor;
+
+ /* When we have strided data accesses with gaps, we may introduce invalid
+ memory accesses. We peel the last iteration of the loop to prevent
+ this. */
+ bool peeling_for_gaps;
+
} *loop_vec_info;
/* Access Functions. */
@@ -266,6 +272,7 @@ typedef struct _loop_vec_info {
#define LOOP_VINFO_STRIDED_STORES(L) (L)->strided_stores
#define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances
#define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor
+#define LOOP_VINFO_PEELING_FOR_GAPS(L) (L)->peeling_for_gaps
#define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \
VEC_length (gimple, (L)->may_misalign_stmts) > 0
===================================================================
@@ -711,6 +711,7 @@ new_loop_vec_info (struct loop *loop)
LOOP_VINFO_STRIDED_STORES (res) = VEC_alloc (gimple, heap, 10);
LOOP_VINFO_SLP_INSTANCES (res) = VEC_alloc (slp_instance, heap, 10);
LOOP_VINFO_SLP_UNROLLING_FACTOR (res) = 1;
+ LOOP_VINFO_PEELING_FOR_GAPS (res) = false;
return res;
}
@@ -2053,6 +2054,10 @@ vect_estimate_min_profitable_iters (loop
peel_iters_prologue = niters < peel_iters_prologue ?
niters : peel_iters_prologue;
peel_iters_epilogue = (niters - peel_iters_prologue) % vf;
+ /* If we need to peel for gaps, but no peeling is required, we have
+ to peel VF iterations. */
+ if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) && !peel_iters_epilogue)
+ peel_iters_epilogue = vf;
}
}
@@ -4212,7 +4217,8 @@ vect_transform_loop (loop_vec_info loop_
do_peeling_for_loop_bound
= (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|| (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
- && LOOP_VINFO_INT_NITERS (loop_vinfo) % vectorization_factor != 0));
+ && LOOP_VINFO_INT_NITERS (loop_vinfo) % vectorization_factor != 0)
+ || LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo));
if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo)
|| LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo))
===================================================================
@@ -1450,7 +1450,7 @@ vect_analyze_group_access (struct data_r
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
HOST_WIDE_INT dr_step = TREE_INT_CST_LOW (step);
- HOST_WIDE_INT stride;
+ HOST_WIDE_INT stride, last_accessed_element = 1;
bool slp_impossible = false;
/* For interleaving, STRIDE is STEP counted in elements, i.e., the size of the
@@ -1479,6 +1479,16 @@ vect_analyze_group_access (struct data_r
fprintf (vect_dump, " step ");
print_generic_expr (vect_dump, step, TDF_SLIM);
}
+
+ if (loop_vinfo)
+ {
+ LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true;
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "Data access with gaps requires scalar "
+ "epilogue loop");
+ }
+
return true;
}
if (vect_print_dump_info (REPORT_DETAILS))
@@ -1531,6 +1541,7 @@ vect_analyze_group_access (struct data_r
next = DR_GROUP_NEXT_DR (vinfo_for_stmt (next));
continue;
}
+
prev = next;
/* Check that all the accesses have the same STEP. */
@@ -1561,6 +1572,8 @@ vect_analyze_group_access (struct data_r
gaps += diff - 1;
}
+ last_accessed_element += diff;
+
/* Store the gap from the previous member of the group. If there is no
gap in the access, DR_GROUP_GAP is always 1. */
DR_GROUP_GAP (vinfo_for_stmt (next)) = diff;
@@ -1652,6 +1665,15 @@ vect_analyze_group_access (struct data_r
VEC_safe_push (gimple, heap, BB_VINFO_STRIDED_STORES (bb_vinfo),
stmt);
}
+
+ /* There is a gap in the end of the group. */
+ if (stride - last_accessed_element > 0 && loop_vinfo)
+ {
+ LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true;
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "Data access with gaps requires scalar "
+ "epilogue loop");
+ }
}
return true;