| Message ID | 8760fayo60.fsf@linaro.org |
|---|---|
| State | New |
| Headers | show |
| Series | [1/7] Use innermost_loop_behavior for outer loop vectorisation | expand |
On Mon, Jul 3, 2017 at 9:28 AM, Richard Sandiford <richard.sandiford@linaro.org> wrote: > This patch replaces the individual stmt_vinfo dr_* fields with > an innermost_loop_behavior, so that the changes in later patches > get picked up automatically. It also adds a helper function for > getting the behavior of a data reference wrt the vectorised loop. > > Tested on aarch64-linux-gnu and x86_64-linux-gnu. OK to install? Ok. Thanks, Richard. > Richard > > > 2017-07-03 Richard Sandiford <richard.sandiford@linaro.org> > > gcc/ > * tree-vectorizer.h (_stmt_vec_info): Replace individual dr_* > fields with dr_wrt_vec_loop. > (STMT_VINFO_DR_BASE_ADDRESS, STMT_VINFO_DR_INIT, STMT_VINFO_DR_OFFSET) > (STMT_VINFO_DR_STEP, STMT_VINFO_DR_ALIGNED_TO): Update accordingly. > (STMT_VINFO_DR_WRT_VEC_LOOP): New macro. > (vect_dr_behavior): New function. > (vect_create_addr_base_for_vector_ref): Remove loop parameter. > * tree-vect-data-refs.c (vect_compute_data_ref_alignment): Use > vect_dr_behavior. Use a step_preserves_misalignment_p boolean to > track whether the step preserves the misalignment. > (vect_create_addr_base_for_vector_ref): Remove loop parameter. > Use vect_dr_behavior. > (vect_setup_realignment): Update call accordingly. > (vect_create_data_ref_ptr): Likewise. Use vect_dr_behavior. > * tree-vect-loop-manip.c (vect_gen_prolog_loop_niters): Update > call to vect_create_addr_base_for_vector_ref. > (vect_create_cond_for_align_checks): Likewise. > * tree-vect-patterns.c (vect_recog_bool_pattern): Copy > STMT_VINFO_DR_WRT_VEC_LOOP as a block. > (vect_recog_mask_conversion_pattern): Likewise. > * tree-vect-stmts.c (compare_step_with_zero): Use vect_dr_behavior. > (new_stmt_vec_info): Remove redundant zeroing. > > Index: gcc/tree-vectorizer.h > =================================================================== > --- gcc/tree-vectorizer.h 2017-07-03 07:53:58.155555242 +0100 > +++ gcc/tree-vectorizer.h 2017-07-03 07:57:56.883079731 +0100 > @@ -554,11 +554,7 @@ typedef struct _stmt_vec_info { > > /* Information about the data-ref relative to this loop > nest (the loop that is being considered for vectorization). */ > - tree dr_base_address; > - tree dr_init; > - tree dr_offset; > - tree dr_step; > - tree dr_aligned_to; > + innermost_loop_behavior dr_wrt_vec_loop; > > /* For loop PHI nodes, the base and evolution part of it. This makes sure > this information is still available in vect_update_ivs_after_vectorizer > @@ -706,11 +702,12 @@ #define STMT_VINFO_SIMD_LANE_ACCESS_P(S) > #define STMT_VINFO_VEC_REDUCTION_TYPE(S) (S)->v_reduc_type > #define STMT_VINFO_VEC_CONST_COND_REDUC_CODE(S) (S)->const_cond_reduc_code > > -#define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_base_address > -#define STMT_VINFO_DR_INIT(S) (S)->dr_init > -#define STMT_VINFO_DR_OFFSET(S) (S)->dr_offset > -#define STMT_VINFO_DR_STEP(S) (S)->dr_step > -#define STMT_VINFO_DR_ALIGNED_TO(S) (S)->dr_aligned_to > +#define STMT_VINFO_DR_WRT_VEC_LOOP(S) (S)->dr_wrt_vec_loop > +#define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_wrt_vec_loop.base_address > +#define STMT_VINFO_DR_INIT(S) (S)->dr_wrt_vec_loop.init > +#define STMT_VINFO_DR_OFFSET(S) (S)->dr_wrt_vec_loop.offset > +#define STMT_VINFO_DR_STEP(S) (S)->dr_wrt_vec_loop.step > +#define STMT_VINFO_DR_ALIGNED_TO(S) (S)->dr_wrt_vec_loop.aligned_to > > #define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p > #define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt > @@ -1012,6 +1009,22 @@ known_alignment_for_access_p (struct dat > return (DR_MISALIGNMENT (data_ref_info) != DR_MISALIGNMENT_UNKNOWN); > } > > +/* Return the behavior of DR with respect to the vectorization context > + (which for outer loop vectorization might not be the behavior recorded > + in DR itself). */ > + > +static inline innermost_loop_behavior * > +vect_dr_behavior (data_reference *dr) > +{ > + gimple *stmt = DR_STMT (dr); > + stmt_vec_info stmt_info = vinfo_for_stmt (stmt); > + loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); > + if (loop_vinfo == NULL > + || !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt)) > + return &DR_INNERMOST (dr); > + else > + return &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info); > +} > > /* Return true if the vect cost model is unlimited. */ > static inline bool > @@ -1138,8 +1151,7 @@ extern tree vect_get_new_vect_var (tree, > extern tree vect_get_new_ssa_name (tree, enum vect_var_kind, > const char * = NULL); > extern tree vect_create_addr_base_for_vector_ref (gimple *, gimple_seq *, > - tree, struct loop *, > - tree = NULL_TREE); > + tree, tree = NULL_TREE); > > /* In tree-vect-loop.c. */ > /* FORNOW: Used in tree-parloops.c. */ > Index: gcc/tree-vect-data-refs.c > =================================================================== > --- gcc/tree-vect-data-refs.c 2017-07-03 07:53:58.155555242 +0100 > +++ gcc/tree-vect-data-refs.c 2017-07-03 07:57:47.758408141 +0100 > @@ -666,11 +666,8 @@ vect_compute_data_ref_alignment (struct > loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); > struct loop *loop = NULL; > tree ref = DR_REF (dr); > - tree vectype; > - tree base, base_addr; > - tree misalign = NULL_TREE; > - tree aligned_to; > - tree step; > + tree vectype = STMT_VINFO_VECTYPE (stmt_info); > + tree base; > unsigned HOST_WIDE_INT alignment; > > if (dump_enabled_p ()) > @@ -683,11 +680,15 @@ vect_compute_data_ref_alignment (struct > /* Initialize misalignment to unknown. */ > SET_DR_MISALIGNMENT (dr, DR_MISALIGNMENT_UNKNOWN); > > - if (tree_fits_shwi_p (DR_STEP (dr))) > - misalign = DR_INIT (dr); > - aligned_to = DR_ALIGNED_TO (dr); > - base_addr = DR_BASE_ADDRESS (dr); > - vectype = STMT_VINFO_VECTYPE (stmt_info); > + innermost_loop_behavior *drb = vect_dr_behavior (dr); > + bool step_preserves_misalignment_p; > + > + /* No step for BB vectorization. */ > + if (!loop) > + { > + gcc_assert (integer_zerop (drb->step)); > + step_preserves_misalignment_p = true; > + } > > /* In case the dataref is in an inner-loop of the loop that is being > vectorized (LOOP), we use the base and misalignment information > @@ -695,26 +696,21 @@ vect_compute_data_ref_alignment (struct > stays the same throughout the execution of the inner-loop, which is why > we have to check that the stride of the dataref in the inner-loop evenly > divides by the vector size. */ > - if (loop && nested_in_vect_loop_p (loop, stmt)) > + else if (nested_in_vect_loop_p (loop, stmt)) > { > tree step = DR_STEP (dr); > + step_preserves_misalignment_p > + = (tree_fits_shwi_p (step) > + && tree_to_shwi (step) % GET_MODE_SIZE (TYPE_MODE (vectype)) == 0); > > - if (tree_fits_shwi_p (step) > - && tree_to_shwi (step) % GET_MODE_SIZE (TYPE_MODE (vectype)) == 0) > - { > - if (dump_enabled_p ()) > - dump_printf_loc (MSG_NOTE, vect_location, > - "inner step divides the vector-size.\n"); > - misalign = STMT_VINFO_DR_INIT (stmt_info); > - aligned_to = STMT_VINFO_DR_ALIGNED_TO (stmt_info); > - base_addr = STMT_VINFO_DR_BASE_ADDRESS (stmt_info); > - } > - else > + if (dump_enabled_p ()) > { > - if (dump_enabled_p ()) > + if (step_preserves_misalignment_p) > + dump_printf_loc (MSG_NOTE, vect_location, > + "inner step divides the vector-size.\n"); > + else > dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, > - "inner step doesn't divide the vector-size.\n"); > - misalign = NULL_TREE; > + "inner step doesn't divide the vector-size.\n"); > } > } > > @@ -725,18 +721,17 @@ vect_compute_data_ref_alignment (struct > else > { > tree step = DR_STEP (dr); > - unsigned vf = loop ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1; > + unsigned vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); > + step_preserves_misalignment_p > + = (tree_fits_shwi_p (step) > + && ((tree_to_shwi (step) * vf) > + % GET_MODE_SIZE (TYPE_MODE (vectype)) == 0)); > > - if (tree_fits_shwi_p (step) > - && ((tree_to_shwi (step) * vf) > - % GET_MODE_SIZE (TYPE_MODE (vectype)) != 0)) > - { > - if (dump_enabled_p ()) > - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, > - "step doesn't divide the vector-size.\n"); > - misalign = NULL_TREE; > - } > + if (!step_preserves_misalignment_p && dump_enabled_p ()) > + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, > + "step doesn't divide the vector-size.\n"); > } > + tree base_addr = drb->base_address; > > /* To look at alignment of the base we have to preserve an inner MEM_REF > as that carries alignment information of the actual access. */ > @@ -777,8 +772,8 @@ vect_compute_data_ref_alignment (struct > > alignment = TYPE_ALIGN_UNIT (vectype); > > - if ((compare_tree_int (aligned_to, alignment) < 0) > - || !misalign) > + if ((compare_tree_int (drb->aligned_to, alignment) < 0) > + || !step_preserves_misalignment_p) > { > if (dump_enabled_p ()) > { > @@ -835,19 +830,16 @@ vect_compute_data_ref_alignment (struct > DR_VECT_AUX (dr)->base_element_aligned = true; > } > > - if (loop && nested_in_vect_loop_p (loop, stmt)) > - step = STMT_VINFO_DR_STEP (stmt_info); > - else > - step = DR_STEP (dr); > /* If this is a backward running DR then first access in the larger > vectype actually is N-1 elements before the address in the DR. > Adjust misalign accordingly. */ > - if (tree_int_cst_sgn (step) < 0) > + tree misalign = drb->init; > + if (tree_int_cst_sgn (drb->step) < 0) > { > tree offset = ssize_int (TYPE_VECTOR_SUBPARTS (vectype) - 1); > /* DR_STEP(dr) is the same as -TYPE_SIZE of the scalar type, > otherwise we wouldn't be here. */ > - offset = fold_build2 (MULT_EXPR, ssizetype, offset, step); > + offset = fold_build2 (MULT_EXPR, ssizetype, offset, drb->step); > /* PLUS because STEP was negative. */ > misalign = size_binop (PLUS_EXPR, misalign, offset); > } > @@ -3973,38 +3965,22 @@ vect_duplicate_ssa_name_ptr_info (tree n > vect_create_addr_base_for_vector_ref (gimple *stmt, > gimple_seq *new_stmt_list, > tree offset, > - struct loop *loop, > tree byte_offset) > { > stmt_vec_info stmt_info = vinfo_for_stmt (stmt); > struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); > - tree data_ref_base; > const char *base_name; > tree addr_base; > tree dest; > gimple_seq seq = NULL; > - tree base_offset; > - tree init; > tree vect_ptr_type; > tree step = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dr))); > loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); > + innermost_loop_behavior *drb = vect_dr_behavior (dr); > > - if (loop_vinfo && loop && loop != (gimple_bb (stmt))->loop_father) > - { > - struct loop *outer_loop = LOOP_VINFO_LOOP (loop_vinfo); > - > - gcc_assert (nested_in_vect_loop_p (outer_loop, stmt)); > - > - data_ref_base = unshare_expr (STMT_VINFO_DR_BASE_ADDRESS (stmt_info)); > - base_offset = unshare_expr (STMT_VINFO_DR_OFFSET (stmt_info)); > - init = unshare_expr (STMT_VINFO_DR_INIT (stmt_info)); > - } > - else > - { > - data_ref_base = unshare_expr (DR_BASE_ADDRESS (dr)); > - base_offset = unshare_expr (DR_OFFSET (dr)); > - init = unshare_expr (DR_INIT (dr)); > - } > + tree data_ref_base = unshare_expr (drb->base_address); > + tree base_offset = unshare_expr (drb->offset); > + tree init = unshare_expr (drb->init); > > if (loop_vinfo) > base_name = get_name (data_ref_base); > @@ -4169,11 +4145,7 @@ vect_create_data_ref_ptr (gimple *stmt, > > /* Check the step (evolution) of the load in LOOP, and record > whether it's invariant. */ > - if (nested_in_vect_loop) > - step = STMT_VINFO_DR_STEP (stmt_info); > - else > - step = DR_STEP (STMT_VINFO_DATA_REF (stmt_info)); > - > + step = vect_dr_behavior (dr)->step; > if (integer_zerop (step)) > *inv_p = true; > else > @@ -4271,7 +4243,7 @@ vect_create_data_ref_ptr (gimple *stmt, > /* Create: (&(base[init_val+offset]+byte_offset) in the loop preheader. */ > > new_temp = vect_create_addr_base_for_vector_ref (stmt, &new_stmt_list, > - offset, loop, byte_offset); > + offset, byte_offset); > if (new_stmt_list) > { > if (pe) > @@ -4985,7 +4957,7 @@ vect_setup_realignment (gimple *stmt, gi > { > /* Generate the INIT_ADDR computation outside LOOP. */ > init_addr = vect_create_addr_base_for_vector_ref (stmt, &stmts, > - NULL_TREE, loop); > + NULL_TREE); > if (loop) > { > pe = loop_preheader_edge (loop); > Index: gcc/tree-vect-loop-manip.c > =================================================================== > --- gcc/tree-vect-loop-manip.c 2017-07-03 07:53:58.155555242 +0100 > +++ gcc/tree-vect-loop-manip.c 2017-07-03 07:57:47.758408141 +0100 > @@ -949,7 +949,6 @@ vect_gen_prolog_loop_niters (loop_vec_in > basic_block bb, int *bound) > { > struct data_reference *dr = LOOP_VINFO_UNALIGNED_DR (loop_vinfo); > - struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); > tree var; > tree niters_type = TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)); > gimple_seq stmts = NULL, new_stmts = NULL; > @@ -977,7 +976,7 @@ vect_gen_prolog_loop_niters (loop_vec_in > tree offset = negative > ? size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1) : size_zero_node; > tree start_addr = vect_create_addr_base_for_vector_ref (dr_stmt, > - &stmts, offset, loop); > + &stmts, offset); > tree type = unsigned_type_for (TREE_TYPE (start_addr)); > tree vectype_align_minus_1 = build_int_cst (type, vectype_align - 1); > HOST_WIDE_INT elem_size = > @@ -1975,7 +1974,6 @@ vect_create_cond_for_align_checks (loop_ > tree *cond_expr, > gimple_seq *cond_expr_stmt_list) > { > - struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); > vec<gimple *> may_misalign_stmts > = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo); > gimple *ref_stmt; > @@ -2016,7 +2014,7 @@ vect_create_cond_for_align_checks (loop_ > /* create: addr_tmp = (int)(address_of_first_vector) */ > addr_base = > vect_create_addr_base_for_vector_ref (ref_stmt, &new_stmt_list, > - offset, loop); > + offset); > if (new_stmt_list != NULL) > gimple_seq_add_seq (cond_expr_stmt_list, new_stmt_list); > > Index: gcc/tree-vect-patterns.c > =================================================================== > --- gcc/tree-vect-patterns.c 2017-07-03 07:53:58.155555242 +0100 > +++ gcc/tree-vect-patterns.c 2017-07-03 07:57:47.759408106 +0100 > @@ -3789,14 +3789,8 @@ vect_recog_bool_pattern (vec<gimple *> * > set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info); > STMT_VINFO_DATA_REF (pattern_stmt_info) > = STMT_VINFO_DATA_REF (stmt_vinfo); > - STMT_VINFO_DR_BASE_ADDRESS (pattern_stmt_info) > - = STMT_VINFO_DR_BASE_ADDRESS (stmt_vinfo); > - STMT_VINFO_DR_INIT (pattern_stmt_info) = STMT_VINFO_DR_INIT (stmt_vinfo); > - STMT_VINFO_DR_OFFSET (pattern_stmt_info) > - = STMT_VINFO_DR_OFFSET (stmt_vinfo); > - STMT_VINFO_DR_STEP (pattern_stmt_info) = STMT_VINFO_DR_STEP (stmt_vinfo); > - STMT_VINFO_DR_ALIGNED_TO (pattern_stmt_info) > - = STMT_VINFO_DR_ALIGNED_TO (stmt_vinfo); > + STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info) > + = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo); > DR_STMT (STMT_VINFO_DATA_REF (stmt_vinfo)) = pattern_stmt; > *type_out = vectype; > *type_in = vectype; > @@ -3930,14 +3924,8 @@ vect_recog_mask_conversion_pattern (vec< > set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info); > STMT_VINFO_DATA_REF (pattern_stmt_info) > = STMT_VINFO_DATA_REF (stmt_vinfo); > - STMT_VINFO_DR_BASE_ADDRESS (pattern_stmt_info) > - = STMT_VINFO_DR_BASE_ADDRESS (stmt_vinfo); > - STMT_VINFO_DR_INIT (pattern_stmt_info) = STMT_VINFO_DR_INIT (stmt_vinfo); > - STMT_VINFO_DR_OFFSET (pattern_stmt_info) > - = STMT_VINFO_DR_OFFSET (stmt_vinfo); > - STMT_VINFO_DR_STEP (pattern_stmt_info) = STMT_VINFO_DR_STEP (stmt_vinfo); > - STMT_VINFO_DR_ALIGNED_TO (pattern_stmt_info) > - = STMT_VINFO_DR_ALIGNED_TO (stmt_vinfo); > + STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info) > + = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo); > DR_STMT (STMT_VINFO_DATA_REF (stmt_vinfo)) = pattern_stmt; > > *type_out = vectype1; > Index: gcc/tree-vect-stmts.c > =================================================================== > --- gcc/tree-vect-stmts.c 2017-07-03 07:53:58.155555242 +0100 > +++ gcc/tree-vect-stmts.c 2017-07-03 07:57:47.759408106 +0100 > @@ -1698,13 +1698,9 @@ static tree permute_vec_elements (tree, > compare_step_with_zero (gimple *stmt) > { > stmt_vec_info stmt_info = vinfo_for_stmt (stmt); > - loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); > - tree step; > - if (loop_vinfo && nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt)) > - step = STMT_VINFO_DR_STEP (stmt_info); > - else > - step = DR_STEP (STMT_VINFO_DATA_REF (stmt_info)); > - return tree_int_cst_compare (step, size_zero_node); > + data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); > + return tree_int_cst_compare (vect_dr_behavior (dr)->step, > + size_zero_node); > } > > /* If the target supports a permute mask that reverses the elements in > @@ -8851,12 +8847,6 @@ new_stmt_vec_info (gimple *stmt, vec_inf > STMT_VINFO_VEC_REDUCTION_TYPE (res) = TREE_CODE_REDUCTION; > STMT_VINFO_VEC_CONST_COND_REDUC_CODE (res) = ERROR_MARK; > > - STMT_VINFO_DR_BASE_ADDRESS (res) = NULL; > - STMT_VINFO_DR_OFFSET (res) = NULL; > - STMT_VINFO_DR_INIT (res) = NULL; > - STMT_VINFO_DR_STEP (res) = NULL; > - STMT_VINFO_DR_ALIGNED_TO (res) = NULL; > - > if (gimple_code (stmt) == GIMPLE_PHI > && is_loop_header_bb_p (gimple_bb (stmt))) > STMT_VINFO_DEF_TYPE (res) = vect_unknown_def_type;
Index: gcc/tree-vectorizer.h =================================================================== --- gcc/tree-vectorizer.h 2017-07-03 07:53:58.155555242 +0100 +++ gcc/tree-vectorizer.h 2017-07-03 07:57:56.883079731 +0100 @@ -554,11 +554,7 @@ typedef struct _stmt_vec_info { /* Information about the data-ref relative to this loop nest (the loop that is being considered for vectorization). */ - tree dr_base_address; - tree dr_init; - tree dr_offset; - tree dr_step; - tree dr_aligned_to; + innermost_loop_behavior dr_wrt_vec_loop; /* For loop PHI nodes, the base and evolution part of it. This makes sure this information is still available in vect_update_ivs_after_vectorizer @@ -706,11 +702,12 @@ #define STMT_VINFO_SIMD_LANE_ACCESS_P(S) #define STMT_VINFO_VEC_REDUCTION_TYPE(S) (S)->v_reduc_type #define STMT_VINFO_VEC_CONST_COND_REDUC_CODE(S) (S)->const_cond_reduc_code -#define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_base_address -#define STMT_VINFO_DR_INIT(S) (S)->dr_init -#define STMT_VINFO_DR_OFFSET(S) (S)->dr_offset -#define STMT_VINFO_DR_STEP(S) (S)->dr_step -#define STMT_VINFO_DR_ALIGNED_TO(S) (S)->dr_aligned_to +#define STMT_VINFO_DR_WRT_VEC_LOOP(S) (S)->dr_wrt_vec_loop +#define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_wrt_vec_loop.base_address +#define STMT_VINFO_DR_INIT(S) (S)->dr_wrt_vec_loop.init +#define STMT_VINFO_DR_OFFSET(S) (S)->dr_wrt_vec_loop.offset +#define STMT_VINFO_DR_STEP(S) (S)->dr_wrt_vec_loop.step +#define STMT_VINFO_DR_ALIGNED_TO(S) (S)->dr_wrt_vec_loop.aligned_to #define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p #define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt @@ -1012,6 +1009,22 @@ known_alignment_for_access_p (struct dat return (DR_MISALIGNMENT (data_ref_info) != DR_MISALIGNMENT_UNKNOWN); } +/* Return the behavior of DR with respect to the vectorization context + (which for outer loop vectorization might not be the behavior recorded + in DR itself). */ + +static inline innermost_loop_behavior * +vect_dr_behavior (data_reference *dr) +{ + gimple *stmt = DR_STMT (dr); + stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); + if (loop_vinfo == NULL + || !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt)) + return &DR_INNERMOST (dr); + else + return &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info); +} /* Return true if the vect cost model is unlimited. */ static inline bool @@ -1138,8 +1151,7 @@ extern tree vect_get_new_vect_var (tree, extern tree vect_get_new_ssa_name (tree, enum vect_var_kind, const char * = NULL); extern tree vect_create_addr_base_for_vector_ref (gimple *, gimple_seq *, - tree, struct loop *, - tree = NULL_TREE); + tree, tree = NULL_TREE); /* In tree-vect-loop.c. */ /* FORNOW: Used in tree-parloops.c. */ Index: gcc/tree-vect-data-refs.c =================================================================== --- gcc/tree-vect-data-refs.c 2017-07-03 07:53:58.155555242 +0100 +++ gcc/tree-vect-data-refs.c 2017-07-03 07:57:47.758408141 +0100 @@ -666,11 +666,8 @@ vect_compute_data_ref_alignment (struct loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); struct loop *loop = NULL; tree ref = DR_REF (dr); - tree vectype; - tree base, base_addr; - tree misalign = NULL_TREE; - tree aligned_to; - tree step; + tree vectype = STMT_VINFO_VECTYPE (stmt_info); + tree base; unsigned HOST_WIDE_INT alignment; if (dump_enabled_p ()) @@ -683,11 +680,15 @@ vect_compute_data_ref_alignment (struct /* Initialize misalignment to unknown. */ SET_DR_MISALIGNMENT (dr, DR_MISALIGNMENT_UNKNOWN); - if (tree_fits_shwi_p (DR_STEP (dr))) - misalign = DR_INIT (dr); - aligned_to = DR_ALIGNED_TO (dr); - base_addr = DR_BASE_ADDRESS (dr); - vectype = STMT_VINFO_VECTYPE (stmt_info); + innermost_loop_behavior *drb = vect_dr_behavior (dr); + bool step_preserves_misalignment_p; + + /* No step for BB vectorization. */ + if (!loop) + { + gcc_assert (integer_zerop (drb->step)); + step_preserves_misalignment_p = true; + } /* In case the dataref is in an inner-loop of the loop that is being vectorized (LOOP), we use the base and misalignment information @@ -695,26 +696,21 @@ vect_compute_data_ref_alignment (struct stays the same throughout the execution of the inner-loop, which is why we have to check that the stride of the dataref in the inner-loop evenly divides by the vector size. */ - if (loop && nested_in_vect_loop_p (loop, stmt)) + else if (nested_in_vect_loop_p (loop, stmt)) { tree step = DR_STEP (dr); + step_preserves_misalignment_p + = (tree_fits_shwi_p (step) + && tree_to_shwi (step) % GET_MODE_SIZE (TYPE_MODE (vectype)) == 0); - if (tree_fits_shwi_p (step) - && tree_to_shwi (step) % GET_MODE_SIZE (TYPE_MODE (vectype)) == 0) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "inner step divides the vector-size.\n"); - misalign = STMT_VINFO_DR_INIT (stmt_info); - aligned_to = STMT_VINFO_DR_ALIGNED_TO (stmt_info); - base_addr = STMT_VINFO_DR_BASE_ADDRESS (stmt_info); - } - else + if (dump_enabled_p ()) { - if (dump_enabled_p ()) + if (step_preserves_misalignment_p) + dump_printf_loc (MSG_NOTE, vect_location, + "inner step divides the vector-size.\n"); + else dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "inner step doesn't divide the vector-size.\n"); - misalign = NULL_TREE; + "inner step doesn't divide the vector-size.\n"); } } @@ -725,18 +721,17 @@ vect_compute_data_ref_alignment (struct else { tree step = DR_STEP (dr); - unsigned vf = loop ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1; + unsigned vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); + step_preserves_misalignment_p + = (tree_fits_shwi_p (step) + && ((tree_to_shwi (step) * vf) + % GET_MODE_SIZE (TYPE_MODE (vectype)) == 0)); - if (tree_fits_shwi_p (step) - && ((tree_to_shwi (step) * vf) - % GET_MODE_SIZE (TYPE_MODE (vectype)) != 0)) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "step doesn't divide the vector-size.\n"); - misalign = NULL_TREE; - } + if (!step_preserves_misalignment_p && dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "step doesn't divide the vector-size.\n"); } + tree base_addr = drb->base_address; /* To look at alignment of the base we have to preserve an inner MEM_REF as that carries alignment information of the actual access. */ @@ -777,8 +772,8 @@ vect_compute_data_ref_alignment (struct alignment = TYPE_ALIGN_UNIT (vectype); - if ((compare_tree_int (aligned_to, alignment) < 0) - || !misalign) + if ((compare_tree_int (drb->aligned_to, alignment) < 0) + || !step_preserves_misalignment_p) { if (dump_enabled_p ()) { @@ -835,19 +830,16 @@ vect_compute_data_ref_alignment (struct DR_VECT_AUX (dr)->base_element_aligned = true; } - if (loop && nested_in_vect_loop_p (loop, stmt)) - step = STMT_VINFO_DR_STEP (stmt_info); - else - step = DR_STEP (dr); /* If this is a backward running DR then first access in the larger vectype actually is N-1 elements before the address in the DR. Adjust misalign accordingly. */ - if (tree_int_cst_sgn (step) < 0) + tree misalign = drb->init; + if (tree_int_cst_sgn (drb->step) < 0) { tree offset = ssize_int (TYPE_VECTOR_SUBPARTS (vectype) - 1); /* DR_STEP(dr) is the same as -TYPE_SIZE of the scalar type, otherwise we wouldn't be here. */ - offset = fold_build2 (MULT_EXPR, ssizetype, offset, step); + offset = fold_build2 (MULT_EXPR, ssizetype, offset, drb->step); /* PLUS because STEP was negative. */ misalign = size_binop (PLUS_EXPR, misalign, offset); } @@ -3973,38 +3965,22 @@ vect_duplicate_ssa_name_ptr_info (tree n vect_create_addr_base_for_vector_ref (gimple *stmt, gimple_seq *new_stmt_list, tree offset, - struct loop *loop, tree byte_offset) { stmt_vec_info stmt_info = vinfo_for_stmt (stmt); struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); - tree data_ref_base; const char *base_name; tree addr_base; tree dest; gimple_seq seq = NULL; - tree base_offset; - tree init; tree vect_ptr_type; tree step = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dr))); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); + innermost_loop_behavior *drb = vect_dr_behavior (dr); - if (loop_vinfo && loop && loop != (gimple_bb (stmt))->loop_father) - { - struct loop *outer_loop = LOOP_VINFO_LOOP (loop_vinfo); - - gcc_assert (nested_in_vect_loop_p (outer_loop, stmt)); - - data_ref_base = unshare_expr (STMT_VINFO_DR_BASE_ADDRESS (stmt_info)); - base_offset = unshare_expr (STMT_VINFO_DR_OFFSET (stmt_info)); - init = unshare_expr (STMT_VINFO_DR_INIT (stmt_info)); - } - else - { - data_ref_base = unshare_expr (DR_BASE_ADDRESS (dr)); - base_offset = unshare_expr (DR_OFFSET (dr)); - init = unshare_expr (DR_INIT (dr)); - } + tree data_ref_base = unshare_expr (drb->base_address); + tree base_offset = unshare_expr (drb->offset); + tree init = unshare_expr (drb->init); if (loop_vinfo) base_name = get_name (data_ref_base); @@ -4169,11 +4145,7 @@ vect_create_data_ref_ptr (gimple *stmt, /* Check the step (evolution) of the load in LOOP, and record whether it's invariant. */ - if (nested_in_vect_loop) - step = STMT_VINFO_DR_STEP (stmt_info); - else - step = DR_STEP (STMT_VINFO_DATA_REF (stmt_info)); - + step = vect_dr_behavior (dr)->step; if (integer_zerop (step)) *inv_p = true; else @@ -4271,7 +4243,7 @@ vect_create_data_ref_ptr (gimple *stmt, /* Create: (&(base[init_val+offset]+byte_offset) in the loop preheader. */ new_temp = vect_create_addr_base_for_vector_ref (stmt, &new_stmt_list, - offset, loop, byte_offset); + offset, byte_offset); if (new_stmt_list) { if (pe) @@ -4985,7 +4957,7 @@ vect_setup_realignment (gimple *stmt, gi { /* Generate the INIT_ADDR computation outside LOOP. */ init_addr = vect_create_addr_base_for_vector_ref (stmt, &stmts, - NULL_TREE, loop); + NULL_TREE); if (loop) { pe = loop_preheader_edge (loop); Index: gcc/tree-vect-loop-manip.c =================================================================== --- gcc/tree-vect-loop-manip.c 2017-07-03 07:53:58.155555242 +0100 +++ gcc/tree-vect-loop-manip.c 2017-07-03 07:57:47.758408141 +0100 @@ -949,7 +949,6 @@ vect_gen_prolog_loop_niters (loop_vec_in basic_block bb, int *bound) { struct data_reference *dr = LOOP_VINFO_UNALIGNED_DR (loop_vinfo); - struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); tree var; tree niters_type = TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)); gimple_seq stmts = NULL, new_stmts = NULL; @@ -977,7 +976,7 @@ vect_gen_prolog_loop_niters (loop_vec_in tree offset = negative ? size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1) : size_zero_node; tree start_addr = vect_create_addr_base_for_vector_ref (dr_stmt, - &stmts, offset, loop); + &stmts, offset); tree type = unsigned_type_for (TREE_TYPE (start_addr)); tree vectype_align_minus_1 = build_int_cst (type, vectype_align - 1); HOST_WIDE_INT elem_size = @@ -1975,7 +1974,6 @@ vect_create_cond_for_align_checks (loop_ tree *cond_expr, gimple_seq *cond_expr_stmt_list) { - struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); vec<gimple *> may_misalign_stmts = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo); gimple *ref_stmt; @@ -2016,7 +2014,7 @@ vect_create_cond_for_align_checks (loop_ /* create: addr_tmp = (int)(address_of_first_vector) */ addr_base = vect_create_addr_base_for_vector_ref (ref_stmt, &new_stmt_list, - offset, loop); + offset); if (new_stmt_list != NULL) gimple_seq_add_seq (cond_expr_stmt_list, new_stmt_list); Index: gcc/tree-vect-patterns.c =================================================================== --- gcc/tree-vect-patterns.c 2017-07-03 07:53:58.155555242 +0100 +++ gcc/tree-vect-patterns.c 2017-07-03 07:57:47.759408106 +0100 @@ -3789,14 +3789,8 @@ vect_recog_bool_pattern (vec<gimple *> * set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info); STMT_VINFO_DATA_REF (pattern_stmt_info) = STMT_VINFO_DATA_REF (stmt_vinfo); - STMT_VINFO_DR_BASE_ADDRESS (pattern_stmt_info) - = STMT_VINFO_DR_BASE_ADDRESS (stmt_vinfo); - STMT_VINFO_DR_INIT (pattern_stmt_info) = STMT_VINFO_DR_INIT (stmt_vinfo); - STMT_VINFO_DR_OFFSET (pattern_stmt_info) - = STMT_VINFO_DR_OFFSET (stmt_vinfo); - STMT_VINFO_DR_STEP (pattern_stmt_info) = STMT_VINFO_DR_STEP (stmt_vinfo); - STMT_VINFO_DR_ALIGNED_TO (pattern_stmt_info) - = STMT_VINFO_DR_ALIGNED_TO (stmt_vinfo); + STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info) + = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo); DR_STMT (STMT_VINFO_DATA_REF (stmt_vinfo)) = pattern_stmt; *type_out = vectype; *type_in = vectype; @@ -3930,14 +3924,8 @@ vect_recog_mask_conversion_pattern (vec< set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info); STMT_VINFO_DATA_REF (pattern_stmt_info) = STMT_VINFO_DATA_REF (stmt_vinfo); - STMT_VINFO_DR_BASE_ADDRESS (pattern_stmt_info) - = STMT_VINFO_DR_BASE_ADDRESS (stmt_vinfo); - STMT_VINFO_DR_INIT (pattern_stmt_info) = STMT_VINFO_DR_INIT (stmt_vinfo); - STMT_VINFO_DR_OFFSET (pattern_stmt_info) - = STMT_VINFO_DR_OFFSET (stmt_vinfo); - STMT_VINFO_DR_STEP (pattern_stmt_info) = STMT_VINFO_DR_STEP (stmt_vinfo); - STMT_VINFO_DR_ALIGNED_TO (pattern_stmt_info) - = STMT_VINFO_DR_ALIGNED_TO (stmt_vinfo); + STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info) + = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo); DR_STMT (STMT_VINFO_DATA_REF (stmt_vinfo)) = pattern_stmt; *type_out = vectype1; Index: gcc/tree-vect-stmts.c =================================================================== --- gcc/tree-vect-stmts.c 2017-07-03 07:53:58.155555242 +0100 +++ gcc/tree-vect-stmts.c 2017-07-03 07:57:47.759408106 +0100 @@ -1698,13 +1698,9 @@ static tree permute_vec_elements (tree, compare_step_with_zero (gimple *stmt) { stmt_vec_info stmt_info = vinfo_for_stmt (stmt); - loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); - tree step; - if (loop_vinfo && nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt)) - step = STMT_VINFO_DR_STEP (stmt_info); - else - step = DR_STEP (STMT_VINFO_DATA_REF (stmt_info)); - return tree_int_cst_compare (step, size_zero_node); + data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); + return tree_int_cst_compare (vect_dr_behavior (dr)->step, + size_zero_node); } /* If the target supports a permute mask that reverses the elements in @@ -8851,12 +8847,6 @@ new_stmt_vec_info (gimple *stmt, vec_inf STMT_VINFO_VEC_REDUCTION_TYPE (res) = TREE_CODE_REDUCTION; STMT_VINFO_VEC_CONST_COND_REDUC_CODE (res) = ERROR_MARK; - STMT_VINFO_DR_BASE_ADDRESS (res) = NULL; - STMT_VINFO_DR_OFFSET (res) = NULL; - STMT_VINFO_DR_INIT (res) = NULL; - STMT_VINFO_DR_STEP (res) = NULL; - STMT_VINFO_DR_ALIGNED_TO (res) = NULL; - if (gimple_code (stmt) == GIMPLE_PHI && is_loop_header_bb_p (gimple_bb (stmt))) STMT_VINFO_DEF_TYPE (res) = vect_unknown_def_type;