@@ -17648,6 +17648,45 @@ aarch64_select_early_remat_modes (sbitmap modes)
}
}
+/* Override the default target speculation_safe_value. */
+static rtx
+aarch64_speculation_safe_value (machine_mode mode,
+ rtx result, rtx val, rtx failval)
+{
+ /* Maybe we should warn if falling back to hard barriers. They are
+ likely to be noticably more expensive than the alternative below. */
+ if (!aarch64_track_speculation)
+ return default_speculation_safe_value (mode, result, val, failval);
+
+ if (!REG_P (val))
+ val = copy_to_mode_reg (mode, val);
+
+ if (!aarch64_reg_or_zero (failval, mode))
+ failval = copy_to_mode_reg (mode, failval);
+
+ switch (mode)
+ {
+ case E_QImode:
+ emit_insn (gen_despeculate_copyqi (result, val, failval));
+ break;
+ case E_HImode:
+ emit_insn (gen_despeculate_copyhi (result, val, failval));
+ break;
+ case E_SImode:
+ emit_insn (gen_despeculate_copysi (result, val, failval));
+ break;
+ case E_DImode:
+ emit_insn (gen_despeculate_copydi (result, val, failval));
+ break;
+ case E_TImode:
+ emit_insn (gen_despeculate_copyti (result, val, failval));
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ return result;
+}
+
/* Target-specific selftests. */
#if CHECKING_P
@@ -18117,6 +18156,9 @@ aarch64_libgcc_floating_mode_supported_p
#undef TARGET_SELECT_EARLY_REMAT_MODES
#define TARGET_SELECT_EARLY_REMAT_MODES aarch64_select_early_remat_modes
+#undef TARGET_SPECULATION_SAFE_VALUE
+#define TARGET_SPECULATION_SAFE_VALUE aarch64_speculation_safe_value
+
#if CHECKING_P
#undef TARGET_RUN_TARGET_SELFTESTS
#define TARGET_RUN_TARGET_SELFTESTS selftest::aarch64_run_selftests
@@ -6129,6 +6129,102 @@ (define_insn "speculation_barrier"
(set_attr "speculation_barrier" "true")]
)
+;; Support for __builtin_speculation_safe_value when we have speculation
+;; tracking enabled. Use the speculation tracker to decide whether to
+;; copy operand 1 to the target, or to copy the fail value (operand 2).
+(define_expand "despeculate_copy<ALLI_TI:mode>"
+ [(set (match_operand:ALLI_TI 0 "register_operand" "=r")
+ (unspec_volatile:ALLI_TI
+ [(match_operand:ALLI_TI 1 "register_operand" "r")
+ (match_operand:ALLI_TI 2 "aarch64_reg_or_zero" "rZ")
+ (use (reg:DI SPECULATION_TRACKER_REGNUM))
+ (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
+ ""
+ "
+ {
+ if (operands[2] == const0_rtx)
+ {
+ rtx tracker;
+ if (<MODE>mode == TImode)
+ tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
+ else
+ tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
+
+ emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
+ tracker));
+ DONE;
+ }
+ }
+ "
+)
+
+;; Pattern to match despeculate_copy<mode>
+(define_insn "*despeculate_copy<ALLI:mode>_insn"
+ [(set (match_operand:ALLI 0 "register_operand" "=r")
+ (unspec_volatile:ALLI
+ [(match_operand:ALLI 1 "register_operand" "r")
+ (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
+ (use (reg:DI SPECULATION_TRACKER_REGNUM))
+ (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
+ ""
+ {
+ operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
+ output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;csdb",
+ operands);
+ return "";
+ }
+ [(set_attr "length" "12")
+ (set_attr "type" "block")
+ (set_attr "speculation_barrier" "true")]
+)
+
+;; Pattern to match despeculate_copyti
+(define_insn "*despeculate_copyti_insn"
+ [(set (match_operand:TI 0 "register_operand" "=r")
+ (unspec_volatile:TI
+ [(match_operand:TI 1 "register_operand" "r")
+ (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
+ (use (reg:DI SPECULATION_TRACKER_REGNUM))
+ (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
+ ""
+ {
+ operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
+ output_asm_insn
+ ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%R0, %R1, %R2, ne\;csdb",
+ operands);
+ return "";
+ }
+ [(set_attr "length" "16")
+ (set_attr "type" "block")
+ (set_attr "speculation_barrier" "true")]
+)
+
+(define_insn "despeculate_simple<ALLI:mode>"
+ [(set (match_operand:ALLI 0 "register_operand" "=r")
+ (unspec_volatile:ALLI
+ [(match_operand:ALLI 1 "register_operand" "r")
+ (use (match_operand:ALLI 2 "register_operand" ""))]
+ UNSPECV_SPECULATION_BARRIER))]
+ ""
+ "and\\t%<w>0, %<w>1, %<w>2\;csdb"
+ [(set_attr "type" "block")
+ (set_attr "length" "8")
+ (set_attr "speculation_barrier" "true")]
+)
+
+(define_insn "despeculate_simpleti"
+ [(set (match_operand:TI 0 "register_operand" "=r")
+ (unspec_volatile:TI
+ [(match_operand:TI 1 "register_operand" "r")
+ (use (match_operand:DI 2 "register_operand" ""))]
+ UNSPECV_SPECULATION_BARRIER))]
+ ""
+ "and\\t%0, %1, %2\;and\\t%R0, %R1, %2\;csdb"
+ [(set_attr "type" "block")
+ (set_attr "length" "12")
+ (set_attr "speculation_barrier" "true")]
+)
+
;; AdvSIMD Stuff
(include "aarch64-simd.md")
@@ -35,6 +35,9 @@ (define_mode_iterator SHORT [QI HI])
;; Iterator for all integer modes (up to 64-bit)
(define_mode_iterator ALLI [QI HI SI DI])
+;; Iterator for all integer modes (up to 128-bit)
+(define_mode_iterator ALLI_TI [QI HI SI DI TI])
+
;; Iterator for all integer modes that can be extended (up to 64-bit)
(define_mode_iterator ALLX [QI HI SI])