@@ -584,16 +584,15 @@ static __always_inline u64 kvm_get_reset_cptr_el2(struct kvm_vcpu *vcpu)
if (has_vhe()) {
val = (CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN |
- CPACR_EL1_ZEN_EL1EN);
- if (cpus_have_final_cap(ARM64_SME))
- val |= CPACR_EL1_SMEN_EL1EN;
+ CPACR_EL1_ZEN_EL1EN | CPACR_EL1_SMEN_EL1EN);
} else if (has_hvhe()) {
val = (CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN);
if (!vcpu_has_sve(vcpu) ||
(vcpu->arch.fp_state != FP_STATE_GUEST_OWNED))
val |= CPACR_EL1_ZEN_EL1EN | CPACR_EL1_ZEN_EL0EN;
- if (cpus_have_final_cap(ARM64_SME))
+ if (!vcpu_has_sme(vcpu) ||
+ (vcpu->arch.fp_state != FP_STATE_GUEST_OWNED))
val |= CPACR_EL1_SMEN_EL1EN | CPACR_EL1_SMEN_EL0EN;
} else {
val = CPTR_NVHE_EL2_RES1;
@@ -602,8 +601,9 @@ static __always_inline u64 kvm_get_reset_cptr_el2(struct kvm_vcpu *vcpu)
if (vcpu_has_sve(vcpu) &&
(vcpu->arch.fp_state == FP_STATE_GUEST_OWNED))
val |= CPTR_EL2_TZ;
- if (cpus_have_final_cap(ARM64_SME))
- val &= ~CPTR_EL2_TSM;
+ if (vcpu_has_sme(vcpu) &&
+ (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED))
+ val |= CPTR_EL2_TSM;
}
return val;
@@ -206,6 +206,16 @@ static int handle_sve(struct kvm_vcpu *vcpu)
return 1;
}
+/*
+ * Guest access to SME registers should be routed to this handler only
+ * when the system doesn't support SME.
+ */
+static int handle_sme(struct kvm_vcpu *vcpu)
+{
+ kvm_inject_undefined(vcpu);
+ return 1;
+}
+
/*
* Guest usage of a ptrauth instruction (which the guest EL1 did not turn into
* a NOP). If we get here, it is that we didn't fixup ptrauth on exit, and all
@@ -268,6 +278,7 @@ static exit_handle_fn arm_exit_handlers[] = {
[ESR_ELx_EC_SVC64] = handle_svc,
[ESR_ELx_EC_SYS64] = kvm_handle_sys_reg,
[ESR_ELx_EC_SVE] = handle_sve,
+ [ESR_ELx_EC_SME] = handle_sme,
[ESR_ELx_EC_ERET] = kvm_handle_eret,
[ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort,
[ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort,
@@ -411,6 +411,52 @@ static void handle_host_smc(struct kvm_cpu_context *host_ctxt)
kvm_skip_host_instr();
}
+static void handle_host_vec(void)
+{
+ u64 old_smcr, new_smcr;
+ u64 mask = 0;
+
+ /*
+ * Handle lazy restore of the EL2 configuration for host SVE
+ * and SME usage. It is likely that when a host supports both
+ * SVE and SME it will use both in quick succession (eg,
+ * saving guest state) so we restore both when either traps.
+ */
+ if (has_hvhe()) {
+ if (cpus_have_final_cap(ARM64_SVE))
+ mask |= CPACR_EL1_ZEN_EL1EN | CPACR_EL1_ZEN_EL0EN;
+ if (cpus_have_final_cap(ARM64_SME))
+ mask |= CPACR_EL1_SMEN_EL1EN | CPACR_EL1_SMEN_EL0EN;
+
+ sysreg_clear_set(cpacr_el1, 0, mask);
+ } else {
+ if (cpus_have_final_cap(ARM64_SVE))
+ mask |= CPTR_EL2_TZ;
+ if (cpus_have_final_cap(ARM64_SME))
+ mask |= CPTR_EL2_TSM;
+
+ sysreg_clear_set(cptr_el2, mask, 0);
+ }
+
+ isb();
+
+ if (cpus_have_final_cap(ARM64_SVE))
+ sve_cond_update_zcr_vq(ZCR_ELx_LEN_MASK, SYS_ZCR_EL2);
+
+ if (cpus_have_final_cap(ARM64_SME)) {
+ old_smcr = read_sysreg_s(SYS_SMCR_EL2);
+ new_smcr = SMCR_ELx_LEN_MASK;
+
+ if (cpus_have_final_cap(ARM64_SME_FA64))
+ new_smcr |= SMCR_ELx_FA64_MASK;
+ if (cpus_have_final_cap(ARM64_SME2))
+ new_smcr |= SMCR_ELx_EZT0_MASK;
+
+ if (old_smcr != new_smcr)
+ write_sysreg_s(new_smcr, SYS_SMCR_EL2);
+ }
+}
+
void handle_trap(struct kvm_cpu_context *host_ctxt)
{
u64 esr = read_sysreg_el2(SYS_ESR);
@@ -423,14 +469,8 @@ void handle_trap(struct kvm_cpu_context *host_ctxt)
handle_host_smc(host_ctxt);
break;
case ESR_ELx_EC_SVE:
- /* Handle lazy restore of the host VL */
- if (has_hvhe())
- sysreg_clear_set(cpacr_el1, 0, (CPACR_EL1_ZEN_EL1EN |
- CPACR_EL1_ZEN_EL0EN));
- else
- sysreg_clear_set(cptr_el2, CPTR_EL2_TZ, 0);
- isb();
- sve_cond_update_zcr_vq(ZCR_ELx_LEN_MASK, SYS_ZCR_EL2);
+ case ESR_ELx_EC_SME:
+ handle_host_vec();
break;
case ESR_ELx_EC_IABT_LOW:
case ESR_ELx_EC_DABT_LOW:
@@ -46,19 +46,14 @@ static void __activate_traps(struct kvm_vcpu *vcpu)
val = vcpu->arch.cptr_el2;
val |= CPTR_EL2_TAM; /* Same bit irrespective of E2H */
val |= has_hvhe() ? CPACR_EL1_TTA : CPTR_EL2_TTA;
- if (cpus_have_final_cap(ARM64_SME)) {
- if (has_hvhe())
- val &= ~(CPACR_EL1_SMEN_EL1EN | CPACR_EL1_SMEN_EL0EN);
- else
- val |= CPTR_EL2_TSM;
- }
if (!guest_owns_fp_regs(vcpu)) {
if (has_hvhe())
val &= ~(CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN |
- CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN);
+ CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN |
+ CPACR_EL1_SMEN_EL0EN | CPACR_EL1_SMEN_EL1EN);
else
- val |= CPTR_EL2_TFP | CPTR_EL2_TZ;
+ val |= CPTR_EL2_TFP | CPTR_EL2_TZ | CPTR_EL2_TSM;
__activate_traps_fpsimd32(vcpu);
}
@@ -186,6 +181,7 @@ static const exit_handler_fn hyp_exit_handlers[] = {
[0 ... ESR_ELx_EC_MAX] = NULL,
[ESR_ELx_EC_CP15_32] = kvm_hyp_handle_cp15_32,
[ESR_ELx_EC_SYS64] = kvm_hyp_handle_sysreg,
+ [ESR_ELx_EC_SME] = kvm_hyp_handle_fpsimd,
[ESR_ELx_EC_SVE] = kvm_hyp_handle_fpsimd,
[ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd,
[ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low,
@@ -198,6 +194,7 @@ static const exit_handler_fn hyp_exit_handlers[] = {
static const exit_handler_fn pvm_exit_handlers[] = {
[0 ... ESR_ELx_EC_MAX] = NULL,
[ESR_ELx_EC_SYS64] = kvm_handle_pvm_sys64,
+ [ESR_ELx_EC_SME] = kvm_handle_pvm_restricted,
[ESR_ELx_EC_SVE] = kvm_handle_pvm_restricted,
[ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd,
[ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low,
@@ -78,6 +78,8 @@ static void __activate_traps(struct kvm_vcpu *vcpu)
if (guest_owns_fp_regs(vcpu)) {
if (vcpu_has_sve(vcpu))
val |= CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN;
+ if (vcpu_has_sme(vcpu))
+ val |= CPACR_EL1_SMEN_EL0EN | CPACR_EL1_SMEN_EL1EN;
} else {
val &= ~(CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN);
__activate_traps_fpsimd32(vcpu);
@@ -177,6 +179,7 @@ static const exit_handler_fn hyp_exit_handlers[] = {
[0 ... ESR_ELx_EC_MAX] = NULL,
[ESR_ELx_EC_CP15_32] = kvm_hyp_handle_cp15_32,
[ESR_ELx_EC_SYS64] = kvm_hyp_handle_sysreg,
+ [ESR_ELx_EC_SME] = kvm_hyp_handle_fpsimd,
[ESR_ELx_EC_SVE] = kvm_hyp_handle_fpsimd,
[ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd,
[ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low,
Now that we have support for managing SME state for KVM guests add handling for SME exceptions generated by guests. As with SVE these are routed to the generic floating point exception handlers for both VHE and nVHE, the floating point state is handled as a uniform block. Since we do not presently support SME for protected VMs handle exceptions from protected guests as UNDEF. For nVHE and hVHE modes we currently do a lazy restore of the host EL2 setup for SVE, do the same for SME. Since it is likely that there will be common situations where SVE and SME are both used in quick succession by the host (eg, saving the guest state) restore the configuration for both at once in order to minimise the number of EL2 entries. Signed-off-by: Mark Brown <broonie@kernel.org> --- arch/arm64/include/asm/kvm_emulate.h | 12 ++++---- arch/arm64/kvm/handle_exit.c | 11 +++++++ arch/arm64/kvm/hyp/nvhe/hyp-main.c | 56 ++++++++++++++++++++++++++++++------ arch/arm64/kvm/hyp/nvhe/switch.c | 13 ++++----- arch/arm64/kvm/hyp/vhe/switch.c | 3 ++ 5 files changed, 73 insertions(+), 22 deletions(-)