diff --git a/arch/arm64/include/asm/Kbuild b/arch/arm64/include/asm/Kbuild
index 44e1d7f10add..39ca0409e157 100644
--- a/arch/arm64/include/asm/Kbuild
+++ b/arch/arm64/include/asm/Kbuild
@@ -33,7 +33,6 @@ generic-y += segment.h
 generic-y += sembuf.h
 generic-y += serial.h
 generic-y += shmbuf.h
-generic-y += simd.h
 generic-y += sizes.h
 generic-y += socket.h
 generic-y += sockios.h
diff --git a/arch/arm64/include/asm/simd.h b/arch/arm64/include/asm/simd.h
new file mode 100644
index 000000000000..142eca29125a
--- /dev/null
+++ b/arch/arm64/include/asm/simd.h
@@ -0,0 +1,16 @@
+
+#include <linux/hardirq.h>
+
+/*
+ * may_use_simd - whether it is allowable at this time to issue SIMD
+ *                instructions or access the SIMD register file
+ *
+ * On arm64, we allow kernel mode NEON in softirq context but not in hardirq
+ * context, due to the fact that the NEON register file may be shared with SVE,
+ * whose state may too large to preserve/restore efficiently at each invocation
+ * of kernel_neon_begin()/_end() in hardirq context.
+ */
+static __must_check inline bool may_use_simd(void)
+{
+	return !in_irq();
+}
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 394c61db5566..97344c94acae 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -127,6 +127,8 @@ void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
 
 void fpsimd_thread_switch(struct task_struct *next)
 {
+	BUG_ON(!irqs_disabled());
+
 	/*
 	 * Save the current FPSIMD state to memory, but only if whatever is in
 	 * the registers is in fact the most recent userland FPSIMD state of
@@ -169,8 +171,10 @@ void fpsimd_flush_thread(void)
 void fpsimd_preserve_current_state(void)
 {
 	preempt_disable();
+	local_bh_disable();
 	if (!test_thread_flag(TIF_FOREIGN_FPSTATE))
 		fpsimd_save_state(&current->thread.fpsimd_state);
+	local_bh_enable();
 	preempt_enable();
 }
 
@@ -182,6 +186,7 @@ void fpsimd_preserve_current_state(void)
 void fpsimd_restore_current_state(void)
 {
 	preempt_disable();
+	local_bh_disable();
 	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
 		struct fpsimd_state *st = &current->thread.fpsimd_state;
 
@@ -189,6 +194,7 @@ void fpsimd_restore_current_state(void)
 		this_cpu_write(fpsimd_last_state, st);
 		st->cpu = smp_processor_id();
 	}
+	local_bh_enable();
 	preempt_enable();
 }
 
@@ -200,6 +206,7 @@ void fpsimd_restore_current_state(void)
 void fpsimd_update_current_state(struct fpsimd_state *state)
 {
 	preempt_disable();
+	local_bh_disable();
 	fpsimd_load_state(state);
 	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
 		struct fpsimd_state *st = &current->thread.fpsimd_state;
@@ -207,6 +214,7 @@ void fpsimd_update_current_state(struct fpsimd_state *state)
 		this_cpu_write(fpsimd_last_state, st);
 		st->cpu = smp_processor_id();
 	}
+	local_bh_enable();
 	preempt_enable();
 }
 
@@ -220,45 +228,68 @@ void fpsimd_flush_task_state(struct task_struct *t)
 
 #ifdef CONFIG_KERNEL_MODE_NEON
 
-static DEFINE_PER_CPU(struct fpsimd_partial_state, hardirq_fpsimdstate);
-static DEFINE_PER_CPU(struct fpsimd_partial_state, softirq_fpsimdstate);
+static DEFINE_PER_CPU(struct fpsimd_partial_state, nested_fpsimdstate);
+static DEFINE_PER_CPU(int, kernel_neon_nesting_level);
 
 /*
  * Kernel-side NEON support functions
  */
 void kernel_neon_begin_partial(u32 num_regs)
 {
-	if (in_interrupt()) {
-		struct fpsimd_partial_state *s = this_cpu_ptr(
-			in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate);
+	int level;
 
-		BUG_ON(num_regs > 32);
-		fpsimd_save_partial_state(s, roundup(num_regs, 2));
-	} else {
+	/*
+	 * We don't allow kernel mode NEON in hard IRQ context because we'd
+	 * have to assume that any sequence involving preserve/restore of the
+	 * FP/SIMD register file could be interrupted by nested use of the NEON.
+	 *
+	 * On SVE capable hardware, that would necessitate executing all
+	 * manipulation of the preserved FP/SIMD state with interrupts disabled,
+	 * unless we preserve/restore the *entire* SVE state in interrupt
+	 * context as well.
+	 */
+	BUG_ON(in_irq());
+
+	preempt_disable();
+	if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE)) {
 		/*
 		 * Save the userland FPSIMD state if we have one and if we
 		 * haven't done so already. Clear fpsimd_last_state to indicate
 		 * that there is no longer userland FPSIMD state in the
 		 * registers.
 		 */
-		preempt_disable();
-		if (current->mm &&
-		    !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE))
+		local_bh_disable();
+		if (!test_and_set_thread_flag(TIF_FOREIGN_FPSTATE))
 			fpsimd_save_state(&current->thread.fpsimd_state);
-		this_cpu_write(fpsimd_last_state, NULL);
+		local_bh_enable();
+	}
+	this_cpu_write(fpsimd_last_state, NULL);
+
+	level = this_cpu_inc_return(kernel_neon_nesting_level);
+	BUG_ON(level > 2);
+
+	if (level > 1) {
+		WARN_ON_ONCE(num_regs > 32);
+		num_regs = max(roundup(num_regs, 2), 32U);
+
+		fpsimd_save_partial_state(this_cpu_ptr(&nested_fpsimdstate),
+					  num_regs);
 	}
 }
 EXPORT_SYMBOL(kernel_neon_begin_partial);
 
 void kernel_neon_end(void)
 {
-	if (in_interrupt()) {
-		struct fpsimd_partial_state *s = this_cpu_ptr(
-			in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate);
-		fpsimd_load_partial_state(s);
-	} else {
-		preempt_enable();
-	}
+	int level;
+
+	level = this_cpu_read(kernel_neon_nesting_level);
+	BUG_ON(level < 1);
+
+	if (level > 1)
+		fpsimd_load_partial_state(this_cpu_ptr(&nested_fpsimdstate));
+
+	this_cpu_dec(kernel_neon_nesting_level);
+	preempt_enable();
 }
 EXPORT_SYMBOL(kernel_neon_end);
 
@@ -270,8 +301,12 @@ static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
 {
 	switch (cmd) {
 	case CPU_PM_ENTER:
-		if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE))
-			fpsimd_save_state(&current->thread.fpsimd_state);
+		if (current->mm) {
+			local_bh_disable();
+			if (!test_thread_flag(TIF_FOREIGN_FPSTATE))
+				fpsimd_save_state(&current->thread.fpsimd_state);
+			local_bh_enable();
+		}
 		this_cpu_write(fpsimd_last_state, NULL);
 		break;
 	case CPU_PM_EXIT:
