iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

arm64: Add percpu vectors for EL1

Browse Source 
commit bd09128d16fac3c34b80bd6a29088ac632e8ce09 upstream.

The Spectre-BHB workaround adds a firmware call to the vectors. This
is needed on some CPUs, but not others. To avoid the unaffected CPU in
a big/little pair from making the firmware call, create per cpu vectors.

The per-cpu vectors only apply when returning from EL0.

Systems using KPTI can use the canonical 'full-fat' vectors directly at
EL1, the trampoline exit code will switch to this_cpu_vector on exit to
EL0. Systems not using KPTI should always use this_cpu_vector.

this_cpu_vector will point at a vector in tramp_vecs or
__bp_harden_el1_vectors, depending on whether KPTI is in use.

Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: James Morse <james.morse@arm.com>
Signed-off-by: Sasha Levin <sashal@kernel.org>
This commit is contained in:
James Morse 2022-03-15 18:24:10 +00:00 committed by Greg Kroah-Hartman
parent 1bb1944970
commit c45d885c5a
5 changed files with 55 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
arch/arm64/include/asm/mmu.h
View File

@ -29,7 +29,7 @@ typedef struct {
*/
#define ASID(mm) ((mm)->context.id.counter & 0xffff)
static inline bool arm64_kernel_unmapped_at_el0(void)
static __always_inline bool arm64_kernel_unmapped_at_el0(void)
{
return IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0) &&
cpus_have_const_cap(ARM64_UNMAP_KERNEL_AT_EL0);

27
arch/arm64/include/asm/vectors.h
View File

@ -5,6 +5,15 @@
#ifndef __ASM_VECTORS_H
#define __ASM_VECTORS_H
#include <linux/bug.h>
#include <linux/percpu.h>
#include <asm/fixmap.h>
extern char vectors[];
extern char tramp_vectors[];
extern char __bp_harden_el1_vectors[];
/*
* Note: the order of this enum corresponds to two arrays in entry.S:
* tramp_vecs and __bp_harden_el1_vectors. By default the canonical
@ -31,4 +40,22 @@ enum arm64_bp_harden_el1_vectors {
EL1_VECTOR_KPTI,
};
/* The vectors to use on return from EL0. e.g. to remap the kernel */
DECLARE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector);
#ifndef CONFIG_UNMAP_KERNEL_AT_EL0
#define TRAMP_VALIAS 0
#endif
static inline const char *
arm64_get_bp_hardening_vector(enum arm64_bp_harden_el1_vectors slot)
{
if (arm64_kernel_unmapped_at_el0())
return (char *)TRAMP_VALIAS + SZ_2K * slot;
WARN_ON_ONCE(slot == EL1_VECTOR_KPTI);
return __bp_harden_el1_vectors + SZ_2K * slot;
}
#endif /* __ASM_VECTORS_H */

11
arch/arm64/kernel/cpufeature.c
View File

@ -10,11 +10,13 @@
#include <linux/bsearch.h>
#include <linux/cpumask.h>
#include <linux/crash_dump.h>
#include <linux/percpu.h>
#include <linux/sort.h>
#include <linux/stop_machine.h>
#include <linux/types.h>
#include <linux/mm.h>
#include <linux/cpu.h>
#include <asm/cpu.h>
#include <asm/cpufeature.h>
#include <asm/cpu_ops.h>
@ -23,6 +25,7 @@
#include <asm/processor.h>
#include <asm/sysreg.h>
#include <asm/traps.h>
#include <asm/vectors.h>
#include <asm/virt.h>
/* Kernel representation of AT_HWCAP and AT_HWCAP2 */
@ -45,6 +48,8 @@ static struct arm64_cpu_capabilities const __ro_after_init *cpu_hwcaps_ptrs[ARM6
/* Need also bit for ARM64_CB_PATCH */
DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE);
DEFINE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector) = vectors;
/*
* Flag to indicate if we have computed the system wide
* capabilities based on the boot time active CPUs. This
@ -1047,6 +1052,12 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
static bool kpti_applied = false;
int cpu = smp_processor_id();
if (__this_cpu_read(this_cpu_vector) == vectors) {
const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI);
__this_cpu_write(this_cpu_vector, v);
}
/*
* We don't need to rewrite the page-tables if either we've done
* it already or we have KASLR enabled and therefore have not

16
arch/arm64/kernel/entry.S
View File

@ -60,7 +60,6 @@
.macro kernel_ventry, el, label, regsize = 64
.align 7
.Lventry_start\@:
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
.if \el == 0
/*
* This must be the first instruction of the EL0 vector entries. It is
@ -75,7 +74,6 @@
.endif
.Lskip_tramp_vectors_cleanup\@:
.endif
#endif
sub sp, sp, #S_FRAME_SIZE
#ifdef CONFIG_VMAP_STACK
@ -1129,10 +1127,14 @@ alternative_else_nop_endif
.endm
.macro tramp_exit, regsize = 64
adr x30, tramp_vectors
#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
add x30, x30, SZ_4K
#endif
tramp_data_read_var x30, this_cpu_vector
alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
mrs x29, tpidr_el1
alternative_else
mrs x29, tpidr_el2
alternative_endif
ldr x30, [x30, x29]
msr vbar_el1, x30
ldr lr, [sp, #S_LR]
tramp_unmap_kernel x29
@ -1193,6 +1195,8 @@ __entry_tramp_data_vectors:
__entry_tramp_data___sdei_asm_trampoline_next_handler:
.quad __sdei_asm_handler
#endif /* CONFIG_ARM_SDE_INTERFACE */
__entry_tramp_data_this_cpu_vector:
.quad this_cpu_vector
.popsection // .rodata
#endif /* CONFIG_RANDOMIZE_BASE */
#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */

8
arch/arm64/kvm/hyp/switch.c
View File

@ -25,6 +25,7 @@
#include <asm/debug-monitors.h>
#include <asm/processor.h>
#include <asm/thread_info.h>
#include <asm/vectors.h>
extern struct exception_table_entry __start___kvm_ex_table;
extern struct exception_table_entry __stop___kvm_ex_table;
@ -152,7 +153,7 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
static void deactivate_traps_vhe(void)
{
extern char vectors[]; /* kernel exception vectors */
const char *host_vectors = vectors;
write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
/*
@ -163,7 +164,10 @@ static void deactivate_traps_vhe(void)
asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_1165522));
write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1);
write_sysreg(vectors, vbar_el1);
if (!arm64_kernel_unmapped_at_el0())
host_vectors = __this_cpu_read(this_cpu_vector);
write_sysreg(host_vectors, vbar_el1);
}
NOKPROBE_SYMBOL(deactivate_traps_vhe);