af64543977
In arm64_apply_bp_hardening() we use cpus_have_const_cap() to check for ARM64_SPECTRE_V2 , but this is not necessary and alternative_has_cap_*() would be preferable. For historical reasons, cpus_have_const_cap() is more complicated than it needs to be. Before cpucaps are finalized, it will perform a bitmap test of the system_cpucaps bitmap, and once cpucaps are finalized it will use an alternative branch. This used to be necessary to handle some race conditions in the window between cpucap detection and the subsequent patching of alternatives and static branches, where different branches could be out-of-sync with one another (or w.r.t. alternative sequences). Now that we use alternative branches instead of static branches, these are all patched atomically w.r.t. one another, and there are only a handful of cases that need special care in the window between cpucap detection and alternative patching. Due to the above, it would be nice to remove cpus_have_const_cap(), and migrate callers over to alternative_has_cap_*(), cpus_have_final_cap(), or cpus_have_cap() depending on when their requirements. This will remove redundant instructions and improve code generation, and will make it easier to determine how each callsite will behave before, during, and after alternative patching. The cpus_have_const_cap() check in arm64_apply_bp_hardening() is intended to avoid the overhead of looking up and invoking a per-cpu function pointer when no branch predictor hardening is required. The arm64_apply_bp_hardening() function itself is called in two distinct flows: 1) When handling certain exceptions taken from EL0, where the PC could be a TTBR1 address and hence might have trained a branch predictor. As cpucaps are detected and alternatives are patched long before it is possible to execute userspace, it is not necessary to use cpus_have_const_cap() for these cases, and cpus_have_final_cap() or alternative_has_cap() would be preferable. 2) When switching between tasks in check_and_switch_context(). This can be called before cpucaps are detected and alternatives are patched, but this is long before the kernel mounts filesystems or accepts any input. At this stage the kernel hasn't loaded any secrets and there is no potential for hostile branch predictor training. Once cpucaps have been finalized and alternatives have been patched, switching tasks will invalidate any prior predictions. Hence it is not necessary to use cpus_have_const_cap() for this case. This patch replaces the use of cpus_have_const_cap() with alternative_has_cap_unlikely(), which will avoid generating code to test the system_cpucaps bitmap and should be better for all subsequent calls at runtime. Signed-off-by: Mark Rutland <mark.rutland@arm.com> Cc: Suzuki K Poulose <suzuki.poulose@arm.com> Cc: Will Deacon <will@kernel.org> Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
121 lines
3.7 KiB
C
121 lines
3.7 KiB
C
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* Interface for managing mitigations for Spectre vulnerabilities.
|
|
*
|
|
* Copyright (C) 2020 Google LLC
|
|
* Author: Will Deacon <will@kernel.org>
|
|
*/
|
|
|
|
#ifndef __ASM_SPECTRE_H
|
|
#define __ASM_SPECTRE_H
|
|
|
|
#define BP_HARDEN_EL2_SLOTS 4
|
|
#define __BP_HARDEN_HYP_VECS_SZ ((BP_HARDEN_EL2_SLOTS - 1) * SZ_2K)
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
#include <linux/percpu.h>
|
|
|
|
#include <asm/cpufeature.h>
|
|
#include <asm/virt.h>
|
|
|
|
/* Watch out, ordering is important here. */
|
|
enum mitigation_state {
|
|
SPECTRE_UNAFFECTED,
|
|
SPECTRE_MITIGATED,
|
|
SPECTRE_VULNERABLE,
|
|
};
|
|
|
|
struct pt_regs;
|
|
struct task_struct;
|
|
|
|
/*
|
|
* Note: the order of this enum corresponds to __bp_harden_hyp_vecs and
|
|
* we rely on having the direct vectors first.
|
|
*/
|
|
enum arm64_hyp_spectre_vector {
|
|
/*
|
|
* Take exceptions directly to __kvm_hyp_vector. This must be
|
|
* 0 so that it used by default when mitigations are not needed.
|
|
*/
|
|
HYP_VECTOR_DIRECT,
|
|
|
|
/*
|
|
* Bounce via a slot in the hypervisor text mapping of
|
|
* __bp_harden_hyp_vecs, which contains an SMC call.
|
|
*/
|
|
HYP_VECTOR_SPECTRE_DIRECT,
|
|
|
|
/*
|
|
* Bounce via a slot in a special mapping of __bp_harden_hyp_vecs
|
|
* next to the idmap page.
|
|
*/
|
|
HYP_VECTOR_INDIRECT,
|
|
|
|
/*
|
|
* Bounce via a slot in a special mapping of __bp_harden_hyp_vecs
|
|
* next to the idmap page, which contains an SMC call.
|
|
*/
|
|
HYP_VECTOR_SPECTRE_INDIRECT,
|
|
};
|
|
|
|
typedef void (*bp_hardening_cb_t)(void);
|
|
|
|
struct bp_hardening_data {
|
|
enum arm64_hyp_spectre_vector slot;
|
|
bp_hardening_cb_t fn;
|
|
};
|
|
|
|
DECLARE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
|
|
|
|
/* Called during entry so must be __always_inline */
|
|
static __always_inline void arm64_apply_bp_hardening(void)
|
|
{
|
|
struct bp_hardening_data *d;
|
|
|
|
if (!alternative_has_cap_unlikely(ARM64_SPECTRE_V2))
|
|
return;
|
|
|
|
d = this_cpu_ptr(&bp_hardening_data);
|
|
if (d->fn)
|
|
d->fn();
|
|
}
|
|
|
|
enum mitigation_state arm64_get_spectre_v2_state(void);
|
|
bool has_spectre_v2(const struct arm64_cpu_capabilities *cap, int scope);
|
|
void spectre_v2_enable_mitigation(const struct arm64_cpu_capabilities *__unused);
|
|
|
|
bool has_spectre_v3a(const struct arm64_cpu_capabilities *cap, int scope);
|
|
void spectre_v3a_enable_mitigation(const struct arm64_cpu_capabilities *__unused);
|
|
|
|
enum mitigation_state arm64_get_spectre_v4_state(void);
|
|
bool has_spectre_v4(const struct arm64_cpu_capabilities *cap, int scope);
|
|
void spectre_v4_enable_mitigation(const struct arm64_cpu_capabilities *__unused);
|
|
void spectre_v4_enable_task_mitigation(struct task_struct *tsk);
|
|
|
|
enum mitigation_state arm64_get_meltdown_state(void);
|
|
|
|
enum mitigation_state arm64_get_spectre_bhb_state(void);
|
|
bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope);
|
|
u8 spectre_bhb_loop_affected(int scope);
|
|
void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused);
|
|
bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr);
|
|
|
|
void spectre_v4_patch_fw_mitigation_enable(struct alt_instr *alt, __le32 *origptr,
|
|
__le32 *updptr, int nr_inst);
|
|
void smccc_patch_fw_mitigation_conduit(struct alt_instr *alt, __le32 *origptr,
|
|
__le32 *updptr, int nr_inst);
|
|
void spectre_bhb_patch_loop_mitigation_enable(struct alt_instr *alt, __le32 *origptr,
|
|
__le32 *updptr, int nr_inst);
|
|
void spectre_bhb_patch_fw_mitigation_enabled(struct alt_instr *alt, __le32 *origptr,
|
|
__le32 *updptr, int nr_inst);
|
|
void spectre_bhb_patch_loop_iter(struct alt_instr *alt,
|
|
__le32 *origptr, __le32 *updptr, int nr_inst);
|
|
void spectre_bhb_patch_wa3(struct alt_instr *alt,
|
|
__le32 *origptr, __le32 *updptr, int nr_inst);
|
|
void spectre_bhb_patch_clearbhb(struct alt_instr *alt,
|
|
__le32 *origptr, __le32 *updptr, int nr_inst);
|
|
|
|
#endif /* __ASSEMBLY__ */
|
|
#endif /* __ASM_SPECTRE_H */
|