- Allow CPUs affected by erratum 1418040 to come online late (previously
we only fixed the other case - CPUs not affected by the erratum coming up late). - Fix branch offset in BPF JIT. - Defer the stolen time initialisation to the CPU online time from the CPU starting time to avoid a (sleep-able) memory allocation in an atomic context. -----BEGIN PGP SIGNATURE----- iQIzBAABCgAdFiEE5RElWfyWxS+3PLO2a9axLQDIXvEFAl9k278ACgkQa9axLQDI XvHBsQ//Y5QpZifzLKM4b2Zcs2/lk67UPd/8rt3p8+WP+T+lQwIxWUxhQmctiYGI iNkLsCVWD3S27eCHWsAKK9h7tlaqnbmQdQuJbFMUF8IjQhx+D+vt14CkYzDnajcS Ob4uqZKCN2WAkMOaL9SEFEXWZh0v5n83PjlorrIz0MzZlNHbAxR53TXHoPcM6rnF xfVwYuk5+FaHKa9YfAIUAtaV5ncTyF6YIk5wqXW/hH1jvfjdF3CpQEog3Oamg8Ik BcH1MKzc2/7k/E916ppPke90ys6w7HdLGm7NaR5cbDwAStxYDqASiAplgsms28HJ 8BbC0RY3FGCRnq40z/2DBBRGy2uP4iifw6gXWyUZnViO+ulbWRmcqvlpii43JDm4 2tLqq225gBgsbKaiLAbnnVpv+VAkodG0MDUy5J4BLK+dxv3Z3UAj0qo3q9+4x/87 Q07GTsAa/e0m0tPjVSwtmikQuJJc2uu7jk7loOLNsO8NfrU2/SJF8mEf0e6IP0JN ZceoReLvWArERkSQqjk4jSijqo+SfUTNQenjt+F5hciBB3dD0fC3Rgslbm96cmUw oSAvrIfKpNWbdZ3LUAQlPItkDKXb5aXkh1WIQ4gvt7OoCBVVCpwLLjqshZ39TaHA EafBjNqv4FvC030VOA3o/DibNJUY1baOOE8Cj1C9/ghA4p1pX1w= =dCba -----END PGP SIGNATURE----- Merge tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux Pull arm64 fixes from Catalin Marinas: - Allow CPUs affected by erratum1418040to come online late (previously we only fixed the other case - CPUs not affected by the erratum coming up late). - Fix branch offset in BPF JIT. - Defer the stolen time initialisation to the CPU online time from the CPU starting time to avoid a (sleep-able) memory allocation in an atomic context. * tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: arm64: paravirt: Initialize steal time when cpu is online arm64: bpf: Fix branch offset in JIT arm64: Allow CPUs unffected by ARM erratum1418040to come in late
This commit is contained in:
Linus Torvalds
commit
69828c475d
@ -910,8 +910,12 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
|
||||
.desc = "ARM erratum 1418040",
|
||||
.capability = ARM64_WORKAROUND_1418040,
|
||||
ERRATA_MIDR_RANGE_LIST(erratum_1418040_list),
|
||||
.type = (ARM64_CPUCAP_SCOPE_LOCAL_CPU |
|
||||
ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU),
|
||||
/*
|
||||
* We need to allow affected CPUs to come in late, but
|
||||
* also need the non-affected CPUs to be able to come
|
||||
* in at any point in time. Wonderful.
|
||||
*/
|
||||
.type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
|
||||
},
|
||||
#endif
|
||||
#ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_AT
|
||||
|
||||
@ -50,16 +50,19 @@ static u64 pv_steal_clock(int cpu)
|
||||
struct pv_time_stolen_time_region *reg;
|
||||
|
||||
reg = per_cpu_ptr(&stolen_time_region, cpu);
|
||||
if (!reg->kaddr) {
|
||||
pr_warn_once("stolen time enabled but not configured for cpu %d\n",
|
||||
cpu);
|
||||
|
||||
/*
|
||||
* paravirt_steal_clock() may be called before the CPU
|
||||
* online notification callback runs. Until the callback
|
||||
* has run we just return zero.
|
||||
*/
|
||||
if (!reg->kaddr)
|
||||
return 0;
|
||||
}
|
||||
|
||||
return le64_to_cpu(READ_ONCE(reg->kaddr->stolen_time));
|
||||
}
|
||||
|
||||
static int stolen_time_dying_cpu(unsigned int cpu)
|
||||
static int stolen_time_cpu_down_prepare(unsigned int cpu)
|
||||
{
|
||||
struct pv_time_stolen_time_region *reg;
|
||||
|
||||
@ -73,7 +76,7 @@ static int stolen_time_dying_cpu(unsigned int cpu)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int init_stolen_time_cpu(unsigned int cpu)
|
||||
static int stolen_time_cpu_online(unsigned int cpu)
|
||||
{
|
||||
struct pv_time_stolen_time_region *reg;
|
||||
struct arm_smccc_res res;
|
||||
@ -103,19 +106,20 @@ static int init_stolen_time_cpu(unsigned int cpu)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int pv_time_init_stolen_time(void)
|
||||
static int __init pv_time_init_stolen_time(void)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = cpuhp_setup_state(CPUHP_AP_ARM_KVMPV_STARTING,
|
||||
"hypervisor/arm/pvtime:starting",
|
||||
init_stolen_time_cpu, stolen_time_dying_cpu);
|
||||
ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
|
||||
"hypervisor/arm/pvtime:online",
|
||||
stolen_time_cpu_online,
|
||||
stolen_time_cpu_down_prepare);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool has_pv_steal_clock(void)
|
||||
static bool __init has_pv_steal_clock(void)
|
||||
{
|
||||
struct arm_smccc_res res;
|
||||
|
||||
|
||||
@ -143,14 +143,17 @@ static inline void emit_addr_mov_i64(const int reg, const u64 val,
|
||||
}
|
||||
}
|
||||
|
||||
static inline int bpf2a64_offset(int bpf_to, int bpf_from,
|
||||
static inline int bpf2a64_offset(int bpf_insn, int off,
|
||||
const struct jit_ctx *ctx)
|
||||
{
|
||||
int to = ctx->offset[bpf_to];
|
||||
/* -1 to account for the Branch instruction */
|
||||
int from = ctx->offset[bpf_from] - 1;
|
||||
|
||||
return to - from;
|
||||
/* BPF JMP offset is relative to the next instruction */
|
||||
bpf_insn++;
|
||||
/*
|
||||
* Whereas arm64 branch instructions encode the offset
|
||||
* from the branch itself, so we must subtract 1 from the
|
||||
* instruction offset.
|
||||
*/
|
||||
return ctx->offset[bpf_insn + off] - (ctx->offset[bpf_insn] - 1);
|
||||
}
|
||||
|
||||
static void jit_fill_hole(void *area, unsigned int size)
|
||||
@ -642,7 +645,7 @@ emit_bswap_uxt:
|
||||
|
||||
/* JUMP off */
|
||||
case BPF_JMP | BPF_JA:
|
||||
jmp_offset = bpf2a64_offset(i + off, i, ctx);
|
||||
jmp_offset = bpf2a64_offset(i, off, ctx);
|
||||
check_imm26(jmp_offset);
|
||||
emit(A64_B(jmp_offset), ctx);
|
||||
break;
|
||||
@ -669,7 +672,7 @@ emit_bswap_uxt:
|
||||
case BPF_JMP32 | BPF_JSLE | BPF_X:
|
||||
emit(A64_CMP(is64, dst, src), ctx);
|
||||
emit_cond_jmp:
|
||||
jmp_offset = bpf2a64_offset(i + off, i, ctx);
|
||||
jmp_offset = bpf2a64_offset(i, off, ctx);
|
||||
check_imm19(jmp_offset);
|
||||
switch (BPF_OP(code)) {
|
||||
case BPF_JEQ:
|
||||
@ -908,10 +911,21 @@ static int build_body(struct jit_ctx *ctx, bool extra_pass)
|
||||
const struct bpf_prog *prog = ctx->prog;
|
||||
int i;
|
||||
|
||||
/*
|
||||
* - offset[0] offset of the end of prologue,
|
||||
* start of the 1st instruction.
|
||||
* - offset[1] - offset of the end of 1st instruction,
|
||||
* start of the 2nd instruction
|
||||
* [....]
|
||||
* - offset[3] - offset of the end of 3rd instruction,
|
||||
* start of 4th instruction
|
||||
*/
|
||||
for (i = 0; i < prog->len; i++) {
|
||||
const struct bpf_insn *insn = &prog->insnsi[i];
|
||||
int ret;
|
||||
|
||||
if (ctx->image == NULL)
|
||||
ctx->offset[i] = ctx->idx;
|
||||
ret = build_insn(insn, ctx, extra_pass);
|
||||
if (ret > 0) {
|
||||
i++;
|
||||
@ -919,11 +933,16 @@ static int build_body(struct jit_ctx *ctx, bool extra_pass)
|
||||
ctx->offset[i] = ctx->idx;
|
||||
continue;
|
||||
}
|
||||
if (ctx->image == NULL)
|
||||
ctx->offset[i] = ctx->idx;
|
||||
if (ret)
|
||||
return ret;
|
||||
}
|
||||
/*
|
||||
* offset is allocated with prog->len + 1 so fill in
|
||||
* the last element with the offset after the last
|
||||
* instruction (end of program)
|
||||
*/
|
||||
if (ctx->image == NULL)
|
||||
ctx->offset[i] = ctx->idx;
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -1002,7 +1021,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
|
||||
memset(&ctx, 0, sizeof(ctx));
|
||||
ctx.prog = prog;
|
||||
|
||||
ctx.offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
|
||||
ctx.offset = kcalloc(prog->len + 1, sizeof(int), GFP_KERNEL);
|
||||
if (ctx.offset == NULL) {
|
||||
prog = orig_prog;
|
||||
goto out_off;
|
||||
@ -1089,7 +1108,7 @@ skip_init_ctx:
|
||||
prog->jited_len = prog_size;
|
||||
|
||||
if (!prog->is_func || extra_pass) {
|
||||
bpf_prog_fill_jited_linfo(prog, ctx.offset);
|
||||
bpf_prog_fill_jited_linfo(prog, ctx.offset + 1);
|
||||
out_off:
|
||||
kfree(ctx.offset);
|
||||
kfree(jit_data);
|
||||
|
||||
@ -142,7 +142,6 @@ enum cpuhp_state {
|
||||
/* Must be the last timer callback */
|
||||
CPUHP_AP_DUMMY_TIMER_STARTING,
|
||||
CPUHP_AP_ARM_XEN_STARTING,
|
||||
CPUHP_AP_ARM_KVMPV_STARTING,
|
||||
CPUHP_AP_ARM_CORESIGHT_STARTING,
|
||||
CPUHP_AP_ARM_CORESIGHT_CTI_STARTING,
|
||||
CPUHP_AP_ARM64_ISNDEP_STARTING,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user