bpf: Support inlining bpf_kptr_xchg() helper
The motivation of inlining bpf_kptr_xchg() comes from the performance profiling of bpf memory allocator benchmark. The benchmark uses bpf_kptr_xchg() to stash the allocated objects and to pop the stashed objects for free. After inling bpf_kptr_xchg(), the performance for object free on 8-CPUs VM increases about 2%~10%. The inline also has downside: both the kasan and kcsan checks on the pointer will be unavailable. bpf_kptr_xchg() can be inlined by converting the calling of bpf_kptr_xchg() into an atomic_xchg() instruction. But the conversion depends on two conditions: 1) JIT backend supports atomic_xchg() on pointer-sized word 2) For the specific arch, the implementation of xchg is the same as atomic_xchg() on pointer-sized words. It seems most 64-bit JIT backends satisfies these two conditions. But as a precaution, defining a weak function bpf_jit_supports_ptr_xchg() to state whether such conversion is safe and only supporting inline for 64-bit host. For x86-64, it supports BPF_XCHG atomic operation and both xchg() and atomic_xchg() use arch_xchg() to implement the exchange, so enabling the inline of bpf_kptr_xchg() on x86-64 first. Reviewed-by: Eduard Zingerman <eddyz87@gmail.com> Signed-off-by: Hou Tao <houtao1@huawei.com> Link: https://lore.kernel.org/r/20240105104819.3916743-2-houtao@huaweicloud.com Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
parent
2121c43f88
commit
7c05e7f3e7
@ -3242,3 +3242,8 @@ void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,
|
||||
BUG_ON(ret < 0);
|
||||
}
|
||||
}
|
||||
|
||||
bool bpf_jit_supports_ptr_xchg(void)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
@ -955,6 +955,7 @@ bool bpf_jit_supports_subprog_tailcalls(void);
|
||||
bool bpf_jit_supports_kfunc_call(void);
|
||||
bool bpf_jit_supports_far_kfunc_call(void);
|
||||
bool bpf_jit_supports_exceptions(void);
|
||||
bool bpf_jit_supports_ptr_xchg(void);
|
||||
void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);
|
||||
bool bpf_helper_changes_pkt_data(void *func);
|
||||
|
||||
|
@ -2925,6 +2925,16 @@ bool __weak bpf_jit_supports_far_kfunc_call(void)
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Return TRUE if the JIT backend satisfies the following two conditions:
|
||||
* 1) JIT backend supports atomic_xchg() on pointer-sized words.
|
||||
* 2) Under the specific arch, the implementation of xchg() is the same
|
||||
* as atomic_xchg() on pointer-sized words.
|
||||
*/
|
||||
bool __weak bpf_jit_supports_ptr_xchg(void)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
/* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call
|
||||
* skb_copy_bits(), so provide a weak definition of it for NET-less config.
|
||||
*/
|
||||
|
@ -1414,6 +1414,7 @@ BPF_CALL_2(bpf_kptr_xchg, void *, map_value, void *, ptr)
|
||||
{
|
||||
unsigned long *kptr = map_value;
|
||||
|
||||
/* This helper may be inlined by verifier. */
|
||||
return xchg(kptr, (unsigned long)ptr);
|
||||
}
|
||||
|
||||
|
@ -19809,6 +19809,23 @@ patch_map_ops_generic:
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Implement bpf_kptr_xchg inline */
|
||||
if (prog->jit_requested && BITS_PER_LONG == 64 &&
|
||||
insn->imm == BPF_FUNC_kptr_xchg &&
|
||||
bpf_jit_supports_ptr_xchg()) {
|
||||
insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2);
|
||||
insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0);
|
||||
cnt = 2;
|
||||
|
||||
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
|
||||
if (!new_prog)
|
||||
return -ENOMEM;
|
||||
|
||||
delta += cnt - 1;
|
||||
env->prog = prog = new_prog;
|
||||
insn = new_prog->insnsi + i + delta;
|
||||
continue;
|
||||
}
|
||||
patch_call_imm:
|
||||
fn = env->ops->get_func_proto(insn->imm, env->prog);
|
||||
/* all functions that have prototype and verifier allowed
|
||||
|
Loading…
Reference in New Issue
Block a user