bpf: Support inlining bpf_kptr_xchg() helper
The motivation of inlining bpf_kptr_xchg() comes from the performance profiling of bpf memory allocator benchmark. The benchmark uses bpf_kptr_xchg() to stash the allocated objects and to pop the stashed objects for free. After inling bpf_kptr_xchg(), the performance for object free on 8-CPUs VM increases about 2%~10%. The inline also has downside: both the kasan and kcsan checks on the pointer will be unavailable. bpf_kptr_xchg() can be inlined by converting the calling of bpf_kptr_xchg() into an atomic_xchg() instruction. But the conversion depends on two conditions: 1) JIT backend supports atomic_xchg() on pointer-sized word 2) For the specific arch, the implementation of xchg is the same as atomic_xchg() on pointer-sized words. It seems most 64-bit JIT backends satisfies these two conditions. But as a precaution, defining a weak function bpf_jit_supports_ptr_xchg() to state whether such conversion is safe and only supporting inline for 64-bit host. For x86-64, it supports BPF_XCHG atomic operation and both xchg() and atomic_xchg() use arch_xchg() to implement the exchange, so enabling the inline of bpf_kptr_xchg() on x86-64 first. Reviewed-by: Eduard Zingerman <eddyz87@gmail.com> Signed-off-by: Hou Tao <houtao1@huawei.com> Link: https://lore.kernel.org/r/20240105104819.3916743-2-houtao@huaweicloud.com Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
parent
2121c43f88
commit
7c05e7f3e7
@ -3242,3 +3242,8 @@ void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,
|
|||||||
BUG_ON(ret < 0);
|
BUG_ON(ret < 0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool bpf_jit_supports_ptr_xchg(void)
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
@ -955,6 +955,7 @@ bool bpf_jit_supports_subprog_tailcalls(void);
|
|||||||
bool bpf_jit_supports_kfunc_call(void);
|
bool bpf_jit_supports_kfunc_call(void);
|
||||||
bool bpf_jit_supports_far_kfunc_call(void);
|
bool bpf_jit_supports_far_kfunc_call(void);
|
||||||
bool bpf_jit_supports_exceptions(void);
|
bool bpf_jit_supports_exceptions(void);
|
||||||
|
bool bpf_jit_supports_ptr_xchg(void);
|
||||||
void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);
|
void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);
|
||||||
bool bpf_helper_changes_pkt_data(void *func);
|
bool bpf_helper_changes_pkt_data(void *func);
|
||||||
|
|
||||||
|
@ -2925,6 +2925,16 @@ bool __weak bpf_jit_supports_far_kfunc_call(void)
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Return TRUE if the JIT backend satisfies the following two conditions:
|
||||||
|
* 1) JIT backend supports atomic_xchg() on pointer-sized words.
|
||||||
|
* 2) Under the specific arch, the implementation of xchg() is the same
|
||||||
|
* as atomic_xchg() on pointer-sized words.
|
||||||
|
*/
|
||||||
|
bool __weak bpf_jit_supports_ptr_xchg(void)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
/* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call
|
/* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call
|
||||||
* skb_copy_bits(), so provide a weak definition of it for NET-less config.
|
* skb_copy_bits(), so provide a weak definition of it for NET-less config.
|
||||||
*/
|
*/
|
||||||
|
@ -1414,6 +1414,7 @@ BPF_CALL_2(bpf_kptr_xchg, void *, map_value, void *, ptr)
|
|||||||
{
|
{
|
||||||
unsigned long *kptr = map_value;
|
unsigned long *kptr = map_value;
|
||||||
|
|
||||||
|
/* This helper may be inlined by verifier. */
|
||||||
return xchg(kptr, (unsigned long)ptr);
|
return xchg(kptr, (unsigned long)ptr);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -19809,6 +19809,23 @@ patch_map_ops_generic:
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Implement bpf_kptr_xchg inline */
|
||||||
|
if (prog->jit_requested && BITS_PER_LONG == 64 &&
|
||||||
|
insn->imm == BPF_FUNC_kptr_xchg &&
|
||||||
|
bpf_jit_supports_ptr_xchg()) {
|
||||||
|
insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2);
|
||||||
|
insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0);
|
||||||
|
cnt = 2;
|
||||||
|
|
||||||
|
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
|
||||||
|
if (!new_prog)
|
||||||
|
return -ENOMEM;
|
||||||
|
|
||||||
|
delta += cnt - 1;
|
||||||
|
env->prog = prog = new_prog;
|
||||||
|
insn = new_prog->insnsi + i + delta;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
patch_call_imm:
|
patch_call_imm:
|
||||||
fn = env->ops->get_func_proto(insn->imm, env->prog);
|
fn = env->ops->get_func_proto(insn->imm, env->prog);
|
||||||
/* all functions that have prototype and verifier allowed
|
/* all functions that have prototype and verifier allowed
|
||||||
|
Loading…
Reference in New Issue
Block a user