iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'Support BTF-powered BPF tracing programs for kernel modules'

Browse Source 
Andrii Nakryiko says:

====================

This patch sets extends kernel and libbpf with support for attaching
BTF-powered raw tracepoint (tp_btf) and tracing (fentry/fexit/fmod_ret/lsm)
BPF programs to BPF hooks defined in kernel modules. As part of that, libbpf
now supports performing CO-RE relocations against types in kernel module BTFs,
in addition to existing vmlinux BTF support.

Kernel UAPI for BPF_PROG_LOAD now allows to specify kernel module (or vmlinux)
BTF object FD in attach_btf_obj_fd field, aliased to attach_prog_fd. This is
used to identify which BTF object needs to be used for finding BTF type by
provided attach_btf_id.

This patch set also sets up a convenient and fully-controlled custom kernel
module (called "bpf_testmod"), that is a predictable playground for all the
BPF selftests, that rely on module BTFs. Currently pahole doesn't generate
BTF_KIND_FUNC info for ftrace-able static functions in kernel modules, so
expose traced function in bpf_sidecar.ko. Once pahole is enhanced, we can go
back to static function.

From end user perspective there are no extra actions that need to happen.
Libbpf will continue searching across all kernel module BTFs, if desired
attach BTF type is not found in vmlinux. That way it doesn't matter if BPF
hook that user is trying to attach to is built into vmlinux image or is
loaded in kernel module.

v5->v6:
  - move btf_put() back to syscall.c (kernel test robot);
  - added close(fd) in patch #5 (John);
v4->v5:
  - use FD to specify BTF object (Alexei);
  - move prog->aux->attach_btf putting into bpf_prog_free() for consistency
    with putting prog->aux->dst_prog;
  - fix BTF FD leak(s) in libbpf;
v3->v4:
  - merge together patch sets [0] and [1];
  - avoid increasing bpf_reg_state by reordering fields (Alexei);
  - preserve btf_data_size in struct module;
v2->v3:
  - fix subtle uninitialized variable use in BTF ID iteration code;
v1->v2:
  - module_put() inside preempt_disable() region (Alexei);
  - bpf_sidecar -> bpf_testmod rename (Alexei);
  - test_progs more relaxed handling of bpf_testmod;
  - test_progs marks skipped sub-tests properly as SKIP now.

  [0] https://patchwork.kernel.org/project/netdevbpf/list/?series=393677&state=*
  [1] https://patchwork.kernel.org/project/netdevbpf/list/?series=393679&state=*
====================

Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Alexei Starovoitov 2020-12-03 17:38:22 -08:00
commit 8158c5fd61
29 changed files with 1232 additions and 282 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
include/linux/bpf.h
View File

@ -421,7 +421,10 @@ struct bpf_insn_access_aux {
enum bpf_reg_type reg_type;
union {
int ctx_field_size;
u32 btf_id;
struct {
struct btf *btf;
u32 btf_id;
};
};
struct bpf_verifier_log *log; /* for verbose logs */
};
@ -458,6 +461,7 @@ struct bpf_verifier_ops {
struct bpf_insn *dst,
struct bpf_prog *prog, u32 *target_size);
int (*btf_struct_access)(struct bpf_verifier_log *log,
const struct btf *btf,
const struct btf_type *t, int off, int size,
enum bpf_access_type atype,
u32 *next_btf_id);
@ -771,6 +775,7 @@ struct bpf_prog_aux {
u32 ctx_arg_info_size;
u32 max_rdonly_access;
u32 max_rdwr_access;
struct btf *attach_btf;
const struct bpf_ctx_arg_aux *ctx_arg_info;
struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */
struct bpf_prog *dst_prog;
@ -1005,7 +1010,6 @@ struct bpf_event_entry {
bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp);
int bpf_prog_calc_tag(struct bpf_prog *fp);
const char *kernel_type_name(u32 btf_type_id);
const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
@ -1450,12 +1454,13 @@ int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
bool btf_ctx_access(int off, int size, enum bpf_access_type type,
const struct bpf_prog *prog,
struct bpf_insn_access_aux *info);
int btf_struct_access(struct bpf_verifier_log *log,
int btf_struct_access(struct bpf_verifier_log *log, const struct btf *btf,
const struct btf_type *t, int off, int size,
enum bpf_access_type atype,
u32 *next_btf_id);
bool btf_struct_ids_match(struct bpf_verifier_log *log,
int off, u32 id, u32 need_type_id);
const struct btf *btf, u32 id, int off,
const struct btf *need_btf, u32 need_type_id);
int btf_distill_func_proto(struct bpf_verifier_log *log,
struct btf *btf,

28
include/linux/bpf_verifier.h
View File

@ -5,6 +5,7 @@
#define _LINUX_BPF_VERIFIER_H 1
#include <linux/bpf.h> /* for enum bpf_reg_type */
#include <linux/btf.h> /* for struct btf and btf_id() */
#include <linux/filter.h> /* for MAX_BPF_STACK */
#include <linux/tnum.h>
@ -43,6 +44,8 @@ enum bpf_reg_liveness {
struct bpf_reg_state {
/* Ordering of fields matters. See states_equal() */
enum bpf_reg_type type;
/* Fixed part of pointer offset, pointer types only */
s32 off;
union {
/* valid when type == PTR_TO_PACKET */
int range;
@ -52,15 +55,20 @@ struct bpf_reg_state {
*/
struct bpf_map *map_ptr;
u32 btf_id; /* for PTR_TO_BTF_ID */
/* for PTR_TO_BTF_ID */
struct {
struct btf *btf;
u32 btf_id;
};
u32 mem_size; /* for PTR_TO_MEM | PTR_TO_MEM_OR_NULL */
/* Max size from any of the above. */
unsigned long raw;
struct {
unsigned long raw1;
unsigned long raw2;
} raw;
};
/* Fixed part of pointer offset, pointer types only */
s32 off;
/* For PTR_TO_PACKET, used to find other pointers with the same variable
* offset, so they can share range knowledge.
* For PTR_TO_MAP_VALUE_OR_NULL this is used to share which map value we
@ -311,7 +319,10 @@ struct bpf_insn_aux_data {
struct {
enum bpf_reg_type reg_type; /* type of pseudo_btf_id */
union {
u32 btf_id; /* btf_id for struct typed var */
struct {
struct btf *btf;
u32 btf_id; /* btf_id for struct typed var */
};
u32 mem_size; /* mem_size for non-struct typed var */
};
} btf_var;
@ -459,9 +470,12 @@ int check_ctx_reg(struct bpf_verifier_env *env,
/* this lives here instead of in bpf.h because it needs to dereference tgt_prog */
static inline u64 bpf_trampoline_compute_key(const struct bpf_prog *tgt_prog,
u32 btf_id)
struct btf *btf, u32 btf_id)
{
return tgt_prog ? (((u64)tgt_prog->aux->id) << 32 | btf_id) : btf_id;
if (tgt_prog)
return ((u64)tgt_prog->aux->id << 32) | btf_id;
else
return ((u64)btf_obj_id(btf) << 32) | 0x80000000 | btf_id;
}
int bpf_check_attach_target(struct bpf_verifier_log *log,

6
include/linux/btf.h
View File

@ -18,6 +18,7 @@ struct btf_show;
extern const struct file_operations btf_fops;
void btf_get(struct btf *btf);
void btf_put(struct btf *btf);
int btf_new_fd(const union bpf_attr *attr);
struct btf *btf_get_by_fd(int fd);
@ -88,7 +89,8 @@ int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj,
char *buf, int len, u64 flags);
int btf_get_fd_by_id(u32 id);
u32 btf_id(const struct btf *btf);
u32 btf_obj_id(const struct btf *btf);
bool btf_is_kernel(const struct btf *btf);
bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s,
const struct btf_member *m,
u32 expected_offset, u32 expected_size);
@ -206,6 +208,8 @@ static inline const struct btf_var_secinfo *btf_type_var_secinfo(
}
#ifdef CONFIG_BPF_SYSCALL
struct bpf_prog;
const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id);
const char *btf_name_by_offset(const struct btf *btf, u32 offset);
struct btf *btf_parse_vmlinux(void);

7
include/uapi/linux/bpf.h
View File

@ -557,7 +557,12 @@ union bpf_attr {
__aligned_u64 line_info; /* line info */
__u32 line_info_cnt; /* number of bpf_line_info records */
__u32 attach_btf_id; /* in-kernel BTF type id to attach to */
__u32 attach_prog_fd; /* 0 to attach to vmlinux */
union {
/* valid prog_fd to attach to bpf prog */
__u32 attach_prog_fd;
/* or valid module BTF object fd or 0 to attach to vmlinux */
__u32 attach_btf_obj_fd;
};
};
struct { /* anonymous struct used by BPF_OBJ_* commands */

70
kernel/bpf/btf.c
View File

@ -1524,6 +1524,11 @@ static void btf_free_rcu(struct rcu_head *rcu)
btf_free(btf);
}
void btf_get(struct btf *btf)
{
refcount_inc(&btf->refcnt);
}
void btf_put(struct btf *btf)
{
if (btf && refcount_dec_and_test(&btf->refcnt)) {
@ -4555,11 +4560,10 @@ struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog)
{
struct bpf_prog *tgt_prog = prog->aux->dst_prog;
if (tgt_prog) {
if (tgt_prog)
return tgt_prog->aux->btf;
} else {
return btf_vmlinux;
}
else
return prog->aux->attach_btf;
}
static bool is_string_ptr(struct btf *btf, const struct btf_type *t)
@ -4700,6 +4704,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
if (ctx_arg_info->offset == off) {
info->reg_type = ctx_arg_info->reg_type;
info->btf = btf_vmlinux;
info->btf_id = ctx_arg_info->btf_id;
return true;
}
@ -4716,6 +4721,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
ret = btf_translate_to_vmlinux(log, btf, t, tgt_type, arg);
if (ret > 0) {
info->btf = btf_vmlinux;
info->btf_id = ret;
return true;
} else {
@ -4723,6 +4729,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
}
}
info->btf = btf;
info->btf_id = t->type;
t = btf_type_by_id(btf, t->type);
/* skip modifiers */
@ -4749,7 +4756,7 @@ enum bpf_struct_walk_result {
WALK_STRUCT,
};
static int btf_struct_walk(struct bpf_verifier_log *log,
static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf,
const struct btf_type *t, int off, int size,
u32 *next_btf_id)
{
@ -4760,7 +4767,7 @@ static int btf_struct_walk(struct bpf_verifier_log *log,
u32 vlen, elem_id, mid;
again:
tname = __btf_name_by_offset(btf_vmlinux, t->name_off);
tname = __btf_name_by_offset(btf, t->name_off);
if (!btf_type_is_struct(t)) {
bpf_log(log, "Type '%s' is not a struct\n", tname);
return -EINVAL;
@ -4777,7 +4784,7 @@ again:
goto error;
member = btf_type_member(t) + vlen - 1;
mtype = btf_type_skip_modifiers(btf_vmlinux, member->type,
mtype = btf_type_skip_modifiers(btf, member->type,
NULL);
if (!btf_type_is_array(mtype))
goto error;
@ -4793,7 +4800,7 @@ again:
/* Only allow structure for now, can be relaxed for
* other types later.
*/
t = btf_type_skip_modifiers(btf_vmlinux, array_elem->type,
t = btf_type_skip_modifiers(btf, array_elem->type,
NULL);
if (!btf_type_is_struct(t))
goto error;
@ -4851,10 +4858,10 @@ error:
/* type of the field */
mid = member->type;
mtype = btf_type_by_id(btf_vmlinux, member->type);
mname = __btf_name_by_offset(btf_vmlinux, member->name_off);
mtype = btf_type_by_id(btf, member->type);
mname = __btf_name_by_offset(btf, member->name_off);
mtype = __btf_resolve_size(btf_vmlinux, mtype, &msize,
mtype = __btf_resolve_size(btf, mtype, &msize,
&elem_type, &elem_id, &total_nelems,
&mid);
if (IS_ERR(mtype)) {
@ -4949,7 +4956,7 @@ error:
mname, moff, tname, off, size);
return -EACCES;
}
stype = btf_type_skip_modifiers(btf_vmlinux, mtype->type, &id);
stype = btf_type_skip_modifiers(btf, mtype->type, &id);
if (btf_type_is_struct(stype)) {
*next_btf_id = id;
return WALK_PTR;
@ -4975,7 +4982,7 @@ error:
return -EINVAL;
}
int btf_struct_access(struct bpf_verifier_log *log,
int btf_struct_access(struct bpf_verifier_log *log, const struct btf *btf,
const struct btf_type *t, int off, int size,
enum bpf_access_type atype __maybe_unused,
u32 *next_btf_id)
@ -4984,7 +4991,7 @@ int btf_struct_access(struct bpf_verifier_log *log,
u32 id;
do {
err = btf_struct_walk(log, t, off, size, &id);
err = btf_struct_walk(log, btf, t, off, size, &id);
switch (err) {
case WALK_PTR:
@ -5000,7 +5007,7 @@ int btf_struct_access(struct bpf_verifier_log *log,
* by diving in it. At this point the offset is
* aligned with the new type, so set it to 0.
*/
t = btf_type_by_id(btf_vmlinux, id);
t = btf_type_by_id(btf, id);
off = 0;
break;
default:
@ -5016,21 +5023,37 @@ int btf_struct_access(struct bpf_verifier_log *log,
return -EINVAL;
}
/* Check that two BTF types, each specified as an BTF object + id, are exactly
* the same. Trivial ID check is not enough due to module BTFs, because we can
* end up with two different module BTFs, but IDs point to the common type in
* vmlinux BTF.
*/
static bool btf_types_are_same(const struct btf *btf1, u32 id1,
const struct btf *btf2, u32 id2)
{
if (id1 != id2)
return false;
if (btf1 == btf2)
return true;
return btf_type_by_id(btf1, id1) == btf_type_by_id(btf2, id2);
}
bool btf_struct_ids_match(struct bpf_verifier_log *log,
int off, u32 id, u32 need_type_id)
const struct btf *btf, u32 id, int off,
const struct btf *need_btf, u32 need_type_id)
{
const struct btf_type *type;
int err;
/* Are we already done? */
if (need_type_id == id && off == 0)
if (off == 0 && btf_types_are_same(btf, id, need_btf, need_type_id))
return true;
again:
type = btf_type_by_id(btf_vmlinux, id);
type = btf_type_by_id(btf, id);
if (!type)
return false;
err = btf_struct_walk(log, type, off, 1, &id);
err = btf_struct_walk(log, btf, type, off, 1, &id);
if (err != WALK_STRUCT)
return false;
@ -5039,7 +5062,7 @@ again:
* continue the search with offset 0 in the new
* type.
*/
if (need_type_id != id) {
if (!btf_types_are_same(btf, id, need_btf, need_type_id)) {
off = 0;
goto again;
}
@ -5710,11 +5733,16 @@ int btf_get_fd_by_id(u32 id)
return fd;
}
u32 btf_id(const struct btf *btf)
u32 btf_obj_id(const struct btf *btf)
{
return btf->id;
}
bool btf_is_kernel(const struct btf *btf)
{
return btf->kernel_btf;
}
static int btf_id_cmp_func(const void *a, const void *b)
{
const int *pa = a, *pb = b;

80
kernel/bpf/syscall.c
View File

@ -1691,6 +1691,8 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred)
bpf_prog_kallsyms_del_all(prog);
btf_put(prog->aux->btf);
bpf_prog_free_linfo(prog);
if (prog->aux->attach_btf)
btf_put(prog->aux->attach_btf);
if (deferred) {
if (prog->aux->sleepable)
@ -1924,12 +1926,16 @@ static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr)
static int
bpf_prog_load_check_attach(enum bpf_prog_type prog_type,
enum bpf_attach_type expected_attach_type,
u32 btf_id, u32 prog_fd)
struct btf *attach_btf, u32 btf_id,
struct bpf_prog *dst_prog)
{
if (btf_id) {
if (btf_id > BTF_MAX_TYPE)
return -EINVAL;
if (!attach_btf && !dst_prog)
return -EINVAL;
switch (prog_type) {
case BPF_PROG_TYPE_TRACING:
case BPF_PROG_TYPE_LSM:
@ -1941,7 +1947,10 @@ bpf_prog_load_check_attach(enum bpf_prog_type prog_type,
}
}
if (prog_fd && prog_type != BPF_PROG_TYPE_TRACING &&
if (attach_btf && (!btf_id || dst_prog))
return -EINVAL;
if (dst_prog && prog_type != BPF_PROG_TYPE_TRACING &&
prog_type != BPF_PROG_TYPE_EXT)
return -EINVAL;
@ -2058,7 +2067,8 @@ static bool is_perfmon_prog_type(enum bpf_prog_type prog_type)
static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr)
{
enum bpf_prog_type type = attr->prog_type;
struct bpf_prog *prog;
struct bpf_prog *prog, *dst_prog = NULL;
struct btf *attach_btf = NULL;
int err;
char license[128];
bool is_gpl;
@ -2100,30 +2110,56 @@ static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr)
if (is_perfmon_prog_type(type) && !perfmon_capable())
return -EPERM;
/* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog
* or btf, we need to check which one it is
*/
if (attr->attach_prog_fd) {
dst_prog = bpf_prog_get(attr->attach_prog_fd);
if (IS_ERR(dst_prog)) {
dst_prog = NULL;
attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd);
if (IS_ERR(attach_btf))
return -EINVAL;
if (!btf_is_kernel(attach_btf)) {
btf_put(attach_btf);
return -EINVAL;
}
}
} else if (attr->attach_btf_id) {
/* fall back to vmlinux BTF, if BTF type ID is specified */
attach_btf = bpf_get_btf_vmlinux();
if (IS_ERR(attach_btf))
return PTR_ERR(attach_btf);
if (!attach_btf)
return -EINVAL;
btf_get(attach_btf);
}
bpf_prog_load_fixup_attach_type(attr);
if (bpf_prog_load_check_attach(type, attr->expected_attach_type,
attr->attach_btf_id,
attr->attach_prog_fd))
attach_btf, attr->attach_btf_id,
dst_prog)) {
if (dst_prog)
bpf_prog_put(dst_prog);
if (attach_btf)
btf_put(attach_btf);
return -EINVAL;
}
/* plain bpf_prog allocation */
prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
if (!prog)
if (!prog) {
if (dst_prog)
bpf_prog_put(dst_prog);
if (attach_btf)
btf_put(attach_btf);
return -ENOMEM;
prog->expected_attach_type = attr->expected_attach_type;
prog->aux->attach_btf_id = attr->attach_btf_id;
if (attr->attach_prog_fd) {
struct bpf_prog *dst_prog;
dst_prog = bpf_prog_get(attr->attach_prog_fd);
if (IS_ERR(dst_prog)) {
err = PTR_ERR(dst_prog);
goto free_prog;
}
prog->aux->dst_prog = dst_prog;
}
prog->expected_attach_type = attr->expected_attach_type;
prog->aux->attach_btf = attach_btf;
prog->aux->attach_btf_id = attr->attach_btf_id;
prog->aux->dst_prog = dst_prog;
prog->aux->offload_requested = !!attr->prog_ifindex;
prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE;
@ -2209,6 +2245,8 @@ free_prog_sec:
free_uid(prog->aux->user);
security_bpf_prog_free(prog->aux);
free_prog:
if (prog->aux->attach_btf)
btf_put(prog->aux->attach_btf);
bpf_prog_free(prog);
return err;
}
@ -2566,7 +2604,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
goto out_put_prog;
}
key = bpf_trampoline_compute_key(tgt_prog, btf_id);
key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id);
}
link = kzalloc(sizeof(*link), GFP_USER);
@ -3543,7 +3581,7 @@ static int bpf_prog_get_info_by_fd(struct file *file,
}
if (prog->aux->btf)
info.btf_id = btf_id(prog->aux->btf);
info.btf_id = btf_obj_id(prog->aux->btf);
ulen = info.nr_func_info;
info.nr_func_info = prog->aux->func_info_cnt;
@ -3646,7 +3684,7 @@ static int bpf_map_get_info_by_fd(struct file *file,
memcpy(info.name, map->name, sizeof(map->name));
if (map->btf) {
info.btf_id = btf_id(map->btf);
info.btf_id = btf_obj_id(map->btf);
info.btf_key_type_id = map->btf_key_type_id;
info.btf_value_type_id = map->btf_value_type_id;
}

77
kernel/bpf/verifier.c
View File

@ -238,7 +238,9 @@ struct bpf_call_arg_meta {
u64 msize_max_value;
int ref_obj_id;
int func_id;
struct btf *btf;
u32 btf_id;
struct btf *ret_btf;
u32 ret_btf_id;
};
@ -556,10 +558,9 @@ static struct bpf_func_state *func(struct bpf_verifier_env *env,
return cur->frame[reg->frameno];
}
const char *kernel_type_name(u32 id)
static const char *kernel_type_name(const struct btf* btf, u32 id)
{
return btf_name_by_offset(btf_vmlinux,
btf_type_by_id(btf_vmlinux, id)->name_off);
return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
}
static void print_verifier_state(struct bpf_verifier_env *env,
@ -589,7 +590,7 @@ static void print_verifier_state(struct bpf_verifier_env *env,
if (t == PTR_TO_BTF_ID ||
t == PTR_TO_BTF_ID_OR_NULL ||
t == PTR_TO_PERCPU_BTF_ID)
verbose(env, "%s", kernel_type_name(reg->btf_id));
verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id));
verbose(env, "(id=%d", reg->id);
if (reg_type_may_be_refcounted_or_null(t))
verbose(env, ",ref_obj_id=%d", reg->ref_obj_id);
@ -1383,7 +1384,8 @@ static void mark_reg_not_init(struct bpf_verifier_env *env,
static void mark_btf_ld_reg(struct bpf_verifier_env *env,
struct bpf_reg_state *regs, u32 regno,
enum bpf_reg_type reg_type, u32 btf_id)
enum bpf_reg_type reg_type,
struct btf *btf, u32 btf_id)
{
if (reg_type == SCALAR_VALUE) {
mark_reg_unknown(env, regs, regno);
@ -1391,6 +1393,7 @@ static void mark_btf_ld_reg(struct bpf_verifier_env *env,
}
mark_reg_known_zero(env, regs, regno);
regs[regno].type = PTR_TO_BTF_ID;
regs[regno].btf = btf;
regs[regno].btf_id = btf_id;
}
@ -2764,7 +2767,7 @@ static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,
/* check access to 'struct bpf_context' fields. Supports fixed offsets only */
static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,
enum bpf_access_type t, enum bpf_reg_type *reg_type,
u32 *btf_id)
struct btf **btf, u32 *btf_id)
{
struct bpf_insn_access_aux info = {
.reg_type = *reg_type,
@ -2782,10 +2785,12 @@ static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off,
*/
*reg_type = info.reg_type;
if (*reg_type == PTR_TO_BTF_ID || *reg_type == PTR_TO_BTF_ID_OR_NULL)
if (*reg_type == PTR_TO_BTF_ID || *reg_type == PTR_TO_BTF_ID_OR_NULL) {
*btf = info.btf;
*btf_id = info.btf_id;
else
} else {
env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;
}
/* remember the offset of last byte accessed in ctx */
if (env->prog->aux->max_ctx_offset < off + size)
env->prog->aux->max_ctx_offset = off + size;
@ -3297,8 +3302,8 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
int value_regno)
{
struct bpf_reg_state *reg = regs + regno;
const struct btf_type *t = btf_type_by_id(btf_vmlinux, reg->btf_id);
const char *tname = btf_name_by_offset(btf_vmlinux, t->name_off);
const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id);
const char *tname = btf_name_by_offset(reg->btf, t->name_off);
u32 btf_id;
int ret;
@ -3319,23 +3324,23 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
}
if (env->ops->btf_struct_access) {
ret = env->ops->btf_struct_access(&env->log, t, off, size,
atype, &btf_id);
ret = env->ops->btf_struct_access(&env->log, reg->btf, t,
off, size, atype, &btf_id);
} else {
if (atype != BPF_READ) {
verbose(env, "only read is supported\n");
return -EACCES;
}
ret = btf_struct_access(&env->log, t, off, size, atype,
&btf_id);
ret = btf_struct_access(&env->log, reg->btf, t, off, size,
atype, &btf_id);
}
if (ret < 0)
return ret;
if (atype == BPF_READ && value_regno >= 0)
mark_btf_ld_reg(env, regs, value_regno, ret, btf_id);
mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id);
return 0;
}
@ -3385,12 +3390,12 @@ static int check_ptr_to_map_access(struct bpf_verifier_env *env,
return -EACCES;
}
ret = btf_struct_access(&env->log, t, off, size, atype, &btf_id);
ret = btf_struct_access(&env->log, btf_vmlinux, t, off, size, atype, &btf_id);
if (ret < 0)
return ret;
if (value_regno >= 0)
mark_btf_ld_reg(env, regs, value_regno, ret, btf_id);
mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id);
return 0;
}
@ -3466,6 +3471,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
mark_reg_unknown(env, regs, value_regno);
} else if (reg->type == PTR_TO_CTX) {
enum bpf_reg_type reg_type = SCALAR_VALUE;
struct btf *btf = NULL;
u32 btf_id = 0;
if (t == BPF_WRITE && value_regno >= 0 &&
@ -3478,7 +3484,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
if (err < 0)
return err;
err = check_ctx_access(env, insn_idx, off, size, t, &reg_type, &btf_id);
err = check_ctx_access(env, insn_idx, off, size, t, &reg_type, &btf, &btf_id);
if (err)
verbose_linfo(env, insn_idx, "; ");
if (!err && t == BPF_READ && value_regno >= 0) {
@ -3500,8 +3506,10 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
*/
regs[value_regno].subreg_def = DEF_NOT_SUBREG;
if (reg_type == PTR_TO_BTF_ID ||
reg_type == PTR_TO_BTF_ID_OR_NULL)
reg_type == PTR_TO_BTF_ID_OR_NULL) {
regs[value_regno].btf = btf;
regs[value_regno].btf_id = btf_id;
}
}
regs[value_regno].type = reg_type;
}
@ -4118,11 +4126,11 @@ found:
arg_btf_id = compatible->btf_id;
}
if (!btf_struct_ids_match(&env->log, reg->off, reg->btf_id,
*arg_btf_id)) {
if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,
btf_vmlinux, *arg_btf_id)) {
verbose(env, "R%d is of type %s but %s is expected\n",
regno, kernel_type_name(reg->btf_id),
kernel_type_name(*arg_btf_id));
regno, kernel_type_name(reg->btf, reg->btf_id),
kernel_type_name(btf_vmlinux, *arg_btf_id));
return -EACCES;
}
@ -4244,6 +4252,7 @@ skip_type_check:
verbose(env, "Helper has invalid btf_id in R%d\n", regno);
return -EACCES;
}
meta->ret_btf = reg->btf;
meta->ret_btf_id = reg->btf_id;
} else if (arg_type == ARG_PTR_TO_SPIN_LOCK) {
if (meta->func_id == BPF_FUNC_spin_lock) {
@ -5190,16 +5199,16 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn
const struct btf_type *t;
mark_reg_known_zero(env, regs, BPF_REG_0);
t = btf_type_skip_modifiers(btf_vmlinux, meta.ret_btf_id, NULL);
t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL);
if (!btf_type_is_struct(t)) {
u32 tsize;
const struct btf_type *ret;
const char *tname;
/* resolve the type size of ksym. */
ret = btf_resolve_size(btf_vmlinux, t, &tsize);
ret = btf_resolve_size(meta.ret_btf, t, &tsize);
if (IS_ERR(ret)) {
tname = btf_name_by_offset(btf_vmlinux, t->name_off);
tname = btf_name_by_offset(meta.ret_btf, t->name_off);
verbose(env, "unable to resolve the size of type '%s': %ld\n",
tname, PTR_ERR(ret));
return -EINVAL;
@ -5212,6 +5221,7 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn
regs[BPF_REG_0].type =
fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
PTR_TO_BTF_ID : PTR_TO_BTF_ID_OR_NULL;
regs[BPF_REG_0].btf = meta.ret_btf;
regs[BPF_REG_0].btf_id = meta.ret_btf_id;
}
} else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL ||
@ -5228,6 +5238,10 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn
fn->ret_type, func_id_name(func_id), func_id);
return -EINVAL;
}
/* current BPF helper definitions are only coming from
* built-in code with type IDs from vmlinux BTF
*/
regs[BPF_REG_0].btf = btf_vmlinux;
regs[BPF_REG_0].btf_id = ret_btf_id;
} else {
verbose(env, "unknown return type %d of func %s#%d\n",
@ -5627,7 +5641,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
if (reg_is_pkt_pointer(ptr_reg)) {
dst_reg->id = ++env->id_gen;
/* something was added to pkt_ptr, set range to zero */
dst_reg->raw = 0;
memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
}
break;
case BPF_SUB:
@ -5692,7 +5706,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
dst_reg->id = ++env->id_gen;
/* something was added to pkt_ptr, set range to zero */
if (smin_val < 0)
dst_reg->raw = 0;
memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
}
break;
case BPF_AND:
@ -7744,6 +7758,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
break;
case PTR_TO_BTF_ID:
case PTR_TO_PERCPU_BTF_ID:
dst_reg->btf = aux->btf_var.btf;
dst_reg->btf_id = aux->btf_var.btf_id;
break;
default:
@ -9739,6 +9754,7 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env,
t = btf_type_skip_modifiers(btf_vmlinux, type, NULL);
if (percpu) {
aux->btf_var.reg_type = PTR_TO_PERCPU_BTF_ID;
aux->btf_var.btf = btf_vmlinux;
aux->btf_var.btf_id = type;
} else if (!btf_type_is_struct(t)) {
const struct btf_type *ret;
@ -9757,6 +9773,7 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env,
aux->btf_var.mem_size = tsize;
} else {
aux->btf_var.reg_type = PTR_TO_BTF_ID;
aux->btf_var.btf = btf_vmlinux;
aux->btf_var.btf_id = type;
}
return 0;
@ -11609,7 +11626,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
bpf_log(log, "Tracing programs must provide btf_id\n");
return -EINVAL;
}
btf = tgt_prog ? tgt_prog->aux->btf : btf_vmlinux;
btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf;
if (!btf) {
bpf_log(log,
"FENTRY/FEXIT program can only be attached to another program annotated with BTF\n");
@ -11885,7 +11902,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env)
return ret;
}
key = bpf_trampoline_compute_key(tgt_prog, btf_id);
key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id);
tr = bpf_trampoline_get(key, &tgt_info);
if (!tr)
return -ENOMEM;

1
kernel/module.c
View File

@ -3712,7 +3712,6 @@ static noinline int do_init_module(struct module *mod)
#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
/* .BTF is not SHF_ALLOC and will get removed, so sanitize pointer */
mod->btf_data = NULL;
mod->btf_data_size = 0;
#endif
/*
* We want to free module_init, but be aware that kallsyms may be

8
kernel/trace/bpf_trace.c
View File

@ -2060,10 +2060,12 @@ struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name)
void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp)
{
struct module *mod = __module_address((unsigned long)btp);
struct module *mod;
if (mod)
module_put(mod);
preempt_disable();
mod = __module_address((unsigned long)btp);
module_put(mod);
preempt_enable();
}
static __always_inline

3
net/ipv4/bpf_tcp_ca.c
View File

@ -95,6 +95,7 @@ static bool bpf_tcp_ca_is_valid_access(int off, int size,
}
static int bpf_tcp_ca_btf_struct_access(struct bpf_verifier_log *log,
const struct btf *btf,
const struct btf_type *t, int off,
int size, enum bpf_access_type atype,
u32 *next_btf_id)
@ -102,7 +103,7 @@ static int bpf_tcp_ca_btf_struct_access(struct bpf_verifier_log *log,
size_t end;
if (atype == BPF_READ)
return btf_struct_access(log, t, off, size, atype, next_btf_id);
return btf_struct_access(log, btf, t, off, size, atype, next_btf_id);
if (t != tcp_sock_type) {
bpf_log(log, "only read is supported\n");

7
tools/include/uapi/linux/bpf.h
View File

@ -557,7 +557,12 @@ union bpf_attr {
__aligned_u64 line_info; /* line info */
__u32 line_info_cnt; /* number of bpf_line_info records */
__u32 attach_btf_id; /* in-kernel BTF type id to attach to */
__u32 attach_prog_fd; /* 0 to attach to vmlinux */
union {
/* valid prog_fd to attach to bpf prog */
__u32 attach_prog_fd;
/* or valid module BTF object fd or 0 to attach to vmlinux */
__u32 attach_btf_obj_fd;
};
};
struct { /* anonymous struct used by BPF_OBJ_* commands */

101
tools/lib/bpf/bpf.c
View File

@ -215,59 +215,55 @@ alloc_zero_tailing_info(const void *orecord, __u32 cnt,
return info;
}
int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
char *log_buf, size_t log_buf_sz)
int libbpf__bpf_prog_load(const struct bpf_prog_load_params *load_attr)
{
void *finfo = NULL, *linfo = NULL;
union bpf_attr attr;
__u32 log_level;
int fd;
if (!load_attr || !log_buf != !log_buf_sz)
if (!load_attr->log_buf != !load_attr->log_buf_sz)
return -EINVAL;
log_level = load_attr->log_level;
if (log_level > (4 | 2 | 1) || (log_level && !log_buf))
if (load_attr->log_level > (4 | 2 | 1) || (load_attr->log_level && !load_attr->log_buf))
return -EINVAL;
memset(&attr, 0, sizeof(attr));
attr.prog_type = load_attr->prog_type;
attr.expected_attach_type = load_attr->expected_attach_type;
if (attr.prog_type == BPF_PROG_TYPE_STRUCT_OPS ||
attr.prog_type == BPF_PROG_TYPE_LSM) {
attr.attach_btf_id = load_attr->attach_btf_id;
} else if (attr.prog_type == BPF_PROG_TYPE_TRACING ||
attr.prog_type == BPF_PROG_TYPE_EXT) {
attr.attach_btf_id = load_attr->attach_btf_id;
if (load_attr->attach_prog_fd)
attr.attach_prog_fd = load_attr->attach_prog_fd;
} else {
attr.prog_ifindex = load_attr->prog_ifindex;
attr.kern_version = load_attr->kern_version;
}
attr.insn_cnt = (__u32)load_attr->insns_cnt;
else
attr.attach_btf_obj_fd = load_attr->attach_btf_obj_fd;
attr.attach_btf_id = load_attr->attach_btf_id;
attr.prog_ifindex = load_attr->prog_ifindex;
attr.kern_version = load_attr->kern_version;
attr.insn_cnt = (__u32)load_attr->insn_cnt;
attr.insns = ptr_to_u64(load_attr->insns);
attr.license = ptr_to_u64(load_attr->license);
attr.log_level = log_level;
if (log_level) {
attr.log_buf = ptr_to_u64(log_buf);
attr.log_size = log_buf_sz;
} else {
attr.log_buf = ptr_to_u64(NULL);
attr.log_size = 0;
attr.log_level = load_attr->log_level;
if (attr.log_level) {
attr.log_buf = ptr_to_u64(load_attr->log_buf);
attr.log_size = load_attr->log_buf_sz;
}
attr.prog_btf_fd = load_attr->prog_btf_fd;
attr.prog_flags = load_attr->prog_flags;
attr.func_info_rec_size = load_attr->func_info_rec_size;
attr.func_info_cnt = load_attr->func_info_cnt;
attr.func_info = ptr_to_u64(load_attr->func_info);
attr.line_info_rec_size = load_attr->line_info_rec_size;
attr.line_info_cnt = load_attr->line_info_cnt;
attr.line_info = ptr_to_u64(load_attr->line_info);
if (load_attr->name)
memcpy(attr.prog_name, load_attr->name,
min(strlen(load_attr->name), BPF_OBJ_NAME_LEN - 1));
attr.prog_flags = load_attr->prog_flags;
min(strlen(load_attr->name), (size_t)BPF_OBJ_NAME_LEN - 1));
fd = sys_bpf_prog_load(&attr, sizeof(attr));
if (fd >= 0)
@ -307,19 +303,19 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
}
fd = sys_bpf_prog_load(&attr, sizeof(attr));
if (fd >= 0)
goto done;
}
if (log_level || !log_buf)
if (load_attr->log_level || !load_attr->log_buf)
goto done;
/* Try again with log */
attr.log_buf = ptr_to_u64(log_buf);
attr.log_size = log_buf_sz;
attr.log_buf = ptr_to_u64(load_attr->log_buf);
attr.log_size = load_attr->log_buf_sz;
attr.log_level = 1;
log_buf[0] = 0;
load_attr->log_buf[0] = 0;
fd = sys_bpf_prog_load(&attr, sizeof(attr));
done:
free(finfo);
@ -327,6 +323,49 @@ done:
return fd;
}
int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
char *log_buf, size_t log_buf_sz)
{
struct bpf_prog_load_params p = {};
if (!load_attr || !log_buf != !log_buf_sz)
return -EINVAL;
p.prog_type = load_attr->prog_type;
p.expected_attach_type = load_attr->expected_attach_type;
switch (p.prog_type) {
case BPF_PROG_TYPE_STRUCT_OPS:
case BPF_PROG_TYPE_LSM:
p.attach_btf_id = load_attr->attach_btf_id;
break;
case BPF_PROG_TYPE_TRACING:
case BPF_PROG_TYPE_EXT:
p.attach_btf_id = load_attr->attach_btf_id;
p.attach_prog_fd = load_attr->attach_prog_fd;
break;
default:
p.prog_ifindex = load_attr->prog_ifindex;
p.kern_version = load_attr->kern_version;
}
p.insn_cnt = load_attr->insns_cnt;
p.insns = load_attr->insns;
p.license = load_attr->license;
p.log_level = load_attr->log_level;
p.log_buf = log_buf;
p.log_buf_sz = log_buf_sz;
p.prog_btf_fd = load_attr->prog_btf_fd;
p.func_info_rec_size = load_attr->func_info_rec_size;
p.func_info_cnt = load_attr->func_info_cnt;
p.func_info = load_attr->func_info;
p.line_info_rec_size = load_attr->line_info_rec_size;
p.line_info_cnt = load_attr->line_info_cnt;
p.line_info = load_attr->line_info;
p.name = load_attr->name;
p.prog_flags = load_attr->prog_flags;
return libbpf__bpf_prog_load(&p);
}
int bpf_load_program(enum bpf_prog_type type, const struct bpf_insn *insns,
size_t insns_cnt, const char *license,
__u32 kern_version, char *log_buf,

61
tools/lib/bpf/btf.c
View File

@ -1323,35 +1323,27 @@ const char *btf__name_by_offset(const struct btf *btf, __u32 offset)
return btf__str_by_offset(btf, offset);
}
int btf__get_from_id(__u32 id, struct btf **btf)
struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf)
{
struct bpf_btf_info btf_info = { 0 };
struct bpf_btf_info btf_info;
__u32 len = sizeof(btf_info);
__u32 last_size;
int btf_fd;
struct btf *btf;
void *ptr;
int err;
err = 0;
*btf = NULL;
btf_fd = bpf_btf_get_fd_by_id(id);
if (btf_fd < 0)
return 0;
/* we won't know btf_size until we call bpf_obj_get_info_by_fd(). so
* let's start with a sane default - 4KiB here - and resize it only if
* bpf_obj_get_info_by_fd() needs a bigger buffer.
*/
btf_info.btf_size = 4096;
last_size = btf_info.btf_size;
last_size = 4096;
ptr = malloc(last_size);
if (!ptr) {
err = -ENOMEM;
goto exit_free;
}
if (!ptr)
return ERR_PTR(-ENOMEM);
memset(ptr, 0, last_size);
memset(&btf_info, 0, sizeof(btf_info));
btf_info.btf = ptr_to_u64(ptr);
btf_info.btf_size = last_size;
err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len);
if (!err && btf_info.btf_size > last_size) {
@ -1360,31 +1352,48 @@ int btf__get_from_id(__u32 id, struct btf **btf)
last_size = btf_info.btf_size;
temp_ptr = realloc(ptr, last_size);
if (!temp_ptr) {
err = -ENOMEM;
btf = ERR_PTR(-ENOMEM);
goto exit_free;
}
ptr = temp_ptr;
memset(ptr, 0, last_size);
len = sizeof(btf_info);
memset(&btf_info, 0, sizeof(btf_info));
btf_info.btf = ptr_to_u64(ptr);
btf_info.btf_size = last_size;
err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len);
}
if (err || btf_info.btf_size > last_size) {
err = errno;
btf = err ? ERR_PTR(-errno) : ERR_PTR(-E2BIG);
goto exit_free;
}
*btf = btf__new((__u8 *)(long)btf_info.btf, btf_info.btf_size);
if (IS_ERR(*btf)) {
err = PTR_ERR(*btf);
*btf = NULL;
}
btf = btf_new(ptr, btf_info.btf_size, base_btf);
exit_free:
close(btf_fd);
free(ptr);
return btf;
}
return err;
int btf__get_from_id(__u32 id, struct btf **btf)
{
struct btf *res;
int btf_fd;
*btf = NULL;
btf_fd = bpf_btf_get_fd_by_id(id);
if (btf_fd < 0)
return -errno;
res = btf_get_from_fd(btf_fd, NULL);
close(btf_fd);
if (IS_ERR(res))
return PTR_ERR(res);
*btf = res;
return 0;
}
int btf__get_map_kv_tids(const struct btf *btf, const char *map_name,

502
tools/lib/bpf/libbpf.c
View File

@ -176,6 +176,8 @@ enum kern_feature_id {
FEAT_PROBE_READ_KERN,
/* BPF_PROG_BIND_MAP is supported */
FEAT_PROG_BIND_MAP,
/* Kernel support for module BTFs */
FEAT_MODULE_BTF,
__FEAT_CNT,
};
@ -276,6 +278,7 @@ struct bpf_program {
enum bpf_prog_type type;
enum bpf_attach_type expected_attach_type;
int prog_ifindex;
__u32 attach_btf_obj_fd;
__u32 attach_btf_id;
__u32 attach_prog_fd;
void *func_info;
@ -402,6 +405,13 @@ struct extern_desc {
static LIST_HEAD(bpf_objects_list);
struct module_btf {
struct btf *btf;
char *name;
__u32 id;
int fd;
};
struct bpf_object {
char name[BPF_OBJ_NAME_LEN];
char license[64];
@ -462,11 +472,19 @@ struct bpf_object {
struct list_head list;
struct btf *btf;
struct btf_ext *btf_ext;
/* Parse and load BTF vmlinux if any of the programs in the object need
* it at load time.
*/
struct btf *btf_vmlinux;
struct btf_ext *btf_ext;
/* vmlinux BTF override for CO-RE relocations */
struct btf *btf_vmlinux_override;
/* Lazily initialized kernel module BTFs */
struct module_btf *btf_modules;
bool btf_modules_loaded;
size_t btf_module_cnt;
size_t btf_module_cap;
void *priv;
bpf_object_clear_priv_t clear_priv;
@ -3957,6 +3975,35 @@ static int probe_prog_bind_map(void)
return ret >= 0;
}
static int probe_module_btf(void)
{
static const char strs[] = "\0int";
__u32 types[] = {
/* int */
BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
};
struct bpf_btf_info info;
__u32 len = sizeof(info);
char name[16];
int fd, err;
fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs));
if (fd < 0)
return 0; /* BTF not supported at all */
memset(&info, 0, sizeof(info));
info.name = ptr_to_u64(name);
info.name_len = sizeof(name);
/* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer;
* kernel's module BTF support coincides with support for
* name/name_len fields in struct bpf_btf_info.
*/
err = bpf_obj_get_info_by_fd(fd, &info, &len);
close(fd);
return !err;
}
enum kern_feature_result {
FEAT_UNKNOWN = 0,
FEAT_SUPPORTED = 1,
@ -4000,7 +4047,10 @@ static struct kern_feature_desc {
},
[FEAT_PROG_BIND_MAP] = {
"BPF_PROG_BIND_MAP support", probe_prog_bind_map,
}
},
[FEAT_MODULE_BTF] = {
"module BTF support", probe_module_btf,
},
};
static bool kernel_supports(enum kern_feature_id feat_id)
@ -4600,46 +4650,43 @@ static size_t bpf_core_essential_name_len(const char *name)
return n;
}
/* dynamically sized list of type IDs */
struct ids_vec {
__u32 *data;
struct core_cand
{
const struct btf *btf;
const struct btf_type *t;
const char *name;
__u32 id;
};
/* dynamically sized list of type IDs and its associated struct btf */
struct core_cand_list {
struct core_cand *cands;
int len;
};
static void bpf_core_free_cands(struct ids_vec *cand_ids)
static void bpf_core_free_cands(struct core_cand_list *cands)
{
free(cand_ids->data);
free(cand_ids);
free(cands->cands);
free(cands);
}
static struct ids_vec *bpf_core_find_cands(const struct btf *local_btf,
__u32 local_type_id,
const struct btf *targ_btf)
static int bpf_core_add_cands(struct core_cand *local_cand,
size_t local_essent_len,
const struct btf *targ_btf,
const char *targ_btf_name,
int targ_start_id,
struct core_cand_list *cands)
{
size_t local_essent_len, targ_essent_len;
const char *local_name, *targ_name;
const struct btf_type *t, *local_t;
struct ids_vec *cand_ids;
__u32 *new_ids;
int i, err, n;
local_t = btf__type_by_id(local_btf, local_type_id);
if (!local_t)
return ERR_PTR(-EINVAL);
local_name = btf__name_by_offset(local_btf, local_t->name_off);
if (str_is_empty(local_name))
return ERR_PTR(-EINVAL);
local_essent_len = bpf_core_essential_name_len(local_name);
cand_ids = calloc(1, sizeof(*cand_ids));
if (!cand_ids)
return ERR_PTR(-ENOMEM);
struct core_cand *new_cands, *cand;
const struct btf_type *t;
const char *targ_name;
size_t targ_essent_len;
int n, i;
n = btf__get_nr_types(targ_btf);
for (i = 1; i <= n; i++) {
for (i = targ_start_id; i <= n; i++) {
t = btf__type_by_id(targ_btf, i);
if (btf_kind(t) != btf_kind(local_t))
if (btf_kind(t) != btf_kind(local_cand->t))
continue;
targ_name = btf__name_by_offset(targ_btf, t->name_off);
@ -4650,24 +4697,174 @@ static struct ids_vec *bpf_core_find_cands(const struct btf *local_btf,
if (targ_essent_len != local_essent_len)
continue;
if (strncmp(local_name, targ_name, local_essent_len) == 0) {
pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s\n",
local_type_id, btf_kind_str(local_t),
local_name, i, btf_kind_str(t), targ_name);
new_ids = libbpf_reallocarray(cand_ids->data,
cand_ids->len + 1,
sizeof(*cand_ids->data));
if (!new_ids) {
err = -ENOMEM;
goto err_out;
}
cand_ids->data = new_ids;
cand_ids->data[cand_ids->len++] = i;
}
if (strncmp(local_cand->name, targ_name, local_essent_len) != 0)
continue;
pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
local_cand->id, btf_kind_str(local_cand->t),
local_cand->name, i, btf_kind_str(t), targ_name,
targ_btf_name);
new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
sizeof(*cands->cands));
if (!new_cands)
return -ENOMEM;
cand = &new_cands[cands->len];
cand->btf = targ_btf;
cand->t = t;
cand->name = targ_name;
cand->id = i;
cands->cands = new_cands;
cands->len++;
}
return cand_ids;
return 0;
}
static int load_module_btfs(struct bpf_object *obj)
{
struct bpf_btf_info info;
struct module_btf *mod_btf;
struct btf *btf;
char name[64];
__u32 id = 0, len;
int err, fd;
if (obj->btf_modules_loaded)
return 0;
/* don't do this again, even if we find no module BTFs */
obj->btf_modules_loaded = true;
/* kernel too old to support module BTFs */
if (!kernel_supports(FEAT_MODULE_BTF))
return 0;
while (true) {
err = bpf_btf_get_next_id(id, &id);
if (err && errno == ENOENT)
return 0;
if (err) {
err = -errno;
pr_warn("failed to iterate BTF objects: %d\n", err);
return err;
}
fd = bpf_btf_get_fd_by_id(id);
if (fd < 0) {
if (errno == ENOENT)
continue; /* expected race: BTF was unloaded */
err = -errno;
pr_warn("failed to get BTF object #%d FD: %d\n", id, err);
return err;
}
len = sizeof(info);
memset(&info, 0, sizeof(info));
info.name = ptr_to_u64(name);
info.name_len = sizeof(name);
err = bpf_obj_get_info_by_fd(fd, &info, &len);
if (err) {
err = -errno;
pr_warn("failed to get BTF object #%d info: %d\n", id, err);
goto err_out;
}
/* ignore non-module BTFs */
if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) {
close(fd);
continue;
}
btf = btf_get_from_fd(fd, obj->btf_vmlinux);
if (IS_ERR(btf)) {
pr_warn("failed to load module [%s]'s BTF object #%d: %ld\n",
name, id, PTR_ERR(btf));
err = PTR_ERR(btf);
goto err_out;
}
err = btf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
if (err)
goto err_out;
mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
mod_btf->btf = btf;
mod_btf->id = id;
mod_btf->fd = fd;
mod_btf->name = strdup(name);
if (!mod_btf->name) {
err = -ENOMEM;
goto err_out;
}
continue;
err_out:
bpf_core_free_cands(cand_ids);
close(fd);
return err;
}
return 0;
}
static struct core_cand_list *
bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id)
{
struct core_cand local_cand = {};
struct core_cand_list *cands;
const struct btf *main_btf;
size_t local_essent_len;
int err, i;
local_cand.btf = local_btf;
local_cand.t = btf__type_by_id(local_btf, local_type_id);
if (!local_cand.t)
return ERR_PTR(-EINVAL);
local_cand.name = btf__name_by_offset(local_btf, local_cand.t->name_off);
if (str_is_empty(local_cand.name))
return ERR_PTR(-EINVAL);
local_essent_len = bpf_core_essential_name_len(local_cand.name);
cands = calloc(1, sizeof(*cands));
if (!cands)
return ERR_PTR(-ENOMEM);
/* Attempt to find target candidates in vmlinux BTF first */
main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands);
if (err)
goto err_out;
/* if vmlinux BTF has any candidate, don't got for module BTFs */
if (cands->len)
return cands;
/* if vmlinux BTF was overridden, don't attempt to load module BTFs */
if (obj->btf_vmlinux_override)
return cands;
/* now look through module BTFs, trying to still find candidates */
err = load_module_btfs(obj);
if (err)
goto err_out;
for (i = 0; i < obj->btf_module_cnt; i++) {
err = bpf_core_add_cands(&local_cand, local_essent_len,
obj->btf_modules[i].btf,
obj->btf_modules[i].name,
btf__get_nr_types(obj->btf_vmlinux) + 1,
cands);
if (err)
goto err_out;
}
return cands;
err_out:
bpf_core_free_cands(cands);
return ERR_PTR(err);
}
@ -5661,7 +5858,6 @@ static int bpf_core_apply_relo(struct bpf_program *prog,
const struct bpf_core_relo *relo,
int relo_idx,
const struct btf *local_btf,
const struct btf *targ_btf,
struct hashmap *cand_cache)
{
struct bpf_core_spec local_spec, cand_spec, targ_spec = {};
@ -5669,8 +5865,8 @@ static int bpf_core_apply_relo(struct bpf_program *prog,
struct bpf_core_relo_res cand_res, targ_res;
const struct btf_type *local_type;
const char *local_name;
struct ids_vec *cand_ids;
__u32 local_id, cand_id;
struct core_cand_list *cands = NULL;
__u32 local_id;
const char *spec_str;
int i, j, err;
@ -5717,24 +5913,24 @@ static int bpf_core_apply_relo(struct bpf_program *prog,
return -EOPNOTSUPP;
}
if (!hashmap__find(cand_cache, type_key, (void **)&cand_ids)) {
cand_ids = bpf_core_find_cands(local_btf, local_id, targ_btf);
if (IS_ERR(cand_ids)) {
pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld",
if (!hashmap__find(cand_cache, type_key, (void **)&cands)) {
cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
if (IS_ERR(cands)) {
pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",
prog->name, relo_idx, local_id, btf_kind_str(local_type),
local_name, PTR_ERR(cand_ids));
return PTR_ERR(cand_ids);
local_name, PTR_ERR(cands));
return PTR_ERR(cands);
}
err = hashmap__set(cand_cache, type_key, cand_ids, NULL, NULL);
err = hashmap__set(cand_cache, type_key, cands, NULL, NULL);
if (err) {
bpf_core_free_cands(cand_ids);
bpf_core_free_cands(cands);
return err;
}
}
for (i = 0, j = 0; i < cand_ids->len; i++) {
cand_id = cand_ids->data[i];
err = bpf_core_spec_match(&local_spec, targ_btf, cand_id, &cand_spec);
for (i = 0, j = 0; i < cands->len; i++) {
err = bpf_core_spec_match(&local_spec, cands->cands[i].btf,
cands->cands[i].id, &cand_spec);
if (err < 0) {
pr_warn("prog '%s': relo #%d: error matching candidate #%d ",
prog->name, relo_idx, i);
@ -5778,7 +5974,7 @@ static int bpf_core_apply_relo(struct bpf_program *prog,
return -EINVAL;
}
cand_ids->data[j++] = cand_spec.root_type_id;
cands->cands[j++] = cands->cands[i];
}
/*
@ -5790,7 +5986,7 @@ static int bpf_core_apply_relo(struct bpf_program *prog,
* depending on relo's kind.
*/
if (j > 0)
cand_ids->len = j;
cands->len = j;
/*
* If no candidates were found, it might be both a programmer error,
@ -5834,20 +6030,19 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
struct hashmap_entry *entry;
struct hashmap *cand_cache = NULL;
struct bpf_program *prog;
struct btf *targ_btf;
const char *sec_name;
int i, err = 0, insn_idx, sec_idx;
if (obj->btf_ext->core_relo_info.len == 0)
return 0;
if (targ_btf_path)
targ_btf = btf__parse(targ_btf_path, NULL);
else
targ_btf = obj->btf_vmlinux;
if (IS_ERR_OR_NULL(targ_btf)) {
pr_warn("failed to get target BTF: %ld\n", PTR_ERR(targ_btf));
return PTR_ERR(targ_btf);
if (targ_btf_path) {
obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
if (IS_ERR_OR_NULL(obj->btf_vmlinux_override)) {
err = PTR_ERR(obj->btf_vmlinux_override);
pr_warn("failed to parse target BTF: %d\n", err);
return err;
}
}
cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
@ -5899,8 +6094,7 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
if (!prog->load)
continue;
err = bpf_core_apply_relo(prog, rec, i, obj->btf,
targ_btf, cand_cache);
err = bpf_core_apply_relo(prog, rec, i, obj->btf, cand_cache);
if (err) {
pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
prog->name, i, err);
@ -5910,9 +6104,10 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
}
out:
/* obj->btf_vmlinux is freed at the end of object load phase */
if (targ_btf != obj->btf_vmlinux)
btf__free(targ_btf);
/* obj->btf_vmlinux and module BTFs are freed after object load */
btf__free(obj->btf_vmlinux_override);
obj->btf_vmlinux_override = NULL;
if (!IS_ERR_OR_NULL(cand_cache)) {
hashmap__for_each_entry(cand_cache, entry, i) {
bpf_core_free_cands(entry->value);
@ -6623,7 +6818,7 @@ static int
load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
char *license, __u32 kern_version, int *pfd)
{
struct bpf_load_program_attr load_attr;
struct bpf_prog_load_params load_attr = {};
char *cp, errmsg[STRERR_BUFSIZE];
size_t log_buf_size = 0;
char *log_buf = NULL;
@ -6642,7 +6837,6 @@ load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
if (!insns || !insns_cnt)
return -EINVAL;
memset(&load_attr, 0, sizeof(struct bpf_load_program_attr));
load_attr.prog_type = prog->type;
/* old kernels might not support specifying expected_attach_type */
if (!kernel_supports(FEAT_EXP_ATTACH_TYPE) && prog->sec_def &&
@ -6653,19 +6847,17 @@ load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
if (kernel_supports(FEAT_PROG_NAME))
load_attr.name = prog->name;
load_attr.insns = insns;
load_attr.insns_cnt = insns_cnt;
load_attr.insn_cnt = insns_cnt;
load_attr.license = license;
if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
prog->type == BPF_PROG_TYPE_LSM) {
load_attr.attach_btf_id = prog->attach_btf_id;
} else if (prog->type == BPF_PROG_TYPE_TRACING ||
prog->type == BPF_PROG_TYPE_EXT) {
load_attr.attach_btf_id = prog->attach_btf_id;
if (prog->attach_prog_fd)
load_attr.attach_prog_fd = prog->attach_prog_fd;
load_attr.attach_btf_id = prog->attach_btf_id;
} else {
load_attr.kern_version = kern_version;
load_attr.prog_ifindex = prog->prog_ifindex;
}
else
load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
load_attr.attach_btf_id = prog->attach_btf_id;
load_attr.kern_version = kern_version;
load_attr.prog_ifindex = prog->prog_ifindex;
/* specify func_info/line_info only if kernel supports them */
btf_fd = bpf_object__btf_fd(prog->obj);
if (btf_fd >= 0 && kernel_supports(FEAT_BTF_FUNC)) {
@ -6689,7 +6881,9 @@ retry_load:
*log_buf = 0;
}
ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size);
load_attr.log_buf = log_buf;
load_attr.log_buf_sz = log_buf_size;
ret = libbpf__bpf_prog_load(&load_attr);
if (ret >= 0) {
if (log_buf && load_attr.log_level)
@ -6730,9 +6924,9 @@ retry_load:
pr_warn("-- BEGIN DUMP LOG ---\n");
pr_warn("\n%s\n", log_buf);
pr_warn("-- END LOG --\n");
} else if (load_attr.insns_cnt >= BPF_MAXINSNS) {
} else if (load_attr.insn_cnt >= BPF_MAXINSNS) {
pr_warn("Program too large (%zu insns), at most %d insns\n",
load_attr.insns_cnt, BPF_MAXINSNS);
load_attr.insn_cnt, BPF_MAXINSNS);
ret = -LIBBPF_ERRNO__PROG2BIG;
} else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
/* Wrong program type? */
@ -6740,7 +6934,9 @@ retry_load:
load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
load_attr.expected_attach_type = 0;
fd = bpf_load_program_xattr(&load_attr, NULL, 0);
load_attr.log_buf = NULL;
load_attr.log_buf_sz = 0;
fd = libbpf__bpf_prog_load(&load_attr);
if (fd >= 0) {
close(fd);
ret = -LIBBPF_ERRNO__PROGTYPE;
@ -6753,11 +6949,11 @@ out:
return ret;
}
static int libbpf_find_attach_btf_id(struct bpf_program *prog);
static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id);
int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver)
{
int err = 0, fd, i, btf_id;
int err = 0, fd, i;
if (prog->obj->loaded) {
pr_warn("prog '%s': can't load after object was loaded\n", prog->name);
@ -6767,10 +6963,14 @@ int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver)
if ((prog->type == BPF_PROG_TYPE_TRACING ||
prog->type == BPF_PROG_TYPE_LSM ||
prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) {
btf_id = libbpf_find_attach_btf_id(prog);
if (btf_id <= 0)
return btf_id;
prog->attach_btf_id = btf_id;
int btf_obj_fd = 0, btf_type_id = 0;
err = libbpf_find_attach_btf_id(prog, &btf_obj_fd, &btf_type_id);
if (err)
return err;
prog->attach_btf_obj_fd = btf_obj_fd;
prog->attach_btf_id = btf_type_id;
}
if (prog->instances.nr < 0 || !prog->instances.fds) {
@ -7281,6 +7481,15 @@ int bpf_object__load_xattr(struct bpf_object_load_attr *attr)
err = err ? : bpf_object__relocate(obj, attr->target_btf_path);
err = err ? : bpf_object__load_progs(obj, attr->log_level);
/* clean up module BTFs */
for (i = 0; i < obj->btf_module_cnt; i++) {
close(obj->btf_modules[i].fd);
btf__free(obj->btf_modules[i].btf);
free(obj->btf_modules[i].name);
}
free(obj->btf_modules);
/* clean up vmlinux BTF */
btf__free(obj->btf_vmlinux);
obj->btf_vmlinux = NULL;
@ -8629,8 +8838,8 @@ static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
return btf__find_by_name_kind(btf, btf_type_name, kind);
}
static inline int __find_vmlinux_btf_id(struct btf *btf, const char *name,
enum bpf_attach_type attach_type)
static inline int find_attach_btf_id(struct btf *btf, const char *name,
enum bpf_attach_type attach_type)
{
int err;
@ -8646,9 +8855,6 @@ static inline int __find_vmlinux_btf_id(struct btf *btf, const char *name,
else
err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
if (err <= 0)
pr_warn("%s is not found in vmlinux BTF\n", name);
return err;
}
@ -8664,7 +8870,10 @@ int libbpf_find_vmlinux_btf_id(const char *name,
return -EINVAL;
}
err = __find_vmlinux_btf_id(btf, name, attach_type);
err = find_attach_btf_id(btf, name, attach_type);
if (err <= 0)
pr_warn("%s is not found in vmlinux BTF\n", name);
btf__free(btf);
return err;
}
@ -8702,11 +8911,49 @@ out:
return err;
}
static int libbpf_find_attach_btf_id(struct bpf_program *prog)
static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name,
enum bpf_attach_type attach_type,
int *btf_obj_fd, int *btf_type_id)
{
int ret, i;
ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
if (ret > 0) {
*btf_obj_fd = 0; /* vmlinux BTF */
*btf_type_id = ret;
return 0;
}
if (ret != -ENOENT)
return ret;
ret = load_module_btfs(obj);
if (ret)
return ret;
for (i = 0; i < obj->btf_module_cnt; i++) {
const struct module_btf *mod = &obj->btf_modules[i];
ret = find_attach_btf_id(mod->btf, attach_name, attach_type);
if (ret > 0) {
*btf_obj_fd = mod->fd;
*btf_type_id = ret;
return 0;
}
if (ret == -ENOENT)
continue;
return ret;
}
return -ESRCH;
}
static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id)
{
enum bpf_attach_type attach_type = prog->expected_attach_type;
__u32 attach_prog_fd = prog->attach_prog_fd;
const char *name = prog->sec_name;
const char *name = prog->sec_name, *attach_name;
const struct bpf_sec_def *sec = NULL;
int i, err;
if (!name)
@ -8717,17 +8964,37 @@ static int libbpf_find_attach_btf_id(struct bpf_program *prog)
continue;
if (strncmp(name, section_defs[i].sec, section_defs[i].len))
continue;
if (attach_prog_fd)
err = libbpf_find_prog_btf_id(name + section_defs[i].len,
attach_prog_fd);
else
err = __find_vmlinux_btf_id(prog->obj->btf_vmlinux,
name + section_defs[i].len,
attach_type);
sec = &section_defs[i];
break;
}
if (!sec) {
pr_warn("failed to identify BTF ID based on ELF section name '%s'\n", name);
return -ESRCH;
}
attach_name = name + sec->len;
/* BPF program's BTF ID */
if (attach_prog_fd) {
err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
if (err < 0) {
pr_warn("failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
attach_prog_fd, attach_name, err);
return err;
}
*btf_obj_fd = 0;
*btf_type_id = err;
return 0;
}
/* kernel/module BTF ID */
err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
if (err) {
pr_warn("failed to find kernel BTF type ID of '%s': %d\n", attach_name, err);
return err;
}
pr_warn("failed to identify btf_id based on ELF section name '%s'\n", name);
return -ESRCH;
return 0;
}
int libbpf_attach_type_by_name(const char *name,
@ -10616,6 +10883,7 @@ int bpf_program__set_attach_target(struct bpf_program *prog,
return btf_id;
prog->attach_btf_id = btf_id;
prog->attach_btf_obj_fd = 0;
prog->attach_prog_fd = attach_prog_fd;
return 0;
}

31
tools/lib/bpf/libbpf_internal.h
View File

@ -151,10 +151,41 @@ int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz);
int libbpf__load_raw_btf(const char *raw_types, size_t types_len,
const char *str_sec, size_t str_len);
struct bpf_prog_load_params {
enum bpf_prog_type prog_type;
enum bpf_attach_type expected_attach_type;
const char *name;
const struct bpf_insn *insns;
size_t insn_cnt;
const char *license;
__u32 kern_version;
__u32 attach_prog_fd;
__u32 attach_btf_obj_fd;
__u32 attach_btf_id;
__u32 prog_ifindex;
__u32 prog_btf_fd;
__u32 prog_flags;
__u32 func_info_rec_size;
const void *func_info;
__u32 func_info_cnt;
__u32 line_info_rec_size;
const void *line_info;
__u32 line_info_cnt;
__u32 log_level;
char *log_buf;
size_t log_buf_sz;
};
int libbpf__bpf_prog_load(const struct bpf_prog_load_params *load_attr);
int bpf_object__section_size(const struct bpf_object *obj, const char *name,
__u32 *size);
int bpf_object__variable_offset(const struct bpf_object *obj, const char *name,
__u32 *off);
struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf);
struct btf_ext_info {
/*

1
tools/testing/selftests/bpf/.gitignore vendored
View File

@ -35,3 +35,4 @@ test_cpp
/tools
/runqslower
/bench
*.ko

12
tools/testing/selftests/bpf/Makefile
View File

@ -80,7 +80,7 @@ TEST_PROGS_EXTENDED := with_addr.sh \
# Compile but not part of 'make run_tests'
TEST_GEN_PROGS_EXTENDED = test_sock_addr test_skb_cgroup_id_user \
flow_dissector_load test_flow_dissector test_tcp_check_syncookie_user \
test_lirc_mode2_user xdping test_cpp runqslower bench
test_lirc_mode2_user xdping test_cpp runqslower bench bpf_testmod.ko
TEST_CUSTOM_PROGS = urandom_read
@ -104,6 +104,7 @@ OVERRIDE_TARGETS := 1
override define CLEAN
$(call msg,CLEAN)
$(Q)$(RM) -r $(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED) $(TEST_GEN_FILES) $(EXTRA_CLEAN)
$(Q)$(MAKE) -C bpf_testmod clean
endef
include ../lib.mk
@ -136,6 +137,11 @@ $(OUTPUT)/urandom_read: urandom_read.c
$(call msg,BINARY,,$@)
$(Q)$(CC) $(LDFLAGS) -o $@ $< $(LDLIBS) -Wl,--build-id=sha1
$(OUTPUT)/bpf_testmod.ko: $(VMLINUX_BTF) $(wildcard bpf_testmod/Makefile bpf_testmod/*.[ch])
$(call msg,MOD,,$@)
$(Q)$(MAKE) $(submake_extras) -C bpf_testmod
$(Q)cp bpf_testmod/bpf_testmod.ko $@
$(OUTPUT)/test_stub.o: test_stub.c $(BPFOBJ)
$(call msg,CC,,$@)
$(Q)$(CC) -c $(CFLAGS) -o $@ $<
@ -388,7 +394,7 @@ TRUNNER_BPF_PROGS_DIR := progs
TRUNNER_EXTRA_SOURCES := test_progs.c cgroup_helpers.c trace_helpers.c \
network_helpers.c testing_helpers.c \
btf_helpers.c flow_dissector_load.h
TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \
TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read $(OUTPUT)/bpf_testmod.ko \
ima_setup.sh \
$(wildcard progs/btf_dump_test_case_*.c)
TRUNNER_BPF_BUILD_RULE := CLANG_BPF_BUILD_RULE
@ -460,4 +466,4 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o $(OUTPUT)/testing_helpers.o \
EXTRA_CLEAN := $(TEST_CUSTOM_PROGS) $(SCRATCH_DIR) \
prog_tests/tests.h map_tests/tests.h verifier/tests.h \
feature \
$(addprefix $(OUTPUT)/,*.o *.skel.h no_alu32 bpf_gcc)
$(addprefix $(OUTPUT)/,*.o *.skel.h no_alu32 bpf_gcc bpf_testmod.ko)

6
tools/testing/selftests/bpf/bpf_testmod/.gitignore vendored Normal file
View File

@ -0,0 +1,6 @@
*.mod
*.mod.c
*.o
.ko
/Module.symvers
/modules.order

20
tools/testing/selftests/bpf/bpf_testmod/Makefile Normal file
View File

@ -0,0 +1,20 @@
BPF_TESTMOD_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
KDIR ?= $(abspath $(BPF_TESTMOD_DIR)/../../../../..)
ifeq ($(V),1)
Q =
else
Q = @
endif
MODULES = bpf_testmod.ko
obj-m += bpf_testmod.o
CFLAGS_bpf_testmod.o = -I$(src)
all:
+$(Q)make -C $(KDIR) M=$(BPF_TESTMOD_DIR) modules
clean:
+$(Q)make -C $(KDIR) M=$(BPF_TESTMOD_DIR) clean

36
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod-events.h Normal file
View File

@ -0,0 +1,36 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2020 Facebook */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM bpf_testmod
#if !defined(_BPF_TESTMOD_EVENTS_H) || defined(TRACE_HEADER_MULTI_READ)
#define _BPF_TESTMOD_EVENTS_H
#include <linux/tracepoint.h>
#include "bpf_testmod.h"
TRACE_EVENT(bpf_testmod_test_read,
TP_PROTO(struct task_struct *task, struct bpf_testmod_test_read_ctx *ctx),
TP_ARGS(task, ctx),
TP_STRUCT__entry(
__field(pid_t, pid)
__array(char, comm, TASK_COMM_LEN)
__field(loff_t, off)
__field(size_t, len)
),
TP_fast_assign(
__entry->pid = task->pid;
memcpy(__entry->comm, task->comm, TASK_COMM_LEN);
__entry->off = ctx->off;
__entry->len = ctx->len;
),
TP_printk("pid=%d comm=%s off=%llu len=%zu",
__entry->pid, __entry->comm, __entry->off, __entry->len)
);
#endif /* _BPF_TESTMOD_EVENTS_H */
#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH .
#define TRACE_INCLUDE_FILE bpf_testmod-events
#include <trace/define_trace.h>

52
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c Normal file
View File

@ -0,0 +1,52 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <linux/error-injection.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/sysfs.h>
#include <linux/tracepoint.h>
#include "bpf_testmod.h"
#define CREATE_TRACE_POINTS
#include "bpf_testmod-events.h"
noinline ssize_t
bpf_testmod_test_read(struct file *file, struct kobject *kobj,
struct bin_attribute *bin_attr,
char *buf, loff_t off, size_t len)
{
struct bpf_testmod_test_read_ctx ctx = {
.buf = buf,
.off = off,
.len = len,
};
trace_bpf_testmod_test_read(current, &ctx);
return -EIO; /* always fail */
}
EXPORT_SYMBOL(bpf_testmod_test_read);
ALLOW_ERROR_INJECTION(bpf_testmod_test_read, ERRNO);
static struct bin_attribute bin_attr_bpf_testmod_file __ro_after_init = {
.attr = { .name = "bpf_testmod", .mode = 0444, },
.read = bpf_testmod_test_read,
};
static int bpf_testmod_init(void)
{
return sysfs_create_bin_file(kernel_kobj, &bin_attr_bpf_testmod_file);
}
static void bpf_testmod_exit(void)
{
return sysfs_remove_bin_file(kernel_kobj, &bin_attr_bpf_testmod_file);
}
module_init(bpf_testmod_init);
module_exit(bpf_testmod_exit);
MODULE_AUTHOR("Andrii Nakryiko");
MODULE_DESCRIPTION("BPF selftests module");
MODULE_LICENSE("Dual BSD/GPL");

14
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.h Normal file
View File

@ -0,0 +1,14 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2020 Facebook */
#ifndef _BPF_TESTMOD_H
#define _BPF_TESTMOD_H
#include <linux/types.h>
struct bpf_testmod_test_read_ctx {
char *buf;
loff_t off;
size_t len;
};
#endif /* _BPF_TESTMOD_H */

80
tools/testing/selftests/bpf/prog_tests/core_reloc.c
View File

@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
#include <test_progs.h>
#include "progs/core_reloc_types.h"
#include "bpf_testmod/bpf_testmod.h"
#include <sys/mman.h>
#include <sys/syscall.h>
#include <bpf/btf.h>
@ -9,6 +10,30 @@ static int duration = 0;
#define STRUCT_TO_CHAR_PTR(struct_name) (const char *)&(struct struct_name)
#define MODULES_CASE(name, sec_name, tp_name) { \
.case_name = name, \
.bpf_obj_file = "test_core_reloc_module.o", \
.btf_src_file = NULL, /* find in kernel module BTFs */ \
.input = "", \
.input_len = 0, \
.output = STRUCT_TO_CHAR_PTR(core_reloc_module_output) { \
.read_ctx_sz = sizeof(struct bpf_testmod_test_read_ctx),\
.read_ctx_exists = true, \
.buf_exists = true, \
.len_exists = true, \
.off_exists = true, \
.len = 123, \
.off = 0, \
.comm = "test_progs", \
.comm_len = sizeof("test_progs"), \
}, \
.output_len = sizeof(struct core_reloc_module_output), \
.prog_sec_name = sec_name, \
.raw_tp_name = tp_name, \
.trigger = trigger_module_test_read, \
.needs_testmod = true, \
}
#define FLAVORS_DATA(struct_name) STRUCT_TO_CHAR_PTR(struct_name) { \
.a = 42, \
.b = 0xc001, \
@ -211,7 +236,7 @@ static int duration = 0;
.output = STRUCT_TO_CHAR_PTR(core_reloc_bitfields_output) \
__VA_ARGS__, \
.output_len = sizeof(struct core_reloc_bitfields_output), \
.direct_raw_tp = true, \
.prog_sec_name = "tp_btf/sys_enter", \
}
@ -222,7 +247,7 @@ static int duration = 0;
}, { \
BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.o", \
"direct:", name), \
.direct_raw_tp = true, \
.prog_sec_name = "tp_btf/sys_enter", \
.fails = true, \
}
@ -309,6 +334,7 @@ static int duration = 0;
struct core_reloc_test_case;
typedef int (*setup_test_fn)(struct core_reloc_test_case *test);
typedef int (*trigger_test_fn)(const struct core_reloc_test_case *test);
struct core_reloc_test_case {
const char *case_name;
@ -319,9 +345,12 @@ struct core_reloc_test_case {
const char *output;
int output_len;
bool fails;
bool needs_testmod;
bool relaxed_core_relocs;
bool direct_raw_tp;
const char *prog_sec_name;
const char *raw_tp_name;
setup_test_fn setup;
trigger_test_fn trigger;
};
static int find_btf_type(const struct btf *btf, const char *name, __u32 kind)
@ -451,6 +480,23 @@ static int setup_type_id_case_failure(struct core_reloc_test_case *test)
return 0;
}
static int trigger_module_test_read(const struct core_reloc_test_case *test)
{
struct core_reloc_module_output *exp = (void *)test->output;
int fd, err;
fd = open("/sys/kernel/bpf_testmod", O_RDONLY);
err = -errno;
if (CHECK(fd < 0, "testmod_file_open", "failed: %d\n", err))
return err;
read(fd, NULL, exp->len); /* request expected number of bytes */
close(fd);
return 0;
}
static struct core_reloc_test_case test_cases[] = {
/* validate we can find kernel image and use its BTF for relocs */
{
@ -467,6 +513,10 @@ static struct core_reloc_test_case test_cases[] = {
.output_len = sizeof(struct core_reloc_kernel_output),
},
/* validate we can find kernel module BTF types for relocs/attach */
MODULES_CASE("module_probed", "raw_tp/bpf_testmod_test_read", "bpf_testmod_test_read"),
MODULES_CASE("module_direct", "tp_btf/bpf_testmod_test_read", NULL),
/* validate BPF program can use multiple flavors to match against
* single target BTF type
*/
@ -779,6 +829,11 @@ void test_core_reloc(void)
if (!test__start_subtest(test_case->case_name))
continue;
if (test_case->needs_testmod && !env.has_testmod) {
test__skip();
continue;
}
if (test_case->setup) {
err = test_case->setup(test_case);
if (CHECK(err, "test_setup", "test #%d setup failed: %d\n", i, err))
@ -790,13 +845,11 @@ void test_core_reloc(void)
test_case->bpf_obj_file, PTR_ERR(obj)))
continue;
/* for typed raw tracepoints, NULL should be specified */
if (test_case->direct_raw_tp) {
probe_name = "tp_btf/sys_enter";
tp_name = NULL;
} else {
probe_name = "raw_tracepoint/sys_enter";
tp_name = "sys_enter";
probe_name = "raw_tracepoint/sys_enter";
tp_name = "sys_enter";
if (test_case->prog_sec_name) {
probe_name = test_case->prog_sec_name;
tp_name = test_case->raw_tp_name; /* NULL for tp_btf */
}
prog = bpf_object__find_program_by_title(obj, probe_name);
@ -837,7 +890,12 @@ void test_core_reloc(void)
goto cleanup;
/* trigger test run */
usleep(1);
if (test_case->trigger) {
if (!ASSERT_OK(test_case->trigger(test_case), "test_trigger"))
goto cleanup;
} else {
usleep(1);
}
if (data->skip) {
test__skip();

53
tools/testing/selftests/bpf/prog_tests/module_attach.c Normal file
View File

@ -0,0 +1,53 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <test_progs.h>
#include "test_module_attach.skel.h"
static int duration;
static int trigger_module_test_read(int read_sz)
{
int fd, err;
fd = open("/sys/kernel/bpf_testmod", O_RDONLY);
err = -errno;
if (CHECK(fd < 0, "testmod_file_open", "failed: %d\n", err))
return err;
read(fd, NULL, read_sz);
close(fd);
return 0;
}
void test_module_attach(void)
{
const int READ_SZ = 456;
struct test_module_attach* skel;
struct test_module_attach__bss *bss;
int err;
skel = test_module_attach__open_and_load();
if (CHECK(!skel, "skel_open", "failed to open skeleton\n"))
return;
bss = skel->bss;
err = test_module_attach__attach(skel);
if (CHECK(err, "skel_attach", "skeleton attach failed: %d\n", err))
goto cleanup;
/* trigger tracepoint */
ASSERT_OK(trigger_module_test_read(READ_SZ), "trigger_read");
ASSERT_EQ(bss->raw_tp_read_sz, READ_SZ, "raw_tp");
ASSERT_EQ(bss->tp_btf_read_sz, READ_SZ, "tp_btf");
ASSERT_EQ(bss->fentry_read_sz, READ_SZ, "fentry");
ASSERT_EQ(bss->fexit_read_sz, READ_SZ, "fexit");
ASSERT_EQ(bss->fexit_ret, -EIO, "fexit_tet");
ASSERT_EQ(bss->fmod_ret_read_sz, READ_SZ, "fmod_ret");
cleanup:
test_module_attach__destroy(skel);
}

17
tools/testing/selftests/bpf/progs/core_reloc_types.h
View File

@ -15,6 +15,23 @@ struct core_reloc_kernel_output {
int comm_len;
};
/*
* MODULE
*/
struct core_reloc_module_output {
long long len;
long long off;
int read_ctx_sz;
bool read_ctx_exists;
bool buf_exists;
bool len_exists;
bool off_exists;
/* we have test_progs[-flavor], so cut flavor part */
char comm[sizeof("test_progs")];
int comm_len;
};
/*
* FLAVORS
*/

96
tools/testing/selftests/bpf/progs/test_core_reloc_module.c Normal file
View File

@ -0,0 +1,96 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_core_read.h>
#include <bpf/bpf_tracing.h>
char _license[] SEC("license") = "GPL";
struct bpf_testmod_test_read_ctx {
/* field order is mixed up */
size_t len;
char *buf;
loff_t off;
} __attribute__((preserve_access_index));
struct {
char in[256];
char out[256];
bool skip;
uint64_t my_pid_tgid;
} data = {};
struct core_reloc_module_output {
long long len;
long long off;
int read_ctx_sz;
bool read_ctx_exists;
bool buf_exists;
bool len_exists;
bool off_exists;
/* we have test_progs[-flavor], so cut flavor part */
char comm[sizeof("test_progs")];
int comm_len;
};
SEC("raw_tp/bpf_testmod_test_read")
int BPF_PROG(test_core_module_probed,
struct task_struct *task,
struct bpf_testmod_test_read_ctx *read_ctx)
{
struct core_reloc_module_output *out = (void *)&data.out;
__u64 pid_tgid = bpf_get_current_pid_tgid();
__u32 real_tgid = (__u32)(pid_tgid >> 32);
__u32 real_pid = (__u32)pid_tgid;
if (data.my_pid_tgid != pid_tgid)
return 0;
if (BPF_CORE_READ(task, pid) != real_pid || BPF_CORE_READ(task, tgid) != real_tgid)
return 0;
out->len = BPF_CORE_READ(read_ctx, len);
out->off = BPF_CORE_READ(read_ctx, off);
out->read_ctx_sz = bpf_core_type_size(struct bpf_testmod_test_read_ctx);
out->read_ctx_exists = bpf_core_type_exists(struct bpf_testmod_test_read_ctx);
out->buf_exists = bpf_core_field_exists(read_ctx->buf);
out->off_exists = bpf_core_field_exists(read_ctx->off);
out->len_exists = bpf_core_field_exists(read_ctx->len);
out->comm_len = BPF_CORE_READ_STR_INTO(&out->comm, task, comm);
return 0;
}
SEC("tp_btf/bpf_testmod_test_read")
int BPF_PROG(test_core_module_direct,
struct task_struct *task,
struct bpf_testmod_test_read_ctx *read_ctx)
{
struct core_reloc_module_output *out = (void *)&data.out;
__u64 pid_tgid = bpf_get_current_pid_tgid();
__u32 real_tgid = (__u32)(pid_tgid >> 32);
__u32 real_pid = (__u32)pid_tgid;
if (data.my_pid_tgid != pid_tgid)
return 0;
if (task->pid != real_pid || task->tgid != real_tgid)
return 0;
out->len = read_ctx->len;
out->off = read_ctx->off;
out->read_ctx_sz = bpf_core_type_size(struct bpf_testmod_test_read_ctx);
out->read_ctx_exists = bpf_core_type_exists(struct bpf_testmod_test_read_ctx);
out->buf_exists = bpf_core_field_exists(read_ctx->buf);
out->off_exists = bpf_core_field_exists(read_ctx->off);
out->len_exists = bpf_core_field_exists(read_ctx->len);
out->comm_len = BPF_CORE_READ_STR_INTO(&out->comm, task, comm);
return 0;
}

66
tools/testing/selftests/bpf/progs/test_module_attach.c Normal file
View File

@ -0,0 +1,66 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_core_read.h>
#include "../bpf_testmod/bpf_testmod.h"
__u32 raw_tp_read_sz = 0;
SEC("raw_tp/bpf_testmod_test_read")
int BPF_PROG(handle_raw_tp,
struct task_struct *task, struct bpf_testmod_test_read_ctx *read_ctx)
{
raw_tp_read_sz = BPF_CORE_READ(read_ctx, len);
return 0;
}
__u32 tp_btf_read_sz = 0;
SEC("tp_btf/bpf_testmod_test_read")
int BPF_PROG(handle_tp_btf,
struct task_struct *task, struct bpf_testmod_test_read_ctx *read_ctx)
{
tp_btf_read_sz = read_ctx->len;
return 0;
}
__u32 fentry_read_sz = 0;
SEC("fentry/bpf_testmod_test_read")
int BPF_PROG(handle_fentry,
struct file *file, struct kobject *kobj,
struct bin_attribute *bin_attr, char *buf, loff_t off, size_t len)
{
fentry_read_sz = len;
return 0;
}
__u32 fexit_read_sz = 0;
int fexit_ret = 0;
SEC("fexit/bpf_testmod_test_read")
int BPF_PROG(handle_fexit,
struct file *file, struct kobject *kobj,
struct bin_attribute *bin_attr, char *buf, loff_t off, size_t len,
int ret)
{
fexit_read_sz = len;
fexit_ret = ret;
return 0;
}
__u32 fmod_ret_read_sz = 0;
SEC("fmod_ret/bpf_testmod_test_read")
int BPF_PROG(handle_fmod_ret,
struct file *file, struct kobject *kobj,
struct bin_attribute *bin_attr, char *buf, loff_t off, size_t len)
{
fmod_ret_read_sz = len;
return 0; /* don't override the exit code */
}
char _license[] SEC("license") = "GPL";

65
tools/testing/selftests/bpf/test_progs.c
View File

@ -149,15 +149,15 @@ void test__end_subtest()
if (sub_error_cnt)
env.fail_cnt++;
else
else if (test->skip_cnt == 0)
env.sub_succ_cnt++;
skip_account();
dump_test_log(test, sub_error_cnt);
fprintf(env.stdout, "#%d/%d %s:%s\n",
test->test_num, test->subtest_num,
test->subtest_name, sub_error_cnt ? "FAIL" : "OK");
test->test_num, test->subtest_num, test->subtest_name,
sub_error_cnt ? "FAIL" : (test->skip_cnt ? "SKIP" : "OK"));
free(test->subtest_name);
test->subtest_name = NULL;
@ -360,6 +360,58 @@ err:
return -1;
}
static int finit_module(int fd, const char *param_values, int flags)
{
return syscall(__NR_finit_module, fd, param_values, flags);
}
static int delete_module(const char *name, int flags)
{
return syscall(__NR_delete_module, name, flags);
}
static void unload_bpf_testmod(void)
{
if (delete_module("bpf_testmod", 0)) {
if (errno == ENOENT) {
if (env.verbosity > VERBOSE_NONE)
fprintf(stdout, "bpf_testmod.ko is already unloaded.\n");
return;
}
fprintf(env.stderr, "Failed to unload bpf_testmod.ko from kernel: %d\n", -errno);
exit(1);
}
if (env.verbosity > VERBOSE_NONE)
fprintf(stdout, "Successfully unloaded bpf_testmod.ko.\n");
}
static int load_bpf_testmod(void)
{
int fd;
/* ensure previous instance of the module is unloaded */
unload_bpf_testmod();
if (env.verbosity > VERBOSE_NONE)
fprintf(stdout, "Loading bpf_testmod.ko...\n");
fd = open("bpf_testmod.ko", O_RDONLY);
if (fd < 0) {
fprintf(env.stderr, "Can't find bpf_testmod.ko kernel module: %d\n", -errno);
return -ENOENT;
}
if (finit_module(fd, "", 0)) {
fprintf(env.stderr, "Failed to load bpf_testmod.ko into the kernel: %d\n", -errno);
close(fd);
return -EINVAL;
}
close(fd);
if (env.verbosity > VERBOSE_NONE)
fprintf(stdout, "Successfully loaded bpf_testmod.ko.\n");
return 0;
}
/* extern declarations for test funcs */
#define DEFINE_TEST(name) extern void test_##name(void);
#include <prog_tests/tests.h>
@ -678,6 +730,11 @@ int main(int argc, char **argv)
save_netns();
stdio_hijack();
env.has_testmod = true;
if (load_bpf_testmod()) {
fprintf(env.stderr, "WARNING! Selftests relying on bpf_testmod.ko will be skipped.\n");
env.has_testmod = false;
}
for (i = 0; i < prog_test_cnt; i++) {
struct prog_test_def *test = &prog_test_defs[i];
@ -722,6 +779,8 @@ int main(int argc, char **argv)
if (test->need_cgroup_cleanup)
cleanup_cgroup_environment();
}
if (env.has_testmod)
unload_bpf_testmod();
stdio_restore();
if (env.get_test_cnt) {

1
tools/testing/selftests/bpf/test_progs.h
View File

@ -66,6 +66,7 @@ struct test_env {
enum verbosity verbosity;
bool jit_enabled;
bool has_testmod;
bool get_test_cnt;
bool list_test_names;