Merge git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next
Pull networking updates from David Miller:
"Another merge window, another pull full of stuff:
1) Support alternative names for network devices, from Jiri Pirko.
2) Introduce per-netns netdev notifiers, also from Jiri Pirko.
3) Support MSG_PEEK in vsock/virtio, from Matias Ezequiel Vara
Larsen.
4) Allow compiling out the TLS TOE code, from Jakub Kicinski.
5) Add several new tracepoints to the kTLS code, also from Jakub.
6) Support set channels ethtool callback in ena driver, from Sameeh
Jubran.
7) New SCTP events SCTP_ADDR_ADDED, SCTP_ADDR_REMOVED,
SCTP_ADDR_MADE_PRIM, and SCTP_SEND_FAILED_EVENT. From Xin Long.
8) Add XDP support to mvneta driver, from Lorenzo Bianconi.
9) Lots of netfilter hw offload fixes, cleanups and enhancements,
from Pablo Neira Ayuso.
10) PTP support for aquantia chips, from Egor Pomozov.
11) Add UDP segmentation offload support to igb, ixgbe, and i40e. From
Josh Hunt.
12) Add smart nagle to tipc, from Jon Maloy.
13) Support L2 field rewrite by TC offloads in bnxt_en, from Venkat
Duvvuru.
14) Add a flow mask cache to OVS, from Tonghao Zhang.
15) Add XDP support to ice driver, from Maciej Fijalkowski.
16) Add AF_XDP support to ice driver, from Krzysztof Kazimierczak.
17) Support UDP GSO offload in atlantic driver, from Igor Russkikh.
18) Support it in stmmac driver too, from Jose Abreu.
19) Support TIPC encryption and auth, from Tuong Lien.
20) Introduce BPF trampolines, from Alexei Starovoitov.
21) Make page_pool API more numa friendly, from Saeed Mahameed.
22) Introduce route hints to ipv4 and ipv6, from Paolo Abeni.
23) Add UDP segmentation offload to cxgb4, Rahul Lakkireddy"
* git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next: (1857 commits)
libbpf: Fix usage of u32 in userspace code
mm: Implement no-MMU variant of vmalloc_user_node_flags
slip: Fix use-after-free Read in slip_open
net: dsa: sja1105: fix sja1105_parse_rgmii_delays()
macvlan: schedule bc_work even if error
enetc: add support Credit Based Shaper(CBS) for hardware offload
net: phy: add helpers phy_(un)lock_mdio_bus
mdio_bus: don't use managed reset-controller
ax88179_178a: add ethtool_op_get_ts_info()
mlxsw: spectrum_router: Fix use of uninitialized adjacency index
mlxsw: spectrum_router: After underlay moves, demote conflicting tunnels
bpf: Simplify __bpf_arch_text_poke poke type handling
bpf: Introduce BPF_TRACE_x helper for the tracing tests
bpf: Add bpf_jit_blinding_enabled for !CONFIG_BPF_JIT
bpf, testing: Add various tail call test cases
bpf, x86: Emit patchable direct jump as tail call
bpf: Constant map key tracking for prog array pokes
bpf: Add poke dependency tracking for prog array maps
bpf: Add initial poke descriptor table for jit images
bpf: Move owner type, jited info into array auxiliary data
...
This commit is contained in:
Linus Torvalds
4
tools/lib/bpf/.gitignore
vendored
4
tools/lib/bpf/.gitignore
vendored
@ -3,3 +3,7 @@ libbpf.pc
|
||||
FEATURE-DUMP.libbpf
|
||||
test_libbpf
|
||||
libbpf.so.*
|
||||
TAGS
|
||||
tags
|
||||
cscope.*
|
||||
/bpf_helper_defs.h
|
||||
|
||||
@ -56,7 +56,7 @@ ifndef VERBOSE
|
||||
endif
|
||||
|
||||
FEATURE_USER = .libbpf
|
||||
FEATURE_TESTS = libelf libelf-mmap bpf reallocarray cxx
|
||||
FEATURE_TESTS = libelf libelf-mmap bpf reallocarray
|
||||
FEATURE_DISPLAY = libelf bpf
|
||||
|
||||
INCLUDES = -I. -I$(srctree)/tools/include -I$(srctree)/tools/arch/$(ARCH)/include/uapi -I$(srctree)/tools/include/uapi
|
||||
@ -143,6 +143,8 @@ LIB_TARGET := $(addprefix $(OUTPUT),$(LIB_TARGET))
|
||||
LIB_FILE := $(addprefix $(OUTPUT),$(LIB_FILE))
|
||||
PC_FILE := $(addprefix $(OUTPUT),$(PC_FILE))
|
||||
|
||||
TAGS_PROG := $(if $(shell which etags 2>/dev/null),etags,ctags)
|
||||
|
||||
GLOBAL_SYM_COUNT = $(shell readelf -s --wide $(BPF_IN_SHARED) | \
|
||||
cut -d "@" -f1 | sed 's/_v[0-9]_[0-9]_[0-9].*//' | \
|
||||
awk '/GLOBAL/ && /DEFAULT/ && !/UND/ {print $$8}' | \
|
||||
@ -150,22 +152,14 @@ GLOBAL_SYM_COUNT = $(shell readelf -s --wide $(BPF_IN_SHARED) | \
|
||||
VERSIONED_SYM_COUNT = $(shell readelf -s --wide $(OUTPUT)libbpf.so | \
|
||||
grep -Eo '[^ ]+@LIBBPF_' | cut -d@ -f1 | sort -u | wc -l)
|
||||
|
||||
CMD_TARGETS = $(LIB_TARGET) $(PC_FILE)
|
||||
|
||||
CXX_TEST_TARGET = $(OUTPUT)test_libbpf
|
||||
|
||||
ifeq ($(feature-cxx), 1)
|
||||
CMD_TARGETS += $(CXX_TEST_TARGET)
|
||||
endif
|
||||
|
||||
TARGETS = $(CMD_TARGETS)
|
||||
CMD_TARGETS = $(LIB_TARGET) $(PC_FILE) $(OUTPUT)test_libbpf
|
||||
|
||||
all: fixdep
|
||||
$(Q)$(MAKE) all_cmd
|
||||
|
||||
all_cmd: $(CMD_TARGETS) check
|
||||
|
||||
$(BPF_IN_SHARED): force elfdep bpfdep
|
||||
$(BPF_IN_SHARED): force elfdep bpfdep bpf_helper_defs.h
|
||||
@(test -f ../../include/uapi/linux/bpf.h -a -f ../../../include/uapi/linux/bpf.h && ( \
|
||||
(diff -B ../../include/uapi/linux/bpf.h ../../../include/uapi/linux/bpf.h >/dev/null) || \
|
||||
echo "Warning: Kernel ABI header at 'tools/include/uapi/linux/bpf.h' differs from latest version at 'include/uapi/linux/bpf.h'" >&2 )) || true
|
||||
@ -183,22 +177,27 @@ $(BPF_IN_SHARED): force elfdep bpfdep
|
||||
echo "Warning: Kernel ABI header at 'tools/include/uapi/linux/if_xdp.h' differs from latest version at 'include/uapi/linux/if_xdp.h'" >&2 )) || true
|
||||
$(Q)$(MAKE) $(build)=libbpf OUTPUT=$(SHARED_OBJDIR) CFLAGS="$(CFLAGS) $(SHLIB_FLAGS)"
|
||||
|
||||
$(BPF_IN_STATIC): force elfdep bpfdep
|
||||
$(BPF_IN_STATIC): force elfdep bpfdep bpf_helper_defs.h
|
||||
$(Q)$(MAKE) $(build)=libbpf OUTPUT=$(STATIC_OBJDIR)
|
||||
|
||||
bpf_helper_defs.h: $(srctree)/include/uapi/linux/bpf.h
|
||||
$(Q)$(srctree)/scripts/bpf_helpers_doc.py --header \
|
||||
--file $(srctree)/include/uapi/linux/bpf.h > bpf_helper_defs.h
|
||||
|
||||
$(OUTPUT)libbpf.so: $(OUTPUT)libbpf.so.$(LIBBPF_VERSION)
|
||||
|
||||
$(OUTPUT)libbpf.so.$(LIBBPF_VERSION): $(BPF_IN_SHARED)
|
||||
$(QUIET_LINK)$(CC) --shared -Wl,-soname,libbpf.so.$(LIBBPF_MAJOR_VERSION) \
|
||||
-Wl,--version-script=$(VERSION_SCRIPT) $^ -lelf -o $@
|
||||
$(QUIET_LINK)$(CC) $(LDFLAGS) \
|
||||
--shared -Wl,-soname,libbpf.so.$(LIBBPF_MAJOR_VERSION) \
|
||||
-Wl,--version-script=$(VERSION_SCRIPT) $^ -lelf -o $@
|
||||
@ln -sf $(@F) $(OUTPUT)libbpf.so
|
||||
@ln -sf $(@F) $(OUTPUT)libbpf.so.$(LIBBPF_MAJOR_VERSION)
|
||||
|
||||
$(OUTPUT)libbpf.a: $(BPF_IN_STATIC)
|
||||
$(QUIET_LINK)$(RM) $@; $(AR) rcs $@ $^
|
||||
|
||||
$(OUTPUT)test_libbpf: test_libbpf.cpp $(OUTPUT)libbpf.a
|
||||
$(QUIET_LINK)$(CXX) $(INCLUDES) $^ -lelf -o $@
|
||||
$(OUTPUT)test_libbpf: test_libbpf.c $(OUTPUT)libbpf.a
|
||||
$(QUIET_LINK)$(CC) $(CFLAGS) $(LDFLAGS) $(INCLUDES) $^ -lelf -o $@
|
||||
|
||||
$(OUTPUT)libbpf.pc:
|
||||
$(QUIET_GEN)sed -e "s|@PREFIX@|$(prefix)|" \
|
||||
@ -247,13 +246,18 @@ install_lib: all_cmd
|
||||
$(call do_install_mkdir,$(libdir_SQ)); \
|
||||
cp -fpR $(LIB_FILE) $(DESTDIR)$(libdir_SQ)
|
||||
|
||||
install_headers:
|
||||
install_headers: bpf_helper_defs.h
|
||||
$(call QUIET_INSTALL, headers) \
|
||||
$(call do_install,bpf.h,$(prefix)/include/bpf,644); \
|
||||
$(call do_install,libbpf.h,$(prefix)/include/bpf,644); \
|
||||
$(call do_install,btf.h,$(prefix)/include/bpf,644); \
|
||||
$(call do_install,libbpf_util.h,$(prefix)/include/bpf,644); \
|
||||
$(call do_install,xsk.h,$(prefix)/include/bpf,644);
|
||||
$(call do_install,xsk.h,$(prefix)/include/bpf,644); \
|
||||
$(call do_install,bpf_helpers.h,$(prefix)/include/bpf,644); \
|
||||
$(call do_install,bpf_helper_defs.h,$(prefix)/include/bpf,644); \
|
||||
$(call do_install,bpf_tracing.h,$(prefix)/include/bpf,644); \
|
||||
$(call do_install,bpf_endian.h,$(prefix)/include/bpf,644); \
|
||||
$(call do_install,bpf_core_read.h,$(prefix)/include/bpf,644);
|
||||
|
||||
install_pkgconfig: $(PC_FILE)
|
||||
$(call QUIET_INSTALL, $(PC_FILE)) \
|
||||
@ -268,14 +272,15 @@ config-clean:
|
||||
$(Q)$(MAKE) -C $(srctree)/tools/build/feature/ clean >/dev/null
|
||||
|
||||
clean:
|
||||
$(call QUIET_CLEAN, libbpf) $(RM) -rf $(TARGETS) $(CXX_TEST_TARGET) \
|
||||
$(call QUIET_CLEAN, libbpf) $(RM) -rf $(CMD_TARGETS) \
|
||||
*.o *~ *.a *.so *.so.$(LIBBPF_MAJOR_VERSION) .*.d .*.cmd \
|
||||
*.pc LIBBPF-CFLAGS $(SHARED_OBJDIR) $(STATIC_OBJDIR)
|
||||
*.pc LIBBPF-CFLAGS bpf_helper_defs.h \
|
||||
$(SHARED_OBJDIR) $(STATIC_OBJDIR)
|
||||
$(call QUIET_CLEAN, core-gen) $(RM) $(OUTPUT)FEATURE-DUMP.libbpf
|
||||
|
||||
|
||||
|
||||
PHONY += force elfdep bpfdep
|
||||
PHONY += force elfdep bpfdep cscope tags
|
||||
force:
|
||||
|
||||
elfdep:
|
||||
@ -284,6 +289,17 @@ elfdep:
|
||||
bpfdep:
|
||||
@if [ "$(feature-bpf)" != "1" ]; then echo "BPF API too old"; exit 1 ; fi
|
||||
|
||||
cscope:
|
||||
ls *.c *.h > cscope.files
|
||||
cscope -b -q -I $(srctree)/include -f cscope.out
|
||||
|
||||
tags:
|
||||
rm -f TAGS tags
|
||||
ls *.c *.h | xargs $(TAGS_PROG) -a
|
||||
|
||||
# Declare the contents of the .PHONY variable as phony. We keep that
|
||||
# information in a variable so we can use it in if_changed and friends.
|
||||
.PHONY: $(PHONY)
|
||||
|
||||
# Delete partially updated (corrupted) files on error
|
||||
.DELETE_ON_ERROR:
|
||||
|
||||
@ -189,7 +189,7 @@ static void *
|
||||
alloc_zero_tailing_info(const void *orecord, __u32 cnt,
|
||||
__u32 actual_rec_size, __u32 expected_rec_size)
|
||||
{
|
||||
__u64 info_len = actual_rec_size * cnt;
|
||||
__u64 info_len = (__u64)actual_rec_size * cnt;
|
||||
void *info, *nrecord;
|
||||
int i;
|
||||
|
||||
@ -228,6 +228,13 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
|
||||
memset(&attr, 0, sizeof(attr));
|
||||
attr.prog_type = load_attr->prog_type;
|
||||
attr.expected_attach_type = load_attr->expected_attach_type;
|
||||
if (attr.prog_type == BPF_PROG_TYPE_TRACING) {
|
||||
attr.attach_btf_id = load_attr->attach_btf_id;
|
||||
attr.attach_prog_fd = load_attr->attach_prog_fd;
|
||||
} else {
|
||||
attr.prog_ifindex = load_attr->prog_ifindex;
|
||||
attr.kern_version = load_attr->kern_version;
|
||||
}
|
||||
attr.insn_cnt = (__u32)load_attr->insns_cnt;
|
||||
attr.insns = ptr_to_u64(load_attr->insns);
|
||||
attr.license = ptr_to_u64(load_attr->license);
|
||||
@ -241,8 +248,6 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
|
||||
attr.log_size = 0;
|
||||
}
|
||||
|
||||
attr.kern_version = load_attr->kern_version;
|
||||
attr.prog_ifindex = load_attr->prog_ifindex;
|
||||
attr.prog_btf_fd = load_attr->prog_btf_fd;
|
||||
attr.func_info_rec_size = load_attr->func_info_rec_size;
|
||||
attr.func_info_cnt = load_attr->func_info_cnt;
|
||||
|
||||
@ -77,8 +77,14 @@ struct bpf_load_program_attr {
|
||||
const struct bpf_insn *insns;
|
||||
size_t insns_cnt;
|
||||
const char *license;
|
||||
__u32 kern_version;
|
||||
__u32 prog_ifindex;
|
||||
union {
|
||||
__u32 kern_version;
|
||||
__u32 attach_prog_fd;
|
||||
};
|
||||
union {
|
||||
__u32 prog_ifindex;
|
||||
__u32 attach_btf_id;
|
||||
};
|
||||
__u32 prog_btf_fd;
|
||||
__u32 func_info_rec_size;
|
||||
const void *func_info;
|
||||
|
||||
263
tools/lib/bpf/bpf_core_read.h
Normal file
263
tools/lib/bpf/bpf_core_read.h
Normal file
@ -0,0 +1,263 @@
|
||||
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
|
||||
#ifndef __BPF_CORE_READ_H__
|
||||
#define __BPF_CORE_READ_H__
|
||||
|
||||
/*
|
||||
* enum bpf_field_info_kind is passed as a second argument into
|
||||
* __builtin_preserve_field_info() built-in to get a specific aspect of
|
||||
* a field, captured as a first argument. __builtin_preserve_field_info(field,
|
||||
* info_kind) returns __u32 integer and produces BTF field relocation, which
|
||||
* is understood and processed by libbpf during BPF object loading. See
|
||||
* selftests/bpf for examples.
|
||||
*/
|
||||
enum bpf_field_info_kind {
|
||||
BPF_FIELD_BYTE_OFFSET = 0, /* field byte offset */
|
||||
BPF_FIELD_BYTE_SIZE = 1,
|
||||
BPF_FIELD_EXISTS = 2, /* field existence in target kernel */
|
||||
BPF_FIELD_SIGNED = 3,
|
||||
BPF_FIELD_LSHIFT_U64 = 4,
|
||||
BPF_FIELD_RSHIFT_U64 = 5,
|
||||
};
|
||||
|
||||
#define __CORE_RELO(src, field, info) \
|
||||
__builtin_preserve_field_info((src)->field, BPF_FIELD_##info)
|
||||
|
||||
#if __BYTE_ORDER == __LITTLE_ENDIAN
|
||||
#define __CORE_BITFIELD_PROBE_READ(dst, src, fld) \
|
||||
bpf_probe_read((void *)dst, \
|
||||
__CORE_RELO(src, fld, BYTE_SIZE), \
|
||||
(const void *)src + __CORE_RELO(src, fld, BYTE_OFFSET))
|
||||
#else
|
||||
/* semantics of LSHIFT_64 assumes loading values into low-ordered bytes, so
|
||||
* for big-endian we need to adjust destination pointer accordingly, based on
|
||||
* field byte size
|
||||
*/
|
||||
#define __CORE_BITFIELD_PROBE_READ(dst, src, fld) \
|
||||
bpf_probe_read((void *)dst + (8 - __CORE_RELO(src, fld, BYTE_SIZE)), \
|
||||
__CORE_RELO(src, fld, BYTE_SIZE), \
|
||||
(const void *)src + __CORE_RELO(src, fld, BYTE_OFFSET))
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Extract bitfield, identified by s->field, and return its value as u64.
|
||||
* All this is done in relocatable manner, so bitfield changes such as
|
||||
* signedness, bit size, offset changes, this will be handled automatically.
|
||||
* This version of macro is using bpf_probe_read() to read underlying integer
|
||||
* storage. Macro functions as an expression and its return type is
|
||||
* bpf_probe_read()'s return value: 0, on success, <0 on error.
|
||||
*/
|
||||
#define BPF_CORE_READ_BITFIELD_PROBED(s, field) ({ \
|
||||
unsigned long long val = 0; \
|
||||
\
|
||||
__CORE_BITFIELD_PROBE_READ(&val, s, field); \
|
||||
val <<= __CORE_RELO(s, field, LSHIFT_U64); \
|
||||
if (__CORE_RELO(s, field, SIGNED)) \
|
||||
val = ((long long)val) >> __CORE_RELO(s, field, RSHIFT_U64); \
|
||||
else \
|
||||
val = val >> __CORE_RELO(s, field, RSHIFT_U64); \
|
||||
val; \
|
||||
})
|
||||
|
||||
/*
|
||||
* Extract bitfield, identified by s->field, and return its value as u64.
|
||||
* This version of macro is using direct memory reads and should be used from
|
||||
* BPF program types that support such functionality (e.g., typed raw
|
||||
* tracepoints).
|
||||
*/
|
||||
#define BPF_CORE_READ_BITFIELD(s, field) ({ \
|
||||
const void *p = (const void *)s + __CORE_RELO(s, field, BYTE_OFFSET); \
|
||||
unsigned long long val; \
|
||||
\
|
||||
switch (__CORE_RELO(s, field, BYTE_SIZE)) { \
|
||||
case 1: val = *(const unsigned char *)p; \
|
||||
case 2: val = *(const unsigned short *)p; \
|
||||
case 4: val = *(const unsigned int *)p; \
|
||||
case 8: val = *(const unsigned long long *)p; \
|
||||
} \
|
||||
val <<= __CORE_RELO(s, field, LSHIFT_U64); \
|
||||
if (__CORE_RELO(s, field, SIGNED)) \
|
||||
val = ((long long)val) >> __CORE_RELO(s, field, RSHIFT_U64); \
|
||||
else \
|
||||
val = val >> __CORE_RELO(s, field, RSHIFT_U64); \
|
||||
val; \
|
||||
})
|
||||
|
||||
/*
|
||||
* Convenience macro to check that field actually exists in target kernel's.
|
||||
* Returns:
|
||||
* 1, if matching field is present in target kernel;
|
||||
* 0, if no matching field found.
|
||||
*/
|
||||
#define bpf_core_field_exists(field) \
|
||||
__builtin_preserve_field_info(field, BPF_FIELD_EXISTS)
|
||||
|
||||
/*
|
||||
* Convenience macro to get byte size of a field. Works for integers,
|
||||
* struct/unions, pointers, arrays, and enums.
|
||||
*/
|
||||
#define bpf_core_field_size(field) \
|
||||
__builtin_preserve_field_info(field, BPF_FIELD_BYTE_SIZE)
|
||||
|
||||
/*
|
||||
* bpf_core_read() abstracts away bpf_probe_read() call and captures offset
|
||||
* relocation for source address using __builtin_preserve_access_index()
|
||||
* built-in, provided by Clang.
|
||||
*
|
||||
* __builtin_preserve_access_index() takes as an argument an expression of
|
||||
* taking an address of a field within struct/union. It makes compiler emit
|
||||
* a relocation, which records BTF type ID describing root struct/union and an
|
||||
* accessor string which describes exact embedded field that was used to take
|
||||
* an address. See detailed description of this relocation format and
|
||||
* semantics in comments to struct bpf_field_reloc in libbpf_internal.h.
|
||||
*
|
||||
* This relocation allows libbpf to adjust BPF instruction to use correct
|
||||
* actual field offset, based on target kernel BTF type that matches original
|
||||
* (local) BTF, used to record relocation.
|
||||
*/
|
||||
#define bpf_core_read(dst, sz, src) \
|
||||
bpf_probe_read(dst, sz, \
|
||||
(const void *)__builtin_preserve_access_index(src))
|
||||
|
||||
/*
|
||||
* bpf_core_read_str() is a thin wrapper around bpf_probe_read_str()
|
||||
* additionally emitting BPF CO-RE field relocation for specified source
|
||||
* argument.
|
||||
*/
|
||||
#define bpf_core_read_str(dst, sz, src) \
|
||||
bpf_probe_read_str(dst, sz, \
|
||||
(const void *)__builtin_preserve_access_index(src))
|
||||
|
||||
#define ___concat(a, b) a ## b
|
||||
#define ___apply(fn, n) ___concat(fn, n)
|
||||
#define ___nth(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, __11, N, ...) N
|
||||
|
||||
/*
|
||||
* return number of provided arguments; used for switch-based variadic macro
|
||||
* definitions (see ___last, ___arrow, etc below)
|
||||
*/
|
||||
#define ___narg(...) ___nth(_, ##__VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
|
||||
/*
|
||||
* return 0 if no arguments are passed, N - otherwise; used for
|
||||
* recursively-defined macros to specify termination (0) case, and generic
|
||||
* (N) case (e.g., ___read_ptrs, ___core_read)
|
||||
*/
|
||||
#define ___empty(...) ___nth(_, ##__VA_ARGS__, N, N, N, N, N, N, N, N, N, N, 0)
|
||||
|
||||
#define ___last1(x) x
|
||||
#define ___last2(a, x) x
|
||||
#define ___last3(a, b, x) x
|
||||
#define ___last4(a, b, c, x) x
|
||||
#define ___last5(a, b, c, d, x) x
|
||||
#define ___last6(a, b, c, d, e, x) x
|
||||
#define ___last7(a, b, c, d, e, f, x) x
|
||||
#define ___last8(a, b, c, d, e, f, g, x) x
|
||||
#define ___last9(a, b, c, d, e, f, g, h, x) x
|
||||
#define ___last10(a, b, c, d, e, f, g, h, i, x) x
|
||||
#define ___last(...) ___apply(___last, ___narg(__VA_ARGS__))(__VA_ARGS__)
|
||||
|
||||
#define ___nolast2(a, _) a
|
||||
#define ___nolast3(a, b, _) a, b
|
||||
#define ___nolast4(a, b, c, _) a, b, c
|
||||
#define ___nolast5(a, b, c, d, _) a, b, c, d
|
||||
#define ___nolast6(a, b, c, d, e, _) a, b, c, d, e
|
||||
#define ___nolast7(a, b, c, d, e, f, _) a, b, c, d, e, f
|
||||
#define ___nolast8(a, b, c, d, e, f, g, _) a, b, c, d, e, f, g
|
||||
#define ___nolast9(a, b, c, d, e, f, g, h, _) a, b, c, d, e, f, g, h
|
||||
#define ___nolast10(a, b, c, d, e, f, g, h, i, _) a, b, c, d, e, f, g, h, i
|
||||
#define ___nolast(...) ___apply(___nolast, ___narg(__VA_ARGS__))(__VA_ARGS__)
|
||||
|
||||
#define ___arrow1(a) a
|
||||
#define ___arrow2(a, b) a->b
|
||||
#define ___arrow3(a, b, c) a->b->c
|
||||
#define ___arrow4(a, b, c, d) a->b->c->d
|
||||
#define ___arrow5(a, b, c, d, e) a->b->c->d->e
|
||||
#define ___arrow6(a, b, c, d, e, f) a->b->c->d->e->f
|
||||
#define ___arrow7(a, b, c, d, e, f, g) a->b->c->d->e->f->g
|
||||
#define ___arrow8(a, b, c, d, e, f, g, h) a->b->c->d->e->f->g->h
|
||||
#define ___arrow9(a, b, c, d, e, f, g, h, i) a->b->c->d->e->f->g->h->i
|
||||
#define ___arrow10(a, b, c, d, e, f, g, h, i, j) a->b->c->d->e->f->g->h->i->j
|
||||
#define ___arrow(...) ___apply(___arrow, ___narg(__VA_ARGS__))(__VA_ARGS__)
|
||||
|
||||
#define ___type(...) typeof(___arrow(__VA_ARGS__))
|
||||
|
||||
#define ___read(read_fn, dst, src_type, src, accessor) \
|
||||
read_fn((void *)(dst), sizeof(*(dst)), &((src_type)(src))->accessor)
|
||||
|
||||
/* "recursively" read a sequence of inner pointers using local __t var */
|
||||
#define ___rd_first(src, a) ___read(bpf_core_read, &__t, ___type(src), src, a);
|
||||
#define ___rd_last(...) \
|
||||
___read(bpf_core_read, &__t, \
|
||||
___type(___nolast(__VA_ARGS__)), __t, ___last(__VA_ARGS__));
|
||||
#define ___rd_p1(...) const void *__t; ___rd_first(__VA_ARGS__)
|
||||
#define ___rd_p2(...) ___rd_p1(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__)
|
||||
#define ___rd_p3(...) ___rd_p2(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__)
|
||||
#define ___rd_p4(...) ___rd_p3(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__)
|
||||
#define ___rd_p5(...) ___rd_p4(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__)
|
||||
#define ___rd_p6(...) ___rd_p5(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__)
|
||||
#define ___rd_p7(...) ___rd_p6(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__)
|
||||
#define ___rd_p8(...) ___rd_p7(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__)
|
||||
#define ___rd_p9(...) ___rd_p8(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__)
|
||||
#define ___read_ptrs(src, ...) \
|
||||
___apply(___rd_p, ___narg(__VA_ARGS__))(src, __VA_ARGS__)
|
||||
|
||||
#define ___core_read0(fn, dst, src, a) \
|
||||
___read(fn, dst, ___type(src), src, a);
|
||||
#define ___core_readN(fn, dst, src, ...) \
|
||||
___read_ptrs(src, ___nolast(__VA_ARGS__)) \
|
||||
___read(fn, dst, ___type(src, ___nolast(__VA_ARGS__)), __t, \
|
||||
___last(__VA_ARGS__));
|
||||
#define ___core_read(fn, dst, src, a, ...) \
|
||||
___apply(___core_read, ___empty(__VA_ARGS__))(fn, dst, \
|
||||
src, a, ##__VA_ARGS__)
|
||||
|
||||
/*
|
||||
* BPF_CORE_READ_INTO() is a more performance-conscious variant of
|
||||
* BPF_CORE_READ(), in which final field is read into user-provided storage.
|
||||
* See BPF_CORE_READ() below for more details on general usage.
|
||||
*/
|
||||
#define BPF_CORE_READ_INTO(dst, src, a, ...) \
|
||||
({ \
|
||||
___core_read(bpf_core_read, dst, src, a, ##__VA_ARGS__) \
|
||||
})
|
||||
|
||||
/*
|
||||
* BPF_CORE_READ_STR_INTO() does same "pointer chasing" as
|
||||
* BPF_CORE_READ() for intermediate pointers, but then executes (and returns
|
||||
* corresponding error code) bpf_core_read_str() for final string read.
|
||||
*/
|
||||
#define BPF_CORE_READ_STR_INTO(dst, src, a, ...) \
|
||||
({ \
|
||||
___core_read(bpf_core_read_str, dst, src, a, ##__VA_ARGS__) \
|
||||
})
|
||||
|
||||
/*
|
||||
* BPF_CORE_READ() is used to simplify BPF CO-RE relocatable read, especially
|
||||
* when there are few pointer chasing steps.
|
||||
* E.g., what in non-BPF world (or in BPF w/ BCC) would be something like:
|
||||
* int x = s->a.b.c->d.e->f->g;
|
||||
* can be succinctly achieved using BPF_CORE_READ as:
|
||||
* int x = BPF_CORE_READ(s, a.b.c, d.e, f, g);
|
||||
*
|
||||
* BPF_CORE_READ will decompose above statement into 4 bpf_core_read (BPF
|
||||
* CO-RE relocatable bpf_probe_read() wrapper) calls, logically equivalent to:
|
||||
* 1. const void *__t = s->a.b.c;
|
||||
* 2. __t = __t->d.e;
|
||||
* 3. __t = __t->f;
|
||||
* 4. return __t->g;
|
||||
*
|
||||
* Equivalence is logical, because there is a heavy type casting/preservation
|
||||
* involved, as well as all the reads are happening through bpf_probe_read()
|
||||
* calls using __builtin_preserve_access_index() to emit CO-RE relocations.
|
||||
*
|
||||
* N.B. Only up to 9 "field accessors" are supported, which should be more
|
||||
* than enough for any practical purpose.
|
||||
*/
|
||||
#define BPF_CORE_READ(src, a, ...) \
|
||||
({ \
|
||||
___type(src, a, ##__VA_ARGS__) __r; \
|
||||
BPF_CORE_READ_INTO(&__r, src, a, ##__VA_ARGS__); \
|
||||
__r; \
|
||||
})
|
||||
|
||||
#endif
|
||||
|
||||
72
tools/lib/bpf/bpf_endian.h
Normal file
72
tools/lib/bpf/bpf_endian.h
Normal file
@ -0,0 +1,72 @@
|
||||
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
|
||||
#ifndef __BPF_ENDIAN__
|
||||
#define __BPF_ENDIAN__
|
||||
|
||||
#include <linux/stddef.h>
|
||||
#include <linux/swab.h>
|
||||
|
||||
/* LLVM's BPF target selects the endianness of the CPU
|
||||
* it compiles on, or the user specifies (bpfel/bpfeb),
|
||||
* respectively. The used __BYTE_ORDER__ is defined by
|
||||
* the compiler, we cannot rely on __BYTE_ORDER from
|
||||
* libc headers, since it doesn't reflect the actual
|
||||
* requested byte order.
|
||||
*
|
||||
* Note, LLVM's BPF target has different __builtin_bswapX()
|
||||
* semantics. It does map to BPF_ALU | BPF_END | BPF_TO_BE
|
||||
* in bpfel and bpfeb case, which means below, that we map
|
||||
* to cpu_to_be16(). We could use it unconditionally in BPF
|
||||
* case, but better not rely on it, so that this header here
|
||||
* can be used from application and BPF program side, which
|
||||
* use different targets.
|
||||
*/
|
||||
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
|
||||
# define __bpf_ntohs(x) __builtin_bswap16(x)
|
||||
# define __bpf_htons(x) __builtin_bswap16(x)
|
||||
# define __bpf_constant_ntohs(x) ___constant_swab16(x)
|
||||
# define __bpf_constant_htons(x) ___constant_swab16(x)
|
||||
# define __bpf_ntohl(x) __builtin_bswap32(x)
|
||||
# define __bpf_htonl(x) __builtin_bswap32(x)
|
||||
# define __bpf_constant_ntohl(x) ___constant_swab32(x)
|
||||
# define __bpf_constant_htonl(x) ___constant_swab32(x)
|
||||
# define __bpf_be64_to_cpu(x) __builtin_bswap64(x)
|
||||
# define __bpf_cpu_to_be64(x) __builtin_bswap64(x)
|
||||
# define __bpf_constant_be64_to_cpu(x) ___constant_swab64(x)
|
||||
# define __bpf_constant_cpu_to_be64(x) ___constant_swab64(x)
|
||||
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
|
||||
# define __bpf_ntohs(x) (x)
|
||||
# define __bpf_htons(x) (x)
|
||||
# define __bpf_constant_ntohs(x) (x)
|
||||
# define __bpf_constant_htons(x) (x)
|
||||
# define __bpf_ntohl(x) (x)
|
||||
# define __bpf_htonl(x) (x)
|
||||
# define __bpf_constant_ntohl(x) (x)
|
||||
# define __bpf_constant_htonl(x) (x)
|
||||
# define __bpf_be64_to_cpu(x) (x)
|
||||
# define __bpf_cpu_to_be64(x) (x)
|
||||
# define __bpf_constant_be64_to_cpu(x) (x)
|
||||
# define __bpf_constant_cpu_to_be64(x) (x)
|
||||
#else
|
||||
# error "Fix your compiler's __BYTE_ORDER__?!"
|
||||
#endif
|
||||
|
||||
#define bpf_htons(x) \
|
||||
(__builtin_constant_p(x) ? \
|
||||
__bpf_constant_htons(x) : __bpf_htons(x))
|
||||
#define bpf_ntohs(x) \
|
||||
(__builtin_constant_p(x) ? \
|
||||
__bpf_constant_ntohs(x) : __bpf_ntohs(x))
|
||||
#define bpf_htonl(x) \
|
||||
(__builtin_constant_p(x) ? \
|
||||
__bpf_constant_htonl(x) : __bpf_htonl(x))
|
||||
#define bpf_ntohl(x) \
|
||||
(__builtin_constant_p(x) ? \
|
||||
__bpf_constant_ntohl(x) : __bpf_ntohl(x))
|
||||
#define bpf_cpu_to_be64(x) \
|
||||
(__builtin_constant_p(x) ? \
|
||||
__bpf_constant_cpu_to_be64(x) : __bpf_cpu_to_be64(x))
|
||||
#define bpf_be64_to_cpu(x) \
|
||||
(__builtin_constant_p(x) ? \
|
||||
__bpf_constant_be64_to_cpu(x) : __bpf_be64_to_cpu(x))
|
||||
|
||||
#endif /* __BPF_ENDIAN__ */
|
||||
47
tools/lib/bpf/bpf_helpers.h
Normal file
47
tools/lib/bpf/bpf_helpers.h
Normal file
@ -0,0 +1,47 @@
|
||||
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
|
||||
#ifndef __BPF_HELPERS__
|
||||
#define __BPF_HELPERS__
|
||||
|
||||
#include "bpf_helper_defs.h"
|
||||
|
||||
#define __uint(name, val) int (*name)[val]
|
||||
#define __type(name, val) typeof(val) *name
|
||||
|
||||
/* Helper macro to print out debug messages */
|
||||
#define bpf_printk(fmt, ...) \
|
||||
({ \
|
||||
char ____fmt[] = fmt; \
|
||||
bpf_trace_printk(____fmt, sizeof(____fmt), \
|
||||
##__VA_ARGS__); \
|
||||
})
|
||||
|
||||
/*
|
||||
* Helper macro to place programs, maps, license in
|
||||
* different sections in elf_bpf file. Section names
|
||||
* are interpreted by elf_bpf loader
|
||||
*/
|
||||
#define SEC(NAME) __attribute__((section(NAME), used))
|
||||
|
||||
#ifndef __always_inline
|
||||
#define __always_inline __attribute__((always_inline))
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Helper structure used by eBPF C program
|
||||
* to describe BPF map attributes to libbpf loader
|
||||
*/
|
||||
struct bpf_map_def {
|
||||
unsigned int type;
|
||||
unsigned int key_size;
|
||||
unsigned int value_size;
|
||||
unsigned int max_entries;
|
||||
unsigned int map_flags;
|
||||
};
|
||||
|
||||
enum libbpf_pin_type {
|
||||
LIBBPF_PIN_NONE,
|
||||
/* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
|
||||
LIBBPF_PIN_BY_NAME,
|
||||
};
|
||||
|
||||
#endif
|
||||
@ -101,6 +101,7 @@ struct bpf_prog_linfo *bpf_prog_linfo__new(const struct bpf_prog_info *info)
|
||||
{
|
||||
struct bpf_prog_linfo *prog_linfo;
|
||||
__u32 nr_linfo, nr_jited_func;
|
||||
__u64 data_sz;
|
||||
|
||||
nr_linfo = info->nr_line_info;
|
||||
|
||||
@ -122,11 +123,11 @@ struct bpf_prog_linfo *bpf_prog_linfo__new(const struct bpf_prog_info *info)
|
||||
/* Copy xlated line_info */
|
||||
prog_linfo->nr_linfo = nr_linfo;
|
||||
prog_linfo->rec_size = info->line_info_rec_size;
|
||||
prog_linfo->raw_linfo = malloc(nr_linfo * prog_linfo->rec_size);
|
||||
data_sz = (__u64)nr_linfo * prog_linfo->rec_size;
|
||||
prog_linfo->raw_linfo = malloc(data_sz);
|
||||
if (!prog_linfo->raw_linfo)
|
||||
goto err_free;
|
||||
memcpy(prog_linfo->raw_linfo, (void *)(long)info->line_info,
|
||||
nr_linfo * prog_linfo->rec_size);
|
||||
memcpy(prog_linfo->raw_linfo, (void *)(long)info->line_info, data_sz);
|
||||
|
||||
nr_jited_func = info->nr_jited_ksyms;
|
||||
if (!nr_jited_func ||
|
||||
@ -142,13 +143,12 @@ struct bpf_prog_linfo *bpf_prog_linfo__new(const struct bpf_prog_info *info)
|
||||
/* Copy jited_line_info */
|
||||
prog_linfo->nr_jited_func = nr_jited_func;
|
||||
prog_linfo->jited_rec_size = info->jited_line_info_rec_size;
|
||||
prog_linfo->raw_jited_linfo = malloc(nr_linfo *
|
||||
prog_linfo->jited_rec_size);
|
||||
data_sz = (__u64)nr_linfo * prog_linfo->jited_rec_size;
|
||||
prog_linfo->raw_jited_linfo = malloc(data_sz);
|
||||
if (!prog_linfo->raw_jited_linfo)
|
||||
goto err_free;
|
||||
memcpy(prog_linfo->raw_jited_linfo,
|
||||
(void *)(long)info->jited_line_info,
|
||||
nr_linfo * prog_linfo->jited_rec_size);
|
||||
(void *)(long)info->jited_line_info, data_sz);
|
||||
|
||||
/* Number of jited_line_info per jited func */
|
||||
prog_linfo->nr_jited_linfo_per_func = malloc(nr_jited_func *
|
||||
|
||||
195
tools/lib/bpf/bpf_tracing.h
Normal file
195
tools/lib/bpf/bpf_tracing.h
Normal file
@ -0,0 +1,195 @@
|
||||
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
|
||||
#ifndef __BPF_TRACING_H__
|
||||
#define __BPF_TRACING_H__
|
||||
|
||||
/* Scan the ARCH passed in from ARCH env variable (see Makefile) */
|
||||
#if defined(__TARGET_ARCH_x86)
|
||||
#define bpf_target_x86
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_s390)
|
||||
#define bpf_target_s390
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_arm)
|
||||
#define bpf_target_arm
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_arm64)
|
||||
#define bpf_target_arm64
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_mips)
|
||||
#define bpf_target_mips
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_powerpc)
|
||||
#define bpf_target_powerpc
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_sparc)
|
||||
#define bpf_target_sparc
|
||||
#define bpf_target_defined
|
||||
#else
|
||||
#undef bpf_target_defined
|
||||
#endif
|
||||
|
||||
/* Fall back to what the compiler says */
|
||||
#ifndef bpf_target_defined
|
||||
#if defined(__x86_64__)
|
||||
#define bpf_target_x86
|
||||
#elif defined(__s390__)
|
||||
#define bpf_target_s390
|
||||
#elif defined(__arm__)
|
||||
#define bpf_target_arm
|
||||
#elif defined(__aarch64__)
|
||||
#define bpf_target_arm64
|
||||
#elif defined(__mips__)
|
||||
#define bpf_target_mips
|
||||
#elif defined(__powerpc__)
|
||||
#define bpf_target_powerpc
|
||||
#elif defined(__sparc__)
|
||||
#define bpf_target_sparc
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if defined(bpf_target_x86)
|
||||
|
||||
#ifdef __KERNEL__
|
||||
#define PT_REGS_PARM1(x) ((x)->di)
|
||||
#define PT_REGS_PARM2(x) ((x)->si)
|
||||
#define PT_REGS_PARM3(x) ((x)->dx)
|
||||
#define PT_REGS_PARM4(x) ((x)->cx)
|
||||
#define PT_REGS_PARM5(x) ((x)->r8)
|
||||
#define PT_REGS_RET(x) ((x)->sp)
|
||||
#define PT_REGS_FP(x) ((x)->bp)
|
||||
#define PT_REGS_RC(x) ((x)->ax)
|
||||
#define PT_REGS_SP(x) ((x)->sp)
|
||||
#define PT_REGS_IP(x) ((x)->ip)
|
||||
#else
|
||||
#ifdef __i386__
|
||||
/* i386 kernel is built with -mregparm=3 */
|
||||
#define PT_REGS_PARM1(x) ((x)->eax)
|
||||
#define PT_REGS_PARM2(x) ((x)->edx)
|
||||
#define PT_REGS_PARM3(x) ((x)->ecx)
|
||||
#define PT_REGS_PARM4(x) 0
|
||||
#define PT_REGS_PARM5(x) 0
|
||||
#define PT_REGS_RET(x) ((x)->esp)
|
||||
#define PT_REGS_FP(x) ((x)->ebp)
|
||||
#define PT_REGS_RC(x) ((x)->eax)
|
||||
#define PT_REGS_SP(x) ((x)->esp)
|
||||
#define PT_REGS_IP(x) ((x)->eip)
|
||||
#else
|
||||
#define PT_REGS_PARM1(x) ((x)->rdi)
|
||||
#define PT_REGS_PARM2(x) ((x)->rsi)
|
||||
#define PT_REGS_PARM3(x) ((x)->rdx)
|
||||
#define PT_REGS_PARM4(x) ((x)->rcx)
|
||||
#define PT_REGS_PARM5(x) ((x)->r8)
|
||||
#define PT_REGS_RET(x) ((x)->rsp)
|
||||
#define PT_REGS_FP(x) ((x)->rbp)
|
||||
#define PT_REGS_RC(x) ((x)->rax)
|
||||
#define PT_REGS_SP(x) ((x)->rsp)
|
||||
#define PT_REGS_IP(x) ((x)->rip)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#elif defined(bpf_target_s390)
|
||||
|
||||
/* s390 provides user_pt_regs instead of struct pt_regs to userspace */
|
||||
struct pt_regs;
|
||||
#define PT_REGS_S390 const volatile user_pt_regs
|
||||
#define PT_REGS_PARM1(x) (((PT_REGS_S390 *)(x))->gprs[2])
|
||||
#define PT_REGS_PARM2(x) (((PT_REGS_S390 *)(x))->gprs[3])
|
||||
#define PT_REGS_PARM3(x) (((PT_REGS_S390 *)(x))->gprs[4])
|
||||
#define PT_REGS_PARM4(x) (((PT_REGS_S390 *)(x))->gprs[5])
|
||||
#define PT_REGS_PARM5(x) (((PT_REGS_S390 *)(x))->gprs[6])
|
||||
#define PT_REGS_RET(x) (((PT_REGS_S390 *)(x))->gprs[14])
|
||||
/* Works only with CONFIG_FRAME_POINTER */
|
||||
#define PT_REGS_FP(x) (((PT_REGS_S390 *)(x))->gprs[11])
|
||||
#define PT_REGS_RC(x) (((PT_REGS_S390 *)(x))->gprs[2])
|
||||
#define PT_REGS_SP(x) (((PT_REGS_S390 *)(x))->gprs[15])
|
||||
#define PT_REGS_IP(x) (((PT_REGS_S390 *)(x))->psw.addr)
|
||||
|
||||
#elif defined(bpf_target_arm)
|
||||
|
||||
#define PT_REGS_PARM1(x) ((x)->uregs[0])
|
||||
#define PT_REGS_PARM2(x) ((x)->uregs[1])
|
||||
#define PT_REGS_PARM3(x) ((x)->uregs[2])
|
||||
#define PT_REGS_PARM4(x) ((x)->uregs[3])
|
||||
#define PT_REGS_PARM5(x) ((x)->uregs[4])
|
||||
#define PT_REGS_RET(x) ((x)->uregs[14])
|
||||
#define PT_REGS_FP(x) ((x)->uregs[11]) /* Works only with CONFIG_FRAME_POINTER */
|
||||
#define PT_REGS_RC(x) ((x)->uregs[0])
|
||||
#define PT_REGS_SP(x) ((x)->uregs[13])
|
||||
#define PT_REGS_IP(x) ((x)->uregs[12])
|
||||
|
||||
#elif defined(bpf_target_arm64)
|
||||
|
||||
/* arm64 provides struct user_pt_regs instead of struct pt_regs to userspace */
|
||||
struct pt_regs;
|
||||
#define PT_REGS_ARM64 const volatile struct user_pt_regs
|
||||
#define PT_REGS_PARM1(x) (((PT_REGS_ARM64 *)(x))->regs[0])
|
||||
#define PT_REGS_PARM2(x) (((PT_REGS_ARM64 *)(x))->regs[1])
|
||||
#define PT_REGS_PARM3(x) (((PT_REGS_ARM64 *)(x))->regs[2])
|
||||
#define PT_REGS_PARM4(x) (((PT_REGS_ARM64 *)(x))->regs[3])
|
||||
#define PT_REGS_PARM5(x) (((PT_REGS_ARM64 *)(x))->regs[4])
|
||||
#define PT_REGS_RET(x) (((PT_REGS_ARM64 *)(x))->regs[30])
|
||||
/* Works only with CONFIG_FRAME_POINTER */
|
||||
#define PT_REGS_FP(x) (((PT_REGS_ARM64 *)(x))->regs[29])
|
||||
#define PT_REGS_RC(x) (((PT_REGS_ARM64 *)(x))->regs[0])
|
||||
#define PT_REGS_SP(x) (((PT_REGS_ARM64 *)(x))->sp)
|
||||
#define PT_REGS_IP(x) (((PT_REGS_ARM64 *)(x))->pc)
|
||||
|
||||
#elif defined(bpf_target_mips)
|
||||
|
||||
#define PT_REGS_PARM1(x) ((x)->regs[4])
|
||||
#define PT_REGS_PARM2(x) ((x)->regs[5])
|
||||
#define PT_REGS_PARM3(x) ((x)->regs[6])
|
||||
#define PT_REGS_PARM4(x) ((x)->regs[7])
|
||||
#define PT_REGS_PARM5(x) ((x)->regs[8])
|
||||
#define PT_REGS_RET(x) ((x)->regs[31])
|
||||
#define PT_REGS_FP(x) ((x)->regs[30]) /* Works only with CONFIG_FRAME_POINTER */
|
||||
#define PT_REGS_RC(x) ((x)->regs[1])
|
||||
#define PT_REGS_SP(x) ((x)->regs[29])
|
||||
#define PT_REGS_IP(x) ((x)->cp0_epc)
|
||||
|
||||
#elif defined(bpf_target_powerpc)
|
||||
|
||||
#define PT_REGS_PARM1(x) ((x)->gpr[3])
|
||||
#define PT_REGS_PARM2(x) ((x)->gpr[4])
|
||||
#define PT_REGS_PARM3(x) ((x)->gpr[5])
|
||||
#define PT_REGS_PARM4(x) ((x)->gpr[6])
|
||||
#define PT_REGS_PARM5(x) ((x)->gpr[7])
|
||||
#define PT_REGS_RC(x) ((x)->gpr[3])
|
||||
#define PT_REGS_SP(x) ((x)->sp)
|
||||
#define PT_REGS_IP(x) ((x)->nip)
|
||||
|
||||
#elif defined(bpf_target_sparc)
|
||||
|
||||
#define PT_REGS_PARM1(x) ((x)->u_regs[UREG_I0])
|
||||
#define PT_REGS_PARM2(x) ((x)->u_regs[UREG_I1])
|
||||
#define PT_REGS_PARM3(x) ((x)->u_regs[UREG_I2])
|
||||
#define PT_REGS_PARM4(x) ((x)->u_regs[UREG_I3])
|
||||
#define PT_REGS_PARM5(x) ((x)->u_regs[UREG_I4])
|
||||
#define PT_REGS_RET(x) ((x)->u_regs[UREG_I7])
|
||||
#define PT_REGS_RC(x) ((x)->u_regs[UREG_I0])
|
||||
#define PT_REGS_SP(x) ((x)->u_regs[UREG_FP])
|
||||
|
||||
/* Should this also be a bpf_target check for the sparc case? */
|
||||
#if defined(__arch64__)
|
||||
#define PT_REGS_IP(x) ((x)->tpc)
|
||||
#else
|
||||
#define PT_REGS_IP(x) ((x)->pc)
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
#if defined(bpf_target_powerpc)
|
||||
#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = (ctx)->link; })
|
||||
#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP
|
||||
#elif defined(bpf_target_sparc)
|
||||
#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = PT_REGS_RET(ctx); })
|
||||
#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP
|
||||
#else
|
||||
#define BPF_KPROBE_READ_RET_IP(ip, ctx) \
|
||||
({ bpf_probe_read(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); })
|
||||
#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) \
|
||||
({ bpf_probe_read(&(ip), sizeof(ip), \
|
||||
(void *)(PT_REGS_FP(ctx) + sizeof(ip))); })
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@ -269,10 +269,9 @@ __s64 btf__resolve_size(const struct btf *btf, __u32 type_id)
|
||||
t = btf__type_by_id(btf, type_id);
|
||||
}
|
||||
|
||||
done:
|
||||
if (size < 0)
|
||||
return -EINVAL;
|
||||
|
||||
done:
|
||||
if (nelems && size > UINT32_MAX / nelems)
|
||||
return -E2BIG;
|
||||
|
||||
@ -317,6 +316,28 @@ __s32 btf__find_by_name(const struct btf *btf, const char *type_name)
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
__s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name,
|
||||
__u32 kind)
|
||||
{
|
||||
__u32 i;
|
||||
|
||||
if (kind == BTF_KIND_UNKN || !strcmp(type_name, "void"))
|
||||
return 0;
|
||||
|
||||
for (i = 1; i <= btf->nr_types; i++) {
|
||||
const struct btf_type *t = btf->types[i];
|
||||
const char *name;
|
||||
|
||||
if (btf_kind(t) != kind)
|
||||
continue;
|
||||
name = btf__name_by_offset(btf, t->name_off);
|
||||
if (name && !strcmp(type_name, name))
|
||||
return i;
|
||||
}
|
||||
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
void btf__free(struct btf *btf)
|
||||
{
|
||||
if (!btf)
|
||||
@ -390,14 +411,14 @@ struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext)
|
||||
GElf_Ehdr ehdr;
|
||||
|
||||
if (elf_version(EV_CURRENT) == EV_NONE) {
|
||||
pr_warning("failed to init libelf for %s\n", path);
|
||||
pr_warn("failed to init libelf for %s\n", path);
|
||||
return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
|
||||
}
|
||||
|
||||
fd = open(path, O_RDONLY);
|
||||
if (fd < 0) {
|
||||
err = -errno;
|
||||
pr_warning("failed to open %s: %s\n", path, strerror(errno));
|
||||
pr_warn("failed to open %s: %s\n", path, strerror(errno));
|
||||
return ERR_PTR(err);
|
||||
}
|
||||
|
||||
@ -405,19 +426,19 @@ struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext)
|
||||
|
||||
elf = elf_begin(fd, ELF_C_READ, NULL);
|
||||
if (!elf) {
|
||||
pr_warning("failed to open %s as ELF file\n", path);
|
||||
pr_warn("failed to open %s as ELF file\n", path);
|
||||
goto done;
|
||||
}
|
||||
if (!gelf_getehdr(elf, &ehdr)) {
|
||||
pr_warning("failed to get EHDR from %s\n", path);
|
||||
pr_warn("failed to get EHDR from %s\n", path);
|
||||
goto done;
|
||||
}
|
||||
if (!btf_check_endianness(&ehdr)) {
|
||||
pr_warning("non-native ELF endianness is not supported\n");
|
||||
pr_warn("non-native ELF endianness is not supported\n");
|
||||
goto done;
|
||||
}
|
||||
if (!elf_rawdata(elf_getscn(elf, ehdr.e_shstrndx), NULL)) {
|
||||
pr_warning("failed to get e_shstrndx from %s\n", path);
|
||||
pr_warn("failed to get e_shstrndx from %s\n", path);
|
||||
goto done;
|
||||
}
|
||||
|
||||
@ -427,29 +448,29 @@ struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext)
|
||||
|
||||
idx++;
|
||||
if (gelf_getshdr(scn, &sh) != &sh) {
|
||||
pr_warning("failed to get section(%d) header from %s\n",
|
||||
idx, path);
|
||||
pr_warn("failed to get section(%d) header from %s\n",
|
||||
idx, path);
|
||||
goto done;
|
||||
}
|
||||
name = elf_strptr(elf, ehdr.e_shstrndx, sh.sh_name);
|
||||
if (!name) {
|
||||
pr_warning("failed to get section(%d) name from %s\n",
|
||||
idx, path);
|
||||
pr_warn("failed to get section(%d) name from %s\n",
|
||||
idx, path);
|
||||
goto done;
|
||||
}
|
||||
if (strcmp(name, BTF_ELF_SEC) == 0) {
|
||||
btf_data = elf_getdata(scn, 0);
|
||||
if (!btf_data) {
|
||||
pr_warning("failed to get section(%d, %s) data from %s\n",
|
||||
idx, name, path);
|
||||
pr_warn("failed to get section(%d, %s) data from %s\n",
|
||||
idx, name, path);
|
||||
goto done;
|
||||
}
|
||||
continue;
|
||||
} else if (btf_ext && strcmp(name, BTF_EXT_ELF_SEC) == 0) {
|
||||
btf_ext_data = elf_getdata(scn, 0);
|
||||
if (!btf_ext_data) {
|
||||
pr_warning("failed to get section(%d, %s) data from %s\n",
|
||||
idx, name, path);
|
||||
pr_warn("failed to get section(%d, %s) data from %s\n",
|
||||
idx, name, path);
|
||||
goto done;
|
||||
}
|
||||
continue;
|
||||
@ -600,9 +621,9 @@ int btf__load(struct btf *btf)
|
||||
log_buf, log_buf_size, false);
|
||||
if (btf->fd < 0) {
|
||||
err = -errno;
|
||||
pr_warning("Error loading BTF: %s(%d)\n", strerror(errno), errno);
|
||||
pr_warn("Error loading BTF: %s(%d)\n", strerror(errno), errno);
|
||||
if (*log_buf)
|
||||
pr_warning("%s\n", log_buf);
|
||||
pr_warn("%s\n", log_buf);
|
||||
goto done;
|
||||
}
|
||||
|
||||
@ -707,8 +728,8 @@ int btf__get_map_kv_tids(const struct btf *btf, const char *map_name,
|
||||
|
||||
if (snprintf(container_name, max_name, "____btf_map_%s", map_name) ==
|
||||
max_name) {
|
||||
pr_warning("map:%s length of '____btf_map_%s' is too long\n",
|
||||
map_name, map_name);
|
||||
pr_warn("map:%s length of '____btf_map_%s' is too long\n",
|
||||
map_name, map_name);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
@ -721,14 +742,14 @@ int btf__get_map_kv_tids(const struct btf *btf, const char *map_name,
|
||||
|
||||
container_type = btf__type_by_id(btf, container_id);
|
||||
if (!container_type) {
|
||||
pr_warning("map:%s cannot find BTF type for container_id:%u\n",
|
||||
map_name, container_id);
|
||||
pr_warn("map:%s cannot find BTF type for container_id:%u\n",
|
||||
map_name, container_id);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (!btf_is_struct(container_type) || btf_vlen(container_type) < 2) {
|
||||
pr_warning("map:%s container_name:%s is an invalid container struct\n",
|
||||
map_name, container_name);
|
||||
pr_warn("map:%s container_name:%s is an invalid container struct\n",
|
||||
map_name, container_name);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
@ -737,25 +758,25 @@ int btf__get_map_kv_tids(const struct btf *btf, const char *map_name,
|
||||
|
||||
key_size = btf__resolve_size(btf, key->type);
|
||||
if (key_size < 0) {
|
||||
pr_warning("map:%s invalid BTF key_type_size\n", map_name);
|
||||
pr_warn("map:%s invalid BTF key_type_size\n", map_name);
|
||||
return key_size;
|
||||
}
|
||||
|
||||
if (expected_key_size != key_size) {
|
||||
pr_warning("map:%s btf_key_type_size:%u != map_def_key_size:%u\n",
|
||||
map_name, (__u32)key_size, expected_key_size);
|
||||
pr_warn("map:%s btf_key_type_size:%u != map_def_key_size:%u\n",
|
||||
map_name, (__u32)key_size, expected_key_size);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
value_size = btf__resolve_size(btf, value->type);
|
||||
if (value_size < 0) {
|
||||
pr_warning("map:%s invalid BTF value_type_size\n", map_name);
|
||||
pr_warn("map:%s invalid BTF value_type_size\n", map_name);
|
||||
return value_size;
|
||||
}
|
||||
|
||||
if (expected_value_size != value_size) {
|
||||
pr_warning("map:%s btf_value_type_size:%u != map_def_value_size:%u\n",
|
||||
map_name, (__u32)value_size, expected_value_size);
|
||||
pr_warn("map:%s btf_value_type_size:%u != map_def_value_size:%u\n",
|
||||
map_name, (__u32)value_size, expected_value_size);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
@ -888,14 +909,14 @@ static int btf_ext_setup_line_info(struct btf_ext *btf_ext)
|
||||
return btf_ext_setup_info(btf_ext, ¶m);
|
||||
}
|
||||
|
||||
static int btf_ext_setup_offset_reloc(struct btf_ext *btf_ext)
|
||||
static int btf_ext_setup_field_reloc(struct btf_ext *btf_ext)
|
||||
{
|
||||
struct btf_ext_sec_setup_param param = {
|
||||
.off = btf_ext->hdr->offset_reloc_off,
|
||||
.len = btf_ext->hdr->offset_reloc_len,
|
||||
.min_rec_size = sizeof(struct bpf_offset_reloc),
|
||||
.ext_info = &btf_ext->offset_reloc_info,
|
||||
.desc = "offset_reloc",
|
||||
.off = btf_ext->hdr->field_reloc_off,
|
||||
.len = btf_ext->hdr->field_reloc_len,
|
||||
.min_rec_size = sizeof(struct bpf_field_reloc),
|
||||
.ext_info = &btf_ext->field_reloc_info,
|
||||
.desc = "field_reloc",
|
||||
};
|
||||
|
||||
return btf_ext_setup_info(btf_ext, ¶m);
|
||||
@ -975,9 +996,9 @@ struct btf_ext *btf_ext__new(__u8 *data, __u32 size)
|
||||
goto done;
|
||||
|
||||
if (btf_ext->hdr->hdr_len <
|
||||
offsetofend(struct btf_ext_header, offset_reloc_len))
|
||||
offsetofend(struct btf_ext_header, field_reloc_len))
|
||||
goto done;
|
||||
err = btf_ext_setup_offset_reloc(btf_ext);
|
||||
err = btf_ext_setup_field_reloc(btf_ext);
|
||||
if (err)
|
||||
goto done;
|
||||
|
||||
|
||||
@ -60,8 +60,8 @@ struct btf_ext_header {
|
||||
__u32 line_info_len;
|
||||
|
||||
/* optional part of .BTF.ext header */
|
||||
__u32 offset_reloc_off;
|
||||
__u32 offset_reloc_len;
|
||||
__u32 field_reloc_off;
|
||||
__u32 field_reloc_len;
|
||||
};
|
||||
|
||||
LIBBPF_API void btf__free(struct btf *btf);
|
||||
@ -72,6 +72,8 @@ LIBBPF_API int btf__finalize_data(struct bpf_object *obj, struct btf *btf);
|
||||
LIBBPF_API int btf__load(struct btf *btf);
|
||||
LIBBPF_API __s32 btf__find_by_name(const struct btf *btf,
|
||||
const char *type_name);
|
||||
LIBBPF_API __s32 btf__find_by_name_kind(const struct btf *btf,
|
||||
const char *type_name, __u32 kind);
|
||||
LIBBPF_API __u32 btf__get_nr_types(const struct btf *btf);
|
||||
LIBBPF_API const struct btf_type *btf__type_by_id(const struct btf *btf,
|
||||
__u32 id);
|
||||
|
||||
@ -428,7 +428,7 @@ static int btf_dump_order_type(struct btf_dump *d, __u32 id, bool through_ptr)
|
||||
/* type loop, but resolvable through fwd declaration */
|
||||
if (btf_is_composite(t) && through_ptr && t->name_off != 0)
|
||||
return 0;
|
||||
pr_warning("unsatisfiable type cycle, id:[%u]\n", id);
|
||||
pr_warn("unsatisfiable type cycle, id:[%u]\n", id);
|
||||
return -ELOOP;
|
||||
}
|
||||
|
||||
@ -636,8 +636,8 @@ static void btf_dump_emit_type(struct btf_dump *d, __u32 id, __u32 cont_id)
|
||||
if (id == cont_id)
|
||||
return;
|
||||
if (t->name_off == 0) {
|
||||
pr_warning("anonymous struct/union loop, id:[%u]\n",
|
||||
id);
|
||||
pr_warn("anonymous struct/union loop, id:[%u]\n",
|
||||
id);
|
||||
return;
|
||||
}
|
||||
btf_dump_emit_struct_fwd(d, id, t);
|
||||
@ -782,7 +782,7 @@ static int btf_align_of(const struct btf *btf, __u32 id)
|
||||
return align;
|
||||
}
|
||||
default:
|
||||
pr_warning("unsupported BTF_KIND:%u\n", btf_kind(t));
|
||||
pr_warn("unsupported BTF_KIND:%u\n", btf_kind(t));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@ -876,7 +876,6 @@ static void btf_dump_emit_struct_def(struct btf_dump *d,
|
||||
__u16 vlen = btf_vlen(t);
|
||||
|
||||
packed = is_struct ? btf_is_struct_packed(d->btf, id, t) : 0;
|
||||
align = packed ? 1 : btf_align_of(d->btf, id);
|
||||
|
||||
btf_dump_printf(d, "%s%s%s {",
|
||||
is_struct ? "struct" : "union",
|
||||
@ -906,6 +905,13 @@ static void btf_dump_emit_struct_def(struct btf_dump *d,
|
||||
btf_dump_printf(d, ";");
|
||||
}
|
||||
|
||||
/* pad at the end, if necessary */
|
||||
if (is_struct) {
|
||||
align = packed ? 1 : btf_align_of(d->btf, id);
|
||||
btf_dump_emit_bit_padding(d, off, t->size * 8, 0, align,
|
||||
lvl + 1);
|
||||
}
|
||||
|
||||
if (vlen)
|
||||
btf_dump_printf(d, "\n");
|
||||
btf_dump_printf(d, "%s}", pfx(lvl));
|
||||
@ -969,6 +975,17 @@ static void btf_dump_emit_typedef_def(struct btf_dump *d, __u32 id,
|
||||
{
|
||||
const char *name = btf_dump_ident_name(d, id);
|
||||
|
||||
/*
|
||||
* Old GCC versions are emitting invalid typedef for __gnuc_va_list
|
||||
* pointing to VOID. This generates warnings from btf_dump() and
|
||||
* results in uncompilable header file, so we are fixing it up here
|
||||
* with valid typedef into __builtin_va_list.
|
||||
*/
|
||||
if (t->type == 0 && strcmp(name, "__gnuc_va_list") == 0) {
|
||||
btf_dump_printf(d, "typedef __builtin_va_list __gnuc_va_list");
|
||||
return;
|
||||
}
|
||||
|
||||
btf_dump_printf(d, "typedef ");
|
||||
btf_dump_emit_type_decl(d, t->type, name, lvl);
|
||||
}
|
||||
@ -1050,7 +1067,7 @@ static void btf_dump_emit_type_decl(struct btf_dump *d, __u32 id,
|
||||
* chain, restore stack, emit warning, and try to
|
||||
* proceed nevertheless
|
||||
*/
|
||||
pr_warning("not enough memory for decl stack:%d", err);
|
||||
pr_warn("not enough memory for decl stack:%d", err);
|
||||
d->decl_stack_cnt = stack_start;
|
||||
return;
|
||||
}
|
||||
@ -1079,8 +1096,8 @@ static void btf_dump_emit_type_decl(struct btf_dump *d, __u32 id,
|
||||
case BTF_KIND_TYPEDEF:
|
||||
goto done;
|
||||
default:
|
||||
pr_warning("unexpected type in decl chain, kind:%u, id:[%u]\n",
|
||||
btf_kind(t), id);
|
||||
pr_warn("unexpected type in decl chain, kind:%u, id:[%u]\n",
|
||||
btf_kind(t), id);
|
||||
goto done;
|
||||
}
|
||||
}
|
||||
@ -1306,8 +1323,8 @@ static void btf_dump_emit_type_chain(struct btf_dump *d,
|
||||
return;
|
||||
}
|
||||
default:
|
||||
pr_warning("unexpected type in decl chain, kind:%u, id:[%u]\n",
|
||||
kind, id);
|
||||
pr_warn("unexpected type in decl chain, kind:%u, id:[%u]\n",
|
||||
kind, id);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -67,18 +67,80 @@ struct bpf_object_open_attr {
|
||||
enum bpf_prog_type prog_type;
|
||||
};
|
||||
|
||||
/* Helper macro to declare and initialize libbpf options struct
|
||||
*
|
||||
* This dance with uninitialized declaration, followed by memset to zero,
|
||||
* followed by assignment using compound literal syntax is done to preserve
|
||||
* ability to use a nice struct field initialization syntax and **hopefully**
|
||||
* have all the padding bytes initialized to zero. It's not guaranteed though,
|
||||
* when copying literal, that compiler won't copy garbage in literal's padding
|
||||
* bytes, but that's the best way I've found and it seems to work in practice.
|
||||
*
|
||||
* Macro declares opts struct of given type and name, zero-initializes,
|
||||
* including any extra padding, it with memset() and then assigns initial
|
||||
* values provided by users in struct initializer-syntax as varargs.
|
||||
*/
|
||||
#define DECLARE_LIBBPF_OPTS(TYPE, NAME, ...) \
|
||||
struct TYPE NAME = ({ \
|
||||
memset(&NAME, 0, sizeof(struct TYPE)); \
|
||||
(struct TYPE) { \
|
||||
.sz = sizeof(struct TYPE), \
|
||||
__VA_ARGS__ \
|
||||
}; \
|
||||
})
|
||||
|
||||
struct bpf_object_open_opts {
|
||||
/* size of this struct, for forward/backward compatiblity */
|
||||
size_t sz;
|
||||
/* object name override, if provided:
|
||||
* - for object open from file, this will override setting object
|
||||
* name from file path's base name;
|
||||
* - for object open from memory buffer, this will specify an object
|
||||
* name and will override default "<addr>-<buf-size>" name;
|
||||
*/
|
||||
const char *object_name;
|
||||
/* parse map definitions non-strictly, allowing extra attributes/data */
|
||||
bool relaxed_maps;
|
||||
/* process CO-RE relocations non-strictly, allowing them to fail */
|
||||
bool relaxed_core_relocs;
|
||||
/* maps that set the 'pinning' attribute in their definition will have
|
||||
* their pin_path attribute set to a file in this directory, and be
|
||||
* auto-pinned to that path on load; defaults to "/sys/fs/bpf".
|
||||
*/
|
||||
const char *pin_root_path;
|
||||
__u32 attach_prog_fd;
|
||||
};
|
||||
#define bpf_object_open_opts__last_field attach_prog_fd
|
||||
|
||||
LIBBPF_API struct bpf_object *bpf_object__open(const char *path);
|
||||
LIBBPF_API struct bpf_object *
|
||||
bpf_object__open_file(const char *path, struct bpf_object_open_opts *opts);
|
||||
LIBBPF_API struct bpf_object *
|
||||
bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
|
||||
struct bpf_object_open_opts *opts);
|
||||
|
||||
/* deprecated bpf_object__open variants */
|
||||
LIBBPF_API struct bpf_object *
|
||||
bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz,
|
||||
const char *name);
|
||||
LIBBPF_API struct bpf_object *
|
||||
bpf_object__open_xattr(struct bpf_object_open_attr *attr);
|
||||
struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr,
|
||||
int flags);
|
||||
LIBBPF_API struct bpf_object *bpf_object__open_buffer(void *obj_buf,
|
||||
size_t obj_buf_sz,
|
||||
const char *name);
|
||||
|
||||
int bpf_object__section_size(const struct bpf_object *obj, const char *name,
|
||||
__u32 *size);
|
||||
int bpf_object__variable_offset(const struct bpf_object *obj, const char *name,
|
||||
__u32 *off);
|
||||
|
||||
enum libbpf_pin_type {
|
||||
LIBBPF_PIN_NONE,
|
||||
/* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
|
||||
LIBBPF_PIN_BY_NAME,
|
||||
};
|
||||
|
||||
/* pin_maps and unpin_maps can both be called with a NULL path, in which case
|
||||
* they will use the pin_path attribute of each map (and ignore all maps that
|
||||
* don't have a pin_path set).
|
||||
*/
|
||||
LIBBPF_API int bpf_object__pin_maps(struct bpf_object *obj, const char *path);
|
||||
LIBBPF_API int bpf_object__unpin_maps(struct bpf_object *obj,
|
||||
const char *path);
|
||||
@ -127,6 +189,8 @@ libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
|
||||
enum bpf_attach_type *expected_attach_type);
|
||||
LIBBPF_API int libbpf_attach_type_by_name(const char *name,
|
||||
enum bpf_attach_type *attach_type);
|
||||
LIBBPF_API int libbpf_find_vmlinux_btf_id(const char *name,
|
||||
enum bpf_attach_type attach_type);
|
||||
|
||||
/* Accessors of bpf_program */
|
||||
struct bpf_program;
|
||||
@ -153,6 +217,9 @@ LIBBPF_API void bpf_program__set_ifindex(struct bpf_program *prog,
|
||||
LIBBPF_API const char *bpf_program__title(const struct bpf_program *prog,
|
||||
bool needs_copy);
|
||||
|
||||
/* returns program size in bytes */
|
||||
LIBBPF_API size_t bpf_program__size(const struct bpf_program *prog);
|
||||
|
||||
LIBBPF_API int bpf_program__load(struct bpf_program *prog, char *license,
|
||||
__u32 kern_version);
|
||||
LIBBPF_API int bpf_program__fd(const struct bpf_program *prog);
|
||||
@ -187,6 +254,8 @@ LIBBPF_API struct bpf_link *
|
||||
bpf_program__attach_raw_tracepoint(struct bpf_program *prog,
|
||||
const char *tp_name);
|
||||
|
||||
LIBBPF_API struct bpf_link *
|
||||
bpf_program__attach_trace(struct bpf_program *prog);
|
||||
struct bpf_insn;
|
||||
|
||||
/*
|
||||
@ -262,8 +331,14 @@ LIBBPF_API int bpf_program__set_sched_cls(struct bpf_program *prog);
|
||||
LIBBPF_API int bpf_program__set_sched_act(struct bpf_program *prog);
|
||||
LIBBPF_API int bpf_program__set_xdp(struct bpf_program *prog);
|
||||
LIBBPF_API int bpf_program__set_perf_event(struct bpf_program *prog);
|
||||
LIBBPF_API int bpf_program__set_tracing(struct bpf_program *prog);
|
||||
|
||||
LIBBPF_API enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog);
|
||||
LIBBPF_API void bpf_program__set_type(struct bpf_program *prog,
|
||||
enum bpf_prog_type type);
|
||||
|
||||
LIBBPF_API enum bpf_attach_type
|
||||
bpf_program__get_expected_attach_type(struct bpf_program *prog);
|
||||
LIBBPF_API void
|
||||
bpf_program__set_expected_attach_type(struct bpf_program *prog,
|
||||
enum bpf_attach_type type);
|
||||
@ -276,6 +351,7 @@ LIBBPF_API bool bpf_program__is_sched_cls(const struct bpf_program *prog);
|
||||
LIBBPF_API bool bpf_program__is_sched_act(const struct bpf_program *prog);
|
||||
LIBBPF_API bool bpf_program__is_xdp(const struct bpf_program *prog);
|
||||
LIBBPF_API bool bpf_program__is_perf_event(const struct bpf_program *prog);
|
||||
LIBBPF_API bool bpf_program__is_tracing(const struct bpf_program *prog);
|
||||
|
||||
/*
|
||||
* No need for __attribute__((packed)), all members of 'bpf_map_def'
|
||||
@ -335,6 +411,9 @@ LIBBPF_API int bpf_map__resize(struct bpf_map *map, __u32 max_entries);
|
||||
LIBBPF_API bool bpf_map__is_offload_neutral(const struct bpf_map *map);
|
||||
LIBBPF_API bool bpf_map__is_internal(const struct bpf_map *map);
|
||||
LIBBPF_API void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex);
|
||||
LIBBPF_API int bpf_map__set_pin_path(struct bpf_map *map, const char *path);
|
||||
LIBBPF_API const char *bpf_map__get_pin_path(const struct bpf_map *map);
|
||||
LIBBPF_API bool bpf_map__is_pinned(const struct bpf_map *map);
|
||||
LIBBPF_API int bpf_map__pin(struct bpf_map *map, const char *path);
|
||||
LIBBPF_API int bpf_map__unpin(struct bpf_map *map, const char *path);
|
||||
|
||||
@ -356,8 +435,18 @@ LIBBPF_API int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
|
||||
LIBBPF_API int bpf_prog_load(const char *file, enum bpf_prog_type type,
|
||||
struct bpf_object **pobj, int *prog_fd);
|
||||
|
||||
struct xdp_link_info {
|
||||
__u32 prog_id;
|
||||
__u32 drv_prog_id;
|
||||
__u32 hw_prog_id;
|
||||
__u32 skb_prog_id;
|
||||
__u8 attach_mode;
|
||||
};
|
||||
|
||||
LIBBPF_API int bpf_set_link_xdp_fd(int ifindex, int fd, __u32 flags);
|
||||
LIBBPF_API int bpf_get_link_xdp_id(int ifindex, __u32 *prog_id, __u32 flags);
|
||||
LIBBPF_API int bpf_get_link_xdp_info(int ifindex, struct xdp_link_info *info,
|
||||
size_t info_size, __u32 flags);
|
||||
|
||||
struct perf_buffer;
|
||||
|
||||
|
||||
@ -190,3 +190,21 @@ LIBBPF_0.0.5 {
|
||||
global:
|
||||
bpf_btf_get_next_id;
|
||||
} LIBBPF_0.0.4;
|
||||
|
||||
LIBBPF_0.0.6 {
|
||||
global:
|
||||
bpf_get_link_xdp_info;
|
||||
bpf_map__get_pin_path;
|
||||
bpf_map__is_pinned;
|
||||
bpf_map__set_pin_path;
|
||||
bpf_object__open_file;
|
||||
bpf_object__open_mem;
|
||||
bpf_program__attach_trace;
|
||||
bpf_program__get_expected_attach_type;
|
||||
bpf_program__get_type;
|
||||
bpf_program__is_tracing;
|
||||
bpf_program__set_tracing;
|
||||
bpf_program__size;
|
||||
btf__find_by_name_kind;
|
||||
libbpf_find_vmlinux_btf_id;
|
||||
} LIBBPF_0.0.5;
|
||||
|
||||
@ -59,10 +59,42 @@ do { \
|
||||
libbpf_print(level, "libbpf: " fmt, ##__VA_ARGS__); \
|
||||
} while (0)
|
||||
|
||||
#define pr_warning(fmt, ...) __pr(LIBBPF_WARN, fmt, ##__VA_ARGS__)
|
||||
#define pr_warn(fmt, ...) __pr(LIBBPF_WARN, fmt, ##__VA_ARGS__)
|
||||
#define pr_info(fmt, ...) __pr(LIBBPF_INFO, fmt, ##__VA_ARGS__)
|
||||
#define pr_debug(fmt, ...) __pr(LIBBPF_DEBUG, fmt, ##__VA_ARGS__)
|
||||
|
||||
static inline bool libbpf_validate_opts(const char *opts,
|
||||
size_t opts_sz, size_t user_sz,
|
||||
const char *type_name)
|
||||
{
|
||||
if (user_sz < sizeof(size_t)) {
|
||||
pr_warn("%s size (%zu) is too small\n", type_name, user_sz);
|
||||
return false;
|
||||
}
|
||||
if (user_sz > opts_sz) {
|
||||
size_t i;
|
||||
|
||||
for (i = opts_sz; i < user_sz; i++) {
|
||||
if (opts[i]) {
|
||||
pr_warn("%s has non-zero extra bytes",
|
||||
type_name);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
#define OPTS_VALID(opts, type) \
|
||||
(!(opts) || libbpf_validate_opts((const char *)opts, \
|
||||
offsetofend(struct type, \
|
||||
type##__last_field), \
|
||||
(opts)->sz, #type))
|
||||
#define OPTS_HAS(opts, field) \
|
||||
((opts) && opts->sz >= offsetofend(typeof(*(opts)), field))
|
||||
#define OPTS_GET(opts, field, fallback_value) \
|
||||
(OPTS_HAS(opts, field) ? (opts)->field : fallback_value)
|
||||
|
||||
int libbpf__load_raw_btf(const char *raw_types, size_t types_len,
|
||||
const char *str_sec, size_t str_len);
|
||||
|
||||
@ -94,7 +126,7 @@ struct btf_ext {
|
||||
};
|
||||
struct btf_ext_info func_info;
|
||||
struct btf_ext_info line_info;
|
||||
struct btf_ext_info offset_reloc_info;
|
||||
struct btf_ext_info field_reloc_info;
|
||||
__u32 data_size;
|
||||
};
|
||||
|
||||
@ -119,13 +151,27 @@ struct bpf_line_info_min {
|
||||
__u32 line_col;
|
||||
};
|
||||
|
||||
/* The minimum bpf_offset_reloc checked by the loader
|
||||
/* bpf_field_info_kind encodes which aspect of captured field has to be
|
||||
* adjusted by relocations. Currently supported values are:
|
||||
* - BPF_FIELD_BYTE_OFFSET: field offset (in bytes);
|
||||
* - BPF_FIELD_EXISTS: field existence (1, if field exists; 0, otherwise);
|
||||
*/
|
||||
enum bpf_field_info_kind {
|
||||
BPF_FIELD_BYTE_OFFSET = 0, /* field byte offset */
|
||||
BPF_FIELD_BYTE_SIZE = 1,
|
||||
BPF_FIELD_EXISTS = 2, /* field existence in target kernel */
|
||||
BPF_FIELD_SIGNED = 3,
|
||||
BPF_FIELD_LSHIFT_U64 = 4,
|
||||
BPF_FIELD_RSHIFT_U64 = 5,
|
||||
};
|
||||
|
||||
/* The minimum bpf_field_reloc checked by the loader
|
||||
*
|
||||
* Offset relocation captures the following data:
|
||||
* Field relocation captures the following data:
|
||||
* - insn_off - instruction offset (in bytes) within a BPF program that needs
|
||||
* its insn->imm field to be relocated with actual offset;
|
||||
* its insn->imm field to be relocated with actual field info;
|
||||
* - type_id - BTF type ID of the "root" (containing) entity of a relocatable
|
||||
* offset;
|
||||
* field;
|
||||
* - access_str_off - offset into corresponding .BTF string section. String
|
||||
* itself encodes an accessed field using a sequence of field and array
|
||||
* indicies, separated by colon (:). It's conceptually very close to LLVM's
|
||||
@ -156,15 +202,16 @@ struct bpf_line_info_min {
|
||||
* bpf_probe_read(&dst, sizeof(dst),
|
||||
* __builtin_preserve_access_index(&src->a.b.c));
|
||||
*
|
||||
* In this case Clang will emit offset relocation recording necessary data to
|
||||
* In this case Clang will emit field relocation recording necessary data to
|
||||
* be able to find offset of embedded `a.b.c` field within `src` struct.
|
||||
*
|
||||
* [0] https://llvm.org/docs/LangRef.html#getelementptr-instruction
|
||||
*/
|
||||
struct bpf_offset_reloc {
|
||||
struct bpf_field_reloc {
|
||||
__u32 insn_off;
|
||||
__u32 type_id;
|
||||
__u32 access_str_off;
|
||||
enum bpf_field_info_kind kind;
|
||||
};
|
||||
|
||||
#endif /* __LIBBPF_LIBBPF_INTERNAL_H */
|
||||
|
||||
@ -102,6 +102,7 @@ probe_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns,
|
||||
case BPF_PROG_TYPE_FLOW_DISSECTOR:
|
||||
case BPF_PROG_TYPE_CGROUP_SYSCTL:
|
||||
case BPF_PROG_TYPE_CGROUP_SOCKOPT:
|
||||
case BPF_PROG_TYPE_TRACING:
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
|
||||
#include "bpf.h"
|
||||
#include "libbpf.h"
|
||||
#include "libbpf_internal.h"
|
||||
#include "nlattr.h"
|
||||
|
||||
#ifndef SOL_NETLINK
|
||||
@ -24,7 +25,7 @@ typedef int (*__dump_nlmsg_t)(struct nlmsghdr *nlmsg, libbpf_dump_nlmsg_t,
|
||||
struct xdp_id_md {
|
||||
int ifindex;
|
||||
__u32 flags;
|
||||
__u32 id;
|
||||
struct xdp_link_info info;
|
||||
};
|
||||
|
||||
int libbpf_netlink_open(__u32 *nl_pid)
|
||||
@ -43,7 +44,7 @@ int libbpf_netlink_open(__u32 *nl_pid)
|
||||
|
||||
if (setsockopt(sock, SOL_NETLINK, NETLINK_EXT_ACK,
|
||||
&one, sizeof(one)) < 0) {
|
||||
fprintf(stderr, "Netlink error reporting not supported\n");
|
||||
pr_warn("Netlink error reporting not supported\n");
|
||||
}
|
||||
|
||||
if (bind(sock, (struct sockaddr *)&sa, sizeof(sa)) < 0) {
|
||||
@ -202,26 +203,11 @@ static int __dump_link_nlmsg(struct nlmsghdr *nlh,
|
||||
return dump_link_nlmsg(cookie, ifi, tb);
|
||||
}
|
||||
|
||||
static unsigned char get_xdp_id_attr(unsigned char mode, __u32 flags)
|
||||
{
|
||||
if (mode != XDP_ATTACHED_MULTI)
|
||||
return IFLA_XDP_PROG_ID;
|
||||
if (flags & XDP_FLAGS_DRV_MODE)
|
||||
return IFLA_XDP_DRV_PROG_ID;
|
||||
if (flags & XDP_FLAGS_HW_MODE)
|
||||
return IFLA_XDP_HW_PROG_ID;
|
||||
if (flags & XDP_FLAGS_SKB_MODE)
|
||||
return IFLA_XDP_SKB_PROG_ID;
|
||||
|
||||
return IFLA_XDP_UNSPEC;
|
||||
}
|
||||
|
||||
static int get_xdp_id(void *cookie, void *msg, struct nlattr **tb)
|
||||
static int get_xdp_info(void *cookie, void *msg, struct nlattr **tb)
|
||||
{
|
||||
struct nlattr *xdp_tb[IFLA_XDP_MAX + 1];
|
||||
struct xdp_id_md *xdp_id = cookie;
|
||||
struct ifinfomsg *ifinfo = msg;
|
||||
unsigned char mode, xdp_attr;
|
||||
int ret;
|
||||
|
||||
if (xdp_id->ifindex && xdp_id->ifindex != ifinfo->ifi_index)
|
||||
@ -237,27 +223,40 @@ static int get_xdp_id(void *cookie, void *msg, struct nlattr **tb)
|
||||
if (!xdp_tb[IFLA_XDP_ATTACHED])
|
||||
return 0;
|
||||
|
||||
mode = libbpf_nla_getattr_u8(xdp_tb[IFLA_XDP_ATTACHED]);
|
||||
if (mode == XDP_ATTACHED_NONE)
|
||||
xdp_id->info.attach_mode = libbpf_nla_getattr_u8(
|
||||
xdp_tb[IFLA_XDP_ATTACHED]);
|
||||
|
||||
if (xdp_id->info.attach_mode == XDP_ATTACHED_NONE)
|
||||
return 0;
|
||||
|
||||
xdp_attr = get_xdp_id_attr(mode, xdp_id->flags);
|
||||
if (!xdp_attr || !xdp_tb[xdp_attr])
|
||||
return 0;
|
||||
if (xdp_tb[IFLA_XDP_PROG_ID])
|
||||
xdp_id->info.prog_id = libbpf_nla_getattr_u32(
|
||||
xdp_tb[IFLA_XDP_PROG_ID]);
|
||||
|
||||
xdp_id->id = libbpf_nla_getattr_u32(xdp_tb[xdp_attr]);
|
||||
if (xdp_tb[IFLA_XDP_SKB_PROG_ID])
|
||||
xdp_id->info.skb_prog_id = libbpf_nla_getattr_u32(
|
||||
xdp_tb[IFLA_XDP_SKB_PROG_ID]);
|
||||
|
||||
if (xdp_tb[IFLA_XDP_DRV_PROG_ID])
|
||||
xdp_id->info.drv_prog_id = libbpf_nla_getattr_u32(
|
||||
xdp_tb[IFLA_XDP_DRV_PROG_ID]);
|
||||
|
||||
if (xdp_tb[IFLA_XDP_HW_PROG_ID])
|
||||
xdp_id->info.hw_prog_id = libbpf_nla_getattr_u32(
|
||||
xdp_tb[IFLA_XDP_HW_PROG_ID]);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int bpf_get_link_xdp_id(int ifindex, __u32 *prog_id, __u32 flags)
|
||||
int bpf_get_link_xdp_info(int ifindex, struct xdp_link_info *info,
|
||||
size_t info_size, __u32 flags)
|
||||
{
|
||||
struct xdp_id_md xdp_id = {};
|
||||
int sock, ret;
|
||||
__u32 nl_pid;
|
||||
__u32 mask;
|
||||
|
||||
if (flags & ~XDP_FLAGS_MASK)
|
||||
if (flags & ~XDP_FLAGS_MASK || !info_size)
|
||||
return -EINVAL;
|
||||
|
||||
/* Check whether the single {HW,DRV,SKB} mode is set */
|
||||
@ -273,14 +272,44 @@ int bpf_get_link_xdp_id(int ifindex, __u32 *prog_id, __u32 flags)
|
||||
xdp_id.ifindex = ifindex;
|
||||
xdp_id.flags = flags;
|
||||
|
||||
ret = libbpf_nl_get_link(sock, nl_pid, get_xdp_id, &xdp_id);
|
||||
if (!ret)
|
||||
*prog_id = xdp_id.id;
|
||||
ret = libbpf_nl_get_link(sock, nl_pid, get_xdp_info, &xdp_id);
|
||||
if (!ret) {
|
||||
size_t sz = min(info_size, sizeof(xdp_id.info));
|
||||
|
||||
memcpy(info, &xdp_id.info, sz);
|
||||
memset((void *) info + sz, 0, info_size - sz);
|
||||
}
|
||||
|
||||
close(sock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static __u32 get_xdp_id(struct xdp_link_info *info, __u32 flags)
|
||||
{
|
||||
if (info->attach_mode != XDP_ATTACHED_MULTI)
|
||||
return info->prog_id;
|
||||
if (flags & XDP_FLAGS_DRV_MODE)
|
||||
return info->drv_prog_id;
|
||||
if (flags & XDP_FLAGS_HW_MODE)
|
||||
return info->hw_prog_id;
|
||||
if (flags & XDP_FLAGS_SKB_MODE)
|
||||
return info->skb_prog_id;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int bpf_get_link_xdp_id(int ifindex, __u32 *prog_id, __u32 flags)
|
||||
{
|
||||
struct xdp_link_info info;
|
||||
int ret;
|
||||
|
||||
ret = bpf_get_link_xdp_info(ifindex, &info, sizeof(info), flags);
|
||||
if (!ret)
|
||||
*prog_id = get_xdp_id(&info, flags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int libbpf_nl_get_link(int sock, unsigned int nl_pid,
|
||||
libbpf_dump_nlmsg_t dump_link_nlmsg, void *cookie)
|
||||
{
|
||||
|
||||
@ -8,6 +8,7 @@
|
||||
|
||||
#include <errno.h>
|
||||
#include "nlattr.h"
|
||||
#include "libbpf_internal.h"
|
||||
#include <linux/rtnetlink.h>
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
@ -121,8 +122,8 @@ int libbpf_nla_parse(struct nlattr *tb[], int maxtype, struct nlattr *head,
|
||||
}
|
||||
|
||||
if (tb[type])
|
||||
fprintf(stderr, "Attribute of type %#x found multiple times in message, "
|
||||
"previous attribute is being ignored.\n", type);
|
||||
pr_warn("Attribute of type %#x found multiple times in message, "
|
||||
"previous attribute is being ignored.\n", type);
|
||||
|
||||
tb[type] = nla;
|
||||
}
|
||||
@ -181,15 +182,14 @@ int libbpf_nla_dump_errormsg(struct nlmsghdr *nlh)
|
||||
|
||||
if (libbpf_nla_parse(tb, NLMSGERR_ATTR_MAX, attr, alen,
|
||||
extack_policy) != 0) {
|
||||
fprintf(stderr,
|
||||
"Failed to parse extended error attributes\n");
|
||||
pr_warn("Failed to parse extended error attributes\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (tb[NLMSGERR_ATTR_MSG])
|
||||
errmsg = (char *) libbpf_nla_data(tb[NLMSGERR_ATTR_MSG]);
|
||||
|
||||
fprintf(stderr, "Kernel error message: %s\n", errmsg);
|
||||
pr_warn("Kernel error message: %s\n", errmsg);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -7,12 +7,14 @@
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
/* libbpf.h */
|
||||
libbpf_set_print(NULL);
|
||||
/* libbpf.h */
|
||||
libbpf_set_print(NULL);
|
||||
|
||||
/* bpf.h */
|
||||
bpf_prog_get_fd_by_id(0);
|
||||
/* bpf.h */
|
||||
bpf_prog_get_fd_by_id(0);
|
||||
|
||||
/* btf.h */
|
||||
btf__new(NULL, 0);
|
||||
/* btf.h */
|
||||
btf__new(NULL, 0);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -73,6 +73,21 @@ struct xsk_nl_info {
|
||||
int fd;
|
||||
};
|
||||
|
||||
/* Up until and including Linux 5.3 */
|
||||
struct xdp_ring_offset_v1 {
|
||||
__u64 producer;
|
||||
__u64 consumer;
|
||||
__u64 desc;
|
||||
};
|
||||
|
||||
/* Up until and including Linux 5.3 */
|
||||
struct xdp_mmap_offsets_v1 {
|
||||
struct xdp_ring_offset_v1 rx;
|
||||
struct xdp_ring_offset_v1 tx;
|
||||
struct xdp_ring_offset_v1 fr;
|
||||
struct xdp_ring_offset_v1 cr;
|
||||
};
|
||||
|
||||
int xsk_umem__fd(const struct xsk_umem *umem)
|
||||
{
|
||||
return umem ? umem->fd : -EINVAL;
|
||||
@ -133,6 +148,58 @@ static int xsk_set_xdp_socket_config(struct xsk_socket_config *cfg,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void xsk_mmap_offsets_v1(struct xdp_mmap_offsets *off)
|
||||
{
|
||||
struct xdp_mmap_offsets_v1 off_v1;
|
||||
|
||||
/* getsockopt on a kernel <= 5.3 has no flags fields.
|
||||
* Copy over the offsets to the correct places in the >=5.4 format
|
||||
* and put the flags where they would have been on that kernel.
|
||||
*/
|
||||
memcpy(&off_v1, off, sizeof(off_v1));
|
||||
|
||||
off->rx.producer = off_v1.rx.producer;
|
||||
off->rx.consumer = off_v1.rx.consumer;
|
||||
off->rx.desc = off_v1.rx.desc;
|
||||
off->rx.flags = off_v1.rx.consumer + sizeof(__u32);
|
||||
|
||||
off->tx.producer = off_v1.tx.producer;
|
||||
off->tx.consumer = off_v1.tx.consumer;
|
||||
off->tx.desc = off_v1.tx.desc;
|
||||
off->tx.flags = off_v1.tx.consumer + sizeof(__u32);
|
||||
|
||||
off->fr.producer = off_v1.fr.producer;
|
||||
off->fr.consumer = off_v1.fr.consumer;
|
||||
off->fr.desc = off_v1.fr.desc;
|
||||
off->fr.flags = off_v1.fr.consumer + sizeof(__u32);
|
||||
|
||||
off->cr.producer = off_v1.cr.producer;
|
||||
off->cr.consumer = off_v1.cr.consumer;
|
||||
off->cr.desc = off_v1.cr.desc;
|
||||
off->cr.flags = off_v1.cr.consumer + sizeof(__u32);
|
||||
}
|
||||
|
||||
static int xsk_get_mmap_offsets(int fd, struct xdp_mmap_offsets *off)
|
||||
{
|
||||
socklen_t optlen;
|
||||
int err;
|
||||
|
||||
optlen = sizeof(*off);
|
||||
err = getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, off, &optlen);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
if (optlen == sizeof(*off))
|
||||
return 0;
|
||||
|
||||
if (optlen == sizeof(struct xdp_mmap_offsets_v1)) {
|
||||
xsk_mmap_offsets_v1(off);
|
||||
return 0;
|
||||
}
|
||||
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
int xsk_umem__create_v0_0_4(struct xsk_umem **umem_ptr, void *umem_area,
|
||||
__u64 size, struct xsk_ring_prod *fill,
|
||||
struct xsk_ring_cons *comp,
|
||||
@ -141,7 +208,6 @@ int xsk_umem__create_v0_0_4(struct xsk_umem **umem_ptr, void *umem_area,
|
||||
struct xdp_mmap_offsets off;
|
||||
struct xdp_umem_reg mr;
|
||||
struct xsk_umem *umem;
|
||||
socklen_t optlen;
|
||||
void *map;
|
||||
int err;
|
||||
|
||||
@ -163,6 +229,7 @@ int xsk_umem__create_v0_0_4(struct xsk_umem **umem_ptr, void *umem_area,
|
||||
umem->umem_area = umem_area;
|
||||
xsk_set_umem_config(&umem->config, usr_config);
|
||||
|
||||
memset(&mr, 0, sizeof(mr));
|
||||
mr.addr = (uintptr_t)umem_area;
|
||||
mr.len = size;
|
||||
mr.chunk_size = umem->config.frame_size;
|
||||
@ -189,8 +256,7 @@ int xsk_umem__create_v0_0_4(struct xsk_umem **umem_ptr, void *umem_area,
|
||||
goto out_socket;
|
||||
}
|
||||
|
||||
optlen = sizeof(off);
|
||||
err = getsockopt(umem->fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
|
||||
err = xsk_get_mmap_offsets(umem->fd, &off);
|
||||
if (err) {
|
||||
err = -errno;
|
||||
goto out_socket;
|
||||
@ -273,33 +339,55 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk)
|
||||
/* This is the C-program:
|
||||
* SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
|
||||
* {
|
||||
* int index = ctx->rx_queue_index;
|
||||
* int ret, index = ctx->rx_queue_index;
|
||||
*
|
||||
* // A set entry here means that the correspnding queue_id
|
||||
* // has an active AF_XDP socket bound to it.
|
||||
* ret = bpf_redirect_map(&xsks_map, index, XDP_PASS);
|
||||
* if (ret > 0)
|
||||
* return ret;
|
||||
*
|
||||
* // Fallback for pre-5.3 kernels, not supporting default
|
||||
* // action in the flags parameter.
|
||||
* if (bpf_map_lookup_elem(&xsks_map, &index))
|
||||
* return bpf_redirect_map(&xsks_map, index, 0);
|
||||
*
|
||||
* return XDP_PASS;
|
||||
* }
|
||||
*/
|
||||
struct bpf_insn prog[] = {
|
||||
/* r1 = *(u32 *)(r1 + 16) */
|
||||
BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_1, 16),
|
||||
/* *(u32 *)(r10 - 4) = r1 */
|
||||
BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_1, -4),
|
||||
BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
|
||||
BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4),
|
||||
/* r2 = *(u32 *)(r1 + 16) */
|
||||
BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 16),
|
||||
/* *(u32 *)(r10 - 4) = r2 */
|
||||
BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_2, -4),
|
||||
/* r1 = xskmap[] */
|
||||
BPF_LD_MAP_FD(BPF_REG_1, xsk->xsks_map_fd),
|
||||
/* r3 = XDP_PASS */
|
||||
BPF_MOV64_IMM(BPF_REG_3, 2),
|
||||
/* call bpf_redirect_map */
|
||||
BPF_EMIT_CALL(BPF_FUNC_redirect_map),
|
||||
/* if w0 != 0 goto pc+13 */
|
||||
BPF_JMP32_IMM(BPF_JSGT, BPF_REG_0, 0, 13),
|
||||
/* r2 = r10 */
|
||||
BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
|
||||
/* r2 += -4 */
|
||||
BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4),
|
||||
/* r1 = xskmap[] */
|
||||
BPF_LD_MAP_FD(BPF_REG_1, xsk->xsks_map_fd),
|
||||
/* call bpf_map_lookup_elem */
|
||||
BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
|
||||
/* r1 = r0 */
|
||||
BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
|
||||
BPF_MOV32_IMM(BPF_REG_0, 2),
|
||||
/* if r1 == 0 goto +5 */
|
||||
/* r0 = XDP_PASS */
|
||||
BPF_MOV64_IMM(BPF_REG_0, 2),
|
||||
/* if r1 == 0 goto pc+5 */
|
||||
BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 5),
|
||||
/* r2 = *(u32 *)(r10 - 4) */
|
||||
BPF_LD_MAP_FD(BPF_REG_1, xsk->xsks_map_fd),
|
||||
BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_10, -4),
|
||||
BPF_MOV32_IMM(BPF_REG_3, 0),
|
||||
/* r1 = xskmap[] */
|
||||
BPF_LD_MAP_FD(BPF_REG_1, xsk->xsks_map_fd),
|
||||
/* r3 = 0 */
|
||||
BPF_MOV64_IMM(BPF_REG_3, 0),
|
||||
/* call bpf_redirect_map */
|
||||
BPF_EMIT_CALL(BPF_FUNC_redirect_map),
|
||||
/* The jumps are to this instruction */
|
||||
BPF_EXIT_INSN(),
|
||||
@ -310,7 +398,7 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk)
|
||||
"LGPL-2.1 or BSD-2-Clause", 0, log_buf,
|
||||
log_buf_size);
|
||||
if (prog_fd < 0) {
|
||||
pr_warning("BPF log buffer:\n%s", log_buf);
|
||||
pr_warn("BPF log buffer:\n%s", log_buf);
|
||||
return prog_fd;
|
||||
}
|
||||
|
||||
@ -343,13 +431,18 @@ static int xsk_get_max_queues(struct xsk_socket *xsk)
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (err || channels.max_combined == 0)
|
||||
if (err) {
|
||||
/* If the device says it has no channels, then all traffic
|
||||
* is sent to a single stream, so max queues = 1.
|
||||
*/
|
||||
ret = 1;
|
||||
else
|
||||
ret = channels.max_combined;
|
||||
} else {
|
||||
/* Take the max of rx, tx, combined. Drivers return
|
||||
* the number of channels in different ways.
|
||||
*/
|
||||
ret = max(channels.max_rx, channels.max_tx);
|
||||
ret = max(ret, (int)channels.max_combined);
|
||||
}
|
||||
|
||||
out:
|
||||
close(fd);
|
||||
@ -465,6 +558,8 @@ static int xsk_setup_xdp_prog(struct xsk_socket *xsk)
|
||||
}
|
||||
} else {
|
||||
xsk->prog_fd = bpf_prog_get_fd_by_id(prog_id);
|
||||
if (xsk->prog_fd < 0)
|
||||
return -errno;
|
||||
err = xsk_lookup_bpf_maps(xsk);
|
||||
if (err) {
|
||||
close(xsk->prog_fd);
|
||||
@ -472,7 +567,8 @@ static int xsk_setup_xdp_prog(struct xsk_socket *xsk)
|
||||
}
|
||||
}
|
||||
|
||||
err = xsk_set_bpf_maps(xsk);
|
||||
if (xsk->rx)
|
||||
err = xsk_set_bpf_maps(xsk);
|
||||
if (err) {
|
||||
xsk_delete_bpf_maps(xsk);
|
||||
close(xsk->prog_fd);
|
||||
@ -491,21 +587,26 @@ int xsk_socket__create(struct xsk_socket **xsk_ptr, const char *ifname,
|
||||
struct sockaddr_xdp sxdp = {};
|
||||
struct xdp_mmap_offsets off;
|
||||
struct xsk_socket *xsk;
|
||||
socklen_t optlen;
|
||||
int err;
|
||||
|
||||
if (!umem || !xsk_ptr || !rx || !tx)
|
||||
if (!umem || !xsk_ptr || !(rx || tx))
|
||||
return -EFAULT;
|
||||
|
||||
if (umem->refcount) {
|
||||
pr_warning("Error: shared umems not supported by libbpf.\n");
|
||||
return -EBUSY;
|
||||
}
|
||||
|
||||
xsk = calloc(1, sizeof(*xsk));
|
||||
if (!xsk)
|
||||
return -ENOMEM;
|
||||
|
||||
err = xsk_set_xdp_socket_config(&xsk->config, usr_config);
|
||||
if (err)
|
||||
goto out_xsk_alloc;
|
||||
|
||||
if (umem->refcount &&
|
||||
!(xsk->config.libbpf_flags & XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)) {
|
||||
pr_warn("Error: shared umems not supported by libbpf supplied XDP program.\n");
|
||||
err = -EBUSY;
|
||||
goto out_xsk_alloc;
|
||||
}
|
||||
|
||||
if (umem->refcount++ > 0) {
|
||||
xsk->fd = socket(AF_XDP, SOCK_RAW, 0);
|
||||
if (xsk->fd < 0) {
|
||||
@ -527,10 +628,6 @@ int xsk_socket__create(struct xsk_socket **xsk_ptr, const char *ifname,
|
||||
memcpy(xsk->ifname, ifname, IFNAMSIZ - 1);
|
||||
xsk->ifname[IFNAMSIZ - 1] = '\0';
|
||||
|
||||
err = xsk_set_xdp_socket_config(&xsk->config, usr_config);
|
||||
if (err)
|
||||
goto out_socket;
|
||||
|
||||
if (rx) {
|
||||
err = setsockopt(xsk->fd, SOL_XDP, XDP_RX_RING,
|
||||
&xsk->config.rx_size,
|
||||
@ -550,8 +647,7 @@ int xsk_socket__create(struct xsk_socket **xsk_ptr, const char *ifname,
|
||||
}
|
||||
}
|
||||
|
||||
optlen = sizeof(off);
|
||||
err = getsockopt(xsk->fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
|
||||
err = xsk_get_mmap_offsets(xsk->fd, &off);
|
||||
if (err) {
|
||||
err = -errno;
|
||||
goto out_socket;
|
||||
@ -599,7 +695,12 @@ int xsk_socket__create(struct xsk_socket **xsk_ptr, const char *ifname,
|
||||
sxdp.sxdp_family = PF_XDP;
|
||||
sxdp.sxdp_ifindex = xsk->ifindex;
|
||||
sxdp.sxdp_queue_id = xsk->queue_id;
|
||||
sxdp.sxdp_flags = xsk->config.bind_flags;
|
||||
if (umem->refcount > 1) {
|
||||
sxdp.sxdp_flags = XDP_SHARED_UMEM;
|
||||
sxdp.sxdp_shared_umem_fd = umem->fd;
|
||||
} else {
|
||||
sxdp.sxdp_flags = xsk->config.bind_flags;
|
||||
}
|
||||
|
||||
err = bind(xsk->fd, (struct sockaddr *)&sxdp, sizeof(sxdp));
|
||||
if (err) {
|
||||
@ -637,7 +738,6 @@ out_xsk_alloc:
|
||||
int xsk_umem__delete(struct xsk_umem *umem)
|
||||
{
|
||||
struct xdp_mmap_offsets off;
|
||||
socklen_t optlen;
|
||||
int err;
|
||||
|
||||
if (!umem)
|
||||
@ -646,8 +746,7 @@ int xsk_umem__delete(struct xsk_umem *umem)
|
||||
if (umem->refcount)
|
||||
return -EBUSY;
|
||||
|
||||
optlen = sizeof(off);
|
||||
err = getsockopt(umem->fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
|
||||
err = xsk_get_mmap_offsets(umem->fd, &off);
|
||||
if (!err) {
|
||||
munmap(umem->fill->ring - off.fr.desc,
|
||||
off.fr.desc + umem->config.fill_size * sizeof(__u64));
|
||||
@ -665,7 +764,6 @@ void xsk_socket__delete(struct xsk_socket *xsk)
|
||||
{
|
||||
size_t desc_sz = sizeof(struct xdp_desc);
|
||||
struct xdp_mmap_offsets off;
|
||||
socklen_t optlen;
|
||||
int err;
|
||||
|
||||
if (!xsk)
|
||||
@ -676,8 +774,7 @@ void xsk_socket__delete(struct xsk_socket *xsk)
|
||||
close(xsk->prog_fd);
|
||||
}
|
||||
|
||||
optlen = sizeof(off);
|
||||
err = getsockopt(xsk->fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
|
||||
err = xsk_get_mmap_offsets(xsk->fd, &off);
|
||||
if (!err) {
|
||||
if (xsk->rx) {
|
||||
munmap(xsk->rx->ring - off.rx.desc,
|
||||
|
||||
Reference in New Issue
Block a user