summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAlexei Starovoitov <ast@kernel.org>2026-10-11 14:18:07 +0000
committerAlexei Starovoitov <ast@kernel.org>2026-10-11 14:18:07 +0000
commitac1dff2daebb04816766106740042e7ad92859dd (patch)
treeabdf06d1a35a148b690237a4390acf81d764c718
parent4411c854408c05f083c828b69e72e3cc22384fbf (diff)
parent400f0881c912af87091973f231c1b573c2b24292 (diff)
Merge branch 'kbuild-bpf-support-inline-info-in-btf'
Alan Maguire says: ==================== kbuild, bpf: Support inline info in BTF If CONFIG_DEBUG_INFO_BTF_INLINE=y|m, add the "inline" BTF feature and encode inline site info in BTF alongside standard BTF data. resolve_btfids will then partition into non- and inline-related info and split BTF will be used to store inline info in a .BTF.inline section in vmlinux and modules. For CONFIG_DEBUG_INFO_BTF=m, vmlinux inline info will be stored in btf_vmlinux_inline.ko and will be loaded on-demand if used via /sys/kernel/btf/vmlinux.inline. A .BTF.inline.link section in vmlinux stores the module name, the SHA of the BTF and the size so that even without loading the kernel can size the sysfs representation correctly. This approach is based on [1] and the .BTF.inline.link section is modeled on .GNU_debuglink. Because a module can be built with .BTF.base to allow for BTF relocation, its inline info needs to be relocated also so a bit more work needs to be done to handle this. Patches 1, 2 add resolve_btfids support to partition BTF into .BTF and .BTF.inline and to support populating a BTF link section for BTF_INLINE=m. Patch 3 handles vmlinux.inline exposure in sysfs. Patch 4 does prep work for relocation needed in patch 5 prior to exposing relocated BTF in sysfs, and patches 6 and 7 test BTF sysfs representations and inline validity using bpf_testmod respectively. To generate inline info the latest pahole is needed [2]. Inline info for vmlinux can be quite large ~10Mb for over 600,000 inline sites in the kernel, so CONFIG_DEBUG_INFO_BTF_INLINE=m is recommended; this brings the on-disk size down below 4Mb for a gzip-compressed btf_vmlinux_inline.ko. Changes since v3 [3]: - Retain original BTF in resolve_btfids and be careful about free order (bots, patch 1) - Add more failure diagnostics to patching of BTF links (bots, patch 2) - Guard vmallocs for zero-sized inline data (bots, patch 3) - Fix potential NULL pointer BTF deref (bots, patch 5) - Fix up leaks in sysfs test (bots, patch 6) - Fix up tests to use kallsyms lookup directly, and improve validation of location info (bots, patch 7) Changes since v2 [4] - Delivered the libbpf, bpftool parts are prerequisites - Switched to using Eduard's neater BTF partitioning approach which is fully in resolve_btfids instead of libbpf (Eduard, patch 1) - Supports mmap()ed vmlinux.inline (Alexei, patch 3, test patch 7) - Moved to using .BTF.inline.link to store inline BTF size which avoids need to switch to using kernfs; also allows us to SHA-verify BTF inline info [1] https://lore.kernel.org/bpf/20260925224229.1850-1-wanjay@amazon.com/ [2] https://git.kernel.org/pub/scm/devel/pahole/pahole.git/ [3] https://lore.kernel.org/bpf/20261003194323.817885-1-alan.maguire@oracle.com/ [4] https://lore.kernel.org/bpf/20260901165757.801449-1-alan.maguire@oracle.com/ ==================== Link: https://patch.msgid.link/20261009112324.1442319-1-alan.maguire@oracle.com Signed-off-by: Alexei Starovoitov <ast@kernel.org>
-rw-r--r--.gitignore1
-rw-r--r--Makefile1
-rw-r--r--include/asm-generic/vmlinux.lds.h31
-rw-r--r--include/linux/btf.h42
-rw-r--r--include/linux/module.h4
-rw-r--r--kernel/bpf/Makefile1
-rw-r--r--kernel/bpf/btf.c397
-rw-r--r--kernel/bpf/btf_vmlinux_inline.c29
-rw-r--r--kernel/bpf/sysfs_btf.c149
-rw-r--r--kernel/module/main.c7
-rw-r--r--lib/Kconfig.debug18
-rw-r--r--scripts/Makefile.btf7
-rwxr-xr-xscripts/gen-btf.sh31
-rwxr-xr-xscripts/link-vmlinux.sh8
-rw-r--r--scripts/package/kernel.spec10
-rw-r--r--tools/bpf/resolve_btfids/Build1
-rw-r--r--tools/bpf/resolve_btfids/btf_colors.c145
-rw-r--r--tools/bpf/resolve_btfids/btf_colors.h24
-rw-r--r--tools/bpf/resolve_btfids/main.c427
-rw-r--r--tools/lib/bpf/btf.c2
-rw-r--r--tools/lib/bpf/btf_relocate.c7
-rw-r--r--tools/lib/bpf/libbpf_internal.h3
-rw-r--r--tools/testing/selftests/bpf/Makefile4
-rw-r--r--tools/testing/selftests/bpf/prog_tests/btf_inline.c113
-rw-r--r--tools/testing/selftests/bpf/prog_tests/btf_sysfs.c87
-rw-r--r--tools/testing/selftests/bpf/test_kmods/bpf_testmod.c11
-rw-r--r--tools/testing/selftests/bpf/trace_helpers.c20
-rw-r--r--tools/testing/selftests/bpf/trace_helpers.h1
-rw-r--r--tools/testing/selftests/hid/Makefile2
29 files changed, 1455 insertions, 128 deletions
diff --git a/.gitignore b/.gitignore
index 9875120ea7bd..d54bdaac9981 100644
--- a/.gitignore
+++ b/.gitignore
@@ -72,6 +72,7 @@ modules.order
/vmlinux.symvers
/vmlinux.thinlto-index
/vmlinux.unstripped
+/vmlinux.BTF.inline
/vmlinux-gdb.py
/vmlinuz
/System.map
diff --git a/Makefile b/Makefile
index 6b8b812f23b9..b3053dd3c105 100644
--- a/Makefile
+++ b/Makefile
@@ -1769,6 +1769,7 @@ endif # CONFIG_MODULES
CLEAN_FILES += vmlinux.symvers modules-only.symvers \
modules.builtin modules.builtin.modinfo modules.nsdeps \
modules.builtin.ranges vmlinux.o.map vmlinux.unstripped \
+ vmlinux.BTF.inline \
vmlinux.thinlto-index builtin.order \
compile_commands.json rust/test \
rust-project.json .vmlinux.objs .vmlinux.export.c \
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index b2988aa12f66..b17251bc63f5 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -675,12 +675,43 @@
/*
* .BTF
*/
+#ifdef CONFIG_DEBUG_INFO_BTF_INLINE
+#define BTF_INLINE \
+ . = ALIGN(PAGE_SIZE); \
+ .BTF.inline : AT(ADDR(.BTF.inline) - LOAD_OFFSET) { \
+ BOUNDED_SECTION_BY(.BTF.inline, _BTF_inline) \
+ }
+#else
+#define BTF_INLINE
+#endif
+
+#ifdef CONFIG_DEBUG_INFO_BTF_INLINE_MODULE
+/* __MODULE_NAME_LEN + SHA-256 digest + u32 BTF size */
+#ifdef CONFIG_64BIT
+#define BTF_LINK_SIZE (64 - 8 + 32 + 4)
+#else
+#define BTF_LINK_SIZE (64 - 4 + 32 + 4)
+#endif
+#define BTF_INLINE_LINK \
+ . = ALIGN(PAGE_SIZE); \
+ .BTF.inline.link : AT(ADDR(.BTF.inline.link) - LOAD_OFFSET) { \
+ __start_BTF_inline_link = .; \
+ BYTE(0) \
+ . += BTF_LINK_SIZE - 1; \
+ __stop_BTF_inline_link = .; \
+ }
+#else
+#define BTF_INLINE_LINK
+#endif
+
#ifdef CONFIG_DEBUG_INFO_BTF
#define BTF \
. = ALIGN(PAGE_SIZE); \
.BTF : AT(ADDR(.BTF) - LOAD_OFFSET) { \
BOUNDED_SECTION_BY(.BTF, _BTF) \
} \
+ BTF_INLINE \
+ BTF_INLINE_LINK \
. = ALIGN(PAGE_SIZE); \
.BTF_ids : AT(ADDR(.BTF_ids) - LOAD_OFFSET) { \
*(.BTF_ids) \
diff --git a/include/linux/btf.h b/include/linux/btf.h
index 4b63bb91550a..b66e20a18c54 100644
--- a/include/linux/btf.h
+++ b/include/linux/btf.h
@@ -5,6 +5,7 @@
#define _LINUX_BTF_H 1
#include <linux/types.h>
+#include <linux/module.h>
#include <linux/bpfptr.h>
#include <linux/bsearch.h>
#include <linux/btf_ids.h>
@@ -118,6 +119,19 @@ struct btf_show;
struct btf_id_set;
struct bpf_prog;
+#define BTF_LINK_SHA256_LEN 32
+
+/*
+ * A .BTF.link or .BTF.inline.link section identifies the module which
+ * carries a vmlinux BTF section. The module name is NUL terminated and the
+ * field is padded to __MODULE_NAME_LEN.
+ */
+struct btf_link {
+ char module_name[__MODULE_NAME_LEN];
+ u8 sha256[BTF_LINK_SHA256_LEN];
+ u32 btf_size;
+} __packed;
+
typedef int (*btf_kfunc_filter_t)(const struct bpf_prog *prog, u32 kfunc_id);
struct btf_kfunc_id_set {
@@ -594,7 +608,8 @@ struct btf_field_iter {
#ifdef CONFIG_BPF_SYSCALL
const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id);
void btf_set_base_btf(struct btf *btf, const struct btf *base_btf);
-int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **map_ids);
+int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **map_ids,
+ __u32 **map_strs);
int btf_field_iter_init(struct btf_field_iter *it, struct btf_type *t,
enum btf_field_iter_kind iter_kind);
__u32 *btf_field_iter_next(struct btf_field_iter *it);
@@ -624,6 +639,29 @@ bool btf_types_are_same(const struct btf *btf1, u32 id1,
const struct btf *btf2, u32 id2);
int btf_check_iter_arg(struct btf *btf, const struct btf_type *func, int arg_idx);
+#if IS_ENABLED(CONFIG_SYSFS)
+struct bin_attribute *sysfs_btf_add(const char *name, void *data, size_t data_size,
+ bool mmap, const char *lazy_module_name);
+bool sysfs_btf_update(struct bin_attribute *attr, void *data);
+void sysfs_btf_remove(struct bin_attribute *attr);
+#else
+static inline struct bin_attribute *
+sysfs_btf_add(const char *name, void *data, size_t data_size, bool mmap,
+ const char *lazy_module_name)
+{
+ return NULL;
+}
+
+static inline bool sysfs_btf_update(struct bin_attribute *attr, void *data)
+{
+ return false;
+}
+
+static inline void sysfs_btf_remove(struct bin_attribute *attr)
+{
+}
+#endif
+
static inline bool btf_type_is_struct_ptr(struct btf *btf, const struct btf_type *t)
{
if (!btf_type_is_ptr(t))
@@ -645,7 +683,7 @@ static inline void btf_set_base_btf(struct btf *btf, const struct btf *base_btf)
}
static inline int btf_relocate(void *log, struct btf *btf, const struct btf *base_btf,
- __u32 **map_ids)
+ __u32 **map_ids, __u32 **map_strs)
{
return -EOPNOTSUPP;
}
diff --git a/include/linux/module.h b/include/linux/module.h
index 96cc98568eea..a6abde12a458 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -503,6 +503,10 @@ struct module {
void *btf_data;
void *btf_base_data;
#endif
+#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE)
+ unsigned int btf_inline_data_size;
+ void *btf_inline_data;
+#endif
#ifdef CONFIG_JUMP_LABEL
struct jump_entry *jump_entries;
unsigned int num_jump_entries;
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index d61cbb59ad08..269810b128c4 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -42,6 +42,7 @@ obj-$(CONFIG_BPF_SYSCALL) += reuseport_array.o
endif
ifeq ($(CONFIG_SYSFS),y)
obj-$(CONFIG_DEBUG_INFO_BTF) += sysfs_btf.o
+obj-$(CONFIG_DEBUG_INFO_BTF_INLINE) += btf_vmlinux_inline.o
endif
ifeq ($(CONFIG_BPF_JIT),y)
obj-$(CONFIG_BPF_SYSCALL) += bpf_struct_ops.o
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index f4d119f3d5ef..b09c04ccdc6d 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -25,11 +25,12 @@
#include <linux/skmsg.h>
#include <linux/perf_event.h>
#include <linux/bsearch.h>
-#include <linux/kobject.h>
#include <linux/string.h>
-#include <linux/sysfs.h>
#include <linux/overflow.h>
#include <linux/bitops.h>
+#include <linux/unaligned.h>
+#include <linux/vmalloc.h>
+#include <crypto/sha2.h>
#include <net/netfilter/nf_bpf_link.h>
@@ -1935,7 +1936,7 @@ void btf_set_base_btf(struct btf *btf, const struct btf *base_btf)
{
btf->base_btf = (struct btf *)base_btf;
btf->start_id = btf_nr_types(base_btf);
- btf->start_str_off = base_btf->hdr.str_len;
+ btf->start_str_off = base_btf->start_str_off + base_btf->hdr.str_len;
}
static int env_resolve_init(struct btf_verifier_env *env)
@@ -6924,15 +6925,137 @@ __u32 btf_relocate_id(const struct btf *btf, __u32 id)
#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
+static struct btf *btf_parse_split(struct btf_verifier_env *env,
+ const char *name, const void *data,
+ unsigned int data_size, struct btf *base_btf)
+{
+ struct btf *btf;
+ int err;
+
+ btf = kzalloc_obj(*btf, GFP_KERNEL | __GFP_NOWARN);
+ if (!btf)
+ return ERR_PTR(-ENOMEM);
+ env->btf = btf;
+
+ btf_set_base_btf(btf, base_btf);
+ btf->kernel_btf = true;
+ btf->named_start_id = 0;
+ strscpy(btf->name, name);
+
+ btf->data = kvmemdup(data, data_size, GFP_KERNEL | __GFP_NOWARN);
+ if (!btf->data) {
+ err = -ENOMEM;
+ goto errout;
+ }
+ btf->data_size = data_size;
+
+ err = btf_parse_hdr(env);
+ if (err)
+ goto errout;
+
+ btf->nohdr_data = btf->data + btf->hdr.hdr_len;
+
+ err = btf_parse_str_sec(env);
+ if (err)
+ goto errout;
+
+ err = btf_check_all_metas(env);
+ if (err)
+ goto errout;
+
+ err = btf_check_modifier_chain_length(env, btf, btf_nr_types(base_btf));
+ if (err)
+ goto errout;
+
+ return btf;
+
+errout:
+ btf_free(btf);
+ return ERR_PTR(err);
+}
+
+static int btf_rebase_inline(struct btf *inline_btf,
+ const struct btf *module_btf,
+ const u32 *module_id_map,
+ const u32 *module_str_map,
+ u32 old_module_type_cnt)
+{
+ u32 old_start_id = inline_btf->start_id;
+ u32 old_start_str_off = inline_btf->start_str_off;
+ u32 old_module_start_str_off = old_start_str_off - module_btf->hdr.str_len;
+ u32 new_start_id = btf_nr_types(module_btf);
+ u32 new_start_str_off = module_btf->start_str_off + module_btf->hdr.str_len;
+ s64 id_delta = (s64)new_start_id - old_start_id;
+ s64 str_delta = (s64)new_start_str_off - old_start_str_off;
+ u32 i;
+
+ /*
+ * The inline BTF was parsed relative to the original module BTF. Its
+ * base IDs must therefore use the map generated when that BTF was
+ * relocated, while IDs for inline-local types only move by the change
+ * in the module BTF's starting ID.
+ */
+ for (i = 0; i < inline_btf->nr_types; i++) {
+ struct btf_field_iter it;
+ struct btf_type *t = inline_btf->types[i];
+ u32 *id, *str_off;
+ int err;
+
+ err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);
+ if (err)
+ return err;
+ while ((id = btf_field_iter_next(&it))) {
+ if (!*id)
+ continue;
+ if (*id < old_module_type_cnt) {
+ if (module_id_map)
+ *id = module_id_map[*id];
+ } else if (*id >= old_start_id) {
+ *id += id_delta;
+ } else {
+ return -EINVAL;
+ }
+ }
+
+ err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS);
+ if (err)
+ return err;
+ while ((str_off = btf_field_iter_next(&it))) {
+ if (!*str_off)
+ continue;
+ if (*str_off < old_module_start_str_off) {
+ /* Vmlinux strings retain their offsets for in-tree modules. */
+ if (!module_id_map)
+ continue;
+ if (!module_str_map || !module_str_map[*str_off])
+ return -EINVAL;
+ *str_off = module_str_map[*str_off];
+ continue;
+ }
+ *str_off += str_delta;
+ }
+ }
+
+ btf_set_base_btf(inline_btf, module_btf);
+ btf_check_sorted(inline_btf);
+ return 0;
+}
+
static struct btf *btf_parse_module(const char *module_name, const void *data,
unsigned int data_size, void *base_data,
- unsigned int base_data_size)
+ unsigned int base_data_size,
+ const void *inline_data,
+ unsigned int inline_data_size,
+ void **relocated_inline_data)
{
- struct btf *btf = NULL, *vmlinux_btf, *base_btf = NULL;
+ struct btf *btf = NULL, *inline_btf = NULL, *vmlinux_btf, *base_btf = NULL;
struct btf_verifier_env *env = NULL;
struct bpf_verifier_log *log;
+ u32 *module_str_map = NULL;
+ u32 old_module_type_cnt;
int err = 0;
+ *relocated_inline_data = NULL;
vmlinux_btf = bpf_get_btf_vmlinux();
if (IS_ERR(vmlinux_btf))
return vmlinux_btf;
@@ -6956,67 +7079,68 @@ static struct btf *btf_parse_module(const char *module_name, const void *data,
base_btf = vmlinux_btf;
}
- btf = kzalloc_obj(*btf, GFP_KERNEL | __GFP_NOWARN);
- if (!btf) {
- err = -ENOMEM;
+ btf = btf_parse_split(env, module_name, data, data_size, base_btf);
+ if (IS_ERR(btf)) {
+ err = PTR_ERR(btf);
+ btf = NULL;
goto errout;
}
- env->btf = btf;
-
- btf->base_btf = base_btf;
- btf->start_id = base_btf->nr_types;
- btf->start_str_off = base_btf->hdr.str_len;
- btf->kernel_btf = true;
- btf->named_start_id = 0;
- strscpy(btf->name, module_name);
+ old_module_type_cnt = btf_nr_types(btf);
- btf->data = kvmemdup(data, data_size, GFP_KERNEL | __GFP_NOWARN);
- if (!btf->data) {
- err = -ENOMEM;
- goto errout;
+ if (inline_data_size) {
+ inline_btf = btf_parse_split(env, module_name, inline_data,
+ inline_data_size, btf);
+ if (IS_ERR(inline_btf)) {
+ pr_warn("failed to validate module [%s] inline BTF: %ld\n",
+ module_name, PTR_ERR(inline_btf));
+ inline_btf = NULL;
+ }
}
- btf->data_size = data_size;
-
- err = btf_parse_hdr(env);
- if (err)
- goto errout;
-
- btf->nohdr_data = btf->data + btf->hdr.hdr_len;
-
- err = btf_parse_str_sec(env);
- if (err)
- goto errout;
-
- err = btf_check_all_metas(env);
- if (err)
- goto errout;
-
- err = btf_check_modifier_chain_length(env, btf, btf_nr_types(base_btf));
- if (err)
- goto errout;
if (base_btf != vmlinux_btf) {
- err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map);
+ err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map,
+ &module_str_map);
if (err)
goto errout;
btf_free(base_btf);
base_btf = vmlinux_btf;
}
- btf_verifier_env_free(env);
+ if (inline_btf) {
+ err = btf_rebase_inline(inline_btf, btf, btf->base_id_map,
+ module_str_map, old_module_type_cnt);
+ if (err) {
+ pr_warn("failed to relocate module [%s] inline BTF: %d\n",
+ module_name, err);
+ btf_free(inline_btf);
+ } else {
+ *relocated_inline_data = inline_btf->data;
+ inline_btf->data = NULL;
+ btf_free(inline_btf);
+ }
+ }
+
+ /*
+ * With a distilled base, btf_relocate() replaces the base BTF and
+ * rewrites string offsets. Check ordering only after that final BTF
+ * view has been established, so named_start_id describes the BTF used
+ * by name lookups.
+ */
btf_check_sorted(btf);
+ btf_verifier_env_free(env);
+ kvfree(module_str_map);
refcount_set(&btf->refcnt, 1);
return btf;
errout:
+ kvfree(module_str_map);
btf_verifier_env_free(env);
+ if (inline_btf)
+ btf_free(inline_btf);
if (!IS_ERR(base_btf) && base_btf != vmlinux_btf)
btf_free(base_btf);
- if (btf) {
- kvfree(btf->data);
- kvfree(btf->types);
- kfree(btf);
- }
+ if (btf)
+ btf_free(btf);
return ERR_PTR(err);
}
@@ -9118,6 +9242,8 @@ struct btf_module {
struct module *module;
struct btf *btf;
struct bin_attribute *sysfs_attr;
+ struct bin_attribute *sysfs_inline_attr;
+ void *btf_inline_data;
int flags;
};
@@ -9126,28 +9252,149 @@ static DEFINE_MUTEX(btf_module_mutex);
static void purge_cand_cache(struct btf *btf);
+#if IS_MODULE(CONFIG_DEBUG_INFO_BTF_INLINE)
+extern char __start_BTF_inline_link[];
+extern char __stop_BTF_inline_link[];
+
+static struct bin_attribute *vmlinux_inline_attr;
+static const struct btf_link *vmlinux_inline_link;
+
+static const struct btf_link *btf_parse_link(void *start, unsigned int size)
+{
+ const struct btf_link *link = start;
+ unsigned int module_name_len;
+
+ if (!link || size != sizeof(*link) || link->btf_size == 0)
+ return NULL;
+
+ module_name_len = strnlen(link->module_name, sizeof(link->module_name));
+ if (!module_name_len || module_name_len == sizeof(link->module_name) ||
+ memchr_inv(link->module_name + module_name_len + 1, 0,
+ sizeof(link->module_name) - module_name_len - 1))
+ return NULL;
+ return link;
+}
+
+static bool btf_verify_link(const struct btf_link *link, const struct module *mod,
+ const void *data, size_t data_size)
+{
+ u8 actual_digest[BTF_LINK_SHA256_LEN];
+
+ if (!link || data_size != link->btf_size) {
+ pr_warn(" BTF link from module [%s] missing or has unexpected size\n", mod->name);
+ return false;
+ }
+ sha256(data, data_size, actual_digest);
+ if (memcmp(actual_digest, link->sha256, sizeof(actual_digest))) {
+ pr_warn("BTF link from module [%s] has unexpected digest\n", mod->name);
+ return false;
+ }
+
+ return true;
+}
+
+static void btf_vmlinux_inline_lazy_init(void)
+{
+ vmlinux_inline_link = btf_parse_link(__start_BTF_inline_link,
+ __stop_BTF_inline_link -
+ __start_BTF_inline_link);
+ if (!vmlinux_inline_link) {
+ pr_warn("invalid vmlinux inline BTF link\n");
+ return;
+ }
+ vmlinux_inline_attr = sysfs_btf_add("vmlinux.inline", NULL,
+ vmlinux_inline_link->btf_size, true,
+ vmlinux_inline_link->module_name);
+ if (IS_ERR(vmlinux_inline_attr)) {
+ pr_warn("failed to register vmlinux inline BTF in sysfs: %ld\n",
+ PTR_ERR(vmlinux_inline_attr));
+ vmlinux_inline_attr = NULL;
+ }
+}
+
+static bool btf_module_is_vmlinux_inline(const struct module *mod)
+{
+ return vmlinux_inline_link && !strcmp(mod->name, vmlinux_inline_link->module_name);
+}
+
+static int btf_module_handle_vmlinux_inline(const struct module *mod)
+{
+ void *data;
+
+ if (!mod->btf_inline_data ||
+ mod->btf_inline_data_size != vmlinux_inline_link->btf_size)
+ return -EINVAL;
+
+ data = vmalloc_user(mod->btf_inline_data_size);
+ if (!data)
+ return -ENOMEM;
+ memcpy(data, mod->btf_inline_data, mod->btf_inline_data_size);
+
+ if (!btf_verify_link(vmlinux_inline_link, mod, data, mod->btf_inline_data_size) ||
+ !sysfs_btf_update(vmlinux_inline_attr, data)) {
+ kvfree(data);
+ return -EINVAL;
+ }
+ return 0;
+}
+
+#else
+
+static bool btf_module_is_vmlinux_inline(const struct module *mod)
+{
+ return false;
+}
+
+static int btf_module_handle_vmlinux_inline(const struct module *mod)
+{
+ return 0;
+}
+
+static void btf_vmlinux_inline_lazy_init(void)
+{
+}
+
+#endif /* IS_MODULE(CONFIG_DEBUG_INFO_BTF_INLINE) */
+
static int btf_module_notify(struct notifier_block *nb, unsigned long op,
void *module)
{
struct btf_module *btf_mod, *tmp;
struct module *mod = module;
+ struct bin_attribute *attr;
struct btf *btf;
+ void *inline_data = NULL, *relocated_inline_data = NULL;
+ unsigned int inline_data_size = 0;
int err = 0;
- if (mod->btf_data_size == 0 ||
- (op != MODULE_STATE_COMING && op != MODULE_STATE_LIVE &&
- op != MODULE_STATE_GOING))
+ if (op != MODULE_STATE_COMING && op != MODULE_STATE_LIVE &&
+ op != MODULE_STATE_GOING)
goto out;
switch (op) {
case MODULE_STATE_COMING:
+ if (btf_module_is_vmlinux_inline(mod)) {
+ err = btf_module_handle_vmlinux_inline(mod);
+ break;
+ }
+
+#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE)
+ if (mod->btf_inline_data_size) {
+ inline_data = mod->btf_inline_data;
+ inline_data_size = mod->btf_inline_data_size;
+ }
+#endif
+ if (!mod->btf_data_size)
+ break;
btf_mod = kzalloc_obj(*btf_mod);
if (!btf_mod) {
err = -ENOMEM;
goto out;
}
btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size,
- mod->btf_base_data, mod->btf_base_data_size);
+ mod->btf_base_data, mod->btf_base_data_size,
+ inline_data, inline_data_size,
+ &relocated_inline_data);
if (IS_ERR(btf)) {
kfree(btf_mod);
if (!IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) {
@@ -9162,6 +9409,7 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op,
err = btf_alloc_id(btf);
if (err) {
btf_free(btf);
+ kvfree(relocated_inline_data);
kfree(btf_mod);
goto out;
}
@@ -9170,37 +9418,35 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op,
mutex_lock(&btf_module_mutex);
btf_mod->module = module;
btf_mod->btf = btf;
+ btf_mod->btf_inline_data = relocated_inline_data;
list_add(&btf_mod->list, &btf_modules);
mutex_unlock(&btf_module_mutex);
- if (IS_ENABLED(CONFIG_SYSFS)) {
- struct bin_attribute *attr;
-
- attr = kzalloc_obj(*attr);
- if (!attr)
- goto out;
+ attr = sysfs_btf_add(btf->name, btf->data, btf->data_size, false, NULL);
+ if (IS_ERR(attr)) {
+ err = 0;
+ goto out;
+ }
+ btf_mod->sysfs_attr = attr;
- sysfs_bin_attr_init(attr);
- attr->attr.name = btf->name;
- attr->attr.mode = 0444;
- attr->size = btf->data_size;
- attr->private = btf->data;
- attr->read = sysfs_bin_attr_simple_read;
+ if (relocated_inline_data) {
+ char name[MODULE_NAME_LEN + sizeof(".inline")];
- err = sysfs_create_bin_file(btf_kobj, attr);
- if (err) {
- pr_warn("failed to register module [%s] BTF in sysfs: %d\n",
- mod->name, err);
- kfree(attr);
+ snprintf(name, sizeof(name), "%s.inline", mod->name);
+ attr = sysfs_btf_add(name, relocated_inline_data,
+ inline_data_size, false, NULL);
+ if (IS_ERR(attr)) {
+ kvfree(relocated_inline_data);
+ btf_mod->btf_inline_data = NULL;
err = 0;
- goto out;
+ break;
}
-
- btf_mod->sysfs_attr = attr;
+ btf_mod->sysfs_inline_attr = attr;
}
-
break;
case MODULE_STATE_LIVE:
+ if (!mod->btf_data_size)
+ break;
mutex_lock(&btf_module_mutex);
list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) {
if (btf_mod->module != module)
@@ -9212,6 +9458,8 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op,
mutex_unlock(&btf_module_mutex);
break;
case MODULE_STATE_GOING:
+ if (!mod->btf_data_size)
+ break;
mutex_lock(&btf_module_mutex);
list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) {
if (btf_mod->module != module)
@@ -9226,10 +9474,12 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op,
btf_free_id(btf_mod->btf);
list_del(&btf_mod->list);
if (btf_mod->sysfs_attr)
- sysfs_remove_bin_file(btf_kobj, btf_mod->sysfs_attr);
+ sysfs_btf_remove(btf_mod->sysfs_attr);
+ if (btf_mod->sysfs_inline_attr)
+ sysfs_btf_remove(btf_mod->sysfs_inline_attr);
+ kvfree(btf_mod->btf_inline_data);
purge_cand_cache(btf_mod->btf);
btf_put(btf_mod->btf);
- kfree(btf_mod->sysfs_attr);
kfree(btf_mod);
break;
}
@@ -9246,6 +9496,7 @@ static struct notifier_block btf_module_nb = {
static int __init btf_module_init(void)
{
+ btf_vmlinux_inline_lazy_init();
register_module_notifier(&btf_module_nb);
return 0;
}
diff --git a/kernel/bpf/btf_vmlinux_inline.c b/kernel/bpf/btf_vmlinux_inline.c
new file mode 100644
index 000000000000..d5c599d9f729
--- /dev/null
+++ b/kernel/bpf/btf_vmlinux_inline.c
@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2026, Oracle and/or its affiliates. */
+/*
+ * Provide kernel BTF inline function information for use by BPF tools.
+ */
+#include <linux/btf.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+
+#if IS_BUILTIN(CONFIG_DEBUG_INFO_BTF_INLINE)
+extern char __start_BTF_inline[];
+extern char __stop_BTF_inline[];
+#endif
+
+static int __init btf_vmlinux_inline_init(void)
+{
+#if IS_BUILTIN(CONFIG_DEBUG_INFO_BTF_INLINE)
+ size_t data_size = __stop_BTF_inline - __start_BTF_inline;
+
+ if (data_size)
+ sysfs_btf_add("vmlinux.inline", __start_BTF_inline, data_size, true, NULL);
+#endif
+ return 0;
+}
+subsys_initcall(btf_vmlinux_inline_init);
+
+MODULE_DESCRIPTION("BTF inline information for vmlinux");
+MODULE_LICENSE("GPL");
diff --git a/kernel/bpf/sysfs_btf.c b/kernel/bpf/sysfs_btf.c
index 9cbe15ce3540..556fe92c3649 100644
--- a/kernel/bpf/sysfs_btf.c
+++ b/kernel/bpf/sysfs_btf.c
@@ -6,8 +6,10 @@
#include <linux/module.h>
#include <linux/kobject.h>
#include <linux/init.h>
+#include <linux/slab.h>
#include <linux/sysfs.h>
#include <linux/mm.h>
+#include <linux/vmalloc.h>
#include <linux/io.h>
#include <linux/btf.h>
@@ -15,16 +17,12 @@
extern char __start_BTF[];
extern char __stop_BTF[];
-static int btf_sysfs_vmlinux_mmap(struct file *filp, struct kobject *kobj,
- const struct bin_attribute *attr,
- struct vm_area_struct *vma)
+static int btf_sysfs_mmap_check(void *data, size_t size, struct vm_area_struct *vma)
{
- unsigned long pages = PAGE_ALIGN(attr->size) >> PAGE_SHIFT;
+ unsigned long pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
size_t vm_size = vma->vm_end - vma->vm_start;
- phys_addr_t addr = __pa_symbol(__start_BTF);
- unsigned long pfn = addr >> PAGE_SHIFT;
- if (attr->private != __start_BTF || !PAGE_ALIGNED(addr))
+ if (!data || !PAGE_ALIGNED((unsigned long)data))
return -EINVAL;
if (vma->vm_pgoff)
@@ -33,24 +31,153 @@ static int btf_sysfs_vmlinux_mmap(struct file *filp, struct kobject *kobj,
if (vma->vm_flags & (VM_WRITE | VM_EXEC | VM_MAYSHARE))
return -EACCES;
- if (pfn + pages < pfn)
- return -EINVAL;
-
if ((vm_size >> PAGE_SHIFT) > pages)
return -EINVAL;
vm_flags_mod(vma, VM_DONTDUMP, VM_MAYEXEC | VM_MAYWRITE);
+ return 0;
+}
+
+static int btf_sysfs_mmap_image(struct file *filp, struct kobject *kobj,
+ const struct bin_attribute *attr,
+ struct vm_area_struct *vma)
+{
+ void *data = READ_ONCE(attr->private);
+ phys_addr_t addr;
+ unsigned long pfn;
+ size_t vm_size = vma->vm_end - vma->vm_start;
+ int err;
+
+ err = btf_sysfs_mmap_check(data, attr->size, vma);
+ if (err)
+ return err;
+
+ /* Kernel image addresses can lie inside the vmalloc range on arm64. */
+ addr = __pa_symbol(data);
+ pfn = addr >> PAGE_SHIFT;
+ if (pfn + (PAGE_ALIGN(attr->size) >> PAGE_SHIFT) < pfn)
+ return -EINVAL;
+
return remap_pfn_range(vma, vma->vm_start, pfn, vm_size, vma->vm_page_prot);
}
static struct bin_attribute bin_attr_btf_vmlinux __ro_after_init = {
.attr = { .name = "vmlinux", .mode = 0444, },
.read = sysfs_bin_attr_simple_read,
- .mmap = btf_sysfs_vmlinux_mmap,
+ .mmap = btf_sysfs_mmap_image,
};
struct kobject *btf_kobj;
+struct btf_sysfs_entry {
+ struct bin_attribute attr;
+ char *module_name;
+};
+
+static void *btf_sysfs_lazy_data(const struct bin_attribute *attr)
+{
+ struct btf_sysfs_entry *entry = container_of(attr, struct btf_sysfs_entry, attr);
+ void *data;
+
+ data = smp_load_acquire(&attr->private);
+ if (!data) {
+ request_module("%s", entry->module_name);
+ data = smp_load_acquire(&attr->private);
+ }
+ return data;
+}
+
+static ssize_t btf_sysfs_read_lazy(struct file *filp, struct kobject *kobj,
+ const struct bin_attribute *attr, char *buf,
+ loff_t off, size_t count)
+{
+ void *data = btf_sysfs_lazy_data(attr);
+
+ if (!data)
+ return -ENODEV;
+
+ return memory_read_from_buffer(buf, count, &off, data, attr->size);
+}
+
+static int btf_sysfs_mmap_lazy(struct file *filp, struct kobject *kobj,
+ const struct bin_attribute *attr,
+ struct vm_area_struct *vma)
+{
+ void *data = btf_sysfs_lazy_data(attr);
+ int err;
+
+ if (!data)
+ return -ENODEV;
+
+ err = btf_sysfs_mmap_check(data, attr->size, vma);
+ if (err)
+ return err;
+
+ return remap_vmalloc_range(vma, data, 0);
+}
+
+struct bin_attribute *sysfs_btf_add(const char *name, void *data, size_t data_size,
+ bool mmap, const char *lazy_module_name)
+{
+ struct btf_sysfs_entry *entry;
+ struct bin_attribute *attr;
+ int err;
+
+ entry = kzalloc_obj(*entry);
+ if (!entry)
+ return ERR_PTR(-ENOMEM);
+
+ attr = &entry->attr;
+ sysfs_bin_attr_init(attr);
+ attr->attr.mode = 0444;
+ attr->size = data_size;
+ attr->private = data;
+ attr->read = lazy_module_name ? btf_sysfs_read_lazy : sysfs_bin_attr_simple_read;
+ if (mmap)
+ attr->mmap = lazy_module_name ? btf_sysfs_mmap_lazy : btf_sysfs_mmap_image;
+ attr->attr.name = kstrdup(name, GFP_KERNEL);
+ if (!attr->attr.name) {
+ err = -ENOMEM;
+ goto err_free;
+ }
+ if (lazy_module_name) {
+ entry->module_name = kstrdup(lazy_module_name, GFP_KERNEL);
+ if (!entry->module_name) {
+ err = -ENOMEM;
+ goto err_free;
+ }
+ }
+
+ err = sysfs_create_bin_file(btf_kobj, attr);
+ if (err) {
+ pr_warn("failed to register [%s] BTF in sysfs: %d\n", name, err);
+ goto err_free;
+ }
+
+ return attr;
+
+err_free:
+ kfree(entry->module_name);
+ kfree(attr->attr.name);
+ kfree(entry);
+ return ERR_PTR(err);
+}
+
+bool sysfs_btf_update(struct bin_attribute *attr, void *data)
+{
+ return attr && !cmpxchg(&attr->private, NULL, data);
+}
+
+void sysfs_btf_remove(struct bin_attribute *attr)
+{
+ struct btf_sysfs_entry *entry = container_of(attr, struct btf_sysfs_entry, attr);
+
+ sysfs_remove_bin_file(btf_kobj, attr);
+ kfree(entry->module_name);
+ kfree(attr->attr.name);
+ kfree(entry);
+}
+
static int __init btf_vmlinux_init(void)
{
bin_attr_btf_vmlinux.private = __start_BTF;
diff --git a/kernel/module/main.c b/kernel/module/main.c
index c1b34dc1e89a..5b9eb56164b2 100644
--- a/kernel/module/main.c
+++ b/kernel/module/main.c
@@ -2723,6 +2723,10 @@ static int find_module_sections(struct module *mod, struct load_info *info)
mod->btf_base_data = any_section_objs(info, ".BTF.base", 1,
&mod->btf_base_data_size);
#endif
+#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE)
+ mod->btf_inline_data = any_section_objs(info, ".BTF.inline", 1,
+ &mod->btf_inline_data_size);
+#endif
#ifdef CONFIG_JUMP_LABEL
mod->jump_entries = section_objs(info, "__jump_table",
sizeof(*mod->jump_entries),
@@ -3177,6 +3181,9 @@ static noinline int do_init_module(struct module *mod)
mod->btf_data = NULL;
mod->btf_base_data = NULL;
#endif
+#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE)
+ mod->btf_inline_data = NULL;
+#endif
/*
* We want to free module_init, but be aware that kallsyms may be
* walking this within an RCU read section. In all the failure paths, we
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 134b15a44625..2f34619905b6 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -425,6 +425,12 @@ config PAHOLE_HAS_LANG_EXCLUDE
otherwise it would emit malformed kernel and module binaries when
using DEBUG_INFO_BTF_MODULES.
+config PAHOLE_HAS_INLINE
+ def_bool PAHOLE_VERSION >= 132
+ help
+ Support for the "inline" BTF feature is available. It encodes
+ information about inline sites and how to retrieve their parameters.
+
config DEBUG_INFO_BTF_MODULES
bool "Generate BTF type information for kernel modules"
default y
@@ -432,6 +438,18 @@ config DEBUG_INFO_BTF_MODULES
help
Generate compact split BTF type information for kernel modules.
+config DEBUG_INFO_BTF_INLINE
+ tristate "Provide information about inline sites in BTF"
+ default n
+ depends on DEBUG_INFO_BTF && PAHOLE_HAS_INLINE && SYSFS
+ select DEBUG_INFO_BTF_MODULES if MODULES
+ help
+ Generate information about inline sites in .BTF.inline sections.
+ These sections contain split BTF relative to the kernel or module BTF
+ and are made available in /sys/kernel/btf with a ".inline" suffix.
+ The information describes inline locations and how to retrieve their
+ associated parameters.
+
config MODULE_ALLOW_BTF_MISMATCH
bool "Allow loading modules with non-matching BTF type info"
depends on DEBUG_INFO_BTF_MODULES
diff --git a/scripts/Makefile.btf b/scripts/Makefile.btf
index a1812985a61a..98012344a81e 100644
--- a/scripts/Makefile.btf
+++ b/scripts/Makefile.btf
@@ -22,7 +22,14 @@ endif
pahole-flags-$(CONFIG_PAHOLE_HAS_LANG_EXCLUDE) += --lang_exclude=rust
+btf-inline := $(CONFIG_DEBUG_INFO_BTF_INLINE)
+ifneq ($(btf-inline),)
+btf-inline-feat := inline
+pahole-flags-$(call test-ge, $(pahole-ver), 132) += --btf_features=$(btf-inline-feat)
+endif
+
export PAHOLE_FLAGS := $(pahole-flags-y)
+export BTF_INLINE := $(btf-inline)
resolve-btfids-flags-y :=
resolve-btfids-flags-$(CONFIG_WERROR) += --fatal_warnings
diff --git a/scripts/gen-btf.sh b/scripts/gen-btf.sh
index 8ca96eb10a69..b17f1189298a 100755
--- a/scripts/gen-btf.sh
+++ b/scripts/gen-btf.sh
@@ -68,6 +68,7 @@ esac
gen_btf_data()
{
+ inline=${BTF_INLINE:+--inline}
btf1="${ELF_FILE}.BTF.1"
${PAHOLE} -J ${PAHOLE_FLAGS} \
${BTF_BASE:+--btf_base ${BTF_BASE}} \
@@ -76,6 +77,7 @@ gen_btf_data()
${RESOLVE_BTFIDS} ${RESOLVE_BTFIDS_FLAGS} \
${BTF_BASE:+--btf_base ${BTF_BASE}} \
+ ${inline} \
--btf ${btf1} "${ELF_FILE}"
}
@@ -83,14 +85,28 @@ gen_btf_o()
{
btf_data=${ELF_FILE}.btf.o
- # Create ${btf_data} which contains just .BTF section but no symbols. Add
+ # Create ${btf_data} which contains just BTF sections but no symbols. Add
# SHF_ALLOC because .BTF will be part of the vmlinux image. --strip-all
# deletes all symbols including __start_BTF and __stop_BTF, which will
# be redefined in the linker script.
echo "" | ${CC} ${CLANG_FLAGS} ${KBUILD_CPPFLAGS} ${KBUILD_CFLAGS} -fno-lto -c -x c -o ${btf_data} -
${OBJCOPY} --add-section .BTF=${ELF_FILE}.BTF \
--set-section-flags .BTF=alloc,readonly ${btf_data}
- ${OBJCOPY} --only-section=.BTF --strip-all ${btf_data}
+ ONLY_SEC="--only-section=.BTF"
+ btf_inline=${ELF_FILE}.BTF.inline
+ if [ "${BTF_INLINE}" = "m" ]; then
+ if [ -f "${btf_inline}" ]; then
+ cp "${btf_inline}" "${objtree}/vmlinux.BTF.inline"
+ else
+ rm -f "${objtree}/vmlinux.BTF.inline"
+ fi
+ fi
+ if [ "${BTF_INLINE}" = "y" ] && [ -f "${btf_inline}" ]; then
+ ${OBJCOPY} --add-section .BTF.inline=${btf_inline} \
+ --set-section-flags .BTF.inline=alloc,readonly ${btf_data}
+ ONLY_SEC="${ONLY_SEC} --only-section=.BTF.inline"
+ fi
+ ${OBJCOPY} ${ONLY_SEC} --strip-all ${btf_data}
# Change e_type to ET_REL so that it can be used to link final vmlinux.
# GNU ld 2.35+ and lld do not allow an ET_EXEC input.
@@ -111,6 +127,15 @@ embed_btf_data()
if [ -f "${btf_base}" ]; then
${OBJCOPY} --add-section .BTF.base=${btf_base} ${ELF_FILE}
fi
+ btf_inline=${ELF_FILE}.BTF.inline
+ case "${ELF_FILE}" in
+ */btf_vmlinux_inline.ko)
+ btf_inline=${BTF_BASE}.BTF.inline
+ ;;
+ esac
+ if [ -n "${BTF_INLINE}" ] && [ -f "${btf_inline}" ]; then
+ ${OBJCOPY} --add-section .BTF.inline=${btf_inline} ${ELF_FILE}
+ fi
btf_ids="${ELF_FILE}.BTF_ids"
if [ -f "${btf_ids}" ]; then
${RESOLVE_BTFIDS} --patch_btfids ${btf_ids} ${ELF_FILE}
@@ -121,6 +146,8 @@ cleanup()
{
rm -f "${ELF_FILE}.BTF.1"
rm -f "${ELF_FILE}.BTF"
+ rm -f "${ELF_FILE}.BTF.inline"
+ rm -f "${ELF_FILE}.BTF.inline.link"
if [ "${BTFGEN_MODE}" = "module" ]; then
rm -f "${ELF_FILE}.BTF.base"
rm -f "${ELF_FILE}.BTF_ids"
diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh
index ab0b8125c8cb..5424caf5135b 100755
--- a/scripts/link-vmlinux.sh
+++ b/scripts/link-vmlinux.sh
@@ -289,7 +289,13 @@ vmlinux_link "${VMLINUX}"
if is_enabled CONFIG_DEBUG_INFO_BTF; then
info BTFIDS ${VMLINUX}
- ${RESOLVE_BTFIDS} --patch_btfids ${btfids_vmlinux} ${VMLINUX}
+ if [ "${BTF_INLINE}" = "m" ]; then
+ ${RESOLVE_BTFIDS} --patch_btfids ${btfids_vmlinux} \
+ --btf_link .BTF.inline:btf_vmlinux_inline:vmlinux.BTF.inline \
+ ${VMLINUX}
+ else
+ ${RESOLVE_BTFIDS} --patch_btfids ${btfids_vmlinux} ${VMLINUX}
+ fi
fi
mksysmap "${VMLINUX}" System.map
diff --git a/scripts/package/kernel.spec b/scripts/package/kernel.spec
index 46e80970f723..8f0933cbbe45 100644
--- a/scripts/package/kernel.spec
+++ b/scripts/package/kernel.spec
@@ -68,13 +68,13 @@ This package provides debug information for the kernel image and modules from th
%undefine _debugsource_packages
%undefine _debuginfo_subpackages
-# Preserve .BTF and .BTF.base sections in kernel modules during debuginfo
-# stripping. find-debuginfo.sh uses eu-strip which removes non-allocated ELF
-# sections like .BTF by default. .BTF.base is required for BTF distillation
-# support; without it, module BTF validation fails.
+# Preserve .BTF, .BTF.base and .BTF.inline sections in kernel modules during
+# debuginfo stripping. find-debuginfo.sh uses eu-strip which removes
+# non-allocated ELF sections like .BTF by default. .BTF.base is required for
+# BTF distillation support; without it, module BTF validation fails.
%global with_keep_section %(%{__find_debuginfo} --help 2>&1 | grep -c keep-section)
%if %{with_keep_section}
-%global _find_debuginfo_opts -r --keep-section .BTF --keep-section .BTF.base
+%global _find_debuginfo_opts -r --keep-section .BTF --keep-section .BTF.base --keep-section .BTF.inline
%else
%global _find_debuginfo_opts -r
%endif
diff --git a/tools/bpf/resolve_btfids/Build b/tools/bpf/resolve_btfids/Build
index 077de3829c72..602e1508bf09 100644
--- a/tools/bpf/resolve_btfids/Build
+++ b/tools/bpf/resolve_btfids/Build
@@ -1,6 +1,7 @@
hostprogs := resolve_btfids
resolve_btfids-y += main.o
+resolve_btfids-y += btf_colors.o
resolve_btfids-y += rbtree.o
resolve_btfids-y += zalloc.o
resolve_btfids-y += string.o
diff --git a/tools/bpf/resolve_btfids/btf_colors.c b/tools/bpf/resolve_btfids/btf_colors.c
new file mode 100644
index 000000000000..5d307d70e7f2
--- /dev/null
+++ b/tools/bpf/resolve_btfids/btf_colors.c
@@ -0,0 +1,145 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <bpf/libbpf_internal.h>
+#include "btf_colors.h"
+
+/*
+ * Functions in this file mainly exist to refer to functions from libbpf_internal.h,
+ * which can't be included in main.c because of the u32 poison and pr_warn macro conflicts.
+ */
+
+void btf_sha256(const void *data, size_t len,
+ __u8 out[BTF_SHA256_DIGEST_LENGTH])
+{
+ libbpf_sha256(data, len, out);
+}
+
+/*
+ * Marks `root_id` and local types reachable from it with `color`.
+ * `colors` is indexed by source ID; worklist has room for every local type.
+ */
+void btf_mark_reachable(struct btf *btf, __u32 root_id, enum btf_color color,
+ __u8 *colors, __u32 *worklist)
+{
+ const struct btf *base = btf__base_btf(btf);
+ __u32 start_id = base ? btf__type_cnt(base) : 1;
+ __u32 pending = 0;
+
+ if (root_id < start_id || (colors[root_id] & color) == color)
+ return;
+ colors[root_id] |= color;
+ worklist[pending++] = root_id;
+ while (pending) {
+ const struct btf_type *t = btf__type_by_id(btf, worklist[--pending]);
+ struct btf_field_iter it;
+ __u32 *id;
+
+ btf_field_iter_init(&it, (struct btf_type *)t, BTF_FIELD_ITER_IDS);
+ while ((id = btf_field_iter_next(&it))) {
+ if (*id < start_id || (colors[*id] & color) == color)
+ continue;
+ colors[*id] |= color;
+ worklist[pending++] = *id;
+ }
+ }
+}
+
+static int cmp_loc(const void *a, const void *b)
+{
+ const struct btf_loc *la = a, *lb = b;
+
+ if (la->func != lb->func)
+ return la->func < lb->func ? -1 : 1;
+ if (la->offset != lb->offset)
+ return la->offset < lb->offset ? -1 : 1;
+ if (la->loc_proto != lb->loc_proto)
+ return la->loc_proto < lb->loc_proto ? -1 : 1;
+ return 0;
+}
+
+static void remap(struct btf *btf, const __u32 *id_map, __u32 src_start_id)
+{
+ const struct btf *base = btf__base_btf(btf);
+ __u32 start_id = base ? btf__type_cnt(base) : 1;
+ __u32 i, type_cnt = btf__type_cnt(btf);
+
+ for (i = start_id; i < type_cnt; i++) {
+ struct btf_type *t = (struct btf_type *)btf__type_by_id(btf, i);
+ struct btf_field_iter it;
+ __u32 *id;
+
+ btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);
+ while ((id = btf_field_iter_next(&it))) {
+ /* Void and types in the original ancestor are unchanged. */
+ if (*id >= src_start_id)
+ *id = id_map[*id - src_start_id];
+ }
+ if (btf_is_locsec(t))
+ qsort(btf_locsec_locs(t), btf_vlen(t), sizeof(struct btf_loc), cmp_loc);
+ }
+}
+
+/*
+ * Split the `src` into `main_out` base and `inline_out`,
+ * according to `colors` array. Relative ordering remains
+ * the same as in `src`.
+ */
+int btf_split_by_color(struct btf *src, const __u8 *colors,
+ struct btf **main_out, struct btf **inline_out)
+{
+ LIBBPF_OPTS(btf_new_opts, opts,
+ .base_btf = (struct btf *)btf__base_btf(src),
+ .add_layout = btf_header(src)->layout_len != 0,
+ );
+ struct btf *main_btf, *inline_btf = NULL;
+ __u32 start_id = opts.base_btf ? btf__type_cnt(opts.base_btf) : 1;
+ __u32 type_cnt = btf__type_cnt(src);
+ __u32 *id_map, i;
+ int err = -ENOMEM;
+
+ *main_out = NULL;
+ *inline_out = NULL;
+ id_map = malloc((type_cnt - start_id ?: 1) * sizeof(*id_map));
+ if (!id_map)
+ return -ENOMEM;
+ main_btf = btf__new_empty_opts(&opts);
+ if (!main_btf)
+ goto out;
+ btf__set_endianness(main_btf, btf__endianness(src));
+ /* Copy MAIN and SHARED marked types to `main_out`. */
+ for (i = start_id; i < type_cnt; i++) {
+ if (colors[i] == BTF_COLOR_LOC)
+ continue;
+ err = btf__add_type(main_btf, src, btf__type_by_id(src, i));
+ if (err < 0)
+ goto out;
+ id_map[i - start_id] = err;
+ }
+ /*
+ * Copy LOC marked types to `inline_out`.
+ * `main_btf` types count is stable at this point.
+ */
+ inline_btf = btf__new_empty_split(main_btf);
+ if (!inline_btf) {
+ err = -ENOMEM;
+ goto out;
+ }
+ for (i = start_id; i < type_cnt; i++) {
+ if (colors[i] != BTF_COLOR_LOC)
+ continue;
+ err = btf__add_type(inline_btf, src, btf__type_by_id(src, i));
+ if (err < 0)
+ goto out;
+ id_map[i - start_id] = err;
+ }
+ remap(main_btf, id_map, start_id);
+ remap(inline_btf, id_map, start_id);
+ *main_out = main_btf;
+ *inline_out = inline_btf;
+ free(id_map);
+ return 0;
+out:
+ btf__free(inline_btf);
+ btf__free(main_btf);
+ free(id_map);
+ return err;
+}
diff --git a/tools/bpf/resolve_btfids/btf_colors.h b/tools/bpf/resolve_btfids/btf_colors.h
new file mode 100644
index 000000000000..56a85a242d6b
--- /dev/null
+++ b/tools/bpf/resolve_btfids/btf_colors.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef __RESOLVE_BTFIDS_BTF_COLORS_H
+#define __RESOLVE_BTFIDS_BTF_COLORS_H
+
+#include <bpf/btf.h>
+
+enum btf_color {
+ BTF_COLOR_NONE = 0,
+ BTF_COLOR_MAIN = 1,
+ BTF_COLOR_LOC = 2,
+ BTF_COLOR_SHARED = BTF_COLOR_MAIN | BTF_COLOR_LOC,
+};
+
+#define BTF_SHA256_DIGEST_LENGTH 32
+
+void btf_sha256(const void *data, size_t len,
+ __u8 out[BTF_SHA256_DIGEST_LENGTH]);
+void btf_mark_reachable(struct btf *btf, __u32 root, enum btf_color color,
+ __u8 *colors, __u32 *worklist);
+
+int btf_split_by_color(struct btf *src, const __u8 *colors,
+ struct btf **main_out, struct btf **inline_out);
+
+#endif
diff --git a/tools/bpf/resolve_btfids/main.c b/tools/bpf/resolve_btfids/main.c
index 37d7e7224207..f737fac60fa2 100644
--- a/tools/bpf/resolve_btfids/main.c
+++ b/tools/bpf/resolve_btfids/main.c
@@ -89,9 +89,11 @@
#include <linux/limits.h>
#include <bpf/btf.h>
#include <bpf/libbpf.h>
+#include "btf_colors.h"
#include <subcmd/parse-options.h>
#define BTF_IDS_SECTION ".BTF_ids"
+#define BTF_LINK_MODULE_NAME_MAX 64
#define BTF_ID_PREFIX "__BTF_ID__"
#define BTF_STRUCT "struct"
@@ -142,7 +144,10 @@ struct object {
struct btf *btf;
struct btf *base_btf;
+ struct btf *inline_btf;
+ struct btf *orig_btf;
bool distill_base;
+ bool extract_inline;
struct {
int fd;
@@ -154,6 +159,11 @@ struct object {
size_t strtabidx;
unsigned long idlist_addr;
int encoding;
+ int elf_class;
+ const char **func_symbols;
+ u32 func_symbols_cnt;
+ u32 func_symbols_cap;
+ bool func_symbols_collected;
} efile;
struct rb_root sets;
@@ -172,6 +182,19 @@ struct object {
u32 addr_syms_cap;
};
+struct btf_link {
+ char *value;
+ char *section;
+ char *module;
+ char *btf_path;
+};
+
+struct btf_links {
+ struct btf_link *links;
+ u32 cnt;
+ u32 cap;
+};
+
#define DECL_TAG_FASTCALL "bpf_fastcall"
#define DECL_TAG_KFUNC "bpf_kfunc"
@@ -259,6 +282,49 @@ static int __ensure_mem(void **data, u32 *cap, u32 cnt, size_t elem_sz)
#define ensure_mem(arr_ptr, cap_ptr, cnt) \
__ensure_mem((void **)(arr_ptr), (cap_ptr), (cnt), sizeof(**(arr_ptr)))
+static int parse_btf_link(const struct option *opt, const char *arg, int unset)
+{
+ struct btf_links *links = opt->value;
+ struct btf_link *link;
+ char *separator;
+
+ if (unset)
+ return -EINVAL;
+ if (__ensure_mem((void **)&links->links, &links->cap, links->cnt + 1,
+ sizeof(*links->links)))
+ return -ENOMEM;
+ link = &links->links[links->cnt];
+ memset(link, 0, sizeof(*link));
+ link->value = strdup(arg);
+ if (!link->value)
+ return -ENOMEM;
+ link->section = link->value;
+ separator = strchr(link->section, ':');
+ if (!separator || separator == link->section)
+ goto err_value;
+ *separator++ = '\0';
+ link->module = separator;
+ separator = strchr(link->module, ':');
+ if (!separator || separator == link->module || !separator[1])
+ goto err_value;
+ *separator++ = '\0';
+ link->btf_path = separator;
+ links->cnt++;
+ return 0;
+err_value:
+ free(link->value);
+ return -EINVAL;
+}
+
+static void free_btf_links(struct btf_links *links)
+{
+ u32 i;
+
+ for (i = 0; i < links->cnt; i++)
+ free(links->links[i].value);
+ free(links->links);
+}
+
static bool is_btf_id(const char *name)
{
return name && !strncmp(name, BTF_ID_PREFIX, sizeof(BTF_ID_PREFIX) - 1);
@@ -480,6 +546,7 @@ static int elf_collect(struct object *obj)
return -1;
}
obj->efile.encoding = ehdr.e_ident[EI_DATA];
+ obj->efile.elf_class = ehdr.e_ident[EI_CLASS];
/*
* Scan all the elf sections and look for save data
@@ -572,6 +639,58 @@ static const char *find_name_by_addr(struct object *obj, Elf64_Addr addr)
return res ? res->name : NULL;
}
+static int cmp_func_symbol(const void *a, const void *b)
+{
+ const char * const *name = a;
+ const char * const *other = b;
+
+ return strcmp(*name, *other);
+}
+
+static int collect_func_symbols(struct object *obj)
+{
+ Elf_Scn *scn;
+ GElf_Shdr sh;
+ int n, i;
+
+ if (obj->efile.symbols_shndx == -1)
+ return 0;
+
+ scn = elf_getscn(obj->efile.elf, obj->efile.symbols_shndx);
+ if (!scn || gelf_getshdr(scn, &sh) != &sh || !sh.sh_entsize)
+ return -EINVAL;
+ n = sh.sh_size / sh.sh_entsize;
+
+ for (i = 0; i < n; i++) {
+ GElf_Sym sym;
+ const char *name;
+
+ if (!gelf_getsym(obj->efile.symbols, i, &sym))
+ return -EINVAL;
+ if (GELF_ST_TYPE(sym.st_info) != STT_FUNC ||
+ sym.st_shndx == SHN_UNDEF || !sym.st_name)
+ continue;
+ name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, sym.st_name);
+ if (!name)
+ return -EINVAL;
+ if (ensure_mem(&obj->efile.func_symbols, &obj->efile.func_symbols_cap,
+ obj->efile.func_symbols_cnt + 1))
+ return -ENOMEM;
+ obj->efile.func_symbols[obj->efile.func_symbols_cnt++] = name;
+ }
+
+ qsort(obj->efile.func_symbols, obj->efile.func_symbols_cnt,
+ sizeof(*obj->efile.func_symbols), cmp_func_symbol);
+ obj->efile.func_symbols_collected = true;
+ return 0;
+}
+
+static bool has_func_symbol(const struct object *obj, const char *name)
+{
+ return bsearch(&name, obj->efile.func_symbols, obj->efile.func_symbols_cnt,
+ sizeof(*obj->efile.func_symbols), cmp_func_symbol) != NULL;
+}
+
static int symbols_collect(struct object *obj)
{
Elf_Scn *scn = NULL;
@@ -688,8 +807,9 @@ static int load_btf(struct object *obj)
if (err) {
pr_err("FAILED: load base BTF from %s: %s\n",
obj->base_btf_path, strerror(-err));
- goto out_err;
+ return err;
}
+ obj->base_btf = base_btf;
}
btf = btf__parse_split(obj->btf_path ?: obj->path, base_btf);
@@ -697,20 +817,12 @@ static int load_btf(struct object *obj)
if (err) {
pr_err("FAILED: load BTF from %s: %s\n",
obj->btf_path ?: obj->path, strerror(-err));
- goto out_err;
+ return err;
}
- obj->base_btf = base_btf;
obj->btf = btf;
return 0;
-
-out_err:
- btf__free(base_btf);
- btf__free(btf);
- obj->base_btf = NULL;
- obj->btf = NULL;
- return err;
}
static int symbols_resolve(struct object *obj)
@@ -977,6 +1089,153 @@ static int dump_raw_btf(struct btf *btf, const char *out_path)
return 0;
}
+static int patch_btf_link(struct object *obj, const struct btf_link *link)
+{
+ char section[128];
+ void *raw_btf_data = NULL;
+ size_t shdrstrndx, module_name_len;
+ int fd = -1, len, err = -1;
+ FILE *btf_file = NULL;
+ Elf_Scn *scn = NULL;
+ u32 raw_btf_size;
+ struct stat st;
+ Elf_Data *data;
+ GElf_Shdr sh;
+ char *name;
+ Elf *elf = NULL;
+ u8 *size;
+
+ if (stat(link->btf_path, &st)) {
+ err = -errno;
+ pr_err("FAILED to stat BTF file %s: %s\n",
+ link->btf_path, strerror(errno));
+ return err;
+ }
+ if (!S_ISREG(st.st_mode) || st.st_size <= 0 || st.st_size > UINT_MAX) {
+ pr_err("FAILED: invalid BTF file size or type: %s\n", link->btf_path);
+ return -EINVAL;
+ }
+ raw_btf_size = st.st_size;
+ raw_btf_data = malloc(raw_btf_size);
+ if (!raw_btf_data) {
+ pr_err("FAILED to allocate %u bytes for BTF file %s\n",
+ raw_btf_size, link->btf_path);
+ return -ENOMEM;
+ }
+ btf_file = fopen(link->btf_path, "rb");
+ if (!btf_file) {
+ err = -errno;
+ pr_err("FAILED to open BTF file %s: %s\n",
+ link->btf_path, strerror(errno));
+ goto out;
+ }
+ if (fread(raw_btf_data, raw_btf_size, 1, btf_file) != 1) {
+ err = -EIO;
+ pr_err("FAILED to read BTF file %s\n", link->btf_path);
+ goto out;
+ }
+ fclose(btf_file);
+ btf_file = NULL;
+ if (obj->efile.elf_class == ELFCLASS32) {
+ module_name_len = BTF_LINK_MODULE_NAME_MAX - 4;
+ } else if (obj->efile.elf_class == ELFCLASS64) {
+ module_name_len = BTF_LINK_MODULE_NAME_MAX - 8;
+ } else {
+ err = -EINVAL;
+ pr_err("FAILED: unsupported ELF class %u in %s\n",
+ obj->efile.elf_class, obj->path);
+ goto out;
+ }
+ if (strlen(link->module) >= module_name_len) {
+ err = -EINVAL;
+ pr_err("FAILED: module name too long for BTF link: %s\n", link->module);
+ goto out;
+ }
+ len = snprintf(section, sizeof(section), "%s.link", link->section);
+ if (len < 0 || len >= sizeof(section)) {
+ err = -ENAMETOOLONG;
+ pr_err("FAILED: BTF link section name too long: %s.link\n", link->section);
+ goto out;
+ }
+
+ fd = open(obj->path, O_RDWR);
+ if (fd < 0) {
+ err = -errno;
+ pr_err("FAILED to open %s: %s\n", obj->path, strerror(errno));
+ goto out;
+ }
+ elf = elf_begin(fd, ELF_C_RDWR_MMAP, NULL);
+ if (!elf) {
+ err = -EINVAL;
+ pr_err("FAILED to create ELF descriptor for %s: %s\n",
+ obj->path, elf_errmsg(-1));
+ goto out;
+ }
+ elf_flagelf(elf, ELF_C_SET, ELF_F_LAYOUT);
+ if (elf_getshdrstrndx(elf, &shdrstrndx)) {
+ pr_err("FAILED to get section name index from %s: %s\n",
+ obj->path, elf_errmsg(-1));
+ goto out;
+ }
+ while ((scn = elf_nextscn(elf, scn))) {
+ if (gelf_getshdr(scn, &sh) != &sh) {
+ pr_err("FAILED to get section header from %s: %s\n",
+ obj->path, elf_errmsg(-1));
+ goto out;
+ }
+ name = elf_strptr(elf, shdrstrndx, sh.sh_name);
+ if (name && !strcmp(name, section))
+ break;
+ }
+ if (!scn) {
+ pr_err("FAILED: section %s not found in %s\n", section, obj->path);
+ goto out;
+ }
+ data = elf_getdata(scn, NULL);
+ if (sh.sh_type != SHT_PROGBITS || !data || !data->d_buf ||
+ data->d_size != module_name_len +
+ BTF_SHA256_DIGEST_LENGTH + sizeof(u32)) {
+ pr_err("FAILED: section %s is not a writable BTF link\n", section);
+ goto out;
+ }
+ memset(data->d_buf, 0, data->d_size);
+ memcpy(data->d_buf, link->module, strlen(link->module) + 1);
+ btf_sha256(raw_btf_data, raw_btf_size,
+ (u8 *)data->d_buf + module_name_len);
+ size = (u8 *)data->d_buf + module_name_len + BTF_SHA256_DIGEST_LENGTH;
+ if (obj->efile.encoding == ELFDATA2LSB) {
+ size[0] = raw_btf_size;
+ size[1] = raw_btf_size >> 8;
+ size[2] = raw_btf_size >> 16;
+ size[3] = raw_btf_size >> 24;
+ } else if (obj->efile.encoding == ELFDATA2MSB) {
+ size[0] = raw_btf_size >> 24;
+ size[1] = raw_btf_size >> 16;
+ size[2] = raw_btf_size >> 8;
+ size[3] = raw_btf_size;
+ } else {
+ pr_err("FAILED: unsupported ELF data encoding %u in %s\n",
+ obj->efile.encoding, obj->path);
+ goto out;
+ }
+ elf_flagdata(data, ELF_C_SET, ELF_F_DIRTY);
+ if (elf_update(elf, ELF_C_WRITE) < 0) {
+ pr_err("FAILED to update ELF file %s: %s\n",
+ obj->path, elf_errmsg(-1));
+ goto out;
+ }
+ err = 0;
+out:
+ if (elf)
+ elf_end(elf);
+ if (fd >= 0)
+ close(fd);
+ if (btf_file)
+ fclose(btf_file);
+ free(raw_btf_data);
+ return err;
+}
+
static const struct btf_type *btf_type_skip_qualifiers(const struct btf *btf, s32 type_id)
{
const struct btf_type *t = btf__type_by_id(btf, type_id);
@@ -1510,6 +1769,84 @@ out:
return err;
}
+static bool keep_in_base_btf(struct object *obj, const struct btf_type *t)
+{
+ const char *name;
+
+ switch (btf_kind(t)) {
+ case BTF_KIND_FUNC:
+ name = btf__name_by_offset(obj->btf, t->name_off);
+ return !obj->efile.func_symbols_collected || has_func_symbol(obj, name) ||
+ btf_id__find(&obj->funcs, name);
+ case BTF_KIND_FUNC_PROTO:
+ case BTF_KIND_LOC_PARAM:
+ case BTF_KIND_LOC_PROTO:
+ case BTF_KIND_LOCSEC:
+ return false;
+ default:
+ /*
+ * Ordinary types stay in base btf even if only inline records use them.
+ * Should we relax this?
+ */
+ return true;
+ }
+}
+
+/* Mark which types should go to .BTF.inline */
+static int color_btf(struct object *obj, __u8 *colors)
+{
+ struct btf *btf = obj->btf;
+ const struct btf *base = btf__base_btf(btf);
+ __u32 start_id = base ? btf__type_cnt(base) : 1;
+ __u32 type_cnt = btf__type_cnt(btf), i;
+ __u32 *worklist;
+
+ worklist = malloc((type_cnt - start_id ?: 1) * sizeof(*worklist));
+ if (!worklist)
+ return -ENOMEM;
+ /* First, mark all types reachable from LOC BTF entries. */
+ for (i = start_id; i < type_cnt; i++) {
+ const struct btf_type *t = btf__type_by_id(btf, i);
+
+ if (!btf_is_locsec(t) && !btf_is_loc_proto(t) && !btf_is_loc_param(t))
+ continue;
+ btf_mark_reachable(btf, i, BTF_COLOR_LOC, colors, worklist);
+ }
+ /* Next, traverse non-marked types converting some LOC markings to SHARED. */
+ for (i = start_id; i < type_cnt; i++) {
+ const struct btf_type *t = btf__type_by_id(btf, i);
+
+ if (colors[i] == BTF_COLOR_LOC && !keep_in_base_btf(obj, t))
+ continue;
+ btf_mark_reachable(btf, i, BTF_COLOR_MAIN, colors, worklist);
+ }
+ free(worklist);
+ return 0;
+}
+
+static int extract_inline_btf(struct object *obj)
+{
+ struct btf *btf = obj->btf, *main_btf, *inline_btf;
+ __u8 *colors;
+ int err;
+
+ colors = calloc(btf__type_cnt(btf), sizeof(*colors));
+ if (!colors)
+ return -ENOMEM;
+ err = color_btf(obj, colors);
+ if (err)
+ goto out;
+ err = btf_split_by_color(btf, colors, &main_btf, &inline_btf);
+ if (!err) {
+ obj->orig_btf = btf;
+ obj->btf = main_btf;
+ obj->inline_btf = inline_btf;
+ }
+out:
+ free(colors);
+ return err;
+}
+
/*
* Sort types by name in ascending order resulting in all
* anonymous types being placed before named types.
@@ -1583,14 +1920,14 @@ static int finalize_btf(struct object *obj)
err = btf__dedup(obj->btf, NULL);
if (err) {
pr_err("FAILED to dedup BTF: %s\n", strerror(errno));
- goto out_err;
+ return err;
}
if (obj->base_btf && obj->distill_base) {
err = btf__distill_base(obj->btf, &base_btf, &btf);
if (err) {
pr_err("FAILED to distill base BTF: %s\n", strerror(errno));
- goto out_err;
+ return err;
}
btf__free(obj->base_btf);
@@ -1602,18 +1939,18 @@ static int finalize_btf(struct object *obj)
err = sort_btf_by_name(obj->btf);
if (err) {
pr_err("FAILED to sort BTF: %s\n", strerror(errno));
- goto out_err;
+ return err;
}
- return 0;
-
-out_err:
- btf__free(base_btf);
- btf__free(btf);
- obj->base_btf = NULL;
- obj->btf = NULL;
+ if (obj->extract_inline) {
+ err = extract_inline_btf(obj);
+ if (err) {
+ pr_err("FAILED to extract inline BTF: %s\n", strerror(-err));
+ return err;
+ }
+ }
- return err;
+ return 0;
}
static inline int make_out_path(char *buf, u32 buf_sz, const char *in_path, const char *suffix)
@@ -1740,7 +2077,7 @@ out:
static const char * const resolve_btfids_usage[] = {
"resolve_btfids [<options>] <ELF object>",
- "resolve_btfids --patch_btfids <.BTF_ids file> <ELF object>",
+ "resolve_btfids --patch_btfids <.BTF_ids file> [--btf_link <section:module:btf-file>] <ELF object>",
NULL
};
@@ -1758,6 +2095,7 @@ int main(int argc, const char **argv)
.sets = RB_ROOT,
};
const char *btfids_path = NULL;
+ struct btf_links btf_links = {};
bool fatal_warnings = false;
bool resolve_btfids = true;
char out_path[PATH_MAX];
@@ -1773,6 +2111,10 @@ int main(int argc, const char **argv)
"turn warnings into errors"),
OPT_BOOLEAN(0, "distill_base", &obj.distill_base,
"distill --btf_base and emit .BTF.base section data"),
+ OPT_BOOLEAN(0, "inline", &obj.extract_inline,
+ "extract location BTF into a .BTF.inline file"),
+ OPT_CALLBACK(0, "btf_link", &btf_links, "section:module:btf-file",
+ "patch a BTF link with --patch_btfids", parse_btf_link),
OPT_STRING(0, "patch_btfids", &btfids_path, "file",
"path to .BTF_ids section data blob to patch into ELF file"),
OPT_END()
@@ -1783,14 +2125,34 @@ int main(int argc, const char **argv)
PARSE_OPT_STOP_AT_NON_OPTION);
if (argc != 1)
usage_with_options(resolve_btfids_usage, btfid_options);
+ if (btf_links.cnt && !btfids_path) {
+ pr_err("--btf_link requires --patch_btfids\n");
+ goto out;
+ }
obj.path = argv[0];
- if (btfids_path)
- return patch_btfids(btfids_path, obj.path);
+ if (btfids_path) {
+ err = patch_btfids(btfids_path, obj.path);
+ if (err || !btf_links.cnt)
+ goto out;
+ if (elf_collect(&obj)) {
+ err = -EINVAL;
+ goto out;
+ }
+ for (u32 i = 0; i < btf_links.cnt; i++) {
+ struct btf_link *link = &btf_links.links[i];
+ err = patch_btf_link(&obj, link);
+ if (err)
+ break;
+ }
+ goto out;
+ }
if (elf_collect(&obj))
goto out;
+ if (obj.extract_inline && collect_func_symbols(&obj))
+ goto out;
/*
* We did not find .BTF_ids section or symbols section,
@@ -1841,18 +2203,33 @@ dump_btf:
if (err)
goto out;
}
+ if (obj.inline_btf) {
+ err = make_out_path(out_path, sizeof(out_path), obj.path, BTF_ELF_SEC ".inline");
+ err = err ?: dump_raw_btf(obj.inline_btf, out_path);
+ if (err)
+ goto out;
+ }
if (!(fatal_warnings && warnings))
err = 0;
out:
- btf__free(obj.base_btf);
+ free_btf_links(&btf_links);
+ /*
+ * Free BTFs in reverse dependency order. inline_btf borrows obj.btf;
+ * obj.btf can borrow a base owned by orig_btf or obj.base_btf, and
+ * orig_btf can also borrow obj.base_btf.
+ */
+ btf__free(obj.inline_btf);
btf__free(obj.btf);
+ btf__free(obj.orig_btf);
+ btf__free(obj.base_btf);
btf_id__free_all(&obj.structs);
btf_id__free_all(&obj.unions);
btf_id__free_all(&obj.typedefs);
btf_id__free_all(&obj.funcs);
btf_id__free_all(&obj.sets);
free(obj.addr_syms);
+ free(obj.efile.func_symbols);
if (obj.efile.elf) {
elf_end(obj.efile.elf);
close(obj.efile.fd);
diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c
index 512b3962f75f..d2ff6f5d6d5b 100644
--- a/tools/lib/bpf/btf.c
+++ b/tools/lib/bpf/btf.c
@@ -6640,7 +6640,7 @@ void btf_set_base_btf(struct btf *btf, const struct btf *base_btf)
int btf__relocate(struct btf *btf, const struct btf *base_btf)
{
- int err = btf_relocate(btf, base_btf, NULL);
+ int err = btf_relocate(btf, base_btf, NULL, NULL);
if (!err)
btf->owns_base = false;
diff --git a/tools/lib/bpf/btf_relocate.c b/tools/lib/bpf/btf_relocate.c
index df5fa4bd87d6..e55dd75f6c95 100644
--- a/tools/lib/bpf/btf_relocate.c
+++ b/tools/lib/bpf/btf_relocate.c
@@ -441,7 +441,8 @@ static int btf_relocate_rewrite_strs(struct btf_relocate *r, __u32 i)
/* If successful, output of relocation is updated BTF with base BTF pointing
* at base_btf, and type ids, strings adjusted accordingly.
*/
-int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **id_map)
+int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **id_map,
+ __u32 **str_map)
{
unsigned int nr_types = btf__type_cnt(btf);
const struct btf_header *dist_base_hdr;
@@ -512,6 +513,10 @@ int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **id_map)
*id_map = r.id_map;
r.id_map = NULL;
}
+ if (str_map) {
+ *str_map = r.str_map;
+ r.str_map = NULL;
+ }
err_out:
free(r.id_map);
free(r.str_map);
diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h
index 546f65b95cf4..4e45dfacb548 100644
--- a/tools/lib/bpf/libbpf_internal.h
+++ b/tools/lib/bpf/libbpf_internal.h
@@ -249,7 +249,8 @@ const char *btf_kind_str(const struct btf_type *t);
const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id);
const struct btf_header *btf_header(const struct btf *btf);
void btf_set_base_btf(struct btf *btf, const struct btf *base_btf);
-int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **id_map);
+int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **id_map,
+ __u32 **str_map);
bool btf_type_is_traceable_func(const struct btf *btf, const struct btf_type *t);
static inline enum btf_func_linkage btf_func_linkage(const struct btf_type *t)
diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile
index a22be7efd1fa..80d8bbbc3084 100644
--- a/tools/testing/selftests/bpf/Makefile
+++ b/tools/testing/selftests/bpf/Makefile
@@ -343,7 +343,9 @@ else
endif
$(RESOLVE_BTFIDS): $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/resolve_btfids \
- $(TOOLSDIR)/bpf/resolve_btfids/main.c \
+ $(TOOLSDIR)/bpf/resolve_btfids/main.c \
+ $(TOOLSDIR)/bpf/resolve_btfids/btf_colors.c \
+ $(TOOLSDIR)/bpf/resolve_btfids/btf_colors.h \
$(TOOLSDIR)/lib/rbtree.c \
$(TOOLSDIR)/lib/zalloc.c \
$(TOOLSDIR)/lib/string.c \
diff --git a/tools/testing/selftests/bpf/prog_tests/btf_inline.c b/tools/testing/selftests/bpf/prog_tests/btf_inline.c
new file mode 100644
index 000000000000..d88fab6429b1
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/btf_inline.c
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2026, Oracle and/or its affiliates. */
+
+#include <test_progs.h>
+#include <bpf/btf.h>
+#include <bpf/libbpf.h>
+
+#define BTF_SYSFS_DIR "/sys/kernel/btf"
+#define BTF_INLINE_SUFFIX ".inline"
+
+/*
+ * For a specific inline site, verify we have the right function,
+ * loc proto and loc param representation and that the offset is
+ * reasonable given the caller where it was inlined.
+ */
+void test_btf_inline(void)
+{
+ struct btf *inline_btf = NULL, *btf = NULL, *vmlinux_btf = NULL;
+ const char *inline_caller = "bpf_testmod_test_write";
+ const char *inline_func = "testmod_inline_len";
+ unsigned long long caller_addr, base_addr;
+ bool skip = false, found_loc = false;
+ const struct btf_loc_param *lp;
+ int locsec_id, func_id, n, i;
+ const struct btf_type *t;
+ struct btf_loc *l;
+ const __u32 *p;
+ int err = 0;
+
+ if (!env.has_testmod) {
+ test__skip();
+ return;
+ }
+
+ base_addr = module_get_base_addr("bpf_testmod");
+ if (!ASSERT_NEQ(base_addr, 0, "base_addr_nonzero"))
+ return;
+
+ if (!ASSERT_OK(kallsyms_find(inline_caller, &caller_addr),
+ "find_inline_caller"))
+ return;
+ if (!ASSERT_NEQ(caller_addr, 0, "caller_addr_nonzero"))
+ return;
+ if (!ASSERT_GT(caller_addr, base_addr, "caller_addr_gt_base_addr"))
+ return;
+ caller_addr -= base_addr;
+
+ vmlinux_btf = btf__load_vmlinux_btf();
+ if (!ASSERT_OK_PTR(vmlinux_btf, "vmlinux_btf"))
+ return;
+
+ btf = btf__parse_split(BTF_SYSFS_DIR "/bpf_testmod", vmlinux_btf);
+ if (!ASSERT_OK_PTR(btf, "bpf_testmod_btf"))
+ goto out;
+
+ inline_btf = btf__parse_split(BTF_SYSFS_DIR "/bpf_testmod" BTF_INLINE_SUFFIX,
+ btf);
+ err = libbpf_get_error(inline_btf);
+ /* pahole may not have inline BTF feature support. */
+ if (err == -ENOENT) {
+ skip = true;
+ goto out;
+ }
+ if (!ASSERT_OK(err, "parse_inline_btf"))
+ goto out;
+ locsec_id = btf__find_by_name_kind(inline_btf, "inline.text", BTF_KIND_LOCSEC);
+ if (locsec_id < 0) {
+ skip = true;
+ goto out;
+ }
+ func_id = btf__find_by_name_kind(inline_btf, inline_func, BTF_KIND_FUNC);
+ if (!ASSERT_GT(func_id, 0, "inline_caller_func"))
+ goto out;
+ t = btf__type_by_id(inline_btf, locsec_id);
+ n = btf_vlen(t);
+ for (i = 0, l = btf_locsec_locs(t); i < n; i++, l++) {
+ if (l->func == func_id) {
+ found_loc = true;
+ break;
+ }
+ }
+ if (!ASSERT_TRUE(found_loc, "found_loc"))
+ goto out;
+ if (!ASSERT_GT(l->loc_proto, 0, "loc_proto_id"))
+ goto out;
+ if (!ASSERT_GT(l->offset, 0, "loc_offset"))
+ goto out;
+ t = btf__type_by_id(inline_btf, l->loc_proto);
+ if (!ASSERT_OK_PTR(t, "loc_proto_ptr"))
+ goto out;
+ if (!ASSERT_EQ(btf_vlen(t), 1, "loc_proto_one_param"))
+ goto out;
+ p = btf_loc_proto_params(t);
+ if (!ASSERT_GT(*p, 0, "loc_proto_has_loc_param"))
+ goto out;
+ t = btf__type_by_id(inline_btf, *p);
+ if (!ASSERT_OK_PTR(t, "loc_param_ptr"))
+ goto out;
+ lp = btf_loc_param(t);
+ if (!ASSERT_TRUE(lp->flags & BTF_LOC_PARAM_REG, "param_has_reg"))
+ goto out;
+ if (!ASSERT_GT(l->offset, caller_addr, "inline_gt_caller"))
+ goto out;
+ /* simple sanity test to roughly ensure inline site still in function */
+ if (!ASSERT_LT(l->offset, caller_addr + 512, "inline_in_caller"))
+ goto out;
+out:
+ btf__free(inline_btf);
+ btf__free(btf);
+ btf__free(vmlinux_btf);
+ if (skip)
+ test__skip();
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/btf_sysfs.c b/tools/testing/selftests/bpf/prog_tests/btf_sysfs.c
index 3923e64c4c1d..3fde8598189f 100644
--- a/tools/testing/selftests/bpf/prog_tests/btf_sysfs.c
+++ b/tools/testing/selftests/bpf/prog_tests/btf_sysfs.c
@@ -3,15 +3,21 @@
#include <test_progs.h>
#include <bpf/btf.h>
+#include <dirent.h>
+#include <limits.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <fcntl.h>
#include <unistd.h>
+#define BTF_SYSFS_DIR "/sys/kernel/btf"
+#define BTF_INLINE_SUFFIX ".inline"
+
static void test_btf_mmap_sysfs(const char *path, struct btf *base)
{
struct stat st;
__u64 btf_size, end;
+ size_t mmap_len = 0;
void *raw_data = NULL;
int fd = -1;
long page_size;
@@ -26,6 +32,7 @@ static void test_btf_mmap_sysfs(const char *path, struct btf *base)
btf_size = st.st_size;
end = (btf_size + page_size - 1) / page_size * page_size;
+ mmap_len = btf_size;
fd = open(path, O_RDONLY);
if (!ASSERT_GE(fd, 0, "open_btf"))
@@ -39,11 +46,13 @@ static void test_btf_mmap_sysfs(const char *path, struct btf *base)
if (!ASSERT_EQ(raw_data, MAP_FAILED, "mmap_btf_shared"))
goto cleanup;
- raw_data = mmap(NULL, end + 1, PROT_READ, MAP_PRIVATE, fd, 0);
+ mmap_len = end + 1;
+ raw_data = mmap(NULL, mmap_len, PROT_READ, MAP_PRIVATE, fd, 0);
if (!ASSERT_EQ(raw_data, MAP_FAILED, "mmap_btf_invalid_size"))
goto cleanup;
- raw_data = mmap(NULL, end, PROT_READ, MAP_PRIVATE, fd, 0);
+ mmap_len = end;
+ raw_data = mmap(NULL, mmap_len, PROT_READ, MAP_PRIVATE, fd, 0);
if (!ASSERT_OK_PTR(raw_data, "mmap_btf"))
goto cleanup;
@@ -70,12 +79,84 @@ static void test_btf_mmap_sysfs(const char *path, struct btf *base)
cleanup:
btf__free(btf);
if (raw_data && raw_data != MAP_FAILED)
- munmap(raw_data, btf_size);
+ munmap(raw_data, mmap_len);
if (fd >= 0)
close(fd);
}
+static void test_btf_inline_sysfs_all(void)
+{
+ struct btf *vmlinux_btf;
+ struct dirent *dentry;
+ DIR *dir;
+ int err = 0;
+
+ dir = opendir(BTF_SYSFS_DIR);
+ if (!ASSERT_OK_PTR(dir, "open_btf_sysfs"))
+ return;
+
+ vmlinux_btf = btf__parse(BTF_SYSFS_DIR "/vmlinux", NULL);
+ if (!ASSERT_OK_PTR(vmlinux_btf, "parse_vmlinux_btf")) {
+ closedir(dir);
+ return;
+ }
+
+ while ((dentry = readdir(dir)) != NULL) {
+ struct btf *base_btf = NULL, *module_btf = NULL, *inline_btf = NULL;
+ char btf_path[PATH_MAX], inline_path[PATH_MAX];
+ struct stat st;
+
+ /* Skip ".", ".." and "foo.inline" */
+ if (strstr(dentry->d_name, "."))
+ continue;
+
+ if (strcmp(dentry->d_name, "vmlinux") == 0)
+ base_btf = vmlinux_btf;
+
+ if (snprintf(btf_path, sizeof(btf_path), "%s/%s",
+ BTF_SYSFS_DIR, dentry->d_name) >= sizeof(btf_path) ||
+ snprintf(inline_path, sizeof(inline_path), "%s/%s%s",
+ BTF_SYSFS_DIR, dentry->d_name, BTF_INLINE_SUFFIX) >=
+ sizeof(inline_path)) {
+ ASSERT_FAIL("BTF sysfs path is too long\n");
+ break;
+ }
+
+ if (!base_btf) {
+ module_btf = btf__parse_split(btf_path, vmlinux_btf);
+ err = libbpf_get_error(module_btf);
+ if (err) {
+ /* A module can be unloaded while its sysfs entry is iterated. */
+ if (err == -ENOENT)
+ continue;
+ ASSERT_OK(err, "parse_module_btf");
+ continue;
+ }
+ base_btf = module_btf;
+ }
+ if (stat(inline_path, &st)) {
+ err = errno;
+ if (err != ENOENT)
+ ASSERT_OK(err, "stat_inline_btf");
+ btf__free(module_btf);
+ continue;
+ }
+ if (base_btf == vmlinux_btf)
+ test_btf_mmap_sysfs(inline_path, base_btf);
+ inline_btf = btf__parse_split(inline_path, base_btf);
+ err = libbpf_get_error(inline_btf);
+ if (!err)
+ btf__free(inline_btf);
+ ASSERT_OK(err, "parse_inline_btf");
+ btf__free(module_btf);
+ }
+ closedir(dir);
+
+ btf__free(vmlinux_btf);
+}
+
void test_btf_sysfs(void)
{
test_btf_mmap_sysfs("/sys/kernel/btf/vmlinux", NULL);
+ test_btf_inline_sysfs_all();
}
diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c
index 93847ca6293b..b4f0af2aeca4 100644
--- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c
+++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c
@@ -735,6 +735,13 @@ out:
EXPORT_SYMBOL(bpf_testmod_test_read);
ALLOW_ERROR_INJECTION(bpf_testmod_test_read, ERRNO);
+static __always_inline size_t testmod_inline_len(size_t len)
+{
+ /* Keep an instruction in the inline site without changing len. */
+ asm volatile("" : "+m"(len));
+ return len;
+}
+
noinline ssize_t
bpf_testmod_test_write(struct file *file, struct kobject *kobj,
const struct bin_attribute *bin_attr,
@@ -743,7 +750,7 @@ bpf_testmod_test_write(struct file *file, struct kobject *kobj,
struct bpf_testmod_test_write_ctx ctx = {
.buf = buf,
.off = off,
- .len = len,
+ .len = testmod_inline_len(len),
};
trace_bpf_testmod_test_write_bare_tp(current, &ctx);
@@ -802,7 +809,7 @@ static struct testmod_uprobe uprobe = {
.consumer.ret_handler = uprobe_ret_handler,
};
-static int testmod_register_uprobe(loff_t offset)
+static __always_inline int testmod_register_uprobe(loff_t offset)
{
int err = -EBUSY;
diff --git a/tools/testing/selftests/bpf/trace_helpers.c b/tools/testing/selftests/bpf/trace_helpers.c
index 679008b310d9..f3a1d627699d 100644
--- a/tools/testing/selftests/bpf/trace_helpers.c
+++ b/tools/testing/selftests/bpf/trace_helpers.c
@@ -244,6 +244,26 @@ out:
return err;
}
+unsigned long long module_get_base_addr(const char *module)
+{
+ unsigned long long addr, base_addr = 0;
+ char name[500];
+ FILE *f;
+
+ f = fopen("/proc/modules", "r");
+ if (!f)
+ return 0;
+
+ while (fscanf(f, "%499s %*s %*s %*s %*s 0x%llx", name, &addr) == 2) {
+ if (strcmp(module, name) == 0) {
+ base_addr = addr;
+ break;
+ }
+ }
+ fclose(f);
+ return base_addr;
+}
+
#ifdef PROCMAP_QUERY
int env_verbosity __weak = 0;
diff --git a/tools/testing/selftests/bpf/trace_helpers.h b/tools/testing/selftests/bpf/trace_helpers.h
index 01c8ecc45627..eb0d0d9804d6 100644
--- a/tools/testing/selftests/bpf/trace_helpers.h
+++ b/tools/testing/selftests/bpf/trace_helpers.h
@@ -38,6 +38,7 @@ typedef int (*ksym_search_cmp_t)(const void *p1, const struct ksym *p2);
int load_kallsyms(void);
struct ksym *ksym_search(long key);
long ksym_get_addr(const char *name);
+unsigned long long module_get_base_addr(const char *module);
struct ksyms *load_kallsyms_local(void);
struct ksym *ksym_search_local(struct ksyms *ksyms, long key);
diff --git a/tools/testing/selftests/hid/Makefile b/tools/testing/selftests/hid/Makefile
index 2f423de83147..d1b65fd37911 100644
--- a/tools/testing/selftests/hid/Makefile
+++ b/tools/testing/selftests/hid/Makefile
@@ -148,6 +148,8 @@ endif
$(RESOLVE_BTFIDS): $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/resolve_btfids \
$(TOOLSDIR)/bpf/resolve_btfids/main.c \
+ $(TOOLSDIR)/bpf/resolve_btfids/btf_colors.c \
+ $(TOOLSDIR)/bpf/resolve_btfids/btf_colors.h \
$(TOOLSDIR)/lib/rbtree.c \
$(TOOLSDIR)/lib/zalloc.c \
$(TOOLSDIR)/lib/string.c \