diff options
Diffstat (limited to 'tools')
200 files changed, 10364 insertions, 1002 deletions
diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile index 271a7dc77273..b0f7168e7943 100644 --- a/tools/bpf/bpftool/Makefile +++ b/tools/bpf/bpftool/Makefile @@ -99,7 +99,7 @@ endif HOST_LDFLAGS := $(LDFLAGS) # Remove warnings for libbpf bootstrap build -LIBBPF_BOOTSTRAP_CFLAGS := $(filter-out -W -Wall -Wextra -Wformat -Wformat-signedness,$(HOST_CFLAGS)) +LIBBPF_BOOTSTRAP_CFLAGS := $(filter-out -W -Wall -Wextra -Wformat%,$(HOST_CFLAGS)) INSTALL ?= install RM ?= rm -f diff --git a/tools/bpf/bpftool/btf.c b/tools/bpf/bpftool/btf.c index 6ef908adf3a4..c9589026da8d 100644 --- a/tools/bpf/bpftool/btf.c +++ b/tools/bpf/bpftool/btf.c @@ -179,7 +179,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, case BTF_KIND_STRUCT: case BTF_KIND_UNION: { const struct btf_member *m = (const void *)(t + 1); - __u32 i, vlen = BTF_INFO_VLEN(t->info); + __u32 i, vlen = btf_vlen(t); if (json_output) { jsonw_uint_field(w, "size", t->size); @@ -193,7 +193,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, const char *name = btf_str(btf, m->name_off); __u32 bit_off, bit_sz; - if (BTF_INFO_KFLAG(t->info)) { + if (btf_kflag(t)) { bit_off = BTF_MEMBER_BIT_OFFSET(m->offset); bit_sz = BTF_MEMBER_BITFIELD_SIZE(m->offset); } else { @@ -224,7 +224,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, } case BTF_KIND_ENUM: { const struct btf_enum *v = (const void *)(t + 1); - __u32 i, vlen = BTF_INFO_VLEN(t->info); + __u32 i, vlen = btf_vlen(t); const char *encoding; encoding = btf_kflag(t) ? "SIGNED" : "UNSIGNED"; @@ -300,8 +300,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, break; } case BTF_KIND_FWD: { - const char *fwd_kind = BTF_INFO_KFLAG(t->info) ? "union" - : "struct"; + const char *fwd_kind = btf_kflag(t) ? "union" : "struct"; if (json_output) jsonw_string_field(w, "fwd_kind", fwd_kind); @@ -322,7 +321,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, } case BTF_KIND_FUNC_PROTO: { const struct btf_param *p = (const void *)(t + 1); - __u32 i, vlen = BTF_INFO_VLEN(t->info); + __u32 i, vlen = btf_vlen(t); if (json_output) { jsonw_uint_field(w, "ret_type_id", t->type); @@ -365,7 +364,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, case BTF_KIND_DATASEC: { const struct btf_var_secinfo *v = (const void *)(t + 1); const struct btf_type *vt; - __u32 i, vlen = BTF_INFO_VLEN(t->info); + __u32 i, vlen = btf_vlen(t); if (json_output) { jsonw_uint_field(w, "size", t->size); diff --git a/tools/bpf/bpftool/btf_dumper.c b/tools/bpf/bpftool/btf_dumper.c index 9dc8425b1789..e4075824343f 100644 --- a/tools/bpf/bpftool/btf_dumper.c +++ b/tools/bpf/bpftool/btf_dumper.c @@ -476,8 +476,8 @@ static int btf_dumper_struct(const struct btf_dumper *d, __u32 type_id, if (!t) return -EINVAL; - kind_flag = BTF_INFO_KFLAG(t->info); - vlen = BTF_INFO_VLEN(t->info); + kind_flag = btf_kflag(t); + vlen = btf_vlen(t); jsonw_start_object(d->jw); m = (struct btf_member *)(t + 1); @@ -535,7 +535,7 @@ static int btf_dumper_datasec(const struct btf_dumper *d, __u32 type_id, if (!t) return -EINVAL; - vlen = BTF_INFO_VLEN(t->info); + vlen = btf_vlen(t); vsi = (struct btf_var_secinfo *)(t + 1); jsonw_start_object(d->jw); @@ -557,7 +557,7 @@ static int btf_dumper_do_type(const struct btf_dumper *d, __u32 type_id, { const struct btf_type *t = btf__type_by_id(d->btf, type_id); - switch (BTF_INFO_KIND(t->info)) { + switch (btf_kind(t)) { case BTF_KIND_INT: return btf_dumper_int(t, bit_offset, data, d->jw, d->is_plain_text); @@ -631,7 +631,7 @@ static int __btf_dumper_type_only(const struct btf *btf, __u32 type_id, t = btf__type_by_id(btf, type_id); - switch (BTF_INFO_KIND(t->info)) { + switch (btf_kind(t)) { case BTF_KIND_INT: case BTF_KIND_TYPEDEF: case BTF_KIND_FLOAT: @@ -661,7 +661,7 @@ static int __btf_dumper_type_only(const struct btf *btf, __u32 type_id, break; case BTF_KIND_FWD: BTF_PRINT_ARG("%s %s ", - BTF_INFO_KFLAG(t->info) ? "union" : "struct", + btf_kflag(t) ? "union" : "struct", btf__name_by_offset(btf, t->name_off)); break; case BTF_KIND_VOLATILE: @@ -718,7 +718,7 @@ static int btf_dump_func(const struct btf *btf, char *func_sig, BTF_PRINT_ARG("%s(", btf__name_by_offset(btf, func->name_off)); else BTF_PRINT_ARG("("); - vlen = BTF_INFO_VLEN(func_proto->info); + vlen = btf_vlen(func_proto); for (i = 0; i < vlen; i++) { struct btf_param *arg = &((struct btf_param *)(func_proto + 1))[i]; diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c index 8bfcff9e2f63..ef366ccc9650 100644 --- a/tools/bpf/bpftool/common.c +++ b/tools/bpf/bpftool/common.c @@ -832,6 +832,8 @@ static int prog_fd_by_nametag(void *nametag, int **fds, bool tag) fd = bpf_prog_get_fd_by_id(id); if (fd < 0) { + if (errno == ENOENT) + continue; p_err("can't get prog by id (%u): %s", id, strerror(errno)); goto err_close_fds; @@ -996,6 +998,8 @@ static int map_fd_by_name(char *name, int **fds, opts_ro.open_flags = BPF_F_RDONLY; fd = bpf_map_get_fd_by_id_opts(id, &opts_ro); if (fd < 0) { + if (errno == ENOENT) + continue; p_err("can't get map by id (%u): %s", id, strerror(errno)); goto err_close_fds; diff --git a/tools/bpf/bpftool/gen.c b/tools/bpf/bpftool/gen.c index 6ae7262ebe0c..a50540ef6521 100644 --- a/tools/bpf/bpftool/gen.c +++ b/tools/bpf/bpftool/gen.c @@ -101,6 +101,12 @@ static bool get_map_ident(const struct bpf_map *map, char *buf, size_t buf_sz) return true; } + if (bpf_map__type(map) == BPF_MAP_TYPE_PERCPU_ARRAY) { + snprintf(buf, buf_sz, "%s", name + 1); + sanitize_identifier(buf); + return true; + } + for (i = 0, n = ARRAY_SIZE(sfxs); i < n; i++) { const char *sfx = sfxs[i], *p; @@ -117,7 +123,7 @@ static bool get_map_ident(const struct bpf_map *map, char *buf, size_t buf_sz) static bool get_datasec_ident(const char *sec_name, char *buf, size_t buf_sz) { - static const char *pfxs[] = { ".data", ".rodata", ".bss", ".kconfig" }; + static const char *pfxs[] = { ".data", ".rodata", ".bss", ".percpu", ".kconfig" }; int i, n; /* recognize hard coded LLVM section name */ @@ -254,7 +260,7 @@ static const struct btf_type *find_type_for_map(struct btf *btf, const char *map return NULL; } -static bool is_mmapable_map(const struct bpf_map *map, char *buf, size_t sz) +static bool is_skel_data(const struct bpf_map *map, char *buf, size_t sz) { size_t tmp_sz; @@ -263,13 +269,24 @@ static bool is_mmapable_map(const struct bpf_map *map, char *buf, size_t sz) return true; } - if (!bpf_map__is_internal(map) || !(bpf_map__map_flags(map) & BPF_F_MMAPABLE)) + if (!bpf_map__is_internal(map)) return false; if (!get_map_ident(map, buf, sz)) return false; - return true; + if (bpf_map__map_flags(map) & BPF_F_MMAPABLE) + return true; + + if (bpf_map__type(map) == BPF_MAP_TYPE_PERCPU_ARRAY) + return bpf_map__btf_value_type_id(map) != 0; + + return false; +} + +static bool is_mmapable_map(const struct bpf_map *map, char *buf, size_t sz) +{ + return is_skel_data(map, buf, sz) && bpf_map__type(map) != BPF_MAP_TYPE_PERCPU_ARRAY; } static int codegen_datasecs(struct bpf_object *obj, const char *obj_name) @@ -287,7 +304,7 @@ static int codegen_datasecs(struct bpf_object *obj, const char *obj_name) bpf_object__for_each_map(map, obj) { /* only generate definitions for memory-mapped internal maps */ - if (!is_mmapable_map(map, map_ident, sizeof(map_ident))) + if (!is_skel_data(map, map_ident, sizeof(map_ident))) continue; sec = find_type_for_map(btf, map_ident); @@ -517,7 +534,7 @@ static void codegen_asserts(struct bpf_object *obj, const char *obj_name) ", obj_name); bpf_object__for_each_map(map, obj) { - if (!is_mmapable_map(map, map_ident, sizeof(map_ident))) + if (!is_skel_data(map, map_ident, sizeof(map_ident))) continue; sec = find_type_for_map(btf, map_ident); @@ -668,8 +685,7 @@ static void codegen_destroy(struct bpf_object *obj, const char *obj_name) bpf_object__for_each_map(map, obj) { if (!get_map_ident(map, ident, sizeof(ident))) continue; - if (bpf_map__is_internal(map) && - (bpf_map__map_flags(map) & BPF_F_MMAPABLE)) + if (is_skel_data(map, ident, sizeof(ident))) printf("\tskel_free_map_data(skel->%1$s, skel->maps.%1$s.initial_value, %2$zu);\n", ident, bpf_map_mmap_sz(map)); codegen("\ @@ -741,7 +757,7 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h const void *mmap_data = NULL; size_t mmap_size = 0; - if (!is_mmapable_map(map, ident, sizeof(ident))) + if (!is_skel_data(map, ident, sizeof(ident))) continue; codegen("\ @@ -793,6 +809,8 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h if (sign_progs) { sopts.insns = opts.insns; sopts.insns_sz = opts.insns_sz; + sopts.data = opts.data; + sopts.data_sz = opts.data_sz; sopts.excl_prog_hash = prog_sha; sopts.excl_prog_hash_sz = sizeof(prog_sha); sopts.signature = sig_buf; @@ -847,9 +865,23 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h bpf_object__for_each_map(map, obj) { const char *mmap_flags; - if (!is_mmapable_map(map, ident, sizeof(ident))) + if (!is_skel_data(map, ident, sizeof(ident))) continue; + if (bpf_map__type(map) == BPF_MAP_TYPE_PERCPU_ARRAY) { + codegen("\ + \n\ + err = skel_protect_map_data(skel->%1$s, &skel->maps.%1$s.initial_value, %2$zd);\n\ + if (err) \n\ + return err; \n\ + #ifdef __KERNEL__ \n\ + skel->%1$s = NULL; \n\ + #endif \n\ + ", + ident, bpf_map_mmap_sz(map)); + continue; + } + if (bpf_map__map_flags(map) & BPF_F_RDONLY_PROG) mmap_flags = "PROT_READ"; else @@ -953,8 +985,7 @@ codegen_maps_skeleton(struct bpf_object *obj, size_t map_cnt, bool mmaped, bool map->map = &obj->maps.%s; \n\ ", i, bpf_map__name(map), ident); - /* memory-mapped internal maps */ - if (mmaped && is_mmapable_map(map, ident, sizeof(ident))) { + if (mmaped && is_skel_data(map, ident, sizeof(ident))) { printf("\tmap->mmaped = (void **)&obj->%s;\n", ident); } diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c index bdcd717b0348..088d1d206065 100644 --- a/tools/bpf/bpftool/link.c +++ b/tools/bpf/bpftool/link.c @@ -377,6 +377,25 @@ static __u64 *u64_to_arr(__u64 val) return (__u64 *) u64_to_ptr(val); } +static __u32 *u64_to_u32_arr(__u64 val) +{ + return (__u32 *)u64_to_ptr(val); +} + +static struct kernel_sym *find_kernel_sym_by_addr(__u64 addr, bool is_ibt_enabled) +{ + struct kernel_sym *sym; + + if (!addr) + return NULL; + + sym = kernel_syms_search(&dd, addr); + if (!sym && is_ibt_enabled && addr >= 4) + sym = kernel_syms_search(&dd, addr - 4); + + return sym; +} + static void show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr) { @@ -404,6 +423,52 @@ show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr) } static void +show_tracing_multi_json(struct bpf_link_info *info, json_writer_t *wtr) +{ + bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol; + __u64 *addrs, *cookies; + __u32 i, *ids; + + if (!dd.sym_count) + kernel_syms_load(&dd); + show_symbol = !!dd.sym_count; + + show_link_attach_type_json(info->tracing_multi.attach_type, wtr); + jsonw_uint_field(wtr, "func_cnt", info->tracing_multi.count); + jsonw_uint_field(wtr, "btf_obj_id", info->tracing_multi.btf_obj_id); + jsonw_name(wtr, "funcs"); + + jsonw_start_array(wtr); + + ids = u64_to_u32_arr(info->tracing_multi.ids); + addrs = u64_to_arr(info->tracing_multi.addrs); + cookies = u64_to_arr(info->tracing_multi.cookies); + + for (i = 0; i < info->tracing_multi.count; i++) { + struct kernel_sym *sym; + __u64 addr = addrs[i]; + + sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL; + + jsonw_start_object(wtr); + jsonw_uint_field(wtr, "id", ids[i]); + jsonw_uint_field(wtr, "addr", addr); + if (sym) { + jsonw_string_field(wtr, "func", sym->name); + if (sym->module[0] == '\0') { + jsonw_name(wtr, "module"); + jsonw_null(wtr); + } else { + jsonw_string_field(wtr, "module", sym->module); + } + } + jsonw_uint_field(wtr, "cookie", cookies[i]); + jsonw_end_object(wtr); + } + jsonw_end_array(wtr); +} + +static void show_perf_event_kprobe_json(struct bpf_link_info *info, json_writer_t *wtr) { jsonw_bool_field(wtr, "retprobe", info->perf_event.type == BPF_PERF_EVENT_KRETPROBE); @@ -589,6 +654,9 @@ static int show_link_close_json(int fd, struct bpf_link_info *info) case BPF_LINK_TYPE_UPROBE_MULTI: show_uprobe_multi_json(info, json_wtr); break; + case BPF_LINK_TYPE_TRACING_MULTI: + show_tracing_multi_json(info, json_wtr); + break; case BPF_LINK_TYPE_PERF_EVENT: switch (info->perf_event.type) { case BPF_PERF_EVENT_EVENT: @@ -833,6 +901,46 @@ static void show_uprobe_multi_plain(struct bpf_link_info *info) } } +static void show_tracing_multi_plain(struct bpf_link_info *info) +{ + bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol; + __u64 *addrs, *cookies; + __u32 i, *ids; + + if (!info->tracing_multi.count) + return; + + if (!dd.sym_count) + kernel_syms_load(&dd); + show_symbol = !!dd.sym_count; + + printf("\n\t"); + show_link_attach_type_plain(info->tracing_multi.attach_type); + printf("btf_obj_id %u ", info->tracing_multi.btf_obj_id); + printf("count %u ", info->tracing_multi.count); + + printf("\n\t%-16s %-16s %-16s %s", + "btf_id", "addr", "cookie", "func [module]"); + + ids = u64_to_u32_arr(info->tracing_multi.ids); + addrs = u64_to_arr(info->tracing_multi.addrs); + cookies = u64_to_arr(info->tracing_multi.cookies); + + for (i = 0; i < info->tracing_multi.count; i++) { + __u64 addr = addrs[i]; + struct kernel_sym *sym; + + sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL; + + printf("\n\t%-16u %016llx %-16llu", ids[i], addr, cookies[i]); + if (sym) { + printf(" %s", sym->name); + if (sym->module[0] != '\0') + printf(" [%s]", sym->module); + } + } +} + static void show_perf_event_kprobe_plain(struct bpf_link_info *info) { const char *buf; @@ -989,6 +1097,9 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info) case BPF_LINK_TYPE_UPROBE_MULTI: show_uprobe_multi_plain(info); break; + case BPF_LINK_TYPE_TRACING_MULTI: + show_tracing_multi_plain(info); + break; case BPF_LINK_TYPE_PERF_EVENT: switch (info->perf_event.type) { case BPF_PERF_EVENT_EVENT: @@ -1029,6 +1140,7 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info) static int do_show_link(int fd) { __u64 *ref_ctr_offsets = NULL, *offsets = NULL, *cookies = NULL; + __u32 *ids = NULL; struct bpf_link_info info; __u32 len = sizeof(info); char path_buf[PATH_MAX]; @@ -1114,6 +1226,26 @@ again: goto again; } } + if (info.type == BPF_LINK_TYPE_TRACING_MULTI && !info.tracing_multi.ids) { + count = info.tracing_multi.count; + if (count) { + ids = calloc(count, sizeof(__u32)); + addrs = calloc(count, sizeof(__u64)); + cookies = calloc(count, sizeof(__u64)); + if (!ids || !addrs || !cookies) { + p_err("mem alloc failed"); + close(fd); + free(cookies); + free(addrs); + free(ids); + return -ENOMEM; + } + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.addrs = ptr_to_u64(addrs); + info.tracing_multi.cookies = ptr_to_u64(cookies); + goto again; + } + } if (info.type == BPF_LINK_TYPE_PERF_EVENT) { switch (info.perf_event.type) { case BPF_PERF_EVENT_TRACEPOINT: @@ -1153,6 +1285,7 @@ again: free(cookies); free(offsets); free(addrs); + free(ids); close(fd); return 0; } diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c index 71a45d96617e..684a8fb72414 100644 --- a/tools/bpf/bpftool/map.c +++ b/tools/bpf/bpftool/map.c @@ -659,8 +659,6 @@ static int do_show_subset(int argc, char **argv) show_map_close_json(fds[i], &info); else show_map_close_plain(fds[i], &info); - - close(fds[i]); } if (json_output && nb_fds > 1) jsonw_end_array(json_wtr); /* root array */ @@ -790,6 +788,12 @@ static int maps_have_btf(int *fds, int nb_fds) static struct btf *btf_vmlinux; +static void free_btf_vmlinux(void) +{ + btf__free(btf_vmlinux); + btf_vmlinux = NULL; +} + static int get_map_kv_btf(const struct bpf_map_info *info, struct btf **btf) { int err = 0; @@ -889,7 +893,6 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, exit_free: free(key); free(value); - close(fd); free_map_kv_btf(btf); return err; @@ -938,6 +941,7 @@ static int do_dump(int argc, char **argv) for (i = 0; i < nb_fds; i++) { if (bpf_map_get_info_by_fd(fds[i], &info, &len)) { p_err("can't get map info: %s", strerror(errno)); + err = -1; break; } err = map_dump(fds[i], &info, wtr, nb_fds > 1); @@ -958,7 +962,7 @@ exit_close: close(fds[i]); exit_free: free(fds); - btf__free(btf_vmlinux); + free_btf_vmlinux(); return err; } @@ -1049,7 +1053,7 @@ static void print_key_value(struct bpf_map_info *info, void *key, btf_wtr = get_btf_writer(); if (!btf_wtr) { p_info("failed to create json writer for btf. falling back to plain output"); - btf__free(btf); + free_map_kv_btf(btf); btf = NULL; print_entry_plain(info, key, value); } else { @@ -1065,7 +1069,7 @@ static void print_key_value(struct bpf_map_info *info, void *key, } else { print_entry_plain(info, key, value); } - btf__free(btf); + free_map_kv_btf(btf); } static int do_lookup(int argc, char **argv) diff --git a/tools/bpf/bpftool/sign.c b/tools/bpf/bpftool/sign.c index f9b742f4bb10..88726a6db6d0 100644 --- a/tools/bpf/bpftool/sign.c +++ b/tools/bpf/bpftool/sign.c @@ -135,9 +135,21 @@ int bpftool_prog_sign(struct bpf_load_and_run_opts *opts) CMS_ContentInfo *cms = NULL; long actual_sig_len = 0; X509 *x509 = NULL; + void *data = NULL; + size_t data_sz; int err = 0; - bd_in = BIO_new_mem_buf(opts->insns, opts->insns_sz); + data_sz = (size_t)opts->insns_sz + opts->data_sz; + data = malloc(data_sz); + if (!data) { + err = -ENOMEM; + goto cleanup; + } + memcpy(data, opts->insns, opts->insns_sz); + if (opts->data_sz) + memcpy((char *)data + opts->insns_sz, opts->data, opts->data_sz); + + bd_in = BIO_new_mem_buf(data, data_sz); if (!bd_in) { err = -ENOMEM; goto cleanup; @@ -175,10 +187,13 @@ int bpftool_prog_sign(struct bpf_load_and_run_opts *opts) goto cleanup; } - EVP_Digest(opts->insns, opts->insns_sz, opts->excl_prog_hash, - &opts->excl_prog_hash_sz, EVP_sha256(), NULL); + if (EVP_Digest(opts->insns, opts->insns_sz, opts->excl_prog_hash, + &opts->excl_prog_hash_sz, EVP_sha256(), NULL) != 1) { + err = -EIO; + goto cleanup; + } - bd_out = BIO_new(BIO_s_mem()); + bd_out = BIO_new(BIO_s_mem()); if (!bd_out) { err = -ENOMEM; goto cleanup; @@ -212,6 +227,7 @@ cleanup: X509_free(x509); EVP_PKEY_free(private_key); BIO_free(bd_in); + free(data); DISPLAY_OSSL_ERR(err < 0); return err; } diff --git a/tools/bpf/bpftool/struct_ops.c b/tools/bpf/bpftool/struct_ops.c index aa43dead249c..835e5e561f7f 100644 --- a/tools/bpf/bpftool/struct_ops.c +++ b/tools/bpf/bpftool/struct_ops.c @@ -643,6 +643,10 @@ int do_struct_ops(int argc, char **argv) err = cmd_select(cmds, argc, argv, do_help); btf__free(btf_vmlinux); + btf_vmlinux = NULL; + map_info_type = NULL; + map_info_alloc_len = 0; + map_info_type_id = 0; return err; } diff --git a/tools/bpf/bpftool/xlated_dumper.c b/tools/bpf/bpftool/xlated_dumper.c index 5e7cb8b36fef..5579173a61e3 100644 --- a/tools/bpf/bpftool/xlated_dumper.c +++ b/tools/bpf/bpftool/xlated_dumper.c @@ -107,14 +107,7 @@ print_insn_for_graph(void *private_data, const char *fmt, ...) p = buf; while (*p != '\0') { - if (*p == '\n') { - memmove(p + 3, p, strlen(buf) + 1 - (p - buf)); - /* Align each instruction dump row left. */ - *p++ = '\\'; - *p++ = 'l'; - /* Output multiline concatenation. */ - *p++ = '\\'; - } else if (*p == '<' || *p == '>' || *p == '|' || *p == '&') { + if (*p == '<' || *p == '>' || *p == '|' || *p == '&') { memmove(p + 1, p, strlen(buf) + 1 - (p - buf)); /* Escape special character. */ *p++ = '\\'; @@ -129,16 +122,10 @@ print_insn_for_graph(void *private_data, const char *fmt, ...) static void __printf(2, 3) print_insn_json(void *private_data, const char *fmt, ...) { - unsigned int l = strlen(fmt); - char chomped_fmt[l]; va_list args; va_start(args, fmt); - if (l > 0) { - strncpy(chomped_fmt, fmt, l - 1); - chomped_fmt[l - 1] = '\0'; - } - jsonw_vprintf_enquote(json_wtr, chomped_fmt, args); + jsonw_vprintf_enquote(json_wtr, fmt, args); va_end(args); } @@ -351,6 +338,7 @@ void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len, printf("%4u: ", i); print_bpf_insn(&cbs, insn + i, true); + printf("\n"); if (opcodes) { printf(" "); @@ -417,6 +405,7 @@ void dump_xlated_for_graph(struct dump_data *dd, void *buf_start, void *buf_end, printf("%u: ", insn_off); print_bpf_insn(&cbs, cur, true); + printf("\\l\\\n"); if (opcodes) { printf("\\ \\ \\ \\ "); diff --git a/tools/bpf/resolve_btfids/Makefile b/tools/bpf/resolve_btfids/Makefile index 7672208f65e4..bb0463b380af 100644 --- a/tools/bpf/resolve_btfids/Makefile +++ b/tools/bpf/resolve_btfids/Makefile @@ -40,7 +40,6 @@ LIBBPF_DESTDIR := $(LIBBPF_OUT) LIBBPF_INCLUDE := $(LIBBPF_DESTDIR)include SUBCMD_DESTDIR := $(SUBCMD_OUT) -SUBCMD_INCLUDE := $(SUBCMD_DESTDIR)include BINARY := $(OUTPUT)/resolve_btfids BINARY_IN := $(BINARY)-in.o @@ -56,7 +55,7 @@ $(OUTPUT) $(OUTPUT)/libsubcmd $(LIBBPF_OUT): $(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd $(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(SUBCMD_OUT) \ DESTDIR=$(SUBCMD_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \ - $(abspath $@) install_headers + $(abspath $@) $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUT) $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT) \ @@ -77,7 +76,7 @@ HOSTCFLAGS_resolve_btfids += -g \ -I$(srctree)/tools/include \ -I$(srctree)/tools/include/uapi \ -I$(LIBBPF_INCLUDE) \ - -I$(SUBCMD_INCLUDE) \ + -I$(srctree)/tools/lib \ $(LIBELF_FLAGS) \ -Wall -Werror diff --git a/tools/bpf/resolve_btfids/main.c b/tools/bpf/resolve_btfids/main.c index f8a91fa7584f..37d7e7224207 100644 --- a/tools/bpf/resolve_btfids/main.c +++ b/tools/bpf/resolve_btfids/main.c @@ -58,6 +58,17 @@ * __BTF_ID__func__vfs_fallocate__5: * .zero 4 * .word (1 << 3) | (1 << 1) | (1 << 2) + * + * In addition to resolving BTF IDs, resolve_btfids performs kernel-specific + * BTF-to-BTF transformations for kfuncs found in BTF_SET8_KFUNCS sets. For + * each such kfunc it: + * + * - emits a "bpf_kfunc" decl tag, and "bpf_fastcall" when KF_FASTCALL is set; + * - wraps the return value and/or arguments that use arena pointers + * with the "address_space(1)" type attribute; + * - rewrites the prototype of KF_IMPLICIT_ARGS kfuncs. + * + * These kfunc annotations were historically produced by pahole. */ #define _GNU_SOURCE @@ -119,6 +130,11 @@ struct btf_id { Elf64_Addr addr[ADDR_CNT]; }; +struct addr_sym { + Elf64_Addr addr; + const char *name; +}; + struct object { const char *path; const char *btf_path; @@ -150,12 +166,27 @@ struct object { int nr_structs; int nr_unions; int nr_typedefs; + + struct addr_sym *addr_syms; + u32 addr_syms_cnt; + u32 addr_syms_cap; }; +#define DECL_TAG_FASTCALL "bpf_fastcall" +#define DECL_TAG_KFUNC "bpf_kfunc" + +#define KF_FASTCALL (1 << 12) +#define KF_ARENA_RET (1 << 13) +#define KF_ARENA_ARG1 (1 << 14) +#define KF_ARENA_ARG2 (1 << 15) #define KF_IMPLICIT_ARGS (1 << 16) #define KF_IMPL_SUFFIX "_impl" +#define TYPE_ATTR_ARENA "address_space(1)" +#define PARAM_SUFFIX_ARENA "__arena" +#define PARAM_SUFFIX_ARENA_NULLABLE "__arena__nullable" struct kfunc { + struct rb_node rb_node; const char *name; u32 btf_id; u32 flags; @@ -166,9 +197,7 @@ struct btf2btf_context { u32 *decl_tags; u32 nr_decl_tags; u32 max_decl_tags; - struct kfunc *kfuncs; - u32 nr_kfuncs; - u32 max_kfuncs; + struct rb_root kfuncs; }; static int verbose; @@ -201,6 +230,35 @@ static int eprintf(int level, int var, const char *fmt, ...) #define pr_info(fmt, ...) \ eprintf(0, verbose, pr_fmt(fmt), ##__VA_ARGS__) +/* + * Grow *data so it can hold at least cnt elements of elem_sz bytes each. + * *cap is the capacity in elements and is updated on growth. + */ +static int __ensure_mem(void **data, u32 *cap, u32 cnt, size_t elem_sz) +{ + u32 new_cap, old_cap = *cap; + void *arr; + + if (cnt <= old_cap) + return 0; + + new_cap = max(old_cap + 256, old_cap * 2); + if (new_cap < cnt) + new_cap = cnt; + + arr = realloc(*data, elem_sz * new_cap); + if (!arr) + return -ENOMEM; + + *data = arr; + *cap = new_cap; + + return 0; +} + +#define ensure_mem(arr_ptr, cap_ptr, cnt) \ + __ensure_mem((void **)(arr_ptr), (cap_ptr), (cnt), sizeof(**(arr_ptr))) + static bool is_btf_id(const char *name) { return name && !strncmp(name, BTF_ID_PREFIX, sizeof(BTF_ID_PREFIX) - 1); @@ -480,6 +538,40 @@ static int elf_collect(struct object *obj) return 0; } +static int push_addr_sym(struct object *obj, Elf64_Addr addr, const char *name) +{ + if (ensure_mem(&obj->addr_syms, &obj->addr_syms_cap, obj->addr_syms_cnt + 1)) + return -ENOMEM; + + obj->addr_syms[obj->addr_syms_cnt++] = (struct addr_sym){ + .addr = addr, + .name = name, + }; + + return 0; +} + +static int cmp_addr_sym(const void *a, const void *b) +{ + Elf64_Addr aa = ((const struct addr_sym *)a)->addr; + Elf64_Addr ab = ((const struct addr_sym *)b)->addr; + + return (aa > ab) - (aa < ab); +} + +static const char *find_name_by_addr(struct object *obj, Elf64_Addr addr) +{ + struct addr_sym key = { .addr = addr }; + struct addr_sym *res; + + if (!obj->addr_syms_cnt) + return NULL; + + res = bsearch(&key, obj->addr_syms, obj->addr_syms_cnt, + sizeof(*obj->addr_syms), cmp_addr_sym); + return res ? res->name : NULL; +} + static int symbols_collect(struct object *obj) { Elf_Scn *scn = NULL; @@ -573,8 +665,15 @@ static int symbols_collect(struct object *obj) return -1; } id->addr[id->addr_cnt++] = sym.st_value; + + if (push_addr_sym(obj, sym.st_value, id->name)) + return -1; } + if (obj->addr_syms_cnt) + qsort(obj->addr_syms, obj->addr_syms_cnt, + sizeof(*obj->addr_syms), cmp_addr_sym); + return 0; } @@ -890,17 +989,8 @@ static const struct btf_type *btf_type_skip_qualifiers(const struct btf *btf, s3 static int push_decl_tag_id(struct btf2btf_context *ctx, u32 decl_tag_id) { - u32 *arr = ctx->decl_tags; - u32 cap = ctx->max_decl_tags; - - if (ctx->nr_decl_tags + 1 > cap) { - cap = max(cap + 256, cap * 2); - arr = realloc(arr, sizeof(u32) * cap); - if (!arr) - return -ENOMEM; - ctx->max_decl_tags = cap; - ctx->decl_tags = arr; - } + if (ensure_mem(&ctx->decl_tags, &ctx->max_decl_tags, ctx->nr_decl_tags + 1)) + return -ENOMEM; ctx->decl_tags[ctx->nr_decl_tags++] = decl_tag_id; @@ -909,23 +999,57 @@ static int push_decl_tag_id(struct btf2btf_context *ctx, u32 decl_tag_id) static int push_kfunc(struct btf2btf_context *ctx, struct kfunc *kfunc) { - struct kfunc *arr = ctx->kfuncs; - u32 cap = ctx->max_kfuncs; + struct rb_node **p = &ctx->kfuncs.rb_node; + struct rb_node *parent = NULL; + struct kfunc *k; - if (ctx->nr_kfuncs + 1 > cap) { - cap = max(cap + 256, cap * 2); - arr = realloc(arr, sizeof(struct kfunc) * cap); - if (!arr) - return -ENOMEM; - ctx->max_kfuncs = cap; - ctx->kfuncs = arr; + /* + * Dedup by BTF ID: collecting the same kfunc twice is a no-op, + * UNLESS the kfunc flags are inconsistent, in which case we + * fail hard because it indicates a bug in a kfunc set declaration. + */ + while (*p) { + parent = *p; + k = rb_entry(parent, struct kfunc, rb_node); + + if (kfunc->btf_id < k->btf_id) { + p = &(*p)->rb_left; + } else if (kfunc->btf_id > k->btf_id) { + p = &(*p)->rb_right; + } else if (k->flags == kfunc->flags) { + return 0; + } else { + pr_err("ERROR: resolve_btfids: kfunc %s has inconsistent flags across BTF ID sets: 0x%x != 0x%x\n", + kfunc->name, k->flags, kfunc->flags); + return -EINVAL; + } } - ctx->kfuncs[ctx->nr_kfuncs++] = *kfunc; + k = zalloc(sizeof(*k)); + if (!k) + return -ENOMEM; + + *k = *kfunc; + rb_link_node(&k->rb_node, parent, p); + rb_insert_color(&k->rb_node, &ctx->kfuncs); return 0; } +static void free_kfuncs(struct rb_root *root) +{ + struct rb_node *next; + struct kfunc *kfunc; + + next = rb_first(root); + while (next) { + kfunc = rb_entry(next, struct kfunc, rb_node); + next = rb_next(&kfunc->rb_node); + rb_erase(&kfunc->rb_node, root); + free(kfunc); + } +} + static int collect_decl_tags(struct btf2btf_context *ctx) { const u32 type_cnt = btf__type_cnt(ctx->btf); @@ -945,94 +1069,76 @@ static int collect_decl_tags(struct btf2btf_context *ctx) return 0; } -/* - * To find the kfunc flags having its struct btf_id (with ELF addresses) - * we need to find the address that is in range of a set8. - * If a set8 is found, then the flags are located at addr + 4 bytes. - * Return 0 (no flags!) if not found. - */ -static u32 find_kfunc_flags(struct object *obj, struct btf_id *kfunc_id) +static bool param_name_has_suffix(const char *name, const char *suffix) { - const u32 *elf_data_ptr = obj->efile.idlist->d_buf; - u64 set_lower_addr, set_upper_addr, addr; - struct btf_id *set_id; - struct rb_node *next; - u32 flags; - u64 idx; + size_t name_len = strlen(name); + size_t suffix_len = strlen(suffix); - for (next = rb_first(&obj->sets); next; next = rb_next(next)) { - set_id = rb_entry(next, struct btf_id, rb_node); - if (set_id->kind != BTF_ID_KIND_SET8 || set_id->addr_cnt != 1) - continue; - - set_lower_addr = set_id->addr[0]; - set_upper_addr = set_lower_addr + set_id->cnt * sizeof(u64); + return name_len >= suffix_len && !strcmp(name + name_len - suffix_len, suffix); +} - for (u32 i = 0; i < kfunc_id->addr_cnt; i++) { - addr = kfunc_id->addr[i]; - /* - * Lower bound is exclusive to skip the 8-byte header of the set. - * Upper bound is inclusive to capture the last entry at offset 8*cnt. - */ - if (set_lower_addr < addr && addr <= set_upper_addr) { - pr_debug("found kfunc %s in BTF_ID_FLAGS %s\n", - kfunc_id->name, set_id->name); - idx = addr - obj->efile.idlist_addr; - idx = idx / sizeof(u32) + 1; - flags = elf_data_ptr[idx]; - - return flags; - } - } - } +static bool is_arena_param(const struct btf *btf, const struct btf_param *param) +{ + const char *name = btf__name_by_offset(btf, param->name_off); - return 0; + return param_name_has_suffix(name, PARAM_SUFFIX_ARENA) || + param_name_has_suffix(name, PARAM_SUFFIX_ARENA_NULLABLE); } static int collect_kfuncs(struct object *obj, struct btf2btf_context *ctx) { - const char *tag_name, *func_name; + Elf_Data *idlist = obj->efile.idlist; struct btf *btf = ctx->btf; - const struct btf_type *t; - u32 flags, func_id; - struct kfunc kfunc; - struct btf_id *id; - int err; + struct rb_node *next; - if (ctx->nr_decl_tags == 0) + if (!idlist || !idlist->d_buf) return 0; - for (u32 i = 0; i < ctx->nr_decl_tags; i++) { - t = btf__type_by_id(btf, ctx->decl_tags[i]); - if (btf_kflag(t) || btf_decl_tag(t)->component_idx != -1) - continue; + for (next = rb_first(&obj->sets); next; next = rb_next(next)) { + struct btf_id_set8 *set8; + struct btf_id *set_id; + u64 set_addr; - tag_name = btf__name_by_offset(btf, t->name_off); - if (strcmp(tag_name, "bpf_kfunc") != 0) + set_id = rb_entry(next, struct btf_id, rb_node); + if (set_id->kind != BTF_ID_KIND_SET8 || set_id->addr_cnt != 1) continue; - func_id = t->type; - t = btf__type_by_id(btf, func_id); - if (!btf_is_func(t)) + set_addr = set_id->addr[0]; + set8 = idlist->d_buf + (set_addr - obj->efile.idlist_addr); + if (!(set8->flags & BTF_SET8_KFUNCS)) continue; - func_name = btf__name_by_offset(btf, t->name_off); - if (!func_name) - continue; + for (u32 i = 0; i < set_id->cnt; i++) { + size_t off = (char *)&set8->pairs[i] - (char *)set8; + const char *name = find_name_by_addr(obj, set_addr + off); + struct kfunc kfunc; + s32 func_id; + int err; - id = btf_id__find(&obj->funcs, func_name); - if (!id || id->kind != BTF_ID_KIND_SYM) - continue; + if (!name) { + pr_err("WARN: resolve_btfids: no BTF ID symbol for %s entry %u\n", + set_id->name, i); + warnings++; + continue; + } - flags = find_kfunc_flags(obj, id); + func_id = btf__find_by_name_kind_own(btf, name, BTF_KIND_FUNC); + if (func_id < 0) { + pr_err("WARN: resolve_btfids: no BTF func for kfunc %s in %s\n", + name, set_id->name); + warnings++; + continue; + } - kfunc.name = id->name; - kfunc.btf_id = func_id; - kfunc.flags = flags; + pr_debug("found kfunc %s in %s\n", name, set_id->name); - err = push_kfunc(ctx, &kfunc); - if (err) - return err; + kfunc.name = name; + kfunc.btf_id = func_id; + kfunc.flags = set8->pairs[i].flags; + err = push_kfunc(ctx, &kfunc); + if (err) + return err; + } } return 0; @@ -1141,7 +1247,7 @@ static int process_kfunc_with_implicit_args(struct btf2btf_context *ctx, struct return -E2BIG; } - if (btf__find_by_name_kind(btf, tmp_name, BTF_KIND_FUNC) > 0) { + if (btf__find_by_name_kind_own(btf, tmp_name, BTF_KIND_FUNC) > 0) { pr_debug("resolve_btfids: function %s already exists in BTF\n", tmp_name); goto add_new_proto; } @@ -1160,7 +1266,7 @@ static int process_kfunc_with_implicit_args(struct btf2btf_context *ctx, struct continue; tag_name = btf__name_by_offset(btf, t->name_off); - if (strcmp(tag_name, "bpf_kfunc") == 0) + if (strcmp(tag_name, DECL_TAG_KFUNC) == 0) continue; idx = btf_decl_tag(t)->component_idx; @@ -1211,22 +1317,187 @@ add_new_proto: return 0; } +static bool is_arena_arg(const struct btf *btf, const struct kfunc *kfunc, + const struct btf_param *param, u32 idx) +{ + if (is_arena_param(btf, param)) + return true; + + switch (idx) { + case 0: + return kfunc->flags & KF_ARENA_ARG1; + case 1: + return kfunc->flags & KF_ARENA_ARG2; + default: + return false; + } +} + +static s32 arena_tag_ptr(struct btf *btf, u32 ptr_id, struct kfunc *kfunc) +{ + const struct btf_type *ptr = btf__type_by_id(btf, ptr_id); + s32 tag_id, new_ptr_id; + + if (!btf_is_ptr(ptr)) { + pr_err("ERROR: resolve_btfids: kfunc %s: arena type is not a pointer\n", + kfunc->name); + return -EINVAL; + } + + tag_id = btf__add_type_attr(btf, TYPE_ATTR_ARENA, ptr->type); + if (tag_id < 0) { + pr_err("ERROR: resolve_btfids: kfunc %s: failed to add a type attr to BTF: %d\n", + kfunc->name, tag_id); + return tag_id; + } + + new_ptr_id = btf__add_ptr(btf, tag_id); + if (new_ptr_id < 0) { + pr_err("ERROR: resolve_btfids: kfunc %s: failed to add a pointer to BTF: %d\n", + kfunc->name, new_ptr_id); + } + + return new_ptr_id; +} + +/* + * Add a FUNC_PROTO for @kfunc with each arena pointer tagged with an + * "address_space(1)" attribute. The original proto may be shared with + * other FUNCs, so it is never modified in place. Returns the original + * proto id when @kfunc has no arena return value or arguments. + */ +static s32 add_arena_tagged_proto(struct btf *btf, struct kfunc *kfunc) +{ + const struct btf_type *func = btf__type_by_id(btf, kfunc->btf_id); + u32 proto_id = func->type; + const struct btf_type *proto = btf__type_by_id(btf, proto_id); + const struct btf_param *params = btf_params(proto); + u32 nr_params = btf_vlen(proto); + s32 ret_type_id = proto->type; + const struct btf_type *t; + struct btf_param *tag_params; + s32 new_proto_id, id; + const char *name; + bool has_arena_arg = false; + int err, i; + + for (i = 0; i < nr_params; i++) { + if (is_arena_arg(btf, kfunc, ¶ms[i], i)) { + has_arena_arg = true; + break; + } + } + + if (!(kfunc->flags & KF_ARENA_RET) && !has_arena_arg) + return proto_id; + + if (kfunc->flags & KF_ARENA_RET) { + ret_type_id = arena_tag_ptr(btf, ret_type_id, kfunc); + if (ret_type_id < 0) + return ret_type_id; + } + + new_proto_id = btf__add_func_proto(btf, ret_type_id); + if (new_proto_id < 0) { + pr_err("ERROR: resolve_btfids: kfunc %s: failed to add a func proto to BTF: %d\n", + kfunc->name, new_proto_id); + return new_proto_id; + } + + for (i = 0; i < nr_params; i++) { + /* btf__add_func_param() below may move the proto, re-fetch */ + proto = btf__type_by_id(btf, proto_id); + name = btf__name_by_offset(btf, btf_params(proto)[i].name_off); + + err = btf__add_func_param(btf, name ?: "", btf_params(proto)[i].type); + if (err < 0) { + pr_err("ERROR: resolve_btfids: kfunc %s: failed to add a proto param to BTF: %d\n", + kfunc->name, err); + return err; + } + } + + for (i = 0; i < nr_params; i++) { + t = btf__type_by_id(btf, new_proto_id); + tag_params = btf_params(t); + if (!is_arena_arg(btf, kfunc, &tag_params[i], i)) + continue; + + id = arena_tag_ptr(btf, tag_params[i].type, kfunc); + if (id < 0) + return id; + + t = btf__type_by_id(btf, new_proto_id); + tag_params = btf_params(t); + tag_params[i].type = id; + } + + pr_debug("added arena-tagged proto for kfunc %s: %d\n", kfunc->name, new_proto_id); + + return new_proto_id; +} + +static int process_kfunc_with_arena_attrs(struct btf2btf_context *ctx, + struct kfunc *kfunc) +{ + struct btf_type *t; + s32 proto_id; + + proto_id = add_arena_tagged_proto(ctx->btf, kfunc); + if (proto_id < 0) + return proto_id; + + t = (struct btf_type *)btf__type_by_id(ctx->btf, kfunc->btf_id); + t->type = proto_id; + + return 0; +} + +static int add_decl_tag(struct btf2btf_context *ctx, const char *tag_name, + u32 target_btf_id, int component_idx) +{ + s32 new_id; + + new_id = btf__add_decl_tag(ctx->btf, tag_name, target_btf_id, component_idx); + if (new_id < 0) { + pr_err("ERROR: resolve_btfids: failed to add '%s' decl tag for BTF id %u: %d\n", + tag_name, target_btf_id, new_id); + return new_id; + } + + return push_decl_tag_id(ctx, new_id); +} + static int btf2btf(struct object *obj) { struct btf2btf_context ctx = {}; + struct rb_node *next; int err; err = build_btf2btf_context(obj, &ctx); if (err) goto out; - for (u32 i = 0; i < ctx.nr_kfuncs; i++) { - struct kfunc *kfunc = &ctx.kfuncs[i]; + for (next = rb_first(&ctx.kfuncs); next; next = rb_next(next)) { + struct kfunc *kfunc = rb_entry(next, struct kfunc, rb_node); - if (!(kfunc->flags & KF_IMPLICIT_ARGS)) - continue; + err = add_decl_tag(&ctx, DECL_TAG_KFUNC, kfunc->btf_id, -1); + if (err) + goto out; + + if (kfunc->flags & KF_FASTCALL) { + err = add_decl_tag(&ctx, DECL_TAG_FASTCALL, kfunc->btf_id, -1); + if (err) + goto out; + } + + if (kfunc->flags & KF_IMPLICIT_ARGS) { + err = process_kfunc_with_implicit_args(&ctx, kfunc); + if (err) + goto out; + } - err = process_kfunc_with_implicit_args(&ctx, kfunc); + err = process_kfunc_with_arena_attrs(&ctx, kfunc); if (err) goto out; } @@ -1234,7 +1505,7 @@ static int btf2btf(struct object *obj) err = 0; out: free(ctx.decl_tags); - free(ctx.kfuncs); + free_kfuncs(&ctx.kfuncs); return err; } @@ -1309,6 +1580,12 @@ static int finalize_btf(struct object *obj) struct btf *base_btf = obj->base_btf, *btf = obj->btf; int err; + err = btf__dedup(obj->btf, NULL); + if (err) { + pr_err("FAILED to dedup BTF: %s\n", strerror(errno)); + goto out_err; + } + if (obj->base_btf && obj->distill_base) { err = btf__distill_base(obj->btf, &base_btf, &btf); if (err) { @@ -1575,6 +1852,7 @@ out: btf_id__free_all(&obj.typedefs); btf_id__free_all(&obj.funcs); btf_id__free_all(&obj.sets); + free(obj.addr_syms); if (obj.efile.elf) { elf_end(obj.efile.elf); close(obj.efile.fd); diff --git a/tools/include/linux/btf_ids.h b/tools/include/linux/btf_ids.h index 72ea363d434d..4fe5c5f1558c 100644 --- a/tools/include/linux/btf_ids.h +++ b/tools/include/linux/btf_ids.h @@ -10,6 +10,9 @@ struct btf_id_set { u32 ids[]; }; +/* This flag implies BTF_SET8 holds kfunc(s) */ +#define BTF_SET8_KFUNCS (1 << 0) + struct btf_id_set8 { u32 cnt; u32 flags; @@ -22,6 +25,7 @@ struct btf_id_set8 { #ifdef CONFIG_DEBUG_INFO_BTF #include <linux/compiler.h> /* for __PASTE */ +#include <linux/stringify.h> /* * Following macros help to define lists of BTF IDs placed @@ -35,7 +39,7 @@ struct btf_id_set8 { #define BTF_IDS_SECTION ".BTF_ids" -#define ____BTF_ID(symbol) \ +#define ____BTF_ID(symbol, word) \ asm( \ ".pushsection " BTF_IDS_SECTION ",\"a\"; \n" \ ".local " #symbol " ; \n" \ @@ -43,10 +47,11 @@ asm( \ ".size " #symbol ", 4; \n" \ #symbol ": \n" \ ".zero 4 \n" \ +word \ ".popsection; \n"); -#define __BTF_ID(symbol) \ - ____BTF_ID(symbol) +#define __BTF_ID(symbol, word) \ + ____BTF_ID(symbol, word) #define __ID(prefix) \ __PASTE(__PASTE(prefix, __COUNTER__), __LINE__) @@ -56,7 +61,14 @@ asm( \ * to 4 zero bytes. */ #define BTF_ID(prefix, name) \ - __BTF_ID(__ID(__BTF_ID__##prefix##__##name##__)) + __BTF_ID(__ID(__BTF_ID__##prefix##__##name##__), "") + +#define ____BTF_ID_FLAGS(prefix, name, flags) \ + __BTF_ID(__ID(__BTF_ID__##prefix##__##name##__), ".long " #flags "\n") +#define __BTF_ID_FLAGS(prefix, name, flags, ...) \ + ____BTF_ID_FLAGS(prefix, name, flags) +#define BTF_ID_FLAGS(prefix, name, ...) \ + __BTF_ID_FLAGS(prefix, name, ##__VA_ARGS__, 0) /* * The BTF_ID_LIST macro defines pure (unsorted) list @@ -155,10 +167,58 @@ asm( \ ".popsection; \n"); \ extern struct btf_id_set name; +/* + * The BTF_SET8_START/END macros pair defines sorted list of + * BTF IDs and their flags plus its members count, with the + * following layout: + * + * BTF_SET8_START(list) + * BTF_ID_FLAGS(type1, name1, flags) + * BTF_ID_FLAGS(type2, name2, flags) + * BTF_SET8_END(list) + * + * __BTF_ID__set8__list: + * .zero 8 + * list: + * __BTF_ID__type1__name1__3: + * .zero 4 + * .word (1 << 0) | (1 << 2) + * __BTF_ID__type2__name2__5: + * .zero 4 + * .word (1 << 3) | (1 << 1) | (1 << 2) + * + */ +#define __BTF_SET8_START(name, scope, flags) \ +__BTF_ID_LIST(name, local) \ +asm( \ +".pushsection " BTF_IDS_SECTION ",\"a\"; \n" \ +"." #scope " __BTF_ID__set8__" #name "; \n" \ +"__BTF_ID__set8__" #name ":; \n" \ +".zero 4 \n" \ +".long " __stringify(flags) "\n" \ +".popsection; \n"); + +#define BTF_SET8_START(name) \ +__BTF_SET8_START(name, local, 0) + +#define BTF_SET8_END(name) \ +asm( \ +".pushsection " BTF_IDS_SECTION ",\"a\"; \n" \ +".size __BTF_ID__set8__" #name ", .-" #name " \n" \ +".popsection; \n"); \ +extern struct btf_id_set8 name; + +#define BTF_KFUNCS_START(name) \ +__BTF_SET8_START(name, local, BTF_SET8_KFUNCS) + +#define BTF_KFUNCS_END(name) \ +BTF_SET8_END(name) + #else -#define BTF_ID_LIST(name) static u32 __maybe_unused name[5]; +#define BTF_ID_LIST(name) static u32 __maybe_unused name[128]; #define BTF_ID(prefix, name) +#define BTF_ID_FLAGS(prefix, name, ...) #define BTF_ID_UNUSED #define BTF_ID_LIST_GLOBAL(name, n) u32 __maybe_unused name[n]; #define BTF_ID_LIST_SINGLE(name, prefix, typename) static u32 __maybe_unused name[1]; @@ -166,6 +226,10 @@ extern struct btf_id_set name; #define BTF_SET_START(name) static struct btf_id_set __maybe_unused name = { 0 }; #define BTF_SET_START_GLOBAL(name) static struct btf_id_set __maybe_unused name = { 0 }; #define BTF_SET_END(name) +#define BTF_SET8_START(name) static struct btf_id_set8 __maybe_unused name = { 0 }; +#define BTF_SET8_END(name) +#define BTF_KFUNCS_START(name) static struct btf_id_set8 __maybe_unused name = { .flags = BTF_SET8_KFUNCS }; +#define BTF_KFUNCS_END(name) #endif /* CONFIG_DEBUG_INFO_BTF */ @@ -215,5 +279,9 @@ MAX_BTF_TRACING_TYPE, }; extern u32 btf_tracing_ids[]; +extern u32 bpf_cgroup_btf_id[]; +extern u32 bpf_local_storage_map_btf_id[]; +extern u32 btf_bpf_map_id[]; +extern u32 bpf_kmem_cache_btf_id[]; #endif diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 89b36de5fdbb..732b35cc08d1 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -3038,8 +3038,34 @@ union bpf_attr { * * * **BPF_F_ADJ_ROOM_DECAP_L3_IPV4**, * **BPF_F_ADJ_ROOM_DECAP_L3_IPV6**: - * Indicate the new IP header version after decapsulating the outer - * IP header. Used when the inner and outer IP versions are different. + * Indicate the new IP header version after decapsulating the + * outer IP header. Used when the inner and outer IP versions + * are different. These flags only trigger a protocol change + * without clearing any tunnel-specific GSO flags. + * + * * **BPF_F_ADJ_ROOM_DECAP_L4_GRE**: + * Clear GRE tunnel GSO flags (SKB_GSO_GRE and SKB_GSO_GRE_CSUM) + * when decapsulating a GRE tunnel. + * + * * **BPF_F_ADJ_ROOM_DECAP_L4_UDP**: + * Clear UDP tunnel GSO flags (SKB_GSO_UDP_TUNNEL and + * SKB_GSO_UDP_TUNNEL_CSUM) when decapsulating a UDP tunnel. + * + * * **BPF_F_ADJ_ROOM_DECAP_IPXIP4**: + * Clear IPIP/SIT tunnel GSO flag (SKB_GSO_IPXIP4) when decapsulating + * a tunnel with an outer IPv4 header (IPv4-in-IPv4 or IPv6-in-IPv4). + * + * * **BPF_F_ADJ_ROOM_DECAP_IPXIP6**: + * Clear IPv6 encapsulation tunnel GSO flag (SKB_GSO_IPXIP6) when + * decapsulating a tunnel with an outer IPv6 header (IPv6-in-IPv6 + * or IPv4-in-IPv6). + * + * When using the decapsulation flags above, the skb->encapsulation + * flag is automatically cleared if all tunnel-specific GSO flags + * (SKB_GSO_UDP_TUNNEL, SKB_GSO_UDP_TUNNEL_CSUM, SKB_GSO_GRE, + * SKB_GSO_GRE_CSUM, SKB_GSO_IPXIP4, SKB_GSO_IPXIP6) have been + * removed from the packet. This handles cases where all tunnel + * layers have been decapsulated. * * A call to this helper is susceptible to change the underlying * packet buffer. Therefore, at load time, all checks on pointers @@ -3532,6 +3558,47 @@ union bpf_attr { * Use the mark present in *params*->mark for the fib lookup. * This option should not be used with BPF_FIB_LOOKUP_DIRECT, * as it only has meaning for full lookups. + * **BPF_FIB_LOOKUP_VLAN** + * If the fib lookup resolves to a VLAN device whose + * parent is a real (non-VLAN) device, set + * *params*->h_vlan_proto and *params*->h_vlan_TCI from + * the VLAN device and replace *params*->ifindex with the + * parent's ifindex. *params*->h_vlan_TCI carries the VID + * only, with PCP and DEI bits zero; a consumer wanting to + * set egress priority writes PCP itself. *params*->smac is + * the VLAN device's own address, which can differ from the + * parent's. Only the immediate parent is resolved; if it + * is itself a VLAN device (QinQ) or in another namespace, + * the egress cannot be reduced to a physical device plus + * one tag and the lookup returns + * **BPF_FIB_LKUP_RET_VLAN_FAILURE** with *params*->ifindex + * left at the input. To obtain the VLAN device's own + * ifindex, repeat the lookup without + * **BPF_FIB_LOOKUP_VLAN**, re-initializing *params* + * first: output fields overwrite the inputs they share + * storage with. The swap and the vlan fields + * are written only on success; other output fields keep + * the helper's existing behaviour, so a frag-needed result + * still reports the route mtu in *params*->mtu_result. + * This flag is only valid for XDP programs; tc programs + * receive -EINVAL since they can redirect to the VLAN + * device directly. + * **BPF_FIB_LOOKUP_VLAN_INPUT** + * Treat *params*->h_vlan_proto and *params*->h_vlan_TCI + * as an input VLAN tag and run the lookup as if ingress + * had happened on the VLAN subinterface carrying that tag + * on *params*->ifindex. The VID is the low 12 bits of + * *params*->h_vlan_TCI; *params*->h_vlan_proto must be + * ETH_P_8021Q or ETH_P_8021AD in network byte order, else + * **-EINVAL**. If *params*->ifindex is itself a VLAN + * device, its inner (QinQ) subinterface is matched; for a + * bond or team, pass the master's ifindex. An unmatched + * tag, a down device, or one in another namespace returns + * **BPF_FIB_LKUP_RET_NOT_FWDED**, mirroring real ingress. + * A VID of 0 is looked up literally, so do not set this + * flag for priority-tagged frames. Cannot be combined with + * **BPF_FIB_LOOKUP_TBID** or **BPF_FIB_LOOKUP_OUTPUT** + * (returns **-EINVAL**). * * *ctx* is either **struct xdp_md** for XDP programs or * **struct sk_buff** tc cls_act programs. @@ -4694,6 +4761,7 @@ union bpf_attr { * * **BPF_RB_RING_SIZE**: The size of ring buffer. * * **BPF_RB_CONS_POS**: Consumer position (can wrap around). * * **BPF_RB_PROD_POS**: Producer(s) position (can wrap around). + * * **BPF_RB_OVERWRITE_POS**: Overwrite position (can wrap around). * * Data returned is just a momentary snapshot of actual values * and could be inaccurate, so this facility should be used to @@ -5079,17 +5147,19 @@ union bpf_attr { * Description * Redirect the packet to another net device of index *ifindex*. * This helper is somewhat similar to **bpf_redirect**\ (), except - * that the redirection happens to the *ifindex*' peer device and - * the netns switch takes place from ingress to ingress without - * going through the CPU's backlog queue. + * that the redirection happens to the *ifindex*' peer device. If + * *flags* is 0, the netns switch takes place from ingress to + * ingress without going through the CPU's backlog queue. If the + * **BPF_F_EGRESS** flag is provided then redirection happens in + * the egress direction of the peer device. * * *skb*\ **->mark** and *skb*\ **->tstamp** are not cleared during * the netns switch. * - * The *flags* argument is reserved and must be 0. The helper is - * currently only supported for tc BPF program types at the - * ingress hook and for veth and netkit target device types. The - * peer device must reside in a different network namespace. + * If the *flags* argument is 0, the helper is currently only + * supported for tc BPF program types at the ingress hook and for + * veth and netkit target device types. The peer device must reside + * in a different network namespace. * Return * The helper returns **TC_ACT_REDIRECT** on success or * **TC_ACT_SHOT** on error. @@ -6239,7 +6309,7 @@ enum { }; /* BPF_FUNC_skb_adjust_room flags. */ -enum { +enum bpf_adj_room_flags { BPF_F_ADJ_ROOM_FIXED_GSO = (1ULL << 0), BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 = (1ULL << 1), BPF_F_ADJ_ROOM_ENCAP_L3_IPV6 = (1ULL << 2), @@ -6249,6 +6319,10 @@ enum { BPF_F_ADJ_ROOM_ENCAP_L2_ETH = (1ULL << 6), BPF_F_ADJ_ROOM_DECAP_L3_IPV4 = (1ULL << 7), BPF_F_ADJ_ROOM_DECAP_L3_IPV6 = (1ULL << 8), + BPF_F_ADJ_ROOM_DECAP_L4_GRE = (1ULL << 9), + BPF_F_ADJ_ROOM_DECAP_L4_UDP = (1ULL << 10), + BPF_F_ADJ_ROOM_DECAP_IPXIP4 = (1ULL << 11), + BPF_F_ADJ_ROOM_DECAP_IPXIP6 = (1ULL << 12), }; enum { @@ -6336,9 +6410,10 @@ enum { /* Flags for bpf_redirect and bpf_redirect_map helpers */ enum { BPF_F_INGRESS = (1ULL << 0), /* used for skb path */ + BPF_F_EGRESS = (1ULL << 1), /* used for skb path */ BPF_F_BROADCAST = (1ULL << 3), /* used for XDP path */ BPF_F_EXCLUDE_INGRESS = (1ULL << 4), /* used for XDP path */ -#define BPF_F_REDIRECT_FLAGS (BPF_F_INGRESS | BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS) +#define BPF_F_REDIRECT_FLAGS (BPF_F_INGRESS | BPF_F_EGRESS | BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS) }; #define __bpf_md_ptr(type, name) \ @@ -6840,6 +6915,15 @@ struct bpf_link_info { __u32 pid; } uprobe_multi; struct { + __u32 attach_type; + __u32 count; /* in/out: tracing_multi target count */ + __u32 btf_obj_id; + __u32 :32; + __aligned_u64 ids; + __aligned_u64 addrs; + __aligned_u64 cookies; + } tracing_multi; + struct { __u32 type; /* enum bpf_perf_event_type */ __u32 :32; union { @@ -7327,6 +7411,8 @@ enum { BPF_FIB_LOOKUP_TBID = (1U << 3), BPF_FIB_LOOKUP_SRC = (1U << 4), BPF_FIB_LOOKUP_MARK = (1U << 5), + BPF_FIB_LOOKUP_VLAN = (1U << 6), + BPF_FIB_LOOKUP_VLAN_INPUT = (1U << 7), }; enum { @@ -7340,6 +7426,7 @@ enum { BPF_FIB_LKUP_RET_NO_NEIGH, /* no neighbor entry for nh */ BPF_FIB_LKUP_RET_FRAG_NEEDED, /* fragmentation required to fwd */ BPF_FIB_LKUP_RET_NO_SRC_ADDR, /* failed to derive IP src addr */ + BPF_FIB_LKUP_RET_VLAN_FAILURE, /* VLAN egress, parent unresolvable */ }; struct bpf_fib_lookup { @@ -7393,7 +7480,13 @@ struct bpf_fib_lookup { union { struct { - /* output */ + /* + * output with BPF_FIB_LOOKUP_VLAN: set from the + * resolved egress VLAN device (see the flag); zeroed + * on other successful lookups. input with + * BPF_FIB_LOOKUP_VLAN_INPUT: the VLAN tag to scope + * the lookup by. + */ __be16 h_vlan_proto; __be16 h_vlan_TCI; }; diff --git a/tools/lib/bpf/bpf_gen_internal.h b/tools/lib/bpf/bpf_gen_internal.h index 49af4260b8e6..6c5ad6c55e8a 100644 --- a/tools/lib/bpf/bpf_gen_internal.h +++ b/tools/lib/bpf/bpf_gen_internal.h @@ -51,7 +51,6 @@ struct bpf_gen { __u32 nr_ksyms; int fd_array; int nr_fd_array; - int hash_insn_offset[SHA256_DWORD_SIZE]; }; void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps); @@ -66,7 +65,8 @@ void bpf_gen__prog_load(struct bpf_gen *gen, enum bpf_prog_type prog_type, const char *prog_name, const char *license, struct bpf_insn *insns, size_t insn_cnt, struct bpf_prog_load_opts *load_attr, int prog_idx); -void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *value, __u32 value_size); +void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *value, __u32 value_size, + __u64 flags); void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx); void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *name, enum bpf_attach_type type); void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 823bce895178..8417de92d028 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -589,7 +589,7 @@ static int btf_parse_type_sec(struct btf *btf) if (type_size < 0) return type_size; if (next_type + type_size > end_type) { - pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types); + pr_warn("BTF type [%u] is malformed\n", btf->start_id + btf->nr_types); return -EINVAL; } @@ -1424,7 +1424,7 @@ static int btf_find_elf_sections(Elf *elf, const char *path, struct btf_elf_secs continue; if (sh.sh_type != SHT_PROGBITS) { - pr_warn("unexpected section type (%d) of section(%d, %s) from %s\n", + pr_warn("unexpected section type (%u) of section(%d, %s) from %s\n", sh.sh_type, idx, name, path); goto err; } @@ -1506,9 +1506,6 @@ static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, dist_base_btf = NULL; } - if (dist_base_btf) - btf->owns_base = true; - switch (gelf_getclass(elf)) { case ELFCLASS32: btf__set_pointer_size(btf, 4); @@ -1523,13 +1520,16 @@ static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, if (btf_ext && secs.btf_ext_data) { *btf_ext = btf_ext__new(secs.btf_ext_data->d_buf, secs.btf_ext_data->d_size); - if (IS_ERR(*btf_ext)) { - err = PTR_ERR(*btf_ext); + if (!*btf_ext) { + err = -errno; goto done; } } else if (btf_ext) { *btf_ext = NULL; } + + if (dist_base_btf) + btf->owns_base = true; done: if (elf) elf_end(elf); @@ -4854,7 +4854,7 @@ recur: continue; if (!btf_dedup_identical_types(d, m1->type, m2->type, depth - 1)) { if (t1->name_off) { - pr_debug("%s '%s' size=%d vlen=%d id1[%u] id2[%u] shallow-equal but not identical for field#%d '%s'\n", + pr_debug("%s '%s' size=%u vlen=%u id1[%u] id2[%u] shallow-equal but not identical for field#%d '%s'\n", k1 == BTF_KIND_STRUCT ? "STRUCT" : "UNION", btf__name_by_offset(d->btf, t1->name_off), t1->size, btf_vlen(t1), id1, id2, i, @@ -5104,7 +5104,7 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type); if (eq <= 0) { if (cand_type->name_off) { - pr_debug("%s '%s' size=%d vlen=%d cand_id[%u] canon_id[%u] shallow-equal but not equiv for field#%d '%s': %d\n", + pr_debug("%s '%s' size=%u vlen=%u cand_id[%u] canon_id[%u] shallow-equal but not equiv for field#%d '%s': %d\n", cand_kind == BTF_KIND_STRUCT ? "STRUCT" : "UNION", btf__name_by_offset(d->btf, cand_type->name_off), cand_type->size, vlen, cand_id, canon_id, i, @@ -6069,7 +6069,7 @@ static int btf_add_distilled_types(struct btf_distill *dist) err = btf_add_type(&dist->pipe, t); break; default: - pr_warn("unexpected kind when adding base type '%s'[%u] of kind [%u] to distilled base BTF.\n", + pr_warn("unexpected kind when adding base type '%s'[%d] of kind [%d] to distilled base BTF.\n", name, i, kind); return -EINVAL; diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index 1a31f2da947f..587172c0de08 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -172,6 +172,8 @@ LIBBPF_API __s32 btf__find_by_name(const struct btf *btf, const char *type_name); LIBBPF_API __s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name, __u32 kind); +LIBBPF_API __s32 btf__find_by_name_kind_own(const struct btf *btf, + const char *type_name, __u32 kind); LIBBPF_API __u32 btf__type_cnt(const struct btf *btf); LIBBPF_API const struct btf *btf__base_btf(const struct btf *btf); LIBBPF_API const struct btf_type *btf__type_by_id(const struct btf *btf, diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index cc1ba65bb6c5..123c448f20c7 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -1776,7 +1776,7 @@ static int btf_dump_get_bitfield_value(struct btf_dump *d, /* Maximum supported bitfield size is 64 bits */ if (t->size > 8) { - pr_warn("unexpected bitfield size %d\n", t->size); + pr_warn("unexpected bitfield size %u\n", t->size); return -EINVAL; } @@ -2251,7 +2251,7 @@ static int btf_dump_get_enum_value(struct btf_dump *d, *value = is_signed ? *(__s8 *)data : *(__u8 *)data; return 0; default: - pr_warn("unexpected size %d for enum, id:[%u]\n", t->size, id); + pr_warn("unexpected size %u for enum, id:[%u]\n", t->size, id); return -EINVAL; } } diff --git a/tools/lib/bpf/btf_relocate.c b/tools/lib/bpf/btf_relocate.c index 53d1f3541bce..df5fa4bd87d6 100644 --- a/tools/lib/bpf/btf_relocate.c +++ b/tools/lib/bpf/btf_relocate.c @@ -280,7 +280,7 @@ static int btf_relocate_map_distilled_base(struct btf_relocate *r) cmp_btf_name_size(&base_info, dist_info) == 0; dist_info++) { if (!dist_info->id || dist_info->id >= r->nr_dist_base_types) { - pr_warn("base BTF id [%d] maps to invalid distilled base BTF id [%d]\n", + pr_warn("base BTF id [%u] maps to invalid distilled base BTF id [%u]\n", id, dist_info->id); err = -EINVAL; goto done; @@ -368,7 +368,7 @@ static int btf_relocate_map_distilled_base(struct btf_relocate *r) continue; dist_t = btf_type_by_id(r->dist_base_btf, id); name = btf__name_by_offset(r->dist_base_btf, dist_t->name_off); - pr_warn("distilled base BTF type '%s' [%d] is not mapped to base BTF id\n", + pr_warn("distilled base BTF type '%s' [%u] is not mapped to base BTF id\n", name, id); err = -EINVAL; break; @@ -397,11 +397,11 @@ static int btf_relocate_validate_distilled_base(struct btf_relocate *r) case BTF_KIND_FWD: if (t->name_off) break; - pr_warn("type [%d], kind [%d] is invalid for distilled base BTF; it is anonymous\n", + pr_warn("type [%u], kind [%d] is invalid for distilled base BTF; it is anonymous\n", i, kind); return -EINVAL; default: - pr_warn("type [%d] in distilled based BTF has unexpected kind [%d]\n", + pr_warn("type [%u] in distilled based BTF has unexpected kind [%d]\n", i, kind); return -EINVAL; } diff --git a/tools/lib/bpf/elf.c b/tools/lib/bpf/elf.c index 295dbda24580..fe136d025967 100644 --- a/tools/lib/bpf/elf.c +++ b/tools/lib/bpf/elf.c @@ -354,7 +354,7 @@ long elf_find_func_offset(Elf *elf, const char *binary_path, const char *name) if (ret > 0) { pr_debug("elf: symbol address match for '%s' in '%s': 0x%lx\n", name, binary_path, - ret); + (unsigned long)ret); } else { if (ret == 0) { pr_warn("elf: '%s' is 0 in symtab for '%s': %s\n", name, binary_path, diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c index e5641fa60163..9f589980beef 100644 --- a/tools/lib/bpf/features.c +++ b/tools/lib/bpf/features.c @@ -620,6 +620,38 @@ static int probe_bpf_syscall_common_attrs(int token_fd) return probe_sys_bpf_ext(); } +static int probe_kern_percpu_data(int token_fd) +{ + struct bpf_insn insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_1, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0), + BPF_EXIT_INSN(), + }; + LIBBPF_OPTS(bpf_map_create_opts, map_opts, + .token_fd = token_fd, + .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, + ); + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, + .token_fd = token_fd, + .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, + ); + int ret, map, insn_cnt = ARRAY_SIZE(insns); + + map = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "libbpf_percpu", sizeof(int), 8, 1, + &map_opts); + if (map < 0) { + pr_warn("Error in %s(): %s. Couldn't create simple percpu_array map.\n", + __func__, errstr(map)); + return map; + } + + insns[0].imm = map; + + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); + close(map); + return probe_fd(ret); +} + typedef int (*feature_probe_fn)(int /* token_fd */); static struct kern_feature_cache feature_cache; @@ -707,6 +739,9 @@ static struct kern_feature_desc { [FEAT_BPF_SYSCALL_COMMON_ATTRS] = { "BPF syscall common attributes support", probe_bpf_syscall_common_attrs, }, + [FEAT_PERCPU_DATA] = { + "kernel supports percpu data", probe_kern_percpu_data, + }, }; bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id) diff --git a/tools/lib/bpf/gen_loader.c b/tools/lib/bpf/gen_loader.c index d79695f01c87..af3a04f161ac 100644 --- a/tools/lib/bpf/gen_loader.c +++ b/tools/lib/bpf/gen_loader.c @@ -111,7 +111,6 @@ static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn in static int add_data(struct bpf_gen *gen, const void *data, __u32 size); static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off); -static void emit_signature_match(struct bpf_gen *gen); void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps) { @@ -154,8 +153,6 @@ void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps /* R7 contains the error code from sys_bpf. Copy it into R0 and exit. */ emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7)); emit(gen, BPF_EXIT_INSN()); - if (OPTS_GET(gen->opts, gen_hash, false)) - emit_signature_match(gen); } static int add_data(struct bpf_gen *gen, const void *data, __u32 size) @@ -377,14 +374,12 @@ static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off) __emit_sys_close(gen); } -static void compute_sha_update_offsets(struct bpf_gen *gen); - int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) { int i; if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) { - pr_warn("nr_progs %d/%d nr_maps %d/%d mismatch\n", + pr_warn("nr_progs %d/%u nr_maps %d/%u mismatch\n", nr_progs, gen->nr_progs, nr_maps, gen->nr_maps); gen->error = -EFAULT; return gen->error; @@ -408,9 +403,6 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) if (!gen->error) { struct gen_loader_opts *opts = gen->opts; - if (OPTS_GET(opts, gen_hash, false)) - compute_sha_update_offsets(gen); - opts->insns = gen->insn_start; opts->insns_sz = gen->insn_cur - gen->insn_start; opts->data = gen->data_start; @@ -460,22 +452,6 @@ void bpf_gen__free(struct bpf_gen *gen) _val; \ }) -static void compute_sha_update_offsets(struct bpf_gen *gen) -{ - __u64 sha[SHA256_DWORD_SIZE]; - __u64 sha_dw; - int i; - - libbpf_sha256(gen->data_start, gen->data_cur - gen->data_start, (__u8 *)sha); - for (i = 0; i < SHA256_DWORD_SIZE; i++) { - struct bpf_insn *insn = - (struct bpf_insn *)(gen->insn_start + gen->hash_insn_offset[i]); - sha_dw = tgt_endian(sha[i]); - insn[0].imm = (__u32)sha_dw; - insn[1].imm = sha_dw >> 32; - } -} - void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data, __u32 btf_raw_size) { @@ -488,7 +464,7 @@ void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data, attr.btf_size = tgt_endian(btf_raw_size); btf_load_attr = add_data(gen, &attr, attr_size); - pr_debug("gen: load_btf: off %d size %d, attr: off %d size %d\n", + pr_debug("gen: load_btf: off %d size %u, attr: off %d size %d\n", btf_data, btf_raw_size, btf_load_attr, attr_size); /* populate union bpf_attr with user provided log details */ @@ -534,7 +510,7 @@ void bpf_gen__map_create(struct bpf_gen *gen, attr.btf_value_type_id = tgt_endian(map_attr->btf_value_type_id); map_create_attr = add_data(gen, &attr, attr_size); - pr_debug("gen: map_create: %s idx %d type %d value_type_id %d, attr: off %d size %d\n", + pr_debug("gen: map_create: %s idx %d type %u value_type_id %u, attr: off %d size %d\n", map_name, map_idx, map_type, map_attr->btf_value_type_id, map_create_attr, attr_size); @@ -557,8 +533,9 @@ void bpf_gen__map_create(struct bpf_gen *gen, * Conditionally update max_entries from the host-supplied loader * ctx. This sizes the map at runtime, but for a signed loader * (gen_hash) it would let an untrusted host re-dimension the - * program's maps after emit_signature_match(), outside what the - * signature attests to. Keep the signer-provided max_entries + * program's maps, outside what the signature attests to: the + * metadata blob is covered by the program signature and verified + * by the kernel at load time. Keep the signer-provided max_entries * baked into the blob in that case. */ if (map_idx >= 0 && !OPTS_GET(gen->opts, gen_hash, false)) @@ -596,45 +573,6 @@ void bpf_gen__map_create(struct bpf_gen *gen, emit_sys_close_stack(gen, stack_off(inner_map_fd)); } -static void emit_signature_match(struct bpf_gen *gen) -{ - __s64 off; - int i; - - /* - * Reject if the metadata map is not exclusive. Without exclusivity - * the cached map->sha[] verified above can be stale: another BPF - * program with map access could have mutated the contents between - * BPF_OBJ_GET_INFO_BY_FD and loader execution. - */ - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX, - 0, 0, 0, 0)); - emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, SHA256_DIGEST_LENGTH)); - off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 2; - if (is_simm16(off)) { - emit(gen, BPF_MOV64_IMM(BPF_REG_7, -EINVAL)); - emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_2, 1, off)); - } else { - gen->error = -ERANGE; - } - - for (i = 0; i < SHA256_DWORD_SIZE; i++) { - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX, - 0, 0, 0, 0)); - emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, i * sizeof(__u64))); - gen->hash_insn_offset[i] = gen->insn_cur - gen->insn_start; - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_3, 0, 0, 0, 0, 0)); - - off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 2; - if (is_simm16(off)) { - emit(gen, BPF_MOV64_IMM(BPF_REG_7, -EINVAL)); - emit(gen, BPF_JMP_REG(BPF_JNE, BPF_REG_2, BPF_REG_3, off)); - } else { - gen->error = -ERANGE; - } - } -} - void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name, enum bpf_attach_type type) { @@ -1082,7 +1020,7 @@ void bpf_gen__prog_load(struct bpf_gen *gen, license_off = add_data(gen, license, strlen(license) + 1); /* add insns to blob of bytes */ insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn)); - pr_debug("gen: prog_load: prog_idx %d type %d insn off %d insns_cnt %zd license off %d\n", + pr_debug("gen: prog_load: prog_idx %d type %u insn off %d insns_cnt %zu license off %d\n", prog_idx, prog_type, insns_off, insn_cnt, license_off); /* convert blob insns to target endianness */ @@ -1105,21 +1043,21 @@ void bpf_gen__prog_load(struct bpf_gen *gen, attr.func_info_rec_size = tgt_endian(load_attr->func_info_rec_size); attr.func_info_cnt = tgt_endian(load_attr->func_info_cnt); func_info = add_data(gen, load_attr->func_info, func_info_tot_sz); - pr_debug("gen: prog_load: func_info: off %d cnt %d rec size %d\n", + pr_debug("gen: prog_load: func_info: off %d cnt %u rec size %u\n", func_info, load_attr->func_info_cnt, load_attr->func_info_rec_size); attr.line_info_rec_size = tgt_endian(load_attr->line_info_rec_size); attr.line_info_cnt = tgt_endian(load_attr->line_info_cnt); line_info = add_data(gen, load_attr->line_info, line_info_tot_sz); - pr_debug("gen: prog_load: line_info: off %d cnt %d rec size %d\n", + pr_debug("gen: prog_load: line_info: off %d cnt %u rec size %u\n", line_info, load_attr->line_info_cnt, load_attr->line_info_rec_size); attr.core_relo_rec_size = tgt_endian((__u32)sizeof(struct bpf_core_relo)); attr.core_relo_cnt = tgt_endian(gen->core_relo_cnt); core_relos = add_data(gen, gen->core_relos, core_relo_tot_sz); - pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zd\n", + pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zu\n", core_relos, gen->core_relo_cnt, sizeof(struct bpf_core_relo)); @@ -1190,7 +1128,7 @@ void bpf_gen__prog_load(struct bpf_gen *gen, } void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, - __u32 value_size) + __u32 value_size, __u64 flags) { int attr_size = offsetofend(union bpf_attr, flags); int map_update_attr, value, key; @@ -1198,6 +1136,7 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, int zero = 0; memset(&attr, 0, attr_size); + attr.flags = tgt_endian(flags); value = add_data(gen, pvalue, value_size); key = add_data(gen, &zero, sizeof(zero)); @@ -1211,10 +1150,10 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, * } * * The runtime initial_value comes from the host-supplied loader - * ctx and would overwrite the blob value after emit_signature_match() - * has already validated map->sha[]. For a signed loader (gen_hash) - * the attested blob value must be authoritative, so skip the override - * and leave the hashed value in place. + * ctx and would overwrite the blob value that the program signature + * covers and the kernel verifies at load time. For a signed loader + * (gen_hash) the attested blob value must be authoritative, so skip + * the override and leave the signed value in place. */ if (!OPTS_GET(gen->opts, gen_hash, false)) { emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, @@ -1234,7 +1173,7 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, } map_update_attr = add_data(gen, &attr, attr_size); - pr_debug("gen: map_update_elem: idx %d, value: off %d size %d, attr: off %d size %d\n", + pr_debug("gen: map_update_elem: idx %d, value: off %d size %u, attr: off %d size %d\n", map_idx, value, value_size, map_update_attr, attr_size); move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, blob_fd_array_off(gen, map_idx)); diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 1368752aa13c..b749c01742ee 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -541,6 +541,7 @@ struct bpf_struct_ops { }; #define DATA_SEC ".data" +#define PERCPU_SEC ".percpu" #define BSS_SEC ".bss" #define RODATA_SEC ".rodata" #define KCONFIG_SEC ".kconfig" @@ -555,6 +556,7 @@ enum libbpf_map_type { LIBBPF_MAP_BSS, LIBBPF_MAP_RODATA, LIBBPF_MAP_KCONFIG, + LIBBPF_MAP_PERCPU, }; struct bpf_map_def { @@ -666,6 +668,7 @@ enum sec_type { SEC_DATA, SEC_RODATA, SEC_ST_OPS, + SEC_PERCPU, }; struct elf_sec_desc { @@ -1486,7 +1489,7 @@ static int init_struct_ops_maps(struct bpf_object *obj, const char *sec_name, type->size); st_ops->type_id = type_id; - pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n", + pr_debug("struct_ops init: struct %s(type_id=%d) %s found at offset %u\n", tname, type_id, var_name, vsi->offset); } @@ -1838,6 +1841,7 @@ static size_t bpf_map_mmap_sz(const struct bpf_map *map) switch (map->def.type) { case BPF_MAP_TYPE_ARRAY: + case BPF_MAP_TYPE_PERCPU_ARRAY: return array_map_mmap_sz(map->def.value_size, map->def.max_entries); case BPF_MAP_TYPE_ARENA: return page_sz * map->def.max_entries; @@ -1866,7 +1870,8 @@ static int bpf_map_mmap_resize(struct bpf_map *map, size_t old_sz, size_t new_sz return 0; } -static char *internal_map_name(struct bpf_object *obj, const char *real_name) +static char *internal_map_name(struct bpf_object *obj, const char *real_name, + enum libbpf_map_type type) { char map_name[BPF_OBJ_NAME_LEN], *p; int pfx_len, sfx_len = max((size_t)7, strlen(real_name)); @@ -1907,8 +1912,11 @@ static char *internal_map_name(struct bpf_object *obj, const char *real_name) if (sfx_len >= BPF_OBJ_NAME_LEN) sfx_len = BPF_OBJ_NAME_LEN - 1; - /* if there are two or more dots in map name, it's a custom dot map */ - if (strchr(real_name + 1, '.') != NULL) + /* + * Don't prefix the bpf_object name if this is a custom dot map + * (containing two or more dots) or a percpu data map. + */ + if (strchr(real_name + 1, '.') != NULL || type == LIBBPF_MAP_PERCPU) pfx_len = 0; else pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name)); @@ -1941,6 +1949,13 @@ static bool map_is_mmapable(struct bpf_object *obj, struct bpf_map *map) if (!map->btf_value_type_id) return false; + /* + * The internal PERCPU maps are not mmapable because the underlying + * percpu_array maps do not have mmap support. + */ + if (map->libbpf_type == LIBBPF_MAP_PERCPU) + return false; + t = btf__type_by_id(obj->btf, map->btf_value_type_id); if (!btf_is_datasec(t)) return false; @@ -1962,6 +1977,7 @@ static int bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, const char *real_name, int sec_idx, void *data, size_t data_sz) { + bool is_percpu = type == LIBBPF_MAP_PERCPU; struct bpf_map_def *def; struct bpf_map *map; size_t mmap_sz; @@ -1975,7 +1991,7 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, map->sec_idx = sec_idx; map->sec_offset = 0; map->real_name = strdup(real_name); - map->name = internal_map_name(obj, real_name); + map->name = internal_map_name(obj, real_name, type); if (!map->real_name || !map->name) { zfree(&map->real_name); zfree(&map->name); @@ -1983,7 +1999,7 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, } def = &map->def; - def->type = BPF_MAP_TYPE_ARRAY; + def->type = is_percpu ? BPF_MAP_TYPE_PERCPU_ARRAY : BPF_MAP_TYPE_ARRAY; def->key_size = sizeof(int); def->value_size = data_sz; def->max_entries = 1; @@ -1996,8 +2012,9 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, if (map_is_mmapable(obj, map)) def->map_flags |= BPF_F_MMAPABLE; - pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n", - map->name, map->sec_idx, map->sec_offset, def->map_flags); + pr_debug("map '%s' (global %sdata): at sec_idx %d, offset %zu, flags %x.\n", + map->name, is_percpu ? "percpu " : "", map->sec_idx, + map->sec_offset, def->map_flags); mmap_sz = bpf_map_mmap_sz(map); map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, @@ -2057,6 +2074,13 @@ static int bpf_object__init_global_data_maps(struct bpf_object *obj) NULL, sec_desc->data->d_size); break; + case SEC_PERCPU: + sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); + err = bpf_object__init_internal_map(obj, LIBBPF_MAP_PERCPU, + sec_name, sec_idx, + sec_desc->data->d_buf, + sec_desc->data->d_size); + break; default: /* skip */ break; @@ -2626,7 +2650,7 @@ int parse_btf_map_def(const char *map_name, struct btf *btf, t = btf__type_by_id(btf, m->type); if (!t) { - pr_warn("map '%s': key type [%d] not found.\n", + pr_warn("map '%s': key type [%u] not found.\n", map_name, m->type); return -EINVAL; } @@ -2666,7 +2690,7 @@ int parse_btf_map_def(const char *map_name, struct btf *btf, t = btf__type_by_id(btf, m->type); if (!t) { - pr_warn("map '%s': value type [%d] not found.\n", + pr_warn("map '%s': value type [%u] not found.\n", map_name, m->type); return -EINVAL; } @@ -2720,7 +2744,7 @@ int parse_btf_map_def(const char *map_name, struct btf *btf, map_def->value_size = 4; t = btf__type_by_id(btf, m->type); if (!t) { - pr_warn("map '%s': %s type [%d] not found.\n", + pr_warn("map '%s': %s type [%u] not found.\n", map_name, desc, m->type); return -EINVAL; } @@ -2806,7 +2830,7 @@ static size_t adjust_ringbuf_sz(size_t sz) return 0; /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be * a power-of-2 multiple of kernel's page size. If user diligently - * satisified these conditions, pass the size through. + * satisfied these conditions, pass the size through. */ if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz)) return sz; @@ -3476,7 +3500,7 @@ static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf, var_name = btf__name_by_offset(btf, t_var->name_off); if (!var_name) { - pr_debug("sec '%s': failed to find name of DATASEC's member #%d\n", + pr_debug("sec '%s': failed to find name of DATASEC's member #%u\n", sec_name, i); return -ENOENT; } @@ -3971,7 +3995,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj) if (!data) return -LIBBPF_ERRNO__FORMAT; - pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", + pr_debug("elf: section(%d) %s, size %lu, link %d, flags %lx, type=%d\n", idx, name, (unsigned long)data->d_size, (int)sh->sh_link, (unsigned long)sh->sh_flags, (int)sh->sh_type); @@ -4016,6 +4040,11 @@ static int bpf_object__elf_collect(struct bpf_object *obj) sec_desc->sec_type = SEC_RODATA; sec_desc->shdr = sh; sec_desc->data = data; + } else if (strcmp(name, PERCPU_SEC) == 0 || + str_has_pfx(name, PERCPU_SEC ".")) { + sec_desc->sec_type = SEC_PERCPU; + sec_desc->shdr = sh; + sec_desc->data = data; } else if (strcmp(name, STRUCT_OPS_SEC) == 0 || strcmp(name, STRUCT_OPS_LINK_SEC) == 0 || strcmp(name, "?" STRUCT_OPS_SEC) == 0 || @@ -4494,7 +4523,7 @@ static int bpf_object__collect_externs(struct bpf_object *obj) ext->kcfg.data_off = roundup(off, ext->kcfg.align); off = ext->kcfg.data_off + ext->kcfg.sz; - pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n", + pr_debug("extern (kcfg) #%d: symbol %d, off %d, name %s\n", i, ext->sym_idx, ext->kcfg.data_off, ext->name); } sec->size = off; @@ -4544,6 +4573,7 @@ static bool bpf_object__shndx_is_data(const struct bpf_object *obj, case SEC_BSS: case SEC_DATA: case SEC_RODATA: + case SEC_PERCPU: return true; default: return false; @@ -4569,6 +4599,8 @@ bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) return LIBBPF_MAP_DATA; case SEC_RODATA: return LIBBPF_MAP_RODATA; + case SEC_PERCPU: + return LIBBPF_MAP_PERCPU; default: return LIBBPF_MAP_UNSPEC; } @@ -4626,7 +4658,7 @@ static int bpf_program__record_reloc(struct bpf_program *prog, struct bpf_map *map; if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) { - pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n", + pr_warn("prog '%s': invalid relo against '%s' for insns[%u].code 0x%x\n", prog->name, sym_name, insn_idx, insn->code); return -LIBBPF_ERRNO__RELOC; } @@ -4749,7 +4781,7 @@ static int bpf_program__record_reloc(struct bpf_program *prog, map->sec_idx != sym->st_shndx || map->sec_offset != sym->st_value) continue; - pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n", + pr_debug("prog '%s': found map %zu (%s, sec %d, off %zu) for insn #%u\n", prog->name, map_idx, map->name, map->sec_idx, map->sec_offset, insn_idx); break; @@ -4776,7 +4808,7 @@ static int bpf_program__record_reloc(struct bpf_program *prog, map = &obj->maps[map_idx]; if (map->libbpf_type != type || map->sec_idx != sym->st_shndx) continue; - pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n", + pr_debug("prog '%s': found data map %zu (%s, sec %d, off %zu) for insn %u\n", prog->name, map_idx, map->name, map->sec_idx, map->sec_offset, insn_idx); break; @@ -4944,7 +4976,7 @@ static int map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map) /* * LLVM annotates global data differently in BTF, that is, - * only as '.data', '.bss' or '.rodata'. + * only as '.data', '.bss', '.percpu' or '.rodata'. */ if (!bpf_map__is_internal(map)) return -ENOENT; @@ -4985,7 +5017,7 @@ static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info) info->value_size = val; else if (sscanf(buff, "max_entries:\t%u", &val) == 1) info->max_entries = val; - else if (sscanf(buff, "map_flags:\t%i", &val) == 1) + else if (sscanf(buff, "map_flags:\t%x", &val) == 1) info->map_flags = val; } @@ -5172,12 +5204,8 @@ bpf_object__probe_loading(struct bpf_object *obj) BPF_EXIT_INSN(), }; int ret, insn_cnt = ARRAY_SIZE(insns); - LIBBPF_OPTS(bpf_prog_load_opts, opts, - .token_fd = obj->token_fd, - .prog_flags = obj->token_fd ? BPF_F_TOKEN_FD : 0, - ); - if (obj->gen_loader) + if (obj->gen_loader || obj->token_fd) return 0; ret = bump_rlimit_memlock(); @@ -5186,9 +5214,9 @@ bpf_object__probe_loading(struct bpf_object *obj) errstr(ret)); /* make sure basic loading works */ - ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &opts); + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); if (ret < 0) - ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts); + ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL); if (ret < 0) { ret = errno; pr_warn("Error in %s(): %s. Couldn't load trivial BPF program. Make sure your kernel supports BPF (CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is set to big enough value.\n", @@ -5297,18 +5325,20 @@ static int bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) { enum libbpf_map_type map_type = map->libbpf_type; + bool is_percpu = map_type == LIBBPF_MAP_PERCPU; + const __u64 update_flags = is_percpu ? BPF_F_ALL_CPUS : 0; int err, zero = 0; size_t mmap_sz; if (obj->gen_loader) { bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps, - map->mmaped, map->def.value_size); + map->mmaped, map->def.value_size, update_flags); if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) bpf_gen__map_freeze(obj->gen_loader, map - obj->maps); return 0; } - err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0); + err = bpf_map_update_elem(map->fd, &zero, map->mmaped, update_flags); if (err) { err = -errno; pr_warn("map '%s': failed to set initial contents: %s\n", @@ -5353,6 +5383,13 @@ bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) return err; } map->mmaped = mmaped; + } else if (is_percpu) { + if (mprotect(map->mmaped, mmap_sz, PROT_READ)) { + err = -errno; + pr_warn("map '%s': failed to mprotect() contents: %s\n", + bpf_map__name(map), errstr(err)); + return err; + } } else if (map->mmaped) { munmap(map->mmaped, mmap_sz); map->mmaped = NULL; @@ -5521,11 +5558,11 @@ static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map) } if (err) { err = -errno; - pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %s\n", + pr_warn("map '%s': failed to initialize slot [%u] to map '%s' fd=%d: %s\n", map->name, i, targ_map->name, fd, errstr(err)); return err; } - pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n", + pr_debug("map '%s': slot [%u] set to map '%s' fd=%d\n", map->name, i, targ_map->name, fd); } @@ -5554,11 +5591,11 @@ static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map) err = bpf_map_update_elem(map->fd, &i, &fd, 0); if (err) { err = -errno; - pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %s\n", + pr_warn("map '%s': failed to initialize slot [%u] to prog '%s' fd=%d: %s\n", map->name, i, targ_prog->name, fd, errstr(err)); return err; } - pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n", + pr_debug("map '%s': slot [%u] set to prog '%s' fd=%d\n", map->name, i, targ_prog->name, fd); } @@ -5628,9 +5665,16 @@ bpf_object__create_maps(struct bpf_object *obj) * runtime due to bpf_program__set_autoload(prog, false), * bpf_object loading will succeed just fine even on old * kernels. + * Same skipping applies to percpu data. */ - if (bpf_map__is_internal(map) && !kernel_supports(obj, FEAT_GLOBAL_DATA)) - map->autocreate = false; + if (bpf_map__is_internal(map)) { + bool is_percpu = map->libbpf_type == LIBBPF_MAP_PERCPU; + enum kern_feature_id feat_id; + + feat_id = is_percpu ? FEAT_PERCPU_DATA : FEAT_GLOBAL_DATA; + if (!kernel_supports(obj, feat_id)) + map->autocreate = false; + } if (!map->autocreate) { pr_debug("map '%s': skipped auto-creating...\n", map->name); @@ -5788,7 +5832,7 @@ int bpf_core_add_cands(struct bpf_core_cand *local_cand, if (strncmp(local_name, targ_name, local_essent_len) != 0) continue; - pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n", + pr_debug("CO-RE relocating [%u] %s %s: found target candidate [%d] %s %s in [%s]\n", local_cand->id, btf_kind_str(local_t), local_name, i, btf_kind_str(t), targ_name, targ_btf_name); @@ -5848,7 +5892,7 @@ static int load_module_btfs(struct bpf_object *obj) if (errno == ENOENT) continue; /* expected race: BTF was unloaded */ err = -errno; - pr_warn("failed to get BTF object #%d FD: %s\n", id, errstr(err)); + pr_warn("failed to get BTF object #%u FD: %s\n", id, errstr(err)); return err; } @@ -5861,7 +5905,7 @@ static int load_module_btfs(struct bpf_object *obj) err = bpf_btf_get_info_by_fd(fd, &info, &len); if (err) { err = -errno; - pr_warn("failed to get BTF object #%d info: %s\n", id, errstr(err)); + pr_warn("failed to get BTF object #%u info: %s\n", id, errstr(err)); break; } @@ -5874,7 +5918,7 @@ static int load_module_btfs(struct bpf_object *obj) btf = btf_get_from_fd(fd, obj->btf_vmlinux); err = libbpf_get_error(btf); if (err) { - pr_warn("failed to load module [%s]'s BTF object #%d: %s\n", + pr_warn("failed to load module [%s]'s BTF object #%u: %s\n", name, id, errstr(err)); break; } @@ -6067,7 +6111,7 @@ static int bpf_core_resolve_relo(struct bpf_program *prog, !hashmap__find(cand_cache, local_id, &cands)) { cands = bpf_core_find_cands(prog->obj, local_btf, local_id); if (IS_ERR(cands)) { - pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n", + pr_warn("prog '%s': relo #%d: target candidate search failed for [%u] %s %s: %ld\n", prog_name, relo_idx, local_id, btf_kind_str(local_type), local_name, PTR_ERR(cands)); return PTR_ERR(cands); @@ -6127,7 +6171,7 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) goto out; } - pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info); + pr_debug("sec '%s': found %u CO-RE relocations\n", sec_name, sec->num_info); for_each_btf_ext_rec(seg, sec, i, rec) { if (rec->insn_off % BPF_INSN_SZ) @@ -6181,7 +6225,7 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res); if (err) { - pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %s\n", + pr_warn("prog '%s': relo #%d: failed to patch insn #%d: %s\n", prog->name, i, insn_idx, errstr(err)); goto out; } @@ -6346,7 +6390,7 @@ static int create_jt_map(struct bpf_object *obj, struct bpf_program *prog, struc goto err_close; } if (sym_off + jt_size > obj->jumptables_data_sz) { - pr_warn("map '.jumptables': jumptables_data size is %zd, trying to access %d\n", + pr_warn("map '.jumptables': jumptables_data size is %zu, trying to access %u\n", obj->jumptables_data_sz, sym_off + jt_size); err = -EINVAL; goto err_close; @@ -6381,7 +6425,7 @@ static int create_jt_map(struct bpf_object *obj, struct bpf_program *prog, struc */ if (insn_off > UINT32_MAX) { pr_warn("map '.jumptables': invalid jump table value 0x%llx at offset %u\n", - (long long)jt[i], sym_off + i * jt_entry_size); + (unsigned long long)jt[i], sym_off + i * jt_entry_size); err = -EINVAL; goto err_close; } @@ -6517,7 +6561,7 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) } break; default: - pr_warn("prog '%s': relo #%d: bad relo type %d\n", + pr_warn("prog '%s': relo #%d: bad relo type %u\n", prog->name, i, relo->type); return -EINVAL; } @@ -6797,7 +6841,7 @@ bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, */ continue; if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) { - pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n", + pr_warn("prog '%s': unexpected relo for insn #%zu, type %u\n", prog->name, insn_idx, relo->type); return -LIBBPF_ERRNO__RELOC; } @@ -6944,7 +6988,7 @@ bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, * +-----------+------+------+ * * At this point, we relocate subA calls, then go one level up and finish with - * relocatin mainA calls. mainA is done. + * relocation mainA calls. mainA is done. * * For mainB process is similar but results in different order. We start with * mainB and skip subA and subB, as mainB never calls them (at least @@ -7587,7 +7631,7 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj, } name = elf_sym_str(obj, sym->st_name) ?: "<?>"; - pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n", + pr_debug(".maps relo #%d: for %zd value %zu rel->r_offset %zu name %u ('%s')\n", i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value, (size_t)rel->r_offset, sym->st_name, name); @@ -7678,7 +7722,7 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj, } map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog; - pr_debug(".maps relo #%d: map '%s' slot [%d] points to %s '%s'\n", + pr_debug(".maps relo #%d: map '%s' slot [%u] points to %s '%s'\n", i, map->name, moff, type, name); } @@ -7891,7 +7935,7 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, prog->attach_btf_id = btf_type_id; /* but by now libbpf common logic is not utilizing - * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because + * prog->attach_btf_obj_fd/prog->attach_btf_id anymore because * this callback is called after opts were populated by * libbpf, so this callback has to update opts explicitly here */ @@ -8738,7 +8782,7 @@ static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj, local_name = btf__name_by_offset(obj->btf, local_type->name_off); targ_name = btf__name_by_offset(btf, targ_type->name_off); - pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n", + pr_warn("extern (var ksym) '%s': incompatible types, expected [%u] %s %s, but kernel has [%u] %s %s\n", ext->name, local_type_id, btf_kind_str(local_type), local_name, targ_type_id, btf_kind_str(targ_type), targ_name); @@ -8915,7 +8959,7 @@ static int bpf_object__resolve_externs(struct bpf_object *obj, if (err) return err; pr_debug("extern (kcfg) '%s': set to 0x%llx\n", - ext->name, (long long)value); + ext->name, (unsigned long long)value); } else { pr_warn("extern '%s': unrecognized extern kind\n", ext->name); return -EINVAL; @@ -10494,7 +10538,7 @@ static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, moff = rel->r_offset - map->sec_offset; shdr_idx = sym->st_shndx; st_ops = map->st_ops; - pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %d (\'%s\')\n", + pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %u (\'%s\')\n", map->name, (long long)(rel->r_info >> 32), (long long)sym->st_value, @@ -10643,7 +10687,7 @@ static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd, int t memset(&info, 0, info_len); err = bpf_prog_get_info_by_fd(attach_prog_fd, &info, &info_len); if (err) { - pr_warn("failed bpf_prog_get_info_by_fd for FD %d: %s\n", + pr_warn("failed bpf_prog_get_info_by_fd for FD %u: %s\n", attach_prog_fd, errstr(err)); return err; } @@ -10656,7 +10700,7 @@ static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd, int t btf = btf_load_from_kernel(info.btf_id, NULL, token_fd); err = libbpf_get_error(btf); if (err) { - pr_warn("Failed to get BTF %d of the program: %s\n", info.btf_id, errstr(err)); + pr_warn("Failed to get BTF %u of the program: %s\n", info.btf_id, errstr(err)); goto out; } err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); @@ -10738,7 +10782,7 @@ static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attac } err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd, prog->obj->token_fd); if (err < 0) { - pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %s\n", + pr_warn("prog '%s': failed to find BPF program (FD %u) BTF ID for '%s': %s\n", prog->name, attach_prog_fd, attach_name, errstr(err)); return err; } @@ -10811,11 +10855,16 @@ static bool map_uses_real_name(const struct bpf_map *map) * such map's corresponding ELF section name as a map name. * This check distinguishes .data/.rodata from .data.* and .rodata.* * maps to know which name has to be returned to the user. + * Map name of the custom .percpu.* maps might be truncated to + * BPF_OBJ_NAME_LEN-1 chars in internal_map_name(). Hence, percpu data + * maps must use real name for their user-visible name. */ if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0) return true; if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0) return true; + if (map->libbpf_type == LIBBPF_MAP_PERCPU) + return true; return false; } @@ -10980,7 +11029,8 @@ int bpf_map__set_value_size(struct bpf_map *map, __u32 size) size_t mmap_old_sz, mmap_new_sz; int err; - if (map->def.type != BPF_MAP_TYPE_ARRAY) + if (map->def.type != BPF_MAP_TYPE_ARRAY && + map->def.type != BPF_MAP_TYPE_PERCPU_ARRAY) return libbpf_err(-EOPNOTSUPP); mmap_old_sz = bpf_map_mmap_sz(map); @@ -11233,7 +11283,7 @@ static int validate_map_op(const struct bpf_map *map, size_t key_sz, } if (value_sz != num_cpu * elem_sz) { - pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n", + pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zu\n", map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz); return -EINVAL; } @@ -11774,7 +11824,7 @@ static void gen_probe_legacy_event_name(char *buf, size_t buf_sz, static int index = 0; int i; - snprintf(buf, buf_sz, "libbpf_%u_%d_%s_0x%zx", getpid(), + snprintf(buf, buf_sz, "libbpf_%d_%d_%s_0x%zx", getpid(), __sync_fetch_and_add(&index, 1), name, offset); /* sanitize name in the probe name */ @@ -12924,8 +12974,8 @@ static long elf_find_func_offset_from_archive(const char *archive_path, const ch ret = elf_find_func_offset(elf, file_name, func_name); if (ret > 0) { pr_debug("elf: symbol address match for %s of %s in %s: 0x%x + 0x%lx = 0x%lx\n", - func_name, file_name, archive_path, entry.data_offset, ret, - ret + entry.data_offset); + func_name, file_name, archive_path, entry.data_offset, (unsigned long)ret, + (unsigned long)(ret + entry.data_offset)); ret += entry.data_offset; } elf_end(elf); @@ -12973,13 +13023,14 @@ static const char *arch_specific_lib_paths(void) /* Get full path to program/shared library. */ static int resolve_full_path(const char *file, char *result, size_t result_sz) { - const char *search_paths[3] = {}; + const char *search_paths[4] = {}; int i, perm; if (str_has_sfx(file, ".so") || strstr(file, ".so.")) { search_paths[0] = getenv("LD_LIBRARY_PATH"); search_paths[1] = "/usr/lib64:/usr/lib"; search_paths[2] = arch_specific_lib_paths(); + search_paths[3] = "/lib64:/lib"; perm = R_OK; } else { search_paths[0] = getenv("PATH"); @@ -14167,7 +14218,7 @@ perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, if (((void *)ehdr) + ehdr_size > base + mmap_size) { void *copy_start = ehdr; size_t len_first = base + mmap_size - copy_start; - size_t len_secnd = ehdr_size - len_first; + size_t len_second = ehdr_size - len_first; if (*copy_size < ehdr_size) { free(*copy_mem); @@ -14181,7 +14232,7 @@ perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, } memcpy(*copy_mem, copy_start, len_first); - memcpy(*copy_mem + len_first, base, len_secnd); + memcpy(*copy_mem + len_first, base, len_second); ehdr = *copy_mem; } @@ -14199,7 +14250,7 @@ struct perf_buffer; struct perf_buffer_params { struct perf_event_attr *attr; - /* if event_cb is specified, it takes precendence */ + /* if event_cb is specified, it takes precedence */ perf_buffer_event_fn event_cb; /* sample_cb and lost_cb are higher-level common-case callbacks */ perf_buffer_sample_fn sample_cb; @@ -14570,7 +14621,7 @@ perf_buffer__process_record(struct perf_event_header *e, void *ctx) break; } default: - pr_warn("unknown perf sample type %d\n", e->type); + pr_warn("unknown perf sample type %u\n", e->type); return LIBBPF_PERF_EVENT_ERROR; } return LIBBPF_PERF_EVENT_CONT; diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index b731df19ae69..08ab2ea881fb 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -460,5 +460,6 @@ LIBBPF_1.8.0 { global: bpf_program__attach_tracing_multi; bpf_program__clone; + btf__find_by_name_kind_own; btf__new_empty_opts; } LIBBPF_1.7.0; diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 04cd303fb5a8..4c46d34fc055 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -401,6 +401,8 @@ enum kern_feature_id { FEAT_BTF_LAYOUT, /* Kernel supports BPF syscall common attributes */ FEAT_BPF_SYSCALL_COMMON_ATTRS, + /* Kernel supports percpu data */ + FEAT_PERCPU_DATA, __FEAT_CNT, }; @@ -596,8 +598,6 @@ typedef int (*type_id_visit_fn)(__u32 *type_id, void *ctx); typedef int (*str_off_visit_fn)(__u32 *str_off, void *ctx); int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx); int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx); -__s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name, - __u32 kind); /* handle direct returned errors */ static inline int libbpf_err(int ret) @@ -768,7 +768,6 @@ int elf_resolve_pattern_offsets(const char *binary_path, const char *pattern, int probe_fd(int fd); #define SHA256_DIGEST_LENGTH 32 -#define SHA256_DWORD_SIZE SHA256_DIGEST_LENGTH / sizeof(__u64) void libbpf_sha256(const void *data, size_t len, __u8 out[SHA256_DIGEST_LENGTH]); int probe_sys_bpf_ext(void); diff --git a/tools/lib/bpf/nlattr.c b/tools/lib/bpf/nlattr.c index 06663f9ea581..007fe17d17b4 100644 --- a/tools/lib/bpf/nlattr.c +++ b/tools/lib/bpf/nlattr.c @@ -123,7 +123,7 @@ int libbpf_nla_parse(struct nlattr *tb[], int maxtype, struct nlattr *head, if (tb[type]) { pr_warn("Attribute of type %#x found multiple times in message, " - "previous attribute is being ignored.\n", type); + "previous attribute is being ignored.\n", (unsigned)type); } tb[type] = nla; diff --git a/tools/lib/bpf/relo_core.c b/tools/lib/bpf/relo_core.c index 6ae3f2a15ad0..8ad2715721cf 100644 --- a/tools/lib/bpf/relo_core.c +++ b/tools/lib/bpf/relo_core.c @@ -216,7 +216,7 @@ recur: goto recur; } default: - pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", + pr_warn("unexpected kind %s relocated, local [%u], target [%u]\n", btf_kind_str(local_type), local_id, targ_id); return 0; } @@ -384,7 +384,7 @@ int bpf_core_parse_spec(const char *prog_name, const struct btf *btf, return sz; spec->bit_offset += access_idx * sz * 8; } else { - pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n", + pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%u] of unexpected kind %s\n", prog_name, relo->type_id, spec_str, i, id, btf_kind_str(t)); return -EINVAL; } @@ -725,7 +725,7 @@ static int bpf_core_calc_field_relo(const char *prog_name, return -EINVAL; *val = sz; } else { - pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n", + pr_warn("prog '%s': relo %u at insn #%u can't be applied to array access\n", prog_name, relo->kind, relo->insn_off / 8); return -EINVAL; } @@ -747,7 +747,7 @@ static int bpf_core_calc_field_relo(const char *prog_name, while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) { if (byte_sz >= 8) { /* bitfield can't be read with 64-bit read */ - pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n", + pr_warn("prog '%s': relo %u at insn #%u can't be satisfied for bitfield\n", prog_name, relo->kind, relo->insn_off / 8); return -E2BIG; } @@ -971,7 +971,7 @@ done: err = 0; } else if (err == -EOPNOTSUPP) { /* EOPNOTSUPP means unknown/unsupported relocation */ - pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%d) at insn #%d\n", + pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%u) at insn #%u\n", prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind, relo->insn_off / 8); } @@ -1067,7 +1067,7 @@ poison: if (BPF_SRC(insn->code) != BPF_K) return -EINVAL; if (res->validate && insn->imm != orig_val) { - pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %llu -> %llu\n", + pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %d, exp %llu -> %llu\n", prog_name, relo_idx, insn_idx, insn->imm, (unsigned long long)orig_val, (unsigned long long)new_val); @@ -1083,7 +1083,7 @@ poison: case BPF_ST: case BPF_STX: if (res->validate && insn->off != orig_val) { - pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %llu -> %llu\n", + pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %d, exp %llu -> %llu\n", prog_name, relo_idx, insn_idx, insn->off, (unsigned long long)orig_val, (unsigned long long)new_val); return -EINVAL; @@ -1159,7 +1159,7 @@ poison: default: pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n", prog_name, relo_idx, insn_idx, insn->code, - insn->src_reg, insn->dst_reg, insn->off, insn->imm); + (unsigned)insn->src_reg, (unsigned)insn->dst_reg, (unsigned)insn->off, (unsigned)insn->imm); return -EINVAL; } @@ -1323,7 +1323,7 @@ int bpf_core_calc_relo_insn(const char *prog_name, const char *spec_str; spec_str = btf__name_by_offset(local_btf, relo->access_str_off); - pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n", + pr_warn("prog '%s': relo #%d: parsing [%u] %s %s + %s failed: %d\n", prog_name, relo_idx, local_id, btf_kind_str(local_type), str_is_empty(local_name) ? "<anon>" : local_name, spec_str ?: "<?>", err); @@ -1346,7 +1346,7 @@ int bpf_core_calc_relo_insn(const char *prog_name, /* libbpf doesn't support candidate search for anonymous types */ if (str_is_empty(local_name)) { - pr_warn("prog '%s': relo #%d: <%s> (%d) relocation doesn't support anonymous types\n", + pr_warn("prog '%s': relo #%d: <%s> (%u) relocation doesn't support anonymous types\n", prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind); return -EOPNOTSUPP; } @@ -1697,7 +1697,7 @@ recur: goto recur; } default: - pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", + pr_warn("unexpected kind %s relocated, local [%u], target [%u]\n", btf_kind_str(local_t), local_id, targ_id); return 0; } diff --git a/tools/lib/bpf/ringbuf.c b/tools/lib/bpf/ringbuf.c index 00ec4837a06d..c8fe57401a8b 100644 --- a/tools/lib/bpf/ringbuf.c +++ b/tools/lib/bpf/ringbuf.c @@ -244,7 +244,7 @@ static int64_t ringbuf_process_ring(struct ring *r, size_t n) do { got_new_data = false; prod_pos = smp_load_acquire(r->producer_pos); - while (cons_pos < prod_pos) { + while (prod_pos - cons_pos > 0) { len_ptr = r->data + (cons_pos & r->mask); len = smp_load_acquire(len_ptr); diff --git a/tools/lib/bpf/skel_internal.h b/tools/lib/bpf/skel_internal.h index 74503d358bc8..1f3f332dffbe 100644 --- a/tools/lib/bpf/skel_internal.h +++ b/tools/lib/bpf/skel_internal.h @@ -18,10 +18,6 @@ #include "bpf.h" #endif -#ifndef SHA256_DIGEST_LENGTH -#define SHA256_DIGEST_LENGTH 32 -#endif - #ifndef __NR_bpf # if defined(__mips__) && defined(_ABIO32) # define __NR_bpf 4355 @@ -135,8 +131,10 @@ static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) { if (addr != ~0ULL) kvfree(p); - /* When addr == ~0ULL the 'p' points to - * ((struct bpf_array *)map)->value. See skel_finalize_map_data. + /* + * When addr == ~0ULL the init buffer has already been released. + * For skel_finalize_map_data(), 'p' points to + * ((struct bpf_array *)map)->value. */ } @@ -174,6 +172,15 @@ out: return addr; } +static inline int skel_protect_map_data(void *p, __u64 *init_val, size_t sz) +{ + (void)sz; + + kvfree(p); + *init_val = ~0ULL; + return 0; +} + #else static inline void *skel_alloc(size_t size) @@ -212,6 +219,15 @@ static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int return NULL; return addr; } + +static inline int skel_protect_map_data(void *p, __u64 *init_val, size_t sz) +{ + (void)init_val; + + if (mprotect(p, sz, PROT_READ)) + return -errno; + return 0; +} #endif static inline int skel_closenz(int fd) @@ -320,25 +336,6 @@ static inline int skel_link_create(int prog_fd, int target_fd, return skel_sys_bpf(BPF_LINK_CREATE, &attr, attr_sz); } -static inline int skel_obj_get_info_by_fd(int fd) -{ - const size_t attr_sz = offsetofend(union bpf_attr, info); - __u8 sha[SHA256_DIGEST_LENGTH]; - struct bpf_map_info info; - __u32 info_len = sizeof(info); - union bpf_attr attr; - - memset(&info, 0, sizeof(info)); - info.hash = (long) &sha; - info.hash_size = SHA256_DIGEST_LENGTH; - - memset(&attr, 0, attr_sz); - attr.info.bpf_fd = fd; - attr.info.info = (long) &info; - attr.info.info_len = info_len; - return skel_sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, attr_sz); -} - static inline int skel_map_freeze(int fd) { const size_t attr_sz = offsetofend(union bpf_attr, map_fd); @@ -384,12 +381,6 @@ static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) set_err; goto out; } - err = skel_obj_get_info_by_fd(map_fd); - if (err < 0) { - opts->errstr = "failed to fetch obj info"; - set_err; - goto out; - } #endif memset(&attr, 0, prog_load_attr_sz); @@ -400,6 +391,8 @@ static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) #ifndef __KERNEL__ attr.signature = (long) opts->signature; attr.signature_size = opts->signature_sz; + if (opts->signature) + attr.fd_array_cnt = 1; #else if (opts->signature || opts->signature_sz) pr_warn("signatures are not supported from bpf_preload\n"); diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c index d2ecd3daab96..2e56e3ab5b6c 100644 --- a/tools/lib/bpf/usdt.c +++ b/tools/lib/bpf/usdt.c @@ -327,7 +327,7 @@ static int sanity_check_usdt_elf(Elf *elf, const char *path) int endianness; if (elf_kind(elf) != ELF_K_ELF) { - pr_warn("usdt: unrecognized ELF kind %d for '%s'\n", elf_kind(elf), path); + pr_warn("usdt: unrecognized ELF kind %u for '%s'\n", elf_kind(elf), path); return -EBADF; } @@ -438,8 +438,9 @@ static int parse_elf_segs(Elf *elf, const char *path, struct elf_seg **segs, siz } pr_debug("usdt: discovered PHDR #%d in '%s': vaddr 0x%lx memsz 0x%lx offset 0x%lx type 0x%lx flags 0x%lx\n", - i, path, (long)phdr.p_vaddr, (long)phdr.p_memsz, (long)phdr.p_offset, - (long)phdr.p_type, (long)phdr.p_flags); + i, path, + (unsigned long)phdr.p_vaddr, (unsigned long)phdr.p_memsz, (unsigned long)phdr.p_offset, + (unsigned long)phdr.p_type, (unsigned long)phdr.p_flags); if (phdr.p_type != PT_LOAD) continue; @@ -719,14 +720,14 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, if (!seg) { err = -ESRCH; pr_warn("usdt: failed to find ELF program segment for '%s:%s' in '%s' at IP 0x%lx\n", - usdt_provider, usdt_name, path, usdt_abs_ip); + usdt_provider, usdt_name, path, (unsigned long)usdt_abs_ip); goto err_out; } if (!seg->is_exec) { err = -ESRCH; pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx) for '%s:%s' at IP 0x%lx is not executable\n", - path, seg->start, seg->end, usdt_provider, usdt_name, - usdt_abs_ip); + path, (unsigned long)seg->start, (unsigned long)seg->end, usdt_provider, usdt_name, + (unsigned long)usdt_abs_ip); goto err_out; } /* translate from virtual address to file offset */ @@ -766,7 +767,7 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, if (!seg) { err = -ESRCH; pr_warn("usdt: failed to find shared lib memory segment for '%s:%s' in '%s' at relative IP 0x%lx\n", - usdt_provider, usdt_name, path, usdt_rel_ip); + usdt_provider, usdt_name, path, (unsigned long)usdt_rel_ip); goto err_out; } @@ -775,8 +776,10 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, pr_debug("usdt: probe for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved abs_ip 0x%lx rel_ip 0x%lx) args '%s' in segment [0x%lx, 0x%lx) at offset 0x%lx\n", usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", path, - note.loc_addr, note.base_addr, usdt_abs_ip, usdt_rel_ip, note.args, - seg ? seg->start : 0, seg ? seg->end : 0, seg ? seg->offset : 0); + (unsigned long)note.loc_addr, (unsigned long)note.base_addr, + (unsigned long)usdt_abs_ip, (unsigned long)usdt_rel_ip, note.args, + (unsigned long)(seg ? seg->start : 0), (unsigned long)(seg ? seg->end : 0), + (unsigned long)(seg ? seg->offset : 0)); /* Adjust semaphore address to be a file offset */ if (note.sema_addr) { @@ -791,14 +794,14 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, if (!seg) { err = -ESRCH; pr_warn("usdt: failed to find ELF loadable segment with semaphore of '%s:%s' in '%s' at 0x%lx\n", - usdt_provider, usdt_name, path, note.sema_addr); + usdt_provider, usdt_name, path, (unsigned long)note.sema_addr); goto err_out; } if (seg->is_exec) { err = -ESRCH; pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx] for semaphore of '%s:%s' at 0x%lx is executable\n", - path, seg->start, seg->end, usdt_provider, usdt_name, - note.sema_addr); + path, (unsigned long)seg->start, (unsigned long)seg->end, usdt_provider, usdt_name, + (unsigned long)note.sema_addr); goto err_out; } @@ -806,8 +809,8 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, pr_debug("usdt: sema for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved 0x%lx) in segment [0x%lx, 0x%lx] at offset 0x%lx\n", usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", - path, note.sema_addr, note.base_addr, usdt_sema_off, - seg->start, seg->end, seg->offset); + path, (unsigned long)note.sema_addr, (unsigned long)note.base_addr, (unsigned long)usdt_sema_off, + (unsigned long)seg->start, (unsigned long)seg->end, (unsigned long)seg->offset); } /* Record adjusted addresses and offsets and parse USDT spec */ @@ -1117,7 +1120,7 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct spec_id, usdt_provider, usdt_name, path); } else { pr_warn("usdt: failed to map IP 0x%lx to spec #%d for '%s:%s' in '%s': %s\n", - target->abs_ip, spec_id, usdt_provider, usdt_name, + (unsigned long)target->abs_ip, spec_id, usdt_provider, usdt_name, path, errstr(err)); } goto err_out; diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h index 87f15f296234..3f74d522f7e7 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -84,7 +84,7 @@ bool scx_bpf_dispatch_vtime_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, st * * Compat macro will be dropped on v6.19 release. */ -int bpf_cpumask_populate(struct cpumask *dst, void *src, size_t src__sz) __ksym __weak; +int bpf_cpumask_populate(struct bpf_cpumask *dst, void *src, size_t src__sz) __ksym __weak; #define __COMPAT_bpf_cpumask_populate(cpumask, src, size__sz) \ (bpf_ksym_exists(bpf_cpumask_populate) ? \ diff --git a/tools/testing/selftests/bpf/DENYLIST.riscv64 b/tools/testing/selftests/bpf/DENYLIST.riscv64 index 4fc4dfdde293..ca1beae7fe8f 100644 --- a/tools/testing/selftests/bpf/DENYLIST.riscv64 +++ b/tools/testing/selftests/bpf/DENYLIST.riscv64 @@ -1,3 +1,2 @@ # riscv64 deny list for BPF CI and local vmtest exceptions # JIT does not support exceptions -tailcalls/tailcall_bpf2bpf* # JIT does not support mixing bpf2bpf and tailcalls diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index b642ee489ea6..2b2f93dec474 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -324,8 +324,6 @@ TRUNNER_BPFTOOL := $(DEFAULT_BPFTOOL) USE_BOOTSTRAP := "bootstrap/" endif -TEST_GEN_PROGS_EXTENDED += $(TRUNNER_BPFTOOL) - $(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED): $(BPFOBJ) TESTING_HELPERS := $(OUTPUT)/testing_helpers.o @@ -466,7 +464,7 @@ endif CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH)) BPF_CFLAGS = -g -Wall -Werror -D__TARGET_ARCH_$(SRCARCH) $(MENDIAN) \ -I$(INCLUDE_DIR) -I$(CURDIR) -I$(APIDIR) \ - -I$(CURDIR)/libarena/include \ + -I$(TOOLSINCDIR) -I$(CURDIR)/libarena/include \ -I$(abspath $(OUTPUT)/../usr/include) \ -std=gnu11 \ -fno-strict-aliasing \ @@ -533,7 +531,7 @@ LSKELS_SIGNED := fentry_test.c fexit_test.c atomics.c # Generate both light skeleton and libbpf skeleton for these LSKELS_EXTRA := test_ksyms_module.c test_ksyms_weak.c kfunc_call_test.c \ - kfunc_call_test_subprog.c + kfunc_call_test_subprog.c test_global_percpu_data.c SKEL_BLACKLIST += $$(LSKELS) $$(LSKELS_SIGNED) test_static_linked.skel.h-deps := test_static_linked1.bpf.o test_static_linked2.bpf.o @@ -934,17 +932,26 @@ $(OUTPUT)/test_verifier: test_verifier.c verifier/tests.h $(BPFOBJ) | $(OUTPUT) $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@ -# Include find_bit.c to compile xskxceiver. -EXTRA_SRC := $(TOOLSDIR)/lib/find_bit.c prog_tests/test_xsk.c prog_tests/test_xsk.h -$(OUTPUT)/xskxceiver: $(EXTRA_SRC) xskxceiver.c xskxceiver.h $(OUTPUT)/network_helpers.o $(OUTPUT)/xsk.o $(OUTPUT)/xsk_xdp_progs.skel.h $(BPFOBJ) | $(OUTPUT) +# Keep xskxceiver independent from test_progs object dependencies. +$(OUTPUT)/xskxceiver: xskxceiver.c xsk.c network_helpers.c \ + $(TOOLSDIR)/lib/find_bit.c prog_tests/test_xsk.c \ + xskxceiver.h xsk.h network_helpers.h \ + prog_tests/test_xsk.h test_progs.h bpf_util.h \ + $(OUTPUT)/xsk_xdp_progs.skel.h $(BPFOBJ) | $(OUTPUT) $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@ -$(OUTPUT)/xdp_hw_metadata: xdp_hw_metadata.c $(OUTPUT)/network_helpers.o $(OUTPUT)/xsk.o $(OUTPUT)/xdp_hw_metadata.skel.h | $(OUTPUT) +$(OUTPUT)/xdp_hw_metadata: xdp_hw_metadata.c xsk.c network_helpers.c \ + $(TOOLSDIR)/lib/find_bit.c xdp_metadata.h \ + xsk.h network_helpers.h test_progs.h bpf_util.h \ + $(OUTPUT)/xdp_hw_metadata.skel.h $(BPFOBJ) | $(OUTPUT) $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@ -$(OUTPUT)/xdp_features: xdp_features.c $(OUTPUT)/network_helpers.o $(OUTPUT)/xdp_features.skel.h | $(OUTPUT) +$(OUTPUT)/xdp_features: xdp_features.c network_helpers.c xdp_features.h \ + network_helpers.h \ + test_progs.h bpf_util.h $(OUTPUT)/xdp_features.skel.h \ + $(BPFOBJ) | $(OUTPUT) $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@ @@ -967,6 +974,7 @@ $(OUTPUT)/bench_ringbufs.o: $(OUTPUT)/ringbuf_bench.skel.h \ $(OUTPUT)/perfbuf_bench.skel.h $(OUTPUT)/bench_bloom_filter_map.o: $(OUTPUT)/bloom_filter_bench.skel.h $(OUTPUT)/bench_bpf_loop.o: $(OUTPUT)/bpf_loop_bench.skel.h +$(OUTPUT)/bench_bpf_for.o: $(OUTPUT)/bpf_for_bench.skel.h $(OUTPUT)/bench_strncmp.o: $(OUTPUT)/strncmp_bench.skel.h $(OUTPUT)/bench_bpf_hashmap_full_update.o: $(OUTPUT)/bpf_hashmap_full_update_bench.skel.h $(OUTPUT)/bench_local_storage.o: $(OUTPUT)/local_storage_bench.skel.h @@ -992,6 +1000,7 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \ $(OUTPUT)/bench_ringbufs.o \ $(OUTPUT)/bench_bloom_filter_map.o \ $(OUTPUT)/bench_bpf_loop.o \ + $(OUTPUT)/bench_bpf_for.o \ $(OUTPUT)/bench_strncmp.o \ $(OUTPUT)/bench_bpf_hashmap_full_update.o \ $(OUTPUT)/bench_local_storage.o \ @@ -1055,10 +1064,13 @@ endif DEFAULT_INSTALL_RULE := $(INSTALL_RULE) override define INSTALL_RULE $(DEFAULT_INSTALL_RULE) + @mkdir -p $(INSTALL_PATH)/tools/sbin + @rsync -a $(if $(PERMISSIVE),--ignore-missing-args) $(TRUNNER_BPFTOOL) $(INSTALL_PATH)/tools/sbin/ + @rsync -a $(if $(PERMISSIVE),--ignore-missing-args) $(OUTPUT)/*.BTF $(INSTALL_PATH)/ @for DIR in $(TEST_INST_SUBDIRS); do \ mkdir -p $(INSTALL_PATH)/$$DIR; \ rsync -a $(if $(PERMISSIVE),--ignore-missing-args) \ - $(OUTPUT)/$$DIR/*.bpf.o \ + $(OUTPUT)/$$DIR/*.bpf.o $(OUTPUT)/$$DIR/*.BTF \ $(INSTALL_PATH)/$$DIR; \ done endef diff --git a/tools/testing/selftests/bpf/README.rst b/tools/testing/selftests/bpf/README.rst index 37164322a102..07c834433b38 100644 --- a/tools/testing/selftests/bpf/README.rst +++ b/tools/testing/selftests/bpf/README.rst @@ -107,12 +107,12 @@ Docker container and local rootfs image. The overall steps are as follows: tools/testing/selftests/bpf/vmtest.sh \ -l <path of local rootfs image> -- \ ./test_progs -d \ - \"$(cat tools/testing/selftests/bpf/DENYLIST.riscv64 \ + "$(cat tools/testing/selftests/bpf/DENYLIST.riscv64 \ | cut -d'#' -f1 \ | sed -e 's/^[[:space:]]*//' \ -e 's/[[:space:]]*$//' \ | tr -s '\n' ',' \ - )\" + )" Link: https://github.com/pulehui/riscv-bpf-vmtest.git [0] Link: https://github.com/libbpf/ci/blob/main/rootfs/mkrootfs_debian.sh [1] diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c index c4a3a6b3eb83..465233ea5f79 100644 --- a/tools/testing/selftests/bpf/bench.c +++ b/tools/testing/selftests/bpf/bench.c @@ -276,6 +276,7 @@ static const struct argp_option opts[] = { extern struct argp bench_ringbufs_argp; extern struct argp bench_bloom_map_argp; extern struct argp bench_bpf_loop_argp; +extern struct argp bench_bpf_for_argp; extern struct argp bench_local_storage_argp; extern struct argp bench_local_storage_rcu_tasks_trace_argp; extern struct argp bench_strncmp_argp; @@ -292,6 +293,7 @@ static const struct argp_child bench_parsers[] = { { &bench_ringbufs_argp, 0, "Ring buffers benchmark", 0 }, { &bench_bloom_map_argp, 0, "Bloom filter map benchmark", 0 }, { &bench_bpf_loop_argp, 0, "bpf_loop helper benchmark", 0 }, + { &bench_bpf_for_argp, 0, "bpf_for loop benchmark", 0 }, { &bench_local_storage_argp, 0, "local_storage benchmark", 0 }, { &bench_strncmp_argp, 0, "bpf_strncmp helper benchmark", 0 }, { &bench_local_storage_rcu_tasks_trace_argp, 0, @@ -557,6 +559,7 @@ extern const struct bench bench_bloom_false_positive; extern const struct bench bench_hashmap_without_bloom; extern const struct bench bench_hashmap_with_bloom; extern const struct bench bench_bpf_loop; +extern const struct bench bench_bpf_for; extern const struct bench bench_strncmp_no_helper; extern const struct bench bench_strncmp_helper; extern const struct bench bench_bpf_hashmap_full_update; @@ -640,6 +643,7 @@ static const struct bench *benchs[] = { &bench_hashmap_without_bloom, &bench_hashmap_with_bloom, &bench_bpf_loop, + &bench_bpf_for, &bench_strncmp_no_helper, &bench_strncmp_helper, &bench_bpf_hashmap_full_update, diff --git a/tools/testing/selftests/bpf/benchs/bench_bpf_for.c b/tools/testing/selftests/bpf/benchs/bench_bpf_for.c new file mode 100644 index 000000000000..730c51ad2dec --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/bench_bpf_for.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <argp.h> +#include "bench.h" +#include "bpf_for_bench.skel.h" + +/* BPF triggering benchmarks */ +static struct ctx { + struct bpf_for_bench *skel; +} ctx; + +static struct { + __u32 nr_loops; +} args = { + /* + * Default to a large loop count so the per-iteration bpf_iter_num_next() cost dominates + * the one-time bpf_iter_num_new()/destroy() setup and teardown. + */ + .nr_loops = 1000, +}; + +enum { + ARG_NR_LOOPS = 4000, +}; + +static const struct argp_option opts[] = { + { "nr_loops", ARG_NR_LOOPS, "nr_loops", 0, + "Set number of iterations for the bpf_for() loop"}, + {}, +}; + +static error_t parse_arg(int key, char *arg, struct argp_state *state) +{ + switch (key) { + case ARG_NR_LOOPS: + args.nr_loops = strtol(arg, NULL, 10); + break; + default: + return ARGP_ERR_UNKNOWN; + } + + return 0; +} + +/* exported into benchmark runner */ +const struct argp bench_bpf_for_argp = { + .options = opts, + .parser = parse_arg, +}; + +static void validate(void) +{ + if (env.consumer_cnt != 0) { + fprintf(stderr, "benchmark doesn't support consumer!\n"); + exit(1); + } +} + +static void *producer(void *input) +{ + while (true) + /* trigger the bpf program */ + syscall(__NR_getpgid); + + return NULL; +} + +static void measure(struct bench_res *res) +{ + res->hits = atomic_swap(&ctx.skel->bss->hits, 0); +} + +static void setup(void) +{ + struct bpf_link *link; + + setup_libbpf(); + + ctx.skel = bpf_for_bench__open_and_load(); + if (!ctx.skel) { + fprintf(stderr, "failed to open skeleton\n"); + exit(1); + } + + link = bpf_program__attach(ctx.skel->progs.benchmark); + if (!link) { + fprintf(stderr, "failed to attach program!\n"); + exit(1); + } + + ctx.skel->bss->nr_loops = args.nr_loops; +} + +const struct bench bench_bpf_for = { + .name = "bpf-for", + .argp = &bench_bpf_for_argp, + .validate = validate, + .setup = setup, + .producer_thread = producer, + .measure = measure, + .report_progress = ops_report_progress, + .report_final = ops_report_final, +}; diff --git a/tools/testing/selftests/bpf/benchs/run_bench_bpf_for.sh b/tools/testing/selftests/bpf/benchs/run_bench_bpf_for.sh new file mode 100755 index 000000000000..7da6453920da --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/run_bench_bpf_for.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +source ./benchs/run_common.sh + +set -eufo pipefail + +for t in 1 4 8 12 16; do +for i in 10 100 500 1000 5000 10000 50000 100000 500000 1000000; do +subtitle "nr_loops: $i, nr_threads: $t" + summarize_ops "bpf_for: " \ + "$($RUN_BENCH -p $t --nr_loops $i bpf-for)" + printf "\n" +done +done diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing/selftests/bpf/bpf_experimental.h index f30a4ac939db..2893bf06ff25 100644 --- a/tools/testing/selftests/bpf/bpf_experimental.h +++ b/tools/testing/selftests/bpf/bpf_experimental.h @@ -431,6 +431,8 @@ static inline int get_preempt_count(void) return bpf_get_lowcore()->preempt_count; #elif defined(bpf_target_loongarch) return bpf_get_current_task_btf()->thread_info.preempt_count; +#elif defined(bpf_target_riscv) + return bpf_get_current_task_btf()->thread_info.preempt_count; #endif return 0; } @@ -442,6 +444,7 @@ static inline int get_preempt_count(void) * * powerpc64 * * s390x * * loongarch + * * riscv */ static inline int bpf_in_interrupt(void) { @@ -464,6 +467,7 @@ static inline int bpf_in_interrupt(void) * * powerpc64 * * s390x * * loongarch + * * riscv */ static inline int bpf_in_nmi(void) { @@ -477,6 +481,7 @@ static inline int bpf_in_nmi(void) * * powerpc64 * * s390x * * loongarch + * * riscv */ static inline int bpf_in_hardirq(void) { @@ -490,6 +495,7 @@ static inline int bpf_in_hardirq(void) * * powerpc64 * * s390x * * loongarch + * * riscv */ static inline int bpf_in_serving_softirq(void) { @@ -511,6 +517,7 @@ static inline int bpf_in_serving_softirq(void) * * powerpc64 * * s390x * * loongarch + * * riscv */ static inline int bpf_in_task(void) { diff --git a/tools/testing/selftests/bpf/config.aarch64 b/tools/testing/selftests/bpf/config.aarch64 index 7efad36ceb26..fc85257701dc 100644 --- a/tools/testing/selftests/bpf/config.aarch64 +++ b/tools/testing/selftests/bpf/config.aarch64 @@ -71,7 +71,6 @@ CONFIG_INPUT_EVDEV=y CONFIG_IP_ADVANCED_ROUTER=y CONFIG_IP_MULTICAST=y CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_IPVLAN=y CONFIG_JUMP_LABEL=y CONFIG_KERNEL_UNCOMPRESSED=y diff --git a/tools/testing/selftests/bpf/config.ppc64el b/tools/testing/selftests/bpf/config.ppc64el index b53afb5e0b71..5685fa4ee82b 100644 --- a/tools/testing/selftests/bpf/config.ppc64el +++ b/tools/testing/selftests/bpf/config.ppc64el @@ -39,7 +39,6 @@ CONFIG_INET=y CONFIG_IP_ADVANCED_ROUTER=y CONFIG_IP_MULTICAST=y CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_JUMP_LABEL=y CONFIG_KALLSYMS_ALL=y CONFIG_KPROBES=y diff --git a/tools/testing/selftests/bpf/config.riscv64 b/tools/testing/selftests/bpf/config.riscv64 index 7bee24a79a71..655cb05a7689 100644 --- a/tools/testing/selftests/bpf/config.riscv64 +++ b/tools/testing/selftests/bpf/config.riscv64 @@ -30,7 +30,6 @@ CONFIG_HARDLOCKUP_DETECTOR=y CONFIG_HIGH_RES_TIMERS=y CONFIG_HUGETLBFS=y CONFIG_INET=y -CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_IP_ADVANCED_ROUTER=y CONFIG_IP_MULTICAST=y CONFIG_IP_MULTIPLE_TABLES=y diff --git a/tools/testing/selftests/bpf/config.s390x b/tools/testing/selftests/bpf/config.s390x index db61878148e4..755d1cfcd9e0 100644 --- a/tools/testing/selftests/bpf/config.s390x +++ b/tools/testing/selftests/bpf/config.s390x @@ -56,7 +56,6 @@ CONFIG_INET=y CONFIG_IP_ADVANCED_ROUTER=y CONFIG_IP_MULTICAST=y CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_IPVLAN=y CONFIG_JUMP_LABEL=y CONFIG_KERNEL_UNCOMPRESSED=y diff --git a/tools/testing/selftests/bpf/config.x86_64 b/tools/testing/selftests/bpf/config.x86_64 index 42ad817b00ae..523e0d29bbd4 100644 --- a/tools/testing/selftests/bpf/config.x86_64 +++ b/tools/testing/selftests/bpf/config.x86_64 @@ -114,7 +114,6 @@ CONFIG_IP_ROUTE_VERBOSE=y CONFIG_IPV6_MIP6=y CONFIG_IPV6_ROUTE_INFO=y CONFIG_IPV6_ROUTER_PREF=y -CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_IPV6_SUBTREES=y CONFIG_IRQ_POLL=y CONFIG_JUMP_LABEL=y diff --git a/tools/testing/selftests/bpf/disasm_helpers.c b/tools/testing/selftests/bpf/disasm_helpers.c index f529f1c8c171..30221352568d 100644 --- a/tools/testing/selftests/bpf/disasm_helpers.c +++ b/tools/testing/selftests/bpf/disasm_helpers.c @@ -55,10 +55,9 @@ struct bpf_insn *disasm_insn(struct bpf_insn *insn, char *buf, size_t buf_sz) * for each instruction (FF stands for instruction `code` byte). * Remove the prefix inplace, and also simplify call instructions. * E.g.: "(85) call foo#10" -> "call foo". - * Also remove newline in the end (the 'max(strlen(buf) - 1, 0)' thing). */ pfx_end = buf + 5; - sfx_start = buf + max((int)strlen(buf) - 1, 0); + sfx_start = buf + (int)strlen(buf); if (strncmp(pfx_end, "call ", 5) == 0 && (tmp = strrchr(buf, '#'))) sfx_start = tmp; len = sfx_start - pfx_end; diff --git a/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h b/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h index ae6b72d15bb6..71d9db610263 100644 --- a/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h @@ -103,12 +103,7 @@ struct arena_qnode { #define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET) #define _Q_PENDING_VAL (1U << _Q_PENDING_OFFSET) -/* - * The qnodes are marked __weak so we can define them in the header - * while still ensuring all compilation units use the same struct - * instance. - */ -struct arena_qnode __weak __arena __hidden qnodes[_Q_MAX_CPUS][_Q_MAX_NODES]; +extern struct arena_qnode __arena __hidden qnodes[_Q_MAX_CPUS][_Q_MAX_NODES]; static inline u32 encode_tail(int cpu, int idx) { diff --git a/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h b/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h index b7b230431929..43c306e17f19 100644 --- a/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h @@ -86,6 +86,25 @@ extern bool CONFIG_X86_64 __kconfig __weak; /* Control dependency provides LOAD->STORE, provide LOAD->LOAD */ #define smp_acquire__after_ctrl_dep() ({ smp_rmb(); }) +#if defined(__BPF_FEATURE_LOAD_ACQ_STORE_REL) +/* + * Clang advertises this feature when it can lower acquire/release atomic + * builtins to BPF_LOAD_ACQ/BPF_STORE_REL. Older compilers keep using the + * barrier-based fallback below. The generated instructions require kernel + * verifier/JIT support added in Linux 6.15; compile for an older BPF CPU to + * keep using the fallback when targeting older kernels. + */ +#define smp_load_acquire(p) \ + ({ \ + __unqual_typeof(*(p)) ___p1 = __atomic_load_n((p), __ATOMIC_ACQUIRE); \ + (typeof(*(p)))___p1; \ + }) + +#define smp_store_release(p, val) \ + ({ \ + __atomic_store_n((p), (val), __ATOMIC_RELEASE); \ + }) +#else #define smp_load_acquire(p) \ ({ \ __unqual_typeof(*(p)) __v = READ_ONCE(*(p)); \ @@ -102,6 +121,7 @@ extern bool CONFIG_X86_64 __kconfig __weak; barrier(); \ WRITE_ONCE(*(p), val); \ }) +#endif #define smp_cond_load_relaxed_label(p, cond_expr, label) \ ({ \ diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h new file mode 100644 index 000000000000..e2431ea6fdd6 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h @@ -0,0 +1,34 @@ +#pragma once + +#define BITS_PER_BYTE 8 +#define BYTES_TO_BITS(nb) ((nb) * BITS_PER_BYTE) + +#define BITS_PER_LONG_LONG (sizeof(long long) * BITS_PER_BYTE) +#define BITS_TO_LONG_LONGS(nr) (((nr) + BITS_PER_LONG_LONG - 1) / BITS_PER_LONG_LONG) +#define BIT_MASK(nr) (1ULL << ((nr) % BITS_PER_LONG_LONG)) +#define BIT_WORD(nr) ((nr) / BITS_PER_LONG_LONG) + +struct arena_bitmap { + u64 bits[0]; +}; + +struct arena_bitmap __arena *bmp_alloc(size_t bits); +void bmp_free(struct arena_bitmap __arena *bmp); + +void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp); +void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); +void bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp); +void bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); +bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp); +bool bmp_test_and_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); +bool bmp_test_and_set_bit(u32 bit, struct arena_bitmap __arena *bmp); + +void bmp_clear(size_t bits, struct arena_bitmap __arena *bmp); +void bmp_and(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src1, struct arena_bitmap __arena *src2); +void bmp_or(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src1, struct arena_bitmap __arena *src2); +bool bmp_empty(size_t bits, struct arena_bitmap __arena *bmp); +void bmp_copy(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src); + +bool bmp_intersects(size_t bits, struct arena_bitmap __arena *arg1, struct arena_bitmap __arena *arg2); +bool bmp_subset(size_t bits, struct arena_bitmap __arena *big, struct arena_bitmap __arena *small); +void bmp_print(size_t bits, struct arena_bitmap __arena *bmp); diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/common.h b/tools/testing/selftests/bpf/libarena/include/libarena/common.h index a3eb1641ac36..931ace9a49e2 100644 --- a/tools/testing/selftests/bpf/libarena/include/libarena/common.h +++ b/tools/testing/selftests/bpf/libarena/include/libarena/common.h @@ -43,7 +43,7 @@ struct { * imprecise. To force the variable to be imprecise, initialize it with * the opaque volatile variable 0 instead of the constant 0. */ -extern const volatile u32 zero; +volatile u32 zero __weak; extern volatile u64 asan_violated; int arena_fls(__u64 word); diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h b/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h index fc27a4bcf5d7..b6676dd67bc0 100644 --- a/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h +++ b/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h @@ -115,7 +115,7 @@ static inline int libarena_asan_init(int arena_asan_init_fd, { LIBBPF_OPTS(bpf_test_run_opts, opts); struct asan_init_args args; - u64 globals_pages; + u64 globals_pages = 0; int ret; ret = libarena_get_globals_pages(arena_asan_init_fd, diff --git a/tools/testing/selftests/bpf/libarena/selftests/st_asan_buddy.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_asan_buddy.bpf.c index 686caba2c643..3266a28f53d7 100644 --- a/tools/testing/selftests/bpf/libarena/selftests/st_asan_buddy.bpf.c +++ b/tools/testing/selftests/bpf/libarena/selftests/test_asan_buddy.bpf.c @@ -12,7 +12,7 @@ extern struct buddy __arena buddy; #ifdef BPF_ARENA_ASAN -#include "st_asan_common.h" +#include "test_asan_common.h" static __always_inline int asan_test_buddy_oob_single(size_t alloc_size) { @@ -154,7 +154,8 @@ __weak int asan_test_buddy_oob(void) size_t sizes[] = { 7, 8, 17, 18, 64, 256, 317, 512, 1024, }; - int ret, i; + int ret; + u32 i; ret = buddy_init(&buddy); if (ret) { @@ -163,6 +164,7 @@ __weak int asan_test_buddy_oob(void) } for (i = zero; i < sizeof(sizes) / sizeof(sizes[0]) && can_loop; i++) { + barrier_var(i); ret = asan_test_buddy_oob_single(sizes[i]); if (ret) { arena_stdout("%s:%d Failed for size %lu", __func__, @@ -190,7 +192,8 @@ __stderr("Call trace:\n" __weak int asan_test_buddy_uaf(void) { size_t sizes[] = { 16, 32, 64, 128, 256, 512, 1024, 16384 }; - int ret, i; + int ret; + u32 i; ret = buddy_init(&buddy); if (ret) { @@ -199,6 +202,7 @@ __weak int asan_test_buddy_uaf(void) } for (i = zero; i < sizeof(sizes) / sizeof(sizes[0]) && can_loop; i++) { + barrier_var(i); ret = asan_test_buddy_uaf_single(sizes[i]); if (ret) { arena_stdout("%s:%d Failed for size %lu", __func__, diff --git a/tools/testing/selftests/bpf/libarena/selftests/st_asan_common.h b/tools/testing/selftests/bpf/libarena/selftests/test_asan_common.h index 34a7918cb4cf..34a7918cb4cf 100644 --- a/tools/testing/selftests/bpf/libarena/selftests/st_asan_common.h +++ b/tools/testing/selftests/bpf/libarena/selftests/test_asan_common.h diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c new file mode 100644 index 000000000000..76319a529f02 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c @@ -0,0 +1,394 @@ +#include <libarena/common.h> + +#include <libarena/asan.h> +#include <libarena/bitmap.h> + +#define TEST_BITS (2 * BITS_PER_LONG_LONG) +#define TEST_WORDS BITS_TO_LONG_LONGS(TEST_BITS) +#define MID_BIT (BITS_PER_LONG_LONG + 1) +#define LAST_BIT (TEST_BITS - 1) + +static void test_bmp_setall(struct arena_bitmap __arena *bmp) +{ + volatile u32 i; + + for (i = zero; i < TEST_WORDS && can_loop; i++) + bmp->bits[i] = ~0ULL; +} + +SEC("syscall") +__weak int test_bitmap_alloc_free(void) +{ + struct arena_bitmap __arena *bmp; + + bmp = bmp_alloc(TEST_BITS); + if (!bmp) + return -ENOMEM; + + if (!bmp_empty(TEST_BITS, bmp)) + goto err; + + __bmp_set_bit(LAST_BIT, bmp); + if (!bmp_test_bit(LAST_BIT, bmp)) + goto err; + + __bmp_clear_bit(LAST_BIT, bmp); + if (bmp_test_bit(LAST_BIT, bmp)) + goto err; + + bmp_free(bmp); + return 0; + +err: + bmp_free(bmp); + return -EINVAL; +} + +SEC("syscall") +__weak int test_bitmap_bit_ops(void) +{ + struct arena_bitmap __arena *bmp; + + bmp = bmp_alloc(TEST_BITS); + if (!bmp) + return -ENOMEM; + + __bmp_set_bit(0, bmp); + if (!bmp_test_bit(0, bmp)) + goto err; + + __bmp_set_bit(MID_BIT, bmp); + if (!bmp_test_bit(MID_BIT, bmp)) + goto err; + + __bmp_set_bit(LAST_BIT, bmp); + if (!bmp_test_bit(LAST_BIT, bmp)) + goto err; + + if (bmp_test_bit(MID_BIT - 1, bmp)) + goto err; + + __bmp_clear_bit(MID_BIT, bmp); + if (bmp_test_bit(MID_BIT, bmp)) + goto err; + + if (!bmp_test_bit(0, bmp)) + goto err; + + if (!bmp_test_bit(LAST_BIT, bmp)) + goto err; + + __bmp_clear_bit(0, bmp); + __bmp_clear_bit(LAST_BIT, bmp); + if (!bmp_empty(TEST_BITS, bmp)) + goto err; + + if (bmp->bits[0]) + goto err; + + if (bmp->bits[1]) + goto err; + + bmp_free(bmp); + return 0; + +err: + bmp_free(bmp); + return -EINVAL; +} + +static bool test_bitmap_test_and_clear_single(struct arena_bitmap __arena *bmp, size_t ind) +{ + if (bmp_test_and_clear_bit(ind, bmp)) + return false; + + __bmp_set_bit(ind, bmp); + + if (!bmp_test_and_clear_bit(ind, bmp)) + return false; + + if (bmp_test_bit(ind, bmp)) + return false; + + if (bmp_test_and_clear_bit(ind, bmp)) + return false; + + return true; +} + +static bool test_bitmap_test_and_set_single(struct arena_bitmap __arena *bmp, size_t ind) +{ + if (bmp_test_and_set_bit(ind, bmp)) + return false; + + if (!bmp_test_and_set_bit(ind, bmp)) + return false; + + if (!bmp_test_bit(ind, bmp)) + return false; + + __bmp_clear_bit(ind, bmp); + + if (bmp_test_and_set_bit(ind, bmp)) + return false; + + return true; +} + +SEC("syscall") +__weak int test_bitmap_test_and_clear_bit(void) +{ + struct arena_bitmap __arena *bmp; + + bmp = bmp_alloc(TEST_BITS); + if (!bmp) + return -ENOMEM; + + if (!test_bitmap_test_and_clear_single(bmp, 0)) + goto err; + + if (!test_bitmap_test_and_clear_single(bmp, MID_BIT)) + goto err; + + if (!test_bitmap_test_and_clear_single(bmp, LAST_BIT)) + goto err; + + if (!bmp_empty(TEST_BITS, bmp)) + goto err; + + bmp_free(bmp); + return 0; + +err: + bmp_free(bmp); + return -EINVAL; +} + +SEC("syscall") +__weak int test_bitmap_test_and_set_bit(void) +{ + struct arena_bitmap __arena *bmp; + + bmp = bmp_alloc(TEST_BITS); + if (!bmp) + return -ENOMEM; + + if (!test_bitmap_test_and_set_single(bmp, 0)) + goto err; + + if (!test_bitmap_test_and_set_single(bmp, MID_BIT)) + goto err; + + if (!test_bitmap_test_and_set_single(bmp, LAST_BIT)) + goto err; + + bmp_free(bmp); + return 0; + +err: + bmp_free(bmp); + return -EINVAL; +} + + +SEC("syscall") +__weak int test_bitmap_and(void) +{ + struct arena_bitmap __arena *src1 = NULL, *src2 = NULL, *dst = NULL; + + src1 = bmp_alloc(TEST_BITS); + src2 = bmp_alloc(TEST_BITS); + dst = bmp_alloc(TEST_BITS); + if (!src1 || !src2 || !dst) + goto err; + + test_bmp_setall(dst); + + __bmp_set_bit(0, src1); + __bmp_set_bit(MID_BIT, src1); + __bmp_set_bit(LAST_BIT, src1); + + __bmp_set_bit(MID_BIT, src2); + __bmp_set_bit(LAST_BIT, src2); + + bmp_and(TEST_BITS, dst, src1, src2); + + if (bmp_test_bit(0, dst)) + goto err; + if (!bmp_test_bit(MID_BIT, dst)) + goto err; + if (!bmp_test_bit(LAST_BIT, dst)) + goto err; + + if (dst->bits[0]) + goto err; + if (dst->bits[1] != (BIT_MASK(MID_BIT) | BIT_MASK(LAST_BIT))) + goto err; + + bmp_free(src1); + bmp_free(src2); + bmp_free(dst); + return 0; + +err: + bmp_free(src1); + bmp_free(src2); + bmp_free(dst); + return -EINVAL; +} + +SEC("syscall") +__weak int test_bitmap_or(void) +{ + struct arena_bitmap __arena *src1 = NULL, *src2 = NULL, *dst = NULL; + + src1 = bmp_alloc(TEST_BITS); + src2 = bmp_alloc(TEST_BITS); + dst = bmp_alloc(TEST_BITS); + if (!src1 || !src2 || !dst) + goto err; + + test_bmp_setall(dst); + + __bmp_set_bit(0, src1); + __bmp_set_bit(LAST_BIT, src1); + + __bmp_set_bit(MID_BIT, src2); + __bmp_set_bit(LAST_BIT, src2); + + bmp_or(TEST_BITS, dst, src1, src2); + + if (!bmp_test_bit(0, dst)) + goto err; + if (!bmp_test_bit(MID_BIT, dst)) + goto err; + if (!bmp_test_bit(LAST_BIT, dst)) + goto err; + + if (dst->bits[0] != BIT_MASK(0)) + goto err; + if (dst->bits[1] != (BIT_MASK(MID_BIT) | BIT_MASK(LAST_BIT))) + goto err; + + bmp_free(src1); + bmp_free(src2); + bmp_free(dst); + return 0; + +err: + bmp_free(src1); + bmp_free(src2); + bmp_free(dst); + return -EINVAL; +} + +SEC("syscall") +__weak int test_bitmap_subset(void) +{ + struct arena_bitmap __arena *big = NULL, *small = NULL; + + big = bmp_alloc(TEST_BITS); + small = bmp_alloc(TEST_BITS); + if (!big || !small) + goto err; + + if (!bmp_subset(TEST_BITS, big, small)) + goto err; + + __bmp_set_bit(0, small); + if (bmp_subset(TEST_BITS, big, small)) + goto err; + + __bmp_set_bit(0, big); + if (!bmp_subset(TEST_BITS, big, small)) + goto err; + + __bmp_set_bit(LAST_BIT, small); + if (bmp_subset(TEST_BITS, big, small)) + goto err; + + __bmp_set_bit(LAST_BIT, big); + __bmp_set_bit(MID_BIT, big); + if (!bmp_subset(TEST_BITS, big, small)) + goto err; + + if (bmp_subset(TEST_BITS, small, big)) + goto err; + + bmp_free(big); + bmp_free(small); + return 0; + +err: + bmp_free(big); + bmp_free(small); + return -EINVAL; + +} + +SEC("syscall") +__weak int test_bitmap_intersects(void) +{ + struct arena_bitmap __arena *arg1 = NULL, *arg2 = NULL; + + arg1 = bmp_alloc(TEST_BITS); + arg2 = bmp_alloc(TEST_BITS); + if (!arg1 || !arg2) + goto err; + + if (bmp_intersects(TEST_BITS, arg1, arg2)) + goto err; + + __bmp_set_bit(0, arg1); + __bmp_set_bit(MID_BIT, arg2); + if (bmp_intersects(TEST_BITS, arg1, arg2)) + goto err; + + __bmp_set_bit(LAST_BIT, arg1); + __bmp_set_bit(LAST_BIT, arg2); + if (!bmp_intersects(TEST_BITS, arg1, arg2)) + goto err; + + bmp_free(arg1); + bmp_free(arg2); + return 0; + +err: + bmp_free(arg1); + bmp_free(arg2); + return -EINVAL; +} + +SEC("syscall") +__weak int test_bitmap_copy(void) +{ + struct arena_bitmap __arena *arg1 = NULL, *arg2 = NULL; + + arg1 = bmp_alloc(TEST_BITS); + arg2 = bmp_alloc(TEST_BITS); + if (!arg1 || !arg2) + goto err; + + __bmp_set_bit(0, arg1); + __bmp_set_bit(MID_BIT, arg1); + + /* Make sure those get overwritten. */ + __bmp_set_bit(1, arg2); + __bmp_set_bit(MID_BIT + 2, arg2); + + bmp_copy(TEST_BITS, arg2, arg1); + + /* Bitmaps are equal if a subset of each other. */ + if (!bmp_subset(TEST_BITS, arg1, arg2) || + !bmp_subset(TEST_BITS, arg2, arg1)) + goto err; + + bmp_free(arg1); + bmp_free(arg2); + return 0; + +err: + bmp_free(arg1); + bmp_free(arg2); + return -EINVAL; +} diff --git a/tools/testing/selftests/bpf/libarena/selftests/st_buddy.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_buddy.bpf.c index b45a306816c0..5628f0987012 100644 --- a/tools/testing/selftests/bpf/libarena/selftests/st_buddy.bpf.c +++ b/tools/testing/selftests/bpf/libarena/selftests/test_buddy.bpf.c @@ -171,7 +171,8 @@ __weak int test_buddy_alloc_multiple(void) SEC("syscall") __weak int test_buddy_alignment(void) { - int ret, i; + int ret; + u32 i; ret = buddy_init(&buddy); if (ret) @@ -179,6 +180,7 @@ __weak int test_buddy_alignment(void) /* Allocate various sizes and check alignment */ for (i = zero; i < 17 && can_loop; i++) { + barrier_var(i); ptrs[i] = buddy_alloc(&buddy, alignment_sizes[i]); if (!ptrs[i]) { arena_stdout("alignment test: alloc failed for size %lu", @@ -198,8 +200,10 @@ __weak int test_buddy_alignment(void) } /* Free all allocations */ - for (i = zero; i < 17 && can_loop; i++) + for (i = zero; i < 17 && can_loop; i++) { + barrier_var(i); buddy_free(&buddy, ptrs[i]); + } buddy_destroy(&buddy); diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c new file mode 100644 index 000000000000..ea1fac95b461 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause + +#include <bpf_atomic.h> + +#include <libarena/common.h> + +#include <libarena/asan.h> +#include <libarena/bitmap.h> + +#define TEST_BITMAP_THREADS 2 +#define TEST_BITMAP_BITS (2 * BITS_PER_LONG_LONG) +#define TEST_BITMAP_SYNC_SPINS BPF_MAX_LOOPS +#define TEST_BITMAP_ITERS 10 * 1000 * 1000 + +static struct arena_bitmap __arena *bitmap; +static volatile u64 started; +static volatile bool test_abort; + +/* + * The test needs cmpxchg atomics on arena memory. + */ +#if defined(ENABLE_ATOMICS_TESTS) && \ + (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ + defined(__TARGET_ARCH_s390) || \ + defined(__TARGET_ARCH_powerpc) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) +static bool bitmap_tests_enabled(void) +{ + return true; +} +#else +static bool bitmap_tests_enabled(void) +{ + return false; +} +#endif + +__weak +int bitmap_wait_for_start(void) +{ + u64 i; + + __sync_fetch_and_add(&started, 1); + + for (i = zero; i < TEST_BITMAP_SYNC_SPINS && can_loop; i++) { + if (test_abort) + return -EINTR; + if (smp_load_acquire(&started) >= TEST_BITMAP_THREADS) + return 0; + } + + test_abort = true; + return -ETIMEDOUT; +} + +/* + * The test makes sure writes don't clobber each other by overwriting + * the same word. One thread always writes on even bits, the other on + * odds. Both should be able to operate on the bitmap oblivious of the + * other's operations. + */ +__weak +int bitmap_test_bit_sequence(u32 bit) +{ + if (bmp_test_and_clear_bit(bit, bitmap)) + return -EINVAL; + + if (bmp_test_and_set_bit(bit, bitmap)) + return -EINVAL; + if (!bmp_test_bit(bit, bitmap)) + return -EINVAL; + + if (!bmp_test_and_set_bit(bit, bitmap)) + return -EINVAL; + if (!bmp_test_bit(bit, bitmap)) + return -EINVAL; + + if (!bmp_test_and_clear_bit(bit, bitmap)) + return -EINVAL; + if (bmp_test_bit(bit, bitmap)) + return -EINVAL; + + if (bmp_test_and_clear_bit(bit, bitmap)) + return -EINVAL; + + bmp_set_bit(bit, bitmap); + if (!bmp_test_bit(bit, bitmap)) + return -EINVAL; + + bmp_clear_bit(bit, bitmap); + if (bmp_test_bit(bit, bitmap)) + return -EINVAL; + + bmp_set_bit(bit, bitmap); + if (!bmp_test_bit(bit, bitmap)) + return -EINVAL; + + return 0; + +} + +static void bitmap_test_reset_single(int parity) +{ + u32 bit; + + for (bit = parity; bit < TEST_BITMAP_BITS && can_loop; bit += 2) + bmp_clear_bit(bit, bitmap); + +} + +static int bitmap_test_common_single(int parity) +{ + u32 bit; + int ret; + + for (bit = parity; bit < TEST_BITMAP_BITS && can_loop; bit += 2) { + if (test_abort) + return -EINTR; + + ret = bitmap_test_bit_sequence(bit); + if (ret) { + test_abort = true; + return ret; + } + } + + return 0; +} + +static int bitmap_test_common(int parity) +{ + int ret; + u32 i; + + arena_subprog_init(); + + ret = bitmap_wait_for_start(); + if (ret) + return ret; + + for (i = zero; i < TEST_BITMAP_ITERS && can_loop; i++) { + ret = bitmap_test_common_single(parity); + if (ret) + return ret; + + if (test_abort) + break; + + bitmap_test_reset_single(parity); + } + + return 0; +} + +SEC("syscall") int parallel_test_bitmap__enabled(void) +{ + return bitmap_tests_enabled() ? 0 : -EOPNOTSUPP; +} + +SEC("syscall") int parallel_test_bitmap__init(void) +{ + bitmap = bmp_alloc(TEST_BITMAP_BITS); + if (!bitmap) + return -ENOMEM; + + return 0; +} + +SEC("syscall") int parallel_test_bitmap__fini(void) +{ + int ret = 0; + + if (!bitmap) + return -EINVAL; + + bmp_free(bitmap); + bitmap = NULL; + + return ret; +} + +SEC("syscall") int parallel_test_bitmap__0(void) +{ + return bitmap_test_common(0); +} + +SEC("syscall") int parallel_test_bitmap__1(void) +{ + return bitmap_test_common(1); +} diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_parallel_spmc.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_spmc.bpf.c index f08f2a92e194..5fa96eb74095 100644 --- a/tools/testing/selftests/bpf/libarena/selftests/test_parallel_spmc.bpf.c +++ b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_spmc.bpf.c @@ -155,7 +155,7 @@ int spmc_quiesce_on_owner(u64 epoch) { u64 i; - bpf_for(i, 0, TEST_SPMC_SYNC_SPINS) { + for (i = zero; i < TEST_SPMC_SYNC_SPINS && can_loop; i++) { if (test_abort) return -EINTR; if (smp_load_acquire(&owner_epoch) >= epoch) @@ -175,8 +175,7 @@ int spmc_quiesce_on_stealer(u64 epoch) int err = -ETIMEDOUT; target = STEALER_EPOCH(epoch); - bpf_for(i, 0, TEST_SPMC_SYNC_SPINS) { - + for (i = zero; i < TEST_SPMC_SYNC_SPINS && can_loop; i++) { if (test_abort) { err = -EINTR; break; @@ -391,7 +390,7 @@ int spmc_wait_for_stealers_to_start(u64 target) { u64 i; - bpf_for(i, 0, TEST_SPMC_SYNC_SPINS) { + for (i = zero; i < TEST_SPMC_SYNC_SPINS && can_loop; i++) { if (test_abort) return -EINTR; if (READ_ONCE(stealers_started) >= target) @@ -537,7 +536,7 @@ static int spmc_wait_for_round_steals(u64 target) arena_subprog_init(); - bpf_for(i, 0, TEST_SPMC_SYNC_SPINS) { + for (i = zero; i < TEST_SPMC_SYNC_SPINS && can_loop; i++) { if (test_abort) return -EINTR; if (round_steals >= target) diff --git a/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c new file mode 100644 index 000000000000..5ff8e688ddc7 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* + * Copyright (c) 2025-2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2025-2026 Emil Tsalapatis <emil@etsalapatis.com> + */ + +#include <libarena/common.h> + +#include <libarena/asan.h> +#include <libarena/bitmap.h> + +__weak +struct arena_bitmap __arena *bmp_alloc(size_t bits) +{ + struct arena_bitmap __arena *bmp; + size_t size = BITS_TO_LONG_LONGS(bits) * sizeof(bmp->bits[0]); + + /* Assume long-aligned masks. */ + if (bits % BITS_PER_LONG_LONG) + return NULL; + + bmp = (struct arena_bitmap __arena *)arena_malloc(size); + if (!bmp) + return NULL; + + bmp_clear(bits, bmp); + + return bmp; +} + +__weak +void bmp_free(struct arena_bitmap __arena *bmp) +{ + arena_free(bmp); +} + +__weak +void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + bmp->bits[BIT_WORD(bit)] |= BIT_MASK(bit); +} + +__weak +void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + bmp->bits[BIT_WORD(bit)] &= ~BIT_MASK(bit); +} + +__weak +bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + return bmp->bits[BIT_WORD(bit)] & BIT_MASK(bit); +} + +__weak +bool bmp_test_and_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + u64 val = BIT_MASK(bit); + u32 idx = BIT_WORD(bit); + u64 old, new, actual; + + do { + old = bmp->bits[idx]; + + if (!(old & val)) + return false; + + new = old & ~val; + actual = cmpxchg(&bmp->bits[idx], old, new); + + if (actual == old) + return true; + + } while (can_loop); + + return false; +} + +__weak +bool bmp_test_and_set_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + u64 val = BIT_MASK(bit); + u32 idx = BIT_WORD(bit); + u64 old, new, actual; + + do { + old = bmp->bits[idx]; + + if ((old & val)) + return true; + + new = old | val; + actual = cmpxchg(&bmp->bits[idx], old, new); + + if (actual == old) + return false; + + } while (can_loop); + + return false; +} + +__weak +void bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + u64 val = BIT_MASK(bit); + u32 idx = BIT_WORD(bit); + u64 old, new, actual; + + do { + old = bmp->bits[idx]; + new = old & ~val; + actual = cmpxchg(&bmp->bits[idx], old, new); + + } while (actual != old && can_loop); +} + +__weak +void bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + u64 val = BIT_MASK(bit); + u32 idx = BIT_WORD(bit); + u64 old, new, actual; + + do { + old = bmp->bits[idx]; + new = old | val; + actual = cmpxchg(&bmp->bits[idx], old, new); + + } while (actual != old && can_loop); +} + +__weak +void bmp_clear(size_t bits, struct arena_bitmap __arena *bmp) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) + bmp->bits[i] = 0; +} + +static __always_inline u64 bmp_last_word_mask(size_t bits) +{ + u32 rem = bits % BITS_PER_LONG_LONG; + + return rem ? (1ULL << rem) - 1 : ~0ULL; +} + +__weak +void bmp_and(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src1, struct arena_bitmap __arena *src2) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) + dst->bits[i] = src1->bits[i] & src2->bits[i]; + + if (nwords && bits % BITS_PER_LONG_LONG) + dst->bits[nwords - 1] &= bmp_last_word_mask(bits); +} + +__weak +void bmp_or(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src1, struct arena_bitmap __arena *src2) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) + dst->bits[i] = src1->bits[i] | src2->bits[i]; + + if (nwords && bits % BITS_PER_LONG_LONG) + dst->bits[nwords - 1] &= bmp_last_word_mask(bits); +} + +__weak +bool bmp_empty(size_t bits, struct arena_bitmap __arena *bmp) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) { + u64 mask = (i == nwords - 1) ? bmp_last_word_mask(bits) : ~0ULL; + + if (bmp->bits[i] & mask) + return false; + } + + return true; +} + +__weak +void bmp_copy(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) + dst->bits[i] = src->bits[i]; + + if (nwords && bits % BITS_PER_LONG_LONG) + dst->bits[nwords - 1] &= bmp_last_word_mask(bits); +} + +__weak +bool bmp_subset(size_t bits, struct arena_bitmap __arena *big, struct arena_bitmap __arena *small) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) { + u64 mask = (i == nwords - 1) ? bmp_last_word_mask(bits) : ~0ULL; + + if (~big->bits[i] & small->bits[i] & mask) + return false; + } + + return true; +} + +__weak +bool bmp_intersects(size_t bits, struct arena_bitmap __arena *arg1, struct arena_bitmap __arena *arg2) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) { + u64 mask = (i == nwords - 1) ? bmp_last_word_mask(bits) : ~0ULL; + + if (arg1->bits[i] & arg2->bits[i] & mask) + return true; + } + + return false; +} + +__weak +void bmp_print(size_t bits, struct arena_bitmap __arena *bmp) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) + arena_stderr("%016llx ", bmp->bits[i]); +} diff --git a/tools/testing/selftests/bpf/libarena/src/common.bpf.c b/tools/testing/selftests/bpf/libarena/src/common.bpf.c index 50be57213dfb..41b1de3452fe 100644 --- a/tools/testing/selftests/bpf/libarena/src/common.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/common.bpf.c @@ -4,9 +4,15 @@ #include <libarena/asan.h> #include <libarena/buddy.h> -const volatile u32 zero = 0; - struct buddy __arena buddy; +volatile u32 zero = 0; + +/* + * Storage for the queue nodes declared by bpf_arena_spin_lock.h. Each program + * linking the arena spinlock provides exactly one definition, so that the array + * is emitted once rather than once per translation unit. + */ +struct arena_qnode __arena __hidden qnodes[_Q_MAX_CPUS][_Q_MAX_NODES]; int arena_fls(__u64 word) { @@ -38,6 +44,12 @@ __weak int arena_buddy_reset(void) return buddy_init(&buddy); } +SEC("syscall") +__weak int arena_buddy_destroy(void) +{ + return buddy_destroy(&buddy); +} + __weak void __arena *arena_malloc(size_t size) { return buddy_alloc(&buddy, size); diff --git a/tools/testing/selftests/bpf/network_helpers.c b/tools/testing/selftests/bpf/network_helpers.c index b82f572641b7..cdf2d7d3ab32 100644 --- a/tools/testing/selftests/bpf/network_helpers.c +++ b/tools/testing/selftests/bpf/network_helpers.c @@ -111,7 +111,7 @@ int start_server_addr(int type, const struct sockaddr_storage *addr, socklen_t a if (settimeo(fd, opts->timeout_ms)) goto error_close; - if (type == SOCK_STREAM && + if ((type & SOCK_TYPE_MASK) == SOCK_STREAM && setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on))) { log_err("Failed to enable SO_REUSEADDR"); goto error_close; @@ -128,7 +128,7 @@ int start_server_addr(int type, const struct sockaddr_storage *addr, socklen_t a goto error_close; } - if (type == SOCK_STREAM) { + if ((type & SOCK_TYPE_MASK) == SOCK_STREAM) { if (listen(fd, opts->backlog ? MAX(opts->backlog, 0) : 1) < 0) { log_err("Failed to listed on socket"); goto error_close; @@ -424,7 +424,8 @@ int make_sockaddr(int family, const char *addr_str, __u16 port, *len = sizeof(*sin6); return 0; } else if (family == AF_UNIX) { - /* Note that we always use abstract unix sockets to avoid having + /* + * Note that we always use abstract unix sockets to avoid having * to clean up leftover files. */ struct sockaddr_un *sun = (void *)addr; @@ -865,7 +866,8 @@ static bool is_ethernet(const u_char *packet) memcpy(&arphdr_type, packet + 8, 2); arphdr_type = ntohs(arphdr_type); - /* Except the following cases, the protocol type contains the + /* + * Except the following cases, the protocol type contains the * Ethernet protocol type for the packet. * * https://www.tcpdump.org/linktypes/LINKTYPE_LINUX_SLL2.html @@ -1033,19 +1035,22 @@ static void *traffic_monitor_thread(void *arg) if (!packet) continue; - /* According to the man page of pcap_dump(), first argument + /* + * According to the man page of pcap_dump(), first argument * is the pcap_dumper_t pointer even it's argument type is * u_char *. */ pcap_dump((u_char *)dumper, &header, packet); - /* Not sure what other types of packets look like. Here, we + /* + * Not sure what other types of packets look like. Here, we * parse only Ethernet and compatible packets. */ if (!is_ethernet(packet)) continue; - /* Skip SLL2 header + /* + * Skip SLL2 header * https://www.tcpdump.org/linktypes/LINKTYPE_LINUX_SLL2.html * * Although the document doesn't mention that, the payload @@ -1079,7 +1084,8 @@ static void *traffic_monitor_thread(void *arg) return NULL; } -/* Prepare the pcap handle to capture packets. +/* + * Prepare the pcap handle to capture packets. * * This pcap is non-blocking and immediate mode is enabled to receive * captured packets as soon as possible. The snaplen is set to 1024 bytes @@ -1150,7 +1156,8 @@ static void encode_test_name(char *buf, size_t len, const char *test_name, const #define PCAP_DIR "/tmp/tmon_pcap" -/* Start to monitor the network traffic in the given network namespace. +/* + * Start to monitor the network traffic in the given network namespace. * * netns: the name of the network namespace to monitor. If NULL, the * current network namespace is monitored. @@ -1255,7 +1262,8 @@ static void traffic_monitor_release(struct tmonitor_ctx *ctx) free(ctx); } -/* Stop the network traffic monitor. +/* + * Stop the network traffic monitor. * * ctx: the context returned by traffic_monitor_start() */ diff --git a/tools/testing/selftests/bpf/network_helpers.h b/tools/testing/selftests/bpf/network_helpers.h index 79a010c88e11..75133119c04a 100644 --- a/tools/testing/selftests/bpf/network_helpers.h +++ b/tools/testing/selftests/bpf/network_helpers.h @@ -25,6 +25,11 @@ typedef __u16 __sum16; #define VIP_NUM 5 #define MAGIC_BYTES 123 +/* include/linux/net.h */ +#ifndef SOCK_TYPE_MASK +#define SOCK_TYPE_MASK 0xf +#endif + struct network_helper_opts { int timeout_ms; int proto; diff --git a/tools/testing/selftests/bpf/prog_tests/arena_atomics.c b/tools/testing/selftests/bpf/prog_tests/arena_atomics.c index d98577a6babc..1ad5d03d07ad 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_atomics.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_atomics.c @@ -222,7 +222,7 @@ static void test_store_release(struct arena_atomics *skel) "store_release64_result"); } -void test_arena_atomics(void) +void serial_test_arena_atomics(void) { struct arena_atomics *skel; int err; diff --git a/tools/testing/selftests/bpf/prog_tests/arena_direct_value.c b/tools/testing/selftests/bpf/prog_tests/arena_direct_value.c index 4b4adb3f4b71..01fcf4965ea4 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_direct_value.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_direct_value.c @@ -66,7 +66,7 @@ cleanup: close(map_fd); } -void test_arena_direct_value(void) +void serial_test_arena_direct_value(void) { if (test__start_subtest("one_past_end")) test_arena_direct_value_one_past_end(); diff --git a/tools/testing/selftests/bpf/prog_tests/arena_htab.c b/tools/testing/selftests/bpf/prog_tests/arena_htab.c index d69fd2465f53..91ccf0402980 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_htab.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_htab.c @@ -81,7 +81,7 @@ static void test_arena_htab_asm(void) arena_htab_asm__destroy(skel); } -void test_arena_htab(void) +void serial_test_arena_htab(void) { if (test__start_subtest("arena_htab_llvm")) test_arena_htab_llvm(); diff --git a/tools/testing/selftests/bpf/prog_tests/arena_list.c b/tools/testing/selftests/bpf/prog_tests/arena_list.c index 4f2866a615ce..2648e06f53d0 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_list.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_list.c @@ -68,7 +68,7 @@ out: arena_list__destroy(skel); } -void test_arena_list(void) +void serial_test_arena_list(void) { if (test__start_subtest("arena_list_1")) test_arena_list_add_del(1, false); diff --git a/tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c b/tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c new file mode 100644 index 000000000000..14c2d1a1d673 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include <sys/user.h> +#ifndef PAGE_SIZE /* on some archs it comes in sys/user.h */ +#include <unistd.h> +#define PAGE_SIZE getpagesize() +#endif + +#include "arena_mem_usage.skel.h" + +/* + * arena_map_mem_usage() is surfaced to user space through the map's + * /proc/<pid>/fdinfo/<fd> "memlock:" line (the same value bpftool map show + * prints). Read it directly so the test has no external dependency. + */ +static long map_memlock(int map_fd) +{ + char path[64], line[128]; + long memlock = -1; + FILE *f; + + snprintf(path, sizeof(path), "/proc/self/fdinfo/%d", map_fd); + f = fopen(path, "r"); + if (!ASSERT_OK_PTR(f, "open_fdinfo")) + return -1; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "memlock:\t%ld", &memlock) == 1) + break; + } + fclose(f); + ASSERT_NEQ(memlock, -1, "parse_memlock"); + return memlock; +} + +static int run(struct bpf_program *prog, const char *name) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + + if (!ASSERT_OK(err, name)) + return -1; + if (!ASSERT_OK(opts.retval, name)) + return -1; + return 0; +} + +void serial_test_arena_mem_usage(void) +{ + struct arena_mem_usage *skel; + const long ps = PAGE_SIZE; + char *base; + size_t sz; + int fd, i; + + skel = arena_mem_usage__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_load")) + return; + fd = bpf_map__fd(skel->maps.arena); + + /* Fresh arena: no data pages, and the scratch page is not counted. */ + ASSERT_EQ(map_memlock(fd), 0, "initial"); + + /* BPF-side allocation of 17 pages. */ + skel->bss->alloc_cnt = 17; + if (run(skel->progs.alloc, "alloc")) + goto out; + /* + * A NULL ptr means bpf_arena_alloc_pages() itself failed (e.g. the host + * is under memory pressure), not a miscount -- flag it distinctly so a + * red CI run is not mistaken for a counting bug. + */ + if (!ASSERT_OK_PTR(skel->bss->ptr, "arena_alloc_pages")) + goto out; + ASSERT_EQ(map_memlock(fd), 17 * ps, "after_alloc"); + + /* Free a single page (arena_free_pages page_cnt==1 path). */ + skel->bss->free_byte_off = 0; + skel->bss->free_cnt = 1; + if (run(skel->progs.free_pages, "free_one")) + goto out; + ASSERT_EQ(map_memlock(fd), 16 * ps, "after_free_one"); + + /* Free ten pages in one call (bulk path); only the freed pages count. */ + skel->bss->free_byte_off = 1 * ps; + skel->bss->free_cnt = 10; + if (run(skel->progs.free_pages, "free_bulk")) + goto out; + ASSERT_EQ(map_memlock(fd), 6 * ps, "after_free_bulk"); + + /* Free the remaining six -> arena empty again. */ + skel->bss->free_byte_off = 11 * ps; + skel->bss->free_cnt = 6; + if (run(skel->progs.free_pages, "free_rest")) + goto out; + ASSERT_EQ(map_memlock(fd), 0, "after_free_rest"); + + /* + * User-space fault-in: touching unallocated arena pages allocates them + * through arena_vm_fault(). libbpf mmap()s the arena at map_extra during + * load, so bpf_map__initial_value() hands back that base. + */ + base = bpf_map__initial_value(skel->maps.arena, &sz); + if (!ASSERT_OK_PTR(base, "arena_base")) + goto out; + for (i = 0; i < 8; i++) + base[i * ps] = 1; + ASSERT_EQ(map_memlock(fd), 8 * ps, "after_faultin"); + + /* + * Free the faulted-in pages from BPF. They are mapped into the user vma + * (elevated refcount), so this also exercises the zap path. + */ + skel->bss->ptr = base; + skel->bss->free_byte_off = 0; + skel->bss->free_cnt = 8; + if (run(skel->progs.free_pages, "free_faulted")) + goto out; + ASSERT_EQ(map_memlock(fd), 0, "after_free_faulted"); +out: + arena_mem_usage__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c b/tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c index acb9d53b5973..545b05d7a0aa 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c @@ -101,7 +101,7 @@ end: return; } -void test_arena_spin_lock(void) +void serial_test_arena_spin_lock(void) { repeat = 1000; if (test__start_subtest("arena_spin_lock_1")) diff --git a/tools/testing/selftests/bpf/prog_tests/arena_strsearch.c b/tools/testing/selftests/bpf/prog_tests/arena_strsearch.c index f81a0c066505..0c1c6cbfa0f1 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_strsearch.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_strsearch.c @@ -23,7 +23,7 @@ static void test_arena_str(void) arena_strsearch__destroy(skel); } -void test_arena_strsearch(void) +void serial_test_arena_strsearch(void) { if (test__start_subtest("arena_strsearch")) test_arena_str(); diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c b/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c index 77f1c0550c9b..6dbd1487343c 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c @@ -11,6 +11,7 @@ #include "bpf_qdisc_fail__invalid_dynptr.skel.h" #include "bpf_qdisc_fail__invalid_dynptr_slice.skel.h" #include "bpf_qdisc_fail__invalid_dynptr_cross_frame.skel.h" +#include "bpf_qdisc_fail__untrusted_write.skel.h" #include "bpf_qdisc_dynptr_use_after_invalidate_clone.skel.h" #define LO_IFINDEX 1 @@ -230,6 +231,7 @@ void test_ns_bpf_qdisc(void) RUN_TESTS(bpf_qdisc_fail__invalid_dynptr); RUN_TESTS(bpf_qdisc_fail__invalid_dynptr_cross_frame); RUN_TESTS(bpf_qdisc_fail__invalid_dynptr_slice); + RUN_TESTS(bpf_qdisc_fail__untrusted_write); RUN_TESTS(bpf_qdisc_dynptr_use_after_invalidate_clone); } diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c index fe30181e6336..eb05fc82f81b 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c @@ -14,6 +14,7 @@ #include "tcp_ca_incompl_cong_ops.skel.h" #include "tcp_ca_unsupp_cong_op.skel.h" #include "tcp_ca_kfunc.skel.h" +#include "tcp_ca_untrusted_btf_write.skel.h" #include "bpf_cc_cubic.skel.h" static const unsigned int total_bytes = 10 * 1024 * 1024; @@ -579,6 +580,15 @@ static void test_tcp_ca_kfunc(void) tcp_ca_kfunc__destroy(skel); } +static void test_untrusted_btf_write(void) +{ + struct tcp_ca_untrusted_btf_write *skel; + + skel = tcp_ca_untrusted_btf_write__open_and_load(); + ASSERT_ERR_PTR(skel, "tcp_ca_untrusted_btf_write__open_and_load"); + tcp_ca_untrusted_btf_write__destroy(skel); +} + static void test_cc_cubic(void) { struct cb_opts cb_opts = { @@ -637,6 +647,8 @@ void test_bpf_tcp_ca(void) test_link_replace(); if (test__start_subtest("tcp_ca_kfunc")) test_tcp_ca_kfunc(); + if (test__start_subtest("untrusted_btf_write")) + test_untrusted_btf_write(); if (test__start_subtest("cc_cubic")) test_cc_cubic(); if (test__start_subtest("dctcp_autoattach_map")) diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 66855cbd6b73..67b9015cbd98 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -4250,6 +4250,54 @@ static struct btf_raw_test raw_tests[] = { .max_entries = 1, }, +/* + * struct inner { + * struct bpf_spin_lock lock; + * }; + * + * struct value { + * struct bpf_spin_lock lock; + * struct inner nested; + * }; + */ +{ + .descr = "struct test duplicate nested unique fields", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_STRUCT_ENC(NAME_TBD, 1, 4), /* [2] */ + BTF_MEMBER_ENC(NAME_TBD, 1, 0), + BTF_STRUCT_ENC(NAME_TBD, 1, 4), /* [3] */ + BTF_MEMBER_ENC(NAME_TBD, 2, 0), + BTF_STRUCT_ENC(NAME_TBD, 2, 8), /* [4] */ + BTF_MEMBER_ENC(NAME_TBD, 2, 0), + BTF_MEMBER_ENC(NAME_TBD, 3, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0bpf_spin_lock\0val\0inner\0lock\0value\0lock\0nested"), + .btf_load_err = true, +}, + +/* + * struct value { + * struct bpf_refcount a; + * struct bpf_refcount b; + * }; + */ +{ + .descr = "struct test duplicate bpf_refcount fields", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_STRUCT_ENC(NAME_TBD, 1, 4), /* [2] */ + BTF_MEMBER_ENC(NAME_TBD, 1, 0), + BTF_STRUCT_ENC(NAME_TBD, 2, 8), /* [3] */ + BTF_MEMBER_ENC(NAME_TBD, 2, 0), + BTF_MEMBER_ENC(NAME_TBD, 2, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0bpf_refcount\0refs\0value\0a\0b"), + .btf_load_err = true, +}, + { .descr = "struct test repeated fields count overflow", .raw_types = { diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c b/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c index b7c18d590b99..5a1e08d39a06 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c @@ -10,6 +10,17 @@ #include "cgroup_iter_memcg.h" #include "cgroup_iter_memcg.skel.h" +/* + * memcg stats are cached per-cpu and only become visible once the periodic + * flusher runs (FLUSH_TIME, 2s), or once pending updates cross + * MEMCG_CHARGE_BATCH * num_online_cpus(). That threshold grows with the CPU + * count, so on a large machine a single pass does not reach it and + * bpf_mem_cgroup_flush_stats() returns without flushing anything. Retry for + * long enough to cover a flusher cycle. + */ +#define MEMCG_STAT_RETRIES 16 +#define MEMCG_STAT_RETRY_DELAY_US (250 * 1000) + static int read_stats(struct bpf_link *link) { int fd, ret = 0; @@ -35,11 +46,13 @@ static int read_stats(struct bpf_link *link) static void test_anon(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; void *map; size_t len; len = sysconf(_SC_PAGESIZE) * 1024; +retry: /* * Increase memcg anon usage by mapping and writing * to a new anon region. @@ -53,6 +66,12 @@ static void test_anon(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup; + if (!memcg_query->nr_anon_mapped && ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + munmap(map, len); + goto retry; + } + ASSERT_GT(memcg_query->nr_anon_mapped, 0, "final anon mapped val"); cleanup: @@ -61,6 +80,7 @@ cleanup: static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; void *map; size_t len; char *path; @@ -76,6 +96,7 @@ static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) fd = open(path, O_CREAT | O_RDWR, 0644); if (!ASSERT_OK_FD(fd, "open fd")) return; +retry: if (!ASSERT_OK(ftruncate(fd, len), "ftruncate")) goto cleanup_fd; @@ -88,6 +109,13 @@ static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup_map; + if ((!memcg_query->nr_file_pages || !memcg_query->nr_file_mapped) && + ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + munmap(map, len); + goto retry; + } + ASSERT_GT(memcg_query->nr_file_pages, 0, "final file value"); ASSERT_GT(memcg_query->nr_file_mapped, 0, "final file mapped value"); @@ -100,6 +128,7 @@ cleanup_fd: static void test_shmem(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; size_t len; int fd; @@ -113,12 +142,18 @@ static void test_shmem(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK_FD(fd, "memfd_create")) return; +retry: if (!ASSERT_OK(fallocate(fd, 0, 0, len), "fallocate")) goto cleanup; if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup; + if (!memcg_query->nr_shmem && ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + goto retry; + } + ASSERT_GT(memcg_query->nr_shmem, 0, "final shmem value"); cleanup: @@ -127,11 +162,13 @@ cleanup: static void test_pgfault(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; void *map; size_t len; len = sysconf(_SC_PAGESIZE) * 1024; +retry: /* Create region to use for triggering a page fault. */ map = mmap(NULL, len, PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (!ASSERT_NEQ(map, MAP_FAILED, "mmap anon")) @@ -143,6 +180,12 @@ static void test_pgfault(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup; + if (!memcg_query->pgfault && ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + munmap(map, len); + goto retry; + } + ASSERT_GT(memcg_query->pgfault, 0, "final pgfault val"); cleanup: diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c index 92c20803ea76..2523c07a16c6 100644 --- a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c +++ b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c @@ -5,6 +5,7 @@ #include <bpf/btf.h> #include "bind4_prog.skel.h" #include "freplace_progmap.skel.h" +#include "fentry_sleepable.skel.h" #include "xdp_dummy.skel.h" typedef int (*test_cb)(struct bpf_object *obj); @@ -335,18 +336,6 @@ out: bpf_object__close(pkt_obj); } - -static void test_func_sockmap_update(void) -{ - const char *prog_name[] = { - "freplace/cls_redirect", - }; - test_fexit_bpf2bpf_common("./freplace_cls_redirect.bpf.o", - "./test_cls_redirect.bpf.o", - ARRAY_SIZE(prog_name), - prog_name, false, NULL); -} - static void test_func_replace_void(void) { const char *prog_name[] = { @@ -588,6 +577,60 @@ out: freplace_progmap__destroy(skel); } +static void test_sleepable_fentry_to_xdp(void) +{ + struct fentry_sleepable *skel = NULL; + struct xdp_dummy *skel_xdp = NULL; + int ifindex, prog_fd, err; + char buff[64] = {}; + +#ifndef __x86_64__ + test__skip(); + return; +#endif + + ifindex = if_nametoindex("lo"); + if (!ASSERT_GT(ifindex, 0, "if_nametoindex")) + return; + + skel_xdp = xdp_dummy__open_and_load(); + if (!ASSERT_OK_PTR(skel_xdp, "xdp_dummy__open_and_load")) + return; + + skel = fentry_sleepable__open(); + if (!ASSERT_OK_PTR(skel, "fentry_sleepable__open")) + goto out; + + skel->bss->user_ptr = buff; + + prog_fd = bpf_program__fd(skel_xdp->progs.__x64_sys_nop); + err = bpf_program__set_attach_target(skel->progs.fentry_xdp, prog_fd, "__x64_sys_nop"); + if (!ASSERT_OK(err, "bpf_program__set_attach_target")) + goto out; + + err = fentry_sleepable__load(skel); + ASSERT_ERR(err, "fentry_sleepable__load"); + if (err) + goto out; + + skel->links.fentry_xdp = bpf_program__attach_trace(skel->progs.fentry_xdp); + if (!ASSERT_OK_PTR(skel->links.fentry_xdp, "bpf_program__attach_trace")) + goto out; + + skel_xdp->links.__x64_sys_nop = bpf_program__attach_xdp(skel_xdp->progs.__x64_sys_nop, + ifindex); + if (!ASSERT_OK_PTR(skel_xdp->links.__x64_sys_nop, "bpf_program__attach_xdp")) + goto out; + + err = system("ping -q -c 1 -W 1 127.0.0.1 > /dev/null"); + ASSERT_OK(err, "ping"); + ASSERT_ERR(skel->bss->retval, "retval"); + +out: + fentry_sleepable__destroy(skel); + xdp_dummy__destroy(skel_xdp); +} + /* NOTE: affect other tests, must run in serial mode */ void serial_test_fexit_bpf2bpf(void) { @@ -599,8 +642,6 @@ void serial_test_fexit_bpf2bpf(void) test_func_replace(); if (test__start_subtest("func_replace_verify")) test_func_replace_verify(); - if (test__start_subtest("func_sockmap_update")) - test_func_sockmap_update(); if (test__start_subtest("func_replace_return_code")) test_func_replace_return_code(); if (test__start_subtest("func_map_prog_compatibility")) @@ -621,4 +662,6 @@ void serial_test_fexit_bpf2bpf(void) test_func_replace_int_with_void(); if (test__start_subtest("freplace_void")) test_func_replace_void(); + if (test__start_subtest("sleepable_fentry_to_xdp")) + test_sleepable_fentry_to_xdp(); } diff --git a/tools/testing/selftests/bpf/prog_tests/fib_lookup.c b/tools/testing/selftests/bpf/prog_tests/fib_lookup.c index bd7658958004..8f4779dd802e 100644 --- a/tools/testing/selftests/bpf/prog_tests/fib_lookup.c +++ b/tools/testing/selftests/bpf/prog_tests/fib_lookup.c @@ -2,6 +2,7 @@ /* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ #include <linux/rtnetlink.h> +#include <linux/if_ether.h> #include <sys/types.h> #include <net/if.h> @@ -23,6 +24,7 @@ #define IPV4_TBID_ADDR "172.0.0.254" #define IPV4_TBID_NET "172.0.0.0" #define IPV4_TBID_DST "172.0.0.2" +#define IPV4_TBID_NONEIGH_DST "172.0.0.5" #define IPV6_TBID_ADDR "fd00::FFFF" #define IPV6_TBID_NET "fd00::" #define IPV6_TBID_DST "fd00::2" @@ -37,6 +39,41 @@ #define IPV6_LOCAL "fd01::3" #define IPV6_GW1 "fd01::1" #define IPV6_GW2 "fd01::2" +#define VLAN_ID 100 +#define VLAN_IFACE "veth1.100" +#define VLAN_ID_DOWN 102 +#define VLAN_IFACE_DOWN "veth1.102" +#define QINQ_OUTER_IFACE "veth1.200" +#define QINQ_INNER_IFACE "veth1.200.300" +#define VLAN_TABLE "300" +#define IPV4_VLAN_IFACE_ADDR "10.5.0.254" +#define IPV4_VLAN_EGRESS_DST "10.5.0.2" +#define IPV4_QINQ_DST "10.7.0.2" +#define IPV4_VLAN_DST "10.6.0.2" +#define IPV4_VLAN_GW "10.5.0.1" +#define IPV6_VLAN_IFACE_ADDR "fd02::254" +#define IPV6_VLAN_EGRESS_DST "fd02::2" +#define IPV6_VLAN_DST "fd03::2" +#define IPV6_VLAN_GW "fd02::1" +#define VLAN_VID_UNUSED 999 +#define VRF_IFACE "vrf-blue" +#define VRF_TABLE "1000" +#define VRF_VLAN_ID 101 +#define VRF_VLAN_IFACE "veth1.101" +#define IPV4_VRF_IFACE_ADDR "10.8.0.254" +#define IPV4_VRF_GW "10.8.0.1" +#define IPV4_VRF_DST "10.9.0.2" +#define TBID_VLAN_ID 50 +#define TBID_VLAN_IFACE "veth2.50" +#define IPV4_TBID_VLAN_DST "172.2.0.2" +#define IPV4_BOND_VLAN_DST "10.11.0.2" +#define IPV4_VLAN_MTU_DST "10.5.9.2" +#define QINQ_AD_VLAN_ID 200 +#define QINQ_INNER_VLAN_ID 300 +#define BOND_IFACE "bond99" +#define BOND_PORT "veth3" +#define BOND_PORT_PEER "veth4" +#define BOND_VLAN_ID 500 #define DMAC "11:11:11:11:11:11" #define DMAC_INIT { 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, } #define DMAC2 "01:01:01:01:01:01" @@ -52,6 +89,17 @@ struct fib_lookup_test { __u32 tbid; __u8 dmac[6]; __u32 mark; + /* + * input tag with BPF_FIB_LOOKUP_VLAN_INPUT; expected output tag + * with BPF_FIB_LOOKUP_VLAN (checked when check_vlan is set) + */ + __u16 vlan_proto; + __u16 vlan_id; + bool check_vlan; + const char *expected_dev; /* expected params->ifindex after lookup */ + const char *iif; /* override the default veth1 input device */ + __u16 tot_len; /* triggers the in-lookup mtu check when set */ + __u16 expected_mtu; /* expected mtu_result (union with tot_len) */ }; static const struct fib_lookup_test tests[] = { @@ -79,6 +127,17 @@ static const struct fib_lookup_test tests[] = { .daddr = IPV4_TBID_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, .lookup_flags = BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID, .tbid = 100, .dmac = DMAC_INIT2, }, + /* + * An error that returns after the egress device is resolved must + * report the egress ifindex, not the input. This routes from input + * veth1 via veth2 (table 100) to a dst with no neighbour, so + * input != egress, pinning NO_NEIGH to the egress device. + */ + { .desc = "IPv4 NO_NEIGH reports the egress ifindex, not the input", + .daddr = IPV4_TBID_NONEIGH_DST, + .expected_ret = BPF_FIB_LKUP_RET_NO_NEIGH, + .lookup_flags = BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID, .tbid = 100, + .expected_dev = "veth2", }, { .desc = "IPv6 TBID lookup failure", .daddr = IPV6_TBID_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, .lookup_flags = BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID, @@ -142,12 +201,237 @@ static const struct fib_lookup_test tests[] = { .expected_dst = IPV6_GW1, .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, .mark = MARK, }, + /* vlan egress resolution */ + /* + * Invariant the VLAN-egress arms jointly enforce: a + * BPF_FIB_LOOKUP_VLAN SUCCESS always carries a physical, + * xmit-capable ifindex; no SUCCESS ever returns a VLAN-device + * ifindex. Reducible arms pin ifindex == the physical parent; the + * QinQ and foreign-netns arms pin VLAN_FAILURE with params->ifindex + * left at the input, so a regression to best-effort (SUCCESS + the + * VLAN ifindex) fails one. + */ + { .desc = "IPv4 VLAN egress, no flag", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = VLAN_IFACE, .check_vlan = true, }, + { .desc = "IPv4 VLAN egress, single VLAN", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + /* + * skb path without tot_len: mtu_result is the VLAN device's mtu + * (1400), not the parent's (1500) + */ + { .desc = "IPv4 VLAN egress, skb-path mtu is the VLAN device's without the flag", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = VLAN_IFACE, .check_vlan = true, .expected_mtu = 1400, }, + { .desc = "IPv4 VLAN egress, flag set but egress is not a VLAN", + .daddr = IPV4_NUD_FAILED_ADDR, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, }, + { .desc = "IPv4 VLAN egress, QinQ not reducible (VLAN_FAILURE)", + .daddr = IPV4_QINQ_DST, + .expected_ret = BPF_FIB_LKUP_RET_VLAN_FAILURE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, }, + { .desc = "IPv4 QinQ egress without the flag (escape hatch)", + .daddr = IPV4_QINQ_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = QINQ_INNER_IFACE, }, + { .desc = "IPv6 VLAN egress, single VLAN", + .daddr = IPV6_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN egress, neighbour on the VLAN device", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, .dmac = DMAC_INIT, }, + { .desc = "IPv4 VLAN egress in OUTPUT mode", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .iif = VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_OUTPUT | BPF_FIB_LOOKUP_VLAN | + BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN egress over a bond", + .daddr = IPV4_BOND_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = BOND_IFACE, .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = BOND_VLAN_ID, }, + { .desc = "IPv4 VLAN egress via TBID table", + .daddr = IPV4_TBID_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID | + BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .tbid = 100, + .expected_dev = "veth2", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = TBID_VLAN_ID, }, + { .desc = "IPv4 VLAN egress, success writes mtu_result with the swap", + .daddr = IPV4_VLAN_MTU_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .tot_len = 500, .expected_mtu = 1000, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN egress, FRAG_NEEDED reports mtu, swap unwritten", + .daddr = IPV4_VLAN_MTU_DST, .expected_ret = BPF_FIB_LKUP_RET_FRAG_NEEDED, + .tot_len = 1400, .expected_mtu = 1000, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, }, + /* vlan tag as lookup input */ + { .desc = "IPv4 VLAN input, no flag", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_GW1, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, }, + { .desc = "IPv4 VLAN input, tag selects subinterface route", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VLAN_GW, .expected_dev = VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv6 VLAN input, tag selects subinterface route", + .daddr = IPV6_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV6_VLAN_GW, .expected_dev = VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input and egress combined", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VLAN_GW, .expected_dev = "veth1", + .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_VLAN | + BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input, neighbour resolved on the route", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VLAN_GW, .expected_dev = VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, .dmac = DMAC_INIT2, }, + { .desc = "IPv4 VLAN input, source address from the subinterface", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_src = IPV4_VLAN_IFACE_ADDR, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SRC | + BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + /* + * VRF: the resolved subinterface is enslaved, so the l3mdev rule + * (full lookup) and l3mdev_fib_table_rcu() (DIRECT) must select + * the VRF table from the resolved ingress + */ + { .desc = "IPv4 VLAN input, VRF subinterface, no flag", + .daddr = IPV4_VRF_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_GW1, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, }, + { .desc = "IPv4 VLAN input, tag selects VRF table", + .daddr = IPV4_VRF_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VRF_GW, .expected_dev = VRF_VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VRF_VLAN_ID, }, + { .desc = "IPv4 VLAN input, DIRECT uses VRF table from resolved ingress", + .daddr = IPV4_VRF_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VRF_GW, .expected_dev = VRF_VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_DIRECT | + BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VRF_VLAN_ID, }, + /* + * failure arms also assert params is left untouched: ifindex still + * names the physical device and the input tag bytes survive + */ + { .desc = "IPv4 VLAN input, invalid proto", + .daddr = IPV4_VLAN_DST, .expected_ret = -EINVAL, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = 0x1234, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input, unmatched VID", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_VID_UNUSED, }, + { .desc = "IPv4 VLAN input, subinterface down", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID_DOWN, }, + /* + * the resolver runs before the forwarding check, so on devices + * with forwarding off FWD_DISABLED (not NOT_FWDED) proves the tag + * resolved to that device and the lookup used it as ingress + */ + { .desc = "IPv4 VLAN input, 802.1ad tag", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_FWD_DISABLED, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021AD, .vlan_id = QINQ_AD_VLAN_ID, }, + { .desc = "IPv4 VLAN input, PCP and DEI bits ignored in TCI", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VLAN_GW, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = 0xe000 | VLAN_ID, }, + { .desc = "IPv4 VLAN input, inner QinQ device from VLAN ifindex", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_FWD_DISABLED, + .iif = QINQ_OUTER_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = QINQ_INNER_VLAN_ID, }, + /* + * bonding: the VLANs live on the master, as on receive, where the + * frame is steered to the master before VLAN processing; a port + * ifindex does not match (ports carry vid state but no VLAN devs) + */ + { .desc = "IPv4 VLAN input, tag on bond master resolves", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_FWD_DISABLED, + .iif = BOND_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = BOND_VLAN_ID, }, + { .desc = "IPv4 VLAN input, tag on bond port does not match", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .iif = BOND_PORT, .expected_dev = BOND_PORT, .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = BOND_VLAN_ID, }, + { .desc = "IPv6 VLAN input, invalid proto", + .daddr = IPV6_VLAN_DST, .expected_ret = -EINVAL, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = 0x1234, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input, VID 0 priority tag fails closed", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = 0, }, + { .desc = "IPv6 VLAN input, unmatched VID", + .daddr = IPV6_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_VID_UNUSED, }, + { .desc = "unknown flag bit rejected", + .daddr = IPV4_VLAN_DST, .expected_ret = -EINVAL, + .lookup_flags = (1 << 14) | BPF_FIB_LOOKUP_SKIP_NEIGH, }, + { .desc = "IPv4 VLAN input rejected with TBID", + .daddr = IPV4_VLAN_DST, .expected_ret = -EINVAL, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_TBID, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input rejected with OUTPUT", + .daddr = IPV4_VLAN_DST, .expected_ret = -EINVAL, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_OUTPUT, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, }; static int setup_netns(void) { int err; + /* + * a new netns copies the IPv4 conf from init_net, so on a host with + * forwarding enabled the arms that expect FWD_DISABLED would see the + * lookup succeed instead; pin it off here and enable it per device + */ + err = write_sysctl("/proc/sys/net/ipv4/conf/all/forwarding", "0"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.conf.all.forwarding)")) + goto fail; + + err = write_sysctl("/proc/sys/net/ipv4/conf/default/forwarding", "0"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.conf.default.forwarding)")) + goto fail; + SYS(fail, "ip link add veth1 type veth peer name veth2"); SYS(fail, "ip link set dev veth1 up"); SYS(fail, "ip link set dev veth2 up"); @@ -204,6 +488,105 @@ static int setup_netns(void) SYS(fail, "ip rule add prio 2 fwmark %d lookup %s", MARK, MARK_TABLE); SYS(fail, "ip -6 rule add prio 2 fwmark %d lookup %s", MARK, MARK_TABLE); + /* + * Setup for vlan tests: a subinterface for egress resolution and + * tag-as-input, a QinQ stack, and an iif rule so the input tests + * observe which device the lookup used as ingress. + */ + SYS(fail, "ip link add link veth1 name %s type vlan id %d", + VLAN_IFACE, VLAN_ID); + SYS(fail, "ip link set dev %s up", VLAN_IFACE); + /* + * lower than the veth1 parent (1500): the skb-path mtu check uses the + * FIB result (VLAN) device, so mtu_result is this value, which the + * no-flag arm below pins + */ + SYS(fail, "ip link set dev %s mtu 1400", VLAN_IFACE); + SYS(fail, "ip addr add %s/24 dev %s", IPV4_VLAN_IFACE_ADDR, VLAN_IFACE); + SYS(fail, "ip addr add %s/64 dev %s nodad", IPV6_VLAN_IFACE_ADDR, VLAN_IFACE); + + /* + * stays down: the input flag must treat its tag the way real + * ingress treats a frame arriving on a down VLAN device (drop) + */ + SYS(fail, "ip link add link veth1 name %s type vlan id %d", + VLAN_IFACE_DOWN, VLAN_ID_DOWN); + + err = write_sysctl("/proc/sys/net/ipv4/conf/" VLAN_IFACE "/forwarding", "1"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.conf." VLAN_IFACE ".forwarding)")) + goto fail; + + err = write_sysctl("/proc/sys/net/ipv6/conf/" VLAN_IFACE "/forwarding", "1"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv6.conf." VLAN_IFACE ".forwarding)")) + goto fail; + + SYS(fail, "ip link add link veth1 name %s type vlan proto 802.1ad id 200", + QINQ_OUTER_IFACE); + SYS(fail, "ip link add link %s name %s type vlan id 300", + QINQ_OUTER_IFACE, QINQ_INNER_IFACE); + SYS(fail, "ip link set dev %s up", QINQ_OUTER_IFACE); + SYS(fail, "ip link set dev %s up", QINQ_INNER_IFACE); + SYS(fail, "ip route add %s/32 dev %s", IPV4_QINQ_DST, QINQ_INNER_IFACE); + + SYS(fail, "ip route add %s/32 via %s", IPV4_VLAN_DST, IPV4_GW1); + SYS(fail, "ip route add table %s %s/32 via %s", + VLAN_TABLE, IPV4_VLAN_DST, IPV4_VLAN_GW); + SYS(fail, "ip rule add prio 3 iif %s lookup %s", VLAN_IFACE, VLAN_TABLE); + SYS(fail, "ip -6 route add %s/128 via %s", IPV6_VLAN_DST, IPV6_GW1); + SYS(fail, "ip -6 route add table %s %s/128 via %s", + VLAN_TABLE, IPV6_VLAN_DST, IPV6_VLAN_GW); + SYS(fail, "ip -6 rule add prio 3 iif %s lookup %s", VLAN_IFACE, VLAN_TABLE); + + /* a bond with one port and a VLAN on the bond */ + SYS(fail, "ip link add %s type bond", BOND_IFACE); + SYS(fail, "ip link add %s type veth peer name %s", BOND_PORT, BOND_PORT_PEER); + SYS(fail, "ip link set %s master %s", BOND_PORT, BOND_IFACE); + SYS(fail, "ip link set dev %s up", BOND_IFACE); + SYS(fail, "ip link set dev %s up", BOND_PORT); + SYS(fail, "ip link add link %s name %s.%d type vlan id %d", + BOND_IFACE, BOND_IFACE, BOND_VLAN_ID, BOND_VLAN_ID); + SYS(fail, "ip link set dev %s.%d up", BOND_IFACE, BOND_VLAN_ID); + SYS(fail, "ip route add %s/32 dev %s.%d", + IPV4_BOND_VLAN_DST, BOND_IFACE, BOND_VLAN_ID); + + /* + * a VRF with its own dedicated subinterface (the iif rules above + * must not see it), for the table-selection-by-ingress cases + */ + SYS(fail, "ip link add %s type vrf table %s", VRF_IFACE, VRF_TABLE); + SYS(fail, "ip link set dev %s up", VRF_IFACE); + SYS(fail, "ip link add link veth1 name %s type vlan id %d", + VRF_VLAN_IFACE, VRF_VLAN_ID); + SYS(fail, "ip link set %s master %s", VRF_VLAN_IFACE, VRF_IFACE); + SYS(fail, "ip link set dev %s up", VRF_VLAN_IFACE); + SYS(fail, "ip addr add %s/24 dev %s", IPV4_VRF_IFACE_ADDR, VRF_VLAN_IFACE); + err = write_sysctl("/proc/sys/net/ipv4/conf/" VRF_VLAN_IFACE "/forwarding", "1"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.conf." VRF_VLAN_IFACE ".forwarding)")) + goto fail; + SYS(fail, "ip route add %s/32 via %s", IPV4_VRF_DST, IPV4_GW1); + SYS(fail, "ip route add table %s %s/32 via %s", + VRF_TABLE, IPV4_VRF_DST, IPV4_VRF_GW); + + /* neighbours on the VLAN subinterface for the non-SKIP_NEIGH cases */ + err = write_sysctl("/proc/sys/net/ipv4/neigh/" VLAN_IFACE "/gc_stale_time", "900"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.neigh." VLAN_IFACE ".gc_stale_time)")) + goto fail; + SYS(fail, "ip neigh add %s dev %s lladdr %s nud stale", + IPV4_VLAN_EGRESS_DST, VLAN_IFACE, DMAC); + SYS(fail, "ip neigh add %s dev %s lladdr %s nud stale", + IPV4_VLAN_GW, VLAN_IFACE, DMAC2); + + /* a VLAN on veth2 with a route in the tbid test table */ + SYS(fail, "ip link add link veth2 name %s type vlan id %d", + TBID_VLAN_IFACE, TBID_VLAN_ID); + SYS(fail, "ip link set dev %s up", TBID_VLAN_IFACE); + SYS(fail, "ip route add table 100 %s/32 dev %s", + IPV4_TBID_VLAN_DST, TBID_VLAN_IFACE); + + /* a locked-mtu route via the subinterface for the FRAG_NEEDED case */ + SYS(fail, "ip route add %s/32 dev %s mtu lock 1000", + IPV4_VLAN_MTU_DST, VLAN_IFACE); + return 0; fail: return -1; @@ -218,9 +601,16 @@ static int set_lookup_params(struct bpf_fib_lookup *params, memset(params, 0, sizeof(*params)); params->l4_protocol = IPPROTO_TCP; - params->ifindex = ifindex; + params->ifindex = test->iif ? if_nametoindex(test->iif) : ifindex; params->tbid = test->tbid; params->mark = test->mark; + params->tot_len = test->tot_len; + + /* h_vlan_proto/h_vlan_TCI union with tbid */ + if (test->lookup_flags & BPF_FIB_LOOKUP_VLAN_INPUT) { + params->h_vlan_proto = htons(test->vlan_proto); + params->h_vlan_TCI = htons(test->vlan_id); + } if (inet_pton(AF_INET6, test->daddr, params->ipv6_dst) == 1) { params->family = AF_INET6; @@ -298,7 +688,7 @@ void test_fib_lookup(void) struct nstoken *nstoken = NULL; struct __sk_buff skb = { }; struct fib_lookup *skel; - int prog_fd, err, ret, i; + int prog_fd, xdp_fd, err, ret, i; /* The test does not use the skb->data, so * use pkt_v6 for both v6 and v4 test. @@ -309,11 +699,16 @@ void test_fib_lookup(void) .ctx_in = &skb, .ctx_size_in = sizeof(skb), ); + LIBBPF_OPTS(bpf_test_run_opts, xdp_opts, + .data_in = &pkt_v6, + .data_size_in = sizeof(pkt_v6), + ); skel = fib_lookup__open_and_load(); if (!ASSERT_OK_PTR(skel, "skel open_and_load")) return; prog_fd = bpf_program__fd(skel->progs.fib_lookup); + xdp_fd = bpf_program__fd(skel->progs.fib_lookup_xdp); SYS(fail, "ip netns add %s", NS_TEST); @@ -343,6 +738,16 @@ void test_fib_lookup(void) if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) continue; + /* + * BPF_FIB_LOOKUP_VLAN is XDP-only; the tc helper rejects it. + * These cases are exercised on the XDP path below. + */ + if (tests[i].lookup_flags & BPF_FIB_LOOKUP_VLAN) { + ASSERT_EQ(skel->bss->fib_lookup_ret, -EINVAL, + "tc rejects BPF_FIB_LOOKUP_VLAN"); + continue; + } + ASSERT_EQ(skel->bss->fib_lookup_ret, tests[i].expected_ret, "fib_lookup_ret"); @@ -352,6 +757,21 @@ void test_fib_lookup(void) if (tests[i].expected_dst) assert_dst_ip(fib_params, tests[i].expected_dst); + if (tests[i].expected_dev) + ASSERT_EQ(fib_params->ifindex, + if_nametoindex(tests[i].expected_dev), "ifindex"); + + if (tests[i].expected_mtu) + ASSERT_EQ(fib_params->mtu_result, tests[i].expected_mtu, + "mtu_result"); + + if (tests[i].check_vlan) { + ASSERT_EQ(fib_params->h_vlan_proto, + htons(tests[i].vlan_proto), "h_vlan_proto"); + ASSERT_EQ(fib_params->h_vlan_TCI, + htons(tests[i].vlan_id), "h_vlan_TCI"); + } + ret = memcmp(tests[i].dmac, fib_params->dmac, sizeof(tests[i].dmac)); if (!ASSERT_EQ(ret, 0, "dmac not match")) { char expected[18], actual[18]; @@ -361,17 +781,330 @@ void test_fib_lookup(void) printf("dmac expected %s actual %s ", expected, actual); } - // ensure tbid is zero'd out after fib lookup. - if (tests[i].lookup_flags & BPF_FIB_LOOKUP_DIRECT) { + /* + * ensure tbid is zero'd out after fib lookup. With + * BPF_FIB_LOOKUP_VLAN the union holds the packed vlan + * fields instead, so skip the check for those. + */ + if ((tests[i].lookup_flags & BPF_FIB_LOOKUP_DIRECT) && + !(tests[i].lookup_flags & BPF_FIB_LOOKUP_VLAN)) { if (!ASSERT_EQ(skel->bss->fib_params.tbid, 0, "expected fib_params.tbid to be zero")) goto fail; } } + /* + * Re-run the cases through bpf_xdp_fib_lookup(). test_run uses the + * current netns' loopback for ctx->rxq->dev, so dev_net() is NS_TEST + * and the lookup runs against its FIB. The path-independent results + * (return code, swapped ifindex, vlan tag, gateway) must match the skb + * path; the no-tot_len mtu_result is skb-specific and not rechecked. + */ + for (i = 0; i < ARRAY_SIZE(tests); i++) { + if (set_lookup_params(fib_params, &tests[i], skb.ifindex)) + continue; + + skel->bss->fib_lookup_ret = -1; + skel->bss->lookup_flags = tests[i].lookup_flags; + + err = bpf_prog_test_run_opts(xdp_fd, &xdp_opts); + if (!ASSERT_OK(err, "xdp test_run")) + continue; + + if (!ASSERT_EQ(skel->bss->fib_lookup_ret, tests[i].expected_ret, + "xdp fib_lookup_ret")) + printf("(xdp) %s\n", tests[i].desc); + + if (tests[i].expected_dev) + ASSERT_EQ(fib_params->ifindex, + if_nametoindex(tests[i].expected_dev), + "xdp ifindex"); + + if (tests[i].expected_dst) + assert_dst_ip(fib_params, tests[i].expected_dst); + + if (tests[i].check_vlan) { + ASSERT_EQ(fib_params->h_vlan_proto, + htons(tests[i].vlan_proto), "xdp h_vlan_proto"); + ASSERT_EQ(fib_params->h_vlan_TCI, + htons(tests[i].vlan_id), "xdp h_vlan_TCI"); + } + + ret = memcmp(tests[i].dmac, fib_params->dmac, sizeof(tests[i].dmac)); + ASSERT_EQ(ret, 0, "xdp dmac"); + + /* + * mtu_result from a tot_len lookup is the route mtu and is + * path-independent; the no-tot_len arm reads dev->mtu and is + * skb-only, so gate on tot_len + */ + if (tests[i].expected_mtu && tests[i].tot_len) + ASSERT_EQ(fib_params->mtu_result, tests[i].expected_mtu, + "xdp mtu_result"); + } + fail: if (nstoken) close_netns(nstoken); SYS_NOFAIL("ip netns del " NS_TEST); fib_lookup__destroy(skel); } + +#define NS_VLAN_A "fib_lookup_vlan_ns_a" +#define NS_VLAN_B "fib_lookup_vlan_ns_b" +#define IPV4_VLAN_NETNS_ADDR "10.66.0.1" +#define IPV4_VLAN_NETNS_DST "10.66.0.2" + +/* + * A VLAN device can be moved to another netns while staying registered + * on its parent. Neither direction may then cross the boundary: the + * egress flag must not publish the foreign parent's ifindex, and the + * input flag must fail closed rather than use a foreign ingress. + */ +void test_fib_lookup_vlan_netns(void) +{ + struct bpf_fib_lookup *fib_params; + struct nstoken *nstoken = NULL; + struct __sk_buff skb = { }; + struct fib_lookup *skel = NULL; + int prog_fd, xdp_fd, err, parent_idx, vlan_idx; + + LIBBPF_OPTS(bpf_test_run_opts, run_opts, + .data_in = &pkt_v6, + .data_size_in = sizeof(pkt_v6), + .ctx_in = &skb, + .ctx_size_in = sizeof(skb), + ); + LIBBPF_OPTS(bpf_test_run_opts, xdp_opts, + .data_in = &pkt_v6, + .data_size_in = sizeof(pkt_v6), + ); + + skel = fib_lookup__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel open_and_load")) + return; + prog_fd = bpf_program__fd(skel->progs.fib_lookup); + xdp_fd = bpf_program__fd(skel->progs.fib_lookup_xdp); + fib_params = &skel->bss->fib_params; + + SYS(fail, "ip netns add %s", NS_VLAN_A); + SYS(fail, "ip netns add %s", NS_VLAN_B); + + nstoken = open_netns(NS_VLAN_A); + if (!ASSERT_OK_PTR(nstoken, "open_netns(a)")) + goto fail; + + SYS(fail, "ip link add veth7 type veth peer name veth8"); + SYS(fail, "ip link set dev veth7 up"); + SYS(fail, "ip link add link veth7 name veth7.66 type vlan id 66"); + SYS(fail, "ip link set veth7.66 netns %s", NS_VLAN_B); + /* + * up it in B before the input lookup: the move closed it, and a + * down device fails the resolver on IFF_UP before reaching the + * netns check this subtest exists to pin + */ + SYS(fail, "ip -n %s link set dev veth7.66 up", NS_VLAN_B); + + parent_idx = if_nametoindex("veth7"); + if (!ASSERT_NEQ(parent_idx, 0, "if_nametoindex(veth7)")) + goto fail; + + /* + * give this netns a route to the destination: the lookup below runs + * against this FIB, so without the route a kernel that resolved the + * moved device anyway would still return NOT_FWDED and the arm would + * pass for the wrong reason + */ + SYS(fail, "ip route add %s/32 dev veth7", IPV4_VLAN_NETNS_DST); + + /* + * input: the moved device is still in veth7's VLAN group, but it + * lives in another netns, so the lookup must fail closed + */ + skb.ifindex = parent_idx; + memset(fib_params, 0, sizeof(*fib_params)); + fib_params->family = AF_INET; + fib_params->l4_protocol = IPPROTO_TCP; + fib_params->ifindex = parent_idx; + fib_params->h_vlan_proto = htons(ETH_P_8021Q); + fib_params->h_vlan_TCI = htons(66); + if (!ASSERT_EQ(inet_pton(AF_INET, IPV4_VLAN_NETNS_DST, &fib_params->ipv4_dst), + 1, "inet_pton(dst)")) + goto fail; + + skel->bss->fib_lookup_ret = -1; + skel->bss->lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | + BPF_FIB_LOOKUP_SKIP_NEIGH; + err = bpf_prog_test_run_opts(prog_fd, &run_opts); + if (!ASSERT_OK(err, "test_run(input)")) + goto fail; + ASSERT_EQ(skel->bss->fib_lookup_ret, BPF_FIB_LKUP_RET_NOT_FWDED, + "input across netns fails closed"); + ASSERT_EQ(fib_params->ifindex, parent_idx, "ifindex untouched"); + ASSERT_EQ(fib_params->h_vlan_TCI, htons(66), "tag untouched"); + + close_netns(nstoken); + nstoken = open_netns(NS_VLAN_B); + if (!ASSERT_OK_PTR(nstoken, "open_netns(b)")) + goto fail; + + /* + * egress: the fib result is the VLAN device here, but its parent + * is in the other netns, so the swap must not happen + */ + SYS(fail, "ip addr add %s/24 dev veth7.66", IPV4_VLAN_NETNS_ADDR); + err = write_sysctl("/proc/sys/net/ipv4/conf/veth7.66/forwarding", "1"); + if (!ASSERT_OK(err, "write_sysctl(forwarding)")) + goto fail; + + vlan_idx = if_nametoindex("veth7.66"); + if (!ASSERT_NEQ(vlan_idx, 0, "if_nametoindex(veth7.66)")) + goto fail; + + memset(fib_params, 0, sizeof(*fib_params)); + fib_params->family = AF_INET; + fib_params->l4_protocol = IPPROTO_TCP; + fib_params->ifindex = vlan_idx; + if (!ASSERT_EQ(inet_pton(AF_INET, IPV4_VLAN_NETNS_DST, &fib_params->ipv4_dst), + 1, "inet_pton(dst)") || + !ASSERT_EQ(inet_pton(AF_INET, IPV4_VLAN_NETNS_ADDR, &fib_params->ipv4_src), + 1, "inet_pton(src)")) + goto fail; + + skel->bss->fib_lookup_ret = -1; + skel->bss->lookup_flags = BPF_FIB_LOOKUP_VLAN | + BPF_FIB_LOOKUP_SKIP_NEIGH; + err = bpf_prog_test_run_opts(xdp_fd, &xdp_opts); + if (!ASSERT_OK(err, "test_run(egress)")) + goto fail; + ASSERT_EQ(skel->bss->fib_lookup_ret, BPF_FIB_LKUP_RET_VLAN_FAILURE, + "egress returns VLAN_FAILURE"); + ASSERT_EQ(fib_params->ifindex, vlan_idx, + "foreign parent not published"); + ASSERT_EQ(fib_params->h_vlan_TCI, 0, "vlan fields zero"); + +fail: + if (nstoken) + close_netns(nstoken); + SYS_NOFAIL("ip netns del " NS_VLAN_A); + SYS_NOFAIL("ip netns del " NS_VLAN_B); + fib_lookup__destroy(skel); +} + +#define REDIRECT_NPKTS 1000 +#define NS_REDIRECT "fib_lookup_redirect_ns" + +/* + * The egress flag exists so an XDP program can redirect to the physical + * parent. A redirect that lands on a VLAN device is dropped at + * xdp_do_flush(), because a VLAN device has no ndo_xdp_xmit. Drive real + * frames with BPF_F_TEST_XDP_LIVE_FRAMES, which runs the native + * xdp_do_redirect() + xdp_do_flush() path: a reducible VLAN egress + * resolves to veth1 and is delivered to its peer veth2, while a QinQ + * egress returns VLAN_FAILURE and is passed to the stack instead of + * redirected to a device that would silently drop it. + */ +void test_fib_lookup_vlan_redirect(void) +{ + int redirect_fd, err, veth1_idx, veth2_idx = -1; + struct bpf_fib_lookup *fib_params; + struct nstoken *nstoken = NULL; + struct fib_lookup *skel = NULL; + bool xdp_attached = false; + + LIBBPF_OPTS(bpf_test_run_opts, lf_opts, + .data_in = &pkt_v4, + .data_size_in = sizeof(pkt_v4), + .flags = BPF_F_TEST_XDP_LIVE_FRAMES, + .repeat = REDIRECT_NPKTS, + ); + + skel = fib_lookup__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel open_and_load")) + return; + redirect_fd = bpf_program__fd(skel->progs.fib_lookup_redirect); + fib_params = &skel->bss->fib_params; + + SYS(fail, "ip netns add %s", NS_REDIRECT); + nstoken = open_netns(NS_REDIRECT); + if (!ASSERT_OK_PTR(nstoken, "open_netns")) + goto fail; + if (setup_netns()) + goto fail; + + veth1_idx = if_nametoindex("veth1"); + veth2_idx = if_nametoindex("veth2"); + if (!ASSERT_NEQ(veth1_idx, 0, "if_nametoindex(veth1)") || + !ASSERT_NEQ(veth2_idx, 0, "if_nametoindex(veth2)")) + goto fail; + + /* + * A redirect to veth1 is delivered to its peer veth2. veth_xdp_xmit() + * only accepts the frame if veth2's NAPI is up, which on veth means + * veth2 carries an XDP program; xdp_count tallies what arrives. + */ + err = bpf_xdp_attach(veth2_idx, bpf_program__fd(skel->progs.xdp_count), + XDP_FLAGS_DRV_MODE, NULL); + if (!ASSERT_OK(err, "attach xdp_count on veth2")) + goto fail; + xdp_attached = true; + + /* reducible VLAN egress: resolves to the physical parent veth1 */ + memset(fib_params, 0, sizeof(*fib_params)); + fib_params->family = AF_INET; + fib_params->l4_protocol = IPPROTO_TCP; + fib_params->ifindex = veth1_idx; + if (!ASSERT_EQ(inet_pton(AF_INET, IPV4_IFACE_ADDR, &fib_params->ipv4_src), + 1, "inet_pton(src)") || + !ASSERT_EQ(inet_pton(AF_INET, IPV4_VLAN_EGRESS_DST, &fib_params->ipv4_dst), + 1, "inet_pton(reducible dst)")) + goto fail; + skel->bss->lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH; + skel->bss->redirected = 0; + skel->bss->passed = 0; + skel->bss->delivered = 0; + + err = bpf_prog_test_run_opts(redirect_fd, &lf_opts); + if (!ASSERT_OK(err, "test_run(reducible egress)")) + goto fail; + ASSERT_EQ(skel->bss->redirected, REDIRECT_NPKTS, "reducible egress redirected"); + ASSERT_EQ(skel->bss->passed, 0, "reducible egress not passed"); + ASSERT_GT(skel->bss->delivered, 0, "reducible egress delivered to veth2"); + + /* + * QinQ egress: not reducible, so the lookup returns VLAN_FAILURE and + * the program passes the frame instead of redirecting to the inner + * VLAN device. redirected == 0 is the assertion that matters: the + * program did not redirect to a device that would drop the frame at + * xdp_do_flush(). veth2's delivered count is not checked here, since + * a passed frame can still reach veth2 through the stack's forwarding + * path, which is unrelated to the redirect under test. + */ + memset(fib_params, 0, sizeof(*fib_params)); + fib_params->family = AF_INET; + fib_params->l4_protocol = IPPROTO_TCP; + fib_params->ifindex = veth1_idx; + if (!ASSERT_EQ(inet_pton(AF_INET, IPV4_IFACE_ADDR, &fib_params->ipv4_src), + 1, "inet_pton(src)") || + !ASSERT_EQ(inet_pton(AF_INET, IPV4_QINQ_DST, &fib_params->ipv4_dst), + 1, "inet_pton(qinq dst)")) + goto fail; + skel->bss->lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH; + skel->bss->redirected = 0; + skel->bss->passed = 0; + + err = bpf_prog_test_run_opts(redirect_fd, &lf_opts); + if (!ASSERT_OK(err, "test_run(qinq egress)")) + goto fail; + ASSERT_EQ(skel->bss->passed, REDIRECT_NPKTS, "qinq egress passed"); + ASSERT_EQ(skel->bss->redirected, 0, "qinq egress not redirected"); + +fail: + if (xdp_attached) + bpf_xdp_detach(veth2_idx, XDP_FLAGS_DRV_MODE, NULL); + if (nstoken) + close_netns(nstoken); + SYS_NOFAIL("ip netns del " NS_REDIRECT); + fib_lookup__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c index f589eefbf9fb..0918321c8e63 100644 --- a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c +++ b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c @@ -4,6 +4,7 @@ #include <string.h> #include <linux/bpf.h> #include <linux/limits.h> +#include <bpf/btf.h> #include <test_progs.h> #include "trace_helpers.h" #include "test_fill_link_info.skel.h" @@ -24,6 +25,22 @@ static __u64 kmulti_cookies[] = { 3, 1, 2 }; #define KPROBE_FUNC "bpf_fentry_test1" static __u64 kprobe_addr; +static const char * const tmulti_syms[] = { + "bpf_fentry_test2", + "bpf_fentry_test1", + "bpf_fentry_test3", +}; + +static __u64 tmulti_cookies[] = { 30, 10, 20 }; +#define TRACING_MULTI_CNT ARRAY_SIZE(tmulti_syms) + +struct tmulti_target { + const char *name; + __u64 addr; + __u64 cookie; + __u32 id; +}; + #define UPROBE_FILE "/proc/self/exe" static ssize_t uprobe_offset; /* uprobe attach point */ @@ -396,6 +413,224 @@ static void test_kprobe_multi_fill_link_info(struct test_fill_link_info *skel, bpf_link__destroy(link); } +static int tmulti_target_cmp(const void *a, const void *b) +{ + const struct tmulti_target *ta = a; + const struct tmulti_target *tb = b; + + return (ta->id > tb->id) - (ta->id < tb->id); +} + +static int setup_tmulti_targets(const struct bpf_program *prog, + struct tmulti_target *targets, + __u32 *btf_obj_id) +{ + struct bpf_prog_info prog_info; + __u32 len = sizeof(prog_info); + struct btf *btf; + int err, i; + __s32 id; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "btf__load_vmlinux_btf")) + return -1; + + for (i = 0; i < TRACING_MULTI_CNT; i++) { + id = btf__find_by_name_kind(btf, tmulti_syms[i], BTF_KIND_FUNC); + if (!ASSERT_GT(id, 0, "btf__find_by_name_kind")) + goto error; + + targets[i].name = tmulti_syms[i]; + targets[i].addr = ksym_get_addr(tmulti_syms[i]); + targets[i].cookie = tmulti_cookies[i]; + targets[i].id = id; + } + + memset(&prog_info, 0, len); + err = bpf_prog_get_info_by_fd(bpf_program__fd(prog), &prog_info, &len); + if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd")) + goto error; + if (!ASSERT_GT(prog_info.attach_btf_obj_id, 0, "attach_btf_obj_id")) + goto error; + *btf_obj_id = prog_info.attach_btf_obj_id; + + /* + * The kernel tracing multi attach sorts ids. We sort as well, + * so we can easily compare ids and cookies later. + */ + qsort(targets, TRACING_MULTI_CNT, sizeof(targets[0]), tmulti_target_cmp); + btf__free(btf); + return 0; + +error: + btf__free(btf); + return -1; +} + +static int verify_tracing_multi_link_info(int fd, const struct bpf_program *prog, + const struct tmulti_target *targets, + __u32 btf_obj_id, bool has_cookies) +{ + enum bpf_attach_type attach_type = bpf_program__expected_attach_type(prog); + __u64 addrs[TRACING_MULTI_CNT], cookies[TRACING_MULTI_CNT]; + __u32 ids[TRACING_MULTI_CNT]; + struct bpf_link_info info; + __u32 len = sizeof(info); + int err, i; + + memset(&info, 0, sizeof(info)); + err = bpf_link_get_info_by_fd(fd, &info, &len); + if (!ASSERT_OK(err, "bpf_link_get_info_by_fd")) + return -1; + + if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type")) + return -1; + + ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type"); + ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count"); + + memset(ids, 0, sizeof(ids)); + memset(cookies, 0, sizeof(cookies)); + memset(addrs, 0, sizeof(addrs)); + + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.addrs = ptr_to_u64(addrs); + info.tracing_multi.cookies = has_cookies ? ptr_to_u64(cookies) : 0; + info.tracing_multi.count = TRACING_MULTI_CNT; + + err = bpf_link_get_info_by_fd(fd, &info, &len); + if (!ASSERT_OK(err, "bpf_link_get_info_by_fd")) + return -1; + + if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type")) + return -1; + + ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type"); + ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count"); + ASSERT_EQ(info.tracing_multi.btf_obj_id, btf_obj_id, "tracing_multi.btf_obj_id"); + + for (i = 0; i < TRACING_MULTI_CNT; i++) { + ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.ids"); + ASSERT_EQ(cookies[i], has_cookies ? targets[i].cookie : 0, "tracing_multi.cookies"); + + if (targets[i].addr) { + struct ksym *ksym; + + if (!ASSERT_NEQ(addrs[i], 0, "tracing_multi.addrs")) + return -1; + ksym = ksym_search(addrs[i]); + if (!ASSERT_OK_PTR(ksym, "ksym_search")) + return -1; + ASSERT_STREQ(ksym->name, targets[i].name, "tracing_multi.addr_name"); + } else { + ASSERT_EQ(addrs[i], 0, "tracing_multi.addrs"); + } + } + + return 0; +} + +static void verify_tracing_multi_invalid_user_buffer(int fd, const struct tmulti_target *targets) +{ + __u32 ids[TRACING_MULTI_CNT] = {}; + struct bpf_link_info info; + __u32 len = sizeof(info); + int err, i; + + /* Wrong info setup (ids != NULL and cnt == 0) -> EINVAL */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = ptr_to_u64(ids); + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EINVAL, "tracing_multi.invalid_count"); + + /* Smaller than actual count provided -> ENOSPC */ + memset(ids, 0, sizeof(ids)); + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.count = TRACING_MULTI_CNT - 1; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -ENOSPC, "tracing_multi.small_count"); + for (i = 0; i < TRACING_MULTI_CNT - 1; i++) + ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.partial_ids"); + /* check that the last entry is not populated */ + ASSERT_EQ(ids[i], 0, "tracing_multi.partial_ids"); + + /* Bigger than actual count provided -> OK */ + memset(ids, 0, sizeof(ids)); + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.count = TRACING_MULTI_CNT + 1; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_OK(err, "tracing_multi.big_count"); + for (i = 0; i < TRACING_MULTI_CNT; i++) + ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.ids"); + + /* Invalid ids pointer -> EFAULT */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = 0x1; + info.tracing_multi.count = TRACING_MULTI_CNT; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_btf_ids"); + + /* Invalid cookies pointer -> EFAULT */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.cookies = 0x1; + info.tracing_multi.count = TRACING_MULTI_CNT; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_cookies"); + + /* Invalid addrs pointer -> EFAULT */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.addrs = 0x1; + info.tracing_multi.count = TRACING_MULTI_CNT; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_addrs"); +} + +static void test_tracing_multi_fill_link_info(struct test_fill_link_info *skel, + bool has_cookies, bool invalid) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tmulti_target targets[TRACING_MULTI_CNT]; + __u32 ids[TRACING_MULTI_CNT], btf_obj_id; + __u64 cookies[TRACING_MULTI_CNT]; + struct bpf_link *link; + int link_fd, err, i; + +#ifndef __x86_64__ + test__skip(); + return; +#endif + + if (setup_tmulti_targets(skel->progs.tmulti_run, targets, &btf_obj_id)) + return; + + for (i = 0; i < TRACING_MULTI_CNT; i++) { + ids[i] = targets[i].id; + cookies[i] = targets[i].cookie; + } + + opts.ids = ids; + opts.cnt = TRACING_MULTI_CNT; + if (has_cookies) + opts.cookies = cookies; + + link = bpf_program__attach_tracing_multi(skel->progs.tmulti_run, NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_tracing_multi")) + return; + + link_fd = bpf_link__fd(link); + if (invalid) { + verify_tracing_multi_invalid_user_buffer(link_fd, targets); + } else { + err = verify_tracing_multi_link_info(link_fd, skel->progs.tmulti_run, + targets, btf_obj_id, has_cookies); + ASSERT_OK(err, "verify_tracing_multi_link_info"); + } + + bpf_link__destroy(link); +} + #define SEC(name) __attribute__((section(name), used)) static short uprobe_link_info_sema_1 SEC(".probes"); @@ -640,6 +875,13 @@ void test_fill_link_info(void) if (test__start_subtest("kprobe_multi_invalid_ubuff")) test_kprobe_multi_fill_link_info(skel, true, true, true); + if (test__start_subtest("tracing_multi_link_info")) { + test_tracing_multi_fill_link_info(skel, false, false); + test_tracing_multi_fill_link_info(skel, true, false); + } + if (test__start_subtest("tracing_multi_invalid_ubuff")) + test_tracing_multi_fill_link_info(skel, true, true); + if (test__start_subtest("uprobe_multi_link_info")) test_uprobe_multi_fill_link_info(skel, false, false); if (test__start_subtest("uretprobe_multi_link_info")) diff --git a/tools/testing/selftests/bpf/prog_tests/get_smp_processor_id.c b/tools/testing/selftests/bpf/prog_tests/get_smp_processor_id.c new file mode 100644 index 000000000000..1b5c738ab81f --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/get_smp_processor_id.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "bpf/libbpf_internal.h" +#include "get_smp_processor_id.skel.h" + +void test_get_smp_processor_id(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts, + .flags = BPF_F_TEST_RUN_ON_CPU, + .cpu = 0, + ); + struct get_smp_processor_id *skel; + int prog_fd, err, online_cpu_nr, i; + bool *online = NULL; + + err = parse_cpu_mask_file("/sys/devices/system/cpu/online", + &online, &online_cpu_nr); + if (!ASSERT_OK(err, "parse_cpu_mask_file")) + return; + + skel = get_smp_processor_id__open_and_load(); + if (!ASSERT_OK_PTR(skel, "get_smp_processor_id__open_and_load")) + goto cleanup; + + prog_fd = bpf_program__fd(skel->progs.call_bpf_get_smp_processor_id); + + for (i = 0; i < online_cpu_nr; i++) { + if (!online[i]) + continue; + + opts.cpu = i; + skel->bss->cpu_nr_result = -1; + + err = bpf_prog_test_run_opts(prog_fd, &opts); + if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) + goto cleanup; + + ASSERT_EQ(skel->bss->cpu_nr_result, opts.cpu, "cpu_nr_result"); + } + +cleanup: + free(online); + get_smp_processor_id__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/global_data_init.c b/tools/testing/selftests/bpf/prog_tests/global_data_init.c index 8466332d7406..5671c31085cd 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_data_init.c +++ b/tools/testing/selftests/bpf/prog_tests/global_data_init.c @@ -1,5 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 #include <test_progs.h> +#include "bpf/libbpf_internal.h" +#include "test_global_percpu_data.skel.h" +#include "test_global_percpu_data.lskel.h" void test_global_data_init(void) { @@ -60,3 +63,336 @@ out: free(newval); bpf_object__close(obj); } + +static void test_percpu_data_on_cpus(struct bpf_map *map, int map_fd, int prog_fd, int *runp) +{ + struct test_global_percpu_data__percpu *data = NULL; + int i, err, key = 0, num_online, run = 0; + __u64 args[2] = {0x1234ULL, 0x5678ULL}; + size_t data_sz; + bool *online; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .ctx_in = args, + .ctx_size_in = sizeof(args), + .flags = BPF_F_TEST_RUN_ON_CPU, + ); + + err = parse_cpu_mask_file("/sys/devices/system/cpu/online", &online, &num_online); + if (!ASSERT_OK(err, "parse_cpu_mask_file")) + return; + + data_sz = map ? bpf_map__value_size(map) : sizeof(*data); + data = calloc(1, data_sz); + if (!ASSERT_OK_PTR(data, "calloc percpu data")) + goto out; + + /* run on every online-CPU */ + for (i = 0; i < num_online; i++) { + __u64 flags; + + if (!online[i]) + continue; + + topts.cpu = i; + topts.retval = -1; + err = bpf_prog_test_run_opts(prog_fd, &topts); + ASSERT_OK(err, "bpf_prog_test_run_opts"); + ASSERT_EQ(topts.retval, 0, "bpf_prog_test_run_opts retval"); + + memset(data, 0, data_sz); + flags = ((__u64) i << 32) | BPF_F_CPU; + if (map) + err = bpf_map__lookup_elem(map, &key, sizeof(key), data, data_sz, flags); + else + err = bpf_map_lookup_elem_flags(map_fd, &key, data, flags); + if (!ASSERT_OK(err, "lookup_elem on cpu")) + break; + + ASSERT_EQ(*runp, ++run, "run"); + ASSERT_EQ(data->cpu_id[0], i, "cpu_id"); + ASSERT_EQ(data->data, 1, "data"); + ASSERT_TRUE(data->set, "set"); + ASSERT_EQ(data->nums[6], 0xc0de, "nums[6]"); + ASSERT_EQ(data->struct_data.i, 1, "struct_data.i"); + ASSERT_TRUE(data->struct_data.set, "struct_data.set"); + ASSERT_EQ(data->struct_data.nums[6], 0xc0de, "struct_data.nums[6]"); + } + +out: + free(data); + free(online); +} + +static void test_global_percpu_data_init(void) +{ + struct test_global_percpu_data__percpu init_value = {}; + struct test_global_percpu_data__percpu *init_data; + const __u32 desired_sz = sysconf(_SC_PAGE_SIZE); + struct test_global_percpu_data *skel = NULL; + size_t init_data_sz; + struct bpf_map *map; + int prog_fd, err; + + skel = test_global_percpu_data__open(); + if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open")) + goto out; + if (!ASSERT_OK_PTR(skel->percpu, "skel->percpu")) + goto out; + if (!ASSERT_OK_PTR(skel->data_percpu, "skel->data_percpu")) + goto out; + if (!ASSERT_OK_PTR(skel->percpu_data, "skel->percpu_data")) + goto out; + if (!ASSERT_OK_PTR(skel->percpu_looooooooong, "skel->percpu_looooooooong")) + goto out; + + ASSERT_STREQ(bpf_map__name(skel->maps.percpu_data), ".percpu.data", + ".percpu.data map name"); + ASSERT_STREQ(bpf_map__name(skel->maps.data_percpu), ".data.percpu", + ".data.percpu map name"); + ASSERT_STREQ(bpf_map__name(skel->maps.percpu_looooooooong), ".percpu.looooooooong", + "long map name"); + ASSERT_STREQ(bpf_map__name(skel->maps.percpu), ".percpu", "map name"); + ASSERT_EQ(skel->percpu->data, -1, "skel->percpu->data"); + ASSERT_FALSE(skel->percpu->set, "skel->percpu->set"); + ASSERT_EQ(skel->percpu->nums[6], 0, "skel->percpu->nums[6]"); + ASSERT_EQ(skel->percpu->struct_data.i, -1, "struct_data.i"); + ASSERT_FALSE(skel->percpu->struct_data.set, "struct_data.set"); + ASSERT_EQ(skel->percpu->struct_data.nums[6], 0, "struct_data.nums[6]"); + + map = skel->maps.percpu; + if (!ASSERT_EQ(bpf_map__type(map), BPF_MAP_TYPE_PERCPU_ARRAY, "bpf_map__type")) + goto out; + + init_value.data = 2; + init_value.nums[6] = -1; + init_value.struct_data.i = 2; + init_value.struct_data.nums[6] = -1; + err = bpf_map__set_initial_value(map, &init_value, sizeof(init_value)); + if (!ASSERT_OK(err, "bpf_map__set_initial_value")) + goto out; + + init_data = bpf_map__initial_value(map, &init_data_sz); + if (!ASSERT_OK_PTR(init_data, "bpf_map__initial_value")) + goto out; + + ASSERT_EQ(init_data->data, init_value.data, "init_value data"); + ASSERT_EQ(init_data->set, init_value.set, "init_value set"); + ASSERT_EQ(init_data->struct_data.i, init_value.struct_data.i, "init_value struct_data.i"); + ASSERT_EQ(init_data->struct_data.nums[6], init_value.struct_data.nums[6], + "init_value struct_data.nums[6]"); + ASSERT_EQ(init_data_sz, sizeof(init_value), "init_value size"); + ASSERT_EQ((void *) init_data, (void *) skel->percpu, "skel->percpu eq init_data"); + ASSERT_EQ(skel->percpu->data, init_value.data, "skel->percpu->data"); + ASSERT_EQ(skel->percpu->set, init_value.set, "skel->percpu->set"); + ASSERT_EQ(skel->percpu->struct_data.i, init_value.struct_data.i, + "skel->percpu->struct_data.i"); + ASSERT_EQ(skel->percpu->struct_data.nums[6], init_value.struct_data.nums[6], + "skel->percpu->struct_data.nums[6]"); + + ASSERT_GT(desired_sz, sizeof(init_value), "desired_sz"); + err = bpf_map__set_value_size(map, desired_sz); + if (!ASSERT_OK(err, "bpf_map__set_value_size")) + goto out; + if (!ASSERT_EQ(bpf_map__value_size(map), desired_sz, "percpu value size")) + goto out; + if (!ASSERT_NEQ(bpf_map__btf_value_type_id(map), 0, "percpu BTF value type")) + goto out; + + init_data = bpf_map__initial_value(map, &init_data_sz); + if (!ASSERT_OK_PTR(init_data, "resized bpf_map__initial_value")) + goto out; + if (!ASSERT_EQ(init_data_sz, desired_sz, "resized initial value size")) + goto out; + if (!ASSERT_EQ(init_data->data, init_value.data, "resized initial value data")) + goto out; + + err = test_global_percpu_data__load(skel); + if (!ASSERT_OK(err, "test_global_percpu_data__load")) + goto out; + + ASSERT_OK_PTR(skel->percpu, "skel->percpu"); + + prog_fd = bpf_program__fd(skel->progs.update_percpu_data); + test_percpu_data_on_cpus(map, bpf_map__fd(map), prog_fd, &skel->bss->run); + +out: + test_global_percpu_data__destroy(skel); +} + +static void test_global_percpu_data_lskel(void) +{ + struct test_global_percpu_data_lskel *lskel = NULL; + int prog_fd, map_fd; + + lskel = test_global_percpu_data_lskel__open_and_load(); + if (!ASSERT_OK_PTR(lskel, "test_global_percpu_data_lskel__open_and_load")) + goto out; + + map_fd = lskel->maps.percpu.map_fd; + prog_fd = lskel->progs.update_percpu_data.prog_fd; + test_percpu_data_on_cpus(NULL, map_fd, prog_fd, &lskel->bss->run); + +out: + test_global_percpu_data_lskel__destroy(lskel); +} + +static int create_rdonly_percpu_array(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, map_opts, + .map_flags = BPF_F_RDONLY_PROG, + ); + int key = 0, map_fd, err; + __u64 value = 0; + + map_fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "percpu_ro_map", sizeof(int), + sizeof(__u64), 1, &map_opts); + if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) + return -1; + + err = bpf_map_update_elem(map_fd, &key, &value, BPF_F_ALL_CPUS); + if (!ASSERT_OK(err, "bpf_map_update_elem")) + goto out; + + err = bpf_map_freeze(map_fd); + if (!ASSERT_OK(err, "bpf_map_freeze")) + goto out; + + return map_fd; + +out: + close(map_fd); + return -1; +} + +static void test_global_percpu_data_rdonly_direct_read(void) +{ + /* + * Raw instructions with manually prepared rdonly percpu_array map + * for testing direct-read global percpu data, because libbpf + * doesn't have rdonly internal percpu_array map support for + * global percpu data. + */ + struct bpf_insn insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_1, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0), + BPF_EXIT_INSN(), + }; + int map_fd, prog_fd; + + map_fd = create_rdonly_percpu_array(); + if (map_fd < 0) + return; + + insns[0].imm = map_fd; + prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "percpu_ro_prog", "GPL", insns, + ARRAY_SIZE(insns), NULL); + if (ASSERT_GE(prog_fd, 0, "bpf_prog_load")) + close(prog_fd); + close(map_fd); +} + +static void test_global_percpu_data_rdonly_direct_write(void) +{ + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts); + /* See the comment in test_global_percpu_data_rdonly_direct_read() */ + struct bpf_insn insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_1, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0), + BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0), + BPF_EXIT_INSN(), + }; + char log_buf[256] = {}; + int map_fd, prog_fd; + + prog_opts.log_buf = log_buf; + prog_opts.log_size = sizeof(log_buf); + prog_opts.log_level = 1; + + map_fd = create_rdonly_percpu_array(); + if (map_fd < 0) + return; + + insns[0].imm = map_fd; + prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "percpu_ro_prog", "GPL", insns, + ARRAY_SIZE(insns), &prog_opts); + if (!ASSERT_LT(prog_fd, 0, "bpf_prog_load")) + close(prog_fd); + else + ASSERT_HAS_SUBSTR(log_buf, "write into map forbidden", "verifier log"); + close(map_fd); +} + +static void test_global_percpu_data_verifier_log(void) +{ + RUN_TESTS(test_global_percpu_data); +} + +static void test_global_percpu_data_iter(void) +{ + DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); + struct test_global_percpu_data *skel; + union bpf_iter_link_info linfo = {}; + struct bpf_link *link = NULL; + int fd, num_cpus, len, err; + char buf[16]; + + num_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(num_cpus, 0, "libbpf_num_possible_cpus")) + return; + + skel = test_global_percpu_data__open(); + if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open")) + return; + + skel->rodata->num_cpus = num_cpus; + skel->rodata->num_off = offsetof(struct test_global_percpu_data__percpu, + struct_data.nums[6]); + skel->rodata->elem_sz = roundup(sizeof(struct test_global_percpu_data__percpu), 8); + skel->percpu->struct_data.nums[6] = 0xc0de; + + err = test_global_percpu_data__load(skel); + if (!ASSERT_OK(err, "test_global_percpu_data__load")) + goto out; + + linfo.map.map_fd = bpf_map__fd(skel->maps.percpu); + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + link = bpf_program__attach_iter(skel->progs.dump_percpu_data, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_iter")) + goto out; + + fd = bpf_iter_create(bpf_link__fd(link)); + if (!ASSERT_GE(fd, 0, "bpf_iter_create")) + goto out; + + while ((len = read(fd, buf, sizeof(buf))) > 0) + do { } while (0); + ASSERT_EQ(len, 0, "read iter"); + ASSERT_TRUE(skel->bss->run_iter, "run_iter"); + ASSERT_EQ(skel->bss->sum, 0xc0de * num_cpus, "sum"); + + close(fd); +out: + bpf_link__destroy(link); + test_global_percpu_data__destroy(skel); +} + +void test_global_percpu_data(void) +{ + if (!feat_supported(NULL, FEAT_PERCPU_DATA)) { + test__skip(); + return; + } + + if (test__start_subtest("init")) + test_global_percpu_data_init(); + if (test__start_subtest("lskel")) + test_global_percpu_data_lskel(); + if (test__start_subtest("rdonly_direct_read")) + test_global_percpu_data_rdonly_direct_read(); + if (test__start_subtest("rdonly_direct_write")) + test_global_percpu_data_rdonly_direct_write(); + test_global_percpu_data_verifier_log(); + if (test__start_subtest("iter")) + test_global_percpu_data_iter(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/global_map_resize.c b/tools/testing/selftests/bpf/prog_tests/global_map_resize.c index 56b5baef35c8..602ce30f1720 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_map_resize.c +++ b/tools/testing/selftests/bpf/prog_tests/global_map_resize.c @@ -23,6 +23,7 @@ static void global_map_resize_bss_subtest(void) struct bpf_map *map; const __u32 desired_sz = sizeof(skel->bss->sum) + sysconf(_SC_PAGE_SIZE) * 2; size_t array_len, actual_sz, new_sz; + int *array; skel = test_global_map_resize__open(); if (!ASSERT_OK_PTR(skel, "test_global_map_resize__open")) @@ -58,10 +59,13 @@ static void global_map_resize_bss_subtest(void) goto teardown; /* fill the newly resized array with ones, - * skipping the first element which was previously set + * skipping the first element which was previously set; + * access through a plain pointer to avoid -Warray-bounds + * since the array was resized beyond its declared length. */ + array = skel->bss->array; for (int i = 1; i < array_len; i++) - skel->bss->array[i] = 1; + array[i] = 1; /* set global const values before loading */ skel->rodata->pid = getpid(); diff --git a/tools/testing/selftests/bpf/prog_tests/icmp_send_kfunc.c b/tools/testing/selftests/bpf/prog_tests/icmp_send_kfunc.c new file mode 100644 index 000000000000..9318d4bc7ce8 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/icmp_send_kfunc.c @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <network_helpers.h> +#include <cgroup_helpers.h> +#include <linux/errqueue.h> +#include <poll.h> +#include <unistd.h> +#include "icmp_send.skel.h" + +#define TIMEOUT_MS 1000 + +#define ICMP_DEST_UNREACH 3 +#define ICMPV6_DEST_UNREACH 1 + +#define ICMP_HOST_UNREACH 1 +#define ICMP_FRAG_NEEDED 4 +#define NR_ICMP_UNREACH 15 +#define ICMPV6_REJECT_ROUTE 6 + +#define KFUNC_RET_UNSET -1 + +static int connect_to_fd_nonblock(int server_fd) +{ + struct sockaddr_storage addr; + socklen_t len = sizeof(addr); + int fd, err, on = 1; + + if (getsockname(server_fd, (struct sockaddr *)&addr, &len)) + return -1; + + fd = socket(addr.ss_family, SOCK_STREAM | SOCK_NONBLOCK, 0); + if (fd < 0) + return -1; + + if (addr.ss_family == AF_INET6 && + setsockopt(fd, IPPROTO_IPV6, IPV6_RECVERR, &on, sizeof(on)) < 0) { + close(fd); + return -1; + } + + err = connect(fd, (struct sockaddr *)&addr, len); + if (err < 0 && errno != EINPROGRESS) { + close(fd); + return -1; + } + + return fd; +} + +static void read_icmp_errqueue(int sockfd, int expected_code, int af) +{ + int expected_ee_type = (af == AF_INET) ? ICMP_DEST_UNREACH : + ICMPV6_DEST_UNREACH; + int expected_origin = (af == AF_INET) ? SO_EE_ORIGIN_ICMP : + SO_EE_ORIGIN_ICMP6; + int expected_level = (af == AF_INET) ? IPPROTO_IP : IPPROTO_IPV6; + int expected_type = (af == AF_INET) ? IP_RECVERR : IPV6_RECVERR; + struct sock_extended_err *sock_err; + char ctrl_buf[512]; + struct msghdr msg = { + .msg_control = ctrl_buf, + .msg_controllen = sizeof(ctrl_buf), + }; + struct pollfd pfd = { + .fd = sockfd, + .events = POLLERR, + }; + struct cmsghdr *cm; + ssize_t n; + + if (!ASSERT_GE(poll(&pfd, 1, TIMEOUT_MS), 1, "poll_errqueue")) + return; + + n = recvmsg(sockfd, &msg, MSG_ERRQUEUE); + if (!ASSERT_GE(n, 0, "recvmsg_errqueue")) + return; + + cm = CMSG_FIRSTHDR(&msg); + if (!ASSERT_NEQ(cm, NULL, "cm_firsthdr_null")) + return; + + for (; cm; cm = CMSG_NXTHDR(&msg, cm)) { + if (cm->cmsg_level != expected_level || + cm->cmsg_type != expected_type) + continue; + + sock_err = (struct sock_extended_err *)CMSG_DATA(cm); + + if (!ASSERT_EQ(sock_err->ee_origin, expected_origin, + "sock_err_origin")) + return; + if (!ASSERT_EQ(sock_err->ee_type, expected_ee_type, + "sock_err_type_dest_unreach")) + return; + ASSERT_EQ(sock_err->ee_code, expected_code, "sock_err_code"); + return; + } + + ASSERT_FAIL("no IP_RECVERR/IPV6_RECVERR control message found"); +} + +static bool valid_unreach_code(int code, int af) +{ + if (code < 0) + return false; + + if (af == AF_INET) + return code <= NR_ICMP_UNREACH && code != ICMP_FRAG_NEEDED; + + return code <= ICMPV6_REJECT_ROUTE; +} + +static void trigger_prog_read_icmp_errqueue(struct icmp_send *skel, int code, + int af, const char *ip) +{ + int srv_fd = -1, client_fd = -1; + int port; + + srv_fd = start_server(af, SOCK_STREAM, ip, 0, TIMEOUT_MS); + if (!ASSERT_OK_FD(srv_fd, "start_server")) + return; + + port = get_socket_local_port(srv_fd); + if (!ASSERT_GE(port, 0, "get_socket_local_port")) { + close(srv_fd); + return; + } + + skel->bss->server_port = ntohs(port); + skel->bss->unreach_type = (af == AF_INET) ? ICMP_DEST_UNREACH : + ICMPV6_DEST_UNREACH; + skel->bss->unreach_code = code; + skel->data->kfunc_ret = KFUNC_RET_UNSET; + + client_fd = connect_to_fd_nonblock(srv_fd); + if (!ASSERT_OK_FD(client_fd, "client_connect_nonblock")) { + close(srv_fd); + return; + } + + if (valid_unreach_code(code, af)) + read_icmp_errqueue(client_fd, code, af); + + close(client_fd); + close(srv_fd); +} + +static void run_icmp_test(struct icmp_send *skel, int af, const char *ip, + int max_code) +{ + for (int code = 0; code <= max_code; code++) { + if (af == AF_INET && code == ICMP_FRAG_NEEDED) + continue; + + trigger_prog_read_icmp_errqueue(skel, code, af, ip); + ASSERT_EQ(skel->data->kfunc_ret, 0, "kfunc_ret"); + } + + /* Test invalid codes */ + trigger_prog_read_icmp_errqueue(skel, -1, af, ip); + ASSERT_EQ(skel->data->kfunc_ret, -EINVAL, "kfunc_ret"); + + trigger_prog_read_icmp_errqueue(skel, max_code + 1, af, ip); + ASSERT_EQ(skel->data->kfunc_ret, -EINVAL, "kfunc_ret"); + + if (af == AF_INET) { + trigger_prog_read_icmp_errqueue(skel, ICMP_FRAG_NEEDED, af, ip); + ASSERT_EQ(skel->data->kfunc_ret, -EINVAL, "kfunc_ret"); + } +} + +static void run_icmp_no_route_test(struct icmp_send *skel, int af) +{ + union { + struct ipv4_packet v4; + struct ipv6_packet v6; + } pkt; + DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts, + .data_in = &pkt, + ); + int err; + + switch (af) { + case AF_INET: + pkt.v4 = pkt_v4; + pkt.v4.iph.version = 4; + pkt.v4.iph.daddr = htonl(INADDR_LOOPBACK); + pkt.v4.tcp.dest = htons(80); + opts.data_size_in = sizeof(pkt.v4); + skel->bss->unreach_type = ICMP_DEST_UNREACH; + break; + case AF_INET6: + pkt.v6 = pkt_v6; + pkt.v6.iph.version = 6; + pkt.v6.iph.daddr = in6addr_loopback; + pkt.v6.tcp.dest = htons(80); + opts.data_size_in = sizeof(pkt.v6); + skel->bss->unreach_type = ICMPV6_DEST_UNREACH; + break; + default: + ASSERT_FAIL("af_not_supported"); + return; + } + + skel->bss->server_port = 80; + skel->data->kfunc_ret = KFUNC_RET_UNSET; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.egress), &opts); + if (!ASSERT_OK(err, "test_run")) + return; + + ASSERT_EQ(skel->data->kfunc_ret, -ENETUNREACH, "kfunc_ret_no_route"); +} + +void test_icmp_send_unreach_cgroup(void) +{ + struct icmp_send *skel; + int cgroup_fd = -1; + + skel = icmp_send__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + cgroup_fd = test__join_cgroup("/icmp_send_unreach_cgroup"); + if (!ASSERT_OK_FD(cgroup_fd, "join_cgroup")) + goto cleanup; + + skel->links.egress = + bpf_program__attach_cgroup(skel->progs.egress, cgroup_fd); + if (!ASSERT_OK_PTR(skel->links.egress, "prog_attach_cgroup")) + goto cleanup; + + if (test__start_subtest("ipv4")) + run_icmp_test(skel, AF_INET, "127.0.0.1", NR_ICMP_UNREACH); + + if (test__start_subtest("ipv6")) + run_icmp_test(skel, AF_INET6, "::1", ICMPV6_REJECT_ROUTE); + + if (test__start_subtest("no_route_ipv4")) + run_icmp_no_route_test(skel, AF_INET); + + if (test__start_subtest("no_route_ipv6")) + run_icmp_no_route_test(skel, AF_INET6); + +cleanup: + icmp_send__destroy(skel); + if (cgroup_fd >= 0) + close(cgroup_fd); +} + +void test_icmp_send_unreach_recursion(void) +{ + struct icmp_send *skel; + int cgroup_fd = -1; + int err; + + err = setup_cgroup_environment(); + if (!ASSERT_OK(err, "setup_cgroup_environment")) + return; + + skel = icmp_send__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + cgroup_fd = get_root_cgroup(); + if (!ASSERT_OK_FD(cgroup_fd, "get_root_cgroup")) + goto cleanup; + + skel->data->target_pid = getpid(); + skel->links.recursion = + bpf_program__attach_cgroup(skel->progs.recursion, cgroup_fd); + if (!ASSERT_OK_PTR(skel->links.recursion, "prog_attach_cgroup")) + goto cleanup; + + trigger_prog_read_icmp_errqueue(skel, ICMP_HOST_UNREACH, AF_INET, + "127.0.0.1"); + + /* + * Because there's recursion involved, the first call will return at + * index 1 since it will return the second, and the second call will + * return at index 0 since it will return the first. + */ + ASSERT_EQ(skel->bss->rec_count, 2, "rec_count"); + ASSERT_EQ(skel->data->rec_kfunc_rets[0], -EBUSY, "kfunc_rets[0]"); + ASSERT_EQ(skel->data->rec_kfunc_rets[1], 0, "kfunc_rets[1]"); + +cleanup: + icmp_send__destroy(skel); + if (cgroup_fd >= 0) + close(cgroup_fd); + cleanup_cgroup_environment(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c index 3df07680f9e0..2b39cc1b09f9 100644 --- a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c +++ b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c @@ -66,11 +66,15 @@ static struct kfunc_test_params kfunc_tests[] = { TC_FAIL(kfunc_call_test_get_mem_fail_rdonly, 0, "R0 cannot write into rdonly_mem"), TC_FAIL(kfunc_call_test_get_mem_fail_use_after_free, 0, "invalid mem access 'scalar'"), TC_FAIL(kfunc_call_test_get_mem_fail_oob, 0, "min value is outside of the allowed memory range"), + TC_FAIL(kfunc_call_test_get_mem_fail_zero_size, 0, "min value is outside of the allowed memory range"), + TC_FAIL(kfunc_call_test_get_mem_fail_oversized, 0, "allocation size exceeds u32 max"), TC_FAIL(kfunc_call_test_get_mem_fail_not_const, 0, "is not a const"), TC_FAIL(kfunc_call_test_mem_acquire_fail, 0, "acquire kernel function does not return PTR_TO_BTF_ID"), TC_FAIL(kfunc_call_test_pointer_arg_type_mismatch, 0, "R1 expected pointer to ctx, but got scalar"), + TC_FAIL(kfunc_call_test_spin_lock_unsafe, 0, "function calls are not allowed while holding a lock"), /* success cases */ + TC_TEST(kfunc_call_test_spin_lock_safe, 0), TC_TEST(kfunc_call_test1, 12), TC_TEST(kfunc_call_test2, 3), TC_TEST(kfunc_call_test4, -1234), diff --git a/tools/testing/selftests/bpf/prog_tests/ksock.c b/tools/testing/selftests/bpf/prog_tests/ksock.c new file mode 100644 index 000000000000..dd6b167623d9 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/ksock.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include <arpa/inet.h> + +#include "test_progs.h" +#include "network_helpers.h" +#include "ksock_lsm.skel.h" +#include "ksock_lsm_verifier.skel.h" + +#define NS_TEST "ksock_lsm_ns" +#define RECV_PORT 7777 +#define RECV_TIMEOUT_SEC 5 + +struct ksock_test_env { + struct nstoken *nstoken; + int rfd; +}; + +static bool ksock_test_env_setup(struct ksock_test_env *env) +{ + struct sockaddr_in addr = { + .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK), + .sin_port = htons(RECV_PORT), + }; + struct timeval tv = { .tv_sec = RECV_TIMEOUT_SEC }; + int err; + + memset(env, 0, sizeof(*env)); + env->rfd = -1; + + if (!ASSERT_OK(make_netns(NS_TEST), "make_netns")) + goto fail; + + env->nstoken = open_netns(NS_TEST); + if (!ASSERT_OK_PTR(env->nstoken, "open_netns")) + goto fail; + + env->rfd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + if (!ASSERT_OK_FD(env->rfd, "receiver socket")) + goto fail; + + err = bind(env->rfd, (struct sockaddr *)&addr, sizeof(addr)); + if (!ASSERT_OK(err, "bind receiver")) + goto fail; + + err = setsockopt(env->rfd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + if (!ASSERT_OK(err, "set rcvtimeo")) + goto fail; + + return true; + +fail: + return false; +} + +void test_ksock_lsm(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct ksock_test_env env; + struct sockaddr_in trigger_addr = { + .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK), + }; + struct ksock_lsm *skel; + char recv_data[sizeof(skel->data->send_data)] = {}; + ssize_t n; + int tfd = -1; + int err; + + skel = ksock_lsm__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel open_and_load")) + return; + + if (!ksock_test_env_setup(&env)) + goto fail; + + /* Step 1: Run the setup SYSCALL prog to create the ksock */ + skel->bss->ipv4_remote = htonl(INADDR_LOOPBACK); + skel->bss->remote_port = RECV_PORT; + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.ksock_setup), + &opts); + if (!ASSERT_OK(err, "ksock_setup run")) + goto fail; + if (!ASSERT_OK(opts.retval, "ksock_setup retval")) + goto fail; + + /* Step 2: Attach LSM prog and trigger socket_bind from userspace */ + skel->links.ksock_socket_bind = + bpf_program__attach_lsm(skel->progs.ksock_socket_bind); + if (!ASSERT_OK_PTR(skel->links.ksock_socket_bind, + "attach socket_bind lsm")) + goto fail; + + tfd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + if (!ASSERT_OK_FD(tfd, "trigger socket")) + goto fail; + + skel->bss->target_pid = getpid(); + err = bind(tfd, (struct sockaddr *)&trigger_addr, sizeof(trigger_addr)); + skel->bss->target_pid = 0; + if (!ASSERT_OK(err, "trigger bind")) + goto fail; + + /* Step 3: Verify the LSM hook sent the notification */ + if (!ASSERT_EQ(skel->data->send_ret, sizeof(skel->data->send_data), + "LSM send bytes")) + goto fail; + + n = recvfrom(env.rfd, recv_data, sizeof(recv_data), 0, NULL, NULL); + if (ASSERT_EQ(n, sizeof(recv_data), "recvfrom len")) + ASSERT_MEMEQ(recv_data, skel->data->send_data, sizeof(recv_data), + "payload match"); + +fail: + if (tfd >= 0) + close(tfd); + if (env.rfd >= 0) + close(env.rfd); + if (env.nstoken) + close_netns(env.nstoken); + remove_netns(NS_TEST); + ksock_lsm__destroy(skel); +} + +void test_ksock_lsm_verifier(void) +{ + RUN_TESTS(ksock_lsm_verifier); +} diff --git a/tools/testing/selftests/bpf/prog_tests/ksock_wq.c b/tools/testing/selftests/bpf/prog_tests/ksock_wq.c new file mode 100644 index 000000000000..d6dc20b8f95b --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/ksock_wq.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include <unistd.h> + +#include "test_progs.h" +#include "ksock_wq.skel.h" + +#define CALLBACK_WAIT_RETRIES 1000 +#define CALLBACK_WAIT_US 1000 + +void test_ksock_wq(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct ksock_wq *skel; + u32 callback_done; + int err, i; + + skel = ksock_wq__open_and_load(); + if (!ASSERT_OK_PTR(skel, "ksock_wq open and load")) + return; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.ksock_wq_start), + &opts); + if (!ASSERT_OK(err, "run ksock_wq_start")) + goto out; + if (!ASSERT_OK(opts.retval, "ksock_wq_start retval")) + goto out; + + for (i = 0; i < CALLBACK_WAIT_RETRIES; i++) { + if (__atomic_load_n(&skel->bss->callback_done, __ATOMIC_ACQUIRE)) + break; + usleep(CALLBACK_WAIT_US); + } + callback_done = __atomic_load_n(&skel->bss->callback_done, + __ATOMIC_ACQUIRE); + if (!ASSERT_EQ(callback_done, 1, "workqueue callback completed")) + goto out; + + ASSERT_EQ(skel->bss->create_err, -EOPNOTSUPP, + "workqueue create rejected"); + +out: + ksock_wq__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/libarena.c b/tools/testing/selftests/bpf/prog_tests/libarena.c index 61ea68dce410..daade4150af6 100644 --- a/tools/testing/selftests/bpf/prog_tests/libarena.c +++ b/tools/testing/selftests/bpf/prog_tests/libarena.c @@ -15,7 +15,12 @@ static void run_libarena_test(struct libarena *skel, struct bpf_program *prog, { int ret; - if (!strstr(name, "test_buddy")) { + if (strstr(name, "test_buddy")) { + /* Buddy tests initialize the allocator directly. */ + ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_destroy)); + if (!ASSERT_OK(ret, "arena_buddy_destroy")) + return; + } else { ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_reset)); if (!ASSERT_OK(ret, "arena_buddy_reset")) return; @@ -24,7 +29,6 @@ static void run_libarena_test(struct libarena *skel, struct bpf_program *prog, ret = libarena_run_prog(bpf_program__fd(prog)); ASSERT_OK(ret, name); - } static void *run_libarena_parallel_prog(void *arg) @@ -69,6 +73,7 @@ static int run_libarena_parallel_test_workers(struct libarena *skel, uint32_t nthreads; void *thread_ret; int ret, err = 0; + int worker_err; int i; for (nthreads = 0; nthreads < UINT_MAX; nthreads++) { @@ -114,7 +119,22 @@ static int run_libarena_parallel_test_workers(struct libarena *skel, continue; } - err = err ?: (long)thread_ret; + worker_err = (long)thread_ret; + + /* + * A worker that bails out because another one already gave up + * reports -EINTR. It is collateral damage that carries no + * information, so skip it entirely: never let it become the + * reported error, and don't log it either. + */ + if (!worker_err || worker_err == -EINTR) + continue; + + if (!err) + err = worker_err; + + fprintf(stdout, "%.*s__%d returned %d\n", (int)prefixlen, name, + i, worker_err); } free(threads); @@ -198,7 +218,7 @@ static void run_libarena_parallel_test(struct libarena *skel, struct bpf_program run_libarena_parallel_fini(skel, name, prefixlen); } -void test_libarena(void) +void serial_test_libarena(void) { struct arena_alloc_reserve_args args; struct libarena *skel; diff --git a/tools/testing/selftests/bpf/prog_tests/libarena_asan.c b/tools/testing/selftests/bpf/prog_tests/libarena_asan.c index d59d9dd12ef2..9c31b17dbf39 100644 --- a/tools/testing/selftests/bpf/prog_tests/libarena_asan.c +++ b/tools/testing/selftests/bpf/prog_tests/libarena_asan.c @@ -17,7 +17,12 @@ static void run_libarena_asan_test(struct libarena_asan *skel, { int ret; - if (!strstr(name, "test_buddy")) { + if (strstr(name, "test_buddy")) { + /* Buddy tests initialize the allocator directly. */ + ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_destroy)); + if (!ASSERT_OK(ret, "arena_buddy_destroy")) + return; + } else { ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_reset)); if (!ASSERT_OK(ret, "arena_buddy_reset")) return; @@ -80,7 +85,7 @@ out: * Run the test depending on whether LLVM can compile arena ASAN * programs. */ -void test_libarena_asan(void) +void serial_test_libarena_asan(void) { #ifdef HAS_BPF_ARENA_ASAN run_test(); @@ -90,4 +95,3 @@ void test_libarena_asan(void) return; } - diff --git a/tools/testing/selftests/bpf/prog_tests/linked_list.c b/tools/testing/selftests/bpf/prog_tests/linked_list.c index 8defea0253ed..c3d133c6a00d 100644 --- a/tools/testing/selftests/bpf/prog_tests/linked_list.c +++ b/tools/testing/selftests/bpf/prog_tests/linked_list.c @@ -68,6 +68,7 @@ static struct { { "obj_type_id_oor", "local type ID argument must be in range [0, U32_MAX]" }, { "obj_new_no_composite", "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct" }, { "obj_new_no_struct", "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct" }, + { "obj_new_flex_array", "access beyond struct obj_new_flex" }, { "obj_drop_non_zero_off", "R1 must have zero offset when passed to release func" }, { "new_null_ret", "R0 invalid mem access 'ptr_or_null_'" }, { "obj_new_acq", "Unreleased reference id=" }, diff --git a/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c b/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c index a970798e1173..28bc4b117f41 100644 --- a/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c +++ b/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c @@ -17,6 +17,7 @@ #include <stdlib.h> #include <string.h> #include <sys/stat.h> +#include <sys/sysmacros.h> #include <sys/types.h> #include <unistd.h> #include "lsm_bdev.skel.h" @@ -172,7 +173,7 @@ void test_lsm_bdev(void) if (!ASSERT_OK(stat(DM_DEV_PATH, &st), "stat dm dev")) goto remove_dm; - dev_key = (__u32)st.st_rdev; + dev_key = (major(st.st_rdev) << 20) | minor(st.st_rdev); /* Look up the device in the BPF map and verify. */ err = bpf_map__lookup_elem(skel->maps.verity_devices, diff --git a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c index 6606f0ed9a9a..39e8a3b8b6af 100644 --- a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c +++ b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c @@ -410,7 +410,8 @@ close_netns: static int check_ping_ok(const char *ns1) { SYS(fail, "ip netns exec %s ping -c 1 -W1 -I veth1 %s > /dev/null", ns1, IP4_ADDR_DST); - SYS(fail, "ip netns exec %s ping6 -c 1 -W1 -I veth1 %s > /dev/null", ns1, IP6_ADDR_DST); + SYS(fail, "ip netns exec %s %s -c 1 -W1 -I veth1 %s > /dev/null", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); return 0; fail: return -1; @@ -424,7 +425,8 @@ static int check_ping_fails(const char *ns1) if (!ret) return -1; - ret = SYS_NOFAIL("ip netns exec %s ping6 -c 1 -W1 -I veth1 %s", ns1, IP6_ADDR_DST); + ret = SYS_NOFAIL("ip netns exec %s %s -c 1 -W1 -I veth1 %s", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); if (!ret) return -1; @@ -657,9 +659,10 @@ static void lwt_ip_encap_vxlan(bool ipv4_encap) skel->bss->fexit_triggered = false; if (ipv4_encap) - SYS(out, "ip netns exec %s ping -c 1 -W1 %s", ns1, IP4_ADDR_DST); + SYS(out, "ip netns exec %s ping -c 1 -W1 %s", ns1, IP4_ADDR_DST); else - SYS(out, "ip netns exec %s ping6 -c 1 -W1 %s", ns1, IP6_ADDR_DST); + SYS(out, "ip netns exec %s %s -c 1 -W1 %s", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); if (!ASSERT_TRUE(skel->bss->fexit_triggered, "fexit_triggered")) goto out; diff --git a/tools/testing/selftests/bpf/prog_tests/mptcp.c b/tools/testing/selftests/bpf/prog_tests/mptcp.c index 8fade8bdc451..32dfc1c511af 100644 --- a/tools/testing/selftests/bpf/prog_tests/mptcp.c +++ b/tools/testing/selftests/bpf/prog_tests/mptcp.c @@ -264,7 +264,7 @@ static int verify_mptcpify(int server_fd, int client_fd) return err; } -static int run_mptcpify(int cgroup_fd) +static int run_mptcpify(int cgroup_fd, int type) { int server_fd, client_fd, err = 0; struct mptcpify *mptcpify_skel; @@ -280,7 +280,7 @@ static int run_mptcpify(int cgroup_fd) goto out; /* without MPTCP */ - server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0); + server_fd = start_server(AF_INET, type, NULL, 0, 0); if (!ASSERT_GE(server_fd, 0, "start_server")) { err = -EIO; goto out; @@ -317,7 +317,14 @@ static void test_mptcpify(void) if (!ASSERT_OK_PTR(netns, "netns_new")) goto fail; - ASSERT_OK(run_mptcpify(cgroup_fd), "run_mptcpify"); + ASSERT_OK(run_mptcpify(cgroup_fd, SOCK_STREAM), "run_mptcpify"); + /* userspace sets flags such as SOCK_CLOEXEC together with the type; + * the BPF prog must still upgrade the socket to MPTCP. See + * update_socket_protocol() in net/socket.c, which runs before the + * type is masked with SOCK_TYPE_MASK. + */ + ASSERT_OK(run_mptcpify(cgroup_fd, SOCK_STREAM | SOCK_CLOEXEC), + "run_mptcpify_cloexec"); fail: netns_free(netns); diff --git a/tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c b/tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c index 246eb259c08a..6a07b2b418d1 100644 --- a/tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +++ b/tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c @@ -34,6 +34,8 @@ static void test_success(void) bpf_program__set_autoload(skel->progs.rcu_read_lock_global_subprog, true); bpf_program__set_autoload(skel->progs.rcu_read_lock_subprog_lock, true); bpf_program__set_autoload(skel->progs.rcu_read_lock_subprog_unlock, true); + bpf_program__set_autoload(skel->progs.non_own_ref_untrusted_ld, true); + bpf_program__set_autoload(skel->progs.rcu_untrusted_union_ld, true); err = rcu_read_lock__load(skel); if (!ASSERT_OK(err, "skel_load")) goto out; diff --git a/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c b/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c index f0a8c828f8f1..7541f4966abc 100644 --- a/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c +++ b/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c @@ -3,6 +3,8 @@ #include <test_progs.h> #include <network_helpers.h> #include <sys/sysinfo.h> +#include <sys/syscall.h> +#include <linux/perf_event.h> #include "res_spin_lock.skel.h" #include "res_spin_lock_fail.skel.h" @@ -102,11 +104,29 @@ end: void serial_test_res_spin_lock_stress(void) { + struct perf_event_attr attr = { + .size = sizeof(attr), + .type = PERF_TYPE_HARDWARE, + .config = PERF_COUNT_HW_CPU_CYCLES, + }; + int pmu_fd; + if (libbpf_num_possible_cpus() < 3) { test__skip(); return; } + pmu_fd = syscall(__NR_perf_event_open, &attr, 0, -1, -1, 0); + if (pmu_fd < 0) { + if (errno == ENOENT || errno == EOPNOTSUPP) { + test__skip(); + return; + } + ASSERT_OK(-errno, "perf_event_open pmu probe"); + return; + } + close(pmu_fd); + ASSERT_OK(load_module("bpf_test_rqspinlock.ko", false), "load module AA"); sleep(5); unload_module("bpf_test_rqspinlock", false); diff --git a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c index 41dfaaabb73f..3f9949e8227d 100644 --- a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c +++ b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c @@ -10,7 +10,25 @@ #include <linux/btf_ids.h> #include "test_progs.h" -static int duration; +#define BTF_DATA_FILE "resolve_btfids.test.o.BTF" + +#define DECL_TAG_FASTCALL "bpf_fastcall" +#define DECL_TAG_KFUNC "bpf_kfunc" +#define TYPE_ATTR_ARENA "address_space(1)" +#define ARENA_ARG(n) (1U << (n)) + +#ifndef KF_FASTCALL +#define KF_FASTCALL (1 << 12) +#endif +#ifndef KF_ARENA_RET +#define KF_ARENA_RET (1 << 13) +#endif +#ifndef KF_ARENA_ARG1 +#define KF_ARENA_ARG1 (1 << 14) +#endif +#ifndef KF_ARENA_ARG2 +#define KF_ARENA_ARG2 (1 << 15) +#endif struct symbol { const char *name; @@ -28,12 +46,50 @@ struct symbol test_symbols[] = { { "func", BTF_KIND_FUNC, -1 }, }; +struct kfunc_symbol { + const char *name; + s32 id; + u32 flags; + u32 arena_args; + bool arena_ret; +}; + +static struct kfunc_symbol kfunc_symbols[] = { + { "kfunc_a", -1, 0, 0, false }, + { "kfunc_b", -1, KF_FASTCALL, 0, false }, + { "kfunc_c", -1, KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2, + ARENA_ARG(0) | ARENA_ARG(1), true }, + { "kfunc_d", -1, KF_ARENA_ARG2, ARENA_ARG(1), false }, + { "kfunc_e", -1, 0, ARENA_ARG(0) | ARENA_ARG(1) | ARENA_ARG(2) | + ARENA_ARG(3) | ARENA_ARG(4), false }, + { "kfunc_f", -1, 0, ARENA_ARG(1), false }, + { "kfunc_g", -1, KF_ARENA_RET, ARENA_ARG(0) | ARENA_ARG(1), true }, +}; + /* Align the .BTF_ids section to 4 bytes */ asm ( ".pushsection " BTF_IDS_SECTION " ,\"a\"; \n" ".balign 4, 0; \n" ".popsection; \n"); +/* + * test_list_local, test_set and test_kfunc_set are .local symbols placed + * in .BTF_ids by inline asm, and are read here directly by C name. To the + * compiler they are plain, default-visibility extern objects. + * + * When test_progs is linked as a position-independent executable (PIE), + * taking the address of such an extern is routed through the GOT. The + * GNU assembler on aarch64 unconditionally converts references to .local + * symbols into section + addend form (".BTF_ids + <offset>"), but a GOT + * slot cannot carry an addend (the AArch64 ELF spec mandates zero), so + * the linker resolves it to the .BTF_ids base. + * + * Mark them hidden so the compiler treats them as non-interposable and + * emits a direct, addend-preserving PC-relative access instead of a GOT + * load, in both PIE and non-PIE builds. test_list_global is .globl and + * not affected, so it is left at default visibility. + */ +#pragma GCC visibility push(hidden) BTF_ID_LIST(test_list_local) BTF_ID_UNUSED BTF_ID(typedef, S) @@ -43,24 +99,49 @@ BTF_ID(struct, S) BTF_ID(union, U) BTF_ID(func, func) -extern __u32 test_list_global[]; -BTF_ID_LIST_GLOBAL(test_list_global, 1) -BTF_ID_UNUSED +BTF_SET_START(test_set) BTF_ID(typedef, S) BTF_ID(typedef, T) BTF_ID(typedef, U) BTF_ID(struct, S) BTF_ID(union, U) BTF_ID(func, func) +BTF_SET_END(test_set) -BTF_SET_START(test_set) +BTF_KFUNCS_START(test_kfunc_set) +BTF_ID_FLAGS(func, kfunc_a) +BTF_ID_FLAGS(func, kfunc_b, KF_FASTCALL) +BTF_ID_FLAGS(func, kfunc_c, KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_d, KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_e) +BTF_ID_FLAGS(func, kfunc_f) +BTF_ID_FLAGS(func, kfunc_g, KF_ARENA_RET) +BTF_KFUNCS_END(test_kfunc_set) + +/* + * Same kfuncs in reverse declaration order, so resolve_btfids has to + * actually sort at least one of the two sets. + */ +BTF_KFUNCS_START(test_kfunc_set_rev) +BTF_ID_FLAGS(func, kfunc_g, KF_ARENA_RET) +BTF_ID_FLAGS(func, kfunc_f) +BTF_ID_FLAGS(func, kfunc_e) +BTF_ID_FLAGS(func, kfunc_d, KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_c, KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_b, KF_FASTCALL) +BTF_ID_FLAGS(func, kfunc_a) +BTF_KFUNCS_END(test_kfunc_set_rev) +#pragma GCC visibility pop + +extern __u32 test_list_global[]; +BTF_ID_LIST_GLOBAL(test_list_global, 1) +BTF_ID_UNUSED BTF_ID(typedef, S) BTF_ID(typedef, T) BTF_ID(typedef, U) BTF_ID(struct, S) BTF_ID(union, U) BTF_ID(func, func) -BTF_SET_END(test_set) static int __resolve_symbol(struct btf *btf, int type_id) @@ -70,10 +151,10 @@ __resolve_symbol(struct btf *btf, int type_id) unsigned int i; type = btf__type_by_id(btf, type_id); - if (!type) { - PRINT_FAIL("Failed to get type for ID %d\n", type_id); + if (!ASSERT_OK_PTR(type, "btf__type_by_id")) return -1; - } + + str = btf__name_by_offset(btf, type->name_off); for (i = 0; i < ARRAY_SIZE(test_symbols); i++) { if (test_symbols[i].id >= 0) @@ -82,64 +163,118 @@ __resolve_symbol(struct btf *btf, int type_id) if (BTF_INFO_KIND(type->info) != test_symbols[i].type) continue; - str = btf__name_by_offset(btf, type->name_off); - if (!str) { - PRINT_FAIL("Failed to get name for BTF ID %d\n", type_id); - return -1; - } - if (!strcmp(str, test_symbols[i].name)) test_symbols[i].id = type_id; } + if (!btf_is_func(type)) + return 0; + + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + if (kfunc_symbols[i].id >= 0) + continue; + if (!strcmp(str, kfunc_symbols[i].name)) + kfunc_symbols[i].id = type_id; + } + return 0; } -static int resolve_symbols(void) +static int resolve_symbols(struct btf *btf) { - struct btf *btf; + __u32 nr = btf__type_cnt(btf); int type_id; - __u32 nr; - btf = btf__parse_raw("resolve_btfids.test.o.BTF"); - if (CHECK(libbpf_get_error(btf), "resolve", - "Failed to load BTF from resolve_btfids.test.o.BTF\n")) - return -1; + for (type_id = 1; type_id < nr; type_id++) { + if (__resolve_symbol(btf, type_id)) + return -1; + } + return 0; +} + +static bool btf_has_decl_tag(struct btf *btf, const char *tag_name, s32 target_id) +{ + const struct btf_type *t; + const char *name; + int nr, id; nr = btf__type_cnt(btf); + for (id = 1; id < nr; id++) { + t = btf__type_by_id(btf, id); + if (!btf_is_decl_tag(t)) + continue; + if (t->type != (__u32)target_id) + continue; + if (btf_decl_tag(t)->component_idx != -1) + continue; + name = btf__name_by_offset(btf, t->name_off); + if (strcmp(name, tag_name) == 0) + return true; + } + return false; +} - for (type_id = 1; type_id < nr; type_id++) { - if (__resolve_symbol(btf, type_id)) - break; +static void check_kfunc_set(struct btf_id_set8 *set) +{ + unsigned int i, j; + + ASSERT_EQ(set->flags, BTF_SET8_KFUNCS, "kfunc_set_flags"); + ASSERT_EQ(set->cnt, ARRAY_SIZE(kfunc_symbols), "kfunc_set_cnt"); + + for (i = 0; i < set->cnt; i++) { + for (j = 0; j < ARRAY_SIZE(kfunc_symbols); j++) { + if (kfunc_symbols[j].id == (s32)set->pairs[i].id) { + ASSERT_EQ(set->pairs[i].flags, + kfunc_symbols[j].flags, "kfunc_flags_check"); + break; + } + } + + ASSERT_TRUE(j < ARRAY_SIZE(kfunc_symbols), "kfunc_id_found"); + + if (i > 0) { + ASSERT_LE(set->pairs[i - 1].id, + set->pairs[i].id, "kfunc_sort_check"); + } } +} - btf__free(btf); - return 0; +/* True if @id is PTR -> TYPE_TAG(kflag=1, "address_space(1)") -> pointee */ +static bool is_arena_tagged_ptr(struct btf *btf, __u32 id) +{ + const struct btf_type *ptr, *tag; + const char *name; + + ptr = btf__type_by_id(btf, id); + if (!btf_is_ptr(ptr)) + return false; + tag = btf__type_by_id(btf, ptr->type); + if (!btf_is_type_tag(tag) || !btf_kflag(tag)) + return false; + name = btf__name_by_offset(btf, tag->name_off); + return strcmp(name, TYPE_ATTR_ARENA) == 0; } void test_resolve_btfids(void) { __u32 *test_list, *test_lists[] = { test_list_local, test_list_global }; unsigned int i, j; - int ret = 0; + struct btf *btf; - if (resolve_symbols()) + btf = btf__parse_raw(BTF_DATA_FILE); + if (!ASSERT_OK_PTR(btf, "btf_parse")) return; + if (resolve_symbols(btf)) + goto out; + /* Check BTF_ID_LIST(test_list_local) and * BTF_ID_LIST_GLOBAL(test_list_global) IDs */ for (j = 0; j < ARRAY_SIZE(test_lists); j++) { test_list = test_lists[j]; - for (i = 0; i < ARRAY_SIZE(test_symbols); i++) { - ret = CHECK(test_list[i] != test_symbols[i].id, - "id_check", - "wrong ID for %s (%d != %d)\n", - test_symbols[i].name, - test_list[i], test_symbols[i].id); - if (ret) - return; - } + for (i = 0; i < ARRAY_SIZE(test_symbols); i++) + ASSERT_EQ(test_list[i], test_symbols[i].id, test_symbols[i].name); } /* Check BTF_SET_START(test_set) IDs */ @@ -153,15 +288,60 @@ void test_resolve_btfids(void) break; } - ret = CHECK(!found, "id_check", - "ID %d not found in test_symbols\n", - test_set.ids[i]); - if (ret) + if (!ASSERT_TRUE(found, "id_in_test_symbols")) break; - if (i > 0) { - if (!ASSERT_LE(test_set.ids[i - 1], test_set.ids[i], "sort_check")) - return; + if (i > 0) + ASSERT_LE(test_set.ids[i - 1], test_set.ids[i], "sort_check"); + } + + check_kfunc_set(&test_kfunc_set); + check_kfunc_set(&test_kfunc_set_rev); + + /* Check resolve_btfids emitted a bpf_kfunc decl_tag for each kfunc */ + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + ASSERT_TRUE(btf_has_decl_tag(btf, DECL_TAG_KFUNC, + kfunc_symbols[i].id), + kfunc_symbols[i].name); + } + + /* Check resolve_btfids emitted bpf_fastcall for KF_FASTCALL kfuncs */ + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + if (kfunc_symbols[i].flags & KF_FASTCALL) { + ASSERT_TRUE(btf_has_decl_tag(btf, DECL_TAG_FASTCALL, + kfunc_symbols[i].id), + kfunc_symbols[i].name); } } + + /* + * Check resolve_btfids wrapped exactly the arena-flagged or suffixed + * return/args with the address_space(1) type attribute, and left other + * pointers/returns untouched. + */ + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + const struct btf_type *fn, *proto; + const struct btf_param *params; + const char *name = kfunc_symbols[i].name; + u32 arena_args = kfunc_symbols[i].arena_args; + __u32 nr; + + fn = btf__type_by_id(btf, kfunc_symbols[i].id); + if (!ASSERT_TRUE(btf_is_func(fn), name)) + continue; + proto = btf__type_by_id(btf, fn->type); + if (!ASSERT_TRUE(btf_is_func_proto(proto), name)) + continue; + params = btf_params(proto); + nr = btf_vlen(proto); + + ASSERT_EQ(is_arena_tagged_ptr(btf, proto->type), + kfunc_symbols[i].arena_ret, name); + for (j = 0; j < nr; j++) + ASSERT_EQ(is_arena_tagged_ptr(btf, params[j].type), + !!(arena_args & ARENA_ARG(j)), name); + } + +out: + btf__free(btf); } diff --git a/tools/testing/selftests/bpf/prog_tests/sha256.c b/tools/testing/selftests/bpf/prog_tests/sha256.c index 604a0b1423d5..5edbc6194b07 100644 --- a/tools/testing/selftests/bpf/prog_tests/sha256.c +++ b/tools/testing/selftests/bpf/prog_tests/sha256.c @@ -25,10 +25,10 @@ void test_sha256(void) size_t i; data = malloc(MAX_LEN); - if (!ASSERT_OK_PTR(data, "malloc")) + if (!ASSERT_NEQ(data, NULL, "malloc")) goto out; digests = malloc((MAX_LEN + 1) * SHA256_DIGEST_LENGTH); - if (!ASSERT_OK_PTR(digests, "malloc")) + if (!ASSERT_NEQ(digests, NULL, "malloc")) goto out; /* Generate MAX_LEN bytes of "random" data deterministically. */ diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c index 5fc417e31fc6..77381d345435 100644 --- a/tools/testing/selftests/bpf/prog_tests/signed_loader.c +++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c @@ -11,6 +11,8 @@ #include <linux/keyctl.h> #include <linux/bpf.h> +#include <bpf/btf.h> + #include "bpf/libbpf_internal.h" /* for libbpf_sha256() */ #include "bpf/skel_internal.h" /* for loader ctx layout (bpf_loader_ctx etc) */ @@ -19,8 +21,6 @@ #include "test_signed_loader_data.skel.h" #include "test_signed_loader_lsm.skel.h" -#define SIG_MATCH_INSNS 33 /* excl (5) + 4 * sha-dword (7) */ - enum { BPF_SIG_UNSIGNED = 0, BPF_SIG_VERIFIED, @@ -35,7 +35,8 @@ enum { }; static int load_loader(const void *insns, __u32 insns_sz, int map_fd, - const void *sig, __u32 sig_sz, __s32 keyring_id) + const void *sig, __u32 sig_sz, __s32 keyring_id, + __u32 fd_array_cnt) { union bpf_attr attr; int fd; @@ -52,6 +53,7 @@ static int load_loader(const void *insns, __u32 insns_sz, int map_fd, attr.signature_size = sig_sz; attr.keyring_id = keyring_id; } + attr.fd_array_cnt = fd_array_cnt; memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, offsetofend(union bpf_attr, keyring_id)); @@ -62,14 +64,12 @@ static int run_gen_loader(const void *insns, __u32 insns_sz, const void *data, __u32 data_sz, const void *excl, __u32 excl_sz, const void *sig, __u32 sig_sz, - bool get_hash, void *ctx, __u32 ctx_sz, bool *loader_ran) + void *ctx, __u32 ctx_sz, bool *loader_ran) { LIBBPF_OPTS(bpf_map_create_opts, mopts, .excl_prog_hash = excl, .excl_prog_hash_size = excl_sz); - __u8 hbuf[SHA256_DIGEST_LENGTH]; - struct bpf_map_info info; - __u32 ilen = sizeof(info), key = 0; + __u32 key = 0; union bpf_attr attr; int map_fd, prog_fd, ret; @@ -87,15 +87,6 @@ static int run_gen_loader(const void *insns, __u32 insns_sz, ret = -errno; goto out_map; } - if (get_hash) { - memset(&info, 0, sizeof(info)); - info.hash = ptr_to_u64(hbuf); - info.hash_size = sizeof(hbuf); - if (bpf_map_get_info_by_fd(map_fd, &info, &ilen)) { - ret = -errno; - goto out_map; - } - } memset(&attr, 0, sizeof(attr)); attr.prog_type = BPF_PROG_TYPE_SYSCALL; @@ -108,6 +99,7 @@ static int run_gen_loader(const void *insns, __u32 insns_sz, attr.signature = ptr_to_u64(sig); attr.signature_size = sig_sz; attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + attr.fd_array_cnt = 1; } memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, @@ -236,79 +228,6 @@ out: return ret; } -static void check_sig_match_shape(const struct bpf_insn *in, int n) -{ - int a = -1, cleanup = -1, i, base, t, br[5], nb = 0; - - /* BPF_PSEUDO_MAP_IDX (the struct bpf_map * form) is used only here. */ - for (i = 0; i + 1 < n; i++) { - if (in[i].code == (BPF_LD | BPF_IMM | BPF_DW) && - in[i].src_reg == BPF_PSEUDO_MAP_IDX) { - a = i; - break; - } - } - if (!ASSERT_GE(a, 0, "emit_signature_match present")) - return; - if (!ASSERT_LE(a + SIG_MATCH_INSNS, n, "block fits in program")) - return; - - /* excl check: r2 = *(u32 *)(map + 32); if r2 != 1 goto cleanup */ - ASSERT_EQ(in[a + 2].code, (BPF_LDX | BPF_MEM | BPF_W), "excl load width"); - ASSERT_EQ(in[a + 2].off, SHA256_DIGEST_LENGTH, "excl field offset"); - ASSERT_EQ(in[a + 4].code, (BPF_JMP | BPF_JNE | BPF_K), "excl branch op"); - ASSERT_EQ(in[a + 4].imm, 1, "excl compared to 1"); - br[nb++] = a + 4; - - /* 4 sha-dword checks: r2 = *(u64 *)(map + i*8); if r2 != r3 goto cleanup */ - for (i = 0; i < 4; i++) { - base = a + 5 + i * 7; - ASSERT_EQ(in[base + 2].code, (BPF_LDX | BPF_MEM | BPF_DW), "sha load width"); - ASSERT_EQ(in[base + 2].off, i * 8, "sha dword offset"); - ASSERT_EQ(in[base + 3].code, (BPF_LD | BPF_IMM | BPF_DW), "sha imm64 (H_meta)"); - ASSERT_EQ(in[base + 6].code, (BPF_JMP | BPF_JNE | BPF_X), "sha branch op"); - br[nb++] = base + 6; - } - - /* - * Locate the real cleanup label so we can pin the exact jump target, - * not just "some backward label". bpf_gen__init() emits the cleanup - * block as a prog-fd close loop whose first instruction is the label - * every error branch jumps to. - */ - for (i = 0; i + 2 < a; i++) { - if (in[i].code == (BPF_LDX | BPF_MEM | BPF_W) && - in[i].dst_reg == BPF_REG_1 && in[i].src_reg == BPF_REG_10 && - in[i + 1].code == (BPF_JMP | BPF_JSLE | BPF_K) && - in[i + 1].dst_reg == BPF_REG_1 && in[i + 1].imm == 0 && - in[i + 1].off == 1 && - in[i + 2].code == (BPF_JMP | BPF_CALL) && - in[i + 2].imm == BPF_FUNC_sys_close) { - cleanup = i; - break; - } - } - if (!ASSERT_GE(cleanup, 0, "cleanup label located")) - return; - for (i = 0; i < nb; i++) { - t = br[i] + 1 + in[br[i]].off; - ASSERT_EQ(t, cleanup, "sig-match lands on cleanup"); - } - /* - * Same invariant for every other cleanup-bound jump in the program: - * emit_check_err() is the only source of "if (r7 < 0) goto cleanup", - * so each of those must also resolve exactly to cleanup. - */ - for (i = 0, t = 0; i < n; i++) { - if (in[i].code != (BPF_JMP | BPF_JSLT | BPF_K) || - in[i].dst_reg != BPF_REG_7 || in[i].imm != 0 || in[i].off >= 0) - continue; - ASSERT_EQ(i + 1 + in[i].off, cleanup, "err-check lands on cleanup"); - t++; - } - ASSERT_GT(t, 0, "found emit_check_err jumps"); -} - struct gen_loader_fixture { struct test_signed_loader *skel; struct gen_loader_opts gopts; @@ -372,16 +291,6 @@ static void gen_loader_fixture_fini(struct gen_loader_fixture *f) test_signed_loader__destroy(f->skel); } -static void metadata_check_shape(void) -{ - struct gen_loader_fixture f; - - if (gen_loader_fixture_init(&f) == 0) - check_sig_match_shape((const struct bpf_insn *)f.gopts.insns, - f.gopts.insns_sz / sizeof(struct bpf_insn)); - gen_loader_fixture_fini(&f); -} - static void metadata_match(void) { struct gen_loader_fixture f; @@ -391,94 +300,265 @@ static void metadata_match(void) if (gen_loader_fixture_init(&f) == 0) { r = run_gen_loader(f.gopts.insns, f.gopts.insns_sz, f.blob, f.data_sz, f.excl, sizeof(f.excl), NULL, 0, - true, f.ctx, f.ctx_sz, &ran); + f.ctx, f.ctx_sz, &ran); ASSERT_TRUE(ran, "loader ran"); ASSERT_EQ(r, 0, "honest loader retval"); } gen_loader_fixture_fini(&f); } -static void metadata_sha_mismatch(void) +static void signature_enforced(void) { + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; struct gen_loader_fixture f; - bool ran; - int r; + int fd; if (gen_loader_fixture_init(&f) == 0) { /* - * blob[0] lives in the loader's fd_array scratch (first add_data in - * bpf_gen__init); a 0-map program never reads it, so flipping it - * changes only map->sha. The metadata check is the only thing that - * can notice -> isolates emit_signature_match. + * A present-but-invalid signature (the cert bytes are not a + * PKCS#7 signature) must be rejected at load: the signature + * path is honored, not ignored. (The valid path is covered by + * the signed lskels.) Pin -EBADMSG, the PKCS#7 parse failure: + * a looser fd < 0 check could also be satisfied by the sparse + * fd_array rejection (-EACCES) that the loader's map reference + * would trip even if the signature were silently ignored. */ - f.blob[0] ^= 0xff; - r = run_gen_loader(f.gopts.insns, f.gopts.insns_sz, f.blob, - f.data_sz, f.excl, sizeof(f.excl), NULL, 0, - true, f.ctx, f.ctx_sz, &ran); - ASSERT_TRUE(ran, "loader ran"); - ASSERT_EQ(r, -EINVAL, "tampered blob rejected by emit_signature_match"); + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, + sizeof(junk), KEY_SPEC_SESSION_KEYRING, 0); + ASSERT_EQ(fd, -EBADMSG, "invalid signature rejected at load"); + if (fd >= 0) + close(fd); } gen_loader_fixture_fini(&f); } -static void metadata_not_exclusive(void) +static void signed_nonexcl_fd_array_rejected(void) { + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; struct gen_loader_fixture f; - bool ran; - int r; + int map_fd, fd; if (gen_loader_fixture_init(&f) == 0) { /* - * Correct blob but a non-exclusive metadata map: the verifier does - * not reject (excl_prog_sha unset), so the runtime map->excl == 1 - * check in the loader must. + * A signed program may only bind exclusive maps through fd_array + * (their contents are folded into the signature). Binding a + * non-exclusive map is rejected, before the signature is even + * examined. */ - r = run_gen_loader(f.gopts.insns, f.gopts.insns_sz, f.blob, - f.data_sz, NULL, 0, NULL, 0, true, f.ctx, - f.ctx_sz, &ran); - ASSERT_TRUE(ran, "loader ran"); - ASSERT_EQ(r, -EINVAL, "non-exclusive metadata map rejected"); + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "nonexcl", 4, + f.data_sz, 1, NULL); + if (ASSERT_OK_FD(map_fd, "nonexcl_map")) { + if (ASSERT_OK(bpf_map_freeze(map_fd), "freeze")) { + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, + map_fd, junk, sizeof(junk), + KEY_SPEC_SESSION_KEYRING, 1); + ASSERT_EQ(fd, -EPERM, + "non-exclusive map in signed fd_array rejected"); + if (fd >= 0) + close(fd); + } + close(map_fd); + } } gen_loader_fixture_fini(&f); } -static void metadata_hash_not_computed(void) +static void signed_unfrozen_fd_array_rejected(void) { + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + LIBBPF_OPTS(bpf_map_create_opts, mopts); struct gen_loader_fixture f; - bool ran; - int r; + __u32 key = 0; + int map_fd, fd; if (gen_loader_fixture_init(&f) == 0) { /* - * Correct, exclusive, frozen map, but its hash was never computed - * (no OBJ_GET_INFO_BY_FD), so map->sha stays zero. The loader must - * fail closed rather than treat an unset hash as a match. + * The metadata map must be frozen before a signed load so the + * folded bytes cannot change afterwards. Bind an exclusive map + * with matching contents but skip the freeze: the load must be + * rejected by the frozen check with -EPERM. The exclusivity + * check right after it would pass, so the errno uniquely pins + * the freeze requirement. */ - r = run_gen_loader(f.gopts.insns, f.gopts.insns_sz, f.blob, - f.data_sz, f.excl, sizeof(f.excl), NULL, 0, - false, f.ctx, f.ctx_sz, &ran); - ASSERT_TRUE(ran, "loader ran"); - ASSERT_EQ(r, -EINVAL, "uncomputed metadata hash rejected"); + mopts.excl_prog_hash = f.excl; + mopts.excl_prog_hash_size = sizeof(f.excl); + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "unfrozen", 4, + f.data_sz, 1, &mopts); + if (ASSERT_OK_FD(map_fd, "unfrozen_map")) { + if (ASSERT_OK(bpf_map_update_elem(map_fd, &key, f.blob, 0), + "update")) { + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, + map_fd, junk, sizeof(junk), + KEY_SPEC_SESSION_KEYRING, 1); + ASSERT_EQ(fd, -EPERM, + "unfrozen map in signed fd_array rejected"); + if (fd >= 0) + close(fd); + } + close(map_fd); + } } gen_loader_fixture_fini(&f); } -static void signature_enforced(void) +static void signed_nonarray_fd_array_rejected(void) +{ + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + LIBBPF_OPTS(bpf_map_create_opts, mopts); + struct gen_loader_fixture f; + int map_fd, fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * Only a plain BPF_MAP_TYPE_ARRAY may be folded into the + * signature. An exclusive map of any other type is rejected + * (-EINVAL) rather than folded - this is the type gate that + * keeps arena maps (map_direct_value_addr() returns a user + * address) and insn-array maps (buffer smaller than value_size) + * out of the hashed region, where the old code would have + * memcpy()'d from them. A hash map stands in here: it is + * exclusive (bound to the loader digest) but not an array. + */ + mopts.excl_prog_hash = f.excl; + mopts.excl_prog_hash_size = sizeof(f.excl); + map_fd = bpf_map_create(BPF_MAP_TYPE_HASH, "excl_hash", 4, 4, 1, + &mopts); + if (ASSERT_OK_FD(map_fd, "excl_hash_map")) { + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, + junk, sizeof(junk), + KEY_SPEC_SESSION_KEYRING, 1); + ASSERT_EQ(fd, -EINVAL, + "non-array map in signed fd_array rejected"); + if (fd >= 0) + close(fd); + close(map_fd); + } + } + gen_loader_fixture_fini(&f); +} + +static int setup_meta_map(const struct gen_loader_fixture *f); + +static void signed_btf_fd_array_rejected(void) +{ + char dir_tmpl[] = "/tmp/signed_loader_btfXXXXXX", *dir = NULL; + __u32 sig_sz = 8192; + int map_fd = -1, prog_fd = -1; + unsigned char *buf = NULL; + struct gen_loader_fixture f; + bool have_fixture = false; + struct btf *btf = NULL; + union bpf_attr attr; + int fds[2]; + __u8 sig[8192]; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + have_fixture = true; + if (gen_loader_fixture_init(&f) != 0) + goto out; + + /* + * fd_array binds maps and BTFs alike, but only exclusive array maps are + * folded into the signature. Build an otherwise genuinely signed load - + * insns || metadata, exclusive frozen map at fd_array[0] - then smuggle + * an extra BTF into fd_array[1]. A signed program may not bind any BTF, + * so resolving the fd_array entries rejects the BTF with -EACCES (in + * __add_used_btf(), before the signature is even verified). + */ + buf = malloc((size_t)f.gopts.insns_sz + f.data_sz); + if (!ASSERT_OK_PTR(buf, "signbuf")) + goto out; + memcpy(buf, f.gopts.insns, f.gopts.insns_sz); + memcpy(buf + f.gopts.insns_sz, f.blob, f.data_sz); + if (!ASSERT_OK(sign_buf(dir, buf, f.gopts.insns_sz + f.data_sz, sig, + &sig_sz), "sign insns||metadata")) + goto out; + + map_fd = setup_meta_map(&f); + if (!ASSERT_OK_FD(map_fd, "meta_map")) + goto out; + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf_new_empty")) + goto out; + btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf_load")) + goto out; + + fds[0] = map_fd; + fds[1] = btf__fd(btf); + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(f.gopts.insns); + attr.insn_cnt = f.gopts.insns_sz / sizeof(struct bpf_insn); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(fds); + attr.fd_array_cnt = 2; + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + ASSERT_EQ(prog_fd < 0 ? -errno : prog_fd, -EACCES, + "BTF in signed fd_array rejected"); + if (prog_fd >= 0) + close(prog_fd); +out: + if (btf) + btf__free(btf); + if (map_fd >= 0) + close(map_fd); + if (have_fixture) + gen_loader_fixture_fini(&f); + if (dir) + run_setup("cleanup", dir); + free(buf); +} + +static void signature_failure_logs(void) { static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + char log_buf[1024] = {}; struct gen_loader_fixture f; + union bpf_attr attr; int fd; if (gen_loader_fixture_init(&f) == 0) { /* - * A present-but-invalid signature (the cert bytes are not a - * PKCS#7 signature) must be rejected at load: the signature - * path is honored, not ignored. (The valid path is covered by - * the signed lskels.) + * Signature verification now runs inside bpf_check(), so a + * failure is reported through the verifier log. A present-but- + * invalid signature is rejected and the log says why. */ - fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, - sizeof(junk), KEY_SPEC_SESSION_KEYRING); + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(f.gopts.insns); + attr.insn_cnt = f.gopts.insns_sz / sizeof(struct bpf_insn); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.signature = ptr_to_u64(junk); + attr.signature_size = sizeof(junk); + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + attr.log_level = 1; + attr.log_buf = ptr_to_u64(log_buf); + attr.log_size = sizeof(log_buf); + memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); + + fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); ASSERT_LT(fd, 0, "invalid signature rejected at load"); + if (fd >= 0) + close(fd); + ASSERT_HAS_SUBSTR(log_buf, "signature verification failed", + "verifier logs signature failure"); } gen_loader_fixture_fini(&f); } @@ -495,8 +575,31 @@ static void signature_too_large(void) * is rejected before the buffer is read. */ fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, - 64 << 20, KEY_SPEC_SESSION_KEYRING); + 64 << 20, KEY_SPEC_SESSION_KEYRING, 0); ASSERT_EQ(fd, -EINVAL, "oversized signature rejected"); + if (fd >= 0) + close(fd); + } + gen_loader_fixture_fini(&f); +} + +static void signature_zero_size(void) +{ + static const __u8 junk[64] = {}; + struct gen_loader_fixture f; + int fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * A present signature with signature_size == 0 is rejected + * up front, before the keyring is resolved or the signature + * buffer is read. + */ + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, + 0, KEY_SPEC_SESSION_KEYRING, 0); + ASSERT_EQ(fd, -EINVAL, "zero-size signature rejected"); + if (fd >= 0) + close(fd); } gen_loader_fixture_fini(&f); } @@ -515,8 +618,10 @@ static void signature_bad_keyring(void) * large positive serial takes the user-keyring path and won't exist. */ fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, - sizeof(junk), INT_MAX); + sizeof(junk), INT_MAX, 0); ASSERT_EQ(fd, -EINVAL, "signature with bad keyring_id rejected"); + if (fd >= 0) + close(fd); } gen_loader_fixture_fini(&f); } @@ -575,7 +680,7 @@ static void metadata_ctx_max_entries_ignored(void) memcpy(blob, gopts.data, data_sz); r = run_gen_loader(gopts.insns, gopts.insns_sz, blob, data_sz, - excl, sizeof(excl), NULL, 0, true, ctx, ctx_sz, &ran); + excl, sizeof(excl), NULL, 0, ctx, ctx_sz, &ran); if (!ASSERT_TRUE(ran, "loader ran") || !ASSERT_EQ(r, 0, "loader retval")) goto free_blob; @@ -661,7 +766,7 @@ static void metadata_ctx_initial_value_ignored(void) memcpy(blob, gopts.data, data_sz); r = run_gen_loader(gopts.insns, gopts.insns_sz, blob, data_sz, - excl, sizeof(excl), NULL, 0, true, ctx, ctx_sz, &ran); + excl, sizeof(excl), NULL, 0, ctx, ctx_sz, &ran); if (!ASSERT_TRUE(ran, "loader ran") || !ASSERT_EQ(r, 0, "loader retval")) goto free_blob; @@ -714,6 +819,7 @@ static void signature_authenticates_insns(void) __u8 excl[SHA256_DIGEST_LENGTH], sig[8192]; __u32 sig_sz = sizeof(sig), insns_sz, data_sz, ctx_sz; unsigned char *insns = NULL, *tampered = NULL, *blob = NULL; + unsigned char *signbuf = NULL; int nr_maps = 0, nr_progs = 0, r; struct bpf_program *p; struct bpf_map *m; @@ -760,29 +866,141 @@ static void signature_authenticates_insns(void) memcpy(blob, gopts.data, data_sz); libbpf_sha256(insns, insns_sz, excl); - if (!ASSERT_OK(sign_buf(dir, insns, insns_sz, sig, &sig_sz), "sign-file")) + signbuf = malloc((size_t)insns_sz + data_sz); + if (!ASSERT_OK_PTR(signbuf, "signbuf")) + goto cleanup; + memcpy(signbuf, insns, insns_sz); + memcpy(signbuf + insns_sz, blob, data_sz); + if (!ASSERT_OK(sign_buf(dir, signbuf, insns_sz + data_sz, sig, &sig_sz), + "sign-file")) goto cleanup; memset(ctx, 0, ctx_sz); ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; r = run_gen_loader(insns, insns_sz, blob, data_sz, excl, sizeof(excl), - sig, sig_sz, true, ctx, ctx_sz, &ran); + sig, sig_sz, ctx, ctx_sz, &ran); ASSERT_TRUE(ran, "valid signature: loader loaded and ran"); ASSERT_EQ(r, 0, "valid signature accepted"); close_loader_ctx_fds(ctx, nr_maps, nr_progs); memcpy(tampered, insns, insns_sz); tampered[insns_sz / 2] ^= 0xff; + /* + * Bind the metadata map to the tampered loader's own digest, so the + * verifier's exclusive-map check (excl_prog_sha == prog->digest) passes + * and the signature - verified after the maps are resolved - is what + * rejects the load. This is the attacker's best case: even after + * re-binding the exclusive map to their tampered loader, the signature + * over the original insns || metadata still fails. (Leaving the map + * bound to the original digest would instead trip the excl check first.) + */ + libbpf_sha256(tampered, insns_sz, excl); memset(ctx, 0, ctx_sz); ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; r = run_gen_loader(tampered, insns_sz, blob, data_sz, excl, sizeof(excl), - sig, sig_sz, true, ctx, ctx_sz, &ran); + sig, sig_sz, ctx, ctx_sz, &ran); ASSERT_FALSE(ran, "tampered loader rejected before run"); ASSERT_EQ(r, -EKEYREJECTED, "signature is bound to the instructions"); cleanup: free(insns); free(tampered); free(blob); + free(signbuf); + free(ctx); + test_signed_loader__destroy(skel); + run_setup("cleanup", dir); +} + +static void signature_authenticates_metadata(void) +{ + LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true); + char dir_tmpl[] = "/tmp/signed_loaderXXXXXX", *dir; + struct test_signed_loader *skel = NULL; + __u8 excl[SHA256_DIGEST_LENGTH], sig[8192]; + __u32 sig_sz = sizeof(sig), insns_sz, data_sz, ctx_sz; + unsigned char *insns = NULL, *blob = NULL; + unsigned char *signbuf = NULL; + int nr_maps = 0, nr_progs = 0, r; + struct bpf_program *p; + struct bpf_map *m; + void *ctx = NULL; + bool ran; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + + skel = test_signed_loader__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + if (!ASSERT_OK(bpf_object__gen_loader(skel->obj, &gopts), "gen_loader")) + goto cleanup; + if (!ASSERT_OK(bpf_object__load(skel->obj), "gen_load")) + goto cleanup; + + bpf_object__for_each_program(p, skel->obj) + nr_progs++; + bpf_object__for_each_map(m, skel->obj) + nr_maps++; + ctx_sz = sizeof(struct bpf_loader_ctx) + + nr_maps * sizeof(struct bpf_map_desc) + + nr_progs * sizeof(struct bpf_prog_desc); + insns_sz = gopts.insns_sz; + data_sz = gopts.data_sz; + ctx = calloc(1, ctx_sz); + insns = malloc(insns_sz); + blob = malloc(data_sz); + if (!ASSERT_OK_PTR(ctx, "ctx") || + !ASSERT_OK_PTR(insns, "insns") || + !ASSERT_OK_PTR(blob, "blob")) + goto cleanup; + memcpy(insns, gopts.insns, insns_sz); + memcpy(blob, gopts.data, data_sz); + libbpf_sha256(insns, insns_sz, excl); + + signbuf = malloc((size_t)insns_sz + data_sz); + if (!ASSERT_OK_PTR(signbuf, "signbuf")) + goto cleanup; + memcpy(signbuf, insns, insns_sz); + memcpy(signbuf + insns_sz, blob, data_sz); + if (!ASSERT_OK(sign_buf(dir, signbuf, insns_sz + data_sz, sig, &sig_sz), + "sign-file")) + goto cleanup; + + memset(ctx, 0, ctx_sz); + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + r = run_gen_loader(insns, insns_sz, blob, data_sz, excl, sizeof(excl), + sig, sig_sz, ctx, ctx_sz, &ran); + ASSERT_TRUE(ran, "valid signature: loader loaded and ran"); + ASSERT_EQ(r, 0, "valid signature accepted"); + close_loader_ctx_fds(ctx, nr_maps, nr_progs); + + /* + * Tamper the metadata after signing while leaving the instructions + * and thus the exclusive hash binding untouched: the map freezes + * fine and excl_prog_sha still matches the loader's digest, so the + * load reaches signature verification, which folds the live frozen + * map bytes into the checked payload and must reject the modified + * blob. A kernel folding anything but the map contents themselves + * would wrongly accept this load. + */ + blob[data_sz / 2] ^= 0xff; + memset(ctx, 0, ctx_sz); + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + r = run_gen_loader(insns, insns_sz, blob, data_sz, excl, sizeof(excl), + sig, sig_sz, ctx, ctx_sz, &ran); + ASSERT_FALSE(ran, "tampered metadata rejected before run"); + ASSERT_EQ(r, -EKEYREJECTED, "signature is bound to the metadata"); +cleanup: + free(insns); + free(blob); + free(signbuf); free(ctx); test_signed_loader__destroy(skel); run_setup("cleanup", dir); @@ -1007,10 +1225,11 @@ static void lsm_signature_verdict(void) { char dir_tmpl[] = "/tmp/signed_loader_lsmXXXXXX", *dir = NULL; struct test_signed_loader_lsm *lsm = NULL; + __u32 sig_sz = 8192, msig_sz = 8192; int map_fd = -1, prog_fd = -1; bool have_fixture = false; struct gen_loader_fixture f; - __u32 sig_sz = 8192; + unsigned char *buf; __s32 ses_serial; __u8 sig[8192]; @@ -1029,7 +1248,7 @@ static void lsm_signature_verdict(void) if (!ASSERT_OK_FD(map_fd, "meta_map_unsigned")) goto out; lsm->bss->seen = 0; - prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, NULL, 0, 0); + prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, NULL, 0, 0, 0); close(map_fd); map_fd = -1; if (!ASSERT_OK_FD(prog_fd, "unsigned loader load")) @@ -1062,22 +1281,51 @@ static void lsm_signature_verdict(void) goto out; lsm->bss->seen = 0; prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, sig, - sig_sz, KEY_SPEC_SESSION_KEYRING); + sig_sz, KEY_SPEC_SESSION_KEYRING, 0); close(map_fd); map_fd = -1; - if (!ASSERT_OK_FD(prog_fd, "signed loader load")) - goto out; - close(prog_fd); + ASSERT_EQ(prog_fd, -EACCES, "unfolded metadata rejected"); + if (prog_fd >= 0) + close(prog_fd); prog_fd = -1; ses_serial = syscall(__NR_keyctl, KEYCTL_GET_KEYRING_ID, KEY_SPEC_SESSION_KEYRING, 0); ASSERT_EQ(lsm->bss->seen, 1, "signed: one observed load"); - ASSERT_EQ(lsm->bss->sig_verdict, BPF_SIG_VERIFIED, "signed verdict"); + ASSERT_EQ(lsm->bss->sig_verdict, BPF_SIG_VERIFIED, + "admission saw a valid signature"); ASSERT_EQ(lsm->bss->sig_keyring_type, BPF_SIG_KEYRING_USER, "signed keyring type"); ASSERT_GT(ses_serial, 0, "session keyring serial resolved"); ASSERT_EQ(lsm->bss->sig_keyring_serial, ses_serial, "signed: validated against session keyring"); + + buf = malloc((size_t)f.gopts.insns_sz + f.data_sz); + if (!ASSERT_OK_PTR(buf, "meta_signbuf")) + goto out; + memcpy(buf, f.gopts.insns, f.gopts.insns_sz); + memcpy(buf + f.gopts.insns_sz, f.blob, f.data_sz); + if (!ASSERT_OK(sign_buf(dir, buf, f.gopts.insns_sz + f.data_sz, + sig, &msig_sz), "sign insns||metadata")) { + free(buf); + goto out; + } + free(buf); + + map_fd = setup_meta_map(&f); + if (!ASSERT_OK_FD(map_fd, "meta_map_bound")) + goto out; + lsm->bss->seen = 0; + prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, sig, + msig_sz, KEY_SPEC_SESSION_KEYRING, 1); + close(map_fd); + map_fd = -1; + if (!ASSERT_OK_FD(prog_fd, "metadata-bound loader load")) + goto out; + close(prog_fd); + prog_fd = -1; + ASSERT_EQ(lsm->bss->seen, 1, "metadata: one observed load"); + ASSERT_EQ(lsm->bss->sig_verdict, BPF_SIG_VERIFIED, + "metadata-bound verdict"); out: if (map_fd >= 0) close(map_fd); @@ -1090,22 +1338,471 @@ out: test_signed_loader_lsm__destroy(lsm); } +/* + * Load-time metadata verification: the kernel folds the frozen metadata map + * into the signature (insns || metadata) and checks it at BPF_PROG_LOAD via + * fd_array_cnt, rather than the loader checking from within BPF. Sign that + * concatenation, hand the kernel the map, and confirm the signed loader loads, + * runs, and installs its target. + */ +static int loadtime_drive(const char *dir, const void *insns, __u32 insns_sz, + const void *data, __u32 data_sz, const __u8 *excl, + void *ctx, __u32 ctx_sz, int *load_ret, bool *ran) +{ + LIBBPF_OPTS(bpf_map_create_opts, mopts, + .excl_prog_hash = excl, + .excl_prog_hash_size = SHA256_DIGEST_LENGTH); + __u32 sig_sz = 8192, key = 0; + unsigned char *buf = NULL; + int map_fd, prog_fd, ret = 0; + union bpf_attr attr; + __u8 sig[8192]; + + *ran = false; + *load_ret = 0; + + /* + * Metadata map, bound to the loader digest and frozen, exactly as + * skel_internal.h's bpf_load_and_run() sets it up. + */ + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", 4, + data_sz, 1, &mopts); + if (map_fd < 0) { + ret = -errno; + goto out_load; + } + if (bpf_map_update_elem(map_fd, &key, data, 0) || bpf_map_freeze(map_fd)) { + ret = -errno; + goto out_load; + } + + /* Sign insns || metadata, the same bytes the kernel reconstructs. */ + buf = malloc((size_t)insns_sz + data_sz); + if (!buf) { + ret = -ENOMEM; + goto out_load; + } + memcpy(buf, insns, insns_sz); + memcpy(buf + insns_sz, data, data_sz); + ret = sign_buf(dir, buf, insns_sz + data_sz, sig, &sig_sz); + if (ret) + goto out_load; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = insns_sz / sizeof(struct bpf_insn); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(&map_fd); + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + attr.fd_array_cnt = 1; + memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + if (prog_fd < 0) { + ret = -errno; + goto out_load; + } + + memset(&attr, 0, sizeof(attr)); + attr.test.prog_fd = prog_fd; + attr.test.ctx_in = ptr_to_u64(ctx); + attr.test.ctx_size_in = ctx_sz; + if (syscall(__NR_bpf, BPF_PROG_RUN, &attr, + offsetofend(union bpf_attr, test)) < 0) { + ret = -errno; + goto out_prog; + } + *ran = true; + ret = (int)attr.test.retval; +out_prog: + close(prog_fd); + goto out_map; +out_load: + *load_ret = ret; +out_map: + free(buf); + if (map_fd >= 0) + close(map_fd); + return ret; +} + +static void loadtime_verify(struct bpf_object *obj, int expect_maps) +{ + LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true); + char dir_tmpl[] = "/tmp/signed_loader_ltXXXXXX", *dir = NULL; + int nr_maps = 0, nr_progs = 0, load_ret = 0, r; + __u8 excl[SHA256_DIGEST_LENGTH]; + struct bpf_prog_desc *pd; + struct bpf_map_desc *md; + unsigned char *blob = NULL; + struct bpf_program *p; + struct bpf_map *m; + __u32 ctx_sz, data_sz; + void *ctx = NULL; + bool ran = false; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + + if (!ASSERT_OK(bpf_object__gen_loader(obj, &gopts), "gen_loader")) + goto out; + if (!ASSERT_OK(bpf_object__load(obj), "gen_load")) + goto out; + + bpf_object__for_each_program(p, obj) + nr_progs++; + bpf_object__for_each_map(m, obj) + nr_maps++; + if (!ASSERT_EQ(nr_maps, expect_maps, "fixture map count")) + goto out; + + ctx_sz = sizeof(struct bpf_loader_ctx) + + nr_maps * sizeof(struct bpf_map_desc) + + nr_progs * sizeof(struct bpf_prog_desc); + ctx = calloc(1, ctx_sz); + if (!ASSERT_OK_PTR(ctx, "ctx_alloc")) + goto out; + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + + data_sz = gopts.data_sz; + blob = malloc(data_sz); + if (!ASSERT_OK_PTR(blob, "blob_alloc")) + goto out; + memcpy(blob, gopts.data, data_sz); + + /* excl_prog_hash = SHA256(loader insns) == the loader's prog->digest. */ + libbpf_sha256(gopts.insns, gopts.insns_sz, excl); + + r = loadtime_drive(dir, gopts.insns, gopts.insns_sz, blob, data_sz, + excl, ctx, ctx_sz, &load_ret, &ran); + ASSERT_OK(load_ret, "signed loader loaded (insns || metadata)"); + ASSERT_TRUE(ran, "loader ran"); + ASSERT_EQ(r, 0, "loader installed its target"); + + md = (struct bpf_map_desc *)((char *)ctx + sizeof(struct bpf_loader_ctx)); + pd = (struct bpf_prog_desc *)(md + nr_maps); + ASSERT_GT(pd[0].prog_fd, 0, "target program installed"); + if (nr_maps) + ASSERT_GT(md[0].map_fd, 0, "target map installed"); + + close_loader_ctx_fds(ctx, nr_maps, nr_progs); +out: + free(blob); + free(ctx); + if (dir) + run_setup("cleanup", dir); +} + +static void loadtime_no_map(void) +{ + struct test_signed_loader *skel = test_signed_loader__open(); + + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + loadtime_verify(skel->obj, 0); + test_signed_loader__destroy(skel); +} + +static void loadtime_with_map(void) +{ + struct test_signed_loader_map *skel = test_signed_loader_map__open(); + + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + loadtime_verify(skel->obj, 1); + test_signed_loader_map__destroy(skel); +} + +/* + * A signed program need not bind any map. A plain BPF_PROG_TYPE_SYSCALL + * program with no fd_array is signed over its instructions alone: the kernel + * verifies the signature, folds no metadata, and the program loads. Exercise + * the fd_array == NULL / fd_array_cnt == 0 path, and confirm the signature + * still authenticates the instructions (a tampered copy is rejected). + */ +static void signed_no_fd_array(void) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char dir_tmpl[] = "/tmp/signed_loaderXXXXXX", *dir; + __u32 sig_sz = 8192; + union bpf_attr attr; + __u8 sig[8192]; + int prog_fd, err; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + + /* No metadata map: the signed payload is the instructions alone. */ + if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz), + "sign-file")) + goto cleanup; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + /* fd_array and fd_array_cnt deliberately left NULL/0. */ + memcpy(attr.prog_name, "signed_nomap", sizeof("signed_nomap")); + + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + if (!ASSERT_GE(prog_fd, 0, "map-less signed program loaded")) { + if (prog_fd >= 0) + close(prog_fd); + goto cleanup; + } + close(prog_fd); + + /* The signature covers the instructions, so tampering must be rejected. */ + insns[0].imm = 1; + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + err = prog_fd < 0 ? -errno : prog_fd; + ASSERT_EQ(err, -EKEYREJECTED, "tampered map-less program rejected"); + if (prog_fd >= 0) + close(prog_fd); +cleanup: + run_setup("cleanup", dir); +} + +/* + * A signed program may reach maps only through fd_array indices, so the kernel + * folds (and thus attests) them. A direct BPF_PSEUDO_MAP_FD reference - a raw, + * unfolded fd baked into the signed instructions - is rejected by the verifier. + */ +static void signed_map_by_fd_rejected(void) +{ + struct bpf_insn insns[] = { + BPF_LD_MAP_FD(BPF_REG_1, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char dir_tmpl[] = "/tmp/signed_loaderXXXXXX", *dir; + __u32 sig_sz = 8192; + union bpf_attr attr; + __u8 sig[8192]; + int map_fd, prog_fd, err; + + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "sig_mapfd", 4, 4, 1, NULL); + if (!ASSERT_GE(map_fd, 0, "map_create")) + return; + insns[0].imm = map_fd; /* bake the raw map fd into the ld_imm64 */ + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + goto out_map; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + goto out_map; + } + + /* Sign the instructions, raw map fd and all. */ + if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz), + "sign-file")) + goto cleanup; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + /* No fd_array: the map is reached by a raw fd in the instructions. */ + memcpy(attr.prog_name, "signed_mapfd", sizeof("signed_mapfd")); + + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + err = prog_fd < 0 ? -errno : prog_fd; + ASSERT_EQ(err, -EINVAL, "signed program referencing a map by fd rejected"); + if (prog_fd >= 0) + close(prog_fd); +cleanup: + run_setup("cleanup", dir); +out_map: + close(map_fd); +} + +/* + * A signed program may reach maps only through the continuous fd_array, so the + * kernel folds (and thus attests) them. Referencing a map by fd_array *index* + * while leaving fd_array_cnt at 0 selects the sparse path, which resolves a map + * the signature never covered; the verifier rejects it up front with -EACCES. + */ +static void signed_sparse_fd_array_rejected(void) +{ + struct bpf_insn insns[] = { + BPF_LD_IMM64_RAW(BPF_REG_1, BPF_PSEUDO_MAP_IDX, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char dir_tmpl[] = "/tmp/signed_loader_spXXXXXX", *dir; + __u32 sig_sz = 8192; + union bpf_attr attr; + __u8 sig[8192]; + int map_fd, prog_fd, err; + + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "sig_sparse", 4, 4, 1, NULL); + if (!ASSERT_GE(map_fd, 0, "map_create")) + return; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + goto out_map; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + goto out_map; + } + + /* Sign the instructions alone; the sparse map is not folded. */ + if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz), + "sign-file")) + goto cleanup; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(&map_fd); + attr.fd_array_cnt = 0; /* sparse: force lazy map resolution */ + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + memcpy(attr.prog_name, "signed_sparse", sizeof("signed_sparse")); + + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + err = prog_fd < 0 ? -errno : prog_fd; + ASSERT_EQ(err, -EACCES, "signed program binding a sparse fd_array map rejected"); + if (prog_fd >= 0) + close(prog_fd); +cleanup: + run_setup("cleanup", dir); +out_map: + close(map_fd); +} + +static void signed_module_kfunc_rejected(void) +{ + struct bpf_insn insns[] = { + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 1, 1), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char dir_tmpl[] = "/tmp/signed_loader_kfnXXXXXX", *dir; + int prog_fd, err, fds[2]; + struct btf *btf = NULL; + __u32 sig_sz = 8192; + union bpf_attr attr; + __u8 sig[8192]; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz), + "sign-file")) + goto cleanup; + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf_new_empty")) + goto cleanup; + btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf_load")) + goto cleanup; + fds[0] = -1; + fds[1] = btf__fd(btf); + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(fds); + attr.fd_array_cnt = 0; /* sparse: force lazy kfunc BTF resolution */ + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + memcpy(attr.prog_name, "signed_kfunc", sizeof("signed_kfunc")); + + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + err = prog_fd < 0 ? -errno : prog_fd; + if (prog_fd >= 0) + close(prog_fd); + + ASSERT_EQ(err, -EACCES, "module kfunc BTF in signed program rejected"); +cleanup: + if (btf) + btf__free(btf); + run_setup("cleanup", dir); +} + void test_signed_loader(void) { - if (test__start_subtest("metadata_check_shape")) - metadata_check_shape(); + if (test__start_subtest("loadtime_no_map")) + loadtime_no_map(); + if (test__start_subtest("loadtime_with_map")) + loadtime_with_map(); if (test__start_subtest("metadata_match")) metadata_match(); - if (test__start_subtest("metadata_sha_mismatch")) - metadata_sha_mismatch(); - if (test__start_subtest("metadata_not_exclusive")) - metadata_not_exclusive(); - if (test__start_subtest("metadata_hash_not_computed")) - metadata_hash_not_computed(); if (test__start_subtest("signature_enforced")) signature_enforced(); + if (test__start_subtest("signed_nonexcl_fd_array_rejected")) + signed_nonexcl_fd_array_rejected(); + if (test__start_subtest("signed_unfrozen_fd_array_rejected")) + signed_unfrozen_fd_array_rejected(); + if (test__start_subtest("signed_nonarray_fd_array_rejected")) + signed_nonarray_fd_array_rejected(); + if (test__start_subtest("signed_btf_fd_array_rejected")) + signed_btf_fd_array_rejected(); + if (test__start_subtest("signed_module_kfunc_rejected")) + signed_module_kfunc_rejected(); + if (test__start_subtest("signature_failure_logs")) + signature_failure_logs(); if (test__start_subtest("signature_too_large")) signature_too_large(); + if (test__start_subtest("signature_zero_size")) + signature_zero_size(); if (test__start_subtest("signature_bad_keyring")) signature_bad_keyring(); if (test__start_subtest("metadata_ctx_max_entries_ignored")) @@ -1114,6 +1811,8 @@ void test_signed_loader(void) metadata_ctx_initial_value_ignored(); if (test__start_subtest("signature_authenticates_insns")) signature_authenticates_insns(); + if (test__start_subtest("signature_authenticates_metadata")) + signature_authenticates_metadata(); if (test__start_subtest("hash_requires_frozen")) hash_requires_frozen(); if (test__start_subtest("no_update_after_freeze")) @@ -1132,4 +1831,10 @@ void test_signed_loader(void) map_hash_unsupported_type(); if (test__start_subtest("lsm_signature_verdict")) lsm_signature_verdict(); + if (test__start_subtest("signed_no_fd_array")) + signed_no_fd_array(); + if (test__start_subtest("signed_map_by_fd_rejected")) + signed_map_by_fd_rejected(); + if (test__start_subtest("signed_sparse_fd_array_rejected")) + signed_sparse_fd_array_rejected(); } diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c index e5fc038d747b..1fef6ec2ba7a 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c @@ -7,7 +7,6 @@ #include "test_progs.h" #include "test_skmsg_load_helpers.skel.h" -#include "test_sockmap_update.skel.h" #include "test_sockmap_invalid_update.skel.h" #include "test_sockmap_skb_verdict_attach.skel.h" #include "test_sockmap_progs_query.skel.h" @@ -235,53 +234,6 @@ out: test_skmsg_load_helpers__destroy(skel); } -static void test_sockmap_update(enum bpf_map_type map_type) -{ - int err, prog, src; - struct test_sockmap_update *skel; - struct bpf_map *dst_map; - const __u32 zero = 0; - char dummy[14] = {0}; - LIBBPF_OPTS(bpf_test_run_opts, topts, - .data_in = dummy, - .data_size_in = sizeof(dummy), - .repeat = 1, - ); - __s64 sk; - - sk = connected_socket_v4(); - if (!ASSERT_NEQ(sk, -1, "connected_socket_v4")) - return; - - skel = test_sockmap_update__open_and_load(); - if (!ASSERT_OK_PTR(skel, "open_and_load")) - goto close_sk; - - prog = bpf_program__fd(skel->progs.copy_sock_map); - src = bpf_map__fd(skel->maps.src); - if (map_type == BPF_MAP_TYPE_SOCKMAP) - dst_map = skel->maps.dst_sock_map; - else - dst_map = skel->maps.dst_sock_hash; - - err = bpf_map_update_elem(src, &zero, &sk, BPF_NOEXIST); - if (!ASSERT_OK(err, "update_elem(src)")) - goto out; - - err = bpf_prog_test_run_opts(prog, &topts); - if (!ASSERT_OK(err, "test_run")) - goto out; - if (!ASSERT_NEQ(topts.retval, 0, "test_run retval")) - goto out; - - compare_cookies(skel->maps.src, dst_map); - -out: - test_sockmap_update__destroy(skel); -close_sk: - close(sk); -} - static void test_sockmap_invalid_update(void) { struct test_sockmap_invalid_update *skel; @@ -1422,10 +1374,6 @@ void test_sockmap_basic(void) test_skmsg_helpers(BPF_MAP_TYPE_SOCKMAP); if (test__start_subtest("sockhash sk_msg load helpers")) test_skmsg_helpers(BPF_MAP_TYPE_SOCKHASH); - if (test__start_subtest("sockmap update")) - test_sockmap_update(BPF_MAP_TYPE_SOCKMAP); - if (test__start_subtest("sockhash update")) - test_sockmap_update(BPF_MAP_TYPE_SOCKHASH); if (test__start_subtest("sockmap update in unsafe context")) test_sockmap_invalid_update(); if (test__start_subtest("sockmap copy")) diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt.c b/tools/testing/selftests/bpf/prog_tests/sockopt.c index eaac83a7f388..6c96f2d9fccf 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockopt.c +++ b/tools/testing/selftests/bpf/prog_tests/sockopt.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include <linux/tcp.h> #include <test_progs.h> #include <io_uring/mini_liburing.h> #include "cgroup_helpers.h" @@ -284,6 +285,27 @@ static struct sockopt_test { .io_uring_support = true, }, { + .descr = "getsockopt: deny negative ctx->optlen in TCP_ZEROCOPY_RECEIVE", + .insns = { + /* ctx->optlen = -1 */ + BPF_MOV64_IMM(BPF_REG_0, -1), + BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, + offsetof(struct bpf_sockopt, optlen)), + + /* return 1 */ + BPF_MOV64_IMM(BPF_REG_0, 1), + BPF_EXIT_INSN(), + }, + .attach_type = BPF_CGROUP_GETSOCKOPT, + .expected_attach_type = BPF_CGROUP_GETSOCKOPT, + + .get_level = IPPROTO_TCP, + .get_optname = TCP_ZEROCOPY_RECEIVE, + .get_optlen = sizeof(struct tcp_zerocopy_receive), + + .error = EFAULT_GETSOCKOPT, + }, + { .descr = "getsockopt: ignore >PAGE_SIZE optlen", .insns = { /* write 0xFF to the first optval byte */ diff --git a/tools/testing/selftests/bpf/prog_tests/stream.c b/tools/testing/selftests/bpf/prog_tests/stream.c index c3cce5c292bd..e4e9374309e2 100644 --- a/tools/testing/selftests/bpf/prog_tests/stream.c +++ b/tools/testing/selftests/bpf/prog_tests/stream.c @@ -103,6 +103,12 @@ void test_stream_arena_fault_address(void) test_address(skel->progs.stream_arena_read_fault, &skel->bss->fault_addr); if (test__start_subtest("write_fault")) test_address(skel->progs.stream_arena_write_fault, &skel->bss->fault_addr); + if (test__start_subtest("load_acquire_fault")) + test_address(skel->progs.stream_arena_load_acquire_fault, &skel->bss->fault_addr); + if (test__start_subtest("xchg_fault")) + test_address(skel->progs.stream_arena_xchg_fault, &skel->bss->fault_addr); + if (test__start_subtest("cmpxchg_fault")) + test_address(skel->progs.stream_arena_cmpxchg_fault, &skel->bss->fault_addr); stream__destroy(skel); } diff --git a/tools/testing/selftests/bpf/prog_tests/tailcalls.c b/tools/testing/selftests/bpf/prog_tests/tailcalls.c index a5a226d0104c..c5c9d6c359bb 100644 --- a/tools/testing/selftests/bpf/prog_tests/tailcalls.c +++ b/tools/testing/selftests/bpf/prog_tests/tailcalls.c @@ -12,6 +12,9 @@ #include "tailcall_cgrp_storage_no_storage.skel.h" #include "tailcall_cgrp_storage.skel.h" #include "tailcall_sleepable.skel.h" +#include "tailcall_callback.skel.h" +#include "tailcall_bpf2bpf2.skel.h" +#include "tailcall_bpf2bpf_fexit.skel.h" /* test_tailcall_1 checks basic functionality by patching multiple locations * in a single program for a single tail call slot with nop->jmp, jmp->nop @@ -1901,6 +1904,55 @@ out: tailcall_sleepable__destroy(skel); } +static void test_tailcall_callback(void) +{ + RUN_TESTS(tailcall_callback); +} + +static void test_tailcall_bpf2bpf_fexit_links(void) +{ + struct tailcall_bpf2bpf_fexit *skel1 = NULL, *skel2 = NULL; + struct tailcall_bpf2bpf2 *skel_tc; + int err, prog_fd; + + skel_tc = tailcall_bpf2bpf2__open_and_load(); + if (!ASSERT_OK_PTR(skel_tc, "tailcall_bpf2bpf2__open_and_load")) + return; + + skel1 = tailcall_bpf2bpf_fexit__open(); + if (!ASSERT_OK_PTR(skel1, "tailcall_bpf2bpf_fexit__open")) + goto out; + + prog_fd = bpf_program__fd(skel_tc->progs.classifier_0); + err = bpf_program__set_attach_target(skel1->progs.fexit, prog_fd, "subprog_tail"); + if (!ASSERT_OK(err, "bpf_program__set_attach_target")) + goto out; + + err = tailcall_bpf2bpf_fexit__load(skel1); + if (!ASSERT_OK(err, "tailcall_bpf2bpf_fexit__load")) + goto out; + + skel1->links.fexit = bpf_program__attach_trace(skel1->progs.fexit); + if (!ASSERT_OK_PTR(skel1->links.fexit, "bpf_program__attach_trace")) + goto out; + + skel2 = tailcall_bpf2bpf_fexit__open(); + if (!ASSERT_OK_PTR(skel2, "tailcall_bpf2bpf_fexit__open")) + goto out; + + err = bpf_program__set_attach_target(skel2->progs.fexit, prog_fd, "subprog_tail"); + if (!ASSERT_OK(err, "bpf_program__set_attach_target")) + goto out; + + err = tailcall_bpf2bpf_fexit__load(skel2); + ASSERT_OK(err, "tailcall_bpf2bpf_fexit__load"); + +out: + tailcall_bpf2bpf_fexit__destroy(skel1); + tailcall_bpf2bpf_fexit__destroy(skel2); + tailcall_bpf2bpf2__destroy(skel_tc); +} + void test_tailcalls(void) { if (test__start_subtest("tailcall_1")) @@ -1967,4 +2019,7 @@ void test_tailcalls(void) test_tailcall_cgrp_storage_no_storage_leaf(); if (test__start_subtest("tailcall_cgrp_storage_no_storage_bridge")) test_tailcall_cgrp_storage_no_storage_bridge(); + test_tailcall_callback(); + if (test__start_subtest("tailcall_bpf2bpf_fexit_links")) + test_tailcall_bpf2bpf_fexit_links(); } diff --git a/tools/testing/selftests/bpf/prog_tests/task_kfunc.c b/tools/testing/selftests/bpf/prog_tests/task_kfunc.c index e6e95c1416e6..30d403028f98 100644 --- a/tools/testing/selftests/bpf/prog_tests/task_kfunc.c +++ b/tools/testing/selftests/bpf/prog_tests/task_kfunc.c @@ -176,6 +176,14 @@ static const char * const success_tests[] = { "test_task_from_pid_current", "test_task_from_pid_invalid", "task_kfunc_acquire_trusted_walked", + "task_kfunc_acquire_after_spin_unlock_non_sleepable", + "task_kfunc_acquire_after_spin_unlock_explicit_rcu", + "task_kfunc_acquire_after_spin_unlock_preempt_disabled", + "task_kfunc_acquire_after_spin_unlock_irq_disabled", + "task_kfunc_acquire_after_rcu_unlock_preempt_disabled", + "task_kfunc_acquire_after_rcu_unlock_irq_disabled", + "task_kfunc_acquire_after_preempt_enable_explicit_rcu", + "task_kfunc_acquire_after_irq_restore_explicit_rcu", "test_task_kfunc_flavor_relo", "test_task_kfunc_flavor_relo_not_found", }; diff --git a/tools/testing/selftests/bpf/prog_tests/tc_redirect.c b/tools/testing/selftests/bpf/prog_tests/tc_redirect.c index 64fbda082309..af8968b89ad7 100644 --- a/tools/testing/selftests/bpf/prog_tests/tc_redirect.c +++ b/tools/testing/selftests/bpf/prog_tests/tc_redirect.c @@ -192,6 +192,8 @@ static int create_netkit(int mode, char *prim, char *peer) req.n.nlmsg_len += sizeof(struct ifinfomsg); addattr_l(&req.n, sizeof(req), IFLA_IFNAME, peer, strlen(peer)); addattr_nest_end(&req.n, peer_info); + addattr32(&req.n, sizeof(req), IFLA_NETKIT_SCRUB, + NETKIT_SCRUB_NONE); addattr_nest_end(&req.n, data); addattr_nest_end(&req.n, linkinfo); @@ -405,6 +407,24 @@ fail: return -1; } +static struct bpf_link *netns_attach_nk(const char *ns, int ifindex, + struct bpf_program *prog) +{ + LIBBPF_OPTS(bpf_netkit_opts, optl); + struct nstoken *nstoken = NULL; + struct bpf_link *link = NULL; + + nstoken = open_netns(ns); + if (!ASSERT_OK_PTR(nstoken, "setns")) + goto cleanup; + + link = bpf_program__attach_netkit(prog, ifindex, &optl); +cleanup: + if (nstoken) + close_netns(nstoken); + return link; +} + static void test_tcp(int family, const char *addr, __u16 port) { int listen_fd = -1, accept_fd = -1, client_fd = -1; @@ -1082,6 +1102,53 @@ done: close_netns(nstoken); } +static void test_tc_redirect_peer_ing(struct netns_setup_result *setup_result) +{ + struct test_tc_peer *skel; + struct nstoken *nstoken; + int err; + + nstoken = open_netns(NS_FWD); + if (!ASSERT_OK_PTR(nstoken, "setns fwd")) + return; + + skel = test_tc_peer__open(); + if (!ASSERT_OK_PTR(skel, "test_tc_peer__open")) + goto done; + + skel->rodata->IFINDEX_SRC = setup_result->ifindex_src_fwd; + skel->rodata->IFINDEX_DST = setup_result->ifindex_dst_fwd; + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc_src_ing, + BPF_NETKIT_PRIMARY), 0, "src_prog_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc_dst_ing, + BPF_NETKIT_PRIMARY), 0, "dst_prog_attach_type"); + + err = test_tc_peer__load(skel); + if (!ASSERT_OK(err, "test_tc_peer__load")) + goto done; + + skel->links.tc_src_ing = netns_attach_nk(NS_SRC, + setup_result->ifindex_src, + skel->progs.tc_src_ing); + if (!ASSERT_OK_PTR(skel->links.tc_src_ing, "attach_src")) + goto done; + skel->links.tc_dst_ing = netns_attach_nk(NS_DST, + setup_result->ifindex_dst, + skel->progs.tc_dst_ing); + if (!ASSERT_OK_PTR(skel->links.tc_dst_ing, "attach_dst")) + goto done; + + if (!ASSERT_OK(set_forwarding(false), "disable forwarding")) + goto done; + + test_connectivity(); + +done: + if (skel) + test_tc_peer__destroy(skel); + close_netns(nstoken); +} + static int tun_open(char *name) { struct ifreq ifr; @@ -1280,6 +1347,7 @@ static void *test_tc_redirect_run_tests(void *arg) RUN_TEST(tc_redirect_peer, MODE_VETH); RUN_TEST(tc_redirect_peer, MODE_NETKIT); + RUN_TEST(tc_redirect_peer_ing, MODE_NETKIT); RUN_TEST(tc_redirect_peer_l3, MODE_VETH); RUN_TEST(tc_redirect_peer_l3, MODE_NETKIT); RUN_TEST(tc_redirect_neigh, MODE_VETH); diff --git a/tools/testing/selftests/bpf/prog_tests/test_map_uninit.c b/tools/testing/selftests/bpf/prog_tests/test_map_uninit.c new file mode 100644 index 000000000000..d0ba2ca587b0 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/test_map_uninit.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> + +#include "map_kptr.skel.h" + +void test_map_uninit_mem_exposure(void) +{ + size_t value_sz, slot_sz, lookup_sz, tail_sz; + int err, key, nr_cpus, cpu, map_fd; + __u8 *value = NULL, *zero = NULL; + struct bpf_program *prog; + struct map_kptr *skel; + + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) + return; + + skel = map_kptr__open(); + if (!ASSERT_OK_PTR(skel, "map_kptr__open")) + return; + + bpf_object__for_each_program(prog, skel->obj) { + err = bpf_program__set_autoload(prog, false); + if (!ASSERT_OK(err, "bpf_program__set_autoload")) + goto out; + } + + err = map_kptr__load(skel); + if (!ASSERT_OK(err, "map_kptr__load")) + goto out; + + value_sz = bpf_map__value_size((skel)->maps.pcpu_array); + slot_sz = roundup(value_sz, 8); + tail_sz = slot_sz - value_sz; + if (!ASSERT_NEQ(tail_sz, 0, "tail_sz")) + goto out; + + lookup_sz = slot_sz * nr_cpus; + map_fd = bpf_map__fd(skel->maps.pcpu_array); + + value = malloc(lookup_sz); + zero = calloc(1, tail_sz); + if (!ASSERT_OK_PTR(value, "malloc value") || !ASSERT_OK_PTR(zero, "calloc zero")) + goto out; + + key = 0; + memset(value, 0x2B, lookup_sz); + err = bpf_map_update_elem(map_fd, &key, value, BPF_ANY); + if (!ASSERT_OK(err, "bpf_map_update_elem")) + goto out; + + memset(value, 0xFF, lookup_sz); + err = bpf_map_lookup_elem(map_fd, &key, value); + if (!ASSERT_OK(err, "bpf_map_lookup_elem")) + goto out; + + for (cpu = 0; cpu < nr_cpus; cpu++) { + __u8 *tail = value + cpu * slot_sz + value_sz; + + if (!ASSERT_MEMEQ(tail, zero, tail_sz, "zeroed tail bytes")) + goto out; + } + +out: + free(zero); + free(value); + map_kptr__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c new file mode 100644 index 000000000000..7f9f54ba3fbe --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> + +#include "struct_ops_arena.skel.h" +#include "struct_ops_arena_attach.skel.h" +#include "struct_ops_arena_fail.skel.h" + +#if defined(__x86_64__) || defined(__aarch64__) +/* + * Attach callbacks with __arena and __arena__nullable arguments and drive + * them through the bpf_testmod_ops3_call_test_arena*() kfuncs. + */ +static void arena_arg(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct struct_ops_arena *skel; + struct bpf_link *link = NULL; + int err; + + skel = struct_ops_arena__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena__open_and_load")) + return; + + link = bpf_map__attach_struct_ops(skel->maps.testmod_arena); + if (!ASSERT_OK_PTR(link, "attach_struct_ops")) + goto out; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.trigger), + &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "trigger_retval"); + +out: + bpf_link__destroy(link); + struct_ops_arena__destroy(skel); +} + +/* + * A program with no arena cannot attach to a member with an __arena + * argument. + */ +static void arena_arg_fail(void) +{ + struct struct_ops_arena_fail *skel; + + skel = struct_ops_arena_fail__open_and_load(); + if (ASSERT_ERR_PTR(skel, "struct_ops_arena_fail__open_and_load")) + return; + + struct_ops_arena_fail__destroy(skel); +} + +static void arena_arg_attach_one(int target_fd, const char *prog_name) +{ + struct struct_ops_arena_attach *skel; + struct bpf_program *prog, *pos; + char log_buf[64 * 1024]; + int err; + + skel = struct_ops_arena_attach__open(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_attach__open")) + return; + + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, prog_name)) + goto out; + + bpf_object__for_each_program(pos, skel->obj) + bpf_program__set_autoload(pos, pos == prog); + + err = bpf_program__set_attach_target(prog, target_fd, "test_arena_cb"); + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + log_buf[0] = '\0'; + bpf_program__set_log_buf(prog, log_buf, sizeof(log_buf)); + err = struct_ops_arena_attach__load(skel); + + ASSERT_EQ(err, -EOPNOTSUPP, prog_name); + ASSERT_HAS_SUBSTR(log_buf, "Cannot attach to a target with arena context arguments", + "verifier_log"); + +out: + struct_ops_arena_attach__destroy(skel); +} + +static void arena_arg_attach(void) +{ + struct struct_ops_arena *skel; + int target_fd; + + skel = struct_ops_arena__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena__open_and_load")) + return; + + target_fd = bpf_program__fd(skel->progs.test_arena_cb); + arena_arg_attach_one(target_fd, "fentry_test_arena"); + arena_arg_attach_one(target_fd, "fexit_test_arena"); + arena_arg_attach_one(target_fd, "freplace_test_arena"); + + struct_ops_arena__destroy(skel); +} +#endif + +/* + * Serialized because it attaches the singleton bpf_testmod_ops3, which + * test_struct_ops_private_stack also attaches; registering it twice fails + * with -EEXIST. + */ +void serial_test_struct_ops_arena(void) +{ + /* + * Arena struct_ops arguments need JIT support, currently x86-64 and + * arm64 only. Elsewhere verification fails with "JIT does not support + * arena arguments", so the programs cannot even load. + */ +#if defined(__x86_64__) || defined(__aarch64__) + if (test__start_subtest("arena_arg")) + arena_arg(); + if (test__start_subtest("arena_arg_fail")) + arena_arg_fail(); + if (test__start_subtest("arena_arg_attach")) + arena_arg_attach(); +#else + test__skip(); +#endif +} diff --git a/tools/testing/selftests/bpf/prog_tests/test_tc_tunnel.c b/tools/testing/selftests/bpf/prog_tests/test_tc_tunnel.c index 1aa7c9463980..67ba27d69347 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_tc_tunnel.c +++ b/tools/testing/selftests/bpf/prog_tests/test_tc_tunnel.c @@ -438,7 +438,6 @@ static int setup(void) SYS(fail_close_ns_client, "ip link add %s type veth peer name %s", "veth1 mtu 1500 netns " CLIENT_NS " address " MAC_ADDR_VETH1, "veth2 mtu 1500 netns " SERVER_NS " address " MAC_ADDR_VETH2); - SYS(fail_close_ns_client, "ethtool -K veth1 tso off"); SYS(fail_close_ns_client, "ip link set veth1 up"); nstoken_server = open_netns(SERVER_NS); if (!ASSERT_OK_PTR(nstoken_server, "open server ns")) diff --git a/tools/testing/selftests/bpf/prog_tests/test_veristat.c b/tools/testing/selftests/bpf/prog_tests/test_veristat.c index 9aff08ac55c0..11f3de2b66ad 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_veristat.c +++ b/tools/testing/selftests/bpf/prog_tests/test_veristat.c @@ -37,6 +37,14 @@ static struct fixture *init_fixture(void) return fix; } +static void read_output(struct fixture *fix) +{ + ssize_t len = pread(fix->fd, fix->output, fix->sz - 1, 0); + + fix->output[len < 0 ? 0 : len] = 0; + ASSERT_GE(len, 0, "pread"); +} + static void teardown_fixture(struct fixture *fix) { free(fix->output); @@ -74,7 +82,7 @@ static void test_set_global_vars_succeeds(void) " -G \"struct11 [ 7 ] [ 5 ] .struct2[0][1].u.mat[3][0] = 175\" " \ " -vl2 > %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("=0xf000000000000001 ", "var_s64 = 0xf000000000000001"); __CHECK_STR("=0xfedcba9876543210 ", "var_u64 = 0xfedcba9876543210"); __CHECK_STR("=0x80000000 ", "var_s32 = -0x80000000"); @@ -116,7 +124,7 @@ static void test_set_global_vars_from_file_succeeds(void) syncfs(fd); SYS(out, "%s set_global_vars.bpf.o -G \"@%s\" -vl2 > %s", fix->veristat, input_file, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("=0x8000 ", "var_s16 = -32768"); __CHECK_STR("=0xecec ", "var_u16 = 60652"); @@ -134,7 +142,7 @@ static void test_set_global_vars_out_of_range(void) "%s set_global_vars.bpf.o -G \"var_s32 = 2147483648\" -vl2 2> %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("is out of range [-2147483648; 2147483647]", "out of range"); out: @@ -149,7 +157,7 @@ static void test_unsupported_ptr_array_type(void) "%s set_global_vars.bpf.o -G \"ptr_arr[0] = 0\" -vl2 2> %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("Can't set ptr_arr[0]. Only ints and enums are supported", "ptr_arr"); out: @@ -164,7 +172,7 @@ static void test_array_out_of_bounds(void) "%s set_global_vars.bpf.o -G \"arr[99] = 0\" -vl2 2> %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("Array index 99 is out of bounds", "arr[99]"); out: @@ -179,7 +187,7 @@ static void test_array_index_not_found(void) "%s set_global_vars.bpf.o -G \"arr[EG2] = 0\" -vl2 2> %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("Can't resolve enum value EG2", "arr[EG2]"); out: @@ -230,6 +238,97 @@ out: teardown_fixture(fix); } +/* + * Name filter tests below run veristat on veristat_foo.bpf.o and + * veristat_bar.bpf.o, both defining programs 'foo', 'bar' and 'buz'. + * Every entry describes a single (filters, file, prog) combination and + * tells whether that program is expected in the veristat output: + * 'true' if it is, 'false' if it is not and -1 if veristat is expected + * to reject the filter. + */ +#define FILTER_OBJS "veristat_foo.bpf.o veristat_bar.bpf.o" + +static const struct name_filter_case { + const char *filters; + const char *file; + const char *prog; + int included; +} name_filter_cases[] = { + /* no filters, every program is processed */ + { "", "foo", "foo", true }, + { "", "foo", "bar", true }, + { "", "foo", "buz", true }, + { "", "bar", "foo", true }, + { "", "bar", "bar", true }, + { "", "bar", "buz", true }, + /* deny filters */ + { "-f '!*foo*'", "foo", "bar", false }, + { "-f '!*foo*'", "bar", "foo", false }, + { "-f '!*foo*'", "bar", "bar", true }, + { "-f '!*foo*/bar'", "foo", "bar", false }, + { "-f '!*foo*/bar'", "foo", "buz", true }, + { "-f '!*foo*/bar'", "bar", "bar", true }, + { "-f '!*foo*/'", "foo", "bar", false }, + { "-f '!*foo*/'", "bar", "bar", true }, + { "-f '!/bar'", "foo", "bar", false }, + { "-f '!/bar'", "foo", "foo", true }, + { "-f '!/'", "foo", "bar", -1 }, + { "-f '!'", "foo", "bar", -1 }, + /* allow filters */ + { "-f '*foo*'", "foo", "bar", true }, + { "-f '*foo*'", "bar", "foo", true }, + { "-f '*foo*'", "bar", "bar", false }, + { "-f '*foo*/bar'", "foo", "bar", true }, + { "-f '*foo*/bar'", "foo", "buz", false }, + { "-f '*foo*/bar'", "bar", "bar", false }, + { "-f '*foo*/'", "foo", "bar", true }, + { "-f '*foo*/'", "bar", "bar", false }, + { "-f '/bar'", "foo", "bar", true }, + { "-f '/bar'", "foo", "foo", false }, + { "-f '/'", "foo", "bar", -1 }, + { "-f ''", "foo", "bar", -1 }, + /* allow and deny filters combined */ + { "-f '*foo*/' -f '!/bar'", "foo", "foo", true }, + { "-f '*foo*/' -f '!/bar'", "foo", "bar", false }, + { "-f '*foo*/' -f '!/bar'", "bar", "foo", false }, +}; + +static void test_name_filters(void) +{ + struct fixture *fix = init_fixture(); + const struct name_filter_case *t; + char cmd[512], row[64], name[128]; + int i, err; + + for (i = 0; i < ARRAY_SIZE(name_filter_cases); i++) { + t = &name_filter_cases[i]; + /* stderr is merged with stdout in order to catch error messages */ + snprintf(cmd, sizeof(cmd), "%s " FILTER_OBJS " -q -o csv -e file,prog %s > %s 2>&1", + fix->veristat, t->filters, fix->tmpfile); + err = system(cmd); + read_output(fix); + + snprintf(row, sizeof(row), "veristat_%s.bpf.o,%s", t->file, t->prog); + snprintf(name, sizeof(name), "veristat %s: %s", t->filters, row); + switch (t->included) { + case true: + ASSERT_OK(err, name); + ASSERT_HAS_SUBSTR(fix->output, row, name); + break; + case false: + ASSERT_OK(err, name); + ASSERT_FALSE(!!strstr(fix->output, row), name); + break; + case -1: + ASSERT_NEQ(err, 0, name); + ASSERT_HAS_SUBSTR(fix->output, "Invalid filter", name); + break; + } + } + + teardown_fixture(fix); +} + void test_veristat(void) { if (test__start_subtest("set_global_vars_succeeds")) @@ -256,6 +355,8 @@ void test_veristat(void) if (test__start_subtest("test_no_array_index_for_array")) test_no_array_index_for_array(); + if (test__start_subtest("name_filters")) + test_name_filters(); } #undef __CHECK_STR diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_failure.c b/tools/testing/selftests/bpf/prog_tests/tracing_failure.c index f9f9e1cb87bf..eb585918f0d4 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_failure.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_failure.c @@ -76,6 +76,24 @@ static void test_fexit_noreturns(void) "Attaching fexit/fsession/fmod_ret to __noreturn function 'do_exit' is rejected."); } +static void test_fexit_int128_ret(void) +{ + /* + * __int128 is returned in a register pair on x86_64 and arm64, so + * bpf_testmod_test_int128_ret() is BTF-encoded and attachable and the + * verifier can reject its >8 byte return value. Other architectures + * return a __int128 differently (e.g. s390x returns larger values by + * reference, which makes pahole skip BTF encoding of the function), so + * only exercise this on x86_64 and arm64. + */ +#if defined(__x86_64__) || defined(__aarch64__) + test_tracing_fail_prog("fexit_int128_ret", + "with a >8 byte return value is not supported for this attach type"); +#else + test__skip(); +#endif +} + void test_tracing_failure(void) { if (test__start_subtest("bpf_spin_lock")) @@ -86,4 +104,6 @@ void test_tracing_failure(void) test_tracing_deny(); if (test__start_subtest("fexit_noreturns")) test_fexit_noreturns(); + if (test__start_subtest("fexit_int128_ret")) + test_fexit_int128_ret(); } diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c index f02ffc7f41d7..0aa9532a05cf 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c @@ -460,6 +460,73 @@ static void test_intersect(void) tracing_multi_intersect__destroy(skel); } +static void test_fentry_after_multi(void) +{ + static const char * const funcs[] = { + "bpf_fentry_test1", + }; + struct bpf_link *fentry_link = NULL, *multi_link = NULL; + struct tracing_multi_intersect *skel = NULL; + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + LIBBPF_OPTS(bpf_test_run_opts, topts); + __u32 *ids = NULL; + int err; + + skel = tracing_multi_intersect__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_intersect__open_and_load")) + return; + + skel->bss->pid = getpid(); + + ids = get_ids(funcs, ARRAY_SIZE(funcs), NULL); + if (!ASSERT_OK_PTR(ids, "get_ids")) + goto cleanup; + + opts.ids = ids; + opts.cnt = ARRAY_SIZE(funcs); + multi_link = bpf_program__attach_tracing_multi(skel->progs.fentry_1, NULL, &opts); + if (!ASSERT_OK_PTR(multi_link, "attach_multi")) + goto cleanup; + + fentry_link = bpf_program__attach(skel->progs.fentry); + if (!ASSERT_OK_PTR(fentry_link, "attach_fentry")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.fentry_1), &topts); + if (!ASSERT_OK(err, "test_run")) + goto cleanup; + ASSERT_EQ(skel->bss->test_result_fentry_1, 1, "multi_fentry"); + ASSERT_EQ(skel->bss->test_result_fentry, 1, "fentry"); + + err = bpf_link__destroy(fentry_link); + fentry_link = NULL; + if (!ASSERT_OK(err, "destroy_fentry")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.fentry_1), &topts); + if (!ASSERT_OK(err, "test_run_multi")) + goto cleanup; + ASSERT_EQ(skel->bss->test_result_fentry_1, 2, "multi_fentry_only"); + ASSERT_EQ(skel->bss->test_result_fentry, 1, "fentry_detached"); + + err = bpf_link__destroy(multi_link); + multi_link = NULL; + if (!ASSERT_OK(err, "destroy_multi")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.fentry_1), &topts); + if (!ASSERT_OK(err, "test_run_detached")) + goto cleanup; + ASSERT_EQ(skel->bss->test_result_fentry_1, 2, "multi_fentry_detached"); + ASSERT_EQ(skel->bss->test_result_fentry, 1, "fentry_still_detached"); + +cleanup: + bpf_link__destroy(fentry_link); + bpf_link__destroy(multi_link); + free(ids); + tracing_multi_intersect__destroy(skel); +} + static void test_session(void) { LIBBPF_OPTS(bpf_test_run_opts, topts); @@ -957,4 +1024,6 @@ void test_tracing_multi_test(void) if (test__start_subtest("attach_api_fails")) test_attach_api_fails(); RUN_TESTS(tracing_multi_verifier); + if (test__start_subtest("fentry_after_multi")) + test_fentry_after_multi(); } diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c index 6f8c0bfb0415..15b95d0235b5 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c @@ -4,6 +4,7 @@ #include <test_progs.h> #include "tracing_struct.skel.h" #include "tracing_struct_many_args.skel.h" +#include "tracing_struct_int128.skel.h" static void test_struct_args(void) { @@ -112,6 +113,39 @@ destroy_skel: tracing_struct_many_args__destroy(skel); } +static void test_int128_args(void) +{ + /* + * __int128 arguments are passed in a register pair on x86_64 and + * arm64, which the trampoline packs into two context slots. Other + * architectures pass a __int128 differently (e.g. s390x passes larger + * arguments by reference), so only exercise this on x86_64 and arm64. + */ +#if defined(__x86_64__) || defined(__aarch64__) + struct tracing_struct_int128 *skel; + int err; + + skel = tracing_struct_int128__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_struct_int128__open_and_load")) + return; + + err = tracing_struct_int128__attach(skel); + if (!ASSERT_OK(err, "tracing_struct_int128__attach")) + goto destroy_skel; + + ASSERT_OK(trigger_module_test_read(256), "trigger_read"); + + ASSERT_EQ(skel->bss->t_b, 2, "t:b"); + ASSERT_EQ(skel->bss->t_c, 3, "t:c"); + ASSERT_EQ(skel->bss->t_ret, 6, "t ret"); + +destroy_skel: + tracing_struct_int128__destroy(skel); +#else + test__skip(); +#endif +} + static void test_union_args(void) { struct tracing_struct *skel; @@ -145,6 +179,8 @@ void test_tracing_struct(void) test_struct_args(); if (test__start_subtest("struct_many_args")) test_struct_many_args(); + if (test__start_subtest("int128_args")) + test_int128_args(); if (test__start_subtest("union_args")) test_union_args(); } diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index be97f6887f0e..64ac49ad67e6 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -2,6 +2,8 @@ #include <test_progs.h> +#include "arena_kfunc.skel.h" +#include "arena_kfunc_jit.skel.h" #include "cap_helpers.h" #include "verifier_align.skel.h" #include "verifier_and.skel.h" @@ -61,12 +63,14 @@ #include "verifier_loops1.skel.h" #include "verifier_lwt.skel.h" #include "verifier_map_in_map.skel.h" +#include "verifier_map_lookup_refine.skel.h" #include "verifier_map_ptr.skel.h" #include "verifier_map_ptr_mixing.skel.h" #include "verifier_map_ret_val.skel.h" #include "verifier_masking.skel.h" #include "verifier_may_goto_1.skel.h" #include "verifier_may_goto_2.skel.h" +#include "verifier_mem_size_reg.skel.h" #include "verifier_meta_access.skel.h" #include "verifier_movsx.skel.h" #include "verifier_mtu.skel.h" @@ -75,6 +79,7 @@ #include "verifier_netfilter_retcode.skel.h" #include "verifier_bpf_fastcall.skel.h" #include "verifier_or_jmp32_k.skel.h" +#include "verifier_percpu_addr.skel.h" #include "verifier_precision.skel.h" #include "verifier_prevent_map_lookup.skel.h" #include "verifier_private_stack.skel.h" @@ -98,6 +103,7 @@ #include "verifier_stack_arg_order.skel.h" #include "verifier_stack_ptr.skel.h" #include "verifier_store_release.skel.h" +#include "verifier_subprog_insn_stats.skel.h" #include "verifier_subprog_precision.skel.h" #include "verifier_subprog_topo.skel.h" #include "verifier_subreg.skel.h" @@ -124,6 +130,7 @@ #include "verifier_jit_inline.skel.h" #include "irq.skel.h" #include "verifier_ctx_ptr_param.skel.h" +#include "verifier_zext.skel.h" #define MAX_ENTRIES 11 @@ -159,6 +166,10 @@ static void run_tests_aux(const char *skel_name, #define RUN(skel) run_tests_aux(#skel, skel##__elf_bytes, NULL) +void test_arena_kfunc(void) { RUN_TESTS(arena_kfunc); } + +void test_arena_kfunc_jit(void) { RUN_TESTS(arena_kfunc_jit); } + void test_verifier_align(void) { RUN(verifier_align); } void test_verifier_and(void) { RUN(verifier_and); } void test_verifier_arena(void) { RUN(verifier_arena); } @@ -215,12 +226,14 @@ void test_verifier_liveness_exp(void) { RUN(verifier_liveness_exp); } void test_verifier_loops1(void) { RUN(verifier_loops1); } void test_verifier_lwt(void) { RUN(verifier_lwt); } void test_verifier_map_in_map(void) { RUN(verifier_map_in_map); } +void test_verifier_map_lookup_refine(void) { RUN(verifier_map_lookup_refine); } void test_verifier_map_ptr(void) { RUN(verifier_map_ptr); } void test_verifier_map_ptr_mixing(void) { RUN(verifier_map_ptr_mixing); } void test_verifier_map_ret_val(void) { RUN(verifier_map_ret_val); } void test_verifier_masking(void) { RUN(verifier_masking); } void test_verifier_may_goto_1(void) { RUN(verifier_may_goto_1); } void test_verifier_may_goto_2(void) { RUN(verifier_may_goto_2); } +void test_verifier_mem_size_reg(void) { RUN(verifier_mem_size_reg); } void test_verifier_meta_access(void) { RUN(verifier_meta_access); } void test_verifier_movsx(void) { RUN(verifier_movsx); } void test_verifier_mul(void) { RUN(verifier_mul); } @@ -228,6 +241,7 @@ void test_verifier_netfilter_ctx(void) { RUN(verifier_netfilter_ctx); } void test_verifier_netfilter_retcode(void) { RUN(verifier_netfilter_retcode); } void test_verifier_bpf_fastcall(void) { RUN(verifier_bpf_fastcall); } void test_verifier_or_jmp32_k(void) { RUN(verifier_or_jmp32_k); } +void test_verifier_percpu_addr(void) { RUN(verifier_percpu_addr); } void test_verifier_precision(void) { RUN(verifier_precision); } void test_verifier_prevent_map_lookup(void) { RUN(verifier_prevent_map_lookup); } void test_verifier_private_stack(void) { RUN(verifier_private_stack); } @@ -251,6 +265,7 @@ void test_verifier_stack_arg(void) { RUN(verifier_stack_arg); } void test_verifier_stack_arg_order(void) { RUN(verifier_stack_arg_order); } void test_verifier_stack_ptr(void) { RUN(verifier_stack_ptr); } void test_verifier_store_release(void) { RUN(verifier_store_release); } +void test_verifier_subprog_insn_stats(void) { RUN(verifier_subprog_insn_stats); } void test_verifier_subprog_precision(void) { RUN(verifier_subprog_precision); } void test_verifier_subprog_topo(void) { RUN(verifier_subprog_topo); } void test_verifier_subreg(void) { RUN(verifier_subreg); } @@ -277,6 +292,7 @@ void test_irq(void) { RUN(irq); } void test_verifier_mtu(void) { RUN(verifier_mtu); } void test_verifier_jit_inline(void) { RUN(verifier_jit_inline); } void test_verifier_ctx_ptr_param(void) { RUN(verifier_ctx_ptr_param); } +void test_verifier_zext(void) { RUN_TESTS(verifier_zext); } static int init_test_val_map(struct bpf_object *obj, char *map_name) { diff --git a/tools/testing/selftests/bpf/progs/arena_atomics.c b/tools/testing/selftests/bpf/progs/arena_atomics.c index 2e7751a85399..73bc2b835f3f 100644 --- a/tools/testing/selftests/bpf/progs/arena_atomics.c +++ b/tools/testing/selftests/bpf/progs/arena_atomics.c @@ -28,8 +28,10 @@ bool skip_all_tests = true; #if defined(ENABLE_ATOMICS_TESTS) && \ defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) + (defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ + defined(__TARGET_ARCH_s390)) bool skip_lacq_srel_tests __attribute((__section__(".data"))) = false; #else bool skip_lacq_srel_tests = true; @@ -315,8 +317,10 @@ int load_acquire(const void *ctx) { #if defined(ENABLE_ATOMICS_TESTS) && \ defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) + (defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ + defined(__TARGET_ARCH_s390)) #define LOAD_ACQUIRE_ARENA(SIZEOP, SIZE, SRC, DST) \ { asm volatile ( \ @@ -367,8 +371,10 @@ int store_release(const void *ctx) { #if defined(ENABLE_ATOMICS_TESTS) && \ defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) + (defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ + defined(__TARGET_ARCH_s390)) #define STORE_RELEASE_ARENA(SIZEOP, DST, VAL) \ { asm volatile ( \ diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc.c b/tools/testing/selftests/bpf/progs/arena_kfunc.c new file mode 100644 index 000000000000..bf0d304e0e59 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_kfunc.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_experimental.h" +#include <bpf_arena_common.h> +#include "../test_kmods/bpf_testmod_kfunc.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + /* page 0 hosts the arena global, page 1 is for allocations */ + __uint(max_entries, 2); +} arena SEC(".maps"); + +/* + * Occupies page 0 so no allocation lands at arena offset 0, which the + * nullable tests below must be able to tell apart from NULL. + */ +u64 __arena arena_pad; + +/* volatile to force the scalar reloads below */ +volatile u64 stash; + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_arg_forms(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + u64 ret; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + /* PTR_TO_ARENA argument */ + *val = 41; + ret = bpf_kfunc_arena_arg_test((u64 *)val); + if (ret != 41 || *val != 42) + return 2; + + /* the low 32 bits as a scalar */ + stash = (u32)(u64)val; + ret = bpf_kfunc_arena_arg_test((u64 *)stash); + if (ret != 42 || *val != 43) + return 3; + + /* the full user address as a scalar */ + stash = (u64)val; + bpf_addr_space_cast(stash, 1, 0); + ret = bpf_kfunc_arena_arg_test((u64 *)stash); + if (ret != 43 || *val != 44) + return 4; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} + +/* + * Pin the rebase semantics using the capture kfuncs, which return the raw + * argument value: __arena rebases unconditionally, so zero low 32 bits + * arrive as the arena kernel base, while __arena__nullable turns them into + * NULL. + */ +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_arg_rebase(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + u64 base, off; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + base = bpf_kfunc_arena_cap_test(NULL); + if (!base) + return 2; + + /* only the low 32 bits contribute */ + stash = 0xbadc0ffe00000000; + if (bpf_kfunc_arena_cap_test((u64 *)stash) != base) + return 3; + + off = (u32)(u64)val; + if (bpf_kfunc_arena_cap_test((u64 *)val) != base + off) + return 4; + + if (bpf_kfunc_arena_cap_nullable_test(NULL) != 0) + return 5; + + stash = 0xbadc0ffe00000000; + if (bpf_kfunc_arena_cap_nullable_test((u64 *)stash) != 0) + return 6; + + if (bpf_kfunc_arena_cap_nullable_test((u64 *)val) != base + off) + return 7; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_args5(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + val[0] = 1; + val[1] = 2; + val[2] = 4; + val[3] = 8; + val[4] = 16; + + if (bpf_kfunc_arena_args5_test((u64 *)&val[0], (u64 *)&val[1], + (u64 *)&val[2], (u64 *)&val[3], + (u64 *)&val[4]) != 31) + return 2; + if (bpf_kfunc_arena_args5_test((u64 *)&val[0], (u64 *)&val[1], + (u64 *)&val[2], (u64 *)&val[3], NULL) != 15) + return 3; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_arg_mixed(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + val[0] = 7; + val[1] = 5; + + if (bpf_kfunc_arena_mixed_test((u64 *)&val[0], NULL) != 7) + return 2; + + if (bpf_kfunc_arena_mixed_test((u64 *)&val[0], (u64 *)&val[1]) != 12) + return 3; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} + +/* kernel-side faults on unpopulated pages recover via the scratch page */ +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_arg_unpopulated(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + stash = (u64)val + PAGE_SIZE; + bpf_kfunc_arena_arg_test((u64 *)stash); + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__failure __msg("arena pointer requires a program with an associated arena") +int arena_arg_no_arena(void *ctx) +{ + bpf_kfunc_arena_arg_test((u64 *)1); + return 0; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__failure __msg("is not a pointer to arena or scalar") +int arena_arg_bad_reg(void *ctx) +{ + u64 buf = 0; + + /* use the arena so the program passes the arena presence check */ + bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + bpf_kfunc_arena_arg_test(&buf); + return 0; +} + +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__failure __msg("arena pointer cannot be a stack argument") +int arena_arg_stack(void *ctx) +{ + bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + bpf_kfunc_arena_stack_arg_test(1, 2, 3, 4, 5, (u64 *)1); + return 0; +} +#else +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__description("arena_arg_stack: not supported, dummy test") +__success +int arena_arg_stack(void *ctx) +{ + return 0; +} +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc_jit.c b/tools/testing/selftests/bpf/progs/arena_kfunc_jit.c new file mode 100644 index 000000000000..b5a01cbc33a7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_kfunc_jit.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +/* + * Verify the JIT-emitted rebase sequences for __arena and __arena__nullable + * kfunc arguments. The capture kfuncs take the argument without + * dereferencing it, so these tests pin only the emitted code. + */ +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_experimental.h" +#include <bpf_arena_common.h> +#include "../test_kmods/bpf_testmod_kfunc.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* volatile to force the scalar reloads below */ +volatile u64 stash; + +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + +SEC("syscall") +__arch_x86_64 +__jited("...") +__jited(" movl %edi, %edi") +__jited(" addq %r12, %rdi") +__jited("...") +__jited(" callq {{.*}}") +__arch_arm64 +__jited("...") +__jited(" add x0, x28, w0, uxtw") +__jited(" {{(bl|mov) .*}}") +__success +int arena_arg_jit_rebase(void *ctx) +{ + stash = (u64)bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + bpf_kfunc_arena_cap_test((u64 *)stash); + return 0; +} + +SEC("syscall") +__arch_x86_64 +__jited("...") +__jited(" movl %edi, %edi") +__jited(" testl %edi, %edi") +__jited(" je L0") +__jited(" addq %r12, %rdi") +__jited("L0: callq {{.*}}") +__arch_arm64 +__jited("...") +__jited(" mov w0, w0") +__jited(" cbz w0, L0") +__jited(" add x0, x28, w0, uxtw") +__jited("L0: {{.*}}") +__success +int arena_arg_jit_nullable(void *ctx) +{ + stash = (u64)bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + bpf_kfunc_arena_cap_nullable_test((u64 *)stash); + return 0; +} + +SEC("syscall") +__arch_x86_64 +__jited("...") +__jited(" movl %edi, %edi") +__jited(" addq %r12, %rdi") +__jited(" movl %esi, %esi") +__jited(" addq %r12, %rsi") +__jited(" movl %edx, %edx") +__jited(" addq %r12, %rdx") +__jited(" movl %ecx, %ecx") +__jited(" addq %r12, %rcx") +__jited(" movl %r8d, %r8d") +__jited(" testl %r8d, %r8d") +__jited(" je L0") +__jited(" addq %r12, %r8") +__jited("L0: callq {{.*}}") +__arch_arm64 +__jited("...") +__jited(" add x0, x28, w0, uxtw") +__jited(" add x1, x28, w1, uxtw") +__jited(" add x2, x28, w2, uxtw") +__jited(" add x3, x28, w3, uxtw") +__jited(" mov w4, w4") +__jited(" cbz w4, L0") +__jited(" add x4, x28, w4, uxtw") +__jited("L0: {{.*}}") +__success +int arena_arg_jit_args5(void *ctx) +{ + u64 __arena *val; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + val[0] = 1; + val[1] = 2; + val[2] = 4; + val[3] = 8; + val[4] = 16; + + bpf_kfunc_arena_args5_test((u64 *)&val[0], (u64 *)&val[1], + (u64 *)&val[2], (u64 *)&val[3], + (u64 *)&val[4]); + return 0; +} + +#endif /* __BPF_FEATURE_ADDR_SPACE_CAST */ + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/arena_mem_usage.c b/tools/testing/selftests/bpf/progs/arena_mem_usage.c new file mode 100644 index 000000000000..455ecd669a5a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_mem_usage.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_arena_common.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1000); /* number of pages */ +#ifdef __TARGET_ARCH_arm64 + __ulong(map_extra, 0x1ull << 32); /* start of mmap() region */ +#else + __ulong(map_extra, 0x1ull << 44); /* start of mmap() region */ +#endif +} arena SEC(".maps"); + +void __arena *ptr; +int alloc_cnt; /* in: pages to allocate */ +long free_byte_off; /* in: byte offset within ptr to start freeing */ +int free_cnt; /* in: pages to free */ + +SEC("syscall") +int alloc(void *ctx) +{ + ptr = bpf_arena_alloc_pages(&arena, NULL, alloc_cnt, NUMA_NO_NODE, 0); + /* Success/failure is checked from user space via skel->bss->ptr. */ + return 0; +} + +SEC("syscall") +int free_pages(void *ctx) +{ + if (!ptr) + return 1; + bpf_arena_free_pages(&arena, (char __arena *)ptr + free_byte_off, free_cnt); + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/arena_spin_lock.c b/tools/testing/selftests/bpf/progs/arena_spin_lock.c index cf7cda79c16c..92e75ec3844c 100644 --- a/tools/testing/selftests/bpf/progs/arena_spin_lock.c +++ b/tools/testing/selftests/bpf/progs/arena_spin_lock.c @@ -23,6 +23,13 @@ int cs_count; #if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST) arena_spinlock_t __arena lock; int test_skip = 1; + +/* + * Storage for the queue nodes declared by bpf_arena_spin_lock.h. Each program + * linking the arena spinlock provides exactly one definition; libarena's lives + * in libarena/src/common.bpf.c. + */ +struct arena_qnode __arena __hidden qnodes[_Q_MAX_CPUS][_Q_MAX_NODES]; #else int test_skip = 2; #endif diff --git a/tools/testing/selftests/bpf/progs/bpf_for_bench.c b/tools/testing/selftests/bpf/progs/bpf_for_bench.c new file mode 100644 index 000000000000..f9c723051fc7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_for_bench.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +int nr_loops; +long hits; + +static int outer_loop(__u32 index, void *data) +{ + int i; + + /* + * Empty body: the work being measured is the open-coded numeric iterator itself + * (bpf_iter_num_new/next/destroy behind bpf_for()). + */ + bpf_for(i, 0, nr_loops) + ; + __sync_add_and_fetch(&hits, nr_loops); + return 0; +} + +SEC("fentry/" SYS_PREFIX "sys_getpgid") +int benchmark(void *ctx) +{ + bpf_loop(1000, outer_loop, NULL, 0); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index b0c441384f20..5eacf1b43252 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h @@ -264,9 +264,12 @@ #endif #if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ - defined(__TARGET_ARCH_powerpc) || defined(__TARGET_ARCH_loongarch)) + (defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ + defined(__TARGET_ARCH_s390) || \ + defined(__TARGET_ARCH_powerpc) || \ + defined(__TARGET_ARCH_loongarch)) #define CAN_USE_LOAD_ACQ_STORE_REL #endif diff --git a/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__untrusted_write.c b/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__untrusted_write.c new file mode 100644 index 000000000000..688c2a049ae3 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__untrusted_write.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include "bpf_experimental.h" +#include "bpf_qdisc_common.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +SEC("struct_ops") +__failure __msg("only read is supported") +int BPF_PROG(untrusted_write, struct sk_buff *skb, struct Qdisc *sch, + struct bpf_sk_buff_ptr *to_free) +{ + struct Qdisc *next = sch->next_sched; + + /* + * sch is trusted, but the walk of next_sched yields a plain + * PTR_TO_BTF_ID which may fault on a dereference. A store through + * it does not get an exception table entry, there is no probed + * store to rewrite it into, hence it has to be rejected before + * bpf_qdisc_btf_struct_access() gets to allow the write to limit. + */ + next->limit = 1000; + + bpf_qdisc_skb_drop(skb, to_free); + return NET_XMIT_DROP; +} + +SEC("struct_ops") +__auxiliary +struct sk_buff *BPF_PROG(bpf_qdisc_test_dequeue, struct Qdisc *sch) +{ + return NULL; +} + +SEC("struct_ops") +__auxiliary +int BPF_PROG(bpf_qdisc_test_init, struct Qdisc *sch, struct nlattr *opt, + struct netlink_ext_ack *extack) +{ + return 0; +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_reset, struct Qdisc *sch) +{ +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_destroy, struct Qdisc *sch) +{ +} + +SEC(".struct_ops") +struct Qdisc_ops test = { + .enqueue = (void *)untrusted_write, + .dequeue = (void *)bpf_qdisc_test_dequeue, + .init = (void *)bpf_qdisc_test_init, + .reset = (void *)bpf_qdisc_test_reset, + .destroy = (void *)bpf_qdisc_test_destroy, + .id = "bpf_qdisc_test", +}; diff --git a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h index d8dacef37c16..593b38f90417 100644 --- a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h +++ b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h @@ -8,6 +8,9 @@ #define AF_INET 2 #define AF_INET6 10 +/* include/linux/net.h */ +#define SOCK_TYPE_MASK 0xf + #define SOL_SOCKET 1 #define SO_REUSEADDR 2 #define SO_SNDBUF 7 @@ -28,10 +31,12 @@ #define __SO_ACCEPTCON (1 << 16) #define IP_TOS 1 +#define IP_TRANSPARENT 19 #define SOL_IPV6 41 #define IPV6_TCLASS 67 #define IPV6_AUTOFLOWLABEL 70 +#define IPV6_TRANSPARENT 75 #define TC_ACT_UNSPEC (-1) #define TC_ACT_OK 0 diff --git a/tools/testing/selftests/bpf/progs/btf_data.c b/tools/testing/selftests/bpf/progs/btf_data.c index baa525275bde..8082c13490ab 100644 --- a/tools/testing/selftests/bpf/progs/btf_data.c +++ b/tools/testing/selftests/bpf/progs/btf_data.c @@ -48,3 +48,43 @@ int func(struct root_struct *root) { return 0; } + +int kfunc_a(struct root_struct *root) +{ + return 0; +} + +int kfunc_b(struct root_struct *root) +{ + return 0; +} + +struct root_struct *kfunc_c(struct root_struct *a, struct root_struct *b) +{ + return a; +} + +int kfunc_d(struct root_struct *a, struct root_struct *b) +{ + return 0; +} + +int kfunc_e(struct root_struct *a__arena, + struct root_struct *b__arena__nullable, + struct root_struct *c__arena, + struct root_struct *d__arena__nullable, + struct root_struct *e__arena) +{ + return 0; +} + +int kfunc_f(struct root_struct *a, struct root_struct *b__arena, int flags) +{ + return 0; +} + +struct root_struct *kfunc_g(struct root_struct *a__arena, + struct root_struct *b__arena__nullable) +{ + return a__arena; +} diff --git a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c index d0d65d6d450c..efe7bcae70f8 100644 --- a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c @@ -64,7 +64,7 @@ int BPF_PROG(cgrp_kfunc_acquire_no_null_check, struct cgroup *cgrp, const char * } SEC("tp_btf/cgroup_mkdir") -__failure __msg("R1 pointer type STRUCT cgroup must point") +__failure __msg("R1 is fp expected STRUCT cgroup") int BPF_PROG(cgrp_kfunc_acquire_fp, struct cgroup *cgrp, const char *path) { struct cgroup *acquired, *stack_cgrp = (struct cgroup *)&path; diff --git a/tools/testing/selftests/bpf/progs/cpumask_common.h b/tools/testing/selftests/bpf/progs/cpumask_common.h index 86085b79f5ca..8fe01308d210 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_common.h +++ b/tools/testing/selftests/bpf/progs/cpumask_common.h @@ -61,7 +61,7 @@ u32 bpf_cpumask_any_distribute(const struct cpumask *src) __ksym __weak; u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1, const struct cpumask *src2) __ksym __weak; u32 bpf_cpumask_weight(const struct cpumask *cpumask) __ksym __weak; -int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t src__sz) __ksym __weak; +int bpf_cpumask_populate(struct bpf_cpumask *cpumask, void *src, size_t src__sz) __ksym __weak; void bpf_rcu_read_lock(void) __ksym __weak; void bpf_rcu_read_unlock(void) __ksym __weak; diff --git a/tools/testing/selftests/bpf/progs/cpumask_failure.c b/tools/testing/selftests/bpf/progs/cpumask_failure.c index 4c45346fe6f7..4628feb53d86 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_failure.c +++ b/tools/testing/selftests/bpf/progs/cpumask_failure.c @@ -116,9 +116,9 @@ int BPF_PROG(test_cpumask_null, struct task_struct *task, u64 clone_flags) return 0; } -SEC("tp_btf/task_newtask") +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") __failure __msg("R2 must be a rcu pointer") -int BPF_PROG(test_global_mask_out_of_rcu, struct task_struct *task, u64 clone_flags) +int BPF_PROG(test_global_mask_out_of_rcu) { struct bpf_cpumask *local, *prev; @@ -133,6 +133,10 @@ int BPF_PROG(test_global_mask_out_of_rcu, struct task_struct *task, u64 clone_fl return 0; } + /* + * Use a sleepable program so explicit RCU is the only source of RCU + * protection. + */ bpf_rcu_read_lock(); local = global_mask; if (!local) { @@ -231,7 +235,7 @@ int BPF_PROG(test_populate_invalid_destination, struct task_struct *task, u64 cl u64 bits; int ret; - ret = bpf_cpumask_populate((struct cpumask *)invalid, &bits, sizeof(bits)); + ret = bpf_cpumask_populate(invalid, &bits, sizeof(bits)); if (!ret) err = 2; @@ -252,7 +256,7 @@ int BPF_PROG(test_populate_invalid_source, struct task_struct *task, u64 clone_f return 0; } - ret = bpf_cpumask_populate((struct cpumask *)local, garbage, 8); + ret = bpf_cpumask_populate(local, garbage, 8); if (!ret) err = 2; @@ -260,3 +264,22 @@ int BPF_PROG(test_populate_invalid_source, struct task_struct *task, u64 clone_f return 0; } + +SEC("tp_btf/task_newtask") +__failure __msg("expected pointer to STRUCT bpf_cpumask but R1 has a pointer to STRUCT cpumask") +int BPF_PROG(test_populate_borrowed_destination, struct task_struct *task, u64 clone_flags) +{ + u64 bits; + int ret; + + /* + * task->cpus_ptr is a borrowed, read-only struct cpumask *, not an + * owned struct bpf_cpumask *. The verifier must reject it as a + * writable destination for bpf_cpumask_populate(). + */ + ret = bpf_cpumask_populate((struct bpf_cpumask *)task->cpus_ptr, &bits, sizeof(bits)); + if (!ret) + err = 2; + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/cpumask_success.c b/tools/testing/selftests/bpf/progs/cpumask_success.c index 774706e7b058..36f77b9732d4 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_success.c +++ b/tools/testing/selftests/bpf/progs/cpumask_success.c @@ -785,7 +785,7 @@ int BPF_PROG(test_populate_reject_small_mask, struct task_struct *task, u64 clon return 0; /* The kfunc should prevent this operation */ - ret = bpf_cpumask_populate((struct cpumask *)local, &toofewbits, sizeof(toofewbits)); + ret = bpf_cpumask_populate(local, &toofewbits, sizeof(toofewbits)); if (ret != -EACCES) err = 2; @@ -824,7 +824,7 @@ int BPF_PROG(test_populate_reject_unaligned, struct task_struct *task, u64 clone /* Misalign the source array by a byte. */ src = &((char *)bits)[1]; - ret = bpf_cpumask_populate((struct cpumask *)mask, src, CPUMASK_TEST_MASKLEN); + ret = bpf_cpumask_populate(mask, src, CPUMASK_TEST_MASKLEN); if (ret != -EINVAL) err = 2; @@ -855,7 +855,7 @@ int BPF_PROG(test_populate, struct task_struct *task, u64 clone_flags) } /* Pass the entire bits array, the kfunc will only copy the valid bits. */ - ret = bpf_cpumask_populate((struct cpumask *)mask, bits, CPUMASK_TEST_MASKLEN); + ret = bpf_cpumask_populate(mask, bits, CPUMASK_TEST_MASKLEN); if (ret) { err = 2; goto out; diff --git a/tools/testing/selftests/bpf/progs/dynptr_fail.c b/tools/testing/selftests/bpf/progs/dynptr_fail.c index 29c6361d8820..1cd61d72c166 100644 --- a/tools/testing/selftests/bpf/progs/dynptr_fail.c +++ b/tools/testing/selftests/bpf/progs/dynptr_fail.c @@ -1112,7 +1112,7 @@ int dynptr_overwrite_ref(void *ctx) /* Reject writes to dynptr slot from bpf_dynptr_read */ SEC("?raw_tp") -__failure __msg("potential write to dynptr at off=-16") +__failure __msg("cannot overwrite referenced dynptr") int dynptr_read_into_slot(void *ctx) { union { @@ -1558,7 +1558,7 @@ int BPF_PROG(skb_invalid_ctx_fexit, void *skb) /* Reject writes to dynptr slot for uninit arg */ SEC("?raw_tp") -__failure __msg("potential write to dynptr at off=-16") +__failure __msg("cannot overwrite referenced dynptr") int uninit_write_into_slot(void *ctx) { struct { @@ -1589,7 +1589,8 @@ int xdp_invalid_ctx(void *ctx) __u32 hdr_size = sizeof(struct ethhdr); /* Can't pass in variable-sized len to bpf_dynptr_slice */ SEC("?tc") -__failure __msg("unbounded memory access") +__failure __msg("must be a known constant") +__msg("requires this memory size to be a verifier-known constant") int dynptr_slice_var_len1(struct __sk_buff *skb) { struct bpf_dynptr ptr; @@ -1609,6 +1610,7 @@ int dynptr_slice_var_len1(struct __sk_buff *skb) /* Can't pass in variable-sized len to bpf_dynptr_slice */ SEC("?tc") __failure __msg("must be a known constant") +__msg("requires this memory size to be a verifier-known constant") int dynptr_slice_var_len2(struct __sk_buff *skb) { char buffer[sizeof(struct ethhdr)] = {}; diff --git a/tools/testing/selftests/bpf/progs/fentry_sleepable.c b/tools/testing/selftests/bpf/progs/fentry_sleepable.c new file mode 100644 index 000000000000..8c0fc691d329 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/fentry_sleepable.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char LICENSE[] SEC("license") = "GPL"; + +void *user_ptr; +int retval; + +SEC("fentry.s") +int BPF_PROG(fentry_xdp) +{ + char buff[64]; + + retval = bpf_copy_from_user(buff, sizeof(buff), user_ptr); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/fib_lookup.c b/tools/testing/selftests/bpf/progs/fib_lookup.c index 7b5dd2214ff4..36b7218d9ae2 100644 --- a/tools/testing/selftests/bpf/progs/fib_lookup.c +++ b/tools/testing/selftests/bpf/progs/fib_lookup.c @@ -4,7 +4,11 @@ #include <linux/types.h> #include <linux/bpf.h> #include <linux/pkt_cls.h> +#include <linux/if_ether.h> +#include <linux/ip.h> +#include <linux/in.h> #include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> struct bpf_fib_lookup fib_params = {}; int fib_lookup_ret = 0; @@ -19,4 +23,57 @@ int fib_lookup(struct __sk_buff *skb) return TC_ACT_SHOT; } +SEC("xdp") +int fib_lookup_xdp(struct xdp_md *ctx) +{ + fib_lookup_ret = bpf_fib_lookup(ctx, &fib_params, sizeof(fib_params), + lookup_flags); + + return XDP_DROP; +} + +int redirected = 0; +int passed = 0; +int delivered = 0; + +SEC("xdp") +int fib_lookup_redirect(struct xdp_md *ctx) +{ + struct bpf_fib_lookup params = fib_params; + long ret; + + ret = bpf_fib_lookup(ctx, ¶ms, sizeof(params), lookup_flags); + if (ret == BPF_FIB_LKUP_RET_SUCCESS) { + redirected++; + return bpf_redirect(params.ifindex, 0); + } + + passed++; + return XDP_PASS; +} + +SEC("xdp") +int xdp_count(struct xdp_md *ctx) +{ + void *data = (void *)(long)ctx->data; + void *data_end = (void *)(long)ctx->data_end; + struct ethhdr *eth = data; + struct iphdr *iph; + + /* + * count only the test's TCP frames: the netns has live + * link-local traffic (DAD, MLD) that would satisfy a bare + * counter + */ + if ((void *)(eth + 1) > data_end || + eth->h_proto != bpf_htons(ETH_P_IP)) + return XDP_DROP; + iph = (void *)(eth + 1); + if ((void *)(iph + 1) > data_end || iph->protocol != IPPROTO_TCP) + return XDP_DROP; + + delivered++; + return XDP_DROP; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/freplace_cls_redirect.c b/tools/testing/selftests/bpf/progs/freplace_cls_redirect.c deleted file mode 100644 index 7e94412d47a5..000000000000 --- a/tools/testing/selftests/bpf/progs/freplace_cls_redirect.c +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (c) 2020 Facebook - -#include <linux/stddef.h> -#include <linux/bpf.h> -#include <linux/pkt_cls.h> -#include <bpf/bpf_endian.h> -#include <bpf/bpf_helpers.h> - -struct { - __uint(type, BPF_MAP_TYPE_SOCKMAP); - __type(key, int); - __type(value, int); - __uint(max_entries, 2); -} sock_map SEC(".maps"); - -SEC("freplace/cls_redirect") -int freplace_cls_redirect_test(struct __sk_buff *skb) -{ - int ret = 0; - const int zero = 0; - struct bpf_sock *sk; - - sk = bpf_map_lookup_elem(&sock_map, &zero); - if (!sk) - return TC_ACT_SHOT; - - ret = bpf_map_update_elem(&sock_map, &zero, sk, 0); - bpf_sk_release(sk); - - return ret == 0 ? TC_ACT_OK : TC_ACT_SHOT; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/get_smp_processor_id.c b/tools/testing/selftests/bpf/progs/get_smp_processor_id.c new file mode 100644 index 000000000000..cf4791a5cf07 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/get_smp_processor_id.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +__u64 cpu_nr_result; + +SEC("raw_tp") +void call_bpf_get_smp_processor_id(void) +{ + register __u64 r0 asm("r0") = -1; + asm volatile ("call %[bpf_get_smp_processor_id];" + : "+r"(r0) + : __imm(bpf_get_smp_processor_id) + : "r1", "r2", "r3", "r4", "r5", "memory"); + cpu_nr_result = r0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/icmp_send.c b/tools/testing/selftests/bpf/progs/icmp_send.c new file mode 100644 index 000000000000..c642ccdf9fd5 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/icmp_send.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> + +/* 127.0.0.1 in host byte order */ +#define SERVER_IP 0x7F000001 +/* ::1 in host byte order (last 32-bit word) */ +#define SERVER_IP6_LO 0x00000001 + +__u16 server_port = 0; +int unreach_type = 0; +int unreach_code = 0; +int kfunc_ret = -1; +int target_pid = -1; + +unsigned int rec_count = 0; +int rec_kfunc_rets[] = { -1, -1 }; + +SEC("cgroup_skb/egress") +int egress(struct __sk_buff *skb) +{ + void *data = (void *)(long)skb->data; + void *data_end = (void *)(long)skb->data_end; + struct iphdr *iph; + struct ipv6hdr *ip6h; + struct tcphdr *tcph; + __u8 version; + + if (data + 1 > data_end) + return SK_PASS; + + version = (*((__u8 *)data)) >> 4; + + if (version == 4) { + iph = data; + if ((void *)(iph + 1) > data_end || + iph->protocol != IPPROTO_TCP || + iph->daddr != bpf_htonl(SERVER_IP)) + return SK_PASS; + + tcph = (void *)iph + iph->ihl * 4; + if ((void *)(tcph + 1) > data_end || + tcph->dest != bpf_htons(server_port)) + return SK_PASS; + + } else if (version == 6) { + ip6h = data; + if ((void *)(ip6h + 1) > data_end || + ip6h->nexthdr != IPPROTO_TCP) + return SK_PASS; + + if (ip6h->daddr.in6_u.u6_addr32[0] != 0 || + ip6h->daddr.in6_u.u6_addr32[1] != 0 || + ip6h->daddr.in6_u.u6_addr32[2] != 0 || + ip6h->daddr.in6_u.u6_addr32[3] != bpf_htonl(SERVER_IP6_LO)) + return SK_PASS; + + tcph = (void *)(ip6h + 1); + if ((void *)(tcph + 1) > data_end || + tcph->dest != bpf_htons(server_port)) + return SK_PASS; + } else { + return SK_PASS; + } + + kfunc_ret = bpf_icmp_send(skb, unreach_type, unreach_code); + + return SK_DROP; +} + +SEC("cgroup_skb/egress") +int recursion(struct __sk_buff *skb) +{ + void *data = (void *)(long)skb->data; + void *data_end = (void *)(long)skb->data_end; + struct icmphdr *icmph; + struct tcphdr *tcph; + struct iphdr *iph; + int ret; + + if ((bpf_get_current_pid_tgid() >> 32) != target_pid) + return SK_PASS; + + iph = data; + if ((void *)(iph + 1) > data_end || iph->version != 4) + return SK_PASS; + + if (iph->daddr != bpf_htonl(SERVER_IP)) + return SK_PASS; + + if (iph->protocol == IPPROTO_TCP) { + tcph = (void *)iph + iph->ihl * 4; + if ((void *)(tcph + 1) > data_end || + tcph->dest != bpf_htons(server_port)) + return SK_PASS; + } else if (iph->protocol == IPPROTO_ICMP) { + icmph = (void *)iph + iph->ihl * 4; + if ((void *)(icmph + 1) > data_end || + icmph->type != unreach_type || icmph->code != unreach_code) + return SK_PASS; + } else { + return SK_PASS; + } + + /* + * This call will provoke a recursion: the ICMP packet generated by the + * kfunc will re-trigger this program since we are in the root cgroup in + * which the kernel ICMP socket belongs. However when re-entering the + * kfunc, it should return EBUSY. + */ + ret = bpf_icmp_send(skb, unreach_type, unreach_code); + rec_kfunc_rets[rec_count & 1] = ret; + __sync_fetch_and_add(&rec_count, 1); + + /* Let the first ICMP error message pass */ + if (iph->protocol == IPPROTO_ICMP) + return SK_PASS; + + return SK_DROP; +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/testing/selftests/bpf/progs/iters.c b/tools/testing/selftests/bpf/progs/iters.c index 0fa70b133d93..62d7df9e80be 100644 --- a/tools/testing/selftests/bpf/progs/iters.c +++ b/tools/testing/selftests/bpf/progs/iters.c @@ -88,6 +88,89 @@ int iter_err_unsafe_asm_loop(const void *ctx) return 0; } +/* + * Naked function, so there is no compiler-generated glue and the whole inlined program can be + * matched. Pinned to arches whose JITs zero-extend 32-bit writes implicitly + * (bpf_jit_needs_zext() == false); on arches that need explicit zero-extension the verifier + * interleaves "wN = wN" insns and the fixed shape below would not match. The inlining itself is + * arch independent, so checking it on these arches is sufficient. + * + * bpf_iter_num_new() emits the full range check (distance computation and both the -EINVAL and + * -E2BIG error paths); bpf_iter_num_next() and bpf_iter_num_destroy() are inlined too. + */ +SEC("raw_tp") +__arch_x86_64 +__arch_arm64 +__success +__xlated("r6 = r10") +__xlated("r6 += -8") +__xlated("call unknown") +__xlated("r3 = r0") +__xlated("r3 &= 65535") +__xlated("r1 = r6") +__xlated("r2 = 0") +/* bpf_iter_num_new(&it, 0, <non-const>) with the range check kept */ +__xlated("if w2 s> w3 goto pc+8") +__xlated("w0 = w3") +__xlated("w0 -= w2") +__xlated("if r0 > 0x800000 goto pc+8") +__xlated("w2 += -1") +__xlated("*(u32 *)(r1 +0) = r2") +__xlated("*(u32 *)(r1 +4) = r3") +__xlated("r0 = 0") +__xlated("goto pc+5") +__xlated("*(u64 *)(r1 +0) = 0") +__xlated("r0 = -22") +__xlated("goto pc+2") +__xlated("*(u64 *)(r1 +0) = 0") +__xlated("r0 = -7") +__xlated("r1 = r6") +/* bpf_iter_num_next(&it) */ +__xlated("r0 = *(u32 *)(r1 +0)") +__xlated("w0 += 1") +__xlated("r2 = *(u32 *)(r1 +4)") +__xlated("if w0 s>= w2 goto pc+3") +__xlated("*(u32 *)(r1 +0) = r0") +__xlated("r0 = r1") +__xlated("goto pc+2") +__xlated("*(u64 *)(r1 +0) = 0") +__xlated("r0 = 0") +__xlated("if r0 != 0x0 goto pc-11") +__xlated("r1 = r6") +/* bpf_iter_num_destroy(&it) is inlined to a nop */ +__xlated("goto pc+0") +__xlated("r0 = 0") +__xlated("exit") +int __naked iter_num_new_inlined(void) +{ + asm volatile ( + /* r6 points to struct bpf_iter_num on the stack */ + "r6 = r10;" + "r6 += -8;" + /* non-constant end so the range checks are kept */ + "call %[bpf_get_prandom_u32];" + "r3 = r0;" + "r3 &= 0xffff;" + "r1 = r6;" + "r2 = 0;" + "call %[bpf_iter_num_new];" + "1:" + "r1 = r6;" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r6;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy) + : __clobber_common, "r6" + ); +} + SEC("raw_tp") __success int iter_while_loop(const void *ctx) diff --git a/tools/testing/selftests/bpf/progs/iters_task_failure.c b/tools/testing/selftests/bpf/progs/iters_task_failure.c index fe3663dedbe1..566a1d3dffea 100644 --- a/tools/testing/selftests/bpf/progs/iters_task_failure.c +++ b/tools/testing/selftests/bpf/progs/iters_task_failure.c @@ -62,6 +62,52 @@ int BPF_PROG(iter_tasks_lock_and_unlock) } SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__failure __msg("expected an RCU CS when using bpf_iter_task_next") +__flag(BPF_F_TEST_STATE_FREQ) +int BPF_PROG(iter_tasks_rcu_state_pruning) +{ + struct bpf_iter_task it; + + asm volatile ( + "call %[bpf_rcu_read_lock];" + "r1 = %[it];" + "r2 = 0;" + "r3 = 0;" /* BPF_TASK_ITER_ALL_PROCS */ + "call %[bpf_iter_task_new];" + + "call %[bpf_get_prandom_u32];" + "if w0 == 0 goto unprotected_%=;" + + /* Keep the outer RCU lock active on the straight-line path. */ + "call %[bpf_rcu_read_lock];" + "call %[bpf_rcu_read_unlock];" + "goto merge_%=;" + + "unprotected_%=:" + /* Create an unprotected gap on the taken path. */ + "call %[bpf_rcu_read_unlock];" + "call %[bpf_rcu_read_lock];" + + "merge_%=: r1 = %[it];" + "call %[bpf_iter_task_next];" + "r1 = %[it];" + "call %[bpf_iter_task_destroy];" + "call %[bpf_rcu_read_unlock];" + : + : __imm_ptr(it), + __imm(bpf_get_prandom_u32), + __imm(bpf_iter_task_new), + __imm(bpf_iter_task_next), + __imm(bpf_iter_task_destroy), + __imm(bpf_rcu_read_lock), + __imm(bpf_rcu_read_unlock) + : __clobber_common + ); + + return 0; +} + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") __failure __msg("expected an RCU CS when using bpf_iter_css_next") int BPF_PROG(iter_css_lock_and_unlock) { diff --git a/tools/testing/selftests/bpf/progs/kfunc_call_fail.c b/tools/testing/selftests/bpf/progs/kfunc_call_fail.c index a1963497f0bf..7e93f7fb1329 100644 --- a/tools/testing/selftests/bpf/progs/kfunc_call_fail.c +++ b/tools/testing/selftests/bpf/progs/kfunc_call_fail.c @@ -4,6 +4,18 @@ #include <bpf/bpf_helpers.h> #include "../test_kmods/bpf_testmod_kfunc.h" +static struct bpf_spin_lock kfunc_call_lock SEC(".data.A"); + +SEC("?tc") +int kfunc_call_test_spin_lock_unsafe(struct __sk_buff *skb) +{ + bpf_spin_lock(&kfunc_call_lock); + bpf_kfunc_trigger_ctx_check(); + bpf_spin_unlock(&kfunc_call_lock); + + return 0; +} + struct syscall_test_args { __u8 data[16]; size_t size; @@ -103,6 +115,66 @@ int kfunc_call_test_get_mem_fail_oob(struct __sk_buff *skb) return ret; } +SEC("?tc") +int kfunc_call_test_get_mem_fail_zero_size(struct __sk_buff *skb) +{ + struct prog_test_ref_kfunc *pt; + unsigned long s = 0; + int *p = NULL; + int ret = 0; + + pt = bpf_kfunc_call_test_acquire(&s); + if (pt) { + /* + * An explicit rdwr_buf_size of 0 gives R0 a zero-sized buffer, + * so any access is out of bounds, hence -EACCES. Previously the + * verifier treated a zero size as "no size argument" and sized + * R0 after the pointed-to return type, wrongly allowing the read. + */ + p = bpf_kfunc_call_test_get_rdwr_mem(pt, 0); + if (p) + ret = p[0]; + else + ret = -1; + + bpf_kfunc_call_test_release(pt); + } + return ret; +} + +SEC("?tc") +int kfunc_call_test_get_mem_fail_oversized(struct __sk_buff *skb) +{ + struct prog_test_ref_kfunc *pt; + unsigned long s = 0; + int *p = NULL; + int ret = 0; + + pt = bpf_kfunc_call_test_acquire(&s); + if (pt) { + /* + * rdwr_buf_size is a const int, so a C literal is narrowed to + * 32 bits before the call. Force the full 64-bit value 2^64 - 192 + * (0xffffffffffffff40, > U32_MAX) into the argument register with + * a 64-bit immediate load. The verifier records r0_size from the + * full register value and must reject it before that value is + * truncated into R0's u32 mem_size. + */ + asm volatile ( + "r1 = %[pt];" + "r2 = %[oversized] ll;" + "call %[get_rdwr_mem];" + "%[p] = r0;" + : [p] "=r"(p) + : [pt] "r"(pt), + [oversized] "i"(0xffffffffffffff40LL), + [get_rdwr_mem] "i"(bpf_kfunc_call_test_get_rdwr_mem) + : "r0", "r1", "r2", "r3", "r4", "r5"); + bpf_kfunc_call_test_release(pt); + } + return ret; +} + int not_const_size = 2 * sizeof(int); SEC("?tc") diff --git a/tools/testing/selftests/bpf/progs/kfunc_call_test.c b/tools/testing/selftests/bpf/progs/kfunc_call_test.c index 5edc51564f71..8e6560c31e78 100644 --- a/tools/testing/selftests/bpf/progs/kfunc_call_test.c +++ b/tools/testing/selftests/bpf/progs/kfunc_call_test.c @@ -5,6 +5,18 @@ #include "bpf_misc.h" #include "../test_kmods/bpf_testmod_kfunc.h" +static struct bpf_spin_lock kfunc_call_lock SEC(".data.A"); + +SEC("tc") +int kfunc_call_test_spin_lock_safe(struct __sk_buff *skb) +{ + bpf_spin_lock(&kfunc_call_lock); + bpf_testmod_test_mod_kfunc(42); + bpf_spin_unlock(&kfunc_call_lock); + + return 0; +} + SEC("tc") int kfunc_call_test5(struct __sk_buff *skb) { diff --git a/tools/testing/selftests/bpf/progs/ksock_common.h b/tools/testing/selftests/bpf/progs/ksock_common.h new file mode 100644 index 000000000000..01edaeb9fdd4 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/ksock_common.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 Isovalent */ + +#ifndef _KSOCK_COMMON_H +#define _KSOCK_COMMON_H + +#include "errno.h" + +#define SOCK_DGRAM 2 +#define IPPROTO_UDP 17 + +struct bpf_ksock *bpf_ksock_create(const struct bpf_ksock_create_opts *opts, + u32 opts__sz, int *err__uninit) __ksym; +int bpf_ksock_connect(struct bpf_ksock *ks, const union bpf_ksock_addr *addr, + u32 addr__sz) __ksym; +struct bpf_ksock *bpf_ksock_acquire(struct bpf_ksock *ks) __ksym; +void bpf_ksock_release(struct bpf_ksock *ks) __ksym; +int bpf_ksock_send(struct bpf_ksock *ks, const void *data, u32 data__sz) __ksym; +void bpf_rcu_read_lock(void) __ksym; +void bpf_rcu_read_unlock(void) __ksym; + +struct __ksock_ctx_value { + struct bpf_ksock __kptr * ctx; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, struct __ksock_ctx_value); + __uint(max_entries, 1); +} __ksock_ctx_map SEC(".maps"); + +static inline struct __ksock_ctx_value *ksock_ctx_value_lookup(void) +{ + u32 key = 0; + + return bpf_map_lookup_elem(&__ksock_ctx_map, &key); +} + +static inline struct bpf_ksock *ksock_ctx_get(void) +{ + struct __ksock_ctx_value *v; + struct bpf_ksock *ks = NULL, *tmp; + + v = ksock_ctx_value_lookup(); + if (!v) + return NULL; + + bpf_rcu_read_lock(); + tmp = v->ctx; + if (tmp) + ks = bpf_ksock_acquire(tmp); + bpf_rcu_read_unlock(); + + return ks; +} + +static inline int ksock_ctx_insert(struct bpf_ksock *ctx) +{ + struct __ksock_ctx_value *v; + struct bpf_ksock *old; + + v = ksock_ctx_value_lookup(); + if (!v) { + bpf_ksock_release(ctx); + return -ENOENT; + } + + old = bpf_kptr_xchg(&v->ctx, ctx); + if (old) { + bpf_ksock_release(old); + return -EEXIST; + } + + return 0; +} + +#endif /* _KSOCK_COMMON_H */ diff --git a/tools/testing/selftests/bpf/progs/ksock_lsm.c b/tools/testing/selftests/bpf/progs/ksock_lsm.c new file mode 100644 index 000000000000..9808451098ef --- /dev/null +++ b/tools/testing/selftests/bpf/progs/ksock_lsm.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_endian.h> +#include "bpf_tracing_net.h" +#include "ksock_common.h" + +char send_data[32] = "hello from bpf ksock"; + +__be32 ipv4_remote; +__u16 remote_port; +int target_pid; +int send_ret = -1; + +SEC("syscall") +int ksock_setup(void *ctx) +{ + struct bpf_ksock_create_opts create_opts = {}; + union bpf_ksock_addr addr = {}; + struct bpf_ksock *ks; + int err = 0; + + create_opts.family = AF_INET; + create_opts.type = SOCK_DGRAM; + create_opts.protocol = IPPROTO_UDP; + + ks = bpf_ksock_create(&create_opts, sizeof(create_opts), &err); + if (!ks) + return err; + + addr.sin.sin_family = AF_INET; + addr.sin.sin_port = bpf_htons(remote_port); + addr.sin.sin_addr.s_addr = ipv4_remote; + + err = bpf_ksock_connect(ks, &addr, sizeof(addr)); + if (err) { + bpf_ksock_release(ks); + return err; + } + + err = ksock_ctx_insert(ks); + if (err && err != -EEXIST) + return err; + return 0; +} + +SEC("lsm.s/socket_bind") +int BPF_PROG(ksock_socket_bind, struct socket *sock, struct sockaddr *address, + int addrlen, int ret) +{ + struct bpf_ksock *ks; + u32 pid = bpf_get_current_pid_tgid() >> 32; + + if (ret || pid != target_pid) + return ret; + + ks = ksock_ctx_get(); + if (!ks) { + send_ret = -ENOENT; + return ret; + } + + send_ret = bpf_ksock_send(ks, send_data, sizeof(send_data)); + bpf_ksock_release(ks); + + return ret; +} + +char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/ksock_lsm_verifier.c b/tools/testing/selftests/bpf/progs/ksock_lsm_verifier.c new file mode 100644 index 000000000000..fd2ccfdb5802 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/ksock_lsm_verifier.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" +#include "ksock_common.h" + +char send_data[11] = "dummy data"; + +SEC("lsm.s/socket_sendmsg") +__description("bpf_ksock_send is rejected from socket_sendmsg LSM hook") +__failure __msg("calling kernel function bpf_ksock_send is not allowed") +int BPF_PROG(ksock_socket_sendmsg, struct socket *sock, struct msghdr *msg, + int size, int ret) +{ + struct __ksock_ctx_value *v; + struct bpf_ksock *ks; + + v = ksock_ctx_value_lookup(); + if (!v) + return ret; + + ks = bpf_kptr_xchg(&v->ctx, NULL); + if (!ks) + return ret; + + bpf_ksock_send(ks, send_data, sizeof(send_data)); + bpf_ksock_release(ks); + + return ret; +} + +char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/ksock_wq.c b/tools/testing/selftests/bpf/progs/ksock_wq.c new file mode 100644 index 000000000000..16a1873d132e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/ksock_wq.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include "bpf_experimental.h" +#include "bpf_tracing_net.h" +#include "errno.h" +#include "ksock_common.h" + +struct ksock_wq_value { + struct bpf_wq work; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, u32); + __type(value, struct ksock_wq_value); +} work_map SEC(".maps"); + +int create_err; +u32 callback_done; + +static int ksock_wq_callback(void *map, int *key, void *value) +{ + struct bpf_ksock_create_opts opts = { + .family = AF_INET, + .type = SOCK_DGRAM, + .protocol = IPPROTO_UDP, + }; + struct bpf_ksock *ks; + int err = 0; + + ks = bpf_ksock_create(&opts, sizeof(opts), &err); + if (ks) + bpf_ksock_release(ks); + create_err = err; + __sync_fetch_and_add(&callback_done, 1); + return 0; +} + +SEC("syscall") +int ksock_wq_start(void *ctx) +{ + struct ksock_wq_value *value; + u32 key = 0; + int err; + + value = bpf_map_lookup_elem(&work_map, &key); + if (!value) + return -ENOENT; + err = bpf_wq_init(&value->work, &work_map, 0); + if (err) + return err; + err = bpf_wq_set_callback(&value->work, ksock_wq_callback, 0); + if (err) + return err; + return bpf_wq_start(&value->work, 0); +} + +char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/linked_list_fail.c b/tools/testing/selftests/bpf/progs/linked_list_fail.c index ddd26d1a083f..031e77a288ee 100644 --- a/tools/testing/selftests/bpf/progs/linked_list_fail.c +++ b/tools/testing/selftests/bpf/progs/linked_list_fail.c @@ -167,6 +167,16 @@ CHECK_OP(push_back); #undef CHECK_OP #undef INIT +struct obj_new_flex_elem { + int lo; + int hi; +}; + +struct obj_new_flex { + int hdr; + struct obj_new_flex_elem cells[]; +}; + SEC("?kprobe/xyz") int map_compat_kprobe(void *ctx) { @@ -231,6 +241,19 @@ int obj_new_no_struct(void *ctx) } SEC("?tc") +int obj_new_flex_array(void *ctx) +{ + struct obj_new_flex *p; + + p = bpf_obj_new_impl(bpf_core_type_id_local(struct obj_new_flex), NULL); + if (!p) + return 0; + p->cells[0].hi = 42; + bpf_obj_drop_impl(p, NULL); + return 0; +} + +SEC("?tc") int obj_drop_non_zero_off(void *ctx) { void *f; diff --git a/tools/testing/selftests/bpf/progs/map_kptr.c b/tools/testing/selftests/bpf/progs/map_kptr.c index 3fbefc568e0a..0d87c97dac99 100644 --- a/tools/testing/selftests/bpf/progs/map_kptr.c +++ b/tools/testing/selftests/bpf/progs/map_kptr.c @@ -4,6 +4,18 @@ #include <bpf/bpf_helpers.h> #include "../test_kmods/bpf_testmod_kfunc.h" +struct map_uninit_value { + struct prog_test_ref_kfunc __kptr_untrusted *unref_ptr; + __u32 data; +} __attribute__((packed)); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, int); + __type(value, struct map_uninit_value); + __uint(max_entries, 1); +} pcpu_array SEC(".maps"); + struct map_value { struct prog_test_ref_kfunc __kptr_untrusted *unref_ptr; struct prog_test_ref_kfunc __kptr *ref_ptr; diff --git a/tools/testing/selftests/bpf/progs/map_kptr_fail.c b/tools/testing/selftests/bpf/progs/map_kptr_fail.c index f11848dfa78f..5e25ca806060 100644 --- a/tools/testing/selftests/bpf/progs/map_kptr_fail.c +++ b/tools/testing/selftests/bpf/progs/map_kptr_fail.c @@ -386,7 +386,16 @@ int kptr_xchg_possibly_null(struct __sk_buff *ctx) } SEC("?tc") +/* + * A compiler with BPF_ST folds the constant into a store-immediate, which the + * verifier rejects on a different path (and with a different message) than the + * BPF_STX form. + */ +#ifdef __BPF_FEATURE_ST +__failure __msg("BPF_ST imm must be 0 when storing to kptr at off=8") +#else __failure __msg("invalid kptr access, R") +#endif int reject_scalar_store_to_kptr(struct __sk_buff *ctx) { struct map_value *v; diff --git a/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c b/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c index f166fff8f217..3e0d4f687aaa 100644 --- a/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c +++ b/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c @@ -3,6 +3,7 @@ #include <vmlinux.h> #include <bpf/bpf_core_read.h> #include "bpf_misc.h" +#include "bpf_kfuncs.h" #include "../test_kmods/bpf_testmod_kfunc.h" SEC("tp_btf/sys_enter") @@ -137,7 +138,7 @@ int helper_param_not_ok(void *ctx) p = bpf_rdonly_cast(0, 0); /* - * Any helper with ARG_CONST_SIZE_OR_ZERO constraint will do, + * Any helper with ARG_MEM_SIZE_OR_ZERO constraint will do, * the most permissive constraint */ bpf_copy_from_user(p, 0, (void *)42); @@ -164,6 +165,239 @@ int mixed_mem_type(void *ctx) return *p; } +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 4096); +} ringbuf SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, u32); + __type(value, u64); +} array SEC(".maps"); + +char dynptr_data[8]; + +int zero; + +SEC("socket") +__success +__log_level(2) +__msg("r8 = *(u64 *)(r7 +0){{.*}}R7=untrusted_ptr_sock") +__msg("r8 = *(u64 *)(r7 +0){{.*}}R7=ringbuf_mem") +__retval(0) +int mixed_mem_untrusted_btf_id_type(void *ctx) +{ + u64 *p, *q, v; + + p = bpf_ringbuf_reserve(&ringbuf, sizeof(*p), 0); + if (!p) + return 1; + *p = 42; + q = bpf_rdonly_cast(0, bpf_core_type_id_kernel(struct sock)); + /* + * The load below is reached with PTR_TO_MEM | MEM_RINGBUF on one + * path and with PTR_TO_BTF_ID | PTR_UNTRUSTED on the other. The + * merged type has to keep the BPF_PROBE_MEM rewrite, otherwise + * the NULL deref taken at runtime panics the kernel instead of + * returning 0. + */ + asm volatile ( + "r7 = %[p];" + "if %[zero] != 0 goto +1;" + "r7 = %[q];" + "r8 = *(u64 *)(r7 + 0);" + "%[v] = r8;" + : [v]"=r"(v) + : [p]"r"(p), + [q]"r"(q), + [zero]"r"(zero) + : "r7", "r8"); + bpf_ringbuf_discard(p, 0); + return v; +} + +SEC("socket") +__success +__log_level(2) +__msg("r8 = *(u32 *)(r7 +0){{.*}}R7=ptr_nameidata") +__msg("r8 = *(u32 *)(r7 +0){{.*}}R7=ringbuf_mem") +__retval(0) +int mixed_mem_btf_id_type(void *ctx) +{ + struct task_struct *task; + u32 *p, *q; + u64 v; + + p = bpf_ringbuf_reserve(&ringbuf, sizeof(*p), 0); + if (!p) + return 1; + *p = 42; + task = bpf_get_current_task_btf(); + /* + * A plain BTF pointer walk yields a bare PTR_TO_BTF_ID, and + * task->nameidata is NULL unless the task currently is in the + * middle of a path lookup. + */ + q = (u32 *)&task->nameidata->flags; + /* + * Same as above, except that the other path yields a bare + * PTR_TO_BTF_ID. Merging it with PTR_TO_MEM used to drop the + * BPF_PROBE_MEM rewrite the bare PTR_TO_BTF_ID would have + * gotten on its own. + */ + asm volatile ( + "r7 = %[p];" + "if %[zero] != 0 goto +1;" + "r7 = %[q];" + "r8 = *(u32 *)(r7 + 0);" + "%[v] = r8;" + : [v]"=r"(v) + : [p]"r"(p), + [q]"r"(q), + [zero]"r"(zero) + : "r7", "r8"); + bpf_ringbuf_discard(p, 0); + return v; +} + +SEC("socket") +__success +__log_level(2) +__msg("r8 = *(u32 *)(r7 +0){{.*}}R7=ptr_nameidata") +__msg("r8 = *(u32 *)(r7 +0){{.*}}R7=rdonly_mem") +__retval(0) +int mixed_rdonly_mem_btf_id_type(void *ctx) +{ + struct task_struct *task; + struct bpf_dynptr dptr; + char buf[sizeof(u32)]; + u32 *p, *q; + u64 v; + + if (bpf_dynptr_from_mem(dynptr_data, sizeof(dynptr_data), 0, &dptr)) + return 1; + p = bpf_dynptr_slice(&dptr, 0, buf, sizeof(buf)); + if (!p) + return 1; + task = bpf_get_current_task_btf(); + q = (u32 *)&task->nameidata->flags; + /* + * Same as above, except that the PTR_TO_MEM side already carries + * MEM_RDONLY. Merging it with a bare PTR_TO_BTF_ID used to yield + * PTR_TO_MEM | MEM_RDONLY, which is not rewritten either since + * only its PTR_UNTRUSTED variant is. + */ + asm volatile ( + "r7 = %[p];" + "if %[zero] != 0 goto +1;" + "r7 = %[q];" + "r8 = *(u32 *)(r7 + 0);" + "%[v] = r8;" + : [v]"=r"(v) + : [p]"r"(p), + [q]"r"(q), + [zero]"r"(zero) + : "r7", "r8"); + return v; +} + +SEC("socket") +__success +__log_level(2) +__msg("r8 = *(u64 *)(r7 +0){{.*}}R7=ringbuf_mem") +__msg("r8 = *(u64 *)(r7 +0){{.*}}R7=rdonly_untrusted_mem") +__retval(0) +int mixed_mem_mem_type(void *ctx) +{ + u64 *p, *q, v; + + p = bpf_ringbuf_reserve(&ringbuf, sizeof(*p), 0); + if (!p) + return 1; + *p = 42; + q = bpf_rdonly_cast(0, 0); + /* + * Both paths are PTR_TO_MEM based, so they used to not trip the + * type mismatch check and skipped the merge altogether, leaving + * the insn with the PTR_TO_MEM | MEM_RINGBUF recorded first and + * hence without the BPF_PROBE_MEM rewrite the other path needs. + */ + asm volatile ( + "r7 = %[q];" + "if %[zero] == 0 goto +1;" + "r7 = %[p];" + "r8 = *(u64 *)(r7 + 0);" + "%[v] = r8;" + : [v]"=r"(v) + : [p]"r"(p), + [q]"r"(q), + [zero]"r"(zero) + : "r7", "r8"); + bpf_ringbuf_discard(p, 0); + return v; +} + +SEC("socket") +__failure +__msg("same insn cannot be used with different pointers") +int mixed_map_value_mem_type(void *ctx) +{ + u64 *p, *q, v; + u32 key = 0; + + p = bpf_map_lookup_elem(&array, &key); + if (!p) + return 1; + q = bpf_rdonly_cast(0, 0); + /* + * PTR_TO_MAP_VALUE is neither PTR_TO_MEM nor PTR_TO_BTF_ID based, + * so it cannot be merged into a type which keeps the BPF_PROBE_MEM + * rewrite the PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED of the other + * path needs. Both bases were mismatch ok, hence the load used to be + * accepted with the PTR_TO_MAP_VALUE recorded and the NULL deref on + * the second path panicked the kernel. + */ + asm volatile ( + "r7 = %[q];" + "if %[zero] == 0 goto +1;" + "r7 = %[p];" + "r8 = *(u64 *)(r7 + 0);" + "%[v] = r8;" + : [v]"=r"(v) + : [p]"r"(p), + [q]"r"(q), + [zero]"r"(zero) + : "r7", "r8"); + return v; +} + +SEC("socket") +__failure +__msg("same insn cannot be used with different pointers") +int mixed_stack_mem_type(void *ctx) +{ + u64 *p = bpf_rdonly_cast(0, 0); + u64 s = 42, v; + + /* + * Same as above, but for a PTR_TO_STACK on the other path. + */ + asm volatile ( + "r7 = %[p];" + "if %[zero] == 0 goto +1;" + "r7 = %[s];" + "r8 = *(u64 *)(r7 + 0);" + "%[v] = r8;" + : [v]"=r"(v) + : [p]"r"(p), + [s]"r"(&s), + [zero]"r"(zero) + : "r7", "r8"); + return v; +} + __attribute__((__aligned__(8))) u8 global[] = { 0x11, 0x22, 0x33, 0x44, diff --git a/tools/testing/selftests/bpf/progs/mptcpify.c b/tools/testing/selftests/bpf/progs/mptcpify.c index cbdc730c3a47..e3f8cb54dbe9 100644 --- a/tools/testing/selftests/bpf/progs/mptcpify.c +++ b/tools/testing/selftests/bpf/progs/mptcpify.c @@ -15,7 +15,7 @@ int BPF_PROG(mptcpify, int family, int type, int protocol) return protocol; if ((family == AF_INET || family == AF_INET6) && - type == SOCK_STREAM && + (type & SOCK_TYPE_MASK) == SOCK_STREAM && (!protocol || protocol == IPPROTO_TCP)) { return IPPROTO_MPTCP; } diff --git a/tools/testing/selftests/bpf/progs/pyperf.h b/tools/testing/selftests/bpf/progs/pyperf.h index 86484f07e1d1..fd93a96e5901 100644 --- a/tools/testing/selftests/bpf/progs/pyperf.h +++ b/tools/testing/selftests/bpf/progs/pyperf.h @@ -85,9 +85,11 @@ static void *get_thread_state(void *tls_base, PidData *pidData) return thread_state; } -static __always_inline bool get_frame_data(void *frame_ptr, PidData *pidData, - FrameData *frame, Symbol *symbol) +__weak bool __get_frame_data(long frame_ptr_, PidData *pidData __arg_nonnull, + FrameData *frame __arg_nonnull, Symbol *symbol __arg_nonnull) { + void *frame_ptr = (void *)frame_ptr_; + // read data from PyFrameObject bpf_probe_read_user(&frame->f_back, sizeof(frame->f_back), @@ -119,6 +121,12 @@ static __always_inline bool get_frame_data(void *frame_ptr, PidData *pidData, return true; } +static __always_inline bool get_frame_data(void *frame_ptr, PidData *pidData, + FrameData *frame, Symbol *symbol) +{ + return __get_frame_data((long)frame_ptr, pidData, frame, symbol); +} + struct { __uint(type, BPF_MAP_TYPE_HASH); __uint(max_entries, 1); diff --git a/tools/testing/selftests/bpf/progs/rcu_read_lock.c b/tools/testing/selftests/bpf/progs/rcu_read_lock.c index b4e073168fb1..31d4081c3a9f 100644 --- a/tools/testing/selftests/bpf/progs/rcu_read_lock.c +++ b/tools/testing/selftests/bpf/progs/rcu_read_lock.c @@ -549,3 +549,79 @@ int rcu_read_lock_sleepable_global_subprog_indirect(void *ctx) bpf_rcu_read_unlock(); return 0; } + +struct rcu_node_data { + long key; + struct bpf_rb_node node; +}; + +struct rcu_node_stash { + struct rcu_node_data __kptr *node; +}; + +/* + * Necessary so that LLVM emits BTF for rcu_node_data rather than just a + * fwd reference to it, same as in progs/local_kptr_stash.c. + */ +struct rcu_node_data *just_here_because_btf_bug; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct rcu_node_stash); +} node_stash SEC(".maps"); + +long non_own_ref_key; + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +int non_own_ref_untrusted_ld(void *ctx) +{ + struct rcu_node_stash *stash; + struct rcu_node_data *node; + int key = 0; + + stash = bpf_map_lookup_elem(&node_stash, &key); + if (!stash) + return 0; + bpf_rcu_read_lock(); + node = stash->node; + if (!node) { + bpf_rcu_read_unlock(); + return 0; + } + bpf_rcu_read_unlock(); + /* + * The unlock leaves node as PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED + * | NON_OWN_REF, and the load below has to get the BPF_PROBE_MEM + * rewrite for it, otherwise a bad address panics the kernel. + */ + non_own_ref_key = node->key; + return 0; +} + +long rcu_untrusted_wq_flags; + +SEC("?tp_btf/tcp_probe") +int BPF_PROG(rcu_untrusted_union_ld, struct sock *sk) +{ + struct socket_wq *wq; + + /* + * sk_wq sits in a two member union, so btf_struct_walk() marks the + * pointer PTR_UNTRUSTED, and the __rcu tag on the member adds MEM_RCU + * on top of it. struct sock is not on the __safe_rcu_or_null allow + * list, hence the two stay combined and the load below has to get the + * BPF_PROBE_MEM rewrite for PTR_TO_BTF_ID | PTR_UNTRUSTED | MEM_RCU, + * otherwise a bad address panics the kernel. + * + * The __rcu tag only reaches BTF on a clang built kernel, that is, one + * with CONFIG_PAHOLE_HAS_BTF_TAG. On a gcc built kernel the walk yields + * a plain untrusted pointer, which is rewritten either way. + */ + wq = sk->sk_wq; + if (!wq) + return 0; + rcu_untrusted_wq_flags = wq->flags; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c index 024ef2aae200..eaaed0859f94 100644 --- a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c +++ b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c @@ -63,6 +63,7 @@ long rbtree_refcounted_node_ref_escapes(void *ctx) SEC("?tc") __failure __msg("Possibly NULL pointer passed to trusted R1") +__msg("requires a non-NULL value of type (void *)") long refcount_acquire_maybe_null(void *ctx) { struct node_acquire *n, *m; @@ -81,6 +82,14 @@ long refcount_acquire_maybe_null(void *ctx) } SEC("?tc") +__failure __msg("R1 is neither owning or non-owning ref") +__msg("expects a pointer to a BPF-managed refcounted object, but R1 is a context pointer") +long refcount_acquire_non_object(void *ctx) +{ + return bpf_refcount_acquire(ctx) != NULL; +} + +SEC("?tc") __failure __msg("Unreleased reference id=3 alloc_insn={{[0-9]+}}") long rbtree_refcounted_node_ref_escapes_owning_input(void *ctx) { diff --git a/tools/testing/selftests/bpf/progs/setget_sockopt.c b/tools/testing/selftests/bpf/progs/setget_sockopt.c index 636a7cd8e2fa..d96e99b67aeb 100644 --- a/tools/testing/selftests/bpf/progs/setget_sockopt.c +++ b/tools/testing/selftests/bpf/progs/setget_sockopt.c @@ -69,12 +69,14 @@ static const struct sockopt_test sol_tcp_tests[] = { static const struct sockopt_test sol_ip_tests[] = { { .opt = IP_TOS, .new = 0xe1, .expected = 0xe1, .tcp_expected = 0xe0, }, + { .opt = IP_TRANSPARENT, .flip = 1, }, { .opt = 0, }, }; static const struct sockopt_test sol_ipv6_tests[] = { { .opt = IPV6_TCLASS, .new = 0xe1, .expected = 0xe1, .tcp_expected = 0xe0, }, { .opt = IPV6_AUTOFLOWLABEL, .flip = 1, }, + { .opt = IPV6_TRANSPARENT, .flip = 1, }, { .opt = 0, }, }; diff --git a/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c b/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c index 0660f29dca95..3177bc5b733a 100644 --- a/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c +++ b/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c @@ -44,8 +44,18 @@ int bpf_prog2(struct __sk_buff *skb) __sink(lport); __sink(rport); - if (data + 8 > data_end) - return SK_DROP; + if (data + 8 > data_end) { + if (bpf_skb_pull_data(skb, 8)) + return SK_DROP; + + data = (void *)(long)skb->data; + data_end = (void *)(long)skb->data_end; + + if (data + 8 > data_end) + return SK_DROP; + + d = data; + } map = d[0]; sk = d[1]; diff --git a/tools/testing/selftests/bpf/progs/stream.c b/tools/testing/selftests/bpf/progs/stream.c index 92ba1d72e0ec..8e8e1339dc74 100644 --- a/tools/testing/selftests/bpf/progs/stream.c +++ b/tools/testing/selftests/bpf/progs/stream.c @@ -64,6 +64,8 @@ SEC("syscall") __arch_x86_64 __arch_arm64 __arch_s390x +__arch_riscv64 +__arch_loongarch __success __retval(0) __stderr("ERROR: Timeout detected for may_goto instruction") __stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}") @@ -184,6 +186,151 @@ int stream_arena_read_fault(void *ctx) return 0; } +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +__stderr("ERROR: Arena READ access at unmapped address 0x{{.*}}") +__stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}") +__stderr("Call trace:\n" +"{{([a-zA-Z_][a-zA-Z0-9_]*\\+0x[0-9a-fA-F]+/0x[0-9a-fA-F]+\n" +"|[ \t]+[^\n]+\n)*}}") +int stream_arena_load_acquire_fault(void *ctx) +{ + static const struct bpf_insn load_acquire_insn = { + .code = 0xc3, /* BPF_STX | BPF_ATOMIC | BPF_W */ + .dst_reg = 0, /* BPF_REG_0 */ + .src_reg = 1, /* BPF_REG_1 */ + .off = 0x7fff, + .imm = 0x100, /* BPF_LOAD_ACQ */ + }; + struct bpf_arena *ptr = (void *)&arena; + u64 user_vm_start, val; + + /* + * Prevent GCC bounds warning: casting &arena to struct bpf_arena * + * triggers bounds checking since the map definition is smaller than + * struct bpf_arena. barrier_var() makes the pointer opaque to GCC, + * preventing the bounds analysis. + */ + barrier_var(ptr); + user_vm_start = ptr->user_vm_start; + fault_addr = user_vm_start + 0x7fff; + bpf_addr_space_cast(user_vm_start, 0, 1); + asm volatile ( + "r1 = %[user_vm_start];" + "r0 = 1;" + ".8byte %[load_acquire_insn];" /* r0 = load_acquire((u32 *)(r1 + 0x7fff)) */ + "%[val] = r0;" + : [val] "=r" (val) + : [user_vm_start] "r" (user_vm_start), + __imm_insn(load_acquire_insn, load_acquire_insn) + : "r0", "r1" + ); + return val; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +__stderr("ERROR: Arena WRITE access at unmapped address 0x{{.*}}") +__stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}") +__stderr("Call trace:\n" +"{{([a-zA-Z_][a-zA-Z0-9_]*\\+0x[0-9a-fA-F]+/0x[0-9a-fA-F]+\n" +"|[ \t]+[^\n]+\n)*}}") +int stream_arena_xchg_fault(void *ctx) +{ + static const struct bpf_insn xchg_insn = { + .code = 0xc3, /* BPF_STX | BPF_ATOMIC | BPF_W */ + .dst_reg = 1, /* BPF_REG_1 */ + .src_reg = 2, /* BPF_REG_2 */ + .off = 0x7fff, + .imm = 0xe1, /* BPF_XCHG */ + }; + struct bpf_arena *ptr = (void *)&arena; + u64 user_vm_start, val; + + /* + * Prevent GCC bounds warning: casting &arena to struct bpf_arena * + * triggers bounds checking since the map definition is smaller than + * struct bpf_arena. barrier_var() makes the pointer opaque to GCC, + * preventing the bounds analysis. + */ + barrier_var(ptr); + user_vm_start = ptr->user_vm_start; + fault_addr = user_vm_start + 0x7fff; + bpf_addr_space_cast(user_vm_start, 0, 1); + /* + * A read-modify-write carrying BPF_FETCH writes to memory, so the fault + * has to be reported as a WRITE from the dst_reg address, but it also + * reads the old value into src_reg, so the exception handler has to + * clear src_reg. Poison it up front, the returned value must be 0. + */ + asm volatile ( + "r1 = %[user_vm_start];" + "r2 = 1;" + ".8byte %[xchg_insn];" /* r2 = xchg((u32 *)(r1 + 0x7fff), r2) */ + "%[val] = r2;" + : [val] "=r" (val) + : [user_vm_start] "r" (user_vm_start), + __imm_insn(xchg_insn, xchg_insn) + : "r1", "r2" + ); + return val; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +__stderr("ERROR: Arena WRITE access at unmapped address 0x{{.*}}") +__stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}") +__stderr("Call trace:\n" +"{{([a-zA-Z_][a-zA-Z0-9_]*\\+0x[0-9a-fA-F]+/0x[0-9a-fA-F]+\n" +"|[ \t]+[^\n]+\n)*}}") +int stream_arena_cmpxchg_fault(void *ctx) +{ + static const struct bpf_insn cmpxchg_insn = { + .code = 0xc3, /* BPF_STX | BPF_ATOMIC | BPF_W */ + .dst_reg = 1, /* BPF_REG_1 */ + .src_reg = 2, /* BPF_REG_2 */ + .off = 0x7fff, + .imm = 0xf1, /* BPF_CMPXCHG */ + }; + struct bpf_arena *ptr = (void *)&arena; + u64 user_vm_start, val; + + /* + * Prevent GCC bounds warning: casting &arena to struct bpf_arena * + * triggers bounds checking since the map definition is smaller than + * struct bpf_arena. barrier_var() makes the pointer opaque to GCC, + * preventing the bounds analysis. + */ + barrier_var(ptr); + user_vm_start = ptr->user_vm_start; + fault_addr = user_vm_start + 0x7fff; + bpf_addr_space_cast(user_vm_start, 0, 1); + /* + * Same as the exchange above, except that a BPF_CMPXCHG reads the old + * value into r0 rather than into src_reg, so r0 is the register the + * exception handler has to clear. It doubles as the compare value, but + * the comparison never happens since the access faults first. + */ + asm volatile ( + "r1 = %[user_vm_start];" + "r0 = 1;" + "r2 = 2;" + ".8byte %[cmpxchg_insn];" /* r0 = cmpxchg((u32 *)(r1 + 0x7fff), r0, r2) */ + "%[val] = r0;" + : [val] "=r" (val) + : [user_vm_start] "r" (user_vm_start), + __imm_insn(cmpxchg_insn, cmpxchg_insn) + : "r0", "r1", "r2" + ); + return val; +} + static __noinline void subprog(void) { int __arena *addr = (int __arena *)0xdeadbeef; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena.c b/tools/testing/selftests/bpf/progs/struct_ops_arena.c new file mode 100644 index 000000000000..8aa8639df91f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_experimental.h" +#include <bpf_arena_common.h> +#include "../test_kmods/bpf_testmod.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + /* page 0 hosts the arena globals, page 1 is for allocations */ + __uint(max_entries, 2); +} arena SEC(".maps"); + +/* also associates the callbacks with the arena */ +u64 __arena arena_touch; +/* raw value of the last __arena ctx argument, captured by test_arena_cb */ +u64 __arena cb_ptr_val; + +SEC("struct_ops/test_arena") +int test_arena_cb(unsigned long long *ctx) +{ + u64 __arena *ptr = (u64 __arena *)ctx[0]; + + arena_touch++; + cb_ptr_val = ctx[0]; + *ptr += 1; + return 0; +} + +SEC("struct_ops/test_arena_nullable") +int test_arena_nullable_cb(unsigned long long *ctx) +{ + u64 __arena *ptr = (u64 __arena *)ctx[0]; + + arena_touch++; + if (!ptr) + return 0xbee; + *ptr += 1; + return 0; +} + +SEC("struct_ops/test_arena_stack") +int test_arena_stack_cb(unsigned long long *ctx) +{ + u64 __arena *ptr = (u64 __arena *)ctx[8]; + + arena_touch++; + /* pin the slot layout: the leading args fill ctx[0]..ctx[7] */ + if (ctx[0] != 1 || ctx[7] != 8) + return 0xbad; + *ptr += 1; + return 0; +} + +SEC("struct_ops/test_arena_multislot") +int test_arena_multislot_cb(unsigned long long *ctx) +{ + u64 __arena *ptr = (u64 __arena *)ctx[2]; + + arena_touch++; + /* + * The 16-byte struct occupies ctx[0] and ctx[1], so @ptr is argument + * one but slot two. Getting that wrong hands the callback a scalar. + */ + if (ctx[0] != 11 || ctx[1] != 22) + return 0xbad; + *ptr += 1; + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena = { + .test_arena = (void *)test_arena_cb, + .test_arena_nullable = (void *)test_arena_nullable_cb, + .test_arena_stack = (void *)test_arena_stack_cb, + .test_arena_multislot = (void *)test_arena_multislot_cb, +}; + +SEC("syscall") +int trigger(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + int ret; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + ret = bpf_testmod_ops3_call_test_arena((u64 *)val); + if (ret) + return 2; + if (*val != 42) + return 3; + + /* + * The callback must have seen exactly (u32)(kaddr - kern_vm_start), + * which is the arena offset of val with the upper 32 bits clear. + */ + if (cb_ptr_val != (u32)(u64)val) + return 4; + + ret = bpf_testmod_ops3_call_test_arena_nullable((u64 *)val); + if (ret) + return 5; + if (*val != 43) + return 6; + + /* NULL survives the nullable kfunc and the trampoline as NULL */ + ret = bpf_testmod_ops3_call_test_arena_nullable(NULL); + if (ret != 0xbee) + return 7; + + /* the arena pointer is stack-passed into the trampoline here */ + ret = bpf_testmod_ops3_call_test_arena_stack((u64 *)val); + if (ret) + return 8; + if (*val != 44) + return 9; + + /* a multi-slot arg precedes the arena pointer here */ + ret = bpf_testmod_ops3_call_test_arena_multislot((u64 *)val); + if (ret) + return 10; + if (*val != 45) + return 11; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_attach.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_attach.c new file mode 100644 index 000000000000..081a770307e5 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_attach.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +SEC("fentry") +int BPF_PROG(fentry_test_arena, unsigned long long *st_ops_ctx) +{ + return 0; +} + +SEC("fexit") +int BPF_PROG(fexit_test_arena, unsigned long long *st_ops_ctx, int ret) +{ + return 0; +} + +SEC("freplace") +int freplace_test_arena(unsigned long long *st_ops_ctx) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_fail.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_fail.c new file mode 100644 index 000000000000..1c0ec727d637 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_fail.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* No arena in the program: attaching to test_arena must be rejected. */ +SEC("struct_ops/test_arena") +int test_arena_no_arena(unsigned long long *ctx) +{ + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_fail = { + .test_arena = (void *)test_arena_no_arena, +}; diff --git a/tools/testing/selftests/bpf/progs/tailcall_callback.c b/tools/testing/selftests/bpf/progs/tailcall_callback.c new file mode 100644 index 000000000000..c41632cf423b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tailcall_callback.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_test_utils.h" + +int classifier_0(struct __sk_buff *skb); + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __array(values, void (void)); +} jmp_table SEC(".maps") = { + .values = { + [0] = (void *) &classifier_0, + }, +}; + +__auxiliary +SEC("tc") +int classifier_0(struct __sk_buff *skb) +{ + return 0; +} + +static __noinline +int subprog_tail0(struct __sk_buff *skb) +{ + int ret = 0; + + bpf_tail_call_static(skb, &jmp_table, 0); + barrier_var(ret); + return ret; +} + +static __noinline +int callback_loop(int index, void **cb_ctx) +{ + int ret; + + ret = subprog_tail0(*cb_ctx); + barrier_var(ret); + return ret ? 1 : 0; +} + +static __noinline +int callback_empty(int index, void *data) +{ + return 0; +} + +/* callback involving subprog with tail call is rejected */ +SEC("tc") +__failure __msg("cannot tail call within callback") +int tailcall_callback_1(struct __sk_buff *skb) +{ + clobber_regs_stack(); + + bpf_loop(1, callback_loop, &skb, 0); + return 0; +} + +/* subprogs with tailcall do not affect no-tailcall callback */ +SEC("tc") +__success +__retval(0) +int tailcall_callback_2(struct __sk_buff *skb) +{ + int ret; + + clobber_regs_stack(); + + ret = subprog_tail0(skb); + __sink(ret); + + bpf_loop(1, callback_empty, NULL, 0); + return 0; +} + +char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/task_kfunc_common.h b/tools/testing/selftests/bpf/progs/task_kfunc_common.h index e9c4fea7a4bb..a0c599b58c29 100644 --- a/tools/testing/selftests/bpf/progs/task_kfunc_common.h +++ b/tools/testing/selftests/bpf/progs/task_kfunc_common.h @@ -20,12 +20,26 @@ struct { __uint(max_entries, 1); } __tasks_kfunc_map SEC(".maps"); +struct task_kptr_lock_value { + struct bpf_spin_lock lock; + struct task_struct __kptr * task; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, struct task_kptr_lock_value); + __uint(max_entries, 1); +} task_kptr_lock_map SEC(".maps"); + struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; void bpf_task_release(struct task_struct *p) __ksym; struct task_struct *bpf_task_from_pid(s32 pid) __ksym; struct task_struct *bpf_task_from_vpid(s32 vpid) __ksym; void bpf_rcu_read_lock(void) __ksym; void bpf_rcu_read_unlock(void) __ksym; +void bpf_local_irq_save(unsigned long *flags) __weak __ksym; +void bpf_local_irq_restore(unsigned long *flags) __weak __ksym; static inline struct __tasks_kfunc_map_value *tasks_kfunc_map_value_lookup(struct task_struct *p) { diff --git a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c index 8942b5478129..f96b0c13ed1a 100644 --- a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c @@ -50,7 +50,7 @@ int BPF_PROG(task_kfunc_acquire_untrusted, struct task_struct *task, u64 clone_f } SEC("tp_btf/task_newtask") -__failure __msg("R1 pointer type STRUCT task_struct must point") +__failure __msg("R1 is fp expected STRUCT task_struct") int BPF_PROG(task_kfunc_acquire_fp, struct task_struct *task, u64 clone_flags) { struct task_struct *acquired, *stack_task = (struct task_struct *)&clone_flags; @@ -378,3 +378,76 @@ int BPF_PROG(task_kfunc_release_in_map, struct task_struct *task, u64 clone_flag return 0; } + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__failure __msg("R1 must be a rcu pointer") +int BPF_PROG(task_kfunc_acquire_after_final_spin_unlock) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_spin_lock(&v->lock); + task = v->task; + bpf_spin_unlock(&v->lock); + if (!task) + return 0; + + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + return 0; +} + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__failure __msg("R1 must be a rcu pointer") +int BPF_PROG(task_kfunc_acquire_after_preempt_enable) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_preempt_disable(); + task = v->task; + bpf_preempt_enable(); + if (!task) + return 0; + + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + return 0; +} + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__failure __msg("R1 must be a rcu pointer") +int BPF_PROG(task_kfunc_acquire_after_irq_restore) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + unsigned long flags; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_local_irq_save(&flags); + task = v->task; + bpf_local_irq_restore(&flags); + if (!task) + return 0; + + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/task_kfunc_success.c b/tools/testing/selftests/bpf/progs/task_kfunc_success.c index d63a79ee33dc..6545b124dee1 100644 --- a/tools/testing/selftests/bpf/progs/task_kfunc_success.c +++ b/tools/testing/selftests/bpf/progs/task_kfunc_success.c @@ -6,6 +6,7 @@ #include <bpf/bpf_helpers.h> #include "../bpf_experimental.h" +#include "bpf_misc.h" #include "task_kfunc_common.h" char _license[] SEC("license") = "GPL"; @@ -366,6 +367,200 @@ int BPF_PROG(task_kfunc_acquire_trusted_walked, struct task_struct *task, u64 cl return 0; } +SEC("fentry/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_spin_unlock_non_sleepable) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_spin_lock(&v->lock); + task = v->task; + bpf_spin_unlock(&v->lock); + if (!task) + return 0; + + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_spin_unlock_explicit_rcu) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_rcu_read_lock(); + bpf_spin_lock(&v->lock); + task = v->task; + bpf_spin_unlock(&v->lock); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_rcu_read_unlock(); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_spin_unlock_preempt_disabled) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_preempt_disable(); + bpf_spin_lock(&v->lock); + task = v->task; + bpf_spin_unlock(&v->lock); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_preempt_enable(); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_spin_unlock_irq_disabled) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + unsigned long flags; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_local_irq_save(&flags); + bpf_spin_lock(&v->lock); + task = v->task; + bpf_spin_unlock(&v->lock); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_local_irq_restore(&flags); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_rcu_unlock_preempt_disabled) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_preempt_disable(); + bpf_rcu_read_lock(); + task = v->task; + bpf_rcu_read_unlock(); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_preempt_enable(); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_rcu_unlock_irq_disabled) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + unsigned long flags; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_local_irq_save(&flags); + bpf_rcu_read_lock(); + task = v->task; + bpf_rcu_read_unlock(); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_local_irq_restore(&flags); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_preempt_enable_explicit_rcu) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_preempt_disable(); + task = v->task; + bpf_rcu_read_lock(); + bpf_preempt_enable(); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_rcu_read_unlock(); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_irq_restore_explicit_rcu) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + unsigned long flags; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_local_irq_save(&flags); + task = v->task; + bpf_rcu_read_lock(); + bpf_local_irq_restore(&flags); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_rcu_read_unlock(); + return 0; +} + SEC("syscall") int test_task_from_vpid_current(const void *ctx) { diff --git a/tools/testing/selftests/bpf/progs/tcp_ca_untrusted_btf_write.c b/tools/testing/selftests/bpf/progs/tcp_ca_untrusted_btf_write.c new file mode 100644 index 000000000000..eda4697aac80 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tcp_ca_untrusted_btf_write.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "bpf_tracing_net.h" +#include <bpf/bpf_core_read.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +SEC("struct_ops") +void BPF_PROG(untrusted_btf_write_init, struct sock *sk) +{ + struct tcp_sock *tp; + int v = 1; + void *p; + + p = bpf_rdonly_cast(&v, 0); + tp = bpf_rdonly_cast(p, bpf_core_type_id_kernel(struct tcp_sock)); + tp->snd_cwnd = 1; +} + +SEC(".struct_ops") +struct tcp_congestion_ops untrusted_btf_write = { + .init = (void *)untrusted_btf_write_init, + .name = "bpf_ro_btf", +}; diff --git a/tools/testing/selftests/bpf/progs/test_fill_link_info.c b/tools/testing/selftests/bpf/progs/test_fill_link_info.c index 137bd6292163..c85081538e93 100644 --- a/tools/testing/selftests/bpf/progs/test_fill_link_info.c +++ b/tools/testing/selftests/bpf/progs/test_fill_link_info.c @@ -58,4 +58,10 @@ int BPF_PROG(umulti_run) return 0; } +SEC("fentry.multi") +int BPF_PROG(tmulti_run) +{ + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_global_percpu_data.c b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c new file mode 100644 index 000000000000..5dc21b3b4cb5 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +/* Used for testing map name. */ +int loong SEC(".percpu.looooooooong"); +int data3 SEC(".data.percpu"); +int data2 SEC(".percpu.data"); + +int run; +/* cpu_id as array to verify map value resizing. */ +int cpu_id[1] SEC(".percpu"); +int data SEC(".percpu") = -1; +int nums[7] SEC(".percpu"); +bool set SEC(".percpu") = false; +struct { + char set; + int i; + int nums[7]; +} struct_data SEC(".percpu") = { + .set = 0, + .i = -1, +}; + +SEC("raw_tp/task_rename") +__auxiliary +int update_percpu_data(void *ctx) +{ + struct_data.nums[6] = 0xc0de; + struct_data.set = 1; + struct_data.i = 1; + nums[6] = 0xc0de; + data = 1; + run++; + set = true; + cpu_id[0] = bpf_get_smp_processor_id(); + return 0; +} + +static const char fmt[] SEC(".percpu.fmt") = "data %d\n"; + +SEC("?kprobe") +__failure __msg("R{{[0-9]+}} points to percpu_array map which cannot be used as const string") +int verifier_strncmp(void *ctx) +{ + return bpf_strncmp("test", 5, fmt); +} + +SEC("?kprobe") +__failure __msg("R{{[0-9]+}} points to percpu_array map which cannot be used as const string") +int verifier_snprintf(void *ctx) +{ + u64 args[] = { data }; + char buf[128]; + int len; + + len = bpf_snprintf(buf, sizeof(buf), fmt, args, sizeof(args)); + if (len > 0) + bpf_printk("snprintf: %s\n", buf); + return 0; +} + +volatile const __u32 num_cpus = 0; +volatile const int num_off; +volatile const int elem_sz; +__u32 sum = 0; +bool run_iter = false; + +SEC("iter/bpf_map_elem") +__auxiliary +int dump_percpu_data(struct bpf_iter__bpf_map_elem *ctx) +{ + void *pptr = ctx->value; + int i; + + if (!pptr) + return 0; + + run_iter = true; + + for (i = 0; i < num_cpus; i++) { + sum += *(int *) (pptr + num_off); + pptr += elem_sz; + } + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_signed_loader.c b/tools/testing/selftests/bpf/progs/test_signed_loader.c index d9a4b85f9391..50451a69b99a 100644 --- a/tools/testing/selftests/bpf/progs/test_signed_loader.c +++ b/tools/testing/selftests/bpf/progs/test_signed_loader.c @@ -4,10 +4,11 @@ /* * Minimal, map-less program. Driven through libbpf's gen_loader (gen_hash) - * by prog_tests/signed_loader.c so the generated light-skeleton loader (with - * the emit_signature_match metadata check) can be exercised against good - * and tampered metadata. A socket filter needs no load-time attach resolution, - * and having no maps keeps the generated loader's ctx trivial (0 maps, 1 prog). + * by prog_tests/signed_loader.c so the generated light-skeleton loader can be + * exercised against good and tampered metadata, which the kernel now verifies + * at load time via the insns||metadata signature. A socket filter needs no + * load-time attach resolution, and having no maps keeps the generated loader's + * ctx trivial (0 maps, 1 prog). */ SEC("socket") int probe(void *ctx) diff --git a/tools/testing/selftests/bpf/progs/test_sockmap_update.c b/tools/testing/selftests/bpf/progs/test_sockmap_update.c deleted file mode 100644 index 6d64ea536e3d..000000000000 --- a/tools/testing/selftests/bpf/progs/test_sockmap_update.c +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (c) 2020 Cloudflare -#include "vmlinux.h" -#include <bpf/bpf_helpers.h> - -struct { - __uint(type, BPF_MAP_TYPE_SOCKMAP); - __uint(max_entries, 1); - __type(key, __u32); - __type(value, __u64); -} src SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_SOCKMAP); - __uint(max_entries, 1); - __type(key, __u32); - __type(value, __u64); -} dst_sock_map SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_SOCKHASH); - __uint(max_entries, 1); - __type(key, __u32); - __type(value, __u64); -} dst_sock_hash SEC(".maps"); - -SEC("tc") -int copy_sock_map(void *ctx) -{ - struct bpf_sock *sk; - bool failed = false; - __u32 key = 0; - - sk = bpf_map_lookup_elem(&src, &key); - if (!sk) - return SK_DROP; - - if (bpf_map_update_elem(&dst_sock_map, &key, sk, 0)) - failed = true; - - if (bpf_map_update_elem(&dst_sock_hash, &key, sk, 0)) - failed = true; - - bpf_sk_release(sk); - return failed ? SK_DROP : SK_PASS; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_tc_peer.c b/tools/testing/selftests/bpf/progs/test_tc_peer.c index 365eacb5dc34..cfb9ef7f467c 100644 --- a/tools/testing/selftests/bpf/progs/test_tc_peer.c +++ b/tools/testing/selftests/bpf/progs/test_tc_peer.c @@ -35,6 +35,28 @@ int tc_src(struct __sk_buff *skb) } SEC("tc") +int tc_dst_ing(struct __sk_buff *skb) +{ + if (!skb->mark) { + skb->mark = 0x1; + return bpf_redirect_peer(IFINDEX_SRC, BPF_F_EGRESS); + } + + return bpf_redirect(IFINDEX_DST, 0); +} + +SEC("tc") +int tc_src_ing(struct __sk_buff *skb) +{ + if (!skb->mark) { + skb->mark = 0x1; + return bpf_redirect_peer(IFINDEX_DST, BPF_F_EGRESS); + } + + return bpf_redirect(IFINDEX_SRC, 0); +} + +SEC("tc") int tc_dst_l3(struct __sk_buff *skb) { return bpf_redirect(IFINDEX_SRC, 0); diff --git a/tools/testing/selftests/bpf/progs/test_tc_tunnel.c b/tools/testing/selftests/bpf/progs/test_tc_tunnel.c index 7376df405a6b..853bca962910 100644 --- a/tools/testing/selftests/bpf/progs/test_tc_tunnel.c +++ b/tools/testing/selftests/bpf/progs/test_tc_tunnel.c @@ -6,6 +6,7 @@ #include <bpf/bpf_helpers.h> #include <bpf/bpf_endian.h> +#include <bpf/bpf_core_read.h> #include "bpf_tracing_net.h" #include "bpf_compiler.h" @@ -37,6 +38,22 @@ struct vxlanhdr___local { #define EXTPROTO_VXLAN 0x1 +#define SKB_GSO_UDP_TUNNEL_MASK (SKB_GSO_UDP_TUNNEL | \ + SKB_GSO_UDP_TUNNEL_CSUM) + +#define SKB_GSO_TUNNEL_MASK (SKB_GSO_UDP_TUNNEL_MASK | \ + SKB_GSO_GRE | \ + SKB_GSO_GRE_CSUM | \ + SKB_GSO_IPXIP4 | \ + SKB_GSO_IPXIP6 | \ + SKB_GSO_ESP) + +#define BPF_F_ADJ_ROOM_DECAP_L4_MASK (BPF_F_ADJ_ROOM_DECAP_L4_UDP | \ + BPF_F_ADJ_ROOM_DECAP_L4_GRE) + +#define BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK (BPF_F_ADJ_ROOM_DECAP_IPXIP4 | \ + BPF_F_ADJ_ROOM_DECAP_IPXIP6) + #define VXLAN_FLAGS bpf_htonl(1<<27) #define VNI_ID 1 #define VXLAN_VNI bpf_htonl(VNI_ID << 8) @@ -589,9 +606,12 @@ int __encap_ip6vxlan_eth(struct __sk_buff *skb) return TC_ACT_OK; } -static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) +static int decap_internal(struct __sk_buff *skb, int off, int len, char proto, + __u64 ipxip_flag) { __u64 flags = BPF_F_ADJ_ROOM_FIXED_GSO; + struct sk_buff *kskb; + struct skb_shared_info *shinfo; struct ipv6_opt_hdr ip6_opt_hdr; struct gre_hdr greh; struct udphdr udph; @@ -599,10 +619,12 @@ static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) switch (proto) { case IPPROTO_IPIP: - flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV4; + flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV4 | + ipxip_flag; break; case IPPROTO_IPV6: - flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV6; + flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV6 | + ipxip_flag; break; case NEXTHDR_DEST: if (bpf_skb_load_bytes(skb, off + len, &ip6_opt_hdr, @@ -610,10 +632,12 @@ static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) return TC_ACT_OK; switch (ip6_opt_hdr.nexthdr) { case IPPROTO_IPIP: - flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV4; + flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV4 | + ipxip_flag; break; case IPPROTO_IPV6: - flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV6; + flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV6 | + ipxip_flag; break; default: return TC_ACT_OK; @@ -621,6 +645,11 @@ static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) break; case IPPROTO_GRE: olen += sizeof(struct gre_hdr); + if (!bpf_core_enum_value_exists(enum bpf_adj_room_flags, + BPF_F_ADJ_ROOM_DECAP_L4_GRE)) + return TC_ACT_SHOT; + flags |= BPF_F_ADJ_ROOM_DECAP_L4_GRE; + if (bpf_skb_load_bytes(skb, off + len, &greh, sizeof(greh)) < 0) return TC_ACT_OK; switch (bpf_ntohs(greh.protocol)) { @@ -634,6 +663,10 @@ static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) break; case IPPROTO_UDP: olen += sizeof(struct udphdr); + if (!bpf_core_enum_value_exists(enum bpf_adj_room_flags, + BPF_F_ADJ_ROOM_DECAP_L4_UDP)) + return TC_ACT_SHOT; + flags |= BPF_F_ADJ_ROOM_DECAP_L4_UDP; if (bpf_skb_load_bytes(skb, off + len, &udph, sizeof(udph)) < 0) return TC_ACT_OK; switch (bpf_ntohs(udph.dest)) { @@ -655,6 +688,40 @@ static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) if (bpf_skb_adjust_room(skb, -olen, BPF_ADJ_ROOM_MAC, flags)) return TC_ACT_SHOT; + kskb = bpf_cast_to_kern_ctx(skb); + shinfo = bpf_core_cast(kskb->head + kskb->end, struct skb_shared_info); + if (shinfo->gso_size) { + if ((flags & BPF_F_ADJ_ROOM_DECAP_L4_UDP) && + (shinfo->gso_type & SKB_GSO_UDP_TUNNEL_MASK)) + return TC_ACT_SHOT; + + if ((flags & BPF_F_ADJ_ROOM_DECAP_L4_GRE) && + (shinfo->gso_type & (SKB_GSO_GRE | SKB_GSO_GRE_CSUM))) + return TC_ACT_SHOT; + + if ((flags & BPF_F_ADJ_ROOM_DECAP_IPXIP4) && + (shinfo->gso_type & SKB_GSO_IPXIP4)) + return TC_ACT_SHOT; + + if ((flags & BPF_F_ADJ_ROOM_DECAP_IPXIP6) && + (shinfo->gso_type & SKB_GSO_IPXIP6)) + return TC_ACT_SHOT; + + if (flags & (BPF_F_ADJ_ROOM_DECAP_L4_MASK | + BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK)) { + if ((shinfo->gso_type & SKB_GSO_TUNNEL_MASK) && + !kskb->encapsulation) + return TC_ACT_SHOT; + if (!(shinfo->gso_type & SKB_GSO_TUNNEL_MASK) && + kskb->encapsulation) + return TC_ACT_SHOT; + } + } else if ((flags & (BPF_F_ADJ_ROOM_DECAP_L4_MASK | + BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK)) && + kskb->encapsulation) { + return TC_ACT_SHOT; + } + return TC_ACT_OK; } @@ -662,6 +729,10 @@ static int decap_ipv4(struct __sk_buff *skb) { struct iphdr iph_outer; + if (!bpf_core_enum_value_exists(enum bpf_adj_room_flags, + BPF_F_ADJ_ROOM_DECAP_IPXIP4)) + return TC_ACT_SHOT; + if (bpf_skb_load_bytes(skb, ETH_HLEN, &iph_outer, sizeof(iph_outer)) < 0) return TC_ACT_OK; @@ -670,19 +741,25 @@ static int decap_ipv4(struct __sk_buff *skb) return TC_ACT_OK; return decap_internal(skb, ETH_HLEN, sizeof(iph_outer), - iph_outer.protocol); + iph_outer.protocol, + BPF_F_ADJ_ROOM_DECAP_IPXIP4); } static int decap_ipv6(struct __sk_buff *skb) { struct ipv6hdr iph_outer; + if (!bpf_core_enum_value_exists(enum bpf_adj_room_flags, + BPF_F_ADJ_ROOM_DECAP_IPXIP6)) + return TC_ACT_SHOT; + if (bpf_skb_load_bytes(skb, ETH_HLEN, &iph_outer, sizeof(iph_outer)) < 0) return TC_ACT_OK; return decap_internal(skb, ETH_HLEN, sizeof(iph_outer), - iph_outer.nexthdr); + iph_outer.nexthdr, + BPF_F_ADJ_ROOM_DECAP_IPXIP6); } SEC("tc") diff --git a/tools/testing/selftests/bpf/progs/tracing_failure.c b/tools/testing/selftests/bpf/progs/tracing_failure.c index 65e485c4468c..f7a095767679 100644 --- a/tools/testing/selftests/bpf/progs/tracing_failure.c +++ b/tools/testing/selftests/bpf/progs/tracing_failure.c @@ -30,3 +30,9 @@ int BPF_PROG(fexit_noreturns) { return 0; } + +SEC("?fexit/bpf_testmod_test_int128_ret") +int BPF_PROG(fexit_int128_ret) +{ + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/tracing_multi_intersect_attach.c b/tools/testing/selftests/bpf/progs/tracing_multi_intersect_attach.c index cd5be0bb6ffd..5b0af8f4c62f 100644 --- a/tools/testing/selftests/bpf/progs/tracing_multi_intersect_attach.c +++ b/tools/testing/selftests/bpf/progs/tracing_multi_intersect_attach.c @@ -11,6 +11,14 @@ __u64 test_result_fentry_1 = 0; __u64 test_result_fentry_2 = 0; __u64 test_result_fexit_1 = 0; __u64 test_result_fexit_2 = 0; +__u64 test_result_fentry = 0; + +SEC("fentry/bpf_fentry_test1") +int BPF_PROG(fentry) +{ + tracing_multi_arg_check(ctx, &test_result_fentry, false); + return 0; +} SEC("fentry.multi") int BPF_PROG(fentry_1) diff --git a/tools/testing/selftests/bpf/progs/tracing_struct_int128.c b/tools/testing/selftests/bpf/progs/tracing_struct_int128.c new file mode 100644 index 000000000000..4638dfec1f38 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_struct_int128.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_helpers.h> + +long t_b, t_c, t_ret; + +SEC("fexit/bpf_testmod_test_int128_arg") +int test_int128_arg_fexit(unsigned long long *ctx) +{ + t_b = (int)ctx[2]; + t_c = (long)ctx[3]; + t_ret = (long)ctx[4]; + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_arena.c b/tools/testing/selftests/bpf/progs/verifier_arena.c index df0e22d1a29b..815f342eb4b0 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena.c @@ -12,15 +12,17 @@ #define private(name) SEC(".bss." #name) __hidden __attribute__((aligned(8))) +#ifdef __TARGET_ARCH_arm64 +#define ARENA_VM_START ((1ull << 32) | (~0u - __PAGE_SIZE * 2 + 1)) +#else +#define ARENA_VM_START ((1ull << 44) | (~0u - __PAGE_SIZE * 2 + 1)) +#endif + struct { __uint(type, BPF_MAP_TYPE_ARENA); __uint(map_flags, BPF_F_MMAPABLE); __uint(max_entries, 2); /* arena of two pages close to 32-bit boundary*/ -#ifdef __TARGET_ARCH_arm64 - __ulong(map_extra, (1ull << 32) | (~0u - __PAGE_SIZE * 2 + 1)); /* start of mmap() region */ -#else - __ulong(map_extra, (1ull << 44) | (~0u - __PAGE_SIZE * 2 + 1)); /* start of mmap() region */ -#endif + __ulong(map_extra, ARENA_VM_START); /* start of mmap() region */ } arena SEC(".maps"); SEC("socket") @@ -93,6 +95,34 @@ int basic_alloc1(void *ctx) return 0; } +SEC("syscall") +__success __retval(0) +int free_scalar_below_arena(void *ctx) +{ + void __arena *page1, *page2, *page3; + __u64 bad_addr = ARENA_VM_START - __PAGE_SIZE; + + page1 = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!page1) + return 1; + + page2 = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!page2) + return 2; + + page3 = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (page3) + return 3; + + bpf_arena_free_pages(&arena, (void __arena *)bad_addr, 1); + + page3 = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (page3) + return 4; + + return 0; +} + SEC("socket") __success __retval(0) int basic_alloc2_nosleep(void *ctx) @@ -605,7 +635,37 @@ int non_arena_ptr_add_to_arena_ptr(void *ctx) return 0; } -#endif +SEC("socket") +__description("arena and stack atomic at the same instruction") +__failure __msg("same insn cannot be used with different pointers") +__arch_x86_64 +__load_if_JITed() +__naked void mixed_arena_stack_atomic(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r6 = r10; \ + r6 += -8; \ + r9 = 0; \ + *(u64 *)(r6 + 0) = r9; \ + r7 = 8192; \ + r7 = addr_space_cast(r7, 0, 1); \ + call %[bpf_get_prandom_u32]; \ + if w0 != 0 goto 1f; \ + r8 = r6; \ + goto 2f; \ +1: r8 = r7; \ +2: r9 = 1; \ + lock *(u64 *)(r8 + 0) += r9; \ + r0 = 0; \ + exit; \ +" : + : __imm_addr(arena), + __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +#endif /* defined(__BPF_FEATURE_ADDR_SPACE_CAST) */ static __noinline u32 __arena *check_arena_arg_nonglobal(u32 __arena *arg) diff --git a/tools/testing/selftests/bpf/progs/verifier_basic_stack.c b/tools/testing/selftests/bpf/progs/verifier_basic_stack.c index d3df7a9f1d8c..0eb495ce85c1 100644 --- a/tools/testing/selftests/bpf/progs/verifier_basic_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_basic_stack.c @@ -27,7 +27,8 @@ __naked void stack_out_of_bounds(void) SEC("socket") __description("uninitialized stack1") -__success __log_level(4) __msg("stack depth 8") +__success __log_level(4) +__msg("subprog 0 (uninitialized_stack1) main {{.*}} stack 8") __failure_unpriv __msg_unpriv("invalid read from stack") __naked void uninitialized_stack1(void) { @@ -45,7 +46,8 @@ __naked void uninitialized_stack1(void) SEC("socket") __description("uninitialized stack2") -__success __log_level(4) __msg("stack depth 8") +__success __log_level(4) +__msg("subprog 0 (uninitialized_stack2) main insns_self {{[0-9]+}} insns_total {{[0-9]+}} stack 8") __failure_unpriv __msg_unpriv("invalid read from stack") __naked void uninitialized_stack2(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds.c b/tools/testing/selftests/bpf/progs/verifier_bounds.c index bc038ac2df98..1a273e416fed 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bounds.c +++ b/tools/testing/selftests/bpf/progs/verifier_bounds.c @@ -1195,7 +1195,7 @@ l0_%=: r1 = r6; \ r3 += -8; \ r5 = 0; \ /* The 4th argument of bpf_skb_store_bytes is defined as \ - * ARG_CONST_SIZE, so 0 is not allowed. The 'r4 != 0' \ + * ARG_MEM_SIZE, so 0 is not allowed. The 'r4 != 0' \ * is providing us this exclusion of zero from initial \ * [0, 7] range. \ */ \ diff --git a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c index 8d7ff38e4c06..328cf630210a 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +++ b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c @@ -10,7 +10,8 @@ SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 8") +__log_level(4) +__msg("subprog 0 (simple) main insns_self {{[0-9]+}} insns_total {{[0-9]+}} stack 8") __xlated("4: r5 = 5") __xlated("5: r0 = ") __xlated("6: r0 = &(void __percpu *)(r0)") @@ -96,7 +97,8 @@ __naked void canary_zero_spills(void) SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 16") +__log_level(4) +__msg("subprog 0 (wrong_reg_in_pattern1) main {{.*}} stack 16") __xlated("1: *(u64 *)(r10 -16) = r1") __xlated("...") __xlated("3: r0 = &(void __percpu *)(r0)") @@ -598,7 +600,8 @@ __naked static void subprogs_use_independent_offsets_aux(void) SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 8") +__log_level(4) +__msg("subprog 0 (helper_call_does_not_prevent_bpf_fastcall) main {{.*}} stack 8") __xlated("2: r0 = &(void __percpu *)(r0)") __success __naked void helper_call_does_not_prevent_bpf_fastcall(void) @@ -620,7 +623,8 @@ __naked void helper_call_does_not_prevent_bpf_fastcall(void) SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 24") +__log_level(4) +__msg("subprog 0 (may_goto_interaction_x86_64) main {{.*}} stack 24") /* may_goto counter at -24 */ __xlated("0: *(u64 *)(r10 -24) =") /* may_goto timestamp at -16 */ @@ -660,7 +664,10 @@ __naked void may_goto_interaction_x86_64(void) SEC("raw_tp") __arch_arm64 -__log_level(4) __msg("stack depth 24") +__arch_riscv64 +__arch_loongarch +__log_level(4) +__msg("subprog 0 (may_goto_interaction) main {{.*}} stack 24") /* may_goto counter at -24 */ __xlated("0: *(u64 *)(r10 -24) =") /* may_goto timestamp at -16 */ @@ -679,7 +686,7 @@ __xlated("10: *(u64 *)(r10 -24) = r12") __xlated("11: *(u64 *)(r10 -8) = r1") __xlated("12: exit") __success -__naked void may_goto_interaction_arm64(void) +__naked void may_goto_interaction(void) { asm volatile ( "r1 = 1;" @@ -707,7 +714,9 @@ __naked static void dummy_loop_callback(void) SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 32+0") +__log_level(4) +__msg("subprog 0 (bpf_loop_interaction1) main {{.*}} stack 32") +__msg("subprog 1 (dummy_loop_callback) static {{.*}} stack 0") __xlated("2: r1 = 1") __xlated("3: r0 =") __xlated("4: r0 = &(void __percpu *)(r0)") @@ -755,7 +764,9 @@ __naked int bpf_loop_interaction1(void) SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 40+0") +__log_level(4) +__msg("subprog 0 (bpf_loop_interaction2) main {{.*}} stack 40") +__msg("subprog 1 (dummy_loop_callback) static {{.*}} stack 0") /* call bpf_get_smp_processor_id */ __xlated("2: r1 = 42") __xlated("3: r0 =") @@ -799,7 +810,10 @@ __naked int bpf_loop_interaction2(void) SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 512+0 max 512") +__log_level(4) +__msg("stack depth max 512") +__msg("subprog 0 (cumulative_stack_depth) main {{.*}} stack 512") +__msg("subprog 1 (cumulative_stack_depth_subprog) static {{.*}} stack 0") /* just to print xlated version when debugging */ __xlated("r0 = &(void __percpu *)(r0)") __success diff --git a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c index 67dc352addfd..966f49348787 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c @@ -52,7 +52,15 @@ __msg("('global_calls_good_only') is global and assumed valid.") /* eventually global_good() is transitively validated as well */ __msg("Validating global_good() func") __msg("('global_good') is safe for any args that match its prototype") -__msg("insns processed {{[0-9]+\\+[0-9]+\\+[0-9]+$}}") +__msg("subprog 0 (chained_global_func_calls_success) main insns_self 7 insns_total 7 stack") +__msg("subprog {{[0-9]+}} (global_calls_good_only) global insns_self 2 insns_total 2 stack") +#if defined(__BPF_CPU_VERSION__) && __BPF_CPU_VERSION__ >= 4 +__msg("subprog {{[0-9]+}} (global_good) global insns_self 3 insns_total 3 stack") +__msg("processed 12 insns") +#else +__msg("subprog {{[0-9]+}} (global_good) global insns_self 5 insns_total 5 stack") +__msg("processed 14 insns") +#endif int chained_global_func_calls_success(void) { int sum = 0; diff --git a/tools/testing/selftests/bpf/progs/verifier_gotox.c b/tools/testing/selftests/bpf/progs/verifier_gotox.c index f88aa4cdb279..5b18c9a27717 100644 --- a/tools/testing/selftests/bpf/progs/verifier_gotox.c +++ b/tools/testing/selftests/bpf/progs/verifier_gotox.c @@ -384,6 +384,31 @@ jt0_%=: \ : __clobber_all); } +/* check valid spill/fill, ptr to insn */ +SEC("socket") +__success +__naked void spill_fill_ptr_to_insn(void) +{ + asm volatile ( + ".pushsection .jumptables,\"\",@progbits;" + "jt0_%=:" + ".quad ret0_%= - socket;" + ".size jt0_%=, 8;" + ".global jt0_%=;" + ".popsection;" + "r0 = jt0_%= ll;" + "r0 = *(u64 *)(r0 + 0);" + "*(u64 *)(r10 - 8) = r0;" + "r0 = *(u64 *)(r10 - 8);" + ".8byte %[gotox_r0];" + "ret0_%=:" + "r0 = 0;" + "exit;" + : + : __imm_insn(gotox_r0, BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_0, 0, 0, 0)) + : __clobber_all); +} + #endif /* __TARGET_ARCH_x86 || __TARGET_ARCH_arm64 || __TARGET_ARCH_powerpc*/ char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c b/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c index f2c54e4d89eb..343fc08d9747 100644 --- a/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c +++ b/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c @@ -85,7 +85,7 @@ __naked void stack_bitwise_and_zero_included(void) r2 += -64; \ r4 = 0; \ /* Call bpf_ringbuf_output(), it is one of a few helper functions with\ - * ARG_CONST_SIZE_OR_ZERO parameter allowed in unpriv mode.\ + * ARG_MEM_SIZE_OR_ZERO parameter allowed in unpriv mode.\ * For unpriv this should signal an error, because memory at &fp[-64] is\ * not initialized. \ */ \ @@ -278,7 +278,7 @@ __naked void stack_jmp_no_min_check(void) r2 += -64; \ r4 = 0; \ /* Call bpf_ringbuf_output(), it is one of a few helper functions with\ - * ARG_CONST_SIZE_OR_ZERO parameter allowed in unpriv mode.\ + * ARG_MEM_SIZE_OR_ZERO parameter allowed in unpriv mode.\ * For unpriv this should signal an error, because memory at &fp[-64] is\ * not initialized. \ */ \ @@ -778,7 +778,7 @@ __naked void variable_memory_8_bytes_leak(void) r3 += 1; \ r4 = 0; \ /* Call bpf_ringbuf_output(), it is one of a few helper functions with\ - * ARG_CONST_SIZE_OR_ZERO parameter allowed in unpriv mode.\ + * ARG_MEM_SIZE_OR_ZERO parameter allowed in unpriv mode.\ * For unpriv this should signal an error, because memory region [1, 64]\ * at &fp[-64] is not fully initialized. \ */ \ diff --git a/tools/testing/selftests/bpf/progs/verifier_helper_value_access.c b/tools/testing/selftests/bpf/progs/verifier_helper_value_access.c index 6d2a38597c34..c6603a118fdc 100644 --- a/tools/testing/selftests/bpf/progs/verifier_helper_value_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_helper_value_access.c @@ -91,7 +91,7 @@ l0_%=: exit; \ /* Call a function taking a pointer and a size which doesn't allow the size to * be zero (i.e. bpf_trace_printk() declares the second argument to be - * ARG_CONST_SIZE, not ARG_CONST_SIZE_OR_ZERO). We attempt to pass zero for the + * ARG_MEM_SIZE, not ARG_MEM_SIZE_OR_ZERO). We attempt to pass zero for the * size and expect to fail. */ SEC("tracepoint") diff --git a/tools/testing/selftests/bpf/progs/verifier_load_acquire.c b/tools/testing/selftests/bpf/progs/verifier_load_acquire.c index 74f4f19c10b8..d17026d7480d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_load_acquire.c +++ b/tools/testing/selftests/bpf/progs/verifier_load_acquire.c @@ -3,6 +3,7 @@ #include <linux/bpf.h> #include <bpf/bpf_helpers.h> +#include <bpf/bpf_core_read.h> #include "../../../include/linux/filter.h" #include "bpf_misc.h" @@ -148,6 +149,22 @@ __naked void load_acquire_from_ctx_pointer(void) : __clobber_all); } +SEC("socket") +__description("load-acquire from ctx pointer, same dst and src register") +__failure __failure_unpriv __msg("BPF_ATOMIC loads from R6 ctx is not allowed") +__naked void load_acquire_ctx_same_dst_src(void) +{ + asm volatile ( + "r6 = r1;" + ".8byte %[load_acquire_insn];" // w6 = load_acquire((u32 *)(r6 + 0)); + "r0 = 0;" + "exit;" + : + : __imm_insn(load_acquire_insn, + BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_6, BPF_REG_6, 0)) + : __clobber_all); +} + SEC("xdp") __description("load-acquire from pkt pointer") __failure __msg("BPF_ATOMIC loads from R2 pkt is not allowed") @@ -206,6 +223,33 @@ __naked void load_acquire_from_sock_pointer(void) } SEC("socket") +__description("load-acquire from rdonly_untrusted_mem pointer") +__failure __msg("BPF_ATOMIC loads from R{{[0-9]+}} rdonly_untrusted_mem is not allowed") +int load_acquire_from_rdonly_untrusted_mem(void *ctx) +{ + __u64 val = 0; + void *p; + + /* + * bpf_rdonly_cast(x, 0) yields PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED. + * A regular BPF_LDX from it is rewritten to BPF_PROBE_MEM, but a + * load-acquire is not, so it must be rejected, otherwise the JIT emits + * a plain load with no exception table entry and a fault would crash + * the kernel. + */ + p = bpf_rdonly_cast(&val, 0); + asm volatile ( + "r1 = %[p];" + ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r1 + 0)); + : + : [p] "r" (p), + __imm_insn(load_acquire_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0)) + : "r0", "r1"); + return 0; +} + +SEC("socket") __description("load-acquire with invalid register R15") __failure __failure_unpriv __msg("R15 is invalid") __naked void load_acquire_with_invalid_reg(void) diff --git a/tools/testing/selftests/bpf/progs/verifier_map_in_map.c b/tools/testing/selftests/bpf/progs/verifier_map_in_map.c index b606b5dca734..d3be69a9a755 100644 --- a/tools/testing/selftests/bpf/progs/verifier_map_in_map.c +++ b/tools/testing/selftests/bpf/progs/verifier_map_in_map.c @@ -154,7 +154,8 @@ l0_%=: r0 = 0; \ SEC("socket") __description("forgot null checking on the inner map pointer") -__failure __msg("R1 type=map_value_or_null expected=map_ptr") +__failure __msg("R1 type=map_ptr_or_null expected=map_ptr") +__msg("map_ptr_or_null, but this argument accepts map_ptr") __failure_unpriv __naked void on_the_inner_map_pointer(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c b/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c new file mode 100644 index 000000000000..c01abf54923d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +struct inner_map { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, int); +} inner_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); + __uint(max_entries, 1); + __type(key, int); + __array(values, struct inner_map); +} outer_map SEC(".maps") = { + .values = { [0] = &inner_map }, +}; + +SEC("?tc") +__failure __msg("type=map_ptr_or_null expected=fp") +int mapofmaps_value_as_kfunc_mem_buf(struct __sk_buff *skb) +{ + struct bpf_dynptr dptr; + __u32 key = 0; + void *inner; + char *p; + + inner = bpf_map_lookup_elem(&outer_map, &key); + /* intentionally NOT NULL-checked: type is map_ptr_or_null */ + + bpf_dynptr_from_skb(skb, 0, &dptr); + /* arg3 is mem+size */ + p = bpf_dynptr_slice(&dptr, 0, inner, 4); + if (p) + return p[0]; + return 0; +} + +SEC("?tc") +__failure __msg("type=map_ptr_or_null expected=fp") +int mapofmaps_value_as_helper_mem_buf(struct __sk_buff *skb) +{ + __u32 key = 0; + void *inner; + + inner = bpf_map_lookup_elem(&outer_map, &key); + /* intentionally NOT NULL-checked: type is map_ptr_or_null */ + + /* arg1 is mem+size */ + return bpf_csum_diff(inner, 4, NULL, 0, 0) + skb->len; +} + +SEC("?tc") +__failure __msg("type=map_ptr_or_null expected=fp") +int mapofmaps_value_as_helper_fixed_mem(struct __sk_buff *skb) +{ + char th[sizeof(struct tcphdr)] = {}; + __u32 key = 0; + void *inner; + + inner = bpf_map_lookup_elem(&outer_map, &key); + /* intentionally NOT NULL-checked: type is map_ptr_or_null */ + + /* arg1 is fixed-sized mem */ + return bpf_tcp_raw_check_syncookie_ipv4(inner, (void *)th); +} diff --git a/tools/testing/selftests/bpf/progs/verifier_map_ptr.c b/tools/testing/selftests/bpf/progs/verifier_map_ptr.c index 166193659870..e0a65835c861 100644 --- a/tools/testing/selftests/bpf/progs/verifier_map_ptr.c +++ b/tools/testing/selftests/bpf/progs/verifier_map_ptr.c @@ -72,14 +72,15 @@ __naked void bpf_map_ptr_write_rejected(void) /* * struct bpf_map starts with the SHA256 hash sha[32] at offset 0 (a readable - * byte array), the u32 excl field at offset 32, and the ops pointer at offset - * 40. Reading a u32 at offset 41 reaches into the middle of the ops pointer, - * i.e. a partial pointer access, which is rejected. + * byte array), followed by the ops pointer at offset 32 and the inner_map_meta + * pointer at offset 40. Reading a u32 at offset 41 reaches into the middle of + * the inner_map_meta pointer, i.e. a partial pointer access, which is + * rejected. */ SEC("socket") __description("bpf_map_ptr: read non-existent field rejected") __failure -__msg("cannot access ptr member ops with moff 40 in struct bpf_map with off 41 size 4") +__msg("cannot access ptr member inner_map_meta with moff 40 in struct bpf_map with off 41 size 4") __failure_unpriv __msg_unpriv("access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN") __flag(BPF_F_ANY_ALIGNMENT) @@ -97,23 +98,23 @@ __naked void read_non_existent_field_rejected(void) } /* - * The u32 excl field spans offsets 32..35 (mend 36). Reading a u32 at offset - * 33 starts inside excl but extends past its end, which the verifier rejects + * The sha byte array spans offsets 0..31 (mend 32). Reading a u32 at offset + * 30 starts inside sha but extends past its end, which the verifier rejects * as an out-of-bounds scalar access. */ SEC("socket") -__description("bpf_map_ptr: read beyond excl field rejected") +__description("bpf_map_ptr: read beyond sha field rejected") __failure -__msg("access beyond the end of member excl (mend:36) in struct bpf_map with off 33 size 4") +__msg("access beyond the end of member sha (mend:32) in struct bpf_map with off 30 size 4") __failure_unpriv __msg_unpriv("access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN") __flag(BPF_F_ANY_ALIGNMENT) -__naked void read_beyond_excl_field_rejected(void) +__naked void read_beyond_sha_field_rejected(void) { asm volatile (" \ r6 = 0; \ r1 = %[map_array_48b] ll; \ - r6 = *(u32*)(r1 + 33); \ + r6 = *(u32*)(r1 + 30); \ r0 = 1; \ exit; \ " : @@ -131,7 +132,7 @@ __naked void ptr_read_ops_field_accepted(void) asm volatile (" \ r6 = 0; \ r1 = %[map_array_48b] ll; \ - r6 = *(u64*)(r1 + 40); \ + r6 = *(u64*)(r1 + 32); \ r0 = 1; \ exit; \ " : diff --git a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c index 4bdf4256a41e..db7e30da234f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c +++ b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c @@ -11,6 +11,8 @@ __description("may_goto 0") __arch_x86_64 __arch_s390x __arch_arm64 +__arch_riscv64 +__arch_loongarch __xlated("0: r0 = 1") __xlated("1: exit") __success @@ -31,6 +33,8 @@ __description("batch 2 of may_goto 0") __arch_x86_64 __arch_s390x __arch_arm64 +__arch_riscv64 +__arch_loongarch __xlated("0: r0 = 1") __xlated("1: exit") __success @@ -53,6 +57,8 @@ __description("may_goto batch with offsets 2/1/0") __arch_x86_64 __arch_s390x __arch_arm64 +__arch_riscv64 +__arch_loongarch __xlated("0: r0 = 1") __xlated("1: exit") __success @@ -79,6 +85,8 @@ __description("may_goto batch with offsets 2/0") __arch_x86_64 __arch_s390x __arch_arm64 +__arch_riscv64 +__arch_loongarch __xlated("0: *(u64 *)(r10 -16) = 65535") __xlated("1: *(u64 *)(r10 -8) = 0") __xlated("2: r12 = *(u64 *)(r10 -16)") @@ -106,4 +114,63 @@ __naked void may_goto_batch_2(void) : __clobber_all); } +/* + * Use bpf_get_prandom_u32() to prevent DCE from removing the checks. + * retval: 0=all ok, 1-6=R0-R5 clobbered. + */ +SEC("syscall") +__description("timed may_goto preserves R0-R5") +__arch_x86_64 +__arch_s390x +__arch_arm64 +__arch_riscv64 +__arch_loongarch +__success +__retval(0) +__naked void timed_may_goto_preserves_regs(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r0 = 0x1111;" + "r0 += r6;" + "r1 = 0x2222;" + "r1 += r6;" + "r2 = 0x3333;" + "r2 += r6;" + "r3 = 0x4444;" + "r3 += r6;" + "r4 = 0x5555;" + "r4 += r6;" + "r5 = 0x6666;" + "r5 += r6;" + ".8byte %[may_goto];" + ".8byte %[loop];" + "r0 -= r6;" + "r1 -= r6;" + "r2 -= r6;" + "r3 -= r6;" + "r4 -= r6;" + "r5 -= r6;" + "if r0 != 0x1111 goto 1f;" + "if r1 != 0x2222 goto 2f;" + "if r2 != 0x3333 goto 3f;" + "if r3 != 0x4444 goto 4f;" + "if r4 != 0x5555 goto 5f;" + "if r5 != 0x6666 goto 6f;" + "r0 = 0;" + "exit;" + "1: r0 = 1; exit;" + "2: r0 = 2; exit;" + "3: r0 = 3; exit;" + "4: r0 = 4; exit;" + "5: r0 = 5; exit;" + "6: r0 = 6; exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 1, 0)), + __imm_insn(loop, BPF_RAW_INSN(BPF_JMP | BPF_JA, 0, 0, -2, 0)) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_mem_size_reg.c b/tools/testing/selftests/bpf/progs/verifier_mem_size_reg.c new file mode 100644 index 000000000000..7e24706a764e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_mem_size_reg.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +/* + * The __szk size of a kfunc memory/size pair must be marked precise even when + * the nullable buffer is passed as NULL. + */ +SEC("?tc") +__success __log_level(2) +__msg("mark_precise: frame0: regs=r4 stack= before") +int dynptr_slice_null_buf_size_precise(struct __sk_buff *skb) +{ + struct bpf_dynptr dptr; + char *p; + + bpf_dynptr_from_skb(skb, 0, &dptr); + + p = bpf_dynptr_slice(&dptr, 0, NULL, 8); + if (p) + return p[0]; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c b/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c new file mode 100644 index 000000000000..967f4e6e3a49 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +#if defined(__TARGET_ARCH_x86) + +int percpu_data SEC(".percpu"); + +/* + * An ld_imm64 of a per-CPU map value is followed by a mov_percpu_addr that + * reuses the same register, so check that the add resolves into the register + * the address was loaded into, for every register. + */ +SEC("raw_tp") +__description("per-CPU address resolution") +__success +__arch_x86_64 +__jited(" movabsq $0x{{.*}}, %rax") +__jited(" addq %gs:{{.*}}, %rax") +__jited(" movabsq $0x{{.*}}, %rdi") +__jited(" addq %gs:{{.*}}, %rdi") +__jited(" movabsq $0x{{.*}}, %rsi") +__jited(" addq %gs:{{.*}}, %rsi") +__jited(" movabsq $0x{{.*}}, %rdx") +__jited(" addq %gs:{{.*}}, %rdx") +__jited(" movabsq $0x{{.*}}, %rcx") +__jited(" addq %gs:{{.*}}, %rcx") +__jited(" movabsq $0x{{.*}}, %r8") +__jited(" addq %gs:{{.*}}, %r8") +__jited(" movabsq $0x{{.*}}, %rbx") +__jited(" addq %gs:{{.*}}, %rbx") +__jited(" movabsq $0x{{.*}}, %r13") +__jited(" addq %gs:{{.*}}, %r13") +__jited(" movabsq $0x{{.*}}, %r14") +__jited(" addq %gs:{{.*}}, %r14") +__jited(" movabsq $0x{{.*}}, %r15") +__jited(" addq %gs:{{.*}}, %r15") +__naked void percpu_addr(void) +{ + asm volatile (" \ + r0 = %[percpu_data] ll; \ + r1 = %[percpu_data] ll; \ + r2 = %[percpu_data] ll; \ + r3 = %[percpu_data] ll; \ + r4 = %[percpu_data] ll; \ + r5 = %[percpu_data] ll; \ + r6 = %[percpu_data] ll; \ + r7 = %[percpu_data] ll; \ + r8 = %[percpu_data] ll; \ + r9 = %[percpu_data] ll; \ + r0 = 0; \ + exit; \ +" : + : __imm_addr(percpu_data) + : __clobber_all); +} + +#else + +SEC("raw_tp") +__description("percpu addr dummy") +__success +int dummy_test(void) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_private_stack.c b/tools/testing/selftests/bpf/progs/verifier_private_stack.c index bb8206e10880..ea0a7e73331d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_private_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_private_stack.c @@ -86,7 +86,9 @@ __naked static void cumulative_stack_depth_subprog(void) SEC("kprobe") __description("Private stack, subtree > MAX_BPF_STACK") __success -__log_level(4) __msg("stack depth 512+32 max 512") +__log_level(4) __msg("stack depth max 512") +__msg("subprog 0 (private_stack_nested_1) main {{.*}} stack 512") +__msg("subprog 1 (cumulative_stack_depth_subprog) static {{.*}} stack 32") __arch_x86_64 /* private stack fp for the main prog */ __jited(" movabsq $0x{{.*}}, %r9") @@ -331,7 +333,11 @@ SEC("fentry/bpf_fentry_test9") __description("Private stack, async callback, potential nesting") __success __retval(0) __load_if_JITed() -__log_level(4) __msg("stack depth 8+0+256+0 max 272") +__log_level(4) __msg("stack depth max 272") +__msg("subprog 0 (private_stack_async_callback_2) main {{.*}} stack 8") +__msg("subprog 1 (timer_cb1) static {{.*}} stack 0") +__msg("subprog 2 (subprog1) static {{.*}} stack 256") +__msg("subprog 3 (subprog2) static {{.*}} stack 0") __arch_x86_64 __jited(" subq $0x100, %rsp") __arch_arm64 @@ -355,7 +361,10 @@ int private_stack_async_callback_2(void) SEC("fentry/bpf_fentry_test9") __description("private stack, max stack depth is private stack") __success -__log_level(4) __msg("stack depth 8+256+0 max 256") +__log_level(4) __msg("stack depth max 256") +__msg("subprog 0 (private_stack_max_depth) main {{.*}} stack 8") +__msg("subprog 1 (subprog1) static insns_self {{[0-9]+}} insns_total {{[0-9]+}} stack 256") +__msg("subprog 2 (subprog2) static insns_self {{[0-9]+}} insns_total {{[0-9]+}} stack 0") int private_stack_max_depth(void) { int x = 0; diff --git a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c index e38f102da45f..663d15fc5fd2 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c +++ b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c @@ -4,6 +4,13 @@ #include <bpf/bpf_helpers.h> #include "bpf_misc.h" +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1); + __type(key, long long); + __type(value, long long); +} map_hash_8b SEC(".maps"); + /* Check that precision marks propagate through scalar IDs. * Registers r{0,1,2} have the same scalar ID. * Range information is propagated for scalars sharing same ID. @@ -915,4 +922,53 @@ __naked void linked_regs_and_subreg_def(void) : __clobber_all); } +/* + * A scalar is spilled to the stack and then filled twice: once via a + * sign-extending load (BPF_MEMSX) into r4 and once via a zero-extending + * load (BPF_MEM) into r5. coerce_reg_to_size_sx() gives r4 a different + * value than the spilled/zero-extended siblings, so r4 must not keep the + * shared scalar id. Otherwise the later 'if r5 == 0x80000000' refines r4 + * through sync_linked_regs() to a known 0x80000000, while at runtime r4 + * is the sign-extended 0xffffffff80000000. The test turns that discrepancy + * into an out-of-bounds map value access (r4 >> 63 is believed 0 but is 1 + * at runtime), which must be rejected. + */ +SEC("socket") +__failure __msg("R0 max value is outside of the allowed memory range") +__naked void ldsx_fill_scalar_id_not_shared(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64*)(r10 - 8) = r1; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map_hash_8b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + /* r7 = unknown u32, keep only bit 31 */ \ + r7 = *(u32*)(r0 + 0); \ + r2 = 0x80000000 ll; \ + r7 &= r2; \ + /* link r6 and r7 via a fresh scalar id */ \ + r6 = r7; \ + /* spill r7 (u32) to the stack */ \ + *(u32*)(r10 - 8) = r7; \ + /* sign-extending fill: must drop the id */ \ + r4 = *(s32*)(r10 - 8); \ + /* zero-extending fill: keeps the id */ \ + r5 = *(u32*)(r10 - 8); \ + /* r5 becomes known 0x80000000 on fall-through */\ + if r5 != r2 goto l0_%=; \ + /* verifier believes r4 == 0 here, runtime is 1 */\ + r4 >>= 63; \ + r0 += r4; \ + r0 = *(u8*)(r0 + 7); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_8b) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_sockmap_mutate.c b/tools/testing/selftests/bpf/progs/verifier_sockmap_mutate.c index fe4b123187b8..20332a731d4e 100644 --- a/tools/testing/selftests/bpf/progs/verifier_sockmap_mutate.c +++ b/tools/testing/selftests/bpf/progs/verifier_sockmap_mutate.c @@ -74,7 +74,7 @@ static __always_inline void test_sockmap_lookup_and_mutate(void) } SEC("action") -__success +__failure __msg("cannot update sockmap in this context") int test_sched_act(struct __sk_buff *skb) { test_sockmap_mutate(skb->sk); @@ -82,7 +82,7 @@ int test_sched_act(struct __sk_buff *skb) } SEC("classifier") -__success +__failure __msg("cannot update sockmap in this context") int test_sched_cls(struct __sk_buff *skb) { test_sockmap_mutate(skb->sk); @@ -90,7 +90,7 @@ int test_sched_cls(struct __sk_buff *skb) } SEC("flow_dissector") -__success +__failure __msg("cannot update sockmap in this context") int test_flow_dissector_delete(struct __sk_buff *skb __always_unused) { test_sockmap_delete(); @@ -98,7 +98,7 @@ int test_flow_dissector_delete(struct __sk_buff *skb __always_unused) } SEC("flow_dissector") -__failure __msg("program of this type cannot use helper bpf_sk_release") +__failure __msg("cannot update sockmap in this context") int test_flow_dissector_update(struct __sk_buff *skb __always_unused) { test_sockmap_lookup_and_update(); /* no access to skb->sk */ @@ -146,7 +146,7 @@ int test_sk_reuseport(struct sk_reuseport_md *ctx) } SEC("socket") -__success +__failure __msg("cannot update sockmap in this context") int test_socket_filter(struct __sk_buff *skb) { test_sockmap_mutate(skb->sk); @@ -179,7 +179,7 @@ int test_sockops_update_dedicated(struct bpf_sock_ops *ctx) } SEC("xdp") -__success +__failure __msg("cannot update sockmap in this context") int test_xdp(struct xdp_md *ctx __always_unused) { test_sockmap_lookup_and_mutate(); diff --git a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c index 0174887e28f5..8b166c42c4e0 100644 --- a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c +++ b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c @@ -634,6 +634,32 @@ __naked void partial_stack_load_preserves_partial_zeros(void) : __clobber_common); } +SEC("raw_tp") +__log_level(2) +__success +__msg("mark_precise: frame0: regs= stack=-8") +__msg("R2=0") +__naked void stack_load_preserves_mixed_zero_and_zero_spill(void) +{ + asm volatile ( + /* fp-8 has scalar const-zero spill bytes and STACK_ZERO bytes. */ + ".8byte %[fp4_st_zero];" /* LLVM-18+: *(u32 *)(r10 -4) = 0; */ + "r0 = 0;" + "*(u32 *)(r10 -8) = r0;" + + "r1 = %[single_byte_buf];" + "r2 = *(u64 *)(r10 -8);" + "r1 += r2;" + "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ + + "r0 = 0;" + "exit;" + : + : __imm_ptr(single_byte_buf), + __imm_insn(fp4_st_zero, BPF_ST_MEM(BPF_W, BPF_REG_FP, -4, 0)) + : __clobber_common); +} + char two_byte_buf[2] SEC(".data.two_byte_buf"); SEC("raw_tp") @@ -1377,4 +1403,46 @@ __naked void partial_fill_from_cleaned_pointer_spill(void) ::: __clobber_all); } +/* check valid spill/fill, ptr to tp buffer */ +SEC("raw_tracepoint.w") +__success +__naked void spill_fill_ptr_to_tp_buffer(void) +{ + asm volatile ( + "r6 = *(u64*)(r1 + 0);" /* r6 is the writable tracepoint buffer */ + "*(u64*)(r10 - 8) = r6;" + "r7 = *(u64*)(r10 - 8);" + "r0 = 0;" + "*(u64*)(r7 + 0) = r0;" /* should be able to write through the buffer */ + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +__noinline int spill_fill_dynptr_subprog(struct bpf_dynptr *dptr) +{ + long *p; + + asm volatile ("*(u64 *)(r10 - 8) = %[dptr];" /* spill the CONST_PTR_TO_DYNPTR argument */ + "%[dptr] = *(u64 *)(r10 - 8);" + : [dptr] "+r"(dptr) :: "memory"); + p = bpf_dynptr_data(dptr, 0, sizeof(*p)); + if (!p) + return 0; + return 0; +} + +static char dptr_mem_buf[16]; + +/* check valid spill/fill, const ptr to dynptr */ +SEC("socket") +__success +int spill_fill_const_ptr_to_dynptr(void) +{ + struct bpf_dynptr ptr; + + bpf_dynptr_from_mem(dptr_mem_buf, sizeof(dptr_mem_buf), 0, &ptr); + return spill_fill_dynptr_subprog(&ptr); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_subprog_insn_stats.c b/tools/testing/selftests/bpf/progs/verifier_subprog_insn_stats.c new file mode 100644 index 000000000000..8f6082fdb5c8 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_subprog_insn_stats.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +struct timer_value { + struct bpf_timer timer; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct timer_value); +} timer_map SEC(".maps"); + +SEC("?raw_tp") +__success __log_level(4) +__msg("subprog 0 (stats_main_only) main insns_self 2 insns_total 2 stack 0") +__msg("processed 2 insns") +__naked int stats_main_only(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +__naked __noinline __used +static int stats_chain_leaf(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +__naked __noinline __used +static int stats_chain_parent(void) +{ + asm volatile ( + "call stats_chain_leaf;" + "exit;" + ); +} + +SEC("?raw_tp") +__success __log_level(4) +/* + * self: 2 + 2 + 2 = 6 + * totals: leaf 2, parent 2 + 2 = 4, main 2 + 4 = 6 + */ +__msg("subprog 0 (stats_static_chain) main insns_self 2 insns_total 6 stack 0") +__msg("subprog {{[0-9]+}} (stats_chain_parent) static insns_self 2 insns_total 4 stack 0") +__msg("subprog {{[0-9]+}} (stats_chain_leaf) static insns_self 2 insns_total 2 stack 0") +__msg("processed 6 insns") +__naked int stats_static_chain(void) +{ + asm volatile ( + "call stats_chain_parent;" + "exit;" + ); +} + +__naked __noinline __used +static int stats_shared_leaf(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +__naked __noinline __used +int stats_global_root(void) +{ + asm volatile ( + "call stats_shared_leaf;" + "exit;" + ); +} + +SEC("?raw_tp") +__success __log_level(4) +/* + * stats_shared_leaf is explored once under each independent root. + * self: main 3 + leaf 4 + global 2 = 9 + * root totals: main 5 + global 4 = 9 + */ +__msg("subprog 0 (stats_shared_roots) main insns_self 3 insns_total 5 stack 0") +__msg("subprog {{[0-9]+}} (stats_shared_leaf) static insns_self 4 insns_total 4 stack 0") +__msg("subprog {{[0-9]+}} (stats_global_root) global insns_self 2 insns_total 4 stack 0") +__msg("processed 9 insns") +__naked int stats_shared_roots(void) +{ + asm volatile ( + "call stats_shared_leaf;" + "call stats_global_root;" + "exit;" + ); +} + +__noinline __used +static int stats_async_leaf(void *map, __u32 *key, struct bpf_timer *timer) +{ + return 0; +} + +__noinline __used +static __u64 stats_async_schedule(struct bpf_map *map, __u32 *key, + struct timer_value *value, void *ctx) +{ + asm volatile ( + "r1 = %[timer];" + "r2 = %[stats_async_leaf];" + "call %[bpf_timer_set_callback];" + : + : [timer] "r" (value), + __imm_ptr(stats_async_leaf), + __imm(bpf_timer_set_callback) + : __clobber_common + ); + return 0; +} + +SEC("?raw_tp") +__success __log_level(4) +/* + * self: 9 + 7 + 2 = 18 + * totals: leaf 2, scheduler 7, main root 18 + */ +__msg("subprog 0 (stats_async_direct) main insns_self 9 insns_total 18 stack 0") +__msg("subprog {{[0-9]+}} (stats_async_schedule) static insns_self 7 insns_total 7 stack 0") +__msg("subprog {{[0-9]+}} (stats_async_leaf) static insns_self 2 insns_total 2 stack 0") +__msg("processed 18 insns") +__naked int stats_async_direct(void) +{ + asm volatile ( + "r1 = %[timer_map] ll;" + "r2 = %[stats_async_schedule];" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_for_each_map_elem];" + "r0 = 0;" + "exit;" + : + : __imm_addr(timer_map), + __imm_ptr(stats_async_schedule), + __imm(bpf_for_each_map_elem) + : __clobber_common + ); +} + +__noinline __used +static int stats_async_nested_leaf(void *map, __u32 *key, struct bpf_timer *timer) +{ + return 0; +} + +__noinline __used +static int stats_async_outer(void *map, __u32 *key, struct bpf_timer *timer) +{ + asm volatile ( + "r1 = %[timer];" + "r2 = %[stats_async_nested_leaf];" + "call %[bpf_timer_set_callback];" + : + : [timer] "r" (timer), + __imm_ptr(stats_async_nested_leaf), + __imm(bpf_timer_set_callback) + : __clobber_common + ); + return 0; +} + +__noinline __used +static __u64 stats_async_nested_schedule(struct bpf_map *map, __u32 *key, + struct timer_value *value, void *ctx) +{ + asm volatile ( + "r1 = %[timer];" + "r2 = %[stats_async_outer];" + "call %[bpf_timer_set_callback];" + : + : [timer] "r" (value), + __imm_ptr(stats_async_outer), + __imm(bpf_timer_set_callback) + : __clobber_common + ); + return 0; +} + +SEC("?raw_tp") +__success __log_level(4) +/* + * self: 9 + 7 + 7 + 2 = 25 + * totals: leaf 2, outer 7, scheduler 7, main root 25 + */ +__msg("subprog 0 (stats_async_nested) main insns_self 9 insns_total 25 stack 0") +__msg("subprog {{[0-9]+}} (stats_async_nested_schedule) static insns_self 7 insns_total 7 stack 0") +__msg("subprog {{[0-9]+}} (stats_async_outer) static insns_self 7 insns_total 7 stack 0") +__msg("subprog {{[0-9]+}} (stats_async_nested_leaf) static insns_self 2 insns_total 2 stack 0") +__msg("processed 25 insns") +__naked int stats_async_nested(void) +{ + asm volatile ( + "r1 = %[timer_map] ll;" + "r2 = %[stats_async_nested_schedule];" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_for_each_map_elem];" + "r0 = 0;" + "exit;" + : + : __imm_addr(timer_map), + __imm_ptr(stats_async_nested_schedule), + __imm(bpf_for_each_map_elem) + : __clobber_common + ); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_uninit.c b/tools/testing/selftests/bpf/progs/verifier_uninit.c index 7718cd7d19ce..691018a46049 100644 --- a/tools/testing/selftests/bpf/progs/verifier_uninit.c +++ b/tools/testing/selftests/bpf/progs/verifier_uninit.c @@ -9,6 +9,7 @@ SEC("socket") __description("read uninitialized register") __failure __msg("R2 !read_ok") +__msg("R2 has never been initialized on this path") __failure_unpriv __naked void read_uninitialized_register(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_unpriv.c b/tools/testing/selftests/bpf/progs/verifier_unpriv.c index 42de5cff7e52..3069e70fbcbd 100644 --- a/tools/testing/selftests/bpf/progs/verifier_unpriv.c +++ b/tools/testing/selftests/bpf/progs/verifier_unpriv.c @@ -96,6 +96,24 @@ __naked void pseudo_btf_id_log_masks_address(void) : __clobber_all); } +static int pseudo_func_callback(__u32 index, void *ctx) +{ + return 0; +} + +SEC("socket") +__description("unpriv: pseudo function policy diagnostic") +__success __failure_unpriv +__msg_unpriv("loading/calling other bpf or kernel functions") +__not_msg_unpriv("BPF-to-BPF function call") +__msg_unpriv("policy check failed for BPF function reference") +__msg_unpriv("avoid BPF function references in unprivileged") +int unpriv_pseudo_func_policy(void *ctx) +{ + bpf_loop(1, pseudo_func_callback, NULL, 0); + return 0; +} + SEC("socket") __description("unpriv: return pointer") __success __failure_unpriv __msg_unpriv("R0 leaks addr") diff --git a/tools/testing/selftests/bpf/progs/verifier_var_off.c b/tools/testing/selftests/bpf/progs/verifier_var_off.c index f345466bca68..a63e33675091 100644 --- a/tools/testing/selftests/bpf/progs/verifier_var_off.c +++ b/tools/testing/selftests/bpf/progs/verifier_var_off.c @@ -60,6 +60,116 @@ __naked void stack_read_priv_vs_unpriv(void) } SEC("cgroup/skb") +__description("variable-offset stack read preserves spilled zero") +__success +__log_level(2) +__msg("mark_precise: frame0: regs= stack=-8") +__msg("R3=0") +__retval(0) +__naked void stack_read_var_off_preserves_spilled_zero(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64*)(r10 - 8) = r0; \ + r2 = *(u32*)(r1 + 0); \ + r2 &= 7; \ + r2 -= 8; \ + r2 += r10; \ + r3 = *(u8*)(r2 + 0); \ + r1 = r10; \ + r1 += -1; \ + r1 += r3; \ + *(u8*)(r1 + 0) = r3; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("cgroup/skb") +__description("variable-offset stack read preserves spilled zero across slots") +__success +__log_level(2) +__msg("mark_precise: frame0: regs= stack=-8,-16") +__msg("R3=0") +__retval(0) +__naked void stack_read_var_off_preserves_spilled_zero_across_slots(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64*)(r10 - 8) = r0; \ + *(u64*)(r10 - 16) = r0; \ + r2 = *(u32*)(r1 + 0); \ + r2 &= 15; \ + r2 -= 16; \ + r2 += r10; \ + r3 = *(u8*)(r2 + 0); \ + r1 = r10; \ + r1 += -1; \ + r1 += r3; \ + *(u8*)(r1 + 0) = r3; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("cgroup/skb") +__description("variable-offset stack read preserves partial spilled zero") +__success +__log_level(2) +__msg("mark_precise: frame0: regs= stack=-8") +__msg("R3=0") +__retval(0) +__naked void stack_read_var_off_preserves_partial_spilled_zero(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u8*)(r10 - 9) = r0; \ + *(u8*)(r10 - 10) = r0; \ + *(u8*)(r10 - 11) = r0; \ + *(u8*)(r10 - 12) = r0; \ + *(u8*)(r10 - 13) = r0; \ + *(u8*)(r10 - 14) = r0; \ + *(u8*)(r10 - 15) = r0; \ + *(u32*)(r10 - 8) = r0; \ + r2 = *(u32*)(r1 + 0); \ + r2 &= 15; \ + if r2 > 10 goto l0_%=; \ + r2 -= 15; \ + r2 += r10; \ + r3 = *(u8*)(r2 + 0); \ + r1 = r10; \ + r1 += -1; \ + r1 += r3; \ + *(u8*)(r1 + 0) = r3; \ +l0_%=: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("cgroup/skb") +__description("variable-offset stack read partial spill with misc data") +__failure +__msg("invalid variable-offset write to stack R1") +__naked void stack_read_var_off_partial_spill_with_misc_data(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u32*)(r10 - 8) = r0; \ + r2 = *(u32*)(r1 + 0); \ + r2 &= 7; \ + r2 -= 8; \ + r2 += r10; \ + r3 = *(u8*)(r2 + 0); \ + r1 = r10; \ + r1 += -1; \ + r1 += r3; \ + *(u8*)(r1 + 0) = 0; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("cgroup/skb") __description("variable-offset stack read, uninitialized") __success __failure_unpriv __msg_unpriv("R2 variable stack access prohibited for !root") @@ -88,7 +198,8 @@ __success /* Check that the maximum stack depth is correctly maintained according to the * maximum possible variable offset. */ -__log_level(4) __msg("stack depth 16") +__log_level(4) +__msg("subprog 0 (stack_write_priv_vs_unpriv) main {{.*}} stack 16") __failure_unpriv /* Variable stack access is rejected for unprivileged. */ @@ -128,7 +239,8 @@ __success /* Check that the maximum stack depth is correctly maintained according to the * maximum possible variable offset. */ -__log_level(4) __msg("stack depth 16") +__log_level(4) +__msg("subprog 0 (stack_write_followed_by_read) main {{.*}} stack 16") __failure_unpriv __msg_unpriv("R2 variable stack access prohibited for !root") __retval(0) diff --git a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c index 2870738d93f7..8f0c45421f89 100644 --- a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c +++ b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c @@ -28,7 +28,7 @@ int BPF_PROG(get_task_exe_file_kfunc_null) } SEC("lsm.s/inode_getxattr") -__failure __msg("R1 pointer type STRUCT task_struct must point to scalar, or struct with scalar") +__failure __msg("R1 is fp expected STRUCT task_struct") int BPF_PROG(get_task_exe_file_kfunc_fp) { u64 x; @@ -98,7 +98,7 @@ int BPF_PROG(path_d_path_kfunc_null) } SEC("lsm.s/task_alloc") -__failure __msg("R1 must be referenced or trusted") +__failure __msg("dereference of modified untrusted_ptr_") int BPF_PROG(path_d_path_kfunc_untrusted_from_argument, struct task_struct *task) { struct path *root; @@ -112,7 +112,7 @@ int BPF_PROG(path_d_path_kfunc_untrusted_from_argument, struct task_struct *task } SEC("lsm.s/file_open") -__failure __msg("R1 must be referenced or trusted") +__failure __msg("dereference of modified untrusted_ptr_") int BPF_PROG(path_d_path_kfunc_untrusted_from_current) { struct path *pwd; diff --git a/tools/testing/selftests/bpf/progs/verifier_zext.c b/tools/testing/selftests/bpf/progs/verifier_zext.c new file mode 100644 index 000000000000..8f2362da91d6 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_zext.c @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "../../../include/linux/filter.h" +#include <bpf_arena_common.h> +#include <bpf/bpf_core_read.h> +#include "bpf_misc.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE | BPF_F_NO_USER_CONV); + __uint(max_entries, 1); +} arena SEC(".maps"); + +extern long bpf_kfunc_call_test4(signed char a, short b, int c, long d) __ksym; + +/* to retain debug info for BTF generation */ +void __kfunc_btf_root(void) +{ + bpf_kfunc_call_test4(0, 0, 0, 0); + bpf_arena_alloc_pages(0, 0, 0, 0, 0); + bpf_rdonly_cast(0, 0); +} + +SEC("socket") +__flag(BPF_F_TEST_STATE_FREQ) +__flag(BPF_F_TEST_RND_HI32) +__success __retval(0) +__naked void zext_lost_across_checkpoint(void) +{ + asm volatile (" \ + call %[bpf_ktime_get_ns]; \ + r8 = r0; \ + r6 = 0xdeadbeefcafebabe ll; /* inject some value for r6's upper half */ \ + if r8 != 0 goto 1f; /* fall-through cached first, branch pruned */ \ + r6 = 32; /* full 64-bit def */ \ + goto 2f; \ +1: w6 = 32; /* 32-bit def, zext mark lost */ \ +2: r0 = r6; /* buggy verifier believed upper 32 bits are 0 */ \ + /* and thus did not zero extended w6 = 32. */ \ + r0 >>= 32; \ + exit; \ +" : + : __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +/* 32-bit ALU result read as 64-bit -> zext */ +SEC("socket") +__success __log_level(2) +__msg("w1 = w0{{ +}}; zext") +__naked void zext_alu32_hi_used(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w1 = w0; \ + r0 = r1; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* 32-bit ALU result read only as 32-bit -> no zext */ +SEC("socket") +__success __log_level(2) +__not_msg("; zext") +__naked void no_zext_alu32_hi_unused(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w1 = w0; /* MOV */ \ + w2 = w1; \ + w2 += w1; /* ALU32, BPF_X */ \ + w2 += 1; /* ALU32, BPF_K */ \ + w2 = w2; /* keep w2 alive for previous instruction */ \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* 64-bit definition is never zero extended */ +SEC("socket") +__success __log_level(2) +__not_msg("r1 = r0{{.*}}; zext") +__naked void no_zext_mov64(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r1 = r0; \ + r0 = r1; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Narrow load result read as 64-bit -> zext */ +SEC("socket") +__success __log_level(2) +__msg("r1 = *(u32 *)(r10 -8){{ +}}; zext") +__naked void zext_narrow_load_hi_used(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64 *)(r10 - 8) = r0; \ + r1 = *(u32 *)(r10 - 8); \ + r0 = r1; \ + exit; \ +" ::: __clobber_all); +} + +/* 32-bit atomic fetch result read as 64-bit -> zext */ +SEC("socket") +__success __log_level(2) +__msg("r1 = atomic_fetch_add((u32 *)(r10 -8), r1){{ +}}; zext") +__naked void zext_atomic_fetch32_hi_used(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64 *)(r10 - 8) = r1; \ + w1 = 1; \ + .8byte %[fetch_add32]; \ + r0 = r1; \ + exit; \ +" : + : __imm_insn(fetch_add32, + BPF_ATOMIC_OP(BPF_W, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_1, -8)) + : __clobber_all); +} + +/* 32-bit atomic cmpxchg result (r0) read as 64-bit -> zext */ +SEC("socket") +__success __log_level(2) +__msg("r0 = atomic_cmpxchg((u32 *)(r10 -8), r0, r1){{ +}}; zext") +__naked void zext_cmpxchg32_hi_used(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64 *)(r10 - 8) = r1; \ + w0 = 0; \ + w1 = 1; \ + .8byte %[cmpxchg32]; \ + r2 = r0; \ + r0 = r2; \ + exit; \ +" : + : __imm_insn(cmpxchg32, + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, BPF_REG_10, BPF_REG_1, -8)) + : __clobber_all); +} + +/* 32-bit def before a branch, upper half used on one branch -> zext */ +SEC("socket") +__success __log_level(2) +__msg("w6 = 32{{ +}}; zext") +__naked void zext_cfg_hi_used_one_branch(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w6 = 32; \ + if r0 == 0 goto 1f; \ + r0 = r6; \ + exit; \ +1: r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* r1's upper half is dead, so 'w1 = 1' must NOT be marked for zero extension. */ +SEC("socket") +__success __log_level(2) +__not_msg("w1 = 1{{.*}}; zext") +__naked void no_zext_other_reg_hi_used(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + r6 <<= 32; \ + w1 = 1; \ + r0 = r6; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* LD_ABS defines r0; when r0 is read as 64-bit it must be zero extended */ +SEC("socket") +__success __log_level(2) +__msg("r0 = *(u8 *)skb[0]{{.*}}; zext") +__naked void zext_ld_abs_hi_used(void) +{ + asm volatile (" \ + r6 = r1; \ + r0 = *(u8 *)skb[0]; \ + r7 = r0; \ + r0 = r7; \ + exit; \ +" ::: __clobber_all); +} + +/* Helper parameters are read as 64-bit (call_use_mask() fallback) */ +SEC("socket") +__success __log_level(2) +__msg("w2 = 1{{ +}}; zext") +__naked void helper_param_read_as_64bit(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -8; \ + w2 = 1; \ + call %[bpf_trace_printk]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_trace_printk) + : __clobber_all); +} + +static __used __naked int subprog_reads_arg_as_64bit(void) +{ + asm volatile (" \ + r0 = r1; \ + exit; \ +" ::: __clobber_all); +} + +/* subprogram parameters are conservatively read as 64-bit */ +SEC("socket") +__success __log_level(2) +__msg("w1 = w0{{ +}}; zext") +__naked void subprog_param_read_as_64bit(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w1 = w0; \ + call subprog_reads_arg_as_64bit; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* kfunc parameters are zero extended */ +SEC("tc") +__success __log_level(2) +__msg("w1 = 1{{ +}}; zext") +__msg("w2 = 1{{ +}}; zext") +__msg("w3 = 1{{ +}}; zext") +__msg("w4 = 1{{ +}}; zext") +__naked void kfunc_param_read_per_btf(void) +{ + asm volatile (" \ + w1 = 1; \ + w2 = 1; \ + w3 = 1; \ + w4 = 1; \ + call bpf_kfunc_call_test4; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__success __log_level(2) +__not_msg("; zext") +__naked void alu32_and_32bit_conditional(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w1 = w0; \ + if w1 > 42 goto 1f; /* BPF_K */ \ + w2 = 28; \ + if w2 > w1 goto 1f; /* BPF_X */ \ + r0 = 0; \ +1: exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__success __log_level(2) +__msg("w1 = w0{{ +}}; zext") +__naked void alu32_and_64bit_conditional(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w1 = w0; \ + if r1 > 42 goto 1f; /* BPF_K */ \ + r2 = 28; \ + if r2 > r1 goto 1f; /* BPF_X */ \ + r0 = 0; \ +1: exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__success __log_level(2) +__not_msg("; zext") +__naked void alu64_and_conditionals(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r1 = r0; \ + if w1 > 42 goto 1f; /* BPF_K */ \ + if r1 > 42 goto 1f; /* BPF_K */ \ + r2 = 28; \ + if w2 > w1 goto 1f; /* BPF_X */ \ + if r2 > r1 goto 1f; /* BPF_X */ \ + r0 = 0; \ +1: exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__arch_s390x +__xlated("7: w1 = w0") +__xlated("8: w1 = w1") +__xlated("9: w1 += 8") +__xlated("10: w1 = w1") +__xlated("11: w2 = w1") +__xlated("12: w2 = w2") +__xlated("13: *(u64 *)(r1 +0) = r2") +__naked void arena_ptr(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = 0; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r1 = addr_space_cast(r0, 0, 1); /* needs zext */ \ + r1 += 8; /* needs zext */ \ + r2 = addr_space_cast(r1, 1, 0); /* needs zext because of BPF_F_NO_USER_CONV */ \ + *(u64 *)(r1 +0) = r2; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_arena_alloc_pages), + __imm_addr(arena) + : __clobber_all); +} + +#endif + +/* Check if probe mem loads keep their zero extension. */ +SEC("socket") +__success __log_level(2) +__arch_s390x +__xlated("3: r1 = *(u64 *)(r0 +0)") +__xlated("4: r2 = *(u32 *)(r0 +0)") +__xlated("5: w2 = w2") +__xlated("6: r3 = *(u16 *)(r0 +0)") +__xlated("7: w3 = w3") +__xlated("8: r4 = *(u8 *)(r0 +0)") +__xlated("9: w4 = w4") +__naked void probe_mem(void) +{ + asm volatile (" \ + r1 = 0; \ + r2 = 0; \ + call %[bpf_rdonly_cast]; \ + r1 = *(u64 *)(r0 + 0); /* BPF_PROBE_MEM */ \ + r2 = *(u32 *)(r0 + 0); /* BPF_PROBE_MEM */ \ + r3 = *(u16 *)(r0 + 0); /* BPF_PROBE_MEM */ \ + r4 = *(u8 *)(r0 + 0); /* BPF_PROBE_MEM */ \ + r0 = r1; /* make the registers used */ \ + r0 += r2; \ + r0 += r3; \ + r0 += r4; \ +1: exit; \ +" : + : __imm(bpf_rdonly_cast) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/veristat_bar.c b/tools/testing/selftests/bpf/progs/veristat_bar.c new file mode 100644 index 000000000000..83d2a2a1dfc9 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/veristat_bar.c @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include "veristat_foo.c" diff --git a/tools/testing/selftests/bpf/progs/veristat_foo.c b/tools/testing/selftests/bpf/progs/veristat_foo.c new file mode 100644 index 000000000000..bd24b97664b4 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/veristat_foo.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +/* + * Programs below exist only to exercise veristat's -f name filters, + * their bodies are irrelevant, only the names matter. + * This file is also included by veristat_bar.c, so that the same set of + * program names is available in two differently named object files. + */ + +SEC("socket") +int foo(void *ctx) +{ + return 0; +} + +SEC("socket") +int bar(void *ctx) +{ + return 0; +} + +SEC("socket") +int buz(void *ctx) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/xdp_dummy.c b/tools/testing/selftests/bpf/progs/xdp_dummy.c index d988b2e0cee8..5f1e0771021d 100644 --- a/tools/testing/selftests/bpf/progs/xdp_dummy.c +++ b/tools/testing/selftests/bpf/progs/xdp_dummy.c @@ -10,4 +10,10 @@ int xdp_dummy_prog(struct xdp_md *ctx) return XDP_PASS; } +SEC("xdp") +int __x64_sys_nop(struct xdp_md *ctx) +{ + return XDP_PASS; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 30f1cd23093c..9366a3c578f1 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -161,6 +161,33 @@ bpf_testmod_test_arg_ptr_to_struct(struct bpf_testmod_struct_arg_1 *a) { return bpf_testmod_test_struct_arg_result; } +#ifdef __SIZEOF_INT128__ +noinline __int128 +bpf_testmod_test_int128_ret(int a) +{ + bpf_testmod_test_struct_arg_result = a; + return (__int128)a; +} + +/* + * The __int128 'a' is the first argument on purpose. On arm64 a 16-byte + * argument must start in an even-numbered register pair, so placing it + * after a single-register scalar would leave a padding register (x1) + * unused. pahole maps parameters to registers positionally and would then + * see the following argument in an "unexpected" register and skip BTF + * encoding of the whole function, making it unattachable. Keeping the + * __int128 first (x0:x1) avoids the padding while still exercising the + * trampoline packing of a 128-bit argument together with the trailing + * int and long arguments. + */ +noinline long +bpf_testmod_test_int128_arg(__int128 a, int b, long c) +{ + bpf_testmod_test_struct_arg_result = (long)a + b + c; + return bpf_testmod_test_struct_arg_result; +} +#endif + __weak noinline void bpf_testmod_looooooooooooooooooooooooooooooong_name(void) { } @@ -210,6 +237,44 @@ __bpf_kfunc void bpf_kfunc_common_test(void) { } +__bpf_kfunc u64 bpf_kfunc_arena_arg_test(u64 *val__arena) +{ + u64 old; + + old = *val__arena; + *val__arena = old + 1; + return old; +} + +__bpf_kfunc u64 bpf_kfunc_arena_cap_test(u64 *val__arena) +{ + return (u64)val__arena; +} + +__bpf_kfunc u64 bpf_kfunc_arena_cap_nullable_test(u64 *val__arena__nullable) +{ + return (u64)val__arena__nullable; +} + +__bpf_kfunc u64 bpf_kfunc_arena_args5_test(u64 *a__arena, u64 *b__arena, + u64 *c__arena, u64 *d__arena, + u64 *e__arena__nullable) +{ + return *a__arena + *b__arena + *c__arena + *d__arena + + (e__arena__nullable ? *e__arena__nullable : 0); +} + +__bpf_kfunc u64 bpf_kfunc_arena_stack_arg_test(u64 a, u64 b, u64 c, u64 d, u64 e, + u64 *f__arena) +{ + return a + b + c + d + e + *f__arena; +} + +__bpf_kfunc u64 bpf_kfunc_arena_mixed_test(u64 *a__arena, u64 *b__arena__nullable) +{ + return *a__arena + (b__arena__nullable ? *b__arena__nullable : 0); +} + __bpf_kfunc void bpf_kfunc_dynptr_test(struct bpf_dynptr *ptr, struct bpf_dynptr *ptr__nullable) { @@ -320,9 +385,36 @@ static int bpf_testmod_test_4(void) return 0; } +static int bpf_testmod_ops3__test_arena(u64 *ptr__arena) +{ + return 0; +} + +static int bpf_testmod_ops3__test_arena_nullable(u64 *ptr__arena__nullable) +{ + return 0; +} + +static int bpf_testmod_ops3__test_arena_stack(u64 a, u64 b, u64 c, u64 d, + u64 e, u64 f, u64 g, u64 h, + u64 *ptr__arena) +{ + return 0; +} + +static int bpf_testmod_ops3__test_arena_multislot(struct bpf_testmod_arena_pair p, + u64 *ptr__arena) +{ + return 0; +} + static struct bpf_testmod_ops3 __bpf_testmod_ops3 = { .test_1 = bpf_testmod_test_3, .test_2 = bpf_testmod_test_4, + .test_arena = bpf_testmod_ops3__test_arena, + .test_arena_nullable = bpf_testmod_ops3__test_arena_nullable, + .test_arena_stack = bpf_testmod_ops3__test_arena_stack, + .test_arena_multislot = bpf_testmod_ops3__test_arena_multislot, }; static void bpf_testmod_test_struct_ops3(void) @@ -341,6 +433,28 @@ __bpf_kfunc void bpf_testmod_ops3_call_test_2(void) st_ops3->test_2(); } +__bpf_kfunc int bpf_testmod_ops3_call_test_arena(u64 *ptr__arena) +{ + return st_ops3->test_arena(ptr__arena); +} + +__bpf_kfunc int bpf_testmod_ops3_call_test_arena_nullable(u64 *ptr__arena__nullable) +{ + return st_ops3->test_arena_nullable(ptr__arena__nullable); +} + +__bpf_kfunc int bpf_testmod_ops3_call_test_arena_stack(u64 *ptr__arena) +{ + return st_ops3->test_arena_stack(1, 2, 3, 4, 5, 6, 7, 8, ptr__arena); +} + +__bpf_kfunc int bpf_testmod_ops3_call_test_arena_multislot(u64 *ptr__arena) +{ + struct bpf_testmod_arena_pair p = { .a = 11, .b = 22 }; + + return st_ops3->test_arena_multislot(p, ptr__arena); +} + struct bpf_testmod_btf_type_tag_1 { int a; }; @@ -514,6 +628,11 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, (void)bpf_testmod_test_arg_ptr_to_struct(&struct_arg1_2); +#ifdef __SIZEOF_INT128__ + (void)bpf_testmod_test_int128_ret(i); + (void)bpf_testmod_test_int128_arg((__int128)1, 2, 3); +#endif + (void)trace_bpf_testmod_test_raw_tp_null_tp(NULL); bpf_testmod_test_struct_ops3(); @@ -723,6 +842,12 @@ BTF_ID_FLAGS(func, bpf_iter_testmod_seq_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_testmod_seq_destroy, KF_ITER_DESTROY) BTF_ID_FLAGS(func, bpf_iter_testmod_seq_value) BTF_ID_FLAGS(func, bpf_kfunc_common_test) +BTF_ID_FLAGS(func, bpf_kfunc_arena_arg_test) +BTF_ID_FLAGS(func, bpf_kfunc_arena_cap_test) +BTF_ID_FLAGS(func, bpf_kfunc_arena_cap_nullable_test) +BTF_ID_FLAGS(func, bpf_kfunc_arena_args5_test) +BTF_ID_FLAGS(func, bpf_kfunc_arena_stack_arg_test) +BTF_ID_FLAGS(func, bpf_kfunc_arena_mixed_test) BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_pass1) BTF_ID_FLAGS(func, bpf_kfunc_dynptr_test) BTF_ID_FLAGS(func, bpf_kfunc_nested_acquire_nonzero_offset_test, KF_ACQUIRE) @@ -738,6 +863,10 @@ BTF_ID_FLAGS(func, bpf_testmod_ctx_create, KF_ACQUIRE | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_testmod_ctx_release, KF_RELEASE) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_1) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_2) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_nullable) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_stack) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_multislot) BTF_ID_FLAGS(func, bpf_kfunc_get_default_trusted_ptr_test); BTF_ID_FLAGS(func, bpf_kfunc_put_default_trusted_ptr_test); BTF_KFUNCS_END(bpf_testmod_common_kfunc_ids) @@ -1352,7 +1481,7 @@ __bpf_kfunc void bpf_kfunc_trigger_ctx_check(void) } BTF_KFUNCS_START(bpf_testmod_check_kfunc_ids) -BTF_ID_FLAGS(func, bpf_testmod_test_mod_kfunc) +BTF_ID_FLAGS(func, bpf_testmod_test_mod_kfunc, KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_kfunc_call_test1) BTF_ID_FLAGS(func, bpf_kfunc_call_test2) BTF_ID_FLAGS(func, bpf_kfunc_call_test3) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h index 863fd10f1619..210b919290cc 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h @@ -103,9 +103,23 @@ struct bpf_testmod_ops2 { int (*test_1)(void); }; +/* 16 bytes, so it takes two argument slots when passed by value */ +struct bpf_testmod_arena_pair { + u64 a; + u64 b; +}; + struct bpf_testmod_ops3 { int (*test_1)(void); int (*test_2)(void); + /* Used to test arena pointer arguments. */ + int (*test_arena)(u64 *ptr); + int (*test_arena_nullable)(u64 *ptr); + /* enough leading args to force @ptr onto the stack on x86 and arm64 */ + int (*test_arena_stack)(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f, + u64 g, u64 h, u64 *ptr); + /* a multi-slot leading arg, so @ptr is not at the slot its arg index suggests */ + int (*test_arena_multislot)(struct bpf_testmod_arena_pair p, u64 *ptr); }; struct st_ops_args { diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index c36bb911defa..7d81070eefe7 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -98,6 +98,15 @@ void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p) __ksym; void bpf_kfunc_call_test_ref(struct prog_test_ref_kfunc *p) __ksym; void bpf_kfunc_call_test_mem_len_pass1(void *mem, int len) __ksym; +__u64 bpf_kfunc_arena_arg_test(__u64 *val__arena) __ksym; +__u64 bpf_kfunc_arena_cap_test(__u64 *val__arena) __ksym; +__u64 bpf_kfunc_arena_cap_nullable_test(__u64 *val__arena__nullable) __ksym; +__u64 bpf_kfunc_arena_args5_test(__u64 *a__arena, __u64 *b__arena, + __u64 *c__arena, __u64 *d__arena, + __u64 *e__arena__nullable) __ksym; +__u64 bpf_kfunc_arena_stack_arg_test(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + __u64 *f__arena) __ksym; +__u64 bpf_kfunc_arena_mixed_test(__u64 *a__arena, __u64 *b__arena__nullable) __ksym; int *bpf_kfunc_call_test_get_rdwr_mem(struct prog_test_ref_kfunc *p, const int rdwr_buf_size) __ksym; int *bpf_kfunc_call_test_get_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size) __ksym; int *bpf_kfunc_call_test_acq_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size) __ksym; @@ -111,6 +120,10 @@ u32 bpf_kfunc_call_test_static_unused_arg(u32 arg, u32 unused) __ksym; #endif void bpf_testmod_test_mod_kfunc(int i) __ksym; +int bpf_testmod_ops3_call_test_arena(__u64 *ptr__arena) __ksym; +int bpf_testmod_ops3_call_test_arena_nullable(__u64 *ptr__arena__nullable) __ksym; +int bpf_testmod_ops3_call_test_arena_stack(__u64 *ptr__arena) __ksym; +int bpf_testmod_ops3_call_test_arena_multislot(__u64 *ptr__arena) __ksym; __u64 bpf_kfunc_call_test1(struct sock *sk, __u32 a, __u64 b, __u32 c, __u64 d) __ksym; diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 3ce32d134e2c..07807757b518 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -807,7 +807,7 @@ static void verify_stderr(int prog_fd, struct expected_msgs *msgs) return; buf = malloc(TEST_LOADER_LOG_BUF_SZ); - if (!ASSERT_OK_PTR(buf, "malloc")) + if (!ASSERT_NEQ(buf, NULL, "malloc")) return; ret = bpf_prog_stream_read(prog_fd, 2, buf, TEST_LOADER_LOG_BUF_SZ - 1, diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index 7ba82974ee78..46eb201b96a3 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -424,10 +424,12 @@ static void jsonw_write_log_message(json_writer_t *w, char *log_buf, size_t log_ } } +/* @quiet elides the human readable output, the JSON report is unaffected */ static void dump_test_log(const struct prog_test_def *test, const struct test_state *test_state, bool skip_ok_subtests, bool par_exec_result, + bool quiet, json_writer_t *w) { bool test_failed = test_state->error_cnt > 0; @@ -449,7 +451,7 @@ static void dump_test_log(const struct prog_test_def *test, if (verbose() && !par_exec_result) return; - if (test_state->log_cnt && print_test) + if (test_state->log_cnt && print_test && !quiet) print_test_log(test_state->log_buf, test_state->log_cnt); if (w && print_test) { @@ -471,15 +473,16 @@ static void dump_test_log(const struct prog_test_def *test, if ((skip_ok_subtests && !subtest_failed) || subtest_filtered) continue; - if (subtest_state->log_cnt && print_subtest) { + if (subtest_state->log_cnt && print_subtest && !quiet) { print_test_log(subtest_state->log_buf, subtest_state->log_cnt); } - print_subtest_name(test->test_num, i + 1, - test->test_name, subtest_state->name, - test_result(subtest_state->error_cnt, - subtest_state->skipped)); + if (!quiet) + print_subtest_name(test->test_num, i + 1, + test->test_name, subtest_state->name, + test_result(subtest_state->error_cnt, + subtest_state->skipped)); if (w && print_subtest) { jsonw_start_object(w); @@ -496,7 +499,8 @@ static void dump_test_log(const struct prog_test_def *test, jsonw_end_object(w); } - print_test_result(test, test_state); + if (!quiet) + print_test_result(test, test_state); } /* A bunch of tests set custom affinity per-thread and/or per-process. Reset @@ -573,18 +577,19 @@ bool test__start_subtest_with_desc(const char *subtest_name, const char *subtest struct subtest_state *subtest_state; const char *subtest_display_name; size_t sub_state_size = sizeof(*subtest_state); + void *tmp; if (env.subtest_state) test__end_subtest(); state->subtest_num++; - state->subtest_states = - realloc(state->subtest_states, - state->subtest_num * sub_state_size); - if (!state->subtest_states) { + tmp = realloc(state->subtest_states, state->subtest_num * sub_state_size); + if (!tmp) { + state->subtest_num--; fprintf(stderr, "Not enough memory to allocate subtest result\n"); return false; } + state->subtest_states = tmp; subtest_state = &state->subtest_states[state->subtest_num - 1]; @@ -730,11 +735,14 @@ int compare_map_keys(int map1_fd, int map2_fd) int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len) { __u32 key, next_key, *cur_key_p, *next_key_p; - char *val_buf1, *val_buf2; - int i, err = 0; + char *val_buf1 = NULL, *val_buf2 = NULL; + int i, err = -ENOMEM; val_buf1 = malloc(stack_trace_len); val_buf2 = malloc(stack_trace_len); + if (!val_buf1 || !val_buf2) + goto out; + err = 0; cur_key_p = NULL; next_key_p = &key; while (bpf_map_get_next_key(smap_fd, cur_key_p, next_key_p) == 0) { @@ -895,6 +903,7 @@ enum ARG_KEYS { ARG_JSON_SUMMARY = 'J', ARG_TRAFFIC_MONITOR = 'm', ARG_WATCHDOG_TIMEOUT = 'w', + ARG_NO_ERROR_SUMMARY = -2, }; static const struct argp_option opts[] = { @@ -927,6 +936,8 @@ static const struct argp_option opts[] = { #endif { "watchdog-timeout", ARG_WATCHDOG_TIMEOUT, "SECONDS", 0, "Kill the process if tests are not making progress for specified number of seconds." }, + { "no-error-summary", ARG_NO_ERROR_SUMMARY, NULL, 0, + "Do not re-print the aggregated error logs of failed tests at the end of the run." }, {}, }; @@ -1128,6 +1139,9 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) case ARG_DEBUG: env->debug = true; break; + case ARG_NO_ERROR_SUMMARY: + env->error_summary = false; + break; case ARG_JSON_SUMMARY: env->json = fopen(arg, "w"); if (env->json == NULL) { @@ -1300,7 +1314,7 @@ static void dump_crash_log(void) if (env.test) { env.test_state->error_cnt++; - dump_test_log(env.test, env.test_state, true, false, NULL); + dump_test_log(env.test, env.test_state, true, false, false, NULL); } } @@ -1458,7 +1472,7 @@ static void run_one_test(int test_num) free(stop_libbpf_log_capture()); - dump_test_log(test, state, false, false, NULL); + dump_test_log(test, state, false, false, false, NULL); } struct dispatch_data { @@ -1513,13 +1527,15 @@ static int dispatch_thread_send_subtests(int sock_fd, struct test_state *state) struct subtest_state *subtest_state; int subtest_num = state->subtest_num; - state->subtest_states = malloc(subtest_num * sizeof(*subtest_state)); + state->subtest_states = calloc(subtest_num, sizeof(*subtest_state)); + if (!state->subtest_states) { + state->subtest_num = 0; + return -ENOMEM; + } for (int i = 0; i < subtest_num; i++) { subtest_state = &state->subtest_states[i]; - memset(subtest_state, 0, sizeof(*subtest_state)); - if (read_prog_test_msg(sock_fd, &msg, MSG_SUBTEST_DONE)) return 1; @@ -1617,7 +1633,7 @@ static void *dispatch_thread(void *ctx) } while (false); pthread_mutex_lock(&stdout_output_lock); - dump_test_log(test, state, false, true, NULL); + dump_test_log(test, state, false, true, false, NULL); pthread_mutex_unlock(&stdout_output_lock); } /* while (true) */ error: @@ -1640,8 +1656,8 @@ done: static void calculate_summary_and_print_errors(struct test_env *env) { - int i; - int succ_cnt = 0, fail_cnt = 0, sub_succ_cnt = 0, skip_cnt = 0; + int i, j; + int succ_cnt = 0, fail_cnt = 0, sub_succ_cnt = 0, sub_fail_cnt = 0, skip_cnt = 0; json_writer_t *w = NULL; for (i = 0; i < prog_test_cnt; i++) { @@ -1654,10 +1670,14 @@ static void calculate_summary_and_print_errors(struct test_env *env) sub_succ_cnt += state->sub_succ_cnt; skip_cnt += state->skip_cnt; - if (state->error_cnt) + if (state->error_cnt) { fail_cnt++; - else if (!test->not_built) + for (j = 0; j < state->subtest_num; j++) + if (state->subtest_states[j].error_cnt) + sub_fail_cnt++; + } else if (!test->not_built) { succ_cnt++; + } } if (env->json) { @@ -1672,6 +1692,7 @@ static void calculate_summary_and_print_errors(struct test_env *env) jsonw_uint_field(w, "success_subtest", sub_succ_cnt); jsonw_uint_field(w, "skipped", skip_cnt); jsonw_uint_field(w, "failed", fail_cnt); + jsonw_uint_field(w, "failed_subtest", sub_fail_cnt); jsonw_name(w, "results"); jsonw_start_array(w); } @@ -1680,9 +1701,14 @@ static void calculate_summary_and_print_errors(struct test_env *env) * We only print error logs summary when there are failed tests and * verbose mode is not enabled. Otherwise, results may be inconsistent. * + * --no-error-summary elides the human readable dump. The walk still + * happens when a JSON report was requested, so the JSON output keeps + * its per-test results; with no JSON report there is nothing left to + * do and the whole loop is skipped. */ - if (!verbose() && fail_cnt) { - printf("\nAll error logs:\n"); + if (!verbose() && fail_cnt && (env->error_summary || w)) { + if (env->error_summary) + printf("\nAll error logs:\n"); /* print error logs again */ for (i = 0; i < prog_test_cnt; i++) { @@ -1692,7 +1718,8 @@ static void calculate_summary_and_print_errors(struct test_env *env) if (!state->tested || !state->error_cnt) continue; - dump_test_log(test, state, true, true, w); + dump_test_log(test, state, true, true, + !env->error_summary, w); } } @@ -1706,12 +1733,12 @@ static void calculate_summary_and_print_errors(struct test_env *env) fclose(env->json); if (env->not_built_cnt) - printf("Summary: %d/%d PASSED, %d SKIPPED (%d not built), %d FAILED\n", + printf("Summary: %d/%d PASSED, %d SKIPPED (%d not built), %d/%d FAILED\n", succ_cnt, sub_succ_cnt, skip_cnt, env->not_built_cnt, - fail_cnt); + fail_cnt, sub_fail_cnt); else - printf("Summary: %d/%d PASSED, %d SKIPPED, %d FAILED\n", - succ_cnt, sub_succ_cnt, skip_cnt, fail_cnt); + printf("Summary: %d/%d PASSED, %d SKIPPED, %d/%d FAILED\n", + succ_cnt, sub_succ_cnt, skip_cnt, fail_cnt, sub_fail_cnt); env->succ_cnt = succ_cnt; env->sub_succ_cnt = sub_succ_cnt; @@ -1741,7 +1768,7 @@ static void server_main(void) data[i].worker_id = i; data[i].sock_fd = env.worker_socks[i]; rc = pthread_create(&dispatcher_threads[i], NULL, dispatch_thread, &data[i]); - if (rc < 0) { + if (rc) { perror("Failed to launch dispatcher thread"); exit(EXIT_ERR_SETUP_INFRA); } @@ -1886,7 +1913,6 @@ static int worker_main_send_subtests(int sock, struct test_state *state) worker_main_send_log(sock, subtest_state->log_buf, subtest_state->log_cnt); free_subtest_state(subtest_state); - free(subtest_state->name); } out: @@ -2023,6 +2049,7 @@ int main(int argc, char **argv) env.secs_till_notify = 10; env.secs_till_kill = 120; + env.error_summary = true; err = argp_parse(&argp, argc, argv, 0, NULL, &env); if (err) return err; diff --git a/tools/testing/selftests/bpf/test_progs.h b/tools/testing/selftests/bpf/test_progs.h index 2cf950afcd85..ea493c477fbd 100644 --- a/tools/testing/selftests/bpf/test_progs.h +++ b/tools/testing/selftests/bpf/test_progs.h @@ -105,6 +105,7 @@ struct test_env { struct test_selector tmon_selector; bool verifier_stats; bool debug; + bool error_summary; enum verbosity verbosity; bool jit_enabled; @@ -123,7 +124,7 @@ struct test_env { int succ_cnt; /* successful tests */ int sub_succ_cnt; /* successful sub-tests */ - int fail_cnt; /* total failed tests + sub-tests */ + int fail_cnt; /* failed tests */ int skip_cnt; /* skipped tests */ int not_built_cnt; /* tests not built */ diff --git a/tools/testing/selftests/bpf/test_sockmap.c b/tools/testing/selftests/bpf/test_sockmap.c index ac814eb63edb..aaf2050e8845 100644 --- a/tools/testing/selftests/bpf/test_sockmap.c +++ b/tools/testing/selftests/bpf/test_sockmap.c @@ -435,7 +435,8 @@ static int msg_alloc_iov(struct msghdr *msg, return 0; unwind_iov: for (i--; i >= 0 ; i--) - free(msg->msg_iov[i].iov_base); + free(iov[i].iov_base); + free(iov); return -ENOMEM; } diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index a8ae03c57bba..bffb7360434c 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -1560,7 +1560,7 @@ static void do_test_single(struct bpf_test *test, bool unpriv, opts.expected_attach_type = test->expected_attach_type; if (expected_ret == VERBOSE_ACCEPT) - opts.log_level = 2; + opts.log_level = 2 | 4; else if (verbose) opts.log_level = verif_log_level | 4; /* force stats */ else diff --git a/tools/testing/selftests/bpf/verifier/calls.c b/tools/testing/selftests/bpf/verifier/calls.c index 302d712e0d7e..eb6e3baef412 100644 --- a/tools/testing/selftests/bpf/verifier/calls.c +++ b/tools/testing/selftests/bpf/verifier/calls.c @@ -31,7 +31,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "R1 pointer type STRUCT prog_test_fail1 must point to scalar", + .errstr = "R1 is fp expected STRUCT prog_test_fail1", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail1", 2 }, }, @@ -46,7 +46,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "max struct nesting depth exceeded\nR1 pointer type STRUCT prog_test_fail2", + .errstr = "max struct nesting depth exceeded\nR1 is fp expected STRUCT prog_test_fail2", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail2", 2 }, }, @@ -61,7 +61,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "R1 pointer type STRUCT prog_test_fail3 must point to scalar", + .errstr = "R1 is fp expected STRUCT prog_test_fail3", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail3", 2 }, }, @@ -1091,7 +1091,17 @@ /* stack_main=32, stack_A=256, stack_B=64 * and max(main+A, main+A+B) < 512 */ - .result = ACCEPT, + .result = VERBOSE_ACCEPT, + .errstr = "stack depth max 352\t" + "subprog 0 (<unknown>) main insns_self \t" + " insns_total \t" + " stack 32\t" + "subprog 1 (<unknown>) static insns_self \t" + " insns_total \t" + " stack 256\t" + "subprog 2 (<unknown>) static insns_self \t" + " insns_total \t" + " stack 64", }, { "calls: stack depth check using three frames. test2", diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c index a7db6f04f7e1..e70741c6b9b7 100644 --- a/tools/testing/selftests/bpf/veristat.c +++ b/tools/testing/selftests/bpf/veristat.c @@ -514,6 +514,40 @@ cleanup: return err == 0; } +/* Exact filter match */ +static bool name_filter_matches(struct filter *f, const char *filename, const char *prog_name) +{ + if (f->any_glob) + return glob_matches(filename, f->any_glob) || + (prog_name && glob_matches(prog_name, f->any_glob)); + if (f->file_glob && f->prog_glob) + return prog_name && + glob_matches(filename, f->file_glob) && + glob_matches(prog_name, f->prog_glob); + if (f->file_glob) + return glob_matches(filename, f->file_glob); + if (f->prog_glob) + return prog_name && glob_matches(prog_name, f->prog_glob); + return false; +} + +/* Check if the filter does not outright reject the file name */ +static bool name_filter_may_match(struct filter *f, const char *filename) +{ + if (f->file_glob) + return glob_matches(filename, f->file_glob); + /* + * If we don't know program name yet, any_glob filter + * has to assume that current BPF object file might be + * relevant; we'll check again later on after opening + * BPF object file, at which point program name will + * be known finally. + */ + if (f->any_glob || f->prog_glob) + return true; + return false; +} + static bool should_process_file_prog(const char *filename, const char *prog_name) { struct filter *f; @@ -521,16 +555,7 @@ static bool should_process_file_prog(const char *filename, const char *prog_name for (i = 0; i < env.deny_filter_cnt; i++) { f = &env.deny_filters[i]; - if (f->kind != FILTER_NAME) - continue; - - if (f->any_glob && glob_matches(filename, f->any_glob)) - return false; - if (f->any_glob && prog_name && glob_matches(prog_name, f->any_glob)) - return false; - if (f->file_glob && glob_matches(filename, f->file_glob)) - return false; - if (f->prog_glob && prog_name && glob_matches(prog_name, f->prog_glob)) + if (f->kind == FILTER_NAME && name_filter_matches(f, filename, prog_name)) return false; } @@ -540,24 +565,15 @@ static bool should_process_file_prog(const char *filename, const char *prog_name continue; allow_cnt++; - if (f->any_glob) { - if (glob_matches(filename, f->any_glob)) - return true; - /* If we don't know program name yet, any_glob filter - * has to assume that current BPF object file might be - * relevant; we'll check again later on after opening - * BPF object file, at which point program name will - * be known finally. - */ - if (!prog_name || glob_matches(prog_name, f->any_glob)) - return true; - } else { - if (f->file_glob && !glob_matches(filename, f->file_glob)) - continue; - if (f->prog_glob && prog_name && !glob_matches(prog_name, f->prog_glob)) - continue; + if (prog_name && name_filter_matches(f, filename, prog_name)) + return true; + /* + * If there is no prog_name and the file name is not blocked by + * the filter, allow to open the file. Afterwards there would be + * a second refining query with prog_name set. + */ + if (!prog_name && name_filter_may_match(f, filename)) return true; - } } /* if there are no file/prog name allow filters, allow all progs, @@ -703,6 +719,12 @@ static int append_filter(struct filter **filters, int *cnt, const char *str) } } + if ((!f->any_glob && !f->file_glob && !f->prog_glob) || + (f->any_glob && strcmp(f->any_glob, "") == 0)) { + fprintf(stderr, "Invalid filter: '%s'\n", str); + return -EINVAL; + } + *cnt += 1; return 0; } @@ -993,13 +1015,15 @@ static void free_verif_stats(struct verif_stats *stats, size_t stat_cnt) static char verif_log_buf[64 * 1024]; -#define MAX_PARSED_LOG_LINES 100 +/* Keep room for all 256 subprogram records and trailing statistics. */ +#define MAX_PARSED_LOG_LINES 300 static int parse_verif_log(char * const buf, size_t buf_sz, struct verif_stats *s) { const char *cur; - int pos, lines, sub_stack, cnt = 0; - char *state = NULL, *token, stack[512]; + long sub_stack; + int pos, lines, cnt = 0; + char *state = NULL, *token, stack[512] = {}; buf[buf_sz - 1] = '\0'; @@ -1025,11 +1049,24 @@ static int parse_verif_log(char * const buf, size_t buf_sz, struct verif_stats * &s->stats[MARK_READ_MAX_LEN])) continue; + /* + * New kernels emit one "subprog <id> (<name>) <kind>" record + * per subprogram with the stack depth at the end, while old + * kernels emit a single "stack depth <a+...+n> max <max>" + * line. Match both formats so veristat works against either + * kernel. + */ + if (sscanf(cur, "stack depth max %ld", &s->stats[MAX_STACK]) == 1) + continue; + if (sscanf(cur, "subprog %*d %*s %*s insns_self %*d insns_total %*d stack %ld", &sub_stack) == 1) { + s->stats[STACK] += sub_stack; + continue; + } if (2 == sscanf(cur, "stack depth %511s max %ld", stack, &s->stats[MAX_STACK])) continue; } while ((token = strtok_r(cnt++ ? NULL : stack, "+", &state))) { - if (sscanf(token, "%d", &sub_stack) == 0) + if (sscanf(token, "%ld", &sub_stack) == 0) break; s->stats[STACK] += sub_stack; } @@ -1248,6 +1285,29 @@ static void fixup_obj_maps(struct bpf_object *obj) /* fix up map size, if necessary */ switch (bpf_map__type(map)) { + /* + * if the verifier doesn't use max_entries + * then set to 1 to avoid -ENOMEM + */ + case BPF_MAP_TYPE_HASH: + case BPF_MAP_TYPE_PERCPU_HASH: + case BPF_MAP_TYPE_LRU_HASH: + case BPF_MAP_TYPE_LRU_PERCPU_HASH: + case BPF_MAP_TYPE_SOCKHASH: + case BPF_MAP_TYPE_DEVMAP_HASH: + case BPF_MAP_TYPE_QUEUE: + case BPF_MAP_TYPE_STACK: + case BPF_MAP_TYPE_BLOOM_FILTER: + case BPF_MAP_TYPE_STACK_TRACE: + bpf_map__set_max_entries(map, 1); + break; + + /* ringbufs must be page-aligned */ + case BPF_MAP_TYPE_RINGBUF: + case BPF_MAP_TYPE_USER_RINGBUF: + bpf_map__set_max_entries(map, sysconf(_SC_PAGESIZE)); + break; + case BPF_MAP_TYPE_SK_STORAGE: case BPF_MAP_TYPE_TASK_STORAGE: case BPF_MAP_TYPE_INODE_STORAGE: diff --git a/tools/testing/selftests/bpf/vmtest.sh b/tools/testing/selftests/bpf/vmtest.sh index 9ca802285393..6a3d026d76bd 100755 --- a/tools/testing/selftests/bpf/vmtest.sh +++ b/tools/testing/selftests/bpf/vmtest.sh @@ -428,8 +428,17 @@ main() if [[ $# -eq 0 && "${debug_shell}" == "no" ]]; then echo "No command specified, will run ${DEFAULT_COMMAND} in the vm" - else - command="$@" + elif [[ $# -gt 0 ]]; then + # Quote each argument so the command survives into the guest: the + # host expands ${command} into the generated init script, which + # the guest bash then parses as shell source. Without the %q + # escapes an argument with a space or a glob (e.g. -a 'verifier_*') + # is re-split and expanded against /root/bpf there. + # + # Skip this when there is no command: printf '%q ' would still + # apply the format once and emit '', which is not the empty + # command that -s (debug shell) expects. + command=$(printf '%q ' "$@") fi local kconfig_file="${OUTPUT_DIR}/latest.config" |