diff options
Diffstat (limited to 'tools/lib')
51 files changed, 4152 insertions, 1096 deletions
diff --git a/tools/lib/api/Makefile b/tools/lib/api/Makefile index 8665c799e0fa..a228fdb5adba 100644 --- a/tools/lib/api/Makefile +++ b/tools/lib/api/Makefile @@ -9,10 +9,6 @@ srctree := $(patsubst %/,%,$(dir $(srctree))) #$(info Determined 'srctree' to be $(srctree)) endif -CC ?= $(CROSS_COMPILE)gcc -AR ?= $(CROSS_COMPILE)ar -LD ?= $(CROSS_COMPILE)ld - MAKEFLAGS += --no-print-directory INSTALL = install diff --git a/tools/lib/api/fd/array.c b/tools/lib/api/fd/array.c index f0f195207fca..67b73481df27 100644 --- a/tools/lib/api/fd/array.c +++ b/tools/lib/api/fd/array.c @@ -31,7 +31,8 @@ int fdarray__grow(struct fdarray *fda, int nr) priv = realloc(fda->priv, psize); if (priv == NULL) { - free(entries); + /* this will be freed by fdarray__exit() */ + fda->entries = entries; return -ENOMEM; } @@ -50,7 +51,7 @@ struct fdarray *fdarray__new(int nr_alloc, int nr_autogrow) if (fda != NULL) { if (fdarray__grow(fda, nr_alloc)) { - free(fda); + fdarray__delete(fda); fda = NULL; } else { fda->nr_autogrow = nr_autogrow; @@ -115,6 +116,9 @@ int fdarray__filter(struct fdarray *fda, short revents, return 0; for (fd = 0; fd < fda->nr; ++fd) { + if (fda->priv[fd].flags & fdarray_flag__nonfilterable) + continue; + if (!fda->entries[fd].events) continue; @@ -122,12 +126,17 @@ int fdarray__filter(struct fdarray *fda, short revents, if (entry_destructor) entry_destructor(fda, fd, arg); + /* + * Set fd to -1 so poll() ignores this entry; otherwise + * POLLHUP/POLLERR are still reported for events=0 fds + * (POSIX: always checked), causing a poll storm. + */ + fda->entries[fd].fd = -1; fda->entries[fd].revents = fda->entries[fd].events = 0; continue; } - if (!(fda->priv[fd].flags & fdarray_flag__nonfilterable)) - ++nr; + ++nr; } return nr; diff --git a/tools/lib/api/fs/fs.c b/tools/lib/api/fs/fs.c index edec23406dbc..cbd8eab0d1df 100644 --- a/tools/lib/api/fs/fs.c +++ b/tools/lib/api/fs/fs.c @@ -261,8 +261,8 @@ static const char *mount_overload(struct fs *fs) /* "PERF_" + name + "_ENVIRONMENT" + '\0' */ char upper_name[5 + name_len + 12 + 1]; - snprintf(upper_name, name_len, "PERF_%s_ENVIRONMENT", fs->name); - mem_toupper(upper_name, name_len); + snprintf(upper_name, sizeof(upper_name), "PERF_%s_ENVIRONMENT", fs->name); + mem_toupper(upper_name, strlen(upper_name)); return getenv(upper_name) ?: *fs->mounts; } @@ -294,11 +294,14 @@ int filename__read_int(const char *filename, int *value) { char line[64]; int fd = open(filename, O_RDONLY), err = -1; + ssize_t n; if (fd < 0) return -errno; - if (read(fd, line, sizeof(line)) > 0) { + n = read(fd, line, sizeof(line) - 1); + if (n > 0) { + line[n] = '\0'; *value = atoi(line); err = 0; } @@ -312,11 +315,14 @@ static int filename__read_ull_base(const char *filename, { char line[64]; int fd = open(filename, O_RDONLY), err = -1; + ssize_t n; if (fd < 0) return -errno; - if (read(fd, line, sizeof(line)) > 0) { + n = read(fd, line, sizeof(line) - 1); + if (n > 0) { + line[n] = '\0'; *value = strtoull(line, NULL, base); if (*value != ULLONG_MAX) err = 0; @@ -370,12 +376,13 @@ int filename__write_int(const char *filename, int value) { int fd = open(filename, O_WRONLY), err = -1; char buf[64]; + int len; if (fd < 0) return -errno; - sprintf(buf, "%d", value); - if (write(fd, buf, sizeof(buf)) == sizeof(buf)) + len = sprintf(buf, "%d", value); + if (write(fd, buf, len) == len) err = 0; close(fd); diff --git a/tools/lib/bitmap.c b/tools/lib/bitmap.c index aa83d22c45e3..fedc9070f0e4 100644 --- a/tools/lib/bitmap.c +++ b/tools/lib/bitmap.c @@ -169,3 +169,13 @@ bool __bitmap_subset(const unsigned long *bitmap1, return false; return true; } + +void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int bits) +{ + unsigned int k; + unsigned int nr = BITS_TO_LONGS(bits); + + for (k = 0; k < nr; k++) + dst[k] = bitmap1[k] ^ bitmap2[k]; +} diff --git a/tools/lib/bpf/Makefile b/tools/lib/bpf/Makefile index 168140f8e646..269fe21fc8da 100644 --- a/tools/lib/bpf/Makefile +++ b/tools/lib/bpf/Makefile @@ -49,6 +49,14 @@ man_dir_SQ = '$(subst ','\'',$(man_dir))' export man_dir man_dir_SQ INSTALL export DESTDIR DESTDIR_SQ +# Defer assigning EXTRA_CFLAGS to CFLAGS until after including +# tools/scripts/Makefile.include, as it may add flags to EXTRA_CFLAGS. +ifdef EXTRA_CFLAGS + CFLAGS := +else + CFLAGS := -g -O2 +endif + include $(srctree)/tools/scripts/Makefile.include # copy a bit from Linux kbuild @@ -70,13 +78,6 @@ LIB_TARGET = libbpf.a libbpf.so.$(LIBBPF_VERSION) LIB_FILE = libbpf.a libbpf.so* PC_FILE = libbpf.pc -# Set compile option CFLAGS -ifdef EXTRA_CFLAGS - CFLAGS := $(EXTRA_CFLAGS) -else - CFLAGS := -g -O2 -endif - # Append required CFLAGS override CFLAGS += -std=gnu89 override CFLAGS += $(EXTRA_WARNINGS) -Wno-switch-enum @@ -84,7 +85,7 @@ override CFLAGS += -Werror -Wall override CFLAGS += $(INCLUDES) override CFLAGS += -fvisibility=hidden override CFLAGS += -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -override CFLAGS += $(CLANG_CROSS_FLAGS) +override CFLAGS += $(EXTRA_CFLAGS) # flags specific for shared library SHLIB_FLAGS := -DSHARED -fPIC @@ -114,12 +115,12 @@ PC_FILE := $(addprefix $(OUTPUT),$(PC_FILE)) TAGS_PROG := $(if $(shell which etags 2>/dev/null),etags,ctags) -GLOBAL_SYM_COUNT = $(shell readelf -s --wide $(BPF_IN_SHARED) | \ +GLOBAL_SYM_COUNT = $(shell $(HOSTREADELF) -s --wide $(BPF_IN_SHARED) | \ cut -d "@" -f1 | sed 's/_v[0-9]_[0-9]_[0-9].*//' | \ sed 's/\[.*\]//' | \ awk '/GLOBAL/ && /DEFAULT/ && !/UND|ABS/ {print $$NF}' | \ sort -u | wc -l) -VERSIONED_SYM_COUNT = $(shell readelf --dyn-syms --wide $(OUTPUT)libbpf.so | \ +VERSIONED_SYM_COUNT = $(shell $(HOSTREADELF) --dyn-syms --wide $(OUTPUT)libbpf.so | \ sed 's/\[.*\]//' | \ awk '/GLOBAL/ && /DEFAULT/ && !/UND|ABS/ {print $$NF}' | \ grep -Eo '[^ ]+@LIBBPF_' | cut -d@ -f1 | sort -u | wc -l) @@ -182,12 +183,12 @@ check_abi: $(OUTPUT)libbpf.so $(VERSION_SCRIPT) "versioned symbols in $^ ($(VERSIONED_SYM_COUNT))." \ "Please make sure all LIBBPF_API symbols are" \ "versioned in $(VERSION_SCRIPT)." >&2; \ - readelf -s --wide $(BPF_IN_SHARED) | \ + $(HOSTREADELF) -s --wide $(BPF_IN_SHARED) | \ cut -d "@" -f1 | sed 's/_v[0-9]_[0-9]_[0-9].*//' | \ sed 's/\[.*\]//' | \ awk '/GLOBAL/ && /DEFAULT/ && !/UND/ {print $$NF}'| \ sort -u > $(OUTPUT)libbpf_global_syms.tmp; \ - readelf --dyn-syms --wide $(OUTPUT)libbpf.so | \ + $(HOSTREADELF) --dyn-syms --wide $(OUTPUT)libbpf.so | \ sed 's/\[.*\]//' | \ awk '/GLOBAL/ && /DEFAULT/ && !/UND|ABS/ {print $$NF}'| \ grep -Eo '[^ ]+@LIBBPF_' | cut -d@ -f1 | \ diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index 5846de364209..96819c082c77 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -59,6 +59,8 @@ # define __NR_bpf 6319 # elif defined(__mips__) && defined(_ABI64) # define __NR_bpf 5315 +# elif defined(__loongarch__) +# define __NR_bpf 280 # else # error __NR_bpf not defined. libbpf does not support your arch. # endif @@ -69,6 +71,42 @@ static inline __u64 ptr_to_u64(const void *ptr) return (__u64) (unsigned long) ptr; } +static inline int sys_bpf_ext(enum bpf_cmd cmd, union bpf_attr *attr, + unsigned int size, + struct bpf_common_attr *attr_common, + unsigned int size_common) +{ + cmd = attr_common ? (cmd | BPF_COMMON_ATTRS) : (cmd & ~BPF_COMMON_ATTRS); + return syscall(__NR_bpf, cmd, attr, size, attr_common, size_common); +} + +static inline int sys_bpf_ext_fd(enum bpf_cmd cmd, union bpf_attr *attr, + unsigned int size, + struct bpf_common_attr *attr_common, + unsigned int size_common) +{ + int fd; + + fd = sys_bpf_ext(cmd, attr, size, attr_common, size_common); + return ensure_good_fd(fd); +} + +int probe_sys_bpf_ext(void) +{ + const size_t attr_sz = offsetofend(union bpf_attr, prog_token_fd); + union bpf_attr attr; + int fd; + + memset(&attr, 0, attr_sz); + fd = syscall(__NR_bpf, BPF_PROG_LOAD | BPF_COMMON_ATTRS, &attr, attr_sz, NULL, + sizeof(struct bpf_common_attr)); + if (fd >= 0) { + close(fd); + return -EINVAL; + } + return errno == EFAULT ? 1 : 0; +} + static inline int sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr, unsigned int size) { @@ -173,6 +211,9 @@ int bpf_map_create(enum bpf_map_type map_type, const struct bpf_map_create_opts *opts) { const size_t attr_sz = offsetofend(union bpf_attr, excl_prog_hash_size); + const size_t attr_common_sz = sizeof(struct bpf_common_attr); + struct bpf_common_attr attr_common; + struct bpf_log_opts *log_opts; union bpf_attr attr; int fd; @@ -206,7 +247,21 @@ int bpf_map_create(enum bpf_map_type map_type, attr.excl_prog_hash = ptr_to_u64(OPTS_GET(opts, excl_prog_hash, NULL)); attr.excl_prog_hash_size = OPTS_GET(opts, excl_prog_hash_size, 0); - fd = sys_bpf_fd(BPF_MAP_CREATE, &attr, attr_sz); + log_opts = OPTS_GET(opts, log_opts, NULL); + if (!OPTS_VALID(log_opts, bpf_log_opts)) + return libbpf_err(-EINVAL); + + if (log_opts && feat_supported(NULL, FEAT_BPF_SYSCALL_COMMON_ATTRS)) { + memset(&attr_common, 0, attr_common_sz); + attr_common.log_buf = ptr_to_u64(OPTS_GET(log_opts, buf, NULL)); + attr_common.log_size = OPTS_GET(log_opts, size, 0); + attr_common.log_level = OPTS_GET(log_opts, level, 0); + fd = sys_bpf_ext_fd(BPF_MAP_CREATE, &attr, attr_sz, &attr_common, attr_common_sz); + OPTS_SET(log_opts, true_size, attr_common.log_true_size); + } else { + fd = sys_bpf_fd(BPF_MAP_CREATE, &attr, attr_sz); + OPTS_SET(log_opts, true_size, 0); + } return libbpf_err_errno(fd); } @@ -787,9 +842,19 @@ int bpf_link_create(int prog_fd, int target_fd, attr.link_create.uprobe_multi.ref_ctr_offsets = ptr_to_u64(OPTS_GET(opts, uprobe_multi.ref_ctr_offsets, 0)); attr.link_create.uprobe_multi.cookies = ptr_to_u64(OPTS_GET(opts, uprobe_multi.cookies, 0)); attr.link_create.uprobe_multi.pid = OPTS_GET(opts, uprobe_multi.pid, 0); + attr.link_create.uprobe_multi.path_fd = OPTS_GET(opts, uprobe_multi.path_fd, 0); if (!OPTS_ZEROED(opts, uprobe_multi)) return libbpf_err(-EINVAL); break; + case BPF_TRACE_FENTRY_MULTI: + case BPF_TRACE_FEXIT_MULTI: + case BPF_TRACE_FSESSION_MULTI: + attr.link_create.tracing_multi.ids = ptr_to_u64(OPTS_GET(opts, tracing_multi.ids, 0)); + attr.link_create.tracing_multi.cookies = ptr_to_u64(OPTS_GET(opts, tracing_multi.cookies, 0)); + attr.link_create.tracing_multi.cnt = OPTS_GET(opts, tracing_multi.cnt, 0); + if (!OPTS_ZEROED(opts, tracing_multi)) + return libbpf_err(-EINVAL); + break; case BPF_TRACE_RAW_TP: case BPF_TRACE_FENTRY: case BPF_TRACE_FEXIT: diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 2c8e88ddb674..7534a593edae 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -37,6 +37,18 @@ extern "C" { LIBBPF_API int libbpf_set_memlock_rlim(size_t memlock_bytes); +struct bpf_log_opts { + size_t sz; /* size of this struct for forward/backward compatibility */ + + char *buf; + __u32 size; + __u32 level; + __u32 true_size; /* out parameter set by kernel */ + + size_t :0; +}; +#define bpf_log_opts__last_field true_size + struct bpf_map_create_opts { size_t sz; /* size of this struct for forward/backward compatibility */ @@ -57,9 +69,12 @@ struct bpf_map_create_opts { const void *excl_prog_hash; __u32 excl_prog_hash_size; + + struct bpf_log_opts *log_opts; + size_t :0; }; -#define bpf_map_create_opts__last_field excl_prog_hash_size +#define bpf_map_create_opts__last_field log_opts LIBBPF_API int bpf_map_create(enum bpf_map_type map_type, const char *map_name, @@ -429,6 +444,7 @@ struct bpf_link_create_opts { const unsigned long *ref_ctr_offsets; const __u64 *cookies; __u32 pid; + __u32 path_fd; } uprobe_multi; struct { __u64 cookie; @@ -454,10 +470,15 @@ struct bpf_link_create_opts { __u32 relative_id; __u64 expected_revision; } cgroup; + struct { + const __u32 *ids; + const __u64 *cookies; + __u32 cnt; + } tracing_multi; }; size_t :0; }; -#define bpf_link_create_opts__last_field uprobe_multi.pid +#define bpf_link_create_opts__last_field uprobe_multi.path_fd LIBBPF_API int bpf_link_create(int prog_fd, int target_fd, enum bpf_attach_type attach_type, diff --git a/tools/lib/bpf/bpf_gen_internal.h b/tools/lib/bpf/bpf_gen_internal.h index 49af4260b8e6..6c5ad6c55e8a 100644 --- a/tools/lib/bpf/bpf_gen_internal.h +++ b/tools/lib/bpf/bpf_gen_internal.h @@ -51,7 +51,6 @@ struct bpf_gen { __u32 nr_ksyms; int fd_array; int nr_fd_array; - int hash_insn_offset[SHA256_DWORD_SIZE]; }; void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps); @@ -66,7 +65,8 @@ void bpf_gen__prog_load(struct bpf_gen *gen, enum bpf_prog_type prog_type, const char *prog_name, const char *license, struct bpf_insn *insns, size_t insn_cnt, struct bpf_prog_load_opts *load_attr, int prog_idx); -void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *value, __u32 value_size); +void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *value, __u32 value_size, + __u64 flags); void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx); void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *name, enum bpf_attach_type type); void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 83fe79ffcb8f..8417de92d028 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -29,6 +29,36 @@ static struct btf_type btf_void; +/* + * Describe how kinds are laid out; some have a singular element following the "struct btf_type", + * some have BTF_INFO_VLEN(t->info) elements. Specify sizes for both. Flags are currently unused. + * Kind layout can be optionally added to the BTF representation in a dedicated section to + * facilitate parsing. New kinds must be added here. + */ +static struct btf_layout layouts[NR_BTF_KINDS] = { +/* singular element size vlen element(s) size flags */ +[BTF_KIND_UNKN] = { 0, 0, 0 }, +[BTF_KIND_INT] = { sizeof(__u32), 0, 0 }, +[BTF_KIND_PTR] = { 0, 0, 0 }, +[BTF_KIND_ARRAY] = { sizeof(struct btf_array), 0, 0 }, +[BTF_KIND_STRUCT] = { 0, sizeof(struct btf_member), 0 }, +[BTF_KIND_UNION] = { 0, sizeof(struct btf_member), 0 }, +[BTF_KIND_ENUM] = { 0, sizeof(struct btf_enum), 0 }, +[BTF_KIND_FWD] = { 0, 0, 0 }, +[BTF_KIND_TYPEDEF] = { 0, 0, 0 }, +[BTF_KIND_VOLATILE] = { 0, 0, 0 }, +[BTF_KIND_CONST] = { 0, 0, 0 }, +[BTF_KIND_RESTRICT] = { 0, 0, 0 }, +[BTF_KIND_FUNC] = { 0, 0, 0 }, +[BTF_KIND_FUNC_PROTO] = { 0, sizeof(struct btf_param), 0 }, +[BTF_KIND_VAR] = { sizeof(struct btf_var), 0, 0 }, +[BTF_KIND_DATASEC] = { 0, sizeof(struct btf_var_secinfo), 0 }, +[BTF_KIND_FLOAT] = { 0, 0, 0 }, +[BTF_KIND_DECL_TAG] = { sizeof(struct btf_decl_tag), 0, 0 }, +[BTF_KIND_TYPE_TAG] = { 0, 0, 0 }, +[BTF_KIND_ENUM64] = { 0, sizeof(struct btf_enum64), 0 }, +}; + struct btf { /* raw BTF data in native endianness */ void *raw_data; @@ -40,42 +70,53 @@ struct btf { /* * When BTF is loaded from an ELF or raw memory it is stored - * in a contiguous memory block. The hdr, type_data, and, strs_data + * in a contiguous memory block. The type_data, layout and strs_data * point inside that memory region to their respective parts of BTF * representation: * - * +--------------------------------+ - * | Header | Types | Strings | - * +--------------------------------+ - * ^ ^ ^ - * | | | - * hdr | | - * types_data-+ | - * strs_data------------+ + * +----------------------------------------+---------------+ + * | Header | Types | Optional layout | Strings | + * +--------------------------------------------------------+ + * ^ ^ ^ ^ + * | | | | + * raw_data | | | + * types_data-+ | | + * layout---------------+ | + * strs_data--------------------------------+ + * + * A separate struct btf_header is embedded as btf->hdr, + * and header information is copied into it. This allows us + * to handle header data for various header formats; the original, + * the extended header with layout info, etc. * * If BTF data is later modified, e.g., due to types added or * removed, BTF deduplication performed, etc, this contiguous - * representation is broken up into three independently allocated - * memory regions to be able to modify them independently. + * representation is broken up into four independent memory + * regions. + * * raw_data is nulled out at that point, but can be later allocated * and cached again if user calls btf__raw_data(), at which point - * raw_data will contain a contiguous copy of header, types, and - * strings: + * raw_data will contain a contiguous copy of header, types, optional + * layout and strings. layout optionally points to a + * btf_layout array - this allows us to encode information about + * the kinds known at encoding time. If layout is NULL no + * layout information is encoded. * - * +----------+ +---------+ +-----------+ - * | Header | | Types | | Strings | - * +----------+ +---------+ +-----------+ - * ^ ^ ^ - * | | | - * hdr | | - * types_data----+ | - * strset__data(strs_set)-----+ + * +----------+ +---------+ +-----------+ +-----------+ + * | Header | | Types | | Layout | | Strings | + * +----------+ +---------+ +-----------+ +-----------+ + * ^ ^ ^ ^ + * | | | | + * hdr | | | + * types_data----+ | | + * layout---------------------+ | + * strset__data(strs_set)---------------------+ * - * +----------+---------+-----------+ - * | Header | Types | Strings | - * raw_data----->+----------+---------+-----------+ + * +----------+---------+-------------------+-----------+ + * | Header | Types | Optional Layout | Strings | + * raw_data----->+----------+---------+-------------------+-----------+ */ - struct btf_header *hdr; + struct btf_header hdr; void *types_data; size_t types_data_cap; /* used size stored in hdr->type_len */ @@ -125,6 +166,17 @@ struct btf { /* whether raw_data is a (read-only) mmap */ bool raw_data_is_mmap; + /* is BTF modifiable? i.e. is it split into separate sections as described above? */ + bool modifiable; + /* does BTF have header information we do not support? If so, disallow + * modification. + */ + bool has_hdr_extra; + /* Points either at raw kind layout data in parsed BTF (if present), or + * at an allocated kind layout array when BTF is modifiable. + */ + void *layout; + /* BTF object FD, if loaded into kernel */ int fd; @@ -216,7 +268,7 @@ static int btf_add_type_idx_entry(struct btf *btf, __u32 type_off) return 0; } -static void btf_bswap_hdr(struct btf_header *h) +static void btf_bswap_hdr(struct btf_header *h, __u32 hdr_len) { h->magic = bswap_16(h->magic); h->hdr_len = bswap_32(h->hdr_len); @@ -224,66 +276,115 @@ static void btf_bswap_hdr(struct btf_header *h) h->type_len = bswap_32(h->type_len); h->str_off = bswap_32(h->str_off); h->str_len = bswap_32(h->str_len); + /* May be operating on raw data with hdr_len that does not include below fields */ + if (hdr_len >= sizeof(struct btf_header)) { + h->layout_off = bswap_32(h->layout_off); + h->layout_len = bswap_32(h->layout_len); + } } static int btf_parse_hdr(struct btf *btf) { - struct btf_header *hdr = btf->hdr; - __u32 meta_left; + struct btf_header *hdr = btf->raw_data; + __u32 hdr_len, meta_left; - if (btf->raw_size < sizeof(struct btf_header)) { + if (btf->raw_size < offsetofend(struct btf_header, str_len)) { pr_debug("BTF header not found\n"); return -EINVAL; } + hdr_len = hdr->hdr_len; + if (hdr->magic == bswap_16(BTF_MAGIC)) { btf->swapped_endian = true; - if (bswap_32(hdr->hdr_len) != sizeof(struct btf_header)) { + hdr_len = bswap_32(hdr->hdr_len); + if (hdr_len < offsetofend(struct btf_header, str_len)) { pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n", - bswap_32(hdr->hdr_len)); + hdr_len); return -ENOTSUP; } - btf_bswap_hdr(hdr); } else if (hdr->magic != BTF_MAGIC) { pr_debug("Invalid BTF magic: %x\n", hdr->magic); return -EINVAL; } - if (btf->raw_size < hdr->hdr_len) { + if (btf->raw_size < hdr_len) { pr_debug("BTF header len %u larger than data size %u\n", - hdr->hdr_len, btf->raw_size); + hdr_len, btf->raw_size); return -EINVAL; } - meta_left = btf->raw_size - hdr->hdr_len; - if (meta_left < (long long)hdr->str_off + hdr->str_len) { + if (btf->swapped_endian) + btf_bswap_hdr(hdr, hdr_len); + + memcpy(&btf->hdr, hdr, min((size_t)hdr_len, sizeof(struct btf_header))); + + /* If unknown header data is found, modification is prohibited in + * btf_ensure_modifiable(). + */ + if (hdr_len > sizeof(struct btf_header)) { + __u8 *h = (__u8 *)hdr; + __u32 i; + + for (i = sizeof(struct btf_header); i < hdr_len; i++) { + if (!h[i]) + continue; + btf->has_hdr_extra = true; + pr_debug("Unknown BTF header data at offset %u; modification is disallowed\n", + i); + break; + } + } + + meta_left = btf->raw_size - hdr_len; + if (meta_left < (long long)btf->hdr.str_off + btf->hdr.str_len) { pr_debug("Invalid BTF total size: %u\n", btf->raw_size); return -EINVAL; } - if ((long long)hdr->type_off + hdr->type_len > hdr->str_off) { + if ((long long)btf->hdr.type_off + btf->hdr.type_len > btf->hdr.str_off) { pr_debug("Invalid BTF data sections layout: type data at %u + %u, strings data at %u + %u\n", - hdr->type_off, hdr->type_len, hdr->str_off, hdr->str_len); + btf->hdr.type_off, btf->hdr.type_len, btf->hdr.str_off, + btf->hdr.str_len); return -EINVAL; } - if (hdr->type_off % 4) { + if (btf->hdr.type_off % 4) { pr_debug("BTF type section is not aligned to 4 bytes\n"); return -EINVAL; } + if (btf->hdr.layout_len == 0) + return 0; + + /* optional layout section sits between types and strings */ + if (btf->hdr.layout_off % 4) { + pr_debug("BTF layout section is not aligned to 4 bytes\n"); + return -EINVAL; + } + if (btf->hdr.layout_off < (long long)btf->hdr.type_off + btf->hdr.type_len) { + pr_debug("Invalid BTF data sections layout: type data at %u + %u, layout data at %u + %u\n", + btf->hdr.type_off, btf->hdr.type_len, + btf->hdr.layout_off, btf->hdr.layout_len); + return -EINVAL; + } + if ((long long)btf->hdr.layout_off + btf->hdr.layout_len > btf->hdr.str_off || + btf->hdr.layout_off > btf->hdr.str_off) { + pr_debug("Invalid BTF data sections layout: layout data at %u + %u, strings data at %u\n", + btf->hdr.layout_off, btf->hdr.layout_len, btf->hdr.str_off); + return -EINVAL; + } return 0; } static int btf_parse_str_sec(struct btf *btf) { - const struct btf_header *hdr = btf->hdr; const char *start = btf->strs_data; - const char *end = start + btf->hdr->str_len; + const char *end = start + btf->hdr.str_len; - if (btf->base_btf && hdr->str_len == 0) + if (btf->base_btf && btf->hdr.str_len == 0) return 0; - if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET || end[-1]) { + if (!btf->hdr.str_len || btf->hdr.str_len - 1 > BTF_MAX_STR_OFFSET || end[-1]) { pr_debug("Invalid BTF string section\n"); return -EINVAL; } @@ -294,10 +395,66 @@ static int btf_parse_str_sec(struct btf *btf) return 0; } -static int btf_type_size(const struct btf_type *t) +static int btf_parse_layout_sec(struct btf *btf) +{ + if (!btf->hdr.layout_len) + return 0; + + if (btf->hdr.layout_len % sizeof(struct btf_layout) != 0) { + pr_debug("Invalid BTF kind layout section\n"); + return -EINVAL; + } + btf->layout = btf->raw_data + btf->hdr.hdr_len + btf->hdr.layout_off; + + if (btf->swapped_endian) { + struct btf_layout *l, *end = btf->layout + btf->hdr.layout_len; + + for (l = btf->layout; l < end; l++) + l->flags = bswap_16(l->flags); + } + + return 0; +} + +/* for unknown kinds, consult kind layout. */ +static int btf_type_size_unknown(const struct btf *btf, const struct btf_type *t) +{ + __u32 l_cnt = btf->hdr.layout_len / sizeof(struct btf_layout); + struct btf_layout *l = btf->layout; + __u32 vlen = btf_vlen(t); + __u32 kind = btf_kind(t); + + /* Fall back to base BTF if needed as they share layout information */ + if (!l) { + struct btf *base_btf = btf->base_btf; + + if (base_btf) { + l = base_btf->layout; + l_cnt = base_btf->hdr.layout_len / sizeof(struct btf_layout); + } + } + if (!l || kind >= l_cnt) { + pr_debug("Unsupported BTF_KIND: %u\n", btf_kind(t)); + return -EINVAL; + } + if (l[kind].info_sz % 4) { + pr_debug("Unsupported info_sz %u for kind %u\n", + l[kind].info_sz, kind); + return -EINVAL; + } + if (l[kind].elem_sz % 4) { + pr_debug("Unsupported elem_sz %u for kind %u\n", + l[kind].elem_sz, kind); + return -EINVAL; + } + + return sizeof(struct btf_type) + l[kind].info_sz + vlen * l[kind].elem_sz; +} + +static int btf_type_size(const struct btf *btf, const struct btf_type *t) { const int base_size = sizeof(struct btf_type); - __u16 vlen = btf_vlen(t); + __u32 vlen = btf_vlen(t); switch (btf_kind(t)) { case BTF_KIND_FWD: @@ -330,8 +487,7 @@ static int btf_type_size(const struct btf_type *t) case BTF_KIND_DECL_TAG: return base_size + sizeof(struct btf_decl_tag); default: - pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t)); - return -EINVAL; + return btf_type_size_unknown(btf, t); } } @@ -350,7 +506,7 @@ static int btf_bswap_type_rest(struct btf_type *t) struct btf_array *a; struct btf_param *p; struct btf_enum *e; - __u16 vlen = btf_vlen(t); + __u32 vlen = btf_vlen(t); int i; switch (btf_kind(t)) { @@ -421,20 +577,19 @@ static int btf_bswap_type_rest(struct btf_type *t) static int btf_parse_type_sec(struct btf *btf) { - struct btf_header *hdr = btf->hdr; void *next_type = btf->types_data; - void *end_type = next_type + hdr->type_len; + void *end_type = next_type + btf->hdr.type_len; int err, type_size; while (next_type + sizeof(struct btf_type) <= end_type) { if (btf->swapped_endian) btf_bswap_type_base(next_type); - type_size = btf_type_size(next_type); + type_size = btf_type_size(btf, next_type); if (type_size < 0) return type_size; if (next_type + type_size > end_type) { - pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types); + pr_warn("BTF type [%u] is malformed\n", btf->start_id + btf->nr_types); return -EINVAL; } @@ -591,8 +746,12 @@ static int btf_validate_type(const struct btf *btf, const struct btf_type *t, __ break; } default: - pr_warn("btf: type [%u]: unrecognized kind %u\n", id, kind); - return -EINVAL; + /* Kind may be represented in kind layout information. */ + if (btf_type_size_unknown(btf, t) < 0) { + pr_warn("btf: type [%u]: unrecognized kind %u\n", id, kind); + return -EINVAL; + } + break; } return 0; } @@ -848,7 +1007,7 @@ int btf__align_of(const struct btf *btf, __u32 id) case BTF_KIND_STRUCT: case BTF_KIND_UNION: { const struct btf_member *m = btf_members(t); - __u16 vlen = btf_vlen(t); + __u32 vlen = btf_vlen(t); int i, max_align = 1, align; for (i = 0; i < vlen; i++, m++) { @@ -1012,7 +1171,8 @@ __s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name, static bool btf_is_modifiable(const struct btf *btf) { - return (void *)btf->hdr != btf->raw_data; + /* BTF is modifiable if split into multiple sections */ + return btf->modifiable; } static void btf_free_raw_data(struct btf *btf) @@ -1036,14 +1196,14 @@ void btf__free(struct btf *btf) if (btf_is_modifiable(btf)) { /* if BTF was modified after loading, it will have a split - * in-memory representation for header, types, and strings + * in-memory representation for types, strings and layout * sections, so we need to free all of them individually. It * might still have a cached contiguous raw data present, * which will be unconditionally freed below. */ - free(btf->hdr); free(btf->types_data); strset__free(btf->strs_set); + free(btf->layout); } btf_free_raw_data(btf); free(btf->raw_data_swapped); @@ -1053,8 +1213,11 @@ void btf__free(struct btf *btf) free(btf); } -static struct btf *btf_new_empty(struct btf *base_btf) +static struct btf *btf_new_empty(struct btf_new_opts *opts) { + bool add_layout = OPTS_GET(opts, add_layout, false); + struct btf *base_btf = OPTS_GET(opts, base_btf, NULL); + struct btf_header *hdr; struct btf *btf; btf = calloc(1, sizeof(*btf)); @@ -1072,26 +1235,42 @@ static struct btf *btf_new_empty(struct btf *base_btf) if (base_btf) { btf->base_btf = base_btf; btf->start_id = btf__type_cnt(base_btf); - btf->start_str_off = base_btf->hdr->str_len + base_btf->start_str_off; + btf->start_str_off = base_btf->hdr.str_len + base_btf->start_str_off; btf->swapped_endian = base_btf->swapped_endian; } /* +1 for empty string at offset 0 */ btf->raw_size = sizeof(struct btf_header) + (base_btf ? 0 : 1); + if (add_layout) + btf->raw_size += sizeof(layouts); btf->raw_data = calloc(1, btf->raw_size); if (!btf->raw_data) { free(btf); return ERR_PTR(-ENOMEM); } - btf->hdr = btf->raw_data; - btf->hdr->hdr_len = sizeof(struct btf_header); - btf->hdr->magic = BTF_MAGIC; - btf->hdr->version = BTF_VERSION; + hdr = btf->raw_data; + hdr->hdr_len = sizeof(struct btf_header); + hdr->magic = BTF_MAGIC; + hdr->version = BTF_VERSION; + + btf->types_data = btf->raw_data + hdr->hdr_len; + btf->strs_data = btf->raw_data + hdr->hdr_len; + hdr->str_len = base_btf ? 0 : 1; /* empty string at offset 0 */ - btf->types_data = btf->raw_data + btf->hdr->hdr_len; - btf->strs_data = btf->raw_data + btf->hdr->hdr_len; - btf->hdr->str_len = base_btf ? 0 : 1; /* empty string at offset 0 */ + if (add_layout) { + hdr->layout_len = sizeof(layouts); + btf->layout = layouts; + /* + * No need to swap endianness here as btf_get_raw_data() + * will do this for us if btf->swapped_endian is true. + */ + memcpy(btf->raw_data + hdr->hdr_len, layouts, sizeof(layouts)); + btf->strs_data += sizeof(layouts); + hdr->str_off += sizeof(layouts); + } + + memcpy(&btf->hdr, hdr, sizeof(*hdr)); return btf; } @@ -1103,7 +1282,19 @@ struct btf *btf__new_empty(void) struct btf *btf__new_empty_split(struct btf *base_btf) { - return libbpf_ptr(btf_new_empty(base_btf)); + LIBBPF_OPTS(btf_new_opts, opts); + + opts.base_btf = base_btf; + + return libbpf_ptr(btf_new_empty(&opts)); +} + +struct btf *btf__new_empty_opts(struct btf_new_opts *opts) +{ + if (!OPTS_VALID(opts, btf_new_opts)) + return libbpf_err_ptr(-EINVAL); + + return libbpf_ptr(btf_new_empty(opts)); } static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf, bool is_mmap) @@ -1124,7 +1315,7 @@ static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf, b if (base_btf) { btf->base_btf = base_btf; btf->start_id = btf__type_cnt(base_btf); - btf->start_str_off = base_btf->hdr->str_len + base_btf->start_str_off; + btf->start_str_off = base_btf->hdr.str_len + base_btf->start_str_off; } if (is_mmap) { @@ -1141,15 +1332,15 @@ static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf, b btf->raw_size = size; - btf->hdr = btf->raw_data; err = btf_parse_hdr(btf); if (err) goto done; - btf->strs_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->str_off; - btf->types_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->type_off; + btf->strs_data = btf->raw_data + btf->hdr.hdr_len + btf->hdr.str_off; + btf->types_data = btf->raw_data + btf->hdr.hdr_len + btf->hdr.type_off; err = btf_parse_str_sec(btf); + err = err ?: btf_parse_layout_sec(btf); err = err ?: btf_parse_type_sec(btf); err = err ?: btf_sanity_check(btf); if (err) @@ -1233,7 +1424,7 @@ static int btf_find_elf_sections(Elf *elf, const char *path, struct btf_elf_secs continue; if (sh.sh_type != SHT_PROGBITS) { - pr_warn("unexpected section type (%d) of section(%d, %s) from %s\n", + pr_warn("unexpected section type (%u) of section(%d, %s) from %s\n", sh.sh_type, idx, name, path); goto err; } @@ -1315,9 +1506,6 @@ static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, dist_base_btf = NULL; } - if (dist_base_btf) - btf->owns_base = true; - switch (gelf_getclass(elf)) { case ELFCLASS32: btf__set_pointer_size(btf, 4); @@ -1332,13 +1520,16 @@ static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, if (btf_ext && secs.btf_ext_data) { *btf_ext = btf_ext__new(secs.btf_ext_data->d_buf, secs.btf_ext_data->d_size); - if (IS_ERR(*btf_ext)) { - err = PTR_ERR(*btf_ext); + if (!*btf_ext) { + err = -errno; goto done; } } else if (btf_ext) { *btf_ext = NULL; } + + if (dist_base_btf) + btf->owns_base = true; done: if (elf) elf_end(elf); @@ -1601,7 +1792,7 @@ static const void *btf_strs_data(const struct btf *btf) static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian) { - struct btf_header *hdr = btf->hdr; + const struct btf_header *hdr = &btf->hdr; struct btf_type *t; void *data, *p; __u32 data_sz; @@ -1614,14 +1805,17 @@ static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endi } data_sz = hdr->hdr_len + hdr->type_len + hdr->str_len; + if (btf->layout) + data_sz += hdr->layout_len; + data = calloc(1, data_sz); if (!data) return NULL; p = data; - memcpy(p, hdr, hdr->hdr_len); + memcpy(p, hdr, min((__u32)sizeof(struct btf_header), hdr->hdr_len)); if (swap_endian) - btf_bswap_hdr(p); + btf_bswap_hdr(p, hdr->hdr_len); p += hdr->hdr_len; memcpy(p, btf->types_data, hdr->type_len); @@ -1639,8 +1833,18 @@ static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endi } p += hdr->type_len; + if (btf->layout) { + memcpy(p, btf->layout, hdr->layout_len); + if (swap_endian) { + struct btf_layout *l, *end = p + hdr->layout_len; + + for (l = p; l < end ; l++) + l->flags = bswap_16(l->flags); + } + p += hdr->layout_len; + } + memcpy(p, btf_strs_data(btf), hdr->str_len); - p += hdr->str_len; *size = data_sz; return data; @@ -1675,7 +1879,7 @@ const char *btf__str_by_offset(const struct btf *btf, __u32 offset) { if (offset < btf->start_str_off) return btf__str_by_offset(btf->base_btf, offset); - else if (offset - btf->start_str_off < btf->hdr->str_len) + else if (offset - btf->start_str_off < btf->hdr.str_len) return btf_strs_data(btf) + (offset - btf->start_str_off); else return errno = EINVAL, NULL; @@ -1783,12 +1987,12 @@ static void btf_invalidate_raw_data(struct btf *btf) } /* Ensure BTF is ready to be modified (by splitting into a three memory - * regions for header, types, and strings). Also invalidate cached + * regions for types, strings and layout. Also invalidate cached * raw_data, if any. */ static int btf_ensure_modifiable(struct btf *btf) { - void *hdr, *types; + void *types, *layout = NULL; struct strset *set = NULL; int err = -ENOMEM; @@ -1798,45 +2002,58 @@ static int btf_ensure_modifiable(struct btf *btf) return 0; } - /* split raw data into three memory regions */ - hdr = malloc(btf->hdr->hdr_len); - types = malloc(btf->hdr->type_len); - if (!hdr || !types) + if (btf->has_hdr_extra) { + /* Additional BTF header data was found; not safe to modify. */ + return -EOPNOTSUPP; + } + + /* split raw data into memory regions; btf->hdr is done already. */ + types = malloc(btf->hdr.type_len); + if (!types) goto err_out; + memcpy(types, btf->types_data, btf->hdr.type_len); - memcpy(hdr, btf->hdr, btf->hdr->hdr_len); - memcpy(types, btf->types_data, btf->hdr->type_len); + if (btf->hdr.layout_len) { + layout = malloc(btf->hdr.layout_len); + if (!layout) + goto err_out; + memcpy(layout, btf->raw_data + btf->hdr.hdr_len + btf->hdr.layout_off, + btf->hdr.layout_len); + } /* build lookup index for all strings */ - set = strset__new(BTF_MAX_STR_OFFSET, btf->strs_data, btf->hdr->str_len); + set = strset__new(BTF_MAX_STR_OFFSET, btf->strs_data, btf->hdr.str_len); if (IS_ERR(set)) { err = PTR_ERR(set); goto err_out; } /* only when everything was successful, update internal state */ - btf->hdr = hdr; btf->types_data = types; - btf->types_data_cap = btf->hdr->type_len; + btf->types_data_cap = btf->hdr.type_len; btf->strs_data = NULL; btf->strs_set = set; + if (layout) + btf->layout = layout; /* if BTF was created from scratch, all strings are guaranteed to be * unique and deduplicated */ - if (btf->hdr->str_len == 0) + if (btf->hdr.str_len == 0) btf->strs_deduped = true; - if (!btf->base_btf && btf->hdr->str_len == 1) + if (!btf->base_btf && btf->hdr.str_len == 1) btf->strs_deduped = true; /* invalidate raw_data representation */ btf_invalidate_raw_data(btf); + btf->modifiable = true; + return 0; err_out: strset__free(set); - free(hdr); free(types); + free(layout); return err; } @@ -1849,6 +2066,7 @@ err_out: int btf__find_str(struct btf *btf, const char *s) { int off; + int err; if (btf->base_btf) { off = btf__find_str(btf->base_btf, s); @@ -1857,8 +2075,9 @@ int btf__find_str(struct btf *btf, const char *s) } /* BTF needs to be in a modifiable state to build string lookup index */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); off = strset__find_str(btf->strs_set, s); if (off < 0) @@ -1875,6 +2094,7 @@ int btf__find_str(struct btf *btf, const char *s) int btf__add_str(struct btf *btf, const char *s) { int off; + int err; if (btf->base_btf) { off = btf__find_str(btf->base_btf, s); @@ -1882,14 +2102,15 @@ int btf__add_str(struct btf *btf, const char *s) return off; } - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); off = strset__add_str(btf->strs_set, s); if (off < 0) return libbpf_err(off); - btf->hdr->str_len = strset__data_size(btf->strs_set); + btf->hdr.str_len = strset__data_size(btf->strs_set); return btf->start_str_off + off; } @@ -1897,24 +2118,42 @@ int btf__add_str(struct btf *btf, const char *s) static void *btf_add_type_mem(struct btf *btf, size_t add_sz) { return libbpf_add_mem(&btf->types_data, &btf->types_data_cap, 1, - btf->hdr->type_len, UINT_MAX, add_sz); + btf->hdr.type_len, UINT_MAX, add_sz); } -static void btf_type_inc_vlen(struct btf_type *t) +static int btf_type_inc_vlen(struct btf_type *t) { + if (btf_vlen(t) == BTF_MAX_VLEN) + return -ENOSPC; t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t)); + return 0; +} + +static void btf_hdr_update_type_len(struct btf *btf, int new_len) +{ + btf->hdr.type_len = new_len; + if (btf->layout) { + btf->hdr.layout_off = btf->hdr.type_off + new_len; + btf->hdr.str_off = btf->hdr.layout_off + btf->hdr.layout_len; + } else { + btf->hdr.str_off = btf->hdr.type_off + new_len; + } +} + +static void btf_hdr_update_str_len(struct btf *btf, int new_len) +{ + btf->hdr.str_len = new_len; } static int btf_commit_type(struct btf *btf, int data_sz) { int err; - err = btf_add_type_idx_entry(btf, btf->hdr->type_len); + err = btf_add_type_idx_entry(btf, btf->hdr.type_len); if (err) return libbpf_err(err); - btf->hdr->type_len += data_sz; - btf->hdr->str_off += data_sz; + btf_hdr_update_type_len(btf, btf->hdr.type_len + data_sz); btf->nr_types++; return btf->start_id + btf->nr_types - 1; } @@ -1963,13 +2202,14 @@ static int btf_add_type(struct btf_pipe *p, const struct btf_type *src_type) __u32 *str_off; int sz, err; - sz = btf_type_size(src_type); + sz = btf_type_size(p->src, src_type); if (sz < 0) return libbpf_err(sz); /* deconstruct BTF, if necessary, and invalidate raw_data */ - if (btf_ensure_modifiable(p->dst)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(p->dst); + if (err) + return libbpf_err(err); t = btf_add_type_mem(p->dst, sz); if (!t) @@ -2004,24 +2244,31 @@ int btf__add_btf(struct btf *btf, const struct btf *src_btf) { struct btf_pipe p = { .src = src_btf, .dst = btf }; int data_sz, sz, cnt, i, err, old_strs_len; + __u32 src_start_id; __u32 *off; void *t; - /* appending split BTF isn't supported yet */ - if (src_btf->base_btf) - return libbpf_err(-ENOTSUP); + /* + * When appending split BTF, the destination must share the same base + * BTF so that base type ID references remain valid. + */ + if (src_btf->base_btf && src_btf->base_btf != btf->base_btf) + return libbpf_err(-EOPNOTSUPP); + + src_start_id = src_btf->base_btf ? btf__type_cnt(src_btf->base_btf) : 1; /* deconstruct BTF, if necessary, and invalidate raw_data */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); /* remember original strings section size if we have to roll back * partial strings section changes */ - old_strs_len = btf->hdr->str_len; + old_strs_len = btf->hdr.str_len; - data_sz = src_btf->hdr->type_len; - cnt = btf__type_cnt(src_btf) - 1; + data_sz = src_btf->hdr.type_len; + cnt = src_btf->nr_types; /* pre-allocate enough memory for new types */ t = btf_add_type_mem(btf, data_sz); @@ -2045,7 +2292,7 @@ int btf__add_btf(struct btf *btf, const struct btf *src_btf) struct btf_field_iter it; __u32 *type_id, *str_off; - sz = btf_type_size(t); + sz = btf_type_size(src_btf, t); if (sz < 0) { /* unlikely, has to be corrupted src_btf */ err = sz; @@ -2060,6 +2307,9 @@ int btf__add_btf(struct btf *btf, const struct btf *src_btf) if (err) goto err_out; while ((str_off = btf_field_iter_next(&it))) { + /* don't remap strings from shared base BTF */ + if (*str_off < src_btf->start_str_off) + continue; err = btf_rewrite_str(&p, str_off); if (err) goto err_out; @@ -2074,11 +2324,11 @@ int btf__add_btf(struct btf *btf, const struct btf *src_btf) if (!*type_id) /* nothing to do for VOID references */ continue; - /* we haven't updated btf's type count yet, so - * btf->start_id + btf->nr_types - 1 is the type ID offset we should - * add to all newly added BTF types - */ - *type_id += btf->start_id + btf->nr_types - 1; + /* don't remap types from shared base BTF */ + if (*type_id < src_start_id) + continue; + + *type_id += btf->start_id + btf->nr_types - src_start_id; } /* go to next type data and type offset index entry */ @@ -2094,8 +2344,7 @@ int btf__add_btf(struct btf *btf, const struct btf *src_btf) * update type count and various internal offsets and sizes to * "commit" the changes and made them visible to the outside world. */ - btf->hdr->type_len += data_sz; - btf->hdr->str_off += data_sz; + btf_hdr_update_type_len(btf, btf->hdr.type_len + data_sz); btf->nr_types += cnt; hashmap__free(p.str_off_map); @@ -2106,13 +2355,14 @@ err_out: /* zero out preallocated memory as if it was just allocated with * libbpf_add_mem() */ - memset(btf->types_data + btf->hdr->type_len, 0, data_sz); - memset(btf->strs_data + old_strs_len, 0, btf->hdr->str_len - old_strs_len); + memset(btf->types_data + btf->hdr.type_len, 0, data_sz); + if (btf->strs_data) + memset(btf->strs_data + old_strs_len, 0, btf->hdr.str_len - old_strs_len); /* and now restore original strings section size; types data size * wasn't modified, so doesn't need restoring, see big comment above */ - btf->hdr->str_len = old_strs_len; + btf_hdr_update_str_len(btf, old_strs_len); hashmap__free(p.str_off_map); @@ -2132,6 +2382,7 @@ int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding { struct btf_type *t; int sz, name_off; + int err; /* non-empty name */ if (str_is_empty(name)) @@ -2143,8 +2394,9 @@ int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding return libbpf_err(-EINVAL); /* deconstruct BTF, if necessary, and invalidate raw_data */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); sz = sizeof(struct btf_type) + sizeof(int); t = btf_add_type_mem(btf, sz); @@ -2180,6 +2432,7 @@ int btf__add_float(struct btf *btf, const char *name, size_t byte_sz) { struct btf_type *t; int sz, name_off; + int err; /* non-empty name */ if (str_is_empty(name)) @@ -2190,8 +2443,9 @@ int btf__add_float(struct btf *btf, const char *name, size_t byte_sz) byte_sz != 16) return libbpf_err(-EINVAL); - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); sz = sizeof(struct btf_type); t = btf_add_type_mem(btf, sz); @@ -2225,12 +2479,14 @@ static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref { struct btf_type *t; int sz, name_off = 0; + int err; if (validate_type_id(ref_type_id)) return libbpf_err(-EINVAL); - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); sz = sizeof(struct btf_type); t = btf_add_type_mem(btf, sz); @@ -2275,13 +2531,15 @@ int btf__add_array(struct btf *btf, int index_type_id, int elem_type_id, __u32 n { struct btf_type *t; struct btf_array *a; + int err; int sz; if (validate_type_id(index_type_id) || validate_type_id(elem_type_id)) return libbpf_err(-EINVAL); - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); sz = sizeof(struct btf_type) + sizeof(struct btf_array); t = btf_add_type_mem(btf, sz); @@ -2305,9 +2563,11 @@ static int btf_add_composite(struct btf *btf, int kind, const char *name, __u32 { struct btf_type *t; int sz, name_off = 0; + int err; - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); sz = sizeof(struct btf_type); t = btf_add_type_mem(btf, sz); @@ -2387,6 +2647,7 @@ int btf__add_field(struct btf *btf, const char *name, int type_id, struct btf_member *m; bool is_bitfield; int sz, name_off = 0; + int err; /* last type should be union/struct */ if (btf->nr_types == 0) @@ -2394,6 +2655,8 @@ int btf__add_field(struct btf *btf, const char *name, int type_id, t = btf_last_type(btf); if (!btf_is_composite(t)) return libbpf_err(-EINVAL); + if (btf_vlen(t) == BTF_MAX_VLEN) + return libbpf_err(-ENOSPC); if (validate_type_id(type_id)) return libbpf_err(-EINVAL); @@ -2407,8 +2670,9 @@ int btf__add_field(struct btf *btf, const char *name, int type_id, return libbpf_err(-EINVAL); /* decompose and invalidate raw data */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); sz = sizeof(struct btf_member); m = btf_add_type_mem(btf, sz); @@ -2427,11 +2691,11 @@ int btf__add_field(struct btf *btf, const char *name, int type_id, /* btf_add_type_mem can invalidate t pointer */ t = btf_last_type(btf); + /* update parent type's vlen and kflag */ t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield || btf_kflag(t)); - btf->hdr->type_len += sz; - btf->hdr->str_off += sz; + btf_hdr_update_type_len(btf, btf->hdr.type_len + sz); return 0; } @@ -2440,13 +2704,15 @@ static int btf_add_enum_common(struct btf *btf, const char *name, __u32 byte_sz, { struct btf_type *t; int sz, name_off = 0; + int err; /* byte_sz must be power of 2 */ if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 8) return libbpf_err(-EINVAL); - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); sz = sizeof(struct btf_type); t = btf_add_type_mem(btf, sz); @@ -2502,6 +2768,7 @@ int btf__add_enum_value(struct btf *btf, const char *name, __s64 value) struct btf_type *t; struct btf_enum *v; int sz, name_off; + int err; /* last type should be BTF_KIND_ENUM */ if (btf->nr_types == 0) @@ -2517,8 +2784,9 @@ int btf__add_enum_value(struct btf *btf, const char *name, __s64 value) return libbpf_err(-E2BIG); /* decompose and invalidate raw data */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); sz = sizeof(struct btf_enum); v = btf_add_type_mem(btf, sz); @@ -2534,14 +2802,15 @@ int btf__add_enum_value(struct btf *btf, const char *name, __s64 value) /* update parent type's vlen */ t = btf_last_type(btf); - btf_type_inc_vlen(t); + err = btf_type_inc_vlen(t); + if (err) + return libbpf_err(err); /* if negative value, set signedness to signed */ if (value < 0) t->info = btf_type_info(btf_kind(t), btf_vlen(t), true); - btf->hdr->type_len += sz; - btf->hdr->str_off += sz; + btf_hdr_update_type_len(btf, btf->hdr.type_len + sz); return 0; } @@ -2579,6 +2848,7 @@ int btf__add_enum64_value(struct btf *btf, const char *name, __u64 value) struct btf_enum64 *v; struct btf_type *t; int sz, name_off; + int err; /* last type should be BTF_KIND_ENUM64 */ if (btf->nr_types == 0) @@ -2592,8 +2862,9 @@ int btf__add_enum64_value(struct btf *btf, const char *name, __u64 value) return libbpf_err(-EINVAL); /* decompose and invalidate raw data */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); sz = sizeof(struct btf_enum64); v = btf_add_type_mem(btf, sz); @@ -2610,10 +2881,11 @@ int btf__add_enum64_value(struct btf *btf, const char *name, __u64 value) /* update parent type's vlen */ t = btf_last_type(btf); - btf_type_inc_vlen(t); + err = btf_type_inc_vlen(t); + if (err) + return libbpf_err(err); - btf->hdr->type_len += sz; - btf->hdr->str_off += sz; + btf_hdr_update_type_len(btf, btf->hdr.type_len + sz); return 0; } @@ -2782,13 +3054,15 @@ int btf__add_func(struct btf *btf, const char *name, int btf__add_func_proto(struct btf *btf, int ret_type_id) { struct btf_type *t; + int err; int sz; if (validate_type_id(ret_type_id)) return libbpf_err(-EINVAL); - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); sz = sizeof(struct btf_type); t = btf_add_type_mem(btf, sz); @@ -2818,6 +3092,7 @@ int btf__add_func_param(struct btf *btf, const char *name, int type_id) struct btf_type *t; struct btf_param *p; int sz, name_off = 0; + int err; if (validate_type_id(type_id)) return libbpf_err(-EINVAL); @@ -2830,8 +3105,9 @@ int btf__add_func_param(struct btf *btf, const char *name, int type_id) return libbpf_err(-EINVAL); /* decompose and invalidate raw data */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); sz = sizeof(struct btf_param); p = btf_add_type_mem(btf, sz); @@ -2849,10 +3125,11 @@ int btf__add_func_param(struct btf *btf, const char *name, int type_id) /* update parent type's vlen */ t = btf_last_type(btf); - btf_type_inc_vlen(t); + err = btf_type_inc_vlen(t); + if (err) + return libbpf_err(err); - btf->hdr->type_len += sz; - btf->hdr->str_off += sz; + btf_hdr_update_type_len(btf, btf->hdr.type_len + sz); return 0; } @@ -2871,6 +3148,7 @@ int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id) struct btf_type *t; struct btf_var *v; int sz, name_off; + int err; /* non-empty name */ if (str_is_empty(name)) @@ -2882,8 +3160,9 @@ int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id) return libbpf_err(-EINVAL); /* deconstruct BTF, if necessary, and invalidate raw_data */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); sz = sizeof(struct btf_type) + sizeof(struct btf_var); t = btf_add_type_mem(btf, sz); @@ -2920,13 +3199,15 @@ int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz) { struct btf_type *t; int sz, name_off; + int err; /* non-empty name */ if (str_is_empty(name)) return libbpf_err(-EINVAL); - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); sz = sizeof(struct btf_type); t = btf_add_type_mem(btf, sz); @@ -2959,6 +3240,7 @@ int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __ { struct btf_type *t; struct btf_var_secinfo *v; + int err; int sz; /* last type should be BTF_KIND_DATASEC */ @@ -2972,8 +3254,9 @@ int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __ return libbpf_err(-EINVAL); /* decompose and invalidate raw data */ - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); sz = sizeof(struct btf_var_secinfo); v = btf_add_type_mem(btf, sz); @@ -2986,10 +3269,11 @@ int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __ /* update parent type's vlen */ t = btf_last_type(btf); - btf_type_inc_vlen(t); + err = btf_type_inc_vlen(t); + if (err) + return libbpf_err(err); - btf->hdr->type_len += sz; - btf->hdr->str_off += sz; + btf_hdr_update_type_len(btf, btf->hdr.type_len + sz); return 0; } @@ -2998,6 +3282,7 @@ static int btf_add_decl_tag(struct btf *btf, const char *value, int ref_type_id, { struct btf_type *t; int sz, value_off; + int err; if (str_is_empty(value) || component_idx < -1) return libbpf_err(-EINVAL); @@ -3005,8 +3290,9 @@ static int btf_add_decl_tag(struct btf *btf, const char *value, int ref_type_id, if (validate_type_id(ref_type_id)) return libbpf_err(-EINVAL); - if (btf_ensure_modifiable(btf)) - return libbpf_err(-ENOMEM); + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); sz = sizeof(struct btf_type) + sizeof(struct btf_decl_tag); t = btf_add_type_mem(btf, sz); @@ -3630,10 +3916,9 @@ int btf__dedup(struct btf *btf, const struct btf_dedup_opts *opts) return libbpf_err(-EINVAL); } - if (btf_ensure_modifiable(btf)) { - err = -ENOMEM; + err = btf_ensure_modifiable(btf); + if (err) goto done; - } err = btf_dedup_prep(d); if (err) { @@ -3953,7 +4238,7 @@ static int btf_dedup_strings(struct btf_dedup *d) /* replace BTF string data and hash with deduped ones */ strset__free(d->btf->strs_set); - d->btf->hdr->str_len = strset__data_size(d->strs_set); + btf_hdr_update_str_len(d->btf, strset__data_size(d->strs_set)); d->btf->strs_set = d->strs_set; d->strs_set = NULL; d->btf->strs_deduped = true; @@ -4040,7 +4325,7 @@ static long btf_hash_enum(struct btf_type *t) static bool btf_equal_enum_members(struct btf_type *t1, struct btf_type *t2) { const struct btf_enum *m1, *m2; - __u16 vlen; + __u32 vlen; int i; vlen = btf_vlen(t1); @@ -4058,7 +4343,7 @@ static bool btf_equal_enum_members(struct btf_type *t1, struct btf_type *t2) static bool btf_equal_enum64_members(struct btf_type *t1, struct btf_type *t2) { const struct btf_enum64 *m1, *m2; - __u16 vlen; + __u32 vlen; int i; vlen = btf_vlen(t1); @@ -4135,7 +4420,7 @@ static long btf_hash_struct(struct btf_type *t) static bool btf_shallow_equal_struct(struct btf_type *t1, struct btf_type *t2) { const struct btf_member *m1, *m2; - __u16 vlen; + __u32 vlen; int i; if (!btf_equal_common(t1, t2)) @@ -4211,7 +4496,7 @@ static bool btf_compat_array(struct btf_type *t1, struct btf_type *t2) static long btf_hash_fnproto(struct btf_type *t) { const struct btf_param *member = btf_params(t); - __u16 vlen = btf_vlen(t); + __u32 vlen = btf_vlen(t); long h = btf_hash_common(t); int i; @@ -4233,7 +4518,7 @@ static long btf_hash_fnproto(struct btf_type *t) static bool btf_equal_fnproto(struct btf_type *t1, struct btf_type *t2) { const struct btf_param *m1, *m2; - __u16 vlen; + __u32 vlen; int i; if (!btf_equal_common(t1, t2)) @@ -4259,7 +4544,7 @@ static bool btf_equal_fnproto(struct btf_type *t1, struct btf_type *t2) static bool btf_compat_fnproto(struct btf_type *t1, struct btf_type *t2) { const struct btf_param *m1, *m2; - __u16 vlen; + __u32 vlen; int i; /* skip return type ID */ @@ -4307,12 +4592,14 @@ static int btf_dedup_prep(struct btf_dedup *d) case BTF_KIND_RESTRICT: case BTF_KIND_PTR: case BTF_KIND_FWD: - case BTF_KIND_TYPEDEF: case BTF_KIND_FUNC: case BTF_KIND_FLOAT: case BTF_KIND_TYPE_TAG: h = btf_hash_common(t); break; + case BTF_KIND_TYPEDEF: + h = btf_hash_typedef(t); + break; case BTF_KIND_INT: case BTF_KIND_DECL_TAG: h = btf_hash_int_decl_tag(t); @@ -4567,7 +4854,7 @@ recur: continue; if (!btf_dedup_identical_types(d, m1->type, m2->type, depth - 1)) { if (t1->name_off) { - pr_debug("%s '%s' size=%d vlen=%d id1[%u] id2[%u] shallow-equal but not identical for field#%d '%s'\n", + pr_debug("%s '%s' size=%u vlen=%u id1[%u] id2[%u] shallow-equal but not identical for field#%d '%s'\n", k1 == BTF_KIND_STRUCT ? "STRUCT" : "UNION", btf__name_by_offset(d->btf, t1->name_off), t1->size, btf_vlen(t1), id1, id2, i, @@ -4806,7 +5093,7 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, case BTF_KIND_STRUCT: case BTF_KIND_UNION: { const struct btf_member *cand_m, *canon_m; - __u16 vlen; + __u32 vlen; if (!btf_shallow_equal_struct(cand_type, canon_type)) return 0; @@ -4817,7 +5104,7 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type); if (eq <= 0) { if (cand_type->name_off) { - pr_debug("%s '%s' size=%d vlen=%d cand_id[%u] canon_id[%u] shallow-equal but not equiv for field#%d '%s': %d\n", + pr_debug("%s '%s' size=%u vlen=%u cand_id[%u] canon_id[%u] shallow-equal but not equiv for field#%d '%s': %d\n", cand_kind == BTF_KIND_STRUCT ? "STRUCT" : "UNION", btf__name_by_offset(d->btf, cand_type->name_off), cand_type->size, vlen, cand_id, canon_id, i, @@ -4834,7 +5121,7 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, case BTF_KIND_FUNC_PROTO: { const struct btf_param *cand_p, *canon_p; - __u16 vlen; + __u32 vlen; if (!btf_compat_fnproto(cand_type, canon_type)) return 0; @@ -5168,7 +5455,7 @@ static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) case BTF_KIND_FUNC_PROTO: { struct btf_param *param; - __u16 vlen; + __u32 vlen; int i; ref_type_id = btf_dedup_ref_type(d, t->type); @@ -5386,7 +5673,7 @@ static int btf_dedup_compact_types(struct btf_dedup *d) continue; t = btf__type_by_id(d->btf, id); - len = btf_type_size(t); + len = btf_type_size(d->btf, t); if (len < 0) return len; @@ -5400,14 +5687,17 @@ static int btf_dedup_compact_types(struct btf_dedup *d) /* shrink struct btf's internal types index and update btf_header */ d->btf->nr_types = next_type_id - d->btf->start_id; d->btf->type_offs_cap = d->btf->nr_types; - d->btf->hdr->type_len = p - d->btf->types_data; + d->btf->hdr.type_len = p - d->btf->types_data; new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap, sizeof(*new_offs)); if (d->btf->type_offs_cap && !new_offs) return -ENOMEM; d->btf->type_offs = new_offs; - d->btf->hdr->str_off = d->btf->hdr->type_len; - d->btf->raw_size = d->btf->hdr->hdr_len + d->btf->hdr->type_len + d->btf->hdr->str_len; + if (d->btf->layout) + d->btf->hdr.layout_off = d->btf->hdr.type_off + d->btf->hdr.type_len; + d->btf->hdr.str_off = d->btf->hdr.type_off + d->btf->hdr.type_len + d->btf->hdr.layout_len; + d->btf->raw_size = d->btf->hdr.hdr_len + d->btf->hdr.type_off + d->btf->hdr.type_len + + d->btf->hdr.layout_len + d->btf->hdr.str_len; return 0; } @@ -5779,7 +6069,7 @@ static int btf_add_distilled_types(struct btf_distill *dist) err = btf_add_type(&dist->pipe, t); break; default: - pr_warn("unexpected kind when adding base type '%s'[%u] of kind [%u] to distilled base BTF.\n", + pr_warn("unexpected kind when adding base type '%s'[%d] of kind [%d] to distilled base BTF.\n", name, i, kind); return -EINVAL; @@ -5865,7 +6155,7 @@ int btf__distill_base(const struct btf *src_btf, struct btf **new_base_btf, goto done; } dist.split_start_id = btf__type_cnt(old_base); - dist.split_start_str = old_base->hdr->str_len; + dist.split_start_str = old_base->hdr.str_len; /* Pass over src split BTF; generate the list of base BTF type ids it * references; these will constitute our distilled BTF set to be @@ -5934,14 +6224,14 @@ done: const struct btf_header *btf_header(const struct btf *btf) { - return btf->hdr; + return &btf->hdr; } void btf_set_base_btf(struct btf *btf, const struct btf *base_btf) { btf->base_btf = (struct btf *)base_btf; btf->start_id = btf__type_cnt(base_btf); - btf->start_str_off = base_btf->hdr->str_len + base_btf->start_str_off; + btf->start_str_off = base_btf->hdr.str_len + base_btf->start_str_off; } int btf__relocate(struct btf *btf, const struct btf *base_btf) @@ -6008,16 +6298,15 @@ int btf__permute(struct btf *btf, __u32 *id_map, __u32 id_map_cnt, goto done; } - new_types = calloc(btf->hdr->type_len, 1); + new_types = calloc(btf->hdr.type_len, 1); if (!new_types) { err = -ENOMEM; goto done; } - if (btf_ensure_modifiable(btf)) { - err = -ENOMEM; + err = btf_ensure_modifiable(btf); + if (err) goto done; - } for (i = start_offs; i < id_map_cnt; i++) { id = id_map[i]; @@ -6046,7 +6335,7 @@ int btf__permute(struct btf *btf, __u32 *id_map, __u32 id_map_cnt, id = order_map[i]; t = btf__type_by_id(btf, id); - type_size = btf_type_size(t); + type_size = btf_type_size(btf, t); memcpy(nt, t, type_size); /* fix up referenced IDs for BTF */ @@ -6072,7 +6361,7 @@ int btf__permute(struct btf *btf, __u32 *id_map, __u32 id_map_cnt, for (nt = new_types, i = 0; i < id_map_cnt - start_offs; i++) { btf->type_offs[i] = nt - new_types; - nt += btf_type_size(nt); + nt += btf_type_size(btf, nt); } free(order_map); diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index b30008c267c0..587172c0de08 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -109,6 +109,26 @@ LIBBPF_API struct btf *btf__new_empty(void); */ LIBBPF_API struct btf *btf__new_empty_split(struct btf *base_btf); +struct btf_new_opts { + size_t sz; + struct btf *base_btf; /* optional base BTF */ + bool add_layout; /* add BTF layout information */ + size_t:0; +}; +#define btf_new_opts__last_field add_layout + +/** + * @brief **btf__new_empty_opts()** creates an unpopulated BTF object with + * optional *base_btf* and BTF kind layout description if *add_layout* + * is set + * @return new BTF object instance which has to be eventually freed with + * **btf__free()** + * + * On error, NULL is returned and the thread-local `errno` variable is + * set to the error code. + */ +LIBBPF_API struct btf *btf__new_empty_opts(struct btf_new_opts *opts); + /** * @brief **btf__distill_base()** creates new versions of the split BTF * *src_btf* and its base BTF. The new base BTF will only contain the types @@ -152,6 +172,8 @@ LIBBPF_API __s32 btf__find_by_name(const struct btf *btf, const char *type_name); LIBBPF_API __s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name, __u32 kind); +LIBBPF_API __s32 btf__find_by_name_kind_own(const struct btf *btf, + const char *type_name, __u32 kind); LIBBPF_API __u32 btf__type_cnt(const struct btf *btf); LIBBPF_API const struct btf *btf__base_btf(const struct btf *btf); LIBBPF_API const struct btf_type *btf__type_by_id(const struct btf *btf, @@ -415,7 +437,7 @@ static inline __u16 btf_kind(const struct btf_type *t) return BTF_INFO_KIND(t->info); } -static inline __u16 btf_vlen(const struct btf_type *t) +static inline __u32 btf_vlen(const struct btf_type *t) { return BTF_INFO_VLEN(t->info); } diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index 53c6624161d7..123c448f20c7 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -316,7 +316,7 @@ static int btf_dump_mark_referenced(struct btf_dump *d) { int i, j, n = btf__type_cnt(d->btf); const struct btf_type *t; - __u16 vlen; + __u32 vlen; for (i = d->last_id + 1; i < n; i++) { t = btf__type_by_id(d->btf, i); @@ -485,7 +485,7 @@ static int btf_dump_order_type(struct btf_dump *d, __u32 id, bool through_ptr) */ struct btf_dump_type_aux_state *tstate = &d->type_states[id]; const struct btf_type *t; - __u16 vlen; + __u32 vlen; int err, i; /* return true, letting typedefs know that it's ok to be emitted */ @@ -798,7 +798,7 @@ static void btf_dump_emit_type(struct btf_dump *d, __u32 id, __u32 cont_id) */ if (top_level_def || t->name_off == 0) { const struct btf_member *m = btf_members(t); - __u16 vlen = btf_vlen(t); + __u32 vlen = btf_vlen(t); int i, new_cont_id; new_cont_id = t->name_off == 0 ? cont_id : id; @@ -820,7 +820,7 @@ static void btf_dump_emit_type(struct btf_dump *d, __u32 id, __u32 cont_id) break; case BTF_KIND_FUNC_PROTO: { const struct btf_param *p = btf_params(t); - __u16 n = btf_vlen(t); + __u32 n = btf_vlen(t); int i; btf_dump_emit_type(d, t->type, cont_id); @@ -839,7 +839,7 @@ static bool btf_is_struct_packed(const struct btf *btf, __u32 id, { const struct btf_member *m; int max_align = 1, align, i, bit_sz; - __u16 vlen; + __u32 vlen; m = btf_members(t); vlen = btf_vlen(t); @@ -973,7 +973,7 @@ static void btf_dump_emit_struct_def(struct btf_dump *d, bool is_struct = btf_is_struct(t); bool packed, prev_bitfield = false; int align, i, off = 0; - __u16 vlen = btf_vlen(t); + __u32 vlen = btf_vlen(t); align = btf__align_of(d->btf, id); packed = is_struct ? btf_is_struct_packed(d->btf, id, t) : 0; @@ -1064,7 +1064,7 @@ static void btf_dump_emit_enum_fwd(struct btf_dump *d, __u32 id, static void btf_dump_emit_enum32_val(struct btf_dump *d, const struct btf_type *t, - int lvl, __u16 vlen) + int lvl, __u32 vlen) { const struct btf_enum *v = btf_enum(t); bool is_signed = btf_kflag(t); @@ -1089,7 +1089,7 @@ static void btf_dump_emit_enum32_val(struct btf_dump *d, static void btf_dump_emit_enum64_val(struct btf_dump *d, const struct btf_type *t, - int lvl, __u16 vlen) + int lvl, __u32 vlen) { const struct btf_enum64 *v = btf_enum64(t); bool is_signed = btf_kflag(t); @@ -1122,7 +1122,7 @@ static void btf_dump_emit_enum_def(struct btf_dump *d, __u32 id, const struct btf_type *t, int lvl) { - __u16 vlen = btf_vlen(t); + __u32 vlen = btf_vlen(t); btf_dump_printf(d, "enum%s%s", t->name_off ? " " : "", @@ -1542,7 +1542,7 @@ static void btf_dump_emit_type_chain(struct btf_dump *d, } case BTF_KIND_FUNC_PROTO: { const struct btf_param *p = btf_params(t); - __u16 vlen = btf_vlen(t); + __u32 vlen = btf_vlen(t); int i; /* @@ -1776,7 +1776,7 @@ static int btf_dump_get_bitfield_value(struct btf_dump *d, /* Maximum supported bitfield size is 64 bits */ if (t->size > 8) { - pr_warn("unexpected bitfield size %d\n", t->size); + pr_warn("unexpected bitfield size %u\n", t->size); return -EINVAL; } @@ -2159,7 +2159,7 @@ static int btf_dump_struct_data(struct btf_dump *d, const void *data) { const struct btf_member *m = btf_members(t); - __u16 n = btf_vlen(t); + __u32 n = btf_vlen(t); int i, err = 0; /* note that we increment depth before calling btf_dump_print() below; @@ -2251,7 +2251,7 @@ static int btf_dump_get_enum_value(struct btf_dump *d, *value = is_signed ? *(__s8 *)data : *(__u8 *)data; return 0; default: - pr_warn("unexpected size %d for enum, id:[%u]\n", t->size, id); + pr_warn("unexpected size %u for enum, id:[%u]\n", t->size, id); return -EINVAL; } } @@ -2449,7 +2449,7 @@ static int btf_dump_type_data_check_zero(struct btf_dump *d, case BTF_KIND_STRUCT: case BTF_KIND_UNION: { const struct btf_member *m = btf_members(t); - __u16 n = btf_vlen(t); + __u32 n = btf_vlen(t); /* if any struct/union member is non-zero, the struct/union * is considered non-zero and dumped. diff --git a/tools/lib/bpf/btf_relocate.c b/tools/lib/bpf/btf_relocate.c index 53d1f3541bce..df5fa4bd87d6 100644 --- a/tools/lib/bpf/btf_relocate.c +++ b/tools/lib/bpf/btf_relocate.c @@ -280,7 +280,7 @@ static int btf_relocate_map_distilled_base(struct btf_relocate *r) cmp_btf_name_size(&base_info, dist_info) == 0; dist_info++) { if (!dist_info->id || dist_info->id >= r->nr_dist_base_types) { - pr_warn("base BTF id [%d] maps to invalid distilled base BTF id [%d]\n", + pr_warn("base BTF id [%u] maps to invalid distilled base BTF id [%u]\n", id, dist_info->id); err = -EINVAL; goto done; @@ -368,7 +368,7 @@ static int btf_relocate_map_distilled_base(struct btf_relocate *r) continue; dist_t = btf_type_by_id(r->dist_base_btf, id); name = btf__name_by_offset(r->dist_base_btf, dist_t->name_off); - pr_warn("distilled base BTF type '%s' [%d] is not mapped to base BTF id\n", + pr_warn("distilled base BTF type '%s' [%u] is not mapped to base BTF id\n", name, id); err = -EINVAL; break; @@ -397,11 +397,11 @@ static int btf_relocate_validate_distilled_base(struct btf_relocate *r) case BTF_KIND_FWD: if (t->name_off) break; - pr_warn("type [%d], kind [%d] is invalid for distilled base BTF; it is anonymous\n", + pr_warn("type [%u], kind [%d] is invalid for distilled base BTF; it is anonymous\n", i, kind); return -EINVAL; default: - pr_warn("type [%d] in distilled based BTF has unexpected kind [%d]\n", + pr_warn("type [%u] in distilled based BTF has unexpected kind [%d]\n", i, kind); return -EINVAL; } diff --git a/tools/lib/bpf/elf.c b/tools/lib/bpf/elf.c index 295dbda24580..fe136d025967 100644 --- a/tools/lib/bpf/elf.c +++ b/tools/lib/bpf/elf.c @@ -354,7 +354,7 @@ long elf_find_func_offset(Elf *elf, const char *binary_path, const char *name) if (ret > 0) { pr_debug("elf: symbol address match for '%s' in '%s': 0x%lx\n", name, binary_path, - ret); + (unsigned long)ret); } else { if (ret == 0) { pr_warn("elf: '%s' is 0 in symtab for '%s': %s\n", name, binary_path, diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c index 2fa434f09cce..9f589980beef 100644 --- a/tools/lib/bpf/features.c +++ b/tools/lib/bpf/features.c @@ -568,6 +568,90 @@ static int probe_ldimm64_full_range_off(int token_fd) return 1; } +#ifdef __x86_64__ + +#ifndef __NR_uprobe +#define __NR_uprobe 336 +#endif + +static int probe_uprobe_syscall(int token_fd) +{ + /* + * If kernel supports uprobe() syscall, it will return -EPROTO when called + * from the outside of a kernel-generated uprobe trampoline. + */ + return syscall(__NR_uprobe) < 0 && errno == EPROTO; +} +#else +static int probe_uprobe_syscall(int token_fd) +{ + return 0; +} +#endif + +static int probe_kern_btf_layout(int token_fd) +{ + static const char strs[] = "\0int"; + __u32 types[] = { + /* int */ + BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), + }; + struct btf_layout layout[] = { + { 0, 0, 0 }, + { sizeof(__u32), 0, 0 }, + }; + struct btf_header hdr = { + .magic = BTF_MAGIC, + .version = BTF_VERSION, + .hdr_len = sizeof(struct btf_header), + .type_len = sizeof(types), + .str_off = sizeof(types) + sizeof(layout), + .str_len = sizeof(strs), + .layout_off = sizeof(types), + .layout_len = sizeof(layout), + }; + + return probe_fd(libbpf__load_raw_btf_hdr(&hdr, (char *)types, strs, + (char *)layout, token_fd)); +} + +static int probe_bpf_syscall_common_attrs(int token_fd) +{ + return probe_sys_bpf_ext(); +} + +static int probe_kern_percpu_data(int token_fd) +{ + struct bpf_insn insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_1, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0), + BPF_EXIT_INSN(), + }; + LIBBPF_OPTS(bpf_map_create_opts, map_opts, + .token_fd = token_fd, + .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, + ); + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, + .token_fd = token_fd, + .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, + ); + int ret, map, insn_cnt = ARRAY_SIZE(insns); + + map = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "libbpf_percpu", sizeof(int), 8, 1, + &map_opts); + if (map < 0) { + pr_warn("Error in %s(): %s. Couldn't create simple percpu_array map.\n", + __func__, errstr(map)); + return map; + } + + insns[0].imm = map; + + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); + close(map); + return probe_fd(ret); +} + typedef int (*feature_probe_fn)(int /* token_fd */); static struct kern_feature_cache feature_cache; @@ -646,6 +730,18 @@ static struct kern_feature_desc { [FEAT_LDIMM64_FULL_RANGE_OFF] = { "full range LDIMM64 support", probe_ldimm64_full_range_off, }, + [FEAT_UPROBE_SYSCALL] = { + "kernel supports uprobe syscall", probe_uprobe_syscall, + }, + [FEAT_BTF_LAYOUT] = { + "kernel supports BTF layout", probe_kern_btf_layout, + }, + [FEAT_BPF_SYSCALL_COMMON_ATTRS] = { + "BPF syscall common attributes support", probe_bpf_syscall_common_attrs, + }, + [FEAT_PERCPU_DATA] = { + "kernel supports percpu data", probe_kern_percpu_data, + }, }; bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id) diff --git a/tools/lib/bpf/gen_loader.c b/tools/lib/bpf/gen_loader.c index cd5c2543f54d..af3a04f161ac 100644 --- a/tools/lib/bpf/gen_loader.c +++ b/tools/lib/bpf/gen_loader.c @@ -63,6 +63,7 @@ static int realloc_insn_buf(struct bpf_gen *gen, __u32 size) gen->error = -ENOMEM; free(gen->insn_start); gen->insn_start = NULL; + gen->insn_cur = NULL; return -ENOMEM; } gen->insn_start = insn_start; @@ -86,6 +87,7 @@ static int realloc_data_buf(struct bpf_gen *gen, __u32 size) gen->error = -ENOMEM; free(gen->data_start); gen->data_start = NULL; + gen->data_cur = NULL; return -ENOMEM; } gen->data_start = data_start; @@ -109,7 +111,6 @@ static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn in static int add_data(struct bpf_gen *gen, const void *data, __u32 size); static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off); -static void emit_signature_match(struct bpf_gen *gen); void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps) { @@ -152,16 +153,20 @@ void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps /* R7 contains the error code from sys_bpf. Copy it into R0 and exit. */ emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7)); emit(gen, BPF_EXIT_INSN()); - if (OPTS_GET(gen->opts, gen_hash, false)) - emit_signature_match(gen); } static int add_data(struct bpf_gen *gen, const void *data, __u32 size) { - __u32 size8 = roundup(size, 8); __u64 zero = 0; + __u32 size8; void *prev; + if (size > INT32_MAX) { + gen->error = -ERANGE; + return 0; + } + size8 = roundup(size, 8); + if (realloc_data_buf(gen, size8)) return 0; prev = gen->data_cur; @@ -293,7 +298,6 @@ static void emit_check_err(struct bpf_gen *gen) emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, off)); } else { gen->error = -ERANGE; - emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, -1)); } } @@ -370,14 +374,12 @@ static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off) __emit_sys_close(gen); } -static void compute_sha_update_offsets(struct bpf_gen *gen); - int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) { int i; if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) { - pr_warn("nr_progs %d/%d nr_maps %d/%d mismatch\n", + pr_warn("nr_progs %d/%u nr_maps %d/%u mismatch\n", nr_progs, gen->nr_progs, nr_maps, gen->nr_maps); gen->error = -EFAULT; return gen->error; @@ -398,10 +400,6 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) blob_fd_array_off(gen, i)); emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0)); emit(gen, BPF_EXIT_INSN()); - if (OPTS_GET(gen->opts, gen_hash, false)) - compute_sha_update_offsets(gen); - - pr_debug("gen: finish %s\n", errstr(gen->error)); if (!gen->error) { struct gen_loader_opts *opts = gen->opts; @@ -419,6 +417,7 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) bpf_insn_bswap(insn++); } } + pr_debug("gen: finish %s\n", errstr(gen->error)); return gen->error; } @@ -453,22 +452,6 @@ void bpf_gen__free(struct bpf_gen *gen) _val; \ }) -static void compute_sha_update_offsets(struct bpf_gen *gen) -{ - __u64 sha[SHA256_DWORD_SIZE]; - __u64 sha_dw; - int i; - - libbpf_sha256(gen->data_start, gen->data_cur - gen->data_start, (__u8 *)sha); - for (i = 0; i < SHA256_DWORD_SIZE; i++) { - struct bpf_insn *insn = - (struct bpf_insn *)(gen->insn_start + gen->hash_insn_offset[i]); - sha_dw = tgt_endian(sha[i]); - insn[0].imm = (__u32)sha_dw; - insn[1].imm = sha_dw >> 32; - } -} - void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data, __u32 btf_raw_size) { @@ -481,7 +464,7 @@ void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data, attr.btf_size = tgt_endian(btf_raw_size); btf_load_attr = add_data(gen, &attr, attr_size); - pr_debug("gen: load_btf: off %d size %d, attr: off %d size %d\n", + pr_debug("gen: load_btf: off %d size %u, attr: off %d size %d\n", btf_data, btf_raw_size, btf_load_attr, attr_size); /* populate union bpf_attr with user provided log details */ @@ -527,7 +510,7 @@ void bpf_gen__map_create(struct bpf_gen *gen, attr.btf_value_type_id = tgt_endian(map_attr->btf_value_type_id); map_create_attr = add_data(gen, &attr, attr_size); - pr_debug("gen: map_create: %s idx %d type %d value_type_id %d, attr: off %d size %d\n", + pr_debug("gen: map_create: %s idx %d type %u value_type_id %u, attr: off %d size %d\n", map_name, map_idx, map_type, map_attr->btf_value_type_id, map_create_attr, attr_size); @@ -545,13 +528,23 @@ void bpf_gen__map_create(struct bpf_gen *gen, default: break; } - /* conditionally update max_entries */ - if (map_idx >= 0) + + /* + * Conditionally update max_entries from the host-supplied loader + * ctx. This sizes the map at runtime, but for a signed loader + * (gen_hash) it would let an untrusted host re-dimension the + * program's maps, outside what the signature attests to: the + * metadata blob is covered by the program signature and verified + * by the kernel at load time. Keep the signer-provided max_entries + * baked into the blob in that case. + */ + if (map_idx >= 0 && !OPTS_GET(gen->opts, gen_hash, false)) move_ctx2blob(gen, attr_field(map_create_attr, max_entries), 4, sizeof(struct bpf_loader_ctx) + sizeof(struct bpf_map_desc) * map_idx + offsetof(struct bpf_map_desc, max_entries), true /* check that max_entries != 0 */); + /* emit MAP_CREATE command */ emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size); debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d", @@ -580,29 +573,6 @@ void bpf_gen__map_create(struct bpf_gen *gen, emit_sys_close_stack(gen, stack_off(inner_map_fd)); } -static void emit_signature_match(struct bpf_gen *gen) -{ - __s64 off; - int i; - - for (i = 0; i < SHA256_DWORD_SIZE; i++) { - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX, - 0, 0, 0, 0)); - emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, i * sizeof(__u64))); - gen->hash_insn_offset[i] = gen->insn_cur - gen->insn_start; - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_3, 0, 0, 0, 0, 0)); - - off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1; - if (is_simm16(off)) { - emit(gen, BPF_MOV64_IMM(BPF_REG_7, -EINVAL)); - emit(gen, BPF_JMP_REG(BPF_JNE, BPF_REG_2, BPF_REG_3, off)); - } else { - gen->error = -ERANGE; - emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, -1)); - } - } -} - void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name, enum bpf_attach_type type) { @@ -1050,11 +1020,11 @@ void bpf_gen__prog_load(struct bpf_gen *gen, license_off = add_data(gen, license, strlen(license) + 1); /* add insns to blob of bytes */ insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn)); - pr_debug("gen: prog_load: prog_idx %d type %d insn off %d insns_cnt %zd license off %d\n", + pr_debug("gen: prog_load: prog_idx %d type %u insn off %d insns_cnt %zu license off %d\n", prog_idx, prog_type, insns_off, insn_cnt, license_off); /* convert blob insns to target endianness */ - if (gen->swapped_endian) { + if (gen->swapped_endian && !gen->error) { struct bpf_insn *insn = gen->data_start + insns_off; int i; @@ -1073,26 +1043,26 @@ void bpf_gen__prog_load(struct bpf_gen *gen, attr.func_info_rec_size = tgt_endian(load_attr->func_info_rec_size); attr.func_info_cnt = tgt_endian(load_attr->func_info_cnt); func_info = add_data(gen, load_attr->func_info, func_info_tot_sz); - pr_debug("gen: prog_load: func_info: off %d cnt %d rec size %d\n", + pr_debug("gen: prog_load: func_info: off %d cnt %u rec size %u\n", func_info, load_attr->func_info_cnt, load_attr->func_info_rec_size); attr.line_info_rec_size = tgt_endian(load_attr->line_info_rec_size); attr.line_info_cnt = tgt_endian(load_attr->line_info_cnt); line_info = add_data(gen, load_attr->line_info, line_info_tot_sz); - pr_debug("gen: prog_load: line_info: off %d cnt %d rec size %d\n", + pr_debug("gen: prog_load: line_info: off %d cnt %u rec size %u\n", line_info, load_attr->line_info_cnt, load_attr->line_info_rec_size); attr.core_relo_rec_size = tgt_endian((__u32)sizeof(struct bpf_core_relo)); attr.core_relo_cnt = tgt_endian(gen->core_relo_cnt); core_relos = add_data(gen, gen->core_relos, core_relo_tot_sz); - pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zd\n", + pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zu\n", core_relos, gen->core_relo_cnt, sizeof(struct bpf_core_relo)); /* convert all info blobs to target endianness */ - if (gen->swapped_endian) + if (gen->swapped_endian && !gen->error) info_blob_bswap(gen, func_info, line_info, core_relos, load_attr); libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name)); @@ -1158,7 +1128,7 @@ void bpf_gen__prog_load(struct bpf_gen *gen, } void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, - __u32 value_size) + __u32 value_size, __u64 flags) { int attr_size = offsetofend(union bpf_attr, flags); int map_update_attr, value, key; @@ -1166,34 +1136,44 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, int zero = 0; memset(&attr, 0, attr_size); + attr.flags = tgt_endian(flags); value = add_data(gen, pvalue, value_size); key = add_data(gen, &zero, sizeof(zero)); - /* if (map_desc[map_idx].initial_value) { + /* + * if (map_desc[map_idx].initial_value) { * if (ctx->flags & BPF_SKEL_KERNEL) * bpf_probe_read_kernel(value, value_size, initial_value); * else * bpf_copy_from_user(value, value_size, initial_value); * } + * + * The runtime initial_value comes from the host-supplied loader + * ctx and would overwrite the blob value that the program signature + * covers and the kernel verifies at load time. For a signed loader + * (gen_hash) the attested blob value must be authoritative, so skip + * the override and leave the signed value in place. */ - emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, - sizeof(struct bpf_loader_ctx) + - sizeof(struct bpf_map_desc) * map_idx + - offsetof(struct bpf_map_desc, initial_value))); - emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8)); - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, value)); - emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); - emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, - offsetof(struct bpf_loader_ctx, flags))); - emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); - emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); + if (!OPTS_GET(gen->opts, gen_hash, false)) { + emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, + sizeof(struct bpf_loader_ctx) + + sizeof(struct bpf_map_desc) * map_idx + + offsetof(struct bpf_map_desc, initial_value))); + emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8)); + emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, + 0, 0, 0, value)); + emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); + emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, + offsetof(struct bpf_loader_ctx, flags))); + emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2)); + emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); + emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); + emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); + } map_update_attr = add_data(gen, &attr, attr_size); - pr_debug("gen: map_update_elem: idx %d, value: off %d size %d, attr: off %d size %d\n", + pr_debug("gen: map_update_elem: idx %d, value: off %d size %u, attr: off %d size %d\n", map_idx, value, value_size, map_update_attr, attr_size); move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, blob_fd_array_off(gen, map_idx)); diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 0be7017800fe..b749c01742ee 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -136,6 +136,9 @@ static const char * const attach_type_name[] = { [BPF_NETKIT_PEER] = "netkit_peer", [BPF_TRACE_KPROBE_SESSION] = "trace_kprobe_session", [BPF_TRACE_UPROBE_SESSION] = "trace_uprobe_session", + [BPF_TRACE_FENTRY_MULTI] = "trace_fentry_multi", + [BPF_TRACE_FEXIT_MULTI] = "trace_fexit_multi", + [BPF_TRACE_FSESSION_MULTI] = "trace_fsession_multi", }; static const char * const link_type_name[] = { @@ -154,6 +157,7 @@ static const char * const link_type_name[] = { [BPF_LINK_TYPE_UPROBE_MULTI] = "uprobe_multi", [BPF_LINK_TYPE_NETKIT] = "netkit", [BPF_LINK_TYPE_SOCKMAP] = "sockmap", + [BPF_LINK_TYPE_TRACING_MULTI] = "tracing_multi", }; static const char * const map_type_name[] = { @@ -192,6 +196,7 @@ static const char * const map_type_name[] = { [BPF_MAP_TYPE_CGRP_STORAGE] = "cgrp_storage", [BPF_MAP_TYPE_ARENA] = "arena", [BPF_MAP_TYPE_INSN_ARRAY] = "insn_array", + [BPF_MAP_TYPE_RHASH] = "rhash", }; static const char * const prog_type_name[] = { @@ -536,6 +541,7 @@ struct bpf_struct_ops { }; #define DATA_SEC ".data" +#define PERCPU_SEC ".percpu" #define BSS_SEC ".bss" #define RODATA_SEC ".rodata" #define KCONFIG_SEC ".kconfig" @@ -550,6 +556,7 @@ enum libbpf_map_type { LIBBPF_MAP_BSS, LIBBPF_MAP_RODATA, LIBBPF_MAP_KCONFIG, + LIBBPF_MAP_PERCPU, }; struct bpf_map_def { @@ -661,6 +668,7 @@ enum sec_type { SEC_DATA, SEC_RODATA, SEC_ST_OPS, + SEC_PERCPU, }; struct elf_sec_desc { @@ -1481,7 +1489,7 @@ static int init_struct_ops_maps(struct bpf_object *obj, const char *sec_name, type->size); st_ops->type_id = type_id; - pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n", + pr_debug("struct_ops init: struct %s(type_id=%d) %s found at offset %u\n", tname, type_id, var_name, vsi->offset); } @@ -1833,6 +1841,7 @@ static size_t bpf_map_mmap_sz(const struct bpf_map *map) switch (map->def.type) { case BPF_MAP_TYPE_ARRAY: + case BPF_MAP_TYPE_PERCPU_ARRAY: return array_map_mmap_sz(map->def.value_size, map->def.max_entries); case BPF_MAP_TYPE_ARENA: return page_sz * map->def.max_entries; @@ -1861,7 +1870,8 @@ static int bpf_map_mmap_resize(struct bpf_map *map, size_t old_sz, size_t new_sz return 0; } -static char *internal_map_name(struct bpf_object *obj, const char *real_name) +static char *internal_map_name(struct bpf_object *obj, const char *real_name, + enum libbpf_map_type type) { char map_name[BPF_OBJ_NAME_LEN], *p; int pfx_len, sfx_len = max((size_t)7, strlen(real_name)); @@ -1902,8 +1912,11 @@ static char *internal_map_name(struct bpf_object *obj, const char *real_name) if (sfx_len >= BPF_OBJ_NAME_LEN) sfx_len = BPF_OBJ_NAME_LEN - 1; - /* if there are two or more dots in map name, it's a custom dot map */ - if (strchr(real_name + 1, '.') != NULL) + /* + * Don't prefix the bpf_object name if this is a custom dot map + * (containing two or more dots) or a percpu data map. + */ + if (strchr(real_name + 1, '.') != NULL || type == LIBBPF_MAP_PERCPU) pfx_len = 0; else pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name)); @@ -1936,6 +1949,13 @@ static bool map_is_mmapable(struct bpf_object *obj, struct bpf_map *map) if (!map->btf_value_type_id) return false; + /* + * The internal PERCPU maps are not mmapable because the underlying + * percpu_array maps do not have mmap support. + */ + if (map->libbpf_type == LIBBPF_MAP_PERCPU) + return false; + t = btf__type_by_id(obj->btf, map->btf_value_type_id); if (!btf_is_datasec(t)) return false; @@ -1957,6 +1977,7 @@ static int bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, const char *real_name, int sec_idx, void *data, size_t data_sz) { + bool is_percpu = type == LIBBPF_MAP_PERCPU; struct bpf_map_def *def; struct bpf_map *map; size_t mmap_sz; @@ -1970,7 +1991,7 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, map->sec_idx = sec_idx; map->sec_offset = 0; map->real_name = strdup(real_name); - map->name = internal_map_name(obj, real_name); + map->name = internal_map_name(obj, real_name, type); if (!map->real_name || !map->name) { zfree(&map->real_name); zfree(&map->name); @@ -1978,7 +1999,7 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, } def = &map->def; - def->type = BPF_MAP_TYPE_ARRAY; + def->type = is_percpu ? BPF_MAP_TYPE_PERCPU_ARRAY : BPF_MAP_TYPE_ARRAY; def->key_size = sizeof(int); def->value_size = data_sz; def->max_entries = 1; @@ -1991,8 +2012,9 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, if (map_is_mmapable(obj, map)) def->map_flags |= BPF_F_MMAPABLE; - pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n", - map->name, map->sec_idx, map->sec_offset, def->map_flags); + pr_debug("map '%s' (global %sdata): at sec_idx %d, offset %zu, flags %x.\n", + map->name, is_percpu ? "percpu " : "", map->sec_idx, + map->sec_offset, def->map_flags); mmap_sz = bpf_map_mmap_sz(map); map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, @@ -2052,6 +2074,13 @@ static int bpf_object__init_global_data_maps(struct bpf_object *obj) NULL, sec_desc->data->d_size); break; + case SEC_PERCPU: + sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); + err = bpf_object__init_internal_map(obj, LIBBPF_MAP_PERCPU, + sec_name, sec_idx, + sec_desc->data->d_buf, + sec_desc->data->d_size); + break; default: /* skip */ break; @@ -2621,7 +2650,7 @@ int parse_btf_map_def(const char *map_name, struct btf *btf, t = btf__type_by_id(btf, m->type); if (!t) { - pr_warn("map '%s': key type [%d] not found.\n", + pr_warn("map '%s': key type [%u] not found.\n", map_name, m->type); return -EINVAL; } @@ -2661,7 +2690,7 @@ int parse_btf_map_def(const char *map_name, struct btf *btf, t = btf__type_by_id(btf, m->type); if (!t) { - pr_warn("map '%s': value type [%d] not found.\n", + pr_warn("map '%s': value type [%u] not found.\n", map_name, m->type); return -EINVAL; } @@ -2715,7 +2744,7 @@ int parse_btf_map_def(const char *map_name, struct btf *btf, map_def->value_size = 4; t = btf__type_by_id(btf, m->type); if (!t) { - pr_warn("map '%s': %s type [%d] not found.\n", + pr_warn("map '%s': %s type [%u] not found.\n", map_name, desc, m->type); return -EINVAL; } @@ -2801,7 +2830,7 @@ static size_t adjust_ringbuf_sz(size_t sz) return 0; /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be * a power-of-2 multiple of kernel's page size. If user diligently - * satisified these conditions, pass the size through. + * satisfied these conditions, pass the size through. */ if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz)) return sz; @@ -3138,12 +3167,14 @@ static bool btf_needs_sanitization(struct bpf_object *obj) bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64); bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC); + bool has_layout = kernel_supports(obj, FEAT_BTF_LAYOUT); return !has_func || !has_datasec || !has_func_global || !has_float || - !has_decl_tag || !has_type_tag || !has_enum64 || !has_qmark_datasec; + !has_decl_tag || !has_type_tag || !has_enum64 || !has_qmark_datasec || + !has_layout; } -static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) +struct btf *bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *orig_btf) { bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); @@ -3153,9 +3184,64 @@ static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64); bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC); + bool has_layout = kernel_supports(obj, FEAT_BTF_LAYOUT); int enum64_placeholder_id = 0; + const struct btf_header *hdr; + struct btf *btf = NULL; + const void *raw_data; struct btf_type *t; int i, j, vlen; + __u32 sz; + int err; + + /* clone BTF to sanitize a copy and leave the original intact */ + raw_data = btf__raw_data(orig_btf, &sz); + if (!raw_data) + return ERR_PTR(-ENOMEM); + /* btf_header() gives us endian-safe header info */ + hdr = btf_header(orig_btf); + + if (!has_layout && hdr->hdr_len >= sizeof(struct btf_header) && + (hdr->layout_len != 0 || hdr->layout_off != 0)) { + const struct btf_header *old_hdr = raw_data; + struct btf_header *new_hdr; + void *new_raw_data; + __u32 new_str_off; + + /* + * Need to rewrite BTF to exclude layout information and + * move string section to immediately after types. + */ + new_raw_data = malloc(sz); + if (!new_raw_data) + return ERR_PTR(-ENOMEM); + + memcpy(new_raw_data, raw_data, sz); + new_hdr = new_raw_data; + new_hdr->layout_off = 0; + new_hdr->layout_len = 0; + new_str_off = hdr->type_off + hdr->type_len; + /* Handle swapped endian case */ + if (old_hdr->magic != hdr->magic) + new_hdr->str_off = bswap_32(new_str_off); + else + new_hdr->str_off = new_str_off; + + memmove(new_raw_data + hdr->hdr_len + new_str_off, + new_raw_data + hdr->hdr_len + hdr->str_off, + hdr->str_len); + sz = hdr->hdr_len + hdr->type_off + hdr->type_len + hdr->str_len; + btf = btf__new(new_raw_data, sz); + free(new_raw_data); + } else { + btf = btf__new(raw_data, sz); + } + err = libbpf_get_error(btf); + if (err) + return ERR_PTR(err); + + /* enforce 8-byte pointers for BPF-targeted BTFs */ + btf__set_pointer_size(btf, 8); for (i = 1; i < btf__type_cnt(btf); i++) { t = (struct btf_type *)btf__type_by_id(btf, i); @@ -3233,9 +3319,10 @@ static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) if (enum64_placeholder_id == 0) { enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0); - if (enum64_placeholder_id < 0) - return enum64_placeholder_id; - + if (enum64_placeholder_id < 0) { + btf__free(btf); + return ERR_PTR(enum64_placeholder_id); + } t = (struct btf_type *)btf__type_by_id(btf, i); } @@ -3249,7 +3336,7 @@ static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) } } - return 0; + return btf; } static bool libbpf_needs_btf(const struct bpf_object *obj) @@ -3413,7 +3500,7 @@ static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf, var_name = btf__name_by_offset(btf, t_var->name_off); if (!var_name) { - pr_debug("sec '%s': failed to find name of DATASEC's member #%d\n", + pr_debug("sec '%s': failed to find name of DATASEC's member #%u\n", sec_name, i); return -ENOENT; } @@ -3600,21 +3687,9 @@ static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) sanitize = btf_needs_sanitization(obj); if (sanitize) { - const void *raw_data; - __u32 sz; - - /* clone BTF to sanitize a copy and leave the original intact */ - raw_data = btf__raw_data(obj->btf, &sz); - kern_btf = btf__new(raw_data, sz); - err = libbpf_get_error(kern_btf); - if (err) - return err; - - /* enforce 8-byte pointers for BPF-targeted BTFs */ - btf__set_pointer_size(obj->btf, 8); - err = bpf_object__sanitize_btf(obj, kern_btf); - if (err) - return err; + kern_btf = bpf_object__sanitize_btf(obj, obj->btf); + if (IS_ERR(kern_btf)) + return PTR_ERR(kern_btf); } if (obj->gen_loader) { @@ -3920,7 +3995,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj) if (!data) return -LIBBPF_ERRNO__FORMAT; - pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", + pr_debug("elf: section(%d) %s, size %lu, link %d, flags %lx, type=%d\n", idx, name, (unsigned long)data->d_size, (int)sh->sh_link, (unsigned long)sh->sh_flags, (int)sh->sh_type); @@ -3965,6 +4040,11 @@ static int bpf_object__elf_collect(struct bpf_object *obj) sec_desc->sec_type = SEC_RODATA; sec_desc->shdr = sh; sec_desc->data = data; + } else if (strcmp(name, PERCPU_SEC) == 0 || + str_has_pfx(name, PERCPU_SEC ".")) { + sec_desc->sec_type = SEC_PERCPU; + sec_desc->shdr = sh; + sec_desc->data = data; } else if (strcmp(name, STRUCT_OPS_SEC) == 0 || strcmp(name, STRUCT_OPS_LINK_SEC) == 0 || strcmp(name, "?" STRUCT_OPS_SEC) == 0 || @@ -4443,7 +4523,7 @@ static int bpf_object__collect_externs(struct bpf_object *obj) ext->kcfg.data_off = roundup(off, ext->kcfg.align); off = ext->kcfg.data_off + ext->kcfg.sz; - pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n", + pr_debug("extern (kcfg) #%d: symbol %d, off %d, name %s\n", i, ext->sym_idx, ext->kcfg.data_off, ext->name); } sec->size = off; @@ -4493,6 +4573,7 @@ static bool bpf_object__shndx_is_data(const struct bpf_object *obj, case SEC_BSS: case SEC_DATA: case SEC_RODATA: + case SEC_PERCPU: return true; default: return false; @@ -4518,6 +4599,8 @@ bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) return LIBBPF_MAP_DATA; case SEC_RODATA: return LIBBPF_MAP_RODATA; + case SEC_PERCPU: + return LIBBPF_MAP_PERCPU; default: return LIBBPF_MAP_UNSPEC; } @@ -4575,7 +4658,7 @@ static int bpf_program__record_reloc(struct bpf_program *prog, struct bpf_map *map; if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) { - pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n", + pr_warn("prog '%s': invalid relo against '%s' for insns[%u].code 0x%x\n", prog->name, sym_name, insn_idx, insn->code); return -LIBBPF_ERRNO__RELOC; } @@ -4698,7 +4781,7 @@ static int bpf_program__record_reloc(struct bpf_program *prog, map->sec_idx != sym->st_shndx || map->sec_offset != sym->st_value) continue; - pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n", + pr_debug("prog '%s': found map %zu (%s, sec %d, off %zu) for insn #%u\n", prog->name, map_idx, map->name, map->sec_idx, map->sec_offset, insn_idx); break; @@ -4725,7 +4808,7 @@ static int bpf_program__record_reloc(struct bpf_program *prog, map = &obj->maps[map_idx]; if (map->libbpf_type != type || map->sec_idx != sym->st_shndx) continue; - pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n", + pr_debug("prog '%s': found data map %zu (%s, sec %d, off %zu) for insn %u\n", prog->name, map_idx, map->name, map->sec_idx, map->sec_offset, insn_idx); break; @@ -4893,7 +4976,7 @@ static int map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map) /* * LLVM annotates global data differently in BTF, that is, - * only as '.data', '.bss' or '.rodata'. + * only as '.data', '.bss', '.percpu' or '.rodata'. */ if (!bpf_map__is_internal(map)) return -ENOENT; @@ -4934,7 +5017,7 @@ static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info) info->value_size = val; else if (sscanf(buff, "max_entries:\t%u", &val) == 1) info->max_entries = val; - else if (sscanf(buff, "map_flags:\t%i", &val) == 1) + else if (sscanf(buff, "map_flags:\t%x", &val) == 1) info->map_flags = val; } @@ -5121,12 +5204,8 @@ bpf_object__probe_loading(struct bpf_object *obj) BPF_EXIT_INSN(), }; int ret, insn_cnt = ARRAY_SIZE(insns); - LIBBPF_OPTS(bpf_prog_load_opts, opts, - .token_fd = obj->token_fd, - .prog_flags = obj->token_fd ? BPF_F_TOKEN_FD : 0, - ); - if (obj->gen_loader) + if (obj->gen_loader || obj->token_fd) return 0; ret = bump_rlimit_memlock(); @@ -5135,9 +5214,9 @@ bpf_object__probe_loading(struct bpf_object *obj) errstr(ret)); /* make sure basic loading works */ - ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &opts); + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); if (ret < 0) - ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts); + ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL); if (ret < 0) { ret = errno; pr_warn("Error in %s(): %s. Couldn't load trivial BPF program. Make sure your kernel supports BPF (CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is set to big enough value.\n", @@ -5157,12 +5236,20 @@ bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) */ return true; - if (obj->token_fd) + if (obj->feat_cache) return feat_supported(obj->feat_cache, feat_id); return feat_supported(NULL, feat_id); } +/* Used in testing to simulate missing features. */ +void bpf_object_set_feat_cache(struct bpf_object *obj, struct kern_feature_cache *cache) +{ + if (obj->feat_cache) + free(obj->feat_cache); + obj->feat_cache = cache; +} + static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) { struct bpf_map_info map_info; @@ -5238,18 +5325,20 @@ static int bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) { enum libbpf_map_type map_type = map->libbpf_type; + bool is_percpu = map_type == LIBBPF_MAP_PERCPU; + const __u64 update_flags = is_percpu ? BPF_F_ALL_CPUS : 0; int err, zero = 0; size_t mmap_sz; if (obj->gen_loader) { bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps, - map->mmaped, map->def.value_size); + map->mmaped, map->def.value_size, update_flags); if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) bpf_gen__map_freeze(obj->gen_loader, map - obj->maps); return 0; } - err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0); + err = bpf_map_update_elem(map->fd, &zero, map->mmaped, update_flags); if (err) { err = -errno; pr_warn("map '%s': failed to set initial contents: %s\n", @@ -5294,6 +5383,13 @@ bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) return err; } map->mmaped = mmaped; + } else if (is_percpu) { + if (mprotect(map->mmaped, mmap_sz, PROT_READ)) { + err = -errno; + pr_warn("map '%s': failed to mprotect() contents: %s\n", + bpf_map__name(map), errstr(err)); + return err; + } } else if (map->mmaped) { munmap(map->mmaped, mmap_sz); map->mmaped = NULL; @@ -5462,11 +5558,11 @@ static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map) } if (err) { err = -errno; - pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %s\n", + pr_warn("map '%s': failed to initialize slot [%u] to map '%s' fd=%d: %s\n", map->name, i, targ_map->name, fd, errstr(err)); return err; } - pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n", + pr_debug("map '%s': slot [%u] set to map '%s' fd=%d\n", map->name, i, targ_map->name, fd); } @@ -5495,11 +5591,11 @@ static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map) err = bpf_map_update_elem(map->fd, &i, &fd, 0); if (err) { err = -errno; - pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %s\n", + pr_warn("map '%s': failed to initialize slot [%u] to prog '%s' fd=%d: %s\n", map->name, i, targ_prog->name, fd, errstr(err)); return err; } - pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n", + pr_debug("map '%s': slot [%u] set to prog '%s' fd=%d\n", map->name, i, targ_prog->name, fd); } @@ -5569,9 +5665,16 @@ bpf_object__create_maps(struct bpf_object *obj) * runtime due to bpf_program__set_autoload(prog, false), * bpf_object loading will succeed just fine even on old * kernels. + * Same skipping applies to percpu data. */ - if (bpf_map__is_internal(map) && !kernel_supports(obj, FEAT_GLOBAL_DATA)) - map->autocreate = false; + if (bpf_map__is_internal(map)) { + bool is_percpu = map->libbpf_type == LIBBPF_MAP_PERCPU; + enum kern_feature_id feat_id; + + feat_id = is_percpu ? FEAT_PERCPU_DATA : FEAT_GLOBAL_DATA; + if (!kernel_supports(obj, feat_id)) + map->autocreate = false; + } if (!map->autocreate) { pr_debug("map '%s': skipped auto-creating...\n", map->name); @@ -5729,7 +5832,7 @@ int bpf_core_add_cands(struct bpf_core_cand *local_cand, if (strncmp(local_name, targ_name, local_essent_len) != 0) continue; - pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n", + pr_debug("CO-RE relocating [%u] %s %s: found target candidate [%d] %s %s in [%s]\n", local_cand->id, btf_kind_str(local_t), local_name, i, btf_kind_str(t), targ_name, targ_btf_name); @@ -5789,7 +5892,7 @@ static int load_module_btfs(struct bpf_object *obj) if (errno == ENOENT) continue; /* expected race: BTF was unloaded */ err = -errno; - pr_warn("failed to get BTF object #%d FD: %s\n", id, errstr(err)); + pr_warn("failed to get BTF object #%u FD: %s\n", id, errstr(err)); return err; } @@ -5798,11 +5901,12 @@ static int load_module_btfs(struct bpf_object *obj) info.name = ptr_to_u64(name); info.name_len = sizeof(name); + btf = NULL; err = bpf_btf_get_info_by_fd(fd, &info, &len); if (err) { err = -errno; - pr_warn("failed to get BTF object #%d info: %s\n", id, errstr(err)); - goto err_out; + pr_warn("failed to get BTF object #%u info: %s\n", id, errstr(err)); + break; } /* ignore non-module BTFs */ @@ -5814,17 +5918,17 @@ static int load_module_btfs(struct bpf_object *obj) btf = btf_get_from_fd(fd, obj->btf_vmlinux); err = libbpf_get_error(btf); if (err) { - pr_warn("failed to load module [%s]'s BTF object #%d: %s\n", + pr_warn("failed to load module [%s]'s BTF object #%u: %s\n", name, id, errstr(err)); - goto err_out; + break; } err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap, sizeof(*obj->btf_modules), obj->btf_module_cnt + 1); if (err) - goto err_out; + break; - mod_btf = &obj->btf_modules[obj->btf_module_cnt++]; + mod_btf = &obj->btf_modules[obj->btf_module_cnt]; mod_btf->btf = btf; mod_btf->id = id; @@ -5832,16 +5936,16 @@ static int load_module_btfs(struct bpf_object *obj) mod_btf->name = strdup(name); if (!mod_btf->name) { err = -ENOMEM; - goto err_out; + break; } - continue; + obj->btf_module_cnt++; + } -err_out: + if (err) { + btf__free(btf); close(fd); - return err; } - - return 0; + return err; } static struct bpf_core_cand_list * @@ -6007,7 +6111,7 @@ static int bpf_core_resolve_relo(struct bpf_program *prog, !hashmap__find(cand_cache, local_id, &cands)) { cands = bpf_core_find_cands(prog->obj, local_btf, local_id); if (IS_ERR(cands)) { - pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n", + pr_warn("prog '%s': relo #%d: target candidate search failed for [%u] %s %s: %ld\n", prog_name, relo_idx, local_id, btf_kind_str(local_type), local_name, PTR_ERR(cands)); return PTR_ERR(cands); @@ -6067,7 +6171,7 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) goto out; } - pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info); + pr_debug("sec '%s': found %u CO-RE relocations\n", sec_name, sec->num_info); for_each_btf_ext_rec(seg, sec, i, rec) { if (rec->insn_off % BPF_INSN_SZ) @@ -6121,7 +6225,7 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res); if (err) { - pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %s\n", + pr_warn("prog '%s': relo #%d: failed to patch insn #%d: %s\n", prog->name, i, insn_idx, errstr(err)); goto out; } @@ -6286,7 +6390,7 @@ static int create_jt_map(struct bpf_object *obj, struct bpf_program *prog, struc goto err_close; } if (sym_off + jt_size > obj->jumptables_data_sz) { - pr_warn("map '.jumptables': jumptables_data size is %zd, trying to access %d\n", + pr_warn("map '.jumptables': jumptables_data size is %zu, trying to access %u\n", obj->jumptables_data_sz, sym_off + jt_size); err = -EINVAL; goto err_close; @@ -6321,7 +6425,7 @@ static int create_jt_map(struct bpf_object *obj, struct bpf_program *prog, struc */ if (insn_off > UINT32_MAX) { pr_warn("map '.jumptables': invalid jump table value 0x%llx at offset %u\n", - (long long)jt[i], sym_off + i * jt_entry_size); + (unsigned long long)jt[i], sym_off + i * jt_entry_size); err = -EINVAL; goto err_close; } @@ -6457,7 +6561,7 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) } break; default: - pr_warn("prog '%s': relo #%d: bad relo type %d\n", + pr_warn("prog '%s': relo #%d: bad relo type %u\n", prog->name, i, relo->type); return -EINVAL; } @@ -6737,7 +6841,7 @@ bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, */ continue; if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) { - pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n", + pr_warn("prog '%s': unexpected relo for insn #%zu, type %u\n", prog->name, insn_idx, relo->type); return -LIBBPF_ERRNO__RELOC; } @@ -6884,7 +6988,7 @@ bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, * +-----------+------+------+ * * At this point, we relocate subA calls, then go one level up and finish with - * relocatin mainA calls. mainA is done. + * relocation mainA calls. mainA is done. * * For mainB process is similar but results in different order. We start with * mainB and skip subA and subB, as mainB never calls them (at least @@ -7527,7 +7631,7 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj, } name = elf_sym_str(obj, sym->st_name) ?: "<?>"; - pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n", + pr_debug(".maps relo #%d: for %zd value %zu rel->r_offset %zu name %u ('%s')\n", i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value, (size_t)rel->r_offset, sym->st_name, name); @@ -7618,7 +7722,7 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj, } map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog; - pr_debug(".maps relo #%d: map '%s' slot [%d] points to %s '%s'\n", + pr_debug(".maps relo #%d: map '%s' slot [%u] points to %s '%s'\n", i, map->name, moff, type, name); } @@ -7712,6 +7816,69 @@ static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name, int *btf_obj_fd, int *btf_type_id); +static inline bool is_tracing_multi(enum bpf_attach_type type) +{ + return type == BPF_TRACE_FENTRY_MULTI || type == BPF_TRACE_FEXIT_MULTI || + type == BPF_TRACE_FSESSION_MULTI; +} + +static const struct module_btf *find_attach_module(struct bpf_object *obj, const char *attach) +{ + const char *sep, *mod_name = NULL; + int i, mod_len, err; + + /* + * We expect attach string in the form of either + * - function_pattern or + * - <module>:function_pattern + */ + sep = strchr(attach, ':'); + if (sep) { + mod_name = attach; + mod_len = sep - mod_name; + } + if (!mod_name) + return NULL; + + err = load_module_btfs(obj); + if (err) + return NULL; + + for (i = 0; i < obj->btf_module_cnt; i++) { + const struct module_btf *mod = &obj->btf_modules[i]; + + if (strncmp(mod->name, mod_name, mod_len) == 0 && mod->name[mod_len] == '\0') + return mod; + } + return NULL; +} + +static int tracing_multi_mod_fd(struct bpf_program *prog, int *btf_obj_fd) +{ + const char *attach_name, *sep; + const struct module_btf *mod; + + *btf_obj_fd = 0; + attach_name = strchr(prog->sec_name, '/'); + + /* Program with no details in spec, using kernel btf. */ + if (!attach_name) + return 0; + + /* Program with no module section, using kernel btf. */ + sep = strchr(++attach_name, ':'); + if (!sep) + return 0; + + /* Program with module specified, get its btf fd. */ + mod = find_attach_module(prog->obj, attach_name); + if (!mod) + return -EINVAL; + + *btf_obj_fd = mod->fd; + return 0; +} + /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */ static int libbpf_prepare_prog_load(struct bpf_program *prog, struct bpf_prog_load_opts *opts, long cookie) @@ -7768,13 +7935,25 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, prog->attach_btf_id = btf_type_id; /* but by now libbpf common logic is not utilizing - * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because + * prog->attach_btf_obj_fd/prog->attach_btf_id anymore because * this callback is called after opts were populated by * libbpf, so this callback has to update opts explicitly here */ opts->attach_btf_obj_fd = btf_obj_fd; opts->attach_btf_id = btf_type_id; } + + if (is_tracing_multi(prog->expected_attach_type)) { + int err, btf_obj_fd = 0; + + err = tracing_multi_mod_fd(prog, &btf_obj_fd); + if (err < 0) + return err; + + prog->attach_btf_obj_fd = btf_obj_fd; + opts->attach_btf_obj_fd = btf_obj_fd; + } + return 0; } @@ -8603,7 +8782,7 @@ static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj, local_name = btf__name_by_offset(obj->btf, local_type->name_off); targ_name = btf__name_by_offset(btf, targ_type->name_off); - pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n", + pr_warn("extern (var ksym) '%s': incompatible types, expected [%u] %s %s, but kernel has [%u] %s %s\n", ext->name, local_type_id, btf_kind_str(local_type), local_name, targ_type_id, btf_kind_str(targ_type), targ_name); @@ -8780,7 +8959,7 @@ static int bpf_object__resolve_externs(struct bpf_object *obj, if (err) return err; pr_debug("extern (kcfg) '%s': set to 0x%llx\n", - ext->name, (long long)value); + ext->name, (unsigned long long)value); } else { pr_warn("extern '%s': unrecognized extern kind\n", ext->name); return -EINVAL; @@ -8881,13 +9060,10 @@ static void bpf_object_unpin(struct bpf_object *obj) bpf_map__unpin(&obj->maps[i], NULL); } -static void bpf_object_post_load_cleanup(struct bpf_object *obj) +static void bpf_object_cleanup_btf(struct bpf_object *obj) { int i; - /* clean up fd_array */ - zfree(&obj->fd_array); - /* clean up module BTFs */ for (i = 0; i < obj->btf_module_cnt; i++) { close(obj->btf_modules[i].fd); @@ -8895,6 +9071,8 @@ static void bpf_object_post_load_cleanup(struct bpf_object *obj) free(obj->btf_modules[i].name); } obj->btf_module_cnt = 0; + obj->btf_module_cap = 0; + obj->btf_modules_loaded = false; zfree(&obj->btf_modules); /* clean up vmlinux BTF */ @@ -8902,6 +9080,15 @@ static void bpf_object_post_load_cleanup(struct bpf_object *obj) obj->btf_vmlinux = NULL; } +static void bpf_object_post_load_cleanup(struct bpf_object *obj) +{ + /* clean up fd_array */ + zfree(&obj->fd_array); + + /* clean up BTF */ + bpf_object_cleanup_btf(obj); +} + static int bpf_object_prepare(struct bpf_object *obj, const char *target_btf_path) { int err; @@ -9802,6 +9989,111 @@ __u32 bpf_program__line_info_cnt(const struct bpf_program *prog) return prog->line_info_cnt; } +int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts *opts) +{ + LIBBPF_OPTS(bpf_prog_load_opts, attr); + struct bpf_object *obj; + const void *info; + __u32 info_cnt, info_rec_size; + int err, fd, prog_btf_fd; + + if (!prog) + return libbpf_err(-EINVAL); + + if (!OPTS_VALID(opts, bpf_prog_load_opts)) + return libbpf_err(-EINVAL); + + obj = prog->obj; + if (obj->state < OBJ_PREPARED) + return libbpf_err(-EINVAL); + + /* + * Caller-provided opts take priority; fall back to + * prog/object defaults when the caller leaves them zero. + */ + attr.attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0) ?: prog->attach_prog_fd; + attr.prog_flags = OPTS_GET(opts, prog_flags, 0) ?: prog->prog_flags; + attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0) ?: prog->prog_ifindex; + attr.kern_version = OPTS_GET(opts, kern_version, 0) ?: obj->kern_version; + attr.fd_array = OPTS_GET(opts, fd_array, NULL) ?: obj->fd_array; + attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0) ?: obj->fd_array_cnt; + attr.token_fd = OPTS_GET(opts, token_fd, 0) ?: obj->token_fd; + if (attr.token_fd) + attr.prog_flags |= BPF_F_TOKEN_FD; + + prog_btf_fd = OPTS_GET(opts, prog_btf_fd, 0); + if (!prog_btf_fd && obj->btf) + prog_btf_fd = btf__fd(obj->btf); + + /* BTF func/line info: only pass if kernel supports it */ + if (kernel_supports(obj, FEAT_BTF_FUNC) && prog_btf_fd > 0) { + attr.prog_btf_fd = prog_btf_fd; + + /* func_info/line_info triples: all-or-nothing from caller */ + info = OPTS_GET(opts, func_info, NULL); + info_cnt = OPTS_GET(opts, func_info_cnt, 0); + info_rec_size = OPTS_GET(opts, func_info_rec_size, 0); + if (!!info != !!info_cnt || !!info != !!info_rec_size) { + pr_warn("prog '%s': func_info, func_info_cnt, and func_info_rec_size must all be specified or all omitted\n", + prog->name); + return libbpf_err(-EINVAL); + } + attr.func_info = info ?: prog->func_info; + attr.func_info_cnt = info ? info_cnt : prog->func_info_cnt; + attr.func_info_rec_size = info ? info_rec_size : prog->func_info_rec_size; + + info = OPTS_GET(opts, line_info, NULL); + info_cnt = OPTS_GET(opts, line_info_cnt, 0); + info_rec_size = OPTS_GET(opts, line_info_rec_size, 0); + if (!!info != !!info_cnt || !!info != !!info_rec_size) { + pr_warn("prog '%s': line_info, line_info_cnt, and line_info_rec_size must all be specified or all omitted\n", + prog->name); + return libbpf_err(-EINVAL); + } + attr.line_info = info ?: prog->line_info; + attr.line_info_cnt = info ? info_cnt : prog->line_info_cnt; + attr.line_info_rec_size = info ? info_rec_size : prog->line_info_rec_size; + } + + /* Logging is caller-controlled; no fallback to prog/obj log settings */ + attr.log_buf = OPTS_GET(opts, log_buf, NULL); + attr.log_size = OPTS_GET(opts, log_size, 0); + attr.log_level = OPTS_GET(opts, log_level, 0); + + /* + * Fields below may be mutated by prog_prepare_load_fn: + * Seed them from prog/obj defaults here; + * Later override with caller-provided opts. + */ + attr.expected_attach_type = prog->expected_attach_type; + attr.attach_btf_id = prog->attach_btf_id; + attr.attach_btf_obj_fd = prog->attach_btf_obj_fd; + + if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) { + err = prog->sec_def->prog_prepare_load_fn(prog, &attr, prog->sec_def->cookie); + if (err) + return libbpf_err(err); + } + + /* Re-apply caller overrides for output fields */ + if (OPTS_GET(opts, expected_attach_type, 0)) + attr.expected_attach_type = OPTS_GET(opts, expected_attach_type, 0); + if (OPTS_GET(opts, attach_btf_id, 0)) + attr.attach_btf_id = OPTS_GET(opts, attach_btf_id, 0); + if (OPTS_GET(opts, attach_btf_obj_fd, 0)) + attr.attach_btf_obj_fd = OPTS_GET(opts, attach_btf_obj_fd, 0); + + /* + * Unlike bpf_object_load_prog(), we intentionally do not call bpf_prog_bind_map() + * for RODATA maps here to avoid mutating the object's state. Callers can bind the + * required maps themselves using bpf_prog_bind_map(). + */ + fd = bpf_prog_load(prog->type, prog->name, obj->license, prog->insns, prog->insns_cnt, + &attr); + + return libbpf_err(fd); +} + #define SEC_DEF(sec_pfx, ptype, atype, flags, ...) { \ .sec = (char *)sec_pfx, \ .prog_type = BPF_PROG_TYPE_##ptype, \ @@ -9823,6 +10115,7 @@ static int attach_kprobe_session(const struct bpf_program *prog, long cookie, st static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link); static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link); static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link); +static int attach_tracing_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link); static const struct bpf_sec_def section_defs[] = { SEC_DEF("socket", SOCKET_FILTER, 0, SEC_NONE), @@ -9858,11 +10151,16 @@ static const struct bpf_sec_def section_defs[] = { SEC_DEF("netkit/peer", SCHED_CLS, BPF_NETKIT_PEER, SEC_NONE), SEC_DEF("tracepoint+", TRACEPOINT, 0, SEC_NONE, attach_tp), SEC_DEF("tp+", TRACEPOINT, 0, SEC_NONE, attach_tp), + SEC_DEF("tracepoint.s+", TRACEPOINT, 0, SEC_SLEEPABLE, attach_tp), + SEC_DEF("tp.s+", TRACEPOINT, 0, SEC_SLEEPABLE, attach_tp), SEC_DEF("raw_tracepoint+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp), SEC_DEF("raw_tp+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp), + SEC_DEF("raw_tracepoint.s+", RAW_TRACEPOINT, 0, SEC_SLEEPABLE, attach_raw_tp), + SEC_DEF("raw_tp.s+", RAW_TRACEPOINT, 0, SEC_SLEEPABLE, attach_raw_tp), SEC_DEF("raw_tracepoint.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp), SEC_DEF("raw_tp.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp), SEC_DEF("tp_btf+", TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF, attach_trace), + SEC_DEF("tp_btf.s+", TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), SEC_DEF("fentry+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF, attach_trace), SEC_DEF("fmod_ret+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF, attach_trace), SEC_DEF("fexit+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF, attach_trace), @@ -9871,6 +10169,12 @@ static const struct bpf_sec_def section_defs[] = { SEC_DEF("fexit.s+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), SEC_DEF("fsession+", TRACING, BPF_TRACE_FSESSION, SEC_ATTACH_BTF, attach_trace), SEC_DEF("fsession.s+", TRACING, BPF_TRACE_FSESSION, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), + SEC_DEF("fsession.multi+", TRACING, BPF_TRACE_FSESSION_MULTI, 0, attach_tracing_multi), + SEC_DEF("fsession.multi.s+", TRACING, BPF_TRACE_FSESSION_MULTI, SEC_SLEEPABLE, attach_tracing_multi), + SEC_DEF("fentry.multi+", TRACING, BPF_TRACE_FENTRY_MULTI, 0, attach_tracing_multi), + SEC_DEF("fexit.multi+", TRACING, BPF_TRACE_FEXIT_MULTI, 0, attach_tracing_multi), + SEC_DEF("fentry.multi.s+", TRACING, BPF_TRACE_FENTRY_MULTI, SEC_SLEEPABLE, attach_tracing_multi), + SEC_DEF("fexit.multi.s+", TRACING, BPF_TRACE_FEXIT_MULTI, SEC_SLEEPABLE, attach_tracing_multi), SEC_DEF("freplace+", EXT, 0, SEC_ATTACH_BTF, attach_trace), SEC_DEF("lsm+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm), SEC_DEF("lsm.s+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm), @@ -10234,7 +10538,7 @@ static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, moff = rel->r_offset - map->sec_offset; shdr_idx = sym->st_shndx; st_ops = map->st_ops; - pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %d (\'%s\')\n", + pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %u (\'%s\')\n", map->name, (long long)(rel->r_info >> 32), (long long)sym->st_value, @@ -10383,7 +10687,7 @@ static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd, int t memset(&info, 0, info_len); err = bpf_prog_get_info_by_fd(attach_prog_fd, &info, &info_len); if (err) { - pr_warn("failed bpf_prog_get_info_by_fd for FD %d: %s\n", + pr_warn("failed bpf_prog_get_info_by_fd for FD %u: %s\n", attach_prog_fd, errstr(err)); return err; } @@ -10396,7 +10700,7 @@ static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd, int t btf = btf_load_from_kernel(info.btf_id, NULL, token_fd); err = libbpf_get_error(btf); if (err) { - pr_warn("Failed to get BTF %d of the program: %s\n", info.btf_id, errstr(err)); + pr_warn("Failed to get BTF %u of the program: %s\n", info.btf_id, errstr(err)); goto out; } err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); @@ -10478,7 +10782,7 @@ static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attac } err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd, prog->obj->token_fd); if (err < 0) { - pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %s\n", + pr_warn("prog '%s': failed to find BPF program (FD %u) BTF ID for '%s': %s\n", prog->name, attach_prog_fd, attach_name, errstr(err)); return err; } @@ -10551,11 +10855,16 @@ static bool map_uses_real_name(const struct bpf_map *map) * such map's corresponding ELF section name as a map name. * This check distinguishes .data/.rodata from .data.* and .rodata.* * maps to know which name has to be returned to the user. + * Map name of the custom .percpu.* maps might be truncated to + * BPF_OBJ_NAME_LEN-1 chars in internal_map_name(). Hence, percpu data + * maps must use real name for their user-visible name. */ if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0) return true; if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0) return true; + if (map->libbpf_type == LIBBPF_MAP_PERCPU) + return true; return false; } @@ -10720,7 +11029,8 @@ int bpf_map__set_value_size(struct bpf_map *map, __u32 size) size_t mmap_old_sz, mmap_new_sz; int err; - if (map->def.type != BPF_MAP_TYPE_ARRAY) + if (map->def.type != BPF_MAP_TYPE_ARRAY && + map->def.type != BPF_MAP_TYPE_PERCPU_ARRAY) return libbpf_err(-EOPNOTSUPP); mmap_old_sz = bpf_map_mmap_sz(map); @@ -10973,7 +11283,7 @@ static int validate_map_op(const struct bpf_map *map, size_t key_sz, } if (value_sz != num_cpu * elem_sz) { - pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n", + pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zu\n", map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz); return -EINVAL; } @@ -11514,7 +11824,7 @@ static void gen_probe_legacy_event_name(char *buf, size_t buf_sz, static int index = 0; int i; - snprintf(buf, buf_sz, "libbpf_%u_%d_%s_0x%zx", getpid(), + snprintf(buf, buf_sz, "libbpf_%d_%d_%s_0x%zx", getpid(), __sync_fetch_and_add(&index, 1), name, offset); /* sanitize name in the probe name */ @@ -11692,6 +12002,8 @@ bpf_program__attach_kprobe_opts(const struct bpf_program *prog, default: return libbpf_err_ptr(-EINVAL); } + if (!func_name && legacy) + return libbpf_err_ptr(-EOPNOTSUPP); if (!legacy) { pfd = perf_event_open_probe(false /* uprobe */, retprobe, @@ -11711,21 +12023,21 @@ bpf_program__attach_kprobe_opts(const struct bpf_program *prog, offset, -1 /* pid */); } if (pfd < 0) { - err = -errno; - pr_warn("prog '%s': failed to create %s '%s+0x%zx' perf event: %s\n", + err = pfd; + pr_warn("prog '%s': failed to create %s '%s%s0x%zx' perf event: %s\n", prog->name, retprobe ? "kretprobe" : "kprobe", - func_name, offset, - errstr(err)); + func_name ?: "", func_name ? "+" : "", + offset, errstr(err)); goto err_out; } link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); err = libbpf_get_error(link); if (err) { close(pfd); - pr_warn("prog '%s': failed to attach to %s '%s+0x%zx': %s\n", + pr_warn("prog '%s': failed to attach to %s '%s%s0x%zx': %s\n", prog->name, retprobe ? "kretprobe" : "kprobe", - func_name, offset, - errstr(err)); + func_name ?: "", func_name ? "+" : "", + offset, errstr(err)); goto err_clean_legacy; } if (legacy) { @@ -12041,7 +12353,16 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, if (addrs && syms) return libbpf_err_ptr(-EINVAL); - if (pattern) { + /* + * Exact function name (no wildcards) without unique_match: + * bypass kallsyms parsing and pass the symbol directly to the + * kernel via syms[] array. When unique_match is set, fall + * through to the slow path which detects duplicate symbols. + */ + if (pattern && !strpbrk(pattern, "*?") && !unique_match) { + syms = &pattern; + cnt = 1; + } else if (pattern) { if (has_available_filter_functions_addrs()) err = libbpf_available_kprobes_parse(&res); else @@ -12084,6 +12405,14 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, link_fd = bpf_link_create(prog_fd, 0, attach_type, &lopts); if (link_fd < 0) { err = -errno; + /* + * Normalize error code: when exact name bypasses kallsyms + * parsing, kernel returns ESRCH from ftrace_lookup_symbols(). + * Convert to ENOENT for API consistency with the pattern + * matching path which returns ENOENT from userspace. + */ + if (err == -ESRCH) + err = -ENOENT; pr_warn("prog '%s': failed to attach: %s\n", prog->name, errstr(err)); goto error; @@ -12101,7 +12430,7 @@ error: static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link) { DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts); - unsigned long offset = 0; + long offset = 0; const char *func_name; char *func; int n; @@ -12123,6 +12452,13 @@ static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf pr_warn("kprobe name is invalid: %s\n", func_name); return -EINVAL; } + + if (offset < 0) { + free(func); + pr_warn("kprobe offset must be a non-negative integer: %li\n", offset); + return -EINVAL; + } + if (opts.retprobe && offset != 0) { free(func); pr_warn("kretprobes do not support offset specification\n"); @@ -12246,6 +12582,279 @@ static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, stru return ret; } +#define MAX_BPF_FUNC_ARGS 12 + +static bool btf_type_is_modifier(const struct btf_type *t) +{ + switch (BTF_INFO_KIND(t->info)) { + case BTF_KIND_TYPEDEF: + case BTF_KIND_VOLATILE: + case BTF_KIND_CONST: + case BTF_KIND_RESTRICT: + case BTF_KIND_TYPE_TAG: + return true; + default: + return false; + } +} + +#define MAX_RESOLVE_DEPTH 32 + +static int btf_get_type_size(const struct btf *btf, __u32 type_id, + const struct btf_type **ret_type) +{ + const struct btf_type *t; + int i; + + *ret_type = btf__type_by_id(btf, 0); + if (!type_id) + return 0; + t = btf__type_by_id(btf, type_id); + for (i = 0; i < MAX_RESOLVE_DEPTH && t && btf_type_is_modifier(t); i++) + t = btf__type_by_id(btf, t->type); + if (!t || i == MAX_RESOLVE_DEPTH) + return -EINVAL; + *ret_type = t; + if (btf_is_ptr(t)) + return btf__pointer_size(btf); + if (btf_is_int(t) || btf_is_any_enum(t) || btf_is_struct(t) || btf_is_union(t)) + return t->size; + return -EINVAL; +} + +bool btf_type_is_traceable_func(const struct btf *btf, const struct btf_type *t) +{ + const struct btf_param *args; + const struct btf_type *proto; + __u32 i, nargs; + int ret; + + if (!btf_is_func(t)) + return false; + proto = btf__type_by_id(btf, t->type); + if (!proto || !btf_is_func_proto(proto)) + return false; + + args = (const struct btf_param *)(proto + 1); + nargs = btf_vlen(proto); + if (nargs > MAX_BPF_FUNC_ARGS) + return false; + + /* No support for struct return type. */ + ret = btf_get_type_size(btf, proto->type, &t); + if (ret < 0 || btf_is_struct(t) || btf_is_union(t)) + return false; + + for (i = 0; i < nargs; i++) { + /* No support for variable args. */ + if (i == nargs - 1 && args[i].type == 0) + return false; + ret = btf_get_type_size(btf, args[i].type, &t); + /* No support of struct argument size greater than 16 bytes. */ + if (ret < 0 || ret > 16) + return false; + /* No support for void argument. */ + if (ret == 0) + return false; + } + + return true; +} + +static int +collect_btf_func_ids_by_glob(const struct btf *btf, const char *pattern, __u32 **ids) +{ + __u32 type_id, nr_types = btf__type_cnt(btf); + size_t cap = 0, cnt = 0; + + if (!pattern) + return -EINVAL; + + for (type_id = 1; type_id < nr_types; type_id++) { + const struct btf_type *t = btf__type_by_id(btf, type_id); + const char *name; + int err; + + if (btf_kind(t) != BTF_KIND_FUNC) + continue; + name = btf__name_by_offset(btf, t->name_off); + if (!name) + continue; + + if (!glob_match(name, pattern)) + continue; + if (!btf_type_is_traceable_func(btf, t)) + continue; + + err = libbpf_ensure_mem((void **) ids, &cap, sizeof(**ids), cnt + 1); + if (err) { + free(*ids); + return -ENOMEM; + } + (*ids)[cnt++] = type_id; + } + + return cnt; +} + +static int collect_func_ids_by_glob(const struct bpf_program *prog, const char *pattern, __u32 **ids) +{ + struct bpf_object *obj = prog->obj; + const struct module_btf *mod; + struct btf *btf = NULL; + const char *sep; + int err; + + err = bpf_object__load_vmlinux_btf(obj, true); + if (err) + return err; + + /* In case we have module specified, we will find its btf and use that. */ + sep = strchr(pattern, ':'); + if (sep) { + mod = find_attach_module(obj, pattern); + if (!mod) { + err = -EINVAL; + goto cleanup; + } + btf = mod->btf; + pattern = sep + 1; + } else { + /* Program is loaded for kernel module. */ + if (prog->attach_btf_obj_fd) { + err = -EINVAL; + goto cleanup; + } + btf = obj->btf_vmlinux; + } + + err = collect_btf_func_ids_by_glob(btf, pattern, ids); + +cleanup: + bpf_object_cleanup_btf(obj); + return err; +} + +struct bpf_link * +bpf_program__attach_tracing_multi(const struct bpf_program *prog, const char *pattern, + const struct bpf_tracing_multi_opts *opts) +{ + LIBBPF_OPTS(bpf_link_create_opts, lopts); + int prog_fd, link_fd, err, cnt; + __u32 *free_ids = NULL; + struct bpf_link *link; + const __u64 *cookies; + const __u32 *ids; + + if (!OPTS_VALID(opts, bpf_tracing_multi_opts)) + return libbpf_err_ptr(-EINVAL); + + prog_fd = bpf_program__fd(prog); + if (prog_fd < 0) { + pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n", + prog->name); + return libbpf_err_ptr(-EINVAL); + } + + cnt = OPTS_GET(opts, cnt, 0); + ids = OPTS_GET(opts, ids, NULL); + cookies = OPTS_GET(opts, cookies, NULL); + + if (!!ids != !!cnt) + return libbpf_err_ptr(-EINVAL); + if (pattern && (ids || cookies)) + return libbpf_err_ptr(-EINVAL); + if (!pattern && !ids) + return libbpf_err_ptr(-EINVAL); + + if (pattern) { + cnt = collect_func_ids_by_glob(prog, pattern, &free_ids); + if (cnt < 0) + return libbpf_err_ptr(cnt); + if (cnt == 0) + return libbpf_err_ptr(-EINVAL); + ids = (const __u32 *) free_ids; + } + + lopts.tracing_multi.ids = ids; + lopts.tracing_multi.cookies = cookies; + lopts.tracing_multi.cnt = cnt; + + link = calloc(1, sizeof(*link)); + if (!link) { + err = -ENOMEM; + goto error; + } + link->detach = &bpf_link__detach_fd; + + link_fd = bpf_link_create(prog_fd, 0, prog->expected_attach_type, &lopts); + if (link_fd < 0) { + err = -errno; + pr_warn("prog '%s': failed to attach: %s\n", prog->name, errstr(err)); + goto error; + } + link->fd = link_fd; + free(free_ids); + return link; + +error: + free(link); + free(free_ids); + return libbpf_err_ptr(err); +} + +static int attach_tracing_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link) +{ + static const char *const prefixes[] = { + "fentry.multi", + "fexit.multi", + "fsession.multi", + "fentry.multi.s", + "fexit.multi.s", + "fsession.multi.s", + }; + const char *spec = NULL; + char *pattern; + size_t i; + int n; + + *link = NULL; + + for (i = 0; i < ARRAY_SIZE(prefixes); i++) { + size_t pfx_len; + + if (!str_has_pfx(prog->sec_name, prefixes[i])) + continue; + + pfx_len = strlen(prefixes[i]); + /* no auto-attach case of, e.g., SEC("fentry.multi") */ + if (prog->sec_name[pfx_len] == '\0') + return 0; + + if (prog->sec_name[pfx_len] != '/') + continue; + + spec = prog->sec_name + pfx_len + 1; + break; + } + + if (!spec) { + pr_warn("prog '%s': invalid section name '%s'\n", + prog->name, prog->sec_name); + return -EINVAL; + } + + n = sscanf(spec, "%m[a-zA-Z0-9_.*?:]", &pattern); + if (n < 1) { + pr_warn("tracing multi pattern is invalid: %s\n", spec); + return -EINVAL; + } + + *link = bpf_program__attach_tracing_multi(prog, pattern, NULL); + free(pattern); + return libbpf_get_error(*link); +} + static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe, const char *binary_path, size_t offset) { @@ -12365,8 +12974,8 @@ static long elf_find_func_offset_from_archive(const char *archive_path, const ch ret = elf_find_func_offset(elf, file_name, func_name); if (ret > 0) { pr_debug("elf: symbol address match for %s of %s in %s: 0x%x + 0x%lx = 0x%lx\n", - func_name, file_name, archive_path, entry.data_offset, ret, - ret + entry.data_offset); + func_name, file_name, archive_path, entry.data_offset, (unsigned long)ret, + (unsigned long)(ret + entry.data_offset)); ret += entry.data_offset; } elf_end(elf); @@ -12414,13 +13023,14 @@ static const char *arch_specific_lib_paths(void) /* Get full path to program/shared library. */ static int resolve_full_path(const char *file, char *result, size_t result_sz) { - const char *search_paths[3] = {}; + const char *search_paths[4] = {}; int i, perm; if (str_has_sfx(file, ".so") || strstr(file, ".so.")) { search_paths[0] = getenv("LD_LIBRARY_PATH"); search_paths[1] = "/usr/lib64:/usr/lib"; search_paths[2] = arch_specific_lib_paths(); + search_paths[3] = "/lib64:/lib"; perm = R_OK; } else { search_paths[0] = getenv("PATH"); @@ -12684,7 +13294,7 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, binary_path, func_offset, pid); } if (pfd < 0) { - err = -errno; + err = pfd; pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n", prog->name, retprobe ? "uretprobe" : "uprobe", binary_path, func_offset, @@ -12966,25 +13576,61 @@ struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog, return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL); } +/* + * Match section name against a prefix array. Returns pointer past + * "prefix/" on match, empty string for bare sections (exact prefix + * match), or NULL if no prefix matches. + */ +static const char *sec_name_match_prefix(const char *sec_name, + const char *const *prefixes, + size_t n) +{ + size_t i; + + for (i = 0; i < n; i++) { + size_t pfx_len; + + if (!str_has_pfx(sec_name, prefixes[i])) + continue; + + pfx_len = strlen(prefixes[i]); + if (sec_name[pfx_len] == '\0') + return sec_name + pfx_len; + + if (sec_name[pfx_len] != '/' || sec_name[pfx_len + 1] == '\0') + continue; + + return sec_name + pfx_len + 1; + } + return NULL; +} + static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link) { + static const char *const prefixes[] = { + "tp.s", + "tp", + "tracepoint.s", + "tracepoint", + }; char *sec_name, *tp_cat, *tp_name; + const char *match; *link = NULL; - /* no auto-attach for SEC("tp") or SEC("tracepoint") */ - if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0) + match = sec_name_match_prefix(prog->sec_name, prefixes, ARRAY_SIZE(prefixes)); + if (!match) { + pr_warn("prog '%s': invalid section name '%s'\n", prog->name, prog->sec_name); + return -EINVAL; + } + if (!match[0]) /* bare section name no autoattach */ return 0; sec_name = strdup(prog->sec_name); if (!sec_name) return -ENOMEM; - /* extract "tp/<category>/<name>" or "tracepoint/<category>/<name>" */ - if (str_has_pfx(prog->sec_name, "tp/")) - tp_cat = sec_name + sizeof("tp/") - 1; - else - tp_cat = sec_name + sizeof("tracepoint/") - 1; + tp_cat = sec_name + (match - prog->sec_name); tp_name = strchr(tp_cat, '/'); if (!tp_name) { free(sec_name); @@ -13048,37 +13694,22 @@ static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf "raw_tracepoint", "raw_tp.w", "raw_tracepoint.w", + "raw_tp.s", + "raw_tracepoint.s", }; - size_t i; - const char *tp_name = NULL; + const char *match; *link = NULL; - for (i = 0; i < ARRAY_SIZE(prefixes); i++) { - size_t pfx_len; - - if (!str_has_pfx(prog->sec_name, prefixes[i])) - continue; - - pfx_len = strlen(prefixes[i]); - /* no auto-attach case of, e.g., SEC("raw_tp") */ - if (prog->sec_name[pfx_len] == '\0') - return 0; - - if (prog->sec_name[pfx_len] != '/') - continue; - - tp_name = prog->sec_name + pfx_len + 1; - break; - } - - if (!tp_name) { - pr_warn("prog '%s': invalid section name '%s'\n", - prog->name, prog->sec_name); + match = sec_name_match_prefix(prog->sec_name, prefixes, ARRAY_SIZE(prefixes)); + if (!match) { + pr_warn("prog '%s': invalid section name '%s'\n", prog->name, prog->sec_name); return -EINVAL; } + if (!match[0]) + return 0; - *link = bpf_program__attach_raw_tracepoint(prog, tp_name); + *link = bpf_program__attach_raw_tracepoint(prog, match); return libbpf_get_error(*link); } @@ -13587,7 +14218,7 @@ perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, if (((void *)ehdr) + ehdr_size > base + mmap_size) { void *copy_start = ehdr; size_t len_first = base + mmap_size - copy_start; - size_t len_secnd = ehdr_size - len_first; + size_t len_second = ehdr_size - len_first; if (*copy_size < ehdr_size) { free(*copy_mem); @@ -13601,7 +14232,7 @@ perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, } memcpy(*copy_mem, copy_start, len_first); - memcpy(*copy_mem + len_first, base, len_secnd); + memcpy(*copy_mem + len_first, base, len_second); ehdr = *copy_mem; } @@ -13619,7 +14250,7 @@ struct perf_buffer; struct perf_buffer_params { struct perf_event_attr *attr; - /* if event_cb is specified, it takes precendence */ + /* if event_cb is specified, it takes precedence */ perf_buffer_event_fn event_cb; /* sample_cb and lost_cb are higher-level common-case callbacks */ perf_buffer_sample_fn sample_cb; @@ -13990,7 +14621,7 @@ perf_buffer__process_record(struct perf_event_header *e, void *ctx) break; } default: - pr_warn("unknown perf sample type %d\n", e->type); + pr_warn("unknown perf sample type %u\n", e->type); return LIBBPF_PERF_EVENT_ERROR; } return LIBBPF_PERF_EVENT_CONT; diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index dfc37a615578..b965ad571540 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -557,7 +557,7 @@ struct bpf_kprobe_opts { size_t sz; /* custom user-provided value fetchable through bpf_get_attach_cookie() */ __u64 bpf_cookie; - /* function's offset to install kprobe to */ + /* function offset, or raw address if func_name == NULL */ size_t offset; /* kprobe is return probe */ bool retprobe; @@ -565,11 +565,36 @@ struct bpf_kprobe_opts { enum probe_attach_mode attach_mode; size_t :0; }; + #define bpf_kprobe_opts__last_field attach_mode +/** + * @brief **bpf_program__attach_kprobe()** attaches a BPF program to a + * kernel function entry or return. + * + * @param prog BPF program to attach + * @param retprobe Attach to function return + * @param func_name Name of the kernel function to attach to + * @return Reference to the newly created BPF link; or NULL is returned on + * error, error code is stored in errno + */ LIBBPF_API struct bpf_link * bpf_program__attach_kprobe(const struct bpf_program *prog, bool retprobe, const char *func_name); + +/** + * @brief **bpf_program__attach_kprobe_opts()** is just like + * bpf_program__attach_kprobe() except with an options struct + * for various configurations. + * + * @param prog BPF program to attach + * @param func_name Name of the kernel function to attach to. If NULL, + * opts->offset is treated as a raw kernel address. Raw-address attach + * is supported with PROBE_ATTACH_MODE_PERF and PROBE_ATTACH_MODE_LINK. + * @param opts Options for altering program attachment + * @return Reference to the newly created BPF link; or NULL is returned on + * error, error code is stored in errno + */ LIBBPF_API struct bpf_link * bpf_program__attach_kprobe_opts(const struct bpf_program *prog, const char *func_name, @@ -701,6 +726,21 @@ bpf_program__attach_ksyscall(const struct bpf_program *prog, const char *syscall_name, const struct bpf_ksyscall_opts *opts); +struct bpf_tracing_multi_opts { + /* size of this struct, for forward/backward compatibility */ + size_t sz; + const __u32 *ids; + const __u64 *cookies; + size_t cnt; + size_t :0; +}; + +#define bpf_tracing_multi_opts__last_field cnt + +LIBBPF_API struct bpf_link * +bpf_program__attach_tracing_multi(const struct bpf_program *prog, const char *pattern, + const struct bpf_tracing_multi_opts *opts); + struct bpf_uprobe_opts { /* size of this struct, for forward/backward compatibility */ size_t sz; @@ -2021,6 +2061,23 @@ LIBBPF_API int libbpf_register_prog_handler(const char *sec, */ LIBBPF_API int libbpf_unregister_prog_handler(int handler_id); +/** + * @brief **bpf_program__clone()** loads a single BPF program from a prepared + * BPF object into the kernel, returning its file descriptor. + * + * The BPF object must have been previously prepared with + * **bpf_object__prepare()**. If @opts is provided, any non-zero field + * overrides the defaults derived from the program/object internals. + * If @opts is NULL, all fields are populated automatically. + * + * The returned FD is owned by the caller and must be closed with close(). + * + * @param prog BPF program from a prepared object + * @param opts Optional load options; non-zero fields override defaults + * @return program FD (>= 0) on success; negative error code on failure + */ +LIBBPF_API int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts *opts); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index d18fbcea7578..08ab2ea881fb 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -455,3 +455,11 @@ LIBBPF_1.7.0 { bpf_program__assoc_struct_ops; btf__permute; } LIBBPF_1.6.0; + +LIBBPF_1.8.0 { + global: + bpf_program__attach_tracing_multi; + bpf_program__clone; + btf__find_by_name_kind_own; + btf__new_empty_opts; +} LIBBPF_1.7.0; diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 974147e8a8aa..4c46d34fc055 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -250,6 +250,7 @@ const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, __u32 id, _ const struct btf_header *btf_header(const struct btf *btf); void btf_set_base_btf(struct btf *btf, const struct btf *base_btf); int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **id_map); +bool btf_type_is_traceable_func(const struct btf *btf, const struct btf_type *t); static inline enum btf_func_linkage btf_func_linkage(const struct btf_type *t) { @@ -394,6 +395,14 @@ enum kern_feature_id { FEAT_BTF_QMARK_DATASEC, /* Kernel supports LDIMM64 imm offsets past 512 MiB. */ FEAT_LDIMM64_FULL_RANGE_OFF, + /* Kernel supports uprobe syscall */ + FEAT_UPROBE_SYSCALL, + /* Kernel supports BTF layout information */ + FEAT_BTF_LAYOUT, + /* Kernel supports BPF syscall common attributes */ + FEAT_BPF_SYSCALL_COMMON_ATTRS, + /* Kernel supports percpu data */ + FEAT_PERCPU_DATA, __FEAT_CNT, }; @@ -410,6 +419,7 @@ struct kern_feature_cache { bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id); bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id); +void bpf_object_set_feat_cache(struct bpf_object *obj, struct kern_feature_cache *cache); int probe_kern_syscall_wrapper(int token_fd); int probe_memcg_account(int token_fd); @@ -420,6 +430,10 @@ int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz); int libbpf__load_raw_btf(const char *raw_types, size_t types_len, const char *str_sec, size_t str_len, int token_fd); +int libbpf__load_raw_btf_hdr(const struct btf_header *hdr, + const char *raw_types, const char *str_sec, + const char *layout_sec, int token_fd); +struct btf *bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *orig_btf); int btf_load_into_kernel(struct btf *btf, char *log_buf, size_t log_sz, __u32 log_level, int token_fd); @@ -584,8 +598,6 @@ typedef int (*type_id_visit_fn)(__u32 *type_id, void *ctx); typedef int (*str_off_visit_fn)(__u32 *str_off, void *ctx); int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx); int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx); -__s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name, - __u32 kind); /* handle direct returned errors */ static inline int libbpf_err(int ret) @@ -756,7 +768,7 @@ int elf_resolve_pattern_offsets(const char *binary_path, const char *pattern, int probe_fd(int fd); #define SHA256_DIGEST_LENGTH 32 -#define SHA256_DWORD_SIZE SHA256_DIGEST_LENGTH / sizeof(__u64) void libbpf_sha256(const void *data, size_t len, __u8 out[SHA256_DIGEST_LENGTH]); +int probe_sys_bpf_ext(void); #endif /* __LIBBPF_LIBBPF_INTERNAL_H */ diff --git a/tools/lib/bpf/libbpf_probes.c b/tools/lib/bpf/libbpf_probes.c index bccf4bb747e1..e40819465ddc 100644 --- a/tools/lib/bpf/libbpf_probes.c +++ b/tools/lib/bpf/libbpf_probes.c @@ -218,18 +218,10 @@ int libbpf_probe_bpf_prog_type(enum bpf_prog_type prog_type, const void *opts) return libbpf_err(ret); } -int libbpf__load_raw_btf(const char *raw_types, size_t types_len, - const char *str_sec, size_t str_len, - int token_fd) +int libbpf__load_raw_btf_hdr(const struct btf_header *hdr, const char *raw_types, + const char *str_sec, const char *layout_sec, + int token_fd) { - struct btf_header hdr = { - .magic = BTF_MAGIC, - .version = BTF_VERSION, - .hdr_len = sizeof(struct btf_header), - .type_len = types_len, - .str_off = types_len, - .str_len = str_len, - }; LIBBPF_OPTS(bpf_btf_load_opts, opts, .token_fd = token_fd, .btf_flags = token_fd ? BPF_F_TOKEN_FD : 0, @@ -237,14 +229,16 @@ int libbpf__load_raw_btf(const char *raw_types, size_t types_len, int btf_fd, btf_len; __u8 *raw_btf; - btf_len = hdr.hdr_len + hdr.type_len + hdr.str_len; + btf_len = hdr->hdr_len + hdr->type_off + hdr->type_len + hdr->str_len + hdr->layout_len; raw_btf = malloc(btf_len); if (!raw_btf) return -ENOMEM; - memcpy(raw_btf, &hdr, sizeof(hdr)); - memcpy(raw_btf + hdr.hdr_len, raw_types, hdr.type_len); - memcpy(raw_btf + hdr.hdr_len + hdr.type_len, str_sec, hdr.str_len); + memcpy(raw_btf, hdr, sizeof(*hdr)); + memcpy(raw_btf + hdr->hdr_len + hdr->type_off, raw_types, hdr->type_len); + memcpy(raw_btf + hdr->hdr_len + hdr->str_off, str_sec, hdr->str_len); + if (layout_sec) + memcpy(raw_btf + hdr->hdr_len + hdr->layout_off, layout_sec, hdr->layout_len); btf_fd = bpf_btf_load(raw_btf, btf_len, &opts); @@ -252,6 +246,22 @@ int libbpf__load_raw_btf(const char *raw_types, size_t types_len, return btf_fd; } +int libbpf__load_raw_btf(const char *raw_types, size_t types_len, + const char *str_sec, size_t str_len, + int token_fd) +{ + struct btf_header hdr = { + .magic = BTF_MAGIC, + .version = BTF_VERSION, + .hdr_len = sizeof(struct btf_header), + .type_len = types_len, + .str_off = types_len, + .str_len = str_len, + }; + + return libbpf__load_raw_btf_hdr(&hdr, raw_types, str_sec, NULL, token_fd); +} + static int load_local_storage_btf(void) { const char strs[] = "\0bpf_spin_lock\0val\0cnt\0l"; @@ -299,6 +309,9 @@ static int probe_map_create(enum bpf_map_type map_type) value_size = sizeof(__u64); opts.map_flags = BPF_F_NO_PREALLOC; break; + case BPF_MAP_TYPE_RHASH: + opts.map_flags = BPF_F_NO_PREALLOC; + break; case BPF_MAP_TYPE_CGROUP_STORAGE: case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: key_size = sizeof(struct bpf_cgroup_storage_key); diff --git a/tools/lib/bpf/libbpf_version.h b/tools/lib/bpf/libbpf_version.h index 99331e317dee..c446c0cd8cf9 100644 --- a/tools/lib/bpf/libbpf_version.h +++ b/tools/lib/bpf/libbpf_version.h @@ -4,6 +4,6 @@ #define __LIBBPF_VERSION_H #define LIBBPF_MAJOR_VERSION 1 -#define LIBBPF_MINOR_VERSION 7 +#define LIBBPF_MINOR_VERSION 8 #endif /* __LIBBPF_VERSION_H */ diff --git a/tools/lib/bpf/nlattr.c b/tools/lib/bpf/nlattr.c index 06663f9ea581..007fe17d17b4 100644 --- a/tools/lib/bpf/nlattr.c +++ b/tools/lib/bpf/nlattr.c @@ -123,7 +123,7 @@ int libbpf_nla_parse(struct nlattr *tb[], int maxtype, struct nlattr *head, if (tb[type]) { pr_warn("Attribute of type %#x found multiple times in message, " - "previous attribute is being ignored.\n", type); + "previous attribute is being ignored.\n", (unsigned)type); } tb[type] = nla; diff --git a/tools/lib/bpf/relo_core.c b/tools/lib/bpf/relo_core.c index 6eea5edba58a..8ad2715721cf 100644 --- a/tools/lib/bpf/relo_core.c +++ b/tools/lib/bpf/relo_core.c @@ -191,8 +191,8 @@ recur: case BTF_KIND_FUNC_PROTO: { struct btf_param *local_p = btf_params(local_type); struct btf_param *targ_p = btf_params(targ_type); - __u16 local_vlen = btf_vlen(local_type); - __u16 targ_vlen = btf_vlen(targ_type); + __u32 local_vlen = btf_vlen(local_type); + __u32 targ_vlen = btf_vlen(targ_type); int i, err; if (local_vlen != targ_vlen) @@ -216,7 +216,7 @@ recur: goto recur; } default: - pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", + pr_warn("unexpected kind %s relocated, local [%u], target [%u]\n", btf_kind_str(local_type), local_id, targ_id); return 0; } @@ -292,6 +292,8 @@ int bpf_core_parse_spec(const char *prog_name, const struct btf *btf, ++spec_str; if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1) return -EINVAL; + if (access_idx < 0) + return -EINVAL; if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) return -E2BIG; spec_str += parsed_len; @@ -382,7 +384,7 @@ int bpf_core_parse_spec(const char *prog_name, const struct btf *btf, return sz; spec->bit_offset += access_idx * sz * 8; } else { - pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n", + pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%u] of unexpected kind %s\n", prog_name, relo->type_id, spec_str, i, id, btf_kind_str(t)); return -EINVAL; } @@ -723,7 +725,7 @@ static int bpf_core_calc_field_relo(const char *prog_name, return -EINVAL; *val = sz; } else { - pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n", + pr_warn("prog '%s': relo %u at insn #%u can't be applied to array access\n", prog_name, relo->kind, relo->insn_off / 8); return -EINVAL; } @@ -745,7 +747,7 @@ static int bpf_core_calc_field_relo(const char *prog_name, while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) { if (byte_sz >= 8) { /* bitfield can't be read with 64-bit read */ - pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n", + pr_warn("prog '%s': relo %u at insn #%u can't be satisfied for bitfield\n", prog_name, relo->kind, relo->insn_off / 8); return -E2BIG; } @@ -969,7 +971,7 @@ done: err = 0; } else if (err == -EOPNOTSUPP) { /* EOPNOTSUPP means unknown/unsupported relocation */ - pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%d) at insn #%d\n", + pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%u) at insn #%u\n", prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind, relo->insn_off / 8); } @@ -1065,7 +1067,7 @@ poison: if (BPF_SRC(insn->code) != BPF_K) return -EINVAL; if (res->validate && insn->imm != orig_val) { - pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %llu -> %llu\n", + pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %d, exp %llu -> %llu\n", prog_name, relo_idx, insn_idx, insn->imm, (unsigned long long)orig_val, (unsigned long long)new_val); @@ -1081,7 +1083,7 @@ poison: case BPF_ST: case BPF_STX: if (res->validate && insn->off != orig_val) { - pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %llu -> %llu\n", + pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %d, exp %llu -> %llu\n", prog_name, relo_idx, insn_idx, insn->off, (unsigned long long)orig_val, (unsigned long long)new_val); return -EINVAL; @@ -1157,7 +1159,7 @@ poison: default: pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n", prog_name, relo_idx, insn_idx, insn->code, - insn->src_reg, insn->dst_reg, insn->off, insn->imm); + (unsigned)insn->src_reg, (unsigned)insn->dst_reg, (unsigned)insn->off, (unsigned)insn->imm); return -EINVAL; } @@ -1321,7 +1323,7 @@ int bpf_core_calc_relo_insn(const char *prog_name, const char *spec_str; spec_str = btf__name_by_offset(local_btf, relo->access_str_off); - pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n", + pr_warn("prog '%s': relo #%d: parsing [%u] %s %s + %s failed: %d\n", prog_name, relo_idx, local_id, btf_kind_str(local_type), str_is_empty(local_name) ? "<anon>" : local_name, spec_str ?: "<?>", err); @@ -1344,7 +1346,7 @@ int bpf_core_calc_relo_insn(const char *prog_name, /* libbpf doesn't support candidate search for anonymous types */ if (str_is_empty(local_name)) { - pr_warn("prog '%s': relo #%d: <%s> (%d) relocation doesn't support anonymous types\n", + pr_warn("prog '%s': relo #%d: <%s> (%u) relocation doesn't support anonymous types\n", prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind); return -EOPNOTSUPP; } @@ -1455,8 +1457,8 @@ static bool bpf_core_names_match(const struct btf *local_btf, size_t local_name_ static int bpf_core_enums_match(const struct btf *local_btf, const struct btf_type *local_t, const struct btf *targ_btf, const struct btf_type *targ_t) { - __u16 local_vlen = btf_vlen(local_t); - __u16 targ_vlen = btf_vlen(targ_t); + __u32 local_vlen = btf_vlen(local_t); + __u32 targ_vlen = btf_vlen(targ_t); int i, j; if (local_t->size != targ_t->size) @@ -1496,8 +1498,8 @@ static int bpf_core_composites_match(const struct btf *local_btf, const struct b bool behind_ptr, int level) { const struct btf_member *local_m = btf_members(local_t); - __u16 local_vlen = btf_vlen(local_t); - __u16 targ_vlen = btf_vlen(targ_t); + __u32 local_vlen = btf_vlen(local_t); + __u32 targ_vlen = btf_vlen(targ_t); int i, j, err; if (local_vlen > targ_vlen) @@ -1672,8 +1674,8 @@ recur: case BTF_KIND_FUNC_PROTO: { struct btf_param *local_p = btf_params(local_t); struct btf_param *targ_p = btf_params(targ_t); - __u16 local_vlen = btf_vlen(local_t); - __u16 targ_vlen = btf_vlen(targ_t); + __u32 local_vlen = btf_vlen(local_t); + __u32 targ_vlen = btf_vlen(targ_t); int i, err; if (local_k != targ_k) @@ -1695,7 +1697,7 @@ recur: goto recur; } default: - pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", + pr_warn("unexpected kind %s relocated, local [%u], target [%u]\n", btf_kind_str(local_t), local_id, targ_id); return 0; } diff --git a/tools/lib/bpf/ringbuf.c b/tools/lib/bpf/ringbuf.c index 00ec4837a06d..c8fe57401a8b 100644 --- a/tools/lib/bpf/ringbuf.c +++ b/tools/lib/bpf/ringbuf.c @@ -244,7 +244,7 @@ static int64_t ringbuf_process_ring(struct ring *r, size_t n) do { got_new_data = false; prod_pos = smp_load_acquire(r->producer_pos); - while (cons_pos < prod_pos) { + while (prod_pos - cons_pos > 0) { len_ptr = r->data + (cons_pos & r->mask); len = smp_load_acquire(len_ptr); diff --git a/tools/lib/bpf/skel_internal.h b/tools/lib/bpf/skel_internal.h index 6a8f5c7a02eb..1f3f332dffbe 100644 --- a/tools/lib/bpf/skel_internal.h +++ b/tools/lib/bpf/skel_internal.h @@ -18,10 +18,6 @@ #include "bpf.h" #endif -#ifndef SHA256_DIGEST_LENGTH -#define SHA256_DIGEST_LENGTH 32 -#endif - #ifndef __NR_bpf # if defined(__mips__) && defined(_ABIO32) # define __NR_bpf 4355 @@ -135,8 +131,10 @@ static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) { if (addr != ~0ULL) kvfree(p); - /* When addr == ~0ULL the 'p' points to - * ((struct bpf_array *)map)->value. See skel_finalize_map_data. + /* + * When addr == ~0ULL the init buffer has already been released. + * For skel_finalize_map_data(), 'p' points to + * ((struct bpf_array *)map)->value. */ } @@ -174,6 +172,15 @@ out: return addr; } +static inline int skel_protect_map_data(void *p, __u64 *init_val, size_t sz) +{ + (void)sz; + + kvfree(p); + *init_val = ~0ULL; + return 0; +} + #else static inline void *skel_alloc(size_t size) @@ -212,6 +219,15 @@ static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int return NULL; return addr; } + +static inline int skel_protect_map_data(void *p, __u64 *init_val, size_t sz) +{ + (void)init_val; + + if (mprotect(p, sz, PROT_READ)) + return -errno; + return 0; +} #endif static inline int skel_closenz(int fd) @@ -243,7 +259,12 @@ static inline int skel_map_create(enum bpf_map_type map_type, attr.excl_prog_hash = (unsigned long) excl_prog_hash; attr.excl_prog_hash_size = excl_prog_hash_sz; +#ifdef __KERNEL__ + if (strscpy(attr.map_name, map_name) < 0) + return -EINVAL; +#else strncpy(attr.map_name, map_name, sizeof(attr.map_name)); +#endif attr.key_size = key_size; attr.value_size = value_size; attr.max_entries = max_entries; @@ -315,25 +336,6 @@ static inline int skel_link_create(int prog_fd, int target_fd, return skel_sys_bpf(BPF_LINK_CREATE, &attr, attr_sz); } -static inline int skel_obj_get_info_by_fd(int fd) -{ - const size_t attr_sz = offsetofend(union bpf_attr, info); - __u8 sha[SHA256_DIGEST_LENGTH]; - struct bpf_map_info info; - __u32 info_len = sizeof(info); - union bpf_attr attr; - - memset(&info, 0, sizeof(info)); - info.hash = (long) &sha; - info.hash_size = SHA256_DIGEST_LENGTH; - - memset(&attr, 0, attr_sz); - attr.info.bpf_fd = fd; - attr.info.info = (long) &info; - attr.info.info_len = info_len; - return skel_sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, attr_sz); -} - static inline int skel_map_freeze(int fd) { const size_t attr_sz = offsetofend(union bpf_attr, map_fd); @@ -379,12 +381,6 @@ static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) set_err; goto out; } - err = skel_obj_get_info_by_fd(map_fd); - if (err < 0) { - opts->errstr = "failed to fetch obj info"; - set_err; - goto out; - } #endif memset(&attr, 0, prog_load_attr_sz); @@ -395,6 +391,8 @@ static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) #ifndef __KERNEL__ attr.signature = (long) opts->signature; attr.signature_size = opts->signature_sz; + if (opts->signature) + attr.fd_array_cnt = 1; #else if (opts->signature || opts->signature_sz) pr_warn("signatures are not supported from bpf_preload\n"); diff --git a/tools/lib/bpf/strset.c b/tools/lib/bpf/strset.c index 2464bcbd04e0..ace73c6b3d62 100644 --- a/tools/lib/bpf/strset.c +++ b/tools/lib/bpf/strset.c @@ -107,6 +107,41 @@ static void *strset_add_str_mem(struct strset *set, size_t add_sz) set->strs_data_len, set->strs_data_max_len, add_sz); } +static long strset_str_append(struct strset *set, const char *s) +{ + uintptr_t old_data = (uintptr_t)set->strs_data; + size_t old_data_len = set->strs_data_len; + uintptr_t old_s = (uintptr_t)s; + long len = strlen(s) + 1; + void *p; + + /* + * Hashmap keys are always offsets within set->strs_data, so to even + * look up some string from the "outside", we need to first append it + * at the end, so that it can be addressed with an offset. Luckily, + * until set->strs_data_len is incremented, that string is just a piece + * of garbage for the rest of the code, so no harm, no foul. On the + * other hand, if the string is unique, it's already appended and + * ready to be used, only a simple set->strs_data_len increment away. + */ + p = strset_add_str_mem(set, len); + if (!p) + return -ENOMEM; + + /* + * The set->strs_data might have reallocated and if 's' pointed + * to an internal string within the old buffer, then it became + * dangling and needs to be reconstructed before the copy. + */ + if (old_data && old_data != (uintptr_t)set->strs_data && + old_s >= old_data && old_s < old_data + old_data_len) + s = set->strs_data + (old_s - old_data); + + memcpy(p, s, len); + + return len; +} + /* Find string offset that corresponds to a given string *s*. * Returns: * - >0 offset into string data, if string is found; @@ -116,16 +151,12 @@ static void *strset_add_str_mem(struct strset *set, size_t add_sz) int strset__find_str(struct strset *set, const char *s) { long old_off, new_off, len; - void *p; - /* see strset__add_str() for why we do this */ - len = strlen(s) + 1; - p = strset_add_str_mem(set, len); - if (!p) - return -ENOMEM; + len = strset_str_append(set, s); + if (len < 0) + return len; new_off = set->strs_data_len; - memcpy(p, s, len); if (hashmap__find(set->strs_hash, new_off, &old_off)) return old_off; @@ -142,24 +173,13 @@ int strset__find_str(struct strset *set, const char *s) int strset__add_str(struct strset *set, const char *s) { long old_off, new_off, len; - void *p; int err; - /* Hashmap keys are always offsets within set->strs_data, so to even - * look up some string from the "outside", we need to first append it - * at the end, so that it can be addressed with an offset. Luckily, - * until set->strs_data_len is incremented, that string is just a piece - * of garbage for the rest of the code, so no harm, no foul. On the - * other hand, if the string is unique, it's already appended and - * ready to be used, only a simple set->strs_data_len increment away. - */ - len = strlen(s) + 1; - p = strset_add_str_mem(set, len); - if (!p) - return -ENOMEM; + len = strset_str_append(set, s); + if (len < 0) + return len; new_off = set->strs_data_len; - memcpy(p, s, len); /* Now attempt to add the string, but only if the string with the same * contents doesn't exist already (HASHMAP_ADD strategy). If such diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c index d1524f6f54ae..2e56e3ab5b6c 100644 --- a/tools/lib/bpf/usdt.c +++ b/tools/lib/bpf/usdt.c @@ -262,6 +262,7 @@ struct usdt_manager { bool has_bpf_cookie; bool has_sema_refcnt; bool has_uprobe_multi; + bool has_uprobe_syscall; }; struct usdt_manager *usdt_manager_new(struct bpf_object *obj) @@ -301,6 +302,13 @@ struct usdt_manager *usdt_manager_new(struct bpf_object *obj) * usdt probes. */ man->has_uprobe_multi = kernel_supports(obj, FEAT_UPROBE_MULTI_LINK); + + /* + * Detect kernel support for uprobe() syscall, it's presence means we can + * take advantage of faster nop10 uprobe handling. + * Added in: 56101b69c919 ("uprobes/x86: Add uprobe syscall to speed up uprobe") + */ + man->has_uprobe_syscall = kernel_supports(obj, FEAT_UPROBE_SYSCALL); return man; } @@ -319,7 +327,7 @@ static int sanity_check_usdt_elf(Elf *elf, const char *path) int endianness; if (elf_kind(elf) != ELF_K_ELF) { - pr_warn("usdt: unrecognized ELF kind %d for '%s'\n", elf_kind(elf), path); + pr_warn("usdt: unrecognized ELF kind %u for '%s'\n", elf_kind(elf), path); return -EBADF; } @@ -430,8 +438,9 @@ static int parse_elf_segs(Elf *elf, const char *path, struct elf_seg **segs, siz } pr_debug("usdt: discovered PHDR #%d in '%s': vaddr 0x%lx memsz 0x%lx offset 0x%lx type 0x%lx flags 0x%lx\n", - i, path, (long)phdr.p_vaddr, (long)phdr.p_memsz, (long)phdr.p_offset, - (long)phdr.p_type, (long)phdr.p_flags); + i, path, + (unsigned long)phdr.p_vaddr, (unsigned long)phdr.p_memsz, (unsigned long)phdr.p_offset, + (unsigned long)phdr.p_type, (unsigned long)phdr.p_flags); if (phdr.p_type != PT_LOAD) continue; @@ -460,10 +469,10 @@ static int parse_elf_segs(Elf *elf, const char *path, struct elf_seg **segs, siz static int parse_vma_segs(int pid, const char *lib_path, struct elf_seg **segs, size_t *seg_cnt) { - char path[PATH_MAX], line[PATH_MAX], mode[16]; + char path[PATH_MAX], line[4096], mode[16]; size_t seg_start, seg_end, seg_off; struct elf_seg *seg; - int tmp_pid, i, err; + int tmp_pid, n, i, err; FILE *f; *seg_cnt = 0; @@ -472,8 +481,13 @@ static int parse_vma_segs(int pid, const char *lib_path, struct elf_seg **segs, * /proc/<pid>/root/<path>. They will be reported as just /<path> in * /proc/<pid>/maps. */ - if (sscanf(lib_path, "/proc/%d/root%s", &tmp_pid, path) == 2 && pid == tmp_pid) + /* %n is not counted in sscanf() return value, so initialize it. */ + n = 0; + if (sscanf(lib_path, "/proc/%d/root%n", &tmp_pid, &n) == 1 && + n > 0 && pid == tmp_pid && lib_path[n] == '/') { + libbpf_strlcpy(path, lib_path + n, sizeof(path)); goto proceed; + } if (!realpath(lib_path, path)) { pr_warn("usdt: failed to get absolute path of '%s' (err %s), using path as is...\n", @@ -496,8 +510,11 @@ proceed: * 7f5c6f5d1000-7f5c6f5d3000 rw-p 001c7000 08:04 21238613 /usr/lib64/libc-2.17.so * 7f5c6f5d3000-7f5c6f5d8000 rw-p 00000000 00:00 0 * 7f5c6f5d8000-7f5c6f5d9000 r-xp 00000000 103:01 362990598 /data/users/andriin/linux/tools/bpf/usdt/libhello_usdt.so + * + * Some VMA names can be longer than the local buffer. Bound the + * writes, but still consume the rest of the line. */ - while (fscanf(f, "%zx-%zx %s %zx %*s %*d%[^\n]\n", + while (fscanf(f, "%zx-%zx %15s %zx %*s %*d%4095[^\n]%*[^\n]\n", &seg_start, &seg_end, mode, &seg_off, line) == 5) { void *tmp; @@ -585,13 +602,34 @@ static int parse_usdt_note(GElf_Nhdr *nhdr, const char *data, size_t name_off, static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, __u64 usdt_cookie); -static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *path, pid_t pid, - const char *usdt_provider, const char *usdt_name, __u64 usdt_cookie, - struct usdt_target **out_targets, size_t *out_target_cnt) +#if defined(__x86_64__) +static bool has_nop_combo(int fd, long off) +{ + unsigned char nop_combo[11] = { + 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, + }; + unsigned char buf[11]; + + if (pread(fd, buf, 11, off) != 11) + return false; + return memcmp(buf, nop_combo, 11) == 0; +} +#else +static bool has_nop_combo(int fd, long off) +{ + return false; +} +#endif + +static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, const char *path, + pid_t pid, const char *usdt_provider, const char *usdt_name, + __u64 usdt_cookie, struct usdt_target **out_targets, + size_t *out_target_cnt) { size_t off, name_off, desc_off, seg_cnt = 0, vma_seg_cnt = 0, target_cnt = 0; struct elf_seg *segs = NULL, *vma_segs = NULL; struct usdt_target *targets = NULL, *target; + Elf *elf = elf_fd->elf; long base_addr = 0; Elf_Scn *notes_scn, *base_scn; GElf_Shdr base_shdr, notes_shdr; @@ -682,14 +720,14 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char * if (!seg) { err = -ESRCH; pr_warn("usdt: failed to find ELF program segment for '%s:%s' in '%s' at IP 0x%lx\n", - usdt_provider, usdt_name, path, usdt_abs_ip); + usdt_provider, usdt_name, path, (unsigned long)usdt_abs_ip); goto err_out; } if (!seg->is_exec) { err = -ESRCH; pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx) for '%s:%s' at IP 0x%lx is not executable\n", - path, seg->start, seg->end, usdt_provider, usdt_name, - usdt_abs_ip); + path, (unsigned long)seg->start, (unsigned long)seg->end, usdt_provider, usdt_name, + (unsigned long)usdt_abs_ip); goto err_out; } /* translate from virtual address to file offset */ @@ -729,7 +767,7 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char * if (!seg) { err = -ESRCH; pr_warn("usdt: failed to find shared lib memory segment for '%s:%s' in '%s' at relative IP 0x%lx\n", - usdt_provider, usdt_name, path, usdt_rel_ip); + usdt_provider, usdt_name, path, (unsigned long)usdt_rel_ip); goto err_out; } @@ -738,8 +776,10 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char * pr_debug("usdt: probe for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved abs_ip 0x%lx rel_ip 0x%lx) args '%s' in segment [0x%lx, 0x%lx) at offset 0x%lx\n", usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", path, - note.loc_addr, note.base_addr, usdt_abs_ip, usdt_rel_ip, note.args, - seg ? seg->start : 0, seg ? seg->end : 0, seg ? seg->offset : 0); + (unsigned long)note.loc_addr, (unsigned long)note.base_addr, + (unsigned long)usdt_abs_ip, (unsigned long)usdt_rel_ip, note.args, + (unsigned long)(seg ? seg->start : 0), (unsigned long)(seg ? seg->end : 0), + (unsigned long)(seg ? seg->offset : 0)); /* Adjust semaphore address to be a file offset */ if (note.sema_addr) { @@ -754,14 +794,14 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char * if (!seg) { err = -ESRCH; pr_warn("usdt: failed to find ELF loadable segment with semaphore of '%s:%s' in '%s' at 0x%lx\n", - usdt_provider, usdt_name, path, note.sema_addr); + usdt_provider, usdt_name, path, (unsigned long)note.sema_addr); goto err_out; } if (seg->is_exec) { err = -ESRCH; pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx] for semaphore of '%s:%s' at 0x%lx is executable\n", - path, seg->start, seg->end, usdt_provider, usdt_name, - note.sema_addr); + path, (unsigned long)seg->start, (unsigned long)seg->end, usdt_provider, usdt_name, + (unsigned long)note.sema_addr); goto err_out; } @@ -769,8 +809,8 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char * pr_debug("usdt: sema for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved 0x%lx) in segment [0x%lx, 0x%lx] at offset 0x%lx\n", usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", - path, note.sema_addr, note.base_addr, usdt_sema_off, - seg->start, seg->end, seg->offset); + path, (unsigned long)note.sema_addr, (unsigned long)note.base_addr, (unsigned long)usdt_sema_off, + (unsigned long)seg->start, (unsigned long)seg->end, (unsigned long)seg->offset); } /* Record adjusted addresses and offsets and parse USDT spec */ @@ -784,6 +824,16 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char * target = &targets[target_cnt]; memset(target, 0, sizeof(*target)); + /* + * We have uprobe syscall and usdt with nop,nop10 instructions combo, + * so we can place the uprobe directly on nop10 (+1) and get this probe + * optimized. + */ + if (man->has_uprobe_syscall && has_nop_combo(elf_fd->fd, usdt_rel_ip)) { + usdt_abs_ip++; + usdt_rel_ip++; + } + target->abs_ip = usdt_abs_ip; target->rel_ip = usdt_rel_ip; target->sema_off = usdt_sema_off; @@ -998,7 +1048,7 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct /* discover USDT in given binary, optionally limiting * activations to a given PID, if pid > 0 */ - err = collect_usdt_targets(man, elf_fd.elf, path, pid, usdt_provider, usdt_name, + err = collect_usdt_targets(man, &elf_fd, path, pid, usdt_provider, usdt_name, usdt_cookie, &targets, &target_cnt); if (err <= 0) { err = (err == 0) ? -ENOENT : err; @@ -1070,7 +1120,7 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct spec_id, usdt_provider, usdt_name, path); } else { pr_warn("usdt: failed to map IP 0x%lx to spec #%d for '%s:%s' in '%s': %s\n", - target->abs_ip, spec_id, usdt_provider, usdt_name, + (unsigned long)target->abs_ip, spec_id, usdt_provider, usdt_name, path, errstr(err)); } goto err_out; diff --git a/tools/lib/perf/TODO b/tools/lib/perf/TODO new file mode 100644 index 000000000000..e179728697d8 --- /dev/null +++ b/tools/lib/perf/TODO @@ -0,0 +1,30 @@ +Future ABI changes +================== + +This file collects items that require a libperf ABI bump. Each entry +should describe the current limitation, the desired end state, and the +scope of the change so that a future ABI revision can batch them +together. + +1. Widen struct perf_cpu.cpu from int16_t to int + - Current limit: 32767 CPUs. No architecture exceeds this today + (x86_64 max is 8192, arm64 is 4096), but NR_CPUS limits keep + growing. perf clamps to INT16_MAX in set_max_cpu_num() as a + safety net. + - Code simplification: the int16_t forces defensive truncation + checks at every boundary where a wider CPU index (int from + sample->cpu, al->cpu, etc.) is narrowed into struct perf_cpu. + Without these checks, values > 32767 silently wrap to negative + numbers (two's complement), bypassing bounds validation. + Widening to int eliminates this entire class of silent + truncation bugs and removes the need for the INT16_MAX clamp + in set_max_cpu_num(). + - Scope: struct perf_cpu is embedded everywhere — perf_cpu_map__cpu(), + perf_cpu_map__min(), perf_cpu_map__max(), perf_cpu_map__has(), the + for_each_cpu macros, and all internal callers. The perf_cpu_map + internal array (RC_CHK_ACCESS(map)->map[]) stores struct perf_cpu + directly. Widening changes the struct layout and every function + that returns or accepts struct perf_cpu by value. + - Migration: bump LIBPERF version in libperf.map, audit all + sizeof(struct perf_cpu) assumptions, update perf.data + serialization if needed. diff --git a/tools/lib/perf/cpumap.c b/tools/lib/perf/cpumap.c index 4160e7d2e120..e51b0490ad57 100644 --- a/tools/lib/perf/cpumap.c +++ b/tools/lib/perf/cpumap.c @@ -15,12 +15,12 @@ #define MAX_NR_CPUS 4096 -void perf_cpu_map__set_nr(struct perf_cpu_map *map, int nr_cpus) +void perf_cpu_map__set_nr(struct perf_cpu_map *map, unsigned int nr_cpus) { RC_CHK_ACCESS(map)->nr = nr_cpus; } -struct perf_cpu_map *perf_cpu_map__alloc(int nr_cpus) +struct perf_cpu_map *perf_cpu_map__alloc(unsigned int nr_cpus) { RC_STRUCT(perf_cpu_map) *cpus; struct perf_cpu_map *result; @@ -78,7 +78,7 @@ void perf_cpu_map__put(struct perf_cpu_map *map) static struct perf_cpu_map *cpu_map__new_sysconf(void) { struct perf_cpu_map *cpus; - int nr_cpus, nr_cpus_conf; + long nr_cpus, nr_cpus_conf; nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); if (nr_cpus < 0) @@ -86,15 +86,13 @@ static struct perf_cpu_map *cpu_map__new_sysconf(void) nr_cpus_conf = sysconf(_SC_NPROCESSORS_CONF); if (nr_cpus != nr_cpus_conf) { - pr_warning("Number of online CPUs (%d) differs from the number configured (%d) the CPU map will only cover the first %d CPUs.", + pr_warning("Number of online CPUs (%ld) differs from the number configured (%ld) the CPU map will only cover the first %ld CPUs.", nr_cpus, nr_cpus_conf, nr_cpus); } cpus = perf_cpu_map__alloc(nr_cpus); if (cpus != NULL) { - int i; - - for (i = 0; i < nr_cpus; ++i) + for (long i = 0; i < nr_cpus; ++i) RC_CHK_ACCESS(cpus)->map[i].cpu = i; } @@ -132,23 +130,23 @@ static int cmp_cpu(const void *a, const void *b) return cpu_a->cpu - cpu_b->cpu; } -static struct perf_cpu __perf_cpu_map__cpu(const struct perf_cpu_map *cpus, int idx) +static struct perf_cpu __perf_cpu_map__cpu(const struct perf_cpu_map *cpus, unsigned int idx) { return RC_CHK_ACCESS(cpus)->map[idx]; } -static struct perf_cpu_map *cpu_map__trim_new(int nr_cpus, const struct perf_cpu *tmp_cpus) +static struct perf_cpu_map *cpu_map__trim_new(unsigned int nr_cpus, const struct perf_cpu *tmp_cpus) { size_t payload_size = nr_cpus * sizeof(struct perf_cpu); struct perf_cpu_map *cpus = perf_cpu_map__alloc(nr_cpus); - int i, j; if (cpus != NULL) { + unsigned int j = 0; + memcpy(RC_CHK_ACCESS(cpus)->map, tmp_cpus, payload_size); qsort(RC_CHK_ACCESS(cpus)->map, nr_cpus, sizeof(struct perf_cpu), cmp_cpu); /* Remove dups */ - j = 0; - for (i = 0; i < nr_cpus; i++) { + for (unsigned int i = 0; i < nr_cpus; i++) { if (i == 0 || __perf_cpu_map__cpu(cpus, i).cpu != __perf_cpu_map__cpu(cpus, i - 1).cpu) { @@ -167,9 +165,8 @@ struct perf_cpu_map *perf_cpu_map__new(const char *cpu_list) struct perf_cpu_map *cpus = NULL; unsigned long start_cpu, end_cpu = 0; char *p = NULL; - int i, nr_cpus = 0; + unsigned int nr_cpus = 0, max_entries = 0; struct perf_cpu *tmp_cpus = NULL, *tmp; - int max_entries = 0; if (!cpu_list) return perf_cpu_map__new_online_cpus(); @@ -208,9 +205,10 @@ struct perf_cpu_map *perf_cpu_map__new(const char *cpu_list) for (; start_cpu <= end_cpu; start_cpu++) { /* check for duplicates */ - for (i = 0; i < nr_cpus; i++) + for (unsigned int i = 0; i < nr_cpus; i++) { if (tmp_cpus[i].cpu == (int16_t)start_cpu) goto invalid; + } if (nr_cpus == max_entries) { max_entries += max(end_cpu - start_cpu + 1, 16UL); @@ -252,12 +250,12 @@ struct perf_cpu_map *perf_cpu_map__new_int(int cpu) return cpus; } -static int __perf_cpu_map__nr(const struct perf_cpu_map *cpus) +static unsigned int __perf_cpu_map__nr(const struct perf_cpu_map *cpus) { return RC_CHK_ACCESS(cpus)->nr; } -struct perf_cpu perf_cpu_map__cpu(const struct perf_cpu_map *cpus, int idx) +struct perf_cpu perf_cpu_map__cpu(const struct perf_cpu_map *cpus, unsigned int idx) { struct perf_cpu result = { .cpu = -1 @@ -269,7 +267,7 @@ struct perf_cpu perf_cpu_map__cpu(const struct perf_cpu_map *cpus, int idx) return result; } -int perf_cpu_map__nr(const struct perf_cpu_map *cpus) +unsigned int perf_cpu_map__nr(const struct perf_cpu_map *cpus) { return cpus ? __perf_cpu_map__nr(cpus) : 1; } @@ -294,7 +292,7 @@ bool perf_cpu_map__is_empty(const struct perf_cpu_map *map) int perf_cpu_map__idx(const struct perf_cpu_map *cpus, struct perf_cpu cpu) { - int low, high; + unsigned int low, high; if (!cpus) return -1; @@ -324,7 +322,7 @@ bool perf_cpu_map__has(const struct perf_cpu_map *cpus, struct perf_cpu cpu) bool perf_cpu_map__equal(const struct perf_cpu_map *lhs, const struct perf_cpu_map *rhs) { - int nr; + unsigned int nr; if (lhs == rhs) return true; @@ -336,7 +334,7 @@ bool perf_cpu_map__equal(const struct perf_cpu_map *lhs, const struct perf_cpu_m if (nr != __perf_cpu_map__nr(rhs)) return false; - for (int idx = 0; idx < nr; idx++) { + for (unsigned int idx = 0; idx < nr; idx++) { if (__perf_cpu_map__cpu(lhs, idx).cpu != __perf_cpu_map__cpu(rhs, idx).cpu) return false; } @@ -353,7 +351,7 @@ struct perf_cpu perf_cpu_map__min(const struct perf_cpu_map *map) struct perf_cpu cpu, result = { .cpu = -1 }; - int idx; + unsigned int idx; perf_cpu_map__for_each_cpu_skip_any(cpu, idx, map) { result = cpu; @@ -384,7 +382,7 @@ bool perf_cpu_map__is_subset(const struct perf_cpu_map *a, const struct perf_cpu if (!a || __perf_cpu_map__nr(b) > __perf_cpu_map__nr(a)) return false; - for (int i = 0, j = 0; i < __perf_cpu_map__nr(a); i++) { + for (unsigned int i = 0, j = 0; i < __perf_cpu_map__nr(a); i++) { if (__perf_cpu_map__cpu(a, i).cpu > __perf_cpu_map__cpu(b, j).cpu) return false; if (__perf_cpu_map__cpu(a, i).cpu == __perf_cpu_map__cpu(b, j).cpu) { @@ -410,8 +408,7 @@ bool perf_cpu_map__is_subset(const struct perf_cpu_map *a, const struct perf_cpu int perf_cpu_map__merge(struct perf_cpu_map **orig, struct perf_cpu_map *other) { struct perf_cpu *tmp_cpus; - int tmp_len; - int i, j, k; + unsigned int tmp_len, i, j, k; struct perf_cpu_map *merged; if (perf_cpu_map__is_subset(*orig, other)) @@ -455,7 +452,7 @@ int perf_cpu_map__merge(struct perf_cpu_map **orig, struct perf_cpu_map *other) struct perf_cpu_map *perf_cpu_map__intersect(struct perf_cpu_map *orig, struct perf_cpu_map *other) { - int i, j, k; + unsigned int i, j, k; struct perf_cpu_map *merged; if (perf_cpu_map__is_subset(other, orig)) diff --git a/tools/lib/perf/evsel.c b/tools/lib/perf/evsel.c index 13a307fc75ae..f747c0bc692d 100644 --- a/tools/lib/perf/evsel.c +++ b/tools/lib/perf/evsel.c @@ -127,7 +127,8 @@ int perf_evsel__open(struct perf_evsel *evsel, struct perf_cpu_map *cpus, struct perf_thread_map *threads) { struct perf_cpu cpu; - int idx, thread, err = 0; + unsigned int idx; + int thread, err = 0; if (cpus == NULL) { static struct perf_cpu_map *empty_cpu_map; @@ -460,7 +461,7 @@ int perf_evsel__enable_cpu(struct perf_evsel *evsel, int cpu_map_idx) int perf_evsel__enable_thread(struct perf_evsel *evsel, int thread) { struct perf_cpu cpu __maybe_unused; - int idx; + unsigned int idx; int err; perf_cpu_map__for_each_cpu(cpu, idx, evsel->cpus) { @@ -499,12 +500,13 @@ int perf_evsel__disable(struct perf_evsel *evsel) int perf_evsel__apply_filter(struct perf_evsel *evsel, const char *filter) { - int err = 0, i; + int err = 0; - for (i = 0; i < perf_cpu_map__nr(evsel->cpus) && !err; i++) + for (unsigned int i = 0; i < perf_cpu_map__nr(evsel->cpus) && !err; i++) { err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_SET_FILTER, (void *)filter, i); + } return err; } diff --git a/tools/lib/perf/include/internal/cpumap.h b/tools/lib/perf/include/internal/cpumap.h index e2be2d17c32b..c19678188b17 100644 --- a/tools/lib/perf/include/internal/cpumap.h +++ b/tools/lib/perf/include/internal/cpumap.h @@ -16,16 +16,16 @@ DECLARE_RC_STRUCT(perf_cpu_map) { refcount_t refcnt; /** Length of the map array. */ - int nr; + unsigned int nr; /** The CPU values. */ struct perf_cpu map[]; }; -struct perf_cpu_map *perf_cpu_map__alloc(int nr_cpus); +struct perf_cpu_map *perf_cpu_map__alloc(unsigned int nr_cpus); int perf_cpu_map__idx(const struct perf_cpu_map *cpus, struct perf_cpu cpu); bool perf_cpu_map__is_subset(const struct perf_cpu_map *a, const struct perf_cpu_map *b); -void perf_cpu_map__set_nr(struct perf_cpu_map *map, int nr_cpus); +void perf_cpu_map__set_nr(struct perf_cpu_map *map, unsigned int nr_cpus); static inline refcount_t *perf_cpu_map__refcnt(struct perf_cpu_map *map) { diff --git a/tools/lib/perf/include/perf/cpumap.h b/tools/lib/perf/include/perf/cpumap.h index 58cc5c5fa47c..e1a0b0d27210 100644 --- a/tools/lib/perf/include/perf/cpumap.h +++ b/tools/lib/perf/include/perf/cpumap.h @@ -6,7 +6,13 @@ #include <stdbool.h> #include <stdint.h> -/** A wrapper around a CPU to avoid confusion with the perf_cpu_map's map's indices. */ +/** + * struct perf_cpu - wrapper around a CPU number. + * @cpu: CPU number, -1 for the "any CPU"/dummy value. + * + * int16_t limits this to 32767 CPUs. Widening to int requires a libperf + * ABI bump — see tools/lib/perf/TODO for the full scope. + */ struct perf_cpu { int16_t cpu; }; @@ -49,7 +55,7 @@ LIBPERF_API void perf_cpu_map__put(struct perf_cpu_map *map); * perf_cpu_map__cpu - get the CPU value at the given index. Returns -1 if index * is invalid. */ -LIBPERF_API struct perf_cpu perf_cpu_map__cpu(const struct perf_cpu_map *cpus, int idx); +LIBPERF_API struct perf_cpu perf_cpu_map__cpu(const struct perf_cpu_map *cpus, unsigned int idx); /** * perf_cpu_map__nr - for an empty map returns 1, as perf_cpu_map__cpu returns a * cpu of -1 for an invalid index, this makes an empty map @@ -57,7 +63,7 @@ LIBPERF_API struct perf_cpu perf_cpu_map__cpu(const struct perf_cpu_map *cpus, i * the result is the number CPUs in the map plus one if the * "any CPU"/dummy value is present. */ -LIBPERF_API int perf_cpu_map__nr(const struct perf_cpu_map *cpus); +LIBPERF_API unsigned int perf_cpu_map__nr(const struct perf_cpu_map *cpus); /** * perf_cpu_map__has_any_cpu_or_is_empty - is map either empty or has the "any CPU"/dummy value. */ diff --git a/tools/lib/perf/include/perf/event.h b/tools/lib/perf/include/perf/event.h index 9043dc72b5d6..fdced574c889 100644 --- a/tools/lib/perf/include/perf/event.h +++ b/tools/lib/perf/include/perf/event.h @@ -8,7 +8,14 @@ #include <linux/bpf.h> #include <sys/types.h> /* pid_t */ -#define event_contains(obj, mem) ((obj).header.size > offsetof(typeof(obj), mem)) +/* + * Verify the full field fits within the event, not just its start offset. + * Only valid for fixed-size scalar fields — for trailing arrays like + * filename[PATH_MAX], sizeof() evaluates to the declared maximum, not + * the actual string length, so this would spuriously return false. + */ +#define event_contains(obj, mem) \ + ((obj).header.size >= offsetof(typeof(obj), mem) + sizeof((obj).mem)) struct perf_record_mmap { struct perf_event_header header; diff --git a/tools/lib/python/abi/abi_regex.py b/tools/lib/python/abi/abi_regex.py index d0c5e3ede6b5..198ecbf49c51 100644 --- a/tools/lib/python/abi/abi_regex.py +++ b/tools/lib/python/abi/abi_regex.py @@ -65,8 +65,7 @@ class AbiRegex(AbiParser): (re.compile(r"\[[^\]]+\]"), "\\\\w\xf7"), (re.compile(r"XX+"), "\\\\w\xf7"), - (re.compile(r"([^A-Z])[XYZ]([^A-Z])"), "\\1\\\\w\xf7\\2"), - (re.compile(r"([^A-Z])[XYZ]$"), "\\1\\\\w\xf7"), + (re.compile(r"(?<![A-Z])[XYZ](?![A-Z])"), "\\\\w\xf7"), (re.compile(r"_[AB]_"), "_\\\\w\xf7_"), # Recover [0-9] type of patterns @@ -155,7 +154,7 @@ class AbiRegex(AbiParser): if self.search_string: if what.find(self.search_string) >= 0: print(f"What: {what}") - except re.PatternError: + except re.error: self.log.warning("Ignoring '%s' as it produced an invalid regex:\n" " '%s'", what, new) @@ -194,7 +193,7 @@ class AbiRegex(AbiParser): try: self.re_string = re.compile(self.search_string) - except re.PatternError as e: + except re.error as e: msg = f"{self.search_string} is not a valid regular expression" raise ValueError(msg) from e @@ -223,9 +222,9 @@ class AbiRegex(AbiParser): for r, s in self.re_whats: try: new = r.sub(s, new) - except re.PatternError as e: + except re.error as e: # Help debugging troubles with new regexes - raise re.PatternError(f"{e}\nwhile re.sub('{r.pattern}', {s}, str)") from e + raise re.error(f"{e}\nwhile re.sub('{r.pattern}', {s}, str)") from e v["regex"].append(new) diff --git a/tools/lib/python/jobserver.py b/tools/lib/python/jobserver.py index aba22c33393d..0b1ffdf9f7a3 100755 --- a/tools/lib/python/jobserver.py +++ b/tools/lib/python/jobserver.py @@ -8,14 +8,14 @@ """ Interacts with the POSIX jobserver during the Kernel build time. -A "normal" jobserver task, like the one initiated by a make subrocess would do: +A "normal" jobserver task, like the one initiated by a make subprocess would do: - open read/write file descriptors to communicate with the job server; - ask for one slot by calling:: claim = os.read(reader, 1) - - when the job finshes, call:: + - when the job finishes, call:: os.write(writer, b"+") # os.write(writer, claim) diff --git a/tools/lib/python/kdoc/c_lex.py b/tools/lib/python/kdoc/c_lex.py new file mode 100644 index 000000000000..cb95f5172448 --- /dev/null +++ b/tools/lib/python/kdoc/c_lex.py @@ -0,0 +1,662 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>. + +""" +Regular expression ancillary classes. + +Those help caching regular expressions and do matching for kernel-doc. + +Please notice that the code here may rise exceptions to indicate bad +usage inside kdoc to indicate problems at the replace pattern. + +Other errors are logged via log instance. +""" + +import logging +import re + +from copy import copy + +from .kdoc_re import KernRe + +log = logging.getLogger(__name__) + +def tokenizer_set_log(logger, prefix = ""): + """ + Replace the module‑level logger with a LoggerAdapter that + prepends *prefix* to every message. + """ + global log + + class PrefixAdapter(logging.LoggerAdapter): + """ + Ancillary class to set prefix on all message logs. + """ + def process(self, msg, kwargs): + return f"{prefix}{msg}", kwargs + + # Wrap the provided logger in our adapter + log = PrefixAdapter(logger, {"prefix": prefix}) + +class CToken(): + """ + Data class to define a C token. + """ + + # Tokens that can be used by the parser. Works like an C enum. + + COMMENT = 0 #: A standard C or C99 comment, including delimiter. + STRING = 1 #: A string, including quotation marks. + CHAR = 2 #: A character, including apostophes. + NUMBER = 3 #: A number. + PUNC = 4 #: A puntuation mark: / ``,`` / ``.``. + BEGIN = 5 #: A begin character: ``{`` / ``[`` / ``(``. + END = 6 #: A end character: ``}`` / ``]`` / ``)``. + CPP = 7 #: A preprocessor macro. + HASH = 8 #: The hash character - useful to handle other macros. + OP = 9 #: A C operator (add, subtract, ...). + STRUCT = 10 #: A ``struct`` keyword. + UNION = 11 #: An ``union`` keyword. + ENUM = 12 #: A ``struct`` keyword. + TYPEDEF = 13 #: A ``typedef`` keyword. + NAME = 14 #: A name. Can be an ID or a type. + SPACE = 15 #: Any space characters, including new lines + ENDSTMT = 16 #: End of an statement (``;``). + + BACKREF = 17 #: Not a valid C sequence, but used at sub regex patterns. + + MISMATCH = 255 #: an error indicator: should never happen in practice. + + # Dict to convert from an enum interger into a string. + _name_by_val = {v: k for k, v in dict(vars()).items() if isinstance(v, int)} + + # Dict to convert from string to an enum-like integer value. + _name_to_val = {k: v for v, k in _name_by_val.items()} + + @staticmethod + def to_name(val): + """Convert from an integer value from CToken enum into a string""" + + return CToken._name_by_val.get(val, f"UNKNOWN({val})") + + @staticmethod + def from_name(name): + """Convert a string into a CToken enum value""" + if name in CToken._name_to_val: + return CToken._name_to_val[name] + + return CToken.MISMATCH + + + def __init__(self, kind, value=None, pos=0, + brace_level=0, paren_level=0, bracket_level=0): + self.kind = kind + self.value = value + self.pos = pos + self.level = (bracket_level, paren_level, brace_level) + + def __repr__(self): + name = self.to_name(self.kind) + if isinstance(self.value, str): + value = '"' + self.value + '"' + else: + value = self.value + + return f"CToken(CToken.{name}, {value}, {self.pos}, {self.level})" + +#: Regexes to parse C code, transforming it into tokens. +RE_SCANNER_LIST = [ + # + # Note that \s\S is different than .*, as it also catches \n + # + (CToken.COMMENT, r"//[^\n]*|/\*[\s\S]*?\*/"), + + (CToken.STRING, r'"(?:\\.|[^"\\])*"'), + (CToken.CHAR, r"'(?:\\.|[^'\\])'"), + + (CToken.NUMBER, r"0[xX][\da-fA-F]+[uUlL]*|0[0-7]+[uUlL]*|" + r"\d+(?:\.\d*)?(?:[eE][+-]?\d+)?[fFlL]*"), + + (CToken.ENDSTMT, r"(?:\s+;|;)"), + + (CToken.PUNC, r"[,\.]"), + + (CToken.BEGIN, r"[\[\(\{]"), + + (CToken.END, r"[\]\)\}]"), + + (CToken.CPP, r"#\s*(?:define|include|ifdef|ifndef|if|else|elif|endif|undef|pragma)\b"), + + (CToken.HASH, r"#"), + + (CToken.OP, r"\+\+|\-\-|\->|==|\!=|<=|>=|&&|\|\||<<|>>|\+=|\-=|\*=|/=|%=" + r"|&=|\|=|\^=|[=\+\-\*/%<>&\|\^~!\?\:]"), + + (CToken.STRUCT, r"\bstruct\b"), + (CToken.UNION, r"\bunion\b"), + (CToken.ENUM, r"\benum\b"), + (CToken.TYPEDEF, r"\btypedef\b"), + + (CToken.NAME, r"[A-Za-z_]\w*"), + + (CToken.SPACE, r"\s+"), + + (CToken.BACKREF, r"\\\d+"), + + (CToken.MISMATCH,r"."), +] + +def fill_re_scanner(token_list): + """Ancillary routine to convert RE_SCANNER_LIST into a finditer regex""" + re_tokens = [] + + for kind, pattern in token_list: + name = CToken.to_name(kind) + re_tokens.append(f"(?P<{name}>{pattern})") + + return KernRe("|".join(re_tokens), re.MULTILINE | re.DOTALL) + +#: Handle C continuation lines. +RE_CONT = KernRe(r"\\\n") + +RE_COMMENT_START = KernRe(r'/\*\s*') + +#: tokenizer regex. Will be filled at the first CTokenizer usage. +RE_SCANNER = fill_re_scanner(RE_SCANNER_LIST) + + +class CTokenizer(): + """ + Scan C statements and definitions and produce tokens. + + When converted to string, it drops comments and handle public/private + values, respecting depth. + """ + + # This class is inspired and follows the basic concepts of: + # https://docs.python.org/3/library/re.html#writing-a-tokenizer + + def __init__(self, source=None): + """ + Create a regular expression to handle RE_SCANNER_LIST. + + While I generally don't like using regex group naming via: + (?P<name>...) + + in this particular case, it makes sense, as we can pick the name + when matching a code via RE_SCANNER. + """ + + # + # Store logger to allow parser classes to re-use it + # + global log + self.log = log + + self.tokens = [] + + if not source: + return + + if isinstance(source, list): + self.tokens = source + return + + # + # While we could just use _tokenize directly via interator, + # As we'll need to use the tokenizer several times inside kernel-doc + # to handle macro transforms, cache the results on a list, as + # re-using it is cheaper than having to parse everytime. + # + for tok in self._tokenize(source): + self.tokens.append(tok) + + def _tokenize(self, source): + """ + Iterator that parses ``source``, splitting it into tokens, as defined + at ``self.RE_SCANNER_LIST``. + + The interactor returns a CToken class object. + """ + + # Handle continuation lines. Note that kdoc_parser already has a + # logic to do that. Still, let's keep it for completeness, as we might + # end re-using this tokenizer outsize kernel-doc some day - or we may + # eventually remove from there as a future cleanup. + source = RE_CONT.sub("", source) + + brace_level = 0 + paren_level = 0 + bracket_level = 0 + + for match in RE_SCANNER.finditer(source): + kind = CToken.from_name(match.lastgroup) + pos = match.start() + value = match.group() + + if kind == CToken.MISMATCH: + log.error(f"Unexpected token '{value}' on pos {pos}:\n\t'{source}'") + elif kind == CToken.BEGIN: + if value == '(': + paren_level += 1 + elif value == '[': + bracket_level += 1 + else: # value == '{' + brace_level += 1 + + elif kind == CToken.END: + if value == ')' and paren_level > 0: + paren_level -= 1 + elif value == ']' and bracket_level > 0: + bracket_level -= 1 + elif brace_level > 0: # value == '}' + brace_level -= 1 + + yield CToken(kind, value, pos, + brace_level, paren_level, bracket_level) + + def __str__(self): + out="" + show_stack = [True] + + for i, tok in enumerate(self.tokens): + if tok.kind == CToken.BEGIN: + show_stack.append(show_stack[-1]) + + elif tok.kind == CToken.END: + prev = show_stack[-1] + if len(show_stack) > 1: + show_stack.pop() + + if not prev and show_stack[-1]: + # + # Try to preserve indent + # + out += "\t" * (len(show_stack) - 1) + + out += str(tok.value) + continue + + elif tok.kind == CToken.COMMENT: + comment = RE_COMMENT_START.sub("", tok.value) + + if comment.startswith("private:"): + show_stack[-1] = False + show = False + elif comment.startswith("public:"): + show_stack[-1] = True + + continue + + if not show_stack[-1]: + continue + + if i < len(self.tokens) - 1: + next_tok = self.tokens[i + 1] + + # Do some cleanups before ";" + + if tok.kind == CToken.SPACE and next_tok.kind == CToken.ENDSTMT: + continue + + if tok.kind == CToken.ENDSTMT and next_tok.kind == tok.kind: + continue + + out += str(tok.value) + + return out + + +class CTokenArgs: + """ + Ancillary class to help using backrefs from sub matches. + + If the highest backref contain a "+" at the last element, + the logic will be greedy, picking all other delims. + + This is needed to parse struct_group macros with end with ``MEMBERS...``. + """ + def __init__(self, sub_str): + self.sub_groups = set() + self.max_group = -1 + self.greedy = None + + for m in KernRe(r'\\(\d+)([+]?)').finditer(sub_str): + group = int(m.group(1)) + if m.group(2) == "+": + if self.greedy and self.greedy != group: + raise ValueError("There are multiple greedy patterns!") + self.greedy = group + + self.sub_groups.add(group) + self.max_group = max(self.max_group, group) + + if self.greedy: + if self.greedy != self.max_group: + raise ValueError("Greedy pattern is not the last one!") + + sub_str = KernRe(r'(\\\d+)[+]').sub(r"\1", sub_str) + + self.sub_str = sub_str + self.sub_tokeninzer = CTokenizer(sub_str) + + def groups(self, new_tokenizer): + r""" + Create replacement arguments for backrefs like: + + ``\0``, ``\1``, ``\2``, ... ``\{number}`` + + It also accepts a ``+`` character to the highest backref, like + ``\4+``. When used, the backref will be greedy, picking all other + arguments afterwards. + + The logic is smart enough to only go up to the maximum required + argument, even if there are more. + + If there is a backref for an argument above the limit, it will + raise an exception. Please notice that, on C, square brackets + don't have any separator on it. Trying to use ``\1``..``\n`` for + brackets also raise an exception. + """ + + level = (0, 0, 0) + + if self.max_group < 0: + return level, [] + + tokens = new_tokenizer.tokens + + # + # Fill \0 with the full token contents + # + groups_list = [ [] ] + + if 0 in self.sub_groups: + inner_level = 0 + + for i in range(0, len(tokens)): + tok = tokens[i] + + if tok.kind == CToken.BEGIN: + inner_level += 1 + + # + # Discard first begin + # + if not groups_list[0]: + continue + elif tok.kind == CToken.END: + inner_level -= 1 + if inner_level < 0: + break + + if inner_level: + groups_list[0].append(tok) + + if not self.max_group: + return level, groups_list + + delim = None + + # + # Ignore everything before BEGIN. The value of begin gives the + # delimiter to be used for the matches + # + for i in range(0, len(tokens)): + tok = tokens[i] + if tok.kind == CToken.BEGIN: + if tok.value == "{": + delim = ";" + elif tok.value == "(": + delim = "," + else: + self.log.error(fr"Can't handle \1..\n on {sub_str}") + + level = tok.level + break + + pos = 1 + groups_list.append([]) + + inner_level = 0 + for i in range(i + 1, len(tokens)): + tok = tokens[i] + + if tok.kind == CToken.BEGIN: + inner_level += 1 + if tok.kind == CToken.END: + inner_level -= 1 + if inner_level < 0: + break + + if tok.kind in [CToken.PUNC, CToken.ENDSTMT] and delim == tok.value: + pos += 1 + if self.greedy and pos > self.max_group: + pos -= 1 + else: + groups_list.append([]) + + if pos > self.max_group: + break + + continue + + groups_list[pos].append(tok) + + if pos < self.max_group: + log.error(fr"{self.sub_str} groups are up to {pos} instead of {self.max_group}") + + return level, groups_list + + def tokens(self, new_tokenizer): + level, groups = self.groups(new_tokenizer) + + new = CTokenizer() + + for tok in self.sub_tokeninzer.tokens: + if tok.kind == CToken.BACKREF: + group = int(tok.value[1:]) + + for group_tok in groups[group]: + new_tok = copy(group_tok) + + new_level = [0, 0, 0] + + for i in range(0, len(level)): + new_level[i] = new_tok.level[i] + level[i] + + new_tok.level = tuple(new_level) + + new.tokens += [ new_tok ] + else: + new.tokens += [ tok ] + + return new.tokens + + +class CMatch: + """ + Finding nested delimiters is hard with regular expressions. It is + even harder on Python with its normal re module, as there are several + advanced regular expressions that are missing. + + This is the case of this pattern:: + + '\\bSTRUCT_GROUP(\\(((?:(?>[^)(]+)|(?1))*)\\))[^;]*;' + + which is used to properly match open/close parentheses of the + string search STRUCT_GROUP(), + + Add a class that counts pairs of delimiters, using it to match and + replace nested expressions. + + The original approach was suggested by: + + https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex + + Although I re-implemented it to make it more generic and match 3 types + of delimiters. The logic checks if delimiters are paired. If not, it + will ignore the search string. + """ + + + def __init__(self, regex, delim="("): + self.regex = KernRe("^" + regex + r"\b") + self.start_delim = delim + + def _search(self, tokenizer): + """ + Finds paired blocks for a regex that ends with a delimiter. + + The suggestion of using finditer to match pairs came from: + https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex + but I ended using a different implementation to align all three types + of delimiters and seek for an initial regular expression. + + The algorithm seeks for open/close paired delimiters and places them + into a stack, yielding a start/stop position of each match when the + stack is zeroed. + + The algorithm should work fine for properly paired lines, but will + silently ignore end delimiters that precede a start delimiter. + This should be OK for kernel-doc parser, as unaligned delimiters + would cause compilation errors. So, we don't need to raise exceptions + to cover such issues. + """ + + start = None + started = False + + import sys + + stack = [] + + for i, tok in enumerate(tokenizer.tokens): + if start is None: + if tok.kind == CToken.NAME and self.regex.match(tok.value): + start = i + stack.append((start, tok.level)) + started = False + + continue + + if not started: + if tok.kind == CToken.SPACE: + continue + + if tok.kind == CToken.BEGIN and tok.value == self.start_delim: + started = True + continue + + # Name only token without BEGIN/END + if i > start: + i -= 1 + yield start, i + start = None + + if tok.kind == CToken.END and tok.level == stack[-1][1]: + start, level = stack.pop() + + yield start, i + start = None + + # + # If an END zeroing levels is not there, return remaining stuff + # This is meant to solve cases where the caller logic might be + # picking an incomplete block. + # + if start and stack: + if started: + s = str(tokenizer) + log.warning(f"can't find a final end at {s}") + + yield start, len(tokenizer.tokens) + + def search(self, source): + """ + This is similar to re.search: + + It matches a regex that it is followed by a delimiter, + returning occurrences only if all delimiters are paired. + """ + + if isinstance(source, CTokenizer): + tokenizer = source + is_token = True + else: + tokenizer = CTokenizer(source) + is_token = False + + for start, end in self._search(tokenizer): + new_tokenizer = CTokenizer(tokenizer.tokens[start:end + 1]) + + if is_token: + yield new_tokenizer + else: + yield str(new_tokenizer) + + def sub(self, sub_str, source, count=0): + """ + This is similar to re.sub: + + It matches a regex that it is followed by a delimiter, + replacing occurrences only if all delimiters are paired. + + if the sub argument contains:: + + r'\0' + + it will work just like re: it places there the matched paired data + with the delimiter stripped. + + If count is different than zero, it will replace at most count + items. + """ + if isinstance(source, CTokenizer): + is_token = True + tokenizer = source + else: + is_token = False + tokenizer = CTokenizer(source) + + # Detect if sub_str contains sub arguments + + args_match = CTokenArgs(sub_str) + + new_tokenizer = CTokenizer() + pos = 0 + n = 0 + + # + # NOTE: the code below doesn't consider overlays at sub. + # We may need to add some extra unit tests to check if those + # would cause problems. When replacing by "", this should not + # be a problem, but other transformations could be problematic + # + for start, end in self._search(tokenizer): + new_tokenizer.tokens += tokenizer.tokens[pos:start] + + new = CTokenizer(tokenizer.tokens[start:end + 1]) + + new_tokenizer.tokens += args_match.tokens(new) + + pos = end + 1 + + n += 1 + if count and n >= count: + break + + new_tokenizer.tokens += tokenizer.tokens[pos:] + + if not is_token: + return str(new_tokenizer) + + return new_tokenizer + + def __repr__(self): + """ + Returns a displayable version of the class init. + """ + + return f'CMatch("{self.regex.regex.pattern}")' diff --git a/tools/lib/python/kdoc/kdoc_files.py b/tools/lib/python/kdoc/kdoc_files.py index 022487ea2cc6..ed82b6e6ab25 100644 --- a/tools/lib/python/kdoc/kdoc_files.py +++ b/tools/lib/python/kdoc/kdoc_files.py @@ -9,13 +9,14 @@ Classes for navigating through the files that kernel-doc needs to handle to generate documentation. """ -import argparse import logging import os import re from kdoc.kdoc_parser import KernelDoc +from kdoc.xforms_lists import CTransforms from kdoc.kdoc_output import OutputFormat +from kdoc.kdoc_yaml_file import KDocTestFile class GlobSourceFiles: @@ -86,11 +87,81 @@ class GlobSourceFiles: file_not_found_cb(fname) +class KdocConfig(): + """ + Stores all configuration attributes that kdoc_parser and kdoc_output + needs. + """ + def __init__(self, verbose=False, werror=False, wreturn=False, + wshort_desc=False, wcontents_before_sections=False, + logger=None): + + self.verbose = verbose + self.werror = werror + self.wreturn = wreturn + self.wshort_desc = wshort_desc + self.wcontents_before_sections = wcontents_before_sections + + if logger: + self.log = logger + else: + self.log = logging.getLogger(__file__) + + self.warning = self.log.warning + class KernelFiles(): """ Parse kernel-doc tags on multiple kernel source files. - There are two type of parsers defined here: + This is the main entry point to run kernel-doc. This class is initialized + using a series of optional arguments: + + ``verbose`` + If True, enables kernel-doc verbosity. Default: False. + + ``out_style`` + Class to be used to format output. If None (default), + only report errors. + + ``xforms`` + Transforms to be applied to C prototypes and data structs. + If not specified, defaults to xforms = CFunction() + + ``werror`` + If True, treat warnings as errors, retuning an error code on warnings. + + Default: False. + + ``wreturn`` + If True, warns about the lack of a return markup on functions. + + Default: False. + ``wshort_desc`` + If True, warns if initial short description is missing. + + Default: False. + + ``wcontents_before_sections`` + If True, warn if there are contents before sections (deprecated). + This option is kept just for backward-compatibility, but it does + nothing, neither here nor at the original Perl script. + + Default: False. + + ``logger`` + Optional logger class instance. + + If not specified, defaults to use: ``logging.getLogger("kernel-doc")`` + + ``yaml_file`` + If defined, stores the output inside a YAML file. + + ``yaml_content`` + Defines what will be inside the YAML file. + + Note: + There are two type of parsers defined here: + - self.parse_file(): parses both kernel-doc markups and ``EXPORT_SYMBOL*`` macros; - self.process_export_file(): parses only ``EXPORT_SYMBOL*`` macros. @@ -117,7 +188,12 @@ class KernelFiles(): if fname in self.files: return - doc = KernelDoc(self.config, fname) + if self.test_file: + store_src = True + else: + store_src = False + + doc = KernelDoc(self.config, fname, self.xforms, store_src=store_src) export_table, entries = doc.parse_kdoc() self.export_table[fname] = export_table @@ -153,16 +229,21 @@ class KernelFiles(): self.error(f"Cannot find file {fname}") - def __init__(self, verbose=False, out_style=None, + def __init__(self, verbose=False, out_style=None, xforms=None, werror=False, wreturn=False, wshort_desc=False, wcontents_before_sections=False, - logger=None): + yaml_file=None, yaml_content=None, logger=None): """ Initialize startup variables and parse all files. """ if not verbose: - verbose = bool(os.environ.get("KBUILD_VERBOSE", 0)) + try: + verbose = bool(int(os.environ.get("KBUILD_VERBOSE", 0))) + except ValueError: + # Handles an eventual case where verbosity is not a number + # like KBUILD_VERBOSE="" + verbose = False if out_style is None: out_style = OutputFormat() @@ -181,29 +262,36 @@ class KernelFiles(): if kdoc_werror: werror = kdoc_werror + if not logger: + logger = logging.getLogger("kernel-doc") + else: + logger = logger + # Some variables are global to the parser logic as a whole as they are # used to send control configuration to KernelDoc class. As such, # those variables are read-only inside the KernelDoc. - self.config = argparse.Namespace + self.config = KdocConfig(verbose, werror, wreturn, wshort_desc, + wcontents_before_sections, logger) - self.config.verbose = verbose - self.config.werror = werror - self.config.wreturn = wreturn - self.config.wshort_desc = wshort_desc - self.config.wcontents_before_sections = wcontents_before_sections + # Override log warning, as we want to count errors + self.config.warning = self.warning - if not logger: - self.config.log = logging.getLogger("kernel-doc") + if yaml_file: + self.test_file = KDocTestFile(self.config, yaml_file, yaml_content) else: - self.config.log = logger + self.test_file = None - self.config.warning = self.warning + if xforms: + self.xforms = xforms + else: + self.xforms = CTransforms() self.config.src_tree = os.environ.get("SRCTREE", None) # Initialize variables that are internal to KernelFiles self.out_style = out_style + self.out_style.set_config(self.config) self.errors = 0 self.results = {} @@ -246,8 +334,6 @@ class KernelFiles(): returning kernel-doc markups on each interaction. """ - self.out_style.set_config(self.config) - if not filenames: filenames = sorted(self.results.keys()) @@ -267,29 +353,28 @@ class KernelFiles(): for s in symbol: function_table.add(s) - self.out_style.set_filter(export, internal, symbol, nosymbol, - function_table, enable_lineno, - no_doc_sections) - - msg = "" if fname not in self.results: self.config.log.warning("No kernel-doc for file %s", fname) continue symbols = self.results[fname] - self.out_style.set_symbols(symbols) - for arg in symbols: - m = self.out_msg(fname, arg.name, arg) + if self.test_file: + self.test_file.set_filter(export, internal, symbol, nosymbol, + function_table, enable_lineno, + no_doc_sections) - if m is None: - ln = arg.get("ln", 0) - dtype = arg.get('type', "") + self.test_file.output_symbols(fname, symbols) - self.config.log.warning("%s:%d Can't handle %s", - fname, ln, dtype) - else: - msg += m + continue + + self.out_style.set_filter(export, internal, symbol, nosymbol, + function_table, enable_lineno, + no_doc_sections) + msg = self.out_style.output_symbols(fname, symbols) if msg: yield fname, msg + + if self.test_file: + self.test_file.write() diff --git a/tools/lib/python/kdoc/kdoc_item.py b/tools/lib/python/kdoc/kdoc_item.py index 2b8a93f79716..a7aa6e1e4c1c 100644 --- a/tools/lib/python/kdoc/kdoc_item.py +++ b/tools/lib/python/kdoc/kdoc_item.py @@ -14,7 +14,8 @@ class KdocItem: then pass into the output modules. """ - def __init__(self, name, fname, type, start_line, **other_stuff): + def __init__(self, name, fname, type, start_line, + **other_stuff): self.name = name self.fname = fname self.type = type @@ -22,15 +23,34 @@ class KdocItem: self.sections = {} self.sections_start_lines = {} self.parameterlist = [] - self.parameterdesc_start_lines = [] + self.parameterdesc_start_lines = {} self.parameterdescs = {} self.parametertypes = {} + + self.warnings = [] + # # Just save everything else into our own dict so that the output # side can grab it directly as before. As we move things into more # structured data, this will, hopefully, fade away. # - self.other_stuff = other_stuff + known_keys = { + 'declaration_start_line', + 'sections', + 'sections_start_lines', + 'parameterlist', + 'parameterdesc_start_lines', + 'parameterdescs', + 'parametertypes', + 'warnings', + } + + self.other_stuff = {} + for k, v in other_stuff.items(): + if k in known_keys: + setattr(self, k, v) # real attribute + else: + self.other_stuff[k] = v def get(self, key, default = None): """ @@ -41,6 +61,23 @@ class KdocItem: def __getitem__(self, key): return self.get(key) + def __repr__(self): + return f"KdocItem({self.name}, {self.fname}, {self.type}, {self.declaration_start_line})" + + @classmethod + def from_dict(cls, d): + """Create a KdocItem from a plain dict.""" + + cp = d.copy() + name = cp.pop('name', None) + fname = cp.pop('fname', None) + type = cp.pop('type', None) + start_line = cp.pop('start_line', 1) + other_stuff = cp.pop('other_stuff', {}) + + # Everything that’s left goes straight to __init__ + return cls(name, fname, type, start_line, **cp, **other_stuff) + # # Tracking of section and parameter information. # @@ -49,7 +86,7 @@ class KdocItem: Set sections and start lines. """ self.sections = sections - self.section_start_lines = start_lines + self.sections_start_lines = start_lines def set_params(self, names, descs, types, starts): """ diff --git a/tools/lib/python/kdoc/kdoc_output.py b/tools/lib/python/kdoc/kdoc_output.py index 4210b91dde5f..618b0d765ef5 100644 --- a/tools/lib/python/kdoc/kdoc_output.py +++ b/tools/lib/python/kdoc/kdoc_output.py @@ -222,6 +222,27 @@ class OutputFormat: return None + def output_symbols(self, fname, symbols): + """ + Handles a set of KdocItem symbols. + """ + self.set_symbols(symbols) + + msg = "" + for arg in symbols: + m = self.msg(fname, arg.name, arg) + + if m is None: + ln = arg.get("ln", 0) + dtype = arg.get('type', "") + + self.config.log.warning("%s:%d Can't handle %s", + fname, ln, dtype) + else: + msg += m + + return msg + # Virtual methods to be overridden by inherited classes # At the base class, those do nothing. def set_symbols(self, symbols): @@ -368,7 +389,7 @@ class RestFormat(OutputFormat): else: self.data += f'{self.lineprefix}**{section}**\n\n' - self.print_lineno(args.section_start_lines.get(section, 0)) + self.print_lineno(args.sections_start_lines.get(section, 0)) self.output_highlight(text) self.data += "\n" self.data += "\n" @@ -492,7 +513,9 @@ class RestFormat(OutputFormat): def out_var(self, fname, name, args): oldprefix = self.lineprefix ln = args.declaration_start_line - full_proto = args.other_stuff["full_proto"] + full_proto = args.other_stuff.get("full_proto") + if not full_proto: + raise KeyError(f"Can't find full proto for {name} variable") self.lineprefix = " " @@ -580,7 +603,35 @@ class RestFormat(OutputFormat): class ManFormat(OutputFormat): - """Consts and functions used by man pages output.""" + """ + Consts and functions used by man pages output. + + This class has one mandatory parameter and some optional ones, which + are needed to define the title header contents: + + ``modulename`` + Defines the module name to be used at the troff ``.TH`` output. + + This argument is optional. If not specified, it will be filled + with the directory which contains the documented file. + + ``section`` + Usually a numeric value from 0 to 9, but man pages also accept + some strings like "p". + + Defauls to ``9`` + + ``manual`` + Defaults to ``Kernel API Manual``. + + The above controls the output of the corresponding fields on troff + title headers, which will be filled like this:: + + .TH "{name}" {section} "{date}" "{modulename}" "{manual}" + + where ``name``` will match the API symbol name, and ``date`` will be + either the date where the Kernel was compiled or the current date + """ highlights = ( (type_constant, r"\1"), @@ -607,7 +658,21 @@ class ManFormat(OutputFormat): "%m %d %Y", ] - def __init__(self, modulename): + def modulename(self, args): + if self._modulename: + return self._modulename + + return os.path.dirname(args.fname) + + def emit_th(self, name, args): + """Emit a title header line.""" + title = name.strip() + module = self.modulename(args) + + self.data += f'.TH "{title}" {self.section} "{self.date}" ' + self.data += f'"{module}" "{self.manual}"\n' + + def __init__(self, modulename=None, section="9", manual="Kernel API Manual"): """ Creates class variables. @@ -616,7 +681,11 @@ class ManFormat(OutputFormat): """ super().__init__() - self.modulename = modulename + + self._modulename = modulename + self.section = section + self.manual = manual + self.symbols = [] dt = None @@ -632,7 +701,7 @@ class ManFormat(OutputFormat): if not dt: dt = datetime.now() - self.man_date = dt.strftime("%B %Y") + self.date = dt.strftime("%B %Y") def arg_name(self, args, name): """ @@ -647,7 +716,8 @@ class ManFormat(OutputFormat): dtype = args.type if dtype == "doc": - return self.modulename + return name +# return os.path.basename(self.modulename(args)) if dtype in ["function", "typedef"]: return name @@ -697,6 +767,185 @@ class ManFormat(OutputFormat): return self.data + def emit_table(self, colspec_row, rows): + + if not rows: + return "" + + out = "" + colspec = "\t".join(["l"] * len(rows[0])) + + out += "\n.TS\n" + out += "box;\n" + out += f"{colspec}.\n" + + if colspec_row: + out_row = [] + + for text in colspec_row: + out_row.append(f"\\fB{text}\\fP") + + out += "\t".join(out_row) + "\n_\n" + + for r in rows: + out += "\t".join(r) + "\n" + + out += ".TE\n" + + return out + + def grid_table(self, lines, start): + """ + Ancillary function to help handling a grid table inside the text. + """ + + i = start + 1 + rows = [] + colspec_row = None + + while i < len(lines): + line = lines[i] + + if KernRe(r"^\s*\|.*\|\s*$").match(line): + parts = [] + + for p in line.strip('|').split('|'): + parts.append(p.strip()) + + rows.append(parts) + + elif KernRe(r'^\+\=[\+\=]+\+\s*$').match(line): + if rows and rows[0]: + if not colspec_row: + colspec_row = [""] * len(rows[0]) + + for j in range(0, len(rows[0])): + content = [] + for row in rows: + content.append(row[j]) + + colspec_row[j] = " ".join(content) + + rows = [] + + elif KernRe(r"^\s*\+[-+]+\+.*$").match(line): + pass + + else: + break + + i += 1 + + return i, self.emit_table(colspec_row, rows) + + def simple_table(self, lines, start): + """ + Ancillary function to help handling a simple table inside the text. + """ + + i = start + rows = [] + colspec_row = None + + pos = [] + for m in KernRe(r'\=+').finditer(lines[i]): + pos.append((m.start(), m.end() - 1)) + + i += 1 + while i < len(lines): + line = lines[i] + + if KernRe(r"^\s*[\=]+[ \t\=]+$").match(line): + i += 1 + break + + elif KernRe(r'^[\s=]+$').match(line): + if rows and rows[0]: + if not colspec_row: + colspec_row = [""] * len(rows[0]) + + for j in range(0, len(rows[0])): + content = [] + for row in rows: + content.append(row[j]) + + colspec_row[j] = " ".join(content) + + rows = [] + + else: + row = [""] * len(pos) + + for j in range(0, len(pos)): + start, end = pos[j] + + row[j] = line[start:end].strip() + + rows.append(row) + + i += 1 + + return i, self.emit_table(colspec_row, rows) + + def code_block(self, lines, start): + """ + Ensure that code blocks won't be messed up at the output. + + By default, troff join lines at the same paragraph. Disable it, + on code blocks. + """ + + line = lines[start] + + if "code-block" in line: + out = "\n.nf\n" + elif line.startswith("..") and line.endswith("::"): + # + # Handle note, warning, error, ... markups + # + line = line[2:-1].strip().upper() + out = f"\n.nf\n\\fB{line}\\fP\n" + elif line.endswith("::"): + out = line[:-1] + out += "\n.nf\n" + else: + # Just in case. Should never happen in practice + out = "\n.nf\n" + + i = start + 1 + ident = None + + while i < len(lines): + line = lines[i] + + m = KernRe(r"\S").match(line) + if not m: + out += line + "\n" + i += 1 + continue + + pos = m.start() + if not ident: + if pos > 0: + ident = pos + else: + out += "\n.fi\n" + if i > start + 1: + return i - 1, out + else: + # Just in case. Should never happen in practice + return i, out + + if pos >= ident: + out += line + "\n" + i += 1 + continue + + break + + out += "\n.fi\n" + return i, out + def output_highlight(self, block): """ Outputs a C symbol that may require being highlighted with @@ -708,15 +957,46 @@ class ManFormat(OutputFormat): if isinstance(contents, list): contents = "\n".join(contents) - for line in contents.strip("\n").split("\n"): - line = KernRe(r"^\s*").sub("", line) - if not line: - continue + lines = contents.strip("\n").split("\n") + i = 0 - if line[0] == ".": - self.data += "\\&" + line + "\n" + while i < len(lines): + org_line = lines[i] + + line = KernRe(r"^\s*").sub("", org_line) + + if line: + if KernRe(r"^\+\-[-+]+\+.*$").match(line): + i, text = self.grid_table(lines, i) + self.data += text + continue + + if KernRe(r"^\=+[ \t]\=[ \t\=]+$").match(line): + i, text = self.simple_table(lines, i) + self.data += text + continue + + if line.endswith("::") or KernRe(r"\.\.\s+code-block.*::").match(line): + i, text = self.code_block(lines, i) + self.data += text + continue + + if line[0] == ".": + self.data += "\\&" + line + "\n" + i += 1 + continue + + # + # Handle lists + # + line = KernRe(r'^[-*]\s+').sub(r'.IP \[bu]\n', line) + line = KernRe(r'^(\d+|a-z)[\.\)]\s+').sub(r'.IP \1\n', line) else: - self.data += line + "\n" + line = ".PP\n" + + i += 1 + + self.data += line + "\n" def out_doc(self, fname, name, args): if not self.check_doc(name, args): @@ -724,7 +1004,7 @@ class ManFormat(OutputFormat): out_name = self.arg_name(args, name) - self.data += f'.TH "{self.modulename}" 9 "{out_name}" "{self.man_date}" "API Manual" LINUX' + "\n" + self.emit_th(out_name, args) for section, text in args.sections.items(): self.data += f'.SH "{section}"' + "\n" @@ -734,7 +1014,7 @@ class ManFormat(OutputFormat): out_name = self.arg_name(args, name) - self.data += f'.TH "{name}" 9 "{out_name}" "{self.man_date}" "Kernel Hacker\'s Manual" LINUX' + "\n" + self.emit_th(out_name, args) self.data += ".SH NAME\n" self.data += f"{name} \\- {args['purpose']}\n" @@ -780,7 +1060,7 @@ class ManFormat(OutputFormat): def out_enum(self, fname, name, args): out_name = self.arg_name(args, name) - self.data += f'.TH "{self.modulename}" 9 "{out_name}" "{self.man_date}" "API Manual" LINUX' + "\n" + self.emit_th(out_name, args) self.data += ".SH NAME\n" self.data += f"enum {name} \\- {args['purpose']}\n" @@ -813,7 +1093,7 @@ class ManFormat(OutputFormat): out_name = self.arg_name(args, name) full_proto = args.other_stuff["full_proto"] - self.data += f'.TH "{self.modulename}" 9 "{out_name}" "{self.man_date}" "API Manual" LINUX' + "\n" + self.emit_th(out_name, args) self.data += ".SH NAME\n" self.data += f"{name} \\- {args['purpose']}\n" @@ -830,11 +1110,11 @@ class ManFormat(OutputFormat): self.output_highlight(text) def out_typedef(self, fname, name, args): - module = self.modulename + module = self.modulename(args) purpose = args.get('purpose') out_name = self.arg_name(args, name) - self.data += f'.TH "{module}" 9 "{out_name}" "{self.man_date}" "API Manual" LINUX' + "\n" + self.emit_th(out_name, args) self.data += ".SH NAME\n" self.data += f"typedef {name} \\- {purpose}\n" @@ -844,12 +1124,12 @@ class ManFormat(OutputFormat): self.output_highlight(text) def out_struct(self, fname, name, args): - module = self.modulename + module = self.modulename(args) purpose = args.get('purpose') definition = args.get('definition') out_name = self.arg_name(args, name) - self.data += f'.TH "{module}" 9 "{out_name}" "{self.man_date}" "API Manual" LINUX' + "\n" + self.emit_th(out_name, args) self.data += ".SH NAME\n" self.data += f"{args.type} {name} \\- {purpose}\n" diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index ca00695b47b3..d9ad1ddc87dd 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -11,9 +11,11 @@ and extract embedded documentation comments from it. import sys import re +import difflib from pprint import pformat -from kdoc.kdoc_re import NestedMatch, KernRe +from kdoc.c_lex import CTokenizer, tokenizer_set_log +from kdoc.kdoc_re import KernRe from kdoc.kdoc_item import KdocItem # @@ -70,140 +72,9 @@ doc_begin_func = KernRe(str(doc_com) + # initial " * ' cache = False) # -# Here begins a long set of transformations to turn structure member prefixes -# and macro invocations into something we can parse and generate kdoc for. -# -struct_args_pattern = r'([^,)]+)' - -struct_xforms = [ - # Strip attributes - (KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", flags=re.I | re.S, cache=False), ' '), - (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), - (KernRe(r'\s*__counted_by\s*\([^;]*\)', re.S), ' '), - (KernRe(r'\s*__counted_by_(le|be)\s*\([^;]*\)', re.S), ' '), - (KernRe(r'\s*__packed\s*', re.S), ' '), - (KernRe(r'\s*CRYPTO_MINALIGN_ATTR', re.S), ' '), - (KernRe(r'\s*__private', re.S), ' '), - (KernRe(r'\s*__rcu', re.S), ' '), - (KernRe(r'\s*____cacheline_aligned_in_smp', re.S), ' '), - (KernRe(r'\s*____cacheline_aligned', re.S), ' '), - (KernRe(r'\s*__cacheline_group_(begin|end)\([^\)]+\);'), ''), - # - # Unwrap struct_group macros based on this definition: - # __struct_group(TAG, NAME, ATTRS, MEMBERS...) - # which has variants like: struct_group(NAME, MEMBERS...) - # Only MEMBERS arguments require documentation. - # - # Parsing them happens on two steps: - # - # 1. drop struct group arguments that aren't at MEMBERS, - # storing them as STRUCT_GROUP(MEMBERS) - # - # 2. remove STRUCT_GROUP() ancillary macro. - # - # The original logic used to remove STRUCT_GROUP() using an - # advanced regex: - # - # \bSTRUCT_GROUP(\(((?:(?>[^)(]+)|(?1))*)\))[^;]*; - # - # with two patterns that are incompatible with - # Python re module, as it has: - # - # - a recursive pattern: (?1) - # - an atomic grouping: (?>...) - # - # I tried a simpler version: but it didn't work either: - # \bSTRUCT_GROUP\(([^\)]+)\)[^;]*; - # - # As it doesn't properly match the end parenthesis on some cases. - # - # So, a better solution was crafted: there's now a NestedMatch - # class that ensures that delimiters after a search are properly - # matched. So, the implementation to drop STRUCT_GROUP() will be - # handled in separate. - # - (KernRe(r'\bstruct_group\s*\(([^,]*,)', re.S), r'STRUCT_GROUP('), - (KernRe(r'\bstruct_group_attr\s*\(([^,]*,){2}', re.S), r'STRUCT_GROUP('), - (KernRe(r'\bstruct_group_tagged\s*\(([^,]*),([^,]*),', re.S), r'struct \1 \2; STRUCT_GROUP('), - (KernRe(r'\b__struct_group\s*\(([^,]*,){3}', re.S), r'STRUCT_GROUP('), - # - # Replace macros - # - # TODO: use NestedMatch for FOO($1, $2, ...) matches - # - # it is better to also move those to the NestedMatch logic, - # to ensure that parentheses will be properly matched. - # - (KernRe(r'__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)', re.S), - r'DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)'), - (KernRe(r'DECLARE_PHY_INTERFACE_MASK\s*\(([^\)]+)\)', re.S), - r'DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)'), - (KernRe(r'DECLARE_BITMAP\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', - re.S), r'unsigned long \1[BITS_TO_LONGS(\2)]'), - (KernRe(r'DECLARE_HASHTABLE\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', - re.S), r'unsigned long \1[1 << ((\2) - 1)]'), - (KernRe(r'DECLARE_KFIFO\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + - r',\s*' + struct_args_pattern + r'\)', re.S), r'\2 *\1'), - (KernRe(r'DECLARE_KFIFO_PTR\s*\(' + struct_args_pattern + r',\s*' + - struct_args_pattern + r'\)', re.S), r'\2 *\1'), - (KernRe(r'(?:__)?DECLARE_FLEX_ARRAY\s*\(' + struct_args_pattern + r',\s*' + - struct_args_pattern + r'\)', re.S), r'\1 \2[]'), - (KernRe(r'DEFINE_DMA_UNMAP_ADDR\s*\(' + struct_args_pattern + r'\)', re.S), r'dma_addr_t \1'), - (KernRe(r'DEFINE_DMA_UNMAP_LEN\s*\(' + struct_args_pattern + r'\)', re.S), r'__u32 \1'), -] -# -# Regexes here are guaranteed to have the end delimiter matching -# the start delimiter. Yet, right now, only one replace group -# is allowed. -# -struct_nested_prefixes = [ - (re.compile(r'\bSTRUCT_GROUP\('), r'\1'), -] - -# -# Transforms for function prototypes -# -function_xforms = [ - (KernRe(r"^static +"), ""), - (KernRe(r"^extern +"), ""), - (KernRe(r"^asmlinkage +"), ""), - (KernRe(r"^inline +"), ""), - (KernRe(r"^__inline__ +"), ""), - (KernRe(r"^__inline +"), ""), - (KernRe(r"^__always_inline +"), ""), - (KernRe(r"^noinline +"), ""), - (KernRe(r"^__FORTIFY_INLINE +"), ""), - (KernRe(r"__init +"), ""), - (KernRe(r"__init_or_module +"), ""), - (KernRe(r"__exit +"), ""), - (KernRe(r"__deprecated +"), ""), - (KernRe(r"__flatten +"), ""), - (KernRe(r"__meminit +"), ""), - (KernRe(r"__must_check +"), ""), - (KernRe(r"__weak +"), ""), - (KernRe(r"__sched +"), ""), - (KernRe(r"_noprof"), ""), - (KernRe(r"__always_unused *"), ""), - (KernRe(r"__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +"), ""), - (KernRe(r"__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +"), ""), - (KernRe(r"__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +"), ""), - (KernRe(r"DECL_BUCKET_PARAMS\s*\(\s*(\S+)\s*,\s*(\S+)\s*\)"), r"\1, \2"), - (KernRe(r"__attribute_const__ +"), ""), - (KernRe(r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+"), ""), -] - -# # Ancillary functions # -def apply_transforms(xforms, text): - """ - Apply a set of transforms to a block of text. - """ - for search, subst in xforms: - text = search.sub(subst, text) - return text - multi_space = KernRe(r'\s\s+') def trim_whitespace(s): """ @@ -215,15 +86,9 @@ def trim_private_members(text): """ Remove ``struct``/``enum`` members that have been marked "private". """ - # First look for a "public:" block that ends a private region, then - # handle the "private until the end" case. - # - text = KernRe(r'/\*\s*private:.*?/\*\s*public:.*?\*/', flags=re.S).sub('', text) - text = KernRe(r'/\*\s*private:.*', flags=re.S).sub('', text) - # - # We needed the comments to do the above, but now we can take them out. - # - return KernRe(r'\s*/\*.*?\*/\s*', flags=re.S).sub('', text).strip() + + tokens = CTokenizer(text) + return str(tokens) class state: """ @@ -276,7 +141,7 @@ class KernelEntry: self.parametertypes = {} self.parameterdesc_start_lines = {} - self.section_start_lines = {} + self.sections_start_lines = {} self.sections = {} self.anon_struct_union = False @@ -356,7 +221,7 @@ class KernelEntry: self.sections[name] += '\n' + contents else: self.sections[name] = contents - self.section_start_lines[name] = self.new_start_line + self.sections_start_lines[name] = self.new_start_line self.new_start_line = 0 # self.config.log.debug("Section: %s : %s", name, pformat(vars(self))) @@ -382,11 +247,15 @@ class KernelDoc: #: String to write when a parameter is not described. undescribed = "-- undescribed --" - def __init__(self, config, fname): + def __init__(self, config, fname, xforms, store_src=False): """Initialize internal variables""" self.fname = fname self.config = config + self.xforms = xforms + self.store_src = store_src + + tokenizer_set_log(self.config.log, f"{self.fname}: CMatch: ") # Initial state for the state machines self.state = state.NORMAL @@ -449,7 +318,7 @@ class KernelDoc: for section in ["Description", "Return"]: if section in sections and not sections[section].rstrip(): del sections[section] - item.set_sections(sections, self.entry.section_start_lines) + item.set_sections(sections, self.entry.sections_start_lines) item.set_params(self.entry.parameterlist, self.entry.parameterdescs, self.entry.parametertypes, self.entry.parameterdesc_start_lines) @@ -512,9 +381,36 @@ class KernelDoc: # if dtype == '': if param.endswith("..."): - if len(param) > 3: # there is a name provided, use that + named_variadic = len(param) > 3 + if named_variadic: # there is a name provided, use that + # + # If the user documented the parameter using the + # ``@name...:`` form, the description is stored in + # parameterdescs under the unstripped key. Migrate + # it to the stripped key so the user's text is not + # silently dropped during output, and so the new + # excess-parameter check in check_sections() does + # not flag the unstripped key as orphaned. + # + orig = self.entry.parameterdescs.pop(param, None) param = param[:-3] + if orig is not None and \ + not self.entry.parameterdescs.get(param): + self.entry.parameterdescs[param] = orig if not self.entry.parameterdescs.get(param): + # + # For a named variadic (e.g. ``args...``), emit the + # standard "not described" warning before auto-filling + # so a missing or mistyped ``@<name>:`` doc tag does + # not go undetected. The bare ``...`` form has no + # natural name for the user to document and so always + # gets the auto-generated text. + # + if named_variadic and decl_type == 'function': + self.emit_msg(ln, + f"function parameter '{param}' " + f"not described in " + f"'{declaration_name}'") self.entry.parameterdescs[param] = "variable arguments" elif (not param) or param == "void": @@ -571,6 +467,11 @@ class KernelDoc: # Ignore argument attributes arg = KernRe(r'\sPOS0?\s').sub(' ', arg) + # Replace '[at_least ' with '[static '. This allows sphinx to parse + # array parameter declarations like 'char A[at_least 4]', where + # 'at_least' is #defined to 'static' by the kernel headers. + arg = arg.replace('[at_least ', '[static ') + # Strip leading/trailing spaces arg = arg.strip() arg = KernRe(r'\s+').sub(' ', arg, count=1) @@ -658,6 +559,51 @@ class KernelDoc: self.push_parameter(ln, decl_type, param, dtype, arg, declaration_name) + def get_suggestions_hint(self, decl_name, possible_names): + # For decl name 'flags' or 'flgas', suggests 'substruct.flags' + submember_exact = [] + submember_substrings = [] + submember_suggestions = [] + for possible_name in possible_names: + parts = possible_name.strip().split('.') + if len(parts) < 2: + continue + + final_part = parts[-1] + if decl_name == final_part: + submember_exact.append(possible_name) + elif decl_name in final_part: + submember_substrings.append(possible_name) + elif difflib.get_close_matches(decl_name, [final_part]): + submember_suggestions.append(possible_name) + + # For decl name 'flgas', suggests 'flags' + full_suggestions = difflib.get_close_matches(decl_name, possible_names) + + # For decl name 'member', suggests 'longer_member' + full_substrings = [name for name in possible_names if decl_name in name] + + ordered_lists = [ + submember_exact, + submember_substrings, + submember_suggestions, + full_suggestions, + full_substrings, + ] + + # Deduplicate but maintain order from most to least likely: + unique_suggestions = {} + for suggestion_list in ordered_lists: + for suggestion in suggestion_list: + unique_suggestions[suggestion] = None + + suggestions = list(unique_suggestions.keys()) + if not suggestions: + return "" + + joined_suggestions = "', '".join(suggestions) + return f"(did you mean one of: '{joined_suggestions}')" + def check_sections(self, ln, decl_name, decl_type): """ Check for errors inside sections, emitting warnings if not found @@ -666,12 +612,39 @@ class KernelDoc: for section in self.entry.sections: if section not in self.entry.parameterlist and \ not known_sections.search(section): + hint = self.get_suggestions_hint(section, self.entry.parameterlist) if decl_type == 'function': dname = f"{decl_type} parameter" else: dname = f"{decl_type} member" self.emit_msg(ln, - f"Excess {dname} '{section}' description in '{decl_name}'") + f"Excess {dname} '{section}' description in '{decl_name}' {hint}".strip()) + + # + # Check that documented parameter names (from doc comments, including + # inline ``/** @member: */`` tags) actually match real members in + # the declaration. This catches mismatched or stale kernel-doc + # member tags that don't correspond to any actual struct/union + # member or function parameter. + # + for param_name, desc in self.entry.parameterdescs.items(): + # Skip auto-generated entries from push_parameter() + if desc == self.undescribed: + continue + if desc in ("no arguments", "anonymous\n", "variable arguments"): + continue + if param_name.startswith("{unnamed_"): + continue + if param_name in self.entry.parameterlist: + continue + + hint = self.get_suggestions_hint(param_name, self.entry.parameterlist) + if decl_type == 'function': + dname = f"{decl_type} parameter" + else: + dname = f"{decl_type} member" + self.emit_msg(ln, + f"Excess {dname} '{param_name}' description in '{decl_name}' {hint}".strip()) def check_return_section(self, ln, declaration_name, return_type): """ @@ -849,13 +822,15 @@ class KernelDoc: return declaration - def dump_struct(self, ln, proto): + def dump_struct(self, ln, proto, source): """ Store an entry for a ``struct`` or ``union`` """ # # Do the basic parse to get the pieces of the declaration. # + source = source + proto = trim_private_members(proto) struct_parts = self.split_struct_proto(proto) if not struct_parts: self.emit_msg(ln, f"{proto} error: Cannot parse struct or union!") @@ -864,17 +839,13 @@ class KernelDoc: if self.entry.identifier != declaration_name: self.emit_msg(ln, f"expecting prototype for {decl_type} {self.entry.identifier}. " - f"Prototype was for {decl_type} {declaration_name} instead\n") + f"Prototype was for {decl_type} {declaration_name} instead") return # # Go through the list of members applying all of our transformations. # - members = trim_private_members(members) - members = apply_transforms(struct_xforms, members) + members = self.xforms.apply("struct", members) - nested = NestedMatch() - for search, sub in struct_nested_prefixes: - members = nested.sub(search, sub, members) # # Deal with embedded struct and union members, and drop enums entirely. # @@ -888,10 +859,11 @@ class KernelDoc: declaration_name) self.check_sections(ln, declaration_name, decl_type) self.output_declaration(decl_type, declaration_name, + source=source, definition=self.format_struct_decl(declaration), purpose=self.entry.declaration_purpose) - def dump_enum(self, ln, proto): + def dump_enum(self, ln, proto, source): """ Store an ``enum`` inside self.entries array. """ @@ -899,6 +871,8 @@ class KernelDoc: # Strip preprocessor directives. Note that this depends on the # trailing semicolon we added in process_proto_type(). # + source = source + proto = trim_private_members(proto) proto = KernRe(r'#\s*((define|ifdef|if)\s+|endif)[^;]*;', flags=re.S).sub('', proto) # # Parse out the name and members of the enum. Typedef form first. @@ -906,7 +880,7 @@ class KernelDoc: r = KernRe(r'typedef\s+enum\s*\{(.*)\}\s*(\w*)\s*;') if r.search(proto): declaration_name = r.group(2) - members = trim_private_members(r.group(1)) + members = r.group(1) # # Failing that, look for a straight enum # @@ -914,7 +888,7 @@ class KernelDoc: r = KernRe(r'enum\s+(\w*)\s*\{(.*)\}') if r.match(proto): declaration_name = r.group(1) - members = trim_private_members(r.group(2)) + members = r.group(2) # # OK, this isn't going to work. # @@ -943,9 +917,10 @@ class KernelDoc: member_set = set() members = KernRe(r'\([^;)]*\)').sub('', members) for arg in members.split(','): - if not arg: - continue arg = KernRe(r'^\s*(\w+).*').sub(r'\1', arg) + if not arg.strip(): + continue + self.entry.parameterlist.append(arg) if arg not in self.entry.parameterdescs: self.entry.parameterdescs[arg] = self.undescribed @@ -961,29 +936,23 @@ class KernelDoc: f"Excess enum value '@{k}' description in '{declaration_name}'") self.output_declaration('enum', declaration_name, + source=source, purpose=self.entry.declaration_purpose) - def dump_var(self, ln, proto): + def dump_var(self, ln, proto, source): """ Store variables that are part of kAPI. """ VAR_ATTRIBS = [ "extern", + "const", ] - OPTIONAL_VAR_ATTR = "^(?:" + "|".join(VAR_ATTRIBS) + ")?" - - sub_prefixes = [ - (KernRe(r"__read_mostly"), ""), - (KernRe(r"__ro_after_init"), ""), - (KernRe(r"(?://.*)$"), ""), - (KernRe(r"(?:/\*.*\*/)"), ""), - (KernRe(r";$"), ""), - (KernRe(r"=.*"), ""), - ] + OPTIONAL_VAR_ATTR = r"^(?:\b(?:" +"|".join(VAR_ATTRIBS) +r")\b\s*)*" # # Store the full prototype before modifying it # + source = source full_proto = proto declaration_name = None @@ -1004,8 +973,7 @@ class KernelDoc: # Drop comments and macros to have a pure C prototype # if not declaration_name: - for r, sub in sub_prefixes: - proto = r.sub(sub, proto) + proto = self.xforms.apply("var", proto) proto = proto.rstrip() @@ -1015,17 +983,17 @@ class KernelDoc: default_val = None - r= KernRe(OPTIONAL_VAR_ATTR + r"\w.*\s+(?:\*+)?([\w_]+)\s*[\d\]\[]*\s*(=.*)?") + r= KernRe(OPTIONAL_VAR_ATTR + r"\s*[\w_\s]*\s+(?:\*+)?([\w_]+)\s*[\d\]\[]*\s*(=.*)?") if r.match(proto): if not declaration_name: declaration_name = r.group(1) default_val = r.group(2) else: - r= KernRe(OPTIONAL_VAR_ATTR + r"(?:\w.*)?\s+(?:\*+)?(?:[\w_]+)\s*[\d\]\[]*\s*(=.*)?") - if r.match(proto): - default_val = r.group(1) + r= KernRe(OPTIONAL_VAR_ATTR + r"(?:[\w_\s]*)?\s+(?:\*+)?(?:[\w_]+)\s*[\d\]\[]*\s*(=.*)?") + if r.match(proto): + default_val = r.group(1) if not declaration_name: self.emit_msg(ln,f"{proto}: can't parse variable") return @@ -1034,39 +1002,38 @@ class KernelDoc: default_val = default_val.lstrip("=").strip() self.output_declaration("var", declaration_name, + source=source, full_proto=full_proto, default_val=default_val, purpose=self.entry.declaration_purpose) - def dump_declaration(self, ln, prototype): + def dump_declaration(self, ln, prototype, source): """ Store a data declaration inside self.entries array. """ if self.entry.decl_type == "enum": - self.dump_enum(ln, prototype) + self.dump_enum(ln, prototype, source) elif self.entry.decl_type == "typedef": - self.dump_typedef(ln, prototype) + self.dump_typedef(ln, prototype, source) elif self.entry.decl_type in ["union", "struct"]: - self.dump_struct(ln, prototype) + self.dump_struct(ln, prototype, source) elif self.entry.decl_type == "var": - self.dump_var(ln, prototype) + self.dump_var(ln, prototype, source) else: # This would be a bug self.emit_message(ln, f'Unknown declaration type: {self.entry.decl_type}') - def dump_function(self, ln, prototype): + def dump_function(self, ln, prototype, source): """ Store a function or function macro inside self.entries array. """ + source = source found = func_macro = False return_type = '' decl_type = 'function' - # - # Apply the initial transformations. - # - prototype = apply_transforms(function_xforms, prototype) + # # If we have a macro, remove the "#define" at the front. # @@ -1085,6 +1052,11 @@ class KernelDoc: declaration_name = r.group(1) func_macro = True found = True + else: + # + # Apply the initial transformations. + # + prototype = self.xforms.apply("func", prototype) # Yes, this truly is vile. We are looking for: # 1. Return type (may be nothing if we're looking at a macro) @@ -1150,13 +1122,14 @@ class KernelDoc: # Store the result. # self.output_declaration(decl_type, declaration_name, + source=source, typedef=('typedef' in return_type), functiontype=return_type, purpose=self.entry.declaration_purpose, func_macro=func_macro) - def dump_typedef(self, ln, proto): + def dump_typedef(self, ln, proto, source): """ Store a ``typedef`` inside self.entries array. """ @@ -1167,6 +1140,8 @@ class KernelDoc: typedef_ident = r'\*?\s*(\w\S+)\s*' typedef_args = r'\s*\((.*)\);' + source = source + typedef1 = KernRe(typedef_type + r'\(' + typedef_ident + r'\)' + typedef_args) typedef2 = KernRe(typedef_type + typedef_ident + typedef_args) @@ -1181,12 +1156,13 @@ class KernelDoc: if self.entry.identifier != declaration_name: self.emit_msg(ln, - f"expecting prototype for typedef {self.entry.identifier}. Prototype was for typedef {declaration_name} instead\n") + f"expecting prototype for typedef {self.entry.identifier}. Prototype was for typedef {declaration_name} instead") return self.create_parameter_list(ln, 'function', args, ',', declaration_name) self.output_declaration('function', declaration_name, + source=source, typedef=True, functiontype=return_type, purpose=self.entry.declaration_purpose) @@ -1200,10 +1176,11 @@ class KernelDoc: if self.entry.identifier != declaration_name: self.emit_msg(ln, - f"expecting prototype for typedef {self.entry.identifier}. Prototype was for typedef {declaration_name} instead\n") + f"expecting prototype for typedef {self.entry.identifier}. Prototype was for typedef {declaration_name} instead") return self.output_declaration('typedef', declaration_name, + source=source, purpose=self.entry.declaration_purpose) return @@ -1241,7 +1218,7 @@ class KernelDoc: function_set.add(symbol) return True - def process_normal(self, ln, line): + def process_normal(self, ln, line, source): """ STATE_NORMAL: looking for the ``/**`` to begin everything. """ @@ -1255,7 +1232,7 @@ class KernelDoc: # next line is always the function name self.state = state.NAME - def process_name(self, ln, line): + def process_name(self, ln, line, source): """ STATE_NAME: Looking for the "name - description" line """ @@ -1388,7 +1365,7 @@ class KernelDoc: return False - def process_decl(self, ln, line): + def process_decl(self, ln, line, source): """ STATE_DECLARATION: We've seen the beginning of a declaration. """ @@ -1417,7 +1394,7 @@ class KernelDoc: self.emit_msg(ln, f"bad line: {line}") - def process_special(self, ln, line): + def process_special(self, ln, line, source): """ STATE_SPECIAL_SECTION: a section ending with a blank line. """ @@ -1468,7 +1445,7 @@ class KernelDoc: # Unknown line, ignore self.emit_msg(ln, f"bad line: {line}") - def process_body(self, ln, line): + def process_body(self, ln, line, source): """ STATE_BODY: the bulk of a kerneldoc comment. """ @@ -1482,7 +1459,7 @@ class KernelDoc: # Unknown line, ignore self.emit_msg(ln, f"bad line: {line}") - def process_inline_name(self, ln, line): + def process_inline_name(self, ln, line, source): """STATE_INLINE_NAME: beginning of docbook comments within a prototype.""" if doc_inline_sect.search(line): @@ -1495,9 +1472,15 @@ class KernelDoc: elif doc_content.search(line): self.emit_msg(ln, f"Incorrect use of kernel-doc format: {line}") self.state = state.PROTO + + # + # Don't let it add partial comments at the code, as breaks the + # logic meant to remove comments from prototypes. + # + self.process_proto_type(ln, "/**\n" + line, source) # else ... ?? - def process_inline_text(self, ln, line): + def process_inline_text(self, ln, line, source): """STATE_INLINE_TEXT: docbook comments within a prototype.""" if doc_inline_end.search(line): @@ -1583,7 +1566,7 @@ class KernelDoc: return proto - def process_proto_function(self, ln, line): + def process_proto_function(self, ln, line, source): """Ancillary routine to process a function prototype.""" # strip C99-style comments to end of line @@ -1625,10 +1608,10 @@ class KernelDoc: # # ... and we're done # - self.dump_function(ln, self.entry.prototype) + self.dump_function(ln, self.entry.prototype, source) self.reset_state(ln) - def process_proto_type(self, ln, line): + def process_proto_type(self, ln, line, source): """ Ancillary routine to process a type. """ @@ -1658,7 +1641,7 @@ class KernelDoc: elif chunk == '}': self.entry.brcount -= 1 elif chunk == ';' and self.entry.brcount <= 0: - self.dump_declaration(ln, self.entry.prototype) + self.dump_declaration(ln, self.entry.prototype, source) self.reset_state(ln) return # @@ -1667,7 +1650,7 @@ class KernelDoc: # self.entry.prototype += ' ' - def process_proto(self, ln, line): + def process_proto(self, ln, line, source): """STATE_PROTO: reading a function/whatever prototype.""" if doc_inline_oneline.search(line): @@ -1679,17 +1662,18 @@ class KernelDoc: self.state = state.INLINE_NAME elif self.entry.decl_type == 'function': - self.process_proto_function(ln, line) + self.process_proto_function(ln, line, source) else: - self.process_proto_type(ln, line) + self.process_proto_type(ln, line, source) - def process_docblock(self, ln, line): + def process_docblock(self, ln, line, source): """STATE_DOCBLOCK: within a ``DOC:`` block.""" if doc_end.search(line): self.dump_section() - self.output_declaration("doc", self.entry.identifier) + self.output_declaration("doc", self.entry.identifier, + source=source) self.reset_state(ln) elif doc_content.search(line): @@ -1740,6 +1724,8 @@ class KernelDoc: prev = "" prev_ln = None export_table = set() + self.state = state.NORMAL + source = "" try: with open(self.fname, "r", encoding="utf8", @@ -1766,6 +1752,12 @@ class KernelDoc: ln, state.name[self.state], line) + if self.store_src: + if source and self.state == state.NORMAL: + source = "" + elif self.state != state.NORMAL: + source += line + "\n" + # This is an optimization over the original script. # There, when export_file was used for the same file, # it was read twice. Here, we use the already-existing @@ -1773,8 +1765,11 @@ class KernelDoc: # if (self.state != state.NORMAL) or \ not self.process_export(export_table, line): + prev_state = self.state # Hand this line to the appropriate state handler - self.state_actions[self.state](self, ln, line) + self.state_actions[self.state](self, ln, line, source) + if prev_state == state.NORMAL and self.state != state.NORMAL: + source += line + "\n" self.emit_unused_warnings() diff --git a/tools/lib/python/kdoc/kdoc_re.py b/tools/lib/python/kdoc/kdoc_re.py index 0bf9e01cdc57..28292efe25a2 100644 --- a/tools/lib/python/kdoc/kdoc_re.py +++ b/tools/lib/python/kdoc/kdoc_re.py @@ -52,7 +52,33 @@ class KernRe: return self.regex.pattern def __repr__(self): - return f're.compile("{self.regex.pattern}")' + """ + Returns a displayable version of the class init. + """ + + flag_map = { + re.IGNORECASE: "re.I", + re.MULTILINE: "re.M", + re.DOTALL: "re.S", + re.VERBOSE: "re.X", + } + + flags = [] + for flag, name in flag_map.items(): + if self.regex.flags & flag: + flags.append(name) + + flags_name = " | ".join(flags) + + max_len = 60 + pattern = "" + for pos in range(0, len(self.regex.pattern), max_len): + pattern += '"' + self.regex.pattern[pos:max_len + pos] + '" ' + + if flags_name: + return f'KernRe({pattern}, {flags_name})' + else: + return f'KernRe({pattern})' def __add__(self, other): """ @@ -78,6 +104,13 @@ class KernRe: self.last_match = self.regex.search(string) return self.last_match + def finditer(self, string): + """ + Alias to re.finditer. + """ + + return self.regex.finditer(string) + def findall(self, string): """ Alias to re.findall. @@ -106,173 +139,9 @@ class KernRe: return self.last_match.group(num) - -class NestedMatch: - """ - Finding nested delimiters is hard with regular expressions. It is - even harder on Python with its normal re module, as there are several - advanced regular expressions that are missing. - - This is the case of this pattern:: - - '\\bSTRUCT_GROUP(\\(((?:(?>[^)(]+)|(?1))*)\\))[^;]*;' - - which is used to properly match open/close parentheses of the - string search STRUCT_GROUP(), - - Add a class that counts pairs of delimiters, using it to match and - replace nested expressions. - - The original approach was suggested by: - - https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex - - Although I re-implemented it to make it more generic and match 3 types - of delimiters. The logic checks if delimiters are paired. If not, it - will ignore the search string. - """ - - # TODO: make NestedMatch handle multiple match groups - # - # Right now, regular expressions to match it are defined only up to - # the start delimiter, e.g.: - # - # \bSTRUCT_GROUP\( - # - # is similar to: STRUCT_GROUP\((.*)\) - # except that the content inside the match group is delimiter-aligned. - # - # The content inside parentheses is converted into a single replace - # group (e.g. r`\1'). - # - # It would be nice to change such definition to support multiple - # match groups, allowing a regex equivalent to: - # - # FOO\((.*), (.*), (.*)\) - # - # it is probably easier to define it not as a regular expression, but - # with some lexical definition like: - # - # FOO(arg1, arg2, arg3) - - DELIMITER_PAIRS = { - '{': '}', - '(': ')', - '[': ']', - } - - RE_DELIM = re.compile(r'[\{\}\[\]\(\)]') - - def _search(self, regex, line): - """ - Finds paired blocks for a regex that ends with a delimiter. - - The suggestion of using finditer to match pairs came from: - https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex - but I ended using a different implementation to align all three types - of delimiters and seek for an initial regular expression. - - The algorithm seeks for open/close paired delimiters and places them - into a stack, yielding a start/stop position of each match when the - stack is zeroed. - - The algorithm should work fine for properly paired lines, but will - silently ignore end delimiters that precede a start delimiter. - This should be OK for kernel-doc parser, as unaligned delimiters - would cause compilation errors. So, we don't need to raise exceptions - to cover such issues. - """ - - stack = [] - - for match_re in regex.finditer(line): - start = match_re.start() - offset = match_re.end() - - d = line[offset - 1] - if d not in self.DELIMITER_PAIRS: - continue - - end = self.DELIMITER_PAIRS[d] - stack.append(end) - - for match in self.RE_DELIM.finditer(line[offset:]): - pos = match.start() + offset - - d = line[pos] - - if d in self.DELIMITER_PAIRS: - end = self.DELIMITER_PAIRS[d] - - stack.append(end) - continue - - # Does the end delimiter match what is expected? - if stack and d == stack[-1]: - stack.pop() - - if not stack: - yield start, offset, pos + 1 - break - - def search(self, regex, line): + def groups(self): """ - This is similar to re.search: - - It matches a regex that it is followed by a delimiter, - returning occurrences only if all delimiters are paired. + Returns the group results of the last match """ - for t in self._search(regex, line): - - yield line[t[0]:t[2]] - - def sub(self, regex, sub, line, count=0): - r""" - This is similar to re.sub: - - It matches a regex that it is followed by a delimiter, - replacing occurrences only if all delimiters are paired. - - if the sub argument contains:: - - r'\1' - - it will work just like re: it places there the matched paired data - with the delimiter stripped. - - If count is different than zero, it will replace at most count - items. - """ - out = "" - - cur_pos = 0 - n = 0 - - for start, end, pos in self._search(regex, line): - out += line[cur_pos:start] - - # Value, ignoring start/end delimiters - value = line[end:pos - 1] - - # replaces \1 at the sub string, if \1 is used there - new_sub = sub - new_sub = new_sub.replace(r'\1', value) - - out += new_sub - - # Drop end ';' if any - if line[pos] == ';': - pos += 1 - - cur_pos = pos - n += 1 - - if count and count >= n: - break - - # Append the remaining string - l = len(line) - out += line[cur_pos:l] - - return out + return self.last_match.groups() diff --git a/tools/lib/python/kdoc/kdoc_yaml_file.py b/tools/lib/python/kdoc/kdoc_yaml_file.py new file mode 100644 index 000000000000..0be020d50df0 --- /dev/null +++ b/tools/lib/python/kdoc/kdoc_yaml_file.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2026: Mauro Carvalho Chehab <mchehab@kernel.org>. + +import os + +from kdoc.kdoc_output import ManFormat, RestFormat + + +class KDocTestFile(): + """ + Handles the logic needed to store kernel‑doc output inside a YAML file. + Useful for unit tests and regression tests. + """ + + def __init__(self, config, yaml_file, yaml_content): + # + # Bail out early if yaml is not available + # + try: + import yaml + except ImportError: + sys.exit("Warning: yaml package not available. Aborting it.") + + self.config = config + self.test_file = os.path.expanduser(yaml_file) + self.yaml_content = yaml_content + self.test_names = set() + + self.tests = [] + + out_dir = os.path.dirname(self.test_file) + if out_dir and not os.path.isdir(out_dir): + sys.exit(f"Directory {out_dir} doesn't exist.") + + self.out_style = [] + + if "man" in self.yaml_content: + out_style = ManFormat() + out_style.set_config(self.config) + + self.out_style.append(out_style) + + if "rst" in self.yaml_content: + out_style = RestFormat() + out_style.set_config(self.config) + + self.out_style.append(out_style) + + def set_filter(self, export, internal, symbol, nosymbol, + function_table, enable_lineno, no_doc_sections): + """ + Set filters at the output classes. + """ + for out_style in self.out_style: + out_style.set_filter(export, internal, symbol, + nosymbol, function_table, + enable_lineno, no_doc_sections) + + @staticmethod + def get_kdoc_item(arg, start_line=1): + + d = vars(arg) + + declaration_start_line = d.get("declaration_start_line") + if not declaration_start_line: + return d + + d["declaration_start_line"] = start_line + + parameterdesc_start_lines = d.get("parameterdesc_start_lines") + if parameterdesc_start_lines: + for key in parameterdesc_start_lines: + ln = parameterdesc_start_lines[key] + ln += start_line - declaration_start_line + + parameterdesc_start_lines[key] = ln + + sections_start_lines = d.get("sections_start_lines") + if sections_start_lines: + for key in sections_start_lines: + ln = sections_start_lines[key] + ln += start_line - declaration_start_line + + sections_start_lines[key] = ln + + return d + + def output_symbols(self, fname, symbols): + """ + Store source, symbols and output strings at self.tests. + """ + + # + # KdocItem needs to be converted into dicts + # + kdoc_item = [] + expected = [] + + # + # Source code didn't produce any symbol + # + if not symbols: + return + + expected_dict = {} + start_line=1 + + for arg in symbols: + source = arg.get("source", "") + + if arg and "KdocItem" in self.yaml_content: + msg = self.get_kdoc_item(arg) + + other_stuff = msg.get("other_stuff", {}) + if "source" in other_stuff: + del other_stuff["source"] + + expected_dict["kdoc_item"] = msg + + base_name = arg.name + if not base_name: + base_name = fname + base_name = base_name.lower().replace(".", "_").replace("/", "_") + + + # Don't add duplicated names + i = 0 + name = base_name + while name in self.test_names: + i += 1 + name = f"{base_name}_{i:03d}" + + self.test_names.add(name) + + for out_style in self.out_style: + if isinstance(out_style, ManFormat): + key = "man" + else: + key = "rst" + + expected_dict[key]= out_style.output_symbols(fname, [arg]).strip() + + test = { + "name": name, + "description": f"{fname} line {arg.declaration_start_line}", + "fname": fname, + "source": source, + "expected": [expected_dict] + } + + self.tests.append(test) + + expected_dict = {} + + def write(self): + """ + Output the content of self.tests to self.test_file. + """ + import yaml + + # Helper function to better handle multilines + def str_presenter(dumper, data): + if "\n" in data: + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + + return dumper.represent_scalar("tag:yaml.org,2002:str", data) + + # Register the representer + yaml.add_representer(str, str_presenter) + + data = {"tests": self.tests} + + with open(self.test_file, "w", encoding="utf-8") as fp: + yaml.dump(data, fp, + sort_keys=False, width=120, indent=2, + default_flow_style=False, allow_unicode=True, + explicit_start=False, explicit_end=False) diff --git a/tools/lib/python/kdoc/xforms_lists.py b/tools/lib/python/kdoc/xforms_lists.py new file mode 100644 index 000000000000..e3dda2fe8a53 --- /dev/null +++ b/tools/lib/python/kdoc/xforms_lists.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2026: Mauro Carvalho Chehab <mchehab@kernel.org>. + +import re + +from kdoc.kdoc_re import KernRe +from kdoc.c_lex import CMatch, CTokenizer + +struct_args_pattern = r"([^,)]+)" + + +class CTransforms: + """ + Data class containing a long set of transformations to turn + structure member prefixes, and macro invocations and variables + into something we can parse and generate kdoc for. + """ + + # + # NOTE: + # Due to performance reasons, place CMatch rules before KernRe, + # as this avoids running the C parser every time. + # + + #: Transforms for structs and unions. + struct_xforms = [ + (CMatch("__attribute__"), ""), + (CMatch("__aligned"), ""), + (CMatch("__counted_by"), ""), + (CMatch("__counted_by_(le|be)"), ""), + (CMatch("__counted_by_ptr"), ""), + (CMatch("__guarded_by"), ""), + (CMatch("__pt_guarded_by"), ""), + (CMatch("__packed"), ""), + (CMatch("CRYPTO_MINALIGN_ATTR"), ""), + (CMatch("__private"), ""), + (CMatch("__rcu"), ""), + (CMatch("____cacheline_aligned_in_smp"), ""), + (CMatch("____cacheline_aligned"), ""), + (CMatch("__cacheline_group_(?:begin|end)"), ""), + (CMatch("__ETHTOOL_DECLARE_LINK_MODE_MASK"), r"DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)"), + (CMatch("DECLARE_PHY_INTERFACE_MASK",),r"DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)"), + (CMatch("DECLARE_BITMAP"), r"unsigned long \1[BITS_TO_LONGS(\2)]"), + (CMatch("DECLARE_HASHTABLE"), r"unsigned long \1[1 << ((\2) - 1)]"), + (CMatch("DECLARE_KFIFO"), r"\2 *\1"), + (CMatch("DECLARE_KFIFO_PTR"), r"\2 *\1"), + (CMatch("(?:__)?DECLARE_FLEX_ARRAY"), r"\1 \2[]"), + (CMatch("DEFINE_DMA_UNMAP_ADDR"), r"dma_addr_t \1"), + (CMatch("DEFINE_DMA_UNMAP_LEN"), r"__u32 \1"), + (CMatch("VIRTIO_DECLARE_FEATURES"), r"union { u64 \1; u64 \1_array[VIRTIO_FEATURES_U64S]; }"), + (CMatch("__SYSFS_FUNCTION_ALTERNATIVE"), r"union { \1+ }"), + (CMatch("__attribute__"), ""), + + # + # Macro __struct_group() creates an union with an anonymous + # and a non-anonymous struct, depending on the parameters. We only + # need one of those at kernel-doc, as we won't be documenting the same + # members twice. + # + (CMatch("struct_group"), r"struct { \2+ };"), + (CMatch("struct_group_attr"), r"struct { \3+ };"), + (CMatch("struct_group_tagged"), r"struct { \3+ };"), + (CMatch("__struct_group"), r"struct { \4+ };"), + ] + + #: Transforms for function prototypes. + function_xforms = [ + (CMatch("static"), ""), + (CMatch("extern"), ""), + (CMatch("asmlinkage"), ""), + (CMatch("inline"), ""), + (CMatch("__inline__"), ""), + (CMatch("__inline"), ""), + (CMatch("__always_inline"), ""), + (CMatch("noinline"), ""), + (CMatch("__FORTIFY_INLINE"), ""), + (CMatch("__init"), ""), + (CMatch("__init_or_module"), ""), + (CMatch("__exit"), ""), + (CMatch("__deprecated"), ""), + (CMatch("__flatten"), ""), + (CMatch("__meminit"), ""), + (CMatch("__must_check"), ""), + (CMatch("__weak"), ""), + (CMatch("__sched"), ""), + (CMatch("__always_unused"), ""), + (CMatch("__maybe_unused"), ""), + (CMatch("__printf"), ""), + (CMatch("__(?:re)?alloc_size"), ""), + (CMatch("__diagnose_as"), ""), + (CMatch("DECL_BUCKET_PARAMS"), r"\1, \2"), + (CMatch("DEFINE_IDTENTRY_IRQ"), r"static void \1(struct pt_regs *regs, u32 vector)"), + (CMatch("__cond_acquires"), ""), + (CMatch("__cond_releases"), ""), + (CMatch("__acquires"), ""), + (CMatch("__releases"), ""), + (CMatch("__must_hold"), ""), + (CMatch("__must_not_hold"), ""), + (CMatch("__must_hold_shared"), ""), + (CMatch("__cond_acquires_shared"), ""), + (CMatch("__acquires_shared"), ""), + (CMatch("__releases_shared"), ""), + (CMatch("__no_context_analysis"), ""), + (CMatch("__attribute_const__"), ""), + (CMatch("__attribute__"), ""), + (CMatch("STATIC_IFN_KUNIT"), ""), + (CMatch("INLINE_IFN_KUNIT"), ""), + + # + # HACK: this is similar to process_export() hack. It is meant to + # drop _noproof from function name. See for instance: + # ahash_request_alloc kernel-doc declaration at include/crypto/hash.h. + # + (KernRe("_noprof"), ""), + ] + + #: Transforms for variable prototypes. + var_xforms = [ + (CMatch("__read_mostly"), ""), + (CMatch("__ro_after_init"), ""), + (CMatch("__guarded_by"), ""), + (CMatch("__pt_guarded_by"), ""), + (CMatch("LIST_HEAD"), r"struct list_head \1"), + (CMatch("DECLARE_PER_CPU"), r"\1 \2[PER_CPU]; }"), + + (KernRe(r"(?://.*)$"), ""), + (KernRe(r"(?:/\*.*\*/)"), ""), + (KernRe(r";$"), ""), + ] + + #: Transforms main dictionary used at apply_transforms(). + xforms = { + "struct": struct_xforms, + "func": function_xforms, + "var": var_xforms, + } + + def apply(self, xforms_type, source): + """ + Apply a set of transforms to a block of source. + + As tokenizer is used here, this function also remove comments + at the end. + """ + if xforms_type not in self.xforms: + return source + + if isinstance(source, str): + source = CTokenizer(source) + + for search, subst in self.xforms[xforms_type]: + # + # KernRe only accept strings. + # + if isinstance(search, KernRe): + source = str(source) + + source = search.sub(subst, source) + return str(source) diff --git a/tools/lib/python/unittest_helper.py b/tools/lib/python/unittest_helper.py new file mode 100755 index 000000000000..f3cba5120401 --- /dev/null +++ b/tools/lib/python/unittest_helper.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2025-2026: Mauro Carvalho Chehab <mchehab@kernel.org>. +# +# pylint: disable=C0103,R0912,R0914,E1101 + +""" +Provides helper functions and classes execute python unit tests. + +Those help functions provide a nice colored output summary of each +executed test and, when a test fails, it shows the different in diff +format when running in verbose mode, like:: + + $ tools/unittests/nested_match.py -v + ... + Traceback (most recent call last): + File "/new_devel/docs/tools/unittests/nested_match.py", line 69, in test_count_limit + self.assertEqual(replaced, "bar(a); bar(b); foo(c)") + ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + AssertionError: 'bar(a) foo(b); foo(c)' != 'bar(a); bar(b); foo(c)' + - bar(a) foo(b); foo(c) + ? ^^^^ + + bar(a); bar(b); foo(c) + ? ^^^^^ + ... + +It also allows filtering what tests will be executed via ``-k`` parameter. + +Typical usage is to do:: + + from unittest_helper import run_unittest + ... + + if __name__ == "__main__": + run_unittest(__file__) + +If passing arguments is needed, on a more complex scenario, it can be +used like on this example:: + + from unittest_helper import TestUnits, run_unittest + ... + env = {'sudo': ""} + ... + if __name__ == "__main__": + runner = TestUnits() + base_parser = runner.parse_args() + base_parser.add_argument('--sudo', action='store_true', + help='Enable tests requiring sudo privileges') + + args = base_parser.parse_args() + + # Update module-level flag + if args.sudo: + env['sudo'] = "1" + + # Run tests with customized arguments + runner.run(__file__, parser=base_parser, args=args, env=env) +""" + +import argparse +import atexit +import os +import re +import unittest +import sys + +from unittest.mock import patch + + +class Summary(unittest.TestResult): + """ + Overrides ``unittest.TestResult`` class to provide a nice colored + summary. When in verbose mode, displays actual/expected difference in + unified diff format. + """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + #: Dictionary to store organized test results. + self.test_results = {} + + #: max length of the test names. + self.max_name_length = 0 + + def startTest(self, test): + super().startTest(test) + test_id = test.id() + parts = test_id.split(".") + + # Extract module, class, and method names + if len(parts) >= 3: + module_name = parts[-3] + else: + module_name = "" + if len(parts) >= 2: + class_name = parts[-2] + else: + class_name = "" + + method_name = parts[-1] + + # Build the hierarchical structure + if module_name not in self.test_results: + self.test_results[module_name] = {} + + if class_name not in self.test_results[module_name]: + self.test_results[module_name][class_name] = [] + + # Track maximum test name length for alignment + display_name = f"{method_name}:" + + self.max_name_length = max(len(display_name), self.max_name_length) + + def _record_test(self, test, status): + test_id = test.id() + parts = test_id.split(".") + if len(parts) >= 3: + module_name = parts[-3] + else: + module_name = "" + if len(parts) >= 2: + class_name = parts[-2] + else: + class_name = "" + method_name = parts[-1] + self.test_results[module_name][class_name].append((method_name, status)) + + def addSuccess(self, test): + super().addSuccess(test) + self._record_test(test, "OK") + + def addFailure(self, test, err): + super().addFailure(test, err) + self._record_test(test, "FAIL") + + def addError(self, test, err): + super().addError(test, err) + self._record_test(test, "ERROR") + + def addSkip(self, test, reason): + super().addSkip(test, reason) + self._record_test(test, f"SKIP ({reason})") + + def printResults(self, verbose): + """ + Print results using colors if tty. + """ + # Check for ANSI color support + use_color = sys.stdout.isatty() + COLORS = { + "OK": "\033[32m", # Green + "FAIL": "\033[31m", # Red + "SKIP": "\033[1;33m", # Yellow + "PARTIAL": "\033[33m", # Orange + "EXPECTED_FAIL": "\033[36m", # Cyan + "reset": "\033[0m", # Reset to default terminal color + } + if not use_color: + for c in COLORS: + COLORS[c] = "" + + # Calculate maximum test name length + if not self.test_results: + return + try: + lengths = [] + for module in self.test_results.values(): + for tests in module.values(): + for test_name, _ in tests: + lengths.append(len(test_name) + 1) # +1 for colon + max_length = max(lengths) + 2 # Additional padding + except ValueError: + sys.exit("Test list is empty") + + # Print results + for module_name, classes in self.test_results.items(): + if verbose: + print(f"{module_name}:") + for class_name, tests in classes.items(): + if verbose: + print(f" {class_name}:") + for test_name, status in tests: + if not verbose and status in [ "OK", "EXPECTED_FAIL" ]: + continue + + # Get base status without reason for SKIP + if status.startswith("SKIP"): + status_code = status.split()[0] + else: + status_code = status + color = COLORS.get(status_code, "") + print( + f" {test_name + ':':<{max_length}}{color}{status}{COLORS['reset']}" + ) + if verbose: + print() + + # Print summary + print(f"\nRan {self.testsRun} tests", end="") + if hasattr(self, "timeTaken"): + print(f" in {self.timeTaken:.3f}s", end="") + print() + + if not self.wasSuccessful(): + print(f"\n{COLORS['FAIL']}FAILED (", end="") + failures = getattr(self, "failures", []) + errors = getattr(self, "errors", []) + if failures: + print(f"failures={len(failures)}", end="") + if errors: + if failures: + print(", ", end="") + print(f"errors={len(errors)}", end="") + print(f"){COLORS['reset']}") + + +def flatten_suite(suite): + """Flatten test suite hierarchy.""" + tests = [] + for item in suite: + if isinstance(item, unittest.TestSuite): + tests.extend(flatten_suite(item)) + else: + tests.append(item) + return tests + + +class TestUnits: + """ + Helper class to set verbosity level. + + This class discover test files, import its unittest classes and + executes the test on it. + """ + def parse_args(self): + """Returns a parser for command line arguments.""" + parser = argparse.ArgumentParser(description="Test runner with regex filtering") + parser.add_argument("-v", "--verbose", action="count", default=1) + parser.add_argument("-q", "--quiet", action="store_true") + parser.add_argument("-f", "--failfast", action="store_true") + parser.add_argument("-k", "--keyword", + help="Regex pattern to filter test methods") + return parser + + def run(self, caller_file=None, pattern=None, + suite=None, parser=None, args=None, env=None): + """ + Execute all tests from the unity test file. + + It contains several optional parameters: + + ``caller_file``: + - name of the file that contains test. + + typical usage is to place __file__ at the caller test, e.g.:: + + if __name__ == "__main__": + TestUnits().run(__file__) + + ``pattern``: + - optional pattern to match multiple file names. Defaults + to basename of ``caller_file``. + + ``suite``: + - an unittest suite initialized by the caller using + ``unittest.TestLoader().discover()``. + + ``parser``: + - an argparse parser. If not defined, this helper will create + one. + + ``args``: + - an ``argparse.Namespace`` data filled by the caller. + + ``env``: + - environment variables that will be passed to the test suite + + At least ``caller_file`` or ``suite`` must be used, otherwise a + ``TypeError`` will be raised. + """ + if not args: + if not parser: + parser = self.parse_args() + args = parser.parse_args() + + if not caller_file and not suite: + raise TypeError("Either caller_file or suite is needed at TestUnits") + + if args.quiet: + verbose = 0 + else: + verbose = args.verbose + + if not env: + env = os.environ.copy() + + env["VERBOSE"] = f"{verbose}" + + patcher = patch.dict(os.environ, env) + patcher.start() + # ensure it gets stopped after + atexit.register(patcher.stop) + + + if verbose >= 2: + unittest.TextTestRunner(verbosity=verbose).run = lambda suite: suite + + # Load ONLY tests from the calling file + if not suite: + if not pattern: + pattern = caller_file + + loader = unittest.TestLoader() + suite = loader.discover(start_dir=os.path.dirname(caller_file), + pattern=os.path.basename(caller_file)) + + # Flatten the suite for environment injection + tests_to_inject = flatten_suite(suite) + + # Filter tests by method name if -k specified + if args.keyword: + try: + pattern = re.compile(args.keyword) + filtered_suite = unittest.TestSuite() + for test in tests_to_inject: # Use the pre-flattened list + method_name = test.id().split(".")[-1] + if pattern.search(method_name): + filtered_suite.addTest(test) + suite = filtered_suite + except re.error as e: + sys.stderr.write(f"Invalid regex pattern: {e}\n") + sys.exit(1) + else: + # Maintain original suite structure if no keyword filtering + suite = unittest.TestSuite(tests_to_inject) + + if verbose >= 2: + resultclass = None + else: + resultclass = Summary + + runner = unittest.TextTestRunner(verbosity=args.verbose, + resultclass=resultclass, + failfast=args.failfast) + result = runner.run(suite) + if resultclass: + result.printResults(verbose) + + sys.exit(not result.wasSuccessful()) + + +def run_unittest(fname): + """ + Basic usage of TestUnits class. + + Use it when there's no need to pass any extra argument to the tests + with. The recommended way is to place this at the end of each + unittest module:: + + if __name__ == "__main__": + run_unittest(__file__) + """ + TestUnits().run(fname) diff --git a/tools/lib/subcmd/Makefile b/tools/lib/subcmd/Makefile index 8703ab487b68..9f1ddcf0504d 100644 --- a/tools/lib/subcmd/Makefile +++ b/tools/lib/subcmd/Makefile @@ -9,10 +9,6 @@ srctree := $(patsubst %/,%,$(dir $(srctree))) #$(info Determined 'srctree' to be $(srctree)) endif -CC ?= $(CROSS_COMPILE)gcc -LD ?= $(CROSS_COMPILE)ld -AR ?= $(CROSS_COMPILE)ar - RM = rm -f MAKEFLAGS += --no-print-directory diff --git a/tools/lib/subcmd/parse-options.c b/tools/lib/subcmd/parse-options.c index 555d617c1f50..e83200e9f56a 100644 --- a/tools/lib/subcmd/parse-options.c +++ b/tools/lib/subcmd/parse-options.c @@ -72,6 +72,7 @@ static int get_value(struct parse_opt_ctx_t *p, const char *s, *arg = NULL; const int unset = flags & OPT_UNSET; int err; + bool force_defval = false; if (unset && p->opt) return opterror(opt, "takes no value", flags); @@ -123,6 +124,42 @@ static int get_value(struct parse_opt_ctx_t *p, } } + if (opt->flags & PARSE_OPT_OPTARG && !p->opt) { + if (!(p->flags & PARSE_OPT_OPTARG_ALLOW_NEXT)) { + /* + * If the option has an optional argument, and the argument is not + * provided in the option itself, do not attempt to get it from + * the next argument, unless PARSE_OPT_OPTARG_ALLOW_NEXT is set. + * + * This prevents a non-option argument from being interpreted as an + * optional argument of a preceding option, for example: + * + * $ cmd --opt val + * -> is "val" argument of "--opt" or a separate non-option + * argument? + * + * With PARSE_OPT_OPTARG_ALLOW_NEXT, "val" is interpreted as + * the argument of "--opt", i.e. the same as "--opt=val". + * Without PARSE_OPT_OPTARG_ALLOW_NEXT, --opt is interpreted + * as having the default value, and "val" as a separate non-option + * argument. + * + * PARSE_OPT_OPTARG_ALLOW_NEXT is useful for commands that take no + * non-option arguments and want to allow more flexibility in + * optional argument passing. + */ + force_defval = true; + } + + if (p->argc <= 1 || p->argv[1][0] == '-') { + /* + * If next argument is an option or does not exist, + * use the default value. + */ + force_defval = true; + } + } + if (opt->flags & PARSE_OPT_NOBUILD) { char reason[128]; bool noarg = false; @@ -148,7 +185,7 @@ static int get_value(struct parse_opt_ctx_t *p, noarg = true; if (opt->flags & PARSE_OPT_NOARG) noarg = true; - if (opt->flags & PARSE_OPT_OPTARG && !p->opt) + if (force_defval) noarg = true; switch (opt->type) { @@ -212,7 +249,7 @@ static int get_value(struct parse_opt_ctx_t *p, err = 0; if (unset) *(const char **)opt->value = NULL; - else if (opt->flags & PARSE_OPT_OPTARG && !p->opt) + else if (force_defval) *(const char **)opt->value = (const char *)opt->defval; else err = get_arg(p, opt, flags, (const char **)opt->value); @@ -244,7 +281,7 @@ static int get_value(struct parse_opt_ctx_t *p, return (*opt->callback)(opt, NULL, 1) ? (-1) : 0; if (opt->flags & PARSE_OPT_NOARG) return (*opt->callback)(opt, NULL, 0) ? (-1) : 0; - if (opt->flags & PARSE_OPT_OPTARG && !p->opt) + if (force_defval) return (*opt->callback)(opt, NULL, 0) ? (-1) : 0; if (get_arg(p, opt, flags, &arg)) return -1; @@ -255,7 +292,7 @@ static int get_value(struct parse_opt_ctx_t *p, *(int *)opt->value = 0; return 0; } - if (opt->flags & PARSE_OPT_OPTARG && !p->opt) { + if (force_defval) { *(int *)opt->value = opt->defval; return 0; } @@ -271,7 +308,7 @@ static int get_value(struct parse_opt_ctx_t *p, *(unsigned int *)opt->value = 0; return 0; } - if (opt->flags & PARSE_OPT_OPTARG && !p->opt) { + if (force_defval) { *(unsigned int *)opt->value = opt->defval; return 0; } @@ -289,7 +326,7 @@ static int get_value(struct parse_opt_ctx_t *p, *(long *)opt->value = 0; return 0; } - if (opt->flags & PARSE_OPT_OPTARG && !p->opt) { + if (force_defval) { *(long *)opt->value = opt->defval; return 0; } @@ -305,7 +342,7 @@ static int get_value(struct parse_opt_ctx_t *p, *(unsigned long *)opt->value = 0; return 0; } - if (opt->flags & PARSE_OPT_OPTARG && !p->opt) { + if (force_defval) { *(unsigned long *)opt->value = opt->defval; return 0; } @@ -321,7 +358,7 @@ static int get_value(struct parse_opt_ctx_t *p, *(u64 *)opt->value = 0; return 0; } - if (opt->flags & PARSE_OPT_OPTARG && !p->opt) { + if (force_defval) { *(u64 *)opt->value = opt->defval; return 0; } @@ -390,7 +427,8 @@ retry: return 0; } if (!rest) { - if (strstarts(options->long_name, "no-")) { + if (strstarts(options->long_name, "no-") && + !(options->flags & PARSE_OPT_NOAUTONEG)) { /* * The long name itself starts with "no-", so * accept the option without "no-" so that users @@ -428,12 +466,12 @@ is_abbreviated: continue; } /* negated and abbreviated very much? */ - if (strstarts("no-", arg)) { + if (strstarts("no-", arg) && !(options->flags & PARSE_OPT_NOAUTONEG)) { flags |= OPT_UNSET; goto is_abbreviated; } /* negated? */ - if (strncmp(arg, "no-", 3)) + if (strncmp(arg, "no-", 3) || (options->flags & PARSE_OPT_NOAUTONEG)) continue; flags |= OPT_UNSET; rest = skip_prefix(arg + 3, options->long_name); @@ -982,7 +1020,8 @@ opt: if (strstarts(opts->long_name, optstr)) print_option_help(opts, 0); if (strstarts("no-", optstr) && - strstarts(opts->long_name, optstr + 3)) + strstarts(opts->long_name, optstr + 3) && + !(opts->flags & PARSE_OPT_NOAUTONEG)) print_option_help(opts, 0); } diff --git a/tools/lib/subcmd/parse-options.h b/tools/lib/subcmd/parse-options.h index 8e9147358a28..38df5fd21963 100644 --- a/tools/lib/subcmd/parse-options.h +++ b/tools/lib/subcmd/parse-options.h @@ -33,6 +33,7 @@ enum parse_opt_flags { PARSE_OPT_KEEP_ARGV0 = 4, PARSE_OPT_KEEP_UNKNOWN = 8, PARSE_OPT_NO_INTERNAL_HELP = 16, + PARSE_OPT_OPTARG_ALLOW_NEXT = 32, }; enum parse_opt_option_flags { @@ -46,6 +47,7 @@ enum parse_opt_option_flags { PARSE_OPT_NOEMPTY = 128, PARSE_OPT_NOBUILD = 256, PARSE_OPT_CANSKIP = 512, + PARSE_OPT_NOAUTONEG = 1024, }; struct option; @@ -148,6 +150,8 @@ struct option { { .type = OPTION_CALLBACK, .short_name = (s), .long_name = (l), .value = (v), .argh = "time", .help = (h), .callback = parse_opt_approxidate_cb } #define OPT_CALLBACK(s, l, v, a, h, f) \ { .type = OPTION_CALLBACK, .short_name = (s), .long_name = (l), .value = (v), .argh = (a), .help = (h), .callback = (f) } +#define OPT_CALLBACK_FLAG(s, l, v, a, h, f, fl) \ + { .type = OPTION_CALLBACK, .short_name = (s), .long_name = (l), .value = (v), .argh = (a), .help = (h), .callback = (f), .flags = (fl) } #define OPT_CALLBACK_SET(s, l, v, os, a, h, f) \ { .type = OPTION_CALLBACK, .short_name = (s), .long_name = (l), .value = (v), .argh = (a), .help = (h), .callback = (f), .set = check_vtype(os, bool *)} #define OPT_CALLBACK_NOOPT(s, l, v, a, h, f) \ diff --git a/tools/lib/subcmd/run-command.c b/tools/lib/subcmd/run-command.c index b7510f83209a..bd21b8bfd58b 100644 --- a/tools/lib/subcmd/run-command.c +++ b/tools/lib/subcmd/run-command.c @@ -169,8 +169,18 @@ int start_command(struct child_process *cmd) static int wait_or_whine(struct child_process *cmd, bool block) { - bool finished = cmd->finished; - int result = cmd->finish_result; + bool finished; + int result; + + if (cmd->pid <= 0) { + cmd->finished = 1; + if (cmd->pid < 0 && cmd->finish_result == 0) + cmd->finish_result = -ERR_RUN_COMMAND_FORK; + return cmd->finish_result; + } + + finished = cmd->finished; + result = cmd->finish_result; while (!finished) { int status, code; @@ -233,7 +243,18 @@ int check_if_command_finished(struct child_process *cmd) char filename[6 + MAX_STRLEN_TYPE(typeof(cmd->pid)) + 7 + 1]; char status_line[256]; FILE *status_file; +#endif + if (cmd->finished) + return 1; + if (cmd->pid <= 0) { + cmd->finished = 1; + if (cmd->pid < 0 && cmd->finish_result == 0) + cmd->finish_result = -ERR_RUN_COMMAND_FORK; + return 1; + } + +#ifdef __linux__ /* * Check by reading /proc/<pid>/status as calling waitpid causes * stdout/stderr to be closed and data lost. @@ -241,8 +262,48 @@ int check_if_command_finished(struct child_process *cmd) sprintf(filename, "/proc/%u/status", cmd->pid); status_file = fopen(filename, "r"); if (status_file == NULL) { - /* Open failed assume finish_command was called. */ - return true; + int status; + pid_t waiting; + + /* + * fopen() can fail with ENOENT if the process has been reaped. + * It can also fail with EMFILE/ENFILE if RLIMIT_NOFILE is reached. + * In those cases, use waitpid(..., WNOHANG) to robustly check + * and reap the process if it has exited. + */ + if (errno == ENOENT) + return 1; + + waiting = waitpid(cmd->pid, &status, WNOHANG); + if (waiting == cmd->pid) { + int result; + int code; + + cmd->finished = 1; + if (WIFSIGNALED(status)) { + result = -ERR_RUN_COMMAND_WAITPID_SIGNAL; + } else if (!WIFEXITED(status)) { + result = -ERR_RUN_COMMAND_WAITPID_NOEXIT; + } else { + code = WEXITSTATUS(status); + switch (code) { + case 127: + result = -ERR_RUN_COMMAND_EXEC; + break; + case 0: + result = 0; + break; + default: + result = -code; + break; + } + } + cmd->finish_result = result; + return 1; + } + if (waiting < 0 && (errno == ECHILD || errno == ESRCH)) + return 1; + return 0; } while (fgets(status_line, sizeof(status_line), status_file) != NULL) { char *p; diff --git a/tools/lib/symbol/Makefile b/tools/lib/symbol/Makefile index 426b845edfac..d692abe8add6 100644 --- a/tools/lib/symbol/Makefile +++ b/tools/lib/symbol/Makefile @@ -9,10 +9,6 @@ srctree := $(patsubst %/,%,$(dir $(srctree))) #$(info Determined 'srctree' to be $(srctree)) endif -CC ?= $(CROSS_COMPILE)gcc -AR ?= $(CROSS_COMPILE)ar -LD ?= $(CROSS_COMPILE)ld - MAKEFLAGS += --no-print-directory INSTALL = install diff --git a/tools/lib/symbol/kallsyms.c b/tools/lib/symbol/kallsyms.c index e335ac2b9e19..d64bd9cc82a9 100644 --- a/tools/lib/symbol/kallsyms.c +++ b/tools/lib/symbol/kallsyms.c @@ -60,7 +60,7 @@ int kallsyms__parse(const char *filename, void *arg, read_to_eol(&io); continue; } - for (i = 0; i < sizeof(symbol_name); i++) { + for (i = 0; i < KSYM_NAME_LEN; i++) { ch = io__get_char(&io); if (ch < 0 || ch == '\n') break; @@ -68,6 +68,9 @@ int kallsyms__parse(const char *filename, void *arg, } symbol_name[i] = '\0'; + if (i == KSYM_NAME_LEN) + read_to_eol(&io); + err = process_symbol(arg, symbol_name, symbol_type, start); if (err) break; diff --git a/tools/lib/thermal/include/thermal.h b/tools/lib/thermal/include/thermal.h index 818ecdfb46e5..d9097271d9fa 100644 --- a/tools/lib/thermal/include/thermal.h +++ b/tools/lib/thermal/include/thermal.h @@ -175,8 +175,8 @@ LIBTHERMAL_API thermal_error_t thermal_sampling_handle(struct thermal_handler *t LIBTHERMAL_API int thermal_sampling_fd(struct thermal_handler *th); -#endif /* __LIBTHERMAL_H */ - #ifdef __cplusplus } #endif + +#endif /* __LIBTHERMAL_H */ |