diff options
Diffstat (limited to 'kernel/bpf')
35 files changed, 5982 insertions, 1828 deletions
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 4dc41bf5780c..90255d80e5be 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -6,7 +6,7 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse endif CFLAGS_core.o += -Wno-override-init $(cflags-nogcse-yy) -obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o +obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o diagnostics.o obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 80b7b8a69446..7b6847200b43 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -55,8 +55,10 @@ struct bpf_arena { struct vm_struct *kern_vm; struct page *scratch_page; struct range_tree rt; - /* protects rt */ + /* protects rt and nr_pages */ rqspinlock_t spinlock; + /* number of pages currently populated in the arena */ + u64 nr_pages; struct list_head vma_list; /* protects vma_list */ struct mutex lock; @@ -143,14 +145,14 @@ static long compute_pgoff(struct bpf_arena *arena, long uaddr) } struct apply_range_data { + struct bpf_arena *arena; struct page **pages; - struct page *scratch_page; int i; }; struct clear_range_data { + struct bpf_arena *arena; struct llist_head *free_pages; - struct page *scratch_page; }; static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) @@ -180,7 +182,7 @@ static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) if (pte_none(old)) continue; - if (WARN_ON_ONCE(pte_page(old) != d->scratch_page)) + if (WARN_ON_ONCE(pte_page(old) != d->arena->scratch_page)) return -EBUSY; ptep_get_and_clear(&init_mm, addr, pte); flush_tlb_before_set(addr); @@ -196,6 +198,7 @@ static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) set_pte_at(&init_mm, addr, pte, pteval); #endif d->i++; + WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages + 1); return 0; } @@ -227,10 +230,11 @@ static int apply_range_clear_cb(pte_t *pte, unsigned long addr, void *data) * scratches its PTE. A later bpf_arena_free_pages() over that range walks * here. Without the skip, scratch_page would be freed. */ - if (page == d->scratch_page) + if (page == d->arena->scratch_page) return 0; __llist_add(&page->pcp_llist, d->free_pages); + WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages - 1); return 0; } @@ -413,7 +417,9 @@ static int arena_map_check_btf(struct bpf_map *map, const struct btf *btf, static u64 arena_map_mem_usage(const struct bpf_map *map) { - return 0; + struct bpf_arena *arena = container_of(map, struct bpf_arena, map); + + return (u64)READ_ONCE(arena->nr_pages) << PAGE_SHIFT; } struct vma_list { @@ -484,8 +490,12 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) kaddr = kbase + (u32)(vmf->address); if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) - /* Make a reasonable effort to address impossible case */ - return VM_FAULT_RETRY; + /* + * A failed lock means a possible deadlock was detected. Don't + * return VM_FAULT_RETRY: this handler never took mmap_lock, but + * the fault path would re-take it on retry and deadlock. Fail. + */ + return VM_FAULT_SIGBUS; page = vmalloc_to_page((void *)kaddr); if (page) { @@ -506,8 +516,7 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) if (ret) goto out_sigsegv_memcg; - struct apply_range_data data = { .pages = &page, .i = 0, - .scratch_page = arena->scratch_page }; + struct apply_range_data data = { .arena = arena, .pages = &page, .i = 0 }; /* Account into memcg of the process that created bpf_arena */ ret = bpf_map_alloc_pages(map, NUMA_NO_NODE, 1, &page); if (ret) { @@ -696,8 +705,8 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt bpf_map_memcg_exit(old_memcg, new_memcg); return 0; } + data.arena = arena; data.pages = pages; - data.scratch_page = arena->scratch_page; if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) goto out_free_pages; @@ -853,6 +862,8 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, uaddr &= PAGE_MASK; kaddr = bpf_arena_get_kern_vm_start(arena) + uaddr; full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; + if (full_uaddr < arena->user_vm_start) + return; uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT)); if (full_uaddr >= uaddr_end) return; @@ -873,8 +884,8 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, range_tree_set(&arena->rt, pgoff, page_cnt); init_llist_head(&free_pages); + cdata.arena = arena; cdata.free_pages = &free_pages; - cdata.scratch_page = arena->scratch_page; /* clear ptes and collect struct pages */ apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT, apply_range_clear_cb, &cdata); @@ -981,8 +992,8 @@ static void arena_free_worker(struct work_struct *work) bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); init_llist_head(&free_pages); + cdata.arena = arena; cdata.free_pages = &free_pages; - cdata.scratch_page = arena->scratch_page; arena_vm_start = bpf_arena_get_kern_vm_start(arena); user_vm_start = bpf_arena_get_user_vm_start(arena); @@ -1107,9 +1118,9 @@ __bpf_kfunc int bpf_arena_reserve_pages(void *p__map, void *ptr__ign, u32 page_c __bpf_kfunc_end_defs(); BTF_KFUNCS_START(arena_kfuncs) -BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_ARENA_RET | KF_ARENA_ARG2) -BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_ARENA_ARG2) -BTF_ID_FLAGS(func, bpf_arena_reserve_pages, KF_ARENA_ARG2) +BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_ARENA_RET | KF_ARENA_ARG2 | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_ARENA_ARG2 | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_arena_reserve_pages, KF_ARENA_ARG2 | KF_SPINLOCK_SAFE) BTF_KFUNCS_END(arena_kfuncs) static const struct btf_kfunc_id_set common_kfunc_set = { diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 248b4818178c..ef315b168b29 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -259,6 +259,37 @@ static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key) return this_cpu_ptr(array->pptrs[index & array->index_mask]); } +static int percpu_array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, u32 off) +{ + struct bpf_array *array = container_of(map, struct bpf_array, map); + + if (!bpf_jit_supports_percpu_insn()) + return -EOPNOTSUPP; + if (map->max_entries != 1) + return -EOPNOTSUPP; + if (off >= map->value_size) + return -EINVAL; + + *imm = (u64)(__force unsigned long) array->pptrs[0]; + return 0; +} + +static int percpu_array_map_direct_value_meta(const struct bpf_map *map, u64 imm, u32 *off) +{ + struct bpf_array *array = container_of(map, struct bpf_array, map); + u64 base = (u64)(__force unsigned long) array->pptrs[0]; + + if (!bpf_jit_supports_percpu_insn()) + return -EOPNOTSUPP; + if (map->max_entries != 1) + return -EOPNOTSUPP; + if (imm < base || imm >= base + array->elem_size) + return -ENOENT; + + *off = imm - base; + return 0; +} + /* emit BPF instructions equivalent to C code of percpu_array_map_lookup_elem() */ static int percpu_array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) { @@ -551,9 +582,10 @@ static int array_map_check_btf(struct bpf_map *map, const struct btf_type *key_type, const struct btf_type *value_type) { - /* One exception for keyless BTF: .bss/.data/.rodata map */ + /* One exception for keyless BTF: .bss/.data/.rodata/.percpu map */ if (btf_type_is_void(key_type)) { - if (map->map_type != BPF_MAP_TYPE_ARRAY || + if ((map->map_type != BPF_MAP_TYPE_ARRAY && + map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) || map->max_entries != 1) return -EINVAL; @@ -576,17 +608,42 @@ static int array_map_check_btf(struct bpf_map *map, static int array_map_mmap(struct bpf_map *map, struct vm_area_struct *vma) { struct bpf_array *array = container_of(map, struct bpf_array, map); - pgoff_t pgoff = PAGE_ALIGN(sizeof(*array)) >> PAGE_SHIFT; if (!(map->map_flags & BPF_F_MMAPABLE)) return -EINVAL; - if (vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) > + /* use u64 math so the offset cannot overflow on 32-bit archs */ + if ((u64)vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) > PAGE_ALIGN((u64)array->map.max_entries * array->elem_size)) return -EINVAL; - return remap_vmalloc_range(vma, array_map_vmalloc_addr(array), - vma->vm_pgoff + pgoff); + /* + * Pages are faulted in on demand by array_map_mmap_fault(). Set the + * same flags that the eager remap_vmalloc_range() path used to set + * via vm_insert_page(), so that e.g. NUMA balancing keeps skipping + * these VMAs. + */ + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP | VM_MIXEDMAP); + + return 0; +} + +static vm_fault_t array_map_mmap_fault(struct bpf_map *map, + struct vm_fault *vmf) +{ + struct bpf_array *array = container_of(map, struct bpf_array, map); + struct page *page; + + page = vmalloc_to_page(array->value + ((u64)vmf->pgoff << PAGE_SHIFT)); + if (!page) + return VM_FAULT_SIGBUS; + + /* the eager remap_vmalloc_range() flushed via vm_insert_page() */ + flush_dcache_folio(page_folio(page)); + get_page(page); + vmf->page = page; + + return 0; } static bool array_map_meta_equal(const struct bpf_map *meta0, @@ -812,6 +869,7 @@ const struct bpf_map_ops array_map_ops = { .map_direct_value_addr = array_map_direct_value_addr, .map_direct_value_meta = array_map_direct_value_meta, .map_mmap = array_map_mmap, + .map_mmap_fault = array_map_mmap_fault, .map_seq_show_elem = array_map_seq_show_elem, .map_check_btf = array_map_check_btf, .map_lookup_batch = generic_map_lookup_batch, @@ -832,6 +890,8 @@ const struct bpf_map_ops percpu_array_map_ops = { .map_get_next_key = bpf_array_get_next_key, .map_lookup_elem = percpu_array_map_lookup_elem, .map_gen_lookup = percpu_array_map_gen_lookup, + .map_direct_value_addr = percpu_array_map_direct_value_addr, + .map_direct_value_meta = percpu_array_map_direct_value_meta, .map_update_elem = array_map_update_elem, .map_delete_elem = array_map_delete_elem, .map_lookup_percpu_elem = percpu_array_map_lookup_percpu_elem, diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 2e4ae0ef0860..a2b18a9f1694 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -214,7 +214,6 @@ static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg) return bt->reg_masks[bt->frame] & (1 << reg); } - /* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */ static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask) { @@ -254,7 +253,6 @@ void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask) } } - /* For given verifier state backtrack_insn() is called from the last insn to * the first insn. Its purpose is to compute a bitmask of registers and * stack slots that needs precision in the parent verifier state. @@ -285,6 +283,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, verbose(env, "stack=%s before ", env->tmp_str_buf); verbose(env, "%d: ", idx); bpf_verbose_insn(env, insn); + verbose(env, "\n"); } /* If there is a history record that some registers gained range at this insn, @@ -636,7 +635,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, * r5 += 1 * ... * call bpf_perf_event_output#25 - * where .arg5_type = ARG_CONST_SIZE_OR_ZERO + * where .arg5_type = ARG_MEM_SIZE_OR_ZERO * * and this case: * r6 = 1 diff --git a/kernel/bpf/bloom_filter.c b/kernel/bpf/bloom_filter.c index b73336c976b7..c6e7559b07de 100644 --- a/kernel/bpf/bloom_filter.c +++ b/kernel/bpf/bloom_filter.c @@ -41,7 +41,7 @@ static long bloom_map_peek_elem(struct bpf_map *map, void *value) for (i = 0; i < bloom->nr_hash_funcs; i++) { h = hash(bloom, value, map->value_size, i); - if (!test_bit(h, bloom->bitset)) + if (!test_bit(h % BITS_PER_LONG, bloom->bitset + BIT_WORD(h))) return -ENOENT; } @@ -57,9 +57,13 @@ static long bloom_map_push_elem(struct bpf_map *map, void *value, u64 flags) if (flags != BPF_ANY) return -EINVAL; + /* + * On 32-bit architectures, hashes larger than INT_MAX would be + * treated as negative by set_bit(). + */ for (i = 0; i < bloom->nr_hash_funcs; i++) { h = hash(bloom, value, map->value_size, i); - set_bit(h, bloom->bitset); + set_bit(h % BITS_PER_LONG, bloom->bitset + BIT_WORD(h)); } return 0; @@ -94,9 +98,10 @@ static int bloom_map_alloc_check(union bpf_attr *attr) static struct bpf_map *bloom_map_alloc(union bpf_attr *attr) { - u32 bitset_bytes, bitset_mask, nr_hash_funcs, nr_bits; + u32 bitset_mask, nr_hash_funcs, nr_bits; int numa_node = bpf_map_attr_numa_node(attr); struct bpf_bloom_filter *bloom; + u64 bitset_bytes; if (attr->key_size != 0 || attr->value_size == 0 || attr->max_entries == 0 || @@ -127,22 +132,16 @@ static struct bpf_map *bloom_map_alloc(union bpf_attr *attr) if (check_mul_overflow(attr->max_entries, nr_hash_funcs, &nr_bits) || check_mul_overflow(nr_bits / 5, (u32)7, &nr_bits) || nr_bits > (1UL << 31)) { - /* The bit array size is 2^32 bits but to avoid overflowing the - * u32, we use U32_MAX, which will round up to the equivalent - * number of bytes - */ - bitset_bytes = BITS_TO_BYTES(U32_MAX); bitset_mask = U32_MAX; } else { if (nr_bits <= BITS_PER_LONG) nr_bits = BITS_PER_LONG; else nr_bits = roundup_pow_of_two(nr_bits); - bitset_bytes = BITS_TO_BYTES(nr_bits); bitset_mask = nr_bits - 1; } - bitset_bytes = roundup(bitset_bytes, sizeof(unsigned long)); + bitset_bytes = BITS_TO_LONGS((u64)bitset_mask + 1) * sizeof(unsigned long); bloom = bpf_map_area_alloc(sizeof(*bloom) + bitset_bytes, numa_node); if (!bloom) diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index f5eaeb2493d4..14a5fdfa0421 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -782,8 +782,8 @@ __bpf_kfunc int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) return -EINVAL; } - /* avoid overflows, e.g., if start == INT_MIN and end == INT_MAX */ - if ((s64)end - (s64)start > BPF_MAX_LOOPS) { + /* start <= end here, so end - start fits in a u32 without overflow */ + if ((u32)(end - start) > BPF_MAX_LOOPS) { s->cur = s->end = 0; return -E2BIG; } @@ -802,12 +802,11 @@ __bpf_kfunc int *bpf_iter_num_next(struct bpf_iter_num* it) { struct bpf_iter_num_kern *s = (void *)it; - /* check failed initialization or if we are done (same behavior); - * need to be careful about overflow, so convert to s64 for checks, - * e.g., if s->cur == s->end == INT_MAX, we can't just do - * s->cur + 1 >= s->end + /* + * s->cur < s->end while iterating, else s->cur == s->end == 0; the signed + * s->cur + 1 >= s->end holds even when s->cur + 1 wraps (start == INT_MIN). */ - if ((s64)(s->cur + 1) >= s->end) { + if (s->cur + 1 >= s->end) { s->cur = s->end = 0; return NULL; } @@ -819,9 +818,7 @@ __bpf_kfunc int *bpf_iter_num_next(struct bpf_iter_num* it) __bpf_kfunc void bpf_iter_num_destroy(struct bpf_iter_num *it) { - struct bpf_iter_num_kern *s = (void *)it; - - s->cur = s->end = 0; + /* no-op */ } __bpf_kfunc_end_defs(); diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 1433809bb166..82c5988417a0 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -186,7 +186,7 @@ static const struct bpf_func_proto bpf_ima_inode_hash_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_ima_inode_hash_btf_ids[0], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .allowed = bpf_ima_inode_hash_allowed, }; @@ -205,7 +205,7 @@ static const struct bpf_func_proto bpf_ima_file_hash_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_ima_file_hash_btf_ids[0], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .allowed = bpf_ima_inode_hash_allowed, }; @@ -295,7 +295,6 @@ BTF_ID(func, bpf_lsm_bpf_map_create) BTF_ID(func, bpf_lsm_bpf_map_free) BTF_ID(func, bpf_lsm_bpf_prog) BTF_ID(func, bpf_lsm_bpf_prog_load) -BTF_ID(func, bpf_lsm_bpf_prog_free) BTF_ID(func, bpf_lsm_bpf_token_create) BTF_ID(func, bpf_lsm_bpf_token_free) BTF_ID(func, bpf_lsm_bpf_token_cmd) diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 51b16e5f5534..d7c3030bc63b 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -147,6 +147,8 @@ void bpf_struct_ops_image_free(void *image) #define MAYBE_NULL_SUFFIX "__nullable" #define REFCOUNTED_SUFFIX "__ref" +#define ARENA_SUFFIX "__arena" +#define ARENA_MAYBE_NULL_SUFFIX "__arena__nullable" /* Prepare argument info for every nullable argument of a member of a * struct_ops type. @@ -159,7 +161,7 @@ void bpf_struct_ops_image_free(void *image) * to provide an array of struct bpf_ctx_arg_aux, which in turn provides * the information that used by the verifier to check the arguments of the * BPF struct_ops program assigned to the member. Here, we only care about - * the arguments that are marked as __nullable. + * the arguments that are marked as __nullable, __ref or __arena. * * The array of struct bpf_ctx_arg_aux is eventually assigned to * prog->aux->ctx_arg_info of BPF struct_ops programs and passed to the @@ -172,10 +174,12 @@ static int prepare_arg_info(struct btf *btf, const char *st_ops_name, const char *member_name, const struct btf_type *func_proto, void *stub_func_addr, + struct btf_func_model *model, struct bpf_struct_ops_arg_info *arg_info) { const struct btf_type *stub_func_proto, *pointed_type; - bool is_nullable = false, is_refcounted = false; + bool is_nullable = false, is_refcounted = false, is_arena = false; + bool is_arena_nullable = false; const struct btf_param *stub_args, *args; struct bpf_ctx_arg_aux *info, *info_buf; u32 nargs, arg_no, info_cnt = 0; @@ -225,27 +229,39 @@ static int prepare_arg_info(struct btf *btf, /* Prepare info for every nullable argument */ info = info_buf; for (arg_no = 0; arg_no < nargs; arg_no++) { - /* Skip arguments that is not suffixed with - * "__nullable or __ref". + bool ptr_to_arena, ptr_to_struct; + + /* + * Skip arguments that are not suffixed with "__arena__nullable", + * "__arena", "__nullable", or "__ref". */ - is_nullable = btf_param_match_suffix(btf, &stub_args[arg_no], - MAYBE_NULL_SUFFIX); + is_arena_nullable = btf_param_match_suffix(btf, &stub_args[arg_no], + ARENA_MAYBE_NULL_SUFFIX); + is_arena = btf_param_match_suffix(btf, &stub_args[arg_no], ARENA_SUFFIX); + is_nullable = !is_arena_nullable && + btf_param_match_suffix(btf, &stub_args[arg_no], MAYBE_NULL_SUFFIX); is_refcounted = btf_param_match_suffix(btf, &stub_args[arg_no], REFCOUNTED_SUFFIX); - if (is_nullable) + if (is_arena_nullable) + suffix = ARENA_MAYBE_NULL_SUFFIX; + else if (is_arena) + suffix = ARENA_SUFFIX; + else if (is_nullable) suffix = MAYBE_NULL_SUFFIX; else if (is_refcounted) suffix = REFCOUNTED_SUFFIX; else continue; - /* Should be a pointer to struct */ - pointed_type = btf_type_resolve_ptr(btf, - args[arg_no].type, - &arg_btf_id); - if (!pointed_type || - !btf_type_is_struct(pointed_type)) { + /* + * Should be a pointer to struct, or any pointer for __arena or + * __arena__nullable. + */ + pointed_type = btf_type_resolve_ptr(btf, args[arg_no].type, &arg_btf_id); + ptr_to_arena = pointed_type && (is_arena || is_arena_nullable); + ptr_to_struct = pointed_type && btf_type_is_struct(pointed_type); + if (!ptr_to_arena && !ptr_to_struct) { pr_warn("stub function %s has %s tagging to an unsupported type\n", stub_fname, suffix); goto err_out; @@ -268,7 +284,18 @@ static int prepare_arg_info(struct btf *btf, info->btf_id = arg_btf_id; info->btf = btf; info->offset = offset; - if (is_nullable) { + if (is_arena || is_arena_nullable) { + /* + * Both types get PTR_TO_ARENA. In verifier state, + * PTR_TO_ARENA encompasses potential NULL values, but + * we do not force the program to check it, or maintain + * precision around it, since it has no safety implication. + */ + info->reg_type = PTR_TO_ARENA; + model->arg_flags[arg_no] |= BTF_FMODEL_ARENA_ARG; + if (is_arena_nullable) + model->arg_flags[arg_no] |= BTF_FMODEL_NULLABLE_ARG; + } else if (is_nullable) { info->reg_type = PTR_TRUSTED | PTR_TO_BTF_ID | PTR_MAYBE_NULL; } else if (is_refcounted) { info->reg_type = PTR_TRUSTED | PTR_TO_BTF_ID; @@ -445,9 +472,22 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, goto errout; } + /* + * A >8 byte return value is passed back in a register pair, + * which the struct_ops trampoline does not preserve (only + * 8 bytes of the return value are saved and restored). + */ + if (st_ops->func_models[i].ret_size > 8) { + pr_warn("func ptr %s in struct %s has a >8 byte return value, which is not supported\n", + mname, st_ops->name); + err = -EOPNOTSUPP; + goto errout; + } + stub_func_addr = *(void **)(st_ops->cfi_stubs + moff); err = prepare_arg_info(btf, st_ops->name, mname, func_proto, stub_func_addr, + &st_ops->func_models[i], arg_info + i); if (err) goto errout; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index f56437e626c9..da36d4b9d31a 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -1169,19 +1169,19 @@ static const char *btf_show_name(struct btf_show *show) id = t->type; break; default: - id = 0; - break; + goto resolved; } + t = btf_type_skip_qualifiers(show->btf, id); if (!id) break; - t = btf_type_skip_qualifiers(show->btf, id); } /* We may not be able to represent this type; bail to be safe */ if (i == BTF_SHOW_MAX_ITER) return ""; +resolved: if (!name) - name = btf_name_by_offset(show->btf, t->name_off); + name = btf_type_is_void(t) ? "void" : btf_name_by_offset(show->btf, t->name_off); switch (BTF_INFO_KIND(t->info)) { case BTF_KIND_STRUCT: @@ -2534,7 +2534,6 @@ static void btf_bitfield_show(void *data, u8 bits_offset, btf_int128_print(show, print_num); } - static void btf_int_bits_show(const struct btf *btf, const struct btf_type *t, void *data, u8 bits_offset, @@ -3669,7 +3668,7 @@ static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_ { BPF_LIST_NODE, "bpf_list_node", false }, { BPF_RB_ROOT, "bpf_rb_root", false }, { BPF_RB_NODE, "bpf_rb_node", false }, - { BPF_REFCOUNT, "bpf_refcount", false }, + { BPF_REFCOUNT, "bpf_refcount", true }, }; int type = 0, i; const char *name = __btf_name_by_offset(btf, var_type->name_off); @@ -3751,7 +3750,7 @@ static int btf_repeat_fields(struct btf_field_info *info, int info_cnt, static int btf_find_struct_field(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, int info_cnt, - u32 level); + u32 level, u32 *seen_mask); /* Find special fields in the struct type of a field. * @@ -3762,7 +3761,7 @@ static int btf_find_struct_field(const struct btf *btf, static int btf_find_nested_struct(const struct btf *btf, const struct btf_type *t, u32 off, u32 nelems, u32 field_mask, struct btf_field_info *info, - int info_cnt, u32 level) + int info_cnt, u32 level, u32 *seen_mask) { int ret, err, i; @@ -3770,7 +3769,7 @@ static int btf_find_nested_struct(const struct btf *btf, const struct btf_type * if (level >= MAX_RESOLVE_DEPTH) return -E2BIG; - ret = btf_find_struct_field(btf, t, field_mask, info, info_cnt, level); + ret = btf_find_struct_field(btf, t, field_mask, info, info_cnt, level, seen_mask); if (ret <= 0) return ret; @@ -3827,7 +3826,7 @@ static int btf_find_field_one(const struct btf *btf, if (expected_size && expected_size != sz * nelems) return 0; ret = btf_find_nested_struct(btf, var_type, off, nelems, field_mask, - &info[0], info_cnt, level); + &info[0], info_cnt, level, seen_mask); return ret; } @@ -3892,11 +3891,11 @@ static int btf_find_field_one(const struct btf *btf, static int btf_find_struct_field(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, int info_cnt, - u32 level) + u32 level, u32 *seen_mask) { int ret, idx = 0; const struct btf_member *member; - u32 i, off, seen_mask = 0; + u32 i, off; for_each_member(i, t, member) { const struct btf_type *member_type = btf_type_by_id(btf, @@ -3910,7 +3909,7 @@ static int btf_find_struct_field(const struct btf *btf, ret = btf_find_field_one(btf, t, member_type, i, off, 0, - field_mask, &seen_mask, + field_mask, seen_mask, &info[idx], info_cnt - idx, level); if (ret < 0) return ret; @@ -3921,11 +3920,11 @@ static int btf_find_struct_field(const struct btf *btf, static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, - int info_cnt, u32 level) + int info_cnt, u32 level, u32 *seen_mask) { int ret, idx = 0; const struct btf_var_secinfo *vsi; - u32 i, off, seen_mask = 0; + u32 i, off; for_each_vsi(i, t, vsi) { const struct btf_type *var = btf_type_by_id(btf, vsi->type); @@ -3933,7 +3932,7 @@ static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t, off = vsi->offset; ret = btf_find_field_one(btf, var, var_type, -1, off, vsi->size, - field_mask, &seen_mask, + field_mask, seen_mask, &info[idx], info_cnt - idx, level); if (ret < 0) @@ -3947,10 +3946,12 @@ static int btf_find_field(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, int info_cnt) { + u32 seen_mask = 0; + if (__btf_type_is_struct(t)) - return btf_find_struct_field(btf, t, field_mask, info, info_cnt, 0); + return btf_find_struct_field(btf, t, field_mask, info, info_cnt, 0, &seen_mask); else if (btf_type_is_datasec(t)) - return btf_find_datasec_var(btf, t, field_mask, info, info_cnt, 0); + return btf_find_datasec_var(btf, t, field_mask, info, info_cnt, 0, &seen_mask); return -EINVAL; } @@ -4168,7 +4169,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type rec->spin_lock_off = rec->fields[i].offset; break; case BPF_RES_SPIN_LOCK: - WARN_ON_ONCE(rec->spin_lock_off >= 0); + WARN_ON_ONCE(rec->res_spin_lock_off >= 0); /* Cache offset for faster lookup at runtime */ rec->res_spin_lock_off = rec->fields[i].offset; break; @@ -6451,7 +6452,7 @@ struct btf *btf_parse_vmlinux(void) if (IS_ERR(btf)) goto err_out; - /* btf_parse_vmlinux() runs under bpf_verifier_lock */ + /* btf_parse_vmlinux() runs under btf_vmlinux_lock */ bpf_ctx_convert.t = btf_type_by_id(btf, bpf_ctx_convert_btf_id[0]); err = btf_alloc_id(btf); if (err) { @@ -6954,15 +6955,19 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, return false; } - /* check for PTR_TO_RDONLY_BUF_OR_NULL or PTR_TO_RDWR_BUF_OR_NULL */ + /* + * Check for PTR_TO_RDONLY_BUF_OR_NULL, PTR_TO_RDWR_BUF_OR_NULL or + * PTR_TO_ARENA (both nullable and non-nullable cases). + */ for (i = 0; i < prog->aux->ctx_arg_info_size; i++) { const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i]; u32 type, flag; type = base_type(ctx_arg_info->reg_type); flag = type_flag(ctx_arg_info->reg_type); - if (ctx_arg_info->offset == off && type == PTR_TO_BUF && - (flag & PTR_MAYBE_NULL)) { + if (ctx_arg_info->offset == off && + (type == PTR_TO_ARENA || + (type == PTR_TO_BUF && (flag & PTR_MAYBE_NULL)))) { info->reg_type = ctx_arg_info->reg_type; return true; } @@ -7101,7 +7106,7 @@ enum bpf_struct_walk_result { static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, const struct btf_type *t, int off, int size, u32 *next_btf_id, enum bpf_type_flag *flag, - const char **field_name) + const char **field_name, bool walk_flex_arrays) { u32 i, moff, mtrue_end, msize = 0, total_nelems = 0; const struct btf_type *mtype, *elem_type = NULL; @@ -7128,11 +7133,14 @@ again: *flag |= PTR_UNTRUSTED; if (off + size > t->size) { + struct btf_array *array_elem; + + if (!walk_flex_arrays) + goto error; + /* If the last element is a variable size array, we may * need to relax the rule. */ - struct btf_array *array_elem; - if (vlen == 0) goto error; @@ -7397,7 +7405,8 @@ int btf_struct_access(struct bpf_verifier_log *log, t = btf_type_by_id(btf, id); do { - err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, field_name); + err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, + field_name, !type_is_alloc(reg->type)); switch (err) { case WALK_PTR: @@ -7456,7 +7465,7 @@ bool btf_types_are_same(const struct btf *btf1, u32 id1, bool btf_struct_ids_match(struct bpf_verifier_log *log, const struct btf *btf, u32 id, int off, const struct btf *need_btf, u32 need_type_id, - bool strict) + bool strict, bool walk_flex_arrays) { const struct btf_type *type; enum bpf_type_flag flag = 0; @@ -7475,7 +7484,8 @@ again: type = btf_type_by_id(btf, id); if (!type) return false; - err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL); + err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL, + walk_flex_arrays); if (err != WALK_STRUCT) return false; @@ -7519,14 +7529,28 @@ static u8 __get_type_fmodel_flags(const struct btf_type *t) { u8 flags = 0; - if (btf_type_is_struct(t)) - flags |= BTF_FMODEL_STRUCT_ARG; if (btf_type_is_signed_int(t)) flags |= BTF_FMODEL_SIGNED_ARG; return flags; } +static u8 __get_arg_fmodel_flags(const struct btf *btf, + const struct btf_param *arg, + const struct btf_type *t) +{ + u8 flags = __get_type_fmodel_flags(t); + + if (btf_param_match_suffix(btf, arg, "__arena__nullable")) + flags |= BTF_FMODEL_ARENA_ARG | BTF_FMODEL_NULLABLE_ARG; + else if (btf_param_match_suffix(btf, arg, "__arena")) + flags |= BTF_FMODEL_ARENA_ARG; + else if (btf_param_match_suffix(btf, arg, "__nullable")) + flags |= BTF_FMODEL_NULLABLE_ARG; + + return flags; +} + int btf_distill_func_proto(struct bpf_verifier_log *log, struct btf *btf, const struct btf_type *func, @@ -7592,7 +7616,7 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, return -EINVAL; } m->arg_size[i] = ret; - m->arg_flags[i] = __get_type_fmodel_flags(t); + m->arg_flags[i] = __get_arg_fmodel_flags(btf, &args[i], t); } m->nr_args = nargs; return 0; @@ -8285,6 +8309,16 @@ int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj, return ssnprintf.len; } +int btf_type_name_to_buf(const struct btf *btf, u32 type_id, char *buf, int len) +{ + struct btf_show show = { + .btf = btf, + .state.type_id = type_id, + }; + + return snprintf(buf, len, "%s", btf_show_name(&show)); +} + #ifdef CONFIG_PROC_FS static void bpf_btf_show_fdinfo(struct seq_file *m, struct file *filp) { @@ -8695,7 +8729,7 @@ const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index 26d37066465f..0f13c13f4133 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -5,6 +5,8 @@ #include <linux/filter.h> #include <linux/sort.h> +#include "diagnostics.h" + #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) /* non-recursive DFS pseudo code @@ -47,7 +49,6 @@ enum { BRANCH = 2, }; - static void mark_subprog_changes_pkt_data(struct bpf_verifier_env *env, int off) { struct bpf_subprog_info *subprog; @@ -113,6 +114,10 @@ static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) if (w < 0 || w >= env->prog->len) { verbose_linfo(env, t, "%d: ", t); verbose(env, "jump out of range from insn %d to %d\n", t, w); + bpf_diag_program_structure( + env, t, "jump out of range", "Keep branch targets inside the program.", + "Instruction %d jumps to instruction %d, but the program only contains instructions 0 through %d.", + t, w, env->prog->len - 1); return -EINVAL; } @@ -136,6 +141,11 @@ static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) verbose_linfo(env, t, "%d: ", t); verbose_linfo(env, w, "%d: ", w); verbose(env, "back-edge from insn %d to %d\n", t, w); + bpf_diag_program_structure( + env, t, "back-edge is not allowed", + "Load with privileges that allow this back-edge, or rewrite the control flow so it does not branch backward.", + "Instruction %d branches back to instruction %d. This program is being rejected without the privilege needed for this back-edge.", + t, w); return -EINVAL; } else if (insn_state[w] == EXPLORED) { /* forward- or cross-edge */ @@ -316,6 +326,11 @@ static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env, if (!jt) { verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start); + bpf_diag_program_structure( + env, subprog_start, "missing jump table", + "Make sure subprograms containing gotox instructions are accompanied by jump tables referencing these subprograms.", + "No jump table was found for the subprogram that starts at instruction %u.", + subprog_start); return ERR_PTR(-EINVAL); } @@ -343,6 +358,11 @@ create_jt(int t, struct bpf_verifier_env *env) if (jt->items[i] < subprog_start || jt->items[i] >= subprog_end) { verbose(env, "jump table for insn %d points outside of the subprog [%u,%u]\n", t, subprog_start, subprog_end); + bpf_diag_program_structure( + env, t, "jump table target out of range", + "Keep every jump-table target inside the same subprogram.", + "The jump table for instruction %d points outside subprogram range [%u,%u).", + t, subprog_start, subprog_end); kvfree(jt); return ERR_PTR(-EINVAL); } @@ -374,6 +394,11 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) w = jt->items[i]; if (w < 0 || w >= env->prog->len) { verbose(env, "indirect jump out of range from insn %d to %d\n", t, w); + bpf_diag_program_structure( + env, t, "indirect jump out of range", + "Keep indirect jump targets inside the program.", + "Instruction %d can jump indirectly to instruction %d, but the program only contains instructions 0 through %d.", + t, w, env->prog->len - 1); return -EINVAL; } @@ -491,7 +516,7 @@ static int visit_insn(int t, struct bpf_verifier_env *env) return ret; } } else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { - struct bpf_kfunc_call_arg_meta meta; + struct bpf_call_arg_meta meta; ret = bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta); if (ret == 0 && bpf_is_iter_next_kfunc(&meta)) { @@ -624,12 +649,21 @@ walk_cfg: if (insn_state[i] != EXPLORED) { verbose(env, "unreachable insn %d\n", i); + bpf_diag_program_structure( + env, i, "unreachable instruction", + "Remove the unreachable instruction or add valid control flow that reaches it.", + "Instruction %d is not reachable from the program entry point.", i); ret = -EINVAL; goto err_free; } if (bpf_is_ldimm64(insn)) { if (insn_state[i + 1] != 0) { verbose(env, "jump into the middle of ldimm64 insn %d\n", i); + bpf_diag_program_structure( + env, i, "jump into ldimm64 immediate", + "Target the first instruction of the ldimm64 pair, or restructure the jump target.", + "Control flow reaches the second half of the ldimm64 instruction pair that starts at instruction %d.", + i); ret = -EINVAL; goto err_free; } diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 4355ccb78a9c..149672c76c49 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -813,8 +813,10 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *old_prog = NULL; struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; struct bpf_cgroup_storage *new_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; + struct bpf_cgroup_storage *old_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; struct bpf_prog *new_prog = prog ? : link->link.prog; enum cgroup_bpf_attach_type atype; + u32 old_flags, old_pl_flags; struct bpf_prog_list *pl; struct hlist_head *progs; int err; @@ -865,6 +867,8 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, if (pl) { old_prog = pl->prog; + old_pl_flags = pl->flags; + bpf_cgroup_storages_assign(old_storage, pl->storage); } else { pl = kmalloc_obj(*pl); if (!pl) { @@ -884,6 +888,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, pl->link = link; pl->flags = flags; bpf_cgroup_storages_assign(pl->storage, storage); + old_flags = cgrp->bpf.flags[atype]; cgrp->bpf.flags[atype] = saved_flags; if (type == BPF_LSM_CGROUP) { @@ -915,12 +920,15 @@ cleanup: if (old_prog) { pl->prog = old_prog; pl->link = NULL; + pl->flags = old_pl_flags; + bpf_cgroup_storages_assign(pl->storage, old_storage); } bpf_cgroup_storages_free(new_storage); if (!old_prog) { hlist_del(&pl->node); kfree(pl); } + cgrp->bpf.flags[atype] = old_flags; return err; } @@ -1018,6 +1026,20 @@ static void replace_effective_prog(struct cgroup *cgrp, } } +static bool cgroup_bpf_storages_compatible(struct bpf_prog *old_prog, + struct bpf_prog *new_prog) +{ + enum bpf_cgroup_storage_type stype; + + for_each_cgroup_storage_type(stype) { + if (old_prog->aux->cgroup_storage[stype] != + new_prog->aux->cgroup_storage[stype]) + return false; + } + + return true; +} + /** * __cgroup_bpf_replace() - Replace link's program and propagate the change * to descendants @@ -1056,6 +1078,9 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp, if (!found) return -ENOENT; + if (!cgroup_bpf_storages_compatible(link->link.prog, new_prog)) + return -EINVAL; + cgrp->bpf.revisions[atype] += 1; old_prog = xchg(&link->link.prog, new_prog); replace_effective_prog(cgrp, atype, pl); @@ -2235,7 +2260,7 @@ int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level, if (ret < 0) return ret; - if (ctx.optlen > *optlen) + if (ctx.optlen > *optlen || ctx.optlen < 0) return -EFAULT; /* BPF programs can shrink the buffer, export the modifications. @@ -2305,7 +2330,7 @@ static const struct bpf_func_proto bpf_sysctl_get_name_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -2347,7 +2372,7 @@ static const struct bpf_func_proto bpf_sysctl_get_current_value_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; BPF_CALL_3(bpf_sysctl_get_new_value, struct bpf_sysctl_kern *, ctx, char *, buf, @@ -2367,7 +2392,7 @@ static const struct bpf_func_proto bpf_sysctl_get_new_value_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; BPF_CALL_3(bpf_sysctl_set_new_value, struct bpf_sysctl_kern *, ctx, @@ -2393,7 +2418,7 @@ static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; static const struct bpf_func_proto * diff --git a/kernel/bpf/check_btf.c b/kernel/bpf/check_btf.c index 93bebe6fe12e..0e8b3ccc7a5b 100644 --- a/kernel/bpf/check_btf.c +++ b/kernel/bpf/check_btf.c @@ -28,9 +28,9 @@ static int check_abnormal_return(struct bpf_verifier_env *env) #define MIN_BPF_FUNCINFO_SIZE 8 #define MAX_FUNCINFO_REC_SIZE 252 -static int check_btf_func_early(struct bpf_verifier_env *env, - const union bpf_attr *attr, - bpfptr_t uattr) +static int prepare_btf_func(struct bpf_verifier_env *env, + const union bpf_attr *attr, + bpfptr_t uattr) { u32 krec_size = sizeof(struct bpf_func_info); const struct btf_type *type, *func_proto; @@ -407,9 +407,9 @@ static int check_core_relo(struct bpf_verifier_env *env, return err; } -int bpf_check_btf_info_early(struct bpf_verifier_env *env, - const union bpf_attr *attr, - bpfptr_t uattr) +int bpf_prepare_btf_info(struct bpf_verifier_env *env, + const union bpf_attr *attr, + bpfptr_t uattr) { struct btf *btf; int err; @@ -429,7 +429,7 @@ int bpf_check_btf_info_early(struct bpf_verifier_env *env, } env->prog->aux->btf = btf; - err = check_btf_func_early(env, attr, uattr); + err = prepare_btf_func(env, attr, uattr); if (err) return err; return 0; diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c index b2a19acadb91..7f1b30059cc8 100644 --- a/kernel/bpf/const_fold.c +++ b/kernel/bpf/const_fold.c @@ -182,7 +182,6 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info * u64 val = 0; if (!bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr || - map->map_type == BPF_MAP_TYPE_INSN_ARRAY || off < 0 || off + size > map->value_size || bpf_map_direct_read(map, off, size, &val, is_ldsx)) { *dst = unknown; @@ -199,14 +198,9 @@ process_call: ci_out[r] = unknown; break; case BPF_STX: - if (mode != BPF_ATOMIC) - break; - if (insn->imm == BPF_CMPXCHG) - ci_out[BPF_REG_0] = unknown; - else if (insn->imm == BPF_LOAD_ACQ) - *dst = unknown; - else if (insn->imm & BPF_FETCH) - *src = unknown; + r = bpf_atomic_load_reg(insn); + if (r >= 0) + ci_out[r] = unknown; break; } } diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 6e19a030da6f..d55e737ed75a 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -126,6 +126,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag fp->aux->main_prog_aux = aux; fp->aux->prog = fp; fp->jit_requested = ebpf_jit_enabled(); + fp->jit_required = IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON); fp->blinding_requested = bpf_jit_blinding_enabled(fp); #ifdef CONFIG_CGROUP_BPF aux->cgroup_atype = CGROUP_BPF_ATTACH_TYPE_INVALID; @@ -305,7 +306,7 @@ int bpf_prog_calc_tag(struct bpf_prog *fp) bool was_ld_map; u32 i; - dst = vmalloc(size); + dst = __vmalloc(size, GFP_KERNEL_ACCOUNT); if (!dst) return -ENOMEM; @@ -915,6 +916,11 @@ static LIST_HEAD(pack_list); #define BPF_PROG_CHUNK_COUNT (BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE) +static bool bpf_jit_mem_is_rox(void) +{ + return execmem_is_rox(EXECMEM_BPF); +} + static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_insns) { struct bpf_prog_pack *pack; @@ -926,16 +932,18 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins pack->ptr = bpf_jit_alloc_exec(BPF_PROG_PACK_SIZE); if (!pack->ptr) goto out; - bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE); bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE); if (static_branch_unlikely(&bpf_pred_flush_enabled)) pack->arch_flush_needed = true; - set_vm_flush_reset_perms(pack->ptr); - err = set_memory_rox((unsigned long)pack->ptr, - BPF_PROG_PACK_SIZE / PAGE_SIZE); - if (err) - goto out; + if (!bpf_jit_mem_is_rox()) { + bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE); + set_vm_flush_reset_perms(pack->ptr); + err = set_memory_rox((unsigned long)pack->ptr, + BPF_PROG_PACK_SIZE / PAGE_SIZE); + if (err) + goto out; + } list_add_tail(&pack->list, &pack_list); return pack; @@ -964,7 +972,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool pr_warn_once("BPF: Predictors not flushed for allocations greater than BPF_PROG_PACK_SIZE\n"); size = round_up(size, PAGE_SIZE); ptr = bpf_jit_alloc_exec(size); - if (ptr) { + if (ptr && !bpf_jit_mem_is_rox()) { int err; bpf_fill_ill_insns(ptr, size); @@ -1115,12 +1123,17 @@ void bpf_jit_uncharge_modmem(u32 size) atomic_long_sub(size, &bpf_jit_current); } -void *__weak bpf_jit_alloc_exec(unsigned long size) +void *bpf_jit_alloc_exec(unsigned long size) { return execmem_alloc(EXECMEM_BPF, size); } -void __weak bpf_jit_free_exec(void *addr) +void *bpf_jit_alloc_exec_rw(unsigned long size) +{ + return execmem_alloc_rw(EXECMEM_BPF, size); +} + +void bpf_jit_free_exec(void *addr) { execmem_free(addr); } @@ -2621,22 +2634,10 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc { #ifdef CONFIG_BPF_JIT struct bpf_prog *orig_prog; - struct bpf_insn_aux_data *orig_insn_aux; if (!bpf_prog_need_blind(prog)) return bpf_int_jit_compile(env, prog); - if (env) { - /* - * If env is not NULL, we are called from the end of bpf_check(), at this - * point, only insn_aux_data is used after failure, so it should be restored - * on failure. - */ - orig_insn_aux = bpf_dup_insn_aux_data(env); - if (!orig_insn_aux) - return prog; - } - orig_prog = prog; prog = bpf_jit_blind_constants(env, prog); /* @@ -2649,8 +2650,6 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc prog = bpf_int_jit_compile(env, prog); if (prog->jited) { bpf_jit_prog_release_other(prog, orig_prog); - if (env) - vfree(orig_insn_aux); return prog; } @@ -2658,8 +2657,6 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc out_restore: prog = orig_prog; - if (env) - bpf_restore_insn_aux_data(env, orig_insn_aux); #endif return prog; } @@ -2670,15 +2667,11 @@ struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct /* In case of BPF to BPF calls, verifier did all the prep * work with regards to JITing, etc. */ - bool jit_needed = false; + bool jit_needed = fp->jit_required; if (fp->bpf_func) goto finalize; - if (IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) || - bpf_prog_has_kfunc_call(fp)) - jit_needed = true; - if (!bpf_prog_select_interpreter(fp)) jit_needed = true; @@ -3299,6 +3292,11 @@ bool __weak bpf_jit_supports_stack_args(void) return false; } +bool __weak bpf_jit_supports_arena_args(void) +{ + return false; +} + bool __weak bpf_jit_supports_far_kfunc_call(void) { return false; @@ -3463,24 +3461,14 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_bulk_tx); #ifdef CONFIG_BPF_SYSCALL -void bpf_get_linfo_file_line(struct btf *btf, const struct bpf_line_info *linfo, - const char **filep, const char **linep, int *nump) +void bpf_get_linfo_source(struct btf *btf, const struct bpf_line_info *linfo, + struct bpf_linfo_source *src) { - /* Get base component of the file path. */ - if (filep) { - *filep = btf_name_by_offset(btf, linfo->file_name_off); - *filep = kbasename(*filep); - } - - /* Obtain the source line, and strip whitespace in prefix. */ - if (linep) { - *linep = btf_name_by_offset(btf, linfo->line_off); - while (isspace(**linep)) - *linep += 1; - } - - if (nump) - *nump = BPF_LINE_INFO_LINE_NUM(linfo->line_col); + src->file = kbasename(btf_name_by_offset(btf, linfo->file_name_off)); + src->line = btf_name_by_offset(btf, linfo->line_off); + src->file_name_off = linfo->file_name_off; + src->line_num = BPF_LINE_INFO_LINE_NUM(linfo->line_col); + src->line_col = BPF_LINE_INFO_LINE_COL(linfo->line_col); } const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn_off) @@ -3523,6 +3511,7 @@ const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep, const char **linep, int *nump) { + struct bpf_linfo_source src; int idx = -1, insn_start, insn_end, len; struct bpf_line_info *linfo; void **jited_linfo; @@ -3554,7 +3543,15 @@ int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char * if (idx == -1) return -ENOENT; - bpf_get_linfo_file_line(btf, &linfo[idx], filep, linep, nump); + bpf_get_linfo_source(btf, &linfo[idx], &src); + while (isspace(*src.line)) + src.line++; + if (filep) + *filep = src.file; + if (linep) + *linep = src.line; + if (nump) + *nump = src.line_num; return 0; } diff --git a/kernel/bpf/cpumask.c b/kernel/bpf/cpumask.c index b8c805b4b06a..1336a4efa755 100644 --- a/kernel/bpf/cpumask.c +++ b/kernel/bpf/cpumask.c @@ -449,12 +449,12 @@ __bpf_kfunc u32 bpf_cpumask_weight(const struct cpumask *cpumask) * @src__sz: Length of the BPF memory region in bytes. * * Return: - * * 0 if the struct cpumask * instance was populated successfully. + * * 0 if the struct bpf_cpumask * instance was populated successfully. * * -EACCES if the memory region is too small to populate the cpumask. * * -EINVAL if the memory region is not aligned to the size of a long * and the architecture does not support efficient unaligned accesses. */ -__bpf_kfunc int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t src__sz) +__bpf_kfunc int bpf_cpumask_populate(struct bpf_cpumask *cpumask, void *src, size_t src__sz) { unsigned long source = (unsigned long)src; @@ -467,7 +467,7 @@ __bpf_kfunc int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t !IS_ALIGNED(source, sizeof(long))) return -EINVAL; - bitmap_copy(cpumask_bits(cpumask), src, nr_cpu_ids); + bitmap_copy(cpumask_bits(&cpumask->cpumask), src, nr_cpu_ids); return 0; } diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c new file mode 100644 index 000000000000..b682fd2be443 --- /dev/null +++ b/kernel/bpf/diagnostics.c @@ -0,0 +1,2359 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + +#include <linux/bpf.h> +#include <linux/bpf_verifier.h> +#include <linux/btf.h> +#include <linux/ctype.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/seq_buf.h> +#include <linux/overflow.h> +#include <linux/slab.h> +#include <linux/stdarg.h> +#include <linux/string.h> + +#include "disasm.h" +#include "diagnostics.h" + +#define REGISTER_TYPE_SAFETY "Register Type Safety" +#define MEMORY_SAFETY "Memory Safety" +#define RESOURCE_LIFETIME_SAFETY "Resource Lifetime Safety" +#define CALL_TYPE_SAFETY "Call Type Safety" +#define EXECUTION_CONTEXT_SAFETY "Execution Context Safety" +#define PROGRAM_STRUCTURE "Program Structure" +#define POLICY "Policy" + +#define BPF_DIAG_TEXT_WIDTH 100 +#define BPF_DIAG_TEXT_INDENT " " +#define BPF_DIAG_CONTEXT 2 +#define BPF_DIAG_CONTEXT_CNT (1 + BPF_DIAG_CONTEXT * 2) +#define BPF_DIAG_HISTORY_RENDER_MAX 64 +#define BPF_DIAG_SOURCE_LANE_WIDTH 88 +#define BPF_DIAG_TAB_WIDTH 8 +#define BPF_DIAG_FMT_CHUNK_SIZE (PAGE_SIZE - sizeof(struct diag_fmt_chunk)) +#define BPF_DIAG_FMT_BUF_SIZE 256 +#define BPF_DIAG_EVENT_LOG_MAX_SIZE (64U << 20) +#define DISASM_LINE_LEN 160 + +enum bpf_diag_mod_target_kind { + BPF_DIAG_MOD_TARGET_NONE, + BPF_DIAG_MOD_TARGET_REG, + BPF_DIAG_MOD_TARGET_STACK_ARG, + BPF_DIAG_MOD_TARGET_STACK_SLOT, + BPF_DIAG_MOD_TARGET_STACK_RANGE, +}; + +struct bpf_diag_mod_target { + u32 frame_id; + union { + struct { + s16 min_off; + s16 max_off; + } range; + u16 spi; + u8 regno; + u8 stack_arg; + }; + u8 frameno; + u8 kind; +}; + +static struct bpf_diag_mod_target diag_reg_target(u32 frame_id, u8 frameno, u8 regno) +{ + return (struct bpf_diag_mod_target){ + .frame_id = frame_id, + .frameno = frameno, + .kind = BPF_DIAG_MOD_TARGET_REG, + .regno = regno, + }; +} + +static struct bpf_diag_mod_target diag_stack_arg_target(u32 frame_id, u8 frameno, u8 slot) +{ + return (struct bpf_diag_mod_target){ + .frame_id = frame_id, + .frameno = frameno, + .kind = BPF_DIAG_MOD_TARGET_STACK_ARG, + .stack_arg = slot, + }; +} + +static struct bpf_diag_mod_target diag_stack_slot_target(u32 frame_id, u8 frameno, u16 spi) +{ + return (struct bpf_diag_mod_target){ + .frame_id = frame_id, + .frameno = frameno, + .kind = BPF_DIAG_MOD_TARGET_STACK_SLOT, + .spi = spi, + }; +} + +static struct bpf_diag_mod_target diag_stack_range_target(u32 frame_id, u8 frameno, + s16 min_off, s16 max_off) +{ + return (struct bpf_diag_mod_target){ + .frame_id = frame_id, + .frameno = frameno, + .kind = BPF_DIAG_MOD_TARGET_STACK_RANGE, + .range.min_off = min_off, + .range.max_off = max_off, + }; +} + +struct bpf_diag_reg_snapshot { + u32 type; + u32 btf_id; + const struct bpf_map *map_ptr; + const struct btf *btf; + struct tnum var_off; + struct cnum64 r64; +}; + +enum bpf_diag_history_kind { + BPF_DIAG_HISTORY_BRANCH, + BPF_DIAG_HISTORY_MOD, + BPF_DIAG_HISTORY_REF_ACQUIRE, + BPF_DIAG_HISTORY_REF_RELEASE, + BPF_DIAG_HISTORY_CONTEXT, +}; + +struct bpf_diag_history_event { + u32 insn_idx : 24; + u32 kind : 8; + u8 in_lineage : 1; + union { + struct { + bool cond_true; + } branch; + struct { + struct bpf_diag_mod_target target; + struct bpf_diag_mod_target origin; + struct bpf_diag_reg_snapshot old, new; + u8 reason; + bool origin_valid; + } mod; + struct { + u32 ref_id; + } ref; + struct { + u32 depth; + u8 kind; + bool enter; + } ctx; + }; +}; + +enum bpf_diag_history_scope { + BPF_DIAG_HISTORY_SCOPE_REG, + BPF_DIAG_HISTORY_SCOPE_STACK_ARG, + BPF_DIAG_HISTORY_SCOPE_REF, + BPF_DIAG_HISTORY_SCOPE_CONTEXT, +}; + +struct bpf_diag_history_opts { + enum bpf_diag_history_scope scope; + u32 frame_id; + u32 frameno; + int regno; + int stack_arg_slot; + u32 ref_id; + enum bpf_diag_context_kind ctx_kind; + u32 ctx_depth; +}; + +static void diag_print_history(struct bpf_verifier_env *env, + const struct bpf_diag_history_opts *opts); +static bool diag_target_matches(const struct bpf_diag_mod_target *event_target, + const struct bpf_diag_mod_target *target); +static const char *diag_context_name(enum bpf_diag_context_kind kind); +struct disasm_line { + char text[DISASM_LINE_LEN]; + int idx; + bool valid; +}; + +struct disasm_ctx { + struct bpf_verifier_env *env; + struct seq_buf seq; +}; + +struct diag_fmt_chunk { + struct list_head node; + struct seq_buf seq; + char data[]; +}; + +struct diag_fmt_mark { + struct diag_fmt_chunk *chunk; + size_t len; +}; + +struct bpf_diag_log { + struct bpf_diag_history_event *events; + /* Sequence number of the oldest retained event on the active path. */ + u64 first_seq; + u32 cnt; + u32 cap; + u32 head; + bool growth_failed; +}; + +struct bpf_diag_scratch { + struct bpf_linfo_source source_lines[BPF_DIAG_CONTEXT_CNT]; + struct disasm_line disasm_lines[BPF_DIAG_CONTEXT_CNT]; +}; + +struct bpf_diag_mod_scope { + struct bpf_reg_state target_reg_snapshot; + struct bpf_diag_mod_target target; + struct bpf_diag_mod_target origin; + enum bpf_diag_mod_reason reason; + u32 insn_idx; + bool active; + bool origin_valid; +}; + +struct bpf_diag { + struct bpf_diag_log log; + struct bpf_diag_scratch scratch; + struct list_head fmt_chunks; + struct bpf_diag_mod_scope mod; + u32 frame_id_gen; +}; + +bool bpf_diag_enabled(const struct bpf_verifier_env *env) +{ + return env->log.level & BPF_LOG_LEVEL; +} + +static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); + +int bpf_diag_init(struct bpf_verifier_env *env) +{ + if (!bpf_diag_enabled(env)) + return 0; + + env->diag = kzalloc_obj(struct bpf_diag, GFP_KERNEL_ACCOUNT); + if (!env->diag) + return -ENOMEM; + + INIT_LIST_HEAD(&env->diag->fmt_chunks); + return 0; +} + +void bpf_diag_init_frame(struct bpf_verifier_env *env, struct bpf_func_state *state) +{ + if (env->diag) + state->diag_frame_id = ++env->diag->frame_id_gen; +} + +static char *diag_fmt_alloc(struct bpf_verifier_env *env, size_t size) +{ + struct bpf_diag *diag = env->diag; + struct diag_fmt_chunk *chunk; + size_t capacity, available; + char *buf; + + if (!diag || !size || size > INT_MAX) + return NULL; + + if (!list_empty(&diag->fmt_chunks)) { + chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); + available = seq_buf_get_buf(&chunk->seq, &buf); + if (available >= size) + goto commit; + } + + capacity = max_t(size_t, BPF_DIAG_FMT_CHUNK_SIZE, size); + chunk = kmalloc(struct_size(chunk, data, capacity), GFP_KERNEL_ACCOUNT); + if (!chunk) + return NULL; + + seq_buf_init(&chunk->seq, chunk->data, capacity); + list_add_tail(&chunk->node, &diag->fmt_chunks); + available = seq_buf_get_buf(&chunk->seq, &buf); + if (WARN_ON_ONCE(available < size)) + return NULL; + +commit: + seq_buf_commit(&chunk->seq, size); + return buf; +} + +char *bpf_diag_fmt_buf(struct bpf_verifier_env *env, size_t size) +{ + char *buf; + + buf = diag_fmt_alloc(env, size); + if (buf) + buf[0] = '\0'; + return buf; +} + +const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list args) +{ + va_list copy; + char *buf; + int len; + + va_copy(copy, args); + len = vsnprintf(NULL, 0, fmt, copy); + va_end(copy); + if (len < 0 || len == INT_MAX) + return ""; + + buf = diag_fmt_alloc(env, len + 1); + if (buf) + vsnprintf(buf, len + 1, fmt, args); + return buf ?: ""; +} + +const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) +{ + const char *buf; + va_list args; + + va_start(args, fmt); + buf = bpf_diag_vfmt(env, fmt, args); + va_end(args); + return buf; +} + +static struct diag_fmt_mark diag_fmt_save(struct bpf_verifier_env *env) +{ + struct bpf_diag *diag = env->diag; + struct diag_fmt_mark mark = {}; + + if (!diag || list_empty(&diag->fmt_chunks)) + return mark; + + mark.chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); + mark.len = mark.chunk->seq.len; + return mark; +} + +static void diag_fmt_restore(struct bpf_verifier_env *env, struct diag_fmt_mark mark) +{ + struct bpf_diag *diag = env->diag; + struct diag_fmt_chunk *chunk; + + if (!diag) + return; + + while (!list_empty(&diag->fmt_chunks)) { + chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); + if (chunk == mark.chunk) + break; + list_del(&chunk->node); + kfree(chunk); + } + + if (mark.chunk) { + mark.chunk->seq.len = mark.len; + seq_buf_str(&mark.chunk->seq); + } +} + +void bpf_diag_free(struct bpf_verifier_env *env) +{ + struct bpf_diag *diag = env->diag; + + if (!diag) + return; + + diag_fmt_restore(env, (struct diag_fmt_mark){}); + kvfree(diag->log.events); + kfree(diag); + env->diag = NULL; +} + +static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) +{ + va_list args; + + if (!bpf_diag_enabled(env)) + return; + + va_start(args, fmt); + bpf_verifier_vlog(&env->log, fmt, args); + va_end(args); +} + +static u64 log_end(const struct bpf_diag_log *log) +{ + return log->first_seq + log->cnt; +} + +static u32 log_pos(const struct bpf_diag_log *log, u32 idx) +{ + u32 pos = log->head + idx; + + return pos < log->cap ? pos : pos - log->cap; +} + +u64 bpf_diag_event_log_save(struct bpf_verifier_env *env) +{ + struct bpf_diag *diag = env->diag; + + return diag ? log_end(&diag->log) : 0; +} + +void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos) +{ + struct bpf_diag *diag = env->diag; + struct bpf_diag_log *log; + u64 end_seq; + + if (!diag) + return; + + log = &diag->log; + end_seq = log_end(log); + if (WARN_ON_ONCE(log_pos > end_seq)) + log_pos = end_seq; + + /* + * A deep abandoned path may have rotated away the shared prefix. In + * that case, restart with an empty retained suffix and remember that + * every event before the restored mark is unavailable. + */ + if (log_pos <= log->first_seq) { + log->first_seq = log_pos; + log->head = 0; + log->cnt = 0; + return; + } + + log->cnt = log_pos - log->first_seq; +} + +u32 bpf_diag_irq_depth(const struct bpf_verifier_state *state) +{ + u32 depth = 0; + int i; + + for (i = 0; i < state->acquired_refs; i++) { + if (state->refs[i].type == REF_TYPE_IRQ) + depth++; + } + + return depth; +} + +static void diag_append_history(struct bpf_verifier_env *env, + const struct bpf_diag_history_event *event) +{ + struct bpf_diag_history_event *events; + struct bpf_diag *diag = env->diag; + struct bpf_diag_log *log; + u32 cap, max_events; + + if (!diag) + return; + log = &diag->log; + + if (log->cnt < log->cap) { + log->events[log_pos(log, log->cnt++)] = *event; + return; + } + + max_events = BPF_DIAG_EVENT_LOG_MAX_SIZE / sizeof(*events); + if (log->growth_failed || log->cap == max_events) + goto rotate; + + cap = min(log->cap ? log->cap * 2 : 64, max_events); + events = kvrealloc(log->events, array_size(cap, sizeof(*events)), GFP_KERNEL_ACCOUNT); + if (!events) { + log->growth_failed = true; + goto rotate; + } + log->events = events; + log->cap = cap; + log->events[log->cnt++] = *event; + return; + +rotate: + if (log->cap) { + log->events[log->head++] = *event; + if (log->head == log->cap) + log->head = 0; + } + log->first_seq++; +} + +static void diag_print_wrapped_prefixed(struct bpf_verifier_env *env, const char *first_prefix, + const char *next_prefix, const char *text) +{ + const char *prefix = first_prefix; + + while (*text) { + const char *line = text; + int prefix_len = strlen(prefix); + int text_width = BPF_DIAG_TEXT_WIDTH - prefix_len; + int len = 0, last_space = -1; + + if (text_width < 1) + text_width = 1; + + while (line[len] && line[len] != '\n' && len < text_width) { + if (line[len] == ' ') + last_space = len; + len++; + } + + if (line[len] && line[len] != '\n' && line[len] != ' ' && last_space > 0) + len = last_space; + + diag_write(env, "%s%.*s\n", prefix, len, line); + + text = line + len; + while (*text == ' ') + text++; + if (*text == '\n') + text++; + + prefix = next_prefix; + } +} + +const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id) +{ + char *buf = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); + size_t len; + int ret; + + if (!buf) + return ""; + + buf[0] = '\0'; + ret = btf_type_name_to_buf(btf, type_id, buf, BPF_DIAG_FMT_BUF_SIZE); + if (ret < 0 || !buf[0]) { + scnprintf(buf, BPF_DIAG_FMT_BUF_SIZE, "BTF type ID %u", type_id); + return buf; + } + + len = strlen(buf); + if (len && buf[len - 1] == '{') + buf[len - 1] = '\0'; + return buf; +} + +static void diag_vprint_indented(struct bpf_verifier_env *env, const char *fmt, va_list args) + __printf(2, 0); + +static void diag_vprint_indented(struct bpf_verifier_env *env, const char *fmt, va_list args) +{ + char *buf; + + if (!bpf_diag_enabled(env)) + return; + + buf = kvasprintf(GFP_KERNEL_ACCOUNT, fmt, args); + if (!buf) { + diag_write(env, "%s<failed to allocate diagnostic text>\n", BPF_DIAG_TEXT_INDENT); + return; + } + + diag_print_wrapped_prefixed(env, BPF_DIAG_TEXT_INDENT, BPF_DIAG_TEXT_INDENT, buf); + kfree(buf); +} + +static int diag_line_width(unsigned int line) +{ + int width = 1; + + while (line >= 10) { + line /= 10; + width++; + } + + return width; +} + +static int diag_line_indent(const char *line) +{ + int indent = 0; + + while (*line == ' ' || *line == '\t') { + if (*line == '\t') + indent = round_up(indent + 1, BPF_DIAG_TAB_WIDTH); + else + indent++; + line++; + } + + return indent; +} + +static void disasm_print(void *private_data, const char *fmt, ...) __printf(2, 3); + +static void disasm_print(void *private_data, const char *fmt, ...) +{ + struct disasm_ctx *ctx = private_data; + va_list args; + + va_start(args, fmt); + seq_buf_vprintf(&ctx->seq, fmt, args); + va_end(args); +} + +static const char *disasm_kfunc_name(void *private_data, const struct bpf_insn *insn) +{ + struct disasm_ctx *ctx = private_data; + + return bpf_disasm_kfunc_name(ctx->env, insn); +} + +static void format_disasm_line(struct bpf_verifier_env *env, int insn_idx, + struct disasm_line *line) +{ + struct disasm_ctx ctx = { .env = env }; + struct bpf_insn *insn; + const struct bpf_insn_cbs cbs = { + .cb_call = disasm_kfunc_name, + .cb_print = disasm_print, + .private_data = &ctx, + }; + + line->idx = insn_idx; + line->valid = false; + seq_buf_init(&ctx.seq, line->text, sizeof(line->text)); + + if (insn_idx < 0 || insn_idx >= env->prog->len) + return; + + if (insn_idx > 0 && bpf_is_ldimm64(&env->prog->insnsi[insn_idx - 1])) + return; + + insn = &env->prog->insnsi[insn_idx]; + if (bpf_is_ldimm64(insn) && insn_idx + 1 >= env->prog->len) + return; + + print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); + seq_buf_str(&ctx.seq); + ctx.seq.len = strnlen(line->text, sizeof(line->text)); + while (ctx.seq.len && line->text[ctx.seq.len - 1] == '\n') + seq_buf_pop(&ctx.seq); + seq_buf_str(&ctx.seq); + + line->valid = true; +} + +static void diag_format_source_text(char *buf, size_t size, const char *line, int width) +{ + int col = 0, len = 0; + + if (!size) + return; + if (width <= 0) { + buf[0] = '\0'; + return; + } + + line = line ?: "..."; + while (*line && col < width && len + 1 < size) { + if (*line == '\t') { + int next = round_up(col + 1, BPF_DIAG_TAB_WIDTH); + + while (col < next && col < width && len + 1 < size) { + buf[len++] = ' '; + col++; + } + line++; + continue; + } + + buf[len++] = *line++; + col++; + } + + if (*line) { + int ellipsis_len = min(3, width); + + while (len > 0 && col > width - ellipsis_len) { + len--; + col--; + } + while (ellipsis_len-- && len + 1 < size) + buf[len++] = '.'; + } + + buf[len] = '\0'; +} + +static void diag_format_source_lane(char *buf, size_t size, const char *source_prefix, + int source_line_width, int line_num, const char *line) +{ + int len, text_width; + + if (line_num <= 0) { + buf[0] = '\0'; + return; + } + + len = scnprintf(buf, size, "%s%*d | ", source_prefix, source_line_width, line_num); + text_width = BPF_DIAG_SOURCE_LANE_WIDTH - len; + diag_format_source_text(buf + len, size - len, line, text_width); +} + +static void bpf_diag_header(struct bpf_verifier_env *env, const char *category, + const char *problem) +{ + char first; + + if (!bpf_diag_enabled(env)) + return; + + category = category ?: "Verifier Error"; + problem = problem ?: ""; + + if (!problem[0]) { + diag_write(env, "\nVerification failed: %s\n", category); + return; + } + + first = toupper(problem[0]); + diag_write(env, "\nVerification failed: %s: %c%s\n", category, first, problem + 1); +} + +static void diag_reason(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); +static void diag_suggestion(struct bpf_verifier_env *env, const char *fmt, ...) + __printf(2, 3); + +static void diag_section(struct bpf_verifier_env *env, const char *title) +{ + if (!bpf_diag_enabled(env)) + return; + + diag_write(env, "\n%s:\n", title); +} + +static void diag_reason(struct bpf_verifier_env *env, const char *fmt, ...) +{ + va_list args; + + if (!bpf_diag_enabled(env)) + return; + + diag_section(env, "Reason"); + + va_start(args, fmt); + diag_vprint_indented(env, fmt, args); + va_end(args); +} + +static void diag_suggestion(struct bpf_verifier_env *env, const char *fmt, ...) +{ + va_list args; + + if (!bpf_diag_enabled(env)) + return; + + diag_section(env, "Suggestion"); + + va_start(args, fmt); + diag_vprint_indented(env, fmt, args); + va_end(args); + diag_write(env, "\n"); +} + +static void diag_print_source_annotation(struct bpf_verifier_env *env, int line_width, int indent, + const char *label, const char *msg) +{ + const char *first_prefix, *next_prefix, *text; + + indent = min_t(int, indent, max_t(int, 0, BPF_DIAG_SOURCE_LANE_WIDTH - line_width - 8)); + text = bpf_diag_fmt(env, "%s: %s", label, msg); + first_prefix = bpf_diag_fmt(env, " %*s | %*s^-- ", line_width + 4, "", indent, ""); + next_prefix = bpf_diag_fmt(env, " %*s | %*s ", line_width + 4, "", indent, ""); + + diag_print_wrapped_prefixed(env, first_prefix, next_prefix, text); +} + +static void diag_print_insn_context(struct bpf_verifier_env *env, u32 insn_idx, + struct disasm_line *disasm_lines) +{ + int insn_width = diag_line_width(env->prog->len ? env->prog->len - 1 : 0); + int i; + + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { + int row = i - BPF_DIAG_CONTEXT; + + format_disasm_line(env, insn_idx + row, &disasm_lines[i]); + } + + diag_write(env, " Instruction context:\n"); + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { + struct disasm_line *line = &disasm_lines[i]; + + if (line->valid) + diag_write(env, " %s%*d | %s\n", + line->idx == insn_idx ? ">>> " : " ", + insn_width, line->idx, line->text); + } +} + +static void bpf_diag_source(struct bpf_verifier_env *env, u32 insn_idx, const char *label, + const char *fmt, ...) +{ + struct bpf_diag_scratch *scratch; + struct bpf_linfo_source *source_lines; + struct disasm_line *disasm_lines; + struct bpf_linfo_source src = {}; + struct diag_fmt_mark mark; + const struct bpf_line_info *linfo; + const struct bpf_subprog_info *subprog; + struct btf *btf = env->prog->aux->btf; + char *source_lane; + const char *msg; + const char *func; + int start_line, end_line, width, indent, subprogno, linfo_start, linfo_end, i; + va_list args; + + if (!bpf_diag_enabled(env)) + return; + if (!env->diag) + return; + + mark = diag_fmt_save(env); + label = label ?: "note"; + scratch = &env->diag->scratch; + source_lines = scratch->source_lines; + disasm_lines = scratch->disasm_lines; + memset(source_lines, 0, sizeof(scratch->source_lines)); + memset(disasm_lines, 0, sizeof(scratch->disasm_lines)); + + va_start(args, fmt); + msg = bpf_diag_vfmt(env, fmt, args); + va_end(args); + if (!*msg) + msg = "<failed to allocate diagnostic text>"; + + linfo = bpf_find_linfo(env->prog, insn_idx); + if (btf && linfo) + bpf_get_linfo_source(btf, linfo, &src); + if (!src.file || !*src.file) { + diag_write(env, " insn %u\n", insn_idx); + goto out_annotation; + } + + subprog = bpf_find_containing_subprog(env, insn_idx); + subprogno = subprog ? subprog - env->subprog_info : -ENOENT; + func = subprogno >= 0 ? bpf_subprog_name(env, subprogno) : NULL; + if (func && *func) + diag_write(env, " %s @ %s:%d:%d\n", func, src.file, src.line_num, src.line_col); + else + diag_write(env, " %s:%d:%d\n", src.file, src.line_num, src.line_col); + if (!src.line || !*src.line) + goto out_annotation; + + start_line = src.line_num - BPF_DIAG_CONTEXT; + end_line = src.line_num + BPF_DIAG_CONTEXT; + width = diag_line_width(end_line); + indent = diag_line_indent(src.line); + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) + source_lines[i].line_num = start_line + i; + + linfo = env->prog->aux->linfo; + linfo_start = subprog ? subprog->linfo_idx : 0; + linfo_end = subprogno >= 0 && subprogno + 1 < env->subprog_cnt ? + env->subprog_info[subprogno + 1].linfo_idx : env->prog->aux->nr_linfo; + for (i = linfo_start; i < linfo_end; i++) { + struct bpf_linfo_source line_src; + int idx; + + bpf_get_linfo_source(btf, &linfo[i], &line_src); + if (line_src.file_name_off != src.file_name_off || + line_src.line_num < start_line || line_src.line_num > end_line || + !line_src.line || !*line_src.line) + continue; + + idx = line_src.line_num - start_line; + if (!source_lines[idx].line) + source_lines[idx] = line_src; + } + + diag_write(env, " Source context:\n"); + source_lane = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); + if (!source_lane) + goto out_restore; + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { + const char *source_prefix; + + source_prefix = source_lines[i].line_num == src.line_num ? ">>> " : " "; + diag_format_source_lane(source_lane, BPF_DIAG_FMT_BUF_SIZE, source_prefix, width, + source_lines[i].line_num, source_lines[i].line); + diag_write(env, " %s\n", source_lane); + if (source_lines[i].line_num == src.line_num) + diag_print_source_annotation(env, width, indent, label, msg); + } + diag_print_insn_context(env, insn_idx, disasm_lines); + goto out_restore; + +out_annotation: + diag_print_source_annotation(env, 0, 0, label, msg); + diag_print_insn_context(env, insn_idx, disasm_lines); +out_restore: + diag_fmt_restore(env, mark); +} + +static const struct bpf_func_state *diag_current_frame(const struct bpf_verifier_env *env) +{ + return env->cur_state->frame[env->cur_state->curframe]; +} + +void bpf_diag_register_type(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *problem, const char *reason, const char *suggestion) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .regno = regno, + }; + + bpf_diag_header(env, REGISTER_TYPE_SAFETY, problem); + diag_reason(env, "%s", reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); + + if (regno >= 0) + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_type type) +{ + switch (base_type(type)) { + case NOT_INIT: + return "an uninitialized value"; + case SCALAR_VALUE: + return "an integer scalar"; + case PTR_TO_CTX: + return "a context pointer"; + case PTR_TO_STACK: + return "a stack pointer"; + case PTR_TO_MAP_VALUE: + if (type_may_be_null(type)) + return "a nullable map value pointer"; + return "a map value pointer"; + case PTR_TO_MEM: + if (type_may_be_null(type)) + return "a nullable memory pointer"; + return "a memory pointer"; + case PTR_TO_BTF_ID: + if (type_may_be_null(type)) + return "a nullable kernel object pointer"; + if (type_is_non_owning_ref(type)) + return "a borrowed allocated object pointer"; + if (type_is_ptr_alloc_obj(type)) + return "an owned allocated object pointer"; + if (type_flag(type) & PTR_UNTRUSTED) + return "an untrusted kernel object pointer"; + return "a kernel object pointer"; + default: + return reg_type_str(env, type); + } +} + +static const char *diag_arg_ordinal(int argno) +{ + switch (argno) { + case 1: + return "first"; + case 2: + return "second"; + case 3: + return "third"; + case 4: + return "fourth"; + case 5: + return "fifth"; + case 6: + return "sixth"; + case 7: + return "seventh"; + case 8: + return "eighth"; + case 9: + return "ninth"; + case 10: + return "tenth"; + case 11: + return "eleventh"; + case 12: + return "twelfth"; + default: + return NULL; + } +} + +void bpf_diag_call_type(struct bpf_verifier_env *env, u32 insn_idx, int argno, int regno, + int stack_arg_slot, const char *call_name, const char *arg_name, + const char *reason, const char *suggestion) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + }; + const char *ordinal = diag_arg_ordinal(argno); + const char *arg_desc; + bool print_history = true; + + if (regno >= 0) { + opts.scope = BPF_DIAG_HISTORY_SCOPE_REG; + opts.regno = regno; + } else if (stack_arg_slot >= 0) { + opts.scope = BPF_DIAG_HISTORY_SCOPE_STACK_ARG; + opts.stack_arg_slot = stack_arg_slot; + } else { + print_history = false; + } + + if (ordinal && arg_name) + arg_desc = bpf_diag_fmt(env, "%s argument (%s)", ordinal, arg_name); + else if (ordinal) + arg_desc = bpf_diag_fmt(env, "%s argument", ordinal); + else if (arg_name) + arg_desc = bpf_diag_fmt(env, "argument %s", arg_name); + else + arg_desc = "argument"; + + bpf_diag_header(env, CALL_TYPE_SAFETY, "invalid call argument"); + diag_reason(env, "The %s to %s does not satisfy the verifier contract: %s.", + arg_desc, call_name, reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "invalid %s for %s", arg_desc, call_name); + + if (print_history) + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +static const char *diag_context_constraint(enum bpf_diag_context_kind kind) +{ + switch (kind) { + case BPF_DIAG_CONTEXT_RCU: + return "RCU read-side critical sections cannot call operations that may sleep"; + case BPF_DIAG_CONTEXT_PREEMPT: + return "preemption-disabled code cannot call operations that may sleep"; + case BPF_DIAG_CONTEXT_IRQ: + return "IRQ-disabled code cannot call operations that may sleep"; + case BPF_DIAG_CONTEXT_LOCK: + return "code holding a BPF spin lock cannot call operations that may sleep"; + case BPF_DIAG_CONTEXT_NONE: + default: + return NULL; + } +} + +static const char *diag_active_context(struct bpf_verifier_env *env, u32 depth, + const char *context) +{ + if (depth == 1) + return bpf_diag_fmt(env, "an active %s (depth 1)", context); + return bpf_diag_fmt(env, "%u active %ss (depth %u)", depth, context, depth); +} + +static u32 diag_context_depth(struct bpf_verifier_env *env, enum bpf_diag_context_kind kind) +{ + switch (kind) { + case BPF_DIAG_CONTEXT_RCU: + return env->cur_state->active_rcu_locks; + case BPF_DIAG_CONTEXT_PREEMPT: + return env->cur_state->active_preempt_locks; + case BPF_DIAG_CONTEXT_IRQ: + return bpf_diag_irq_depth(env->cur_state); + case BPF_DIAG_CONTEXT_LOCK: + return env->cur_state->active_locks; + case BPF_DIAG_CONTEXT_NONE: + default: + return 0; + } +} + +void bpf_diag_ctx_forbidden(struct bpf_verifier_env *env, u32 insn_idx, + const char *operation, const char *suggestion) +{ + struct bpf_diag_history_opts opts; + enum bpf_diag_context_kind ctx_kind; + const char *constraint, *context; + u32 depth; + + if (env->cur_state->active_rcu_locks) + ctx_kind = BPF_DIAG_CONTEXT_RCU; + else if (env->cur_state->active_preempt_locks) + ctx_kind = BPF_DIAG_CONTEXT_PREEMPT; + else if (env->cur_state->active_irq_id) + ctx_kind = BPF_DIAG_CONTEXT_IRQ; + else if (env->cur_state->active_locks) + ctx_kind = BPF_DIAG_CONTEXT_LOCK; + else + ctx_kind = BPF_DIAG_CONTEXT_NONE; + + depth = diag_context_depth(env, ctx_kind); + opts = (struct bpf_diag_history_opts) { + .scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT, + .ctx_kind = ctx_kind, + .ctx_depth = depth, + }; + constraint = diag_context_constraint(ctx_kind); + context = diag_context_name(ctx_kind); + + bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY, + "operation is not allowed in this context"); + if (constraint) { + if (depth) { + diag_reason( + env, "The operation %s cannot be used in %s because %s. This path is still inside %s.", + operation, context, constraint, diag_active_context(env, depth, context)); + } else { + diag_reason(env, "The operation %s cannot be used in %s because %s.", + operation, context, constraint); + } + } else { + diag_reason(env, "The operation %s cannot be used in %s.", operation, + context); + } + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s is not allowed in %s", operation, + context); + + if (ctx_kind != BPF_DIAG_CONTEXT_NONE) + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_ctx_active(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + enum bpf_diag_context_kind ctx_kind, const char *suggestion) +{ + u32 depth = diag_context_depth(env, ctx_kind); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT, + .ctx_kind = ctx_kind, + .ctx_depth = depth, + }; + const char *context = diag_context_name(ctx_kind); + + bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY, + "operation is not allowed in this context"); + diag_reason( + env, "The operation %s cannot be used while this path is still inside %s. Leave the region before this operation.", + operation, diag_active_context(env, depth, context)); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s is not allowed before leaving %s", + operation, context); + + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_ctx_required(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + enum bpf_diag_context_kind ctx_kind, const char *suggestion) +{ + const char *context = diag_context_name(ctx_kind); + + bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY, "required context is not active"); + diag_reason(env, "The operation %s requires an active %s, but this path is outside one.", + operation, context); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s requires %s", operation, context); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_ctx_underflow(struct bpf_verifier_env *env, u32 insn_idx, + const char *operation, enum bpf_diag_context_kind ctx_kind, + const char *suggestion) +{ + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT, + .ctx_kind = ctx_kind, + }; + const char *context = diag_context_name(ctx_kind); + + bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY, "unmatched context exit"); + diag_reason( + env, "The operation %s tries to leave %s, but this path has no active %s to leave. The current depth is 0.", + operation, context, context); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s has no matching enter on this path", + operation); + + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_program_structure(struct bpf_verifier_env *env, u32 insn_idx, + const char *problem, const char *suggestion, + const char *reason_fmt, ...) +{ + va_list args; + + bpf_diag_header(env, PROGRAM_STRUCTURE, problem); + diag_section(env, "Reason"); + + va_start(args, reason_fmt); + diag_vprint_indented(env, reason_fmt, args); + va_end(args); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_policy(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + const char *reason, const char *suggestion) +{ + bpf_diag_header(env, POLICY, "operation is not allowed"); + diag_reason(env, "The %s is not allowed: %s.", operation, reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "policy check failed for %s", operation); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_invalid_deref(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *reg_name, const struct bpf_reg_state *reg, + enum bpf_diag_invalid_deref_kind kind, s64 offset) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .regno = regno, + }; + const char *type_name = bpf_diag_reg_type_plain(env, reg->type); + + bpf_diag_header(env, REGISTER_TYPE_SAFETY, "invalid dereference"); + + switch (kind) { + case BPF_DIAG_DEREF_SCALAR: + diag_reason(env, "%s is an integer scalar here, not a pointer to memory.", + reg_name); + break; + case BPF_DIAG_DEREF_NULLABLE_PTR: + diag_reason( + env, "%s may be NULL here (%s). The program could dereference NULL on this path, so the verifier cannot prove this access is safe.", + reg_name, type_name); + break; + case BPF_DIAG_DEREF_MODIFIED_PTR: + diag_reason( + env, "%s has offset %lld here, but this pointer type must be dereferenced in its original form.", + reg_name, offset); + break; + case BPF_DIAG_DEREF_INVALID_PTR: + default: + diag_reason( + env, "%s has type %s here, which is not valid for this memory access.", + reg_name, type_name); + break; + } + + diag_section(env, "At"); + if (kind == BPF_DIAG_DEREF_MODIFIED_PTR) + bpf_diag_source(env, insn_idx, "error", + "dereference requires the original %s pointer", type_name); + else + bpf_diag_source(env, insn_idx, "error", "invalid dereference of %s (%s)", + reg_name, type_name); + + if (regno >= 0) + diag_print_history(env, &opts); + + switch (kind) { + case BPF_DIAG_DEREF_NULLABLE_PTR: + diag_suggestion( + env, "Add a NULL check before the access and dereference the pointer only on the non-NULL path."); + break; + case BPF_DIAG_DEREF_MODIFIED_PTR: + diag_suggestion( + env, "Preserve the original pointer in another register, or use only offsets this pointer type permits before dereferencing it."); + break; + case BPF_DIAG_DEREF_SCALAR: + case BPF_DIAG_DEREF_INVALID_PTR: + default: + diag_suggestion( + env, "Preserve a pointer-valued register where needed, or reload and revalidate the pointer after scalar arithmetic, helper calls, or other operations that can invalidate it."); + break; + } +} + +void bpf_diag_unreadable_reg(struct bpf_verifier_env *env, u32 insn_idx, int regno) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .regno = regno, + }; + const struct bpf_diag_log *log = env->diag ? &env->diag->log : NULL; + struct bpf_diag_mod_target target; + bool invalidated = false; + int i; + + target = diag_reg_target(opts.frame_id, opts.frameno, regno); + for (i = log ? log->cnt : 0; i > 0; i--) { + const struct bpf_diag_history_event *event; + + event = &log->events[log_pos(log, i - 1)]; + + if (event->kind != BPF_DIAG_HISTORY_MOD || + !diag_target_matches(&event->mod.target, &target)) + continue; + invalidated = event->mod.new.type == NOT_INIT; + break; + } + + bpf_diag_header(env, REGISTER_TYPE_SAFETY, "unreadable register"); + if (invalidated) + diag_reason( + env, "R%d is not readable here. A previous operation invalidated this register, so the verifier cannot use it as an input.", + regno); + else if (log && !log->first_seq) + diag_reason(env, + "R%d has never been initialized on this path, so the verifier cannot use it as an input.", + regno); + else + diag_reason( + env, "R%d is not readable here. It may never have been initialized, or an earlier operation may have invalidated it.", + regno); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "R%d is not readable", regno); + + if (regno >= 0) + diag_print_history(env, &opts); + + if (invalidated) + diag_suggestion( + env, "Avoid using the register after it is invalidated, or initialize it again before this instruction."); + else if (log && !log->first_seq) + diag_suggestion(env, "Initialize R%d on every path before this instruction.", regno); + else + diag_suggestion( + env, "Initialize the register on every path, or initialize it again after any operation that invalidates it."); +} + +static int diag_stack_argno(u8 slot) +{ + return MAX_BPF_FUNC_REG_ARGS + slot + 1; +} + +static void diag_format_stack_arg(char *buf, size_t size, u8 slot, const char *arg_name) +{ + int argno = diag_stack_argno(slot); + const char *ordinal = diag_arg_ordinal(argno); + + if (ordinal && arg_name) + scnprintf(buf, size, "outgoing stack argument %u (%s argument, %s)", slot + 1, + ordinal, arg_name); + else if (ordinal) + scnprintf(buf, size, "outgoing stack argument %u (%s argument)", slot + 1, ordinal); + else if (arg_name) + scnprintf(buf, size, "outgoing stack argument %u (%s)", slot + 1, arg_name); + else + scnprintf(buf, size, "outgoing stack argument %u", slot + 1); +} + +void bpf_diag_stack_arg_uninit(struct bpf_verifier_env *env, u32 insn_idx, int nargs, + int stack_arg_slot, const char *callee_name, + const char *arg_name) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_STACK_ARG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .stack_arg_slot = stack_arg_slot, + }; + const char *arg_buf; + + arg_buf = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); + if (arg_buf) + diag_format_stack_arg((char *)arg_buf, BPF_DIAG_FMT_BUF_SIZE, stack_arg_slot, + arg_name); + else + arg_buf = ""; + bpf_diag_header(env, REGISTER_TYPE_SAFETY, "missing stack argument"); + if (callee_name && *callee_name) + diag_reason( + env, "Function %s expects %d arguments, but %s is not initialized at this call.", + callee_name, nargs, arg_buf); + else + diag_reason( + env, "The callee expects %d arguments, but %s is not initialized at this call.", + nargs, arg_buf); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s is not initialized", arg_buf); + + if (stack_arg_slot >= 0) + diag_print_history(env, &opts); + + diag_suggestion( + env, "Write the outgoing stack argument after any operation that may invalidate stored pointer values, and before making this call."); +} + +void bpf_diag_memory(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion) +{ + bpf_diag_header(env, MEMORY_SAFETY, problem); + diag_reason(env, "%s", reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true) +{ + struct bpf_diag_history_event event = { + .insn_idx = insn_idx, + .kind = BPF_DIAG_HISTORY_BRANCH, + .branch = { + .cond_true = cond_true, + }, + }; + + diag_append_history(env, &event); +} + +static void diag_snapshot_reg(struct bpf_diag_reg_snapshot *snapshot, + const struct bpf_reg_state *reg) +{ + snapshot->type = reg->type; + if (type_is_map_ptr(reg->type)) + snapshot->map_ptr = reg->map_ptr; + if (base_type(reg->type) == PTR_TO_BTF_ID && reg->btf && reg->btf_id) { + snapshot->btf_id = reg->btf_id; + snapshot->btf = reg->btf; + } + snapshot->var_off = reg->var_off; + snapshot->r64 = reg->r64; +} + +static bool diag_mod_insn_origin(struct bpf_verifier_env *env, u32 insn_idx, + const struct bpf_diag_mod_target *target, + struct bpf_diag_mod_target *origin) +{ + const struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; + u8 class = BPF_CLASS(insn->code); + const struct bpf_func_state *state; + + if (target->kind == BPF_DIAG_MOD_TARGET_REG && (class == BPF_ALU || class == BPF_ALU64) && + BPF_OP(insn->code) == BPF_MOV && BPF_SRC(insn->code) == BPF_X) { + *origin = diag_reg_target(target->frame_id, target->frameno, insn->src_reg); + return true; + } + + if ((target->kind != BPF_DIAG_MOD_TARGET_STACK_ARG && + target->kind != BPF_DIAG_MOD_TARGET_STACK_SLOT) || + class != BPF_STX) + return false; + + state = env->cur_state->frame[env->cur_state->curframe]; + *origin = diag_reg_target(state->diag_frame_id, state->frameno, insn->src_reg); + return true; +} + +static bool diag_mod_keeps_lineage(struct bpf_verifier_env *env, + const struct bpf_diag_history_event *event) +{ + const struct bpf_insn *insn; + u8 class; + + if (event->mod.reason != BPF_DIAG_MOD_WRITE || + event->mod.target.kind != BPF_DIAG_MOD_TARGET_REG) + return false; + + insn = &env->prog->insnsi[event->insn_idx]; + class = BPF_CLASS(insn->code); + if (class != BPF_ALU && class != BPF_ALU64) + return false; + + switch (BPF_OP(insn->code)) { + case BPF_ADD: + case BPF_SUB: + case BPF_MUL: + case BPF_OR: + case BPF_AND: + case BPF_LSH: + case BPF_RSH: + case BPF_ARSH: + case BPF_XOR: + case BPF_NEG: + case BPF_END: + return true; + default: + return false; + } +} + +static void diag_record_mod(struct bpf_verifier_env *env, u32 insn_idx, + struct bpf_diag_mod_target target, + enum bpf_diag_mod_reason reason, + const struct bpf_reg_state *old_reg, + const struct bpf_reg_state *new_reg, + const struct bpf_diag_mod_target *origin) +{ + struct bpf_diag_history_event event = { + .insn_idx = insn_idx, + .kind = BPF_DIAG_HISTORY_MOD, + .mod = { + .target = target, + .reason = reason, + }, + }; + + if (old_reg) + diag_snapshot_reg(&event.mod.old, old_reg); + if (new_reg) + diag_snapshot_reg(&event.mod.new, new_reg); + if (origin) { + event.mod.origin = *origin; + event.mod.origin_valid = true; + } else if (diag_mod_insn_origin(env, insn_idx, &target, &event.mod.origin)) { + event.mod.origin_valid = true; + } + if (old_reg && new_reg && + (reason == BPF_DIAG_MOD_WRITE || reason == BPF_DIAG_MOD_SPILL) && + !memcmp(&event.mod.old, &event.mod.new, sizeof(event.mod.old)) && + !event.mod.origin_valid && + diag_mod_keeps_lineage(env, &event)) + return; + + diag_append_history(env, &event); +} + +static struct bpf_reg_state *target_to_reg(struct bpf_verifier_env *env, + const struct bpf_diag_mod_target *target) +{ + struct bpf_verifier_state *vstate = env->cur_state; + struct bpf_func_state *state; + + state = target->frameno <= vstate->curframe ? vstate->frame[target->frameno] : NULL; + + if (!state) + return NULL; + if (state->diag_frame_id != target->frame_id) + return NULL; + + switch (target->kind) { + case BPF_DIAG_MOD_TARGET_REG: + if (target->regno >= MAX_BPF_REG) + return NULL; + return &state->regs[target->regno]; + case BPF_DIAG_MOD_TARGET_STACK_ARG: + if (target->stack_arg >= state->out_stack_arg_cnt) + return NULL; + return &state->stack_arg_regs[target->stack_arg]; + case BPF_DIAG_MOD_TARGET_STACK_SLOT: + if (target->spi >= state->allocated_stack / BPF_REG_SIZE) + return NULL; + return &state->stack[target->spi].spilled_ptr; + default: + return NULL; + } +} + +static bool reg_to_target(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + struct bpf_diag_mod_target *target) +{ + struct bpf_verifier_state *vstate = env->cur_state; + unsigned long addr = (unsigned long)reg; + int frame; + + for (frame = 0; frame <= vstate->curframe; frame++) { + struct bpf_func_state *state = vstate->frame[frame]; + unsigned long start, end; + u32 nslots = state->allocated_stack / BPF_REG_SIZE; + int spi; + + start = (unsigned long)state->regs; + end = (unsigned long)(state->regs + MAX_BPF_REG); + if (addr >= start && addr < end) { + *target = diag_reg_target(state->diag_frame_id, state->frameno, + reg - state->regs); + return true; + } + + start = (unsigned long)state->stack_arg_regs; + end = (unsigned long)(state->stack_arg_regs + state->out_stack_arg_cnt); + if (state->out_stack_arg_cnt && addr >= start && addr < end) { + *target = diag_stack_arg_target(state->diag_frame_id, state->frameno, + reg - state->stack_arg_regs); + return true; + } + + start = (unsigned long)state->stack; + end = (unsigned long)(state->stack + nslots); + if (nslots && addr >= start && addr < end) { + spi = ((const char *)reg - (const char *)state->stack) / + sizeof(*state->stack); + *target = diag_stack_slot_target(state->diag_frame_id, state->frameno, spi); + return true; + } + } + return false; +} + +void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason) +{ + struct bpf_diag *diag = env->diag; + + if (!diag) + return; + diag->mod.active = reg_to_target(env, reg, &diag->mod.target); + if (!diag->mod.active) + return; + diag->mod.target_reg_snapshot = *reg; + diag->mod.insn_idx = env->insn_idx; + diag->mod.reason = reason; + diag->mod.origin_valid = origin && reg_to_target(env, origin, &diag->mod.origin); +} + +void bpf_diag_mod_end(struct bpf_verifier_env *env) +{ + struct bpf_diag *diag = env->diag; + const struct bpf_reg_state *new_reg; + + if (!diag || !diag->mod.active) + return; + diag->mod.active = false; + /* + * Resolve the target again because the enclosing function state's stack + * may have been reallocated while the modification was in progress. + */ + new_reg = target_to_reg(env, &diag->mod.target); + if (!new_reg) + return; + diag_record_mod(env, diag->mod.insn_idx, diag->mod.target, diag->mod.reason, + &diag->mod.target_reg_snapshot, new_reg, + diag->mod.origin_valid ? &diag->mod.origin : NULL); +} + +void bpf_diag_record_scrub(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + enum bpf_diag_mod_reason reason) +{ + struct bpf_diag_mod_target target; + + if (!env->diag || reg->type == NOT_INIT || !reg_to_target(env, reg, &target)) + return; + diag_record_mod(env, env->insn_idx, target, reason, reg, NULL, NULL); +} + +void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env, + const struct bpf_func_state *state, s16 min_off, s16 max_off, + enum bpf_diag_mod_reason reason) +{ + diag_record_mod(env, env->insn_idx, + diag_stack_range_target(state->diag_frame_id, state->frameno, min_off, max_off), + reason, NULL, NULL, NULL); +} + +static void diag_record_ref(struct bpf_verifier_env *env, u32 insn_idx, u8 kind, u32 ref_id) +{ + struct bpf_diag_history_event event = { + .insn_idx = insn_idx, + .kind = kind, + .ref = { + .ref_id = ref_id, + }, + }; + + diag_append_history(env, &event); +} + +void bpf_diag_record_ref_acquire(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id) +{ + diag_record_ref(env, insn_idx, BPF_DIAG_HISTORY_REF_ACQUIRE, ref_id); +} + +void bpf_diag_record_ref_release(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id) +{ + diag_record_ref(env, insn_idx, BPF_DIAG_HISTORY_REF_RELEASE, ref_id); +} + +void bpf_diag_record_context(struct bpf_verifier_env *env, u32 insn_idx, + enum bpf_diag_context_kind ctx_kind, bool enter, u32 depth) +{ + /* + * Keep leave events so context rendering can stop at a depth-zero exit + * and show nested-region depth accurately for the active path. + */ + struct bpf_diag_history_event event = { + .insn_idx = insn_idx, + .kind = BPF_DIAG_HISTORY_CONTEXT, + .ctx = { + .kind = ctx_kind, + .enter = enter, + .depth = depth, + }, + }; + + diag_append_history(env, &event); +} + +static int diag_history_context_start_idx(const struct bpf_diag_log *log, + const struct bpf_diag_history_opts *opts) +{ + int i; + + if (!opts->ctx_depth) + return 0; + + /* Find the most recent outermost entry, or a depth-zero exit. */ + for (i = log->cnt; i > 0; i--) { + const struct bpf_diag_history_event *event; + + event = &log->events[log_pos(log, i - 1)]; + + if (event->kind != BPF_DIAG_HISTORY_CONTEXT || event->ctx.kind != opts->ctx_kind) + continue; + + if (event->ctx.enter && event->ctx.depth == 1) + return i - 1; + if (!event->ctx.enter && event->ctx.depth == 0) + return 0; + } + + return 0; +} + +struct bpf_diag_history_filter { + const struct bpf_diag_history_opts *opts; + u32 lineage_start; + bool lineage_valid; +}; + +static bool diag_target_matches(const struct bpf_diag_mod_target *event_target, + const struct bpf_diag_mod_target *target) +{ + int slot_off; + + if (event_target->frame_id != target->frame_id || event_target->frameno != target->frameno) + return false; + + if (event_target->kind == BPF_DIAG_MOD_TARGET_STACK_RANGE && + target->kind == BPF_DIAG_MOD_TARGET_STACK_SLOT) { + slot_off = -(target->spi + 1) * BPF_REG_SIZE; + return event_target->range.min_off < slot_off + BPF_REG_SIZE && + event_target->range.max_off > slot_off; + } + + if (event_target->kind != target->kind) + return false; + + switch (target->kind) { + case BPF_DIAG_MOD_TARGET_REG: + return event_target->regno == target->regno; + case BPF_DIAG_MOD_TARGET_STACK_ARG: + return event_target->stack_arg == target->stack_arg; + case BPF_DIAG_MOD_TARGET_STACK_SLOT: + return event_target->spi == target->spi; + default: + return false; + } +} + +static void diag_build_lineage(struct bpf_verifier_env *env, struct bpf_diag_log *log, + struct bpf_diag_history_filter *filter) +{ + const struct bpf_diag_history_opts *opts = filter->opts; + struct bpf_diag_mod_target target; + int i; + + for (i = 0; i < log->cnt; i++) + log->events[log_pos(log, i)].in_lineage = false; + + if (opts->scope == BPF_DIAG_HISTORY_SCOPE_REG) + target = diag_reg_target(opts->frame_id, opts->frameno, opts->regno); + else if (opts->scope == BPF_DIAG_HISTORY_SCOPE_STACK_ARG) + target = diag_stack_arg_target(opts->frame_id, opts->frameno, + opts->stack_arg_slot); + else + return; + + /* + * Find the nearest mutation of the active target. A fill or spill changes + * the target to its origin, so the same walk follows register/stack + * lineage recursively until it reaches the write that created the value. + */ + for (i = log->cnt; i > 0; i--) { + struct bpf_diag_history_event *event; + + event = &log->events[log_pos(log, i - 1)]; + if (event->kind != BPF_DIAG_HISTORY_MOD || + !diag_target_matches(&event->mod.target, &target)) + continue; + + event->in_lineage = true; + filter->lineage_start = i - 1; + filter->lineage_valid = true; + + if (event->mod.origin_valid) { + target = event->mod.origin; + continue; + } + if (event->mod.reason != BPF_DIAG_MOD_WRITE && + event->mod.reason != BPF_DIAG_MOD_SPILL) + continue; + if (diag_mod_keeps_lineage(env, event)) + continue; + break; + } +} + +static int diag_history_start_idx(const struct bpf_diag_log *log, + const struct bpf_diag_history_filter *filter) +{ + const struct bpf_diag_history_opts *opts = filter->opts; + int i; + + if (opts->scope == BPF_DIAG_HISTORY_SCOPE_CONTEXT) + return diag_history_context_start_idx(log, opts); + if (filter->lineage_valid) + return filter->lineage_start; + if (opts->scope != BPF_DIAG_HISTORY_SCOPE_REF) + return 0; + + for (i = log->cnt; i > 0; i--) { + const struct bpf_diag_history_event *event; + + event = &log->events[log_pos(log, i - 1)]; + if (event->kind == BPF_DIAG_HISTORY_REF_ACQUIRE && + event->ref.ref_id == opts->ref_id) + return i - 1; + } + + return 0; +} + +static bool diag_history_event_visible(const struct bpf_diag_history_event *event, + const struct bpf_diag_history_filter *filter) +{ + const struct bpf_diag_history_opts *opts = filter->opts; + + switch (event->kind) { + case BPF_DIAG_HISTORY_BRANCH: + return true; + case BPF_DIAG_HISTORY_MOD: + return filter->lineage_valid && event->in_lineage; + case BPF_DIAG_HISTORY_REF_ACQUIRE: + case BPF_DIAG_HISTORY_REF_RELEASE: + return opts->scope == BPF_DIAG_HISTORY_SCOPE_REF && + event->ref.ref_id == opts->ref_id; + case BPF_DIAG_HISTORY_CONTEXT: + return opts->scope == BPF_DIAG_HISTORY_SCOPE_CONTEXT && + event->ctx.kind == opts->ctx_kind; + default: + return false; + } +} + +static const char *diag_s64_bound_name(s64 value) +{ + if (value == S64_MIN) + return "S64_MIN"; + if (value == S64_MAX) + return "S64_MAX"; + return NULL; +} + +static const char *diag_u64_bound_name(u64 value) +{ + if (value == U64_MAX) + return "U64_MAX"; + return NULL; +} + +static const char *diag_s64_str(struct bpf_verifier_env *env, s64 value) +{ + return diag_s64_bound_name(value) ?: bpf_diag_fmt(env, "%lld", value); +} + +static const char *diag_u64_str(struct bpf_verifier_env *env, u64 value) +{ + return diag_u64_bound_name(value) ?: bpf_diag_fmt(env, "%llu", value); +} + +static bool diag_cnum64_unknown(struct cnum64 range) +{ + return cnum64_smin(range) == S64_MIN && cnum64_smax(range) == S64_MAX && + cnum64_umin(range) == 0 && cnum64_umax(range) == U64_MAX; +} + +static bool diag_snapshot_unknown(const struct bpf_diag_reg_snapshot *snapshot) +{ + return tnum_is_unknown(snapshot->var_off) && diag_cnum64_unknown(snapshot->r64); +} + +static const char *diag_scalar_range(struct bpf_verifier_env *env, struct cnum64 range) +{ + return bpf_diag_fmt(env, "signed range [%s, %s], unsigned range [%s, %s]", + diag_s64_str(env, cnum64_smin(range)), + diag_s64_str(env, cnum64_smax(range)), + diag_u64_str(env, cnum64_umin(range)), + diag_u64_str(env, cnum64_umax(range))); +} + +const char *bpf_diag_fmt_s64_sum(struct bpf_verifier_env *env, s64 value, int addend) +{ + s64 sum; + + if (check_add_overflow(value, (s64)addend, &sum)) + return bpf_diag_fmt(env, "%lld plus %d (%s)", value, addend, + addend < 0 ? "below S64_MIN" : "above S64_MAX"); + + return bpf_diag_fmt(env, "%lld", sum); +} + +static const char *diag_access_offset(struct bpf_verifier_env *env, int off, + const struct bpf_reg_state *reg) +{ + if (tnum_is_const(reg->var_off)) + return bpf_diag_fmt(env, "constant %s", + bpf_diag_fmt_s64_sum(env, (s64)reg->var_off.value, off)); + + if (tnum_is_unknown(reg->var_off) && diag_cnum64_unknown(reg->r64)) + return bpf_diag_fmt(env, "unbounded"); + + if (off) + return bpf_diag_fmt(env, + "variable: known bits %#llx, unknown mask %#llx, plus fixed offset %d; %s", + (u64)reg->var_off.value, reg->var_off.mask, off, + diag_scalar_range(env, reg->r64)); + return bpf_diag_fmt(env, "variable: known bits %#llx, unknown mask %#llx; %s", + (u64)reg->var_off.value, reg->var_off.mask, + diag_scalar_range(env, reg->r64)); +} + +void bpf_diag_mem_bounds(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *reg_name, const char *type_name, const char *proof, + int off, int size, u32 mem_size, const struct bpf_reg_state *reg) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .regno = regno, + }; + const char *offset_desc; + + if (!bpf_diag_enabled(env)) + return; + + offset_desc = diag_access_offset(env, off, reg); + + bpf_diag_header(env, MEMORY_SAFETY, "access outside bounds"); + diag_reason( + env, "The verifier cannot prove offset + access_size <= object_size. Here, %s. %s is %s; offset is %s; access_size is %d; object_size is %u.", + proof, reg_name, type_name, offset_desc, size, mem_size); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "access may be outside object bounds"); + + if (regno >= 0) + diag_print_history(env, &opts); + + diag_suggestion( + env, "Add or adjust a bounds check that proves offset + access_size stays within the object."); +} + +static const char *diag_lock_name(const struct bpf_reference_state *lock) +{ + switch (lock->type) { + case REF_TYPE_LOCK: + return "bpf_spin_lock"; + case REF_TYPE_RES_LOCK: + return "resource spin lock"; + case REF_TYPE_RES_LOCK_IRQ: + return "IRQ-saving resource spin lock"; + default: + return "lock"; + } +} + +static void diag_res_report(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason) +{ + bpf_diag_header(env, RESOURCE_LIFETIME_SAFETY, problem); + diag_reason(env, "%s", reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); +} + +void bpf_diag_res(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion) +{ + diag_res_report(env, insn_idx, problem, reason); + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_lock(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion, + const struct bpf_reference_state *active_lock) +{ + diag_res_report(env, insn_idx, problem, reason); + + if (active_lock) { + diag_section(env, "Active lock"); + bpf_diag_source(env, active_lock->insn_idx, "acquired", + "active %s has verifier identity %d", + diag_lock_name(active_lock), active_lock->id); + } + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_irq(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion, u32 depth) +{ + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT, + .ctx_kind = BPF_DIAG_CONTEXT_IRQ, + .ctx_depth = depth, + }; + + bpf_diag_header(env, RESOURCE_LIFETIME_SAFETY, problem); + diag_reason(env, "%s", reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); + + if (depth) + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_leak(struct bpf_verifier_env *env, u32 ref_id, u32 alloc_insn, u32 fail_insn) +{ + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REF, + .ref_id = ref_id, + }; + + bpf_diag_header(env, RESOURCE_LIFETIME_SAFETY, "unreleased resource"); + diag_reason( + env, "Owned resource (id=%u) was acquired at instruction %u and still needs to be released before this exit path.", + ref_id, alloc_insn); + + diag_section(env, "At"); + bpf_diag_source(env, fail_insn, "error", + "owned resource (id=%u) still needs release", ref_id); + + diag_print_history(env, &opts); + + diag_suggestion( + env, "Release or transfer ownership of the acquired resource on every path before the program exits."); +} + +static const char *diag_var_offset(struct bpf_verifier_env *env, + const struct bpf_diag_reg_snapshot *snapshot) +{ + if (tnum_is_const(snapshot->var_off)) + return bpf_diag_fmt(env, "at offset %lld", (s64)snapshot->var_off.value); + + if (diag_snapshot_unknown(snapshot)) + return bpf_diag_fmt(env, "with unknown offset"); + + return bpf_diag_fmt(env, + "with variable offset: known bits %#llx, unknown mask %#llx, %s", + snapshot->var_off.value, snapshot->var_off.mask, + diag_scalar_range(env, snapshot->r64)); +} + +static const char *diag_reg_map_name(const struct bpf_map *map) +{ + if (!map || !map->name[0]) + return NULL; + + return map->name; +} + +static const char *diag_reg_snapshot(struct bpf_verifier_env *env, + const struct bpf_diag_reg_snapshot *snapshot) +{ + const char *type_name = reg_type_str(env, snapshot->type); + const char *offset = diag_var_offset(env, snapshot); + const char *btf = snapshot->btf && snapshot->btf_id ? + bpf_diag_fmt_btf_type(env, snapshot->btf, snapshot->btf_id) : NULL; + const char *map_name; + + if (snapshot->type == SCALAR_VALUE) { + if (tnum_is_const(snapshot->var_off)) + return bpf_diag_fmt(env, "integer scalar value %lld", + (s64)snapshot->var_off.value); + if (diag_snapshot_unknown(snapshot)) + return bpf_diag_fmt(env, "integer scalar with unknown value"); + if (cnum64_is_const(snapshot->r64)) + return bpf_diag_fmt(env, "integer scalar value %lld", + cnum64_smin(snapshot->r64)); + return bpf_diag_fmt(env, "integer scalar with %s", + diag_scalar_range(env, snapshot->r64)); + } + + if (snapshot->type == NOT_INIT) + return bpf_diag_fmt(env, "uninitialized value"); + + if (base_type(snapshot->type) == PTR_TO_CTX) + return bpf_diag_fmt(env, "context pointer %s", offset); + + if (base_type(snapshot->type) == PTR_TO_STACK) + return bpf_diag_fmt(env, "stack pointer %s", offset); + + if (base_type(snapshot->type) == PTR_TO_MAP_VALUE) { + const char *kind = type_may_be_null(snapshot->type) ? "nullable map value" : + "map value"; + + map_name = diag_reg_map_name(snapshot->map_ptr); + if (map_name) + return bpf_diag_fmt(env, "%s from %s %s", kind, map_name, offset); + return bpf_diag_fmt(env, "%s %s", kind, offset); + } + + if (base_type(snapshot->type) == CONST_PTR_TO_MAP) { + map_name = diag_reg_map_name(snapshot->map_ptr); + if (map_name) + return bpf_diag_fmt(env, "map pointer for map %s", map_name); + return bpf_diag_fmt(env, "map pointer"); + } + + if (type_is_non_owning_ref(snapshot->type)) { + if (btf) + return bpf_diag_fmt(env, "borrowed allocated object pointer type=%s", btf); + return bpf_diag_fmt(env, "borrowed allocated object pointer"); + } + + if (type_is_ptr_alloc_obj(snapshot->type)) { + if (btf) + return bpf_diag_fmt(env, "owned allocated object pointer type=%s", btf); + return bpf_diag_fmt(env, "owned allocated object pointer"); + } + + if (base_type(snapshot->type) == PTR_TO_BTF_ID && btf) + return bpf_diag_fmt(env, "%s type=%s %s", type_name, btf, offset); + + return bpf_diag_fmt(env, "%s %s", type_name, offset); +} + +static const char *diag_mod_target_desc(struct bpf_verifier_env *env, + const struct bpf_diag_mod_target *target) +{ + switch (target->kind) { + case BPF_DIAG_MOD_TARGET_REG: + return bpf_diag_fmt(env, "R%u", target->regno); + case BPF_DIAG_MOD_TARGET_STACK_ARG: + return bpf_diag_fmt(env, "*(R11-%u)", (target->stack_arg + 1) * BPF_REG_SIZE); + case BPF_DIAG_MOD_TARGET_STACK_SLOT: + return bpf_diag_fmt(env, "stack slot fp%d", -(target->spi + 1) * BPF_REG_SIZE); + default: + return "value"; + } +} + +static void diag_print_mod(struct bpf_verifier_env *env, const struct bpf_diag_history_event *event) +{ + const struct bpf_diag_mod_target *target = &event->mod.target; + const char *target_desc, *reason = NULL, *old, *new; + const char *label = "update"; + + if (target->kind == BPF_DIAG_MOD_TARGET_STACK_RANGE) { + bpf_diag_source( + env, event->insn_idx, "invalidated", + "variable-offset stack write may affect bytes fp%d through fp%d", + target->range.min_off, target->range.max_off - 1); + return; + } + + old = diag_reg_snapshot(env, &event->mod.old); + new = diag_reg_snapshot(env, &event->mod.new); + target_desc = diag_mod_target_desc(env, target); + + switch (event->mod.reason) { + case BPF_DIAG_MOD_REF_RELEASE: + reason = target->kind == BPF_DIAG_MOD_TARGET_REG ? "resource release invalidated " + "this pointer" : + "resource release invalidated " + "this value"; + break; + case BPF_DIAG_MOD_PKT_DATA_CHANGE: + reason = "packet data may have moved"; + break; + case BPF_DIAG_MOD_NON_OWN_REF: + reason = "leaving the protected region invalidated this borrowed pointer"; + break; + case BPF_DIAG_MOD_CALLER_SAVED: + reason = target->kind == BPF_DIAG_MOD_TARGET_STACK_ARG ? + "call invalidated this outgoing stack argument" : + "call invalidated this caller-saved register"; + break; + case BPF_DIAG_MOD_WRITE: + if (target->kind == BPF_DIAG_MOD_TARGET_STACK_SLOT) + reason = "a later stack write overwrote this spilled value"; + break; + case BPF_DIAG_MOD_SPILL: + label = "spilled"; + break; + case BPF_DIAG_MOD_VAR_WRITE: + default: + break; + } + + if (reason) { + bpf_diag_source(env, event->insn_idx, "invalidated", + "%s: %s; previous value was %s", target_desc, reason, old); + return; + } + + bpf_diag_source(env, event->insn_idx, label, "%s changed from %s to %s", target_desc, + old, new); +} + +static void diag_print_ref_event(struct bpf_verifier_env *env, + const struct bpf_diag_history_event *event) +{ + const char *label; + + label = event->kind == BPF_DIAG_HISTORY_REF_ACQUIRE ? "acquired" : "released"; + bpf_diag_source(env, event->insn_idx, label, "owned resource (id=%u)", + event->ref.ref_id); +} + +static const char *diag_context_name(enum bpf_diag_context_kind kind) +{ + switch (kind) { + case BPF_DIAG_CONTEXT_RCU: + return "RCU read lock region"; + case BPF_DIAG_CONTEXT_PREEMPT: + return "non-preemptible region"; + case BPF_DIAG_CONTEXT_IRQ: + return "IRQ-disabled region"; + case BPF_DIAG_CONTEXT_LOCK: + return "lock region"; + case BPF_DIAG_CONTEXT_NONE: + default: + return "non-sleepable program"; + } +} + +static void diag_print_context_event(struct bpf_verifier_env *env, + const struct bpf_diag_history_event *event) +{ + bpf_diag_source(env, event->insn_idx, "context", "%s %s; depth is now %u", + event->ctx.enter ? "entered" : "left", + diag_context_name(event->ctx.kind), event->ctx.depth); +} + +static void diag_print_history(struct bpf_verifier_env *env, + const struct bpf_diag_history_opts *opts) +{ + const struct bpf_diag_history_event *event; + struct bpf_diag_history_filter filter = { + .opts = opts, + }; + struct bpf_diag_log *log; + struct diag_fmt_mark mark; + bool first = true; + int start_idx; + u32 i, visible_cnt = 0, visible_idx = 0; + + if (!bpf_diag_enabled(env)) + return; + + if (!env->diag) + return; + log = &env->diag->log; + + diag_build_lineage(env, log, &filter); + + start_idx = diag_history_start_idx(log, &filter); + for (i = start_idx; i < log->cnt; i++) { + event = &log->events[log_pos(log, i)]; + if (diag_history_event_visible(event, &filter)) + visible_cnt++; + } + + if (!visible_cnt && !log->first_seq && opts->scope == BPF_DIAG_HISTORY_SCOPE_STACK_ARG) + return; + + diag_section(env, "Causal path"); + mark = diag_fmt_save(env); + for (i = start_idx; i < log->cnt; i++) { + event = &log->events[log_pos(log, i)]; + if (!diag_history_event_visible(event, &filter)) + continue; + + diag_fmt_restore(env, mark); + if (visible_cnt > BPF_DIAG_HISTORY_RENDER_MAX && + visible_idx >= BPF_DIAG_HISTORY_RENDER_MAX / 2 && + visible_idx < visible_cnt - BPF_DIAG_HISTORY_RENDER_MAX / 2) { + if (visible_idx++ != BPF_DIAG_HISTORY_RENDER_MAX / 2) + continue; + if (!first) + diag_write(env, "\n"); + first = false; + diag_write(env, " %u intermediate causal-history events omitted\n", + visible_cnt - BPF_DIAG_HISTORY_RENDER_MAX); + continue; + } + visible_idx++; + + if (!first) + diag_write(env, "\n"); + first = false; + + switch (event->kind) { + case BPF_DIAG_HISTORY_BRANCH: + bpf_diag_source(env, event->insn_idx, "branch", + "took the %s branch of this conditional, goto %s", + event->branch.cond_true ? "true" : "false", + event->branch.cond_true ? "followed" : "not followed"); + break; + case BPF_DIAG_HISTORY_MOD: + diag_print_mod(env, event); + break; + case BPF_DIAG_HISTORY_REF_ACQUIRE: + case BPF_DIAG_HISTORY_REF_RELEASE: + diag_print_ref_event(env, event); + break; + case BPF_DIAG_HISTORY_CONTEXT: + diag_print_context_event(env, event); + break; + default: + break; + } + } + + if (!visible_cnt) + diag_write(env, " no retained diagnostic events on this path\n"); + if (log->first_seq) + diag_write(env, " %llu older causal-history event%s not retained because diagnostic " + "event storage reached capacity\n", + log->first_seq, log->first_seq == 1 ? "" : "s"); + diag_fmt_restore(env, mark); +} diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h new file mode 100644 index 000000000000..d1b79945008a --- /dev/null +++ b/kernel/bpf/diagnostics.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#ifndef __BPF_DIAGNOSTICS_H +#define __BPF_DIAGNOSTICS_H + +#include <linux/bpf.h> +#include <linux/compiler_attributes.h> +#include <linux/stdarg.h> +#include <linux/types.h> + +struct bpf_reference_state; +struct bpf_func_state; +struct bpf_reg_state; +struct bpf_verifier_env; +struct bpf_verifier_state; +struct btf; + +const char *bpf_diag_fmt_s64_sum(struct bpf_verifier_env *env, s64 value, int addend); +enum bpf_diag_mod_reason { + BPF_DIAG_MOD_WRITE, + BPF_DIAG_MOD_SPILL, + BPF_DIAG_MOD_VAR_WRITE, + BPF_DIAG_MOD_REF_RELEASE, + BPF_DIAG_MOD_PKT_DATA_CHANGE, + BPF_DIAG_MOD_NON_OWN_REF, + BPF_DIAG_MOD_CALLER_SAVED, +}; + +enum bpf_diag_context_kind { + BPF_DIAG_CONTEXT_NONE, + BPF_DIAG_CONTEXT_RCU, + BPF_DIAG_CONTEXT_PREEMPT, + BPF_DIAG_CONTEXT_IRQ, + BPF_DIAG_CONTEXT_LOCK, +}; + +enum bpf_diag_invalid_deref_kind { + BPF_DIAG_DEREF_SCALAR, + BPF_DIAG_DEREF_NULLABLE_PTR, + BPF_DIAG_DEREF_MODIFIED_PTR, + BPF_DIAG_DEREF_INVALID_PTR, +}; + +bool bpf_diag_enabled(const struct bpf_verifier_env *env); +int bpf_diag_init(struct bpf_verifier_env *env); +void bpf_diag_init_frame(struct bpf_verifier_env *env, struct bpf_func_state *state); +char *bpf_diag_fmt_buf(struct bpf_verifier_env *env, size_t size); +const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list args) + __printf(2, 0); +const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); +const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id); +const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_type type); +u64 bpf_diag_event_log_save(struct bpf_verifier_env *env); +void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos); +u32 bpf_diag_irq_depth(const struct bpf_verifier_state *state); +void bpf_diag_free(struct bpf_verifier_env *env); +void bpf_diag_register_type(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *problem, const char *reason, const char *suggestion); +void bpf_diag_invalid_deref(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *reg_name, const struct bpf_reg_state *reg, + enum bpf_diag_invalid_deref_kind kind, s64 offset); +void bpf_diag_unreadable_reg(struct bpf_verifier_env *env, u32 insn_idx, int regno); +void bpf_diag_stack_arg_uninit(struct bpf_verifier_env *env, u32 insn_idx, int nargs, + int stack_arg_slot, const char *callee_name, + const char *arg_name); +void bpf_diag_memory(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion); +void bpf_diag_mem_bounds(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *reg_name, const char *type_name, const char *proof, + int off, int size, u32 mem_size, const struct bpf_reg_state *reg); +void bpf_diag_res(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion); +void bpf_diag_lock(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion, + const struct bpf_reference_state *active_lock); +void bpf_diag_irq(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion, u32 depth); +void bpf_diag_leak(struct bpf_verifier_env *env, u32 ref_id, u32 alloc_insn, u32 fail_insn); +void bpf_diag_call_type(struct bpf_verifier_env *env, u32 insn_idx, int argno, int regno, + int stack_arg_slot, const char *call_name, const char *arg_name, + const char *reason, const char *suggestion); +void bpf_diag_ctx_forbidden(struct bpf_verifier_env *env, u32 insn_idx, + const char *operation, const char *suggestion); +void bpf_diag_ctx_active(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + enum bpf_diag_context_kind ctx_kind, const char *suggestion); +void bpf_diag_ctx_required(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + enum bpf_diag_context_kind ctx_kind, const char *suggestion); +void bpf_diag_ctx_underflow(struct bpf_verifier_env *env, u32 insn_idx, + const char *operation, enum bpf_diag_context_kind ctx_kind, + const char *suggestion); +void bpf_diag_program_structure(struct bpf_verifier_env *env, u32 insn_idx, + const char *problem, const char *suggestion, + const char *reason_fmt, ...) __printf(5, 6); +void bpf_diag_policy(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + const char *reason, const char *suggestion); +void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true); +void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason); +void bpf_diag_mod_end(struct bpf_verifier_env *env); +void bpf_diag_record_scrub(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + enum bpf_diag_mod_reason reason); +void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env, + const struct bpf_func_state *state, s16 min_off, s16 max_off, + enum bpf_diag_mod_reason reason); +void bpf_diag_record_ref_acquire(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id); +void bpf_diag_record_ref_release(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id); +void bpf_diag_record_context(struct bpf_verifier_env *env, u32 insn_idx, + enum bpf_diag_context_kind ctx_kind, bool enter, u32 depth); + +#endif /* __BPF_DIAGNOSTICS_H */ diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index 0391b3bc0073..50b3ca5149a0 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -139,7 +139,7 @@ static void print_bpf_end_insn(bpf_insn_print_t verbose, void *private_data, const struct bpf_insn *insn) { - verbose(private_data, "(%02x) r%d = %s%d r%d\n", + verbose(private_data, "(%02x) r%d = %s%d r%d", insn->code, insn->dst_reg, BPF_SRC(insn->code) == BPF_TO_BE ? "be" : "le", insn->imm, insn->dst_reg); @@ -149,7 +149,7 @@ static void print_bpf_bswap_insn(bpf_insn_print_t verbose, void *private_data, const struct bpf_insn *insn) { - verbose(private_data, "(%02x) r%d = bswap%d r%d\n", + verbose(private_data, "(%02x) r%d = bswap%d r%d", insn->code, insn->dst_reg, insn->imm, insn->dst_reg); } @@ -197,19 +197,19 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, else print_bpf_end_insn(verbose, cbs->private_data, insn); } else if (BPF_OP(insn->code) == BPF_NEG) { - verbose(cbs->private_data, "(%02x) %c%d = -%c%d\n", + verbose(cbs->private_data, "(%02x) %c%d = -%c%d", insn->code, class == BPF_ALU ? 'w' : 'r', insn->dst_reg, class == BPF_ALU ? 'w' : 'r', insn->dst_reg); } else if (is_addr_space_cast(insn)) { - verbose(cbs->private_data, "(%02x) r%d = addr_space_cast(r%d, %u, %u)\n", + verbose(cbs->private_data, "(%02x) r%d = addr_space_cast(r%d, %u, %u)", insn->code, insn->dst_reg, insn->src_reg, ((u32)insn->imm) >> 16, (u16)insn->imm); } else if (is_mov_percpu_addr(insn)) { - verbose(cbs->private_data, "(%02x) r%d = &(void __percpu *)(r%d)\n", + verbose(cbs->private_data, "(%02x) r%d = &(void __percpu *)(r%d)", insn->code, insn->dst_reg, insn->src_reg); } else if (BPF_SRC(insn->code) == BPF_X) { - verbose(cbs->private_data, "(%02x) %c%d %s %s%c%d\n", + verbose(cbs->private_data, "(%02x) %c%d %s %s%c%d", insn->code, class == BPF_ALU ? 'w' : 'r', insn->dst_reg, is_sdiv_smod(insn) ? bpf_alu_sign_string[BPF_OP(insn->code) >> 4] @@ -218,7 +218,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, class == BPF_ALU ? 'w' : 'r', insn->src_reg); } else { - verbose(cbs->private_data, "(%02x) %c%d %s %d\n", + verbose(cbs->private_data, "(%02x) %c%d %s %d", insn->code, class == BPF_ALU ? 'w' : 'r', insn->dst_reg, is_sdiv_smod(insn) ? bpf_alu_sign_string[BPF_OP(insn->code) >> 4] @@ -227,7 +227,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, } } else if (class == BPF_STX) { if (BPF_MODE(insn->code) == BPF_MEM) - verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = r%d\n", + verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = r%d", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, @@ -235,7 +235,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, else if (BPF_MODE(insn->code) == BPF_ATOMIC && (insn->imm == BPF_ADD || insn->imm == BPF_AND || insn->imm == BPF_OR || insn->imm == BPF_XOR)) { - verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) %s r%d\n", + verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) %s r%d", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, @@ -246,7 +246,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->imm == (BPF_AND | BPF_FETCH) || insn->imm == (BPF_OR | BPF_FETCH) || insn->imm == (BPF_XOR | BPF_FETCH))) { - verbose(cbs->private_data, "(%02x) r%d = atomic%s_fetch_%s((%s *)(r%d %+d), r%d)\n", + verbose(cbs->private_data, "(%02x) r%d = atomic%s_fetch_%s((%s *)(r%d %+d), r%d)", insn->code, insn->src_reg, BPF_SIZE(insn->code) == BPF_DW ? "64" : "", bpf_atomic_alu_string[BPF_OP(insn->imm) >> 4], @@ -254,7 +254,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->dst_reg, insn->off, insn->src_reg); } else if (BPF_MODE(insn->code) == BPF_ATOMIC && insn->imm == BPF_CMPXCHG) { - verbose(cbs->private_data, "(%02x) r0 = atomic%s_cmpxchg((%s *)(r%d %+d), r0, r%d)\n", + verbose(cbs->private_data, "(%02x) r0 = atomic%s_cmpxchg((%s *)(r%d %+d), r0, r%d)", insn->code, BPF_SIZE(insn->code) == BPF_DW ? "64" : "", bpf_ldst_string[BPF_SIZE(insn->code) >> 3], @@ -262,44 +262,44 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->src_reg); } else if (BPF_MODE(insn->code) == BPF_ATOMIC && insn->imm == BPF_XCHG) { - verbose(cbs->private_data, "(%02x) r%d = atomic%s_xchg((%s *)(r%d %+d), r%d)\n", + verbose(cbs->private_data, "(%02x) r%d = atomic%s_xchg((%s *)(r%d %+d), r%d)", insn->code, insn->src_reg, BPF_SIZE(insn->code) == BPF_DW ? "64" : "", bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); } else if (BPF_MODE(insn->code) == BPF_ATOMIC && insn->imm == BPF_LOAD_ACQ) { - verbose(cbs->private_data, "(%02x) r%d = load_acquire((%s *)(r%d %+d))\n", + verbose(cbs->private_data, "(%02x) r%d = load_acquire((%s *)(r%d %+d))", insn->code, insn->dst_reg, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->src_reg, insn->off); } else if (BPF_MODE(insn->code) == BPF_ATOMIC && insn->imm == BPF_STORE_REL) { - verbose(cbs->private_data, "(%02x) store_release((%s *)(r%d %+d), r%d)\n", + verbose(cbs->private_data, "(%02x) store_release((%s *)(r%d %+d), r%d)", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); } else { - verbose(cbs->private_data, "BUG_%02x\n", insn->code); + verbose(cbs->private_data, "BUG_%02x", insn->code); } } else if (class == BPF_ST) { if (BPF_MODE(insn->code) == BPF_MEM) { - verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = %d\n", + verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = %d", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->imm); } else if (BPF_MODE(insn->code) == 0xc0 /* BPF_NOSPEC, no UAPI */) { - verbose(cbs->private_data, "(%02x) nospec\n", insn->code); + verbose(cbs->private_data, "(%02x) nospec", insn->code); } else { - verbose(cbs->private_data, "BUG_st_%02x\n", insn->code); + verbose(cbs->private_data, "BUG_st_%02x", insn->code); } } else if (class == BPF_LDX) { if (BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) { - verbose(cbs->private_data, "BUG_ldx_%02x\n", insn->code); + verbose(cbs->private_data, "BUG_ldx_%02x", insn->code); return; } - verbose(cbs->private_data, "(%02x) r%d = *(%s *)(r%d %+d)\n", + verbose(cbs->private_data, "(%02x) r%d = *(%s *)(r%d %+d)", insn->code, insn->dst_reg, BPF_MODE(insn->code) == BPF_MEM ? bpf_ldst_string[BPF_SIZE(insn->code) >> 3] : @@ -307,12 +307,12 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->src_reg, insn->off); } else if (class == BPF_LD) { if (BPF_MODE(insn->code) == BPF_ABS) { - verbose(cbs->private_data, "(%02x) r0 = *(%s *)skb[%d]\n", + verbose(cbs->private_data, "(%02x) r0 = *(%s *)skb[%d]", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->imm); } else if (BPF_MODE(insn->code) == BPF_IND) { - verbose(cbs->private_data, "(%02x) r0 = *(%s *)skb[r%d + %d]\n", + verbose(cbs->private_data, "(%02x) r0 = *(%s *)skb[r%d + %d]", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->src_reg, insn->imm); @@ -332,12 +332,12 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, if (is_ptr && !allow_ptr_leaks) imm = 0; - verbose(cbs->private_data, "(%02x) r%d = %s\n", + verbose(cbs->private_data, "(%02x) r%d = %s", insn->code, insn->dst_reg, __func_imm_name(cbs, insn, imm, tmp, sizeof(tmp))); } else { - verbose(cbs->private_data, "BUG_ld_%02x\n", insn->code); + verbose(cbs->private_data, "BUG_ld_%02x", insn->code); return; } } else if (class == BPF_JMP32 || class == BPF_JMP) { @@ -347,35 +347,35 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, char tmp[64]; if (insn->src_reg == BPF_PSEUDO_CALL) { - verbose(cbs->private_data, "(%02x) call pc%s\n", + verbose(cbs->private_data, "(%02x) call pc%s", insn->code, __func_get_name(cbs, insn, tmp, sizeof(tmp))); } else { strcpy(tmp, "unknown"); - verbose(cbs->private_data, "(%02x) call %s#%d\n", insn->code, + verbose(cbs->private_data, "(%02x) call %s#%d", insn->code, __func_get_name(cbs, insn, tmp, sizeof(tmp)), insn->imm); } } else if (insn->code == (BPF_JMP | BPF_JA)) { - verbose(cbs->private_data, "(%02x) goto pc%+d\n", + verbose(cbs->private_data, "(%02x) goto pc%+d", insn->code, insn->off); } else if (insn->code == (BPF_JMP | BPF_JA | BPF_X)) { - verbose(cbs->private_data, "(%02x) gotox r%d\n", + verbose(cbs->private_data, "(%02x) gotox r%d", insn->code, insn->dst_reg); } else if (insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO) { - verbose(cbs->private_data, "(%02x) may_goto pc%+d\n", + verbose(cbs->private_data, "(%02x) may_goto pc%+d", insn->code, insn->off); } else if (insn->code == (BPF_JMP32 | BPF_JA)) { - verbose(cbs->private_data, "(%02x) gotol pc%+d\n", + verbose(cbs->private_data, "(%02x) gotol pc%+d", insn->code, insn->imm); } else if (insn->code == (BPF_JMP | BPF_EXIT)) { - verbose(cbs->private_data, "(%02x) exit\n", insn->code); + verbose(cbs->private_data, "(%02x) exit", insn->code); } else if (BPF_SRC(insn->code) == BPF_X) { verbose(cbs->private_data, - "(%02x) if %c%d %s %c%d goto pc%+d\n", + "(%02x) if %c%d %s %c%d goto pc%+d", insn->code, class == BPF_JMP32 ? 'w' : 'r', insn->dst_reg, bpf_jmp_string[BPF_OP(insn->code) >> 4], @@ -383,14 +383,14 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->src_reg, insn->off); } else { verbose(cbs->private_data, - "(%02x) if %c%d %s 0x%x goto pc%+d\n", + "(%02x) if %c%d %s 0x%x goto pc%+d", insn->code, class == BPF_JMP32 ? 'w' : 'r', insn->dst_reg, bpf_jmp_string[BPF_OP(insn->code) >> 4], (u32)insn->imm, insn->off); } } else { - verbose(cbs->private_data, "(%02x) %s\n", + verbose(cbs->private_data, "(%02x) %s", insn->code, bpf_class_string[class]); } } diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c index ea2d60dc1fee..79f0c222c583 100644 --- a/kernel/bpf/dispatcher.c +++ b/kernel/bpf/dispatcher.c @@ -148,7 +148,10 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero, false); if (!d->image) goto out; - d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE); + /* d->rw_image doesn't need to be in module memory range, so we + * can use vzalloc. + */ + d->rw_image = vzalloc(PAGE_SIZE); if (!d->rw_image) { bpf_prog_pack_free(d->image, PAGE_SIZE); d->image = NULL; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 3cf2cc6e3ab6..65b441e4a351 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -20,6 +20,26 @@ static bool is_cmpxchg_insn(const struct bpf_insn *insn) insn->imm == BPF_CMPXCHG; } +/* Returns true if 'insn' is an address space cast instruction translated as BPF_ALU op */ +static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *insn) +{ + struct bpf_map *arena = (struct bpf_map *)prog->aux->arena; + + if (insn->code != (BPF_ALU64 | BPF_MOV | BPF_X) || insn->off != BPF_ADDR_SPACE_CAST) + return false; + + /* cast from as(1) to as(0) */ + if (insn->imm == 1) + return true; + + /* cast from as(0) to as(1) */ + if (insn->imm == 1 << 16) + return arena && arena->map_flags & BPF_F_NO_USER_CONV; + + /* non-BPF_F_NO_USER_CONV cast from as(0) to as(1) should be handled by JIT */ + return false; +} + /* Return the regno defined by the insn, or -1. */ static int insn_def_regno(const struct bpf_insn *insn) { @@ -29,30 +49,66 @@ static int insn_def_regno(const struct bpf_insn *insn) case BPF_ST: return -1; case BPF_STX: - if (BPF_MODE(insn->code) == BPF_ATOMIC || - BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) { - if (insn->imm == BPF_CMPXCHG) - return BPF_REG_0; - else if (insn->imm == BPF_LOAD_ACQ) - return insn->dst_reg; - else if (insn->imm & BPF_FETCH) - return insn->src_reg; - } - return -1; + return bpf_atomic_load_reg(insn); default: return insn->dst_reg; } } -/* Return TRUE if INSN has defined any 32-bit value explicitly. */ -static bool insn_has_def32(struct bpf_insn *insn) +/* + * For use only in combination with insn_def_regno() >= 0. + * Returns TRUE if the destination register operates on 64-bit, + * otherwise return FALSE. + */ +static bool bpf_is_reg64(struct bpf_prog *prog, struct bpf_insn *insn) { - int dst_reg = insn_def_regno(insn); + u8 class = BPF_CLASS(insn->code); + u8 mode = BPF_MODE(insn->code); + u8 size = BPF_SIZE(insn->code); + u8 op = BPF_OP(insn->code); + bool mode_mem; + + /* subregister endiness swap */ + if ((class == BPF_ALU || class == BPF_ALU64) && op == BPF_END && insn->imm != 64) + return false; + + /* w0 += 1 */ + if (class == BPF_ALU && op != BPF_END) + return false; + + /* address space casts converted to BPF_ALU, see bpf_do_misc_fixups() */ + if (is_addr_space_cast32(prog, insn)) + return false; - if (dst_reg == -1) + /* non 64-bit, non signed extended loads */ + mode_mem = mode == BPF_MEM || mode == BPF_PROBE_MEM || mode == BPF_PROBE_MEM32; + if (class == BPF_LDX && mode_mem && size != BPF_DW) return false; - return !bpf_is_reg64(insn, dst_reg, NULL, DST_OP); + /* atomics, see insn_def_regno() */ + if (class == BPF_STX && size != BPF_DW) + return false; + + /* both LD_IND and LD_ABS return 32-bit data. */ + if (class == BPF_LD && (mode == BPF_IND || mode == BPF_ABS)) + return false; + + /* Conservatively return true at default. */ + return true; +} + +/* + * Return the 32-bit subregister defined by INSN, or -1 if INSN does not + * explicitly define a 32-bit value. + */ +int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn) +{ + int dst_reg = insn_def_regno(insn); + + if (dst_reg < 0 || bpf_is_reg64(prog, insn)) + return -1; + + return dst_reg; } static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b) @@ -169,11 +225,12 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the * original insn at old prog. */ - data[off].zext_dst = insn_has_def32(insn + off + cnt - 1); + data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0; if (cnt == 1) return; prog_len = new_prog->len; + env->insn_aux_data_len = prog_len; memmove(data + off + cnt - 1, data + off, sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); @@ -181,7 +238,7 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, for (i = off; i < off + cnt - 1; i++) { /* Expand insni[off]'s seen count to the patched range. */ data[i].seen = old_seen; - data[i].zext_dst = insn_has_def32(insn + i); + data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; } /* @@ -345,13 +402,17 @@ static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, sizeof(*env->subprog_info) * move); env->subprog_cnt -= j - i; - /* remove func_info */ + /* remove func_info and its aux */ if (aux->func_info) { move = aux->func_info_cnt - j; memmove(aux->func_info + i, aux->func_info + j, sizeof(*aux->func_info) * move); + if (aux->func_info_aux) + memmove(aux->func_info_aux + i, + aux->func_info_aux + j, + sizeof(*aux->func_info_aux) * move); aux->func_info_cnt -= j - i; /* func_info->insn_off is set after all code rewrites, * in adjust_btf_func() - no need to adjust @@ -440,7 +501,6 @@ static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) { struct bpf_insn_aux_data *aux_data = env->insn_aux_data; - struct bpf_insn *insns = env->prog->insnsi; int end = start + len; int i; @@ -449,9 +509,6 @@ void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) kvfree(aux_data[i].jt); aux_data[i].jt = NULL; } - - if (bpf_is_ldimm64(&insns[i])) - i++; } } @@ -464,7 +521,6 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) if (bpf_prog_is_offloaded(env->prog->aux)) bpf_prog_offload_remove_insns(env, off, cnt); - /* Should be called before bpf_remove_insns, as it uses prog->insnsi */ bpf_clear_insn_aux_data(env, off, cnt); err = bpf_remove_insns(env->prog, off, cnt); @@ -483,6 +539,7 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) memmove(aux_data + off, aux_data + off + cnt, sizeof(*aux_data) * (orig_prog_len - off - cnt)); + env->insn_aux_data_len -= cnt; return 0; } @@ -616,11 +673,7 @@ int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, if (load_reg == -1) continue; - /* NOTE: arg "reg" (the fourth one) is only used for - * BPF_STX + SRC_OP, so it is safe to pass NULL - * here. - */ - if (bpf_is_reg64(&insn, load_reg, NULL, DST_OP)) { + if (bpf_is_reg64(env->prog, &insn)) { if (class == BPF_LD && BPF_MODE(code) == BPF_IMM) i++; @@ -759,6 +812,7 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) for (i = 0; i < insn_cnt; i++, insn++) { bpf_convert_ctx_access_t convert_ctx_access; + enum bpf_reg_type ptr_type; u8 mode; if (env->insn_aux_data[i + delta].nospec) { @@ -851,7 +905,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) continue; } - switch ((int)env->insn_aux_data[i + delta].ptr_type) { + ptr_type = env->insn_aux_data[i + delta].ptr_type; + switch ((int)ptr_type) { case PTR_TO_CTX: if (!ops->convert_ctx_access) continue; @@ -867,26 +922,6 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) case PTR_TO_XDP_SOCK: convert_ctx_access = bpf_xdp_sock_convert_ctx_access; break; - case PTR_TO_BTF_ID: - case PTR_TO_BTF_ID | PTR_UNTRUSTED: - /* PTR_TO_BTF_ID | MEM_ALLOC always has a valid lifetime, unlike - * PTR_TO_BTF_ID, and an active referenced id, but the same cannot - * be said once it is marked PTR_UNTRUSTED, hence we must handle - * any faults for loads into such types. BPF_WRITE is disallowed - * for this case. - */ - case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED: - case PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED: - if (type == BPF_READ) { - if (BPF_MODE(insn->code) == BPF_MEM) - insn->code = BPF_LDX | BPF_PROBE_MEM | - BPF_SIZE((insn)->code); - else - insn->code = BPF_LDX | BPF_PROBE_MEMSX | - BPF_SIZE((insn)->code); - env->prog->aux->num_exentries++; - } - continue; case PTR_TO_ARENA: if (BPF_MODE(insn->code) == BPF_MEMSX) { if (!bpf_jit_supports_insn(insn, true)) { @@ -900,6 +935,29 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) env->prog->aux->num_exentries++; continue; default: + /* + * A pointer which may fault on a dereference must not + * be loaded from without fault protection, hence turn + * the BPF_LDX into a BPF_PROBE_MEM one so that a bad + * address is handled rather than panicking the kernel. + * A store through one is rejected earlier, there is no + * probed counterpart to rewrite it into. + */ + if (bpf_is_ptr_to_mem_or_btf_id(ptr_type) && + bpf_may_fault_on_deref(ptr_type) && + type == BPF_READ) { + if (BPF_MODE(insn->code) == BPF_MEM) + insn->code = BPF_LDX | BPF_PROBE_MEM | + BPF_SIZE(insn->code); + else + insn->code = BPF_LDX | BPF_PROBE_MEMSX | + BPF_SIZE(insn->code); + env->prog->aux->num_exentries++; + continue; + } + if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, + "access to a fault prone pointer is not rewritten as a probed one")) + return -EFAULT; continue; } @@ -1005,26 +1063,6 @@ static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_s env->subprog_info[env->subprog_cnt].start = env->prog->len; } -struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env) -{ - size_t size; - void *new_aux; - - size = array_size(sizeof(struct bpf_insn_aux_data), env->prog->len); - new_aux = __vmalloc(size, GFP_KERNEL_ACCOUNT); - if (new_aux) - memcpy(new_aux, env->insn_aux_data, size); - return new_aux; -} - -void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_insn_aux_data *orig_insn_aux) -{ - /* the expanded elements are zero-filled, so no special handling is required */ - vfree(env->insn_aux_data); - env->insn_aux_data = orig_insn_aux; -} - static int jit_subprogs(struct bpf_verifier_env *env) { struct bpf_prog *prog = env->prog, **func, *tmp; @@ -1299,7 +1337,6 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) bool blinded = false; struct bpf_insn *insn; struct bpf_prog *prog, *orig_prog; - struct bpf_insn_aux_data *orig_insn_aux; u32 *orig_subprog_starts; if (env->subprog_cnt <= 1) @@ -1307,14 +1344,8 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) prog = orig_prog = env->prog; if (bpf_prog_need_blind(prog)) { - orig_insn_aux = bpf_dup_insn_aux_data(env); - if (!orig_insn_aux) { - err = -ENOMEM; - goto out_cleanup; - } orig_subprog_starts = bpf_dup_subprog_starts(env); if (!orig_subprog_starts) { - vfree(orig_insn_aux); err = -ENOMEM; goto out_cleanup; } @@ -1334,7 +1365,6 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) if (blinded) { bpf_jit_prog_release_other(prog, orig_prog); kvfree(orig_subprog_starts); - vfree(orig_insn_aux); } return 0; @@ -1364,7 +1394,6 @@ out_jit_err: out_restore: bpf_restore_subprog_starts(env, orig_subprog_starts); - bpf_restore_insn_aux_data(env, orig_insn_aux); kvfree(orig_subprog_starts); out_cleanup: /* cleanup main prog to be interpreted */ @@ -1378,7 +1407,6 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) #ifndef CONFIG_BPF_JIT_ALWAYS_ON struct bpf_prog *prog = env->prog; struct bpf_insn *insn = prog->insnsi; - bool has_kfunc_call = bpf_prog_has_kfunc_call(prog); int depth; #endif int i, err = 0; @@ -1404,8 +1432,8 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) return err; } #ifndef CONFIG_BPF_JIT_ALWAYS_ON - if (has_kfunc_call) { - verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); + if (prog->jit_required) { + verbose(env, "program requires BPF JIT compiler but it is not available\n"); return -EINVAL; } for (i = 0; i < env->subprog_cnt; i++) { @@ -1447,7 +1475,6 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) return err; } - /* The function requires that first instruction in 'patch' is insnsi[prog->len - 1] */ static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len) { @@ -1514,15 +1541,12 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) } for (i = 0; i < insn_cnt;) { - if (insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) && insn->imm) { - if ((insn->off == BPF_ADDR_SPACE_CAST && insn->imm == 1) || - (((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) { - /* convert to 32-bit mov that clears upper 32-bit */ - insn->code = BPF_ALU | BPF_MOV | BPF_X; - /* clear off and imm, so it's a normal 'wX = wY' from JIT pov */ - insn->off = 0; - insn->imm = 0; - } /* cast from as(0) to as(1) should be handled by JIT */ + if (is_addr_space_cast32(env->prog, insn)) { + /* convert to 32-bit mov that clears upper 32-bit */ + insn->code = BPF_ALU | BPF_MOV | BPF_X; + /* clear off and imm, so it's a normal 'wX = wY' from JIT pov */ + insn->off = 0; + insn->imm = 0; goto next_insn; } @@ -1819,6 +1843,43 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) goto next_insn; } + if (bpf_jit_supports_percpu_insn() && + insn->code == (BPF_LD | BPF_IMM | BPF_DW) && + (insn->src_reg == BPF_PSEUDO_MAP_VALUE || + insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE)) { + struct bpf_map *map; + + aux = &env->insn_aux_data[i + delta]; + map = env->used_maps[aux->map_index]; + if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) + goto next_insn; + + prog->jit_required = true; + + /* + * We are *skipping* first half of ld_imm64 insn + * with 'i++;', patching over second half of it + * with that same half + mov64_percpu_reg insn. + * All because bpf_patch_insn_data() can only + * replace one 8-byte insn, which does not work + * well for ld_imm64 insn. + */ + + insn_buf[0] = insn[1]; + insn_buf[1] = BPF_MOV64_PERCPU_REG(insn->dst_reg, insn->dst_reg); + cnt = 2; + + i++; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = prog = new_prog; + insn = new_prog->insnsi + i + delta; + goto next_insn; + } + if (insn->code != (BPF_JMP | BPF_CALL)) goto next_insn; if (insn->src_reg == BPF_PSEUDO_CALL) @@ -1841,8 +1902,10 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) } /* Skip inlining the helper call if the JIT does it. */ - if (bpf_jit_inlines_helper_call(insn->imm)) + if (bpf_jit_inlines_helper_call(insn->imm)) { + prog->jit_required = 1; goto next_insn; + } if (insn->imm == BPF_FUNC_get_route_realm) prog->dst_needed = 1; @@ -2007,6 +2070,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) return -EFAULT; } + if (bpf_map_is_percpu_map(map_ptr->map_type)) + prog->jit_required = true; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -2111,6 +2177,7 @@ patch_map_ops_generic: * way, it's fine to back out this inlining logic */ #ifdef CONFIG_SMP + prog->jit_required = true; insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&cpu_number); insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 0); @@ -2132,6 +2199,7 @@ patch_map_ops_generic: /* Implement bpf_get_current_task() and bpf_get_current_task_btf() inline. */ if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) && bpf_verifier_inlines_helper_call(env, insn->imm)) { + prog->jit_required = true; insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)¤t_task); insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0); @@ -2338,7 +2406,7 @@ patch_call_imm: func_id_name(insn->imm), insn->imm); return -EFAULT; } - insn->imm = fn->func - __bpf_call_base; + insn->imm = BPF_CALL_IMM(fn->func); next_insn: if (subprogs[cur_subprog + 1].start == i + delta + 1) { subprogs[cur_subprog].stack_depth += stack_depth_extra; diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 9f394e1aa2e8..d40cb5dd446c 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -998,7 +998,6 @@ static void dec_elem_count(struct bpf_htab *htab) atomic_dec(&htab->count); } - static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) { htab_put_fd_value(htab, l); @@ -2970,7 +2969,6 @@ static int rhtab_delete_elem(struct bpf_rhtab *rhtab, struct rhtab_elem *elem, v return 0; } - static long rhtab_map_delete_elem(struct bpf_map *map, void *key) { struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index c18f1e16edee..b3cc5c8fc875 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -278,7 +278,7 @@ const struct bpf_func_proto bpf_get_current_comm_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, }; #if defined(CONFIG_QUEUED_SPINLOCKS) || defined(CONFIG_BPF_ARCH_SPINLOCK) @@ -539,7 +539,7 @@ const struct bpf_func_proto bpf_strtol_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, .arg4_size = sizeof(s64), @@ -567,7 +567,7 @@ const struct bpf_func_proto bpf_strtoul_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, .arg4_size = sizeof(u64), @@ -583,7 +583,7 @@ static const struct bpf_func_proto bpf_strncmp_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_PTR_TO_CONST_STR, }; @@ -627,7 +627,7 @@ const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto = { .arg1_type = ARG_ANYTHING, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; static const struct bpf_func_proto bpf_get_raw_smp_processor_id_proto = { @@ -653,7 +653,7 @@ const struct bpf_func_proto bpf_event_output_data_proto = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(bpf_copy_from_user, void *, dst, u32, size, @@ -675,7 +675,7 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -706,7 +706,7 @@ const struct bpf_func_proto bpf_copy_from_user_task_proto = { .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_BTF_ID, .arg4_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], @@ -1093,10 +1093,10 @@ const struct bpf_func_proto bpf_snprintf_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_CONST_STR, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; static void *map_key_from_value(struct bpf_map *map, void *value, u32 *arr_idx) @@ -1888,7 +1888,7 @@ static const struct bpf_func_proto bpf_dynptr_from_mem_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL | MEM_UNINIT | MEM_WRITE, }; @@ -1943,7 +1943,7 @@ static const struct bpf_func_proto bpf_dynptr_read_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_DYNPTR, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, @@ -2004,7 +2004,7 @@ static const struct bpf_func_proto bpf_dynptr_write_proto = { .arg1_type = ARG_PTR_TO_DYNPTR, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE_OR_ZERO, + .arg4_type = ARG_MEM_SIZE_OR_ZERO, .arg5_type = ARG_ANYTHING, }; @@ -3395,19 +3395,21 @@ __bpf_kfunc void bpf_throw(u64 cookie) WARN_ON_ONCE(!ctx.aux->exception_boundary); WARN_ON_ONCE(!ctx.bp); WARN_ON_ONCE(!ctx.cnt); - /* Prevent KASAN false positives for CONFIG_KASAN_STACK by unpoisoning + /* + * Prevent KASAN false positives for CONFIG_KASAN_STACK by unpoisoning * deeper stack depths than ctx.sp as we do not return from bpf_throw, - * which skips compiler generated instrumentation to do the same. + * which skips compiler generated instrumentation to do the same. Some + * architectures cannot recover sp while unwinding, so fall back to bp. */ - kasan_unpoison_task_stack_below((void *)(long)ctx.sp); + kasan_unpoison_task_stack_below((void *)(long)(ctx.sp ?: ctx.bp)); ctx.aux->bpf_exception_cb(cookie, ctx.sp + ctx.aux->stack_arg_sp_adjust, ctx.bp, 0, 0); WARN(1, "A call to BPF exception callback should never return\n"); } -__bpf_kfunc int bpf_wq_init(struct bpf_wq *wq, void *p__map, unsigned int flags) +__bpf_kfunc int bpf_wq_init(struct bpf_wq *wq, void *p__const_map, unsigned int flags) { struct bpf_async_kern *async = (struct bpf_async_kern *)wq; - struct bpf_map *map = p__map; + struct bpf_map *map = p__const_map; BUILD_BUG_ON(sizeof(struct bpf_async_kern) > sizeof(struct bpf_wq)); BUILD_BUG_ON(__alignof__(struct bpf_async_kern) != __alignof__(struct bpf_wq)); @@ -4388,7 +4390,7 @@ struct bpf_task_work_ctx { struct bpf_map *map; void *map_val; enum task_work_notify_mode mode; - bpf_task_work_callback_t callback_fn; + bpf_callback_t callback_fn; struct rcu_head rcu; } __aligned(8); @@ -4471,7 +4473,8 @@ static void bpf_task_work_callback(struct callback_head *cb) key = (void *)map_key_from_value(ctx->map, ctx->map_val, &idx); migrate_disable(); - ctx->callback_fn(ctx->map, key, ctx->map_val); + ctx->callback_fn((u64)(long)ctx->map, (u64)(long)key, + (u64)(long)ctx->map_val, 0, 0); migrate_enable(); bpf_task_work_ctx_reset(ctx); @@ -4594,7 +4597,7 @@ static struct bpf_task_work_ctx *bpf_task_work_acquire_ctx(struct bpf_task_work } static int bpf_task_work_schedule(struct task_struct *task, struct bpf_task_work *tw, - struct bpf_map *map, bpf_task_work_callback_t callback_fn, + struct bpf_map *map, void *callback_fn, struct bpf_prog_aux *aux, enum task_work_notify_mode mode) { struct bpf_prog *prog; @@ -4619,7 +4622,7 @@ static int bpf_task_work_schedule(struct task_struct *task, struct bpf_task_work } ctx->task = task; - ctx->callback_fn = callback_fn; + ctx->callback_fn = (bpf_callback_t)callback_fn; ctx->prog = prog; ctx->mode = mode; ctx->map = map; @@ -4642,17 +4645,17 @@ release_prog: * mode * @task: Task struct for which callback should be scheduled * @tw: Pointer to struct bpf_task_work in BPF map value for internal bookkeeping - * @map__map: bpf_map that embeds struct bpf_task_work in the values + * @map__const_map: bpf_map that embeds struct bpf_task_work in the values * @callback: pointer to BPF subprogram to call * @aux: pointer to bpf_prog_aux of the caller BPF program, implicitly set by the verifier * * Return: 0 if task work has been scheduled successfully, negative error code otherwise */ __bpf_kfunc int bpf_task_work_schedule_signal(struct task_struct *task, struct bpf_task_work *tw, - void *map__map, bpf_task_work_callback_t callback, + void *map__const_map, bpf_task_work_callback_t callback, struct bpf_prog_aux *aux) { - return bpf_task_work_schedule(task, tw, map__map, callback, aux, TWA_SIGNAL); + return bpf_task_work_schedule(task, tw, map__const_map, callback, aux, TWA_SIGNAL); } /** @@ -4660,17 +4663,17 @@ __bpf_kfunc int bpf_task_work_schedule_signal(struct task_struct *task, struct b * mode * @task: Task struct for which callback should be scheduled * @tw: Pointer to struct bpf_task_work in BPF map value for internal bookkeeping - * @map__map: bpf_map that embeds struct bpf_task_work in the values + * @map__const_map: bpf_map that embeds struct bpf_task_work in the values * @callback: pointer to BPF subprogram to call * @aux: pointer to bpf_prog_aux of the caller BPF program, implicitly set by the verifier * * Return: 0 if task work has been scheduled successfully, negative error code otherwise */ __bpf_kfunc int bpf_task_work_schedule_resume(struct task_struct *task, struct bpf_task_work *tw, - void *map__map, bpf_task_work_callback_t callback, + void *map__const_map, bpf_task_work_callback_t callback, struct bpf_prog_aux *aux) { - return bpf_task_work_schedule(task, tw, map__map, callback, aux, TWA_RESUME); + return bpf_task_work_schedule(task, tw, map__const_map, callback, aux, TWA_RESUME); } static int make_file_dynptr(struct file *file, u32 flags, bool may_sleep, @@ -4811,30 +4814,32 @@ BTF_ID_FLAGS(func, bpf_obj_drop, KF_RELEASE | KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_obj_drop_impl, KF_RELEASE) BTF_ID_FLAGS(func, bpf_percpu_obj_drop, KF_RELEASE | KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_percpu_obj_drop_impl, KF_RELEASE) -BTF_ID_FLAGS(func, bpf_refcount_acquire, KF_ACQUIRE | KF_RET_NULL | KF_RCU | KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_refcount_acquire_impl, KF_ACQUIRE | KF_RET_NULL | KF_RCU) -BTF_ID_FLAGS(func, bpf_list_push_front, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_list_push_front_impl) -BTF_ID_FLAGS(func, bpf_list_push_back, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_list_push_back_impl) -BTF_ID_FLAGS(func, bpf_list_add, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_list_pop_front, KF_ACQUIRE | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_pop_back, KF_ACQUIRE | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_del, KF_ACQUIRE | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_front, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_back, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_is_first) -BTF_ID_FLAGS(func, bpf_list_is_last) -BTF_ID_FLAGS(func, bpf_list_empty) +BTF_ID_FLAGS(func, bpf_refcount_acquire, + KF_ACQUIRE | KF_RET_NULL | KF_RCU | KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_refcount_acquire_impl, + KF_ACQUIRE | KF_RET_NULL | KF_RCU | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_push_front, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_push_front_impl, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_push_back, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_push_back_impl, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_add, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_pop_front, KF_ACQUIRE | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_pop_back, KF_ACQUIRE | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_del, KF_ACQUIRE | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_front, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_back, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_is_first, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_is_last, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_empty, KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_task_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_task_release, KF_RELEASE) -BTF_ID_FLAGS(func, bpf_rbtree_remove, KF_ACQUIRE | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_rbtree_add, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_rbtree_add_impl) -BTF_ID_FLAGS(func, bpf_rbtree_first, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_rbtree_root, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_rbtree_left, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_rbtree_right, KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_rbtree_remove, KF_ACQUIRE | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_add, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_add_impl, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_first, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_root, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_left, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_right, KF_RET_NULL | KF_SPINLOCK_SAFE) #ifdef CONFIG_CGROUPS BTF_ID_FLAGS(func, bpf_cgroup_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL) @@ -4868,7 +4873,6 @@ static const struct btf_kfunc_id_set generic_kfunc_set = { .set = &generic_btf_ids, }; - BTF_ID_LIST(generic_dtor_ids) BTF_ID(struct, task_struct) BTF_ID(func, bpf_task_release_dtor) @@ -4884,9 +4888,9 @@ BTF_ID_FLAGS(func, bpf_rcu_read_lock) BTF_ID_FLAGS(func, bpf_rcu_read_unlock) BTF_ID_FLAGS(func, bpf_dynptr_slice, KF_RET_NULL) BTF_ID_FLAGS(func, bpf_dynptr_slice_rdwr, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_iter_num_new, KF_ITER_NEW) -BTF_ID_FLAGS(func, bpf_iter_num_next, KF_ITER_NEXT | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_iter_num_destroy, KF_ITER_DESTROY) +BTF_ID_FLAGS(func, bpf_iter_num_new, KF_ITER_NEW | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_iter_num_next, KF_ITER_NEXT | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_iter_num_destroy, KF_ITER_DESTROY | KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_iter_task_vma_new, KF_ITER_NEW | KF_RCU) BTF_ID_FLAGS(func, bpf_iter_task_vma_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_task_vma_destroy, KF_ITER_DESTROY) @@ -4961,8 +4965,8 @@ BTF_ID_FLAGS(func, bpf_strncasestr); #if defined(CONFIG_BPF_LSM) && defined(CONFIG_CGROUPS) BTF_ID_FLAGS(func, bpf_cgroup_read_xattr, KF_RCU) #endif -BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_task_work_schedule_signal, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_task_work_schedule_resume, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_dynptr_from_file) diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 0aadfbae0acc..74fc4b3f80d6 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -269,7 +269,6 @@ bpf_insn_successors(struct bpf_verifier_env *env, u32 idx) __diag_pop(); - static inline bool update_insn(struct bpf_verifier_env *env, struct func_instance *instance, u32 frame, u32 insn_idx) { @@ -497,7 +496,6 @@ static void print_instance(struct bpf_verifier_env *env, struct func_instance *i pos = env->log.end_pos; verbose(env, "%3d: ", insn_idx); bpf_verbose_insn(env, &insns[insn_idx]); - bpf_vlog_reset(&env->log, env->log.end_pos - 1); /* remove \n */ insn_pos = env->log.end_pos; verbose(env, "%*c;", bpf_vlog_alignment(insn_pos - pos), ' '); pos = env->log.end_pos; @@ -1043,7 +1041,6 @@ static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, i if (!printed) { verbose(env, "%3d: ", idx); bpf_verbose_insn(env, insn); - bpf_vlog_reset(&env->log, env->log.end_pos - 1); printed = true; } verbose(env, "\tr%d: ", i); verbose_arg_track(env, &at_in[i]); @@ -1058,7 +1055,6 @@ static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, i if (!printed) { verbose(env, "%3d: ", idx); bpf_verbose_insn(env, insn); - bpf_vlog_reset(&env->log, env->log.end_pos - 1); printed = true; } verbose(env, "\tsa%d: ", i); verbose_arg_track(env, &at_in[ai]); @@ -1070,7 +1066,6 @@ static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, i if (!printed) { verbose(env, "%3d: ", idx); bpf_verbose_insn(env, insn); - bpf_vlog_reset(&env->log, env->log.end_pos - 1); printed = true; } verbose(env, "\tfp%+d: ", -(i + 1) * 8); verbose_arg_track(env, &at_stack_in[i]); @@ -1213,12 +1208,9 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, clear_stack_for_all_offs(insn, at_out, insn->dst_reg, at_stack_out, sz); - if (insn->imm == BPF_CMPXCHG) - at_out[BPF_REG_0] = none; - else if (insn->imm == BPF_LOAD_ACQ) - *dst = none; - else if (insn->imm & BPF_FETCH) - *src = none; + r = bpf_atomic_load_reg(insn); + if (r >= 0) + at_out[r] = none; } } else if (class == BPF_ST && BPF_MODE(insn->code) == BPF_MEM) { u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); @@ -1545,6 +1537,7 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, verbose(env, "%3d: ", idx); bpf_verbose_insn(env, &insns[idx]); + verbose(env, "\n"); /* Collect what needs printing */ if (is_ldx_stx_call && @@ -1868,7 +1861,6 @@ static int analyze_subprog(struct bpf_verifier_env *env, if (need_resched()) cond_resched(); - /* * When an instance is reused (must_write_initialized == true), * record into a fresh instance and merge afterward. This avoids @@ -2050,29 +2042,38 @@ out: /* Each field is a register bitmask */ struct insn_live_regs { - u16 use; /* registers read by instruction */ - u16 def; /* registers written by instruction */ - u16 in; /* registers that may be alive before instruction */ - u16 out; /* registers that may be alive after instruction */ + u32 use; /* registers read by instruction */ + u32 def; /* registers written by instruction */ + u32 in; /* registers that may be alive before instruction */ + u32 out; /* registers that may be alive after instruction */ }; /* Bitmask with 1s for all caller saved registers */ #define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1) +static inline u32 reg32_mask(u32 n) { return BIT(n); } +static inline u32 reg64_mask(u32 n) { return BIT(n) | BIT(n + 16); } +static inline u32 mask_widen(u32 m) { return m | (m << 16); } +static inline u16 mask_lo(u32 m) { return (u16)m; } +static inline u16 mask_hi(u32 m) { return (u16)(m >> 16); } + /* Compute info->{use,def} fields for the instruction */ static void compute_insn_live_regs(struct bpf_verifier_env *env, struct bpf_insn *insn, struct insn_live_regs *info) { struct bpf_call_summary cs; - u8 class = BPF_CLASS(insn->code); - u8 code = BPF_OP(insn->code); - u8 mode = BPF_MODE(insn->code); - u16 src = BIT(insn->src_reg); - u16 dst = BIT(insn->dst_reg); - u16 r0 = BIT(0); - u16 def = 0; - u16 use = 0xffff; + const u8 class = BPF_CLASS(insn->code); + const u8 code = BPF_OP(insn->code); + const u8 mode = BPF_MODE(insn->code); + const u8 size = BPF_SIZE(insn->code); + const u32 src = reg64_mask(insn->src_reg); + const u32 dst = reg64_mask(insn->dst_reg); + const u32 src32 = mask_lo(src); + const u32 dst32 = mask_lo(dst); + const u32 r0 = reg64_mask(0); + u32 def = 0; + u32 use = U32_MAX; switch (class) { case BPF_LD: @@ -2083,8 +2084,8 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, use = 0; } break; - case BPF_LD | BPF_ABS: - case BPF_LD | BPF_IND: + case BPF_ABS: + case BPF_IND: /* stick with defaults */ break; } @@ -2092,7 +2093,15 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, case BPF_LDX: switch (mode) { case BPF_MEM: + /* a narrow load still redefines the whole register */ + def = dst; + use = src; + break; case BPF_MEMSX: + /* + * sign extension defines the whole register; + * src holds a pointer, hence is used as 64-bit. + */ def = dst; use = src; break; @@ -2110,12 +2119,19 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, switch (mode) { case BPF_MEM: def = 0; - use = dst | src; + use = dst | (size == BPF_DW ? src : src32); break; - case BPF_ATOMIC: + case BPF_ATOMIC: { + /* + * dst holds a pointer and is always used as 64-bit; + * the value operand and r0 are read as 32-bit for BPF_W atomics. + */ + u32 srcv = size == BPF_DW ? src : src32; + u32 r0v = size == BPF_DW ? r0 : mask_lo(r0); + switch (insn->imm) { case BPF_CMPXCHG: - use = r0 | dst | src; + use = r0v | dst | srcv; def = r0; break; case BPF_LOAD_ACQ: @@ -2124,10 +2140,10 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, break; case BPF_STORE_REL: def = 0; - use = dst | src; + use = dst | srcv; break; default: - use = dst | src; + use = dst | srcv; if (insn->imm & BPF_FETCH) def = src; else @@ -2135,6 +2151,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, } break; } + } break; case BPF_ALU: case BPF_ALU64: @@ -2148,14 +2165,14 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, if (BPF_SRC(insn->code) == BPF_K) use = 0; else - use = src; + use = class == BPF_ALU64 ? src : src32; break; default: def = dst; if (BPF_SRC(insn->code) == BPF_K) - use = dst; + use = class == BPF_ALU64 ? dst : dst32; else - use = dst | src; + use = class == BPF_ALU64 ? (dst | src) : (dst32 | src32); } break; case BPF_JMP: @@ -2181,13 +2198,14 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, use = def & ~BIT(BPF_REG_0); if (bpf_get_call_summary(env, insn, &cs)) use = GENMASK(min_t(u8, cs.num_params, MAX_BPF_FUNC_REG_ARGS), 1); + def = mask_widen(def); + use = mask_widen(use); break; default: def = 0; - if (BPF_SRC(insn->code) == BPF_K) - use = dst; - else - use = dst | src; + use = class == BPF_JMP ? dst : dst32; + if (BPF_SRC(insn->code) == BPF_X) + use |= class == BPF_JMP ? src : src32; } break; } @@ -2209,6 +2227,7 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) struct bpf_insn *insns = env->prog->insnsi; struct insn_live_regs *state; int insn_cnt = env->prog->len; + u64 pos, insn_pos; int err = 0, i, j; bool changed; @@ -2252,8 +2271,8 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) int insn_idx = env->cfg.insn_postorder[i]; struct insn_live_regs *live = &state[insn_idx]; struct bpf_iarray *succ; - u16 new_out = 0; - u16 new_in = 0; + u32 new_out = 0; + u32 new_in = 0; succ = bpf_insn_successors(env, insn_idx); for (int s = 0; s < succ->cnt; ++s) @@ -2267,8 +2286,20 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) } } - for (i = 0; i < insn_cnt; ++i) - insn_aux[i].live_regs_before = state[i].in; + for (i = 0; i < insn_cnt; ++i) { + int def32 = bpf_insn_def32(env->prog, &insns[i]); + u32 out = state[i].out; + u32 in = state[i].in; + + insn_aux[i].live_regs_before = mask_lo(in) | mask_hi(in); + /* + * On architectures where 32-bit operations do not reset upper halves + * of the registers, the verifier needs to zero extend a destination + * register if an instruction defines a 32-bit subregister and the + * upper half of that register is alive after the instruction. + */ + insn_aux[i].zext_dst = def32 >= 0 && (mask_hi(out) & BIT(def32)); + } if (env->log.level & BPF_LOG_LEVEL2) { verbose(env, "Live regs before insn:\n"); @@ -2284,7 +2315,12 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) else verbose(env, "."); verbose(env, " "); + pos = env->log.end_pos; bpf_verbose_insn(env, &insns[i]); + insn_pos = env->log.end_pos; + if (insn_aux[i].zext_dst) + verbose(env, "%*c; zext", bpf_vlog_alignment(insn_pos - pos), ' '); + verbose(env, "\n"); if (bpf_is_ldimm64(&insns[i])) i++; } diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index b740fa73ee26..589770ca3d3a 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -615,17 +615,6 @@ static void print_scalar_ranges(struct bpf_verifier_env *env, } } -static bool type_is_map_ptr(enum bpf_reg_type t) { - switch (base_type(t)) { - case CONST_PTR_TO_MAP: - case PTR_TO_MAP_KEY: - case PTR_TO_MAP_VALUE: - return true; - default: - return false; - } -} - /* * _a stands for append, was shortened to avoid multiline statements below. * This macro is used to output a comma separated list of attributes. diff --git a/kernel/bpf/mmap_unlock_work.h b/kernel/bpf/mmap_unlock_work.h index 5d18d7d85bef..1834db20b861 100644 --- a/kernel/bpf/mmap_unlock_work.h +++ b/kernel/bpf/mmap_unlock_work.h @@ -4,12 +4,15 @@ #ifndef __MMAP_UNLOCK_WORK_H__ #define __MMAP_UNLOCK_WORK_H__ +#include <linux/atomic.h> +#include <linux/err.h> #include <linux/irq_work.h> /* irq_work to run mmap_read_unlock() in irq_work */ struct mmap_unlock_irq_work { struct irq_work irq_work; struct mm_struct *mm; + atomic_t active; }; DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); @@ -18,32 +21,36 @@ DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); * We cannot do mmap_read_unlock() when the irq is disabled, because of * risk to deadlock with rq_lock. To look up vma when the irqs are * disabled, we need to run mmap_read_unlock() in irq_work. We use a - * percpu variable to do the irq_work. If the irq_work is already used - * by another lookup, we fall over. + * percpu variable to do the irq_work. The active flag reserves the slot + * before mmap_read_trylock() and until the irq_work callback consumes mm. */ -static inline bool bpf_mmap_unlock_get_irq_work(struct mmap_unlock_irq_work **work_ptr) +static inline struct mmap_unlock_irq_work *bpf_mmap_unlock_guard_get(void) { - struct mmap_unlock_irq_work *work = NULL; - bool irq_work_busy = false; + struct mmap_unlock_irq_work *work; - if (irqs_disabled()) { - if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { - work = this_cpu_ptr(&mmap_unlock_work); - if (irq_work_is_busy(&work->irq_work)) { - /* cannot queue more up_read, fallback */ - irq_work_busy = true; - } - } else { - /* - * PREEMPT_RT does not allow to trylock mmap sem in - * interrupt disabled context. Force the fallback code. - */ - irq_work_busy = true; - } - } + if (!irqs_disabled()) + return NULL; + + /* + * PREEMPT_RT does not allow to trylock mmap sem in interrupt + * disabled context. Force the fallback code. + */ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + return ERR_PTR(-EBUSY); + + work = this_cpu_ptr(&mmap_unlock_work); + if (irq_work_is_busy(&work->irq_work) || + atomic_cmpxchg_acquire(&work->active, 0, 1)) + return ERR_PTR(-EBUSY); - *work_ptr = work; - return irq_work_busy; + return work; +} + +static inline void +bpf_mmap_unlock_guard_put(struct mmap_unlock_irq_work *work) +{ + if (work) + atomic_set_release(&work->active, 0); } static inline void bpf_mmap_unlock_mm(struct mmap_unlock_irq_work *work, struct mm_struct *mm) diff --git a/kernel/bpf/net_namespace.c b/kernel/bpf/net_namespace.c index 25f30f9edaef..81006a242618 100644 --- a/kernel/bpf/net_namespace.c +++ b/kernel/bpf/net_namespace.c @@ -171,33 +171,28 @@ static int bpf_netns_link_update_prog(struct bpf_link *link, struct net *net; int idx, ret; + guard(mutex)(&netns_bpf_mutex); + if (old_prog && old_prog != link->prog) return -EPERM; if (new_prog->type != link->prog->type) return -EINVAL; - mutex_lock(&netns_bpf_mutex); - net = net_link->net; - if (!net || !check_net(net)) { + if (!net || !check_net(net)) /* Link auto-detached or netns dying */ - ret = -ENOLINK; - goto out_unlock; - } + return -ENOLINK; run_array = rcu_dereference_protected(net->bpf.run_array[type], lockdep_is_held(&netns_bpf_mutex)); idx = link_index(net, type, net_link); ret = bpf_prog_array_update_at(run_array, idx, new_prog); if (ret) - goto out_unlock; + return ret; old_prog = xchg(&link->prog, new_prog); bpf_prog_put(old_prog); - -out_unlock: - mutex_unlock(&netns_bpf_mutex); - return ret; + return 0; } static int bpf_netns_link_fill_info(const struct bpf_link *link, diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c index 9a5f94371e50..6e8b18c32a10 100644 --- a/kernel/bpf/queue_stack_maps.c +++ b/kernel/bpf/queue_stack_maps.c @@ -99,8 +99,10 @@ static long __queue_map_get(struct bpf_map *map, void *value, bool delete) int err = 0; void *ptr; - if (raw_res_spin_lock_irqsave(&qs->lock, flags)) + if (raw_res_spin_lock_irqsave(&qs->lock, flags)) { + memset(value, 0, qs->map.value_size); return -EBUSY; + } if (queue_stack_map_is_empty(qs)) { memset(value, 0, qs->map.value_size); @@ -121,7 +123,6 @@ out: return err; } - static long __stack_map_get(struct bpf_map *map, void *value, bool delete) { struct bpf_queue_stack *qs = bpf_queue_stack(map); @@ -130,8 +131,10 @@ static long __stack_map_get(struct bpf_map *map, void *value, bool delete) void *ptr; u32 index; - if (raw_res_spin_lock_irqsave(&qs->lock, flags)) + if (raw_res_spin_lock_irqsave(&qs->lock, flags)) { + memset(value, 0, qs->map.value_size); return -EBUSY; + } if (queue_stack_map_is_empty(qs)) { memset(value, 0, qs->map.value_size); diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index 35ae64ade36b..3f1013d80544 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -321,7 +321,7 @@ static unsigned long ringbuf_avail_data_sz(struct bpf_ringbuf *rb) if (unlikely(rb->overwrite_mode)) { over_pos = smp_load_acquire(&rb->overwrite_pos); prod_pos = smp_load_acquire(&rb->producer_pos); - return prod_pos - max(cons_pos, over_pos); + return min(prod_pos - cons_pos, prod_pos - over_pos); } else { prod_pos = smp_load_acquire(&rb->producer_pos); return prod_pos - cons_pos; @@ -482,7 +482,7 @@ static void *__bpf_ringbuf_reserve(struct bpf_ringbuf *rb, u64 size) prod_pos = rb->producer_pos; new_prod_pos = prod_pos + len; - while (pend_pos < prod_pos) { + while (prod_pos - pend_pos > 0) { hdr = (void *)rb->data + (pend_pos & rb->mask); hdr_len = READ_ONCE(hdr->len); if (hdr_len & BPF_RINGBUF_BUSY_BIT) @@ -634,7 +634,7 @@ const struct bpf_func_proto bpf_ringbuf_output_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_CONST_MAP_PTR, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/rqspinlock.c b/kernel/bpf/rqspinlock.c index 2129defc4a9a..111ec80ea958 100644 --- a/kernel/bpf/rqspinlock.c +++ b/kernel/bpf/rqspinlock.c @@ -745,10 +745,10 @@ __bpf_kfunc void bpf_res_spin_unlock_irqrestore(struct bpf_res_spin_lock *lock, __bpf_kfunc_end_defs(); BTF_KFUNCS_START(rqspinlock_kfunc_ids) -BTF_ID_FLAGS(func, bpf_res_spin_lock, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_res_spin_unlock) -BTF_ID_FLAGS(func, bpf_res_spin_lock_irqsave, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_res_spin_unlock_irqrestore) +BTF_ID_FLAGS(func, bpf_res_spin_lock, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_res_spin_unlock, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_res_spin_lock_irqsave, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_res_spin_unlock_irqrestore, KF_SPINLOCK_SAFE) BTF_KFUNCS_END(rqspinlock_kfunc_ids) static const struct btf_kfunc_id_set rqspinlock_kfunc_set = { diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 41fe87d7302f..a839041e0d00 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -414,8 +414,7 @@ static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *i static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, u32 trace_nr, bool user, bool may_fault) { - struct mmap_unlock_irq_work *work = NULL; - bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); + struct mmap_unlock_irq_work *work; bool has_user_ctx = user && current && current->mm; struct stack_map_build_id_cache cache = {}; struct vm_area_struct *vma; @@ -426,15 +425,16 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, return; } - /* If the irq_work is in use, fall back to report ips. Same - * fallback is used for kernel stack (!user) on a stackmap with - * build_id. - */ - if (!has_user_ctx || irq_work_busy || !mmap_read_trylock(current->mm)) { - /* cannot access current->mm, fall back to ips */ - for (i = 0; i < trace_nr; i++) - stack_map_build_id_set_ip(&id_offs[i]); - return; + if (!has_user_ctx) + goto fallback; + + work = bpf_mmap_unlock_guard_get(); + if (IS_ERR(work)) + goto fallback; + + if (!mmap_read_trylock(current->mm)) { + bpf_mmap_unlock_guard_put(work); + goto fallback; } for (i = 0; i < trace_nr; i++) { @@ -465,6 +465,12 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, vma->vm_pgoff); } bpf_mmap_unlock_mm(work, current->mm); + return; + +fallback: + /* cannot access current->mm, fall back to ips */ + for (i = 0; i < trace_nr; i++) + stack_map_build_id_set_ip(&id_offs[i]); } static struct perf_callchain_entry * @@ -504,78 +510,116 @@ get_callchain_entry_for_task(struct task_struct *task, u32 max_depth) #endif } -static long __bpf_get_stackid(struct bpf_map *map, - struct perf_callchain_entry *trace, u64 flags) +struct stackid { + struct stack_map_bucket *bucket; + const u64 *ips; + u32 nr; + u32 len; + u32 hash; + u32 id; + bool hash_matches; +}; + +static int stackid_init(struct stackid *stackid, struct bpf_map *map, + const struct perf_callchain_entry *trace, u32 trace_nr, u64 flags) { struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); - struct stack_map_bucket *bucket, *new_bucket, *old_bucket; - u32 hash, id, trace_nr, trace_len, i, max_depth; u32 skip = flags & BPF_F_SKIP_FIELD_MASK; - bool user = flags & BPF_F_USER_STACK; - u64 *ips; - bool hash_matches; + u32 max_depth; - if (trace->nr <= skip) + if (trace_nr <= skip) /* skipping more than usable stack trace */ return -EFAULT; max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags); - trace_nr = min_t(u32, trace->nr - skip, max_depth - skip); - trace_len = trace_nr * sizeof(u64); - ips = trace->ip + skip; - hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0); - id = hash & (smap->n_buckets - 1); - bucket = READ_ONCE(smap->buckets[id]); + stackid->nr = min_t(u32, trace_nr - skip, max_depth - skip); + stackid->len = stackid->nr * sizeof(u64); + stackid->ips = trace->ip + skip; + stackid->hash = jhash2((const u32 *)stackid->ips, stackid->len / sizeof(u32), 0); + stackid->id = stackid->hash & (smap->n_buckets - 1); + stackid->bucket = READ_ONCE(smap->buckets[stackid->id]); + stackid->hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash; + return 0; +} + +static int stackid_fastpath(struct stackid *stackid, struct bpf_map *map, + const struct perf_callchain_entry *trace, u32 trace_nr, + u64 flags) +{ + int err; + + err = stackid_init(stackid, map, trace, trace_nr, flags); + if (err) + return err; - hash_matches = bucket && bucket->hash == hash; /* fast cmp */ - if (hash_matches && flags & BPF_F_FAST_STACK_CMP) - return id; + if (stackid->hash_matches && flags & BPF_F_FAST_STACK_CMP) + return stackid->id; + + if (stack_map_use_build_id(map)) + return -ENOENT; + if (stackid->hash_matches && stackid->bucket->nr == stackid->nr && + memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0) + return stackid->id; + if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) + return -EEXIST; + return -ENOENT; +} + +static struct stack_map_bucket * +stackid_new_bucket(struct stackid *stackid, struct bpf_map *map) +{ + struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); + struct bpf_stack_build_id *id_offs; + struct stack_map_bucket *bucket; + u32 i; + + bucket = (struct stack_map_bucket *) pcpu_freelist_pop(&smap->freelist); + if (unlikely(!bucket)) + return NULL; + + if (stack_map_use_build_id(map)) { + id_offs = (struct bpf_stack_build_id *)bucket->data; + for (i = 0; i < stackid->nr; i++) + id_offs[i].ip = stackid->ips[i]; + } else { + memcpy(bucket->data, stackid->ips, stackid->len); + } + + bucket->hash = stackid->hash; + bucket->nr = stackid->nr; + return bucket; +} + +static long stackid_install(struct stackid *stackid, struct bpf_map *map, + struct stack_map_bucket *new_bucket, u64 flags) +{ + struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); + bool user = flags & BPF_F_USER_STACK; + struct stack_map_bucket *old_bucket; + u32 trace_len; if (stack_map_use_build_id(map)) { struct bpf_stack_build_id *id_offs; - /* for build_id+offset, pop a bucket before slow cmp */ - new_bucket = (struct stack_map_bucket *) - pcpu_freelist_pop(&smap->freelist); - if (unlikely(!new_bucket)) - return -ENOMEM; - new_bucket->nr = trace_nr; id_offs = (struct bpf_stack_build_id *)new_bucket->data; - for (i = 0; i < trace_nr; i++) - id_offs[i].ip = ips[i]; - stack_map_get_build_id_offset(id_offs, trace_nr, user, false /* !may_fault */); - trace_len = trace_nr * sizeof(struct bpf_stack_build_id); - if (hash_matches && bucket->nr == trace_nr && - memcmp(bucket->data, new_bucket->data, trace_len) == 0) { + stack_map_get_build_id_offset(id_offs, stackid->nr, user, false /* !may_fault */); + trace_len = stackid->nr * sizeof(struct bpf_stack_build_id); + if (stackid->hash_matches && stackid->bucket->nr == stackid->nr && + memcmp(stackid->bucket->data, new_bucket->data, trace_len) == 0) { pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); - return id; + return stackid->id; } - if (bucket && !(flags & BPF_F_REUSE_STACKID)) { + if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) { pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); return -EEXIST; } - } else { - if (hash_matches && bucket->nr == trace_nr && - memcmp(bucket->data, ips, trace_len) == 0) - return id; - if (bucket && !(flags & BPF_F_REUSE_STACKID)) - return -EEXIST; - - new_bucket = (struct stack_map_bucket *) - pcpu_freelist_pop(&smap->freelist); - if (unlikely(!new_bucket)) - return -ENOMEM; - memcpy(new_bucket->data, ips, trace_len); } - new_bucket->hash = hash; - new_bucket->nr = trace_nr; - - old_bucket = xchg(&smap->buckets[id], new_bucket); + old_bucket = xchg(&smap->buckets[stackid->id], new_bucket); if (old_bucket) pcpu_freelist_push(&smap->freelist, &old_bucket->fnode); - return id; + return stackid->id; } BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, @@ -583,23 +627,36 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, { u32 elem_size = stack_map_data_size(map); bool user = flags & BPF_F_USER_STACK; + struct stack_map_bucket *new_bucket; struct perf_callchain_entry *trace; + struct stackid stackid; bool kernel = !user; u32 max_depth; + int err; if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID))) return -EINVAL; max_depth = stack_map_calculate_max_depth(map->value_size, elem_size, flags); - trace = get_perf_callchain(regs, kernel, user, max_depth, - false, false, 0); - if (unlikely(!trace)) - /* couldn't fetch the stack trace */ - return -EFAULT; + scoped_guard(preempt) { + trace = get_perf_callchain(regs, kernel, user, max_depth, + false, false, 0); + if (unlikely(!trace)) + /* couldn't fetch the stack trace */ + return -EFAULT; + + err = stackid_fastpath(&stackid, map, trace, trace->nr, flags); + if (err != -ENOENT) + return err; + + new_bucket = stackid_new_bucket(&stackid, map); + if (!new_bucket) + return -ENOMEM; + } - return __bpf_get_stackid(map, trace, flags); + return stackid_install(&stackid, map, new_bucket, flags); } const struct bpf_func_proto bpf_get_stackid_proto = { @@ -611,7 +668,7 @@ const struct bpf_func_proto bpf_get_stackid_proto = { .arg3_type = ARG_ANYTHING, }; -static __u64 count_kernel_ip(struct perf_callchain_entry *trace) +static __u64 count_kernel_ip(const struct perf_callchain_entry *trace) { __u64 nr_kernel = 0; @@ -626,10 +683,13 @@ static __u64 count_kernel_ip(struct perf_callchain_entry *trace) BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, struct bpf_map *, map, u64, flags) { + const struct perf_callchain_entry *trace; struct perf_event *event = ctx->event; - struct perf_callchain_entry *trace; + struct stack_map_bucket *new_bucket; + struct stackid stackid; bool kernel, user; __u64 nr_kernel; + u32 trace_nr; int ret; /* perf_sample_data doesn't have callchain, use bpf_get_stackid */ @@ -649,26 +709,28 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, return -EFAULT; nr_kernel = count_kernel_ip(trace); - __u64 nr = trace->nr; /* save original */ if (kernel) { - trace->nr = nr_kernel; - ret = __bpf_get_stackid(map, trace, flags); + trace_nr = nr_kernel; } else { /* user */ u64 skip = flags & BPF_F_SKIP_FIELD_MASK; + trace_nr = trace->nr; skip += nr_kernel; if (skip > BPF_F_SKIP_FIELD_MASK) return -EFAULT; flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip; - ret = __bpf_get_stackid(map, trace, flags); } - /* restore nr */ - trace->nr = nr; + ret = stackid_fastpath(&stackid, map, trace, trace_nr, flags); + if (ret != -ENOENT) + return ret; - return ret; + new_bucket = stackid_new_bucket(&stackid, map); + if (new_bucket) + return stackid_install(&stackid, map, new_bucket, flags); + return -ENOMEM; } const struct bpf_func_proto bpf_get_stackid_proto_pe = { @@ -680,19 +742,55 @@ const struct bpf_func_proto bpf_get_stackid_proto_pe = { .arg3_type = ARG_ANYTHING, }; +static u32 callchain_store(const struct perf_callchain_entry *trace, u32 trace_nr, + void *buf, u32 elem_size, u64 flags) +{ + bool user_build_id = flags & BPF_F_USER_BUILD_ID; + u32 skip = flags & BPF_F_SKIP_FIELD_MASK; + const u64 *ips; + u32 copy_len; + + trace_nr = trace_nr - skip; + copy_len = trace_nr * elem_size; + + ips = trace->ip + skip; + if (user_build_id) { + struct bpf_stack_build_id *id_offs = buf; + + for (u32 i = 0; i < trace_nr; i++) + id_offs[i].ip = ips[i]; + } else { + memcpy(buf, ips, copy_len); + } + return trace_nr; +} + +static long callchain_finalize(void *buf, u32 size, u32 trace_nr, u32 elem_size, + u64 flags, bool may_fault) +{ + bool user_build_id = flags & BPF_F_USER_BUILD_ID; + bool user = flags & BPF_F_USER_STACK; + u32 copy_len = trace_nr * elem_size; + + if (user_build_id) + stack_map_get_build_id_offset(buf, trace_nr, user, may_fault); + + if (size > copy_len) + memset(buf + copy_len, 0, size - copy_len); + return copy_len; +} + static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, - struct perf_callchain_entry *trace_in, void *buf, u32 size, u64 flags, bool may_fault) { - u32 trace_nr, copy_len, elem_size, max_depth; bool user_build_id = flags & BPF_F_USER_BUILD_ID; bool crosstask = task && task != current; u32 skip = flags & BPF_F_SKIP_FIELD_MASK; bool user = flags & BPF_F_USER_STACK; struct perf_callchain_entry *trace; + u32 trace_nr, elem_size, max_depth; bool kernel = !user; int err = -EINVAL; - u64 *ips; if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | BPF_F_USER_BUILD_ID))) @@ -718,13 +816,11 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, max_depth = stack_map_calculate_max_depth(size, elem_size, flags); + preempt_disable(); if (may_fault) rcu_read_lock(); /* need RCU for perf's callchain below */ - if (trace_in) { - trace = trace_in; - trace->nr = min_t(u32, trace->nr, max_depth); - } else if (kernel && task) { + if (kernel && task) { trace = get_callchain_entry_for_task(task, max_depth); } else { trace = get_perf_callchain(regs, kernel, user, max_depth, @@ -734,33 +830,18 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, if (unlikely(!trace) || trace->nr < skip) { if (may_fault) rcu_read_unlock(); + preempt_enable(); goto err_fault; } - trace_nr = trace->nr - skip; - copy_len = trace_nr * elem_size; - - ips = trace->ip + skip; - if (user_build_id) { - struct bpf_stack_build_id *id_offs = buf; - u32 i; - - for (i = 0; i < trace_nr; i++) - id_offs[i].ip = ips[i]; - } else { - memcpy(buf, ips, copy_len); - } + trace_nr = callchain_store(trace, trace->nr, buf, elem_size, flags); - /* trace/ips should not be dereferenced after this point */ + /* trace should not be dereferenced after this point */ if (may_fault) rcu_read_unlock(); + preempt_enable(); - if (user_build_id) - stack_map_get_build_id_offset(buf, trace_nr, user, may_fault); - - if (size > copy_len) - memset(buf + copy_len, 0, size - copy_len); - return copy_len; + return callchain_finalize(buf, size, trace_nr, elem_size, flags, may_fault); err_fault: err = -EFAULT; @@ -772,7 +853,7 @@ clear: BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size, u64, flags) { - return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */); + return __bpf_get_stack(regs, NULL, buf, size, flags, false /* !may_fault */); } const struct bpf_func_proto bpf_get_stack_proto = { @@ -781,14 +862,14 @@ const struct bpf_func_proto bpf_get_stack_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; BPF_CALL_4(bpf_get_stack_sleepable, struct pt_regs *, regs, void *, buf, u32, size, u64, flags) { - return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, true /* may_fault */); + return __bpf_get_stack(regs, NULL, buf, size, flags, true /* may_fault */); } const struct bpf_func_proto bpf_get_stack_sleepable_proto = { @@ -797,7 +878,7 @@ const struct bpf_func_proto bpf_get_stack_sleepable_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -807,14 +888,17 @@ static long __bpf_get_task_stack(struct task_struct *task, void *buf, u32 size, struct pt_regs *regs; long res = -EINVAL; - if (!try_get_task_stack(task)) + if (!try_get_task_stack(task)) { + memset(buf, 0, size); return -EFAULT; + } regs = task_pt_regs(task); if (regs) - res = __bpf_get_stack(regs, task, NULL, buf, size, flags, may_fault); + res = __bpf_get_stack(regs, task, buf, size, flags, may_fault); + else + memset(buf, 0, size); put_task_stack(task); - return res; } @@ -831,7 +915,7 @@ const struct bpf_func_proto bpf_get_task_stack_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -848,22 +932,48 @@ const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; +static int __bpf_get_stack_pe(const struct perf_callchain_entry *trace, u32 trace_nr, + void *buf, u32 size, u64 flags) +{ + bool user_build_id = flags & BPF_F_USER_BUILD_ID; + u64 skip = flags & BPF_F_SKIP_FIELD_MASK; + bool user = flags & BPF_F_USER_STACK; + u32 elem_size, max_depth, nr_trace; + bool kernel = !user; + + if (kernel && user_build_id) + return -EINVAL; + + elem_size = user_build_id ? sizeof(struct bpf_stack_build_id) : sizeof(u64); + if (unlikely(size % elem_size)) + return -EINVAL; + + max_depth = stack_map_calculate_max_depth(size, elem_size, flags); + trace_nr = min_t(u32, trace_nr, max_depth); + + if (trace_nr < skip) + return -EFAULT; + + nr_trace = callchain_store(trace, trace_nr, buf, elem_size, flags); + return callchain_finalize(buf, size, nr_trace, elem_size, flags, false /* !may_fault */); +} + BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, void *, buf, u32, size, u64, flags) { struct pt_regs *regs = (struct pt_regs *)(ctx->regs); + const struct perf_callchain_entry *trace; struct perf_event *event = ctx->event; - struct perf_callchain_entry *trace; bool kernel, user; int err = -EINVAL; __u64 nr_kernel; if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)) - return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */); + return __bpf_get_stack(regs, NULL, buf, size, flags, false /* !may_fault */); if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | BPF_F_USER_BUILD_ID))) @@ -880,27 +990,20 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, nr_kernel = count_kernel_ip(trace); if (kernel) { - __u64 nr = trace->nr; - - trace->nr = nr_kernel; - err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */); - - /* restore nr */ - trace->nr = nr; + err = __bpf_get_stack_pe(trace, nr_kernel, buf, size, flags); } else { /* user */ u64 skip = flags & BPF_F_SKIP_FIELD_MASK; skip += nr_kernel; if (skip > BPF_F_SKIP_FIELD_MASK) goto clear; - flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip; - err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */); + err = __bpf_get_stack_pe(trace, trace->nr, buf, size, flags); } - return err; clear: - memset(buf, 0, size); + if (err < 0) + memset(buf, 0, size); return err; } @@ -911,7 +1014,7 @@ const struct bpf_func_proto bpf_get_stack_proto_pe = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index ea2153cf28d0..4e6aafad33bd 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -812,7 +812,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * infinite loop check triggering, see * iter_active_depths_differ() */ - if (old_reg->iter.btf != cur_reg->iter.btf || + if (old_reg->type != cur_reg->type || + old_reg->iter.btf != cur_reg->iter.btf || old_reg->iter.btf_id != cur_reg->iter.btf_id || old_reg->iter.state != cur_reg->iter.state || /* ignore {old_reg,cur_reg}->iter.depth, see above */ diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 6db306d23b47..6874ba1424af 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -40,7 +40,6 @@ #include <linux/tracepoint.h> #include <linux/overflow.h> #include <linux/cookie.h> -#include <linux/verification.h> #include <linux/btf_ids.h> #include <net/netfilter/nf_bpf_link.h> @@ -637,7 +636,6 @@ int bpf_map_alloc_pages(const struct bpf_map *map, int nid, return ret; } - static int btf_field_cmp(const void *a, const void *b) { const struct btf_field *f1 = a, *f2 = b; @@ -1078,11 +1076,24 @@ static void bpf_map_mmap_close(struct vm_area_struct *vma) bpf_map_write_active_dec(map); } +static vm_fault_t bpf_map_mmap_fault(struct vm_fault *vmf) +{ + struct bpf_map *map = vmf->vma->vm_private_data; + + return map->ops->map_mmap_fault(map, vmf); +} + static const struct vm_operations_struct bpf_map_default_vmops = { .open = bpf_map_mmap_open, .close = bpf_map_mmap_close, }; +static const struct vm_operations_struct bpf_map_lazy_vmops = { + .open = bpf_map_mmap_open, + .close = bpf_map_mmap_close, + .fault = bpf_map_mmap_fault, +}; + static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma) { struct bpf_map *map = filp->private_data; @@ -1118,7 +1129,7 @@ out: return err; /* set default open/close callbacks */ - vma->vm_ops = &bpf_map_default_vmops; + vma->vm_ops = map->ops->map_mmap_fault ? &bpf_map_lazy_vmops : &bpf_map_default_vmops; vma->vm_private_data = map; vm_flags_clear(vma, VM_MAYEXEC); /* If mapping is read-only, then disallow potentially re-mapping with @@ -1599,13 +1610,6 @@ static int map_create_alloc(union bpf_attr *attr, bpfptr_t uattr, struct bpf_ver err = -EFAULT; goto free_map; } - - /* See libbpf: emit_signature_match() */ - BUILD_BUG_ON(offsetof(struct bpf_map, excl) != SHA256_DIGEST_SIZE); - BUILD_BUG_ON(!__same_type(map->excl, u32)); - BUILD_BUG_ON(offsetof(struct bpf_map, sha) != 0); - BUILD_BUG_ON(!__same_type(map->sha, u8[SHA256_DIGEST_SIZE])); - map->excl = 1; } else if (attr->excl_prog_hash_size) { bpf_log(log, "Invalid excl_prog_hash_size.\n"); err = -EINVAL; @@ -1657,7 +1661,7 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr, struct bpf_common_at err = security_bpf_map_create(map, attr, token, uattr.is_kernel); if (err) - goto free_map_sec; + goto free_map; err = bpf_map_alloc_id(map); if (err) @@ -1838,7 +1842,6 @@ free_key: return err; } - #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) @@ -2886,64 +2889,6 @@ static bool is_perfmon_prog_type(enum bpf_prog_type prog_type) } } -static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) -{ - switch (keyring_id) { - case 0: - return BPF_SIG_KEYRING_BUILTIN; - case (s32)(unsigned long)VERIFY_USE_SECONDARY_KEYRING: - return BPF_SIG_KEYRING_SECONDARY; - case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING: - return BPF_SIG_KEYRING_PLATFORM; - default: - return BPF_SIG_KEYRING_USER; - } -} - -static int bpf_prog_verify_signature(struct bpf_prog *prog, union bpf_attr *attr, - bool is_kernel, s32 *keyring_serial) -{ - bpfptr_t usig = make_bpfptr(attr->signature, is_kernel); - struct bpf_dynptr_kern sig_ptr, insns_ptr; - struct bpf_key *key = NULL; - void *sig; - int err = 0; - - /* - * Don't attempt to use kmalloc_large or vmalloc for signatures. - * Practical signature for BPF program should be below this limit. - */ - if (attr->signature_size > KMALLOC_MAX_CACHE_SIZE) - return -EINVAL; - - if (system_keyring_id_check(attr->keyring_id) == 0) - key = bpf_lookup_system_key(attr->keyring_id); - else - key = bpf_lookup_user_key(attr->keyring_id, 0); - - if (!key) - return -EINVAL; - - sig = kvmemdup_bpfptr(usig, attr->signature_size); - if (IS_ERR(sig)) { - bpf_key_put(key); - return PTR_ERR(sig); - } - - bpf_dynptr_init(&sig_ptr, sig, BPF_DYNPTR_TYPE_LOCAL, 0, - attr->signature_size); - bpf_dynptr_init(&insns_ptr, prog->insnsi, BPF_DYNPTR_TYPE_LOCAL, 0, - prog->len * sizeof(struct bpf_insn)); - - err = bpf_verify_pkcs7_signature((struct bpf_dynptr *)&insns_ptr, - (struct bpf_dynptr *)&sig_ptr, key); - if (!err) - *keyring_serial = bpf_key_serial(key); - bpf_key_put(key); - kvfree(sig); - return err; -} - static int bpf_prog_mark_insn_arrays_ready(struct bpf_prog *prog) { int err; @@ -3109,6 +3054,10 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at prog->aux->attach_btf = attach_btf; prog->aux->attach_btf_id = multi_func ? bpf_multi_func_btf_id[0] : attr->attach_btf_id; prog->aux->dst_prog = dst_prog; + if (dst_prog) { + prog->aux->saved_dst_prog_type = dst_prog->type; + prog->aux->saved_dst_attach_type = dst_prog->expected_attach_type; + } prog->aux->dev_bound = !!attr->prog_ifindex; prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; @@ -3133,17 +3082,8 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at /* eBPF programs must be GPL compatible to use GPL-ed functions */ prog->gpl_compatible = license_is_gpl_compatible(license) ? 1 : 0; - if (attr->signature) { - err = bpf_prog_verify_signature(prog, attr, uattr.is_kernel, - &prog->aux->sig.keyring_serial); - if (err) - goto free_prog; - prog->aux->sig.keyring_type = bpf_classify_keyring(attr->keyring_id); - prog->aux->sig.verdict = BPF_SIG_VERIFIED; - } else { - prog->aux->sig.keyring_type = BPF_SIG_KEYRING_NONE; - prog->aux->sig.verdict = BPF_SIG_UNSIGNED; - } + prog->aux->sig.keyring_type = BPF_SIG_KEYRING_NONE; + prog->aux->sig.verdict = BPF_SIG_UNSIGNED; prog->orig_prog = NULL; prog->jited = 0; @@ -3189,10 +3129,6 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at if (err < 0) goto free_prog; - err = security_bpf_prog_load(prog, attr, token, uattr.is_kernel); - if (err) - goto free_prog; - /* run eBPF verifier */ err = bpf_check(&prog, attr, uattr, attr_log); if (err < 0) @@ -3471,9 +3407,10 @@ static const char *bpf_link_type_strs[] = { static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) { const struct bpf_link *link = filp->private_data; - const struct bpf_prog *prog = link->prog; + const struct bpf_prog *prog; enum bpf_link_type type = link->type; char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; + u32 prog_id = 0; if (type < ARRAY_SIZE(bpf_link_type_strs) && bpf_link_type_strs[type]) { if (link->type == BPF_LINK_TYPE_KPROBE_MULTI) @@ -3490,13 +3427,20 @@ static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) } seq_printf(m, "link_id:\t%u\n", link->id); + rcu_read_lock(); + prog = READ_ONCE(link->prog); if (prog) { bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); + prog_id = prog->aux->id; + } + rcu_read_unlock(); + + if (prog) { seq_printf(m, "prog_tag:\t%s\n" "prog_id:\t%u\n", prog_tag, - prog->aux->id); + prog_id); } if (link->ops->show_fdinfo) link->ops->show_fdinfo(link, m); @@ -3564,7 +3508,6 @@ int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer) if (fd < 0) return fd; - id = bpf_link_alloc_id(link); if (id < 0) { put_unused_fd(fd); @@ -3626,10 +3569,12 @@ static void bpf_tracing_link_release(struct bpf_link *link) { struct bpf_tracing_link *tr_link = container_of(link, struct bpf_tracing_link, link.link); + int err; - WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link.node, - tr_link->trampoline, - tr_link->tgt_prog)); + err = bpf_trampoline_unlink_prog(&tr_link->link.node, + tr_link->trampoline, + tr_link->tgt_prog); + WARN_ONCE(err, "bpf_trampoline_unlink_prog failed: %d\n", err); bpf_trampoline_put(tr_link->trampoline); @@ -5535,6 +5480,7 @@ static int bpf_link_get_info_by_fd(struct file *file, { struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info); struct bpf_link_info info; + const struct bpf_prog *prog; u32 info_len = attr->info.info_len; int err; @@ -5549,8 +5495,12 @@ static int bpf_link_get_info_by_fd(struct file *file, info.type = link->type; info.id = link->id; - if (link->prog) - info.prog_id = link->prog->aux->id; + + rcu_read_lock(); + prog = READ_ONCE(link->prog); + if (prog) + info.prog_id = prog->aux->id; + rcu_read_unlock(); if (link->ops->fill_link_info) { err = link->ops->fill_link_info(link, &info); @@ -5565,7 +5515,6 @@ static int bpf_link_get_info_by_fd(struct file *file, return 0; } - static int token_get_info_by_fd(struct file *file, struct bpf_token *token, const union bpf_attr *attr, @@ -6567,7 +6516,6 @@ BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size) return __sys_bpf(cmd, KERNEL_BPFPTR(attr), attr_size, KERNEL_BPFPTR(NULL), 0); } - /* To shut up -Wmissing-prototypes. * This function is used by the kernel light skeleton * to load bpf programs when modules are loaded or during kernel boot. @@ -6623,7 +6571,7 @@ static const struct bpf_func_proto bpf_sys_bpf_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; const struct bpf_func_proto * __weak @@ -6670,7 +6618,7 @@ static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, .arg4_size = sizeof(u64), diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index e791ae065c39..13e1aabe6f88 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -753,9 +753,9 @@ static struct bpf_iter_reg task_vma_reg_info = { BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, bpf_callback_t, callback_fn, void *, callback_ctx, u64, flags) { - struct mmap_unlock_irq_work *work = NULL; + struct mmap_unlock_irq_work *work; struct vm_area_struct *vma; - bool irq_work_busy = false; + bool __maybe_unused mmput_needed = false; struct mm_struct *mm; int ret = -ENOENT; @@ -765,14 +765,43 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, if (!task) return -ENOENT; - mm = task->mm; + if (task == current) { + mm = task->mm; + } else { + /* + * Foreign task: pin task->mm against a concurrent exit_mm(). + * Use trylock on alloc_lock instead of get_task_mm()'s + * blocking task_lock() to avoid deadlocking the target task. + */ + if (!IS_ENABLED(CONFIG_MMU)) + return -EOPNOTSUPP; + if (irqs_disabled()) + return -EBUSY; + if (!spin_trylock(&task->alloc_lock)) + return -EBUSY; + mm = task->mm; + if (mm && !(task->flags & PF_KTHREAD)) { + mmget(mm); + mmput_needed = true; + } else { + mm = NULL; + } + spin_unlock(&task->alloc_lock); + } if (!mm) return -ENOENT; - irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); + work = bpf_mmap_unlock_guard_get(); + if (IS_ERR(work)) { + ret = PTR_ERR(work); + goto out; + } - if (irq_work_busy || !mmap_read_trylock(mm)) - return -EBUSY; + if (!mmap_read_trylock(mm)) { + bpf_mmap_unlock_guard_put(work); + ret = -EBUSY; + goto out; + } vma = find_vma(mm, start); @@ -782,6 +811,11 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, ret = 0; } bpf_mmap_unlock_mm(work, mm); +out: +#ifdef CONFIG_MMU + if (mmput_needed) + mmput_async(mm); +#endif return ret; } @@ -1161,6 +1195,8 @@ static void do_mmap_read_unlock(struct irq_work *entry) work = container_of(entry, struct mmap_unlock_irq_work, irq_work); mmap_read_unlock_non_owner(work->mm); + work->mm = NULL; + bpf_mmap_unlock_guard_put(work); } static int __init task_iter_init(void) diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 1a721fc4bef5..90b70ea0d370 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -529,6 +529,36 @@ bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_a return tnodes; } +/* + * The arena base against which save_args() converts the arguments marked + * with BTF_FMODEL_ARENA_ARG. Only the struct_ops indirect trampoline + * converts: it dispatches to a single prog whose arena is known at + * generation time. Return 0 when there is nothing to convert. + */ +u64 bpf_tramp_arena_base(const struct btf_func_model *m, + struct bpf_tramp_nodes *tnodes, u32 flags) +{ + const struct bpf_prog *prog; + int i; + + if (!(flags & BPF_TRAMP_F_INDIRECT) || + tnodes[BPF_TRAMP_FENTRY].nr_nodes != 1) + return 0; + + for (i = 0; i < m->nr_args; i++) + if (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG) + break; + if (i == m->nr_args) + return 0; + + /* Verification rejects an arena argument without an arena. */ + prog = tnodes[BPF_TRAMP_FENTRY].nodes[0]->link->prog; + if (WARN_ON_ONCE(!prog->aux->arena)) + return 0; + + return bpf_arena_get_kern_vm_start(prog->aux->arena); +} + static void bpf_tramp_image_free(struct bpf_tramp_image *im) { bpf_image_ksym_del(&im->ksym); @@ -670,6 +700,13 @@ out: return ERR_PTR(err); } +void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) +{ + trampoline_lock(tr); + tr->flags |= flags; + trampoline_unlock(tr); +} + static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex, const struct bpf_trampoline_ops *ops, void *data) { @@ -913,6 +950,13 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node, int cnt = 0, i; kind = bpf_attach_type_to_tramp(node->link->prog); + /* + * Arena ctx args are converted only by struct_ops indirect + * trampolines. They must never be attached to a generic trampoline. + */ + if (WARN_ON_ONCE(bpf_prog_has_arena_ctx_arg(node->link->prog))) + return -ENOTSUPP; + if (tr->extension_prog) /* cannot attach fentry/fexit if extension prog is attached. * cannot overwrite extension prog either. @@ -997,12 +1041,15 @@ static void bpf_shim_tramp_link_release(struct bpf_link *link) { struct bpf_shim_tramp_link *shim_link = container_of(link, struct bpf_shim_tramp_link, link.link); + int err; /* paired with 'shim_link->trampoline = tr' in bpf_trampoline_link_cgroup_shim */ if (!shim_link->trampoline) return; - WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link.node, shim_link->trampoline, NULL)); + err = bpf_trampoline_unlink_prog(&shim_link->link.node, shim_link->trampoline, NULL); + WARN_ONCE(err, "bpf_trampoline_unlink_prog failed: %d\n", err); + bpf_trampoline_put(shim_link->trampoline); } @@ -1536,6 +1583,7 @@ static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_ima if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); + tr->func.ftrace_managed = true; ftrace_hash_add(data->reg, data->entry, ip, addr); tr->cur_image = im; return 0; @@ -1584,7 +1632,17 @@ static void bpf_trampoline_multi_attach_init(struct bpf_trampoline *tr) static void bpf_trampoline_multi_attach_free(struct bpf_trampoline *tr) { - if (tr->multi_attach.old_image) + /* + * Only free old_image if it is no longer the active image. + * When bpf_trampoline_update() fails before modify_fentry_multi()/ + * unregister_fentry_multi() is called, cur_image is unchanged + * (cur_image == old_image) and ftrace still points to it. Freeing + * it would cause a UAF when ftrace calls into the freed memory. + * On success, cur_image is either a new image or NULL, so + * old_image != cur_image means the image is stale. + */ + if (tr->multi_attach.old_image && + tr->multi_attach.old_image != tr->cur_image) bpf_tramp_image_put(tr->multi_attach.old_image); tr->multi_attach.old_image = NULL; @@ -1708,19 +1766,21 @@ rollback_put: return err; } -int bpf_trampoline_multi_detach(struct bpf_prog *prog, struct bpf_tracing_multi_link *link) +void bpf_trampoline_multi_detach(struct bpf_prog *prog, + struct bpf_tracing_multi_link *link) { struct bpf_tracing_multi_data *data = &link->data; struct bpf_tracing_multi_node *mnode; - int i; + int i, err; trampoline_lock_all(); for_each_mnode(mnode, link) { data->entry = &mnode->entry; bpf_trampoline_multi_attach_init(mnode->trampoline); - WARN_ON_ONCE(__bpf_trampoline_unlink_prog(&mnode->node, mnode->trampoline, - NULL, &trampoline_multi_ops, data)); + err = __bpf_trampoline_unlink_prog(&mnode->node, mnode->trampoline, NULL, + &trampoline_multi_ops, data); + WARN_ONCE(err, "__bpf_trampoline_unlink_prog failed: %d\n", err); } if (ftrace_hash_count(data->unreg)) @@ -1737,7 +1797,6 @@ int bpf_trampoline_multi_detach(struct bpf_prog *prog, struct bpf_tracing_multi_ bpf_trampoline_put(mnode->trampoline); clear_tracing_multi_data(data); - return 0; } #undef for_each_mnode_cnt diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index fdc5fbb1f78c..e421ea2b80c3 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -22,6 +22,8 @@ #include <linux/ctype.h> #include <linux/error-injection.h> #include <linux/bpf_lsm.h> +#include <linux/security.h> +#include <linux/verification.h> #include <linux/btf_ids.h> #include <linux/poison.h> #include <linux/module.h> @@ -32,6 +34,7 @@ #include <linux/trace_events.h> #include <linux/kallsyms.h> +#include "diagnostics.h" #include "disasm.h" static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { @@ -191,6 +194,7 @@ struct bpf_verifier_stack_elem { struct bpf_verifier_stack_elem *next; /* length of verifier log at the time this state was pushed on stack */ u32 log_pos; + u64 diag_log_pos; }; #define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192 @@ -201,9 +205,11 @@ struct bpf_verifier_stack_elem { #define BPF_PRIV_STACK_MIN_SIZE 64 static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int parent_id); -static int release_reference_nomark(struct bpf_verifier_state *state, int id); +static int __release_reference_nomark(struct bpf_verifier_state *state, int id); +static int release_reference_nomark(struct bpf_verifier_env *env, int id); static int release_reference(struct bpf_verifier_env *env, int id); static void invalidate_non_owning_refs(struct bpf_verifier_env *env); +static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env); static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env); static bool is_tracing_prog_type(enum bpf_prog_type type); static int ref_set_non_owning(struct bpf_verifier_env *env, @@ -250,27 +256,6 @@ static int validate_ref_obj(struct bpf_verifier_env *env, struct ref_obj_desc *r return 0; } -struct bpf_call_arg_meta { - struct bpf_map_desc map; - struct bpf_dynptr_desc dynptr; - struct ref_obj_desc ref_obj; - bool raw_mode; - bool pkt_access; - u8 release_regno; - int regno; - int access_size; - int mem_size; - u64 msize_max_value; - int func_id; - struct btf *btf; - u32 btf_id; - struct btf *ret_btf; - u32 ret_btf_id; - u32 subprogno; - struct btf_field *kptr_field; - s64 const_map_key; -}; - struct bpf_kfunc_meta { struct btf *btf; const struct btf_type *proto; @@ -322,6 +307,7 @@ static const char *btf_type_name(const struct btf *btf, u32 id) } static DEFINE_MUTEX(bpf_verifier_lock); +static DEFINE_MUTEX(btf_vmlinux_lock); static DEFINE_MUTEX(bpf_percpu_ma_lock); __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) @@ -422,7 +408,7 @@ static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) return btf_type_is_void(type); } -static const char *subprog_name(const struct bpf_verifier_env *env, int subprog) +const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog) { struct bpf_func_info *info; @@ -652,7 +638,6 @@ static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type type, bool first_slot, int id, int parent_id); - static void mark_dynptr_stack_regs(struct bpf_verifier_env *env, struct bpf_reg_state *sreg1, struct bpf_reg_state *sreg2, @@ -831,6 +816,10 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, if (dynptr_type_referenced(state->stack[spi].spilled_ptr.dynptr.type) && dynptr_ref_cnt(env, state->stack[spi].spilled_ptr.parent_id) <= 1) { verbose(env, "cannot overwrite referenced dynptr\n"); + bpf_diag_res( + env, env->insn_idx, "referenced dynptr overwrite", + "This stack slot contains a dynptr that owns or protects a referenced resource. Overwriting the last dynptr for that resource would lose the verifier-tracked release path.", + "Release or clone the dynptr so another live dynptr still tracks the referenced resource before overwriting this stack slot."); return -EINVAL; } @@ -901,36 +890,39 @@ static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_re return true; } -static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - enum bpf_arg_type arg_type) +static enum bpf_dynptr_type dynptr_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg) { - struct bpf_func_state *state = bpf_func(env, reg); - enum bpf_dynptr_type dynptr_type; + struct bpf_func_state *state; int spi; + if (reg->type == CONST_PTR_TO_DYNPTR) + return reg->dynptr.type; + + spi = dynptr_get_spi(env, reg); + if (spi < 0) + return BPF_DYNPTR_TYPE_INVALID; + state = bpf_func(env, reg); + return state->stack[spi].spilled_ptr.dynptr.type; +} + +static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + enum bpf_arg_type arg_type) +{ /* ARG_PTR_TO_DYNPTR takes any type of dynptr */ if (arg_type == ARG_PTR_TO_DYNPTR) return true; - dynptr_type = arg_to_dynptr_type(arg_type); - if (reg->type == CONST_PTR_TO_DYNPTR) { - return reg->dynptr.type == dynptr_type; - } else { - spi = dynptr_get_spi(env, reg); - if (spi < 0) - return false; - return state->stack[spi].spilled_ptr.dynptr.type == dynptr_type; - } + return dynptr_reg_type(env, reg) == arg_to_dynptr_type(arg_type); } static void __mark_reg_known_zero(struct bpf_reg_state *reg); static bool in_rcu_cs(struct bpf_verifier_env *env); -static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta); +static bool is_kfunc_rcu_protected(struct bpf_call_arg_meta *meta); static int mark_stack_slots_iter(struct bpf_verifier_env *env, - struct bpf_kfunc_call_arg_meta *meta, + struct bpf_call_arg_meta *meta, struct bpf_reg_state *reg, int insn_idx, struct btf *btf, u32 btf_id, int nr_slots) { @@ -1060,10 +1052,10 @@ static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_s } static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx); -static int release_irq_state(struct bpf_verifier_state *state, int id); +static int release_irq_state(struct bpf_verifier_env *env, int id); static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env, - struct bpf_kfunc_call_arg_meta *meta, + struct bpf_call_arg_meta *meta, struct bpf_reg_state *reg, int insn_idx, int kfunc_class) { @@ -1113,13 +1105,23 @@ static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_r if (st->irq.kfunc_class != kfunc_class) { const char *flag_kfunc = st->irq.kfunc_class == IRQ_NATIVE_KFUNC ? "native" : "lock"; const char *used_kfunc = kfunc_class == IRQ_NATIVE_KFUNC ? "native" : "lock"; + const char *reason; verbose(env, "irq flag acquired by %s kfuncs cannot be restored with %s kfuncs\n", flag_kfunc, used_kfunc); + reason = bpf_diag_fmt(env, + "This IRQ flag was saved by %s IRQ kfuncs, but the restore call " + "belongs to the %s IRQ kfunc family. Save and restore operations " + "must use the same family.", + flag_kfunc, used_kfunc); + bpf_diag_irq(env, env->insn_idx, "IRQ flag restore mismatch", reason, + "Restore the flag with the matching IRQ restore kfunc for the save " + "operation that created it.", + bpf_diag_irq_depth(env->cur_state)); return -EINVAL; } - err = release_irq_state(env->cur_state, st->id); + err = release_irq_state(env, st->id); WARN_ON_ONCE(err && err != -EACCES); if (err) { int insn_idx = 0; @@ -1133,6 +1135,11 @@ static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_r verbose(env, "cannot restore irq state out of order, expected id=%d acquired at insn_idx=%d\n", env->cur_state->active_irq_id, insn_idx); + bpf_diag_irq(env, env->insn_idx, "IRQ flag restore out of order", + "IRQ-disabled regions must be restored in last-in, first-out order, " + "but this restore does not match the currently active IRQ flag.", + "Restore nested IRQ flags in the reverse order they were saved.", + bpf_diag_irq_depth(env->cur_state)); return err; } @@ -1434,6 +1441,7 @@ static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int par s->type = REF_TYPE_PTR; s->id = ++env->id_gen; s->parent_id = parent_id; + bpf_diag_record_ref_acquire(env, insn_idx, s->id); return s->id; } @@ -1453,6 +1461,8 @@ static int acquire_lock_state(struct bpf_verifier_env *env, int insn_idx, enum r state->active_locks++; state->active_lock_id = id; state->active_lock_ptr = ptr; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_LOCK, true, + state->active_locks); return 0; } @@ -1468,6 +1478,8 @@ static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx) s->id = ++env->id_gen; state->active_irq_id = s->id; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_IRQ, true, + bpf_diag_irq_depth(state)); return s->id; } @@ -1509,8 +1521,9 @@ static bool reg_is_referenced(struct bpf_verifier_env *env, const struct bpf_reg return find_reference_state(env->cur_state, reg->id); } -static int release_lock_state(struct bpf_verifier_state *state, int type, int id, void *ptr) +static int release_lock_state(struct bpf_verifier_env *env, int type, int id, void *ptr) { + struct bpf_verifier_state *state = env->cur_state; void *prev_ptr = NULL; u32 prev_id = 0; int i; @@ -1523,6 +1536,8 @@ static int release_lock_state(struct bpf_verifier_state *state, int type, int id /* Reassign active lock (id, ptr). */ state->active_lock_id = prev_id; state->active_lock_ptr = prev_ptr; + bpf_diag_record_context(env, env->insn_idx, BPF_DIAG_CONTEXT_LOCK, + false, state->active_locks); return 0; } if (state->refs[i].type & REF_TYPE_LOCK_MASK) { @@ -1533,8 +1548,9 @@ static int release_lock_state(struct bpf_verifier_state *state, int type, int id return -EINVAL; } -static int release_irq_state(struct bpf_verifier_state *state, int id) +static int release_irq_state(struct bpf_verifier_env *env, int id) { + struct bpf_verifier_state *state = env->cur_state; u32 prev_id = 0; int i; @@ -1547,6 +1563,8 @@ static int release_irq_state(struct bpf_verifier_state *state, int id) if (state->refs[i].id == id) { release_reference_state(state, i); state->active_irq_id = prev_id; + bpf_diag_record_context(env, env->insn_idx, BPF_DIAG_CONTEXT_IRQ, + false, bpf_diag_irq_depth(state)); return 0; } else { prev_id = state->refs[i].id; @@ -1610,6 +1628,8 @@ static int copy_func_state(struct bpf_func_state *dst, const struct bpf_func_state *src) { memcpy(dst, src, offsetof(struct bpf_func_state, stack)); + /* Instruction accounting is path-local, not part of verifier state. */ + dst->insns_subtotal = 0; return copy_stack_state(dst, src); } @@ -1689,7 +1709,6 @@ static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_sta return true; } - void bpf_free_backedges(struct bpf_scc_visit *visit) { struct bpf_scc_backedge *backedge, *next; @@ -1716,6 +1735,7 @@ static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, err = bpf_copy_verifier_state(cur, &head->st); if (err) return err; + bpf_diag_event_log_restore(env, head->diag_log_pos); } if (pop_log) bpf_vlog_reset(&env->log, head->log_pos); @@ -1759,6 +1779,7 @@ static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, elem->prev_insn_idx = prev_insn_idx; elem->next = env->head; elem->log_pos = env->log.end_pos; + elem->diag_log_pos = bpf_diag_event_log_save(env); env->head = elem; env->stack_size++; err = bpf_copy_verifier_state(&elem->st, cur); @@ -1805,6 +1826,17 @@ static const int caller_saved[CALLER_SAVED_REGS] = { BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 }; +static void bpf_diag_record_caller_saved(struct bpf_verifier_env *env, + struct bpf_reg_state *regs) +{ + int i; + + for (i = 1; i < CALLER_SAVED_REGS; i++) { + bpf_diag_record_scrub(env, ®s[caller_saved[i]], + BPF_DIAG_MOD_CALLER_SAVED); + } +} + /* This helper doesn't clear reg->id */ static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) { @@ -1872,32 +1904,34 @@ static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type ty reg->dynptr.first_slot = first_slot; } -static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) +/* + * Refine the return type of the bpf_map_lookup_elem() for special map types: + * map-in-map, xskmap, sockmap and sockhash. + */ +static void refine_map_lookup_value(struct bpf_reg_state *reg) { - if (base_type(reg->type) == PTR_TO_MAP_VALUE) { - const struct bpf_map *map = reg->map_ptr; + enum bpf_type_flag maybe_null = reg->type & PTR_MAYBE_NULL; + const struct bpf_map *map = reg->map_ptr; - if (map->inner_map_meta) { - reg->type = CONST_PTR_TO_MAP; - reg->map_ptr = map->inner_map_meta; - /* transfer reg's id which is unique for every map_lookup_elem - * as UID of the inner map. - */ - if (btf_record_has_field(map->inner_map_meta->record, - BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) { - reg->map_uid = reg->id; - } - } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { - reg->type = PTR_TO_XDP_SOCK; - } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || - map->map_type == BPF_MAP_TYPE_SOCKHASH) { - reg->type = PTR_TO_SOCKET; - } else { - reg->type = PTR_TO_MAP_VALUE; - } - return; + if (map->inner_map_meta) { + reg->type = CONST_PTR_TO_MAP | maybe_null; + reg->map_ptr = map->inner_map_meta; + /* transfer reg's id which is unique for every map_lookup_elem + * as UID of the inner map. + */ + if (btf_record_has_field(map->inner_map_meta->record, + BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) + reg->map_uid = reg->id; + } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { + reg->type = PTR_TO_XDP_SOCK | maybe_null; + } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || + map->map_type == BPF_MAP_TYPE_SOCKHASH) { + reg->type = PTR_TO_SOCKET | maybe_null; } +} +static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) +{ reg->type &= ~PTR_MAYBE_NULL; } @@ -2147,12 +2181,9 @@ out: /* Mark a register as having a completely unknown (scalar) value. */ void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg) { - s32 subreg_def = reg->subreg_def; - memset(reg, 0, sizeof(*reg)); reg->type = SCALAR_VALUE; reg->var_off = tnum_unknown; - reg->subreg_def = subreg_def; __mark_reg_unbounded(reg); } @@ -2228,7 +2259,6 @@ static int mark_btf_ld_reg(struct bpf_verifier_env *env, } } -#define DEF_NOT_SUBREG (0) static void init_reg_state(struct bpf_verifier_env *env, struct bpf_func_state *state) { @@ -2237,7 +2267,6 @@ static void init_reg_state(struct bpf_verifier_env *env, for (i = 0; i < MAX_BPF_REG; i++) { bpf_mark_reg_not_init(env, ®s[i]); - regs[i].subreg_def = DEF_NOT_SUBREG; } /* frame pointer */ @@ -2261,6 +2290,7 @@ static void init_func_state(struct bpf_verifier_env *env, { state->callsite = callsite; state->frameno = frameno; + bpf_diag_init_frame(env, state); state->subprogno = subprogno; state->callback_ret_range = retval_range(0, 0); init_reg_state(env, state); @@ -2283,6 +2313,7 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, elem->prev_insn_idx = prev_insn_idx; elem->next = env->head; elem->log_pos = env->log.end_pos; + elem->diag_log_pos = bpf_diag_event_log_save(env); env->head = elem; env->stack_size++; if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { @@ -2309,7 +2340,6 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, return &elem->st; } - static int cmp_subprogs(const void *a, const void *b) { return ((struct bpf_subprog_info *)a)->start - @@ -2490,6 +2520,83 @@ int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, return 0; } +#define BPF_FD_SLOT_BTF 1UL + +static void fd_slot_set_map(struct bpf_fd_array *slot, struct bpf_map *map) +{ + slot->val = (unsigned long)map; +} + +static void fd_slot_set_btf(struct bpf_fd_array *slot, struct btf *btf) +{ + slot->val = (unsigned long)btf | BPF_FD_SLOT_BTF; +} + +static struct bpf_map *fd_slot_map(struct bpf_fd_array slot) +{ + if (slot.val & BPF_FD_SLOT_BTF) + return NULL; + return (struct bpf_map *)slot.val; +} + +static struct btf *fd_slot_btf(struct bpf_fd_array slot) +{ + if (!(slot.val & BPF_FD_SLOT_BTF)) + return NULL; + return (struct btf *)(slot.val & ~BPF_FD_SLOT_BTF); +} + +static struct btf * +fd_array_get_btf_continuous(struct bpf_verifier_env *env, u32 idx) +{ + struct btf *btf; + + if (idx >= env->fd_array_cnt) { + verbose(env, "kfunc fd_idx %u out of bounds, fd_array_cnt %u\n", + idx, env->fd_array_cnt); + return ERR_PTR(-EINVAL); + } + btf = fd_slot_btf(env->fd_array[idx]); + if (!btf) { + verbose(env, "kfunc fd_idx %u is not a module BTF\n", idx); + return ERR_PTR(-EINVAL); + } + btf_get(btf); + return btf; +} + +static struct btf * +fd_array_get_btf_sparse(struct bpf_verifier_env *env, u32 idx) +{ + struct btf *btf; + int btf_fd; + + if (copy_from_bpfptr_offset(&btf_fd, env->fd_array_raw, + (size_t)idx * sizeof(btf_fd), sizeof(btf_fd))) + return ERR_PTR(-EFAULT); + btf = btf_get_by_fd(btf_fd); + if (IS_ERR(btf)) { + verbose(env, "invalid module BTF fd specified\n"); + return btf; + } + return btf; +} + +static struct btf *fd_array_get_btf(struct bpf_verifier_env *env, u32 idx) +{ + if (env->signature) { + verbose(env, "signed program cannot bind any BTF\n"); + return ERR_PTR(-EACCES); + } + if (env->fd_array) + return fd_array_get_btf_continuous(env, idx); + if (!bpfptr_is_null(env->fd_array_raw)) + return fd_array_get_btf_sparse(env, idx); + + verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); + return ERR_PTR(-EPROTO); +} + static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) { @@ -2498,7 +2605,6 @@ static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, struct bpf_kfunc_btf *b; struct module *mod; struct btf *btf; - int btf_fd; tab = env->prog->aux->kfunc_btf_tab; b = bsearch(&kf_btf, tab->descs, tab->nr_descs, @@ -2509,22 +2615,9 @@ static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, return ERR_PTR(-E2BIG); } - if (bpfptr_is_null(env->fd_array)) { - verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); - return ERR_PTR(-EPROTO); - } - - if (copy_from_bpfptr_offset(&btf_fd, env->fd_array, - offset * sizeof(btf_fd), - sizeof(btf_fd))) - return ERR_PTR(-EFAULT); - - btf = btf_get_by_fd(btf_fd); - if (IS_ERR(btf)) { - verbose(env, "invalid module BTF fd specified\n"); + btf = fd_array_get_btf(env, offset); + if (IS_ERR(btf)) return btf; - } - if (!btf_is_module(btf)) { verbose(env, "BTF fd for kfunc is not a module BTF\n"); btf_put(btf); @@ -2582,6 +2675,26 @@ static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) return btf_vmlinux ?: ERR_PTR(-ENOENT); } +static struct btf *find_kfunc_desc_btf_cached(struct bpf_verifier_env *env, s16 offset) +{ + struct bpf_kfunc_btf kf_btf = { .offset = offset }; + struct bpf_kfunc_btf_tab *tab; + struct bpf_kfunc_btf *b; + + if (!offset) + return btf_vmlinux ?: ERR_PTR(-ENOENT); + if (offset < 0) + return ERR_PTR(-EINVAL); + + tab = env->prog->aux->kfunc_btf_tab; + if (!tab) + return ERR_PTR(-ENOENT); + + b = bsearch(&kf_btf, tab->descs, tab->nr_descs, + sizeof(tab->descs[0]), kfunc_btf_cmp_by_off); + return b ? b->btf : ERR_PTR(-ENOENT); +} + #define KF_IMPL_SUFFIX "_impl" static const struct btf_type *find_kfunc_impl_proto(struct bpf_verifier_log *log, @@ -2674,8 +2787,12 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env, return 0; } +static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + struct bpf_func_proto *proto); + int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) { + struct bpf_call_arg_meta meta; struct bpf_kfunc_btf_tab *btf_tab; struct btf_func_model func_model; struct bpf_kfunc_desc_tab *tab; @@ -2715,6 +2832,8 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) prog_aux->kfunc_tab = tab; } + env->prog->jit_required = 1; + /* func_id == 0 is always invalid, but instead of returning an error, be * conservative and wait until the code elimination pass before returning * error, so that invalid calls that get pruned out can be in BPF programs @@ -2759,26 +2878,41 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) if (err) return err; - desc = &tab->descs[tab->nr_descs++]; + memset(&meta, 0, sizeof(meta)); + meta.btf = kfunc.btf; + meta.func_id = kfunc.id; + meta.func_proto = kfunc.proto; + meta.func_name = kfunc.name; + meta.kfunc_flags = kfunc.flags ? *kfunc.flags : 0; + + tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1), GFP_KERNEL_ACCOUNT); + if (!tab) + return -ENOMEM; + prog_aux->kfunc_tab = tab; + + desc = &tab->descs[tab->nr_descs]; + memset(desc, 0, sizeof(*desc)); + + err = gen_kfunc_arg_proto(env, &meta, &desc->proto); + if (err) + return err; + desc->func_id = func_id; desc->offset = offset; desc->addr = addr; desc->func_model = func_model; + tab->nr_descs++; sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off, NULL); return 0; } -bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) -{ - return !!prog->aux->kfunc_tab; -} - -static int add_subprog_and_kfunc(struct bpf_verifier_env *env) +static int add_subprogs(struct bpf_verifier_env *env) { struct bpf_subprog_info *subprog = env->subprog_info; int i, ret, insn_cnt = env->prog->len, ex_cb_insn; struct bpf_insn *insn = env->prog->insnsi; + const char *operation, *suggestion; /* Add entry function. */ ret = add_subprog(env, 0); @@ -2786,20 +2920,26 @@ static int add_subprog_and_kfunc(struct bpf_verifier_env *env) return ret; for (i = 0; i < insn_cnt; i++, insn++) { - if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) && - !bpf_pseudo_kfunc_call(insn)) + if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) continue; if (!env->bpf_capable) { + if (bpf_pseudo_func(insn)) { + operation = "BPF function reference"; + suggestion = "Load this program with the required capability, or avoid BPF function references in unprivileged programs."; + } else { + operation = "BPF-to-BPF function call"; + suggestion = "Load this program with the required capability, or avoid BPF-to-BPF function calls in unprivileged programs."; + } verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); + bpf_diag_policy( + env, i, operation, + "loading or calling other BPF functions requires CAP_BPF or CAP_SYS_ADMIN", + suggestion); return -EPERM; } - if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) - ret = add_subprog(env, i + insn->imm + 1); - else - ret = bpf_add_kfunc_call(env, insn->imm, insn->off); - + ret = add_subprog(env, i + insn->imm + 1); if (ret < 0) return ret; } @@ -2837,6 +2977,32 @@ static int add_subprog_and_kfunc(struct bpf_verifier_env *env) return 0; } +static int add_kfuncs(struct bpf_verifier_env *env) +{ + struct bpf_insn *insn = env->prog->insnsi; + int i, ret, insn_cnt = env->prog->len; + + for (i = 0; i < insn_cnt; i++, insn++) { + if (!bpf_pseudo_kfunc_call(insn)) + continue; + + if (!env->bpf_capable) { + verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); + bpf_diag_policy( + env, i, "kernel function call", + "calling kernel functions requires CAP_BPF or CAP_SYS_ADMIN", + "Load this program with the required capability, or avoid kernel function calls in unprivileged programs."); + return -EPERM; + } + + ret = bpf_add_kfunc_call(env, insn->imm, insn->off); + if (ret < 0) + return ret; + } + + return 0; +} + static int check_subprogs(struct bpf_verifier_env *env) { int i, subprog_start, subprog_end, off, cur_subprog = 0; @@ -2870,6 +3036,12 @@ static int check_subprogs(struct bpf_verifier_env *env) off = i + bpf_jmp_offset(&insn[i]) + 1; if (off < subprog_start || off >= subprog_end) { verbose(env, "jump out of range from insn %d to %d\n", i, off); + bpf_diag_program_structure( + env, i, "jump out of range", + "Keep branch targets within the same subprogram, or use an explicit subprogram call.", + "Instruction %d jumps to instruction %d, but subprogram %d only contains instructions %d through %d. " + "A branch target must stay inside the same subprogram.", + i, off, cur_subprog, subprog_start, subprog_end - 1); return -EINVAL; } next: @@ -2882,6 +3054,11 @@ next: code != (BPF_JMP32 | BPF_JA) && code != (BPF_JMP | BPF_JA)) { verbose(env, "last insn is not an exit or jmp\n"); + bpf_diag_program_structure( + env, i, "subprogram can fall through", + "End each subprogram with an exit or an explicit jump that keeps control flow inside the subprogram.", + "Subprogram %d reaches its last instruction %d without an exit or jump, so control could continue into the next subprogram.", + cur_subprog, i); return -EINVAL; } subprog_start = subprog_end; @@ -2952,8 +3129,13 @@ static int sort_subprogs_topo(struct bpf_verifier_env *env) if (bpf_pseudo_func(&insn[idx])) continue; verbose(env, "recursive call from %s() to %s()\n", - subprog_name(env, cur), - subprog_name(env, callee)); + bpf_subprog_name(env, cur), + bpf_subprog_name(env, callee)); + bpf_diag_program_structure( + env, idx, "recursive subprogram call", + "Rewrite the recursion as an explicit bounded loop, or split the logic so subprogram calls do not form a cycle.", + "This bpf2bpf call would make the subprogram call graph recursive. " + "The verifier requires a finite, acyclic call graph so it can bound stack depth and analysis."); ret = -EINVAL; goto out; } @@ -2974,7 +3156,7 @@ static int sort_subprogs_topo(struct bpf_verifier_env *env) if (env->log.level & BPF_LOG_LEVEL2) for (i = 0; i < cnt; i++) verbose(env, "topo_order[%d] = %s\n", - i, subprog_name(env, env->subprog_topo_order[i])); + i, bpf_subprog_name(env, env->subprog_topo_order[i])); out: kvfree(dfs_stack); kvfree(color); @@ -2990,133 +3172,25 @@ static void mark_stack_slots_scratched(struct bpf_verifier_env *env, mark_stack_slot_scratched(env, spi - i); } -/* This function is supposed to be used by the following 32-bit optimization - * code only. It returns TRUE if the source or destination register operates - * on 64-bit, otherwise return FALSE. - */ -bool bpf_is_reg64(struct bpf_insn *insn, - u32 regno, struct bpf_reg_state *reg, enum bpf_reg_arg_type t) -{ - u8 code, class, op; - - code = insn->code; - class = BPF_CLASS(code); - op = BPF_OP(code); - if (class == BPF_JMP) { - /* BPF_EXIT for "main" will reach here. Return TRUE - * conservatively. - */ - if (op == BPF_EXIT) - return true; - if (op == BPF_CALL) { - /* BPF to BPF call will reach here because of marking - * caller saved clobber with DST_OP_NO_MARK for which we - * don't care the register def because they are anyway - * marked as NOT_INIT already. - */ - if (insn->src_reg == BPF_PSEUDO_CALL) - return false; - /* Helper call will reach here because of arg type - * check, conservatively return TRUE. - */ - if (t == SRC_OP) - return true; - - return false; - } - } - - if (class == BPF_ALU64 && op == BPF_END && (insn->imm == 16 || insn->imm == 32)) - return false; - - if (class == BPF_ALU64 || class == BPF_JMP || - (class == BPF_ALU && op == BPF_END && insn->imm == 64)) - return true; - - if (class == BPF_ALU || class == BPF_JMP32) - return false; - - if (class == BPF_LDX) { - if (t != SRC_OP) - return BPF_SIZE(code) == BPF_DW || BPF_MODE(code) == BPF_MEMSX; - /* LDX source must be ptr. */ - return true; - } - - if (class == BPF_STX) { - /* BPF_STX (including atomic variants) has one or more source - * operands, one of which is a ptr. Check whether the caller is - * asking about it. - */ - if (t == SRC_OP && reg->type != SCALAR_VALUE) - return true; - return BPF_SIZE(code) == BPF_DW; - } - - if (class == BPF_LD) { - u8 mode = BPF_MODE(code); - - /* LD_IMM64 */ - if (mode == BPF_IMM) - return true; - - /* Both LD_IND and LD_ABS return 32-bit data. */ - if (t != SRC_OP) - return false; - - /* Implicit ctx ptr. */ - if (regno == BPF_REG_6) - return true; - - /* Explicit source could be any width. */ - return true; - } - - if (class == BPF_ST) - /* The only source register for BPF_ST is a ptr. */ - return true; - - /* Conservatively return true at default. */ - return true; -} - -static void mark_insn_zext(struct bpf_verifier_env *env, - struct bpf_reg_state *reg) -{ - s32 def_idx = reg->subreg_def; - - if (def_idx == DEF_NOT_SUBREG) - return; - - env->insn_aux_data[def_idx - 1].zext_dst = true; - /* The dst will be zero extended, so won't be sub-register anymore. */ - reg->subreg_def = DEF_NOT_SUBREG; -} - static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno, enum bpf_reg_arg_type t) { - struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; struct bpf_reg_state *reg; - bool rw64; mark_reg_scratched(env, regno); reg = ®s[regno]; - rw64 = bpf_is_reg64(insn, regno, reg, t); if (t == SRC_OP) { /* check whether register used as source operand can be read */ if (reg->type == NOT_INIT) { verbose(env, "R%d !read_ok\n", regno); + bpf_diag_unreadable_reg(env, env->insn_idx, regno); return -EACCES; } /* We don't need to worry about FP liveness because it's read-only */ if (regno == BPF_REG_FP) return 0; - if (rw64) - mark_insn_zext(env, reg); - return 0; } else { /* check whether register used as dest operand can be written to */ @@ -3124,7 +3198,6 @@ static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *r verbose(env, "frame pointer is read only\n"); return -EACCES; } - reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; if (t == DST_OP) mark_reg_unknown(env, regs, regno); } @@ -3228,7 +3301,7 @@ static void linked_regs_unpack(u64 val, struct linked_regs *s) } } -static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) +const char *bpf_disasm_kfunc_name(void *data, const struct bpf_insn *insn) { const struct btf_type *func; struct btf *desc_btf; @@ -3236,18 +3309,20 @@ static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL) return NULL; - desc_btf = find_kfunc_desc_btf(data, insn->off); + desc_btf = find_kfunc_desc_btf_cached(data, insn->off); if (IS_ERR(desc_btf)) return "<error>"; func = btf_type_by_id(desc_btf, insn->imm); + if (!func || !btf_type_is_func(func)) + return "<error>"; return btf_name_by_offset(desc_btf, func->name_off); } void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn) { const struct bpf_insn_cbs cbs = { - .cb_call = disasm_kfunc_name, + .cb_call = bpf_disasm_kfunc_name, .cb_print = verbose, .private_data = env, }; @@ -3305,34 +3380,6 @@ static int mark_chain_precision_batch(struct bpf_verifier_env *env, return bpf_mark_chain_precision(env, starting_state, -1, NULL); } -static bool is_spillable_regtype(enum bpf_reg_type type) -{ - switch (base_type(type)) { - case PTR_TO_MAP_VALUE: - case PTR_TO_STACK: - case PTR_TO_CTX: - case PTR_TO_PACKET: - case PTR_TO_PACKET_META: - case PTR_TO_PACKET_END: - case PTR_TO_FLOW_KEYS: - case CONST_PTR_TO_MAP: - case PTR_TO_SOCKET: - case PTR_TO_SOCK_COMMON: - case PTR_TO_TCP_SOCK: - case PTR_TO_XDP_SOCK: - case PTR_TO_BTF_ID: - case PTR_TO_BUF: - case PTR_TO_MEM: - case PTR_TO_FUNC: - case PTR_TO_MAP_KEY: - case PTR_TO_ARENA: - return true; - default: - return false; - } -} - - /* check if register is a constant scalar value */ static bool is_reg_const(struct bpf_reg_state *reg, bool subreg32) { @@ -3346,13 +3393,18 @@ static u64 reg_const_value(struct bpf_reg_state *reg, bool subreg32) return subreg32 ? tnum_subreg(reg->var_off).value : reg->var_off.value; } +static bool is_pointer_regtype(enum bpf_reg_type type) +{ + return type != SCALAR_VALUE && type != NOT_INIT; +} + static bool __is_pointer_value(bool allow_ptr_leaks, const struct bpf_reg_state *reg) { if (allow_ptr_leaks) return false; - return reg->type != SCALAR_VALUE; + return is_pointer_regtype(reg->type); } static void clear_scalar_id(struct bpf_reg_state *reg) @@ -3389,6 +3441,7 @@ static void save_register_state(struct bpf_verifier_env *env, { int i; + bpf_diag_mod_begin(env, &state->stack[spi].spilled_ptr, reg, BPF_DIAG_MOD_SPILL); state->stack[spi].spilled_ptr = *reg; for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--) @@ -3397,6 +3450,8 @@ static void save_register_state(struct bpf_verifier_env *env, /* size < 8 bytes spill */ for (; i; i--) mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]); + + bpf_diag_mod_end(env); } static bool is_bpf_st_mem(struct bpf_insn *insn) @@ -3469,7 +3524,16 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, bpf_is_spilled_reg(&state->stack[spi]) && !bpf_is_spilled_scalar_reg(&state->stack[spi]) && size != BPF_REG_SIZE) { + const char *reason; + verbose(env, "attempt to corrupt spilled pointer on stack\n"); + reason = bpf_diag_fmt(env, + "This store writes %d bytes at stack offset %d into a stack slot that currently holds a spilled pointer. " + "Partial writes to spilled pointers are rejected because they can corrupt pointer metadata and leak kernel pointers.", + size, off); + bpf_diag_memory( + env, insn_idx, "stack spill corruption", reason, + "Write the full 8-byte spilled pointer slot, or use a separate stack slot for scalar data before overwriting only part of it."); return -EACCES; } @@ -3477,7 +3541,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, if (value_regno >= 0) reg = &cur->regs[value_regno]; if (!env->bypass_spec_v4) { - bool sanitize = reg && is_spillable_regtype(reg->type); + bool sanitize = reg && is_pointer_regtype(reg->type); for (i = 0; i < size; i++) { u8 type = state->stack[spi].slot_type[(slot - i) % @@ -3518,7 +3582,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, __mark_reg_known(tmp_reg, insn->imm); tmp_reg->type = SCALAR_VALUE; save_register_state(env, state, spi, tmp_reg, size); - } else if (reg && is_spillable_regtype(reg->type)) { + } else if (reg && is_pointer_regtype(reg->type)) { /* register containing pointer is being spilled into stack */ if (size != BPF_REG_SIZE) { verbose_linfo(env, insn_idx, "; "); @@ -3533,6 +3597,9 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, } else { u8 type = STACK_MISC; + if (bpf_is_spilled_reg(&state->stack[spi])) + bpf_diag_record_scrub(env, &state->stack[spi].spilled_ptr, + BPF_DIAG_MOD_WRITE); scrub_special_slot(state, spi); /* when we zero initialize stack slots mark them as such */ @@ -3693,6 +3760,8 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, if (err) return err; } + bpf_diag_record_scrub_stack(env, state, min_off, max_off, + BPF_DIAG_MOD_VAR_WRITE); return 0; } @@ -3703,14 +3772,21 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, * SCALAR. This function does not deal with register filling; the caller must * ensure that all spilled registers in the stack range have been marked as * read. + * + * STACK_SPILL bytes backed by spilled scalar const zeroes are also considered + * zero bytes. In that case, mark the contributing stack slots precise so + * pruning cannot reuse a zero-spill state for a later non-zero spill state. + * + * Returns an error if precision backtracking fails. */ -static void mark_reg_stack_read(struct bpf_verifier_env *env, - /* func where src register points to */ - struct bpf_func_state *ptr_state, - int min_off, int max_off, int dst_regno) +static int mark_reg_stack_read(struct bpf_verifier_env *env, + /* func where src register points to */ + struct bpf_func_state *ptr_state, + int min_off, int max_off, int dst_regno) { struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; + u64 zero_spill_mask = 0; int i, slot, spi; u8 *stype; int zeros = 0; @@ -3720,19 +3796,48 @@ static void mark_reg_stack_read(struct bpf_verifier_env *env, spi = slot / BPF_REG_SIZE; mark_stack_slot_scratched(env, spi); stype = ptr_state->stack[spi].slot_type; - if (stype[slot % BPF_REG_SIZE] != STACK_ZERO) - break; - zeros++; + if (stype[slot % BPF_REG_SIZE] == STACK_ZERO) { + zeros++; + continue; + } + if (stype[slot % BPF_REG_SIZE] == STACK_SPILL && + bpf_register_is_null(&ptr_state->stack[spi].spilled_ptr)) { + zero_spill_mask |= 1ull << spi; + zeros++; + continue; + } + break; } if (zeros == max_off - min_off) { /* Any access_size read into register is zero extended, * so the whole register == const_zero. */ __mark_reg_const_zero(env, &state->regs[dst_regno]); + if (zero_spill_mask) { + bpf_bt_set_frame_slot_mask(&env->bt, ptr_state->frameno, zero_spill_mask); + return mark_chain_precision_batch(env, env->cur_state); + } } else { /* have read misc data from the stack */ mark_reg_unknown(env, state->regs, dst_regno); } + + return 0; +} + +static void bpf_diag_stack_read_uninit(struct bpf_verifier_env *env, int off, int i, + int size) +{ + const char *reason; + + reason = bpf_diag_fmt(env, + "This rejected read uses %d bytes at stack offset %d, but byte %d in that range is uninitialized on this path. " + "Programs loaded with CAP_PERFMON can be allowed to read uninitialized stack bytes, but this program is being rejected without that allowance.", + size, off, i); + bpf_diag_memory( + env, env->insn_idx, "uninitialized stack read", reason, + "Initialize every byte in the stack range before reading it, adjust the offset and size so the read covers only initialized bytes, " + "or load with CAP_PERFMON if uninitialized stack reads are intended."); } /* Read the stack at 'off' and put the results into the register indicated by @@ -3754,6 +3859,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; struct bpf_reg_state *reg; u8 *stype, type; + int err; int insn_flags = INSN_F_STACK_ACCESS; int hist_spi = spi, hist_frame = reg_state->frameno; @@ -3763,6 +3869,12 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, mark_stack_slot_scratched(env, spi); check_fastcall_stack_contract(env, state, env->insn_idx, off); + /* + * Refine the in-progress load record's origin to the source stack slot. + */ + if (dst_regno >= 0) + bpf_diag_mod_begin(env, &state->regs[dst_regno], reg, BPF_DIAG_MOD_WRITE); + if (bpf_is_spilled_reg(®_state->stack[spi])) { u8 spill_size = 1; @@ -3781,11 +3893,6 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, if (size <= spill_size && bpf_stack_narrow_access_ok(off, size, spill_size)) { - /* The earlier check_reg_arg() has decided the - * subreg_def for this insn. Save it first. - */ - s32 subreg_def = state->regs[dst_regno].subreg_def; - if (env->bpf_capable && size == 4 && spill_size == 4 && get_reg_width(reg) <= 32) /* Ensure stack slot has an ID to build a relation @@ -3793,7 +3900,6 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, */ assign_scalar_id_before_mov(env, reg); state->regs[dst_regno] = *reg; - state->regs[dst_regno].subreg_def = subreg_def; /* Break the relation on a narrowing fill. * coerce_reg_to_size will adjust the boundaries. @@ -3823,6 +3929,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } else { verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); + bpf_diag_stack_read_uninit(env, off, i, size); } return -EACCES; } @@ -3836,7 +3943,10 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, __mark_reg_const_zero(env, &state->regs[dst_regno]); insn_flags = 0; /* not restoring original register state */ } else { - mark_reg_unknown(env, state->regs, dst_regno); + err = mark_reg_stack_read(env, reg_state, off, off + size, + dst_regno); + if (err) + return err; insn_flags = 0; /* not restoring original register state */ } } @@ -3878,11 +3988,15 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } else { verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); + bpf_diag_stack_read_uninit(env, off, i, size); } return -EACCES; } - if (dst_regno >= 0) - mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); + if (dst_regno >= 0) { + err = mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); + if (err) + return err; + } insn_flags = 0; /* we are not restoring spilled register */ } if (insn_flags) @@ -3936,7 +4050,10 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg min_off = reg_smin(reg) + off; max_off = reg_smax(reg) + off; - mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); + err = mark_reg_stack_read(env, ptr_state, min_off, max_off + size, + dst_regno); + if (err) + return err; check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off); return 0; } @@ -3964,11 +4081,19 @@ static int check_stack_read(struct bpf_verifier_env *env, * check_stack_read_fixed_off). */ if (dst_regno < 0 && var_off) { + const char *reason; char tn_buf[48]; tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", tn_buf, off, size); + reason = bpf_diag_fmt(env, + "The helper would access the stack through variable offset %s plus fixed offset %d and size %d. " + "Helper stack memory arguments require a constant stack offset and a precise initialized range.", + tn_buf, off, size); + bpf_diag_memory( + env, env->insn_idx, "variable stack access", reason, + "Use a fixed stack offset for helper memory arguments, or copy the needed bytes into a fixed stack slot first."); return -EACCES; } /* Variable offset is prohibited for unprivileged mode for simplicity @@ -3995,7 +4120,6 @@ static int check_stack_read(struct bpf_verifier_env *env, return err; } - /* check_stack_write dispatches to check_stack_write_fixed_off or * check_stack_write_var_off. * @@ -4054,14 +4178,17 @@ static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_s if (spi + 1 > subprog->max_out_stack_arg_cnt) subprog->max_out_stack_arg_cnt = spi + 1; + arg = &state->stack_arg_regs[spi]; + bpf_diag_mod_begin(env, arg, value_reg, BPF_DIAG_MOD_WRITE); + if (value_reg) { state->stack_arg_regs[spi] = *value_reg; } else { /* BPF_ST: store immediate, treat as scalar */ - arg = &state->stack_arg_regs[spi]; arg->type = SCALAR_VALUE; __mark_reg_known(arg, env->prog->insnsi[env->insn_idx].imm); } + bpf_diag_mod_end(env); state->no_stack_arg_load = true; return bpf_push_jmp_history(env, env->cur_state, INSN_F_STACK_ARG_ACCESS, spi, 0, 0); @@ -4094,7 +4221,9 @@ static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_st caller = vstate->frame[vstate->curframe - 1]; arg = &caller->stack_arg_regs[spi]; cur = vstate->frame[vstate->curframe]; + bpf_diag_mod_begin(env, &cur->regs[dst_regno], arg, BPF_DIAG_MOD_WRITE); cur->regs[dst_regno] = *arg; + bpf_diag_mod_end(env); return bpf_push_jmp_history(env, env->cur_state, INSN_F_STACK_ARG_ACCESS, spi, 0, 0); } @@ -4109,7 +4238,8 @@ static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx) } static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, - int nargs) + int nargs, const char *callee_name, const struct btf *btf, + const struct btf_param *args) { int i, spi; @@ -4117,8 +4247,14 @@ static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_fu spi = i - MAX_BPF_FUNC_REG_ARGS; if (spi >= caller->out_stack_arg_cnt || caller->stack_arg_regs[spi].type == NOT_INIT) { + const char *arg_name = NULL; + + if (args && args[i].name_off) + arg_name = btf_name_by_offset(btf, args[i].name_off); verbose(env, "callee expects %d args, stack arg%d is not initialized\n", nargs, spi + 1); + bpf_diag_stack_arg_uninit(env, env->insn_idx, nargs, spi, + callee_name, arg_name); return -EFAULT; } } @@ -4199,6 +4335,9 @@ static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_ int off, int size, u32 mem_size, bool zero_size_allowed) { + const char *proof = ""; + const char *start; + s64 max_start, max_end; int err; /* We may have adjusted the register pointing to memory region, so we @@ -4217,14 +4356,28 @@ static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_ reg_smin(reg) + off < 0)) { verbose(env, "%s min value is negative, either use unsigned index or do a if (index >=0) check.\n", reg_arg_name(env, argno)); - return -EACCES; + err = -EACCES; + if (bpf_diag_enabled(env)) { + start = bpf_diag_fmt_s64_sum(env, reg_smin(reg), off); + proof = bpf_diag_fmt( + env, "the minimal bound for a memory access is a negative value: %s", + start); + } + goto report_error; } + err = __check_mem_access(env, reg, argno, reg_smin(reg) + off, size, mem_size, zero_size_allowed); if (err) { verbose(env, "%s min value is outside of the allowed memory range\n", reg_arg_name(env, argno)); - return err; + if (bpf_diag_enabled(env)) { + start = bpf_diag_fmt_s64_sum(env, reg_smin(reg), off); + proof = bpf_diag_fmt( + env, "the minimal bound for a memory access is %s and is outside of the object of size %u", + start, mem_size); + } + goto report_error; } /* If we haven't set a max value then we need to bail since we can't be @@ -4234,17 +4387,36 @@ static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_ if (reg_umax(reg) >= BPF_MAX_VAR_OFF) { verbose(env, "%s unbounded memory access, make sure to bounds check any such access\n", reg_arg_name(env, argno)); - return -EACCES; + err = -EACCES; + if (bpf_diag_enabled(env)) + proof = bpf_diag_fmt( + env, "the maximal bound for a memory access is %llu and exceeds maximum allowed offset of %u", + reg_umax(reg), BPF_MAX_VAR_OFF); + goto report_error; } + err = __check_mem_access(env, reg, argno, reg_umax(reg) + off, size, mem_size, zero_size_allowed); if (err) { verbose(env, "%s max value is outside of the allowed memory range\n", reg_arg_name(env, argno)); - return err; + if (bpf_diag_enabled(env)) { + max_start = (s64)reg_umax(reg) + off; + max_end = max_start + size; + proof = bpf_diag_fmt( + env, "the maximal bound for a memory access is %lld: start %lld + access_size %d, beyond object_size %u", + max_end, max_start, size, mem_size); + } + goto report_error; } return 0; + +report_error: + bpf_diag_mem_bounds(env, env->insn_idx, reg_from_argno(argno), + reg_arg_name(env, argno), reg_type_str(env, reg->type), proof, + off, size, mem_size, reg); + return err; } static int __check_ptr_off_reg(struct bpf_verifier_env *env, @@ -4273,6 +4445,9 @@ static int __check_ptr_off_reg(struct bpf_verifier_env *env, if (!fixed_off_ok && reg->var_off.value != 0) { verbose(env, "dereference of modified %s ptr %s off=%lld disallowed\n", reg_type_str(env, reg->type), reg_arg_name(env, argno), reg->var_off.value); + bpf_diag_invalid_deref(env, env->insn_idx, reg_from_argno(argno), + reg_arg_name(env, argno), reg, + BPF_DIAG_DEREF_MODIFIED_PTR, reg->var_off.value); return -EACCES; } @@ -4349,7 +4524,8 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, */ if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value, kptr_field->kptr.btf, kptr_field->kptr.btf_id, - kptr_field->type != BPF_KPTR_UNREF)) + kptr_field->type != BPF_KPTR_UNREF, + !type_is_alloc(reg->type))) goto bad_type; return 0; bad_type: @@ -4375,7 +4551,9 @@ static bool in_sleepable(struct bpf_verifier_env *env) static bool in_rcu_cs(struct bpf_verifier_env *env) { return env->cur_state->active_rcu_locks || + env->cur_state->active_preempt_locks || env->cur_state->active_locks || + env->cur_state->active_irq_id || !in_sleepable(env); } @@ -4394,6 +4572,9 @@ BTF_ID(struct, task_struct) #ifdef CONFIG_CRYPTO BTF_ID(struct, bpf_crypto_ctx) #endif +#ifdef CONFIG_INET +BTF_ID(struct, bpf_ksock) +#endif BTF_SET_END(rcu_protected_types) static bool rcu_protected_object(const struct btf *btf, u32 btf_id) @@ -4604,7 +4785,7 @@ static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state * } static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, - const struct bpf_call_arg_meta *meta, + const struct bpf_func_proto *fn, enum bpf_access_type t) { enum bpf_prog_type prog_type = resolve_prog_type(env->prog); @@ -4628,8 +4809,8 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, case BPF_PROG_TYPE_LWT_XMIT: case BPF_PROG_TYPE_SK_SKB: case BPF_PROG_TYPE_SK_MSG: - if (meta) - return meta->pkt_access; + if (fn) + return fn->pkt_access; env->seen_direct_write = true; return true; @@ -4790,7 +4971,6 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, valid = false; } - if (valid) { env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; @@ -4844,6 +5024,22 @@ static bool is_arena_reg(struct bpf_verifier_env *env, int regno) return reg->type == PTR_TO_ARENA; } +static bool is_load_acq_unsafe(struct bpf_verifier_env *env, int regno, + struct bpf_insn *insn) +{ + const struct bpf_reg_state *reg = reg_state(env, regno); + + /* + * A BPF_LOAD_ACQ is not rewritten to a BPF_PROBE_MEM load by the + * verifier, unlike a regular BPF_LDX. The JIT would emit a plain load + * with no exception table entry, so a fault (e.g. NULL deref) crashes + * the kernel instead of being handled. Reject the source pointer types + * that would have needed that protection, the remaining ones stay + * allowed. + */ + return insn->imm == BPF_LOAD_ACQ && bpf_may_fault_on_deref(reg->type); +} + /* Return false if @regno contains a pointer whose type isn't supported for * atomic instruction @insn. */ @@ -4860,7 +5056,8 @@ static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno, return false; if (is_arena_reg(env, regno)) return bpf_jit_supports_insn(insn, true); - + if (is_load_acq_unsafe(env, regno, insn)) + return false; return true; } @@ -4873,6 +5070,18 @@ static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { [CONST_PTR_TO_MAP] = btf_bpf_map_id, }; +static enum bpf_reg_type lookup_reg2btf_ids(u32 ref_id) +{ + enum bpf_reg_type type; + + for (type = 0; type < __BPF_REG_TYPE_MAX; type++) { + if (reg2btf_ids[type] && *reg2btf_ids[type] == ref_id) + return type; + } + + return NOT_INIT; +} + static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg) { /* A referenced register is always trusted. */ @@ -5187,10 +5396,6 @@ continue_func: if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn)) return -EFAULT; if (subprog[sidx].is_async_cb) { - if (subprog[sidx].has_tail_call) { - verifier_bug(env, "subprog has tail_call and async cb"); - return -EFAULT; - } /* async callbacks don't increase bpf prog stack size unless called directly */ if (!bpf_pseudo_call(insn + i)) continue; @@ -5213,8 +5418,8 @@ continue_func: if (!priv_stack_supported) subprog[idx].priv_stack_mode = NO_PRIV_STACK; - if (subprog[idx].has_tail_call) - tail_call_reachable = true; + /* sync tail_call_reachable with callee state on entry */ + tail_call_reachable = subprog[idx].has_tail_call; frame = bpf_subprog_is_global(env, idx) ? 0 : frame + 1; if (frame >= MAX_CALL_FRAMES) { @@ -5231,8 +5436,8 @@ continue_func: */ if (tail_call_reachable) { for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { - if (subprog[tmp].is_exception_cb) { - verbose(env, "cannot tail call within exception cb\n"); + if (subprog[tmp].is_cb) { + verbose(env, "cannot tail call within callback\n"); return -EINVAL; } if (subprog[tmp].stack_arg_cnt) { @@ -5577,6 +5782,8 @@ int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, u64 addr; int err; + if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY || map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) + return -EINVAL; err = map->ops->map_direct_value_addr(map, &addr, off); if (err) return err; @@ -5789,6 +5996,11 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, return -EACCES; } + if (atype != BPF_READ && bpf_may_fault_on_deref(reg->type)) { + verbose(env, "only read is supported\n"); + return -EACCES; + } + if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) { if (!btf_is_kernel(reg->btf)) { verifier_bug(env, "reg->btf must be kernel btf"); @@ -5801,8 +6013,7 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, reg_arg_name(env, argno), tname, off, size); } else { /* Writes are permitted with default btf_struct_access for - * program allocated objects (which always have id > 0), - * but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC. + * program allocated objects (which always have id > 0). */ if (atype != BPF_READ && !type_is_ptr_alloc_obj(reg->type)) { verbose(env, "only read is supported\n"); @@ -6069,6 +6280,51 @@ static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val) reg_bounds_sync(dst_reg); } +static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, + int bpf_size, int value_regno, bool is_ldsx) +{ + struct bpf_reg_state *regs = cur_regs(env); + int size = bpf_size_to_bytes(bpf_size); + struct bpf_map *map = reg->map_ptr; + + switch (map->map_type) { + case BPF_MAP_TYPE_INSN_ARRAY: + if (bpf_size != BPF_DW) { + verbose(env, "Invalid read of %d bytes from insn_array\n", size); + return -EACCES; + } + regs[value_regno] = *reg; + add_scalar_to_reg(®s[value_regno], off); + regs[value_regno].type = PTR_TO_INSN; + return 0; + case BPF_MAP_TYPE_PERCPU_ARRAY: + goto reg_unknown; + default: + break; + } + + /* If map is read-only, track its contents as scalars. */ + if (tnum_is_const(reg->var_off) && + bpf_map_is_rdonly(map) && + map->ops->map_direct_value_addr) { + int map_off = off + reg->var_off.value; + u64 val = 0; + int err; + + err = bpf_map_direct_read(map, map_off, size, &val, is_ldsx); + if (err) + return err; + + regs[value_regno].type = SCALAR_VALUE; + __mark_reg_known(®s[value_regno], val); + return 0; + } + +reg_unknown: + mark_reg_unknown(env, regs, value_regno); + return 0; +} + /* check whether memory at (regno + off) is accessible for t = (read | write) * if t==write, value_regno is a register which value is stored into memory * if t==read, value_regno is a register which will receive the value from memory @@ -6123,38 +6379,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (kptr_field) { err = check_map_kptr_access(env, value_regno, insn_idx, kptr_field); } else if (t == BPF_READ && value_regno >= 0) { - struct bpf_map *map = reg->map_ptr; - - /* - * If map is read-only, track its contents as scalars, - * unless it is an insn array (see the special case below) - */ - if (tnum_is_const(reg->var_off) && - bpf_map_is_rdonly(map) && - map->ops->map_direct_value_addr && - map->map_type != BPF_MAP_TYPE_INSN_ARRAY) { - int map_off = off + reg->var_off.value; - u64 val = 0; - - err = bpf_map_direct_read(map, map_off, size, - &val, is_ldsx); - if (err) - return err; - - regs[value_regno].type = SCALAR_VALUE; - __mark_reg_known(®s[value_regno], val); - } else if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) { - if (bpf_size != BPF_DW) { - verbose(env, "Invalid read of %d bytes from insn_array\n", - size); - return -EACCES; - } - regs[value_regno] = *reg; - add_scalar_to_reg(®s[value_regno], off); - regs[value_regno].type = PTR_TO_INSN; - } else { - mark_reg_unknown(env, regs, value_regno); - } + err = check_map_mem_read(env, reg, off, bpf_size, value_regno, is_ldsx); } } else if (base_type(reg->type) == PTR_TO_MEM) { bool rdonly_mem = type_is_rdonly_mem(reg->type); @@ -6163,6 +6388,9 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (type_may_be_null(reg->type)) { verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); + bpf_diag_invalid_deref(env, insn_idx, reg_from_argno(argno), + reg_arg_name(env, argno), reg, + BPF_DIAG_DEREF_NULLABLE_PTR, 0); return -EACCES; } @@ -6220,12 +6448,6 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b } else { mark_reg_known_zero(env, regs, value_regno); - /* A load of ctx field could have different - * actual load size with the one encoded in the - * insn. When the dst is PTR, it is for sure not - * a sub-register. - */ - regs[value_regno].subreg_def = DEF_NOT_SUBREG; if (base_type(info.reg_type) == PTR_TO_BTF_ID) { regs[value_regno].btf = info.btf; regs[value_regno].btf_id = info.btf_id; @@ -6319,18 +6541,38 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (t == BPF_READ && value_regno >= 0) mark_reg_unknown(env, regs, value_regno); } else { + enum bpf_diag_invalid_deref_kind kind = BPF_DIAG_DEREF_INVALID_PTR; + verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); + if (reg->type == SCALAR_VALUE) + kind = BPF_DIAG_DEREF_SCALAR; + else if (type_may_be_null(reg->type)) + kind = BPF_DIAG_DEREF_NULLABLE_PTR; + bpf_diag_invalid_deref(env, insn_idx, reg_from_argno(argno), + reg_arg_name(env, argno), reg, kind, 0); return -EACCES; } if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && regs[value_regno].type == SCALAR_VALUE) { - if (!is_ldsx) + if (!is_ldsx) { /* b/h/w load zero-extends, mark upper bits as known 0 */ coerce_reg_to_size(®s[value_regno], size); - else + } else { + /* + * Sign-extension can change the register value relative + * to a scalar it is linked with by id (e.g. a zero- + * extending fill of the same spilled stack slot), thus + * drop the shared id in that case. + */ + bool no_sext = reg_umax(®s[value_regno]) < + (1ULL << (size * BITS_PER_BYTE - 1)); + coerce_reg_to_size_sx(®s[value_regno], size); + if (!no_sext) + clear_scalar_id(®s[value_regno]); + } } return err; } @@ -6368,15 +6610,19 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, src_reg_type = regs[insn->src_reg].type; - /* Check if (src_reg + off) is readable. The state of dst_reg will be - * updated by this call. + /* + * check_stack_read_fixed_off() may refine the modification's origin to + * the source stack slot. */ + bpf_diag_mod_begin(env, ®s[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE); err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, argno_from_reg(insn->src_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, strict_alignment_once, is_ldsx); err = err ?: save_aux_ptr_type(env, src_reg_type, allow_trust_mismatch); err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], ctx); + if (!err) + bpf_diag_mod_end(env); return err; } @@ -6467,21 +6713,12 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return -EACCES; } - if (insn->imm & BPF_FETCH) { - if (insn->imm == BPF_CMPXCHG) - load_reg = BPF_REG_0; - else - load_reg = insn->src_reg; - + load_reg = bpf_atomic_load_reg(insn); + if (load_reg >= 0) { /* check and record load of old value */ err = check_reg_arg(env, load_reg, DST_OP); if (err) return err; - } else { - /* This instruction accesses a memory location but doesn't - * actually load it into a register. - */ - load_reg = -1; } dst_reg = cur_regs(env) + insn->dst_reg; @@ -6491,18 +6728,20 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, */ err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, -1, true, false); - if (!err && load_reg >= 0) + if (!err && load_reg >= 0) { + bpf_diag_mod_begin(env, cur_regs(env) + load_reg, NULL, BPF_DIAG_MOD_WRITE); err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, load_reg, true, false); + if (!err) + bpf_diag_mod_end(env); + } if (err) return err; - if (is_arena_reg(env, insn->dst_reg)) { - err = save_aux_ptr_type(env, PTR_TO_ARENA, false); - if (err) - return err; - } + err = save_aux_ptr_type(env, dst_reg->type, false); + if (err) + return err; /* Check whether we can write into the same memory. */ err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, true, false); @@ -6516,7 +6755,7 @@ static int check_atomic_load(struct bpf_verifier_env *env, { int err; - err = check_load_mem(env, insn, true, false, false, "atomic_load"); + err = check_reg_arg(env, insn->src_reg, SRC_OP); if (err) return err; @@ -6527,7 +6766,7 @@ static int check_atomic_load(struct bpf_verifier_env *env, return -EACCES; } - return 0; + return check_load_mem(env, insn, true, false, false, "atomic_load"); } static int check_atomic_store(struct bpf_verifier_env *env, @@ -6604,12 +6843,14 @@ static int check_stack_range_initialized( */ bool clobber = type == BPF_WRITE; /* - * Negative access_size signals global subprog/kfunc arg check where + * Negative access_size signals global subprog arg check where * STACK_POISON slots are acceptable. static stack liveness * might have determined that subprog doesn't read them, * but BTF based global subprog validation isn't accurate enough. */ bool allow_poison = access_size < 0 || clobber; + /* The call will initialize the memory; uninitialized stack allowed */ + bool raw_mode = meta && meta->arg_raw_mem.regno == reg_from_argno(argno); access_size = abs(access_size); @@ -6622,7 +6863,6 @@ static int check_stack_range_initialized( if (err) return err; - if (tnum_is_const(reg->var_off)) { min_off = max_off = reg->var_off.value + off; } else { @@ -6645,41 +6885,14 @@ static int check_stack_range_initialized( * helper return since specific bounds are unknown what may * cause uninitialized stack leaking. */ - if (meta && meta->raw_mode) - meta = NULL; + raw_mode = false; min_off = reg_smin(reg) + off; max_off = reg_smax(reg) + off; } - if (meta && meta->raw_mode) { - /* Ensure we won't be overwriting dynptrs when simulating byte - * by byte access in check_helper_call using meta.access_size. - * This would be a problem if we have a helper in the future - * which takes: - * - * helper(uninit_mem, len, dynptr) - * - * Now, uninint_mem may overlap with dynptr pointer. Hence, it - * may end up writing to dynptr itself when touching memory from - * arg 1. This can be relaxed on a case by case basis for known - * safe cases, but reject due to the possibilitiy of aliasing by - * default. - */ - for (i = min_off; i < max_off + access_size; i++) { - int stack_off = -i - 1; - - spi = bpf_get_spi(i); - /* raw_mode may write past allocated_stack */ - if (state->allocated_stack <= stack_off) - continue; - if (state->stack[spi].slot_type[stack_off % BPF_REG_SIZE] == STACK_DYNPTR) { - verbose(env, "potential write to dynptr at off=%d disallowed\n", i); - return -EACCES; - } - } - meta->access_size = access_size; - meta->regno = reg_from_argno(argno); + if (raw_mode) { + meta->arg_raw_mem.size = access_size; return 0; } @@ -6738,14 +6951,17 @@ mark: return 0; } -static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - int access_size, enum bpf_access_type access_type, - bool zero_size_allowed, - struct bpf_call_arg_meta *meta) +static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, int access_size, + enum bpf_access_type access_type, bool zero_size_allowed, + struct bpf_call_arg_meta *meta, bool *known_memory) { struct bpf_reg_state *regs = cur_regs(env); u32 *max_access; + if (known_memory) + *known_memory = true; + switch (base_type(reg->type)) { case PTR_TO_PACKET: case PTR_TO_PACKET_META: @@ -6815,6 +7031,8 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ if (zero_size_allowed && access_size == 0 && bpf_register_is_null(reg)) return 0; + if (known_memory && base_type(reg->type) != PTR_TO_CTX) + *known_memory = false; verbose(env, "%s type=%s ", reg_arg_name(env, argno), reg_type_str(env, reg->type)); @@ -6823,6 +7041,12 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ } } +enum bpf_mem_size_failure { + BPF_MEM_SIZE_FAIL_NONE, + BPF_MEM_SIZE_FAIL_MEMORY, + BPF_MEM_SIZE_FAIL_SIZE, +}; + /* verify arguments to helpers or kfuncs consisting of a pointer and an access * size. * @@ -6831,11 +7055,15 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ static int check_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg, struct bpf_reg_state *size_reg, argno_t mem_argno, - argno_t size_argno, enum bpf_access_type access_type, + argno_t size_argno, u32 access_type, bool zero_size_allowed, - struct bpf_call_arg_meta *meta) + struct bpf_call_arg_meta *meta, + enum bpf_mem_size_failure *failure) { - int err; + int err = 0; + + if (failure) + *failure = BPF_MEM_SIZE_FAIL_NONE; /* This is used to refine r0 return value bounds for helpers * that enforce this value as an upper bound on return values. @@ -6858,22 +7086,33 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, if (reg_smin(size_reg) < 0) { verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n", reg_arg_name(env, size_argno)); - return -EACCES; + err = -EACCES; + goto size_error; } if (reg_umin(size_reg) == 0 && !zero_size_allowed) { verbose(env, "%s invalid zero-sized read: u64=[%lld,%lld]\n", reg_arg_name(env, size_argno), reg_umin(size_reg), reg_umax(size_reg)); - return -EACCES; + err = -EACCES; + goto size_error; } if (reg_umax(size_reg) >= BPF_MAX_VAR_SIZ) { verbose(env, "%s unbounded memory access, use 'var &= const' or 'if (var < const)'\n", reg_arg_name(env, size_argno)); - return -EACCES; + err = -EACCES; + goto size_error; } - err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), - access_type, zero_size_allowed, meta); + + if (access_type & BPF_READ) + err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), + BPF_READ, zero_size_allowed, meta, NULL); + if (!err && access_type & BPF_WRITE) + err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), + BPF_WRITE, zero_size_allowed, meta, NULL); + if (err && failure) + *failure = BPF_MEM_SIZE_FAIL_MEMORY; + if (!err) { int regno = reg_from_argno(size_argno); @@ -6882,18 +7121,25 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, else err = mark_stack_arg_precision(env, arg_idx_from_argno(size_argno)); } + + return err; + +size_error: + if (failure) + *failure = BPF_MEM_SIZE_FAIL_SIZE; return err; } static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - argno_t argno, u32 mem_size) + argno_t argno, u32 mem_size, enum bpf_access_type access_type, + struct bpf_call_arg_meta *meta, bool *known_memory) { - bool may_be_null = type_may_be_null(reg->type); - struct bpf_reg_state saved_reg; - int err; + int size, err = 0; if (bpf_register_is_null(reg)) return 0; + if (known_memory) + *known_memory = true; if (mem_size > S32_MAX) { verbose(env, "%s memory size %u is too large\n", @@ -6901,48 +7147,83 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg return -EACCES; } - /* Assuming that the register contains a value check if the memory - * access is safe. Temporarily save and restore the register's state as - * the conversion shouldn't be visible to a caller. + /* + * Only a global subprog (meta == NULL) may read poisoned stack slots: + * its static stack liveness proved the callee body skips them. */ - if (may_be_null) { - saved_reg = *reg; - mark_ptr_not_null_reg(reg); + size = (!meta && base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size; + + if (access_type & BPF_READ) + err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta, + known_memory); + if (!err && (access_type & BPF_WRITE)) + err = check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, meta, + known_memory); + + return err; +} + +static int process_const_alloc_mem_size(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, struct ret_mem_desc *ret_mem) +{ + int regno = reg_from_argno(argno); + int err; + + if (ret_mem->found) { + verifier_bug(env, "only one allocation size argument permitted"); + return -EFAULT; } - int size = base_type(reg->type) == PTR_TO_STACK ? -(int)mem_size : mem_size; + if (!tnum_is_const(reg->var_off)) { + verbose(env, "%s is not a const\n", reg_arg_name(env, argno)); + return -EINVAL; + } - err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, NULL); - err = err ?: check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, NULL); + if (reg->var_off.value > U32_MAX) { + verbose(env, "%s allocation size exceeds u32 max\n", reg_arg_name(env, argno)); + return -EINVAL; + } - if (may_be_null) - *reg = saved_reg; + if (regno >= 0) + err = mark_chain_precision(env, regno); + else + err = mark_stack_arg_precision(env, arg_idx_from_argno(argno)); + if (err) + return err; - return err; + ret_mem->size = reg->var_off.value; + ret_mem->found = true; + + return 0; } -static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg, - struct bpf_reg_state *size_reg, argno_t mem_argno, argno_t size_argno) +static int process_const_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, struct bpf_call_arg_meta *meta) { - bool may_be_null = type_may_be_null(mem_reg->type); - struct bpf_reg_state saved_reg; - struct bpf_call_arg_meta meta; + int regno = reg_from_argno(argno); int err; - memset(&meta, 0, sizeof(meta)); + if (meta->arg_constant.found) { + verifier_bug(env, "only one constant argument permitted"); + return -EFAULT; + } - if (may_be_null) { - saved_reg = *mem_reg; - mark_ptr_not_null_reg(mem_reg); + if (!tnum_is_const(reg->var_off)) { + verbose(env, "%s must be a known constant\n", reg_arg_name(env, argno)); + return -EINVAL; } - err = check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_READ, true, &meta); - err = err ?: check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_WRITE, true, &meta); + if (regno >= 0) + err = mark_chain_precision(env, regno); + else + err = mark_stack_arg_precision(env, arg_idx_from_argno(argno)); + if (err < 0) + return err; - if (may_be_null) - *mem_reg = saved_reg; + meta->arg_constant.found = true; + meta->arg_constant.value = reg->var_off.value; - return err; + return 0; } enum { @@ -6978,6 +7259,7 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state bool is_lock = flags & PROCESS_SPIN_LOCK, is_res_lock = flags & PROCESS_RES_LOCK; const char *lock_str = is_res_lock ? "bpf_res_spin" : "bpf_spin"; struct bpf_verifier_state *cur = env->cur_state; + struct bpf_reference_state *lock; bool is_const = tnum_is_const(reg->var_off); bool is_irq = flags & PROCESS_LOCK_IRQ; u64 val = reg->var_off.value; @@ -7027,14 +7309,25 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state ptr = btf; if (!is_res_lock && cur->active_locks) { - if (find_lock_state(env->cur_state, REF_TYPE_LOCK, 0, NULL)) { + lock = find_lock_state(cur, REF_TYPE_LOCK, 0, NULL); + if (lock) { verbose(env, "Locking two bpf_spin_locks are not allowed\n"); + bpf_diag_lock( + env, env->insn_idx, "nested spin lock", + "This path already holds a bpf_spin_lock. The verifier allows only one regular BPF spin lock at a time.", + "Unlock the current bpf_spin_lock before taking another one.", lock); return -EINVAL; } } else if (is_res_lock && cur->active_locks) { - if (find_lock_state(env->cur_state, REF_TYPE_RES_LOCK | REF_TYPE_RES_LOCK_IRQ, reg->id, ptr)) { + lock = find_lock_state(cur, REF_TYPE_RES_LOCK | REF_TYPE_RES_LOCK_IRQ, + reg->id, ptr); + if (lock) { verbose(env, "Acquiring the same lock again, AA deadlock detected\n"); + bpf_diag_lock( + env, env->insn_idx, "recursive resource spin lock", + "This path already holds the same resource spin lock. Taking it again would deadlock.", + "Avoid reacquiring the same resource spin lock before it is unlocked.", lock); return -EINVAL; } } @@ -7061,6 +7354,10 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state if (!cur->active_locks) { verbose(env, "%s_unlock without taking a lock\n", lock_str); + bpf_diag_res( + env, env->insn_idx, "unlock without lock", + "This unlock operation has no matching active lock on the current path.", + "Take the matching lock before this unlock, or remove the unmatched unlock path."); return -EINVAL; } @@ -7070,18 +7367,39 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state type = REF_TYPE_RES_LOCK; else type = REF_TYPE_LOCK; - if (!find_lock_state(cur, type, reg->id, ptr)) { + + lock = find_lock_state(cur, type, reg->id, ptr); + if (!lock) { verbose(env, "%s_unlock of different lock\n", lock_str); + lock = find_lock_state(cur, REF_TYPE_LOCK_MASK, cur->active_lock_id, + cur->active_lock_ptr); + bpf_diag_lock( + env, env->insn_idx, "unlock of a different lock", + "This unlock does not match any active lock with the same tracked identity on the current path.", + "Unlock the same lock object that was most recently acquired.", lock); return -EINVAL; } if (reg->id != cur->active_lock_id || ptr != cur->active_lock_ptr) { verbose(env, "%s_unlock cannot be out of order\n", lock_str); + lock = find_lock_state(cur, REF_TYPE_LOCK_MASK, cur->active_lock_id, + cur->active_lock_ptr); + bpf_diag_lock( + env, env->insn_idx, "unlock out of order", + "Locks must be released in last-in, first-out order, but this unlock does not match the currently active lock.", + "Release nested locks in the reverse order they were acquired.", lock); return -EINVAL; } - if (release_lock_state(cur, type, reg->id, ptr)) { + if (release_lock_state(env, type, reg->id, ptr)) { verbose(env, "%s_unlock of different lock\n", lock_str); + bpf_diag_lock( + env, env->insn_idx, "unlock of a different lock", + "The verifier could not release a lock state matching this unlock operation.", + "Pass the same lock object and lock kind that were used for the matching lock operation.", + lock); return -EINVAL; } + if (!in_rcu_cs(env)) + invalidate_rcu_protected_refs(env); invalidate_non_owning_refs(env); } @@ -7152,18 +7470,6 @@ static int process_timer_func(struct bpf_verifier_env *env, struct bpf_reg_state return check_map_field_pointer(env, reg, argno, BPF_TIMER, map); } -static int process_timer_helper(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_call_arg_meta *meta) -{ - return process_timer_func(env, reg, argno, &meta->map); -} - -static int process_timer_kfunc(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta) -{ - return process_timer_func(env, reg, argno, &meta->map); -} - static int process_kptr_func(struct bpf_verifier_env *env, int regno, struct bpf_call_arg_meta *meta) { @@ -7212,6 +7518,12 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno, return 0; } +static void bpf_diag_call_arg(struct bpf_verifier_env *env, u32 insn_idx, argno_t argno, + const char *call_name, const char *reason, const char *suggestion); +__printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u32 insn_idx, + argno_t argno, const char *call_name, + const char *suggestion, const char *fmt, ...); + /* * Validate dynptr arguments for helper, kfunc and subprog. * @@ -7236,7 +7548,8 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno, * and checked dynamically during runtime. */ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - argno_t argno, int insn_idx, enum bpf_arg_type arg_type, + argno_t argno, int insn_idx, const char *call_name, + enum bpf_arg_type arg_type, struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr) { int spi, err = 0; @@ -7245,6 +7558,11 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat verbose(env, "%s expected pointer to stack or const struct bpf_dynptr\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt( + env, insn_idx, argno, call_name, + "Pass the address of a stack dynptr object, or use a const dynptr pointer returned by the verifier-supported path.", + "a dynptr argument must be a pointer to a dynptr stack slot or a verifier-provided const struct bpf_dynptr, but %s is %s", + reg_arg_name(env, argno), bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } @@ -7267,6 +7585,10 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat if (!is_dynptr_reg_valid_uninit(env, reg)) { verbose(env, "Dynptr has to be an uninitialized dynptr\n"); + bpf_diag_res( + env, insn_idx, "dynptr is already initialized", + "This kfunc constructs a dynptr and requires an uninitialized dynptr stack slot, but the selected slot already holds dynptr state.", + "Use a fresh stack dynptr slot, or release/destroy the existing dynptr before reusing the slot."); return -EINVAL; } @@ -7283,21 +7605,35 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat /* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */ if (reg->type == CONST_PTR_TO_DYNPTR && (arg_type & OBJ_RELEASE)) { verbose(env, "CONST_PTR_TO_DYNPTR cannot be released\n"); + bpf_diag_res( + env, insn_idx, "const dynptr release", + "This release operation was given a const dynptr. Const dynptr values are verifier-provided views and cannot be released by the program.", + "Release only mutable dynptrs that the program initialized or reserved."); return -EINVAL; } if (!is_dynptr_reg_valid_init(env, reg)) { verbose(env, "Expected an initialized dynptr as %s\n", reg_arg_name(env, argno)); + bpf_diag_res( + env, insn_idx, "uninitialized dynptr use", + "This operation requires an initialized dynptr, but the stack slot does not currently hold a valid dynptr on this path.", + "Initialize the dynptr on every path before this call, and avoid overwriting or releasing it before this use."); return -EINVAL; } /* Fold modifiers (in this case, OBJ_RELEASE) when checking expected type */ if (!is_dynptr_type_expected(env, reg, arg_type & ~OBJ_RELEASE)) { - verbose(env, - "Expected a dynptr of type %s as %s\n", - dynptr_type_str(arg_to_dynptr_type(arg_type)), - reg_arg_name(env, argno)); + enum bpf_dynptr_type expected_type = arg_to_dynptr_type(arg_type); + enum bpf_dynptr_type actual_type = dynptr_reg_type(env, reg); + + verbose(env, "Expected a dynptr of type %s as %s\n", + dynptr_type_str(expected_type), reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt( + env, insn_idx, argno, call_name, + "Use a dynptr constructor that matches this operation, or call an operation that accepts the dynptr's current type.", + "the dynptr is initialized with backing object type %s, but this operation expects dynptr type %s", + dynptr_type_str(actual_type), dynptr_type_str(expected_type)); return -EINVAL; } @@ -7308,10 +7644,6 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat if (spi < 0) return spi; - /* - * For CONST_PTR_TO_DYNPTR, reg is already scratched by check_reg_arg - * in check_helper_call and mark_btf_func_reg_size in check_kfunc_call. - */ mark_stack_slots_scratched(env, spi, BPF_DYNPTR_NR_SLOTS); reg = &state->stack[spi].spilled_ptr; @@ -7326,23 +7658,22 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat return err; } -static bool is_iter_kfunc(struct bpf_kfunc_call_arg_meta *meta) +static bool is_iter_kfunc(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY); } -static bool is_iter_new_kfunc(struct bpf_kfunc_call_arg_meta *meta) +static bool is_iter_new_kfunc(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ITER_NEW; } - -static bool is_iter_destroy_kfunc(struct bpf_kfunc_call_arg_meta *meta) +static bool is_iter_destroy_kfunc(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ITER_DESTROY; } -static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg_idx, +static bool is_kfunc_arg_iter(struct bpf_call_arg_meta *meta, int arg_idx, const struct btf_param *arg) { /* btf_check_iter_kfuncs() guarantees that first argument of any iter @@ -7356,7 +7687,7 @@ static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg_idx, } static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { struct bpf_func_state *state = bpf_func(env, reg); const struct btf_type *t; @@ -7366,6 +7697,11 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * if (reg->type != PTR_TO_STACK) { verbose(env, "%s expected pointer to an iterator on stack\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt( + env, insn_idx, argno, meta->func_name, + "Pass the address of a stack iterator object for iterator new, next, and destroy calls.", + "iterator state must live in verifier-tracked stack memory, but %s is %s", + reg_arg_name(env, argno), bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } @@ -7379,6 +7715,10 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * if (btf_id < 0) { verbose(env, "expected valid iter pointer as %s\n", reg_arg_name(env, argno)); + bpf_diag_call_arg( + env, insn_idx, argno, meta->func_name, + "the kfunc expects a recognized iterator state pointer, but this argument does not match a valid iterator type", + "Pass the exact iterator state type expected by this kfunc."); return -EINVAL; } t = btf_type_by_id(meta->btf, btf_id); @@ -7389,6 +7729,10 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) { verbose(env, "expected uninitialized iter_%s as %s\n", iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno)); + bpf_diag_res( + env, insn_idx, "iterator is already initialized", + "Iterator creation requires an uninitialized iterator stack object, but this stack range already contains iterator state.", + "Use a fresh iterator stack slot, or destroy the existing iterator before reusing the slot."); return -EINVAL; } @@ -7413,9 +7757,16 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * case -EINVAL: verbose(env, "expected an initialized iter_%s as %s\n", iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno)); + bpf_diag_res( + env, insn_idx, "uninitialized iterator use", + "This iterator operation requires an initialized iterator state object, but the stack range does not contain a live iterator on this path.", + "Call the matching iterator new kfunc on every path before calling next or destroy, and do not destroy the iterator before this use."); return err; case -EPROTO: verbose(env, "expected an RCU CS when using %s\n", meta->func_name); + bpf_diag_ctx_required( + env, insn_idx, meta->func_name, BPF_DIAG_CONTEXT_RCU, + "Wrap iterator use in bpf_rcu_read_lock() and bpf_rcu_read_unlock(), keeping all exit paths balanced."); return err; default: return err; @@ -7523,7 +7874,7 @@ static int widen_imprecise_scalars(struct bpf_verifier_env *env, } static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_st, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { int iter_frameno = meta->iter.frameno; int iter_spi = meta->iter.spi; @@ -7610,7 +7961,7 @@ static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_ * bpf_iter_num_destroy(&it); */ static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st; struct bpf_func_state *cur_fr = cur_st->frame[cur_st->curframe], *queued_fr; @@ -7666,13 +8017,18 @@ static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx, static bool arg_type_is_mem_size(enum bpf_arg_type type) { - return type == ARG_CONST_SIZE || - type == ARG_CONST_SIZE_OR_ZERO; + return type == ARG_MEM_SIZE || type == ARG_MEM_SIZE_OR_ZERO; } static bool arg_type_is_raw_mem(enum bpf_arg_type type) { - return base_type(type) == ARG_PTR_TO_MEM && + /* + * A map value output buffer (e.g. bpf_map_pop_elem) is also a raw + * (uninitialized) memory argument, and like ARG_PTR_TO_MEM it may be + * passed as a PTR_TO_STACK that reaches check_stack_range_initialized(). + */ + return (base_type(type) == ARG_PTR_TO_MEM || + base_type(type) == ARG_PTR_TO_MAP_VALUE) && type & MEM_UNINIT; } @@ -7807,8 +8163,8 @@ static const struct bpf_reg_types dynptr_types = { static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_MAP_KEY] = &mem_types, [ARG_PTR_TO_MAP_VALUE] = &mem_types, - [ARG_CONST_SIZE] = &scalar_types, - [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, + [ARG_MEM_SIZE] = &scalar_types, + [ARG_MEM_SIZE_OR_ZERO] = &scalar_types, [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, [ARG_CONST_MAP_PTR] = &const_map_ptr_types, [ARG_PTR_TO_CTX] = &context_types, @@ -7830,13 +8186,70 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_DYNPTR] = &dynptr_types, }; +static void bpf_diag_call_arg(struct bpf_verifier_env *env, u32 insn_idx, argno_t argno, + const char *call_name, const char *reason, + const char *suggestion) +{ + int arg = arg_from_argno(argno); + int regno = reg_from_argno(argno); + int stack_slot = -1; + + if (arg < 0 && regno >= BPF_REG_1 && regno <= BPF_REG_5) + arg = regno; + if (arg > MAX_BPF_FUNC_REG_ARGS) + stack_slot = arg - MAX_BPF_FUNC_REG_ARGS - 1; + + bpf_diag_call_type(env, insn_idx, arg, regno, stack_slot, + call_name && *call_name ? call_name : "call", + reg_arg_name(env, argno), reason, suggestion); +} + +static const char *bpf_diag_arg_name(struct bpf_verifier_env *env, argno_t argno) +{ + return bpf_diag_fmt(env, "%s", reg_arg_name(env, argno)); +} + +__printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u32 insn_idx, + argno_t argno, const char *call_name, + const char *suggestion, const char *fmt, ...) +{ + const char *reason; + va_list args; + + va_start(args, fmt); + reason = bpf_diag_vfmt(env, fmt, args); + va_end(args); + + bpf_diag_call_arg(env, insn_idx, argno, call_name, reason, suggestion); +} + +static const char *bpf_diag_expected_reg_types(struct bpf_verifier_env *env, + const enum bpf_reg_type *types, int count) +{ + size_t len = 0, size = 1; + char *buf; + int i; + + for (i = 0; i < count; i++) + size += strlen(reg_type_str(env, types[i])) + (i ? 2 : 0); + + buf = bpf_diag_fmt_buf(env, size); + if (!buf) + return ""; + + for (i = 0; i < count; i++) + len += scnprintf(buf + len, size - len, "%s%s", i ? ", " : "", + reg_type_str(env, types[i])); + return buf; +} + static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type, - const u32 *arg_btf_id, - struct bpf_call_arg_meta *meta) + enum bpf_arg_type arg_type, const u32 *arg_btf_id, + struct bpf_call_arg_meta *meta, const char *call_name) { enum bpf_reg_type expected, type = reg->type; const struct bpf_reg_types *compatible; + const char *actual, *accepted; int i, j, err; compatible = compatible_reg_types[base_type(arg_type)]; @@ -7883,6 +8296,12 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re for (j = 0; j + 1 < i; j++) verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); + actual = bpf_diag_fmt(env, "%s", reg_type_str(env, reg->type)); + accepted = bpf_diag_expected_reg_types(env, compatible->types, i); + bpf_diag_call_arg_fmt(env, env->insn_idx, argno, call_name, + "Pass a value with one of the accepted pointer or scalar types for this call.", + "it has type %s, but this argument accepts %s", + actual, accepted); return -EACCES; found: @@ -7919,6 +8338,10 @@ found: (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) { verbose(env, "Possibly NULL pointer passed to helper %s\n", reg_arg_name(env, argno)); + bpf_diag_call_arg( + env, env->insn_idx, argno, call_name, + "the pointer may be NULL, but this call requires a non-NULL pointer", + "Add a NULL check and make the call only on the non-NULL path."); return -EACCES; } @@ -7947,7 +8370,7 @@ found: if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value, btf_vmlinux, *arg_btf_id, - strict_type_match)) { + strict_type_match, !type_is_alloc(reg->type))) { verbose(env, "%s is of type %s but %s is expected\n", reg_arg_name(env, argno), btf_type_name(reg->btf, reg->btf_id), @@ -8109,6 +8532,12 @@ static int check_arg_const_str(struct bpf_verifier_env *env, return -EACCES; } + if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { + verbose(env, "%s points to percpu_array map which cannot be used as const string\n", + reg_arg_name(env, argno)); + return -EACCES; + } + if (!bpf_map_is_rdonly(map)) { verbose(env, "%s does not point to a readonly map'\n", reg_arg_name(env, argno)); return -EACCES; @@ -8208,15 +8637,53 @@ static int get_constant_map_key(struct bpf_verifier_env *env, static bool can_elide_value_nullness(const struct bpf_map *map); +static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, struct bpf_call_arg_meta *meta) +{ + /* Use map_uid (which is unique id of inner map) to reject: + * inner_map1 = bpf_map_lookup_elem(outer_map, key1) + * inner_map2 = bpf_map_lookup_elem(outer_map, key2) + * if (inner_map1 && inner_map2) { + * timer = bpf_map_lookup_elem(inner_map1); + * if (timer) + * // mismatch would have been allowed + * bpf_timer_init(timer, inner_map2); + * } + * + * Comparing map_ptr is enough to distinguish normal and outer maps. + */ + if (meta->map.ptr && + (meta->map.ptr != reg->map_ptr || meta->map.uid != reg->map_uid)) { + argno_t obj_argno = argno_from_reg(reg_from_argno(argno) - 1); + struct btf_record *rec = meta->map.ptr->record; + const char *obj_name = "workqueue"; + + if (rec->timer_off >= 0) + obj_name = "timer"; + else if (rec->task_work_off >= 0) + obj_name = "bpf_task_work"; + + verbose(env, "%s pointer in %s map_uid=%d ", + obj_name, reg_arg_name(env, obj_argno), meta->map.uid); + verbose(env, "doesn't match map pointer in %s map_uid=%d\n", + reg_arg_name(env, argno), reg->map_uid); + return -EINVAL; + } + + meta->map.ptr = reg->map_ptr; + meta->map.uid = reg->map_uid; + return 0; +} + static int check_func_arg(struct bpf_verifier_env *env, u32 arg, struct bpf_call_arg_meta *meta, - const struct bpf_func_proto *fn, int insn_idx) { + const struct bpf_func_proto *fn = meta->fn; u32 regno = BPF_REG_1 + arg; struct bpf_reg_state *reg = reg_state(env, regno); enum bpf_arg_type arg_type = fn->arg_type[arg]; - argno_t argno = argno_from_arg(arg + 1); + argno_t argno = argno_from_reg(regno); enum bpf_reg_type type = reg->type; u32 *arg_btf_id = NULL; u32 key_size; @@ -8239,7 +8706,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, } if (type_is_pkt_pointer(type) && - !may_access_direct_pkt_data(env, meta, BPF_READ)) { + !may_access_direct_pkt_data(env, fn, BPF_READ)) { verbose(env, "helper access to the packet is not allowed\n"); return -EACCES; } @@ -8261,11 +8728,12 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) arg_btf_id = fn->arg_btf_id[arg]; - err = check_reg_type(env, reg, argno_from_reg(regno), arg_type, arg_btf_id, meta); + err = check_reg_type(env, reg, argno, arg_type, arg_btf_id, meta, + func_id_name(meta->func_id)); if (err) return err; - err = check_func_arg_reg_off(env, reg, argno_from_reg(regno), arg_type); + err = check_func_arg_reg_off(env, reg, argno, arg_type); if (err) return err; @@ -8274,6 +8742,10 @@ skip_type_check: !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { verbose(env, "release helper %s expects referenced PTR_TO_BTF_ID passed to %s\n", func_id_name(meta->func_id), reg_arg_name(env, argno)); + bpf_diag_call_arg( + env, insn_idx, argno, func_id_name(meta->func_id), + "release helpers require a value that owns a live resource returned by a matching acquire helper", + "Pass the resource-owning pointer returned by the matching acquire helper, and avoid calling the release helper after ownership has already been transferred or released."); return -EINVAL; } @@ -8283,29 +8755,9 @@ skip_type_check: switch (base_type(arg_type)) { case ARG_CONST_MAP_PTR: /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ - if (meta->map.ptr) { - /* Use map_uid (which is unique id of inner map) to reject: - * inner_map1 = bpf_map_lookup_elem(outer_map, key1) - * inner_map2 = bpf_map_lookup_elem(outer_map, key2) - * if (inner_map1 && inner_map2) { - * timer = bpf_map_lookup_elem(inner_map1); - * if (timer) - * // mismatch would have been allowed - * bpf_timer_init(timer, inner_map2); - * } - * - * Comparing map_ptr is enough to distinguish normal and outer maps. - */ - if (meta->map.ptr != reg->map_ptr || - meta->map.uid != reg->map_uid) { - verbose(env, - "timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", - meta->map.uid, reg->map_uid); - return -EINVAL; - } - } - meta->map.ptr = reg->map_ptr; - meta->map.uid = reg->map_uid; + err = process_map_ptr_arg(env, reg, argno, meta); + if (err) + return err; break; case ARG_PTR_TO_MAP_KEY: /* bpf_map_xxx(..., map_ptr, ..., key) call: @@ -8322,7 +8774,8 @@ skip_type_check: return -EFAULT; } key_size = meta->map.ptr->key_size; - err = check_helper_mem_access(env, reg, argno_from_reg(regno), key_size, BPF_READ, false, NULL); + err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, NULL, + NULL); if (err) return err; if (can_elide_value_nullness(meta->map.ptr)) { @@ -8348,10 +8801,18 @@ skip_type_check: verifier_bug(env, "invalid map_ptr to access map->value"); return -EFAULT; } - meta->raw_mode = arg_type & MEM_UNINIT; - err = check_helper_mem_access(env, reg, argno_from_reg(regno), meta->map.ptr->value_size, + + /* + * Disable raw mode for bpf_map_peek_elem() on a bloom filter. The helper reads + * the value buffer as an input rather than filling it. + */ + if (meta->func_id == BPF_FUNC_map_peek_elem && + meta->map.ptr->map_type == BPF_MAP_TYPE_BLOOM_FILTER) + meta->arg_raw_mem.regno = 0; + + err = check_helper_mem_access(env, reg, argno, meta->map.ptr->value_size, arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, - false, meta); + false, meta, NULL); break; case ARG_PTR_TO_PERCPU_BTF_ID: if (!reg->btf_id) { @@ -8367,11 +8828,11 @@ skip_type_check: return -EACCES; } if (meta->func_id == BPF_FUNC_spin_lock) { - err = process_spin_lock(env, reg, argno_from_reg(regno), PROCESS_SPIN_LOCK); + err = process_spin_lock(env, reg, argno, PROCESS_SPIN_LOCK); if (err) return err; } else if (meta->func_id == BPF_FUNC_spin_unlock) { - err = process_spin_lock(env, reg, argno_from_reg(regno), 0); + err = process_spin_lock(env, reg, argno, 0); if (err) return err; } else { @@ -8380,7 +8841,7 @@ skip_type_check: } break; case ARG_PTR_TO_TIMER: - err = process_timer_helper(env, reg, argno_from_reg(regno), meta); + err = process_timer_func(env, reg, argno, &meta->map); if (err) return err; break; @@ -8391,51 +8852,41 @@ skip_type_check: /* The access to this pointer is only checked when we hit the * next is_mem_size argument below. */ - meta->raw_mode = arg_type & MEM_UNINIT; if (arg_type & MEM_FIXED_SIZE) { - err = check_helper_mem_access(env, reg, argno_from_reg(regno), fn->arg_size[arg], - arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, - false, meta); + err = check_mem_reg(env, reg, argno_from_reg(regno), fn->arg_size[arg], + arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, meta, NULL); if (err) return err; if (arg_type & MEM_ALIGNED) err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true); } break; - case ARG_CONST_SIZE: - err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), - argno_from_reg(regno), - fn->arg_type[arg - 1] & MEM_WRITE ? - BPF_WRITE : BPF_READ, - false, meta); + case ARG_MEM_SIZE: + err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, + argno_from_reg(regno - 1), argno, + fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, + false, meta, NULL); break; - case ARG_CONST_SIZE_OR_ZERO: - err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), - argno_from_reg(regno), - fn->arg_type[arg - 1] & MEM_WRITE ? - BPF_WRITE : BPF_READ, - true, meta); + case ARG_MEM_SIZE_OR_ZERO: + err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, + argno_from_reg(regno - 1), argno, + fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, + true, meta, NULL); break; case ARG_PTR_TO_DYNPTR: - err = process_dynptr_func(env, reg, argno_from_reg(regno), insn_idx, arg_type, &meta->ref_obj, - &meta->dynptr); + err = process_dynptr_func(env, reg, argno, insn_idx, func_id_name(meta->func_id), + arg_type, &meta->ref_obj, &meta->dynptr); if (err) return err; break; case ARG_CONST_ALLOC_SIZE_OR_ZERO: - if (!tnum_is_const(reg->var_off)) { - verbose(env, "R%d is not a known constant'\n", - regno); - return -EACCES; - } - meta->mem_size = reg->var_off.value; - err = mark_chain_precision(env, regno); + err = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); if (err) return err; break; case ARG_PTR_TO_CONST_STR: { - err = check_arg_const_str(env, reg, argno_from_reg(regno)); + err = check_arg_const_str(env, reg, argno); if (err) return err; break; @@ -8472,12 +8923,7 @@ static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) if (func_id == BPF_FUNC_map_delete_elem) return true; break; - case BPF_PROG_TYPE_SOCKET_FILTER: - case BPF_PROG_TYPE_SCHED_CLS: - case BPF_PROG_TYPE_SCHED_ACT: - case BPF_PROG_TYPE_XDP: case BPF_PROG_TYPE_SK_REUSEPORT: - case BPF_PROG_TYPE_FLOW_DISSECTOR: case BPF_PROG_TYPE_SK_LOOKUP: return true; default: @@ -8747,26 +9193,21 @@ error: return -EINVAL; } -static bool check_raw_mode_ok(const struct bpf_func_proto *fn) +static bool check_raw_mode_ok(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta) { - int count = 0; + int i; - if (arg_type_is_raw_mem(fn->arg1_type)) - count++; - if (arg_type_is_raw_mem(fn->arg2_type)) - count++; - if (arg_type_is_raw_mem(fn->arg3_type)) - count++; - if (arg_type_is_raw_mem(fn->arg4_type)) - count++; - if (arg_type_is_raw_mem(fn->arg5_type)) - count++; + for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { + if (fn->arg_type[i] == ARG_DONTCARE) + break; + if (!arg_type_is_raw_mem(fn->arg_type[i])) + continue; + if (meta->arg_raw_mem.regno) + return false; + meta->arg_raw_mem.regno = i + 1; + } - /* We only support one arg being in raw mode at the moment, - * which is sufficient for the helper functions we have - * right now. - */ - return count <= 1; + return true; } static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg) @@ -8807,6 +9248,8 @@ static bool check_btf_id_ok(const struct bpf_func_proto *fn) int i; for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { + if (fn->arg_type[i] == ARG_DONTCARE) + break; if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID) return !!fn->arg_btf_id[i]; if (base_type(fn->arg_type[i]) == ARG_PTR_TO_SPIN_LOCK) @@ -8828,6 +9271,8 @@ static bool check_mem_arg_rw_flag_ok(const struct bpf_func_proto *fn) for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i]; + if (arg_type == ARG_DONTCARE) + break; if (base_type(arg_type) != ARG_PTR_TO_MEM) continue; if (!(arg_type & (MEM_WRITE | MEM_RDONLY))) @@ -8844,6 +9289,8 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_ for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i]; + if (arg_type == ARG_DONTCARE) + break; if (arg_type_is_release(arg_type)) { if (meta->release_regno) return false; @@ -8856,7 +9303,7 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_ static int check_func_proto(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta) { - return check_raw_mode_ok(fn) && + return check_raw_mode_ok(fn, meta) && check_arg_pair_ok(fn) && check_mem_arg_rw_flag_ok(fn) && check_proto_release_reg(fn, meta) && @@ -8875,8 +9322,10 @@ static void clear_all_pkt_pointers(struct bpf_verifier_env *env) struct bpf_reg_state *reg; bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ - if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) + if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) { + bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_PKT_DATA_CHANGE); mark_reg_invalid(env, reg); + } })); } @@ -8906,7 +9355,7 @@ static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range reg->range = AT_PKT_END; } -static int release_reference_nomark(struct bpf_verifier_state *state, int id) +static int __release_reference_nomark(struct bpf_verifier_state *state, int id) { int i; @@ -8921,6 +9370,16 @@ static int release_reference_nomark(struct bpf_verifier_state *state, int id) return -EINVAL; } +static int release_reference_nomark(struct bpf_verifier_env *env, int id) +{ + int err; + + err = __release_reference_nomark(env->cur_state, id); + if (!err) + bpf_diag_record_ref_release(env, env->insn_idx, id); + return err; +} + static int idstack_push(struct bpf_idmap *idmap, u32 id) { int i; @@ -8963,8 +9422,10 @@ static int release_reference(struct bpf_verifier_env *env, int id) if (err) return err; - if (find_reference_state(vstate, id)) - WARN_ON_ONCE(release_reference_nomark(vstate, id)); + if (find_reference_state(vstate, id)) { + err = release_reference_nomark(env, id); + WARN_ON_ONCE(err); + } while ((id = idstack_pop(idstack))) { /* @@ -8992,10 +9453,25 @@ static int release_reference(struct bpf_verifier_env *env, int id) return err; } + /* + * A dynptr occupies two stack slots that invalidate_dynptr() + * clears together. Record both scrubs before invalidating it. + */ + if (stack && stack->slot_type[BPF_REG_SIZE - 1] == STACK_DYNPTR) { + struct bpf_stack_state *dyn_stack = stack; + + if (reg->dynptr.first_slot) + dyn_stack--; + bpf_diag_record_scrub(env, &dyn_stack[0].spilled_ptr, + BPF_DIAG_MOD_REF_RELEASE); + bpf_diag_record_scrub(env, &dyn_stack[1].spilled_ptr, + BPF_DIAG_MOD_REF_RELEASE); + invalidate_dynptr(env, dyn_stack); + continue; + } + bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_REF_RELEASE); if (!stack || stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL) mark_reg_invalid(env, reg); - else if (stack->slot_type[BPF_REG_SIZE - 1] == STACK_DYNPTR) - invalidate_dynptr(env, stack); })); } @@ -9008,8 +9484,10 @@ static void invalidate_non_owning_refs(struct bpf_verifier_env *env) struct bpf_reg_state *reg; bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ - if (type_is_non_owning_ref(reg->type)) + if (type_is_non_owning_ref(reg->type)) { + bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_NON_OWN_REF); mark_reg_invalid(env, reg); + } })); } @@ -9022,8 +9500,10 @@ static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env) bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, clear_mask, ({ if (reg->type & MEM_RCU) { + bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE); reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL); reg->type |= PTR_UNTRUSTED; + bpf_diag_mod_end(env); } })); } @@ -9034,15 +9514,19 @@ static int ref_convert_alloc_rcu_protected(struct bpf_verifier_env *env, u32 id) struct bpf_reg_state *reg; int err; - err = release_reference_nomark(env->cur_state, id); + err = release_reference_nomark(env, id); + if (err) + return err; bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ if (reg->id != id) continue; if ((reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) { + bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE); reg->id = 0; reg->type &= ~MEM_ALLOC; reg->type |= MEM_RCU; + bpf_diag_mod_end(env); } })); @@ -9054,6 +9538,8 @@ static void clear_caller_saved_regs(struct bpf_verifier_env *env, { int i; + bpf_diag_record_caller_saved(env, regs); + /* after the call registers r0 - r5 were scratched */ for (i = 0; i < CALLER_SAVED_REGS; i++) { bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); @@ -9061,13 +9547,15 @@ static void clear_caller_saved_regs(struct bpf_verifier_env *env, } } -static void invalidate_outgoing_stack_args(const struct bpf_verifier_env *env, +static void invalidate_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *state) { int i, nslots = state->out_stack_arg_cnt; - for (i = 0; i < nslots; i++) + for (i = 0; i < nslots; i++) { + bpf_diag_record_scrub(env, &state->stack_arg_regs[i], BPF_DIAG_MOD_CALLER_SAVED); bpf_mark_reg_not_init(env, &state->stack_arg_regs[i]); + } } typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env, @@ -9135,20 +9623,28 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct bpf_func_state *caller = cur_func(env); struct bpf_verifier_log *log = &env->log; struct ref_obj_desc ref_obj = {}; + const struct btf_param *args; + const struct btf_type *func, *func_proto; u32 i; int ret, err; ret = btf_prepare_func_args(env, subprog); if (ret) { if (bpf_in_stack_arg_cnt(sub) > 0) { - err = check_outgoing_stack_args(env, caller, sub->arg_cnt); + err = check_outgoing_stack_args(env, caller, sub->arg_cnt, + bpf_subprog_name(env, subprog), + NULL, NULL); if (err) return err; } return ret; } - ret = check_outgoing_stack_args(env, caller, sub->arg_cnt); + func = btf_type_by_id(btf, env->prog->aux->func_info[subprog].type_id); + func_proto = btf_type_by_id(btf, func->type); + args = btf_params(func_proto); + ret = check_outgoing_stack_args(env, caller, sub->arg_cnt, + bpf_subprog_name(env, subprog), btf, args); if (ret) return ret; @@ -9187,7 +9683,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, ret = check_func_arg_reg_off(env, reg, argno, ARG_DONTCARE); if (ret < 0) return ret; - if (check_mem_reg(env, reg, argno, arg->mem_size)) + if (check_mem_reg(env, reg, argno, arg->mem_size, BPF_READ | BPF_WRITE, NULL, + NULL)) return -EINVAL; if (!(arg->arg_type & PTR_MAYBE_NULL) && (type_may_be_null(reg->type) || bpf_register_is_null(reg))) { @@ -9213,7 +9710,9 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, if (ret) return ret; - ret = process_dynptr_func(env, reg, argno, -1, arg->arg_type, &ref_obj, NULL); + ret = process_dynptr_func(env, reg, argno, env->insn_idx, + bpf_subprog_name(env, subprog), arg->arg_type, + &ref_obj, NULL); if (ret) return ret; } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { @@ -9224,7 +9723,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, continue; memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */ - err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta); + err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta, + bpf_subprog_name(env, subprog)); err = err ?: check_func_arg_reg_off(env, reg, argno, arg->arg_type); if (err) return err; @@ -9365,17 +9865,25 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err == -EFAULT) return err; if (bpf_subprog_is_global(env, subprog)) { - const char *sub_name = subprog_name(env, subprog); + const char *sub_name = bpf_subprog_name(env, subprog); + const char *operation; + bool returns_void; if (env->cur_state->active_locks) { verbose(env, "global function calls are not allowed while holding a lock,\n" "use static function instead\n"); + operation = bpf_diag_fmt(env, "global function %s()", sub_name); + bpf_diag_ctx_active(env, *insn_idx, operation, BPF_DIAG_CONTEXT_LOCK, + "Release the lock before calling the global function, or use a static function instead."); return -EINVAL; } if (env->subprog_info[subprog].might_sleep && !in_sleepable_context(env)) { verbose(env, "sleepable global function %s() called in %s\n", sub_name, non_sleepable_context_description(env)); + operation = bpf_diag_fmt(env, "sleepable global function %s()", sub_name); + bpf_diag_ctx_forbidden(env, *insn_idx, operation, + "Move the call outside the critical section, or use a non-sleepable function."); return -EINVAL; } @@ -9388,17 +9896,22 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (env->log.level & BPF_LOG_LEVEL) verbose(env, "Func#%d ('%s') is global and assumed valid.\n", subprog, sub_name); + returns_void = subprog_returns_void(env, subprog); if (env->subprog_info[subprog].changes_pkt_data) clear_all_pkt_pointers(env); /* mark global subprog for verifying after main prog */ subprog_aux(env, subprog)->called = true; + if (returns_void) + bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); + else + bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); clear_caller_saved_regs(env, caller->regs); invalidate_outgoing_stack_args(env, cur_func(env)); /* All non-void global functions return a 64-bit SCALAR_VALUE. */ - if (!subprog_returns_void(env, subprog)) { + if (!returns_void) { mark_reg_unknown(env, caller->regs, BPF_REG_0); - caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; + bpf_diag_mod_end(env); } if (env->subprog_info[subprog].might_throw) { @@ -9433,6 +9946,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; + bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); clear_caller_saved_regs(env, caller->regs); /* and go analyze first insn of the callee */ @@ -9683,6 +10197,42 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, static bool is_rbtree_lock_required_kfunc(u32 btf_id); +static void account_processed_insn(struct bpf_verifier_env *env) +{ + struct bpf_func_state *frame = cur_func(env); + + env->insn_processed++; + frame->insns_subtotal++; + env->subprog_info[frame->subprogno].insns_self++; +} + +static void account_processed_insns(struct bpf_verifier_env *env, + struct bpf_func_state *callee, + struct bpf_func_state *caller) +{ + u32 insns; + + if (!callee) + return; + + insns = callee->insns_subtotal; + + env->subprog_info[callee->subprogno].insns_total += insns; + if (caller) + caller->insns_subtotal += insns; + callee->insns_subtotal = 0; +} + +static void account_current_path(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *state = env->cur_state; + int frame; + + for (frame = state->curframe; frame >= 0; frame--) + account_processed_insns(env, state->frame[frame], + frame ? state->frame[frame - 1] : NULL); +} + /* Are we currently verifying the callback for a rbtree helper that must * be called with lock held? If so, no need to complain about unreleased * lock @@ -9760,7 +10310,9 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) } } else { /* return to the caller whatever r0 had in the callee */ + bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE); caller->regs[BPF_REG_0] = *r0; + bpf_diag_mod_end(env); } /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite, @@ -9779,6 +10331,7 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) verbose(env, "to caller at %d:\n", *insn_idx); print_verifier_state(env, state, caller->frameno, true); } + account_processed_insns(env, callee, caller); /* clear everything in the callee. In case of exceptional exits using * bpf_throw, this will be done by copy_verifier_state for extra frames. */ free_func_state(callee); @@ -9965,6 +10518,7 @@ static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exi continue; verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", state->refs[i].id, state->refs[i].insn_idx); + bpf_diag_leak(env, state->refs[i].id, state->refs[i].insn_idx, env->insn_idx); refs_lingering = true; } return refs_lingering ? -EINVAL : 0; @@ -9976,6 +10530,8 @@ static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit if (check_lock && env->cur_state->active_locks) { verbose(env, "%s cannot be used inside bpf_spin_lock-ed region\n", prefix); + bpf_diag_ctx_active(env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_LOCK, + "Release the BPF spin lock before this operation on every path."); return -EINVAL; } @@ -9987,16 +10543,23 @@ static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit if (check_lock && env->cur_state->active_irq_id) { verbose(env, "%s cannot be used inside bpf_local_irq_save-ed region\n", prefix); + bpf_diag_ctx_active(env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_IRQ, + "Restore the saved IRQ state before this operation on every path."); return -EINVAL; } if (check_lock && env->cur_state->active_rcu_locks) { verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix); + bpf_diag_ctx_active(env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_RCU, + "Call bpf_rcu_read_unlock() before this operation on every path."); return -EINVAL; } if (check_lock && env->cur_state->active_preempt_locks) { verbose(env, "%s cannot be used inside bpf_preempt_disable-ed region\n", prefix); + bpf_diag_ctx_active( + env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_PREEMPT, + "Call bpf_preempt_enable() before this operation on every path."); return -EINVAL; } @@ -10178,6 +10741,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn enum bpf_type_flag ret_flag; struct bpf_reg_state *regs; struct bpf_call_arg_meta meta; + const char *operation; int insn_idx = *insn_idx_p; bool changes_data; int i, err, func_id; @@ -10193,17 +10757,31 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (err) { verbose(env, "program of this type cannot use helper %s#%d\n", func_id_name(func_id), func_id); + operation = bpf_diag_fmt(env, "helper %s#%d", func_id_name(func_id), func_id); + bpf_diag_policy( + env, insn_idx, operation, "this program type does not allow the helper", + "Use a helper allowed for this program type, or move the logic to a compatible program type."); return err; } /* eBPF programs must be GPL compatible to use GPL-ed functions */ if (!env->prog->gpl_compatible && fn->gpl_only) { verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); + operation = bpf_diag_fmt(env, "helper %s#%d", func_id_name(func_id), func_id); + bpf_diag_policy( + env, insn_idx, operation, + "this helper is restricted to GPL-compatible programs", + "Use a GPL-compatible license, or replace the helper with one that is available to non-GPL programs."); return -EINVAL; } if (fn->allowed && !fn->allowed(env->prog)) { verbose(env, "helper call is not allowed in probe\n"); + operation = bpf_diag_fmt(env, "helper %s#%d", func_id_name(func_id), func_id); + bpf_diag_policy( + env, insn_idx, operation, + "the helper-specific policy callback rejected this program", + "Use the helper only from an allowed attach point or program configuration."); return -EINVAL; } @@ -10215,7 +10793,6 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn } memset(&meta, 0, sizeof(meta)); - meta.pkt_access = fn->pkt_access; err = check_func_proto(fn, &meta); if (err) { @@ -10226,6 +10803,10 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (fn->might_sleep && !in_sleepable_context(env)) { verbose(env, "sleepable helper %s#%d in %s\n", func_id_name(func_id), func_id, non_sleepable_context_description(env)); + operation = bpf_diag_fmt(env, "sleepable helper %s#%d", + func_id_name(func_id), func_id); + bpf_diag_ctx_forbidden(env, insn_idx, operation, + "Move the helper call outside the critical section, or use a non-sleepable helper."); return -EINVAL; } @@ -10234,9 +10815,10 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn env->insn_aux_data[insn_idx].non_sleepable = true; meta.func_id = func_id; + meta.fn = fn; /* check args */ for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { - err = check_func_arg(env, i, &meta, fn, insn_idx); + err = check_func_arg(env, i, &meta, insn_idx); if (err) return err; } @@ -10254,8 +10836,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn /* Mark slots with STACK_MISC in case of raw mode, stack offset * is inferred from register state. */ - for (i = 0; i < meta.access_size; i++) { - err = check_mem_access(env, insn_idx, regs + meta.regno, argno_from_reg(meta.regno), i, BPF_B, + for (i = 0; i < meta.arg_raw_mem.size; i++) { + err = check_mem_access(env, insn_idx, regs + meta.arg_raw_mem.regno, + argno_from_reg(meta.arg_raw_mem.regno), i, BPF_B, BPF_WRITE, -1, false, false); if (err) return err; @@ -10411,15 +10994,14 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return err; /* reset caller saved regs */ + bpf_diag_record_caller_saved(env, regs); + bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); for (i = 0; i < CALLER_SAVED_REGS; i++) { bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); } invalidate_outgoing_stack_args(env, cur_func(env)); - /* helper call returns 64-bit value. */ - regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; - /* update return register (already marked as written above) */ ret_type = fn->ret_type; ret_flag = type_flag(ret_type); @@ -10453,10 +11035,12 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn regs[BPF_REG_0].map_ptr = meta.map.ptr; regs[BPF_REG_0].map_uid = meta.map.uid; regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag; - if (!type_may_be_null(ret_flag) && + if (type_may_be_null(ret_flag) || btf_record_has_field(meta.map.ptr->record, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK)) { regs[BPF_REG_0].id = ++env->id_gen; } + /* requires regs[BPF_REG_0].id to be set because of the map-in-map case */ + refine_map_lookup_value(®s[BPF_REG_0]); break; case RET_PTR_TO_SOCKET: mark_reg_known_zero(env, regs, BPF_REG_0); @@ -10473,7 +11057,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn case RET_PTR_TO_MEM: mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; - regs[BPF_REG_0].mem_size = meta.mem_size; + regs[BPF_REG_0].mem_size = meta.ret_mem.size; break; case RET_PTR_TO_MEM_OR_BTF_ID: { @@ -10554,7 +11138,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return -EINVAL; } - if (type_may_be_null(regs[BPF_REG_0].type)) + if (type_may_be_null(regs[BPF_REG_0].type) && !regs[BPF_REG_0].id) regs[BPF_REG_0].id = ++env->id_gen; if (is_ptr_cast_function(func_id) && @@ -10566,6 +11150,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (err) return err; + bpf_diag_mod_end(env); + /* * In order for a release of any of the original or cast pointers * to invalidate all other pointers, reuse the same reference id for @@ -10582,6 +11168,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn __mark_reg_known_zero(r0); r0->type = SCALAR_VALUE; + bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); regs[BPF_REG_0].type &= ~PTR_MAYBE_NULL; regs[BPF_REG_0].id = meta.ref_obj.id; } else if (is_acquire_function(func_id, meta.map.ptr)) { @@ -10600,6 +11187,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (err) return err; + bpf_diag_mod_end(env); + err = check_map_func_compatibility(env, meta.map.ptr, func_id); if (err) return err; @@ -10657,76 +11246,50 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return 0; } -/* mark_btf_func_reg_size() is used when the reg size is determined by - * the BTF func_proto's return value size and argument. - */ -static void __mark_btf_func_reg_size(struct bpf_verifier_env *env, struct bpf_reg_state *regs, - u32 regno, size_t reg_size) -{ - struct bpf_reg_state *reg = ®s[regno]; - - if (regno == BPF_REG_0) { - /* Function return value */ - reg->subreg_def = reg_size == sizeof(u64) ? - DEF_NOT_SUBREG : env->insn_idx + 1; - } else if (reg_size == sizeof(u64)) { - /* Function argument */ - mark_insn_zext(env, reg); - } -} - -static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, - size_t reg_size) -{ - return __mark_btf_func_reg_size(env, cur_regs(env), regno, reg_size); -} - -static bool is_kfunc_acquire(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ACQUIRE; } -static bool is_kfunc_release(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_release(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_RELEASE; } -static bool is_kfunc_destructive(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_destructive(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_DESTRUCTIVE; } -static bool is_kfunc_rcu(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_rcu(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_RCU; } -static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_rcu_protected(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_RCU_PROTECTED; } static bool is_kfunc_arg_mem_size(const struct btf *btf, - const struct btf_param *arg, - const struct bpf_reg_state *reg) + const struct btf_param *arg) { const struct btf_type *t; t = btf_type_skip_modifiers(btf, arg->type, NULL); - if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) + if (!btf_type_is_scalar(t)) return false; return btf_param_match_suffix(btf, arg, "__sz"); } static bool is_kfunc_arg_const_mem_size(const struct btf *btf, - const struct btf_param *arg, - const struct bpf_reg_state *reg) + const struct btf_param *arg) { const struct btf_type *t; t = btf_type_skip_modifiers(btf, arg->type, NULL); - if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) + if (!btf_type_is_scalar(t)) return false; return btf_param_match_suffix(btf, arg, "__szk"); @@ -10747,6 +11310,11 @@ static bool is_kfunc_arg_map(const struct btf *btf, const struct btf_param *arg) return btf_param_match_suffix(btf, arg, "__map"); } +static bool is_kfunc_arg_const_map(const struct btf *btf, const struct btf_param *arg) +{ + return btf_param_match_suffix(btf, arg, "__const_map"); +} + static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg) { return btf_param_match_suffix(btf, arg, "__alloc"); @@ -10764,7 +11332,8 @@ static bool is_kfunc_arg_refcounted_kptr(const struct btf *btf, const struct btf static bool is_kfunc_arg_nullable(const struct btf *btf, const struct btf_param *arg) { - return btf_param_match_suffix(btf, arg, "__nullable"); + return btf_param_match_suffix(btf, arg, "__nullable") || + btf_param_match_suffix(btf, arg, "__arena"); } static bool is_kfunc_arg_nonown_allowed(const struct btf *btf, const struct btf_param *arg) @@ -10782,6 +11351,12 @@ static bool is_kfunc_arg_irq_flag(const struct btf *btf, const struct btf_param return btf_param_match_suffix(btf, arg, "__irq_flag"); } +static bool is_kfunc_arg_arena(const struct btf *btf, const struct btf_param *arg) +{ + return btf_param_match_suffix(btf, arg, "__arena__nullable") || + btf_param_match_suffix(btf, arg, "__arena"); +} + static bool is_kfunc_arg_scalar_with_name(const struct btf *btf, const struct btf_param *arg, const char *name) @@ -10922,7 +11497,7 @@ static bool is_kfunc_arg_prog_aux(const struct btf *btf, const struct btf_param * To determine whether an argument is implicit, we compare its position * against the number of arguments in the prototype w/o implicit args. */ -static bool is_kfunc_arg_implicit(const struct bpf_kfunc_call_arg_meta *meta, u32 arg_idx) +static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_idx) { const struct btf_type *func, *func_proto; u32 argn; @@ -10978,6 +11553,11 @@ static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, } enum kfunc_ptr_arg_type { + KF_ARG_CONST_MEM_SIZE, + KF_ARG_MEM_SIZE, + KF_ARG_CONST, + KF_ARG_CONST_ALLOC_SIZE_OR_ZERO, + KF_ARG_ANYTHING, KF_ARG_PTR_TO_CTX, KF_ARG_PTR_TO_ALLOC_BTF_ID, /* Allocated object */ KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */ @@ -10987,18 +11567,17 @@ enum kfunc_ptr_arg_type { KF_ARG_PTR_TO_LIST_NODE, KF_ARG_PTR_TO_BTF_ID, /* Also covers reg2btf_ids conversions */ KF_ARG_PTR_TO_MEM, - KF_ARG_PTR_TO_MEM_SIZE, /* Size derived from next argument, skip it */ KF_ARG_PTR_TO_CALLBACK, KF_ARG_PTR_TO_RB_ROOT, KF_ARG_PTR_TO_RB_NODE, - KF_ARG_PTR_TO_NULL, KF_ARG_PTR_TO_CONST_STR, - KF_ARG_PTR_TO_MAP, + KF_ARG_CONST_MAP_PTR, KF_ARG_PTR_TO_TIMER, KF_ARG_PTR_TO_WORKQUEUE, KF_ARG_PTR_TO_IRQ_FLAG, KF_ARG_PTR_TO_RES_SPIN_LOCK, KF_ARG_PTR_TO_TASK_WORK, + KF_ARG_PTR_TO_ARENA, }; enum special_kfunc_type { @@ -11068,10 +11647,7 @@ enum special_kfunc_type { KF_bpf_task_work_schedule_resume, KF_bpf_arena_alloc_pages, KF_bpf_arena_free_pages, - KF_bpf_arena_reserve_pages, KF_bpf_session_is_return, - KF_bpf_stream_vprintk, - KF_bpf_stream_print_stack, }; BTF_ID_LIST(special_kfunc_list) @@ -11161,14 +11737,11 @@ BTF_ID(func, bpf_task_work_schedule_signal) BTF_ID(func, bpf_task_work_schedule_resume) BTF_ID(func, bpf_arena_alloc_pages) BTF_ID(func, bpf_arena_free_pages) -BTF_ID(func, bpf_arena_reserve_pages) #ifdef CONFIG_BPF_EVENTS BTF_ID(func, bpf_session_is_return) #else BTF_ID_UNUSED #endif -BTF_ID(func, bpf_stream_vprintk) -BTF_ID(func, bpf_stream_print_stack) static bool is_bpf_obj_new_kfunc(u32 func_id) { @@ -11221,7 +11794,7 @@ static bool is_task_work_add_kfunc(u32 func_id) func_id == special_kfunc_list[KF_bpf_task_work_schedule_resume]; } -static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_ret_null(struct bpf_call_arg_meta *meta) { if (is_bpf_refcount_acquire_kfunc(meta->func_id) && meta->arg_owning_ref) return false; @@ -11229,140 +11802,208 @@ static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta) return meta->kfunc_flags & KF_RET_NULL; } -static bool is_kfunc_bpf_rcu_read_lock(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_bpf_rcu_read_lock(struct bpf_call_arg_meta *meta) { return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock]; } -static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_call_arg_meta *meta) { return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock]; } -static bool is_kfunc_bpf_preempt_disable(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_bpf_preempt_disable(struct bpf_call_arg_meta *meta) { return meta->func_id == special_kfunc_list[KF_bpf_preempt_disable]; } -static bool is_kfunc_bpf_preempt_enable(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_bpf_preempt_enable(struct bpf_call_arg_meta *meta) { return meta->func_id == special_kfunc_list[KF_bpf_preempt_enable]; } -bool bpf_is_kfunc_pkt_changing(struct bpf_kfunc_call_arg_meta *meta) +bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) { return meta->func_id == special_kfunc_list[KF_bpf_xdp_pull_data]; } -static enum kfunc_ptr_arg_type -get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_func_state *caller, - struct bpf_reg_state *regs, struct bpf_kfunc_call_arg_meta *meta, - const struct btf_type *t, const struct btf_type *ref_t, - const char *ref_tname, const struct btf_param *args, - int arg, int nargs, argno_t argno, struct bpf_reg_state *reg) +static int +get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + const struct btf_param *args, int arg, int nargs) { - bool arg_mem_size = false; + const struct btf_type *t, *ref_t = NULL; + argno_t argno = argno_from_arg(arg + 1); + const char *ref_tname = NULL; + int arg_type; - if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || - meta->func_id == special_kfunc_list[KF_bpf_session_is_return] || - meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) - return KF_ARG_PTR_TO_CTX; + t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL); - if (arg + 1 < nargs && - (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1], get_func_arg_reg(caller, regs, arg + 1)) || - is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1], get_func_arg_reg(caller, regs, arg + 1)))) - arg_mem_size = true; + /* Scalar arguments are classified from their BTF suffix/name alone. */ + if (btf_type_is_scalar(t)) { + if (is_kfunc_arg_constant(meta->btf, &args[arg])) + return KF_ARG_CONST; + if (is_kfunc_arg_const_mem_size(meta->btf, &args[arg])) + return KF_ARG_CONST_MEM_SIZE; + if (is_kfunc_arg_mem_size(meta->btf, &args[arg])) + return KF_ARG_MEM_SIZE; + if (is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdonly_buf_size") || + is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdwr_buf_size")) + return KF_ARG_CONST_ALLOC_SIZE_OR_ZERO; + return KF_ARG_ANYTHING; + } + + if (!btf_type_is_ptr(t)) { + verbose(env, "Unrecognized %s type %s\n", + reg_arg_name(env, argno), btf_type_str(t)); + return -EINVAL; + } + ref_t = btf_type_skip_modifiers(meta->btf, t->type, NULL); + ref_tname = btf_name_by_offset(meta->btf, ref_t->name_off); /* In this function, we verify the kfunc's BTF as per the argument type, * leaving the rest of the verification with respect to the register * type to our caller. When a set of conditions hold in the BTF type of * arguments, we resolve it to a known kfunc_ptr_arg_type. */ - if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) - return KF_ARG_PTR_TO_CTX; - - if (is_kfunc_arg_nullable(meta->btf, &args[arg]) && bpf_register_is_null(reg) && - !arg_mem_size) - return KF_ARG_PTR_TO_NULL; - - if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_ALLOC_BTF_ID; - - if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_REFCOUNTED_KPTR; - - if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_DYNPTR; - - if (is_kfunc_arg_iter(meta, arg, &args[arg])) - return KF_ARG_PTR_TO_ITER; - - if (is_kfunc_arg_list_head(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_LIST_HEAD; - - if (is_kfunc_arg_list_node(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_LIST_NODE; - - if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_RB_ROOT; - - if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_RB_NODE; - - if (is_kfunc_arg_const_str(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_CONST_STR; + if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || + meta->func_id == special_kfunc_list[KF_bpf_session_is_return] || + meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) + arg_type = KF_ARG_PTR_TO_CTX; + else if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) + arg_type = KF_ARG_PTR_TO_CTX; + else if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_ALLOC_BTF_ID; + else if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_REFCOUNTED_KPTR; + else if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_DYNPTR; + else if (is_kfunc_arg_iter(meta, arg, &args[arg])) + arg_type = KF_ARG_PTR_TO_ITER; + else if (is_kfunc_arg_list_head(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_LIST_HEAD; + else if (is_kfunc_arg_list_node(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_LIST_NODE; + else if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_RB_ROOT; + else if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_RB_NODE; + else if (is_kfunc_arg_const_str(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_CONST_STR; + else if (is_kfunc_arg_const_map(meta->btf, &args[arg])) + arg_type = KF_ARG_CONST_MAP_PTR; + else if (is_kfunc_arg_map(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_BTF_ID; + else if (is_kfunc_arg_wq(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_WORKQUEUE; + else if (is_kfunc_arg_timer(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_TIMER; + else if (is_kfunc_arg_task_work(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_TASK_WORK; + else if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_IRQ_FLAG; + else if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_RES_SPIN_LOCK; + else if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_CALLBACK; + else if (is_kfunc_arg_arena(meta->btf, &args[arg])) { + if (!bpf_jit_supports_arena_args()) { + verbose(env, "JIT does not support kfunc %s() with arena pointer arguments\n", + meta->func_name); + return -ENOTSUPP; + } + if (!env->prog->aux->arena) { + verbose(env, + "%s arena pointer requires a program with an associated arena\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + if (reg_from_argno(argno) < 0) { + verbose(env, "%s arena pointer cannot be a stack argument\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + /* + * Both suffixes accept a constant zero. The function model determines + * whether the JIT rebases it to the arena base or preserves NULL. + * The common nullable path below records that verifier property. + */ + arg_type = KF_ARG_PTR_TO_ARENA; + } else if (arg + 1 < nargs && + (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || + is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { + if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && + !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", + reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); + return -EINVAL; + } + arg_type = KF_ARG_PTR_TO_MEM; + } else if (btf_type_is_struct(ref_t)) + /* A pointer to a struct without a size argument is classified as KF_ARG_PTR_TO_BTF_ID */ + arg_type = KF_ARG_PTR_TO_BTF_ID; + else { + /* + * Otherwise this is a fixed-size memory buffer supported by + * check_helper_mem_access(): a pointer to a scalar or a struct of + * scalars. The access size is derived from the pointed-to BTF type. + */ + if (!btf_type_is_scalar(ref_t) && + !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", + reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); + return -EINVAL; + } + arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; + } - if (is_kfunc_arg_map(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_MAP; + if (is_kfunc_arg_nullable(meta->btf, &args[arg])) + arg_type |= PTR_MAYBE_NULL; - if (is_kfunc_arg_wq(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_WORKQUEUE; + return arg_type; +} - if (is_kfunc_arg_timer(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_TIMER; +static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + struct bpf_func_proto *proto) +{ + const struct btf *btf = meta->btf; + const struct btf_param *args; + u32 i, nargs; + int arg_type; - if (is_kfunc_arg_task_work(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_TASK_WORK; + args = (const struct btf_param *)(meta->func_proto + 1); + nargs = btf_type_vlen(meta->func_proto); + if (nargs > MAX_BPF_FUNC_ARGS) { + verbose(env, "Function %s has %d > %d args\n", meta->func_name, + nargs, MAX_BPF_FUNC_ARGS); + return -EINVAL; + } + if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { + verbose(env, "JIT does not support kfunc %s() with %d args\n", + meta->func_name, nargs); + return -ENOTSUPP; + } - if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_IRQ_FLAG; + for (i = 0; i < nargs; i++) { + if (is_kfunc_arg_prog_aux(btf, &args[i]) || + is_kfunc_arg_ignore(btf, &args[i]) || + is_kfunc_arg_implicit(meta, i)) + continue; - if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_RES_SPIN_LOCK; + arg_type = get_kfunc_arg_type(env, meta, args, i, nargs); + if (arg_type < 0) + return arg_type; - if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) { - if (!btf_type_is_struct(ref_t)) { - verbose(env, "kernel function %s %s pointer type %s %s is not supported\n", - meta->func_name, reg_arg_name(env, argno), - btf_type_str(ref_t), ref_tname); - return -EINVAL; - } - return KF_ARG_PTR_TO_BTF_ID; + proto->arg_type[i] = arg_type; } - if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) - return KF_ARG_PTR_TO_CALLBACK; - - /* This is the catch all argument type of register types supported by - * check_helper_mem_access. However, we only allow when argument type is - * pointer to scalar, or struct composed (recursively) of scalars. When - * arg_mem_size is true, the pointer can be void *. - */ - if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) && - (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) { - verbose(env, "%s pointer type %s %s must point to %sscalar, or struct with scalar\n", - reg_arg_name(env, argno), - btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : ""); - return -EINVAL; - } - return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM; + return 0; } static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, const struct btf_type *ref_t, const char *ref_tname, u32 ref_id, - struct bpf_kfunc_call_arg_meta *meta, + struct bpf_call_arg_meta *meta, int arg, argno_t argno) { const struct btf_type *reg_ref_t; @@ -11414,7 +12055,8 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, ®_ref_id); reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off); struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->var_off.value, - meta->btf, ref_id, strict_type_match); + meta->btf, ref_id, strict_type_match, + !type_is_alloc(reg->type)); /* If kfunc is accepting a projection type (ie. __sk_buff), it cannot * actually use it -- it must cast to the underlying type. So we allow * caller to pass in the underlying type. @@ -11431,7 +12073,7 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, } static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { int err, spi, kfunc_class = IRQ_NATIVE_KFUNC; bool irq_save; @@ -11455,6 +12097,12 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * if (!is_irq_flag_reg_valid_uninit(env, reg)) { verbose(env, "expected uninitialized irq flag as %s\n", reg_arg_name(env, argno)); + bpf_diag_res(env, env->insn_idx, "IRQ flag is already initialized", + "Saving IRQ state requires an uninitialized stack slot for " + "the IRQ flag, but this slot already contains tracked IRQ " + "flag state.", + "Use a fresh stack slot for this save operation, or restore " + "the existing IRQ flag before reusing the slot."); return -EINVAL; } @@ -11471,6 +12119,11 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * if (err) { verbose(env, "expected an initialized irq flag as %s\n", reg_arg_name(env, argno)); + bpf_diag_res(env, env->insn_idx, "uninitialized IRQ flag restore", + "Restoring IRQ state requires a stack slot that was " + "initialized by a matching IRQ save operation on this path.", + "Pass the same stack slot that was previously initialized by " + "the matching IRQ save kfunc."); return err; } @@ -11483,11 +12136,13 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * err = unmark_stack_slot_irq_flag(env, reg, kfunc_class); if (err) return err; + + if (!in_rcu_cs(env)) + invalidate_rcu_protected_refs(env); } return 0; } - static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg) { struct btf_record *rec = reg_btf_record(reg); @@ -11513,8 +12168,10 @@ static void ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 id) { struct bpf_func_state *unused; struct bpf_reg_state *reg; + int err; - WARN_ON_ONCE(release_reference_nomark(env->cur_state, id)); + err = release_reference_nomark(env, id); + WARN_ON_ONCE(err); bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ if (reg->id == id) { @@ -11622,20 +12279,6 @@ static bool is_bpf_rbtree_api_kfunc(u32 btf_id) btf_id == special_kfunc_list[KF_bpf_rbtree_right]; } -static bool is_bpf_iter_num_api_kfunc(u32 btf_id) -{ - return btf_id == special_kfunc_list[KF_bpf_iter_num_new] || - btf_id == special_kfunc_list[KF_bpf_iter_num_next] || - btf_id == special_kfunc_list[KF_bpf_iter_num_destroy]; -} - -static bool is_bpf_graph_api_kfunc(u32 btf_id) -{ - return is_bpf_list_api_kfunc(btf_id) || - is_bpf_rbtree_api_kfunc(btf_id) || - is_bpf_refcount_acquire_kfunc(btf_id); -} - static bool is_bpf_res_spin_lock_kfunc(u32 btf_id) { return btf_id == special_kfunc_list[KF_bpf_res_spin_lock] || @@ -11644,24 +12287,16 @@ static bool is_bpf_res_spin_lock_kfunc(u32 btf_id) btf_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]; } -static bool is_bpf_arena_kfunc(u32 btf_id) +static bool kfunc_spin_allowed(struct bpf_verifier_env *env, s32 func_id, s16 offset) { - return btf_id == special_kfunc_list[KF_bpf_arena_alloc_pages] || - btf_id == special_kfunc_list[KF_bpf_arena_free_pages] || - btf_id == special_kfunc_list[KF_bpf_arena_reserve_pages]; -} + struct bpf_kfunc_meta kfunc; + int err; -static bool is_bpf_stream_kfunc(u32 btf_id) -{ - return btf_id == special_kfunc_list[KF_bpf_stream_vprintk] || - btf_id == special_kfunc_list[KF_bpf_stream_print_stack]; -} + err = fetch_kfunc_meta(env, func_id, offset, &kfunc); + if (err || !kfunc.flags) + return false; -static bool kfunc_spin_allowed(u32 btf_id) -{ - return is_bpf_graph_api_kfunc(btf_id) || is_bpf_iter_num_api_kfunc(btf_id) || - is_bpf_res_spin_lock_kfunc(btf_id) || is_bpf_arena_kfunc(btf_id) || - is_bpf_stream_kfunc(btf_id); + return *kfunc.flags & KF_SPINLOCK_SAFE; } static bool is_sync_callback_calling_kfunc(u32 btf_id) @@ -11756,7 +12391,7 @@ static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env, static int __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta, + struct bpf_call_arg_meta *meta, enum btf_field_type head_field_type, struct btf_field **head_field) { @@ -11806,7 +12441,7 @@ __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env, static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_LIST_HEAD, &meta->arg_list_head.field); @@ -11814,7 +12449,7 @@ static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_RB_ROOT, &meta->arg_rbtree_root.field); @@ -11823,7 +12458,7 @@ static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, static int __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta, + struct bpf_call_arg_meta *meta, enum btf_field_type head_field_type, enum btf_field_type node_field_type, struct btf_field **node_field) @@ -11861,7 +12496,8 @@ __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, et = btf_type_by_id(field->graph_root.btf, field->graph_root.value_btf_id); t = btf_type_by_id(reg->btf, reg->btf_id); if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf, - field->graph_root.value_btf_id, true)) { + field->graph_root.value_btf_id, true, + !type_is_alloc(reg->type))) { verbose(env, "operation on %s expects arg#1 %s at offset=%d " "in struct %s, but arg is at offset=%d in struct %s\n", btf_field_type_name(head_field_type), @@ -11887,7 +12523,7 @@ __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta, BPF_LIST_HEAD, BPF_LIST_NODE, @@ -11896,7 +12532,7 @@ static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta, BPF_RB_ROOT, BPF_RB_NODE, @@ -11925,7 +12561,7 @@ static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env) } } -static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, +static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, int insn_idx) { const char *func_name = meta->func_name, *ref_tname; @@ -11939,18 +12575,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ args = (const struct btf_param *)(meta->func_proto + 1); nargs = btf_type_vlen(meta->func_proto); - if (nargs > MAX_BPF_FUNC_ARGS) { - verbose(env, "Function %s has %d > %d args\n", func_name, nargs, - MAX_BPF_FUNC_ARGS); - return -EINVAL; - } - if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { - verbose(env, "JIT does not support kfunc %s() with %d args\n", - func_name, nargs); - return -ENOTSUPP; - } - ret = check_outgoing_stack_args(env, caller, nargs); + ret = check_outgoing_stack_args(env, caller, nargs, func_name, btf, args); if (ret) return ret; @@ -11964,9 +12590,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ argno_t argno = argno_from_arg(i + 1); int regno = reg_from_argno(argno); bool btf_id_fixed_off_ok = true; - u32 ref_id, type_size; - bool is_ret_buf_sz = false; - int kf_arg_type; + u32 ref_id = args[i].type, type_size; + int kf_arg_type = meta->fn->arg_type[i]; if (is_kfunc_arg_prog_aux(btf, &args[i])) { /* Reject repeated use bpf_prog_aux */ @@ -11989,153 +12614,67 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ t = btf_type_skip_modifiers(btf, args[i].type, NULL); - if (btf_type_is_scalar(t)) { - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - return -EINVAL; - } - - if (is_kfunc_arg_constant(meta->btf, &args[i])) { - if (meta->arg_constant.found) { - verifier_bug(env, "only one constant argument permitted"); - return -EFAULT; - } - if (!tnum_is_const(reg->var_off)) { - verbose(env, "%s must be a known constant\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (regno >= 0) - ret = mark_chain_precision(env, regno); - else - ret = mark_stack_arg_precision(env, i); - if (ret < 0) - return ret; - meta->arg_constant.found = true; - meta->arg_constant.value = reg->var_off.value; - } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) { - meta->r0_rdonly = true; - is_ret_buf_sz = true; - } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdwr_buf_size")) { - is_ret_buf_sz = true; - } - - if (is_ret_buf_sz) { - if (meta->r0_size) { - verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc"); - return -EINVAL; - } - - if (!tnum_is_const(reg->var_off)) { - verbose(env, "%s is not a const\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - - meta->r0_size = reg->var_off.value; - if (regno >= 0) - ret = mark_chain_precision(env, regno); - else - ret = mark_stack_arg_precision(env, i); - if (ret) - return ret; - } - continue; + if (btf_type_is_ptr(t)) { + ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); + ref_tname = btf_name_by_offset(btf, ref_t->name_off); } - if (!btf_type_is_ptr(t)) { - verbose(env, "Unrecognized %s type %s\n", - reg_arg_name(env, argno), btf_type_str(t)); - return -EINVAL; - } + if (btf_type_is_ptr(t) && + (bpf_register_is_null(reg) || type_may_be_null(reg->type)) && + !type_may_be_null(kf_arg_type)) { + const char *expected_type; - if ((bpf_register_is_null(reg) || type_may_be_null(reg->type)) && - !is_kfunc_arg_nullable(meta->btf, &args[i])) { + expected_type = bpf_diag_fmt_btf_type(env, btf, args[i].type); verbose(env, "Possibly NULL pointer passed to trusted %s\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Add a NULL check and call the kfunc only on the non-NULL path.", + "the pointer may be NULL, but this kfunc requires a non-NULL value of type %s", + expected_type); return -EACCES; } if (regno == meta->release_regno && !is_kfunc_arg_dynptr(meta->btf, &args[i]) && !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); verbose(env, "release kfunc %s expects referenced PTR_TO_BTF_ID passed to %s\n", func_name, reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass the resource-owning pointer returned by the matching acquire kfunc, and avoid calling the release kfunc after ownership has already been transferred or released.", + "release kfuncs require a resource-owning value of type %s returned by a matching acquire kfunc", + expected_type); return -EINVAL; } if (reg_is_referenced(env, reg)) update_ref_obj(&meta->ref_obj, reg); - ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); + if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) + continue; - kf_arg_type = get_kfunc_ptr_arg_type(env, caller, regs, meta, t, ref_t, ref_tname, - args, i, nargs, argno, reg); - if (kf_arg_type < 0) - return kf_arg_type; + if (is_kfunc_arg_map(btf, &args[i])) { + ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; + ref_t = btf_type_by_id(btf_vmlinux, ref_id); + ref_tname = btf_name_by_offset(btf, ref_t->name_off); + } - switch (kf_arg_type) { - case KF_ARG_PTR_TO_NULL: - continue; - case KF_ARG_PTR_TO_MAP: - if (!reg->map_ptr) { - verbose(env, "pointer in %s isn't map pointer\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (meta->map.ptr && (reg->map_ptr->record->wq_off >= 0 || - reg->map_ptr->record->task_work_off >= 0)) { - /* Use map_uid (which is unique id of inner map) to reject: - * inner_map1 = bpf_map_lookup_elem(outer_map, key1) - * inner_map2 = bpf_map_lookup_elem(outer_map, key2) - * if (inner_map1 && inner_map2) { - * wq = bpf_map_lookup_elem(inner_map1); - * if (wq) - * // mismatch would have been allowed - * bpf_wq_init(wq, inner_map2); - * } - * - * Comparing map_ptr is enough to distinguish normal and outer maps. - */ - if (meta->map.ptr != reg->map_ptr || - meta->map.uid != reg->map_uid) { - if (reg->map_ptr->record->task_work_off >= 0) { - verbose(env, - "bpf_task_work pointer in R2 map_uid=%d doesn't match map pointer in R3 map_uid=%d\n", - meta->map.uid, reg->map_uid); - return -EINVAL; - } - verbose(env, - "workqueue pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", - meta->map.uid, reg->map_uid); - return -EINVAL; - } - } - meta->map.ptr = reg->map_ptr; - meta->map.uid = reg->map_uid; - fallthrough; + switch (base_type(kf_arg_type)) { + case KF_ARG_CONST: + case KF_ARG_CONST_MEM_SIZE: + case KF_ARG_MEM_SIZE: + case KF_ARG_ANYTHING: + case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: case KF_ARG_PTR_TO_ALLOC_BTF_ID: case KF_ARG_PTR_TO_BTF_ID: - if (!is_trusted_reg(env, reg)) { - if (!is_kfunc_rcu(meta)) { - verbose(env, "%s must be referenced or trusted\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (!is_rcu_reg(reg)) { - verbose(env, "%s must be a rcu pointer\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } - fallthrough; + case KF_ARG_CONST_MAP_PTR: case KF_ARG_PTR_TO_ITER: case KF_ARG_PTR_TO_LIST_HEAD: case KF_ARG_PTR_TO_LIST_NODE: case KF_ARG_PTR_TO_RB_ROOT: case KF_ARG_PTR_TO_RB_NODE: case KF_ARG_PTR_TO_MEM: - case KF_ARG_PTR_TO_MEM_SIZE: case KF_ARG_PTR_TO_CALLBACK: case KF_ARG_PTR_TO_CONST_STR: case KF_ARG_PTR_TO_WORKQUEUE: @@ -12143,6 +12682,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ case KF_ARG_PTR_TO_TASK_WORK: case KF_ARG_PTR_TO_IRQ_FLAG: case KF_ARG_PTR_TO_RES_SPIN_LOCK: + case KF_ARG_PTR_TO_ARENA: break; case KF_ARG_PTR_TO_DYNPTR: arg_type = ARG_PTR_TO_DYNPTR; @@ -12166,11 +12706,71 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ if (ret < 0) return ret; - switch (kf_arg_type) { + switch (base_type(kf_arg_type)) { + case KF_ARG_CONST: + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass an integer scalar value for this argument, not a pointer or resource object.", + "the kfunc expects an integer scalar, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); + return -EINVAL; + } + + ret = process_const_arg(env, reg, argno, meta); + if (ret < 0) { + if (ret == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a compile-time constant or a value the verifier can prove is constant at this call.", + "the kfunc requires this scalar argument to be a verifier-known constant, but %s is variable on this path", + reg_arg_name(env, argno)); + return ret; + } + break; + case KF_ARG_ANYTHING: + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass an integer scalar value for this argument, not a pointer or resource object.", + "the kfunc expects an integer scalar, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); + return -EINVAL; + } + break; + case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass an integer scalar value for this argument, not a pointer or resource object.", + "the kfunc expects an integer scalar, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); + return -EINVAL; + } + + if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) + meta->r0_rdonly = true; + ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); + if (ret < 0) { + if (ret == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a verifier-known constant size for this kfunc buffer argument.", + "the kfunc uses this argument as a return-buffer size, but %s is invalid or variable on this path", + reg_arg_name(env, argno)); + return ret; + } + break; case KF_ARG_PTR_TO_CTX: if (reg->type != PTR_TO_CTX) { verbose(env, "%s expected pointer to ctx, but got %s\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass the original program context pointer or preserve it before modifying registers.", + "the kfunc expects a context pointer, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } @@ -12181,6 +12781,13 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ meta->ret_btf_id = ret; } break; + case KF_ARG_PTR_TO_ARENA: + if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a pointer to arena or scalar\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + break; case KF_ARG_PTR_TO_ALLOC_BTF_ID: if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { if (!is_bpf_obj_drop_kfunc(meta->func_id)) { @@ -12197,10 +12804,19 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } else { verbose(env, "%s expected pointer to allocated object\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a pointer returned by the matching BPF object allocation path.", + "the kfunc expects an allocated object pointer, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } if (!reg_is_referenced(env, reg)) { verbose(env, "allocated object must be referenced\n"); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass the owned object pointer before it is released or transferred.", + "the allocated object pointer in %s must still carry verifier-tracked ownership, but this pointer no longer owns a live resource", + reg_arg_name(env, argno)); return -EINVAL; } if (meta->btf == btf_vmlinux) { @@ -12237,8 +12853,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type); } - ret = process_dynptr_func(env, reg, argno, insn_idx, dynptr_arg_type, - &meta->ref_obj, &meta->dynptr); + ret = process_dynptr_func(env, reg, argno, insn_idx, func_name, + dynptr_arg_type, &meta->ref_obj, &meta->dynptr); if (ret < 0) return ret; break; @@ -12335,75 +12951,181 @@ check_ok: if (ret < 0) return ret; break; - case KF_ARG_PTR_TO_MAP: - /* If argument has '__map' suffix expect 'struct bpf_map *' */ - ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; - ref_t = btf_type_by_id(btf_vmlinux, ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); - fallthrough; - case KF_ARG_PTR_TO_BTF_ID: - /* Only base_type is checked, further checks are done here */ - if ((base_type(reg->type) != PTR_TO_BTF_ID || - (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) && - !reg2btf_ids[base_type(reg->type)]) { - verbose(env, "%s is %s ", reg_arg_name(env, argno), - reg_type_str(env, reg->type)); - verbose(env, "expected %s or socket\n", - reg_type_str(env, base_type(reg->type) | - (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS))); + case KF_ARG_CONST_MAP_PTR: + if (base_type(reg->type) != CONST_PTR_TO_MAP || + type_may_be_null(reg->type)) { + verbose(env, "pointer in %s isn't map pointer\n", + reg_arg_name(env, argno)); return -EINVAL; } - ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno); + ret = process_map_ptr_arg(env, reg, argno, meta); if (ret < 0) return ret; break; - case KF_ARG_PTR_TO_MEM: - resolve_ret = btf_resolve_size(btf, ref_t, &type_size); - if (IS_ERR(resolve_ret)) { - verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", - reg_arg_name(env, argno), btf_type_str(ref_t), - ref_tname, PTR_ERR(resolve_ret)); + case KF_ARG_PTR_TO_BTF_ID: + /* Only base_type is checked, further checks are done here */ + if (base_type(reg->type) == PTR_TO_BTF_ID || + reg2btf_ids[base_type(reg->type)]) { + if (!is_trusted_reg(env, reg) || + bpf_type_has_unsafe_modifiers(reg->type)) { + if (!is_kfunc_rcu(meta)) { + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); + verbose(env, "%s must be referenced or trusted\n", + reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a pointer acquired from a verifier-tracked source, or call this kfunc only inside the required protection if it accepts RCU pointers.", + "the kfunc requires a trusted or resource-owning pointer to %s, but %s is %s", + expected_type, + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); + return -EINVAL; + } + if (!is_rcu_reg(reg)) { + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); + verbose(env, "%s must be a rcu pointer\n", + reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Use this kfunc with a pointer that is valid in an RCU read lock region.", + "the kfunc requires an RCU-protected pointer to %s, but %s is %s", + expected_type, + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); + return -EINVAL; + } + } + + ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno); + if (ret < 0) + return ret; + break; + } + + if (!__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); + const char *expected_type; + + verbose(env, "%s is %s expected %s %s", + reg_arg_name(env, argno), reg_type_str(env, reg->type), + btf_type_str(ref_t), ref_tname); + if (reg2btf_type != NOT_INIT) + verbose(env, " or %s", reg_type_str(env, reg2btf_type)); + verbose(env, "\n"); + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a verifier-tracked pointer to the expected kernel object type, not a pointer to stack storage or another memory buffer.", + "the kfunc expects a pointer to %s, but this argument is %s and cannot be used as that kernel object pointer", + expected_type, + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } - ret = check_mem_reg(env, reg, argno, type_size); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_MEM_SIZE: - { - struct bpf_reg_state *buff_reg = reg; - const struct btf_param *buff_arg = &args[i]; - struct bpf_reg_state *size_reg = get_func_arg_reg(caller, regs, i + 1); - const struct btf_param *size_arg = &args[i + 1]; - argno_t next_argno = argno_from_arg(i + 2); - - if (!bpf_register_is_null(buff_reg) || !is_kfunc_arg_nullable(meta->btf, buff_arg)) { - ret = check_kfunc_mem_size_reg(env, buff_reg, size_reg, - argno, next_argno); + + /* + * If the register does not contain btf id but the argument type is a pointer to + * scalar-only struct, allow verifying it as a fixed size memory. + */ + kf_arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; + fallthrough; + case KF_ARG_PTR_TO_MEM: + if (kf_arg_type & MEM_FIXED_SIZE) { + bool known_memory; + + resolve_ret = btf_resolve_size(btf, ref_t, &type_size); + if (IS_ERR(resolve_ret)) { + verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), + ref_tname, PTR_ERR(resolve_ret)); + return -EINVAL; + } + ret = check_mem_reg(env, reg, argno, type_size, BPF_READ | BPF_WRITE, + meta, &known_memory); if (ret < 0) { - verbose(env, "%s and ", reg_arg_name(env, argno)); - verbose(env, "%s memory, len pair leads to invalid memory access\n", - reg_arg_name(env, next_argno)); + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); + if (known_memory) + bpf_diag_call_arg_fmt( + env, insn_idx, argno, func_name, + "Pass memory with at least the required number of accessible bytes and suitable read and write access.", + "the kfunc expects %u bytes of memory for %s, but the verifier cannot prove that %s provides a readable and writable range of that size", + type_size, expected_type, + bpf_diag_reg_type_plain(env, reg->type)); + else + bpf_diag_call_arg_fmt( + env, insn_idx, argno, func_name, + "Pass stack, map, context, or other verifier-known memory of the expected type and size, not an integer cast to a pointer.", + "the kfunc expects %u bytes of memory for %s, but it is %s and not verifier-known memory", + type_size, expected_type, + bpf_diag_reg_type_plain(env, reg->type)); return ret; } } + break; + case KF_ARG_CONST_MEM_SIZE: + ret = process_const_arg(env, reg, argno, meta); + if (ret < 0) { + if (ret == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a compile-time constant or a value the verifier can prove is constant at this call.", + "the kfunc requires this memory size to be a verifier-known constant, but %s is variable on this path", + reg_arg_name(env, argno)); + return ret; + } + fallthrough; + case KF_ARG_MEM_SIZE: + { + struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, i - 1); + struct bpf_reg_state *size_reg = reg; + argno_t buff_argno = argno_from_arg(i); + enum bpf_mem_size_failure failure; - if (is_kfunc_arg_const_mem_size(meta->btf, size_arg, size_reg)) { - if (meta->arg_constant.found) { - verifier_bug(env, "only one constant argument permitted"); - return -EFAULT; - } - if (!tnum_is_const(size_reg->var_off)) { - verbose(env, "%s must be a known constant\n", - reg_arg_name(env, next_argno)); - return -EINVAL; - } - meta->arg_constant.found = true; - meta->arg_constant.value = size_reg->var_off.value; + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass an integer scalar length for this memory argument.", + "the kfunc expects a scalar memory size, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); + return -EINVAL; } - /* Skip next '__sz' or '__szk' argument */ - i++; + if (bpf_register_is_null(buff_reg)) + break; + + ret = check_mem_size_reg(env, buff_reg, size_reg, buff_argno, argno, + BPF_READ | BPF_WRITE, true, meta, &failure); + if (ret < 0) { + const char *buff_arg, *size_arg; + + buff_arg = bpf_diag_arg_name(env, buff_argno); + size_arg = bpf_diag_arg_name(env, argno); + verbose(env, "%s and ", reg_arg_name(env, buff_argno)); + verbose(env, "%s memory, len pair leads to invalid memory access\n", + reg_arg_name(env, argno)); + if (failure == BPF_MEM_SIZE_FAIL_MEMORY) { + bpf_diag_call_arg_fmt(env, insn_idx, buff_argno, func_name, + "Pass a stack, map, context, or other verifier-known memory pointer, and keep the paired length within that object.", + "it is the memory pointer in a memory/length pair with %s, but %s does not describe verifier-readable memory for the requested length", + size_arg, buff_arg); + } else if (failure == BPF_MEM_SIZE_FAIL_SIZE) { + if (reg_smin(size_reg) < 0) + bpf_diag_call_arg_fmt( + env, insn_idx, argno, func_name, + "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", + "the memory size in %s may be negative because its signed minimum is %lld", + size_arg, reg_smin(size_reg)); + else + bpf_diag_call_arg_fmt( + env, insn_idx, argno, func_name, + "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", + "the memory size in %s may reach %llu bytes, but variable memory accesses must stay below %u bytes", + size_arg, reg_umax(size_reg), BPF_MAX_VAR_SIZ); + } + return ret; + } break; } case KF_ARG_PTR_TO_CALLBACK: @@ -12417,6 +13139,11 @@ check_ok: if (!type_is_ptr_alloc_obj(reg->type)) { verbose(env, "%s is neither owning or non-owning ref\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass an owning or non-owning pointer to a BPF-managed object containing a bpf_refcount field.", + "the kfunc expects a pointer to a BPF-managed refcounted object, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } if (!type_is_non_owning_ref(reg->type)) @@ -12441,6 +13168,11 @@ check_ok: if (reg->type != PTR_TO_MAP_VALUE) { verbose(env, "%s doesn't point to a const string\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a constant string pointer that the verifier recognizes, such as a string stored in a read-only map value.", + "the kfunc expects a pointer to a constant string stored in verifier-known memory, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } ret = check_arg_const_str(env, reg, argno); @@ -12463,7 +13195,7 @@ check_ok: reg_arg_name(env, argno)); return -EINVAL; } - ret = process_timer_kfunc(env, reg, argno, meta); + ret = process_timer_func(env, reg, argno, &meta->map); if (ret < 0) return ret; break; @@ -12481,6 +13213,11 @@ check_ok: if (reg->type != PTR_TO_STACK) { verbose(env, "%s doesn't point to an irq flag on stack\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass the same stack slot used by bpf_local_irq_save() or bpf_res_spin_lock_irqsave().", + "the kfunc expects a stack pointer to an IRQ flag slot, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } ret = process_irq_flag(env, reg, argno, meta); @@ -12519,16 +13256,17 @@ check_ok: int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, s32 func_id, s16 offset, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { struct bpf_kfunc_meta kfunc; int err; + memset(meta, 0, sizeof(*meta)); + err = fetch_kfunc_meta(env, func_id, offset, &kfunc); if (err) return err; - memset(meta, 0, sizeof(*meta)); meta->btf = kfunc.btf; meta->func_id = kfunc.id; meta->func_proto = kfunc.proto; @@ -12678,7 +13416,7 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * int arg, int insn_idx) { struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; - struct bpf_kfunc_call_arg_meta meta; + struct bpf_call_arg_meta meta; const struct btf_param *args; const struct btf_type *t, *ref_t; const struct btf *btf; @@ -12739,7 +13477,7 @@ out: * 0 - fall-through to 'else' branch * < 0 - not fall-through to 'else' branch, return error */ -static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, +static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, struct bpf_reg_state *regs, struct bpf_insn_aux_data *insn_aux, const struct btf_type *ptr_type, struct btf *desc_btf) { @@ -12918,11 +13656,12 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_reg_state *regs = cur_regs(env); const char *func_name, *ptr_type_name; const struct btf_type *t, *ptr_type; - struct bpf_kfunc_call_arg_meta meta; + struct bpf_call_arg_meta meta; struct bpf_insn_aux_data *insn_aux; + const char *operation; int err, insn_idx = *insn_idx_p; - const struct btf_param *args; u32 i, nargs, ptr_type_id; + struct bpf_kfunc_desc *desc; struct btf *desc_btf; int id; @@ -12931,14 +13670,26 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; err = bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta); - if (err == -EACCES && meta.func_name) + if (err == -EACCES && meta.func_name) { verbose(env, "calling kernel function %s is not allowed\n", meta.func_name); + operation = bpf_diag_fmt(env, "kfunc %s", meta.func_name); + bpf_diag_policy( + env, insn_idx, operation, "this program cannot call the kfunc", + "Use a kfunc allowed for this program type and attach point, or change the program context."); + } if (err) return err; desc_btf = meta.btf; func_name = meta.func_name; insn_aux = &env->insn_aux_data[insn_idx]; + desc = find_kfunc_desc(env->prog, insn->imm, insn->off); + if (!desc) { + verifier_bug(env, "kfunc descriptor not found for func_id %u", insn->imm); + return -EFAULT; + } + meta.fn = &desc->proto; + insn_aux->is_iter_next = bpf_is_iter_next_kfunc(&meta); if (!insn->off && @@ -12965,7 +13716,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, verbose(env, "failed to mark s32 range for retval in forked state for lock\n"); return err; } - __mark_btf_func_reg_size(env, regs, BPF_REG_0, sizeof(u32)); } else if (!insn->off && insn->imm == special_kfunc_list[KF___bpf_trap]) { verbose(env, "unexpected __bpf_trap() due to uninitialized variable?\n"); return -EFAULT; @@ -12973,12 +13723,19 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (is_kfunc_destructive(&meta) && !capable(CAP_SYS_BOOT)) { verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n"); + operation = bpf_diag_fmt(env, "destructive kfunc %s", meta.func_name); + bpf_diag_policy( + env, insn_idx, operation, "destructive kfuncs require CAP_SYS_BOOT", + "Load the program with CAP_SYS_BOOT, or avoid destructive kfuncs."); return -EACCES; } sleepable = bpf_is_kfunc_sleepable(&meta); if (sleepable && !in_sleepable(env)) { verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); + operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name); + bpf_diag_ctx_forbidden(env, insn_idx, operation, + "Mark the program sleepable if the program type allows it, or use a non-sleepable kfunc."); return -EACCES; } @@ -13016,11 +13773,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } } - if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) { - meta.r0_size = sizeof(u64); - meta.r0_rdonly = false; - } - if (is_bpf_wq_set_callback_kfunc(meta.func_id)) { err = push_callback_call(env, insn, insn_idx, meta.subprogno, set_timer_callback_state); @@ -13049,26 +13801,46 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (rcu_lock) { env->cur_state->active_rcu_locks++; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_RCU, true, + env->cur_state->active_rcu_locks); } else if (rcu_unlock) { if (env->cur_state->active_rcu_locks == 0) { verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name); + bpf_diag_ctx_underflow( + env, insn_idx, func_name, BPF_DIAG_CONTEXT_RCU, + "Remove the extra bpf_rcu_read_unlock() call, or ensure this path first enters an RCU read lock region."); return -EINVAL; } - if (--env->cur_state->active_rcu_locks == 0) + env->cur_state->active_rcu_locks--; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_RCU, false, + env->cur_state->active_rcu_locks); + if (!in_rcu_cs(env)) invalidate_rcu_protected_refs(env); } else if (preempt_disable) { env->cur_state->active_preempt_locks++; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_PREEMPT, true, + env->cur_state->active_preempt_locks); } else if (preempt_enable) { if (env->cur_state->active_preempt_locks == 0) { verbose(env, "unmatched attempt to enable preemption (kernel function %s)\n", func_name); + bpf_diag_ctx_underflow( + env, insn_idx, func_name, BPF_DIAG_CONTEXT_PREEMPT, + "Remove the extra bpf_preempt_enable() call, or ensure this path first disables preemption."); return -EINVAL; } env->cur_state->active_preempt_locks--; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_PREEMPT, false, + env->cur_state->active_preempt_locks); + if (!in_rcu_cs(env)) + invalidate_rcu_protected_refs(env); } if (sleepable && !in_sleepable_context(env)) { verbose(env, "kernel func %s is sleepable within %s\n", func_name, non_sleepable_context_description(env)); + operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name); + bpf_diag_ctx_forbidden(env, insn_idx, operation, + "Move the kfunc call outside the critical section, or use a non-sleepable kfunc."); return -EACCES; } @@ -13079,6 +13851,9 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (is_kfunc_rcu_protected(&meta) && !in_rcu_cs(env)) { verbose(env, "kernel func %s requires RCU critical section protection\n", func_name); + bpf_diag_ctx_required( + env, insn_idx, func_name, BPF_DIAG_CONTEXT_RCU, + "Call this kfunc between bpf_rcu_read_lock() and bpf_rcu_read_unlock(), keeping all exit paths balanced."); return -EACCES; } @@ -13116,11 +13891,12 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } } + bpf_diag_record_caller_saved(env, regs); + bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); for (i = 0; i < CALLER_SAVED_REGS; i++) { u32 regno = caller_saved[i]; bpf_mark_reg_not_init(env, ®s[regno]); - regs[regno].subreg_def = DEF_NOT_SUBREG; } invalidate_outgoing_stack_args(env, cur_func(env)); @@ -13142,7 +13918,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] || meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); - mark_btf_func_reg_size(env, BPF_REG_0, t->size); } else if (btf_type_is_ptr(t)) { ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf); @@ -13153,15 +13928,19 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, /* kfunc returning 'void *' is equivalent to returning scalar */ mark_reg_unknown(env, regs, BPF_REG_0); } else if (!__btf_type_is_struct(ptr_type)) { - if (!meta.r0_size) { + if (!meta.ret_mem.found) { __u32 sz; if (!IS_ERR(btf_resolve_size(desc_btf, ptr_type, &sz))) { - meta.r0_size = sz; + meta.ret_mem.found = true; + meta.ret_mem.size = sz; meta.r0_rdonly = true; } + + if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) + meta.r0_rdonly = false; } - if (!meta.r0_size) { + if (!meta.ret_mem.found) { ptr_type_name = btf_name_by_offset(desc_btf, ptr_type->name_off); verbose(env, @@ -13174,7 +13953,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_MEM; - regs[BPF_REG_0].mem_size = meta.r0_size; + regs[BPF_REG_0].mem_size = meta.ret_mem.size; if (meta.r0_rdonly) regs[BPF_REG_0].type |= MEM_RDONLY; @@ -13229,7 +14008,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, /* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */ regs[BPF_REG_0].id = ++env->id_gen; } - mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); if (is_kfunc_acquire(&meta)) { id = acquire_reference(env, insn_idx, 0); if (id < 0) @@ -13266,17 +14044,11 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, caller_info->stack_arg_cnt = stack_arg_cnt; } - args = (const struct btf_param *)(meta.func_proto + 1); - for (i = 0; i < min_t(int, nargs, MAX_BPF_FUNC_REG_ARGS); i++) { - u32 regno = i + 1; - - t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL); - if (btf_type_is_ptr(t)) - mark_btf_func_reg_size(env, regno, sizeof(void *)); - else - /* scalar. ensured by check_kfunc_args() */ - mark_btf_func_reg_size(env, regno, t->size); - } + /* + * Record R0 before process_iter_next_call() snapshots the alternate + * iterator path's diagnostic position. + */ + bpf_diag_mod_end(env); if (bpf_is_iter_next_kfunc(&meta)) { err = process_iter_next_call(env, insn_idx, &meta); @@ -13661,9 +14433,8 @@ static int sanitize_check_bounds(struct bpf_verifier_env *env, * If we return -EACCES, caller may want to try again treating pointer as a * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. */ -static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, - struct bpf_insn *insn, - const struct bpf_reg_state *ptr_reg, +static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn, + u32 ptr_regno, const struct bpf_reg_state *ptr_reg, const struct bpf_reg_state *off_reg) { struct bpf_verifier_state *vstate = env->cur_state; @@ -13675,6 +14446,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_sanitize_info info = {}; u8 opcode = BPF_OP(insn->code); u32 dst = insn->dst_reg; + const char *reason; int ret, bounds_ret; dst_reg = ®s[dst]; @@ -13698,12 +14470,24 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, verbose(env, "R%d 32-bit pointer arithmetic prohibited\n", dst); + reason = bpf_diag_fmt( + env, "R%d holds %s. 32-bit ALU operations on pointers discard pointer tracking, so the verifier cannot keep the result as a safe pointer.", + ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "32-bit pointer arithmetic", reason, + "Use a 64-bit ALU instruction with an allowed, bounded scalar offset."); return -EACCES; } if (ptr_reg->type & PTR_MAYBE_NULL) { verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", dst, reg_type_str(env, ptr_reg->type)); + reason = bpf_diag_fmt( + env, "R%d may be NULL (%s). Pointer arithmetic is allowed only after the program proves the pointer is non-NULL on this path.", + ptr_regno, reg_type_str(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer arithmetic before NULL check", reason, + "Make sure that a NULL check precedes any arithmetic performed on the pointer."); return -EACCES; } @@ -13733,6 +14517,12 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, default: verbose(env, "R%d pointer arithmetic on %s prohibited\n", dst, reg_type_str(env, ptr_reg->type)); + reason = bpf_diag_fmt( + env, "R%d holds %s. This pointer kind does not allow offset arithmetic.", + ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer arithmetic is not allowed", reason, + "Do not change this pointer's offset; use it only in operations accepted for its kind."); return -EACCES; } @@ -13750,9 +14540,25 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, if (base_type(ptr_reg->type) == PTR_TO_MEM && (ptr_reg->type & PTR_UNTRUSTED)) return 0; - if (!check_reg_sane_offset_scalar(env, off_reg, ptr_reg->type) || - !check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type)) + if (!check_reg_sane_offset_scalar(env, off_reg, ptr_reg->type)) { + reason = bpf_diag_fmt( + env, "The scalar offset used with R%d is unbounded or outside the verifier's safe pointer-offset range [-%u, %u].", + ptr_regno, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason, + "Clamp or bounds-check the scalar offset before applying it to the pointer."); + return -EINVAL; + } + if (!check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type)) { + reason = bpf_diag_fmt( + env, "R%d already has an offset outside the verifier's safe range [-%u, %u] for %s.", + ptr_regno, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF, + bpf_diag_reg_type_plain(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason, + "Keep the base pointer within the verifier's allowed offset range before applying more arithmetic."); return -EINVAL; + } /* pointer types do not carry 32-bit bounds at the moment. */ __mark_reg32_unbounded(dst_reg); @@ -13795,6 +14601,13 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, /* scalar -= pointer. Creates an unknown scalar */ verbose(env, "R%d tried to subtract pointer from scalar\n", dst); + reason = bpf_diag_fmt( + env, "This operation subtracts pointer register R%d from scalar register R%d. " + "The verifier only tracks pointer-minus-scalar arithmetic for allowed pointer types.", + ptr_regno, dst); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer subtracted from scalar", reason, + "Keep the pointer as the base; only add or subtract bounded scalars when permitted."); return -EACCES; } /* We don't allow subtraction from FP, because (according to @@ -13804,6 +14617,12 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, if (ptr_reg->type == PTR_TO_STACK) { verbose(env, "R%d subtraction from stack pointer prohibited\n", dst); + reason = bpf_diag_fmt( + env, "R%d is a stack pointer. The verifier does not allow BPF_SUB to move stack pointers.", + ptr_regno); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "subtraction from stack pointer", reason, + "Use addition from R10 to form stack addresses within the tracked stack frame."); return -EACCES; } dst_reg->r64 = cnum64_add(ptr_reg->r64, cnum64_negate(off_reg->r64)); @@ -13829,16 +14648,38 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, /* bitwise ops on pointers are troublesome, prohibit. */ verbose(env, "R%d bitwise operator %s on pointer prohibited\n", dst, bpf_alu_string[opcode >> 4]); + reason = bpf_diag_fmt( + env, "R%d holds %s. Bitwise operator %s would destroy the pointer value the verifier is tracking.", + ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type), + bpf_alu_string[opcode >> 4]); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "bitwise operation on pointer", reason, + "Do bitwise operations on scalar values, not on pointer-valued registers."); return -EACCES; default: /* other operators (e.g. MUL,LSH) produce non-pointer results */ verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", dst, bpf_alu_string[opcode >> 4]); + reason = bpf_diag_fmt( + env, "R%d holds %s. Operator %s is not one of the limited pointer arithmetic operations the verifier can track.", + ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type), + bpf_alu_string[opcode >> 4]); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "invalid pointer arithmetic operator", reason, + "Use only verifier-supported addition or subtraction with a bounded scalar offset, or perform this operation on a scalar value."); return -EACCES; } - if (!check_reg_sane_offset_ptr(env, dst_reg, ptr_reg->type)) + if (!check_reg_sane_offset_ptr(env, dst_reg, ptr_reg->type)) { + reason = bpf_diag_fmt( + env, "After this arithmetic, R%d would be outside the verifier's safe offset range [-%u, %u] for %s.", + dst, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF, + bpf_diag_reg_type_plain(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason, + "Tighten the scalar bounds before the arithmetic so the resulting pointer remains within the allowed range."); return -EINVAL; + } reg_bounds_sync(dst_reg); bounds_ret = sanitize_check_bounds(env, insn, dst_reg); if (bounds_ret == -EACCES) @@ -14770,14 +15611,14 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, if (dst_reg->type != PTR_TO_ARENA) *dst_reg = *src_reg; - dst_reg->subreg_def = env->insn_idx + 1; - - if (BPF_CLASS(insn->code) == BPF_ALU64) + if (BPF_CLASS(insn->code) == BPF_ALU64) { /* * 32-bit operations zero upper bits automatically. * 64-bit operations need to be converted to 32. */ aux->needs_zext = true; + aux->zext_dst = true; + } /* Any arithmetic operations are allowed on arena pointers */ return 0; @@ -14810,15 +15651,15 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, if (err) return err; off_reg = *dst_reg; - return adjust_ptr_min_max_vals(env, insn, src_reg, &off_reg); + return adjust_ptr_min_max_vals(env, insn, insn->src_reg, src_reg, + &off_reg); } } else if (ptr_reg) { /* pointer += scalar */ err = mark_chain_precision(env, insn->src_reg); if (err) return err; - return adjust_ptr_min_max_vals(env, insn, - dst_reg, src_reg); + return adjust_ptr_min_max_vals(env, insn, insn->dst_reg, dst_reg, src_reg); } else if (dst_reg->precise) { /* if dst_reg is precise, src_reg should be precise as well */ err = mark_chain_precision(env, insn->src_reg); @@ -14833,8 +15674,7 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, __mark_reg_known(&off_reg, insn->imm); src_reg = &off_reg; if (ptr_reg) /* pointer += K */ - return adjust_ptr_min_max_vals(env, insn, - ptr_reg, src_reg); + return adjust_ptr_min_max_vals(env, insn, insn->dst_reg, ptr_reg, src_reg); } /* Got here implies adding two SCALAR_VALUEs */ @@ -14920,6 +15760,8 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) u8 opcode = BPF_OP(insn->code); int err; + bpf_diag_mod_begin(env, ®s[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE); + if (opcode == BPF_END || opcode == BPF_NEG) { /* check src operand */ err = check_reg_arg(env, insn->dst_reg, SRC_OP); @@ -14973,18 +15815,14 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) if (insn->imm) { /* off == BPF_ADDR_SPACE_CAST */ mark_reg_unknown(env, regs, insn->dst_reg); - if (insn->imm == 1) { /* cast from as(1) to as(0) */ + if (insn->imm == 1) /* cast from as(1) to as(0) */ dst_reg->type = PTR_TO_ARENA; - /* PTR_TO_ARENA is 32-bit */ - dst_reg->subreg_def = env->insn_idx + 1; - } } else if (insn->off == 0) { /* case: R1 = R2 * copy register state to dest reg */ assign_scalar_id_before_mov(env, src_reg); *dst_reg = *src_reg; - dst_reg->subreg_def = DEF_NOT_SUBREG; } else { /* case: R1 = (s8, s16 s32)R2 */ if (is_pointer_value(env, insn->src_reg)) { @@ -15002,7 +15840,6 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) if (!no_sext) clear_scalar_id(dst_reg); coerce_reg_to_size_sx(dst_reg, insn->off >> 3); - dst_reg->subreg_def = DEF_NOT_SUBREG; } else { mark_reg_unknown(env, regs, insn->dst_reg); } @@ -15027,7 +15864,6 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) */ if (!is_src_reg_u32) clear_scalar_id(dst_reg); - dst_reg->subreg_def = env->insn_idx + 1; } else { /* case: W1 = (s8, s16)W2 */ bool no_sext = reg_umax(src_reg) < (1ULL << (insn->off - 1)); @@ -15037,7 +15873,6 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) *dst_reg = *src_reg; if (!no_sext) clear_scalar_id(dst_reg); - dst_reg->subreg_def = env->insn_idx + 1; coerce_subreg_to_size_sx(dst_reg, insn->off >> 3); } } else { @@ -15100,7 +15935,12 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) return err; } - return reg_bounds_sanity_check(env, ®s[insn->dst_reg], "alu"); + err = reg_bounds_sanity_check(env, ®s[insn->dst_reg], "alu"); + if (err) + return err; + + bpf_diag_mod_end(env); + return 0; } static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, @@ -15715,7 +16555,7 @@ static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, * No one could have freed the reference state before * doing the NULL check. */ - WARN_ON_ONCE(release_reference_nomark(vstate, id)); + WARN_ON_ONCE(__release_reference_nomark(vstate, id)); bpf_for_each_reg_in_vstate(vstate, state, reg, ({ mark_ptr_or_null_reg(state, reg, id, is_null); @@ -15906,12 +16746,8 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s continue; if ((!(reg->id & BPF_ADD_CONST) && !(known_reg->id & BPF_ADD_CONST)) || reg->delta == known_reg->delta) { - s32 saved_subreg_def = reg->subreg_def; - *reg = *known_reg; - reg->subreg_def = saved_subreg_def; } else { - s32 saved_subreg_def = reg->subreg_def; s32 saved_off = reg->delta; u32 saved_id = reg->id; @@ -15921,12 +16757,11 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s /* reg = known_reg; reg += delta */ *reg = *known_reg; /* - * Must preserve off, id and subreg_def flag, - * otherwise another sync_linked_regs() will be incorrect. + * Must preserve off and id, otherwise another sync_linked_regs() + * will be incorrect. */ reg->delta = saved_off; reg->id = saved_id; - reg->subreg_def = saved_subreg_def; scalar32_min_max_add(reg, &fake_reg); scalar_min_max_add(reg, &fake_reg); @@ -16199,11 +17034,13 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) return err; dst_reg = ®s[insn->dst_reg]; + bpf_diag_mod_begin(env, dst_reg, NULL, BPF_DIAG_MOD_WRITE); if (insn->src_reg == 0) { u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; dst_reg->type = SCALAR_VALUE; __mark_reg_known(®s[insn->dst_reg], imm); + bpf_diag_mod_end(env); return 0; } @@ -16227,6 +17064,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) verifier_bug(env, "pseudo btf id: unexpected dst reg type"); return -EFAULT; } + bpf_diag_mod_end(env); return 0; } @@ -16246,6 +17084,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) dst_reg->type = PTR_TO_FUNC; dst_reg->subprogno = subprogno; + bpf_diag_mod_end(env); return 0; } @@ -16256,6 +17095,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) if (map->map_type == BPF_MAP_TYPE_ARENA) { __mark_reg_unknown(env, dst_reg); dst_reg->map_ptr = map; + bpf_diag_mod_end(env); return 0; } __mark_reg_known(dst_reg, aux->map_off); @@ -16273,6 +17113,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) return -EFAULT; } + bpf_diag_mod_end(env); return 0; } @@ -16351,6 +17192,8 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) return err; /* reset caller saved regs to unreadable */ + bpf_diag_record_caller_saved(env, regs); + bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); for (i = 0; i < CALLER_SAVED_REGS; i++) { bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); @@ -16361,8 +17204,7 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) * Already marked as written above. */ mark_reg_unknown(env, regs, BPF_REG_0); - /* ld_abs load up to 32-bit skb data. */ - regs[BPF_REG_0].subreg_def = env->insn_idx + 1; + bpf_diag_mod_end(env); /* * See bpf_gen_ld_abs() which emits a hidden BPF_EXIT with r0=0 * which must be explored by the verifier when in a subprog. @@ -16383,7 +17225,6 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) return 0; } - static bool return_retval_range(struct bpf_verifier_env *env, struct bpf_retval_range *range) { enum bpf_prog_type prog_type = resolve_prog_type(env->prog); @@ -16672,12 +17513,11 @@ bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm) bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, struct bpf_call_summary *cs) { - struct bpf_kfunc_call_arg_meta meta; + struct bpf_call_arg_meta meta; const struct bpf_func_proto *fn; int i; if (bpf_helper_call(call)) { - if (bpf_get_helper_proto(env, call->imm, &fn) < 0) /* error would be reported later */ return false; @@ -16987,6 +17827,8 @@ static bool reg_type_mismatch_ok(enum bpf_reg_type type) case PTR_TO_BTF_ID: case PTR_TO_ARENA: return false; + case PTR_TO_MEM: + return !bpf_may_fault_on_deref(type); default: return true; } @@ -17010,27 +17852,29 @@ static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) !reg_type_mismatch_ok(prev)); } -static bool is_ptr_to_mem_or_btf_id(enum bpf_reg_type type) +static bool is_ptr_to_mem(enum bpf_reg_type type) { - switch (base_type(type)) { - case PTR_TO_MEM: - case PTR_TO_BTF_ID: - return true; - default: - return false; - } + return base_type(type) == PTR_TO_MEM; } -static bool is_ptr_to_mem(enum bpf_reg_type type) +static enum bpf_reg_type merge_ptr_types(enum bpf_reg_type type_a, + enum bpf_reg_type type_b) { - return base_type(type) == PTR_TO_MEM; + bool to_mem = is_ptr_to_mem(type_a) || is_ptr_to_mem(type_b); + enum bpf_reg_type type_merged = to_mem ? PTR_TO_MEM : PTR_TO_BTF_ID; + + if (bpf_may_fault_on_deref(type_a) || bpf_may_fault_on_deref(type_b)) + type_merged |= to_mem ? MEM_RDONLY | PTR_UNTRUSTED : + PTR_UNTRUSTED; + else + type_merged |= ((type_a | type_b) & MEM_RDONLY); + return type_merged; } static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, bool allow_trust_mismatch) { enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type; - enum bpf_reg_type merged_type; if (*prev_type == NOT_INIT) { /* Saw a valid insn @@ -17047,24 +17891,16 @@ static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type typ * Reject it. */ if (allow_trust_mismatch && - is_ptr_to_mem_or_btf_id(type) && - is_ptr_to_mem_or_btf_id(*prev_type)) { + bpf_is_ptr_to_mem_or_btf_id(type) && + bpf_is_ptr_to_mem_or_btf_id(*prev_type)) { /* * Have to support a use case when one path through - * the program yields TRUSTED pointer while another - * is UNTRUSTED. Fallback to UNTRUSTED to generate - * BPF_PROBE_MEM/BPF_PROBE_MEMSX. - * Same behavior of MEM_RDONLY flag. + * the program yields a TRUSTED pointer while another + * is UNTRUSTED. Merge them into a type which keeps + * the BPF_PROBE_MEM/BPF_PROBE_MEMSX rewrite when + * either side needs it. */ - if (is_ptr_to_mem(type) || is_ptr_to_mem(*prev_type)) - merged_type = PTR_TO_MEM; - else - merged_type = PTR_TO_BTF_ID; - if ((type & PTR_UNTRUSTED) || (*prev_type & PTR_UNTRUSTED)) - merged_type |= PTR_UNTRUSTED; - if ((type & MEM_RDONLY) || (*prev_type & MEM_RDONLY)) - merged_type |= MEM_RDONLY; - *prev_type = merged_type; + *prev_type = merge_ptr_types(type, *prev_type); } else { verbose(env, "same insn cannot be used with different pointers\n"); return -EINVAL; @@ -17288,9 +18124,13 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) insn->imm != BPF_FUNC_spin_unlock && insn->imm != BPF_FUNC_kptr_xchg) || (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && - (insn->off != 0 || !kfunc_spin_allowed(insn->imm)))) { + !kfunc_spin_allowed(env, insn->imm, insn->off))) { verbose(env, "function calls are not allowed while holding a lock\n"); + bpf_diag_ctx_active( + env, env->insn_idx, + "function call", BPF_DIAG_CONTEXT_LOCK, + "Release the BPF spin lock before making this call, or move the call outside the locked region."); return -EINVAL; } } @@ -17364,7 +18204,9 @@ static int do_check(struct bpf_verifier_env *env) insn = &insns[env->insn_idx]; insn_aux = &env->insn_aux_data[env->insn_idx]; - if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { + account_processed_insn(env); + + if (env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { verbose(env, "BPF program is too large. Processed %d insn\n", env->insn_processed); @@ -17373,6 +18215,27 @@ static int do_check(struct bpf_verifier_env *env) state->last_insn_idx = env->prev_insn_idx; state->insn_idx = env->insn_idx; + /* + * Record the incoming edge so active and queued paths use the same + * branch-recording path. A zero-offset conditional has identical + * successors, so its outcome cannot be reconstructed from the edge. + */ + if (!state->speculative && prev_insn_idx >= 0 && prev_insn_idx < insn_cnt) { + struct bpf_insn *prev_insn = &insns[prev_insn_idx]; + int fallthrough_idx = prev_insn_idx + 1; + int branch_idx = prev_insn_idx + bpf_jmp_offset(prev_insn) + 1; + u8 class = BPF_CLASS(prev_insn->code); + u8 opcode = BPF_OP(prev_insn->code); + + if ((class == BPF_JMP || class == BPF_JMP32) && + opcode != BPF_JA && opcode != BPF_CALL && opcode != BPF_EXIT && + opcode <= BPF_JCOND && branch_idx != fallthrough_idx) { + if (env->insn_idx == branch_idx) + bpf_diag_record_branch(env, prev_insn_idx, true); + else if (env->insn_idx == fallthrough_idx) + bpf_diag_record_branch(env, prev_insn_idx, false); + } + } if (bpf_is_prune_point(env, env->insn_idx)) { err = bpf_is_state_visited(env, env->insn_idx); @@ -17422,6 +18285,7 @@ static int do_check(struct bpf_verifier_env *env) env->prev_log_pos = env->log.end_pos; verbose(env, "%d: ", env->insn_idx); bpf_verbose_insn(env, insn); + verbose(env, "\n"); env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos; env->prev_log_pos = env->log.end_pos; } @@ -17504,6 +18368,7 @@ static int do_check(struct bpf_verifier_env *env) "speculation barrier after jump instruction may not have the desired effect")) return -EFAULT; process_bpf_exit: + account_current_path(env); mark_verifier_state_scratched(env); err = bpf_update_branch_counts(env, env->cur_state); if (err) @@ -17566,6 +18431,11 @@ static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf) if (env->used_btfs[i].btf == btf) goto ret_put; + if (env->signature) { + verbose(env, "signed program cannot bind any BTF\n"); + ret = -EACCES; + goto ret_put; + } if (env->used_btf_cnt >= MAX_USED_BTFS) { verbose(env, "The total number of btfs per program has reached the limit of %u\n", MAX_USED_BTFS); @@ -17758,7 +18628,9 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); return -EINVAL; } + } + if (btf_record_has_field(map->record, BPF_SPIN_LOCK)) { if (is_tracing_prog_type(prog_type)) { verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); return -EINVAL; @@ -17830,6 +18702,7 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, return -EOPNOTSUPP; } env->prog->aux->arena = (void *)map; + env->prog->jit_required = true; if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) { verbose(env, "arena's user address must be set via map_extra or mmap()\n"); return -EINVAL; @@ -17848,6 +18721,12 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) if (env->used_maps[i] == map) return i; + if (env->signature && + env->prog->aux->sig.verdict == BPF_SIG_VERIFIED) { + verbose(env, "signed program cannot bind map '%s' not covered by the signature\n", + map->name); + return -EACCES; + } if (env->used_map_cnt >= MAX_USED_MAPS) { verbose(env, "The total number of maps per program has reached the limit of %u\n", MAX_USED_MAPS); @@ -17877,6 +18756,7 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) return err; } env->insn_array_maps[env->insn_array_map_cnt++] = map; + env->prog->jit_required = true; } return env->used_map_cnt - 1; @@ -17900,6 +18780,48 @@ static int add_used_map(struct bpf_verifier_env *env, int fd) return __add_used_map(env, map); } +static int fd_array_get_map_idx_continuous(struct bpf_verifier_env *env, u32 idx) +{ + struct bpf_map *map; + + if (idx >= env->fd_array_cnt) { + verbose(env, "fd_idx %u out of bounds, fd_array_cnt %u\n", + idx, env->fd_array_cnt); + return -EINVAL; + } + map = fd_slot_map(env->fd_array[idx]); + if (!map) { + verbose(env, "fd_idx %u is not a map\n", idx); + return -EINVAL; + } + return __add_used_map(env, map); +} + +static int fd_array_get_map_idx_sparse(struct bpf_verifier_env *env, u32 idx) +{ + int fd; + + if (copy_from_bpfptr_offset(&fd, env->fd_array_raw, + (size_t)idx * sizeof(fd), sizeof(fd))) + return -EFAULT; + return add_used_map(env, fd); +} + +static int fd_array_get_map_idx(struct bpf_verifier_env *env, u32 idx) +{ + if (env->fd_array) + return fd_array_get_map_idx_continuous(env, idx); + if (env->signature) { + verbose(env, "signed program must bind maps via a continuous fd_array (fd_array_cnt)\n"); + return -EACCES; + } + if (!bpfptr_is_null(env->fd_array_raw)) + return fd_array_get_map_idx_sparse(env, idx); + + verbose(env, "fd_idx without fd_array is invalid\n"); + return -EPROTO; +} + static int check_alu_fields(struct bpf_verifier_env *env, struct bpf_insn *insn) { u8 class = BPF_CLASS(insn->code); @@ -18117,7 +19039,6 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env) struct bpf_map *map; int map_idx; u64 addr; - u32 fd; if (i == insn_cnt - 1 || insn[1].code != 0 || insn[1].dst_reg != 0 || insn[1].src_reg != 0 || @@ -18169,21 +19090,17 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env) switch (insn[0].src_reg) { case BPF_PSEUDO_MAP_IDX_VALUE: case BPF_PSEUDO_MAP_IDX: - if (bpfptr_is_null(env->fd_array)) { - verbose(env, "fd_idx without fd_array is invalid\n"); - return -EPROTO; - } - if (copy_from_bpfptr_offset(&fd, env->fd_array, - insn[0].imm * sizeof(fd), - sizeof(fd))) - return -EFAULT; + map_idx = fd_array_get_map_idx(env, insn[0].imm); break; default: - fd = insn[0].imm; + if (env->signature) { + verbose(env, "signed program cannot reference a map by fd, only via fd_array index\n"); + return -EINVAL; + } + map_idx = add_used_map(env, insn[0].imm); break; } - map_idx = add_used_map(env, fd); if (map_idx < 0) return map_idx; map = env->used_maps[map_idx]; @@ -18277,8 +19194,6 @@ static void release_insn_arrays(struct bpf_verifier_env *env) bpf_insn_array_release(env->insn_array_maps[i]); } - - /* The verifier does more data flow analysis than llvm and will not * explore branches that are dead at run time. Malicious programs can * have dead code too. Therefore replace all dead at-run-time code @@ -18306,8 +19221,6 @@ static void sanitize_dead_code(struct bpf_verifier_env *env) } } - - static void free_states(struct bpf_verifier_env *env) { struct bpf_verifier_state_list *sl; @@ -18358,6 +19271,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) struct bpf_prog_aux *aux = env->prog->aux; struct bpf_verifier_state *state; struct bpf_reg_state *regs; + u32 insn_processed = env->insn_processed; int ret, i; env->prev_linfo = NULL; @@ -18385,7 +19299,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) regs = state->frame[state->curframe]->regs; if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { - const char *sub_name = subprog_name(env, subprog); + const char *sub_name = bpf_subprog_name(env, subprog); struct bpf_subprog_arg_info *arg; struct bpf_reg_state *reg; @@ -18496,9 +19410,22 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) ret = do_check(env); out: - if (!ret && pop_log) - bpf_vlog_reset(&env->log, 0); + account_current_path(env); + if (!ret) { + if (pop_log) + bpf_vlog_reset(&env->log, 0); + bpf_diag_event_log_restore(env, 0); + } free_states(env); + + /* + * The override is needed to account for async subprograms, which + * are verified with their own set of stack frames and thus are + * not accounted as callees by account_current_path(). + * Accumulate their total counts as total counts of the main or + * global subprog hosting the async call. + */ + env->subprog_info[subprog].insns_total = env->insn_processed - insn_processed; return ret; } @@ -18527,7 +19454,6 @@ static int do_check_subprogs(struct bpf_verifier_env *env) struct bpf_prog_aux *aux = env->prog->aux; struct bpf_func_info_aux *sub_aux; int i, ret, new_cnt; - u32 insn_processed; if (!aux->func_info) return 0; @@ -18542,8 +19468,6 @@ again: if (!bpf_subprog_is_global(env, i)) continue; - insn_processed = env->insn_processed; - sub_aux = subprog_aux(env, i); if (!sub_aux->called || sub_aux->verified) continue; @@ -18551,12 +19475,11 @@ again: env->insn_idx = env->subprog_info[i].start; WARN_ON_ONCE(env->insn_idx == 0); ret = do_check_common(env, i); - env->subprog_info[i].insn_processed = env->insn_processed - insn_processed; if (ret) { return ret; } else if (env->log.level & BPF_LOG_LEVEL) { verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n", - i, subprog_name(env, i)); + i, bpf_subprog_name(env, i)); } /* We verified new global subprog, it might have called some @@ -18578,18 +19501,15 @@ again: static int do_check_main(struct bpf_verifier_env *env) { - u32 insn_processed = env->insn_processed; int ret; env->insn_idx = 0; ret = do_check_common(env, 0); - env->subprog_info[0].insn_processed = env->insn_processed - insn_processed; if (!ret) env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; return ret; } - static void print_verification_stats(struct bpf_verifier_env *env) { /* Skip over hidden subprogs which are not verified. */ @@ -18598,15 +19518,20 @@ static void print_verification_stats(struct bpf_verifier_env *env) if (env->log.level & BPF_LOG_STATS) { verbose(env, "verification time %lld usec\n", div_u64(env->verification_time, 1000)); - verbose(env, "stack depth %d", env->subprog_info[0].stack_depth); - for (i = 1; i < subprog_cnt; i++) - verbose(env, "+%d", env->subprog_info[i].stack_depth); - verbose(env, " max %d\n", env->max_stack_depth); - verbose(env, "insns processed %d", env->subprog_info[0].insn_processed); - for (i = 1; i < subprog_cnt; i++) - if (bpf_subprog_is_global(env, i)) - verbose(env, "+%d", env->subprog_info[i].insn_processed); - verbose(env, "\n"); + verbose(env, "stack depth max %d\n", env->max_stack_depth); + for (i = 0; i < subprog_cnt; i++) { + const char *name = env->subprog_info[i].name; + const char *kind; + + if (!name || !name[0]) + name = "<unknown>"; + kind = i == 0 ? "main" : + bpf_subprog_is_global(env, i) ? "global" : "static"; + verbose(env, "subprog %d (%s) %s insns_self %d insns_total %d stack %d\n", + i, name, kind, env->subprog_info[i].insns_self, + env->subprog_info[i].insns_total, + env->subprog_info[i].stack_depth); + } } verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " "total_states %d peak_states %d mark_read %d\n", @@ -18628,6 +19553,7 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) { const struct btf_type *t, *func_proto; const struct bpf_struct_ops_desc *st_ops_desc; + const struct bpf_struct_ops_arg_info *arg_info; const struct bpf_struct_ops *st_ops; const struct btf_member *member; struct bpf_prog *prog = env->prog; @@ -18706,10 +19632,23 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) return -EACCES; } - for (i = 0; i < st_ops_desc->arg_info[member_idx].cnt; i++) { - if (st_ops_desc->arg_info[member_idx].info[i].refcounted) { + arg_info = &st_ops_desc->arg_info[member_idx]; + for (i = 0; i < arg_info->cnt; i++) { + const struct bpf_ctx_arg_aux *info = &arg_info->info[i]; + + if (info->refcounted) has_refcounted_arg = true; - break; + if (base_type(info->reg_type) == PTR_TO_ARENA) { + if (!bpf_jit_supports_arena_args()) { + verbose(env, "JIT does not support arena arguments\n"); + return -ENOTSUPP; + } + if (!prog->aux->arena) { + verbose(env, + "arena argument of %s requires a program with an associated arena\n", + mname); + return -EINVAL; + } } } @@ -18730,8 +19669,7 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) prog->aux->attach_func_name = mname; env->ops = st_ops->verifier_ops; - return bpf_prog_ctx_arg_info_init(prog, st_ops_desc->arg_info[member_idx].info, - st_ops_desc->arg_info[member_idx].cnt); + return bpf_prog_ctx_arg_info_init(prog, arg_info->info, arg_info->cnt); } #define SECURITY_PREFIX "security_" @@ -18838,6 +19776,9 @@ static int btf_id_allow_sleepable(u32 btf_id, unsigned long addr, const struct b const struct btf_type *t; const char *tname; + if (!btf_is_kernel(btf)) + return -EINVAL; + switch (prog->type) { case BPF_PROG_TYPE_TRACING: t = btf_type_by_id(btf, btf_id); @@ -18922,6 +19863,20 @@ btf_attach_func_proto(struct bpf_verifier_log *log, struct btf *btf, u32 func_id return btf_type_by_id(btf, func->type); } +static bool attach_uses_trampoline_retval(enum bpf_attach_type type) +{ + switch (type) { + case BPF_MODIFY_RETURN: + case BPF_TRACE_FEXIT: + case BPF_TRACE_FEXIT_MULTI: + case BPF_TRACE_FSESSION: + case BPF_TRACE_FSESSION_MULTI: + return true; + default: + return false; + } +} + int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog, @@ -18981,6 +19936,16 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, bpf_log(log, "Subprog %s doesn't exist\n", tname); return -EINVAL; } + /* + * A struct_ops indirect trampoline converts arena arguments + * before invoking its program. A tracing or extension program + * attached to the main program would see the converted offset as a + * regular BTF pointer. + */ + if (subprog == 0 && bpf_prog_has_arena_ctx_arg(tgt_prog)) { + bpf_log(log, "Cannot attach to a target with arena context arguments\n"); + return -EOPNOTSUPP; + } if (aux->func && aux->func[subprog]->aux->exception_cb) { bpf_log(log, "%s programs cannot attach to exception callback\n", @@ -19186,6 +20151,14 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, if (ret < 0) return ret; + if (tgt_info->fmodel.ret_size > 8 && + attach_uses_trampoline_retval(prog->expected_attach_type)) { + bpf_log(log, + "Attach to function %s with a >8 byte return value is not supported for this attach type\n", + tname); + return -EOPNOTSUPP; + } + /* * *.multi programs don't need an address during program * verification, we just take the module ref if needed. @@ -19421,7 +20394,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) return -ENOMEM; if (tgt_prog && tgt_prog->aux->tail_call_reachable) - tr->flags = BPF_TRAMP_F_TAIL_CALL_CTX; + bpf_trampoline_set_flags(tr, BPF_TRAMP_F_TAIL_CALL_CTX); prog->aux->dst_trampoline = tr; return 0; @@ -19460,6 +20433,9 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt err = btf_distill_func_proto(NULL, btf, t, tname, &tgt_info->fmodel); if (err < 0) return err; + if (tgt_info->fmodel.ret_size > 8 && + attach_uses_trampoline_retval(prog->expected_attach_type)) + return -EOPNOTSUPP; if (btf_is_module(btf)) { /* The bpf program already holds reference to module. */ if (WARN_ON_ONCE(!prog->aux->mod)) @@ -19483,13 +20459,25 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt struct btf *bpf_get_btf_vmlinux(void) { - if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { - mutex_lock(&bpf_verifier_lock); - if (!btf_vmlinux) - btf_vmlinux = btf_parse_vmlinux(); - mutex_unlock(&bpf_verifier_lock); + /* Pairs with the smp_store_release() on the parse path below. */ + struct btf *btf = smp_load_acquire(&btf_vmlinux); + + if (!btf && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { + mutex_lock(&btf_vmlinux_lock); + btf = btf_vmlinux; + if (!btf) { + btf = btf_parse_vmlinux(); + /* + * Order the parsed BTF contents and the globals the + * parse populated (e.g. bpf_ctx_convert.t) before + * the pointer publication. Pairs with the acquire + * on the lockless fast path above. + */ + smp_store_release(&btf_vmlinux, btf); + } + mutex_unlock(&btf_vmlinux_lock); } - return btf_vmlinux; + return btf; } /* @@ -19497,7 +20485,7 @@ struct btf *bpf_get_btf_vmlinux(void) * this case expect that every file descriptor in the array is either a map or * a BTF. Everything else is considered to be trash. */ -static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd) +static int add_fd_from_fd_array(struct bpf_verifier_env *env, u32 idx, int fd) { struct bpf_map *map; struct btf *btf; @@ -19509,51 +20497,83 @@ static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd) err = __add_used_map(env, map); if (err < 0) return err; + fd_slot_set_map(&env->fd_array[idx], map); return 0; } btf = __btf_get_by_fd(f); if (!IS_ERR(btf)) { btf_get(btf); - return __add_used_btf(env, btf); + err = __add_used_btf(env, btf); + if (err < 0) + return err; + fd_slot_set_btf(&env->fd_array[idx], btf); + return 0; } verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd); return PTR_ERR(map); } -static int process_fd_array(struct bpf_verifier_env *env, union bpf_attr *attr, bpfptr_t uattr) +/* + * A continuous fd_array is resolved into an in-memory cache with one slot + * per entry. The bound here is deliberately generous and not derived from + * the per-program object limits: Duplicate entries /are/ permitted, and + * the number of distinct maps and BTFs a program can bind is enforced when + * each entry is resolved by __add_used_map() and __add_used_btf(). + */ +#define MAX_FD_ARRAY_CNT 4096 + +static int process_fd_array_continuous(struct bpf_verifier_env *env, + bpfptr_t fd_array, u32 cnt) { - size_t size = sizeof(int); - int ret; - int fd; + int fd, ret; u32 i; - env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); - - /* - * The only difference between old (no fd_array_cnt is given) and new - * APIs is that in the latter case the fd_array is expected to be - * continuous and is scanned for map fds right away - */ - if (!attr->fd_array_cnt) - return 0; - - /* Check for integer overflow */ - if (attr->fd_array_cnt >= (U32_MAX / size)) { - verbose(env, "fd_array_cnt is too big (%u)\n", attr->fd_array_cnt); - return -EINVAL; + if (cnt > MAX_FD_ARRAY_CNT) { + verbose(env, "fd_array has too many entries (%u, max %u)\n", + cnt, MAX_FD_ARRAY_CNT); + return -E2BIG; } - for (i = 0; i < attr->fd_array_cnt; i++) { - if (copy_from_bpfptr_offset(&fd, env->fd_array, i * size, size)) + env->fd_array = kvcalloc(cnt, sizeof(*env->fd_array), + GFP_KERNEL_ACCOUNT); + if (!env->fd_array) + return -ENOMEM; + env->fd_array_cnt = cnt; + for (i = 0; i < cnt; i++) { + if (copy_from_bpfptr_offset(&fd, fd_array, + (size_t)i * sizeof(fd), sizeof(fd))) return -EFAULT; - - ret = add_fd_from_fd_array(env, fd); + ret = add_fd_from_fd_array(env, i, fd); if (ret) return ret; } + return 0; +} + +static int process_fd_array(struct bpf_verifier_env *env, + union bpf_attr *attr, bpfptr_t uattr) +{ + bpfptr_t fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); + if (bpfptr_is_null(fd_array)) { + if (attr->fd_array_cnt) { + verbose(env, "fd_array_cnt %u without fd_array is invalid\n", + attr->fd_array_cnt); + return -EINVAL; + } + return 0; + } + /* + * New API: the caller passes fd_array_cnt and a continuous array that + * is resolved and bound up front. Legacy API (no fd_array_cnt): keep + * the caller's array and resolve entries on the spot at each reference. + */ + if (attr->fd_array_cnt) + return process_fd_array_continuous(env, fd_array, + attr->fd_array_cnt); + env->fd_array_raw = fd_array; return 0; } @@ -19753,6 +20773,51 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn_buf[4] = BPF_ALU64_REG(BPF_SUB, BPF_REG_0, BPF_REG_1); insn_buf[5] = BPF_ALU64_IMM(BPF_NEG, BPF_REG_0, 0); *cnt = 6; + } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_new]) { + /* inline bpf_iter_num_new(&it, start, end); R1=&it, R2=start, R3=end */ + int i = 0; + + /* if (start > end) goto einval; */ + insn_buf[i++] = BPF_JMP32_REG(BPF_JSGT, BPF_REG_2, BPF_REG_3, 8); + /* r0 = (u32)end - (u32)start; if (r0 > BPF_MAX_LOOPS) goto e2big; */ + insn_buf[i++] = BPF_MOV32_REG(BPF_REG_0, BPF_REG_3); + insn_buf[i++] = BPF_ALU32_REG(BPF_SUB, BPF_REG_0, BPF_REG_2); + insn_buf[i++] = BPF_JMP_IMM(BPF_JGT, BPF_REG_0, BPF_MAX_LOOPS, 8); + /* s->cur = start - 1; s->end = end; return 0; */ + insn_buf[i++] = BPF_ALU32_IMM(BPF_ADD, BPF_REG_2, -1); + insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_2, 0); + insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3, 4); + insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, 0); + insn_buf[i++] = BPF_JMP_A(5); + /* einval: s->cur = s->end = 0; return -EINVAL; */ + insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); + insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); + insn_buf[i++] = BPF_JMP_A(2); + /* e2big: s->cur = s->end = 0; return -E2BIG; */ + insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); + insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, -E2BIG); + *cnt = i; + } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_next]) { + /* inline bpf_iter_num_next(&it); R1=&it, returns &s->cur or NULL */ + int i = 0; + + /* r0 = s->cur + 1; if ((s32)r0 >= s->end) goto done; */ + insn_buf[i++] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0); + insn_buf[i++] = BPF_ALU32_IMM(BPF_ADD, BPF_REG_0, 1); + insn_buf[i++] = BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 4); + insn_buf[i++] = BPF_JMP32_REG(BPF_JSGE, BPF_REG_0, BPF_REG_2, 3); + /* s->cur = r0; return &s->cur; */ + insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0); + insn_buf[i++] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); + insn_buf[i++] = BPF_JMP_A(2); + /* done: s->cur = s->end = 0; return NULL; */ + insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); + insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, 0); + *cnt = i; + } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_destroy]) { + /* bpf_iter_num_destroy() is a no-op; emit a nop to drop the call */ + insn_buf[0] = BPF_JMP_A(0); + *cnt = 1; } if (env->insn_aux_data[insn_idx].arg_prog) { @@ -19768,6 +20833,146 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; } +static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) +{ + switch (keyring_id) { + case 0: + return BPF_SIG_KEYRING_BUILTIN; + case (s32)(unsigned long)VERIFY_USE_SECONDARY_KEYRING: + return BPF_SIG_KEYRING_SECONDARY; + case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING: + return BPF_SIG_KEYRING_PLATFORM; + default: + return BPF_SIG_KEYRING_USER; + } +} + +/* + * Verify the PKCS#7 signature of a loaded program. Called from bpf_check() + * once the program's metadata maps have been resolved into used_maps, so + * the exact maps folded into the signature are the ones the program binds. + * + * The signature covers the instructions followed by the frozen contents of + * each map, in @maps order: insns || map_0 || map_1 || [...]. On success the + * verdict and keyring info are recorded on prog->aux. + */ +static int bpf_prog_verify_signature(struct bpf_verifier_env *env, + union bpf_attr *attr, bool is_kernel) +{ + bpfptr_t usig = make_bpfptr(attr->signature, is_kernel); + struct bpf_dynptr_kern sig_ptr, data_ptr; + struct bpf_prog *prog = env->prog; + struct bpf_map **maps = env->used_maps; + struct bpf_key *key = NULL; + void *sig, *data = NULL; + u32 map_cnt = env->used_map_cnt; + u32 i, off, insns_sz; + u64 data_sz; + int err = 0; + + /* + * Don't attempt to use kmalloc_large or vmalloc for signatures. + * Practical signature for BPF program should be below this limit. + */ + if (!attr->signature_size || + attr->signature_size > KMALLOC_MAX_CACHE_SIZE) + return -EINVAL; + if (system_keyring_id_check(attr->keyring_id) == 0) + key = bpf_lookup_system_key(attr->keyring_id); + else + key = bpf_lookup_user_key(attr->keyring_id, 0); + if (!key) { + verbose(env, "cannot resolve signing keyring with keyring_id %d\n", + attr->keyring_id); + return -EINVAL; + } + + sig = kvmemdup_bpfptr(usig, attr->signature_size); + if (IS_ERR(sig)) { + bpf_key_put(key); + return PTR_ERR(sig); + } + + insns_sz = prog->len * sizeof(struct bpf_insn); + data_sz = insns_sz; + for (i = 0; i < map_cnt; i++) { + struct bpf_map *map = maps[i]; + + if (map->map_type != BPF_MAP_TYPE_ARRAY || + !map->ops->map_direct_value_addr) { + verbose(env, "signed program metadata map '%s' must be an array\n", + map->name); + err = -EINVAL; + goto out; + } + if (!READ_ONCE(map->frozen)) { + verbose(env, "signed program metadata map '%s' must be frozen\n", + map->name); + err = -EPERM; + goto out; + } + if (bpf_map_write_active(map)) { + verbose(env, "signed program metadata map '%s' has active writers\n", + map->name); + err = -EBUSY; + goto out; + } + if (!map->excl_prog_sha) { + verbose(env, "signed program metadata map '%s' must be exclusive\n", + map->name); + err = -EPERM; + goto out; + } + data_sz += map->value_size; + } + if (bpf_dynptr_check_size(data_sz)) { + verbose(env, "signed payload too large: %llu bytes\n", data_sz); + err = -E2BIG; + goto out; + } + data = kvmalloc(data_sz, GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!data) { + err = -ENOMEM; + goto out; + } + memcpy(data, prog->insnsi, insns_sz); + off = insns_sz; + for (i = 0; i < map_cnt; i++) { + struct bpf_map *map = maps[i]; + u64 addr; + + err = map->ops->map_direct_value_addr(map, &addr, 0); + if (err) { + verbose(env, "failed to read signed metadata map '%s': %d\n", + map->name, err); + goto out; + } + memcpy(data + off, (void *)(unsigned long)addr, + map->value_size); + off += map->value_size; + } + + bpf_dynptr_init(&data_ptr, data, BPF_DYNPTR_TYPE_LOCAL, 0, data_sz); + bpf_dynptr_init(&sig_ptr, sig, BPF_DYNPTR_TYPE_LOCAL, 0, + attr->signature_size); + + err = bpf_verify_pkcs7_signature((struct bpf_dynptr *)&data_ptr, + (struct bpf_dynptr *)&sig_ptr, key); + if (err) { + verbose(env, "signature verification failed: %d\n", err); + } else { + verbose(env, "signature verification passed\n"); + prog->aux->sig.keyring_serial = bpf_key_serial(key); + prog->aux->sig.keyring_type = bpf_classify_keyring(attr->keyring_id); + prog->aux->sig.verdict = BPF_SIG_VERIFIED; + } +out: + kvfree(data); + bpf_key_put(key); + kvfree(sig); + return err; +} + int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log) { @@ -19790,18 +20995,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, return -ENOMEM; env->bt.env = env; - - len = (*prog)->len; - env->insn_aux_data = - vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); - ret = -ENOMEM; - if (!env->insn_aux_data) - goto err_free_env; - for (i = 0; i < len; i++) - env->insn_aux_data[i].orig_idx = i; - env->succ = bpf_iarray_realloc(NULL, 2); - if (!env->succ) - goto err_free_env; env->prog = *prog; env->ops = bpf_verifier_ops[env->prog->type]; @@ -19810,22 +21003,55 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token); env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token); env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF); - - bpf_get_btf_vmlinux(); - - /* grab the mutex to protect few globals used by verifier */ - if (!is_priv) - mutex_lock(&bpf_verifier_lock); + env->signature = attr->signature; /* user could have requested verbose verifier output * and supplied buffer to store the verification trace */ ret = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size); if (ret) - goto err_unlock; + goto err_free_env; + ret = bpf_diag_init(env); + if (ret) + goto err_prep; + if (env->signature) { + ret = bpf_prog_calc_tag(env->prog); + if (ret < 0) + goto err_prep; + } ret = process_fd_array(env, attr, uattr); if (ret) + goto err_prep; + + if (env->signature) { + ret = bpf_prog_verify_signature(env, attr, uattr.is_kernel); + if (ret) + goto err_prep; + } + + ret = security_bpf_prog_load(env->prog, attr, env->prog->aux->token, + uattr.is_kernel); + if (ret) + goto err_prep; + + bpf_get_btf_vmlinux(); + + /* Serialize verification of unprivileged programs. */ + if (!is_priv) + mutex_lock(&bpf_verifier_lock); + + len = env->insn_aux_data_len = env->prog->len; + env->insn_aux_data = + __vmalloc(array_size(sizeof(struct bpf_insn_aux_data), len), + GFP_KERNEL_ACCOUNT | __GFP_ZERO); + ret = -ENOMEM; + if (!env->insn_aux_data) + goto skip_full_check; + for (i = 0; i < len; i++) + env->insn_aux_data[i].orig_idx = i; + env->succ = bpf_iarray_realloc(NULL, 2); + if (!env->succ) goto skip_full_check; mark_verifier_state_clean(env); @@ -19858,11 +21084,13 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, INIT_LIST_HEAD(&env->explored_states[i]); INIT_LIST_HEAD(&env->free_list); - ret = bpf_check_btf_info_early(env, attr, uattr); + /* Prepare BTF and func_info needed to discover all subprograms. */ + ret = bpf_prepare_btf_info(env, attr, uattr); if (ret < 0) goto skip_full_check; - ret = add_subprog_and_kfunc(env); + /* Discover all subprograms before validating their layout and BTF. */ + ret = add_subprogs(env); if (ret < 0) goto skip_full_check; @@ -19870,14 +21098,21 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + /* Validate BTF against the complete subprogram layout and apply CO-RE. */ ret = bpf_check_btf_info(env, attr, uattr); if (ret < 0) goto skip_full_check; + /* Validate instructions and resolve the program's referenced resources. */ ret = check_and_resolve_insns(env); if (ret < 0) goto skip_full_check; + /* Build kfunc prototypes after resolving program resources. */ + ret = add_kfuncs(env); + if (ret < 0) + goto skip_full_check; + if (bpf_prog_is_offloaded(env->prog->aux)) { ret = bpf_prog_offload_verifier_prep(env->prog); if (ret) @@ -20049,17 +21284,27 @@ err_release_maps: *prog = env->prog; module_put(env->attach_btf_mod); -err_unlock: if (!is_priv) mutex_unlock(&bpf_verifier_lock); - bpf_clear_insn_aux_data(env, 0, env->prog->len); + goto err_free_env; +err_prep: + err = bpf_log_attr_finalize(attr_log, &env->log); + if (err) + ret = err; + release_insn_arrays(env); + release_maps(env); + release_btfs(env); err_free_env: + if (env->insn_aux_data) + bpf_clear_insn_aux_data(env, 0, env->insn_aux_data_len); + vfree(env->insn_aux_data); + kvfree(env->fd_array); bpf_stack_liveness_free(env); kvfree(env->cfg.insn_postorder); kvfree(env->scc_info); kvfree(env->succ); kvfree(env->gotox_tmp_buf); - vfree(env->insn_aux_data); + bpf_diag_free(env); kvfree(env); return ret; } |