diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/bpf/arraymap.c | 5 | ||||
| -rw-r--r-- | kernel/bpf/backtrack.c | 62 | ||||
| -rw-r--r-- | kernel/bpf/bpf_iter.c | 2 | ||||
| -rw-r--r-- | kernel/bpf/btf.c | 27 | ||||
| -rw-r--r-- | kernel/bpf/cfg.c | 3 | ||||
| -rw-r--r-- | kernel/bpf/core.c | 5 | ||||
| -rw-r--r-- | kernel/bpf/disasm.c | 50 | ||||
| -rw-r--r-- | kernel/bpf/fixups.c | 7 | ||||
| -rw-r--r-- | kernel/bpf/hashtab.c | 29 | ||||
| -rw-r--r-- | kernel/bpf/local_storage.c | 5 | ||||
| -rw-r--r-- | kernel/bpf/percpu_freelist.c | 35 | ||||
| -rw-r--r-- | kernel/bpf/percpu_freelist.h | 1 | ||||
| -rw-r--r-- | kernel/bpf/stackmap.c | 2 | ||||
| -rw-r--r-- | kernel/bpf/states.c | 11 | ||||
| -rw-r--r-- | kernel/bpf/syscall.c | 2 | ||||
| -rw-r--r-- | kernel/bpf/verifier.c | 259 |
16 files changed, 360 insertions, 145 deletions
diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index ef315b168b29..0ce26b538075 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -436,7 +436,7 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, void __percpu *pptr; void *ptr, *val; u32 size; - int cpu; + int cpu, off = 0; if (unlikely((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS)) /* unknown flags */ @@ -468,9 +468,10 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, } for_each_possible_cpu(cpu) { ptr = per_cpu_ptr(pptr, cpu); - val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu; + val = (map_flags & BPF_F_ALL_CPUS) ? value : value + off; copy_map_value(map, ptr, val); bpf_obj_cancel_fields(map, ptr); + off += size; } unlock: rcu_read_unlock(); diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index a2b18a9f1694..47282ffeeaf9 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -520,37 +520,34 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, return -EFAULT; } } else if (opcode == BPF_EXIT) { - bool r0_precise; + bool from_subprog_call, r0_precise; + + /* BPF_EXIT in subprog or callback always returns + * right after the call instruction, so by checking + * whether the instruction at subseq_idx-1 is subprog + * call or not we can distinguish actual exit from + * *subprog* from exit from *callback*. In the former + * case, we need to propagate r0 precision, if + * necessary. In the former we never do that. + */ + from_subprog_call = subseq_idx - 1 >= 0 && + bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]); + + r0_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_0); /* Backtracking to a nested function call, 'idx' is a part of * the inner frame 'subseq_idx' is a part of the outer frame. * In case of a regular function call, instructions giving * precision to registers R1-R5 should have been found already. - * In case of a callback, it is ok to have R1-R5 marked for - * backtracking, as these registers are set by the function - * invoking callback. + * In case of a callback from bpf_loop(), R{1,4} in the calling + * frame would be set as precise and that is correct. */ - if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx)) - for (i = BPF_REG_1; i <= BPF_REG_5; i++) - bt_clear_reg(bt, i); - if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { + if (from_subprog_call && (bt_reg_mask(bt) & BPF_REGMASK_ARGS)) { verifier_bug(env, "backtracking exit unexpected regs %x", bt_reg_mask(bt)); return -EFAULT; } - /* BPF_EXIT in subprog or callback always returns - * right after the call instruction, so by checking - * whether the instruction at subseq_idx-1 is subprog - * call or not we can distinguish actual exit from - * *subprog* from exit from *callback*. In the former - * case, we need to propagate r0 precision, if - * necessary. In the former we never do that. - */ - r0_precise = subseq_idx - 1 >= 0 && - bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) && - bt_is_reg_set(bt, BPF_REG_0); - bt_clear_reg(bt, BPF_REG_0); if (bt_subprog_enter(bt)) return -EFAULT; @@ -582,16 +579,29 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, */ } } else if (class == BPF_LD) { - if (!bt_is_reg_set(bt, dreg)) - return 0; - bt_clear_reg(bt, dreg); /* It's ld_imm64 or ld_abs or ld_ind. * For ld_imm64 no further tracking of precision * into parent is necessary */ - if (mode == BPF_IND || mode == BPF_ABS) - /* to be analyzed */ - return -ENOTSUPP; + if (mode == BPF_IMM) { + bt_clear_reg(bt, dreg); + return 0; + } + /* + * BPF_{IND,ABS} are modelled as two branches: + * - fallthrough; + * - implicit subprogram exit. + * It is necessary to switch current frame if + * implicit subprogram exit branch is backtracked. + */ + if (mode == BPF_IND || mode == BPF_ABS) { + if (bt_is_reg_set(bt, dreg)) + return -ENOTSUPP; + if (subseq_idx != idx + 1) + if (bt_subprog_enter(bt)) + return -EFAULT; + return 0; + } } /* Propagate precision marks to linked registers, to account for * registers marked as precise in this function. diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index 14a5fdfa0421..b40eb404adab 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -754,7 +754,7 @@ const struct bpf_func_proto bpf_loop_proto = { .func = bpf_loop, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_ANYTHING, + .arg1_type = ARG_SCALAR, .arg2_type = ARG_PTR_TO_FUNC, .arg3_type = ARG_PTR_TO_STACK_OR_NULL, .arg4_type = ARG_ANYTHING, diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index da36d4b9d31a..9f33e95d5741 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -2911,14 +2911,29 @@ static void btf_modifier_show(const struct btf *btf, else t = btf_type_skip_modifiers(btf, type_id, NULL); - btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); + /* + * A modifier can resolve to void, which has no show op; print a + * placeholder rather than dereferencing NULL. + */ + if (!btf_type_ops(t)) + btf_df_show(btf, t, type_id, data, bits_offset, show); + else + btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); } static void btf_var_show(const struct btf *btf, const struct btf_type *t, u32 type_id, void *data, u8 bits_offset, struct btf_show *show) { - t = btf_type_id_resolve(btf, &type_id); + /* + * btf_type_id_resolve() dereferences btf->resolved_ids, which is NULL + * for a base BTF (e.g. the vmlinux BTF that bpf_snprintf_btf() uses). + * Resolve the var's type directly in that case. + */ + if (btf->resolved_ids) + t = btf_type_id_resolve(btf, &type_id); + else + t = btf_type_skip_modifiers(btf, t->type, &type_id); btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); } @@ -6657,6 +6672,10 @@ struct bpf_raw_tp_null_args { static const struct bpf_raw_tp_null_args raw_tp_null_args[] = { /* sched */ { "sched_pi_setprio", 0x10 }, + /* + * do_wait() passes NULL for wait4(-1) and waitid(P_ALL). + */ + { "sched_process_wait", 0x1 }, /* ... from sched_numa_pair_template event class */ { "sched_stick_numa", 0x100 }, { "sched_swap_numa", 0x100 }, @@ -6717,6 +6736,9 @@ static const struct bpf_raw_tp_null_args raw_tp_null_args[] = { { "rxrpc_resend", 0x10 }, { "rxrpc_tq", 0x10 }, { "rxrpc_client", 0x1 }, + /* signal */ + { "signal_generate", 0x20 }, + { "signal_deliver", 0x20 }, /* skb */ {"kfree_skb", 0x1000}, /* sunrpc */ @@ -8727,6 +8749,7 @@ BPF_CALL_4(bpf_btf_find_by_name_kind, char *, name, int, name_sz, u32, kind, int const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { .func = bpf_btf_find_by_name_kind, .gpl_only = false, + .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg2_type = ARG_MEM_SIZE, diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index 0f13c13f4133..842c7d1eabcc 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -125,6 +125,7 @@ static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) /* mark branch target for state pruning */ mark_prune_point(env, w); mark_jmp_point(env, w); + mark_jump_target(env, w); } if (insn_state[w] == 0) { @@ -403,6 +404,7 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) } mark_jmp_point(env, w); + mark_jump_target(env, w); /* EXPLORED || DISCOVERED */ if (insn_state[w]) @@ -564,6 +566,7 @@ static int visit_insn(int t, struct bpf_verifier_env *env) mark_prune_point(env, t + off + 1); mark_jmp_point(env, t + off + 1); + mark_jump_target(env, t + off + 1); return ret; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index d55e737ed75a..8b294dfc1ad4 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1128,11 +1128,6 @@ void *bpf_jit_alloc_exec(unsigned long size) return execmem_alloc(EXECMEM_BPF, size); } -void *bpf_jit_alloc_exec_rw(unsigned long size) -{ - return execmem_alloc_rw(EXECMEM_BPF, size); -} - void bpf_jit_free_exec(void *addr) { execmem_free(addr); diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index 50b3ca5149a0..3ce8d74b0e40 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -7,6 +7,9 @@ #include "disasm.h" +/* Only defined by the non-UAPI linux/filter.h, which this file cannot use. */ +#define BPF_PROBE_ATOMIC 0xe0 + #define __BPF_FUNC_STR_FN(x) [BPF_FUNC_ ## x] = __stringify(bpf_ ## x) static const char * const func_id_str[] = { __BPF_FUNC_MAPPER(__BPF_FUNC_STR_FN) @@ -226,57 +229,57 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->imm); } } else if (class == BPF_STX) { + const char *probe_pfx = BPF_MODE(insn->code) == BPF_PROBE_ATOMIC ? "probe " : ""; + bool atomic = BPF_MODE(insn->code) == BPF_ATOMIC || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC; + if (BPF_MODE(insn->code) == BPF_MEM) verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = r%d", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); - else if (BPF_MODE(insn->code) == BPF_ATOMIC && + else if (atomic && (insn->imm == BPF_ADD || insn->imm == BPF_AND || insn->imm == BPF_OR || insn->imm == BPF_XOR)) { - verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) %s r%d", - insn->code, + verbose(cbs->private_data, "(%02x) %slock *(%s *)(r%d %+d) %s r%d", + insn->code, probe_pfx, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, bpf_alu_string[BPF_OP(insn->imm) >> 4], insn->src_reg); - } else if (BPF_MODE(insn->code) == BPF_ATOMIC && + } else if (atomic && (insn->imm == (BPF_ADD | BPF_FETCH) || insn->imm == (BPF_AND | BPF_FETCH) || insn->imm == (BPF_OR | BPF_FETCH) || insn->imm == (BPF_XOR | BPF_FETCH))) { - verbose(cbs->private_data, "(%02x) r%d = atomic%s_fetch_%s((%s *)(r%d %+d), r%d)", - insn->code, insn->src_reg, + verbose(cbs->private_data, "(%02x) %sr%d = atomic%s_fetch_%s((%s *)(r%d %+d), r%d)", + insn->code, probe_pfx, insn->src_reg, BPF_SIZE(insn->code) == BPF_DW ? "64" : "", bpf_atomic_alu_string[BPF_OP(insn->imm) >> 4], bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); - } else if (BPF_MODE(insn->code) == BPF_ATOMIC && - insn->imm == BPF_CMPXCHG) { - verbose(cbs->private_data, "(%02x) r0 = atomic%s_cmpxchg((%s *)(r%d %+d), r0, r%d)", - insn->code, + } else if (atomic && insn->imm == BPF_CMPXCHG) { + verbose(cbs->private_data, "(%02x) %sr0 = atomic%s_cmpxchg((%s *)(r%d %+d), r0, r%d)", + insn->code, probe_pfx, BPF_SIZE(insn->code) == BPF_DW ? "64" : "", bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); - } else if (BPF_MODE(insn->code) == BPF_ATOMIC && - insn->imm == BPF_XCHG) { - verbose(cbs->private_data, "(%02x) r%d = atomic%s_xchg((%s *)(r%d %+d), r%d)", - insn->code, insn->src_reg, + } else if (atomic && insn->imm == BPF_XCHG) { + verbose(cbs->private_data, "(%02x) %sr%d = atomic%s_xchg((%s *)(r%d %+d), r%d)", + insn->code, probe_pfx, insn->src_reg, BPF_SIZE(insn->code) == BPF_DW ? "64" : "", bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); - } else if (BPF_MODE(insn->code) == BPF_ATOMIC && - insn->imm == BPF_LOAD_ACQ) { - verbose(cbs->private_data, "(%02x) r%d = load_acquire((%s *)(r%d %+d))", - insn->code, insn->dst_reg, + } else if (atomic && insn->imm == BPF_LOAD_ACQ) { + verbose(cbs->private_data, "(%02x) %sr%d = load_acquire((%s *)(r%d %+d))", + insn->code, probe_pfx, insn->dst_reg, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->src_reg, insn->off); - } else if (BPF_MODE(insn->code) == BPF_ATOMIC && - insn->imm == BPF_STORE_REL) { - verbose(cbs->private_data, "(%02x) store_release((%s *)(r%d %+d), r%d)", - insn->code, + } else if (atomic && insn->imm == BPF_STORE_REL) { + verbose(cbs->private_data, "(%02x) %sstore_release((%s *)(r%d %+d), r%d)", + insn->code, probe_pfx, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); } else { @@ -295,7 +298,8 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, verbose(cbs->private_data, "BUG_st_%02x", insn->code); } } else if (class == BPF_LDX) { - if (BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) { + if ((BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) || + (BPF_MODE(insn->code) == BPF_MEMSX && BPF_SIZE(insn->code) == BPF_DW)) { verbose(cbs->private_data, "BUG_ldx_%02x", insn->code); return; } diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 65b441e4a351..52d3cec33672 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -13,10 +13,15 @@ #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) +/* + * Matches BPF_PROBE_ATOMIC too: bpf_convert_ctx_accesses() rewrites arena + * atomics before bpf_opt_subreg_zext_lo32_rnd_hi32() runs. + */ static bool is_cmpxchg_insn(const struct bpf_insn *insn) { return BPF_CLASS(insn->code) == BPF_STX && - BPF_MODE(insn->code) == BPF_ATOMIC && + (BPF_MODE(insn->code) == BPF_ATOMIC || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) && insn->imm == BPF_CMPXCHG; } diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index c2796e8d29ea..6f331c80130d 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -530,6 +530,9 @@ static int htab_map_check_btf(struct bpf_map *map, const struct btf *btf, { struct bpf_htab *htab = container_of(map, struct bpf_htab, map); + if (btf_type_is_void(key_type)) + return -EINVAL; + if (htab_is_prealloc(htab)) return 0; /* @@ -1025,7 +1028,7 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, } else { u32 size = round_up(htab->map.value_size, 8); void *val; - int cpu; + int cpu, off = 0; if (map_flags & BPF_F_CPU) { cpu = map_flags >> 32; @@ -1037,9 +1040,10 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, for_each_possible_cpu(cpu) { ptr = per_cpu_ptr(pptr, cpu); - val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu; + val = (map_flags & BPF_F_ALL_CPUS) ? value : value + off; copy_map_value(&htab->map, ptr, val); bpf_obj_cancel_fields(&htab->map, ptr); + off += size; } } } @@ -2864,16 +2868,6 @@ static int rhtab_map_alloc_check(union bpf_attr *attr) return htab_map_alloc_check(attr); } -static void rhtab_check_and_free_fields(struct bpf_rhtab *rhtab, - struct rhtab_elem *elem) -{ - if (IS_ERR_OR_NULL(rhtab->map.record)) - return; - - bpf_obj_free_fields(rhtab->map.record, - rhtab_elem_value(elem, rhtab->map.key_size)); -} - static void rhtab_mem_dtor(void *obj, void *ctx) { struct htab_btf_record *hrec = ctx; @@ -2963,8 +2957,8 @@ static int rhtab_delete_elem(struct bpf_rhtab *rhtab, struct rhtab_elem *elem, v rhtab_read_elem_value(&rhtab->map, copy, elem, flags); check_and_init_map_value(&rhtab->map, copy); } - /* Release internal structs: kptr, bpf_timer, task_work, wq */ - rhtab_check_and_free_fields(rhtab, elem); + bpf_obj_cancel_fields(&rhtab->map, + rhtab_elem_value(elem, rhtab->map.key_size)); bpf_mem_cache_free_rcu(&rhtab->ma, elem); return 0; } @@ -3005,7 +2999,6 @@ static int rhtab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, void static long rhtab_map_update_existing(struct bpf_map *map, struct rhtab_elem *elem, void *value, u64 map_flags) { - struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); void *old_val = rhtab_elem_value(elem, map->key_size); if (map_flags & BPF_NOEXIST) @@ -3025,7 +3018,7 @@ static long rhtab_map_update_existing(struct bpf_map *map, struct rhtab_elem *el * kptrs/etc. still sit in the slot. Cancel them after the copy * to match arraymap's update semantics. */ - rhtab_check_and_free_fields(rhtab, elem); + bpf_obj_cancel_fields(map, old_val); return 0; } @@ -3066,7 +3059,6 @@ static long rhtab_map_update_elem(struct bpf_map *map, void *key, void *value, u memcpy(elem->data, key, map->key_size); copy_map_value(map, rhtab_elem_value(elem, map->key_size), value); - check_and_init_map_value(map, rhtab_elem_value(elem, map->key_size)); /* Prevent deadlock for NMI programs attempting to take bucket lock */ bpf_disable_instrumentation(); @@ -3110,6 +3102,9 @@ static int rhtab_map_check_btf(struct bpf_map *map, const struct btf *btf, { struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); + if (btf_type_is_void(key_type)) + return -EINVAL; + return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor); } diff --git a/kernel/bpf/local_storage.c b/kernel/bpf/local_storage.c index 23267213a17f..83cd527a2542 100644 --- a/kernel/bpf/local_storage.c +++ b/kernel/bpf/local_storage.c @@ -220,7 +220,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key, struct bpf_cgroup_storage *storage; void *val; u32 size; - int cpu; + int cpu, off = 0; if ((u32)map_flags & ~(BPF_ANY | BPF_EXIST | BPF_F_CPU | BPF_F_ALL_CPUS)) return -EINVAL; @@ -245,8 +245,9 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key, } size = round_up(_map->value_size, 8); for_each_possible_cpu(cpu) { - val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu; + val = (map_flags & BPF_F_ALL_CPUS) ? value : value + off; copy_map_value(_map, per_cpu_ptr(storage->percpu_buf, cpu), val); + off += size; } unlock: rcu_read_unlock(); diff --git a/kernel/bpf/percpu_freelist.c b/kernel/bpf/percpu_freelist.c index 632762b57299..06ce588d13a3 100644 --- a/kernel/bpf/percpu_freelist.c +++ b/kernel/bpf/percpu_freelist.c @@ -17,6 +17,8 @@ int pcpu_freelist_init(struct pcpu_freelist *s) raw_res_spin_lock_init(&head->lock); head->first = NULL; } + raw_res_spin_lock_init(&s->extralist.lock); + s->extralist.first = NULL; return 0; } @@ -46,22 +48,28 @@ void __pcpu_freelist_push(struct pcpu_freelist *s, struct pcpu_freelist_node *node) { struct pcpu_freelist_head *head; - int cpu; + int cpu, this_cpu; if (___pcpu_freelist_push(this_cpu_ptr(s->freelist), node)) return; + this_cpu = raw_smp_processor_id(); while (true) { - for_each_cpu_wrap(cpu, cpu_possible_mask, raw_smp_processor_id()) { - if (cpu == raw_smp_processor_id()) + for_each_cpu_wrap(cpu, cpu_possible_mask, this_cpu) { + if (cpu == this_cpu) continue; + head = per_cpu_ptr(s->freelist, cpu); - if (raw_res_spin_lock(&head->lock)) - continue; - pcpu_freelist_push_node(head, node); - raw_res_spin_unlock(&head->lock); - return; + if (___pcpu_freelist_push(head, node)) + return; } + + /* + * Push cannot fail. Use the extra list when none of the + * per-CPU freelists can accept the node. + */ + if (___pcpu_freelist_push(&s->extralist, node)) + return; } } @@ -117,6 +125,17 @@ static struct pcpu_freelist_node *___pcpu_freelist_pop(struct pcpu_freelist *s) } raw_res_spin_unlock(&head->lock); } + + /* Per-CPU lists are empty or unavailable, try the extra list. */ + head = &s->extralist; + if (!READ_ONCE(head->first)) + return NULL; + if (raw_res_spin_lock(&head->lock)) + return NULL; + node = head->first; + if (node) + WRITE_ONCE(head->first, node->next); + raw_res_spin_unlock(&head->lock); return node; } diff --git a/kernel/bpf/percpu_freelist.h b/kernel/bpf/percpu_freelist.h index 914798b74967..980cf2884fd2 100644 --- a/kernel/bpf/percpu_freelist.h +++ b/kernel/bpf/percpu_freelist.h @@ -14,6 +14,7 @@ struct pcpu_freelist_head { struct pcpu_freelist { struct pcpu_freelist_head __percpu *freelist; + struct pcpu_freelist_head extralist; }; struct pcpu_freelist_node { diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index a839041e0d00..d09d4c3fe547 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -875,6 +875,7 @@ BPF_CALL_4(bpf_get_stack_sleepable, struct pt_regs *, regs, void *, buf, u32, si const struct bpf_func_proto bpf_get_stack_sleepable_proto = { .func = bpf_get_stack_sleepable, .gpl_only = true, + .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, @@ -928,6 +929,7 @@ BPF_CALL_4(bpf_get_task_stack_sleepable, struct task_struct *, task, void *, buf const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = { .func = bpf_get_task_stack_sleepable, .gpl_only = false, + .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 4e6aafad33bd..66fb11b6c6a7 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -445,22 +445,19 @@ static void __clean_func_state(struct bpf_verifier_env *env, struct bpf_reg_state *spill = &st->stack[i].spilled_ptr; if (lo_live && stype == STACK_SPILL) { - u8 val = STACK_MISC; - if (spill->type != SCALAR_VALUE) continue; - /* - * 8 byte spill of scalar 0 where half slot is dead - * should become STACK_ZERO in lo 4 bytes. + * Can't replace with STACK_ZERO, because + * that requires bpf_mark_chain_precision(). */ if (bpf_register_is_null(spill)) - val = STACK_ZERO; + continue; for (j = 0; j < 4; j++) { u8 *t = &st->stack[i].slot_type[j]; if (*t == STACK_SPILL) - *t = val; + *t = STACK_MISC; } } bpf_mark_reg_not_init(env, spill); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 6874ba1424af..c7bc9ba9b331 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -6568,6 +6568,7 @@ EXPORT_SYMBOL_NS(kern_sys_bpf, "BPF_INTERNAL"); static const struct bpf_func_proto bpf_sys_bpf_proto = { .func = bpf_sys_bpf, .gpl_only = false, + .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, @@ -6593,6 +6594,7 @@ BPF_CALL_1(bpf_sys_close, u32, fd) static const struct bpf_func_proto bpf_sys_close_proto = { .func = bpf_sys_close, .gpl_only = false, + .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a68e5435ca5f..72a3f5998dd2 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -352,9 +352,18 @@ static bool reg_not_null(struct bpf_verifier_env *env, const struct bpf_reg_stat if (type_may_be_null(type)) return false; + /* + * The types below guarantee a non-NULL base, an unbounded offset can + * still wrap base + offset to zero. + */ + if (reg_smin(reg) <= -BPF_MAX_VAR_OFF || reg_smax(reg) >= BPF_MAX_VAR_OFF) + return false; + type = base_type(type); return type == PTR_TO_SOCKET || type == PTR_TO_TCP_SOCK || + type == PTR_TO_XDP_SOCK || + type == PTR_TO_BUF || type == PTR_TO_MAP_VALUE || type == PTR_TO_MAP_KEY || type == PTR_TO_SOCK_COMMON || @@ -4237,6 +4246,15 @@ static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx) return mark_chain_precision_batch(env, env->cur_state); } +static int mark_arg_precision(struct bpf_verifier_env *env, argno_t argno) +{ + int regno = reg_from_argno(argno); + + if (regno >= 0) + return mark_chain_precision(env, regno); + return mark_stack_arg_precision(env, arg_idx_from_argno(argno)); +} + static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, int nargs, const char *callee_name, const struct btf *btf, const struct btf_param *args) @@ -4486,6 +4504,13 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, if (type_flag(reg->type) & ~perm_flags) goto bad_type; + /* + * A BPF_KPTR_PERCPU field is read back as MEM_PERCPU, so the value + * stored in it must carry the same flag. + */ + if ((kptr_field->type == BPF_KPTR_PERCPU) != !!(reg->type & MEM_PERCPU)) + goto bad_type; + /* We need to verify reg->type and reg->btf, before accessing reg->btf */ reg_name = btf_type_name(reg->btf, reg->btf_id); @@ -4692,8 +4717,15 @@ static int check_map_kptr_access(struct bpf_verifier_env *env, return ret; } else if (class == BPF_STX) { val_reg = reg_state(env, value_regno); - if (!bpf_register_is_null(val_reg) && - map_kptr_match_type(env, kptr_field, val_reg, value_regno)) + if (bpf_register_is_null(val_reg)) { + /* + * This store is valid only because the scalar is known to be + * zero. Mark it precise so another scalar cannot be pruned + * against this state. + */ + return mark_chain_precision(env, value_regno); + } + if (map_kptr_match_type(env, kptr_field, val_reg, value_regno)) return -EACCES; } else if (class == BPF_ST) { if (insn->imm) { @@ -5300,6 +5332,15 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, if (!priv_stack_supported) subprog[idx].priv_stack_mode = NO_PRIV_STACK; process_func: + if (subprog[idx].has_ld_abs) { + for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { + if (subprog[tmp].is_cb) { + verbose(env, "cannot use BPF_LD_[ABS|IND] within callback\n"); + return -EINVAL; + } + } + } + /* protect against potential stack overflow that might happen when * bpf2bpf calls get combined with tailcalls. Limit the caller's stack * depth for such case down to 256 so that the worst case scenario @@ -6020,7 +6061,13 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, return -EACCES; } - if (type_is_alloc(reg->type) && !type_is_non_owning_ref(reg->type) && + /* + * A fault-prone allocated object may still be read through a + * BPF_PROBE_MEM load after its lifetime protection ends. Writes + * through such pointers were rejected above. + */ + if (type_is_alloc(reg->type) && !bpf_may_fault_on_deref(reg->type) && + !type_is_non_owning_ref(reg->type) && !(reg->type & MEM_RCU) && !reg_is_referenced(env, reg)) { verifier_bug(env, "allocated object must have a referenced id"); return -EFAULT; @@ -7113,14 +7160,8 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, if (err && failure) *failure = BPF_MEM_SIZE_FAIL_MEMORY; - if (!err) { - int regno = reg_from_argno(size_argno); - - if (regno >= 0) - err = mark_chain_precision(env, regno); - else - err = mark_stack_arg_precision(env, arg_idx_from_argno(size_argno)); - } + if (!err) + err = mark_arg_precision(env, size_argno); return err; @@ -7137,7 +7178,7 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg int size, err = 0; if (bpf_register_is_null(reg)) - return 0; + return mark_arg_precision(env, argno); if (known_memory) *known_memory = true; @@ -7398,10 +7439,14 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state lock); return -EINVAL; } + /* + * Invalidate non-owning refs before RCU demotion clears their + * NON_OWN_REF flag. + */ + invalidate_non_owning_refs(env); + if (!in_rcu_cs(env)) invalidate_rcu_protected_refs(env); - - invalidate_non_owning_refs(env); } return 0; } @@ -8166,6 +8211,7 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_MEM_SIZE] = &scalar_types, [ARG_MEM_SIZE_OR_ZERO] = &scalar_types, [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, + [ARG_SCALAR] = &scalar_types, [ARG_CONST_MAP_PTR] = &const_map_ptr_types, [ARG_PTR_TO_CTX] = &context_types, [ARG_PTR_TO_SOCK_COMMON] = &sock_types, @@ -8717,11 +8763,15 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return err; } - if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) + if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) { /* A NULL register has a SCALAR_VALUE type, so skip * type checking. */ + err = mark_chain_precision(env, regno); + if (err) + return err; goto skip_type_check; + } /* arg_btf_id and arg_size are in a union. */ if (base_type(arg_type) == ARG_PTR_TO_BTF_ID || @@ -9501,7 +9551,7 @@ static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env) bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, clear_mask, ({ if (reg->type & MEM_RCU) { bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE); - reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL); + reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL | NON_OWN_REF); reg->type |= PTR_UNTRUSTED; bpf_diag_mod_end(env); } @@ -9719,8 +9769,12 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct bpf_call_arg_meta meta; int err; - if (bpf_register_is_null(reg) && type_may_be_null(arg->arg_type)) + if (bpf_register_is_null(reg) && type_may_be_null(arg->arg_type)) { + err = mark_arg_precision(env, argno); + if (err) + return err; continue; + } memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */ err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta, @@ -9976,10 +10030,12 @@ int map_set_for_each_callback_args(struct bpf_verifier_env *env, callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; __mark_reg_known_zero(&callee->regs[BPF_REG_2]); callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr; + callee->regs[BPF_REG_2].map_uid = caller->regs[BPF_REG_1].map_uid; callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr; + callee->regs[BPF_REG_3].map_uid = caller->regs[BPF_REG_1].map_uid; /* pointer to stack or null */ callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3]; @@ -10057,6 +10113,7 @@ static int set_timer_callback_state(struct bpf_verifier_env *env, int insn_idx) { struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr; + u32 map_uid = caller->regs[BPF_REG_1].map_uid; /* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn); * callback_fn(struct bpf_map *map, void *key, void *value); @@ -10064,14 +10121,17 @@ static int set_timer_callback_state(struct bpf_verifier_env *env, callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; __mark_reg_known_zero(&callee->regs[BPF_REG_1]); callee->regs[BPF_REG_1].map_ptr = map_ptr; + callee->regs[BPF_REG_1].map_uid = map_uid; callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; __mark_reg_known_zero(&callee->regs[BPF_REG_2]); callee->regs[BPF_REG_2].map_ptr = map_ptr; + callee->regs[BPF_REG_2].map_uid = map_uid; callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = map_ptr; + callee->regs[BPF_REG_3].map_uid = map_uid; /* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); @@ -10171,6 +10231,7 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, int insn_idx) { struct bpf_map *map_ptr = caller->regs[BPF_REG_3].map_ptr; + u32 map_uid = caller->regs[BPF_REG_3].map_uid; /* * callback_fn(struct bpf_map *map, void *key, void *value); @@ -10178,14 +10239,17 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; __mark_reg_known_zero(&callee->regs[BPF_REG_1]); callee->regs[BPF_REG_1].map_ptr = map_ptr; + callee->regs[BPF_REG_1].map_uid = map_uid; callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; __mark_reg_known_zero(&callee->regs[BPF_REG_2]); callee->regs[BPF_REG_2].map_ptr = map_ptr; + callee->regs[BPF_REG_2].map_uid = map_uid; callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = map_ptr; + callee->regs[BPF_REG_3].map_uid = map_uid; /* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); @@ -10233,9 +10297,10 @@ static void account_current_path(struct bpf_verifier_env *env) frame ? state->frame[frame - 1] : NULL); } -/* Are we currently verifying the callback for a rbtree helper that must - * be called with lock held? If so, no need to complain about unreleased - * lock +/* + * Are we currently verifying the callback for an rbtree kfunc that must + * be called with a lock held, or one of that callback's subprogs? If so, + * no need to complain about an unreleased lock. */ static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env) { @@ -10243,17 +10308,19 @@ static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env) struct bpf_insn *insn = env->prog->insnsi; struct bpf_func_state *callee; int kfunc_btf_id; + u32 frame; - if (!state->curframe) - return false; - - callee = state->frame[state->curframe]; + for (frame = state->curframe; frame; frame--) { + callee = state->frame[frame]; + if (!callee->in_callback_fn) + continue; - if (!callee->in_callback_fn) - return false; + kfunc_btf_id = insn[callee->callsite].imm; + if (is_rbtree_lock_required_kfunc(kfunc_btf_id)) + return true; + } - kfunc_btf_id = insn[callee->callsite].imm; - return is_rbtree_lock_required_kfunc(kfunc_btf_id); + return false; } static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg) @@ -10630,33 +10697,45 @@ static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env) return &env->insn_aux_data[env->insn_idx]; } -static bool loop_flag_is_zero(struct bpf_verifier_env *env) +/* Returns 1 if R4 is a known zero, 0 if it is not, a negative errno on error. */ +static int loop_flag_is_zero(struct bpf_verifier_env *env) { struct bpf_reg_state *reg = reg_state(env, BPF_REG_4); - bool reg_is_null = bpf_register_is_null(reg); + int err; - if (reg_is_null) - mark_chain_precision(env, BPF_REG_4); + if (!bpf_register_is_null(reg)) + return 0; - return reg_is_null; + err = mark_chain_precision(env, BPF_REG_4); + if (err) + return err; + return 1; } -static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) +static int update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) { struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state; + int flag_is_zero; if (!state->initialized) { + flag_is_zero = loop_flag_is_zero(env); + if (flag_is_zero < 0) + return flag_is_zero; state->initialized = 1; - state->fit_for_inline = loop_flag_is_zero(env); + state->fit_for_inline = flag_is_zero; state->callback_subprogno = subprogno; - return; + return 0; } if (!state->fit_for_inline) - return; + return 0; - state->fit_for_inline = (loop_flag_is_zero(env) && + flag_is_zero = loop_flag_is_zero(env); + if (flag_is_zero < 0) + return flag_is_zero; + state->fit_for_inline = (flag_is_zero && state->callback_subprogno == subprogno); + return 0; } /* Returns whether or not the given map can potentially elide @@ -10868,6 +10947,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn verbose(env, "get_local_storage() doesn't support non-zero flags\n"); return -EINVAL; } + err = mark_chain_precision(env, BPF_REG_2); + if (err) + return err; break; case BPF_FUNC_for_each_map_elem: err = push_callback_call(env, insn, insn_idx, meta.subprogno, @@ -10885,7 +10967,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn err = check_bpf_snprintf_call(env, regs); break; case BPF_FUNC_loop: - update_loop_inline_state(env, meta.subprogno); + err = update_loop_inline_state(env, meta.subprogno); + if (err) + return err; /* Verifier relies on R1 value to determine if bpf_loop() iteration * is finished, thus mark it precise. */ @@ -11226,6 +11310,17 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (env->cur_state->curframe) { struct bpf_verifier_state *branch; + /* + * A taken tail call is modeled as a return from the current + * frame. A callback frame cannot be left that way because + * prepare_func_exit() would apply its return contract to the + * unknown R0 synthesized below. Stack-depth validation rejects + * this construct anyway. + */ + if (cur_func(env)->in_callback_fn) { + verbose(env, "cannot tail call within callback\n"); + return -EINVAL; + } mark_reg_scratched(env, BPF_REG_0); branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); if (IS_ERR(branch)) @@ -12651,8 +12746,12 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (reg_is_referenced(env, reg)) update_ref_obj(&meta->ref_obj, reg); - if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) + if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) { + ret = mark_arg_precision(env, argno); + if (ret) + return ret; continue; + } if (is_kfunc_arg_map(btf, &args[i])) { ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; @@ -13146,7 +13245,7 @@ check_ok: bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } - if (!type_is_non_owning_ref(reg->type)) + if (!type_is_non_owning_ref(reg->type) && reg_is_referenced(env, reg)) meta->arg_owning_ref = true; rec = reg_btf_record(reg); @@ -13228,6 +13327,11 @@ check_ok: { int flags = PROCESS_RES_LOCK; + if (in_rbtree_lock_required_cb(env)) { + verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n"); + return -EACCES; + } + if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { verbose(env, "%s doesn't point to map value or allocated object\n", reg_arg_name(env, argno)); @@ -14560,9 +14664,6 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn return -EINVAL; } - /* pointer types do not carry 32-bit bounds at the moment. */ - __mark_reg32_unbounded(dst_reg); - if (sanitize_needed(opcode)) { ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, &info, false); @@ -14570,6 +14671,14 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn return sanitize_err(env, insn, ret); } + /* + * Pointer types do not carry 32-bit bounds at the moment. Blank r32 + * only after sanitize_ptr_alu() may have snapshotted dst_reg into a + * speculative path: otherwise reg_bounds_sanity_check() might hit some + * constraints violations. + */ + __mark_reg32_unbounded(dst_reg); + switch (opcode) { case BPF_ADD: /* @@ -16295,6 +16404,13 @@ static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *r if (__is_pointer_value(false, reg1) || __is_pointer_value(false, reg2)) { u64 val; + /* + * The low 32 bits of a valid pointer may well be zero, hence + * nothing below applies to a 32-bit comparison. + */ + if (is_jmp32) + return -1; + /* arrange that reg2 is a scalar, and reg1 is a pointer */ if (!is_reg_const(reg2, is_jmp32)) { opcode = flip_opcode(opcode); @@ -16856,6 +16972,16 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, return err; } + /* + * Collect the linked registers before env->{true,false}_reg{1,2} setup, + * otherwise ids dropped by collect_linked_regs() would be resurrected + * when env->{true,false}_reg{1,2} are copied back. + */ + if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id) + collect_linked_regs(env, this_branch, src_reg->id, &linked_regs); + if (dst_reg->type == SCALAR_VALUE && dst_reg->id) + collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs); + is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; env->false_reg1 = *dst_reg; env->false_reg2 = *src_reg; @@ -16910,10 +17036,6 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, * 'this_branch' and 'other_branch' share this history * if parent state is created. */ - if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id) - collect_linked_regs(env, this_branch, src_reg->id, &linked_regs); - if (dst_reg->type == SCALAR_VALUE && dst_reg->id) - collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs); if (linked_regs.cnt > 1) { err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, linked_regs_pack(&linked_regs)); if (err) @@ -16963,7 +17085,6 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, */ if (!is_jmp32 && BPF_SRC(insn->code) == BPF_X && __is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) && - type_may_be_null(src_reg->type) != type_may_be_null(dst_reg->type) && base_type(src_reg->type) != PTR_TO_BTF_ID && base_type(dst_reg->type) != PTR_TO_BTF_ID) { eq_branch_regs = NULL; @@ -16979,9 +17100,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, break; } if (eq_branch_regs) { - if (type_may_be_null(src_reg->type)) + /* src == dst && dst != NULL => src != NULL */ + if (reg_not_null(env, dst_reg) && type_may_be_null(src_reg->type)) mark_ptr_not_null_reg(&eq_branch_regs[insn->src_reg]); - else + /* src == dst && src != NULL => dst != NULL */ + if (reg_not_null(env, src_reg) && type_may_be_null(dst_reg->type)) mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]); } } @@ -16996,6 +17119,15 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, type_may_be_null(dst_reg->type) && ((BPF_SRC(insn->code) == BPF_K && insn->imm == 0) || (BPF_SRC(insn->code) == BPF_X && bpf_register_is_null(src_reg)))) { + /* + * For BPF_X the zero is a property of this execution path, + * hence src_reg has to be precise. + */ + if (BPF_SRC(insn->code) == BPF_X) { + err = mark_chain_precision(env, insn->src_reg); + if (err) + return err; + } /* Mark all identical registers in each branch as either * safe or unknown depending R == 0 or R != 0 conditional. */ @@ -17081,6 +17213,15 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) verbose(env, "callback function not static\n"); return -EINVAL; } + /* + * When env->subprog_cnt == 1 this instruction won't be rewritten + * to hold a real function address. Assume that no usable program + * combines e.g. main and timer callback and just reject here. + */ + if (subprogno == 0) { + verbose(env, "callback function cannot be the main program\n"); + return -EINVAL; + } dst_reg->type = PTR_TO_FUNC; dst_reg->subprogno = subprogno; @@ -17146,6 +17287,7 @@ static bool may_access_skb(enum bpf_prog_type type) */ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) { + struct bpf_verifier_state *state = env->cur_state; struct bpf_reg_state *regs = cur_regs(env); static const int ctx_reg = BPF_REG_6; u8 mode = BPF_MODE(insn->code); @@ -17156,6 +17298,13 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) return -EINVAL; } + for (i = state->curframe; i; i--) { + if (state->frame[i]->in_callback_fn) { + verbose(env, "cannot use BPF_LD_[ABS|IND] within callback\n"); + return -EINVAL; + } + } + if (!env->ops->gen_ld_abs) { verifier_bug(env, "gen_ld_abs is null"); return -EFAULT; @@ -17623,6 +17772,10 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, * r0 = *(u64 *)(r10 - 8); r0 += r1; * r0 += r1; exit; * exit; + * + * Both uses of the marks assume that a pattern is entered at its first + * spill and thus executes as a unit, hence a pattern is not grown past + * an instruction targeted by a jump. */ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, struct bpf_subprog_info *subprog, @@ -17661,6 +17814,10 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, for (i = 1, off = lowest_off; i <= ARRAY_SIZE(caller_saved); ++i, off += BPF_REG_SIZE) { if (insn_idx - i < 0 || insn_idx + i >= env->prog->len) break; + /* stx/ldx/call must not be a jump targets, a jump to the first stx is fine */ + if (bpf_is_jump_target(env, insn_idx - i + 1) || + bpf_is_jump_target(env, insn_idx + i)) + break; stx = &insns[insn_idx - i]; ldx = &insns[insn_idx + i]; /* must be a stack spill/fill pair */ |