summaryrefslogtreecommitdiff
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/bpf/arena.c16
-rw-r--r--kernel/bpf/btf.c19
-rw-r--r--kernel/bpf/cgroup.c26
-rw-r--r--kernel/bpf/stackmap.c33
-rw-r--r--kernel/bpf/verifier.c66
5 files changed, 131 insertions, 29 deletions
diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c
index 3c61c181a6fa..0b1ec3306ea4 100644
--- a/kernel/bpf/arena.c
+++ b/kernel/bpf/arena.c
@@ -634,7 +634,7 @@ static unsigned long arena_get_unmapped_area(struct file *filp, unsigned long ad
if (pgoff)
return -EINVAL;
- if (len > SZ_4G)
+ if (len > (u64)map->max_entries * PAGE_SIZE)
return -E2BIG;
/* if user_vm_start was specified at arena creation time */
@@ -650,11 +650,11 @@ static unsigned long arena_get_unmapped_area(struct file *filp, unsigned long ad
ret = mm_get_unmapped_area(filp, addr, len * 2, 0, flags);
if (IS_ERR_VALUE(ret))
return ret;
+ if (!ret)
+ /* user_vm_start == 0 means that the user range is not set yet */
+ return -EINVAL;
if ((ret >> 32) == ((ret + len - 1) >> 32))
return ret;
- if (WARN_ON_ONCE(arena->user_vm_start))
- /* checks at map creation time should prevent this */
- return -EFAULT;
return round_up(ret, SZ_4G);
}
@@ -679,7 +679,7 @@ static int arena_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)
return -EBUSY;
/* Earlier checks should prevent this */
- if (WARN_ON_ONCE(vma->vm_end - vma->vm_start > SZ_4G || vma->vm_pgoff))
+ if (WARN_ON_ONCE(vma_pages(vma) > map->max_entries || vma->vm_pgoff))
return -EFAULT;
if (remember_vma(arena, vma))
@@ -988,6 +988,8 @@ static int arena_free_range(struct bpf_arena *arena, long uaddr, long page_cnt)
static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable)
{
+ /* The first mmap() may be setting the user range right now */
+ u64 user_vm_start = READ_ONCE(arena->user_vm_start);
struct mem_cgroup *new_memcg, *old_memcg;
u64 full_uaddr, uaddr_end;
struct arena_free_span *s;
@@ -995,8 +997,8 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt,
/* only aligned lower 32-bit are relevant */
uaddr = (u32)uaddr;
uaddr &= PAGE_MASK;
- full_uaddr = clear_lo32(arena->user_vm_start) + uaddr;
- if (full_uaddr < arena->user_vm_start)
+ full_uaddr = clear_lo32(user_vm_start) + uaddr;
+ if (!user_vm_start || full_uaddr < user_vm_start)
return;
uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT));
if (full_uaddr >= uaddr_end)
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 1be30487a2fa..8bf39098ba97 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -6590,6 +6590,15 @@ static int find_kern_ctx_type_id(enum bpf_prog_type prog_type)
return ctx_type->type;
}
+/*
+ * A projection type is the view that programs have of a kernel object with
+ * another layout: see btf_is_projection_of() for the two types behind them.
+ */
+bool btf_is_projection_type(const char *pname)
+{
+ return strcmp(pname, "__sk_buff") == 0 || strcmp(pname, "xdp_md") == 0;
+}
+
bool btf_is_projection_of(const char *pname, const char *tname)
{
if (strcmp(pname, "__sk_buff") == 0 && strcmp(tname, "sk_buff") == 0)
@@ -8483,9 +8492,8 @@ static int btf_check_func_type_match(struct bpf_verifier_log *log,
}
/* This is an optional check to make program writing easier.
* Compare names of structs and report an error to the user.
- * btf_prepare_func_args() already checked that t2 struct
- * is a context type. btf_prepare_func_args() will check
- * later that t1 struct is a context type as well.
+ * check_ext_args() compares what btf_prepare_func_args()
+ * derives for the two arguments.
*/
s1 = btf_name_by_offset(btf1, t1->name_off);
s2 = btf_name_by_offset(btf2, t2->name_off);
@@ -9019,6 +9027,11 @@ skip_pointer:
return err;
sub->arg_slot_cnt = slots_used;
+ /* check_ext_args() needs them after env is freed */
+ if (is_global)
+ memcpy(subprog_aux(env, subprog)->args, sub->args,
+ sizeof(subprog_aux(env, subprog)->args));
+
sub->args_cached = true;
return 0;
diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index ce82963bcfc7..57a785241d42 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -1279,9 +1279,14 @@ static struct bpf_prog_list *find_detach_entry(struct hlist_head *progs,
return ERR_PTR(-ENOENT);
/* to maintain backward compatibility NONE and OVERRIDE cgroups
- * allow detaching with invalid FD (prog==NULL) in legacy mode
+ * allow detaching with invalid FD (prog==NULL) in legacy mode.
+ * Without the FD of the attached program the caller needs
+ * CAP_NET_ADMIN or CAP_SYS_ADMIN in the initial user namespace.
*/
- return hlist_entry(progs->first, typeof(*pl), node);
+ pl = hlist_entry(progs->first, typeof(*pl), node);
+ if (pl->prog != prog && !bpf_net_capable())
+ return ERR_PTR(-EPERM);
+ return pl;
}
if (!prog && !link)
@@ -1785,7 +1790,7 @@ int __cgroup_bpf_run_filter_skb(struct sock *sk,
struct sk_buff *skb,
enum cgroup_bpf_attach_type atype)
{
- unsigned int offset = -skb_network_offset(skb);
+ int offset = skb_network_offset(skb);
struct sock *save_sk;
void *saved_data_end;
struct cgroup *cgrp;
@@ -1797,7 +1802,15 @@ int __cgroup_bpf_run_filter_skb(struct sock *sk,
cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
save_sk = skb->sk;
skb->sk = sk;
- __skb_push(skb, offset);
+
+ /*
+ * The program sees the packet from its network header on, on
+ * whichever side of skb->data that header lies.
+ */
+ if (offset > 0)
+ __skb_pull(skb, offset);
+ else
+ __skb_push(skb, -offset);
/* compute pointers for the bpf prog */
bpf_compute_and_save_data_end(skb, &saved_data_end);
@@ -1839,7 +1852,10 @@ int __cgroup_bpf_run_filter_skb(struct sock *sk,
ret = -EFAULT;
}
bpf_restore_data_end(skb, saved_data_end);
- __skb_pull(skb, offset);
+ if (offset > 0)
+ __skb_push(skb, offset);
+ else
+ __skb_pull(skb, -offset);
skb->sk = save_sk;
return ret;
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index f7e8d766d128..a19126f5db02 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -522,12 +522,14 @@ static int stackid_init(struct stackid *stackid, struct bpf_map *map,
u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
u32 max_depth;
+ max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags);
+ /* trace_nr can be above max_depth and max_depth below skip: clamp first */
+ trace_nr = min_t(u32, trace_nr, max_depth);
if (trace_nr <= skip)
/* skipping more than usable stack trace */
return -EFAULT;
- max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags);
- stackid->nr = min_t(u32, trace_nr - skip, max_depth - skip);
+ stackid->nr = trace_nr - skip;
stackid->len = stackid->nr * sizeof(u64);
stackid->ips = trace->ip + skip;
stackid->hash = jhash2((const u32 *)stackid->ips, stackid->len / sizeof(u32), 0);
@@ -642,7 +644,8 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
/* couldn't fetch the stack trace */
return -EFAULT;
- err = stackid_fastpath(&stackid, map, trace, trace->nr, flags);
+ /* the entry is released already, an NMI can change trace->nr */
+ err = stackid_fastpath(&stackid, map, trace, READ_ONCE(trace->nr), flags);
if (err != -ENOENT)
return err;
@@ -822,14 +825,20 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
crosstask, false, 0);
}
- if (unlikely(!trace) || trace->nr < skip) {
- if (may_fault)
- rcu_read_unlock();
- preempt_enable();
- goto err_fault;
- }
+ if (unlikely(!trace))
+ goto err_unlock;
- trace_nr = callchain_store(trace, trace->nr, buf, elem_size, flags);
+ /*
+ * trace->nr can be above max_depth. The per-CPU entry is released
+ * already: a tracing program or an NMI which takes a callchain in the
+ * same context before the copy below reuses it. And the unwinders of
+ * PowerPC store context markers on top of max_depth.
+ */
+ trace_nr = min_t(u32, READ_ONCE(trace->nr), max_depth);
+ if (trace_nr < skip)
+ goto err_unlock;
+
+ trace_nr = callchain_store(trace, trace_nr, buf, elem_size, flags);
/* trace should not be dereferenced after this point */
if (may_fault)
@@ -838,6 +847,10 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
return callchain_finalize(buf, size, trace_nr, elem_size, flags, may_fault);
+err_unlock:
+ if (may_fault)
+ rcu_read_unlock();
+ preempt_enable();
err_fault:
err = -EFAULT;
clear:
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index a90bc6c4033a..217bcfa6ebfe 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -13574,7 +13574,12 @@ static int resolve_func_arg_type(struct bpf_verifier_env *env,
if (base_type(*arg_type) != ARG_PTR_TO_BTF_ID)
return 0;
+ /*
+ * A kfunc that releases an object, or that takes a reference to one,
+ * needs the object itself: memory of the program cannot stand in for it.
+ */
if (!is_kfunc(meta) || arg_type_is_release(*arg_type) ||
+ is_kfunc_acquire(meta) ||
base_type(reg->type) == PTR_TO_BTF_ID ||
reg2btf_ids[base_type(reg->type)])
return 0;
@@ -13585,7 +13590,15 @@ static int resolve_func_arg_type(struct bpf_verifier_env *env,
ref_t = btf_type_skip_modifiers(btf, ref_id, &ref_id);
ref_tname = btf_name_by_offset(btf, ref_t->name_off);
- if (!btf_type_is_scalar_struct(env, btf, ref_t))
+ /*
+ * A kfunc that takes a projection type (ie. xdp_md) casts it to the
+ * underlying type. So memory of the program cannot stand in for it,
+ * even if the projection type holds only scalars. A struct without
+ * members (ie. iou_ctx) is a handle for an object of another type.
+ */
+ if (btf_is_projection_type(ref_tname) ||
+ !btf_type_is_scalar_struct(env, btf, ref_t) ||
+ !btf_type_vlen(ref_t))
return 0;
resolve_ret = btf_resolve_size(btf, ref_t, arg_size);
@@ -14340,9 +14353,13 @@ static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_re
/* If kfunc is accepting a projection type (ie. __sk_buff), it cannot
* actually use it -- it must cast to the underlying type. So we allow
- * caller to pass in the underlying type.
+ * caller to pass in the underlying type. Only kernel BTF has that type:
+ * a type of the same name in the BTF of the program is program memory.
+ * And only a pointer to the start of the object points to that type.
*/
- taking_projection = is_kfunc(meta) && btf_is_projection_of(arg_tname, reg_tname);
+ taking_projection = is_kfunc(meta) && btf_is_kernel(reg_btf) &&
+ !reg->var_off.value &&
+ btf_is_projection_of(arg_tname, reg_tname);
if (!taking_projection && !struct_same) {
verbose(env, "%s %s expected pointer to %s %s but %s has a pointer to %s %s\n",
meta->func_name, reg_arg_name(env, argno),
@@ -22510,6 +22527,37 @@ static bool func_has_rcu_read_lock(const struct bpf_prog *prog, int subprog)
!sub_aux->called[BPF_FUNC_CTX_NO_RCU_LOCK];
}
+/*
+ * The callers of a global function are checked against the arguments that its
+ * BTF describes, and an extension is verified with those of its own BTF. It
+ * must not expect more than the callers of the function provide.
+ *
+ * While an extension is loaded, bpf_check_attach_target() runs before its
+ * arguments are known. check_attach_btf_id() does the check when they are.
+ */
+static int check_ext_args(struct bpf_verifier_log *log, const struct bpf_prog *prog,
+ const struct bpf_prog *tgt_prog, int subprog)
+{
+ const struct bpf_subprog_arg_info *tgt = tgt_prog->aux->func_info_aux[subprog].args;
+ const struct bpf_subprog_arg_info *ext = prog->aux->func_info_aux[0].args;
+ int i;
+
+ for (i = 0; i < MAX_BPF_FUNC_REG_ARGS && ext[i].arg_type != ARG_UNUSED; i++) {
+ /* An extension may accept NULL where the callers never pass it. */
+ if ((ext[i].arg_type == tgt[i].arg_type ||
+ ext[i].arg_type == (tgt[i].arg_type | PTR_MAYBE_NULL)) &&
+ (base_type(ext[i].arg_type) == ARG_PTR_TO_MEM ?
+ ext[i].mem_size <= tgt[i].mem_size : ext[i].btf_id == tgt[i].btf_id))
+ continue;
+
+ bpf_log(log, "Extension program's argument in R%d is incompatible with original\n",
+ i + 1);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
int bpf_check_attach_target(struct bpf_verifier_log *log,
const struct bpf_prog *prog,
const struct bpf_prog *tgt_prog,
@@ -22797,7 +22845,8 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
return -EINVAL;
}
if (prog_extension &&
- btf_check_type_match(log, prog, btf, t))
+ (btf_check_type_match(log, prog, btf, t) ||
+ check_ext_args(log, prog, tgt_prog, subprog)))
return -EINVAL;
t = btf_attach_func_proto(log, btf, btf_id);
if (!t || !btf_type_is_func_proto(t))
@@ -23017,6 +23066,15 @@ static int check_attach_btf_id(struct bpf_verifier_env *env)
*/
env->ops = bpf_verifier_ops[tgt_prog->type];
prog->expected_attach_type = tgt_prog->expected_attach_type;
+
+ /* only now it is known what the arguments of prog stand for */
+ ret = btf_prepare_func_args(env, 0);
+ if (ret)
+ return ret;
+ ret = check_ext_args(&env->log, prog, tgt_prog,
+ find_tgt_subprog(tgt_prog, btf_id));
+ if (ret)
+ return ret;
}
/* store info about the attachment target that will be used later */
generated by cgit v1.2.3 (git 2.0.4) at 2026-11-07 00:33:37 +0000