diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2026-11-20 17:41:06 -0800 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2026-11-20 17:41:06 -0800 |
| commit | 84afebd853437e44cb71f718be11d90e4c1a59dd (patch) | |
| tree | 09dc2b20cfa9303bc824b928eb6c9a322ecb318a /kernel | |
| parent | 05718d5c2907cb46fb27bb6580edb89769ea430a (diff) | |
| parent | d5ba86298b45e1a81c578c499e9a222a3b734d5d (diff) | |
Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf
Pull bpf fixes from Alexei Starovoitov:
"Mostly sockmap fixes as promised, plus uprobe ctx rewrite via bpf and
cgroup-bpf fixes.
- Reject writable map iterators on read-only maps (Adethya
Srinivasan)
- Various sockmap fixes (Alexei Starovoitov):
- Clear the redirect target before the sk_msg prog runs
- Check that the redirected message is the cork
- Leave sk_stream_wait_memory() once the socket is dead
- Keep the rest of the cork after a redirect
- Don't let an SK_PASS verdict cover the next message
- Subtract all dropped bytes from the return value of sendmsg()
- Clear the end mark of elements added by bpf_msg_push_data() and
bpf_msg_pop_data()
- Fix ring capacity invariant break in bpf_msg_push_data() (Bo Cao)
- Report RCU-tasks quiescent states in do_check() to fix rcu_tasks
stalls during a long verification (Breno Leitao)
- Various sockmap fixes (Chris Mason):
- Give no progs to a TCP listener
- Check the target of an egress redirect before each send
- Check the TCP state in recvmsg()
- Reset the copy position when corking
- Don't copy 0 bytes into a full cork
- Use a dedicated workqueue for freeing BPF programs to fix a
deadlock between pack_mutex waiters and the vmalloc purge work
(Emil Tsalapatis)
- Fix verifier model mismatch on a narrow read of
__sk_buff.remote_port (Henry Martin)
- Fix bpf-gcc CO-RE relocation in verifier_sockmap_mutate selftest
(Jiayuan Chen)
- Restrict uprobe ctx writes to the registers that ptrace can set
(Jiri Olsa)
- Various sockmap fixes (Josef Bacik):
- Write sg.start when recvmsg() faults
- Don't let an SK_DROP verdict stick
- Clear the resume state of a stopped psock
- Various fixes (Michal Luczaj):
- Fix "unresolved symbol tcp_request_sock" build error with
CONFIG_INET=n
- Restrict iter-context sockmap updates to iter/sockmap
- Restrict bpf_sock_destroy() to iter/{tcp,udp}
- Mark bpf_iter__sockmap::sk as PTR_TRUSTED
- Mark backing_file_alloc/free LSM hooks as untrusted (Sergei
Nizovtsev)
- Various cgroup-bpf fixes (Shakeel Butt):
- Don't read freed effective arrays in BPF_PROG_QUERY
- Don't attach to a removed cgroup
- Protect cgroup_bpf state with its own mutex, so that queries
and the release work don't wait for cgroup_mutex
- Don't lose errors in effective BPF_LSM_CGROUP queries
- Check variable-offset accesses against special fields (Shihuang
Liu)"
* tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: (38 commits)
selftests/bpf: Test which ctx registers a uprobe prog can write
bpf: Restrict uprobe ctx writes to the registers ptrace can set
selftests/bpf: Fix bpf-gcc CO-RE relocation in verifier_sockmap_mutate
bpf, cgroup: Don't lose errors in effective BPF_LSM_CGROUP queries
bpf, cgroup: Protect cgroup_bpf state with its own mutex
bpf, cgroup: Don't attach to a removed cgroup
bpf, cgroup: Don't read freed effective arrays in BPF_PROG_QUERY
bpf: Fix verifier model mismatch on __sk_buff.remote_port narrow read
bpf: Fix "unresolved symbol tcp_request_sock" build error
selftests/bpf: Add iter/sockmap bpf_sock_destroy() negative test
bpf: Mark bpf_iter__sockmap::sk as PTR_TRUSTED
bpf: Restrict bpf_sock_destroy() to iter/{tcp,udp}
bpf, docs: Document BPF_ITER_NOSLEEP
selftests/bpf: Add negative test for sockmap update from non-sockmap iter
bpf, sockmap: Restrict iter-context sockmap updates to iter/sockmap
selftests/bpf: Remove sockmap update from bpf_iter_unix
bpf, lsm: Mark backing_file_alloc/free LSM hooks as untrusted
bpf: Report RCU-tasks quiescent states in do_check()
bpf: Check variable-offset accesses against special fields
bpf: Use dedicated workqueue for freeing BPF programs
...
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/bpf/bpf_lsm.c | 2 | ||||
| -rw-r--r-- | kernel/bpf/btf.c | 7 | ||||
| -rw-r--r-- | kernel/bpf/cgroup.c | 163 | ||||
| -rw-r--r-- | kernel/bpf/core.c | 20 | ||||
| -rw-r--r-- | kernel/bpf/local_storage.c | 8 | ||||
| -rw-r--r-- | kernel/bpf/map_iter.c | 12 | ||||
| -rw-r--r-- | kernel/bpf/trampoline.c | 2 | ||||
| -rw-r--r-- | kernel/bpf/verifier.c | 7 | ||||
| -rw-r--r-- | kernel/events/uprobes.c | 9 | ||||
| -rw-r--r-- | kernel/trace/bpf_trace.c | 10 |
10 files changed, 180 insertions, 60 deletions
diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 468e63dbd922..ae23511fd811 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -411,6 +411,8 @@ BTF_ID(func, bpf_lsm_bpf_map_free) BTF_ID(func, bpf_lsm_bpf_prog_free) BTF_ID(func, bpf_lsm_file_alloc_security) BTF_ID(func, bpf_lsm_file_free_security) +BTF_ID(func, bpf_lsm_backing_file_alloc) +BTF_ID(func, bpf_lsm_backing_file_free) BTF_ID(func, bpf_lsm_namespace_init) BTF_ID(func, bpf_lsm_namespace_free) #ifdef CONFIG_SECURITY_NETWORK diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 7e866bf017b5..49fcd32d418a 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8094,7 +8094,12 @@ int btf_struct_access(struct bpf_verifier_log *log, for (i = 0; i < rec->cnt; i++) { struct btf_field *field = &rec->fields[i]; u32 offset = field->offset; - if (off < offset + field->size && offset < off + size) { + /* An access at 'lo' or above ends after the start of the field. */ + s64 lo = max(min_off, (s64)offset - size + 1); + + /* Round up to min_off + k * step. The step is never 0. */ + lo += imod(min_off - lo, reg->step); + if (lo <= max_off && lo < offset + field->size) { bpf_log(log, "direct access to %s is disallowed\n", btf_field_type_name(field->type)); diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 57a785241d42..f195d5fcae6c 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -206,13 +206,53 @@ unsigned int __cgroup_bpf_run_lsm_current(const void *ctx, return ret; } +/* + * cgroup_bpf_mutex protects the bpf state of all cgroups: prog lists, + * effective arrays, flags, revisions, storage lists, link->cgroup, + * link->map of struct_ops links and cgroup_lsm_atype[]. + * + * Take cgroup_mutex first if you need both. This mutex doesn't keep a + * cgroup alive or the hierarchy stable. Attach, detach and replace walk + * the descendants, so they take both. Everything else takes only this one. + * + * cgroup_bpf_inherit() runs at mkdir with only cgroup_mutex. That's safe: + * it writes the new cgroup, which no one else can see yet, and what it + * reads from the ancestors changes only under cgroup_mutex. The ancestors + * are online, and only rmdir, which needs cgroup_mutex, starts a release. + */ +static DEFINE_MUTEX(cgroup_bpf_mutex); + +#define cgroup_bpf_held() lockdep_is_held(&cgroup_bpf_mutex) + +void cgroup_bpf_lock(void) +{ + mutex_lock(&cgroup_bpf_mutex); +} + +void cgroup_bpf_unlock(void) +{ + mutex_unlock(&cgroup_bpf_mutex); +} + +static void cgroup_bpf_lock_hierarchy(void) +{ + cgroup_lock(); + mutex_lock(&cgroup_bpf_mutex); +} + +static void cgroup_bpf_unlock_hierarchy(void) +{ + mutex_unlock(&cgroup_bpf_mutex); + cgroup_unlock(); +} + #ifdef CONFIG_BPF_LSM static enum cgroup_bpf_attach_type bpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id) { int i; - lockdep_assert_held(&cgroup_mutex); + lockdep_assert_held(&cgroup_bpf_mutex); if (attach_type != BPF_LSM_CGROUP) return to_cgroup_bpf_attach_type(attach_type); @@ -233,7 +273,7 @@ void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) { int i = cgroup_atype - CGROUP_LSM_START; - lockdep_assert_held(&cgroup_mutex); + lockdep_assert_held(&cgroup_bpf_mutex); if (!cgroup_lsm_atype[i].attach_btf_id) { cgroup_lsm_atype[i].attach_btf_id = attach_btf_id; @@ -249,13 +289,13 @@ void bpf_cgroup_atype_put(int cgroup_atype) { int i = cgroup_atype - CGROUP_LSM_START; - cgroup_lock(); + cgroup_bpf_lock(); if (--cgroup_lsm_atype[i].refcnt <= 0) { WRITE_ONCE(cgroup_lsm_atype[i].returns_errno, true); cgroup_lsm_atype[i].attach_btf_id = 0; } WARN_ON_ONCE(cgroup_lsm_atype[i].refcnt < 0); - cgroup_unlock(); + cgroup_bpf_unlock(); } #else static enum cgroup_bpf_attach_type @@ -424,7 +464,7 @@ static int bpf_cgroup_array_copy_to_user(struct bpf_prog_array *array, return -EFAULT; i++; } - return item->prog ? -ENOSPC : 0; + return 0; } static int bpf_cgroup_array_delete_safe_at(struct bpf_prog_array *array, @@ -459,7 +499,7 @@ static void cgroup_bpf_release(struct work_struct *work) unsigned int atype; - cgroup_lock(); + cgroup_bpf_lock(); for (atype = 0; atype < ARRAY_SIZE(cgrp->bpf.progs); atype++) { struct hlist_head *progs = &cgrp->bpf.progs[atype]; @@ -480,7 +520,7 @@ static void cgroup_bpf_release(struct work_struct *work) } old_array = rcu_dereference_protected( cgrp->bpf.effective[atype], - lockdep_is_held(&cgroup_mutex)); + cgroup_bpf_held()); bpf_cgroup_array_free(old_array, atype); } @@ -489,7 +529,7 @@ static void cgroup_bpf_release(struct work_struct *work) bpf_cgroup_storage_free(storage); } - cgroup_unlock(); + cgroup_bpf_unlock(); for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p)) cgroup_bpf_put(p); @@ -665,7 +705,9 @@ static void activate_effective_progs(struct cgroup *cgrp, enum cgroup_bpf_attach_type atype, struct bpf_prog_array *old_array) { + /* cgroup_bpf_inherit() holds only cgroup_mutex, see cgroup_bpf_mutex */ old_array = rcu_replace_pointer(cgrp->bpf.effective[atype], old_array, + lockdep_is_held(&cgroup_bpf_mutex) || lockdep_is_held(&cgroup_mutex)); /* free prog array after grace period, since __cgroup_bpf_run_*() * might be still walking the array @@ -687,6 +729,9 @@ static int cgroup_bpf_inherit(struct cgroup *cgrp) struct cgroup *p; int ret, i; + /* See the comment above cgroup_bpf_mutex */ + lockdep_assert_held(&cgroup_mutex); + ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0, GFP_KERNEL); if (ret) @@ -747,6 +792,8 @@ static int update_effective_progs(struct cgroup *cgrp, struct cgroup_subsys_state *css; int err; + lockdep_assert_held(&cgroup_bpf_mutex); + /* allocate and recompute effective prog arrays */ css_for_each_descendant_pre(css, &cgrp->self) { struct cgroup *desc = container_of(css, struct cgroup, self); @@ -954,7 +1001,7 @@ static int insert_pl_to_hlist(struct bpf_prog_list *pl, struct hlist_head *progs * @revision: bpf_prog_list revision * * Exactly one of @prog or @link can be non-null. - * Must be called with cgroup_mutex held. + * Must be called with cgroup_mutex and cgroup_bpf_mutex held. */ static int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog, struct bpf_prog *replace_prog, @@ -988,6 +1035,13 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, /* replace_prog implies BPF_F_REPLACE, and vice versa */ return -EINVAL; + /* + * The cgroup may have been removed since the caller looked it up. + * Then its bpf state may be freed, and nothing would detach this. + */ + if (cgroup_is_dead(cgrp)) + return -ENOENT; + atype = bpf_cgroup_atype_find(type, new_prog->aux->attach_btf_id); if (atype < 0) return -EINVAL; @@ -1093,10 +1147,10 @@ static int cgroup_bpf_attach(struct cgroup *cgrp, { int ret; - cgroup_lock(); + cgroup_bpf_lock_hierarchy(); ret = __cgroup_bpf_attach(cgrp, prog, replace_prog, link, type, flags, id_or_fd, revision); - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); return ret; } @@ -1174,7 +1228,7 @@ static void replace_effective_prog(struct cgroup *cgrp, progs = rcu_dereference_protected( desc->bpf.effective[atype], - lockdep_is_held(&cgroup_mutex)); + cgroup_bpf_held()); item = &progs->items[pos]; prog_list_replace_item(pl, item); } @@ -1202,7 +1256,7 @@ static bool cgroup_bpf_storages_compatible(struct bpf_prog *old_prog, * @new_prog: &struct bpf_prog for the target BPF program with its refcnt * incremented * - * Must be called with cgroup_mutex held. + * Must be called with cgroup_mutex and cgroup_bpf_mutex held. */ static int __cgroup_bpf_replace(struct cgroup *cgrp, struct bpf_cgroup_link *link, @@ -1250,7 +1304,7 @@ static int cgroup_bpf_replace(struct bpf_link *link, struct bpf_prog *new_prog, cg_link = container_of(link, struct bpf_cgroup_link, link); - cgroup_lock(); + cgroup_bpf_lock_hierarchy(); /* link might have been auto-released by dying cgroup, so fail */ if (!cg_link->cgroup) { ret = -ENOLINK; @@ -1262,7 +1316,7 @@ static int cgroup_bpf_replace(struct bpf_link *link, struct bpf_prog *new_prog, } ret = __cgroup_bpf_replace(cg_link->cgroup, cg_link, new_prog); out_unlock: - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); return ret; } @@ -1333,7 +1387,7 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl, progs = rcu_dereference_protected( desc->bpf.effective[atype], - lockdep_is_held(&cgroup_mutex)); + cgroup_bpf_held()); /* Remove the program from the array */ WARN_ONCE(bpf_cgroup_array_delete_safe_at(progs, pos, atype), @@ -1351,7 +1405,7 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl, * @revision: bpf_prog_list revision * * At most one of @prog or @link can be non-NULL. - * Must be called with cgroup_mutex held. + * Must be called with cgroup_mutex and cgroup_bpf_mutex held. */ static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, struct bpf_cgroup_link *link, enum bpf_attach_type type, @@ -1421,13 +1475,13 @@ static int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, { int ret; - cgroup_lock(); + cgroup_bpf_lock_hierarchy(); ret = __cgroup_bpf_detach(cgrp, prog, NULL, type, revision); - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); return ret; } -/* Must be called with cgroup_mutex held to avoid races. */ +/* Must be called with cgroup_bpf_mutex held. */ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, union bpf_attr __user *uattr, u32 uattr_size) { @@ -1446,6 +1500,14 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, if (effective_query && prog_attach_flags) return -EINVAL; + /* + * cgroup_bpf_release() frees the effective arrays once the bpf + * refcount is zero. It takes this lock too, so a nonzero count means + * the arrays are still valid. + */ + if (percpu_ref_is_zero(&cgrp->bpf.refcnt)) + return -ENOENT; + if (type == BPF_STRUCT_OPS) { u32 type_id = attr->query.type_id; @@ -1473,7 +1535,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, for (atype = from_atype; atype <= to_atype; atype++) { if (effective_query) { effective = rcu_dereference_protected(cgrp->bpf.effective[atype], - lockdep_is_held(&cgroup_mutex)); + cgroup_bpf_held()); total_cnt += bpf_cgroup_array_length(effective, atype); } else { total_cnt += prog_list_length(&cgrp->bpf.progs[atype], NULL); @@ -1503,9 +1565,10 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, for (atype = from_atype; atype <= to_atype && total_cnt; atype++) { if (effective_query) { effective = rcu_dereference_protected(cgrp->bpf.effective[atype], - lockdep_is_held(&cgroup_mutex)); + cgroup_bpf_held()); cnt = min_t(int, bpf_cgroup_array_length(effective, atype), total_cnt); - ret = bpf_cgroup_array_copy_to_user(effective, prog_ids, cnt, atype); + if (bpf_cgroup_array_copy_to_user(effective, prog_ids, cnt, atype)) + return -EFAULT; } else { struct hlist_head *progs; struct bpf_prog_list *pl; @@ -1544,9 +1607,9 @@ static int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, { int ret; - cgroup_lock(); + cgroup_bpf_lock(); ret = __cgroup_bpf_query(cgrp, attr, uattr, uattr_size); - cgroup_unlock(); + cgroup_bpf_unlock(); return ret; } @@ -1614,11 +1677,11 @@ static void bpf_cgroup_link_release(struct bpf_link *link) if (!cg_link->cgroup) return; - cgroup_lock(); + cgroup_bpf_lock_hierarchy(); /* re-check cgroup under lock again */ if (!cg_link->cgroup) { - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); return; } @@ -1630,7 +1693,7 @@ static void bpf_cgroup_link_release(struct bpf_link *link) cg = cg_link->cgroup; cg_link->cgroup = NULL; - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); cgroup_put(cg); } @@ -1657,10 +1720,10 @@ static void bpf_cgroup_link_show_fdinfo(const struct bpf_link *link, container_of(link, struct bpf_cgroup_link, link); u64 cg_id = 0; - cgroup_lock(); + cgroup_bpf_lock(); if (cg_link->cgroup) cg_id = cgroup_id(cg_link->cgroup); - cgroup_unlock(); + cgroup_bpf_unlock(); seq_printf(seq, "cgroup_id:\t%llu\n" @@ -1676,10 +1739,10 @@ static int bpf_cgroup_link_fill_link_info(const struct bpf_link *link, container_of(link, struct bpf_cgroup_link, link); u64 cg_id = 0; - cgroup_lock(); + cgroup_bpf_lock(); if (cg_link->cgroup) cg_id = cgroup_id(cg_link->cgroup); - cgroup_unlock(); + cgroup_bpf_unlock(); info->cgroup.cgroup_id = cg_id; info->cgroup.attach_type = link->attach_type; @@ -2956,11 +3019,11 @@ static int __cgroup_struct_ops_link_detach(struct bpf_link *link, bool wake_poll struct bpf_map *map; struct cgroup *cgrp; - cgroup_lock(); + cgroup_bpf_lock_hierarchy(); cgrp = cg_link->cgroup; if (!cgrp) { - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); return 0; } @@ -2972,7 +3035,7 @@ static int __cgroup_struct_ops_link_detach(struct bpf_link *link, bool wake_poll break; } if (WARN_ON_ONCE(!pl)) { - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); return -ENOENT; } @@ -2990,7 +3053,7 @@ static int __cgroup_struct_ops_link_detach(struct bpf_link *link, bool wake_poll kfree(pl); static_branch_dec(&cgroup_bpf_enabled_key[atype]); - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); return 0; } @@ -3013,15 +3076,15 @@ static void cgroup_struct_ops_link_show_fdinfo(const struct bpf_link *link, stru struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); - cgroup_lock(); + cgroup_bpf_lock(); if (!cg_link->cgroup) { - cgroup_unlock(); + cgroup_bpf_unlock(); return; } seq_printf(seq, "map_id:\t%u\n", cg_link->map->id); seq_printf(seq, "cgroup_id:\t%llu\n", cgroup_id(cg_link->cgroup)); - cgroup_unlock(); + cgroup_bpf_unlock(); } static int cgroup_struct_ops_link_fill_link_info(const struct bpf_link *link, @@ -3029,15 +3092,15 @@ static int cgroup_struct_ops_link_fill_link_info(const struct bpf_link *link, { struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); - cgroup_lock(); + cgroup_bpf_lock(); if (!cg_link->cgroup) { - cgroup_unlock(); + cgroup_bpf_unlock(); return 0; } info->struct_ops.map_id = cg_link->map->id; info->struct_ops.cgroup_id = cgroup_id(cg_link->cgroup); - cgroup_unlock(); + cgroup_bpf_unlock(); return 0; } @@ -3055,7 +3118,7 @@ static int cgroup_struct_ops_link_update(struct bpf_link *link, struct bpf_map * if (!bpf_struct_ops_valid_to_reg(new_map)) return -EINVAL; - cgroup_lock(); + cgroup_bpf_lock_hierarchy(); cgrp = cg_link->cgroup; if (!cgrp) { @@ -3088,7 +3151,7 @@ static int cgroup_struct_ops_link_update(struct bpf_link *link, struct bpf_map * cgrp->bpf.revisions[atype]++; out: - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); return err; } @@ -3160,7 +3223,13 @@ int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr atype = bpf_struct_ops_map_cgroup_atype(map); progs = &cgrp->bpf.progs[atype]; - cgroup_lock(); + cgroup_bpf_lock_hierarchy(); + + /* See the cgroup_is_dead() check in __cgroup_bpf_attach() */ + if (cgroup_is_dead(cgrp)) { + err = -ENOENT; + goto unlock; + } if (attr->link_create.cgroup.expected_revision && attr->link_create.cgroup.expected_revision != cgrp->bpf.revisions[atype]) { @@ -3197,12 +3266,12 @@ int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr cgrp->bpf.revisions[atype]++; static_branch_inc(&cgroup_bpf_enabled_key[atype]); - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); return bpf_link_settle(&link_primer); unlock: - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); cleanup: kfree(pl); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 0b8456f2dce9..0bfd23a41401 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -24,6 +24,7 @@ #include <linux/skbuff.h> #include <linux/static_call.h> #include <linux/vmalloc.h> +#include <linux/workqueue.h> #include <linux/prandom.h> #include <linux/bpf.h> #include <linux/btf.h> @@ -3143,6 +3144,17 @@ static void bpf_free_used_btfs(struct bpf_prog_aux *aux) kfree(aux->used_btfs); } +struct workqueue_struct *bpf_prog_free_wq __ro_after_init; + +static int __init bpf_prog_free_wq_init(void) +{ + bpf_prog_free_wq = alloc_workqueue("bpf_prog_free", WQ_PERCPU, 1); + if (!bpf_prog_free_wq) + panic("Failed to allocate BPF program free workqueue\n"); + return 0; +} +early_initcall(bpf_prog_free_wq_init); + static void bpf_prog_free_images(struct work_struct *work) { struct bpf_prog_aux *aux; @@ -3198,7 +3210,7 @@ static void bpf_prog_free_deferred(struct work_struct *work) /* The symbols are looked up under rcu_read_lock() */ INIT_RCU_WORK(&aux->rwork, bpf_prog_free_images); if (ksyms) - queue_rcu_work(system_percpu_wq, &aux->rwork); + queue_rcu_work(bpf_prog_free_wq, &aux->rwork); else bpf_prog_free_images(&aux->rwork.work); } @@ -3211,7 +3223,11 @@ void bpf_prog_free(struct bpf_prog *fp) bpf_prog_put(aux->dst_prog); bpf_token_put(aux->token); INIT_WORK(&aux->work, bpf_prog_free_deferred); - schedule_work(&aux->work); + /* + * Free BPF programs on a dedicated workqueue to avoid blocking or slowing + * down the system workqueue. + */ + queue_work(bpf_prog_free_wq, &aux->work); } /* RNG for unprivileged user space with separated state from prandom_u32(). */ diff --git a/kernel/bpf/local_storage.c b/kernel/bpf/local_storage.c index 83cd527a2542..2cfbddeb1c7e 100644 --- a/kernel/bpf/local_storage.c +++ b/kernel/bpf/local_storage.c @@ -345,14 +345,18 @@ static void cgroup_storage_map_free(struct bpf_map *_map) struct list_head *storages = &map->list; struct bpf_cgroup_storage *storage, *stmp; - cgroup_lock(); + /* + * Attach and cgroup_bpf_release() change the same cgroup storage + * lists under other maps' locks, so take cgroup_bpf_mutex. + */ + cgroup_bpf_lock(); list_for_each_entry_safe(storage, stmp, storages, list_map) { bpf_cgroup_storage_unlink(storage); bpf_cgroup_storage_free(storage); } - cgroup_unlock(); + cgroup_bpf_unlock(); WARN_ON(!RB_EMPTY_ROOT(&map->root)); WARN_ON(!list_empty(&map->list)); diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c index a58c148692e6..7a769e5f5397 100644 --- a/kernel/bpf/map_iter.c +++ b/kernel/bpf/map_iter.c @@ -99,8 +99,8 @@ static struct bpf_iter_reg bpf_map_reg_info = { /* * For an iterator that hands the live map value to the program as a writable - * buffer: the verifier knows nothing of the map, so the special fields of the - * value have to lie outside of what the program may write. + * buffer: the verifier knows nothing of the map, so read-only maps + * and special fields of the value have to be protected from writes. */ int bpf_iter_map_check_value_write(const struct bpf_prog *prog, const struct bpf_map *map) @@ -109,7 +109,13 @@ int bpf_iter_map_check_value_write(const struct bpf_prog *prog, const struct btf_field *field; u32 i; - if (IS_ERR_OR_NULL(rec) || !prog->aux->max_rdwr_write) + if (!prog->aux->max_rdwr_write) + return 0; + + if (map->map_flags & BPF_F_RDONLY_PROG) + return -EACCES; + + if (IS_ERR_OR_NULL(rec)) return 0; for (i = 0; i < rec->cnt; i++) { diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 21d460866dfe..b0052c7565d8 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -584,7 +584,7 @@ static void __bpf_tramp_image_put_rcu(struct rcu_head *rcu) im = container_of(rcu, struct bpf_tramp_image, rcu); INIT_WORK(&im->work, __bpf_tramp_image_put_deferred); - schedule_work(&im->work); + queue_work(bpf_prog_free_wq, &im->work); } /* callback, fexit step 2. Called after percpu_ref_kill confirms. */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5e4700fda66b..9b228ffbe3fe 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10526,6 +10526,7 @@ static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) { enum bpf_attach_type eatype = env->prog->expected_attach_type; enum bpf_prog_type type = resolve_prog_type(env->prog); + const char *afname = env->prog->aux->attach_func_name; if (func_id != BPF_FUNC_map_update_elem && func_id != BPF_FUNC_map_delete_elem) @@ -10536,7 +10537,8 @@ static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) */ switch (type) { case BPF_PROG_TYPE_TRACING: - if (eatype == BPF_TRACE_ITER) + if (eatype == BPF_TRACE_ITER && + !strcmp(afname + strlen(BPF_ITER_FUNC_PREFIX), "sockmap")) return true; break; case BPF_PROG_TYPE_SOCK_OPS: @@ -20627,8 +20629,7 @@ static int do_check(struct bpf_verifier_env *env) if (signal_pending(current)) return -EAGAIN; - if (need_resched()) - cond_resched(); + cond_resched_tasks_rcu_qs(); /* reset current history entry on each new instruction */ env->cur_hist_ent = NULL; diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index caf444762a27..c47b1ee266d5 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -2704,6 +2704,15 @@ void __weak arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vadd } /* + * Can a bpf prog write these bytes of pt_regs? An arch opts in and lists the + * registers that ptrace can set to any value. + */ +bool __weak arch_uprobe_regs_writable(int off, int size) +{ + return false; +} + +/* * Run handler and ask thread to singlestep. * Ensure all non-fatal signals cannot interrupt thread while it singlesteps. */ diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index d22e78e7f28d..1f35a18ef399 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -1397,8 +1397,16 @@ static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type if (off + size > sizeof(struct pt_regs)) return false; - if (type == BPF_WRITE) + if (type == BPF_WRITE) { + /* + * pt_regs also holds what the kernel restores on the way to + * user space: cs and flags on x86, pstate on arm64, status on + * riscv. A prog can write the registers of the task only. + */ + if (!arch_uprobe_regs_writable(off, size)) + return false; prog->aux->kprobe_write_ctx = true; + } return true; } |
