summaryrefslogtreecommitdiff
path: root/kernel
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2026-11-20 17:41:06 -0800
committerLinus Torvalds <torvalds@linux-foundation.org>2026-11-20 17:41:06 -0800
commit84afebd853437e44cb71f718be11d90e4c1a59dd (patch)
tree09dc2b20cfa9303bc824b928eb6c9a322ecb318a /kernel
parent05718d5c2907cb46fb27bb6580edb89769ea430a (diff)
parentd5ba86298b45e1a81c578c499e9a222a3b734d5d (diff)
Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf
Pull bpf fixes from Alexei Starovoitov: "Mostly sockmap fixes as promised, plus uprobe ctx rewrite via bpf and cgroup-bpf fixes. - Reject writable map iterators on read-only maps (Adethya Srinivasan) - Various sockmap fixes (Alexei Starovoitov): - Clear the redirect target before the sk_msg prog runs - Check that the redirected message is the cork - Leave sk_stream_wait_memory() once the socket is dead - Keep the rest of the cork after a redirect - Don't let an SK_PASS verdict cover the next message - Subtract all dropped bytes from the return value of sendmsg() - Clear the end mark of elements added by bpf_msg_push_data() and bpf_msg_pop_data() - Fix ring capacity invariant break in bpf_msg_push_data() (Bo Cao) - Report RCU-tasks quiescent states in do_check() to fix rcu_tasks stalls during a long verification (Breno Leitao) - Various sockmap fixes (Chris Mason): - Give no progs to a TCP listener - Check the target of an egress redirect before each send - Check the TCP state in recvmsg() - Reset the copy position when corking - Don't copy 0 bytes into a full cork - Use a dedicated workqueue for freeing BPF programs to fix a deadlock between pack_mutex waiters and the vmalloc purge work (Emil Tsalapatis) - Fix verifier model mismatch on a narrow read of __sk_buff.remote_port (Henry Martin) - Fix bpf-gcc CO-RE relocation in verifier_sockmap_mutate selftest (Jiayuan Chen) - Restrict uprobe ctx writes to the registers that ptrace can set (Jiri Olsa) - Various sockmap fixes (Josef Bacik): - Write sg.start when recvmsg() faults - Don't let an SK_DROP verdict stick - Clear the resume state of a stopped psock - Various fixes (Michal Luczaj): - Fix "unresolved symbol tcp_request_sock" build error with CONFIG_INET=n - Restrict iter-context sockmap updates to iter/sockmap - Restrict bpf_sock_destroy() to iter/{tcp,udp} - Mark bpf_iter__sockmap::sk as PTR_TRUSTED - Mark backing_file_alloc/free LSM hooks as untrusted (Sergei Nizovtsev) - Various cgroup-bpf fixes (Shakeel Butt): - Don't read freed effective arrays in BPF_PROG_QUERY - Don't attach to a removed cgroup - Protect cgroup_bpf state with its own mutex, so that queries and the release work don't wait for cgroup_mutex - Don't lose errors in effective BPF_LSM_CGROUP queries - Check variable-offset accesses against special fields (Shihuang Liu)" * tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: (38 commits) selftests/bpf: Test which ctx registers a uprobe prog can write bpf: Restrict uprobe ctx writes to the registers ptrace can set selftests/bpf: Fix bpf-gcc CO-RE relocation in verifier_sockmap_mutate bpf, cgroup: Don't lose errors in effective BPF_LSM_CGROUP queries bpf, cgroup: Protect cgroup_bpf state with its own mutex bpf, cgroup: Don't attach to a removed cgroup bpf, cgroup: Don't read freed effective arrays in BPF_PROG_QUERY bpf: Fix verifier model mismatch on __sk_buff.remote_port narrow read bpf: Fix "unresolved symbol tcp_request_sock" build error selftests/bpf: Add iter/sockmap bpf_sock_destroy() negative test bpf: Mark bpf_iter__sockmap::sk as PTR_TRUSTED bpf: Restrict bpf_sock_destroy() to iter/{tcp,udp} bpf, docs: Document BPF_ITER_NOSLEEP selftests/bpf: Add negative test for sockmap update from non-sockmap iter bpf, sockmap: Restrict iter-context sockmap updates to iter/sockmap selftests/bpf: Remove sockmap update from bpf_iter_unix bpf, lsm: Mark backing_file_alloc/free LSM hooks as untrusted bpf: Report RCU-tasks quiescent states in do_check() bpf: Check variable-offset accesses against special fields bpf: Use dedicated workqueue for freeing BPF programs ...
Diffstat (limited to 'kernel')
-rw-r--r--kernel/bpf/bpf_lsm.c2
-rw-r--r--kernel/bpf/btf.c7
-rw-r--r--kernel/bpf/cgroup.c163
-rw-r--r--kernel/bpf/core.c20
-rw-r--r--kernel/bpf/local_storage.c8
-rw-r--r--kernel/bpf/map_iter.c12
-rw-r--r--kernel/bpf/trampoline.c2
-rw-r--r--kernel/bpf/verifier.c7
-rw-r--r--kernel/events/uprobes.c9
-rw-r--r--kernel/trace/bpf_trace.c10
10 files changed, 180 insertions, 60 deletions
diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c
index 468e63dbd922..ae23511fd811 100644
--- a/kernel/bpf/bpf_lsm.c
+++ b/kernel/bpf/bpf_lsm.c
@@ -411,6 +411,8 @@ BTF_ID(func, bpf_lsm_bpf_map_free)
BTF_ID(func, bpf_lsm_bpf_prog_free)
BTF_ID(func, bpf_lsm_file_alloc_security)
BTF_ID(func, bpf_lsm_file_free_security)
+BTF_ID(func, bpf_lsm_backing_file_alloc)
+BTF_ID(func, bpf_lsm_backing_file_free)
BTF_ID(func, bpf_lsm_namespace_init)
BTF_ID(func, bpf_lsm_namespace_free)
#ifdef CONFIG_SECURITY_NETWORK
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 7e866bf017b5..49fcd32d418a 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -8094,7 +8094,12 @@ int btf_struct_access(struct bpf_verifier_log *log,
for (i = 0; i < rec->cnt; i++) {
struct btf_field *field = &rec->fields[i];
u32 offset = field->offset;
- if (off < offset + field->size && offset < off + size) {
+ /* An access at 'lo' or above ends after the start of the field. */
+ s64 lo = max(min_off, (s64)offset - size + 1);
+
+ /* Round up to min_off + k * step. The step is never 0. */
+ lo += imod(min_off - lo, reg->step);
+ if (lo <= max_off && lo < offset + field->size) {
bpf_log(log,
"direct access to %s is disallowed\n",
btf_field_type_name(field->type));
diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index 57a785241d42..f195d5fcae6c 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -206,13 +206,53 @@ unsigned int __cgroup_bpf_run_lsm_current(const void *ctx,
return ret;
}
+/*
+ * cgroup_bpf_mutex protects the bpf state of all cgroups: prog lists,
+ * effective arrays, flags, revisions, storage lists, link->cgroup,
+ * link->map of struct_ops links and cgroup_lsm_atype[].
+ *
+ * Take cgroup_mutex first if you need both. This mutex doesn't keep a
+ * cgroup alive or the hierarchy stable. Attach, detach and replace walk
+ * the descendants, so they take both. Everything else takes only this one.
+ *
+ * cgroup_bpf_inherit() runs at mkdir with only cgroup_mutex. That's safe:
+ * it writes the new cgroup, which no one else can see yet, and what it
+ * reads from the ancestors changes only under cgroup_mutex. The ancestors
+ * are online, and only rmdir, which needs cgroup_mutex, starts a release.
+ */
+static DEFINE_MUTEX(cgroup_bpf_mutex);
+
+#define cgroup_bpf_held() lockdep_is_held(&cgroup_bpf_mutex)
+
+void cgroup_bpf_lock(void)
+{
+ mutex_lock(&cgroup_bpf_mutex);
+}
+
+void cgroup_bpf_unlock(void)
+{
+ mutex_unlock(&cgroup_bpf_mutex);
+}
+
+static void cgroup_bpf_lock_hierarchy(void)
+{
+ cgroup_lock();
+ mutex_lock(&cgroup_bpf_mutex);
+}
+
+static void cgroup_bpf_unlock_hierarchy(void)
+{
+ mutex_unlock(&cgroup_bpf_mutex);
+ cgroup_unlock();
+}
+
#ifdef CONFIG_BPF_LSM
static enum cgroup_bpf_attach_type
bpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id)
{
int i;
- lockdep_assert_held(&cgroup_mutex);
+ lockdep_assert_held(&cgroup_bpf_mutex);
if (attach_type != BPF_LSM_CGROUP)
return to_cgroup_bpf_attach_type(attach_type);
@@ -233,7 +273,7 @@ void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype)
{
int i = cgroup_atype - CGROUP_LSM_START;
- lockdep_assert_held(&cgroup_mutex);
+ lockdep_assert_held(&cgroup_bpf_mutex);
if (!cgroup_lsm_atype[i].attach_btf_id) {
cgroup_lsm_atype[i].attach_btf_id = attach_btf_id;
@@ -249,13 +289,13 @@ void bpf_cgroup_atype_put(int cgroup_atype)
{
int i = cgroup_atype - CGROUP_LSM_START;
- cgroup_lock();
+ cgroup_bpf_lock();
if (--cgroup_lsm_atype[i].refcnt <= 0) {
WRITE_ONCE(cgroup_lsm_atype[i].returns_errno, true);
cgroup_lsm_atype[i].attach_btf_id = 0;
}
WARN_ON_ONCE(cgroup_lsm_atype[i].refcnt < 0);
- cgroup_unlock();
+ cgroup_bpf_unlock();
}
#else
static enum cgroup_bpf_attach_type
@@ -424,7 +464,7 @@ static int bpf_cgroup_array_copy_to_user(struct bpf_prog_array *array,
return -EFAULT;
i++;
}
- return item->prog ? -ENOSPC : 0;
+ return 0;
}
static int bpf_cgroup_array_delete_safe_at(struct bpf_prog_array *array,
@@ -459,7 +499,7 @@ static void cgroup_bpf_release(struct work_struct *work)
unsigned int atype;
- cgroup_lock();
+ cgroup_bpf_lock();
for (atype = 0; atype < ARRAY_SIZE(cgrp->bpf.progs); atype++) {
struct hlist_head *progs = &cgrp->bpf.progs[atype];
@@ -480,7 +520,7 @@ static void cgroup_bpf_release(struct work_struct *work)
}
old_array = rcu_dereference_protected(
cgrp->bpf.effective[atype],
- lockdep_is_held(&cgroup_mutex));
+ cgroup_bpf_held());
bpf_cgroup_array_free(old_array, atype);
}
@@ -489,7 +529,7 @@ static void cgroup_bpf_release(struct work_struct *work)
bpf_cgroup_storage_free(storage);
}
- cgroup_unlock();
+ cgroup_bpf_unlock();
for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))
cgroup_bpf_put(p);
@@ -665,7 +705,9 @@ static void activate_effective_progs(struct cgroup *cgrp,
enum cgroup_bpf_attach_type atype,
struct bpf_prog_array *old_array)
{
+ /* cgroup_bpf_inherit() holds only cgroup_mutex, see cgroup_bpf_mutex */
old_array = rcu_replace_pointer(cgrp->bpf.effective[atype], old_array,
+ lockdep_is_held(&cgroup_bpf_mutex) ||
lockdep_is_held(&cgroup_mutex));
/* free prog array after grace period, since __cgroup_bpf_run_*()
* might be still walking the array
@@ -687,6 +729,9 @@ static int cgroup_bpf_inherit(struct cgroup *cgrp)
struct cgroup *p;
int ret, i;
+ /* See the comment above cgroup_bpf_mutex */
+ lockdep_assert_held(&cgroup_mutex);
+
ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0,
GFP_KERNEL);
if (ret)
@@ -747,6 +792,8 @@ static int update_effective_progs(struct cgroup *cgrp,
struct cgroup_subsys_state *css;
int err;
+ lockdep_assert_held(&cgroup_bpf_mutex);
+
/* allocate and recompute effective prog arrays */
css_for_each_descendant_pre(css, &cgrp->self) {
struct cgroup *desc = container_of(css, struct cgroup, self);
@@ -954,7 +1001,7 @@ static int insert_pl_to_hlist(struct bpf_prog_list *pl, struct hlist_head *progs
* @revision: bpf_prog_list revision
*
* Exactly one of @prog or @link can be non-null.
- * Must be called with cgroup_mutex held.
+ * Must be called with cgroup_mutex and cgroup_bpf_mutex held.
*/
static int __cgroup_bpf_attach(struct cgroup *cgrp,
struct bpf_prog *prog, struct bpf_prog *replace_prog,
@@ -988,6 +1035,13 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp,
/* replace_prog implies BPF_F_REPLACE, and vice versa */
return -EINVAL;
+ /*
+ * The cgroup may have been removed since the caller looked it up.
+ * Then its bpf state may be freed, and nothing would detach this.
+ */
+ if (cgroup_is_dead(cgrp))
+ return -ENOENT;
+
atype = bpf_cgroup_atype_find(type, new_prog->aux->attach_btf_id);
if (atype < 0)
return -EINVAL;
@@ -1093,10 +1147,10 @@ static int cgroup_bpf_attach(struct cgroup *cgrp,
{
int ret;
- cgroup_lock();
+ cgroup_bpf_lock_hierarchy();
ret = __cgroup_bpf_attach(cgrp, prog, replace_prog, link, type, flags,
id_or_fd, revision);
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
return ret;
}
@@ -1174,7 +1228,7 @@ static void replace_effective_prog(struct cgroup *cgrp,
progs = rcu_dereference_protected(
desc->bpf.effective[atype],
- lockdep_is_held(&cgroup_mutex));
+ cgroup_bpf_held());
item = &progs->items[pos];
prog_list_replace_item(pl, item);
}
@@ -1202,7 +1256,7 @@ static bool cgroup_bpf_storages_compatible(struct bpf_prog *old_prog,
* @new_prog: &struct bpf_prog for the target BPF program with its refcnt
* incremented
*
- * Must be called with cgroup_mutex held.
+ * Must be called with cgroup_mutex and cgroup_bpf_mutex held.
*/
static int __cgroup_bpf_replace(struct cgroup *cgrp,
struct bpf_cgroup_link *link,
@@ -1250,7 +1304,7 @@ static int cgroup_bpf_replace(struct bpf_link *link, struct bpf_prog *new_prog,
cg_link = container_of(link, struct bpf_cgroup_link, link);
- cgroup_lock();
+ cgroup_bpf_lock_hierarchy();
/* link might have been auto-released by dying cgroup, so fail */
if (!cg_link->cgroup) {
ret = -ENOLINK;
@@ -1262,7 +1316,7 @@ static int cgroup_bpf_replace(struct bpf_link *link, struct bpf_prog *new_prog,
}
ret = __cgroup_bpf_replace(cg_link->cgroup, cg_link, new_prog);
out_unlock:
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
return ret;
}
@@ -1333,7 +1387,7 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl,
progs = rcu_dereference_protected(
desc->bpf.effective[atype],
- lockdep_is_held(&cgroup_mutex));
+ cgroup_bpf_held());
/* Remove the program from the array */
WARN_ONCE(bpf_cgroup_array_delete_safe_at(progs, pos, atype),
@@ -1351,7 +1405,7 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl,
* @revision: bpf_prog_list revision
*
* At most one of @prog or @link can be non-NULL.
- * Must be called with cgroup_mutex held.
+ * Must be called with cgroup_mutex and cgroup_bpf_mutex held.
*/
static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
struct bpf_cgroup_link *link, enum bpf_attach_type type,
@@ -1421,13 +1475,13 @@ static int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
{
int ret;
- cgroup_lock();
+ cgroup_bpf_lock_hierarchy();
ret = __cgroup_bpf_detach(cgrp, prog, NULL, type, revision);
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
return ret;
}
-/* Must be called with cgroup_mutex held to avoid races. */
+/* Must be called with cgroup_bpf_mutex held. */
static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
union bpf_attr __user *uattr, u32 uattr_size)
{
@@ -1446,6 +1500,14 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
if (effective_query && prog_attach_flags)
return -EINVAL;
+ /*
+ * cgroup_bpf_release() frees the effective arrays once the bpf
+ * refcount is zero. It takes this lock too, so a nonzero count means
+ * the arrays are still valid.
+ */
+ if (percpu_ref_is_zero(&cgrp->bpf.refcnt))
+ return -ENOENT;
+
if (type == BPF_STRUCT_OPS) {
u32 type_id = attr->query.type_id;
@@ -1473,7 +1535,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
for (atype = from_atype; atype <= to_atype; atype++) {
if (effective_query) {
effective = rcu_dereference_protected(cgrp->bpf.effective[atype],
- lockdep_is_held(&cgroup_mutex));
+ cgroup_bpf_held());
total_cnt += bpf_cgroup_array_length(effective, atype);
} else {
total_cnt += prog_list_length(&cgrp->bpf.progs[atype], NULL);
@@ -1503,9 +1565,10 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
for (atype = from_atype; atype <= to_atype && total_cnt; atype++) {
if (effective_query) {
effective = rcu_dereference_protected(cgrp->bpf.effective[atype],
- lockdep_is_held(&cgroup_mutex));
+ cgroup_bpf_held());
cnt = min_t(int, bpf_cgroup_array_length(effective, atype), total_cnt);
- ret = bpf_cgroup_array_copy_to_user(effective, prog_ids, cnt, atype);
+ if (bpf_cgroup_array_copy_to_user(effective, prog_ids, cnt, atype))
+ return -EFAULT;
} else {
struct hlist_head *progs;
struct bpf_prog_list *pl;
@@ -1544,9 +1607,9 @@ static int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
{
int ret;
- cgroup_lock();
+ cgroup_bpf_lock();
ret = __cgroup_bpf_query(cgrp, attr, uattr, uattr_size);
- cgroup_unlock();
+ cgroup_bpf_unlock();
return ret;
}
@@ -1614,11 +1677,11 @@ static void bpf_cgroup_link_release(struct bpf_link *link)
if (!cg_link->cgroup)
return;
- cgroup_lock();
+ cgroup_bpf_lock_hierarchy();
/* re-check cgroup under lock again */
if (!cg_link->cgroup) {
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
return;
}
@@ -1630,7 +1693,7 @@ static void bpf_cgroup_link_release(struct bpf_link *link)
cg = cg_link->cgroup;
cg_link->cgroup = NULL;
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
cgroup_put(cg);
}
@@ -1657,10 +1720,10 @@ static void bpf_cgroup_link_show_fdinfo(const struct bpf_link *link,
container_of(link, struct bpf_cgroup_link, link);
u64 cg_id = 0;
- cgroup_lock();
+ cgroup_bpf_lock();
if (cg_link->cgroup)
cg_id = cgroup_id(cg_link->cgroup);
- cgroup_unlock();
+ cgroup_bpf_unlock();
seq_printf(seq,
"cgroup_id:\t%llu\n"
@@ -1676,10 +1739,10 @@ static int bpf_cgroup_link_fill_link_info(const struct bpf_link *link,
container_of(link, struct bpf_cgroup_link, link);
u64 cg_id = 0;
- cgroup_lock();
+ cgroup_bpf_lock();
if (cg_link->cgroup)
cg_id = cgroup_id(cg_link->cgroup);
- cgroup_unlock();
+ cgroup_bpf_unlock();
info->cgroup.cgroup_id = cg_id;
info->cgroup.attach_type = link->attach_type;
@@ -2956,11 +3019,11 @@ static int __cgroup_struct_ops_link_detach(struct bpf_link *link, bool wake_poll
struct bpf_map *map;
struct cgroup *cgrp;
- cgroup_lock();
+ cgroup_bpf_lock_hierarchy();
cgrp = cg_link->cgroup;
if (!cgrp) {
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
return 0;
}
@@ -2972,7 +3035,7 @@ static int __cgroup_struct_ops_link_detach(struct bpf_link *link, bool wake_poll
break;
}
if (WARN_ON_ONCE(!pl)) {
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
return -ENOENT;
}
@@ -2990,7 +3053,7 @@ static int __cgroup_struct_ops_link_detach(struct bpf_link *link, bool wake_poll
kfree(pl);
static_branch_dec(&cgroup_bpf_enabled_key[atype]);
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
return 0;
}
@@ -3013,15 +3076,15 @@ static void cgroup_struct_ops_link_show_fdinfo(const struct bpf_link *link, stru
struct bpf_cgroup_link *cg_link =
container_of(link, struct bpf_cgroup_link, link);
- cgroup_lock();
+ cgroup_bpf_lock();
if (!cg_link->cgroup) {
- cgroup_unlock();
+ cgroup_bpf_unlock();
return;
}
seq_printf(seq, "map_id:\t%u\n", cg_link->map->id);
seq_printf(seq, "cgroup_id:\t%llu\n", cgroup_id(cg_link->cgroup));
- cgroup_unlock();
+ cgroup_bpf_unlock();
}
static int cgroup_struct_ops_link_fill_link_info(const struct bpf_link *link,
@@ -3029,15 +3092,15 @@ static int cgroup_struct_ops_link_fill_link_info(const struct bpf_link *link,
{
struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link);
- cgroup_lock();
+ cgroup_bpf_lock();
if (!cg_link->cgroup) {
- cgroup_unlock();
+ cgroup_bpf_unlock();
return 0;
}
info->struct_ops.map_id = cg_link->map->id;
info->struct_ops.cgroup_id = cgroup_id(cg_link->cgroup);
- cgroup_unlock();
+ cgroup_bpf_unlock();
return 0;
}
@@ -3055,7 +3118,7 @@ static int cgroup_struct_ops_link_update(struct bpf_link *link, struct bpf_map *
if (!bpf_struct_ops_valid_to_reg(new_map))
return -EINVAL;
- cgroup_lock();
+ cgroup_bpf_lock_hierarchy();
cgrp = cg_link->cgroup;
if (!cgrp) {
@@ -3088,7 +3151,7 @@ static int cgroup_struct_ops_link_update(struct bpf_link *link, struct bpf_map *
cgrp->bpf.revisions[atype]++;
out:
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
return err;
}
@@ -3160,7 +3223,13 @@ int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr
atype = bpf_struct_ops_map_cgroup_atype(map);
progs = &cgrp->bpf.progs[atype];
- cgroup_lock();
+ cgroup_bpf_lock_hierarchy();
+
+ /* See the cgroup_is_dead() check in __cgroup_bpf_attach() */
+ if (cgroup_is_dead(cgrp)) {
+ err = -ENOENT;
+ goto unlock;
+ }
if (attr->link_create.cgroup.expected_revision &&
attr->link_create.cgroup.expected_revision != cgrp->bpf.revisions[atype]) {
@@ -3197,12 +3266,12 @@ int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr
cgrp->bpf.revisions[atype]++;
static_branch_inc(&cgroup_bpf_enabled_key[atype]);
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
return bpf_link_settle(&link_primer);
unlock:
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
cleanup:
kfree(pl);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 0b8456f2dce9..0bfd23a41401 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -24,6 +24,7 @@
#include <linux/skbuff.h>
#include <linux/static_call.h>
#include <linux/vmalloc.h>
+#include <linux/workqueue.h>
#include <linux/prandom.h>
#include <linux/bpf.h>
#include <linux/btf.h>
@@ -3143,6 +3144,17 @@ static void bpf_free_used_btfs(struct bpf_prog_aux *aux)
kfree(aux->used_btfs);
}
+struct workqueue_struct *bpf_prog_free_wq __ro_after_init;
+
+static int __init bpf_prog_free_wq_init(void)
+{
+ bpf_prog_free_wq = alloc_workqueue("bpf_prog_free", WQ_PERCPU, 1);
+ if (!bpf_prog_free_wq)
+ panic("Failed to allocate BPF program free workqueue\n");
+ return 0;
+}
+early_initcall(bpf_prog_free_wq_init);
+
static void bpf_prog_free_images(struct work_struct *work)
{
struct bpf_prog_aux *aux;
@@ -3198,7 +3210,7 @@ static void bpf_prog_free_deferred(struct work_struct *work)
/* The symbols are looked up under rcu_read_lock() */
INIT_RCU_WORK(&aux->rwork, bpf_prog_free_images);
if (ksyms)
- queue_rcu_work(system_percpu_wq, &aux->rwork);
+ queue_rcu_work(bpf_prog_free_wq, &aux->rwork);
else
bpf_prog_free_images(&aux->rwork.work);
}
@@ -3211,7 +3223,11 @@ void bpf_prog_free(struct bpf_prog *fp)
bpf_prog_put(aux->dst_prog);
bpf_token_put(aux->token);
INIT_WORK(&aux->work, bpf_prog_free_deferred);
- schedule_work(&aux->work);
+ /*
+ * Free BPF programs on a dedicated workqueue to avoid blocking or slowing
+ * down the system workqueue.
+ */
+ queue_work(bpf_prog_free_wq, &aux->work);
}
/* RNG for unprivileged user space with separated state from prandom_u32(). */
diff --git a/kernel/bpf/local_storage.c b/kernel/bpf/local_storage.c
index 83cd527a2542..2cfbddeb1c7e 100644
--- a/kernel/bpf/local_storage.c
+++ b/kernel/bpf/local_storage.c
@@ -345,14 +345,18 @@ static void cgroup_storage_map_free(struct bpf_map *_map)
struct list_head *storages = &map->list;
struct bpf_cgroup_storage *storage, *stmp;
- cgroup_lock();
+ /*
+ * Attach and cgroup_bpf_release() change the same cgroup storage
+ * lists under other maps' locks, so take cgroup_bpf_mutex.
+ */
+ cgroup_bpf_lock();
list_for_each_entry_safe(storage, stmp, storages, list_map) {
bpf_cgroup_storage_unlink(storage);
bpf_cgroup_storage_free(storage);
}
- cgroup_unlock();
+ cgroup_bpf_unlock();
WARN_ON(!RB_EMPTY_ROOT(&map->root));
WARN_ON(!list_empty(&map->list));
diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c
index a58c148692e6..7a769e5f5397 100644
--- a/kernel/bpf/map_iter.c
+++ b/kernel/bpf/map_iter.c
@@ -99,8 +99,8 @@ static struct bpf_iter_reg bpf_map_reg_info = {
/*
* For an iterator that hands the live map value to the program as a writable
- * buffer: the verifier knows nothing of the map, so the special fields of the
- * value have to lie outside of what the program may write.
+ * buffer: the verifier knows nothing of the map, so read-only maps
+ * and special fields of the value have to be protected from writes.
*/
int bpf_iter_map_check_value_write(const struct bpf_prog *prog,
const struct bpf_map *map)
@@ -109,7 +109,13 @@ int bpf_iter_map_check_value_write(const struct bpf_prog *prog,
const struct btf_field *field;
u32 i;
- if (IS_ERR_OR_NULL(rec) || !prog->aux->max_rdwr_write)
+ if (!prog->aux->max_rdwr_write)
+ return 0;
+
+ if (map->map_flags & BPF_F_RDONLY_PROG)
+ return -EACCES;
+
+ if (IS_ERR_OR_NULL(rec))
return 0;
for (i = 0; i < rec->cnt; i++) {
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 21d460866dfe..b0052c7565d8 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -584,7 +584,7 @@ static void __bpf_tramp_image_put_rcu(struct rcu_head *rcu)
im = container_of(rcu, struct bpf_tramp_image, rcu);
INIT_WORK(&im->work, __bpf_tramp_image_put_deferred);
- schedule_work(&im->work);
+ queue_work(bpf_prog_free_wq, &im->work);
}
/* callback, fexit step 2. Called after percpu_ref_kill confirms. */
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 5e4700fda66b..9b228ffbe3fe 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -10526,6 +10526,7 @@ static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
{
enum bpf_attach_type eatype = env->prog->expected_attach_type;
enum bpf_prog_type type = resolve_prog_type(env->prog);
+ const char *afname = env->prog->aux->attach_func_name;
if (func_id != BPF_FUNC_map_update_elem &&
func_id != BPF_FUNC_map_delete_elem)
@@ -10536,7 +10537,8 @@ static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
*/
switch (type) {
case BPF_PROG_TYPE_TRACING:
- if (eatype == BPF_TRACE_ITER)
+ if (eatype == BPF_TRACE_ITER &&
+ !strcmp(afname + strlen(BPF_ITER_FUNC_PREFIX), "sockmap"))
return true;
break;
case BPF_PROG_TYPE_SOCK_OPS:
@@ -20627,8 +20629,7 @@ static int do_check(struct bpf_verifier_env *env)
if (signal_pending(current))
return -EAGAIN;
- if (need_resched())
- cond_resched();
+ cond_resched_tasks_rcu_qs();
/* reset current history entry on each new instruction */
env->cur_hist_ent = NULL;
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index caf444762a27..c47b1ee266d5 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -2704,6 +2704,15 @@ void __weak arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vadd
}
/*
+ * Can a bpf prog write these bytes of pt_regs? An arch opts in and lists the
+ * registers that ptrace can set to any value.
+ */
+bool __weak arch_uprobe_regs_writable(int off, int size)
+{
+ return false;
+}
+
+/*
* Run handler and ask thread to singlestep.
* Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
*/
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index d22e78e7f28d..1f35a18ef399 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -1397,8 +1397,16 @@ static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type
if (off + size > sizeof(struct pt_regs))
return false;
- if (type == BPF_WRITE)
+ if (type == BPF_WRITE) {
+ /*
+ * pt_regs also holds what the kernel restores on the way to
+ * user space: cs and flags on x86, pstate on arm64, status on
+ * riscv. A prog can write the registers of the task only.
+ */
+ if (!arch_uprobe_regs_writable(off, size))
+ return false;
prog->aux->kprobe_write_ctx = true;
+ }
return true;
}