diff options
| author | Martin KaFai Lau <martin.lau@kernel.org> | 2026-09-17 13:05:34 -0700 |
|---|---|---|
| committer | Alexei Starovoitov <ast@kernel.org> | 2026-09-19 05:32:56 +0000 |
| commit | e176821b68d8022180ac7ecce25dff797e66ca97 (patch) | |
| tree | c42567710010ec86878106a6f9b9b85fa9cb80b9 /kernel | |
| parent | 2ef54525d2fc24224c0317e2e662307fcbbd1f9c (diff) | |
bpf: Add a few bpf_cgroup_array_* helper functions
In the upcoming patch, the array can store a struct_ops map.
The array could have a cfi_stubs acting as a dummy instead of
the dummy_bpf_prog. The array logic will need to skip the cfi_stubs
also in order to support storing struct_ops map in the array.
bpf_cgroup_array_length(), bpf_cgroup_array_copy_to_user(), and
bpf_cgroup_array_delete_safe_at() are added as a preparation work
to allow skipping the cfi_stubs in the upcoming patch. This patch
only skips the dummy_bpf_prog which is the same as the existing behavior.
The current bpf_prog_array_*() callers are changed to call the new
bpf_cgroup_array_*(). This is a no-op change.
Unlike bpf_prog_array_copy_to_user(), bpf_cgroup_array_copy_to_user()
does not need a temporary buffer. The cgroup caller already holds
cgroup_mutex and dereferences the effective array with
rcu_dereference_protected(), so it does not copy to userspace
from an RCU read-side critical section. Details in commit 0911287ce32b.
Another addition is the bpf_cgroup_array_free(). This prepares
the array to have a different rcu gp for the struct_ops use case,
for example, a struct_ops could have mix of sleepable ops and
non-sleepable ops. In this patch, bpf_cgroup_array_free() only
goes through the regular rcu gp. This is a no-op change also.
bpf_prog_dummy() is also added to return the global dummy_bpf_prog.
bpf_cgroup_array_dummy() is added to decide the sentinel based on atype.
It now always returns bpf_prog_dummy(). In the upcoming patch,
it can return a cfi_stubs if the atype belongs to a struct_ops.
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Signed-off-by: Amery Hung <ameryhung@gmail.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Reviewed-by: Emil Tsalapatis <emil@etsalapatis.com>
Link: https://patch.msgid.link/20260917200542.3689605-9-ameryhung@gmail.com
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/bpf/cgroup.c | 80 | ||||
| -rw-r--r-- | kernel/bpf/core.c | 5 |
2 files changed, 76 insertions, 9 deletions
diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 0ce58764caae..0c8f6f6be049 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -319,6 +319,68 @@ static void bpf_cgroup_link_auto_detach(struct bpf_cgroup_link *link) link->cgroup = NULL; } +static void bpf_cgroup_array_free(struct bpf_prog_array *array) +{ + if (!array || array == &bpf_empty_prog_array) + return; + kfree_rcu(array, rcu); +} + +static void *bpf_cgroup_array_dummy(enum cgroup_bpf_attach_type atype) +{ + return bpf_prog_dummy(); +} + +static int bpf_cgroup_array_length(struct bpf_prog_array *array, + enum cgroup_bpf_attach_type atype) +{ + struct bpf_prog_array_item *item; + int cnt = 0; + + for (item = array->items; item->prog; item++) { + if (item->prog != bpf_cgroup_array_dummy(atype)) + cnt++; + } + + return cnt; +} + +static int bpf_cgroup_array_copy_to_user(struct bpf_prog_array *array, + __u32 __user *prog_ids, int cnt, + enum cgroup_bpf_attach_type atype) +{ + struct bpf_prog_array_item *item; + int i = 0; + u32 id; + + for (item = array->items; item->prog && i < cnt; item++) { + if (item->prog == bpf_cgroup_array_dummy(atype)) + continue; + id = item->prog->aux->id; + if (copy_to_user(prog_ids + i, &id, sizeof(id))) + return -EFAULT; + i++; + } + return item->prog ? -ENOSPC : 0; +} + +static int bpf_cgroup_array_delete_safe_at(struct bpf_prog_array *array, + int index, enum cgroup_bpf_attach_type atype) +{ + struct bpf_prog_array_item *item; + + for (item = array->items; item->prog; item++) { + if (item->prog == bpf_cgroup_array_dummy(atype)) + continue; + if (!index) { + WRITE_ONCE(item->prog, bpf_cgroup_array_dummy(atype)); + return 0; + } + index--; + } + return -ENOENT; +} + /** * cgroup_bpf_release() - put references of all bpf programs and * release all cgroup bpf data @@ -356,7 +418,7 @@ static void cgroup_bpf_release(struct work_struct *work) old_array = rcu_dereference_protected( cgrp->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); - bpf_prog_array_free(old_array); + bpf_cgroup_array_free(old_array); } list_for_each_entry_safe(storage, stmp, storages, list_cg) { @@ -530,7 +592,7 @@ static void activate_effective_progs(struct cgroup *cgrp, /* free prog array after grace period, since __cgroup_bpf_run_*() * might be still walking the array */ - bpf_prog_array_free(old_array); + bpf_cgroup_array_free(old_array); } /** @@ -570,7 +632,7 @@ static int cgroup_bpf_inherit(struct cgroup *cgrp) return 0; cleanup: for (i = 0; i < NR; i++) - bpf_prog_array_free(arrays[i]); + bpf_cgroup_array_free(arrays[i]); for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p)) cgroup_bpf_put(p); @@ -625,7 +687,7 @@ static int update_effective_progs(struct cgroup *cgrp, if (percpu_ref_is_zero(&desc->bpf.refcnt)) { if (unlikely(desc->bpf.inactive)) { - bpf_prog_array_free(desc->bpf.inactive); + bpf_cgroup_array_free(desc->bpf.inactive); desc->bpf.inactive = NULL; } continue; @@ -644,7 +706,7 @@ cleanup: css_for_each_descendant_pre(css, &cgrp->self) { struct cgroup *desc = container_of(css, struct cgroup, self); - bpf_prog_array_free(desc->bpf.inactive); + bpf_cgroup_array_free(desc->bpf.inactive); desc->bpf.inactive = NULL; } @@ -1191,7 +1253,7 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl, lockdep_is_held(&cgroup_mutex)); /* Remove the program from the array */ - WARN_ONCE(bpf_prog_array_delete_safe_at(progs, pos), + WARN_ONCE(bpf_cgroup_array_delete_safe_at(progs, pos, atype), "Failed to purge a prog from array at index %d", pos); } } @@ -1321,7 +1383,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, if (effective_query) { effective = rcu_dereference_protected(cgrp->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); - total_cnt += bpf_prog_array_length(effective); + total_cnt += bpf_cgroup_array_length(effective, atype); } else { total_cnt += prog_list_length(&cgrp->bpf.progs[atype], NULL); } @@ -1351,8 +1413,8 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, if (effective_query) { effective = rcu_dereference_protected(cgrp->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); - cnt = min_t(int, bpf_prog_array_length(effective), total_cnt); - ret = bpf_prog_array_copy_to_user(effective, prog_ids, cnt); + cnt = min_t(int, bpf_cgroup_array_length(effective, atype), total_cnt); + ret = bpf_cgroup_array_copy_to_user(effective, prog_ids, cnt, atype); } else { struct hlist_head *progs; struct bpf_prog_list *pl; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 4e208cc94752..227211166dcc 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2784,6 +2784,11 @@ void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs) call_rcu_tasks_trace(&progs->rcu, __bpf_prog_array_free_sleepable_cb); } +struct bpf_prog *bpf_prog_dummy(void) +{ + return &dummy_bpf_prog.prog; +} + int bpf_prog_array_length(struct bpf_prog_array *array) { struct bpf_prog_array_item *item; |
