summaryrefslogtreecommitdiff
path: root/kernel
diff options
context:
space:
mode:
authorMartin KaFai Lau <martin.lau@kernel.org>2026-09-17 13:05:34 -0700
committerAlexei Starovoitov <ast@kernel.org>2026-09-19 05:32:56 +0000
commite176821b68d8022180ac7ecce25dff797e66ca97 (patch)
treec42567710010ec86878106a6f9b9b85fa9cb80b9 /kernel
parent2ef54525d2fc24224c0317e2e662307fcbbd1f9c (diff)
bpf: Add a few bpf_cgroup_array_* helper functions
In the upcoming patch, the array can store a struct_ops map. The array could have a cfi_stubs acting as a dummy instead of the dummy_bpf_prog. The array logic will need to skip the cfi_stubs also in order to support storing struct_ops map in the array. bpf_cgroup_array_length(), bpf_cgroup_array_copy_to_user(), and bpf_cgroup_array_delete_safe_at() are added as a preparation work to allow skipping the cfi_stubs in the upcoming patch. This patch only skips the dummy_bpf_prog which is the same as the existing behavior. The current bpf_prog_array_*() callers are changed to call the new bpf_cgroup_array_*(). This is a no-op change. Unlike bpf_prog_array_copy_to_user(), bpf_cgroup_array_copy_to_user() does not need a temporary buffer. The cgroup caller already holds cgroup_mutex and dereferences the effective array with rcu_dereference_protected(), so it does not copy to userspace from an RCU read-side critical section. Details in commit 0911287ce32b. Another addition is the bpf_cgroup_array_free(). This prepares the array to have a different rcu gp for the struct_ops use case, for example, a struct_ops could have mix of sleepable ops and non-sleepable ops. In this patch, bpf_cgroup_array_free() only goes through the regular rcu gp. This is a no-op change also. bpf_prog_dummy() is also added to return the global dummy_bpf_prog. bpf_cgroup_array_dummy() is added to decide the sentinel based on atype. It now always returns bpf_prog_dummy(). In the upcoming patch, it can return a cfi_stubs if the atype belongs to a struct_ops. Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org> Signed-off-by: Amery Hung <ameryhung@gmail.com> Signed-off-by: Alexei Starovoitov <ast@kernel.org> Reviewed-by: Emil Tsalapatis <emil@etsalapatis.com> Link: https://patch.msgid.link/20260917200542.3689605-9-ameryhung@gmail.com
Diffstat (limited to 'kernel')
-rw-r--r--kernel/bpf/cgroup.c80
-rw-r--r--kernel/bpf/core.c5
2 files changed, 76 insertions, 9 deletions
diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index 0ce58764caae..0c8f6f6be049 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -319,6 +319,68 @@ static void bpf_cgroup_link_auto_detach(struct bpf_cgroup_link *link)
link->cgroup = NULL;
}
+static void bpf_cgroup_array_free(struct bpf_prog_array *array)
+{
+ if (!array || array == &bpf_empty_prog_array)
+ return;
+ kfree_rcu(array, rcu);
+}
+
+static void *bpf_cgroup_array_dummy(enum cgroup_bpf_attach_type atype)
+{
+ return bpf_prog_dummy();
+}
+
+static int bpf_cgroup_array_length(struct bpf_prog_array *array,
+ enum cgroup_bpf_attach_type atype)
+{
+ struct bpf_prog_array_item *item;
+ int cnt = 0;
+
+ for (item = array->items; item->prog; item++) {
+ if (item->prog != bpf_cgroup_array_dummy(atype))
+ cnt++;
+ }
+
+ return cnt;
+}
+
+static int bpf_cgroup_array_copy_to_user(struct bpf_prog_array *array,
+ __u32 __user *prog_ids, int cnt,
+ enum cgroup_bpf_attach_type atype)
+{
+ struct bpf_prog_array_item *item;
+ int i = 0;
+ u32 id;
+
+ for (item = array->items; item->prog && i < cnt; item++) {
+ if (item->prog == bpf_cgroup_array_dummy(atype))
+ continue;
+ id = item->prog->aux->id;
+ if (copy_to_user(prog_ids + i, &id, sizeof(id)))
+ return -EFAULT;
+ i++;
+ }
+ return item->prog ? -ENOSPC : 0;
+}
+
+static int bpf_cgroup_array_delete_safe_at(struct bpf_prog_array *array,
+ int index, enum cgroup_bpf_attach_type atype)
+{
+ struct bpf_prog_array_item *item;
+
+ for (item = array->items; item->prog; item++) {
+ if (item->prog == bpf_cgroup_array_dummy(atype))
+ continue;
+ if (!index) {
+ WRITE_ONCE(item->prog, bpf_cgroup_array_dummy(atype));
+ return 0;
+ }
+ index--;
+ }
+ return -ENOENT;
+}
+
/**
* cgroup_bpf_release() - put references of all bpf programs and
* release all cgroup bpf data
@@ -356,7 +418,7 @@ static void cgroup_bpf_release(struct work_struct *work)
old_array = rcu_dereference_protected(
cgrp->bpf.effective[atype],
lockdep_is_held(&cgroup_mutex));
- bpf_prog_array_free(old_array);
+ bpf_cgroup_array_free(old_array);
}
list_for_each_entry_safe(storage, stmp, storages, list_cg) {
@@ -530,7 +592,7 @@ static void activate_effective_progs(struct cgroup *cgrp,
/* free prog array after grace period, since __cgroup_bpf_run_*()
* might be still walking the array
*/
- bpf_prog_array_free(old_array);
+ bpf_cgroup_array_free(old_array);
}
/**
@@ -570,7 +632,7 @@ static int cgroup_bpf_inherit(struct cgroup *cgrp)
return 0;
cleanup:
for (i = 0; i < NR; i++)
- bpf_prog_array_free(arrays[i]);
+ bpf_cgroup_array_free(arrays[i]);
for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))
cgroup_bpf_put(p);
@@ -625,7 +687,7 @@ static int update_effective_progs(struct cgroup *cgrp,
if (percpu_ref_is_zero(&desc->bpf.refcnt)) {
if (unlikely(desc->bpf.inactive)) {
- bpf_prog_array_free(desc->bpf.inactive);
+ bpf_cgroup_array_free(desc->bpf.inactive);
desc->bpf.inactive = NULL;
}
continue;
@@ -644,7 +706,7 @@ cleanup:
css_for_each_descendant_pre(css, &cgrp->self) {
struct cgroup *desc = container_of(css, struct cgroup, self);
- bpf_prog_array_free(desc->bpf.inactive);
+ bpf_cgroup_array_free(desc->bpf.inactive);
desc->bpf.inactive = NULL;
}
@@ -1191,7 +1253,7 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl,
lockdep_is_held(&cgroup_mutex));
/* Remove the program from the array */
- WARN_ONCE(bpf_prog_array_delete_safe_at(progs, pos),
+ WARN_ONCE(bpf_cgroup_array_delete_safe_at(progs, pos, atype),
"Failed to purge a prog from array at index %d", pos);
}
}
@@ -1321,7 +1383,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
if (effective_query) {
effective = rcu_dereference_protected(cgrp->bpf.effective[atype],
lockdep_is_held(&cgroup_mutex));
- total_cnt += bpf_prog_array_length(effective);
+ total_cnt += bpf_cgroup_array_length(effective, atype);
} else {
total_cnt += prog_list_length(&cgrp->bpf.progs[atype], NULL);
}
@@ -1351,8 +1413,8 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
if (effective_query) {
effective = rcu_dereference_protected(cgrp->bpf.effective[atype],
lockdep_is_held(&cgroup_mutex));
- cnt = min_t(int, bpf_prog_array_length(effective), total_cnt);
- ret = bpf_prog_array_copy_to_user(effective, prog_ids, cnt);
+ cnt = min_t(int, bpf_cgroup_array_length(effective, atype), total_cnt);
+ ret = bpf_cgroup_array_copy_to_user(effective, prog_ids, cnt, atype);
} else {
struct hlist_head *progs;
struct bpf_prog_list *pl;
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 4e208cc94752..227211166dcc 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -2784,6 +2784,11 @@ void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs)
call_rcu_tasks_trace(&progs->rcu, __bpf_prog_array_free_sleepable_cb);
}
+struct bpf_prog *bpf_prog_dummy(void)
+{
+ return &dummy_bpf_prog.prog;
+}
+
int bpf_prog_array_length(struct bpf_prog_array *array)
{
struct bpf_prog_array_item *item;