summaryrefslogtreecommitdiff
path: root/kernel/bpf/stackmap.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/bpf/stackmap.c')
-rw-r--r--kernel/bpf/stackmap.c351
1 files changed, 227 insertions, 124 deletions
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index 41fe87d7302f..a839041e0d00 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -414,8 +414,7 @@ static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *i
static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
u32 trace_nr, bool user, bool may_fault)
{
- struct mmap_unlock_irq_work *work = NULL;
- bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work);
+ struct mmap_unlock_irq_work *work;
bool has_user_ctx = user && current && current->mm;
struct stack_map_build_id_cache cache = {};
struct vm_area_struct *vma;
@@ -426,15 +425,16 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
return;
}
- /* If the irq_work is in use, fall back to report ips. Same
- * fallback is used for kernel stack (!user) on a stackmap with
- * build_id.
- */
- if (!has_user_ctx || irq_work_busy || !mmap_read_trylock(current->mm)) {
- /* cannot access current->mm, fall back to ips */
- for (i = 0; i < trace_nr; i++)
- stack_map_build_id_set_ip(&id_offs[i]);
- return;
+ if (!has_user_ctx)
+ goto fallback;
+
+ work = bpf_mmap_unlock_guard_get();
+ if (IS_ERR(work))
+ goto fallback;
+
+ if (!mmap_read_trylock(current->mm)) {
+ bpf_mmap_unlock_guard_put(work);
+ goto fallback;
}
for (i = 0; i < trace_nr; i++) {
@@ -465,6 +465,12 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
vma->vm_pgoff);
}
bpf_mmap_unlock_mm(work, current->mm);
+ return;
+
+fallback:
+ /* cannot access current->mm, fall back to ips */
+ for (i = 0; i < trace_nr; i++)
+ stack_map_build_id_set_ip(&id_offs[i]);
}
static struct perf_callchain_entry *
@@ -504,78 +510,116 @@ get_callchain_entry_for_task(struct task_struct *task, u32 max_depth)
#endif
}
-static long __bpf_get_stackid(struct bpf_map *map,
- struct perf_callchain_entry *trace, u64 flags)
+struct stackid {
+ struct stack_map_bucket *bucket;
+ const u64 *ips;
+ u32 nr;
+ u32 len;
+ u32 hash;
+ u32 id;
+ bool hash_matches;
+};
+
+static int stackid_init(struct stackid *stackid, struct bpf_map *map,
+ const struct perf_callchain_entry *trace, u32 trace_nr, u64 flags)
{
struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
- struct stack_map_bucket *bucket, *new_bucket, *old_bucket;
- u32 hash, id, trace_nr, trace_len, i, max_depth;
u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
- bool user = flags & BPF_F_USER_STACK;
- u64 *ips;
- bool hash_matches;
+ u32 max_depth;
- if (trace->nr <= skip)
+ if (trace_nr <= skip)
/* skipping more than usable stack trace */
return -EFAULT;
max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags);
- trace_nr = min_t(u32, trace->nr - skip, max_depth - skip);
- trace_len = trace_nr * sizeof(u64);
- ips = trace->ip + skip;
- hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0);
- id = hash & (smap->n_buckets - 1);
- bucket = READ_ONCE(smap->buckets[id]);
+ stackid->nr = min_t(u32, trace_nr - skip, max_depth - skip);
+ stackid->len = stackid->nr * sizeof(u64);
+ stackid->ips = trace->ip + skip;
+ stackid->hash = jhash2((const u32 *)stackid->ips, stackid->len / sizeof(u32), 0);
+ stackid->id = stackid->hash & (smap->n_buckets - 1);
+ stackid->bucket = READ_ONCE(smap->buckets[stackid->id]);
+ stackid->hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash;
+ return 0;
+}
+
+static int stackid_fastpath(struct stackid *stackid, struct bpf_map *map,
+ const struct perf_callchain_entry *trace, u32 trace_nr,
+ u64 flags)
+{
+ int err;
+
+ err = stackid_init(stackid, map, trace, trace_nr, flags);
+ if (err)
+ return err;
- hash_matches = bucket && bucket->hash == hash;
/* fast cmp */
- if (hash_matches && flags & BPF_F_FAST_STACK_CMP)
- return id;
+ if (stackid->hash_matches && flags & BPF_F_FAST_STACK_CMP)
+ return stackid->id;
+
+ if (stack_map_use_build_id(map))
+ return -ENOENT;
+ if (stackid->hash_matches && stackid->bucket->nr == stackid->nr &&
+ memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0)
+ return stackid->id;
+ if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID))
+ return -EEXIST;
+ return -ENOENT;
+}
+
+static struct stack_map_bucket *
+stackid_new_bucket(struct stackid *stackid, struct bpf_map *map)
+{
+ struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
+ struct bpf_stack_build_id *id_offs;
+ struct stack_map_bucket *bucket;
+ u32 i;
+
+ bucket = (struct stack_map_bucket *) pcpu_freelist_pop(&smap->freelist);
+ if (unlikely(!bucket))
+ return NULL;
+
+ if (stack_map_use_build_id(map)) {
+ id_offs = (struct bpf_stack_build_id *)bucket->data;
+ for (i = 0; i < stackid->nr; i++)
+ id_offs[i].ip = stackid->ips[i];
+ } else {
+ memcpy(bucket->data, stackid->ips, stackid->len);
+ }
+
+ bucket->hash = stackid->hash;
+ bucket->nr = stackid->nr;
+ return bucket;
+}
+
+static long stackid_install(struct stackid *stackid, struct bpf_map *map,
+ struct stack_map_bucket *new_bucket, u64 flags)
+{
+ struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
+ bool user = flags & BPF_F_USER_STACK;
+ struct stack_map_bucket *old_bucket;
+ u32 trace_len;
if (stack_map_use_build_id(map)) {
struct bpf_stack_build_id *id_offs;
- /* for build_id+offset, pop a bucket before slow cmp */
- new_bucket = (struct stack_map_bucket *)
- pcpu_freelist_pop(&smap->freelist);
- if (unlikely(!new_bucket))
- return -ENOMEM;
- new_bucket->nr = trace_nr;
id_offs = (struct bpf_stack_build_id *)new_bucket->data;
- for (i = 0; i < trace_nr; i++)
- id_offs[i].ip = ips[i];
- stack_map_get_build_id_offset(id_offs, trace_nr, user, false /* !may_fault */);
- trace_len = trace_nr * sizeof(struct bpf_stack_build_id);
- if (hash_matches && bucket->nr == trace_nr &&
- memcmp(bucket->data, new_bucket->data, trace_len) == 0) {
+ stack_map_get_build_id_offset(id_offs, stackid->nr, user, false /* !may_fault */);
+ trace_len = stackid->nr * sizeof(struct bpf_stack_build_id);
+ if (stackid->hash_matches && stackid->bucket->nr == stackid->nr &&
+ memcmp(stackid->bucket->data, new_bucket->data, trace_len) == 0) {
pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
- return id;
+ return stackid->id;
}
- if (bucket && !(flags & BPF_F_REUSE_STACKID)) {
+ if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) {
pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
return -EEXIST;
}
- } else {
- if (hash_matches && bucket->nr == trace_nr &&
- memcmp(bucket->data, ips, trace_len) == 0)
- return id;
- if (bucket && !(flags & BPF_F_REUSE_STACKID))
- return -EEXIST;
-
- new_bucket = (struct stack_map_bucket *)
- pcpu_freelist_pop(&smap->freelist);
- if (unlikely(!new_bucket))
- return -ENOMEM;
- memcpy(new_bucket->data, ips, trace_len);
}
- new_bucket->hash = hash;
- new_bucket->nr = trace_nr;
-
- old_bucket = xchg(&smap->buckets[id], new_bucket);
+ old_bucket = xchg(&smap->buckets[stackid->id], new_bucket);
if (old_bucket)
pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
- return id;
+ return stackid->id;
}
BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
@@ -583,23 +627,36 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
{
u32 elem_size = stack_map_data_size(map);
bool user = flags & BPF_F_USER_STACK;
+ struct stack_map_bucket *new_bucket;
struct perf_callchain_entry *trace;
+ struct stackid stackid;
bool kernel = !user;
u32 max_depth;
+ int err;
if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
return -EINVAL;
max_depth = stack_map_calculate_max_depth(map->value_size, elem_size, flags);
- trace = get_perf_callchain(regs, kernel, user, max_depth,
- false, false, 0);
- if (unlikely(!trace))
- /* couldn't fetch the stack trace */
- return -EFAULT;
+ scoped_guard(preempt) {
+ trace = get_perf_callchain(regs, kernel, user, max_depth,
+ false, false, 0);
+ if (unlikely(!trace))
+ /* couldn't fetch the stack trace */
+ return -EFAULT;
+
+ err = stackid_fastpath(&stackid, map, trace, trace->nr, flags);
+ if (err != -ENOENT)
+ return err;
+
+ new_bucket = stackid_new_bucket(&stackid, map);
+ if (!new_bucket)
+ return -ENOMEM;
+ }
- return __bpf_get_stackid(map, trace, flags);
+ return stackid_install(&stackid, map, new_bucket, flags);
}
const struct bpf_func_proto bpf_get_stackid_proto = {
@@ -611,7 +668,7 @@ const struct bpf_func_proto bpf_get_stackid_proto = {
.arg3_type = ARG_ANYTHING,
};
-static __u64 count_kernel_ip(struct perf_callchain_entry *trace)
+static __u64 count_kernel_ip(const struct perf_callchain_entry *trace)
{
__u64 nr_kernel = 0;
@@ -626,10 +683,13 @@ static __u64 count_kernel_ip(struct perf_callchain_entry *trace)
BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
struct bpf_map *, map, u64, flags)
{
+ const struct perf_callchain_entry *trace;
struct perf_event *event = ctx->event;
- struct perf_callchain_entry *trace;
+ struct stack_map_bucket *new_bucket;
+ struct stackid stackid;
bool kernel, user;
__u64 nr_kernel;
+ u32 trace_nr;
int ret;
/* perf_sample_data doesn't have callchain, use bpf_get_stackid */
@@ -649,26 +709,28 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
return -EFAULT;
nr_kernel = count_kernel_ip(trace);
- __u64 nr = trace->nr; /* save original */
if (kernel) {
- trace->nr = nr_kernel;
- ret = __bpf_get_stackid(map, trace, flags);
+ trace_nr = nr_kernel;
} else { /* user */
u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
+ trace_nr = trace->nr;
skip += nr_kernel;
if (skip > BPF_F_SKIP_FIELD_MASK)
return -EFAULT;
flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
- ret = __bpf_get_stackid(map, trace, flags);
}
- /* restore nr */
- trace->nr = nr;
+ ret = stackid_fastpath(&stackid, map, trace, trace_nr, flags);
+ if (ret != -ENOENT)
+ return ret;
- return ret;
+ new_bucket = stackid_new_bucket(&stackid, map);
+ if (new_bucket)
+ return stackid_install(&stackid, map, new_bucket, flags);
+ return -ENOMEM;
}
const struct bpf_func_proto bpf_get_stackid_proto_pe = {
@@ -680,19 +742,55 @@ const struct bpf_func_proto bpf_get_stackid_proto_pe = {
.arg3_type = ARG_ANYTHING,
};
+static u32 callchain_store(const struct perf_callchain_entry *trace, u32 trace_nr,
+ void *buf, u32 elem_size, u64 flags)
+{
+ bool user_build_id = flags & BPF_F_USER_BUILD_ID;
+ u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
+ const u64 *ips;
+ u32 copy_len;
+
+ trace_nr = trace_nr - skip;
+ copy_len = trace_nr * elem_size;
+
+ ips = trace->ip + skip;
+ if (user_build_id) {
+ struct bpf_stack_build_id *id_offs = buf;
+
+ for (u32 i = 0; i < trace_nr; i++)
+ id_offs[i].ip = ips[i];
+ } else {
+ memcpy(buf, ips, copy_len);
+ }
+ return trace_nr;
+}
+
+static long callchain_finalize(void *buf, u32 size, u32 trace_nr, u32 elem_size,
+ u64 flags, bool may_fault)
+{
+ bool user_build_id = flags & BPF_F_USER_BUILD_ID;
+ bool user = flags & BPF_F_USER_STACK;
+ u32 copy_len = trace_nr * elem_size;
+
+ if (user_build_id)
+ stack_map_get_build_id_offset(buf, trace_nr, user, may_fault);
+
+ if (size > copy_len)
+ memset(buf + copy_len, 0, size - copy_len);
+ return copy_len;
+}
+
static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
- struct perf_callchain_entry *trace_in,
void *buf, u32 size, u64 flags, bool may_fault)
{
- u32 trace_nr, copy_len, elem_size, max_depth;
bool user_build_id = flags & BPF_F_USER_BUILD_ID;
bool crosstask = task && task != current;
u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
bool user = flags & BPF_F_USER_STACK;
struct perf_callchain_entry *trace;
+ u32 trace_nr, elem_size, max_depth;
bool kernel = !user;
int err = -EINVAL;
- u64 *ips;
if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
BPF_F_USER_BUILD_ID)))
@@ -718,13 +816,11 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
max_depth = stack_map_calculate_max_depth(size, elem_size, flags);
+ preempt_disable();
if (may_fault)
rcu_read_lock(); /* need RCU for perf's callchain below */
- if (trace_in) {
- trace = trace_in;
- trace->nr = min_t(u32, trace->nr, max_depth);
- } else if (kernel && task) {
+ if (kernel && task) {
trace = get_callchain_entry_for_task(task, max_depth);
} else {
trace = get_perf_callchain(regs, kernel, user, max_depth,
@@ -734,33 +830,18 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
if (unlikely(!trace) || trace->nr < skip) {
if (may_fault)
rcu_read_unlock();
+ preempt_enable();
goto err_fault;
}
- trace_nr = trace->nr - skip;
- copy_len = trace_nr * elem_size;
-
- ips = trace->ip + skip;
- if (user_build_id) {
- struct bpf_stack_build_id *id_offs = buf;
- u32 i;
-
- for (i = 0; i < trace_nr; i++)
- id_offs[i].ip = ips[i];
- } else {
- memcpy(buf, ips, copy_len);
- }
+ trace_nr = callchain_store(trace, trace->nr, buf, elem_size, flags);
- /* trace/ips should not be dereferenced after this point */
+ /* trace should not be dereferenced after this point */
if (may_fault)
rcu_read_unlock();
+ preempt_enable();
- if (user_build_id)
- stack_map_get_build_id_offset(buf, trace_nr, user, may_fault);
-
- if (size > copy_len)
- memset(buf + copy_len, 0, size - copy_len);
- return copy_len;
+ return callchain_finalize(buf, size, trace_nr, elem_size, flags, may_fault);
err_fault:
err = -EFAULT;
@@ -772,7 +853,7 @@ clear:
BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size,
u64, flags)
{
- return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */);
+ return __bpf_get_stack(regs, NULL, buf, size, flags, false /* !may_fault */);
}
const struct bpf_func_proto bpf_get_stack_proto = {
@@ -781,14 +862,14 @@ const struct bpf_func_proto bpf_get_stack_proto = {
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
- .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg3_type = ARG_MEM_SIZE_OR_ZERO,
.arg4_type = ARG_ANYTHING,
};
BPF_CALL_4(bpf_get_stack_sleepable, struct pt_regs *, regs, void *, buf, u32, size,
u64, flags)
{
- return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, true /* may_fault */);
+ return __bpf_get_stack(regs, NULL, buf, size, flags, true /* may_fault */);
}
const struct bpf_func_proto bpf_get_stack_sleepable_proto = {
@@ -797,7 +878,7 @@ const struct bpf_func_proto bpf_get_stack_sleepable_proto = {
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
- .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg3_type = ARG_MEM_SIZE_OR_ZERO,
.arg4_type = ARG_ANYTHING,
};
@@ -807,14 +888,17 @@ static long __bpf_get_task_stack(struct task_struct *task, void *buf, u32 size,
struct pt_regs *regs;
long res = -EINVAL;
- if (!try_get_task_stack(task))
+ if (!try_get_task_stack(task)) {
+ memset(buf, 0, size);
return -EFAULT;
+ }
regs = task_pt_regs(task);
if (regs)
- res = __bpf_get_stack(regs, task, NULL, buf, size, flags, may_fault);
+ res = __bpf_get_stack(regs, task, buf, size, flags, may_fault);
+ else
+ memset(buf, 0, size);
put_task_stack(task);
-
return res;
}
@@ -831,7 +915,7 @@ const struct bpf_func_proto bpf_get_task_stack_proto = {
.arg1_type = ARG_PTR_TO_BTF_ID,
.arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
- .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg3_type = ARG_MEM_SIZE_OR_ZERO,
.arg4_type = ARG_ANYTHING,
};
@@ -848,22 +932,48 @@ const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = {
.arg1_type = ARG_PTR_TO_BTF_ID,
.arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
- .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg3_type = ARG_MEM_SIZE_OR_ZERO,
.arg4_type = ARG_ANYTHING,
};
+static int __bpf_get_stack_pe(const struct perf_callchain_entry *trace, u32 trace_nr,
+ void *buf, u32 size, u64 flags)
+{
+ bool user_build_id = flags & BPF_F_USER_BUILD_ID;
+ u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
+ bool user = flags & BPF_F_USER_STACK;
+ u32 elem_size, max_depth, nr_trace;
+ bool kernel = !user;
+
+ if (kernel && user_build_id)
+ return -EINVAL;
+
+ elem_size = user_build_id ? sizeof(struct bpf_stack_build_id) : sizeof(u64);
+ if (unlikely(size % elem_size))
+ return -EINVAL;
+
+ max_depth = stack_map_calculate_max_depth(size, elem_size, flags);
+ trace_nr = min_t(u32, trace_nr, max_depth);
+
+ if (trace_nr < skip)
+ return -EFAULT;
+
+ nr_trace = callchain_store(trace, trace_nr, buf, elem_size, flags);
+ return callchain_finalize(buf, size, nr_trace, elem_size, flags, false /* !may_fault */);
+}
+
BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
void *, buf, u32, size, u64, flags)
{
struct pt_regs *regs = (struct pt_regs *)(ctx->regs);
+ const struct perf_callchain_entry *trace;
struct perf_event *event = ctx->event;
- struct perf_callchain_entry *trace;
bool kernel, user;
int err = -EINVAL;
__u64 nr_kernel;
if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN))
- return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */);
+ return __bpf_get_stack(regs, NULL, buf, size, flags, false /* !may_fault */);
if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
BPF_F_USER_BUILD_ID)))
@@ -880,27 +990,20 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
nr_kernel = count_kernel_ip(trace);
if (kernel) {
- __u64 nr = trace->nr;
-
- trace->nr = nr_kernel;
- err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */);
-
- /* restore nr */
- trace->nr = nr;
+ err = __bpf_get_stack_pe(trace, nr_kernel, buf, size, flags);
} else { /* user */
u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
skip += nr_kernel;
if (skip > BPF_F_SKIP_FIELD_MASK)
goto clear;
-
flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
- err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */);
+ err = __bpf_get_stack_pe(trace, trace->nr, buf, size, flags);
}
- return err;
clear:
- memset(buf, 0, size);
+ if (err < 0)
+ memset(buf, 0, size);
return err;
}
@@ -911,7 +1014,7 @@ const struct bpf_func_proto bpf_get_stack_proto_pe = {
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
- .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg3_type = ARG_MEM_SIZE_OR_ZERO,
.arg4_type = ARG_ANYTHING,
};
generated by cgit v1.2.3 (git 2.0.4) at 2026-08-20 18:31:18 +0000