summaryrefslogtreecommitdiff
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/bpf/bpf_lsm.c2
-rw-r--r--kernel/bpf/btf.c7
-rw-r--r--kernel/bpf/cgroup.c163
-rw-r--r--kernel/bpf/core.c20
-rw-r--r--kernel/bpf/local_storage.c8
-rw-r--r--kernel/bpf/map_iter.c12
-rw-r--r--kernel/bpf/trampoline.c2
-rw-r--r--kernel/bpf/verifier.c7
-rw-r--r--kernel/events/uprobes.c9
-rw-r--r--kernel/time/jiffies.c21
-rw-r--r--kernel/trace/bpf_trace.c10
-rw-r--r--kernel/trace/fgraph.c59
-rw-r--r--kernel/trace/ftrace.c3
-rw-r--r--kernel/trace/ring_buffer.c60
-rw-r--r--kernel/trace/simple_ring_buffer.c26
-rw-r--r--kernel/trace/trace.c45
-rw-r--r--kernel/trace/trace.h3
-rw-r--r--kernel/trace/trace_events.c8
-rw-r--r--kernel/trace/trace_events_filter.c203
-rw-r--r--kernel/trace/trace_events_filter_test.h12
-rw-r--r--kernel/trace/trace_events_hist.c25
-rw-r--r--kernel/trace/trace_events_trigger.c2
-rw-r--r--kernel/trace/trace_events_user.c12
-rw-r--r--kernel/trace/trace_hwlat.c41
-rw-r--r--kernel/trace/trace_osnoise.c8
25 files changed, 572 insertions, 196 deletions
diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c
index 468e63dbd922..ae23511fd811 100644
--- a/kernel/bpf/bpf_lsm.c
+++ b/kernel/bpf/bpf_lsm.c
@@ -411,6 +411,8 @@ BTF_ID(func, bpf_lsm_bpf_map_free)
BTF_ID(func, bpf_lsm_bpf_prog_free)
BTF_ID(func, bpf_lsm_file_alloc_security)
BTF_ID(func, bpf_lsm_file_free_security)
+BTF_ID(func, bpf_lsm_backing_file_alloc)
+BTF_ID(func, bpf_lsm_backing_file_free)
BTF_ID(func, bpf_lsm_namespace_init)
BTF_ID(func, bpf_lsm_namespace_free)
#ifdef CONFIG_SECURITY_NETWORK
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 7e866bf017b5..49fcd32d418a 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -8094,7 +8094,12 @@ int btf_struct_access(struct bpf_verifier_log *log,
for (i = 0; i < rec->cnt; i++) {
struct btf_field *field = &rec->fields[i];
u32 offset = field->offset;
- if (off < offset + field->size && offset < off + size) {
+ /* An access at 'lo' or above ends after the start of the field. */
+ s64 lo = max(min_off, (s64)offset - size + 1);
+
+ /* Round up to min_off + k * step. The step is never 0. */
+ lo += imod(min_off - lo, reg->step);
+ if (lo <= max_off && lo < offset + field->size) {
bpf_log(log,
"direct access to %s is disallowed\n",
btf_field_type_name(field->type));
diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index 57a785241d42..f195d5fcae6c 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -206,13 +206,53 @@ unsigned int __cgroup_bpf_run_lsm_current(const void *ctx,
return ret;
}
+/*
+ * cgroup_bpf_mutex protects the bpf state of all cgroups: prog lists,
+ * effective arrays, flags, revisions, storage lists, link->cgroup,
+ * link->map of struct_ops links and cgroup_lsm_atype[].
+ *
+ * Take cgroup_mutex first if you need both. This mutex doesn't keep a
+ * cgroup alive or the hierarchy stable. Attach, detach and replace walk
+ * the descendants, so they take both. Everything else takes only this one.
+ *
+ * cgroup_bpf_inherit() runs at mkdir with only cgroup_mutex. That's safe:
+ * it writes the new cgroup, which no one else can see yet, and what it
+ * reads from the ancestors changes only under cgroup_mutex. The ancestors
+ * are online, and only rmdir, which needs cgroup_mutex, starts a release.
+ */
+static DEFINE_MUTEX(cgroup_bpf_mutex);
+
+#define cgroup_bpf_held() lockdep_is_held(&cgroup_bpf_mutex)
+
+void cgroup_bpf_lock(void)
+{
+ mutex_lock(&cgroup_bpf_mutex);
+}
+
+void cgroup_bpf_unlock(void)
+{
+ mutex_unlock(&cgroup_bpf_mutex);
+}
+
+static void cgroup_bpf_lock_hierarchy(void)
+{
+ cgroup_lock();
+ mutex_lock(&cgroup_bpf_mutex);
+}
+
+static void cgroup_bpf_unlock_hierarchy(void)
+{
+ mutex_unlock(&cgroup_bpf_mutex);
+ cgroup_unlock();
+}
+
#ifdef CONFIG_BPF_LSM
static enum cgroup_bpf_attach_type
bpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id)
{
int i;
- lockdep_assert_held(&cgroup_mutex);
+ lockdep_assert_held(&cgroup_bpf_mutex);
if (attach_type != BPF_LSM_CGROUP)
return to_cgroup_bpf_attach_type(attach_type);
@@ -233,7 +273,7 @@ void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype)
{
int i = cgroup_atype - CGROUP_LSM_START;
- lockdep_assert_held(&cgroup_mutex);
+ lockdep_assert_held(&cgroup_bpf_mutex);
if (!cgroup_lsm_atype[i].attach_btf_id) {
cgroup_lsm_atype[i].attach_btf_id = attach_btf_id;
@@ -249,13 +289,13 @@ void bpf_cgroup_atype_put(int cgroup_atype)
{
int i = cgroup_atype - CGROUP_LSM_START;
- cgroup_lock();
+ cgroup_bpf_lock();
if (--cgroup_lsm_atype[i].refcnt <= 0) {
WRITE_ONCE(cgroup_lsm_atype[i].returns_errno, true);
cgroup_lsm_atype[i].attach_btf_id = 0;
}
WARN_ON_ONCE(cgroup_lsm_atype[i].refcnt < 0);
- cgroup_unlock();
+ cgroup_bpf_unlock();
}
#else
static enum cgroup_bpf_attach_type
@@ -424,7 +464,7 @@ static int bpf_cgroup_array_copy_to_user(struct bpf_prog_array *array,
return -EFAULT;
i++;
}
- return item->prog ? -ENOSPC : 0;
+ return 0;
}
static int bpf_cgroup_array_delete_safe_at(struct bpf_prog_array *array,
@@ -459,7 +499,7 @@ static void cgroup_bpf_release(struct work_struct *work)
unsigned int atype;
- cgroup_lock();
+ cgroup_bpf_lock();
for (atype = 0; atype < ARRAY_SIZE(cgrp->bpf.progs); atype++) {
struct hlist_head *progs = &cgrp->bpf.progs[atype];
@@ -480,7 +520,7 @@ static void cgroup_bpf_release(struct work_struct *work)
}
old_array = rcu_dereference_protected(
cgrp->bpf.effective[atype],
- lockdep_is_held(&cgroup_mutex));
+ cgroup_bpf_held());
bpf_cgroup_array_free(old_array, atype);
}
@@ -489,7 +529,7 @@ static void cgroup_bpf_release(struct work_struct *work)
bpf_cgroup_storage_free(storage);
}
- cgroup_unlock();
+ cgroup_bpf_unlock();
for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))
cgroup_bpf_put(p);
@@ -665,7 +705,9 @@ static void activate_effective_progs(struct cgroup *cgrp,
enum cgroup_bpf_attach_type atype,
struct bpf_prog_array *old_array)
{
+ /* cgroup_bpf_inherit() holds only cgroup_mutex, see cgroup_bpf_mutex */
old_array = rcu_replace_pointer(cgrp->bpf.effective[atype], old_array,
+ lockdep_is_held(&cgroup_bpf_mutex) ||
lockdep_is_held(&cgroup_mutex));
/* free prog array after grace period, since __cgroup_bpf_run_*()
* might be still walking the array
@@ -687,6 +729,9 @@ static int cgroup_bpf_inherit(struct cgroup *cgrp)
struct cgroup *p;
int ret, i;
+ /* See the comment above cgroup_bpf_mutex */
+ lockdep_assert_held(&cgroup_mutex);
+
ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0,
GFP_KERNEL);
if (ret)
@@ -747,6 +792,8 @@ static int update_effective_progs(struct cgroup *cgrp,
struct cgroup_subsys_state *css;
int err;
+ lockdep_assert_held(&cgroup_bpf_mutex);
+
/* allocate and recompute effective prog arrays */
css_for_each_descendant_pre(css, &cgrp->self) {
struct cgroup *desc = container_of(css, struct cgroup, self);
@@ -954,7 +1001,7 @@ static int insert_pl_to_hlist(struct bpf_prog_list *pl, struct hlist_head *progs
* @revision: bpf_prog_list revision
*
* Exactly one of @prog or @link can be non-null.
- * Must be called with cgroup_mutex held.
+ * Must be called with cgroup_mutex and cgroup_bpf_mutex held.
*/
static int __cgroup_bpf_attach(struct cgroup *cgrp,
struct bpf_prog *prog, struct bpf_prog *replace_prog,
@@ -988,6 +1035,13 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp,
/* replace_prog implies BPF_F_REPLACE, and vice versa */
return -EINVAL;
+ /*
+ * The cgroup may have been removed since the caller looked it up.
+ * Then its bpf state may be freed, and nothing would detach this.
+ */
+ if (cgroup_is_dead(cgrp))
+ return -ENOENT;
+
atype = bpf_cgroup_atype_find(type, new_prog->aux->attach_btf_id);
if (atype < 0)
return -EINVAL;
@@ -1093,10 +1147,10 @@ static int cgroup_bpf_attach(struct cgroup *cgrp,
{
int ret;
- cgroup_lock();
+ cgroup_bpf_lock_hierarchy();
ret = __cgroup_bpf_attach(cgrp, prog, replace_prog, link, type, flags,
id_or_fd, revision);
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
return ret;
}
@@ -1174,7 +1228,7 @@ static void replace_effective_prog(struct cgroup *cgrp,
progs = rcu_dereference_protected(
desc->bpf.effective[atype],
- lockdep_is_held(&cgroup_mutex));
+ cgroup_bpf_held());
item = &progs->items[pos];
prog_list_replace_item(pl, item);
}
@@ -1202,7 +1256,7 @@ static bool cgroup_bpf_storages_compatible(struct bpf_prog *old_prog,
* @new_prog: &struct bpf_prog for the target BPF program with its refcnt
* incremented
*
- * Must be called with cgroup_mutex held.
+ * Must be called with cgroup_mutex and cgroup_bpf_mutex held.
*/
static int __cgroup_bpf_replace(struct cgroup *cgrp,
struct bpf_cgroup_link *link,
@@ -1250,7 +1304,7 @@ static int cgroup_bpf_replace(struct bpf_link *link, struct bpf_prog *new_prog,
cg_link = container_of(link, struct bpf_cgroup_link, link);
- cgroup_lock();
+ cgroup_bpf_lock_hierarchy();
/* link might have been auto-released by dying cgroup, so fail */
if (!cg_link->cgroup) {
ret = -ENOLINK;
@@ -1262,7 +1316,7 @@ static int cgroup_bpf_replace(struct bpf_link *link, struct bpf_prog *new_prog,
}
ret = __cgroup_bpf_replace(cg_link->cgroup, cg_link, new_prog);
out_unlock:
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
return ret;
}
@@ -1333,7 +1387,7 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl,
progs = rcu_dereference_protected(
desc->bpf.effective[atype],
- lockdep_is_held(&cgroup_mutex));
+ cgroup_bpf_held());
/* Remove the program from the array */
WARN_ONCE(bpf_cgroup_array_delete_safe_at(progs, pos, atype),
@@ -1351,7 +1405,7 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl,
* @revision: bpf_prog_list revision
*
* At most one of @prog or @link can be non-NULL.
- * Must be called with cgroup_mutex held.
+ * Must be called with cgroup_mutex and cgroup_bpf_mutex held.
*/
static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
struct bpf_cgroup_link *link, enum bpf_attach_type type,
@@ -1421,13 +1475,13 @@ static int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
{
int ret;
- cgroup_lock();
+ cgroup_bpf_lock_hierarchy();
ret = __cgroup_bpf_detach(cgrp, prog, NULL, type, revision);
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
return ret;
}
-/* Must be called with cgroup_mutex held to avoid races. */
+/* Must be called with cgroup_bpf_mutex held. */
static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
union bpf_attr __user *uattr, u32 uattr_size)
{
@@ -1446,6 +1500,14 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
if (effective_query && prog_attach_flags)
return -EINVAL;
+ /*
+ * cgroup_bpf_release() frees the effective arrays once the bpf
+ * refcount is zero. It takes this lock too, so a nonzero count means
+ * the arrays are still valid.
+ */
+ if (percpu_ref_is_zero(&cgrp->bpf.refcnt))
+ return -ENOENT;
+
if (type == BPF_STRUCT_OPS) {
u32 type_id = attr->query.type_id;
@@ -1473,7 +1535,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
for (atype = from_atype; atype <= to_atype; atype++) {
if (effective_query) {
effective = rcu_dereference_protected(cgrp->bpf.effective[atype],
- lockdep_is_held(&cgroup_mutex));
+ cgroup_bpf_held());
total_cnt += bpf_cgroup_array_length(effective, atype);
} else {
total_cnt += prog_list_length(&cgrp->bpf.progs[atype], NULL);
@@ -1503,9 +1565,10 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
for (atype = from_atype; atype <= to_atype && total_cnt; atype++) {
if (effective_query) {
effective = rcu_dereference_protected(cgrp->bpf.effective[atype],
- lockdep_is_held(&cgroup_mutex));
+ cgroup_bpf_held());
cnt = min_t(int, bpf_cgroup_array_length(effective, atype), total_cnt);
- ret = bpf_cgroup_array_copy_to_user(effective, prog_ids, cnt, atype);
+ if (bpf_cgroup_array_copy_to_user(effective, prog_ids, cnt, atype))
+ return -EFAULT;
} else {
struct hlist_head *progs;
struct bpf_prog_list *pl;
@@ -1544,9 +1607,9 @@ static int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
{
int ret;
- cgroup_lock();
+ cgroup_bpf_lock();
ret = __cgroup_bpf_query(cgrp, attr, uattr, uattr_size);
- cgroup_unlock();
+ cgroup_bpf_unlock();
return ret;
}
@@ -1614,11 +1677,11 @@ static void bpf_cgroup_link_release(struct bpf_link *link)
if (!cg_link->cgroup)
return;
- cgroup_lock();
+ cgroup_bpf_lock_hierarchy();
/* re-check cgroup under lock again */
if (!cg_link->cgroup) {
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
return;
}
@@ -1630,7 +1693,7 @@ static void bpf_cgroup_link_release(struct bpf_link *link)
cg = cg_link->cgroup;
cg_link->cgroup = NULL;
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
cgroup_put(cg);
}
@@ -1657,10 +1720,10 @@ static void bpf_cgroup_link_show_fdinfo(const struct bpf_link *link,
container_of(link, struct bpf_cgroup_link, link);
u64 cg_id = 0;
- cgroup_lock();
+ cgroup_bpf_lock();
if (cg_link->cgroup)
cg_id = cgroup_id(cg_link->cgroup);
- cgroup_unlock();
+ cgroup_bpf_unlock();
seq_printf(seq,
"cgroup_id:\t%llu\n"
@@ -1676,10 +1739,10 @@ static int bpf_cgroup_link_fill_link_info(const struct bpf_link *link,
container_of(link, struct bpf_cgroup_link, link);
u64 cg_id = 0;
- cgroup_lock();
+ cgroup_bpf_lock();
if (cg_link->cgroup)
cg_id = cgroup_id(cg_link->cgroup);
- cgroup_unlock();
+ cgroup_bpf_unlock();
info->cgroup.cgroup_id = cg_id;
info->cgroup.attach_type = link->attach_type;
@@ -2956,11 +3019,11 @@ static int __cgroup_struct_ops_link_detach(struct bpf_link *link, bool wake_poll
struct bpf_map *map;
struct cgroup *cgrp;
- cgroup_lock();
+ cgroup_bpf_lock_hierarchy();
cgrp = cg_link->cgroup;
if (!cgrp) {
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
return 0;
}
@@ -2972,7 +3035,7 @@ static int __cgroup_struct_ops_link_detach(struct bpf_link *link, bool wake_poll
break;
}
if (WARN_ON_ONCE(!pl)) {
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
return -ENOENT;
}
@@ -2990,7 +3053,7 @@ static int __cgroup_struct_ops_link_detach(struct bpf_link *link, bool wake_poll
kfree(pl);
static_branch_dec(&cgroup_bpf_enabled_key[atype]);
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
return 0;
}
@@ -3013,15 +3076,15 @@ static void cgroup_struct_ops_link_show_fdinfo(const struct bpf_link *link, stru
struct bpf_cgroup_link *cg_link =
container_of(link, struct bpf_cgroup_link, link);
- cgroup_lock();
+ cgroup_bpf_lock();
if (!cg_link->cgroup) {
- cgroup_unlock();
+ cgroup_bpf_unlock();
return;
}
seq_printf(seq, "map_id:\t%u\n", cg_link->map->id);
seq_printf(seq, "cgroup_id:\t%llu\n", cgroup_id(cg_link->cgroup));
- cgroup_unlock();
+ cgroup_bpf_unlock();
}
static int cgroup_struct_ops_link_fill_link_info(const struct bpf_link *link,
@@ -3029,15 +3092,15 @@ static int cgroup_struct_ops_link_fill_link_info(const struct bpf_link *link,
{
struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link);
- cgroup_lock();
+ cgroup_bpf_lock();
if (!cg_link->cgroup) {
- cgroup_unlock();
+ cgroup_bpf_unlock();
return 0;
}
info->struct_ops.map_id = cg_link->map->id;
info->struct_ops.cgroup_id = cgroup_id(cg_link->cgroup);
- cgroup_unlock();
+ cgroup_bpf_unlock();
return 0;
}
@@ -3055,7 +3118,7 @@ static int cgroup_struct_ops_link_update(struct bpf_link *link, struct bpf_map *
if (!bpf_struct_ops_valid_to_reg(new_map))
return -EINVAL;
- cgroup_lock();
+ cgroup_bpf_lock_hierarchy();
cgrp = cg_link->cgroup;
if (!cgrp) {
@@ -3088,7 +3151,7 @@ static int cgroup_struct_ops_link_update(struct bpf_link *link, struct bpf_map *
cgrp->bpf.revisions[atype]++;
out:
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
return err;
}
@@ -3160,7 +3223,13 @@ int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr
atype = bpf_struct_ops_map_cgroup_atype(map);
progs = &cgrp->bpf.progs[atype];
- cgroup_lock();
+ cgroup_bpf_lock_hierarchy();
+
+ /* See the cgroup_is_dead() check in __cgroup_bpf_attach() */
+ if (cgroup_is_dead(cgrp)) {
+ err = -ENOENT;
+ goto unlock;
+ }
if (attr->link_create.cgroup.expected_revision &&
attr->link_create.cgroup.expected_revision != cgrp->bpf.revisions[atype]) {
@@ -3197,12 +3266,12 @@ int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr
cgrp->bpf.revisions[atype]++;
static_branch_inc(&cgroup_bpf_enabled_key[atype]);
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
return bpf_link_settle(&link_primer);
unlock:
- cgroup_unlock();
+ cgroup_bpf_unlock_hierarchy();
cleanup:
kfree(pl);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 0b8456f2dce9..0bfd23a41401 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -24,6 +24,7 @@
#include <linux/skbuff.h>
#include <linux/static_call.h>
#include <linux/vmalloc.h>
+#include <linux/workqueue.h>
#include <linux/prandom.h>
#include <linux/bpf.h>
#include <linux/btf.h>
@@ -3143,6 +3144,17 @@ static void bpf_free_used_btfs(struct bpf_prog_aux *aux)
kfree(aux->used_btfs);
}
+struct workqueue_struct *bpf_prog_free_wq __ro_after_init;
+
+static int __init bpf_prog_free_wq_init(void)
+{
+ bpf_prog_free_wq = alloc_workqueue("bpf_prog_free", WQ_PERCPU, 1);
+ if (!bpf_prog_free_wq)
+ panic("Failed to allocate BPF program free workqueue\n");
+ return 0;
+}
+early_initcall(bpf_prog_free_wq_init);
+
static void bpf_prog_free_images(struct work_struct *work)
{
struct bpf_prog_aux *aux;
@@ -3198,7 +3210,7 @@ static void bpf_prog_free_deferred(struct work_struct *work)
/* The symbols are looked up under rcu_read_lock() */
INIT_RCU_WORK(&aux->rwork, bpf_prog_free_images);
if (ksyms)
- queue_rcu_work(system_percpu_wq, &aux->rwork);
+ queue_rcu_work(bpf_prog_free_wq, &aux->rwork);
else
bpf_prog_free_images(&aux->rwork.work);
}
@@ -3211,7 +3223,11 @@ void bpf_prog_free(struct bpf_prog *fp)
bpf_prog_put(aux->dst_prog);
bpf_token_put(aux->token);
INIT_WORK(&aux->work, bpf_prog_free_deferred);
- schedule_work(&aux->work);
+ /*
+ * Free BPF programs on a dedicated workqueue to avoid blocking or slowing
+ * down the system workqueue.
+ */
+ queue_work(bpf_prog_free_wq, &aux->work);
}
/* RNG for unprivileged user space with separated state from prandom_u32(). */
diff --git a/kernel/bpf/local_storage.c b/kernel/bpf/local_storage.c
index 83cd527a2542..2cfbddeb1c7e 100644
--- a/kernel/bpf/local_storage.c
+++ b/kernel/bpf/local_storage.c
@@ -345,14 +345,18 @@ static void cgroup_storage_map_free(struct bpf_map *_map)
struct list_head *storages = &map->list;
struct bpf_cgroup_storage *storage, *stmp;
- cgroup_lock();
+ /*
+ * Attach and cgroup_bpf_release() change the same cgroup storage
+ * lists under other maps' locks, so take cgroup_bpf_mutex.
+ */
+ cgroup_bpf_lock();
list_for_each_entry_safe(storage, stmp, storages, list_map) {
bpf_cgroup_storage_unlink(storage);
bpf_cgroup_storage_free(storage);
}
- cgroup_unlock();
+ cgroup_bpf_unlock();
WARN_ON(!RB_EMPTY_ROOT(&map->root));
WARN_ON(!list_empty(&map->list));
diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c
index a58c148692e6..7a769e5f5397 100644
--- a/kernel/bpf/map_iter.c
+++ b/kernel/bpf/map_iter.c
@@ -99,8 +99,8 @@ static struct bpf_iter_reg bpf_map_reg_info = {
/*
* For an iterator that hands the live map value to the program as a writable
- * buffer: the verifier knows nothing of the map, so the special fields of the
- * value have to lie outside of what the program may write.
+ * buffer: the verifier knows nothing of the map, so read-only maps
+ * and special fields of the value have to be protected from writes.
*/
int bpf_iter_map_check_value_write(const struct bpf_prog *prog,
const struct bpf_map *map)
@@ -109,7 +109,13 @@ int bpf_iter_map_check_value_write(const struct bpf_prog *prog,
const struct btf_field *field;
u32 i;
- if (IS_ERR_OR_NULL(rec) || !prog->aux->max_rdwr_write)
+ if (!prog->aux->max_rdwr_write)
+ return 0;
+
+ if (map->map_flags & BPF_F_RDONLY_PROG)
+ return -EACCES;
+
+ if (IS_ERR_OR_NULL(rec))
return 0;
for (i = 0; i < rec->cnt; i++) {
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 21d460866dfe..b0052c7565d8 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -584,7 +584,7 @@ static void __bpf_tramp_image_put_rcu(struct rcu_head *rcu)
im = container_of(rcu, struct bpf_tramp_image, rcu);
INIT_WORK(&im->work, __bpf_tramp_image_put_deferred);
- schedule_work(&im->work);
+ queue_work(bpf_prog_free_wq, &im->work);
}
/* callback, fexit step 2. Called after percpu_ref_kill confirms. */
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 5e4700fda66b..9b228ffbe3fe 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -10526,6 +10526,7 @@ static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
{
enum bpf_attach_type eatype = env->prog->expected_attach_type;
enum bpf_prog_type type = resolve_prog_type(env->prog);
+ const char *afname = env->prog->aux->attach_func_name;
if (func_id != BPF_FUNC_map_update_elem &&
func_id != BPF_FUNC_map_delete_elem)
@@ -10536,7 +10537,8 @@ static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
*/
switch (type) {
case BPF_PROG_TYPE_TRACING:
- if (eatype == BPF_TRACE_ITER)
+ if (eatype == BPF_TRACE_ITER &&
+ !strcmp(afname + strlen(BPF_ITER_FUNC_PREFIX), "sockmap"))
return true;
break;
case BPF_PROG_TYPE_SOCK_OPS:
@@ -20627,8 +20629,7 @@ static int do_check(struct bpf_verifier_env *env)
if (signal_pending(current))
return -EAGAIN;
- if (need_resched())
- cond_resched();
+ cond_resched_tasks_rcu_qs();
/* reset current history entry on each new instruction */
env->cur_hist_ent = NULL;
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index caf444762a27..c47b1ee266d5 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -2704,6 +2704,15 @@ void __weak arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vadd
}
/*
+ * Can a bpf prog write these bytes of pt_regs? An arch opts in and lists the
+ * registers that ptrace can set to any value.
+ */
+bool __weak arch_uprobe_regs_writable(int off, int size)
+{
+ return false;
+}
+
+/*
* Run handler and ask thread to singlestep.
* Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
*/
diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c
index af1c69bf0c23..b861dc860d34 100644
--- a/kernel/time/jiffies.c
+++ b/kernel/time/jiffies.c
@@ -137,11 +137,19 @@ static int sysctl_k2u_int_conv_userhz(bool *negp, ulong *u_ptr, const int *k_ptr
return proc_int_k2u_conv_kop(u_ptr, k_ptr, negp, sysctl_jiffies_to_clock_t);
}
+/*
+ * Calc ms <-> jiffies conversions for sysctls in unsigned long arithmetic.
+ * Think hard about using the non-sysctl converters (msecs_to_jiffies &
+ * jiffies_to_msecs) as the interaction between ulong and uint is not trivial.
+ */
static ulong sysctl_msecs_to_jiffies(const ulong val)
{
- if (val > jiffies_to_msecs(MAX_JIFFY_OFFSET))
- return MAX_JIFFY_OFFSET;
- return msecs_to_jiffies(val);
+ ulong secs = val / MSEC_PER_SEC;
+
+ if (secs >= ULONG_MAX / HZ)
+ return ULONG_MAX;
+
+ return secs * HZ + DIV_ROUND_UP((val % MSEC_PER_SEC) * HZ, MSEC_PER_SEC);
}
static int sysctl_u2k_int_conv_ms(const bool *negp, const ulong *u_ptr, int *k_ptr)
@@ -151,7 +159,12 @@ static int sysctl_u2k_int_conv_ms(const bool *negp, const ulong *u_ptr, int *k_p
static ulong sysctl_jiffies_to_msecs(const ulong val)
{
- return jiffies_to_msecs(val);
+ ulong secs = val / HZ;
+
+ if (secs >= ULONG_MAX / MSEC_PER_SEC)
+ return ULONG_MAX;
+
+ return secs * MSEC_PER_SEC + (val % HZ) * MSEC_PER_SEC / HZ;
}
static int sysctl_k2u_int_conv_ms(bool *negp, ulong *u_ptr, const int *k_ptr)
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index d22e78e7f28d..1f35a18ef399 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -1397,8 +1397,16 @@ static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type
if (off + size > sizeof(struct pt_regs))
return false;
- if (type == BPF_WRITE)
+ if (type == BPF_WRITE) {
+ /*
+ * pt_regs also holds what the kernel restores on the way to
+ * user space: cs and flags on x86, pstate on arm64, status on
+ * riscv. A prog can write the registers of the task only.
+ */
+ if (!arch_uprobe_regs_writable(off, size))
+ return false;
prog->aux->kprobe_write_ctx = true;
+ }
return true;
}
diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c
index 6d87143b3ddb..d1f79a85ed2e 100644
--- a/kernel/trace/fgraph.c
+++ b/kernel/trace/fgraph.c
@@ -568,7 +568,11 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func,
if (unlikely(ftrace_graph_is_dead()))
return -EBUSY;
- if (!current->ret_stack)
+ /*
+ * We must make sure the ret_stack is tested before we read
+ * anything else (See alloc_retstack_tasklist()).
+ */
+ if (!smp_load_acquire(&current->ret_stack))
return -EBUSY;
BUILD_BUG_ON(SHADOW_STACK_SIZE % sizeof(long));
@@ -577,12 +581,6 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func,
val = (FGRAPH_TYPE_RESERVED << FGRAPH_TYPE_SHIFT) | FGRAPH_FRAME_OFFSET;
/*
- * We must make sure the ret_stack is tested before we read
- * anything else.
- */
- smp_rmb();
-
- /*
* Check if there's room on the shadow stack to fit a fraph frame
* and a bitmap word.
*/
@@ -700,11 +698,7 @@ int function_graph_enter_regs(unsigned long ret, unsigned long func,
if (!bitmap)
goto out_ret;
- /*
- * Since this function uses fgraph_idx = 0 as a tail-call checking
- * flag, set that bit always.
- */
- set_bitmap(current, offset, bitmap | BIT(0));
+ set_bitmap(current, offset, bitmap);
ftrace_test_recursion_unlock(bit);
return 0;
out_ret:
@@ -1033,13 +1027,16 @@ void ftrace_stub_graph(struct ftrace_graph_ret *trace, struct fgraph_ops *gops,
trace_func_graph_ret_t ftrace_graph_return = ftrace_stub_graph;
trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
-/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
+/*
+ * Loop thru all the tasks in the system and assign ret_stack.
+ * The @ret_stack_list is a small pre-allocated fallback, in case
+ * the GFP_NOWAIT allocations here fail.
+ */
static int alloc_retstack_tasklist(unsigned long **ret_stack_list)
{
- int i;
- int ret = 0;
int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
struct task_struct *g, *t;
+ int i, ret = 0;
if (WARN_ON_ONCE(!fgraph_stack_cachep))
return -ENOMEM;
@@ -1054,26 +1051,31 @@ static int alloc_retstack_tasklist(unsigned long **ret_stack_list)
}
}
- rcu_read_lock();
- for_each_process_thread(g, t) {
- if (start == end) {
- ret = -EAGAIN;
- goto unlock;
- }
+ scoped_guard (rcu) {
+ for_each_process_thread(g, t) {
+ unsigned long *rs;
+
+ if (t->ret_stack)
+ continue;
+
+ rs = kmem_cache_alloc(fgraph_stack_cachep, GFP_NOWAIT);
+ if (!rs) {
+ if (start == end) {
+ ret = -EAGAIN;
+ goto free;
+ }
+ rs = ret_stack_list[start++];
+ }
- if (t->ret_stack == NULL) {
atomic_set(&t->trace_overrun, 0);
- ret_stack_init_task_vars(ret_stack_list[start]);
+ ret_stack_init_task_vars(rs);
t->curr_ret_stack = 0;
t->curr_ret_depth = -1;
/* Make sure the tasks see the 0 first: */
- smp_wmb();
- t->ret_stack = ret_stack_list[start++];
+ smp_store_release(&t->ret_stack, rs);
}
}
-unlock:
- rcu_read_unlock();
free:
for (i = start; i < end; i++)
kmem_cache_free(fgraph_stack_cachep, ret_stack_list[i]);
@@ -1117,8 +1119,7 @@ graph_init_task(struct task_struct *t, unsigned long *ret_stack)
t->curr_ret_stack = 0;
t->curr_ret_depth = -1;
/* make curr_ret_stack visible before we add the ret_stack */
- smp_wmb();
- t->ret_stack = ret_stack;
+ smp_store_release(&t->ret_stack, ret_stack);
}
/*
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index c7d74b33d36d..6654da9d1bbf 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -5029,7 +5029,8 @@ match_records(struct ftrace_hash *hash, char *func, int len, char *mod)
if (unlikely(ftrace_disabled))
return 0;
- if (func_g.type == MATCH_INDEX)
+ if (func_g.type == MATCH_FULL && !mod &&
+ func_g.search && isdigit(func_g.search[0]))
return add_rec_by_index(hash, &func_g, clear_filter);
do_for_each_ftrace_rec(pg, rec) {
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 75cd8df23c1b..a5a0adc934fa 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -2468,8 +2468,17 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
* memory. It may not be accurate. But we don't care, we just want
* to prevent doing any allocation when it is obvious that it is
* not going to succeed.
+ *
+ * With CONFIG_DEFERRED_STRUCT_PAGE_INIT, si_mem_available()
+ * will reflect the actual available memory only after
+ * SYSTEM_FREEING_INITMEM.
*/
- i = si_mem_available();
+ if (IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT) &&
+ unlikely(system_state < SYSTEM_FREEING_INITMEM))
+ i = LONG_MAX;
+ else
+ i = si_mem_available();
+
if (i < (nr_pages << subbuf_order))
return -ENOMEM;
@@ -5615,14 +5624,12 @@ unsigned long ring_buffer_overruns(struct trace_buffer *buffer)
}
EXPORT_SYMBOL_GPL(ring_buffer_overruns);
-static bool rb_read_remote_meta_page(struct ring_buffer_per_cpu *cpu_buffer)
+static void rb_read_remote_meta_page(struct ring_buffer_per_cpu *cpu_buffer)
{
local_set(&cpu_buffer->entries, READ_ONCE(cpu_buffer->meta_page->entries));
local_set(&cpu_buffer->overrun, READ_ONCE(cpu_buffer->meta_page->overrun));
local_set(&cpu_buffer->pages_touched, READ_ONCE(cpu_buffer->meta_page->pages_touched));
local_set(&cpu_buffer->pages_lost, READ_ONCE(cpu_buffer->meta_page->pages_lost));
-
- return rb_num_of_entries(cpu_buffer);
}
static void rb_update_remote_head(struct ring_buffer_per_cpu *cpu_buffer)
@@ -5825,15 +5832,20 @@ __rb_get_reader_page_from_remote(struct ring_buffer_per_cpu *cpu_buffer)
{
struct buffer_page *new_reader, *prev_reader, *prev_head, *new_head, *last;
- if (!rb_read_remote_meta_page(cpu_buffer))
- return NULL;
+ rb_read_remote_meta_page(cpu_buffer);
+
+ /*
+ * Order meta::entries read before page::commit, most likely paired with
+ * simple_ring_buffer_commit().
+ */
+ smp_rmb();
/* More to read on the reader page */
- if (cpu_buffer->reader_page->read < rb_page_size(cpu_buffer->reader_page)) {
- if (!cpu_buffer->reader_page->read)
- cpu_buffer->read_stamp = cpu_buffer->reader_page->page->time_stamp;
- return cpu_buffer->reader_page;
- }
+ if (cpu_buffer->reader_page->read < rb_page_size(cpu_buffer->reader_page))
+ goto out;
+
+ if (!rb_num_of_entries(cpu_buffer))
+ return NULL;
prev_reader = cpu_buffer->subbuf_ids[cpu_buffer->meta_page->reader.id];
@@ -5848,7 +5860,8 @@ __rb_get_reader_page_from_remote(struct ring_buffer_per_cpu *cpu_buffer)
new_reader = cpu_buffer->subbuf_ids[cpu_buffer->meta_page->reader.id];
- WARN_ON_ONCE(prev_reader == new_reader);
+ if (prev_reader == new_reader)
+ return NULL;
prev_head = new_reader; /* New reader was also the previous head */
new_head = prev_head;
@@ -5876,10 +5889,23 @@ __rb_get_reader_page_from_remote(struct ring_buffer_per_cpu *cpu_buffer)
cpu_buffer->reader_page = new_reader;
cpu_buffer->reader_page->read = 0;
cpu_buffer->pages = &new_head->list;
- cpu_buffer->read_stamp = new_reader->page->time_stamp;
cpu_buffer->lost_events = cpu_buffer->meta_page->reader.lost_events;
+ local_inc(&cpu_buffer->pages_read);
- return rb_page_size(cpu_buffer->reader_page) ? cpu_buffer->reader_page : NULL;
+ if (!rb_page_size(cpu_buffer->reader_page))
+ return NULL;
+
+out:
+ /*
+ * Order page::commit read before page::data and page::time_stamp, most
+ * likely paired with simple_ring_buffer_commit().
+ */
+ smp_rmb();
+
+ if (!cpu_buffer->reader_page->read)
+ cpu_buffer->read_stamp = cpu_buffer->reader_page->page->time_stamp;
+
+ return cpu_buffer->reader_page;
}
static struct buffer_page *
@@ -6908,7 +6934,8 @@ int ring_buffer_poll_remote(struct trace_buffer *buffer, int cpu)
cpu_buffer = buffer->buffers[cpu];
guard(raw_spinlock)(&cpu_buffer->reader_lock);
- if (rb_read_remote_meta_page(cpu_buffer))
+ rb_read_remote_meta_page(cpu_buffer);
+ if (rb_num_of_entries(cpu_buffer))
rb_wakeups(buffer, cpu_buffer);
return 0;
@@ -8091,8 +8118,7 @@ int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node)
buffer->buffers[cpu] =
rb_allocate_cpu_buffer(buffer, nr_pages, cpu);
if (!buffer->buffers[cpu]) {
- WARN(1, "failed to allocate ring buffer on CPU %u\n",
- cpu);
+ pr_warn("failed to allocate ring buffer on CPU %u\n", cpu);
return -ENOMEM;
}
diff --git a/kernel/trace/simple_ring_buffer.c b/kernel/trace/simple_ring_buffer.c
index abfaf9bee12e..2dddaef638dc 100644
--- a/kernel/trace/simple_ring_buffer.c
+++ b/kernel/trace/simple_ring_buffer.c
@@ -139,6 +139,13 @@ int simple_ring_buffer_swap_reader_page(struct simple_rb_per_cpu *cpu_buffer)
reader = cpu_buffer->reader_page;
+ /*
+ * The reader is already the tail page. See simple_rb_move_tail() for
+ * memory ordering.
+ */
+ if (reader == smp_load_acquire(&cpu_buffer->tail_page))
+ return 0;
+
do {
/* Run after the writer to find the head */
ret = simple_rb_find_head(cpu_buffer);
@@ -196,7 +203,13 @@ static struct simple_buffer_page *simple_rb_move_tail(struct simple_rb_per_cpu *
}
simple_bpage_reset(new_tail);
- cpu_buffer->tail_page = new_tail;
+
+ /*
+ * Setting a new tail may unblock the swap in
+ * simple_ring_buffer_swap_reader_page(). Ensure the reader observes the
+ * page::commit reset.
+ */
+ smp_store_release(&cpu_buffer->tail_page, new_tail);
simple_rb_meta_inc(cpu_buffer->meta->pages_touched, 1);
@@ -294,8 +307,19 @@ EXPORT_SYMBOL_GPL(simple_ring_buffer_reserve);
*/
void simple_ring_buffer_commit(struct simple_rb_per_cpu *cpu_buffer)
{
+ /*
+ * Order page::data and page::time_stamp before page::commit. Paired
+ * with __rb_get_reader_page_from_remote().
+ */
+ smp_wmb();
local_set(&cpu_buffer->tail_page->page->commit,
cpu_buffer->tail_page->write);
+
+ /*
+ * Order page::commit before meta::entries. Paired with
+ * __rb_get_reader_page_from_remote().
+ */
+ smp_wmb();
simple_rb_meta_inc(cpu_buffer->meta->entries, 1);
/*
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 4ccdefb5c3ef..65b9d67647e5 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -1456,6 +1456,13 @@ static void free_tracers(struct trace_array *tr)
}
}
+static void free_tracer_options(struct trace_array *tr)
+{
+ for (int i = 0; i < tr->nr_topts; i++)
+ kfree(tr->topts[i].topts);
+ kfree(tr->topts);
+}
+
/**
* register_tracer - register a tracer with the ftrace system.
* @type: the plugin for the tracer
@@ -4028,7 +4035,9 @@ int set_tracer_flag(struct trace_array *tr, u64 mask, int enabled)
break;
case TRACE_ITER(COPY_MARKER):
- update_marker_trace(tr, enabled);
+ /* Writers of the top level trace_marker may still be on this instance */
+ if (update_marker_trace(tr, enabled) && !enabled)
+ synchronize_rcu();
/* update_marker_trace updates the tr->trace_flags */
return 0;
}
@@ -6289,7 +6298,7 @@ char *trace_user_fault_read(struct trace_user_buf_info *tinfo,
if (copy_func) {
ret = copy_func(buffer, ptr, size, data);
} else {
- ret = __copy_from_user(buffer, ptr, size);
+ ret = copy_from_user(buffer, ptr, size);
}
preempt_disable_notrace();
@@ -6345,9 +6354,10 @@ tracing_mark_write(struct file *filp, const char __user *ubuf,
if (tr == &global_trace) {
guard(rcu)();
list_for_each_entry_rcu(tr, &marker_copies, marker_list) {
- written = write_marker_to_buffer(tr, buf, cnt, ip);
- if (written < 0)
- break;
+ ssize_t w = write_marker_to_buffer(tr, buf, cnt, ip);
+
+ if (w >= 0 && w > written)
+ written = w;
}
} else {
written = write_marker_to_buffer(tr, buf, cnt, ip);
@@ -6424,9 +6434,10 @@ tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
if (tr == &global_trace) {
guard(rcu)();
list_for_each_entry_rcu(tr, &marker_copies, marker_list) {
- written = write_raw_marker_to_buffer(tr, buf, cnt);
- if (written < 0)
- break;
+ ssize_t w = write_raw_marker_to_buffer(tr, buf, cnt);
+
+ if (w >= 0 && w > written)
+ written = w;
}
} else {
written = write_raw_marker_to_buffer(tr, buf, cnt);
@@ -7404,8 +7415,12 @@ static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned
return err;
}
- return ring_buffer_map_get_reader(iter->array_buffer->buffer,
- iter->cpu_file);
+ trace_access_lock(iter->cpu_file);
+ err = ring_buffer_map_get_reader(iter->array_buffer->buffer,
+ iter->cpu_file);
+ trace_access_unlock(iter->cpu_file);
+
+ return err;
} else if (cmd) {
return -ENOTTY;
}
@@ -8761,7 +8776,7 @@ trace_array_create_systems(const char *name, const char *systems,
if (trace_instance_dir) {
ret = trace_array_create_dir(tr);
if (ret)
- goto out_free_tr;
+ goto out_free_tracers;
} else
__trace_early_add_events(tr);
@@ -8771,6 +8786,9 @@ trace_array_create_systems(const char *name, const char *systems,
return tr;
+ out_free_tracers:
+ free_tracers(tr);
+ free_tracer_options(tr);
out_free_tr:
ftrace_free_ftrace_ops(tr);
free_trace_buffers(tr);
@@ -8924,10 +8942,7 @@ static int __remove_instance(struct trace_array *tr)
if (tr->flags & TRACE_ARRAY_FL_VMALLOC)
vfree((void *)tr->range_addr_start);
- for (i = 0; i < tr->nr_topts; i++) {
- kfree(tr->topts[i].topts);
- }
- kfree(tr->topts);
+ free_tracer_options(tr);
free_cpumask_var(tr->pipe_cpumask);
free_cpumask_var(tr->tracing_cpumask);
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 0635503a31dd..1fec567e789e 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -1807,7 +1807,6 @@ enum regex_type {
MATCH_MIDDLE_ONLY,
MATCH_END_ONLY,
MATCH_GLOB,
- MATCH_INDEX,
};
struct regex {
@@ -1912,9 +1911,11 @@ extern const struct file_operations event_inject_fops;
#ifdef CONFIG_HIST_TRIGGERS
extern int register_trigger_hist_cmd(void);
extern int register_trigger_hist_enable_disable_cmds(void);
+extern void hist_put_synth_event(struct trace_event_file *file);
#else
static inline int register_trigger_hist_cmd(void) { return 0; }
static inline int register_trigger_hist_enable_disable_cmds(void) { return 0; }
+static inline void hist_put_synth_event(struct trace_event_file *file) { }
#endif
extern int register_trigger_cmds(void);
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c
index 49d135bc2fbf..7f89160aeff3 100644
--- a/kernel/trace/trace_events.c
+++ b/kernel/trace/trace_events.c
@@ -4317,7 +4317,7 @@ static void update_event_probe(struct event_probe_data *data)
set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
}
-static void
+static void notrace
event_enable_probe(unsigned long ip, unsigned long parent_ip,
struct trace_array *tr, struct ftrace_probe_ops *ops,
void *data)
@@ -4334,7 +4334,7 @@ event_enable_probe(unsigned long ip, unsigned long parent_ip,
update_event_probe(edata);
}
-static void
+static void notrace
event_enable_count_probe(unsigned long ip, unsigned long parent_ip,
struct trace_array *tr, struct ftrace_probe_ops *ops,
void *data)
@@ -5026,8 +5026,8 @@ void __init trace_event_init(void)
{
event_trace_memsetup();
init_ftrace_syscalls();
- event_trace_enable();
event_trace_init_fields();
+ event_trace_enable();
}
#ifdef CONFIG_EVENT_TRACE_STARTUP_TEST
@@ -5205,7 +5205,7 @@ static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable);
static struct trace_event_file event_trace_file __initdata;
-static void __init
+static void notrace __init
function_test_events_call(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct ftrace_regs *regs)
{
diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c
index c10951bcfc5f..a9d77df3e19a 100644
--- a/kernel/trace/trace_events_filter.c
+++ b/kernel/trace/trace_events_filter.c
@@ -758,7 +758,7 @@ static int filter_pred_##size##_cpumask(struct filter_pred *pred, void *event) \
unsigned int cpu = *addr; \
\
if (cpu >= nr_cpu_ids) \
- return 0; \
+ return pred->op == OP_NE; \
\
return do_filter_scalar_cpumask(pred->op, cpu, pred->mask); \
}
@@ -891,6 +891,25 @@ static int filter_pred_pchar_user(struct filter_pred *pred, void *event)
return filter_pchar(pred, str);
}
+static __always_inline int filter_dyn_string(struct filter_pred *pred,
+ char *str, int len)
+{
+ int cmp;
+
+ /*
+ * The regex callbacks use zero for an unknown-length, NUL-terminated
+ * string. A zero-length data_loc or rel_loc field has no bytes at all.
+ * Match an empty string instead of reading past the end of the field.
+ */
+ if (!len) {
+ str = "";
+ len = 1;
+ }
+
+ cmp = pred->regex->match(str, pred->regex, len);
+ return cmp ^ pred->not;
+}
+
/*
* Filter predicate for dynamic sized arrays of characters.
* These are implemented through a list of strings at the end
@@ -907,13 +926,8 @@ static int filter_pred_strloc(struct filter_pred *pred, void *event)
int str_loc = str_item & 0xffff;
int str_len = str_item >> 16;
char *addr = (char *)(event + str_loc);
- int cmp, match;
-
- cmp = pred->regex->match(addr, pred->regex, str_len);
-
- match = cmp ^ pred->not;
- return match;
+ return filter_dyn_string(pred, addr, str_len);
}
/*
@@ -930,13 +944,8 @@ static int filter_pred_strrelloc(struct filter_pred *pred, void *event)
int str_loc = str_item & 0xffff;
int str_len = str_item >> 16;
char *addr = (char *)(&item[1]) + str_loc;
- int cmp, match;
- cmp = pred->regex->match(addr, pred->regex, str_len);
-
- match = cmp ^ pred->not;
-
- return match;
+ return filter_dyn_string(pred, addr, str_len);
}
/* Filter predicate for CPUs. */
@@ -1037,7 +1046,8 @@ static int filter_pred_within(struct filter_pred *pred, void *event)
*
* @str: the string to be searched
* @r: the regex structure containing the pattern string
- * @len: the length of the string to be searched (including '\0')
+ * @len: the length of the string to be searched (including '\0'), or zero
+ * for a NUL-terminated string of unknown length
*
* Note:
* - @str might not be NULL-terminated if it's of type DYN_STRING
@@ -1046,7 +1056,7 @@ static int filter_pred_within(struct filter_pred *pred, void *event)
static int regex_match_full(char *str, struct regex *r, int len)
{
- /* len of zero means str is dynamic and ends with '\0' */
+ /* Zero is reserved for a NUL-terminated string of unknown length. */
if (!len)
return strcmp(str, r->pattern) == 0;
@@ -1082,6 +1092,30 @@ static int regex_match_end(char *str, struct regex *r, int len)
return 0;
}
+static int regex_match_end_str(char *str, struct regex *r, int len)
+{
+ int slen;
+
+ if (len < r->len)
+ return 0;
+
+ slen = strnlen(str, len);
+ if (slen >= r->len &&
+ memcmp(str + slen - r->len, r->pattern, r->len) == 0)
+ return 1;
+ return 0;
+}
+
+static int regex_match_end_dynstr(char *str, struct regex *r, int len)
+{
+ int slen = strlen(str);
+
+ if (slen >= r->len &&
+ memcmp(str + slen - r->len, r->pattern, r->len) == 0)
+ return 1;
+ return 0;
+}
+
static int regex_match_glob(char *str, struct regex *r, int len)
{
return glob_match_len(r->pattern, str, len) ? 1 : 0;
@@ -1118,9 +1152,6 @@ enum regex_type filter_parse_regex(char *buff, int len, char **search, int *not)
*search = buff;
- if (isdigit(buff[0]))
- return MATCH_INDEX;
-
for (i = 0; i < len; i++) {
if (buff[i] == '*') {
if (!i) {
@@ -1158,8 +1189,6 @@ static void filter_build_regex(struct filter_pred *pred)
}
switch (type) {
- /* MATCH_INDEX should not happen, but if it does, match full */
- case MATCH_INDEX:
case MATCH_FULL:
r->match = regex_match_full;
break;
@@ -1708,7 +1737,9 @@ static int parse_pred(const char *str, void *data,
/* See if the field is a kernel function name */
if ((len = str_has_prefix(str + i, ".function"))) {
- function = true;
+ /* Ignore if event is already a function */
+ if (!ftrace_event_is_function(call))
+ function = true;
i += len;
}
@@ -1732,6 +1763,11 @@ static int parse_pred(const char *str, void *data,
while (isspace(str[i]))
i++;
+ if (!str[i]) {
+ parse_error(pe, FILT_ERR_INVALID_FILTER, pos + i);
+ goto err_free;
+ }
+
s = i;
pred = kzalloc_obj(*pred);
@@ -2021,10 +2057,18 @@ static int parse_pred(const char *str, void *data,
if (field->filter_type == FILTER_COMM) {
pred->fn_num = FILTER_PRED_FN_COMM;
+ if (pred->regex->match == regex_match_end)
+ pred->regex->match = regex_match_end_str;
} else if (field->filter_type == FILTER_STATIC_STRING) {
pred->fn_num = FILTER_PRED_FN_STRING;
pred->regex->field_len = field->size;
+ if (pred->regex->match == regex_match_end) {
+ if (!field->size)
+ pred->regex->match = regex_match_end_dynstr;
+ else
+ pred->regex->match = regex_match_end_str;
+ }
} else if (field->filter_type == FILTER_DYN_STRING) {
pred->fn_num = FILTER_PRED_FN_STRLOC;
@@ -2305,6 +2349,7 @@ static int process_system_preds(struct trace_subsystem_dir *dir,
struct event_filter *filter = NULL;
struct filter_head *filter_list;
bool fail = true;
+ int ret = -ENOMEM;
int err;
filter_list = kmalloc_obj(*filter_list);
@@ -2331,9 +2376,10 @@ static int process_system_preds(struct trace_subsystem_dir *dir,
filter_disable(file);
parse_error(pe, FILT_ERR_BAD_SUBSYS_FILTER, 0);
append_filter_err(tr, pe, filter);
- } else
+ } else {
event_set_filtered_flag(file);
-
+ fail = false;
+ }
filter_item = kzalloc_obj(*filter_item);
if (!filter_item)
@@ -2347,8 +2393,6 @@ static int process_system_preds(struct trace_subsystem_dir *dir,
filter_item->filter = event_filter(file);
event_set_filter(file, filter);
filter = NULL;
-
- fail = false;
}
if (fail)
@@ -2363,19 +2407,18 @@ static int process_system_preds(struct trace_subsystem_dir *dir,
return 0;
fail:
/* No call succeeded */
- free_filter_list(filter_list);
parse_error(pe, FILT_ERR_BAD_SUBSYS_FILTER, 0);
- return -EINVAL;
+ ret = -EINVAL;
fail_mem:
__free_filter(filter);
- /* If any call succeeded, we still need to sync */
- if (!fail)
+ /* If any filter was replaced, we still need to sync */
+ if (!list_empty(&filter_list->list))
delay_free_filter(filter_list);
else
free_filter_list(filter_list);
- return -ENOMEM;
+ return ret;
}
static int create_filter_start(char *filter_string, bool set_str,
@@ -2749,6 +2792,28 @@ static int ftrace_function_set_filter(struct perf_event *event,
}
#endif /* CONFIG_FUNCTION_TRACER */
+static int ftrace_profile_filter_perm(struct event_filter *filter)
+{
+ struct prog_entry *prog = rcu_dereference_protected(filter->prog,
+ lockdep_is_held(&event_mutex));
+ int i, err;
+
+ for (i = 0; prog[i].pred; i++) {
+ if (prog[i].pred->fn_num != FILTER_PRED_FN_PCHAR)
+ continue;
+
+ /*
+ * Even a counting filter can reveal kernel string data.
+ * Require the same access as raw kernel tracepoints.
+ */
+ err = perf_allow_kernel();
+ if (err)
+ return err;
+ return perf_allow_tracepoint();
+ }
+ return 0;
+}
+
int ftrace_profile_set_filter(struct perf_event *event, int event_id,
char *filter_str)
{
@@ -2770,6 +2835,10 @@ int ftrace_profile_set_filter(struct perf_event *event, int event_id,
if (err)
goto free_filter;
+ err = ftrace_profile_filter_perm(filter);
+ if (err)
+ goto free_filter;
+
if (ftrace_event_is_function(call))
err = ftrace_function_set_filter(event, filter);
else
@@ -2789,6 +2858,8 @@ free_filter:
#include <linux/types.h>
#include <linux/tracepoint.h>
+#include "trace_probe.h"
+
#define CREATE_TRACE_POINTS
#include "trace_events_filter_test.h"
@@ -2800,12 +2871,22 @@ free_filter:
.match = m, \
.not_visited = nvisit, \
}
+
+#define DATA_STR(m, s) \
+{ \
+ .filter = FILTER, \
+ .str = s, \
+ .match = m, \
+ .not_visited = "", \
+}
+
#define YES 1
#define NO 0
static struct test_filter_data_t {
char *filter;
struct trace_event_raw_ftrace_test_filter rec;
+ const char *str;
int match;
char *not_visited;
} test_filter_data[] = {
@@ -2857,9 +2938,46 @@ static struct test_filter_data_t {
DATA_REC(YES, 1, 1, 1, 1, 1, 1, 1, 1, "bdfh"),
DATA_REC(YES, 0, 1, 0, 1, 0, 1, 0, 1, ""),
DATA_REC(YES, 1, 0, 1, 0, 1, 0, 1, 0, "bdfh"),
+#undef FILTER
+#define FILTER "str ~ \"0001:*\""
+ DATA_STR(YES, "0001:01:00.0"),
+ DATA_STR(YES, "0001:"),
+ DATA_STR(NO, "0002:01:00.0"),
+ DATA_STR(NO, "pci:0001:01:00.0"),
+ DATA_STR(NO, "0001"),
+#undef FILTER
+#define FILTER "str ~ \"!0001:*\""
+ DATA_STR(NO, "0001:01:00.0"),
+ DATA_STR(YES, "0002:01:00.0"),
+#undef FILTER
+#define FILTER "str ~ \"0001:*:00.[0-9]\""
+ DATA_STR(YES, "0001:01:00.0"),
+ DATA_STR(YES, "0001:ff:00.7"),
+ DATA_STR(NO, "0001:01:00.a"),
+ DATA_STR(NO, "0002:01:00.0"),
+#undef FILTER
+#define FILTER "str ~ \"0001:01:00.0\""
+ DATA_STR(YES, "0001:01:00.0"),
+ DATA_STR(NO, "0001:01:00.1"),
+ DATA_STR(NO, "0001:01:00.00"),
+ DATA_STR(NO, "0001:01:00"),
+#undef FILTER
+#define FILTER "str == \"0001:01:00.0\""
+ DATA_STR(YES, "0001:01:00.0"),
+ DATA_STR(NO, "0001:01:00.1"),
+#undef FILTER
+#define FILTER "str ~ \"*0001:*\""
+ DATA_STR(YES, "0001:01:00.0"),
+ DATA_STR(YES, "pci:0001:01:00.0"),
+ DATA_STR(NO, "0002:01:00.0"),
+#undef FILTER
+#define FILTER "str ~ \"*:00.0\""
+ DATA_STR(YES, "0001:01:00.0"),
+ DATA_STR(NO, "0001:01:00.1"),
};
#undef DATA_REC
+#undef DATA_STR
#undef FILTER
#undef YES
#undef NO
@@ -2910,8 +3028,29 @@ static __init int ftrace_test_event_filter(void)
for (i = 0; i < DATA_CNT; i++) {
struct event_filter *filter = NULL;
struct test_filter_data_t *d = &test_filter_data[i];
+ union {
+ struct trace_event_raw_ftrace_test_filter rec;
+ /* Add some spare, to append the string contents. */
+ char buf[sizeof(struct trace_event_raw_ftrace_test_filter) + 32];
+ } event = { .rec = d->rec };
int err;
+ if (d->str) {
+ struct ftrace_event_field *field;
+ u32 offset = offsetof(typeof(event.rec), __data);
+ u32 len = strlen(d->str) + 1;
+ u32 *str_item;
+
+ field = trace_find_event_field(&event_ftrace_test_filter, "str");
+ if (WARN_ON_ONCE(!field))
+ break;
+ if (WARN_ON_ONCE(offset + len > sizeof(event.buf)))
+ break;
+ str_item = (u32 *)&event.buf[field->offset];
+ *str_item = make_data_loc(len, offset);
+ memcpy(&event.buf[offset], d->str, len);
+ }
+
err = create_filter(NULL, &event_ftrace_test_filter,
d->filter, false, &filter);
if (err) {
@@ -2933,7 +3072,7 @@ static __init int ftrace_test_event_filter(void)
update_pred_fn(filter, d->not_visited);
test_pred_visited = 0;
- err = filter_match_preds(filter, &d->rec);
+ err = filter_match_preds(filter, &event.rec);
preempt_enable();
mutex_unlock(&event_mutex);
@@ -2960,7 +3099,7 @@ static __init int ftrace_test_event_filter(void)
/* Need to call ftrace_test_filter to prevent a warning */
if (!trace_ftrace_test_filter_enabled())
- trace_ftrace_test_filter(1, 2, 3, 4, 5, 6, 7, 8);
+ trace_ftrace_test_filter(1, 2, 3, 4, 5, 6, 7, 8, "");
return 0;
}
diff --git a/kernel/trace/trace_events_filter_test.h b/kernel/trace/trace_events_filter_test.h
index e651dfbd345e..74a9dba91545 100644
--- a/kernel/trace/trace_events_filter_test.h
+++ b/kernel/trace/trace_events_filter_test.h
@@ -9,9 +9,10 @@
TRACE_EVENT(ftrace_test_filter,
- TP_PROTO(int a, int b, int c, int d, int e, int f, int g, int h),
+ TP_PROTO(int a, int b, int c, int d, int e, int f, int g, int h,
+ const char *str),
- TP_ARGS(a, b, c, d, e, f, g, h),
+ TP_ARGS(a, b, c, d, e, f, g, h, str),
TP_STRUCT__entry(
__field(int, a)
@@ -22,6 +23,7 @@ TRACE_EVENT(ftrace_test_filter,
__field(int, f)
__field(int, g)
__field(int, h)
+ __string(str, str)
),
TP_fast_assign(
@@ -33,11 +35,13 @@ TRACE_EVENT(ftrace_test_filter,
__entry->f = f;
__entry->g = g;
__entry->h = h;
+ __assign_str(str);
),
- TP_printk("a %d, b %d, c %d, d %d, e %d, f %d, g %d, h %d",
+ TP_printk("a %d, b %d, c %d, d %d, e %d, f %d, g %d, h %d, str %s",
__entry->a, __entry->b, __entry->c, __entry->d,
- __entry->e, __entry->f, __entry->g, __entry->h)
+ __entry->e, __entry->f, __entry->g, __entry->h,
+ __get_str(str))
);
#endif /* _TRACE_TEST_H || TRACE_HEADER_MULTI_READ */
diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c
index 1aae7d7c6dc5..d4702ad4e003 100644
--- a/kernel/trace/trace_events_hist.c
+++ b/kernel/trace/trace_events_hist.c
@@ -1430,6 +1430,11 @@ static int parse_map_size(char *str)
if (ret)
goto out;
+ if (!size) {
+ ret = -EINVAL;
+ goto out;
+ }
+
map_bits = ilog2(roundup_pow_of_two(size));
if (map_bits < TRACING_MAP_BITS_MIN ||
map_bits > TRACING_MAP_BITS_MAX)
@@ -6761,6 +6766,17 @@ static bool hist_trigger_check_refs(struct event_trigger_data *data,
return false;
}
+void hist_put_synth_event(struct trace_event_file *file)
+{
+ struct synth_event *se;
+
+ lockdep_assert_held(&event_mutex);
+
+ se = find_synth_event_call(file->event_call);
+ if (se)
+ se->ref--;
+}
+
static void hist_unregister_trigger(char *glob,
struct event_trigger_data *data,
struct trace_event_file *file)
@@ -6816,7 +6832,6 @@ static void hist_unreg_all(struct trace_event_file *file)
{
struct event_trigger_data *test, *n;
struct hist_trigger_data *hist_data;
- struct synth_event *se;
lockdep_assert_held(&event_mutex);
@@ -6829,9 +6844,7 @@ static void hist_unreg_all(struct trace_event_file *file)
list_del_rcu(&test->list);
trace_event_trigger_enable_disable(file, 0);
- se = find_synth_event_call(file->event_call);
- if (se)
- se->ref--;
+ hist_put_synth_event(file);
update_cond_flag(file);
if (hist_data->enable_timestamps)
@@ -6946,9 +6959,7 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops,
}
event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
- se = find_synth_event_call(file->event_call);
- if (se)
- se->ref--;
+ hist_put_synth_event(file);
ret = 0;
goto out_free;
}
diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c
index 4d2fde996c0f..a2487ad7fef8 100644
--- a/kernel/trace/trace_events_trigger.c
+++ b/kernel/trace/trace_events_trigger.c
@@ -643,6 +643,8 @@ clear_event_triggers(struct trace_array *tr)
list_for_each_entry_safe(data, n, &file->triggers, list) {
trace_event_trigger_enable_disable(file, 0);
list_del_rcu(&data->list);
+ if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
+ hist_put_synth_event(file);
if (data->cmd_ops->free)
data->cmd_ops->free(data);
}
diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c
index f658c3a77aa7..e29e1c772aa7 100644
--- a/kernel/trace/trace_events_user.c
+++ b/kernel/trace/trace_events_user.c
@@ -256,6 +256,11 @@ static void delayed_destroy_user_event(struct work_struct *work)
* attempt to delete it later.
*/
pr_warn("user_events: Unable to delete event\n");
+ /*
+ * Keep it as persistent so it can be still deleted
+ * since nobody own this event
+ */
+ user->reg_flags |= USER_EVENT_REG_PERSIST;
refcount_set(&user->refcnt, 1);
}
out:
@@ -1035,6 +1040,9 @@ static int user_event_parse_cmd(struct user_event_group *group,
if (flags)
*flags++ = '\0';
+ if (strstr(name, "/") != NULL)
+ return -EINVAL;
+
return user_event_parse(group, name, args, flags, newuser, reg_flags);
}
@@ -1506,6 +1514,10 @@ static int destroy_user_event(struct user_event *user)
lockdep_assert_held(&event_mutex);
+ /* If there is a reference to a dynamic event */
+ if (trace_event_dyn_busy(&user->call))
+ return -EBUSY;
+
/*
* Detach the fields before removing the call. Removing the event
* frees the field list memory (trace_destroy_fields() is run on
diff --git a/kernel/trace/trace_hwlat.c b/kernel/trace/trace_hwlat.c
index 3fe274b84f1c..5be9031f1ffe 100644
--- a/kernel/trace/trace_hwlat.c
+++ b/kernel/trace/trace_hwlat.c
@@ -203,14 +203,12 @@ static int get_sample(void)
time_type start, t1, t2, last_t2;
s64 diff, outer_diff, total, last_total = 0;
u64 sample = 0;
- u64 sample_width = READ_ONCE(hwlat_data.sample_width);
+ u64 sample_width = READ_ONCE(hwlat_data.sample_width) * NSEC_PER_USEC;
u64 thresh = tracing_thresh;
u64 outer_sample = 0;
- int ret = -1;
+ int ret = 0;
unsigned int count = 0;
- do_div(thresh, NSEC_PER_USEC); /* modifies interval value */
-
kdata->nmi_total_ts = 0;
kdata->nmi_count = 0;
/* Make sure NMIs see this first */
@@ -229,28 +227,37 @@ static int get_sample(void)
if (time_u64(last_t2)) {
/* Check the delta from outer loop (t2 to next t1) */
- outer_diff = time_to_us(time_sub(t1, last_t2));
+ outer_diff = time_sub(t1, last_t2);
/* This shouldn't happen */
if (outer_diff < 0) {
hwlat_err(BANNER "time running backwards\n");
- goto out;
+ ret = -1;
+ break;
}
if (outer_diff > outer_sample)
outer_sample = outer_diff;
}
last_t2 = t2;
- total = time_to_us(time_sub(t2, start)); /* sample width */
+ total = time_sub(t2, start); /* sample width */
/* Check for possible overflows */
if (total < last_total) {
hwlat_err("Time total overflowed\n");
+ /* Keep ret == 0 to preserve any latencies already detected */
break;
}
last_total = total;
/* This checks the inner loop (t1 to t2) */
- diff = time_to_us(time_sub(t2, t1)); /* current diff */
+ diff = time_sub(t2, t1); /* current diff */
+
+ /* This shouldn't happen: time went backwards between t1 and t2 */
+ if (diff < 0) {
+ hwlat_err(BANNER "time running backwards\n");
+ ret = -1;
+ break;
+ }
if (diff > thresh || outer_diff > thresh) {
if (!count)
@@ -258,12 +265,6 @@ static int get_sample(void)
count++;
}
- /* This shouldn't happen */
- if (diff < 0) {
- hwlat_err(BANNER "time running backwards\n");
- goto out;
- }
-
if (diff > sample)
sample = diff; /* only want highest value */
@@ -273,10 +274,11 @@ static int get_sample(void)
trace_hwlat_callback_enabled = false;
barrier(); /* Make sure nmi_total_ts is no longer updated */
- ret = 0;
+ if (ret)
+ return ret;
/* If we exceed the threshold value, we have found a hardware latency */
- if (sample > thresh || outer_sample > thresh) {
+ if (count) {
u64 latency;
ret = 1;
@@ -286,8 +288,8 @@ static int get_sample(void)
do_div(kdata->nmi_total_ts, NSEC_PER_USEC);
s.seqnum = atomic64_inc_return(&hwlat_data.count);
- s.duration = sample;
- s.outer_duration = outer_sample;
+ s.duration = time_to_us(sample);
+ s.outer_duration = time_to_us(outer_sample);
s.nmi_total_ts = kdata->nmi_total_ts;
s.nmi_count = kdata->nmi_count;
s.count = count;
@@ -301,8 +303,6 @@ static int get_sample(void)
latency_fsnotify(tr);
}
}
-
-out:
return ret;
}
@@ -835,7 +835,6 @@ static int hwlat_tracer_init(struct trace_array *tr)
tr->max_latency = 0;
save_tracing_thresh = tracing_thresh;
- /* tracing_thresh is in nsecs, we speak in usecs */
if (!tracing_thresh)
tracing_thresh = last_tracing_thresh;
diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c
index 995da9e0b548..fbba42286747 100644
--- a/kernel/trace/trace_osnoise.c
+++ b/kernel/trace/trace_osnoise.c
@@ -2039,6 +2039,14 @@ static int start_kthread(unsigned int cpu)
return 0;
if (timerlat_enabled()) {
+ /*
+ * In timerlat user-space mode the workload is the user task
+ * reading timerlat_fd, so there is no kernel thread to start.
+ * One started here would set its kthread_cpumask bit, which
+ * stop_kthread() reads as "osnoise workload kthread".
+ */
+ if (!test_bit(OSN_WORKLOAD, &osnoise_options))
+ return 0;
snprintf(comm, 24, "timerlat/%d", cpu);
main = timerlat_main;
} else {