diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/bpf/bpf_lsm.c | 2 | ||||
| -rw-r--r-- | kernel/bpf/btf.c | 7 | ||||
| -rw-r--r-- | kernel/bpf/cgroup.c | 163 | ||||
| -rw-r--r-- | kernel/bpf/core.c | 20 | ||||
| -rw-r--r-- | kernel/bpf/local_storage.c | 8 | ||||
| -rw-r--r-- | kernel/bpf/map_iter.c | 12 | ||||
| -rw-r--r-- | kernel/bpf/trampoline.c | 2 | ||||
| -rw-r--r-- | kernel/bpf/verifier.c | 7 | ||||
| -rw-r--r-- | kernel/events/uprobes.c | 9 | ||||
| -rw-r--r-- | kernel/time/jiffies.c | 21 | ||||
| -rw-r--r-- | kernel/trace/bpf_trace.c | 10 | ||||
| -rw-r--r-- | kernel/trace/fgraph.c | 59 | ||||
| -rw-r--r-- | kernel/trace/ftrace.c | 3 | ||||
| -rw-r--r-- | kernel/trace/ring_buffer.c | 60 | ||||
| -rw-r--r-- | kernel/trace/simple_ring_buffer.c | 26 | ||||
| -rw-r--r-- | kernel/trace/trace.c | 45 | ||||
| -rw-r--r-- | kernel/trace/trace.h | 3 | ||||
| -rw-r--r-- | kernel/trace/trace_events.c | 8 | ||||
| -rw-r--r-- | kernel/trace/trace_events_filter.c | 203 | ||||
| -rw-r--r-- | kernel/trace/trace_events_filter_test.h | 12 | ||||
| -rw-r--r-- | kernel/trace/trace_events_hist.c | 25 | ||||
| -rw-r--r-- | kernel/trace/trace_events_trigger.c | 2 | ||||
| -rw-r--r-- | kernel/trace/trace_events_user.c | 12 | ||||
| -rw-r--r-- | kernel/trace/trace_hwlat.c | 41 | ||||
| -rw-r--r-- | kernel/trace/trace_osnoise.c | 8 |
25 files changed, 572 insertions, 196 deletions
diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 468e63dbd922..ae23511fd811 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -411,6 +411,8 @@ BTF_ID(func, bpf_lsm_bpf_map_free) BTF_ID(func, bpf_lsm_bpf_prog_free) BTF_ID(func, bpf_lsm_file_alloc_security) BTF_ID(func, bpf_lsm_file_free_security) +BTF_ID(func, bpf_lsm_backing_file_alloc) +BTF_ID(func, bpf_lsm_backing_file_free) BTF_ID(func, bpf_lsm_namespace_init) BTF_ID(func, bpf_lsm_namespace_free) #ifdef CONFIG_SECURITY_NETWORK diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 7e866bf017b5..49fcd32d418a 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8094,7 +8094,12 @@ int btf_struct_access(struct bpf_verifier_log *log, for (i = 0; i < rec->cnt; i++) { struct btf_field *field = &rec->fields[i]; u32 offset = field->offset; - if (off < offset + field->size && offset < off + size) { + /* An access at 'lo' or above ends after the start of the field. */ + s64 lo = max(min_off, (s64)offset - size + 1); + + /* Round up to min_off + k * step. The step is never 0. */ + lo += imod(min_off - lo, reg->step); + if (lo <= max_off && lo < offset + field->size) { bpf_log(log, "direct access to %s is disallowed\n", btf_field_type_name(field->type)); diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 57a785241d42..f195d5fcae6c 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -206,13 +206,53 @@ unsigned int __cgroup_bpf_run_lsm_current(const void *ctx, return ret; } +/* + * cgroup_bpf_mutex protects the bpf state of all cgroups: prog lists, + * effective arrays, flags, revisions, storage lists, link->cgroup, + * link->map of struct_ops links and cgroup_lsm_atype[]. + * + * Take cgroup_mutex first if you need both. This mutex doesn't keep a + * cgroup alive or the hierarchy stable. Attach, detach and replace walk + * the descendants, so they take both. Everything else takes only this one. + * + * cgroup_bpf_inherit() runs at mkdir with only cgroup_mutex. That's safe: + * it writes the new cgroup, which no one else can see yet, and what it + * reads from the ancestors changes only under cgroup_mutex. The ancestors + * are online, and only rmdir, which needs cgroup_mutex, starts a release. + */ +static DEFINE_MUTEX(cgroup_bpf_mutex); + +#define cgroup_bpf_held() lockdep_is_held(&cgroup_bpf_mutex) + +void cgroup_bpf_lock(void) +{ + mutex_lock(&cgroup_bpf_mutex); +} + +void cgroup_bpf_unlock(void) +{ + mutex_unlock(&cgroup_bpf_mutex); +} + +static void cgroup_bpf_lock_hierarchy(void) +{ + cgroup_lock(); + mutex_lock(&cgroup_bpf_mutex); +} + +static void cgroup_bpf_unlock_hierarchy(void) +{ + mutex_unlock(&cgroup_bpf_mutex); + cgroup_unlock(); +} + #ifdef CONFIG_BPF_LSM static enum cgroup_bpf_attach_type bpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id) { int i; - lockdep_assert_held(&cgroup_mutex); + lockdep_assert_held(&cgroup_bpf_mutex); if (attach_type != BPF_LSM_CGROUP) return to_cgroup_bpf_attach_type(attach_type); @@ -233,7 +273,7 @@ void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) { int i = cgroup_atype - CGROUP_LSM_START; - lockdep_assert_held(&cgroup_mutex); + lockdep_assert_held(&cgroup_bpf_mutex); if (!cgroup_lsm_atype[i].attach_btf_id) { cgroup_lsm_atype[i].attach_btf_id = attach_btf_id; @@ -249,13 +289,13 @@ void bpf_cgroup_atype_put(int cgroup_atype) { int i = cgroup_atype - CGROUP_LSM_START; - cgroup_lock(); + cgroup_bpf_lock(); if (--cgroup_lsm_atype[i].refcnt <= 0) { WRITE_ONCE(cgroup_lsm_atype[i].returns_errno, true); cgroup_lsm_atype[i].attach_btf_id = 0; } WARN_ON_ONCE(cgroup_lsm_atype[i].refcnt < 0); - cgroup_unlock(); + cgroup_bpf_unlock(); } #else static enum cgroup_bpf_attach_type @@ -424,7 +464,7 @@ static int bpf_cgroup_array_copy_to_user(struct bpf_prog_array *array, return -EFAULT; i++; } - return item->prog ? -ENOSPC : 0; + return 0; } static int bpf_cgroup_array_delete_safe_at(struct bpf_prog_array *array, @@ -459,7 +499,7 @@ static void cgroup_bpf_release(struct work_struct *work) unsigned int atype; - cgroup_lock(); + cgroup_bpf_lock(); for (atype = 0; atype < ARRAY_SIZE(cgrp->bpf.progs); atype++) { struct hlist_head *progs = &cgrp->bpf.progs[atype]; @@ -480,7 +520,7 @@ static void cgroup_bpf_release(struct work_struct *work) } old_array = rcu_dereference_protected( cgrp->bpf.effective[atype], - lockdep_is_held(&cgroup_mutex)); + cgroup_bpf_held()); bpf_cgroup_array_free(old_array, atype); } @@ -489,7 +529,7 @@ static void cgroup_bpf_release(struct work_struct *work) bpf_cgroup_storage_free(storage); } - cgroup_unlock(); + cgroup_bpf_unlock(); for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p)) cgroup_bpf_put(p); @@ -665,7 +705,9 @@ static void activate_effective_progs(struct cgroup *cgrp, enum cgroup_bpf_attach_type atype, struct bpf_prog_array *old_array) { + /* cgroup_bpf_inherit() holds only cgroup_mutex, see cgroup_bpf_mutex */ old_array = rcu_replace_pointer(cgrp->bpf.effective[atype], old_array, + lockdep_is_held(&cgroup_bpf_mutex) || lockdep_is_held(&cgroup_mutex)); /* free prog array after grace period, since __cgroup_bpf_run_*() * might be still walking the array @@ -687,6 +729,9 @@ static int cgroup_bpf_inherit(struct cgroup *cgrp) struct cgroup *p; int ret, i; + /* See the comment above cgroup_bpf_mutex */ + lockdep_assert_held(&cgroup_mutex); + ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0, GFP_KERNEL); if (ret) @@ -747,6 +792,8 @@ static int update_effective_progs(struct cgroup *cgrp, struct cgroup_subsys_state *css; int err; + lockdep_assert_held(&cgroup_bpf_mutex); + /* allocate and recompute effective prog arrays */ css_for_each_descendant_pre(css, &cgrp->self) { struct cgroup *desc = container_of(css, struct cgroup, self); @@ -954,7 +1001,7 @@ static int insert_pl_to_hlist(struct bpf_prog_list *pl, struct hlist_head *progs * @revision: bpf_prog_list revision * * Exactly one of @prog or @link can be non-null. - * Must be called with cgroup_mutex held. + * Must be called with cgroup_mutex and cgroup_bpf_mutex held. */ static int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog, struct bpf_prog *replace_prog, @@ -988,6 +1035,13 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, /* replace_prog implies BPF_F_REPLACE, and vice versa */ return -EINVAL; + /* + * The cgroup may have been removed since the caller looked it up. + * Then its bpf state may be freed, and nothing would detach this. + */ + if (cgroup_is_dead(cgrp)) + return -ENOENT; + atype = bpf_cgroup_atype_find(type, new_prog->aux->attach_btf_id); if (atype < 0) return -EINVAL; @@ -1093,10 +1147,10 @@ static int cgroup_bpf_attach(struct cgroup *cgrp, { int ret; - cgroup_lock(); + cgroup_bpf_lock_hierarchy(); ret = __cgroup_bpf_attach(cgrp, prog, replace_prog, link, type, flags, id_or_fd, revision); - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); return ret; } @@ -1174,7 +1228,7 @@ static void replace_effective_prog(struct cgroup *cgrp, progs = rcu_dereference_protected( desc->bpf.effective[atype], - lockdep_is_held(&cgroup_mutex)); + cgroup_bpf_held()); item = &progs->items[pos]; prog_list_replace_item(pl, item); } @@ -1202,7 +1256,7 @@ static bool cgroup_bpf_storages_compatible(struct bpf_prog *old_prog, * @new_prog: &struct bpf_prog for the target BPF program with its refcnt * incremented * - * Must be called with cgroup_mutex held. + * Must be called with cgroup_mutex and cgroup_bpf_mutex held. */ static int __cgroup_bpf_replace(struct cgroup *cgrp, struct bpf_cgroup_link *link, @@ -1250,7 +1304,7 @@ static int cgroup_bpf_replace(struct bpf_link *link, struct bpf_prog *new_prog, cg_link = container_of(link, struct bpf_cgroup_link, link); - cgroup_lock(); + cgroup_bpf_lock_hierarchy(); /* link might have been auto-released by dying cgroup, so fail */ if (!cg_link->cgroup) { ret = -ENOLINK; @@ -1262,7 +1316,7 @@ static int cgroup_bpf_replace(struct bpf_link *link, struct bpf_prog *new_prog, } ret = __cgroup_bpf_replace(cg_link->cgroup, cg_link, new_prog); out_unlock: - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); return ret; } @@ -1333,7 +1387,7 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl, progs = rcu_dereference_protected( desc->bpf.effective[atype], - lockdep_is_held(&cgroup_mutex)); + cgroup_bpf_held()); /* Remove the program from the array */ WARN_ONCE(bpf_cgroup_array_delete_safe_at(progs, pos, atype), @@ -1351,7 +1405,7 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl, * @revision: bpf_prog_list revision * * At most one of @prog or @link can be non-NULL. - * Must be called with cgroup_mutex held. + * Must be called with cgroup_mutex and cgroup_bpf_mutex held. */ static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, struct bpf_cgroup_link *link, enum bpf_attach_type type, @@ -1421,13 +1475,13 @@ static int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, { int ret; - cgroup_lock(); + cgroup_bpf_lock_hierarchy(); ret = __cgroup_bpf_detach(cgrp, prog, NULL, type, revision); - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); return ret; } -/* Must be called with cgroup_mutex held to avoid races. */ +/* Must be called with cgroup_bpf_mutex held. */ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, union bpf_attr __user *uattr, u32 uattr_size) { @@ -1446,6 +1500,14 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, if (effective_query && prog_attach_flags) return -EINVAL; + /* + * cgroup_bpf_release() frees the effective arrays once the bpf + * refcount is zero. It takes this lock too, so a nonzero count means + * the arrays are still valid. + */ + if (percpu_ref_is_zero(&cgrp->bpf.refcnt)) + return -ENOENT; + if (type == BPF_STRUCT_OPS) { u32 type_id = attr->query.type_id; @@ -1473,7 +1535,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, for (atype = from_atype; atype <= to_atype; atype++) { if (effective_query) { effective = rcu_dereference_protected(cgrp->bpf.effective[atype], - lockdep_is_held(&cgroup_mutex)); + cgroup_bpf_held()); total_cnt += bpf_cgroup_array_length(effective, atype); } else { total_cnt += prog_list_length(&cgrp->bpf.progs[atype], NULL); @@ -1503,9 +1565,10 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, for (atype = from_atype; atype <= to_atype && total_cnt; atype++) { if (effective_query) { effective = rcu_dereference_protected(cgrp->bpf.effective[atype], - lockdep_is_held(&cgroup_mutex)); + cgroup_bpf_held()); cnt = min_t(int, bpf_cgroup_array_length(effective, atype), total_cnt); - ret = bpf_cgroup_array_copy_to_user(effective, prog_ids, cnt, atype); + if (bpf_cgroup_array_copy_to_user(effective, prog_ids, cnt, atype)) + return -EFAULT; } else { struct hlist_head *progs; struct bpf_prog_list *pl; @@ -1544,9 +1607,9 @@ static int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, { int ret; - cgroup_lock(); + cgroup_bpf_lock(); ret = __cgroup_bpf_query(cgrp, attr, uattr, uattr_size); - cgroup_unlock(); + cgroup_bpf_unlock(); return ret; } @@ -1614,11 +1677,11 @@ static void bpf_cgroup_link_release(struct bpf_link *link) if (!cg_link->cgroup) return; - cgroup_lock(); + cgroup_bpf_lock_hierarchy(); /* re-check cgroup under lock again */ if (!cg_link->cgroup) { - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); return; } @@ -1630,7 +1693,7 @@ static void bpf_cgroup_link_release(struct bpf_link *link) cg = cg_link->cgroup; cg_link->cgroup = NULL; - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); cgroup_put(cg); } @@ -1657,10 +1720,10 @@ static void bpf_cgroup_link_show_fdinfo(const struct bpf_link *link, container_of(link, struct bpf_cgroup_link, link); u64 cg_id = 0; - cgroup_lock(); + cgroup_bpf_lock(); if (cg_link->cgroup) cg_id = cgroup_id(cg_link->cgroup); - cgroup_unlock(); + cgroup_bpf_unlock(); seq_printf(seq, "cgroup_id:\t%llu\n" @@ -1676,10 +1739,10 @@ static int bpf_cgroup_link_fill_link_info(const struct bpf_link *link, container_of(link, struct bpf_cgroup_link, link); u64 cg_id = 0; - cgroup_lock(); + cgroup_bpf_lock(); if (cg_link->cgroup) cg_id = cgroup_id(cg_link->cgroup); - cgroup_unlock(); + cgroup_bpf_unlock(); info->cgroup.cgroup_id = cg_id; info->cgroup.attach_type = link->attach_type; @@ -2956,11 +3019,11 @@ static int __cgroup_struct_ops_link_detach(struct bpf_link *link, bool wake_poll struct bpf_map *map; struct cgroup *cgrp; - cgroup_lock(); + cgroup_bpf_lock_hierarchy(); cgrp = cg_link->cgroup; if (!cgrp) { - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); return 0; } @@ -2972,7 +3035,7 @@ static int __cgroup_struct_ops_link_detach(struct bpf_link *link, bool wake_poll break; } if (WARN_ON_ONCE(!pl)) { - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); return -ENOENT; } @@ -2990,7 +3053,7 @@ static int __cgroup_struct_ops_link_detach(struct bpf_link *link, bool wake_poll kfree(pl); static_branch_dec(&cgroup_bpf_enabled_key[atype]); - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); return 0; } @@ -3013,15 +3076,15 @@ static void cgroup_struct_ops_link_show_fdinfo(const struct bpf_link *link, stru struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); - cgroup_lock(); + cgroup_bpf_lock(); if (!cg_link->cgroup) { - cgroup_unlock(); + cgroup_bpf_unlock(); return; } seq_printf(seq, "map_id:\t%u\n", cg_link->map->id); seq_printf(seq, "cgroup_id:\t%llu\n", cgroup_id(cg_link->cgroup)); - cgroup_unlock(); + cgroup_bpf_unlock(); } static int cgroup_struct_ops_link_fill_link_info(const struct bpf_link *link, @@ -3029,15 +3092,15 @@ static int cgroup_struct_ops_link_fill_link_info(const struct bpf_link *link, { struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); - cgroup_lock(); + cgroup_bpf_lock(); if (!cg_link->cgroup) { - cgroup_unlock(); + cgroup_bpf_unlock(); return 0; } info->struct_ops.map_id = cg_link->map->id; info->struct_ops.cgroup_id = cgroup_id(cg_link->cgroup); - cgroup_unlock(); + cgroup_bpf_unlock(); return 0; } @@ -3055,7 +3118,7 @@ static int cgroup_struct_ops_link_update(struct bpf_link *link, struct bpf_map * if (!bpf_struct_ops_valid_to_reg(new_map)) return -EINVAL; - cgroup_lock(); + cgroup_bpf_lock_hierarchy(); cgrp = cg_link->cgroup; if (!cgrp) { @@ -3088,7 +3151,7 @@ static int cgroup_struct_ops_link_update(struct bpf_link *link, struct bpf_map * cgrp->bpf.revisions[atype]++; out: - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); return err; } @@ -3160,7 +3223,13 @@ int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr atype = bpf_struct_ops_map_cgroup_atype(map); progs = &cgrp->bpf.progs[atype]; - cgroup_lock(); + cgroup_bpf_lock_hierarchy(); + + /* See the cgroup_is_dead() check in __cgroup_bpf_attach() */ + if (cgroup_is_dead(cgrp)) { + err = -ENOENT; + goto unlock; + } if (attr->link_create.cgroup.expected_revision && attr->link_create.cgroup.expected_revision != cgrp->bpf.revisions[atype]) { @@ -3197,12 +3266,12 @@ int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr cgrp->bpf.revisions[atype]++; static_branch_inc(&cgroup_bpf_enabled_key[atype]); - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); return bpf_link_settle(&link_primer); unlock: - cgroup_unlock(); + cgroup_bpf_unlock_hierarchy(); cleanup: kfree(pl); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 0b8456f2dce9..0bfd23a41401 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -24,6 +24,7 @@ #include <linux/skbuff.h> #include <linux/static_call.h> #include <linux/vmalloc.h> +#include <linux/workqueue.h> #include <linux/prandom.h> #include <linux/bpf.h> #include <linux/btf.h> @@ -3143,6 +3144,17 @@ static void bpf_free_used_btfs(struct bpf_prog_aux *aux) kfree(aux->used_btfs); } +struct workqueue_struct *bpf_prog_free_wq __ro_after_init; + +static int __init bpf_prog_free_wq_init(void) +{ + bpf_prog_free_wq = alloc_workqueue("bpf_prog_free", WQ_PERCPU, 1); + if (!bpf_prog_free_wq) + panic("Failed to allocate BPF program free workqueue\n"); + return 0; +} +early_initcall(bpf_prog_free_wq_init); + static void bpf_prog_free_images(struct work_struct *work) { struct bpf_prog_aux *aux; @@ -3198,7 +3210,7 @@ static void bpf_prog_free_deferred(struct work_struct *work) /* The symbols are looked up under rcu_read_lock() */ INIT_RCU_WORK(&aux->rwork, bpf_prog_free_images); if (ksyms) - queue_rcu_work(system_percpu_wq, &aux->rwork); + queue_rcu_work(bpf_prog_free_wq, &aux->rwork); else bpf_prog_free_images(&aux->rwork.work); } @@ -3211,7 +3223,11 @@ void bpf_prog_free(struct bpf_prog *fp) bpf_prog_put(aux->dst_prog); bpf_token_put(aux->token); INIT_WORK(&aux->work, bpf_prog_free_deferred); - schedule_work(&aux->work); + /* + * Free BPF programs on a dedicated workqueue to avoid blocking or slowing + * down the system workqueue. + */ + queue_work(bpf_prog_free_wq, &aux->work); } /* RNG for unprivileged user space with separated state from prandom_u32(). */ diff --git a/kernel/bpf/local_storage.c b/kernel/bpf/local_storage.c index 83cd527a2542..2cfbddeb1c7e 100644 --- a/kernel/bpf/local_storage.c +++ b/kernel/bpf/local_storage.c @@ -345,14 +345,18 @@ static void cgroup_storage_map_free(struct bpf_map *_map) struct list_head *storages = &map->list; struct bpf_cgroup_storage *storage, *stmp; - cgroup_lock(); + /* + * Attach and cgroup_bpf_release() change the same cgroup storage + * lists under other maps' locks, so take cgroup_bpf_mutex. + */ + cgroup_bpf_lock(); list_for_each_entry_safe(storage, stmp, storages, list_map) { bpf_cgroup_storage_unlink(storage); bpf_cgroup_storage_free(storage); } - cgroup_unlock(); + cgroup_bpf_unlock(); WARN_ON(!RB_EMPTY_ROOT(&map->root)); WARN_ON(!list_empty(&map->list)); diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c index a58c148692e6..7a769e5f5397 100644 --- a/kernel/bpf/map_iter.c +++ b/kernel/bpf/map_iter.c @@ -99,8 +99,8 @@ static struct bpf_iter_reg bpf_map_reg_info = { /* * For an iterator that hands the live map value to the program as a writable - * buffer: the verifier knows nothing of the map, so the special fields of the - * value have to lie outside of what the program may write. + * buffer: the verifier knows nothing of the map, so read-only maps + * and special fields of the value have to be protected from writes. */ int bpf_iter_map_check_value_write(const struct bpf_prog *prog, const struct bpf_map *map) @@ -109,7 +109,13 @@ int bpf_iter_map_check_value_write(const struct bpf_prog *prog, const struct btf_field *field; u32 i; - if (IS_ERR_OR_NULL(rec) || !prog->aux->max_rdwr_write) + if (!prog->aux->max_rdwr_write) + return 0; + + if (map->map_flags & BPF_F_RDONLY_PROG) + return -EACCES; + + if (IS_ERR_OR_NULL(rec)) return 0; for (i = 0; i < rec->cnt; i++) { diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 21d460866dfe..b0052c7565d8 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -584,7 +584,7 @@ static void __bpf_tramp_image_put_rcu(struct rcu_head *rcu) im = container_of(rcu, struct bpf_tramp_image, rcu); INIT_WORK(&im->work, __bpf_tramp_image_put_deferred); - schedule_work(&im->work); + queue_work(bpf_prog_free_wq, &im->work); } /* callback, fexit step 2. Called after percpu_ref_kill confirms. */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5e4700fda66b..9b228ffbe3fe 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10526,6 +10526,7 @@ static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) { enum bpf_attach_type eatype = env->prog->expected_attach_type; enum bpf_prog_type type = resolve_prog_type(env->prog); + const char *afname = env->prog->aux->attach_func_name; if (func_id != BPF_FUNC_map_update_elem && func_id != BPF_FUNC_map_delete_elem) @@ -10536,7 +10537,8 @@ static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) */ switch (type) { case BPF_PROG_TYPE_TRACING: - if (eatype == BPF_TRACE_ITER) + if (eatype == BPF_TRACE_ITER && + !strcmp(afname + strlen(BPF_ITER_FUNC_PREFIX), "sockmap")) return true; break; case BPF_PROG_TYPE_SOCK_OPS: @@ -20627,8 +20629,7 @@ static int do_check(struct bpf_verifier_env *env) if (signal_pending(current)) return -EAGAIN; - if (need_resched()) - cond_resched(); + cond_resched_tasks_rcu_qs(); /* reset current history entry on each new instruction */ env->cur_hist_ent = NULL; diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index caf444762a27..c47b1ee266d5 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -2704,6 +2704,15 @@ void __weak arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vadd } /* + * Can a bpf prog write these bytes of pt_regs? An arch opts in and lists the + * registers that ptrace can set to any value. + */ +bool __weak arch_uprobe_regs_writable(int off, int size) +{ + return false; +} + +/* * Run handler and ask thread to singlestep. * Ensure all non-fatal signals cannot interrupt thread while it singlesteps. */ diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c index af1c69bf0c23..b861dc860d34 100644 --- a/kernel/time/jiffies.c +++ b/kernel/time/jiffies.c @@ -137,11 +137,19 @@ static int sysctl_k2u_int_conv_userhz(bool *negp, ulong *u_ptr, const int *k_ptr return proc_int_k2u_conv_kop(u_ptr, k_ptr, negp, sysctl_jiffies_to_clock_t); } +/* + * Calc ms <-> jiffies conversions for sysctls in unsigned long arithmetic. + * Think hard about using the non-sysctl converters (msecs_to_jiffies & + * jiffies_to_msecs) as the interaction between ulong and uint is not trivial. + */ static ulong sysctl_msecs_to_jiffies(const ulong val) { - if (val > jiffies_to_msecs(MAX_JIFFY_OFFSET)) - return MAX_JIFFY_OFFSET; - return msecs_to_jiffies(val); + ulong secs = val / MSEC_PER_SEC; + + if (secs >= ULONG_MAX / HZ) + return ULONG_MAX; + + return secs * HZ + DIV_ROUND_UP((val % MSEC_PER_SEC) * HZ, MSEC_PER_SEC); } static int sysctl_u2k_int_conv_ms(const bool *negp, const ulong *u_ptr, int *k_ptr) @@ -151,7 +159,12 @@ static int sysctl_u2k_int_conv_ms(const bool *negp, const ulong *u_ptr, int *k_p static ulong sysctl_jiffies_to_msecs(const ulong val) { - return jiffies_to_msecs(val); + ulong secs = val / HZ; + + if (secs >= ULONG_MAX / MSEC_PER_SEC) + return ULONG_MAX; + + return secs * MSEC_PER_SEC + (val % HZ) * MSEC_PER_SEC / HZ; } static int sysctl_k2u_int_conv_ms(bool *negp, ulong *u_ptr, const int *k_ptr) diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index d22e78e7f28d..1f35a18ef399 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -1397,8 +1397,16 @@ static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type if (off + size > sizeof(struct pt_regs)) return false; - if (type == BPF_WRITE) + if (type == BPF_WRITE) { + /* + * pt_regs also holds what the kernel restores on the way to + * user space: cs and flags on x86, pstate on arm64, status on + * riscv. A prog can write the registers of the task only. + */ + if (!arch_uprobe_regs_writable(off, size)) + return false; prog->aux->kprobe_write_ctx = true; + } return true; } diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 6d87143b3ddb..d1f79a85ed2e 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -568,7 +568,11 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func, if (unlikely(ftrace_graph_is_dead())) return -EBUSY; - if (!current->ret_stack) + /* + * We must make sure the ret_stack is tested before we read + * anything else (See alloc_retstack_tasklist()). + */ + if (!smp_load_acquire(¤t->ret_stack)) return -EBUSY; BUILD_BUG_ON(SHADOW_STACK_SIZE % sizeof(long)); @@ -577,12 +581,6 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func, val = (FGRAPH_TYPE_RESERVED << FGRAPH_TYPE_SHIFT) | FGRAPH_FRAME_OFFSET; /* - * We must make sure the ret_stack is tested before we read - * anything else. - */ - smp_rmb(); - - /* * Check if there's room on the shadow stack to fit a fraph frame * and a bitmap word. */ @@ -700,11 +698,7 @@ int function_graph_enter_regs(unsigned long ret, unsigned long func, if (!bitmap) goto out_ret; - /* - * Since this function uses fgraph_idx = 0 as a tail-call checking - * flag, set that bit always. - */ - set_bitmap(current, offset, bitmap | BIT(0)); + set_bitmap(current, offset, bitmap); ftrace_test_recursion_unlock(bit); return 0; out_ret: @@ -1033,13 +1027,16 @@ void ftrace_stub_graph(struct ftrace_graph_ret *trace, struct fgraph_ops *gops, trace_func_graph_ret_t ftrace_graph_return = ftrace_stub_graph; trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub; -/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */ +/* + * Loop thru all the tasks in the system and assign ret_stack. + * The @ret_stack_list is a small pre-allocated fallback, in case + * the GFP_NOWAIT allocations here fail. + */ static int alloc_retstack_tasklist(unsigned long **ret_stack_list) { - int i; - int ret = 0; int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE; struct task_struct *g, *t; + int i, ret = 0; if (WARN_ON_ONCE(!fgraph_stack_cachep)) return -ENOMEM; @@ -1054,26 +1051,31 @@ static int alloc_retstack_tasklist(unsigned long **ret_stack_list) } } - rcu_read_lock(); - for_each_process_thread(g, t) { - if (start == end) { - ret = -EAGAIN; - goto unlock; - } + scoped_guard (rcu) { + for_each_process_thread(g, t) { + unsigned long *rs; + + if (t->ret_stack) + continue; + + rs = kmem_cache_alloc(fgraph_stack_cachep, GFP_NOWAIT); + if (!rs) { + if (start == end) { + ret = -EAGAIN; + goto free; + } + rs = ret_stack_list[start++]; + } - if (t->ret_stack == NULL) { atomic_set(&t->trace_overrun, 0); - ret_stack_init_task_vars(ret_stack_list[start]); + ret_stack_init_task_vars(rs); t->curr_ret_stack = 0; t->curr_ret_depth = -1; /* Make sure the tasks see the 0 first: */ - smp_wmb(); - t->ret_stack = ret_stack_list[start++]; + smp_store_release(&t->ret_stack, rs); } } -unlock: - rcu_read_unlock(); free: for (i = start; i < end; i++) kmem_cache_free(fgraph_stack_cachep, ret_stack_list[i]); @@ -1117,8 +1119,7 @@ graph_init_task(struct task_struct *t, unsigned long *ret_stack) t->curr_ret_stack = 0; t->curr_ret_depth = -1; /* make curr_ret_stack visible before we add the ret_stack */ - smp_wmb(); - t->ret_stack = ret_stack; + smp_store_release(&t->ret_stack, ret_stack); } /* diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index c7d74b33d36d..6654da9d1bbf 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -5029,7 +5029,8 @@ match_records(struct ftrace_hash *hash, char *func, int len, char *mod) if (unlikely(ftrace_disabled)) return 0; - if (func_g.type == MATCH_INDEX) + if (func_g.type == MATCH_FULL && !mod && + func_g.search && isdigit(func_g.search[0])) return add_rec_by_index(hash, &func_g, clear_filter); do_for_each_ftrace_rec(pg, rec) { diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 75cd8df23c1b..a5a0adc934fa 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -2468,8 +2468,17 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, * memory. It may not be accurate. But we don't care, we just want * to prevent doing any allocation when it is obvious that it is * not going to succeed. + * + * With CONFIG_DEFERRED_STRUCT_PAGE_INIT, si_mem_available() + * will reflect the actual available memory only after + * SYSTEM_FREEING_INITMEM. */ - i = si_mem_available(); + if (IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT) && + unlikely(system_state < SYSTEM_FREEING_INITMEM)) + i = LONG_MAX; + else + i = si_mem_available(); + if (i < (nr_pages << subbuf_order)) return -ENOMEM; @@ -5615,14 +5624,12 @@ unsigned long ring_buffer_overruns(struct trace_buffer *buffer) } EXPORT_SYMBOL_GPL(ring_buffer_overruns); -static bool rb_read_remote_meta_page(struct ring_buffer_per_cpu *cpu_buffer) +static void rb_read_remote_meta_page(struct ring_buffer_per_cpu *cpu_buffer) { local_set(&cpu_buffer->entries, READ_ONCE(cpu_buffer->meta_page->entries)); local_set(&cpu_buffer->overrun, READ_ONCE(cpu_buffer->meta_page->overrun)); local_set(&cpu_buffer->pages_touched, READ_ONCE(cpu_buffer->meta_page->pages_touched)); local_set(&cpu_buffer->pages_lost, READ_ONCE(cpu_buffer->meta_page->pages_lost)); - - return rb_num_of_entries(cpu_buffer); } static void rb_update_remote_head(struct ring_buffer_per_cpu *cpu_buffer) @@ -5825,15 +5832,20 @@ __rb_get_reader_page_from_remote(struct ring_buffer_per_cpu *cpu_buffer) { struct buffer_page *new_reader, *prev_reader, *prev_head, *new_head, *last; - if (!rb_read_remote_meta_page(cpu_buffer)) - return NULL; + rb_read_remote_meta_page(cpu_buffer); + + /* + * Order meta::entries read before page::commit, most likely paired with + * simple_ring_buffer_commit(). + */ + smp_rmb(); /* More to read on the reader page */ - if (cpu_buffer->reader_page->read < rb_page_size(cpu_buffer->reader_page)) { - if (!cpu_buffer->reader_page->read) - cpu_buffer->read_stamp = cpu_buffer->reader_page->page->time_stamp; - return cpu_buffer->reader_page; - } + if (cpu_buffer->reader_page->read < rb_page_size(cpu_buffer->reader_page)) + goto out; + + if (!rb_num_of_entries(cpu_buffer)) + return NULL; prev_reader = cpu_buffer->subbuf_ids[cpu_buffer->meta_page->reader.id]; @@ -5848,7 +5860,8 @@ __rb_get_reader_page_from_remote(struct ring_buffer_per_cpu *cpu_buffer) new_reader = cpu_buffer->subbuf_ids[cpu_buffer->meta_page->reader.id]; - WARN_ON_ONCE(prev_reader == new_reader); + if (prev_reader == new_reader) + return NULL; prev_head = new_reader; /* New reader was also the previous head */ new_head = prev_head; @@ -5876,10 +5889,23 @@ __rb_get_reader_page_from_remote(struct ring_buffer_per_cpu *cpu_buffer) cpu_buffer->reader_page = new_reader; cpu_buffer->reader_page->read = 0; cpu_buffer->pages = &new_head->list; - cpu_buffer->read_stamp = new_reader->page->time_stamp; cpu_buffer->lost_events = cpu_buffer->meta_page->reader.lost_events; + local_inc(&cpu_buffer->pages_read); - return rb_page_size(cpu_buffer->reader_page) ? cpu_buffer->reader_page : NULL; + if (!rb_page_size(cpu_buffer->reader_page)) + return NULL; + +out: + /* + * Order page::commit read before page::data and page::time_stamp, most + * likely paired with simple_ring_buffer_commit(). + */ + smp_rmb(); + + if (!cpu_buffer->reader_page->read) + cpu_buffer->read_stamp = cpu_buffer->reader_page->page->time_stamp; + + return cpu_buffer->reader_page; } static struct buffer_page * @@ -6908,7 +6934,8 @@ int ring_buffer_poll_remote(struct trace_buffer *buffer, int cpu) cpu_buffer = buffer->buffers[cpu]; guard(raw_spinlock)(&cpu_buffer->reader_lock); - if (rb_read_remote_meta_page(cpu_buffer)) + rb_read_remote_meta_page(cpu_buffer); + if (rb_num_of_entries(cpu_buffer)) rb_wakeups(buffer, cpu_buffer); return 0; @@ -8091,8 +8118,7 @@ int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node) buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, nr_pages, cpu); if (!buffer->buffers[cpu]) { - WARN(1, "failed to allocate ring buffer on CPU %u\n", - cpu); + pr_warn("failed to allocate ring buffer on CPU %u\n", cpu); return -ENOMEM; } diff --git a/kernel/trace/simple_ring_buffer.c b/kernel/trace/simple_ring_buffer.c index abfaf9bee12e..2dddaef638dc 100644 --- a/kernel/trace/simple_ring_buffer.c +++ b/kernel/trace/simple_ring_buffer.c @@ -139,6 +139,13 @@ int simple_ring_buffer_swap_reader_page(struct simple_rb_per_cpu *cpu_buffer) reader = cpu_buffer->reader_page; + /* + * The reader is already the tail page. See simple_rb_move_tail() for + * memory ordering. + */ + if (reader == smp_load_acquire(&cpu_buffer->tail_page)) + return 0; + do { /* Run after the writer to find the head */ ret = simple_rb_find_head(cpu_buffer); @@ -196,7 +203,13 @@ static struct simple_buffer_page *simple_rb_move_tail(struct simple_rb_per_cpu * } simple_bpage_reset(new_tail); - cpu_buffer->tail_page = new_tail; + + /* + * Setting a new tail may unblock the swap in + * simple_ring_buffer_swap_reader_page(). Ensure the reader observes the + * page::commit reset. + */ + smp_store_release(&cpu_buffer->tail_page, new_tail); simple_rb_meta_inc(cpu_buffer->meta->pages_touched, 1); @@ -294,8 +307,19 @@ EXPORT_SYMBOL_GPL(simple_ring_buffer_reserve); */ void simple_ring_buffer_commit(struct simple_rb_per_cpu *cpu_buffer) { + /* + * Order page::data and page::time_stamp before page::commit. Paired + * with __rb_get_reader_page_from_remote(). + */ + smp_wmb(); local_set(&cpu_buffer->tail_page->page->commit, cpu_buffer->tail_page->write); + + /* + * Order page::commit before meta::entries. Paired with + * __rb_get_reader_page_from_remote(). + */ + smp_wmb(); simple_rb_meta_inc(cpu_buffer->meta->entries, 1); /* diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 4ccdefb5c3ef..65b9d67647e5 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -1456,6 +1456,13 @@ static void free_tracers(struct trace_array *tr) } } +static void free_tracer_options(struct trace_array *tr) +{ + for (int i = 0; i < tr->nr_topts; i++) + kfree(tr->topts[i].topts); + kfree(tr->topts); +} + /** * register_tracer - register a tracer with the ftrace system. * @type: the plugin for the tracer @@ -4028,7 +4035,9 @@ int set_tracer_flag(struct trace_array *tr, u64 mask, int enabled) break; case TRACE_ITER(COPY_MARKER): - update_marker_trace(tr, enabled); + /* Writers of the top level trace_marker may still be on this instance */ + if (update_marker_trace(tr, enabled) && !enabled) + synchronize_rcu(); /* update_marker_trace updates the tr->trace_flags */ return 0; } @@ -6289,7 +6298,7 @@ char *trace_user_fault_read(struct trace_user_buf_info *tinfo, if (copy_func) { ret = copy_func(buffer, ptr, size, data); } else { - ret = __copy_from_user(buffer, ptr, size); + ret = copy_from_user(buffer, ptr, size); } preempt_disable_notrace(); @@ -6345,9 +6354,10 @@ tracing_mark_write(struct file *filp, const char __user *ubuf, if (tr == &global_trace) { guard(rcu)(); list_for_each_entry_rcu(tr, &marker_copies, marker_list) { - written = write_marker_to_buffer(tr, buf, cnt, ip); - if (written < 0) - break; + ssize_t w = write_marker_to_buffer(tr, buf, cnt, ip); + + if (w >= 0 && w > written) + written = w; } } else { written = write_marker_to_buffer(tr, buf, cnt, ip); @@ -6424,9 +6434,10 @@ tracing_mark_raw_write(struct file *filp, const char __user *ubuf, if (tr == &global_trace) { guard(rcu)(); list_for_each_entry_rcu(tr, &marker_copies, marker_list) { - written = write_raw_marker_to_buffer(tr, buf, cnt); - if (written < 0) - break; + ssize_t w = write_raw_marker_to_buffer(tr, buf, cnt); + + if (w >= 0 && w > written) + written = w; } } else { written = write_raw_marker_to_buffer(tr, buf, cnt); @@ -7404,8 +7415,12 @@ static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned return err; } - return ring_buffer_map_get_reader(iter->array_buffer->buffer, - iter->cpu_file); + trace_access_lock(iter->cpu_file); + err = ring_buffer_map_get_reader(iter->array_buffer->buffer, + iter->cpu_file); + trace_access_unlock(iter->cpu_file); + + return err; } else if (cmd) { return -ENOTTY; } @@ -8761,7 +8776,7 @@ trace_array_create_systems(const char *name, const char *systems, if (trace_instance_dir) { ret = trace_array_create_dir(tr); if (ret) - goto out_free_tr; + goto out_free_tracers; } else __trace_early_add_events(tr); @@ -8771,6 +8786,9 @@ trace_array_create_systems(const char *name, const char *systems, return tr; + out_free_tracers: + free_tracers(tr); + free_tracer_options(tr); out_free_tr: ftrace_free_ftrace_ops(tr); free_trace_buffers(tr); @@ -8924,10 +8942,7 @@ static int __remove_instance(struct trace_array *tr) if (tr->flags & TRACE_ARRAY_FL_VMALLOC) vfree((void *)tr->range_addr_start); - for (i = 0; i < tr->nr_topts; i++) { - kfree(tr->topts[i].topts); - } - kfree(tr->topts); + free_tracer_options(tr); free_cpumask_var(tr->pipe_cpumask); free_cpumask_var(tr->tracing_cpumask); diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 0635503a31dd..1fec567e789e 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -1807,7 +1807,6 @@ enum regex_type { MATCH_MIDDLE_ONLY, MATCH_END_ONLY, MATCH_GLOB, - MATCH_INDEX, }; struct regex { @@ -1912,9 +1911,11 @@ extern const struct file_operations event_inject_fops; #ifdef CONFIG_HIST_TRIGGERS extern int register_trigger_hist_cmd(void); extern int register_trigger_hist_enable_disable_cmds(void); +extern void hist_put_synth_event(struct trace_event_file *file); #else static inline int register_trigger_hist_cmd(void) { return 0; } static inline int register_trigger_hist_enable_disable_cmds(void) { return 0; } +static inline void hist_put_synth_event(struct trace_event_file *file) { } #endif extern int register_trigger_cmds(void); diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 49d135bc2fbf..7f89160aeff3 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -4317,7 +4317,7 @@ static void update_event_probe(struct event_probe_data *data) set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags); } -static void +static void notrace event_enable_probe(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, void *data) @@ -4334,7 +4334,7 @@ event_enable_probe(unsigned long ip, unsigned long parent_ip, update_event_probe(edata); } -static void +static void notrace event_enable_count_probe(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, void *data) @@ -5026,8 +5026,8 @@ void __init trace_event_init(void) { event_trace_memsetup(); init_ftrace_syscalls(); - event_trace_enable(); event_trace_init_fields(); + event_trace_enable(); } #ifdef CONFIG_EVENT_TRACE_STARTUP_TEST @@ -5205,7 +5205,7 @@ static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable); static struct trace_event_file event_trace_file __initdata; -static void __init +static void notrace __init function_test_events_call(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct ftrace_regs *regs) { diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index c10951bcfc5f..a9d77df3e19a 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -758,7 +758,7 @@ static int filter_pred_##size##_cpumask(struct filter_pred *pred, void *event) \ unsigned int cpu = *addr; \ \ if (cpu >= nr_cpu_ids) \ - return 0; \ + return pred->op == OP_NE; \ \ return do_filter_scalar_cpumask(pred->op, cpu, pred->mask); \ } @@ -891,6 +891,25 @@ static int filter_pred_pchar_user(struct filter_pred *pred, void *event) return filter_pchar(pred, str); } +static __always_inline int filter_dyn_string(struct filter_pred *pred, + char *str, int len) +{ + int cmp; + + /* + * The regex callbacks use zero for an unknown-length, NUL-terminated + * string. A zero-length data_loc or rel_loc field has no bytes at all. + * Match an empty string instead of reading past the end of the field. + */ + if (!len) { + str = ""; + len = 1; + } + + cmp = pred->regex->match(str, pred->regex, len); + return cmp ^ pred->not; +} + /* * Filter predicate for dynamic sized arrays of characters. * These are implemented through a list of strings at the end @@ -907,13 +926,8 @@ static int filter_pred_strloc(struct filter_pred *pred, void *event) int str_loc = str_item & 0xffff; int str_len = str_item >> 16; char *addr = (char *)(event + str_loc); - int cmp, match; - - cmp = pred->regex->match(addr, pred->regex, str_len); - - match = cmp ^ pred->not; - return match; + return filter_dyn_string(pred, addr, str_len); } /* @@ -930,13 +944,8 @@ static int filter_pred_strrelloc(struct filter_pred *pred, void *event) int str_loc = str_item & 0xffff; int str_len = str_item >> 16; char *addr = (char *)(&item[1]) + str_loc; - int cmp, match; - cmp = pred->regex->match(addr, pred->regex, str_len); - - match = cmp ^ pred->not; - - return match; + return filter_dyn_string(pred, addr, str_len); } /* Filter predicate for CPUs. */ @@ -1037,7 +1046,8 @@ static int filter_pred_within(struct filter_pred *pred, void *event) * * @str: the string to be searched * @r: the regex structure containing the pattern string - * @len: the length of the string to be searched (including '\0') + * @len: the length of the string to be searched (including '\0'), or zero + * for a NUL-terminated string of unknown length * * Note: * - @str might not be NULL-terminated if it's of type DYN_STRING @@ -1046,7 +1056,7 @@ static int filter_pred_within(struct filter_pred *pred, void *event) static int regex_match_full(char *str, struct regex *r, int len) { - /* len of zero means str is dynamic and ends with '\0' */ + /* Zero is reserved for a NUL-terminated string of unknown length. */ if (!len) return strcmp(str, r->pattern) == 0; @@ -1082,6 +1092,30 @@ static int regex_match_end(char *str, struct regex *r, int len) return 0; } +static int regex_match_end_str(char *str, struct regex *r, int len) +{ + int slen; + + if (len < r->len) + return 0; + + slen = strnlen(str, len); + if (slen >= r->len && + memcmp(str + slen - r->len, r->pattern, r->len) == 0) + return 1; + return 0; +} + +static int regex_match_end_dynstr(char *str, struct regex *r, int len) +{ + int slen = strlen(str); + + if (slen >= r->len && + memcmp(str + slen - r->len, r->pattern, r->len) == 0) + return 1; + return 0; +} + static int regex_match_glob(char *str, struct regex *r, int len) { return glob_match_len(r->pattern, str, len) ? 1 : 0; @@ -1118,9 +1152,6 @@ enum regex_type filter_parse_regex(char *buff, int len, char **search, int *not) *search = buff; - if (isdigit(buff[0])) - return MATCH_INDEX; - for (i = 0; i < len; i++) { if (buff[i] == '*') { if (!i) { @@ -1158,8 +1189,6 @@ static void filter_build_regex(struct filter_pred *pred) } switch (type) { - /* MATCH_INDEX should not happen, but if it does, match full */ - case MATCH_INDEX: case MATCH_FULL: r->match = regex_match_full; break; @@ -1708,7 +1737,9 @@ static int parse_pred(const char *str, void *data, /* See if the field is a kernel function name */ if ((len = str_has_prefix(str + i, ".function"))) { - function = true; + /* Ignore if event is already a function */ + if (!ftrace_event_is_function(call)) + function = true; i += len; } @@ -1732,6 +1763,11 @@ static int parse_pred(const char *str, void *data, while (isspace(str[i])) i++; + if (!str[i]) { + parse_error(pe, FILT_ERR_INVALID_FILTER, pos + i); + goto err_free; + } + s = i; pred = kzalloc_obj(*pred); @@ -2021,10 +2057,18 @@ static int parse_pred(const char *str, void *data, if (field->filter_type == FILTER_COMM) { pred->fn_num = FILTER_PRED_FN_COMM; + if (pred->regex->match == regex_match_end) + pred->regex->match = regex_match_end_str; } else if (field->filter_type == FILTER_STATIC_STRING) { pred->fn_num = FILTER_PRED_FN_STRING; pred->regex->field_len = field->size; + if (pred->regex->match == regex_match_end) { + if (!field->size) + pred->regex->match = regex_match_end_dynstr; + else + pred->regex->match = regex_match_end_str; + } } else if (field->filter_type == FILTER_DYN_STRING) { pred->fn_num = FILTER_PRED_FN_STRLOC; @@ -2305,6 +2349,7 @@ static int process_system_preds(struct trace_subsystem_dir *dir, struct event_filter *filter = NULL; struct filter_head *filter_list; bool fail = true; + int ret = -ENOMEM; int err; filter_list = kmalloc_obj(*filter_list); @@ -2331,9 +2376,10 @@ static int process_system_preds(struct trace_subsystem_dir *dir, filter_disable(file); parse_error(pe, FILT_ERR_BAD_SUBSYS_FILTER, 0); append_filter_err(tr, pe, filter); - } else + } else { event_set_filtered_flag(file); - + fail = false; + } filter_item = kzalloc_obj(*filter_item); if (!filter_item) @@ -2347,8 +2393,6 @@ static int process_system_preds(struct trace_subsystem_dir *dir, filter_item->filter = event_filter(file); event_set_filter(file, filter); filter = NULL; - - fail = false; } if (fail) @@ -2363,19 +2407,18 @@ static int process_system_preds(struct trace_subsystem_dir *dir, return 0; fail: /* No call succeeded */ - free_filter_list(filter_list); parse_error(pe, FILT_ERR_BAD_SUBSYS_FILTER, 0); - return -EINVAL; + ret = -EINVAL; fail_mem: __free_filter(filter); - /* If any call succeeded, we still need to sync */ - if (!fail) + /* If any filter was replaced, we still need to sync */ + if (!list_empty(&filter_list->list)) delay_free_filter(filter_list); else free_filter_list(filter_list); - return -ENOMEM; + return ret; } static int create_filter_start(char *filter_string, bool set_str, @@ -2749,6 +2792,28 @@ static int ftrace_function_set_filter(struct perf_event *event, } #endif /* CONFIG_FUNCTION_TRACER */ +static int ftrace_profile_filter_perm(struct event_filter *filter) +{ + struct prog_entry *prog = rcu_dereference_protected(filter->prog, + lockdep_is_held(&event_mutex)); + int i, err; + + for (i = 0; prog[i].pred; i++) { + if (prog[i].pred->fn_num != FILTER_PRED_FN_PCHAR) + continue; + + /* + * Even a counting filter can reveal kernel string data. + * Require the same access as raw kernel tracepoints. + */ + err = perf_allow_kernel(); + if (err) + return err; + return perf_allow_tracepoint(); + } + return 0; +} + int ftrace_profile_set_filter(struct perf_event *event, int event_id, char *filter_str) { @@ -2770,6 +2835,10 @@ int ftrace_profile_set_filter(struct perf_event *event, int event_id, if (err) goto free_filter; + err = ftrace_profile_filter_perm(filter); + if (err) + goto free_filter; + if (ftrace_event_is_function(call)) err = ftrace_function_set_filter(event, filter); else @@ -2789,6 +2858,8 @@ free_filter: #include <linux/types.h> #include <linux/tracepoint.h> +#include "trace_probe.h" + #define CREATE_TRACE_POINTS #include "trace_events_filter_test.h" @@ -2800,12 +2871,22 @@ free_filter: .match = m, \ .not_visited = nvisit, \ } + +#define DATA_STR(m, s) \ +{ \ + .filter = FILTER, \ + .str = s, \ + .match = m, \ + .not_visited = "", \ +} + #define YES 1 #define NO 0 static struct test_filter_data_t { char *filter; struct trace_event_raw_ftrace_test_filter rec; + const char *str; int match; char *not_visited; } test_filter_data[] = { @@ -2857,9 +2938,46 @@ static struct test_filter_data_t { DATA_REC(YES, 1, 1, 1, 1, 1, 1, 1, 1, "bdfh"), DATA_REC(YES, 0, 1, 0, 1, 0, 1, 0, 1, ""), DATA_REC(YES, 1, 0, 1, 0, 1, 0, 1, 0, "bdfh"), +#undef FILTER +#define FILTER "str ~ \"0001:*\"" + DATA_STR(YES, "0001:01:00.0"), + DATA_STR(YES, "0001:"), + DATA_STR(NO, "0002:01:00.0"), + DATA_STR(NO, "pci:0001:01:00.0"), + DATA_STR(NO, "0001"), +#undef FILTER +#define FILTER "str ~ \"!0001:*\"" + DATA_STR(NO, "0001:01:00.0"), + DATA_STR(YES, "0002:01:00.0"), +#undef FILTER +#define FILTER "str ~ \"0001:*:00.[0-9]\"" + DATA_STR(YES, "0001:01:00.0"), + DATA_STR(YES, "0001:ff:00.7"), + DATA_STR(NO, "0001:01:00.a"), + DATA_STR(NO, "0002:01:00.0"), +#undef FILTER +#define FILTER "str ~ \"0001:01:00.0\"" + DATA_STR(YES, "0001:01:00.0"), + DATA_STR(NO, "0001:01:00.1"), + DATA_STR(NO, "0001:01:00.00"), + DATA_STR(NO, "0001:01:00"), +#undef FILTER +#define FILTER "str == \"0001:01:00.0\"" + DATA_STR(YES, "0001:01:00.0"), + DATA_STR(NO, "0001:01:00.1"), +#undef FILTER +#define FILTER "str ~ \"*0001:*\"" + DATA_STR(YES, "0001:01:00.0"), + DATA_STR(YES, "pci:0001:01:00.0"), + DATA_STR(NO, "0002:01:00.0"), +#undef FILTER +#define FILTER "str ~ \"*:00.0\"" + DATA_STR(YES, "0001:01:00.0"), + DATA_STR(NO, "0001:01:00.1"), }; #undef DATA_REC +#undef DATA_STR #undef FILTER #undef YES #undef NO @@ -2910,8 +3028,29 @@ static __init int ftrace_test_event_filter(void) for (i = 0; i < DATA_CNT; i++) { struct event_filter *filter = NULL; struct test_filter_data_t *d = &test_filter_data[i]; + union { + struct trace_event_raw_ftrace_test_filter rec; + /* Add some spare, to append the string contents. */ + char buf[sizeof(struct trace_event_raw_ftrace_test_filter) + 32]; + } event = { .rec = d->rec }; int err; + if (d->str) { + struct ftrace_event_field *field; + u32 offset = offsetof(typeof(event.rec), __data); + u32 len = strlen(d->str) + 1; + u32 *str_item; + + field = trace_find_event_field(&event_ftrace_test_filter, "str"); + if (WARN_ON_ONCE(!field)) + break; + if (WARN_ON_ONCE(offset + len > sizeof(event.buf))) + break; + str_item = (u32 *)&event.buf[field->offset]; + *str_item = make_data_loc(len, offset); + memcpy(&event.buf[offset], d->str, len); + } + err = create_filter(NULL, &event_ftrace_test_filter, d->filter, false, &filter); if (err) { @@ -2933,7 +3072,7 @@ static __init int ftrace_test_event_filter(void) update_pred_fn(filter, d->not_visited); test_pred_visited = 0; - err = filter_match_preds(filter, &d->rec); + err = filter_match_preds(filter, &event.rec); preempt_enable(); mutex_unlock(&event_mutex); @@ -2960,7 +3099,7 @@ static __init int ftrace_test_event_filter(void) /* Need to call ftrace_test_filter to prevent a warning */ if (!trace_ftrace_test_filter_enabled()) - trace_ftrace_test_filter(1, 2, 3, 4, 5, 6, 7, 8); + trace_ftrace_test_filter(1, 2, 3, 4, 5, 6, 7, 8, ""); return 0; } diff --git a/kernel/trace/trace_events_filter_test.h b/kernel/trace/trace_events_filter_test.h index e651dfbd345e..74a9dba91545 100644 --- a/kernel/trace/trace_events_filter_test.h +++ b/kernel/trace/trace_events_filter_test.h @@ -9,9 +9,10 @@ TRACE_EVENT(ftrace_test_filter, - TP_PROTO(int a, int b, int c, int d, int e, int f, int g, int h), + TP_PROTO(int a, int b, int c, int d, int e, int f, int g, int h, + const char *str), - TP_ARGS(a, b, c, d, e, f, g, h), + TP_ARGS(a, b, c, d, e, f, g, h, str), TP_STRUCT__entry( __field(int, a) @@ -22,6 +23,7 @@ TRACE_EVENT(ftrace_test_filter, __field(int, f) __field(int, g) __field(int, h) + __string(str, str) ), TP_fast_assign( @@ -33,11 +35,13 @@ TRACE_EVENT(ftrace_test_filter, __entry->f = f; __entry->g = g; __entry->h = h; + __assign_str(str); ), - TP_printk("a %d, b %d, c %d, d %d, e %d, f %d, g %d, h %d", + TP_printk("a %d, b %d, c %d, d %d, e %d, f %d, g %d, h %d, str %s", __entry->a, __entry->b, __entry->c, __entry->d, - __entry->e, __entry->f, __entry->g, __entry->h) + __entry->e, __entry->f, __entry->g, __entry->h, + __get_str(str)) ); #endif /* _TRACE_TEST_H || TRACE_HEADER_MULTI_READ */ diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 1aae7d7c6dc5..d4702ad4e003 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -1430,6 +1430,11 @@ static int parse_map_size(char *str) if (ret) goto out; + if (!size) { + ret = -EINVAL; + goto out; + } + map_bits = ilog2(roundup_pow_of_two(size)); if (map_bits < TRACING_MAP_BITS_MIN || map_bits > TRACING_MAP_BITS_MAX) @@ -6761,6 +6766,17 @@ static bool hist_trigger_check_refs(struct event_trigger_data *data, return false; } +void hist_put_synth_event(struct trace_event_file *file) +{ + struct synth_event *se; + + lockdep_assert_held(&event_mutex); + + se = find_synth_event_call(file->event_call); + if (se) + se->ref--; +} + static void hist_unregister_trigger(char *glob, struct event_trigger_data *data, struct trace_event_file *file) @@ -6816,7 +6832,6 @@ static void hist_unreg_all(struct trace_event_file *file) { struct event_trigger_data *test, *n; struct hist_trigger_data *hist_data; - struct synth_event *se; lockdep_assert_held(&event_mutex); @@ -6829,9 +6844,7 @@ static void hist_unreg_all(struct trace_event_file *file) list_del_rcu(&test->list); trace_event_trigger_enable_disable(file, 0); - se = find_synth_event_call(file->event_call); - if (se) - se->ref--; + hist_put_synth_event(file); update_cond_flag(file); if (hist_data->enable_timestamps) @@ -6946,9 +6959,7 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops, } event_trigger_unregister(cmd_ops, file, glob+1, trigger_data); - se = find_synth_event_call(file->event_call); - if (se) - se->ref--; + hist_put_synth_event(file); ret = 0; goto out_free; } diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c index 4d2fde996c0f..a2487ad7fef8 100644 --- a/kernel/trace/trace_events_trigger.c +++ b/kernel/trace/trace_events_trigger.c @@ -643,6 +643,8 @@ clear_event_triggers(struct trace_array *tr) list_for_each_entry_safe(data, n, &file->triggers, list) { trace_event_trigger_enable_disable(file, 0); list_del_rcu(&data->list); + if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) + hist_put_synth_event(file); if (data->cmd_ops->free) data->cmd_ops->free(data); } diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index f658c3a77aa7..e29e1c772aa7 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -256,6 +256,11 @@ static void delayed_destroy_user_event(struct work_struct *work) * attempt to delete it later. */ pr_warn("user_events: Unable to delete event\n"); + /* + * Keep it as persistent so it can be still deleted + * since nobody own this event + */ + user->reg_flags |= USER_EVENT_REG_PERSIST; refcount_set(&user->refcnt, 1); } out: @@ -1035,6 +1040,9 @@ static int user_event_parse_cmd(struct user_event_group *group, if (flags) *flags++ = '\0'; + if (strstr(name, "/") != NULL) + return -EINVAL; + return user_event_parse(group, name, args, flags, newuser, reg_flags); } @@ -1506,6 +1514,10 @@ static int destroy_user_event(struct user_event *user) lockdep_assert_held(&event_mutex); + /* If there is a reference to a dynamic event */ + if (trace_event_dyn_busy(&user->call)) + return -EBUSY; + /* * Detach the fields before removing the call. Removing the event * frees the field list memory (trace_destroy_fields() is run on diff --git a/kernel/trace/trace_hwlat.c b/kernel/trace/trace_hwlat.c index 3fe274b84f1c..5be9031f1ffe 100644 --- a/kernel/trace/trace_hwlat.c +++ b/kernel/trace/trace_hwlat.c @@ -203,14 +203,12 @@ static int get_sample(void) time_type start, t1, t2, last_t2; s64 diff, outer_diff, total, last_total = 0; u64 sample = 0; - u64 sample_width = READ_ONCE(hwlat_data.sample_width); + u64 sample_width = READ_ONCE(hwlat_data.sample_width) * NSEC_PER_USEC; u64 thresh = tracing_thresh; u64 outer_sample = 0; - int ret = -1; + int ret = 0; unsigned int count = 0; - do_div(thresh, NSEC_PER_USEC); /* modifies interval value */ - kdata->nmi_total_ts = 0; kdata->nmi_count = 0; /* Make sure NMIs see this first */ @@ -229,28 +227,37 @@ static int get_sample(void) if (time_u64(last_t2)) { /* Check the delta from outer loop (t2 to next t1) */ - outer_diff = time_to_us(time_sub(t1, last_t2)); + outer_diff = time_sub(t1, last_t2); /* This shouldn't happen */ if (outer_diff < 0) { hwlat_err(BANNER "time running backwards\n"); - goto out; + ret = -1; + break; } if (outer_diff > outer_sample) outer_sample = outer_diff; } last_t2 = t2; - total = time_to_us(time_sub(t2, start)); /* sample width */ + total = time_sub(t2, start); /* sample width */ /* Check for possible overflows */ if (total < last_total) { hwlat_err("Time total overflowed\n"); + /* Keep ret == 0 to preserve any latencies already detected */ break; } last_total = total; /* This checks the inner loop (t1 to t2) */ - diff = time_to_us(time_sub(t2, t1)); /* current diff */ + diff = time_sub(t2, t1); /* current diff */ + + /* This shouldn't happen: time went backwards between t1 and t2 */ + if (diff < 0) { + hwlat_err(BANNER "time running backwards\n"); + ret = -1; + break; + } if (diff > thresh || outer_diff > thresh) { if (!count) @@ -258,12 +265,6 @@ static int get_sample(void) count++; } - /* This shouldn't happen */ - if (diff < 0) { - hwlat_err(BANNER "time running backwards\n"); - goto out; - } - if (diff > sample) sample = diff; /* only want highest value */ @@ -273,10 +274,11 @@ static int get_sample(void) trace_hwlat_callback_enabled = false; barrier(); /* Make sure nmi_total_ts is no longer updated */ - ret = 0; + if (ret) + return ret; /* If we exceed the threshold value, we have found a hardware latency */ - if (sample > thresh || outer_sample > thresh) { + if (count) { u64 latency; ret = 1; @@ -286,8 +288,8 @@ static int get_sample(void) do_div(kdata->nmi_total_ts, NSEC_PER_USEC); s.seqnum = atomic64_inc_return(&hwlat_data.count); - s.duration = sample; - s.outer_duration = outer_sample; + s.duration = time_to_us(sample); + s.outer_duration = time_to_us(outer_sample); s.nmi_total_ts = kdata->nmi_total_ts; s.nmi_count = kdata->nmi_count; s.count = count; @@ -301,8 +303,6 @@ static int get_sample(void) latency_fsnotify(tr); } } - -out: return ret; } @@ -835,7 +835,6 @@ static int hwlat_tracer_init(struct trace_array *tr) tr->max_latency = 0; save_tracing_thresh = tracing_thresh; - /* tracing_thresh is in nsecs, we speak in usecs */ if (!tracing_thresh) tracing_thresh = last_tracing_thresh; diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index 995da9e0b548..fbba42286747 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -2039,6 +2039,14 @@ static int start_kthread(unsigned int cpu) return 0; if (timerlat_enabled()) { + /* + * In timerlat user-space mode the workload is the user task + * reading timerlat_fd, so there is no kernel thread to start. + * One started here would set its kthread_cpumask bit, which + * stop_kthread() reads as "osnoise workload kthread". + */ + if (!test_bit(OSN_WORKLOAD, &osnoise_options)) + return 0; snprintf(comm, 24, "timerlat/%d", cpu); main = timerlat_main; } else { |
