diff options
Diffstat (limited to 'kernel')
28 files changed, 367 insertions, 117 deletions
diff --git a/kernel/audit.c b/kernel/audit.c index 562476937fa7..9412af9144bc 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -2120,7 +2120,8 @@ void audit_log_n_hex(struct audit_buffer *ab, const unsigned char *buf, void audit_log_n_string(struct audit_buffer *ab, const char *string, size_t slen) { - int avail, new_len; + int avail; + size_t new_len; unsigned char *ptr; struct sk_buff *skb; @@ -2130,7 +2131,13 @@ void audit_log_n_string(struct audit_buffer *ab, const char *string, BUG_ON(!ab->skb); skb = ab->skb; avail = skb_tailroom(skb); - new_len = slen + 3; /* enclosing quotes + null terminator */ + + /* enclosing quotes + null terminator */ + if (check_add_overflow(slen, 3, &new_len)) { + audit_log_format(ab, "?"); + return; + } + if (new_len > avail) { avail = audit_expand(ab, new_len); if (!avail) diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c index 4401119b5275..7f791afe5791 100644 --- a/kernel/auditfilter.c +++ b/kernel/auditfilter.c @@ -1045,6 +1045,10 @@ int audit_del_rule(struct audit_entry *entry) goto out; } + list_del_rcu(&e->list); + list_del(&e->rule.list); + synchronize_rcu(); + if (e->rule.watch) audit_remove_watch_rule(&e->rule); @@ -1062,8 +1066,6 @@ int audit_del_rule(struct audit_entry *entry) audit_signals--; #endif - list_del_rcu(&e->list); - list_del(&e->rule.list); call_rcu(&e->rcu, audit_free_rule_rcu); out: diff --git a/kernel/bpf/rqspinlock.c b/kernel/bpf/rqspinlock.c index e527cb425cf4..111ec80ea958 100644 --- a/kernel/bpf/rqspinlock.c +++ b/kernel/bpf/rqspinlock.c @@ -572,9 +572,10 @@ queue: /* Disable queue destruction when we detect deadlocks. */ if (ret == -EDEADLK) { - if (!next) + if (!try_cmpxchg_tail(lock, tail, 0)) { next = smp_cond_load_relaxed(&node->next, (VAL)); - arch_mcs_spin_unlock_contended(&next->locked); + arch_mcs_spin_unlock_contended(&next->locked); + } goto err_release_node; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 648c5784178e..c9533ea700ba 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -13697,23 +13697,21 @@ static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; } -static int sanitize_err(struct bpf_verifier_env *env, - const struct bpf_insn *insn, int reason, - const struct bpf_reg_state *off_reg, - const struct bpf_reg_state *dst_reg) +static int sanitize_err(struct bpf_verifier_env *env, const struct bpf_insn *insn, int reason) { static const char *err = "pointer arithmetic with it prohibited for !root"; const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub"; u32 dst = insn->dst_reg, src = insn->src_reg; + struct bpf_reg_state *regs = cur_regs(env); switch (reason) { case REASON_BOUNDS: verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", - off_reg == dst_reg ? dst : src, err); + regs[src].type == SCALAR_VALUE ? src : dst, err); break; case REASON_TYPE: verbose(env, "R%d has pointer with unsupported alu operation, %s\n", - off_reg == dst_reg ? src : dst, err); + regs[src].type == SCALAR_VALUE ? dst : src, err); break; case REASON_PATHS: verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", @@ -13849,13 +13847,6 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, return -EACCES; } - /* - * Accesses to untrusted PTR_TO_MEM are done through probe - * instructions, hence no need to track offsets. - */ - if (base_type(ptr_reg->type) == PTR_TO_MEM && (ptr_reg->type & PTR_UNTRUSTED)) - return 0; - switch (base_type(ptr_reg->type)) { case PTR_TO_CTX: case PTR_TO_MAP_VALUE: @@ -13885,11 +13876,19 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, return -EACCES; } - /* In case of 'scalar += pointer', dst_reg inherits pointer type and id. - * The id may be overwritten later if we create a new variable offset. + /* For 'scalar += pointer', dst_reg inherits the complete pointer + * register state. Individual fields may be adjusted later by pointer + * arithmetic. Callers guarantee that below does not overwrite off_reg. */ - dst_reg->type = ptr_reg->type; - dst_reg->id = ptr_reg->id; + if (dst_reg != ptr_reg) + *dst_reg = *ptr_reg; + + /* + * Accesses to untrusted PTR_TO_MEM are done through probe + * instructions, hence no need to track offsets. + */ + if (base_type(ptr_reg->type) == PTR_TO_MEM && (ptr_reg->type & PTR_UNTRUSTED)) + return 0; if (!check_reg_sane_offset_scalar(env, off_reg, ptr_reg->type) || !check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type)) @@ -13902,7 +13901,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, &info, false); if (ret < 0) - return sanitize_err(env, insn, ret, off_reg, dst_reg); + return sanitize_err(env, insn, ret); } switch (opcode) { @@ -13932,7 +13931,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, } break; case BPF_SUB: - if (dst_reg == off_reg) { + if (dst_reg != ptr_reg) { /* scalar -= pointer. Creates an unknown scalar */ verbose(env, "R%d tried to subtract pointer from scalar\n", dst); @@ -13995,7 +13994,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, return -EFAULT; } if (ret < 0) - return sanitize_err(env, insn, ret, off_reg, dst_reg); + return sanitize_err(env, insn, ret); } return 0; @@ -14747,7 +14746,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, if (sanitize_needed(opcode)) { ret = sanitize_val_alu(env, insn); if (ret < 0) - return sanitize_err(env, insn, ret, NULL, NULL); + return sanitize_err(env, insn, ret); } /* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops. @@ -14950,8 +14949,8 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, err = mark_chain_precision(env, insn->dst_reg); if (err) return err; - return adjust_ptr_min_max_vals(env, insn, - src_reg, dst_reg); + off_reg = *dst_reg; + return adjust_ptr_min_max_vals(env, insn, src_reg, &off_reg); } } else if (ptr_reg) { /* pointer += scalar */ diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 38f8d9df8fbc..b5b461d4418b 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -3996,6 +3996,7 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf, struct psi_trigger *new; struct cgroup *cgrp; struct psi_group *psi; + bool need_rtpoll_worker; ssize_t ret = 0; cgrp = cgroup_kn_lock_live(of->kn, false); @@ -4015,12 +4016,32 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf, } psi = cgroup_psi(cgrp); - new = psi_trigger_create(psi, buf, res, of->file, of); + new = psi_trigger_create(psi, buf, res, of->file, of, + &need_rtpoll_worker); if (IS_ERR(new)) { ret = PTR_ERR(new); goto out_unlock; } + /* + * The worker fork must run with neither cgroup_mutex nor the file's + * kernfs active reference held. The latter is broken since + * cgroup_kn_lock_live(). @of->priv may be released while unlocked, so + * recheck before publishing @new. + */ + if (need_rtpoll_worker) { + cgroup_unlock(); + ret = psi_trigger_create_rtpoll_worker(psi); + cgroup_lock(); + + if (!ret && !of->priv) + ret = -ENODEV; + if (ret) { + psi_trigger_destroy(new); + goto out_unlock; + } + } + smp_store_release(&ctx->psi.trigger, new); out_unlock: diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 4084e926e284..6300b216012c 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -830,7 +830,7 @@ static struct uprobe *hprobe_expire(struct hprobe *hprobe, bool get) if (try_cmpxchg(&hprobe->state, &hstate, uprobe ? HPROBE_STABLE : HPROBE_GONE)) { /* We won the race, we are the ones to unlock SRCU */ __srcu_read_unlock(&uretprobes_srcu, hprobe->srcu_idx); - return get ? get_uprobe(uprobe) : uprobe; + return get && uprobe ? get_uprobe(uprobe) : uprobe; } /* diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 179b26e9c934..2650d1e52803 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -982,8 +982,11 @@ retry: return -1; /* - * Special case for regular (non PI) futexes. The unlock path in - * user space has two race scenarios: + * Special case for regular (non PI) futexes. Ordinarily, we do + * not perform any processing here unless the current thread was + * the owner of the futex (by the TID check below). + * + * However, the unlock path has three race scenarios: * * 1. The unlock path releases the user space futex value and * before it can execute the futex() syscall to wake up @@ -992,42 +995,70 @@ retry: * 2. A woken up waiter is killed before it can acquire the * futex in user space. * - * In the second case, the wake up notification could be generated - * by the unlock path in user space after setting the futex value - * to zero or by the kernel after setting the OWNER_DIED bit below. + * 3. A woken up waiter is killed in user space after another + * thread has acquired the futex, but before it can set + * FUTEX_WAITERS. + * + * Note that, if userspace uses the FUTEX_ROBUST_UNLOCK flag, we + * will not see case 1 here. + * + * In the second and third case, the wake up notification could + * be generated from any of: + * + * i. An ordinary futex wakeup after unlock (with or + * without FUTEX_ROBUST_UNLOCK) + * ii. A robust wakeup from another thread's death + * iii. A previous round through this special case + * + * As a result, the futex world will be in one of four states: + * + * A. The futex word is 0 (unlocked) + * B. The futex word is owned by another thread + * (FUTEX_WAITERS is not set) + * C. The futex word is owned by another thread + * (FUTEX_WAITERS set) + * D. The futex's owner died and OWNER_DIED is set + * (the owner part of the word is 0) * - * In both cases the TID validation below prevents a wakeup of - * potential waiters which can cause these waiters to block - * forever. + * The key issue is that the kernel usually (at least from + * sources ii. and iii. or when so requested by userspace from + * source i.) only ever wakes *one* waiter at a time. If this + * waiter dies before acquiring the futex (or setting the + * FUTEX_WAITERS bit), the kernel *must* still wake the next + * waiter down the line to uphold the futex invariants and + * avoid lost wakeups. Note we do not need to handle state C, + * as it does not matter to us whether *we* successfully set + * the bit or a third thread did so in the meantime. * - * In both cases the following conditions are met: + * Therefore, in these cases we must issue an additional + * futex_wake(). Note however that we *must not* set OWNER_DIED + * here. Our thread is *not* the owner of the futex. * - * 1) task->futex.robust_list->list_op_pending != NULL - * @pending_op == true - * 2) The owner part of user space futex value == 0 + * Thus to summarize, the conditions for needing the additional + * futex_wake() are: + * + * 1) @pending_op == true (the thread has not finished the + * mutex operation) + * 2) The futex word is in one of the states A, B or D * 3) Regular futex: @pi == false * - * If these conditions are met, it is safe to attempt waking up a - * potential waiter without touching the user space futex value and - * trying to set the OWNER_DIED bit. If the futex value is zero, - * the rest of the user space mutex state is consistent, so a woken - * waiter will just take over the uncontended futex. Setting the - * OWNER_DIED bit would create inconsistent state and malfunction - * of the user space owner died handling. Otherwise, the OWNER_DIED - * bit is already set, and the woken waiter is expected to deal with - * this. + * Note in particular that in all of the states A-D the owner + * portion of the futex word differs from our thread's TID + * (unless the actual owner has the same TID in another PID + * namespace, but we cannot currently distinguish that + * scenario), so this can be a special-case wakeup in the bail + * path of the ordinary TID check. */ owner = uval & FUTEX_TID_MASK; - if (pending_op && !pi && !owner) { - futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, NULL, 1, - FUTEX_BITSET_MATCH_ANY); + if (owner != task_pid_vnr(curr)) { + if (pending_op && !pi && (!owner || !(uval & FUTEX_WAITERS))) { + futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, NULL, 1, + FUTEX_BITSET_MATCH_ANY); + } return 0; } - if (owner != task_pid_vnr(curr)) - return 0; - /* * Ok, this dying thread is truly holding a futex * of interest. Set the OWNER_DIED bit atomically diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c index 4834a809985a..175c08a6e41e 100644 --- a/kernel/liveupdate/kexec_handover.c +++ b/kernel/liveupdate/kexec_handover.c @@ -38,6 +38,16 @@ #include "../kexec_internal.h" #include "kexec_handover_internal.h" +/* + * This is the minimal alignment required by deferred struct page init. + * deferred_init_memmap_chunk frees memory to the buddy allocator, which looks + * at the neighboring pages (up to MAX_PAGE_ORDER) to merge them. + * If KHO scratch is not aligned to that value, buddy can access uninitialized + * struct pages, which can cause a crash. + */ +#define SCRATCH_ALIGNMENT_BYTES (PAGE_SIZE * MAX_ORDER_NR_PAGES) +static_assert(SCRATCH_ALIGNMENT_BYTES >= CMA_MIN_ALIGNMENT_BYTES); + /* The magic token for preserved pages */ #define KHO_PAGE_MAGIC 0x4b484f50U /* ASCII for 'KHOP' */ @@ -640,8 +650,8 @@ static void __init scratch_size_update(void) * Scratch areas are released as MIGRATE_CMA. Round them up to the right * size. */ - scratch_size_lowmem = round_up(scratch_size_lowmem, CMA_MIN_ALIGNMENT_BYTES); - scratch_size_global = round_up(scratch_size_global, CMA_MIN_ALIGNMENT_BYTES); + scratch_size_lowmem = round_up(scratch_size_lowmem, SCRATCH_ALIGNMENT_BYTES); + scratch_size_global = round_up(scratch_size_global, SCRATCH_ALIGNMENT_BYTES); } static phys_addr_t __init scratch_size_node(int nid) @@ -656,7 +666,7 @@ static phys_addr_t __init scratch_size_node(int nid) size = scratch_size_pernode; } - return round_up(size, CMA_MIN_ALIGNMENT_BYTES); + return round_up(size, SCRATCH_ALIGNMENT_BYTES); } /** @@ -692,7 +702,7 @@ static void __init kho_reserve_scratch(void) * next kernel */ size = scratch_size_lowmem; - addr = memblock_phys_alloc_range(size, CMA_MIN_ALIGNMENT_BYTES, 0, + addr = memblock_phys_alloc_range(size, SCRATCH_ALIGNMENT_BYTES, 0, ARCH_LOW_ADDRESS_LIMIT); if (!addr) { pr_err("Failed to reserve lowmem scratch buffer\n"); @@ -705,7 +715,7 @@ static void __init kho_reserve_scratch(void) /* reserve large contiguous area for allocations without nid */ size = scratch_size_global; - addr = memblock_phys_alloc(size, CMA_MIN_ALIGNMENT_BYTES); + addr = memblock_phys_alloc(size, SCRATCH_ALIGNMENT_BYTES); if (!addr) { pr_err("Failed to reserve global scratch buffer\n"); goto err_free_scratch_areas; @@ -721,7 +731,7 @@ static void __init kho_reserve_scratch(void) */ for_each_node_state(nid, N_MEMORY) { size = scratch_size_node(nid); - addr = memblock_alloc_range_nid(size, CMA_MIN_ALIGNMENT_BYTES, + addr = memblock_alloc_range_nid(size, SCRATCH_ALIGNMENT_BYTES, 0, MEMBLOCK_ALLOC_ACCESSIBLE, nid, true); if (!addr) { diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index a4e689eefdfb..dff4ee04694c 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -870,8 +870,19 @@ static int sugov_start(struct cpufreq_policy *policy) memset(sg_cpu, 0, sizeof(*sg_cpu)); sg_cpu->cpu = cpu; sg_cpu->sg_policy = sg_policy; + } + + /* + * Publish the hooks only after all per-CPU data is initialized, so a + * shared policy's sugov_update_shared() never reads an uninitialized + * sibling sugov_cpu. + */ + for_each_cpu(cpu, policy->cpus) { + struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu); + cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, uu); } + return 0; } diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 0f858b98c9aa..200300043fa5 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -1017,7 +1017,8 @@ static void update_dl_entity(struct sched_dl_entity *dl_se) if (dl_time_before(dl_se->deadline, rq_clock(rq)) || dl_entity_overflow(dl_se, rq_clock(rq))) { - if (unlikely((!dl_is_implicit(dl_se) || dl_se->dl_defer) && + if (unlikely((!dl_is_implicit(dl_se) || + (dl_se->dl_defer && dl_se->dl_defer_running)) && !dl_time_before(dl_se->deadline, rq_clock(rq)) && !is_dl_boosted(dl_se))) { update_dl_revised_wakeup(dl_se, rq); diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index e3fa7b2fac9d..18183062f751 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3606,6 +3606,9 @@ static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fo } else if (unlikely(fork)) { scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] during fork", p->comm, p->pid); + } else if (unlikely(scx_enable_state() != SCX_ENABLING)) { + scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] outside the enable path", + p->comm, p->pid); } else { struct rq *rq; struct rq_flags rf; @@ -4577,20 +4580,25 @@ static struct cgroup *root_cgroup(void) return &cgrp_dfl_root.cgrp; } +/* + * cgroup_lock() must nest outside the rwsem write side: a writer waiting + * for cgroup_mutex deadlocks with cgroup teardown, which holds it while + * draining a set_* file write blocked on the rwsem behind the writer. + */ static void scx_cgroup_lock(void) { + cgroup_lock(); #ifdef CONFIG_EXT_GROUP_SCHED percpu_down_write(&scx_cgroup_ops_rwsem); #endif - cgroup_lock(); } static void scx_cgroup_unlock(void) { - cgroup_unlock(); #ifdef CONFIG_EXT_GROUP_SCHED percpu_up_write(&scx_cgroup_ops_rwsem); #endif + cgroup_unlock(); } #else /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */ static inline struct cgroup *root_cgroup(void) { return NULL; } @@ -5929,6 +5937,15 @@ static void scx_sub_disable(struct scx_sched *sch) percpu_down_write(&scx_fork_rwsem); scx_cgroup_lock(); + /* + * An enable that failed before scx_link_sched() never owned a cgroup or + * task and won't be waited on by an ancestor's drain_descendants(). + * Nothing to reparent and walking the tasks can misbehave as the task + * ownership invariant (either owned by self or parent) does not hold. + */ + if (list_empty(&sch->sibling)) + goto dump; + set_cgroup_sched(sch_cgroup(sch), parent); scx_task_iter_start(&sti, sch->cgrp); @@ -5941,8 +5958,8 @@ static void scx_sub_disable(struct scx_sched *sch) continue; /* - * By the time control reaches here, all descendant schedulers - * should already have been disabled. + * By the time control reaches here, all linked descendant + * schedulers should have been disabled. */ WARN_ON_ONCE(!scx_task_on_sched(sch, p)); @@ -5993,15 +6010,22 @@ static void scx_sub_disable(struct scx_sched *sch) /* * $p is initialized for $parent and still attached to * @sch. Disable and exit for @sch, switch over to - * $parent, override the state to READY to account for - * $p having already been initialized, and then enable. + * $parent and override the state to READY to account + * for $p having already been initialized. */ scx_disable_and_exit_task(sch, p); scx_set_task_state(p, SCX_TASK_INIT_BEGIN); scx_set_task_state(p, SCX_TASK_INIT); scx_set_task_sched(p, parent); scx_set_task_state(p, SCX_TASK_READY); - scx_enable_task(parent, p); + + /* + * A task on a non-ext class, possible under an + * %SCX_OPS_SWITCH_PARTIAL root, stays READY and is + * enabled by switching_to_scx() if it switches over. + */ + if (p->sched_class == &ext_sched_class) + scx_enable_task(parent, p); } task_rq_unlock(rq, p, &rf); @@ -6009,6 +6033,7 @@ static void scx_sub_disable(struct scx_sched *sch) } scx_task_iter_stop(&sti); +dump: scx_disable_dump(sch); scx_cgroup_unlock(); @@ -7708,10 +7733,14 @@ static void scx_sub_enable_workfn(struct kthread_work *work) /* * $p is now only initialized for @sch and READY, which - * is what we want. Assign it to @sch and enable. + * is what we want. Assign it to @sch and, if it's on + * the ext class, enable. A non-ext task, possible under + * an %SCX_OPS_SWITCH_PARTIAL root, stays READY and is + * enabled by switching_to_scx() if it switches over. */ scx_set_task_sched(p, sch); - scx_enable_task(sch, p); + if (p->sched_class == &ext_sched_class) + scx_enable_task(sch, p); p->scx.flags &= ~SCX_TASK_SUB_INIT; } diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index 8e8c6201b7df..6f93cc32b650 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -554,8 +554,10 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, cpu_rq(cpu)->scx.local_dsq.nr == 0 && (!(flags & SCX_PICK_IDLE_IN_NODE) || (waker_node == node)) && !cpumask_empty(idle_cpumask(waker_node)->cpu)) { - if (cpumask_test_cpu(cpu, allowed)) + if (cpumask_test_cpu(cpu, allowed)) { + scx_idle_test_and_clear_cpu(cpu); goto out_unlock; + } } } diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index d9c9d9480a45..e2e825dcd088 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -1134,6 +1134,12 @@ void psi_cgroup_free(struct cgroup *cgroup) return; cancel_delayed_work_sync(&cgroup->psi->avgs_work); + /* + * A psi_schedule_rtpoll_work() call racing the last trigger's + * destruction may have re-armed the timer after psi_trigger_destroy() + * deleted it. Spurious firing while the group is alive is harmless. + */ + timer_shutdown_sync(&cgroup->psi->rtpoll_timer); free_percpu(cgroup->psi->pcpu); /* All triggers must be removed by now */ WARN_ONCE(cgroup->psi->rtpoll_states, "psi: trigger leak\n"); @@ -1292,9 +1298,44 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) return 0; } +/* + * Create @group's rtpoll worker after psi_trigger_create() reported the need + * for one. kthread creation depends on the whole fork path and we don't want + * all of that nested inside cgroup_mutex, so the caller must drop it and any + * other lock that forks can wait behind. If two callers race, the loser stops + * its never-woken kthread. + */ +int psi_trigger_create_rtpoll_worker(struct psi_group *group) +{ + struct task_struct *task; + + task = kthread_create(psi_rtpoll_worker, group, "psimon"); + if (IS_ERR(task)) + return PTR_ERR(task); + + scoped_guard(mutex, &group->rtpoll_trigger_lock) { + if (!rcu_access_pointer(group->rtpoll_task)) { + atomic_set(&group->rtpoll_wakeup, 0); + wake_up_process(task); + rcu_assign_pointer(group->rtpoll_task, task); + + /* + * Poll once to catch up on scheduling attempts dropped + * while there was no rtpoll worker. + */ + psi_schedule_rtpoll_work(group, 1, true); + return 0; + } + } + + kthread_stop(task); + return 0; +} + struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, enum psi_res res, struct file *file, - struct kernfs_open_file *of) + struct kernfs_open_file *of, + bool *need_rtpoll_worker) { struct psi_trigger *t; enum psi_states state; @@ -1302,6 +1343,8 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, bool privileged; u32 window_us; + *need_rtpoll_worker = false; + if (static_branch_likely(&psi_disabled)) return ERR_PTR(-EOPNOTSUPP); @@ -1362,26 +1405,14 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, if (privileged) { mutex_lock(&group->rtpoll_trigger_lock); - if (!rcu_access_pointer(group->rtpoll_task)) { - struct task_struct *task; - - task = kthread_create(psi_rtpoll_worker, group, "psimon"); - if (IS_ERR(task)) { - kfree(t); - mutex_unlock(&group->rtpoll_trigger_lock); - return ERR_CAST(task); - } - atomic_set(&group->rtpoll_wakeup, 0); - wake_up_process(task); - rcu_assign_pointer(group->rtpoll_task, task); - } - list_add(&t->node, &group->rtpoll_triggers); group->rtpoll_min_period = min(group->rtpoll_min_period, div_u64(t->win.size, UPDATES_PER_WINDOW)); group->rtpoll_nr_triggers[t->state]++; group->rtpoll_states |= (1 << t->state); + *need_rtpoll_worker = !rcu_access_pointer(group->rtpoll_task); + mutex_unlock(&group->rtpoll_trigger_lock); } else { mutex_lock(&group->avgs_lock); @@ -1541,6 +1572,8 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf, size_t buf_size; struct seq_file *seq; struct psi_trigger *new; + bool need_rtpoll_worker; + int ret; if (static_branch_likely(&psi_disabled)) return -EOPNOTSUPP; @@ -1565,12 +1598,22 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf, return -EBUSY; } - new = psi_trigger_create(&psi_system, buf, res, file, NULL); + new = psi_trigger_create(&psi_system, buf, res, file, NULL, + &need_rtpoll_worker); if (IS_ERR(new)) { mutex_unlock(&seq->lock); return PTR_ERR(new); } + if (need_rtpoll_worker) { + ret = psi_trigger_create_rtpoll_worker(&psi_system); + if (ret) { + psi_trigger_destroy(new); + mutex_unlock(&seq->lock); + return ret; + } + } + smp_store_release(&seq->private, new); mutex_unlock(&seq->lock); diff --git a/kernel/smp.c b/kernel/smp.c index a0bb56bd8dda..52dffc86555c 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -137,10 +137,10 @@ csd_do_func(smp_call_func_t func, void *info, call_single_data_t *csd) trace_csd_function_exit(func, csd); } -#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG - static DEFINE_STATIC_KEY_MAYBE(CONFIG_CSD_LOCK_WAIT_DEBUG_DEFAULT, csdlock_debug_enabled); +#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG + /* * Parse the csdlock_debug= kernel boot parameter. * @@ -342,6 +342,10 @@ static __always_inline void csd_lock_wait(call_single_data_t *csd) smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); } #else +static __always_inline void __csd_lock_wait(call_single_data_t *csd) +{ +} + static void csd_lock_record(call_single_data_t *csd) { } @@ -354,8 +358,23 @@ static __always_inline void csd_lock_wait(call_single_data_t *csd) static __always_inline void csd_lock(call_single_data_t *csd) { - csd_lock_wait(csd); - csd->node.u_flags |= CSD_FLAG_LOCK; + if (IS_ENABLED(CONFIG_CSD_LOCK_WAIT_DEBUG) && + static_branch_unlikely(&csdlock_debug_enabled)) { + + for (;;) { + unsigned int flags; + + __csd_lock_wait(csd); + flags = READ_ONCE(csd->node.u_flags); + + if (!(flags & CSD_FLAG_LOCK) && + try_cmpxchg_acquire(&csd->node.u_flags, &flags, flags | CSD_FLAG_LOCK)) + break; + } + } else { + csd_lock_wait(csd); + csd->node.u_flags |= CSD_FLAG_LOCK; + } /* * prevent CPU from reordering the above assignment @@ -380,7 +399,8 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data); #ifdef CONFIG_CSD_LOCK_WAIT_DEBUG static call_single_data_t *get_single_csd_data(int cpu) { - if (static_branch_unlikely(&csdlock_debug_enabled)) + if (static_branch_unlikely(&csdlock_debug_enabled) && + (unsigned int)cpu < nr_cpu_ids) return per_cpu_ptr(&csd_data, cpu); return this_cpu_ptr(&csd_data); } diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index f215990b9061..f681015413b8 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -961,10 +961,8 @@ int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter return -ENOMEM; ret = get_ips_from_filter(filter, notfilter, addrs, mods, num); - if (ret < 0) - return ret; - - ret = register_fprobe_ips(fp, addrs, ret); + if (ret >= 0) + ret = register_fprobe_ips(fp, addrs, ret); for (int i = 0; i < num; i++) { if (mods[i]) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index f93e34dd2328..6c47a94f5924 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1098,6 +1098,12 @@ struct ftrace_ops global_ops = { }; /* + * parser_lock - Protects trace_parser state against concurrent operations. + * Held across trace_get_user() and subsequent buffer parsing to prevent races. + */ +static DEFINE_MUTEX(parser_lock); + +/* * Used by the stack unwinder to know about dynamic ftrace trampolines. */ struct ftrace_ops *ftrace_ops_trampoline(unsigned long addr) @@ -5842,6 +5848,8 @@ ftrace_regex_write(struct file *file, const char __user *ubuf, /* iter->hash is a local copy, so we don't need regex_lock */ parser = &iter->parser; + + guard(mutex)(&parser_lock); read = trace_get_user(parser, ubuf, cnt, ppos); if (read >= 0 && trace_parser_loaded(parser) && @@ -6984,12 +6992,14 @@ int ftrace_regex_release(struct inode *inode, struct file *file) iter = file->private_data; parser = &iter->parser; + mutex_lock(&parser_lock); if (trace_parser_loaded(parser)) { int enable = !(iter->flags & FTRACE_ITER_NOTRACE); ftrace_process_regex(iter, parser->buffer, parser->idx, enable); } + mutex_unlock(&parser_lock); trace_parser_put(parser); @@ -7321,10 +7331,12 @@ ftrace_graph_release(struct inode *inode, struct file *file) parser = &fgd->parser; + mutex_lock(&parser_lock); if (trace_parser_loaded((parser))) { ret = ftrace_graph_set_hash(fgd->new_hash, parser->buffer); } + mutex_unlock(&parser_lock); trace_parser_put(parser); @@ -7437,6 +7449,7 @@ ftrace_graph_write(struct file *file, const char __user *ubuf, parser = &fgd->parser; + guard(mutex)(&parser_lock); read = trace_get_user(parser, ubuf, cnt, ppos); if (read >= 0 && trace_parser_loaded(parser) && diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 804ccae694d2..8e2485bb3aa8 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -2599,6 +2599,7 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) return_ptr(cpu_buffer); fail_free_reader: + kfree(cpu_buffer->subbuf_ids); free_buffer_page(cpu_buffer->reader_page); return NULL; @@ -5783,6 +5784,7 @@ __rb_get_reader_page_from_remote(struct ring_buffer_per_cpu *cpu_buffer) cpu_buffer->head_page = new_head; cpu_buffer->reader_page = new_reader; + cpu_buffer->reader_page->read = 0; cpu_buffer->pages = &new_head->list; cpu_buffer->read_stamp = new_reader->page->time_stamp; cpu_buffer->lost_events = cpu_buffer->meta_page->reader.lost_events; diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 18710c190c92..01a5e87af299 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -6187,7 +6187,7 @@ char *trace_user_fault_read(struct trace_user_buf_info *tinfo, { int cpu = smp_processor_id(); char *buffer = per_cpu_ptr(tinfo->tbuf, cpu)->buf; - unsigned int cnt; + unsigned long long cnt; int trys = 0; int ret; diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 80fe152af1dd..bf77331f56a4 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -1941,6 +1941,7 @@ struct event_trigger_data { struct list_head named_list; struct event_trigger_data *named_data; struct llist_node llist; + void (*private_data_free)(struct event_trigger_data *data); }; /* Avoid typos */ diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index c46e623e7e0d..c01b10b99f67 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -1350,7 +1350,9 @@ __ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match, call = file->event_call; /* If a module is specified, skip events that are not that module */ - if (module && (!call->module || strcmp(module_name(call->module), module))) + if (module && + ((call->flags & TRACE_EVENT_FL_DYNAMIC) || + !call->module || strcmp(module_name(call->module), module))) continue; name = trace_event_name(call); @@ -3931,8 +3933,8 @@ static void trace_module_add_events(struct module *mod) end = mod->trace_events + mod->num_trace_events; for_each_event(call, start, end) { - __register_event(*call, mod); - __add_event_to_tracers(*call); + if (!__register_event(*call, mod)) + __add_event_to_tracers(*call); } update_cache_events(mod); diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 6385cd662d8d..2b46ca536045 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -1027,6 +1027,9 @@ static int regex_match_full(char *str, struct regex *r, int len) if (!len) return strcmp(str, r->pattern) == 0; + if (len < r->len) + return 0; + return strncmp(str, r->pattern, len) == 0; } diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 82ce492ab268..58d28cd1afa3 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -6349,6 +6349,7 @@ static void event_hist_trigger_free(struct event_trigger_data *data) trigger_data_free(data); + tracepoint_synchronize_unregister(); remove_hist_vars(hist_data); unregister_field_var_hists(hist_data); @@ -6388,6 +6389,7 @@ static void event_hist_trigger_named_free(struct event_trigger_data *data) del_named_trigger(data); trigger_data_free(data); + tracepoint_synchronize_unregister(); kfree(cmd_ops); } } diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c index 655db2e82513..ad83419cb420 100644 --- a/kernel/trace/trace_events_trigger.c +++ b/kernel/trace/trace_events_trigger.c @@ -38,6 +38,13 @@ static void trigger_create_kthread_locked(void) } } +static void trigger_data_free_one(struct event_trigger_data *data) +{ + if (data->private_data_free) + data->private_data_free(data); + kfree(data); +} + static void trigger_data_free_queued_locked(void) { struct event_trigger_data *data, *tmp; @@ -52,7 +59,7 @@ static void trigger_data_free_queued_locked(void) tracepoint_synchronize_unregister(); llist_for_each_entry_safe(data, tmp, llnodes, llist) - kfree(data); + trigger_data_free_one(data); } /* Bulk garbage collection of event_trigger_data elements */ @@ -75,7 +82,7 @@ static int trigger_kthread_fn(void *ignore) tracepoint_synchronize_unregister(); llist_for_each_entry_safe(data, tmp, llnodes, llist) - kfree(data); + trigger_data_free_one(data); } return 0; @@ -1717,6 +1724,14 @@ int event_enable_trigger_print(struct seq_file *m, return 0; } +static void enable_trigger_private_data_free(struct event_trigger_data *data) +{ + struct enable_trigger_data *enable_data = data->private_data; + + trace_event_put_ref(enable_data->file->event_call); + kfree(enable_data); +} + void event_enable_trigger_free(struct event_trigger_data *data) { struct enable_trigger_data *enable_data = data->private_data; @@ -1728,9 +1743,8 @@ void event_enable_trigger_free(struct event_trigger_data *data) if (!data->ref) { /* Remove the SOFT_MODE flag */ trace_event_enable_disable(enable_data->file, 0, 1); - trace_event_put_ref(enable_data->file->event_call); + data->private_data_free = enable_trigger_private_data_free; trigger_data_free(data); - kfree(enable_data); } } diff --git a/kernel/trace/trace_fprobe.c b/kernel/trace/trace_fprobe.c index 9f5f08c0e7c2..3120403a5b60 100644 --- a/kernel/trace/trace_fprobe.c +++ b/kernel/trace/trace_fprobe.c @@ -1481,11 +1481,21 @@ static int enable_trace_fprobe(struct trace_event_call *call, list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) { ret = __register_trace_fprobe(tf); if (ret < 0) - return ret; + goto err; } } return 0; + +err: + /* Failed to enable one of them. Roll back all */ + list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) + __unregister_trace_fprobe(tf); + if (file) + trace_probe_remove_file(tp, file); + else + trace_probe_clear_flag(tp, TP_FLAG_PROFILE); + return ret; } /* diff --git a/kernel/trace/trace_mmiotrace.c b/kernel/trace/trace_mmiotrace.c index 226cf66e0d68..df8692c2dea8 100644 --- a/kernel/trace/trace_mmiotrace.c +++ b/kernel/trace/trace_mmiotrace.c @@ -29,6 +29,7 @@ static void mmio_reset_data(struct trace_array *tr) { overrun_detected = false; prev_overruns = 0; + atomic_set(&dropped_count, 0); tracing_reset_online_cpus(&tr->array_buffer); } @@ -109,7 +110,6 @@ static void mmio_pipe_open(struct trace_iterator *iter) iter->private = hiter; } -/* XXX: This is not called when the pipe is closed! */ static void mmio_close(struct trace_iterator *iter) { struct header_iter *hiter = iter->private; @@ -146,7 +146,7 @@ static ssize_t mmio_read(struct trace_iterator *iter, struct file *filp, goto print_out; } - if (!hiter) + if (!hiter || !hiter->dev) return 0; mmio_print_pcidev(s, hiter->dev); @@ -279,6 +279,7 @@ static struct tracer mmio_tracer __read_mostly = .start = mmio_trace_start, .pipe_open = mmio_pipe_open, .close = mmio_close, + .pipe_close = mmio_close, .read = mmio_read, .print_line = mmio_print_line, .noboot = true, @@ -293,11 +294,15 @@ device_initcall(init_mmio_trace); static void __trace_mmiotrace_rw(struct trace_array *tr, struct mmiotrace_rw *rw) { - struct trace_buffer *buffer = tr->array_buffer.buffer; + struct trace_buffer *buffer; struct ring_buffer_event *event; struct trace_mmiotrace_rw *entry; unsigned int trace_ctx; + if (!tr) + return; + + buffer = tr->array_buffer.buffer; trace_ctx = tracing_gen_ctx_flags(0); event = trace_buffer_lock_reserve(buffer, TRACE_MMIO_RW, sizeof(*entry), trace_ctx); @@ -320,11 +325,15 @@ void mmio_trace_rw(struct mmiotrace_rw *rw) static void __trace_mmiotrace_map(struct trace_array *tr, struct mmiotrace_map *map) { - struct trace_buffer *buffer = tr->array_buffer.buffer; + struct trace_buffer *buffer; struct ring_buffer_event *event; struct trace_mmiotrace_map *entry; unsigned int trace_ctx; + if (!tr) + return; + + buffer = tr->array_buffer.buffer; trace_ctx = tracing_gen_ctx_flags(0); event = trace_buffer_lock_reserve(buffer, TRACE_MMIO_MAP, sizeof(*entry), trace_ctx); diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index 506e6037e163..c8fd9b946f44 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -1901,7 +1901,11 @@ const char **traceprobe_expand_meta_args(int argc, const char *argv[], trace_probe_log_err(0, BAD_VAR); return ERR_PTR(-ENOENT); } - /* Note: $argN starts from $arg1 */ + /* Note: $argN starts from $arg1, so $arg0 is invalid. */ + if (n == 0) { + trace_probe_log_err(0, BAD_ARG_NUM); + return ERR_PTR(-EINVAL); + } ret = sprint_nth_btf_arg(n - 1, type, buf + used, bufsize - used, ctx); if (ret < 0) diff --git a/kernel/trace/trace_remote.c b/kernel/trace/trace_remote.c index 0f6ef5c36d84..e6724f947170 100644 --- a/kernel/trace/trace_remote.c +++ b/kernel/trace/trace_remote.c @@ -1004,11 +1004,10 @@ int trace_remote_alloc_buffer(struct trace_buffer_desc *desc, size_t desc_size, desc->nr_cpus++; for (id = 0; id < nr_pages; id++) { + rb_desc->nr_page_va++; rb_desc->page_va[id] = (unsigned long)__get_free_page(GFP_KERNEL); if (!rb_desc->page_va[id]) goto err; - - rb_desc->nr_page_va++; } rb_desc = __next_ring_buffer_desc(rb_desc); } @@ -1150,10 +1149,21 @@ static ssize_t remote_events_dir_enable_write(struct file *filp, const char __us for (i = 0; i < remote->nr_events; i++) { struct remote_event *evt = &remote->events[i]; + int eret; - trace_remote_enable_event(remote, evt, enable); + eret = trace_remote_enable_event(remote, evt, enable); + /* + * Save the first error and return that. Some events + * may still have been enabled, but let the user + * know that something went wrong. + */ + if (!ret && eret) + ret = eret; } + if (ret) + return ret; + return count; } diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c index e98ee7e1e66f..8a4f3c75e39f 100644 --- a/kernel/trace/trace_syscalls.c +++ b/kernel/trace/trace_syscalls.c @@ -1451,6 +1451,11 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) if (syscall_get_data(sys_data, args, &user_ptr, &size, user_sizes, &uargs, buf_size) < 0) return; + + /* The above may have caused a migration */ + head = this_cpu_ptr(sys_data->enter_event->perf_events); + if (hlist_empty(head)) + return; } /* get the size after alignment with the u32 buffer size field */ |