From 72dd0ec09e7cc98ed58ddeac26575e5d1ab8a93d Mon Sep 17 00:00:00 2001 From: Tetsuo Handa Date: Tue, 4 Aug 2026 22:52:19 +0900 Subject: printk: Don't WARN on kthread_run failure. Since __kthread_create_on_node() returns -EINTR upon SIGKILL, we should not use WARN_ON() in order to catch kthread_run() failure. Reported-by: syzbot+1ebbc20f223b99446034@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=1ebbc20f223b99446034 Fixes: 5f53ca3ff83b ("printk: Implement legacy printer kthread for PREEMPT_RT") Fixes: 76f258bf3f2a ("printk: nbcon: Introduce printer kthreads") Signed-off-by: Tetsuo Handa Reviewed-by: John Ogness Reviewed-by: Petr Mladek Link: https://patch.msgid.link/76bb4c1c-5d85-4635-b3bb-fc06f292c59e@I-love.SAKURA.ne.jp Signed-off-by: Petr Mladek --- kernel/printk/nbcon.c | 2 +- kernel/printk/printk.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 4b03b019cd5e..a5921a84a80e 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -1382,7 +1382,7 @@ bool nbcon_kthread_create(struct console *con) return true; kt = kthread_run(nbcon_kthread_func, con, "pr/%s%d", con->name, con->index); - if (WARN_ON(IS_ERR(kt))) { + if (IS_ERR(kt)) { con_printk(KERN_ERR, con, "failed to start printing thread\n"); return false; } diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 3fcdf4b4e2e5..6d3d18a50da7 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3732,7 +3732,7 @@ static bool legacy_kthread_create(void) lockdep_assert_console_list_lock_held(); kt = kthread_run(legacy_kthread_func, NULL, "pr/legacy"); - if (WARN_ON(IS_ERR(kt))) { + if (IS_ERR(kt)) { pr_err("failed to start legacy printing thread\n"); return false; } -- cgit v1.2.3 From 5a88f78df753993469dab4d1831f8fb4256a9468 Mon Sep 17 00:00:00 2001 From: "Cen Zhang (Microsoft)" Date: Fri, 14 Aug 2026 00:09:44 -0400 Subject: reboot: fix cad_pid use-after-free race cad_pid is a single kernel-wide struct pid pointer. proc_do_cad_pid() reads it and passes it to pid_vnr() without protecting the lifetime of the referenced struct pid. A concurrent writer can replace cad_pid and drop the final reference to the old struct pid after the reader has loaded the pointer but before pid_vnr() has finished dereferencing it, causing a use-after-free. kill_cad_pid() has the same lifetime race when it passes cad_pid to kill_pid(). At the time this issue was reported, an unprivileged user could reach the sysctl through user and PID namespaces because cad_pid was registered in pid_table[]. Moving cad_pid back to the global reboot sysctl table corrected that namespace and permission mismatch, but did not fix the underlying lifetime race. Fix this by treating cad_pid as an RCU-protected pointer at both read sites and by waiting for a grace period before dropping the old reference on the write side. call_rcu(&old_pid->rcu, ...) cannot be used here because free_pid() also queues pid->rcu; queueing the same rcu_head twice can corrupt the RCU callback list. Original KASAN crash stack: kernel/pid.c:545 pid_nr_ns() # reads freed pid->level kernel/pid.c:556 pid_vnr() # calls pid_nr_ns() kernel/pid.c:775 proc_do_cad_pid() # calls pid_vnr(cad_pid) Fixes: 9ec52099e4b8 ("[PATCH] replace cad_pid by a struct pid") Reported-by: AutonomousCodeSecurity@microsoft.com Closes: https://lore.kernel.org/all/20260717210143.4734-1-blbllhy@gmail.com/ Link: https://lore.kernel.org/all/alz5ZYLE4kaq_v2P@redhat.com/ Link: https://lore.kernel.org/all/al4ICz9biJKtdZc4@redhat.com/ Suggested-by: Mateusz Guzik Suggested-by: Bradley Morgan Suggested-by: Oleg Nesterov Suggested-by: Eric W. Biederman Suggested-by: Pavel Tikhomirov Cc: stable@vger.kernel.org Signed-off-by: Cen Zhang (Microsoft) Link: https://patch.msgid.link/20260814040944.16561-1-blbllhy@gmail.com Reviewed-by: Bradley Morgan Reviewed-by: Oleg Nesterov Reviewed-by: Pavel Tikhomirov Signed-off-by: Christian Brauner (Amutable) --- kernel/reboot.c | 19 +++++++++++++++---- kernel/signal.c | 12 ++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/reboot.c b/kernel/reboot.c index f070c5c1103a..d177d89fcc33 100644 --- a/kernel/reboot.c +++ b/kernel/reboot.c @@ -13,7 +13,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -24,8 +26,7 @@ */ static int C_A_D = 1; -struct pid *cad_pid; -EXPORT_SYMBOL(cad_pid); +struct pid __rcu *cad_pid; #if defined(CONFIG_ARM) #define DEFAULT_REBOOT_MODE = REBOOT_HARD @@ -1371,10 +1372,14 @@ static int proc_do_cad_pid(const struct ctl_table *table, int write, void *buffe { struct ctl_table tmp_table = *table; struct pid *new_pid; + struct pid *old_pid; pid_t tmp_pid; int r; - tmp_pid = pid_vnr(cad_pid); + rcu_read_lock(); + tmp_pid = pid_vnr(rcu_dereference(cad_pid)); + rcu_read_unlock(); + tmp_table.data = &tmp_pid; r = proc_dointvec(&tmp_table, write, buffer, lenp, ppos); @@ -1385,7 +1390,13 @@ static int proc_do_cad_pid(const struct ctl_table *table, int write, void *buffe if (!new_pid) return -ESRCH; - put_pid(xchg(&cad_pid, new_pid)); + old_pid = unrcu_pointer(xchg(&cad_pid, RCU_INITIALIZER(new_pid))); + /* + * Wait for cad_pid readers before put_pid(). We cannot use + * call_rcu() here because free_pid() already owns pid->rcu. + */ + synchronize_rcu(); + put_pid(old_pid); return 0; } diff --git a/kernel/signal.c b/kernel/signal.c index bbc0fd4cc4d7..2162fad7b940 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -1899,6 +1899,18 @@ int kill_pid(struct pid *pid, int sig, int priv) } EXPORT_SYMBOL(kill_pid); +int kill_cad_pid(int sig, int priv) +{ + int ret; + + rcu_read_lock(); + ret = kill_pid(rcu_dereference(cad_pid), sig, priv); + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL(kill_cad_pid); + #ifdef CONFIG_POSIX_TIMERS /* * These functions handle POSIX timer signals. POSIX timers use -- cgit v1.2.3 From 93d88ac4a448f200d18d1d71de0074b8e716bbee Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Tue, 25 Aug 2026 10:35:56 +0800 Subject: sched_ext: Pass the initial cpu.idle state in scx_cgroup_init_args scx_cgroup_init_args carries the initial weight and bandwidth control parameters of a cgroup to ops.cgroup_init(), but not its cpu.idle state. A cgroup that was already configured idle before the scheduler was loaded (or before it was onlined under it) is presented as non-idle, and the BPF scheduler only learns about it if cpu.idle is written again later. Add the sched_idle state to scx_cgroup_init_args and fill it in all four places that build the args: scx_tg_online() for cgroups onlined under the scheduler, scx_cgroup_init() for cgroups that already exist when the scheduler is loaded, and the sub-scheduler handover paths scx_cgroup_claim_subtree() and scx_cgroup_return_subtree(). Verified in a VM with a probe scheduler printing the init args: a cgroup configured cpu.idle=1 before loading shows sched_idle=1 in ops.cgroup_init(), the default shows 0, and later cpu.idle writes still come through ops.cgroup_set_idle(). The sub-scheduler paths are compile-tested only. Fixes: 347ed2d566da ("sched/ext: Implement cgroup_set_idle() callback") Signed-off-by: Tao Cui Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 4 +++- kernel/sched/ext/internal.h | 3 +++ kernel/sched/ext/sub.c | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 713aa26b2828..901fb0f8b976 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4766,7 +4766,8 @@ int scx_tg_online(struct task_group *tg) { .weight = tg->scx.weight, .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, - .bw_burst_us = tg->scx.bw_burst_us }; + .bw_burst_us = tg->scx.bw_burst_us, + .sched_idle = tg->scx.idle }; ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, tg->css.cgroup, &args); @@ -5187,6 +5188,7 @@ static int scx_cgroup_init(struct scx_sched *sch) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, + .sched_idle = tg->scx.idle, }; ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 0967b99a4948..076a351bb3f2 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -259,6 +259,9 @@ struct scx_cgroup_init_args { u64 bw_period_us; u64 bw_quota_us; u64 bw_burst_us; + + /* whether the cgroup is configured SCHED_IDLE via cpu.idle */ + bool sched_idle; }; enum scx_cpu_preempt_reason { diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 0554448835bd..385302d19914 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1361,6 +1361,7 @@ static s32 scx_cgroup_claim_subtree(struct scx_sched *sch) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, + .sched_idle = tg->scx.idle, }; if (tg->scx.sched != parent || @@ -1464,6 +1465,7 @@ static void scx_cgroup_return_subtree(struct scx_sched *sch) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, + .sched_idle = tg->scx.idle, }; /* the first pass must have transferred everything */ -- cgit v1.2.3 From 29871903f3a38e75e896f9a644713859490cb85f Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Tue, 1 Sep 2026 20:43:47 +0800 Subject: sched_ext: Don't deliver duplicate ops.cgroup_set_idle() for same value ops.cgroup_set_idle() is documented to be invoked when a cgroup transitions between idle and non-idle states, and scx_group_set_weight() already skips value-preserving writes. scx_group_set_idle() delivers every write unconditionally, so rewriting an already-correct cpu.idle value feeds the BPF scheduler a transition callback each time, which toggle- or accounting-based schedulers miscount. Mirror the weight guard and only deliver on an actual change. Verified with a probe scheduler printing each callback: rewriting cpu.idle=1 twice on an already-idle cgroup delivered two callbacks before and none after. Fixes: 347ed2d566da ("sched/ext: Implement cgroup_set_idle() callback") Link: https://lore.kernel.org/r/b53c61a1-4d7d-4232-941f-d48b0563d4ed@linux.dev Signed-off-by: Tao Cui Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 901fb0f8b976..5f242ab69cf4 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4933,7 +4933,8 @@ void scx_group_set_idle(struct task_group *tg, bool idle) percpu_down_read(&scx_cgroup_ops_rwsem); sch = scx_tg_knob_sched(tg); - if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle)) + if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle) && + tg->scx.idle != idle) SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); /* Update the task group's idle state */ -- cgit v1.2.3 From 0433632bbe8d279e978f3f85212f36281a89946c Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 1 Sep 2026 11:37:42 +0206 Subject: printk/nbcon: Flush nbcon_irq_work in nbcon_free() Ensure any pending nbcon_irq_work is flushed before allowing the console to be recycled. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://patch.msgid.link/20260901093245.344455-2-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/nbcon.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 4b03b019cd5e..354f274d8a42 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -1837,6 +1837,8 @@ void nbcon_free(struct console *con) /* Synchronize the kthread stop. */ lockdep_assert_console_list_lock_held(); + irq_work_sync(&con->irq_work); + if (printk_kthreads_running) { nbcon_kthread_stop(con); -- cgit v1.2.3 From 560f4deda32785e260056200f8bb911c475c5b88 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 1 Sep 2026 11:37:43 +0206 Subject: printk/nbcon: Change nbcon_irq_work to IRQ_WORK_LAZY Change the nbcon_irq_work to be IRQ_WORK_LAZY, thus not raising an IRQ upon irq_work queuing. The irq_work is then handled on the next kernel tick. This additional delay is acceptable because nbcon_irq_work is only responsible for non-emergency deferred printing, which is delayed anyway. This has the benefit of not needing to raise an IRQ for each printk() call. On a side note, the Tegra20 and Tegra30 platforms can hang if an irq_work IRQ is raised while entering cpuidle states. This problem was reproducible by calling printk() while entering cpuidle. So this change also provides a workaround for these platforms (as long as they are not running tickless). Link: https://lore.kernel.org/lkml/f3757a75-0ba1-4558-bf57-f19ab7e59a4c@nvidia.com Fixes: 76f258bf3f2a ("printk: nbcon: Introduce printer kthreads") Signed-off-by: John Ogness Reviewed-by: Sebastian Andrzej Siewior Reviewed-by: Petr Mladek Tested-by: Jon Hunter Link: https://patch.msgid.link/20260901093245.344455-3-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/nbcon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 354f274d8a42..c8502fc4f4e5 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -1782,7 +1782,7 @@ bool nbcon_alloc(struct console *con) } rcuwait_init(&con->rcuwait); - init_irq_work(&con->irq_work, nbcon_irq_work); + con->irq_work = IRQ_WORK_INIT_LAZY(nbcon_irq_work); atomic_long_set(&ACCESS_PRIVATE(con, nbcon_prev_seq), -1UL); nbcon_state_set(con, &state); -- cgit v1.2.3 From 0a85182723b65ad8bee8131bc38fcf0347d6679b Mon Sep 17 00:00:00 2001 From: Wanwu Li Date: Thu, 3 Sep 2026 14:06:26 +0800 Subject: sched_ext: Fix NULL sched deref in kfunc sub-sched error paths When the root scheduler has sub-scheds attached, the COMPAT kfunc wrappers scx_bpf_select_cpu_and() and scx_bpf_dsq_insert_vtime() refuse the call and report to @p's scheduler: scx_error(scx_task_sched(p), "... must be used"); The wrappers are reachable with tasks that have no scheduler. scx_bpf_select_cpu_and() is in the select_cpu kfunc group, which scx_kfunc_context_filter() opens to BPF_PROG_TYPE_SYSCALL programs; scx_bpf_dsq_insert_vtime() is in the enqueue_dispatch group, which ops.enqueue() and ops.dispatch() may call with any KF_RCU task -- the group has no kf_tasks validation, and scx_dsq_insert_preamble() checks task ownership with scx_task_on_sched() precisely because @p may be an arbitrary task. scx_task_sched(p) is p->scx.sched, which is NULL for tasks past sched_ext_dead() -- which clears it via scx_disable_and_exit_task() on exit -- and for idle tasks, which the enable paths skip as they are never scheduled through SCX. It is also an rcu_dereference_protected() that expects @p's pi_lock or rq lock, which neither wrapper holds. Passing NULL to scx_error() reaches scx_vexit(), which dereferences sch->exit_info, oopsing the kernel. One concrete trigger exercised while developing the fix: a BPF_PROG_TYPE_SYSCALL program calling the select_cpu_and wrapper on an exited-but-not-reaped task while a sub-scheduler was attached (its pid stays findable while the zombie is unreaped; faulting instruction is the scx_vexit() prologue "mov r15,[rdi+0x398]" with RDI=NULL and 0x398 the offset of sch->exit_info): sched_ext: BPF scheduler "kfunc_subsched_null" enabled sched_ext: BPF sub-scheduler "kfunc_subsched_null" enabled sched_ext: Unassociated program run_select_cpu_ (id 76) BUG: kernel NULL pointer dereference, address: 0000000000000398 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page Oops: Oops: 0000 [#1] SMP NOPTI CPU: 7 UID: 0 PID: 8201 Comm: kfunc_test_runn Tainted: G W RIP: 0010:scx_vexit+0x25/0xa0 Code: ... <4c> 8b bf 98 03 00 00 ... CR2: 0000000000000398 Call Trace: __scx_exit+0x4f/0x70 scx_bpf_select_cpu_and+0xab/0xb0 bpf_prog_430ed61a7b66e03a_run_select_cpu_and+0x9c/0xe7 ? __x64_sys_bpf+0x2c/0x40 bpf_prog_test_run_syscall+0x130/0x2f0 __sys_bpf+0x930/0x10d0 ? __x64_sys_bpf+0x2c/0x40 __x64_sys_bpf+0x2c/0x40 do_syscall_64+0xbc/0x460 entry_SYSCALL_64_after_hwframe+0x76/0x7e Read @p's scheduler under RCU instead, which the wrappers can do from their guard(rcu)(): fault it when it can be determined, and when it can't be determined -- @p is a task past sched_ext_dead() or an idle task -- there is nothing obviously wrong to report, so just refuse the call as before without faulting any scheduler. These COMPAT wrappers are scheduled for eventual removal once the deprecation grace period elapses, but until then -- and regardless of their removal timeline -- they must not oops the kernel on a task they are handed. Cc: stable@vger.kernel.org # v7.1+ Fixes: a5fa0708cbfd ("sched_ext: Enforce scheduling authority in dispatch and select_cpu operations") Suggested-by: Andrea Righi Signed-off-by: Wanwu Li Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 11 +++++++++-- kernel/sched/ext/idle.c | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 5f242ab69cf4..7e414a7c53fc 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -8949,10 +8949,17 @@ __bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, #ifdef CONFIG_EXT_SUB_SCHED /* * Disallow if any sub-scheds are attached. There is no way to tell - * which scheduler called us, just error out @p's scheduler. + * which scheduler called us, so error out @p's scheduler -- read it + * under RCU as @p's locks aren't necessarily held here. @p may be a + * task past sched_ext_dead() or an idle task, in which case its + * scheduler can't be determined and there is nothing obviously wrong + * to report; just refuse the call. */ if (unlikely(!list_empty(&sch->children))) { - scx_error(scx_task_sched(p), "__scx_bpf_dsq_insert_vtime() must be used"); + struct scx_sched *tsch = scx_task_sched_rcu(p); + + if (tsch) + scx_error(tsch, "__scx_bpf_dsq_insert_vtime() must be used"); return; } #endif diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index d2973fb3af6d..aa9fb6de0ad6 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -1142,10 +1142,17 @@ __bpf_kfunc s32 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64 #ifdef CONFIG_EXT_SUB_SCHED /* * Disallow if any sub-scheds are attached. There is no way to tell - * which scheduler called us, just error out @p's scheduler. + * which scheduler called us, so error out @p's scheduler -- read it + * under RCU as @p's locks aren't necessarily held here. @p may be a + * task past sched_ext_dead() or an idle task, in which case its + * scheduler can't be determined and there is nothing obviously wrong + * to report; just refuse the call. */ if (unlikely(!list_empty(&sch->children))) { - scx_error(scx_task_sched(p), "__scx_bpf_select_cpu_and() must be used"); + struct scx_sched *tsch = scx_task_sched_rcu(p); + + if (tsch) + scx_error(tsch, "__scx_bpf_select_cpu_and() must be used"); return -EINVAL; } #endif -- cgit v1.2.3 From cdd812d0683dee14ead02c9eded568685e61b23f Mon Sep 17 00:00:00 2001 From: Daehyeon Ko <4ncienth@gmail.com> Date: Mon, 31 Aug 2026 09:12:21 +0900 Subject: exit: hold a reference to thread_pid across proc_flush_pid Commit 0a36bad01731 ("release_task: kill the no longer needed get/put_pid(thread_pid)") removed the reference around proc_flush_pid(). It assumed that free_pids(post.pids) at the end of release_task() would keep thread_pid alive until then. That assumption is wrong. __change_pid() only records a detached PID in post.pids when pid_has_task() is false for every PIDTYPE. If another task still uses the exiting task's PID as its process group or session ID, __unhash_process() removes the exiting task's PIDTYPE_PID link but leaves the PID out of post.pids. release_task() therefore holds no reference to it after dropping tasklist_lock. The other task can then remove the remaining PIDTYPE links. Its free_pids() call schedules delayed_put_pid(), and the RCU callback can free the PID before the first release_task() reaches proc_flush_pid(). An unprivileged reproducer races wait4(-1) against setsid() to trigger this ordering. Three of three fresh v7.2 KASAN boots reported: BUG: KASAN: slab-use-after-free in proc_invalidate_siblings_dcache+0x3e2/0x3f0 Read of size 8 by task h7_pid_reaper/1921 Call Trace: proc_invalidate_siblings_dcache release_task wait_consider_task __do_wait do_wait kernel_wait4 Freed by task 0: kmem_cache_free put_pid delayed_put_pid rcu_core Last potentially related work creation: __call_rcu_common free_pids ksys_setsid KASAN identified a 144-byte object from the pid cache and located the bad read 80 bytes into the freed object, matching pid->inodes. With an explicit reference, three of three fresh boots completed without a KASAN report. The concurrent RCU callback dropped its reference while proc_flush_pid() was protected, and the balancing put_pid() performed the final free afterward. Take a reference before __unhash_process() clears p->thread_pid and release it after proc_flush_pid() completes. A tested source reproducer is available privately on request. No controlled read or write, information leak, or privilege escalation is claimed. The mainline patch applies directly to v6.19.y and newer; v6.16.y through v6.18.y need a context-adjusted backport. Fixes: 0a36bad01731 ("release_task: kill the no longer needed get/put_pid(thread_pid)") Reported-by: syzbot+0aee5e8066eddbbe7397@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=0aee5e8066eddbbe7397 Reported-by: syzbot+e8b3520b53e78e90034e@syzkaller.appspotmail.com Link: https://syzkaller.appspot.com/bug?extid=e8b3520b53e78e90034e Cc: stable@vger.kernel.org # see patch description, needs adjustments for 6.16.y-6.18.y Signed-off-by: Daehyeon Ko <4ncienth@gmail.com> Link: https://patch.msgid.link/20260831001221.3755948-1-4ncienth@gmail.com Acked-by: Oleg Nesterov Reviewed-by: Bradley Morgan Signed-off-by: Christian Brauner (Amutable) --- kernel/exit.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/exit.c b/kernel/exit.c index 182c06671c78..e12072f1507c 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -261,8 +261,11 @@ repeat: pidfs_exit(p); cgroup_task_release(p); - /* Retrieve @thread_pid before __unhash_process() may set it to NULL. */ - thread_pid = task_pid(p); + /* + * Pin @thread_pid before __unhash_process() clears it. The last + * PIDTYPE detach can otherwise free it before proc_flush_pid(). + */ + thread_pid = get_pid(task_pid(p)); write_lock_irq(&tasklist_lock); ptrace_release_task(p); @@ -291,8 +294,8 @@ repeat: } write_unlock_irq(&tasklist_lock); - /* @thread_pid can't go away until free_pids() below */ proc_flush_pid(thread_pid); + put_pid(thread_pid); exit_cred_namespaces(p); add_device_randomness(&p->se.sum_exec_runtime, sizeof(p->se.sum_exec_runtime)); -- cgit v1.2.3 From 3265ef0b670180b0b946d73ee9825d6d91e98a08 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 5 Sep 2026 06:09:55 -1000 Subject: sched_ext: Rename sch to root_sch in dispatch_one() dispatch_one() uses the root scheduler for everything it does, including the two decisions to keep running @prev, which are wrong when @prev belongs to a sub-scheduler. The function has to deal with @prev's scheduler too. Rename the root's local from sch to root_sch for clarity and to make room for it. No functional change. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 7e414a7c53fc..120540cdda74 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2919,7 +2919,7 @@ static inline void maybe_queue_balance_callback(struct rq *rq) static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev) { - struct scx_sched *sch = scx_root_protected_live(); + struct scx_sched *root_sch = scx_root_protected_live(); enum scx_dsp_verdict verdict; s32 cpu = cpu_of(rq); @@ -2928,7 +2928,7 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev scx_process_sync_ecaps(rq, prev); - if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && + if ((root_sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && unlikely(rq->scx.cpu_released)) { /* * If the previous sched_class for the current CPU was not SCX, @@ -2936,8 +2936,8 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev * core. This callback complements ->cpu_release(), which is * emitted in switch_class(). */ - if (sch->ops.cpu_acquire) - SCX_CALL_OP(sch, cpu_acquire, rq, cpu, NULL); + if (root_sch->ops.cpu_acquire) + SCX_CALL_OP(root_sch, cpu_acquire, rq, cpu, NULL); rq->scx.cpu_released = false; } @@ -2955,7 +2955,7 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev * test. */ if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice && - !scx_bypassing(sch, cpu)) { + !scx_bypassing(root_sch, cpu)) { verdict = SCX_DSP_PREV; goto has_tasks; } @@ -2967,7 +2967,7 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev goto has_tasks; } - verdict = scx_dispatch_sched(sch, rq, prev, false); + verdict = scx_dispatch_sched(root_sch, rq, prev, false); if (verdict != SCX_DSP_NONE) goto has_tasks; @@ -2976,9 +2976,9 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev * %SCX_OPS_ENQ_LAST is in effect. */ if ((prev->scx.flags & SCX_TASK_QUEUED) && - (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu)) && + (!(root_sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(root_sch, cpu)) && scx_task_can_stay_on_cpu(rq, prev)) { - __scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1); + __scx_add_event(root_sch, SCX_EV_DISPATCH_KEEP_LAST, 1); verdict = SCX_DSP_PREV; goto has_tasks; } -- cgit v1.2.3 From 90f19b2816f5d243a8daf36ca83d9d9d04f00e4c Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 5 Sep 2026 06:09:56 -1000 Subject: sched_ext: Use @prev's scheduler for the keep decisions in dispatch_one() dispatch_one() tests ops flags and bypass state against the root scheduler in both places where it decides to keep running @prev: the early keep of a @prev with slice left tests the root's bypass state, and the keep-last at the end tests the root's SCX_OPS_ENQ_LAST and bypass state. Both are properties of the scheduler @prev belongs to, and put_prev_task_scx(), which acts on the outcome, reads them from that scheduler. When @prev belongs to a sub-scheduler the two sides disagree. The keep-last case is visible. The root set SCX_OPS_ENQ_LAST, so a lone @prev of a sub-scheduler is not kept and is enqueued with SCX_ENQ_LAST to a sub-scheduler that never opted in. This trips the WARN_ON_ONCE in put_prev_task_scx() for the missing flag, and the sub-scheduler queues the task like any other and triggers no follow-up scheduling event, which can lead to stalls. Test SCX_OPS_ENQ_LAST and bypass state on @prev's sched in both places and charge SCX_EV_DISPATCH_KEEP_LAST to it. Read the sched at each decision, as the dispatch in between can drop the rq lock. Fixes: 88234b075c3f ("sched_ext: Introduce scx_task_sched[_rcu]()") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 120540cdda74..adf5993fa597 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2955,7 +2955,7 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev * test. */ if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice && - !scx_bypassing(root_sch, cpu)) { + !scx_bypassing(scx_task_sched(prev), cpu)) { verdict = SCX_DSP_PREV; goto has_tasks; } @@ -2972,15 +2972,20 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev goto has_tasks; /* - * Didn't find another task to run. Keep running @prev unless - * %SCX_OPS_ENQ_LAST is in effect. + * Didn't find another task to run. Keep running @prev unless its own + * scheduler set %SCX_OPS_ENQ_LAST and takes the enqueue instead, see + * put_prev_task_scx(). Read the scheduler here as the dispatch above + * may have dropped the rq lock while @prev changed class or scheduler. */ - if ((prev->scx.flags & SCX_TASK_QUEUED) && - (!(root_sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(root_sch, cpu)) && - scx_task_can_stay_on_cpu(rq, prev)) { - __scx_add_event(root_sch, SCX_EV_DISPATCH_KEEP_LAST, 1); - verdict = SCX_DSP_PREV; - goto has_tasks; + if (prev->scx.flags & SCX_TASK_QUEUED) { + struct scx_sched *prev_sch = scx_task_sched(prev); + + if ((!(prev_sch->ops.flags & SCX_OPS_ENQ_LAST) || + scx_bypassing(prev_sch, cpu)) && scx_task_can_stay_on_cpu(rq, prev)) { + __scx_add_event(prev_sch, SCX_EV_DISPATCH_KEEP_LAST, 1); + verdict = SCX_DSP_PREV; + goto has_tasks; + } } rq->scx.flags &= ~SCX_RQ_IN_DISPATCH; return SCX_DSP_NONE; -- cgit v1.2.3 From 113a9796effe3376d2ec5aabcca1fef4fef4cd62 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 12 Aug 2024 16:19:48 +0200 Subject: tick/broadcast: Plug clockevents replacement race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 朱恺乾 reported and decoded the following race condition when a broadcast device is replaced: CPUA CPUB __tick_broadcast_oneshot_control() bc = tick_broadcast_device.evtdev; tick_install_broadcast_device(dev) clockevents_exchange_device(cur, dev) shutdown(cur); detach(cur); cur->handler = noop; tick_broadcast_device.evtdev = dev; tick_broadcast_set_event(bc, next_event); <- FAIL: arms a detached device. If the original broadcast device has a restricted interrupt affinity mask and the last CPU in that mask goes offline then the BUG() in tick_cleanup_dead_cpu() triggers because the clockevent device is not in detached state. The reason for this is that tick_install_broadcast_device() is not serialized vs. tick broadcast operations. The obvious cure is to serialize tick_install_broadcast_device() with tick_broadcast_lock against a concurrent tick broadcast operation. That requires to split clockevents_exchange_device() into two parts, one which does the exchange, shutdown and detach operation and the other which drops the module reference count. This is required because the module reference cannot be dropped while holding tick_broadcast_lock. Let clockevents_exchange_device() do both operations as before, but let the broadcast device code take the two step approach and do the device exchange under tick_broadcast_lock and drop the module reference count after releasing it. Fixes: f8381cba04ba ("[PATCH] tick-management: broadcast functionality") Reported-by: 朱恺乾 Signed-off-by: Thomas Gleixner Signed-off-by: Thomas Gleixner Reviewed-by: Bradley Morgan Tested-by: 刘术高 Cc: stable@vger.kernel.org Link: https://patch.msgid.link/87cymdsu0r.ffs@tglx --- kernel/time/clockevents.c | 33 ++++++++++++++++++++------------- kernel/time/tick-broadcast.c | 36 ++++++++++++++++++++++-------------- kernel/time/tick-internal.h | 2 ++ 3 files changed, 44 insertions(+), 27 deletions(-) (limited to 'kernel') diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c index 0014d163f989..62ad7c000386 100644 --- a/kernel/time/clockevents.c +++ b/kernel/time/clockevents.c @@ -615,33 +615,40 @@ void clockevents_handle_noop(struct clock_event_device *dev) { } -/** - * clockevents_exchange_device - release and request clock devices - * @old: device to release (can be NULL) - * @new: device to request (can be NULL) - * - * Called from various tick functions with clockevents_lock held and - * interrupts disabled. - */ -void clockevents_exchange_device(struct clock_event_device *old, - struct clock_event_device *new) +void __clockevents_exchange_device(struct clock_event_device *old, + struct clock_event_device *new) { /* * Caller releases a clock event device. We queue it into the * released list and do a notify add later. */ if (old) { - module_put(old->owner); clockevents_switch_state(old, CLOCK_EVT_STATE_DETACHED); list_move(&old->list, &clockevents_released); } if (new) { - BUG_ON(!clockevent_state_detached(new)); + WARN_ON(!clockevent_state_detached(new)); clockevents_shutdown(new); } } +/** + * clockevents_exchange_device - release and request clock devices + * @old: device to release (can be NULL) + * @new: device to request (can be NULL) + * + * Called from various tick functions with clockevents_lock held and + * interrupts disabled. + */ +void clockevents_exchange_device(struct clock_event_device *old, + struct clock_event_device *new) +{ + __clockevents_exchange_device(old, new); + if (old) + module_put(old->owner); +} + /** * clockevents_suspend - suspend clock devices */ @@ -699,7 +706,7 @@ void tick_offline_cpu(unsigned int cpu) if (cpumask_test_cpu(cpu, dev->cpumask) && cpumask_weight(dev->cpumask) == 1 && !tick_is_broadcast_device(dev)) { - BUG_ON(!clockevent_state_detached(dev)); + WARN_ON(!clockevent_state_detached(dev)); list_del(&dev->list); } } diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index 115e0bf01276..bda3d2391a60 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -165,23 +165,31 @@ static bool tick_set_oneshot_wakeup_device(struct clock_event_device *newdev, */ void tick_install_broadcast_device(struct clock_event_device *dev, int cpu) { - struct clock_event_device *cur = tick_broadcast_device.evtdev; + struct clock_event_device *cur; - if (tick_set_oneshot_wakeup_device(dev, cpu)) - return; + scoped_guard(raw_spinlock_irqsave, &tick_broadcast_lock) { - if (!tick_check_broadcast_device(cur, dev)) - return; + if (tick_set_oneshot_wakeup_device(dev, cpu)) + return; - if (!try_module_get(dev->owner)) - return; + cur = tick_broadcast_device.evtdev; + if (!tick_check_broadcast_device(cur, dev)) + return; - clockevents_exchange_device(cur, dev); + if (!try_module_get(dev->owner)) + return; + + __clockevents_exchange_device(cur, dev); + if (cur) + cur->event_handler = clockevents_handle_noop; + WRITE_ONCE(tick_broadcast_device.evtdev, dev); + if (!cpumask_empty(tick_broadcast_mask)) + tick_broadcast_start_periodic(dev); + } + + /* Module release must be outside of the lock */ if (cur) - cur->event_handler = clockevents_handle_noop; - tick_broadcast_device.evtdev = dev; - if (!cpumask_empty(tick_broadcast_mask)) - tick_broadcast_start_periodic(dev); + module_put(cur->owner); if (!(dev->features & CLOCK_EVT_FEAT_ONESHOT)) return; @@ -1218,7 +1226,7 @@ int tick_broadcast_oneshot_active(void) */ bool tick_broadcast_oneshot_available(void) { - struct clock_event_device *bc = tick_broadcast_device.evtdev; + struct clock_event_device *bc = READ_ONCE(tick_broadcast_device.evtdev); return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false; } @@ -1226,7 +1234,7 @@ bool tick_broadcast_oneshot_available(void) #else int __tick_broadcast_oneshot_control(enum tick_broadcast_state state) { - struct clock_event_device *bc = tick_broadcast_device.evtdev; + struct clock_event_device *bc = READ_ONCE(tick_broadcast_device.evtdev); if (!bc || (bc->features & CLOCK_EVT_FEAT_HRTIMER)) return -EBUSY; diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h index 182974c4f21b..65680db95053 100644 --- a/kernel/time/tick-internal.h +++ b/kernel/time/tick-internal.h @@ -55,6 +55,8 @@ static inline void clockevent_set_state(struct clock_event_device *dev, } extern void clockevents_shutdown(struct clock_event_device *dev); +extern void __clockevents_exchange_device(struct clock_event_device *old, + struct clock_event_device *new); extern void clockevents_exchange_device(struct clock_event_device *old, struct clock_event_device *new); extern void clockevents_switch_state(struct clock_event_device *dev, -- cgit v1.2.3 From 353a95f1cd8da8a5436a3f070be07d2f484486cd Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 22 Aug 2026 09:23:25 +0200 Subject: syscall_user_dispatch: Use CONFIG_SYSCTL for sysctl guard Commit 8d75c338f0bc ("sysctl: remove CONFIG_PROC_SYSCTL, it just mirrors CONFIG_SYSCTL") removed CONFIG_PROC_SYSCTL, but the sysctl added by commit 5b6e32ba7b59 ("syscall_user_dispatch: Add kernel.syscall_user_dispatch sysctl") is still guarded by it. Now that both commits are merged, kernel.syscall_user_dispatch is no longer registered. syscall_user_dispatch_allowed defaults to true. SUD therefore remains available, but administrators cannot disable new activations. Use CONFIG_SYSCTL for the guard and documentation. Fixes: 5b6e32ba7b59 ("syscall_user_dispatch: Add kernel.syscall_user_dispatch sysctl") Assisted-by: Codex:gpt-5.6-sol Acked-by: Oleg Nesterov Reviewed-by: Joel Granados Signed-off-by: Karl Mehltretter Acked-by: Randy Dunlap Reviewed-by: Bradley Morgan Signed-off-by: Joel Granados --- kernel/entry/syscall_user_dispatch.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/entry/syscall_user_dispatch.c b/kernel/entry/syscall_user_dispatch.c index 2002c7aae435..59c861866941 100644 --- a/kernel/entry/syscall_user_dispatch.c +++ b/kernel/entry/syscall_user_dispatch.c @@ -178,7 +178,7 @@ int syscall_user_dispatch_set_config(struct task_struct *task, unsigned long siz (char __user *)(uintptr_t)cfg.selector); } -#ifdef CONFIG_PROC_SYSCTL +#ifdef CONFIG_SYSCTL static const struct ctl_table syscall_user_dispatch_sysctls[] = { { .procname = "syscall_user_dispatch", @@ -195,4 +195,4 @@ static int __init syscall_user_dispatch_sysctl_init(void) return 0; } late_initcall(syscall_user_dispatch_sysctl_init); -#endif /* CONFIG_PROC_SYSCTL */ +#endif /* CONFIG_SYSCTL */ -- cgit v1.2.3 From 504981db4f69bdd28054fb98c96a3a67f7248dde Mon Sep 17 00:00:00 2001 From: Donggeun Yoo Date: Sat, 5 Sep 2026 16:47:27 +0900 Subject: dma-coherent: report a failed reserved memory assignment rmem_dma_device_init() drops the return value of dma_assign_coherent_memory() and always reports success. That call fails with -EBUSY when the device already has a coherent pool, and the file allows only "*one* such region of memory" per device. of_reserved_mem_device_init_by_idx() reads the zero as success. It logs "assigned reserved memory node" for a region that was not assigned and records the pairing, so of_reserved_mem_device_release() later runs rmem_dma_device_release() for it. That clears dev->dma_mem without looking at which region it was called for, dropping the pool the device did get and leaving it on ordinary memory. dma_declare_coherent_memory() checks the same call and releases the memory on failure, and rmem_swiotlb_device_init() propagates its own errors. Return the error here as well, so a device tree that assigns two pools to one device fails the probe instead of half working. Fixes: 7bfa5ab6fa1b ("drivers: dma-coherent: add initialization from device tree") Signed-off-by: Donggeun Yoo Link: https://lore.kernel.org/r/20260905074727.108029-1-donggeunyoo.kernel@gmail.com Signed-off-by: Marek Szyprowski --- kernel/dma/coherent.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/dma/coherent.c b/kernel/dma/coherent.c index 45bbae947f4b..4d0266893bcc 100644 --- a/kernel/dma/coherent.c +++ b/kernel/dma/coherent.c @@ -352,8 +352,7 @@ static int rmem_dma_device_init(struct reserved_mem *rmem, struct device *dev) min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit)) dev_warn(dev, "reserved memory is beyond device's set DMA address range\n"); - dma_assign_coherent_memory(dev, mem); - return 0; + return dma_assign_coherent_memory(dev, mem); } static void rmem_dma_device_release(struct reserved_mem *rmem, -- cgit v1.2.3 From b7d7914a9ae3097e63d113007e4fb44d33d515b1 Mon Sep 17 00:00:00 2001 From: Donggeun Yoo Date: Sat, 5 Sep 2026 17:42:10 +0900 Subject: swiotlb: use the adjusted address for the highmem page lookup swiotlb_bounce() reads the page frame number from the slot's recorded orig_addr, then advances orig_addr by tlb_offset to reach the address the caller asked about. The highmem branch mixes the two: the offset within the page comes from the adjusted address, the page from the value before it. Once the adjustment crosses a page boundary the pair no longer describes one location, and the whole copy lands one page below the intended one for a positive tlb_offset, one above for a negative one. DMA_FROM_DEVICE writes the device data over the wrong page and leaves the intended one stale, DMA_TO_DEVICE feeds the device from a page the mapping may not cover. Partial syncs through dma_sync_single_range_for_*() are what make tlb_offset non-zero. The branch test is picked the same way, so a slot recorded in lowmem can be adjusted into highmem and the lowmem path then hands a highmem address to phys_to_virt(). Take both from orig_addr once it is final and keep pfn in the branch that uses it. PhysHighMem() asks the question straight from the address, as dma-debug already does. Fixes: 5f89468e2f06 ("swiotlb: manipulate orig_addr when tlb_addr has offset") Cc: stable@vger.kernel.org Signed-off-by: Donggeun Yoo Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20260905084210.148255-1-donggeunyoo.kernel@gmail.com Signed-off-by: Marek Szyprowski --- kernel/dma/swiotlb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index ded7016a46a7..aa2f1c4588b9 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -1019,7 +1019,6 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT; phys_addr_t orig_addr = mem->slots[index].orig_addr; size_t alloc_size = mem->slots[index].alloc_size; - unsigned long pfn = PFN_DOWN(orig_addr); unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start; int tlb_offset; @@ -1052,7 +1051,8 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size size = alloc_size; } - if (PageHighMem(pfn_to_page(pfn))) { + if (PhysHighMem(orig_addr)) { + unsigned long pfn = PFN_DOWN(orig_addr); unsigned int offset = orig_addr & ~PAGE_MASK; struct page *page; unsigned int sz = 0; -- cgit v1.2.3 From 56ea4e86832d8abe8930394473566c194d189f85 Mon Sep 17 00:00:00 2001 From: Norbert Szetei Date: Mon, 7 Sep 2026 16:22:17 +0200 Subject: nstree: check listing permission before taking a namespace reference legitimize_ns() takes a reference on the candidate namespace before may_list_ns() has decided whether the caller may see it. The __free(ns_put) cleanup on the denied path can drop the last reference to a mount namespace while we still hold the rcu read lock, and put_mnt_ns() may sleep there. This is the same problem commit 2ec2aff3c8e2 ("ns: make sure reference are dropped outside of rcu lock") fixed for the put_user() path. Neither ns_requested() nor may_list_ns() needs a reference, both only look at the namespace type and at the caller's own namespaces, so do the checks first and take the reference last. Splat: Voluntary context switch within RCU read-side critical section! WARNING: kernel/rcu/tree_plugin.h:332 at rcu_note_context_switch+0x238/0x2a0, CPU#5: a/3442 CPU: 5 UID: 1000 PID: 3442 Comm: a Not tainted 7.0.0-30-generic #30-Ubuntu PREEMPT(lazy) RIP: 0010:rcu_note_context_switch+0x238/0x2a0 Call Trace: __schedule+0xcf/0x650 schedule+0x27/0x90 schedule_preempt_disabled+0x15/0x30 __mutex_lock.constprop.0+0x550/0xaf0 __mutex_lock_slowpath+0x13/0x20 mutex_lock+0x3b/0x50 exp_funnel_lock+0xb2/0x260 synchronize_rcu_expedited+0xe7/0x220 namespace_unlock+0x26a/0x320 put_mnt_ns+0xd3/0x120 mntns_put+0xe/0x20 do_listns+0x13e/0x560 __do_sys_listns+0x126/0x2d0 __x64_sys_listns+0x20/0x30 x64_sys_call+0x2366/0x2390 do_syscall_64+0x105/0x5a0 entry_SYSCALL_64_after_hwframe+0x76/0x7e Fixes: 76b6f5dfb3fd ("nstree: add listns()") Signed-off-by: Norbert Szetei Link: https://patch.msgid.link/ABA32239-733B-438C-B95A-B13ED69FF0F3@doyensec.com Reviewed-by: Bradley Morgan Signed-off-by: Christian Brauner (Amutable) --- kernel/nstree.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/nstree.c b/kernel/nstree.c index 6d12e5900ac0..831f279d174a 100644 --- a/kernel/nstree.c +++ b/kernel/nstree.c @@ -533,19 +533,13 @@ DEFINE_FREE(ns_put, struct ns_common *, if (!IS_ERR_OR_NULL(_T)) ns_put(_T)) static inline struct ns_common *__must_check legitimize_ns(const struct klistns *kls, struct ns_common *candidate) { - struct ns_common *ns __free(ns_put) = NULL; - if (!ns_requested(kls, candidate)) return NULL; - ns = ns_get_unless_inactive(candidate); - if (!ns) - return NULL; - - if (!may_list_ns(kls, ns)) + if (!may_list_ns(kls, candidate)) return NULL; - return no_free_ptr(ns); + return ns_get_unless_inactive(candidate); } static ssize_t do_listns_userns(struct klistns *kls) -- cgit v1.2.3 From 59e63416f5153e7d58652c616fbdcb7d5e01fff7 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Tue, 8 Sep 2026 12:56:37 +0200 Subject: perf/core: Allow list_del during perf_event_overflow() A PMU might use perf_sched_cb_inc() and perf_sched_cb_dec() interface to get the PMU call back function pmu::sched_task invoked at schedule in and schedule out. This is achieved by walking along the list anchored by sched_cb_list. The following scenario might lead to a list corruption. perf_pmu_sched_task() for_each_list_entry(..., &sched_cb_list) +--> __perf_pmu_sched_task() +--> event->pmu->sched_task()) +--> PMU_push_sample() +--> perf_event_overflow() +--> __perf_event_overflow() +--> pmu->stop() +--> perf_sched_cb_dec() remove entry from sched_cb_list while list node in use. This happens when ioctl(fd, PERF_EVENT_IOC_REFRESH, xxx) has been invoked and perf_event::event_limit hits zero. Prevent the list corruption and convert for_each_list_entry() to for_each_list_entry_safe(). Fixes: bd2756811766 ("perf: Rewrite core context handling") Signed-off-by: Thomas Richter Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260908105637.627004-1-tmricht@linux.ibm.com --- kernel/events/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index 33210aff3ee6..fe33fe15689d 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -3925,13 +3925,13 @@ static void perf_pmu_sched_task(struct task_struct *prev, bool sched_in) { struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); - struct perf_cpu_pmu_context *cpc; + struct perf_cpu_pmu_context *cpc, *cpc2; /* cpuctx->task_ctx will be handled in perf_event_context_sched_in/out */ if (prev == next || cpuctx->task_ctx) return; - list_for_each_entry(cpc, this_cpu_ptr(&sched_cb_list), sched_cb_entry) + list_for_each_entry_safe(cpc, cpc2, this_cpu_ptr(&sched_cb_list), sched_cb_entry) __perf_pmu_sched_task(cpc, sched_in ? next : prev, sched_in); } -- cgit v1.2.3 From 9a8bc9bb4c3fb3218b4f151f98a722fbeb5b5c34 Mon Sep 17 00:00:00 2001 From: Vincent Guittot Date: Mon, 7 Sep 2026 14:38:55 +0200 Subject: sched/eevdf: Fix augmented max_slice Similarly to se->min_slice, init se->max_slice with se->slice before enqueueing the entity so the augmented callback computes it correctly at parent level. Fixes: 6e3c0a4e1ad1 ("sched/fair: Fix lag clamp") Signed-off-by: Vincent Guittot Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: K Prateek Nayak Link: https://patch.msgid.link/20260907123855.1297976-1-vincent.guittot@linaro.org --- kernel/sched/fair.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ade1eceb39b8..5b944f9a8a00 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1073,6 +1073,8 @@ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) sum_w_vruntime_add(cfs_rq, se); se->min_vruntime = se->vruntime; se->min_slice = se->slice; + se->max_slice = se->slice; + rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, __entity_less, &min_vruntime_cb); } -- cgit v1.2.3 From 51b0e68cfa0ac69e3c3ea9d6753af7e15dfaab22 Mon Sep 17 00:00:00 2001 From: Vincent Guittot Date: Wed, 9 Sep 2026 17:05:22 +0200 Subject: sched/eevdf: Fix rb augmented with multi fields The eevdf rb tree maintains 3 augmented fields but only one is currently copied when balancing the tree. Add a more generic define that can be used when there are several augmented fields. In this case, we provide a function that takes care of copying all fields. Fixes: aef6987d8954 ("sched/eevdf: Propagate min_slice up the cgroup hierarchy") Signed-off-by: Vincent Guittot Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: K Prateek Nayak Tested-by: K Prateek Nayak Link: https://patch.msgid.link/20260909150522.858312-1-vincent.guittot@linaro.org --- kernel/sched/fair.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 5b944f9a8a00..944833e8056f 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1032,6 +1032,13 @@ static inline void __max_slice_update(struct sched_entity *se, struct rb_node *n } } +static inline void min_vruntime_copy(struct sched_entity *new, struct sched_entity *old) +{ + new->min_vruntime = old->min_vruntime; + new->min_slice = old->min_slice; + new->max_slice = old->max_slice; +} + /* * se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime) */ @@ -1059,8 +1066,9 @@ static inline bool min_vruntime_update(struct sched_entity *se, bool exit) se->max_slice == old_max_slice; } -RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity, - run_node, min_vruntime, min_vruntime_update); + +RB_DECLARE_CALLBACKS_MULTI(static, min_vruntime_cb, struct sched_entity, + run_node, min_vruntime_copy, min_vruntime_update); /* * Enqueue an entity into the rb-tree: -- cgit v1.2.3 From c23810313bdf6b02f39a1f2a1464c4b18bd39e31 Mon Sep 17 00:00:00 2001 From: Hui Su Date: Fri, 4 Sep 2026 11:47:07 +0800 Subject: sched: Account cgroup CPU time to the execution context Proxy execution separates the scheduling context from the execution context. Commit aa4f74dfd42b ("sched: Fix runtime accounting w/ split exec & sched contexts") made per-task and thread-group runtime accounting follow the task that actually executes, while cgroup CPU usage is charged to the donor. When the donor and execution task belong to different cgroups, this makes a task's execution time count against a different cgroup from the one the task belongs to. Cgroup CPU usage should follow the execution context, matching the per-task, thread-group, and cgroup user/system accounting. Keep scheduling state associated with the donor, but charge cgroup CPU usage to rq->curr. A reproducer with the donor and execution task in separate cgroups showed the execution task accumulating runtime while cgroup CPU usage was charged to the donor's cgroup. With this change, the execution task's cgroup accumulates the CPU usage instead. The same behavior was verified with an RT donor and with legacy cpuacct accounting. Fixes: aa4f74dfd42b ("sched: Fix runtime accounting w/ split exec & sched contexts") Suggested-by: Tejun Heo Signed-off-by: Hui Su Signed-off-by: Peter Zijlstra (Intel) Acked-by: Tejun Heo Acked-by: John Stultz Link: https://patch.msgid.link/20260904034707.268416-1-sh_def@163.com --- kernel/sched/fair.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 944833e8056f..7455a83a6a99 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1414,7 +1414,6 @@ static s64 update_se(struct rq *rq, struct sched_entity *se) se->exec_start = now; if (entity_is_task(se)) { - struct task_struct *donor = task_of(se); struct task_struct *running = rq->curr; /* * If se is a task, we account the time against the running @@ -1427,8 +1426,7 @@ static s64 update_se(struct rq *rq, struct sched_entity *se) account_group_exec_runtime(running, delta_exec); account_mm_sched(rq, running, delta_exec); - /* cgroup time is always accounted against the donor */ - cgroup_account_cputime(donor, delta_exec); + cgroup_account_cputime(running, delta_exec); } else { /* If not task, account the time against donor se */ se->sum_exec_runtime += delta_exec; -- cgit v1.2.3 From f5741d2b34519d387edf6e9798fc7030c20a35f3 Mon Sep 17 00:00:00 2001 From: Hui Su Date: Wed, 2 Sep 2026 23:02:09 +0800 Subject: sched/core: Call wq_worker_tick() for the execution context wq_worker_tick() accounts CPU time and detects CPU-intensive work for the kworker that is actually running. With proxy execution, rq->donor is the scheduling context while rq->curr is the execution context. Calling the hook with rq->donor can skip workqueue accounting when a kworker is executing on behalf of a donor task. It can also account a blocked kworker when the donor is a worker but rq->curr is the task actually executing. The former can delay WORKER_CPU_INTENSIVE handling and pool concurrency management, which can delay pending kernel work and userspace operations depending on it. Use rq->curr for the workqueue tick hook while retaining rq->donor for scheduler accounting. Fixes: af0c8b2bf67b ("sched: Split scheduler and execution contexts") Signed-off-by: Hui Su Signed-off-by: Peter Zijlstra (Intel) Acked-by: Tejun Heo Link: https://patch.msgid.link/20260902150208.1209922-2-sh_def@163.com --- kernel/sched/core.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b998ef6b87af..7885ff76e69f 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5776,8 +5776,8 @@ void sched_tick(void) { int cpu = smp_processor_id(); struct rq *rq = cpu_rq(cpu); - /* accounting goes to the donor task */ - struct task_struct *donor; + /* scheduler accounting goes to the donor task */ + struct task_struct *curr, *donor; struct rq_flags rf; unsigned long hw_pressure; u64 resched_latency; @@ -5788,6 +5788,7 @@ void sched_tick(void) sched_clock_tick(); rq_lock(rq, &rf); + curr = rq->curr; donor = rq->donor; psi_account_irqtime(rq, donor, NULL); @@ -5813,8 +5814,8 @@ void sched_tick(void) perf_event_task_tick(); - if (donor->flags & PF_WQ_WORKER) - wq_worker_tick(donor); + if (curr->flags & PF_WQ_WORKER) + wq_worker_tick(curr); if (!scx_switched_all()) { rq->idle_balance = idle_cpu(cpu); -- cgit v1.2.3 From 88ce88e933e40f4ab4b81a5e9d0a10328583f593 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Thu, 10 Sep 2026 23:35:07 +0200 Subject: bpf: Add KF_PERFMON kfunc flag Tracing related BPF helpers e.g. under bpf_base_func_proto() are gated behind CAP_PERFMON. However, the same is currently not true for kfuncs and they are accessible via plain CAP_BPF. Add a new KF_PERFMON flag which can be used such that check_kfunc_call() ensures env->allow_ptr_leaks is permitted. This follows similar pattern to existing KF_DESTRUCTIVE flag. The rejection returns -EPERM to match the other CAP_PERFMON gates in the verifier, that is, check_ptr_to_btf_access() and check_ptr_to_map_access(), which report the very same policy to user space. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260910213510.49358-1-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 72a3f5998dd2..939e535a3442 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11356,6 +11356,11 @@ static bool is_kfunc_destructive(struct bpf_call_arg_meta *meta) return meta->kfunc_flags & KF_DESTRUCTIVE; } +static bool is_kfunc_perfmon(struct bpf_call_arg_meta *meta) +{ + return meta->kfunc_flags & KF_PERFMON; +} + static bool is_kfunc_rcu(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_RCU; @@ -13834,6 +13839,15 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return -EACCES; } + if (is_kfunc_perfmon(&meta) && !env->allow_ptr_leaks) { + verbose(env, "%s is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", + func_name); + operation = bpf_diag_fmt(env, "kfunc %s", func_name); + bpf_diag_policy(env, insn_idx, operation, "the kfunc requires CAP_PERFMON", + "Load the program with CAP_PERFMON, or avoid the kfunc."); + return -EPERM; + } + sleepable = bpf_is_kfunc_sleepable(&meta); if (sleepable && !in_sleepable(env)) { verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); -- cgit v1.2.3 From 81c975aae375d2053fa2926ee3730f02e713140c Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Thu, 10 Sep 2026 23:35:08 +0200 Subject: bpf: Require CAP_PERFMON for kfuncs reading memory Mark fault-safe probe reading kfuncs as KF_PERFMON, similarly as we do for the old-style BPF helper equivalents. bpf_rdonly_cast() is included in this list as well as it returns PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED for an unchecked object and is using fault-safe BPF_PROBE_MEM. Note that only the void form of bpf_rdonly_cast() produced a register that was readable without CAP_PERFMON. For a struct type id the kfunc returns PTR_TO_BTF_ID | PTR_UNTRUSTED, whose dereference has always been gated in check_ptr_to_btf_access(). The flag is not conditional on the type id, so for the latter it only moves the rejection from the dereference to the call itself, which is the better place to report it anyway. The bpf_stream_vprintk() and bpf_stream_print_stack() kfuncs are marked as well. The former ends up in the same bpf_bprintf_prepare() as the bpf_snprintf() helper, where %pks, %pus and %pI4 read through a program- supplied address and %pB resolves one into a symbol. The latter walks the stack and prints each instruction pointer via %pS. Lastly, bpf_get_kmem_cache() takes a raw scalar address that the verifier does not constrain and dereferences the page/slab metadata for it. Field reads on the returned pointer are still blocked (PTR_TO_BTF_ID | PTR_UNTRUSTED -> check_ptr_to_btf_access() results in -EPERM without the CAP), but the NULL/non-NULL result is observable. Reported-by: STAR Labs SG Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260910213510.49358-2-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- kernel/bpf/helpers.c | 60 ++++++++++++++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index b3cc5c8fc875..712dca5a2c5b 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -4883,7 +4883,7 @@ BTF_ID(func, bpf_cgroup_release_dtor) BTF_KFUNCS_START(common_btf_ids) BTF_ID_FLAGS(func, bpf_cast_to_kern_ctx, KF_FASTCALL) -BTF_ID_FLAGS(func, bpf_rdonly_cast, KF_FASTCALL) +BTF_ID_FLAGS(func, bpf_rdonly_cast, KF_FASTCALL | KF_PERFMON) BTF_ID_FLAGS(func, bpf_rcu_read_lock) BTF_ID_FLAGS(func, bpf_rcu_read_unlock) BTF_ID_FLAGS(func, bpf_dynptr_slice, KF_RET_NULL) @@ -4920,26 +4920,26 @@ BTF_ID_FLAGS(func, bpf_wq_set_callback, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_wq_start) BTF_ID_FLAGS(func, bpf_preempt_disable) BTF_ID_FLAGS(func, bpf_preempt_enable) -BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW) +BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW | KF_PERFMON) BTF_ID_FLAGS(func, bpf_iter_bits_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_bits_destroy, KF_ITER_DESTROY) -BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE) -BTF_ID_FLAGS(func, bpf_copy_from_user_task_str, KF_SLEEPABLE) -BTF_ID_FLAGS(func, bpf_get_kmem_cache) +BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_task_str, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_get_kmem_cache, KF_PERFMON) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_local_irq_save) BTF_ID_FLAGS(func, bpf_local_irq_restore) #ifdef CONFIG_BPF_EVENTS -BTF_ID_FLAGS(func, bpf_probe_read_user_dynptr) -BTF_ID_FLAGS(func, bpf_probe_read_kernel_dynptr) -BTF_ID_FLAGS(func, bpf_probe_read_user_str_dynptr) -BTF_ID_FLAGS(func, bpf_probe_read_kernel_str_dynptr) -BTF_ID_FLAGS(func, bpf_copy_from_user_dynptr, KF_SLEEPABLE) -BTF_ID_FLAGS(func, bpf_copy_from_user_str_dynptr, KF_SLEEPABLE) -BTF_ID_FLAGS(func, bpf_copy_from_user_task_dynptr, KF_SLEEPABLE) -BTF_ID_FLAGS(func, bpf_copy_from_user_task_str_dynptr, KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_probe_read_user_dynptr, KF_PERFMON) +BTF_ID_FLAGS(func, bpf_probe_read_kernel_dynptr, KF_PERFMON) +BTF_ID_FLAGS(func, bpf_probe_read_user_str_dynptr, KF_PERFMON) +BTF_ID_FLAGS(func, bpf_probe_read_kernel_str_dynptr, KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_dynptr, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_str_dynptr, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_task_dynptr, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_task_str_dynptr, KF_SLEEPABLE | KF_PERFMON) #endif #ifdef CONFIG_DMA_SHARED_BUFFER BTF_ID_FLAGS(func, bpf_iter_dmabuf_new, KF_ITER_NEW | KF_SLEEPABLE) @@ -4947,26 +4947,26 @@ BTF_ID_FLAGS(func, bpf_iter_dmabuf_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPAB BTF_ID_FLAGS(func, bpf_iter_dmabuf_destroy, KF_ITER_DESTROY | KF_SLEEPABLE) #endif BTF_ID_FLAGS(func, __bpf_trap) -BTF_ID_FLAGS(func, bpf_strcmp); -BTF_ID_FLAGS(func, bpf_strcasecmp); -BTF_ID_FLAGS(func, bpf_strncasecmp); -BTF_ID_FLAGS(func, bpf_strchr); -BTF_ID_FLAGS(func, bpf_strchrnul); -BTF_ID_FLAGS(func, bpf_strnchr); -BTF_ID_FLAGS(func, bpf_strrchr); -BTF_ID_FLAGS(func, bpf_strlen); -BTF_ID_FLAGS(func, bpf_strnlen); -BTF_ID_FLAGS(func, bpf_strspn); -BTF_ID_FLAGS(func, bpf_strcspn); -BTF_ID_FLAGS(func, bpf_strstr); -BTF_ID_FLAGS(func, bpf_strcasestr); -BTF_ID_FLAGS(func, bpf_strnstr); -BTF_ID_FLAGS(func, bpf_strncasestr); +BTF_ID_FLAGS(func, bpf_strcmp, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strcasecmp, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strncasecmp, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strchr, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strchrnul, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strnchr, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strrchr, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strlen, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strnlen, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strspn, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strcspn, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strstr, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strcasestr, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strnstr, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strncasestr, KF_PERFMON); #if defined(CONFIG_BPF_LSM) && defined(CONFIG_CGROUPS) BTF_ID_FLAGS(func, bpf_cgroup_read_xattr, KF_RCU) #endif -BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) -BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE | KF_PERFMON) BTF_ID_FLAGS(func, bpf_task_work_schedule_signal, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_task_work_schedule_resume, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_dynptr_from_file) -- cgit v1.2.3 From f9191460cd805e84eace0884c75c67ec719fa1d8 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Thu, 10 Sep 2026 23:35:09 +0200 Subject: bpf: Require CAP_PERFMON for untrusted read-only memory reads Marking bpf_rdonly_cast() KF_PERFMON CAP-limits one producer of PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED, but not the type itself. A global subprogram argument tagged __arg_untrusted results in the same register with no kfunc call. Reported-by: STAR Labs SG Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260910213510.49358-3-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 939e535a3442..8058f684a9ea 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6453,6 +6453,15 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b return -EACCES; } + if (rdonly_untrusted && !env->allow_ptr_leaks) { + verbose(env, "%s access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", + reg_type_str(env, reg->type)); + bpf_diag_policy(env, insn_idx, "read from untrusted read-only memory", + "the access requires CAP_PERFMON", + "Load the program with CAP_PERFMON, or avoid dereferencing untrusted pointers."); + return -EPERM; + } + /* * Accesses to untrusted PTR_TO_MEM are done through probe * instructions, hence no need to check bounds in that case. -- cgit v1.2.3 From 2936aed9b02162df7fbe08fcd6bfbb3270ad9f95 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Thu, 10 Sep 2026 20:56:04 +0200 Subject: bpf: Clear scalar delta on narrowing stack spill check_stack_write_fixed_off() breaks the scalar link on a narrowing spill by zeroing the id of the destination stack slot, but leaves the delta in place. save_register_state() has just copied the source register verbatim, so the slot keeps the BPF_ADD_CONST delta of a register it is no longer linked to. This is the same shape as the case addressed in 1b327732c846 ("bpf: Clear delta when clearing reg id for non-{add,sub} ops"). Unlike the latter, no miscomputation seems reachable, so mainly consistency. Signed-off-by: Daniel Borkmann Acked-by: Eduard Zingerman Link: https://lore.kernel.org/r/20260910185604.44949-1-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8058f684a9ea..1b0b1fb62878 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3582,7 +3582,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, save_register_state(env, state, spi, reg, size); /* Break the relation on a narrowing spill. */ if (!reg_value_fits) - state->stack[spi].spilled_ptr.id = 0; + clear_scalar_id(&state->stack[spi].spilled_ptr); } else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) && env->bpf_capable) { struct bpf_reg_state *tmp_reg = &env->fake_reg[0]; -- cgit v1.2.3 From b0b3dc66529676228cb938cbcad66920f735c223 Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Thu, 10 Sep 2026 20:22:55 +0800 Subject: bpf: Fix divide-by-zero in btf_struct_walk() When an access goes past the struct and the last member is a flexible array, btf_struct_walk() folds the offset back into a single element with (off - moff) % t->size, but never checks that the element type has a size. BTF takes an empty struct, so this in program BTF /* event could be empty */ struct event { #ifdef HAVE_TIMESTAMP __u64 ts; #endif }; struct batch { int nr; struct event events[]; }; divides by zero at prog load time. Getting there needs a PTR_TO_BTF_ID that is not MEM_ALLOC, e.g. a plain read of a local kptr stashed in a map from a sleepable program. Oops: divide error: 0000 [#1] SMP KASAN PTI RIP: 0010:btf_struct_walk+0x53f/0x1570 Call Trace: btf_struct_access+0x42a/0xcd0 check_ptr_to_btf_access+0x4dc/0x1160 check_mem_access+0x3a45/0x8740 check_load_mem+0x36a/0xd10 do_check_common+0x3ef0/0xb210 bpf_check+0x6d3b/0x8580 bpf_prog_load+0xf7c/0x2720 __sys_bpf+0xa83/0x3690 __x64_sys_bpf+0xc7/0x150 x64_sys_call+0x1f3f/0x27e0 do_syscall_64+0xe5/0x610 entry_SYSCALL_64_after_hwframe+0x76/0x7e Reject a zero-sized element type. The fixed array path in the same function already bails out on the same thing: btf_struct_walk() ... /* skip empty array */ if (moff == mtrue_end) continue; msize /= total_nelems; Fixes: 9c5f8a1008a1 ("bpf: Support variable length array in tracing programs") Signed-off-by: Jiayuan Chen Acked-by: Eduard Zingerman Link: https://lore.kernel.org/r/20260910122316.186384-1-jiayuan.chen@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 9f33e95d5741..d67a169ba2bb 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7187,7 +7187,7 @@ again: if (btf_type_is_int(t)) return WALK_SCALAR; - if (!btf_type_is_struct(t)) + if (!btf_type_is_struct(t) || !t->size) goto error; off = (off - moff) % t->size; -- cgit v1.2.3 From c5dcb3aadc18d7b82ba64790721b005d18193d35 Mon Sep 17 00:00:00 2001 From: Andrea Parri Date: Thu, 10 Sep 2026 16:34:42 +0200 Subject: hrtimer: Use hard expiry when updating timers on the same base Rearming a queued timer with nonzero slack can leave the timerqueue out of order. remove_and_enqueue_same_base() checks the new soft expiry against its neighbours' hard expiries, then stores the new hard expiry in the node without requeueing it. For example, with A at 10 and B at 20, rearming A at 11 with slack 30 passes the neighbour check but leaves A's hard expiry of 41 before B's 20. The same function also caches the soft expiry in base->expires_next when updating or inserting the first timer, giving next-event selection an earlier deadline than the queue head's hard expiry. Set the timer expiry before handling the queue. Use its stored hard expiry for the in-place ordering check and both updates to base->expires_next. The early update is safe because remove_and_enqueue_same_base() runs with base->cpu_base->lock held. The lock keeps the queue stable while hrtimer_can_update_in_place() checks the new expiry against both neighbours. If the check fails, timerqueue_linked_del() removes the node without comparing expiry values before it is reinserted. Fixes: eddffab8282e3 ("hrtimer: Keep track of first expiring timer per clock base") Fixes: 343f2f4dc5425 ("hrtimer: Try to modify timers in place") Signed-off-by: Andrea Parri Signed-off-by: Thomas Gleixner Assisted-by: LLM Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260910143442.2018-1-parri.andrea@gmail.com --- kernel/time/hrtimer.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 530d61257b9a..cbf1693c86b3 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1263,13 +1263,23 @@ remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *b { bool was_first = false; + /* + * Updating the sort key while @timer is queued can temporarily + * make the tree inconsistent. This is safe under cpu_base->lock: + * no other queue operation can observe that state. + * hrtimer_can_update_in_place() either confirms that the new expiry + * fits between the neighbours or timerqueue_linked_del() removes the + * timer without consulting the expiry. + */ + hrtimer_set_expires_range_ns(timer, expires, delta_ns); + expires = hrtimer_get_expires(timer); + /* Remove it from the timer queue if active */ if (timer->is_queued) { was_first = !timerqueue_linked_prev(&timer->node); /* Try to update in place to avoid the de/enqueue dance */ if (hrtimer_can_update_in_place(timer, base, expires)) { - hrtimer_set_expires_range_ns(timer, expires, delta_ns); trace_hrtimer_start(timer, mode, true); if (was_first) base->expires_next = expires; @@ -1280,9 +1290,6 @@ remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *b timerqueue_linked_del(&base->active, &timer->node); } - /* Set the new expiry time */ - hrtimer_set_expires_range_ns(timer, expires, delta_ns); - debug_activate(timer, mode, timer->is_queued); base->cpu_base->active_bases |= 1 << base->index; -- cgit v1.2.3 From 2deb753127d7b7035e893955c5e91875e767d1f8 Mon Sep 17 00:00:00 2001 From: Henry Martin Date: Fri, 4 Sep 2026 19:52:23 +0800 Subject: tracing/user_events: Don't destroy fields when event removal fails destroy_user_event() destroys the event's fields before attempting to remove the trace event call. If user_event_set_call_visible() fails, e.g. because the event is still enabled and trace_remove_event_call() returns -EBUSY, the event is left registered with an irreversibly destroyed field list. Any subsequent interaction with the event then operates on an empty field list while it is still fully visible in tracefs. Move the field destruction after the call removal, and splice the field list back onto the event when the removal fails so the event remains in a consistent state. Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260904115223.2976446-1-bsdhenrymartin@gmail.com Fixes: 7f5a08c79df35 ("user_events: Add minimal support for trace_event into ftrace") Signed-off-by: Henry Martin Reviewed-by: Beau Belgrave Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_user.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index 93cda2f6f269..f658c3a77aa7 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -1122,10 +1122,9 @@ static void user_event_destroy_validators(struct user_event *user) } } -static void user_event_destroy_fields(struct user_event *user) +static void user_event_destroy_fields(struct list_head *head) { struct ftrace_event_field *field, *next; - struct list_head *head = &user->fields; list_for_each_entry_safe(field, next, head, link) { list_del(&field->link); @@ -1502,17 +1501,32 @@ static int user_event_set_call_visible(struct user_event *user, bool visible) static int destroy_user_event(struct user_event *user) { + LIST_HEAD(fields); int ret = 0; lockdep_assert_held(&event_mutex); - /* Must destroy fields before call removal */ - user_event_destroy_fields(user); + /* + * Detach the fields before removing the call. Removing the event + * frees the field list memory (trace_destroy_fields() is run on + * successful removal and kmem_cache_free()s the fields), but the + * fields here are allocated and owned by user_events. Destroy + * them separately once removal has succeeded. + */ + list_splice_init(&user->fields, &fields); ret = user_event_set_call_visible(user, false); - if (ret) + if (ret) { + /* + * Removal failed and the event stays registered, recover + * the fields so it is left in a consistent state. + */ + list_splice(&fields, &user->fields); return ret; + } + + user_event_destroy_fields(&fields); dyn_event_remove(&user->devent); hash_del(&user->node); @@ -2212,7 +2226,7 @@ static int user_event_parse(struct user_event_group *group, char *name, put_user_lock: mutex_unlock(&event_mutex); put_user: - user_event_destroy_fields(user); + user_event_destroy_fields(&user->fields); user_event_destroy_validators(user); kfree(user->call.print_fmt); -- cgit v1.2.3 From 08cacffeef8f64f1a222c93467ca84f24a46c953 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Jean?= Date: Sat, 22 Aug 2026 19:53:22 +0000 Subject: ftrace: fork: Initialize function graph state before copy_exec_state() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dup_task_struct() copies the parent's task_struct, including ret_stack. ftrace_graph_init_task() clears the copied function graph state, but it currently runs after copy_exec_state(). For non-CLONE_VM forks, copy_exec_state() allocates a new task_exec_state. If that allocation fails, copy_process() reaches bad_fork_free and free_task() calls ftrace_graph_exit_task(). Since the child still carries the parent's ret_stack pointer, the unwind frees the parent's active function graph return stack. The parent subsequently accesses freed memory from function_graph_enter_regs(). KASAN reports: [ 22.190920] ================================================================== [ 22.195899] BUG: KASAN: slab-use-after-free in function_graph_enter_regs+0xa76/0xb90 [ 22.200747] Write of size 8 at addr ff110000054dc0a8 by task repro/1 [ 22.205134] [ 22.210770] CPU: 0 UID: 0 PID: 1 Comm: repro Not tainted 7.2.0-07732-g9328b3b03bdc-dirty #3 PREEMPT(lazy) [ 22.212576] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.16.3-debian-1.16.3-2 04/01/2014 [ 22.213750] Call Trace: [ 22.215271] [ 22.216242] ? ftrace_stub_direct_tramp+0x10/0x10 [ 22.217774] dump_stack_lvl+0x4e/0x70 [ 22.220531] print_report+0x157/0x4b4 [ 22.223202] ? fixup_red_left+0x9/0x30 [ 22.224407] ? complete_report_info+0x83/0x110 [ 22.226679] ? function_graph_enter_regs+0xa76/0xb90 [ 22.228084] kasan_report+0xce/0x100 [ 22.230109] ? function_graph_enter_regs+0xa76/0xb90 [ 22.232860] ? stack_trace_save+0x4/0xd0 [ 22.234156] function_graph_enter_regs+0xa76/0xb90 [ 22.236090] ? kasan_save_stack+0x30/0x50 [ 22.237752] ? __pfx_function_graph_enter_regs+0x10/0x10 [ 22.238694] ? ring_buffer_lock_reserve+0x345/0xf80 [ 22.239628] ? stack_trace_save+0x4/0xd0 [ 22.242121] ? stack_trace_save+0x4/0xd0 [ 22.243588] ftrace_graph_func+0xda/0x160 [ 22.245362] ? ftrace_stub_direct_tramp+0x10/0x10 [ 22.246520] 0xffffffffa0000095 [ 22.250528] ? stack_trace_save+0x9/0xd0 [ 22.251757] ? ring_buffer_unlock_commit+0x11d/0x5c0 [ 22.253152] stack_trace_save+0x9/0xd0 [ 22.254264] kasan_save_stack+0x30/0x50 [ 22.273631] kasan_save_track+0x14/0x30 [ 22.276763] kasan_save_free_info+0x3b/0x70 [ 22.278296] __kasan_slab_free+0x43/0x70 [ 22.280157] kmem_cache_free+0xbf/0x3b0 [ 22.282963] ? ftrace_stub_direct_tramp+0x10/0x10 [ 22.284001] free_task+0xa2/0x160 [ 22.285699] ? ftrace_stub_direct_tramp+0x10/0x10 [ 22.286752] copy_process+0x2aae/0x7bc0 Initialize the child function graph state immediately after dup_task_struct(), before the first fallible operation. Cc: stable@vger.kernel.org Fixes: 6b1c66c9cca9 ("exec_state: relocate dumpable information") Reviewed-by: Bradley Morgan Link: https://patch.msgid.link/20260822195321.962383-2-Jeremy.Jean@oss.cyber.gouv.fr Assisted-by: Codex:gpt-5 Signed-off-by: Jérémy Jean Signed-off-by: Steven Rostedt --- kernel/fork.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index 416758c8a3d4..a5934a317634 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2133,6 +2133,11 @@ __latent_entropy struct task_struct *copy_process( p = dup_task_struct(current, node); if (!p) goto fork_out; + /* + * Must run before the first fallible op, so error paths never + * free the parent's ret_stack. + */ + ftrace_graph_init_task(p); retval = copy_exec_state(clone_flags, p); if (retval) goto bad_fork_free; @@ -2159,8 +2164,6 @@ __latent_entropy struct task_struct *copy_process( */ p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL; - ftrace_graph_init_task(p); - rt_mutex_init_task(p); raw_spin_lock_init(&p->blocked_lock); -- cgit v1.2.3 From b22845487096247f5370c412376a472304627847 Mon Sep 17 00:00:00 2001 From: Donggeun Yoo Date: Sun, 6 Sep 2026 06:19:22 +0900 Subject: fgraph: Remove unused FGRAPH_MAX_INDEX FGRAPH_MAX_INDEX has no user, and it expands to FGRAPH_INDEX_SIZE and FGRAPH_RET_INDEX, neither of which is defined anywhere in the tree. It was added in that form by commit 91c46b0aa917 ("function_graph: Implement fgraph_reserve_data() and fgraph_retrieve_data()"), which introduced the current data word layout under new names, so anything referencing it would have failed to build ever since. Remove it. Link: https://patch.msgid.link/20260905211922.1196366-1-donggeunyoo.kernel@gmail.com Signed-off-by: Donggeun Yoo Acked-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/fgraph.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 40d373d65f9b..ed455b53513b 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -143,9 +143,6 @@ enum { #define FGRAPH_DATA_INDEX_MASK GENMASK(FGRAPH_DATA_INDEX_BITS - 1, 0) #define FGRAPH_DATA_INDEX_SHIFT (FGRAPH_DATA_SHIFT + FGRAPH_DATA_BITS) -#define FGRAPH_MAX_INDEX \ - ((FGRAPH_INDEX_SIZE << FGRAPH_DATA_BITS) + FGRAPH_RET_INDEX) - #define FGRAPH_ARRAY_SIZE FGRAPH_INDEX_BITS /* -- cgit v1.2.3 From 0701995aaf8fc2281154db829ca85e231951e51d Mon Sep 17 00:00:00 2001 From: Donggeun Yoo Date: Sun, 6 Sep 2026 12:44:06 +0900 Subject: function_graph: Use the saved entry's size when reprinting it When a graph entry does not fit in the trace_seq, print_graph_entry() saves it in the iterator's fgraph_data and reprints it on the next read. The entry has already been consumed from the ring buffer by then, so the copy is all that is left of it. The copy is sized with iter->ent_size, which no longer describes the saved entry but whatever entry the iterator has moved on to. The argument count is derived from the same field, so a 72 byte entry saved and then reprinted ahead of a 48 byte return entry loses its arguments. Record the size next to the failure flag, so that the two are always set together, and restore it before reprinting. Cc: stable@vger.kernel.org Fixes: ff5c9c576e75 ("ftrace: Add support for function argument to graph tracer") Link: https://patch.msgid.link/20260906034406.1335316-1-donggeunyoo.kernel@gmail.com Signed-off-by: Donggeun Yoo Signed-off-by: Steven Rostedt --- kernel/trace/trace_functions_graph.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'kernel') diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index ff7cb1a76b95..c5befd0c5b9a 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -52,6 +52,7 @@ struct fgraph_data { }; struct ftrace_graph_ret_entry ret; int failed; + int ent_size; int cpu; }; @@ -1274,6 +1275,7 @@ print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s, if (s->full) { data->failed = 1; data->cpu = cpu; + data->ent_size = iter->ent_size; } else data->failed = 0; } @@ -1457,6 +1459,7 @@ print_graph_function_flags(struct trace_iterator *iter, u32 flags) if (data && data->failed) { field = &data->ent.ent; iter->cpu = data->cpu; + iter->ent_size = data->ent_size; ret = print_graph_entry(field, s, iter, flags); if (ret == TRACE_TYPE_HANDLED && iter->cpu != cpu) { per_cpu_ptr(data->cpu_data, iter->cpu)->ignore = 1; -- cgit v1.2.3 From 4bddcb346a6cf4615ca77f69a589623b877ca267 Mon Sep 17 00:00:00 2001 From: Donggeun Yoo Date: Sun, 6 Sep 2026 21:40:25 +0900 Subject: tracing: Free histogram var refs regardless of how often they are referenced Using the same variable three or more times in one hist trigger leaks the variable reference and its strings when the trigger is removed. commit 656fe2ba85e8 ("tracing: Use hist trigger's var_ref array to destroy var_refs") made a trigger's var_refs[] array the only owner of a var ref: destroy_hist_field() returns early for HIST_FIELD_FL_VAR_REF, so the field expressions never destroy one. One entry, freed once, no count needed. commit 8bcebc77e85f ("tracing: Fix histogram code when expression has same var as value") then made repeated references share one object and added a count of them. Only the increment side exists, since those expressions still return early and never drop a reference, so __destroy_hist_field() sees how many references were created rather than how many are left. It frees when the decremented count is 0 or 1, so two references work and three or more leak. Sharing kept one array entry per object, and create_var_ref() searches and appends within a single trigger, so nothing outside it holds the object. Removing a trigger whose variables are still referenced is already refused by check_var_refs() with -EBUSY. Drop the count and free unconditionally. Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260906124025.3550596-1-donggeunyoo.kernel@gmail.com Fixes: 8bcebc77e85f ("tracing: Fix histogram code when expression has same var as value") Signed-off-by: Donggeun Yoo Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_hist.c | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 963e0d6b61fd..f90680b33a37 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -169,7 +169,6 @@ struct hist_field { struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; struct hist_trigger_data *hist_data; enum hist_field_fn fn_num; - unsigned int ref; unsigned int size; unsigned int offset; unsigned int is_signed; @@ -1913,16 +1912,8 @@ out: return field_op; } -static void get_hist_field(struct hist_field *hist_field) -{ - hist_field->ref++; -} - static void __destroy_hist_field(struct hist_field *hist_field) { - if (--hist_field->ref > 1) - return; - kfree(hist_field->var.name); kfree(hist_field->name); @@ -1969,8 +1960,6 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, if (!hist_field) return NULL; - hist_field->ref = 1; - hist_field->hist_data = hist_data; if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS) @@ -2223,10 +2212,8 @@ static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data, for (i = 0; i < hist_data->n_var_refs; i++) { ref_field = hist_data->var_refs[i]; if (ref_field->var.idx == var_field->var.idx && - ref_field->var.hist_data == var_field->hist_data) { - get_hist_field(ref_field); + ref_field->var.hist_data == var_field->hist_data) return ref_field; - } } /* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */ if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX) @@ -3276,7 +3263,6 @@ static struct hist_field *create_var(struct hist_trigger_data *hist_data, goto out; } - var->ref = 1; var->flags = HIST_FIELD_FL_VAR; var->var.idx = idx; var->var.hist_data = var->hist_data = hist_data; -- cgit v1.2.3 From 516001d53e6b2ea95a251ee2ef54a1a689a3fd58 Mon Sep 17 00:00:00 2001 From: Donggeun Yoo Date: Sun, 6 Sep 2026 22:33:52 +0900 Subject: tracing: Free histogram the var ref when its initialization fails create_var_ref() allocates a VAR_REF hist_field and then calls init_var_ref() to fill it in. When that fails the field is leaked. commit 656fe2ba85e8 ("tracing: Use hist trigger's var_ref array to destroy var_refs") made destroy_hist_field() return early for HIST_FIELD_FL_VAR_REF, since var refs are freed by walking the trigger's var_refs[] array instead. create_var_ref() adds the field to that array only after init_var_ref() has succeeded, so on this path the field is in neither place and nothing frees it. The call was correct when it was written, before var refs were taken out of destroy_hist_field(). init_var_ref() cannot free it either. The caller owns the field, so init_var_ref() undoes only its own string allocations and leaves the field alone. Freeing it there would leave create_var_ref() passing freed memory to destroy_hist_field(), which reads its flags. Call __destroy_hist_field(), which frees the field without consulting the flag. Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260906133352.3815019-1-donggeunyoo.kernel@gmail.com Fixes: 656fe2ba85e8 ("tracing: Use hist trigger's var_ref array to destroy var_refs") Signed-off-by: Donggeun Yoo Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_hist.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index f90680b33a37..bbdd56208eff 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -2221,7 +2221,7 @@ static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data, ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL); if (ref_field) { if (init_var_ref(ref_field, var_field, system, event_name)) { - destroy_hist_field(ref_field, 0); + __destroy_hist_field(ref_field); return NULL; } -- cgit v1.2.3 From 230234d12ce42ab04132a32c3a848f07a5d27a71 Mon Sep 17 00:00:00 2001 From: Donggeun Yoo Date: Mon, 7 Sep 2026 12:49:48 +0900 Subject: tracing: Free histogram the field rejected for a bad modifier Writing a hist trigger whose value or variable carries a modifier that is not allowed there leaks the fields that were built for it. __create_val_field() takes the field from parse_expr() and stores it in hist_data->fields[] only after the modifier checks have run: hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs); ... if (hist_field->flags & HIST_FIELD_FL_VAR) { if (hist_field->flags & (...)) goto err; } else { if (hist_field->flags & (...)) goto err; } hist_data->fields[val_idx] = hist_field; Both checks jump past that store, and the err label returns without freeing anything. The error unwinds to create_hist_data(), which calls destroy_hist_data() -> destroy_hist_fields(), and that reaches a field only by walking fields[]. A field that never got there is unreachable. commit e0213434fe3e ("tracing: Do not let histogram values have some modifiers") set ret to -EINVAL and fell through to the store, which left the field owned by fields[] and freed along with the rest of hist_data. Splitting the check into a value case and a variable case replaced that fall-through with a goto that skips it. With CONFIG_DEBUG_KMEMLEAK, 200 writes of # echo 'hist:keys=prev_pid:vals=next_pid.log2' > \ events/sched/sched_switch/trigger each correctly rejected with -EINVAL, leave 332 unreferenced objects (63744 bytes) reported at create_hist_field(); 200 install and remove cycles of a valid trigger leave none. A '.log2' field is two allocations, since create_hist_field() puts the plain field in operands[0] of the log2 field, and both are reported. Use destroy_hist_field() rather than __destroy_hist_field() so that operands[0] is freed as well. It returns early for HIST_FIELD_FL_VAR_REF, which is what an operand owned by hist_data->var_refs[] needs; the rejected field itself is never a var ref, because a var ref never carries a modifier flag. Cc: stable@vger.kernel.org Fixes: e30fbc618e97 ("tracing/histograms: Allow variables to have some modifiers") Link: https://patch.msgid.link/20260907034948.240387-1-donggeunyoo.kernel@gmail.com Signed-off-by: Donggeun Yoo Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_hist.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index bbdd56208eff..8cad99a8d01e 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -4317,6 +4317,7 @@ static int __create_val_field(struct hist_trigger_data *hist_data, return ret; err: hist_err(file->tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str)); + destroy_hist_field(hist_field, 0); return -EINVAL; } -- cgit v1.2.3 From 06f5634ec5584954177f9a22e36b3bfb398a971b Mon Sep 17 00:00:00 2001 From: Donggeun Yoo Date: Mon, 7 Sep 2026 15:03:23 +0900 Subject: tracing: Keep the entry count when the histogram stats allocation fails print_entries() uses n_entries both as the number of sort entries and as its own return value, so the -ENOMEM it stores when the stats allocation fails overwrites the count that the cleanup still needs: n_entries = tracing_map_sort_entries(map, ...); if (n_entries < 0) return n_entries; ... if (!stats) { n_entries = -ENOMEM; goto out; } ... out: tracing_map_destroy_sort_entries(sort_entries, n_entries); tracing_map_destroy_sort_entries() takes an unsigned int and loops up to it, so -ENOMEM arrives as 4294967284. It walks an array of at most map->max_elts pointers and calls destroy_sort_entry(), which dereferences and frees, on whatever lies past the end. Reading the hist file of a trigger with a .percent value, with that allocation forced to fail: BUG: KASAN: vmalloc-out-of-bounds in tracing_map_destroy_sort_entries+0xa0/0xb0 Read of size 8 at addr ffffc90000045000 by task init/1 tracing_map_destroy_sort_entries+0xa0/0xb0 hist_show+0x6f7/0x1df0 seq_read_iter+0x2b8/0x1190 vfs_read+0x176/0xa40 The buggy address belongs to a 4-page vmalloc region starting at ffffc90000041000 allocated at tracing_map_sort_entries+0x5c/0xd50 A few pages further the fault is fatal. The registers at the oops confirm the bound: the loop's end pointer less the array start, over the pointer size, is 4294967284. Return the error in a separate variable and leave n_entries holding the count, the way tracing_map_sort_entries() does on its own error path. The stats block is only entered for a value carrying .percent or .graph, which __create_val_field() has rejected since v6.3, so this cannot be reached in mainline as it stands. It becomes reachable again with "tracing: hist: let values keep the percent and graph modifiers", so it should be applied first. Cc: stable@vger.kernel.org Fixes: abaa5258ce5e ("tracing: Add .percent suffix option to histogram values") Link: https://patch.msgid.link/20260907060323.480728-1-donggeunyoo.kernel@gmail.com Reported-by: sashiko-bot@kernel.org Closes: https://lore.kernel.org/all/20260907053113.1CED91F00A3A@smtp.kernel.org/ Signed-off-by: Donggeun Yoo Acked-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_hist.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 8cad99a8d01e..8d80562fb502 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -5677,7 +5677,7 @@ static int print_entries(struct seq_file *m, { struct tracing_map_sort_entry **sort_entries = NULL; struct tracing_map *map = hist_data->map; - int i, j, n_entries; + int i, j, n_entries, ret; struct hist_val_stat *stats = NULL; u64 val; @@ -5687,6 +5687,8 @@ static int print_entries(struct seq_file *m, if (n_entries < 0) return n_entries; + ret = n_entries; + /* Calculate the max and the total for each field if needed. */ for (j = 0; j < hist_data->n_vals; j++) { if (!(hist_data->fields[j]->flags & @@ -5695,7 +5697,7 @@ static int print_entries(struct seq_file *m, if (!stats) { stats = kzalloc_objs(*stats, hist_data->n_vals); if (!stats) { - n_entries = -ENOMEM; + ret = -ENOMEM; goto out; } } @@ -5716,7 +5718,7 @@ static int print_entries(struct seq_file *m, out: tracing_map_destroy_sort_entries(sort_entries, n_entries); - return n_entries; + return ret; } static void hist_trigger_show(struct seq_file *m, -- cgit v1.2.3 From 3d617bfd79330ae3acf94862c18bb3ccf5f5a0f9 Mon Sep 17 00:00:00 2001 From: Donggeun Yoo Date: Mon, 7 Sep 2026 14:21:13 +0900 Subject: tracing: Let histogram values keep the percent and graph modifiers The .percent and .graph modifiers exist only for histogram values, but a value carrying either of them has been rejected since v6.3. The example in Documentation/trace/histogram.rst, # echo 'hist:keys=prev_comm:vals=hitcount.percent:nohitcount' > \ events/sched/sched_switch/trigger returns -EINVAL. parse_field() sets the two flags only when the field is neither a key nor a variable, that is, only on a value: } else if (strncmp(modifier, "percent", 7) == 0) { if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY)) goto error; *flags |= HIST_FIELD_FL_PERCENT; __create_val_field() then rejects a value for carrying them, so no field can reach hist_trigger_print_val(), where both are implemented. commit e0213434fe3e ("tracing: Do not let histogram values have some modifiers") added the check after a value with .buckets oopsed in hist_field_name(). That happens because .buckets and .log2 make create_hist_field() build a nested field in operands[0] which hist_field_name() then walks into. The percent and graph flags do not create an operand and are not read by hist_field_name(); they are only used when printing a value. Stop rejecting the two flags on a value. The check for variables is left alone, where they are unreachable anyway because parse_field() rejects a variable carrying them first. With the two flags removed, the trigger above installs and prints as documented: { prev_comm: rcu_preempt } hitcount (%): 0.00 { prev_comm: init } hitcount (%): 99.98 Totals: Hits: 237896 Cc: stable@vger.kernel.org Fixes: e0213434fe3e ("tracing: Do not let histogram values have some modifiers") Link: https://patch.msgid.link/20260907052113.430818-1-donggeunyoo.kernel@gmail.com Signed-off-by: Donggeun Yoo Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_hist.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 8d80562fb502..5e00da2d5b1a 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -4299,8 +4299,7 @@ static int __create_val_field(struct hist_trigger_data *hist_data, goto err; } else { /* Value */ - if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT | - HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 | + if (hist_field->flags & (HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_SYM | HIST_FIELD_FL_SYM_OFFSET | HIST_FIELD_FL_SYSCALL | HIST_FIELD_FL_STACKTRACE)) goto err; -- cgit v1.2.3 From 89b000ba0796593aa61f6eec24d369337594588b Mon Sep 17 00:00:00 2001 From: Hemanth Selam Date: Mon, 7 Sep 2026 11:56:08 +0530 Subject: tracing: Fix typo "availabe" in comment Correct "availabe" to "available", reported by scripts/checkpatch.pl using the misspelling list in scripts/spelling.txt. Only touches comments, no code changes. Link: https://patch.msgid.link/20260907062608.13924-1-hemanth.selam@gmail.com Assisted-by: Cursor:claude-opus-5 Signed-off-by: Hemanth Selam Signed-off-by: Steven Rostedt --- kernel/trace/rethook.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/rethook.c b/kernel/trace/rethook.c index 5a8bdf88999a..87a27f3aa4a6 100644 --- a/kernel/trace/rethook.c +++ b/kernel/trace/rethook.c @@ -171,7 +171,7 @@ struct rethook_node *rethook_try_get(struct rethook *rh) * This expects the caller will set up a rethook on a function entry. * When the function returns, the rethook will eventually be reclaimed * or released in the rethook_recycle() with call_rcu(). - * This means the caller must be run in the RCU-availabe context. + * This means the caller must be run in the RCU-available context. */ if (unlikely(!rcu_is_watching())) return NULL; -- cgit v1.2.3 From 6ede78d0563a2a3ae3e46f9c07cedb5d79645429 Mon Sep 17 00:00:00 2001 From: Donggeun Yoo Date: Mon, 7 Sep 2026 18:14:15 +0900 Subject: tracing: Set the trace clock before registering the histogram trigger hist_register_trigger() puts the trigger on the global named_triggers list in cmd_ops->init(), and only then sets the trace clock: if (data->cmd_ops->init) { ret = data->cmd_ops->init(data); if (ret < 0) goto out; } if (hist_data->enable_timestamps) { ret = tracing_set_clock(file->tr, hist_data->attrs->clock); if (ret) { hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock)); goto out; } The clock string is not checked anywhere before that call, so a named trigger using common_timestamp with an unknown clock fails after it has already become findable. event_hist_trigger_parse() then frees it without taking it off the list, and the next lookup by name reads the freed object: ~# cd /sys/kernel/tracing/events/sched/sched_switch ~# echo 'hist:name=foo:keys=common_pid:ts=common_timestamp:clock=bogus' > trigger bash: echo: write error: Invalid argument ~# echo 'hist:name=foo:keys=common_pid' > trigger BUG: KASAN: slab-use-after-free in find_named_trigger+0xac/0xc0 Read of size 8 at addr ffff88800915d760 by task init/1 find_named_trigger+0xac/0xc0 hist_register_trigger+0xc1/0x900 event_hist_trigger_parse+0x3146/0x6af0 event_trigger_write+0xce/0x160 Freed by task 63: kfree+0x154/0x420 trigger_kthread_fn+0xfd/0x160 Set the clock before the trigger is registered, so that nothing which can fail runs after it is published, the way commit 6f86bdeab633 ("tracing: Fix bad hist from corrupting named_triggers list") moved the registration below the rest of the setup. tracing_set_filter_buffering() is reference counted, so the init failure path has to drop the reference that the clock block now takes first. Cc: stable@vger.kernel.org Fixes: a4072fe85ba3 ("tracing: Add a clock attribute for hist triggers") Link: https://patch.msgid.link/20260907091415.554535-1-donggeunyoo.kernel@gmail.com Signed-off-by: Donggeun Yoo Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_hist.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 5e00da2d5b1a..1889e310b73c 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -6631,12 +6631,6 @@ static int hist_register_trigger(char *glob, data->cmd_ops = cmd_ops; } - if (data->cmd_ops->init) { - ret = data->cmd_ops->init(data); - if (ret < 0) - goto out; - } - if (hist_data->enable_timestamps) { char *clock = hist_data->attrs->clock; @@ -6649,6 +6643,15 @@ static int hist_register_trigger(char *glob, tracing_set_filter_buffering(file->tr, true); } + if (data->cmd_ops->init) { + ret = data->cmd_ops->init(data); + if (ret < 0) { + if (hist_data->enable_timestamps) + tracing_set_filter_buffering(file->tr, false); + goto out; + } + } + if (named_data) { remove_hist_vars(hist_data); destroy_hist_data(hist_data); -- cgit v1.2.3 From 0fe23b8eaba0d3372c66b7b31204408da0715edc Mon Sep 17 00:00:00 2001 From: Donggeun Yoo Date: Mon, 7 Sep 2026 21:44:19 +0900 Subject: tracing: Take the reference before publishing the named histogram trigger event_hist_trigger_named_init() puts the trigger on the global named_triggers list and only then takes the reference on the trigger it shares its histogram with: data->ref++; save_named_trigger(data->named_data->name, data); ret = event_hist_trigger_init(data->named_data); if (ret < 0) { kfree(data->cmd_ops); data->cmd_ops = &trigger_hist_cmd; } return ret; event_hist_trigger_init() fails when alloc_hist_pad() cannot allocate, and nothing takes the trigger back off the list on the way out. event_hist_trigger_parse() frees it, and the next lookup by name reads the freed object: BUG: KASAN: slab-use-after-free in find_named_trigger+0xac/0xc0 Read of size 8 at addr ffff888009346860 by task init/1 find_named_trigger+0xac/0xc0 hist_register_trigger+0xc1/0xa00 event_hist_trigger_parse+0x3146/0x6af0 event_trigger_write+0xce/0x160 Freed by task 67: kfree+0x154/0x420 trigger_kthread_fn+0xfd/0x160 Do the reference first and publish once it has succeeded, so that nothing which can fail runs after the trigger becomes findable. Cc: stable@vger.kernel.org Fixes: 7ab0fc61ce73 ("tracing: Move histogram trigger variables from stack to per CPU structure") Reported-by: Sashiko AI Closes: https://lore.kernel.org/linux-trace-kernel/20260907092944.3950E1F00A3D@smtp.kernel.org/ Link: https://patch.msgid.link/20260907124420.607097-2-donggeunyoo.kernel@gmail.com Signed-off-by: Donggeun Yoo Acked-by: Tom Zanussi Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_hist.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 1889e310b73c..54c95f9bd0a3 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -6371,17 +6371,18 @@ static int event_hist_trigger_named_init(struct event_trigger_data *data) { int ret; - data->ref++; - - save_named_trigger(data->named_data->name, data); - ret = event_hist_trigger_init(data->named_data); if (ret < 0) { kfree(data->cmd_ops); data->cmd_ops = &trigger_hist_cmd; + return ret; } - return ret; + data->ref++; + + save_named_trigger(data->named_data->name, data); + + return 0; } static void event_hist_trigger_named_free(struct event_trigger_data *data) -- cgit v1.2.3 From 92383cef66791a0c63a2f27755cadbdb2fbf270b Mon Sep 17 00:00:00 2001 From: Donggeun Yoo Date: Mon, 7 Sep 2026 21:44:20 +0900 Subject: tracing: Undo the registration when enabling the histogram trigger fails Commit 6f86bdeab633 ("tracing: Fix bad hist from corrupting named_triggers list") described how a trigger that is registered but not on file->triggers ends up freed while still on the global named_triggers list, and moved the registration down so that hist_trigger_enable() follows it immediately. One path still gets there. hist_trigger_enable() adds the trigger and takes it straight back out when the event cannot be enabled: list_add_tail_rcu(&data->list, &file->triggers); update_cond_flag(file); if (trace_event_trigger_enable_disable(file, 1) < 0) { list_del_rcu(&data->list); update_cond_flag(file); ret--; } so the list walk in hist_unregister_trigger() matches nothing, test stays NULL, and the ->free() that would call del_named_trigger() is skipped. out_unreg falls through to out_free, which frees the trigger anyway: BUG: KASAN: slab-use-after-free in find_named_trigger+0xac/0xc0 Read of size 8 at addr ffff8880091d3160 by task init/1 find_named_trigger+0xac/0xc0 hist_register_trigger+0xc1/0xa00 event_hist_trigger_parse+0x3146/0x6af0 event_trigger_write+0xce/0x160 Freed by task 69: kfree+0x154/0x420 trigger_kthread_fn+0xfd/0x160 Leave the trigger where hist_unregister_trigger() can find it and let that undo the registration, which is the only code that knows all of what cmd_ops->init() took: the named list entry, the hist_pad reference, the reference on the trigger a named histogram is shared with, and the copied cmd_ops. It also pairs the failed trace_event_trigger_enable_disable(), whose sm_ref and buffered event reference are otherwise left behind. Since ->free() releases trigger_data and, for a trigger that does not share its histogram, hist_data with it, out_unreg can no longer fall through to out_free. For a trigger that does share, hist_register_trigger() has already destroyed the caller's hist_data, so the fall-through was reading freed memory there as well. Move the enable_timestamps check in hist_unregister_trigger() above the ->free() call for the same reason: hist_data does not outlive it once the trigger being removed is the one that owns it. Cc: stable@vger.kernel.org Fixes: 067fe038e70f ("tracing: Add variable reference handling to hist triggers") Reported-by: Sashiko AI Closes: https://lore.kernel.org/linux-trace-kernel/20260907092944.3950E1F00A3D@smtp.kernel.org/ Link: https://patch.msgid.link/20260907124420.607097-3-donggeunyoo.kernel@gmail.com Signed-off-by: Donggeun Yoo Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_hist.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 54c95f9bd0a3..53100287466f 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -6670,11 +6670,12 @@ static int hist_trigger_enable(struct event_trigger_data *data, update_cond_flag(file); - if (trace_event_trigger_enable_disable(file, 1) < 0) { - list_del_rcu(&data->list); - update_cond_flag(file); + /* + * On failure the caller undoes the registration, and + * hist_unregister_trigger() can only find the trigger here. + */ + if (trace_event_trigger_enable_disable(file, 1) < 0) ret--; - } return ret; } @@ -6752,13 +6753,13 @@ static void hist_unregister_trigger(char *glob, } } - if (test && test->cmd_ops->free) - test->cmd_ops->free(test); - if (hist_data->enable_timestamps) { if (!hist_data->remove || test) tracing_set_filter_buffering(file->tr, false); } + + if (test && test->cmd_ops->free) + test->cmd_ops->free(test); } static bool hist_file_check_refs(struct trace_event_file *file) @@ -6963,6 +6964,8 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops, return ret; out_unreg: event_trigger_unregister(cmd_ops, file, glob+1, trigger_data); + /* The unregister frees trigger_data, skip out_free */ + goto out; out_free: remove_hist_vars(hist_data); -- cgit v1.2.3 From a5e70ba87ca8ebc79b4e63de302d03b0625fe153 Mon Sep 17 00:00:00 2001 From: Donggeun Yoo Date: Tue, 8 Sep 2026 00:50:44 +0900 Subject: tracing: Fix memory corruption from the histogram stacktrace modifier parse_field() sets HIST_FIELD_FL_STACKTRACE from the ".stacktrace" modifier before it looks the field name up, and nothing afterwards checks that the name resolved to a field which holds a stacktrace. create_hist_field() picks HIST_FIELD_FN_STACK on the strength of the field pointer alone, which reads a __data_loc word from the record and follows its low 16 bits as an offset into the same record. event_hist_trigger() takes the first word there as an entry count and copies that many longs into a 31 entry array: n_entries = *stack; memcpy(entries, ++stack, n_entries * sizeof(unsigned long)); Neither end of that copy is bounded, and the count is whatever the event holds at the offset, so any field will do: # cd /sys/kernel/tracing/events/sched/sched_process_fork # echo 'hist:keys=parent_pid.stacktrace' > trigger # (true) BUG: kernel NULL pointer dereference, address: 0000000000000008 RIP: 0010:rb_insert_color+0x18/0x130 timerqueue_linked_add+0x7e/0xd0 enqueue_hrtimer+0x39/0xb0 __hrtimer_run_queues+0x10f/0x1f0 RIP: 0010:memcpy+0xc/0x30 event_hist_trigger+0x165/0x690 The timer interrupt landed on the rbtree the copy had already run over. No debug options are needed for this; KASAN reports the same write as an out-of-bounds read of 13835058055416381440 bytes. Documentation/trace/histogram.rst already states the rule, "must be a long[] type", so enforce it once the name has been resolved. Names which resolve to no field at all, "hitcount.stacktrace" and the common_* pseudo-fields, are refused for the same reason: they hold no stacktrace to read. Cc: stable@vger.kernel.org Fixes: cc5fc8bfc961 ("tracing/histogram: Add stacktrace type") Link: https://patch.msgid.link/20260907155045.692664-2-donggeunyoo.kernel@gmail.com Signed-off-by: Donggeun Yoo Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_hist.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 53100287466f..9bc829c1e876 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -2317,6 +2317,7 @@ parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, struct ftrace_event_field *field = NULL; char *field_name, *modifier, *str; struct trace_array *tr = file->tr; + bool stack_modifier = false; modifier = str = kstrdup(field_str, GFP_KERNEL); if (!modifier) @@ -2339,9 +2340,10 @@ parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, *flags |= HIST_FIELD_FL_EXECNAME; else if (strcmp(modifier, "syscall") == 0) *flags |= HIST_FIELD_FL_SYSCALL; - else if (strcmp(modifier, "stacktrace") == 0) + else if (strcmp(modifier, "stacktrace") == 0) { *flags |= HIST_FIELD_FL_STACKTRACE; - else if (strcmp(modifier, "log2") == 0) + stack_modifier = true; + } else if (strcmp(modifier, "log2") == 0) *flags |= HIST_FIELD_FL_LOG2; else if (strcmp(modifier, "usecs") == 0) *flags |= HIST_FIELD_FL_TIMESTAMP_USECS; @@ -2412,6 +2414,12 @@ parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, } } } + + if (stack_modifier && + (!field || field->filter_type != FILTER_STACKTRACE)) { + hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str)); + field = ERR_PTR(-EINVAL); + } out: kfree(str); -- cgit v1.2.3 From 7f711e62355bb3123a2ca2f97a2facbfebc678c6 Mon Sep 17 00:00:00 2001 From: Donggeun Yoo Date: Tue, 8 Sep 2026 00:50:45 +0900 Subject: tracing: Fix memory corruption from a "STACKTRACE" histogram key "cpu", "CPU", "stacktrace" and "STACKTRACE" are generic fields, defined with an offset and a size of zero so that the filter code can match them by name. parse_field() maps them onto their common_* equivalents for backward compatibility, but unlike the common_* names it hands the placeholder back to the caller instead of NULL. create_hist_field() takes a non-NULL field as a promise that the record carries a stacktrace and picks HIST_FIELD_FN_STACK, so the __data_loc word is read from offset 0, that is from common_type, and its low 16 bits are followed as an offset into the record. What is found there becomes the length of an unbounded memcpy. Pick an event whose id is small enough that the offset stays inside its own record and the length is a kernel text address: # cd /sys/kernel/tracing # echo 'hist:keys=STACKTRACE' > events/ftrace/print/trigger # echo hello > trace_marker Oops: general protection fault, probably for non-canonical address RIP: 0010:rb_next+0x23/0x60 RIP: 0010:memcpy+0xc/0x30 event_hist_trigger+0x2e7/0x12c0 Kernel panic - not syncing: Fatal exception in interrupt Leave the field NULL, which is what the comment above the branch says the code does and what common_stacktrace already does. FILTER_CPU and FILTER_COMM are left alone, their create_hist_field() branches never look at the field. Cc: stable@vger.kernel.org Fixes: 4b512860bdbd ("tracing: Rename stacktrace field to common_stacktrace") Link: https://patch.msgid.link/20260907155045.692664-3-donggeunyoo.kernel@gmail.com Signed-off-by: Donggeun Yoo Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_hist.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 9bc829c1e876..8af97fd4ee2d 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -2404,6 +2404,7 @@ parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, *flags |= HIST_FIELD_FL_CPU; } else if (field && field->filter_type == FILTER_STACKTRACE) { *flags |= HIST_FIELD_FL_STACKTRACE; + field = NULL; } else if (field && field->filter_type == FILTER_COMM) { *flags |= HIST_FIELD_FL_COMM | HIST_FIELD_FL_STRING; } else { -- cgit v1.2.3 From 911002e99e15f640f1fdc6d276206beaef59e790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Tue, 8 Sep 2026 08:22:15 +0200 Subject: tracing: Restore :mod: trailer after parsing in ftrace_set_clr_event() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While ftrace_set_clr_event() modifies its input buffer during parsing, before returning to the caller the buffer is supposed to be restored to its original state. This works correctly for the colon between the subsystem and event but not the colon at the beginning of :mod:. Restore the colon, so the :mod: trailer is not stripped after ftrace_set_clr_event(). Cc: stable@vger.kernel.org Fixes: 4c86bc531e60 ("tracing: Add :mod: command to enabled module events") Link: https://patch.msgid.link/20260908-tracing-cli-event-filter-v2-1-05396a3fb663@linutronix.de Signed-off-by: Thomas Weißschuh Signed-off-by: Steven Rostedt --- kernel/trace/trace_events.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 9dbc2441763b..30c0ddf90887 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -1489,6 +1489,8 @@ int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set) /* Put back the colon to allow this to be called again */ if (buf) *(buf - 1) = ':'; + if (mod) + *(mod - 5) = ':'; return ret; } -- cgit v1.2.3 From 7e645147dfba67edb3ed3090a1ed1d89df77fc27 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Wed, 9 Sep 2026 08:29:17 +0200 Subject: tracing: Fix ring_buffer_read_page_size() kernel-doc ring_buffer_read_page_size() takes a parameter named rpage, but its kernel-doc describes page. As a result, kernel-doc reports rpage as undescribed and page as an excess parameter description. Rename the documentation entry to match the function. Link: https://patch.msgid.link/20260909062917.89482-1-kmehltretter@gmail.com Fixes: dae8dda341d2 ("tracing: Fix subbuf resize races with trace_pipe_raw readers") Assisted-by: LLM Signed-off-by: Karl Mehltretter Reviewed-by: Vincent Donnefort Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 9c03a555a6ba..0e30c7bd6045 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -7384,7 +7384,7 @@ EXPORT_SYMBOL_GPL(ring_buffer_read_page_data); /** * ring_buffer_read_page_size - get size of the read page. - * @page: the page to get the size from + * @rpage: the page to get the size from * * Returns size of the page in bytes. */ -- cgit v1.2.3 From ed0aff60f83a9bdc2f6556376ac79c96b3ce7e80 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Thu, 10 Sep 2026 22:12:09 -0400 Subject: tracing: Take trace_array reference when opening a tracer options file When a tracer option file is opened, it is passed a descriptor that points to an element on the trace_array's topts array. This element has information to find the trace array and other information. It uses this element to take a reference of the trace_array so that the trace_array does not get removed while this file is opened. Unfortunately, there's a race condition where the element itself could be freed by the removal of the instance the trace_array represents causing a use-after-free as this element that is used to find the trace_array to increment its reference counter is also freed when the instance is removed. To solve this, add a trace_array_tracer_options_get() helper function that will take the address of the element that is passed to the open function by the inode->i_private pointer and search all the trace_arrays under a lock to find the one that the element's address is in the range of the trace_arrays topts array elements. When a match happens, that trace_array's reference would be increased. Note, there's a race where if an admin was deleting and creating trace instances at the same time and the memory of the old trace_array's array matched the memory of the new trace_array that it could in theory open the option from the wrong trace array. But we do not care because it would be stupid to perform that kind of action. As long as the only thing that can happen is that the option from the wrong trace array is used and doesn't crash the kernel it will only make the user confused. But if they are doing something stupid like this, they are already confused, so no harm done. Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260910221209.62dad8d3@robin Fixes: 7e2cfbd2d3c86 ("tracing: Have option files inc the trace array ref count") Reported-by: sashiko-bot@kernel.org Closes: https://lore.kernel.org/linux-trace-kernel/20260902121918.5a9e9d1b@gandalf.local.home/ Signed-off-by: Steven Rostedt --- kernel/trace/trace.c | 46 +++++++++++++++++++++++++++++++++++++++++++++- kernel/trace/trace.h | 1 + 2 files changed, 46 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 8658cad53cb5..e4a490d3d08c 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -7717,12 +7717,55 @@ trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt, return cnt; } +static bool tr_option_match(struct trace_array *tr, void *topt) +{ + for (int i = 0; i < tr->nr_topts; i++) { + struct trace_options *tr_topts = &tr->topts[i]; + + if (topt >= (void *)&tr_topts->topts[0] && + topt < (void *)&tr_topts->topts[tr_topts->nr_topts]) + return true; + } + return false; +} + +/* + * The topt is the address of a trace_array->topts[] element that holds the + * the tracer options descriptor. But since the trace_array reference has not + * been taken yet, it cannot be dereferenced as it could have been freed by + * a rmdir of the instance the trace_array represents. + * + * Search the list of trace_arrays and compare the topt to the address of + * the entire trace_array topts array for each trace_array in the list. + * If one is matched, then take the reference and return it. If not, the + * trace_array no longer exits. + */ +static int trace_array_tracer_options_get(void *topt) +{ + struct trace_array *tr; + int ret; + + ret = security_locked_down(LOCKDOWN_TRACEFS); + if (ret) + return ret; + + if (tracing_disabled) + return -ENODEV; + + guard(mutex)(&trace_types_lock); + list_for_each_entry(tr, &ftrace_trace_arrays, list) { + if (tr_option_match(tr, topt)) + return __trace_array_get(tr); + } + return -ENODEV; +} + static int tracing_open_options(struct inode *inode, struct file *filp) { struct trace_option_dentry *topt = inode->i_private; int ret; - ret = tracing_check_open_get_tr(topt->tr); + ret = trace_array_tracer_options_get(topt); if (ret) return ret; @@ -7984,6 +8027,7 @@ create_trace_option_files(struct trace_array *tr, struct tracer *tracer, tr->topts = tr_topts; tr->topts[tr->nr_topts].tracer = tracer; tr->topts[tr->nr_topts].topts = topts; + tr->topts[tr->nr_topts].nr_topts = cnt; tr->nr_topts++; for (cnt = 0; opts[cnt].name; cnt++) { diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 5e76f94e7a80..bd3c8f80300f 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -227,6 +227,7 @@ struct array_buffer { struct trace_options { struct tracer *tracer; struct trace_option_dentry *topts; + int nr_topts; }; struct trace_pid_list *trace_pid_list_alloc(void); -- cgit v1.2.3 From 815e07c8fe885a87751c2496a30ae0dcd4118210 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 11 Sep 2026 12:21:52 +0200 Subject: ring-buffer: Acquire the lock with irqsave in rb_wake_up_waiters() rb_wake_up_waiters() is a irq_work callback which is initialized with init_irq_work(). As such it will be invoked in thread context on PREEMPT_RT. Invoking the callback in IRQ context on PREEMPT_RT is not an option due its usage of wake_up_all(). Since this callback may run in thread context, it needs to acquire ring_buffer_per_cpu::reader_lock with disabling interrupts and may not assume that they are disabled. Use raw_spinlock_irqsave() to acquire ring_buffer_per_cpu::reader_lock. Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260911102152.YEtwkBj9@linutronix.de Fixes: 68282dd930ea3 ("ring-buffer: Fix resetting of shortest_full") Reviewed-by: Vincent Donnefort Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 0e30c7bd6045..9bc8ce8c5676 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -904,14 +904,13 @@ static void rb_wake_up_waiters(struct irq_work *work) struct ring_buffer_per_cpu *cpu_buffer = container_of(rbwork, struct ring_buffer_per_cpu, irq_work); - /* Called from interrupt context */ - raw_spin_lock(&cpu_buffer->reader_lock); - rbwork->wakeup_full = false; - rbwork->full_waiters_pending = false; + scoped_guard(raw_spinlock_irqsave, &cpu_buffer->reader_lock) { + rbwork->wakeup_full = false; + rbwork->full_waiters_pending = false; - /* Waking up all waiters, they will reset the shortest full */ - cpu_buffer->shortest_full = 0; - raw_spin_unlock(&cpu_buffer->reader_lock); + /* Waking up all waiters, they will reset the shortest full */ + cpu_buffer->shortest_full = 0; + } wake_up_all(&rbwork->full_waiters); } -- cgit v1.2.3 From 0d7823cd4cda35f2060a685fa276ff7709915abc Mon Sep 17 00:00:00 2001 From: Siddharth Chintamaneni Date: Wed, 2 Sep 2026 17:14:13 +0000 Subject: bpf: Allow terminal gotox instructions check_subprogs() treats gotox as a direct jump and validates its reserved zero offset. When gotox is the final instruction, this produces a synthetic successor one instruction past the end of the subprogram and rejects an otherwise valid program. Skip direct-offset validation for gotox and accept it as a non-fallthrough terminal instruction. Its actual targets remain validated from the instruction-array jump table during CFG construction. Fixes: 493d9e0d6083 ("bpf, x86: add support for indirect jumps") Signed-off-by: Siddharth Chintamaneni Reviewed-by: Anton Protopopov Link: https://lore.kernel.org/r/20260902171414.96165-1-sidchintamaneni@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 1b0b1fb62878..ddba53eaa333 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3042,6 +3042,8 @@ static int check_subprogs(struct bpf_verifier_env *env) subprog[cur_subprog].exit_idx = i; goto next; } + if (insn_is_gotox(&insn[i])) + goto next; off = i + bpf_jmp_offset(&insn[i]) + 1; if (off < subprog_start || off >= subprog_end) { verbose(env, "jump out of range from insn %d to %d\n", i, off); @@ -3061,7 +3063,8 @@ next: */ if (code != (BPF_JMP | BPF_EXIT) && code != (BPF_JMP32 | BPF_JA) && - code != (BPF_JMP | BPF_JA)) { + code != (BPF_JMP | BPF_JA) && + !insn_is_gotox(&insn[i])) { verbose(env, "last insn is not an exit or jmp\n"); bpf_diag_program_structure( env, i, "subprogram can fall through", -- cgit v1.2.3 From b4dcc18b97913888e8d009624e07c8014ce41b84 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Fri, 11 Sep 2026 22:25:12 +0800 Subject: ftrace: Use rcu_assign_pointer() for tmp_ops filter hash tmp_ops.func_hash->filter_hash is annotated __rcu, but update_ftrace_direct_mod() assigns hash to it directly. Sparse reports an address-space mismatch. Use rcu_assign_pointer() for the assignment. Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260911142512.19344-1-leon.hwang@linux.dev Fixes: 50b35c9e50a8 ("ftrace: Use hash argument for tmp_ops in update_ftrace_direct_mod") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202609110704.Q3M5vCDV-lkp@intel.com/ Signed-off-by: Leon Hwang Signed-off-by: Steven Rostedt --- kernel/trace/ftrace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 53d5db60bfa5..673a54fdf392 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -6675,7 +6675,7 @@ int update_ftrace_direct_mod(struct ftrace_ops *ops, struct ftrace_hash *hash, b /* Enable the tmp_ops to have the same functions as the hash object. */ ftrace_ops_init(&tmp_ops); - tmp_ops.func_hash->filter_hash = hash; + rcu_assign_pointer(tmp_ops.func_hash->filter_hash, hash); err = register_ftrace_function_nolock(&tmp_ops); if (err) -- cgit v1.2.3 From bcfe2816e6ec46c3f4c58aa4264476665ddb3f69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20Ahmet=20Memi=C5=9F?= Date: Fri, 11 Sep 2026 18:56:47 +0300 Subject: tracing: Don't dereference trace_event_file in deferred trigger free MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The enable_event trigger defers trace_event_put_ref() to the trigger free kthread, but the trace_event_file can already be freed when the instance is removed. Keep the trace_event_call directly in enable_trigger_data so the deferred free does not access the freed trace_event_file. Cc: stable@vger.kernel.org Fixes: e091351b3881 ("tracing: Delay module ref count for "enable_event" trigger") Reported-by: Alexander Gordeev Closes: https://lore.kernel.org/all/20260828134340.2501683A24-agordeev@linux.ibm.com/ Link: https://patch.msgid.link/20260911155650.354844-1-aliamemis@disroot.org Signed-off-by: Ali Ahmet Memiş Signed-off-by: Steven Rostedt --- kernel/trace/trace.h | 1 + kernel/trace/trace_events_trigger.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index bd3c8f80300f..3749485a7d85 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -1953,6 +1953,7 @@ struct event_trigger_data { struct enable_trigger_data { struct trace_event_file *file; + struct trace_event_call *call; bool enable; bool hist; }; diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c index 149300cc5e8a..4d2fde996c0f 100644 --- a/kernel/trace/trace_events_trigger.c +++ b/kernel/trace/trace_events_trigger.c @@ -1728,7 +1728,8 @@ static void enable_trigger_private_data_free(struct event_trigger_data *data) { struct enable_trigger_data *enable_data = data->private_data; - trace_event_put_ref(enable_data->file->event_call); + /* The file may already be freed here, only the call is kept alive */ + trace_event_put_ref(enable_data->call); kfree(enable_data); } @@ -1801,6 +1802,7 @@ int event_enable_trigger_parse(struct event_command *cmd_ops, enable_data->hist = hist; enable_data->enable = enable; enable_data->file = event_enable_file; + enable_data->call = event_enable_file->event_call; trigger_data = trigger_data_alloc(cmd_ops, cmd, param, enable_data); if (!trigger_data) -- cgit v1.2.3 From 85136bf22404474a815fc0ed26ec0d1cbc1bc3f9 Mon Sep 17 00:00:00 2001 From: "Jose Fernandez (Anthropic)" Date: Wed, 9 Sep 2026 17:51:04 +0000 Subject: bpf: Avoid soft lockup in __htab_map_lookup_and_delete_batch() __htab_map_lookup_and_delete_batch() has no rescheduling point. The batch count bounds how many entries are copied out, not how many buckets are visited, so one BPF_MAP_LOOKUP_BATCH call can walk the map end to end. The empty-bucket fast path is worse: it stays inside a single rcu_read_lock() / bpf_disable_instrumentation() section for any run of consecutive empty buckets. That holds up on small maps, but it falls apart at scale. On a 144-CPU arm64 host running a CONFIG_PREEMPT_NONE kernel, periodic BPF_MAP_LOOKUP_BATCH calls against an LRU hash map with 16,777,216 buckets held a CPU inside the batch op for 77+ seconds and triggered the soft lockup watchdog. Commit 75134f16e7dd ("bpf: Add schedule points in batch ops") fixed this same problem in the generic batch ops, but not in this htab-native path, which every htab-based hash map variant uses for its lookup[_and_delete] batch ops. Complete that fix here. Leave the critical section after 64 consecutive empty buckets, call cond_resched_tasks_rcu_qs(), and resume at the saved bucket cursor. No locks are held at that point, and resuming from the cursor is already the function's behavior for non-empty buckets. Add the same call to the per-bucket loop after copy_to_user(), where every lock has been dropped. cond_resched_rcu() is not enough here: sleeping with bpf_prog_active elevated makes tracing programs on that CPU silently skip their invocations. Plain cond_resched() is not enough either. It is a no-op under PREEMPT and PREEMPT_LAZY, the only models arm64 and x86 have offered since commit 7dadeaa6e851 ("sched: Further restrict the preemption modes"). It is also never a Tasks RCU quiescent state, in any model: the reschedule counts as a preemption. The walking task stays a holdout and stalls every synchronize_rcu_tasks() caller, ftrace and BPF trampoline teardown included, until the syscall returns [1]. cond_resched_tasks_rcu_qs() is the usual tool for that [2]. It reports the quiescent state at each yield and still reschedules as cond_resched() does on PREEMPT_NONE and PREEMPT_VOLUNTARY kernels. Fixes: 057996380a42 ("bpf: Add batch ops to all htab bpf map") Cc: "Paul E. McKenney" Cc: Rik van Riel Link: https://lore.kernel.org/bpf/20260715215314.44423f47@fangorn/ [1] Link: https://lore.kernel.org/bpf/9d444098-7c03-4163-af12-bd0a79a51443@paulmck-laptop/ [2] Assisted-by: LLM Signed-off-by: Jose Fernandez (Anthropic) Signed-off-by: Josef Bacik Reviewed-by: Rik van Riel Link: https://lore.kernel.org/r/20260909-b4-htab-batch-resched-v2-1-0cb529d8f95a@toxicpanda.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/hashtab.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 6f331c80130d..a72dc5b9f184 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -1772,6 +1772,12 @@ static int htab_lru_percpu_map_lookup_and_delete_elem(struct bpf_map *map, flags); } +/* + * Max consecutive empty buckets to walk in one RCU + + * instrumentation-disabled section before rescheduling. + */ +#define HTAB_BATCH_EMPTY_RESCHED 64 + static int __htab_map_lookup_and_delete_batch(struct bpf_map *map, const union bpf_attr *attr, @@ -1793,6 +1799,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, unsigned long flags = 0; bool locked = false; struct htab_elem *l; + u32 empty_cnt = 0; struct bucket *b; int ret = 0; @@ -1971,12 +1978,21 @@ again_nocopy: } next_batch: - /* If we are not copying data, we can go to next bucket and avoid - * unlocking the rcu. + /* + * If we are not copying data, we can go to next bucket and avoid + * unlocking the rcu. Bound the walk though: after + * HTAB_BATCH_EMPTY_RESCHED consecutive empty buckets, fully exit + * the critical section (no locks are held here) and reschedule. */ if (!bucket_cnt && (batch + 1 < htab->n_buckets)) { batch++; - goto again_nocopy; + if (++empty_cnt < HTAB_BATCH_EMPTY_RESCHED) + goto again_nocopy; + empty_cnt = 0; + rcu_read_unlock(); + bpf_enable_instrumentation(); + cond_resched_tasks_rcu_qs(); + goto again; } rcu_read_unlock(); @@ -1990,11 +2006,13 @@ next_batch: } total += bucket_cnt; + empty_cnt = 0; batch++; if (batch >= htab->n_buckets) { ret = -ENOENT; goto after_loop; } + cond_resched_tasks_rcu_qs(); goto again; after_loop: -- cgit v1.2.3 From c7a1c6e8004ab12a9c9bfdcb603f60f9bf4a3cee Mon Sep 17 00:00:00 2001 From: fangqiurong Date: Sat, 12 Sep 2026 21:15:18 +0800 Subject: sched_ext: Close the pre-enable ops error claim window scx_alloc_and_add_sched() publishes ops->priv before scx_root_enable_workfn() switches the state to SCX_ENABLING. An error claimed via scx_bpf_error_bstr() from an associated BPF program in that window is consumed by scx_disable_workfn(), which takes the pre-enable shortcut in scx_root_disable(). The shortcut returns without any teardown and restores SCX_DISABLED with an unconditional scx_set_enable_state() xchg racing the enable workfn's own transition. The enable then completes with the claim consumed: the scheduler stays up but can never be disabled again, and bpf_scx_unreg() frees it while still in use, resulting in a use-after-free. Both WARN_ON_ONCE()s fire back to back: WARNING: kernel/sched/ext/ext.c:7522 at scx_root_enable_workfn+0xeec/0x1be0, CPU#3: scx_enable_help/276 WARNING: kernel/sched/ext/ext.c:6398 at scx_root_disable+0xb50/0xdb8, CPU#0: sched_ext_helpe/664 scx_root_enable_workfn() switches to SCX_ENABLING before the scheduler allocation, so ops->priv is never visible while SCX_DISABLED. The allocation failure path restores SCX_DISABLED. Fixes: 105dcd005be2 ("sched_ext: Introduce scx_prog_sched()") Cc: stable@vger.kernel.org Signed-off-by: fangqiurong Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index adf5993fa597..83999203a63a 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -7516,22 +7516,24 @@ static void scx_root_enable_workfn(struct kthread_work *work) #ifdef CONFIG_EXT_SUB_SCHED cgroup_get(cgrp); #endif + /* + * Transition to ENABLING to arm the disable path. Allocation failure + * still unwinds locally. Full disabling on failure applies only after + * scx_alloc_and_add_sched() succeeds. + */ + WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED); + WARN_ON_ONCE(scx_root); + sch = scx_alloc_and_add_sched(cmd, cgrp, NULL); if (IS_ERR(sch)) { ret = PTR_ERR(sch); + WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_ENABLING); goto err_free_tid_hash; } if (sch->is_cid_type) static_branch_enable(&__scx_is_cid_type); - /* - * Transition to ENABLING and clear exit info to arm the disable path. - * Failure triggers full disabling from here on. - */ - WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED); - WARN_ON_ONCE(scx_root); - atomic_long_set(&scx_nr_rejected, 0); for_each_possible_cpu(cpu) { -- cgit v1.2.3 From 442ffa742daa65a0e8fe003abe9fbe472366e4de Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Fri, 11 Sep 2026 20:39:35 +0100 Subject: tracing/remotes: Account for ring buffer page header in size calculation trace_buffer_desc_size() and trace_remote_alloc_buffer() undercount the required pages because every ring buffer page contains a header (BUF_PAGE_HDR_SIZE). Account for that header to ensure allocated remote ring buffers aren't smaller than requested by the user. The newly introduced helper __calc_nr_pages_ring_buffer_desc() can return a value that overflows the descriptor nr_pages field (32 bits). Link: https://patch.msgid.link/20260911193937.602202-2-vdonnefort@google.com Fixes: 2e67fabd8b77 ("ring-buffer: Introduce ring-buffer remotes") Signed-off-by: Vincent Donnefort Signed-off-by: Steven Rostedt --- kernel/trace/trace_remote.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/trace_remote.c b/kernel/trace/trace_remote.c index 75fa1ffc4c96..c17902e42ef1 100644 --- a/kernel/trace/trace_remote.c +++ b/kernel/trace/trace_remote.c @@ -980,7 +980,7 @@ int trace_remote_alloc_buffer(struct trace_buffer_desc *desc, size_t desc_size, const struct cpumask *cpumask) { size_t min_desc_size = trace_buffer_desc_size(buffer_size, cpumask_weight(cpumask)); - unsigned int nr_pages = max(DIV_ROUND_UP(buffer_size, PAGE_SIZE), 2UL) + 1; + unsigned int nr_pages = __calc_nr_pages_ring_buffer_desc(buffer_size); struct ring_buffer_desc *rb_desc; int cpu, ret = -ENOMEM; -- cgit v1.2.3 From d059d8bf2c9b5d563d15e7552d73e17d7535013a Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Fri, 11 Sep 2026 20:39:36 +0100 Subject: tracing/remotes: Catch nr_page_va overflow in ring_buffer_desc sizing The number of pages per remote ring buffer is capped by ring_buffer_desc::nr_page_va (32 bits). A buffer_size large enough to overflow that field would silently allocate a descriptor smaller than what was asked for. Return SIZE_MAX from trace_buffer_desc_size() on nr_page_va overflow. Link: https://patch.msgid.link/20260911193937.602202-3-vdonnefort@google.com Fixes: 2e67fabd8b77 ("ring-buffer: Introduce ring-buffer remotes") Signed-off-by: Vincent Donnefort Signed-off-by: Steven Rostedt --- kernel/trace/trace_remote.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/trace_remote.c b/kernel/trace/trace_remote.c index c17902e42ef1..2d5bc423efca 100644 --- a/kernel/trace/trace_remote.c +++ b/kernel/trace/trace_remote.c @@ -980,9 +980,12 @@ int trace_remote_alloc_buffer(struct trace_buffer_desc *desc, size_t desc_size, const struct cpumask *cpumask) { size_t min_desc_size = trace_buffer_desc_size(buffer_size, cpumask_weight(cpumask)); - unsigned int nr_pages = __calc_nr_pages_ring_buffer_desc(buffer_size); struct ring_buffer_desc *rb_desc; int cpu, ret = -ENOMEM; + unsigned int nr_pages; + + if (min_desc_size == SIZE_MAX) + return -E2BIG; if (desc_size < min_desc_size) return -EINVAL; @@ -991,6 +994,7 @@ int trace_remote_alloc_buffer(struct trace_buffer_desc *desc, size_t desc_size, desc->struct_len = min_desc_size; rb_desc = __first_ring_buffer_desc(desc); + nr_pages = __calc_nr_pages_ring_buffer_desc(buffer_size); for_each_cpu(cpu, cpumask) { unsigned int id; -- cgit v1.2.3 From d860c67c051685abb0460b593b193f0f45f4fa92 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sat, 12 Sep 2026 11:39:38 +0100 Subject: ring-buffer: Check resize_disabled before publishing the new subbuf order ring_buffer_subbuf_order_set() stores the new order and only then walks the CPUs, returning -EBUSY if any of them has resizing disabled. A user mapped buffer has resizing disabled, and __rb_map_vma() reads buffer->subbuf_order without buffer->mutex, so an mmap of an already mapped CPU racing the failing order change sizes the mapping with the new order and inserts pages past the sub-buffer into the VMA. Check the CPUs before storing the new order. Cc: stable@vger.kernel.org Fixes: 117c39200d9d ("ring-buffer: Introducing ring-buffer mapping functions") Link: https://patch.msgid.link/20260912103938.1127021-1-devnexen@gmail.com Signed-off-by: David Carlier Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'kernel') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 9bc8ce8c5676..04bb94c29f58 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -7473,6 +7473,14 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) old_capacity = rb_subbuf_capacity(buffer); + /* The mmap fast path reads subbuf_order without buffer->mutex. */ + for_each_buffer_cpu(buffer, cpu) { + if (!cpumask_test_cpu(cpu, buffer->cpumask)) + continue; + if (atomic_read(&buffer->buffers[cpu]->resize_disabled)) + return -EBUSY; + } + atomic_inc(&buffer->record_disabled); /* Make sure all commits have finished */ -- cgit v1.2.3 From 1c21452d02eec2f008e2c5535820f85adbd7587a Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Sat, 5 Sep 2026 02:11:39 +0000 Subject: bpf: Fix UAF due to concurrent consumption of ttrace lists in alloc_bulk Syzkaller repeatedly triggered UAF splats related to nodes in waiting_for_gp_ttrace within the bpf memalloc: BUG: KASAN: slab-use-after-free in llist_del_first+0x85/0x110 lib/llist.c:61 Read of size 8 at addr ffff8881572cd080 by task syz.4.470/5112 ... llist_del_first+0x85/0x110 lib/llist.c:61 alloc_bulk+0x193/0x460 kernel/bpf/memalloc.c:229 bpf_mem_refill+0x386/0x560 kernel/bpf/memalloc.c:436 Freed by task 14: ... __free_rcu kernel/bpf/memalloc.c:281 [inline] __free_rcu_tasks_trace+0x48/0xd0 kernel/bpf/memalloc.c:291 rcu_tasks_invoke_cbs+0x1ec/0x3e0 kernel/rcu/tasks.h:571 rcu_tasks_one_gp+0x13d/0x220 kernel/rcu/tasks.h:621 rcu_tasks_kthread+0xf3/0x120 kernel/rcu/tasks.h:651 The reason is that the UAF occurs after the RCU Tasks Trace GP expires: when the __free_rcu() callback runs, there is no synchronization protecting llist_del_all() against concurrent alloc_bulk() operating on waiting_for_gp_ttrace, leading to the race condition below: CPU0 CPU1 __free_rcu (RCU Tasks Trace callback) alloc_bulk llist_del_first(&c->waiting_for_gp_ttrace) entry = smp_load_acquire(&head->first); do { if (entry == NULL) return NULL; free_all(llist_del_all(&c->waiting_for_gp_ttrace)) llist_for_each_safe(pos, t, llnode) free_one(pos); next = READ_ONCE(entry->next); <-- trigger UAF } while (!try_cmpxchg(&head->first, &entry, next)); In addition, there is also a theoretical race condition on the free_by_rcu_ttrace list. This race requires two preconditions: an in-flight Tasks Trace GP keeping c->call_rcu_ttrace_in_progress == 1, and concurrent cross-CPU frees repopulating c->free_by_rcu_ttrace with new nodes. Under these conditions, the following scenario triggers UAF: // CPU0 // irq work is still busy (on PREEMPT_RT) alloc_bulk() llist_del_first(&c->free_by_rcu_ttrace) entry = smp_load_acquire(&head->first); do { if (entry == NULL) return NULL; // CPU1 bpf_mem_alloc_destroy() WRITE_ONCE(c->draining, true) // wait for CPU0 irq_work_sync() // CPU2 do_call_rcu_ttrace(tgt(CPU0)) if (c->draining) { llist_del_all(&c->free_by_rcu_ttrace) free_all() } // CPU0 continue next = READ_ONCE(entry->next); <-- trigger UAF while (!try_cmpxchg(&head->first, &entry, next)); Fix this by introducing a raw spinlock to synchronize the concurrent consumption on waiting_for_gp_ttrace and free_by_rcu_ttrace. Fixes: 04fabf00b4d3 ("bpf: Allow reuse from waiting_for_gp_ttrace list.") Suggested-by: Alexei Starovoitov Suggested-by: Hou Tao Signed-off-by: Pu Lehui Acked-by: Hou Tao Link: https://lore.kernel.org/r/20260905021139.4116529-1-pulehui@huaweicloud.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/memalloc.c | 50 ++++++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 22 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c index e9662db7198f..8a8f088e83e6 100644 --- a/kernel/bpf/memalloc.c +++ b/kernel/bpf/memalloc.c @@ -119,6 +119,7 @@ struct bpf_mem_cache { struct llist_head waiting_for_gp_ttrace; struct rcu_head rcu_ttrace; atomic_t call_rcu_ttrace_in_progress; + raw_spinlock_t lock; }; struct bpf_mem_caches { @@ -214,25 +215,24 @@ static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node, bool atomic) gfp = __GFP_NOWARN | __GFP_ACCOUNT; gfp |= atomic ? GFP_NOWAIT : GFP_KERNEL; - for (i = 0; i < cnt; i++) { - /* - * For every 'c' llist_del_first(&c->free_by_rcu_ttrace); is - * done only by one CPU == current CPU. Other CPUs might - * llist_add() and llist_del_all() in parallel. - */ - obj = llist_del_first(&c->free_by_rcu_ttrace); - if (!obj) - break; - add_obj_to_free_list(c, obj); - } - if (i >= cnt) - return; + /* + * c->lock serializes concurrent llist_del_first() against + * llist_del_all() in __free_rcu() and do_call_rcu_ttrace(). + */ + scoped_guard(raw_spinlock_irqsave, &c->lock) { + for (i = 0; i < cnt; i++) { + obj = llist_del_first(&c->free_by_rcu_ttrace); + if (!obj) + break; + add_obj_to_free_list(c, obj); + } - for (; i < cnt; i++) { - obj = llist_del_first(&c->waiting_for_gp_ttrace); - if (!obj) - break; - add_obj_to_free_list(c, obj); + for (; i < cnt; i++) { + obj = llist_del_first(&c->waiting_for_gp_ttrace); + if (!obj) + break; + add_obj_to_free_list(c, obj); + } } if (i >= cnt) return; @@ -279,8 +279,12 @@ static int free_all(struct bpf_mem_cache *c, struct llist_node *llnode, bool per static void __free_rcu(struct rcu_head *head) { struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu_ttrace); + struct llist_node *llnode; + + scoped_guard(raw_spinlock_irqsave, &c->lock) + llnode = llist_del_all(&c->waiting_for_gp_ttrace); - free_all(c, llist_del_all(&c->waiting_for_gp_ttrace), !!c->percpu_size); + free_all(c, llnode, !!c->percpu_size); atomic_set(&c->call_rcu_ttrace_in_progress, 0); } @@ -300,7 +304,8 @@ static void do_call_rcu_ttrace(struct bpf_mem_cache *c) if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 1)) { if (unlikely(READ_ONCE(c->draining))) { - llnode = llist_del_all(&c->free_by_rcu_ttrace); + scoped_guard(raw_spinlock_irqsave, &c->lock) + llnode = llist_del_all(&c->free_by_rcu_ttrace); free_all(c, llnode, !!c->percpu_size); } return; @@ -535,6 +540,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) c->objcg = objcg; c->percpu_size = percpu_size; c->tgt = c; + raw_spin_lock_init(&c->lock); init_refill_work(c); prefill_mem_cache(c, cpu); } @@ -557,7 +563,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) c->objcg = objcg; c->percpu_size = percpu_size; c->tgt = c; - + raw_spin_lock_init(&c->lock); init_refill_work(c); prefill_mem_cache(c, cpu); } @@ -609,7 +615,7 @@ int bpf_mem_alloc_percpu_unit_init(struct bpf_mem_alloc *ma, int size) c->objcg = objcg; c->percpu_size = percpu_size; c->tgt = c; - + raw_spin_lock_init(&c->lock); init_refill_work(c); prefill_mem_cache(c, cpu); } -- cgit v1.2.3 From 953824e508b27d12837e32ef37ef6248e1f6fc7a Mon Sep 17 00:00:00 2001 From: Masoud Aghasi Date: Thu, 3 Sep 2026 09:27:34 +0100 Subject: bpf: Fix u32 overflow issue in map batch operations Several map batch operation implementations such as generic_map_lookup_batch() use calculations in the form of "values + cp * map->value_size" to compute the desired userspace memory address for reading or writing. This can overflow the u32 type (the result of "cp * map->value_size") when the map size exceeds 4GB. generic_map_lookup_batch() may corrupt values for some keys in userspace memory, and in some cases it mismatches values for some keys while still reporting success. Other batch operations may fail to delete or update some keys, or the syscall may return unexpected errors. Add size_t casts to prevent the affected offset and size calculations from overflowing. Fixes: cb4d03ab499d ("bpf: Add generic support for lookup batch op") Fixes: aa2e93b8e58e ("bpf: Add generic support for update and delete batch ops") Fixes: 057996380a42 ("bpf: Add batch ops to all htab bpf map") Signed-off-by: Masoud Aghasi Link: https://lore.kernel.org/r/20260903082734.623904-1-maghasi@disroot.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/hashtab.c | 8 ++++---- kernel/bpf/syscall.c | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index a72dc5b9f184..4f495dcbf670 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -1997,10 +1997,10 @@ next_batch: rcu_read_unlock(); bpf_enable_instrumentation(); - if (bucket_cnt && (copy_to_user(ukeys + total * key_size, keys, - key_size * bucket_cnt) || - copy_to_user(uvalues + total * value_size, values, - value_size * bucket_cnt))) { + if (bucket_cnt && (copy_to_user(ukeys + (size_t)total * key_size, keys, + (size_t)key_size * bucket_cnt) || + copy_to_user(uvalues + (size_t)total * value_size, values, + (size_t)value_size * bucket_cnt))) { ret = -EFAULT; goto after_loop; } diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index c7bc9ba9b331..853b47f81384 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2036,7 +2036,7 @@ int generic_map_delete_batch(struct bpf_map *map, for (cp = 0; cp < max_count; cp++) { err = -EFAULT; - if (copy_from_user(key, keys + cp * map->key_size, + if (copy_from_user(key, keys + (size_t)cp * map->key_size, map->key_size)) break; @@ -2098,9 +2098,9 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file, for (cp = 0; cp < max_count; cp++) { err = -EFAULT; - if (copy_from_user(key, keys + cp * map->key_size, + if (copy_from_user(key, keys + (size_t)cp * map->key_size, map->key_size) || - copy_from_user(value, values + cp * value_size, value_size)) + copy_from_user(value, values + (size_t)cp * value_size, value_size)) break; err = bpf_map_update_value(map, map_file, key, value, @@ -2179,12 +2179,12 @@ int generic_map_lookup_batch(struct bpf_map *map, if (err) goto free_buf; - if (copy_to_user(keys + cp * map->key_size, key, + if (copy_to_user(keys + (size_t)cp * map->key_size, key, map->key_size)) { err = -EFAULT; goto free_buf; } - if (copy_to_user(values + cp * value_size, value, value_size)) { + if (copy_to_user(values + (size_t)cp * value_size, value, value_size)) { err = -EFAULT; goto free_buf; } -- cgit v1.2.3 From 057dac23d329d5c5ed62352f2659a39fd46c6d4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Koutn=C3=BD?= Date: Mon, 14 Sep 2026 14:19:10 +0200 Subject: cgroup: Avoid iteration of dying tasks with zero refcount MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The commit 260fbcb92bbea ("cgroup: Move dying_tasks cleanup from cgroup_task_release() to cgroup_task_free()") extended the lifetime of tasks on the dying_tasks list. The iterators have provision to go through dying_tasks because of dying threadgroup leaders or explicit CSS_TASK_ITER_WITH_DEAD, however, it was expected that such tasks can obtain a new reference (that is possible before cgroup_task_release()/put_task_struct_rcu_user()). The tasks after cgroup_task_release() and before cgroup_task_free() are subject to race when they may or may not have ->usage count > 0. The race window is between css_task_iter_next() invocations when css_set_lock is released and we may arrive at a new ->task_pos. The iterator should not attempt to resurrect tasks whose ->usage count dropped to zero. (When that happens, __put_task_struct_rcu_cb() is already imminent and the returned task_struct would could be used after free.) As for the fix, we cannot simply check the signal->live count of a task on the dying list because that won't distinguish regular zombies waiting to be reaped from RCU remnant tasks that are going to be free'd. Therefore add an extra check to rule out ->usage==0 tasks from any iteration. The repeat: loop in css_task_iter_advance() doesn't consider ->usage count, so add a new loop to css_task_iter_next() to skip de-used tasks on the dying_list. Rough illustration of the possible race R (reader of cgroup.procs) T (thread) L (group leader) --------------------------------- -------------------------------- -------------------------------- L exits, signal->live > 0 cgroup_task_dead(L) css_set_skip_task_iters() // skips only cset->tasks list_add_tail(&L->cg_list, &cset->dying_tasks) css_task_iter_next() take css_set_lock css_task_iter_advance() leader && signal->live != 0 => it->task_pos = &L->cg_list release css_set_lock T exits --signal->live == 0 cgroup_task_dead(T) // css_set_lock release_task(T) cgroup_task_release(T) release_task(L) // zap_leader cgroup_task_release(L) put_task_struct_rcu_user(L) ...RCU... put_task_struct(L) L->usage = 0 /* L still on dying_tasks */ ...RCU... __put_task_struct(L) css_task_iter_next() // another iteration take css_set_lock it->task_pos = &L->cg_list get_task_struct(L) => addition on 0 drop css_set_lock cgroup_task_free(L) css_set_skip_task_iters() // dying skip comes too late free_task(L) cgroup_procs_show() task_pid_vnr(L) Fixes: 260fbcb92bbea ("cgroup: Move dying_tasks cleanup from cgroup_task_release() to cgroup_task_free()") Cc: stable@vger.kernel.org # v6.19+ Link: https://lists.debian.org/debian-kernel/2026/08/msg00220.html Reported-by: Noah Elias Feldt Reported-by: Salvatore Bonaccorso Tested-by: Salvatore Bonaccorso Signed-off-by: Michal Koutný Signed-off-by: Tejun Heo --- kernel/cgroup/cgroup.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 353c8f83439a..a3d363502b7b 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -5207,10 +5207,13 @@ struct task_struct *css_task_iter_next(struct css_task_iter *it) if (it->flags & CSS_TASK_ITER_SKIPPED) css_task_iter_advance(it); - if (it->task_pos) { + while (it->task_pos && !it->cur_task) { it->cur_task = list_entry(it->task_pos, struct task_struct, cg_list); - get_task_struct(it->cur_task); + /* a task on dying_tasks with zero refcount is only valid for + * RCU readers, not even interesting for + * CSS_TASK_ITER_WITH_DEAD, find another one */ + it->cur_task = tryget_task_struct(it->cur_task); css_task_iter_advance(it); } -- cgit v1.2.3 From 82431877d837a6c2593efd8e66ba28b35a220ca4 Mon Sep 17 00:00:00 2001 From: Kuniyuki Iwashima Date: Sat, 5 Sep 2026 23:36:30 +0000 Subject: sysctl: Check range in proc_dointvec_ms_jiffies_minmax Add the range check to do_proc_int_conv_ms_jiffies_minmax that commit d174174c6776 ("sysctl: replace SYSCTL_INT_CONV_CUSTOM macro with functions") incorrectly removed. Fixes: d174174c6776 ("sysctl: replace SYSCTL_INT_CONV_CUSTOM macro with functions") Signed-off-by: Kuniyuki Iwashima Signed-off-by: Joel Granados --- kernel/time/jiffies.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c index 213ae1d6a014..70926b73905a 100644 --- a/kernel/time/jiffies.c +++ b/kernel/time/jiffies.c @@ -181,7 +181,7 @@ static int do_proc_int_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr, int *k_ptr, int dir, const struct ctl_table *tbl) { - return proc_int_conv(negp, u_ptr, k_ptr, dir, tbl, false, + return proc_int_conv(negp, u_ptr, k_ptr, dir, tbl, true, sysctl_u2k_int_conv_ms, sysctl_k2u_int_conv_ms); } -- cgit v1.2.3 From 318012c56576e09806747f64f89f8f3cde1f999f Mon Sep 17 00:00:00 2001 From: Kuniyuki Iwashima Date: Sat, 5 Sep 2026 23:36:31 +0000 Subject: sysctl: Check range in do_proc_ulong_conv_ms_jiffies Add the range check back to do_proc_ulong_conv_ms_jiffies that commit b96b5c6708ea ("sysctl: Replace do_proc_do{int,ulong,uint}vec with do_proc_vec") incorrectly removed. Append "_minmax" to the end of do_proc_ulong_conv_ms_jiffies so it is clear that there should be a range check. Fixes: b96b5c6708ea ("sysctl: Replace do_proc_do{int,ulong,uint}vec with do_proc_vec") Signed-off-by: Kuniyuki Iwashima Signed-off-by: Joel Granados --- kernel/time/jiffies.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c index 70926b73905a..41f88ebac74a 100644 --- a/kernel/time/jiffies.c +++ b/kernel/time/jiffies.c @@ -195,10 +195,10 @@ static int sysctl_k2u_ulong_conv_ms(ulong *u_ptr, const ulong *k_ptr) return proc_ulong_k2u_conv_kop(u_ptr, k_ptr, sysctl_jiffies_to_msecs); } -static int do_proc_ulong_conv_ms_jiffies(bool *negp, ulong *u_ptr, ulong *k_ptr, - int dir, const struct ctl_table *tbl) +static int do_proc_ulong_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr, ulong *k_ptr, + int dir, const struct ctl_table *tbl) { - return proc_ulong_conv(u_ptr, k_ptr, dir, tbl, false, + return proc_ulong_conv(u_ptr, k_ptr, dir, tbl, true, sysctl_u2k_ulong_conv_ms, sysctl_k2u_ulong_conv_ms); } @@ -229,8 +229,8 @@ static int do_proc_int_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr, return -ENOSYS; } -static int do_proc_ulong_conv_ms_jiffies(bool *negp, ulong *u_ptr, ulong *k_ptr, - int dir, const struct ctl_table *tbl) +static int do_proc_ulong_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr, ulong *k_ptr, + int dir, const struct ctl_table *tbl) { return -ENOSYS; } @@ -333,7 +333,7 @@ int proc_doulongvec_ms_jiffies_minmax(const struct ctl_table *table, int dir, void *buffer, size_t *lenp, loff_t *ppos) { return proc_doulongvec_conv(table, dir, buffer, lenp, ppos, - do_proc_ulong_conv_ms_jiffies); + do_proc_ulong_conv_ms_jiffies_minmax); } EXPORT_SYMBOL(proc_doulongvec_ms_jiffies_minmax); -- cgit v1.2.3 From afdf35cfae0d039a4a6c907fa5d8391f1ef0a0aa Mon Sep 17 00:00:00 2001 From: Joel Granados Date: Thu, 10 Sep 2026 12:22:16 +0200 Subject: sysctl: Fix type truncation in sysctl_msec_to_jiffies Return MAX_JIFFY_OFFSET for all the values truncated when val (u64) is passed to msecs_to_jiffies (u32). This aligns with how very large millisecond values get translated into MAX_JIFFY_OFFSET. Fixes: b96b5c6708ea ("sysctl: Replace do_proc_do{int,ulong,uint}vec with do_proc_vec") Suggested-by: Kuniyuki Iwashima Signed-off-by: Joel Granados --- kernel/time/jiffies.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c index 41f88ebac74a..80c354811538 100644 --- a/kernel/time/jiffies.c +++ b/kernel/time/jiffies.c @@ -136,6 +136,8 @@ static int sysctl_k2u_int_conv_userhz(bool *negp, ulong *u_ptr, const int *k_ptr static ulong sysctl_msecs_to_jiffies(const ulong val) { + if (val > jiffies_to_msecs(MAX_JIFFY_OFFSET)) + return MAX_JIFFY_OFFSET; return msecs_to_jiffies(val); } -- cgit v1.2.3 From 9a0b159ff18c8f6fcf982bb81e15a9ceb14db43a Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 14 Sep 2026 22:12:34 -1000 Subject: sched_ext: scx_qmap: Restore unused idle claims from ops.dispatch() scx_qmap tracks idle cids itself. pick_direct_dispatch_cid() claims a cid by clearing its bit and the task is inserted into that cid's local DSQ, which kicks the CPU. When the task does not arrive, for example because the insert fell back to the global DSQ after an affinity change, the CPU wakes, finds nothing and picks idle again. That is not an idle transition, so ops.update_idle() is not called and the cid stays marked busy until an unrelated task runs on it. Restore the claim from ops.dispatch(). The kick guarantees a dispatch on the kicked CPU, and when it finds nothing to run with a NULL @prev, the CPU is going back to idle. Document the pattern in ops.update_idle(), which reports only actual transitions. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi Cc: Andrea Righi --- kernel/sched/ext/internal.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 076a351bb3f2..0adaf649d5e0 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -572,6 +572,12 @@ struct sched_ext_ops { * * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle * tracking. + * + * Only actual transitions are reported. A CPU that is claimed with an + * idle pick and kicked but dispatches no task returns to idle without a + * transition. A scheduler tracking idle CPUs itself must restore the + * idle state from ops.dispatch() when it returns without the next task + * to run. */ void (*update_idle)(s32 cpu, bool idle); -- cgit v1.2.3 From a9e3760b0838299649c0d57cca44daaf40ba3c33 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 14 Sep 2026 22:12:34 -1000 Subject: sched_ext: Maintain an online cid mask in the scheduler arena Schedulers on the default cid mapping treat [0, nr_online_cids) as the online set and restart on hotplug. Schedulers that install their own mapping with scx_bpf_cid_override() have no way to learn which cids are online: the count no longer identifies members and the CPU-form cpumask is unusable from cid programs. This is an obvious hole in the cid API. Add scx_bpf_online_cmask(), a kernel-maintained cmask in the scheduler's arena, allocated alongside the per-CPU scratch masks and populated after the cid mapping is finalized and before ops.init(), for child schedulers too. The pointer stays valid through ops.exit() with no reference to take. It is the arena offset as a void pointer, the same form struct_ops arena arguments arrive in. The verifier types the void return as a scalar for the program's arena cast. The mask follows the SCX hotplug notifications: seeded from cpu_active_mask and updated before ops.cid_online/offline() runs, so it lags cpu_online_mask only inside a hotplug transition. Updates walk the scheduler list under the lock that also serializes unlinking. Reads are live, not atomic snapshots. Root initialization excludes hotplug. v2: Reworded the getter kerneldoc (Andrea Righi). Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 75 +++++++++++++++++++++++++++++++++++++++++---- kernel/sched/ext/internal.h | 3 +- kernel/sched/ext/sub.c | 10 +++--- 3 files changed, 77 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 83999203a63a..70b711c4de6e 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3670,8 +3670,20 @@ static void handle_hotplug(struct rq *rq, bool online) s16 *tbl = rcu_dereference_check(scx_cpu_to_cid_tbl, lockdep_is_cpus_held()); - if (tbl) + if (tbl) { + struct scx_sched *pos; + cpu_or_cid = tbl[cpu]; + + guard(raw_spinlock_irqsave)(&scx_sched_lock); + list_for_each_entry(pos, &scx_sched_all, all) { + struct scx_cmask *mask = pos->online_cmask; + + if (mask) + __assign_bit(cpu_or_cid, (unsigned long *)mask->bits, + online); + } + } } if (online && SCX_HAS_OP(sch, cpu_online)) @@ -5280,12 +5292,17 @@ static void free_exit_info(struct scx_exit_info *ei); static const char *scx_exit_reason(enum scx_exit_kind kind); static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind); -s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) +s32 scx_alloc_kern_arena_objs(struct scx_sched *sch) { size_t size = struct_size_t(struct scx_cmask, bits, SCX_CMASK_NR_WORDS(num_possible_cpus())); + struct scx_cmask *online; + struct scx_cmask_ref ref; int cpu; + /* hotplug stays excluded until the online mask is published */ + lockdep_assert_cpus_held(); + if (!sch->is_cid_type || !sch->arena_pool) return 0; @@ -5301,15 +5318,28 @@ s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) return -ENOMEM; scx_cmask_init(*slot, 0, num_possible_cpus()); } + + /* pack the online mask alongside the scratch masks */ + online = scx_arena_alloc(sch, size); + if (!online) + return -ENOMEM; + + scoped_guard(rcu) { + scx_cmask_ref_init_kern(sch, online, 0, num_possible_cpus(), &ref); + scx_cmask_ref_from_cpumask(&ref, cpu_active_mask); + } + sch->online_cmask = online; + return 0; } -static void scx_set_cmask_scratch_free(struct scx_sched *sch) +static void scx_free_kern_arena_objs(struct scx_sched *sch) { size_t size = struct_size_t(struct scx_cmask, bits, SCX_CMASK_NR_WORDS(num_possible_cpus())); int cpu; + scx_arena_free(sch, sch->online_cmask, size); if (!sch->set_cmask_scratch) return; @@ -5396,7 +5426,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); free_exit_info(sch->exit_info); - scx_set_cmask_scratch_free(sch); + scx_free_kern_arena_objs(sch); scx_arena_pool_destroy(sch); if (sch->arena_map) bpf_map_put(sch->arena_map); @@ -7601,7 +7631,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) goto err_disable; } - ret = scx_set_cmask_scratch_alloc(sch); + ret = scx_alloc_kern_arena_objs(sch); if (ret) { cpus_read_unlock(); goto err_disable; @@ -10338,13 +10368,45 @@ __bpf_kfunc u32 scx_bpf_nr_cids(void) * hotplug, which lets schedulers treat [0, nr_online_cids) as the online * range. Schedulers that prefer to handle hotplug without a restart should * install a custom mapping via scx_bpf_cid_override() and track onlining - * through the ops.cid_online / ops.cid_offline callbacks. + * through the ops.cid_online / ops.cid_offline callbacks, starting from the + * mask scx_bpf_online_cmask() returns. */ __bpf_kfunc u32 scx_bpf_nr_online_cids(void) { return num_online_cpus(); } +/** + * scx_bpf_online_cmask - Return the online cid mask in the scheduler arena + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return a kernel-maintained cmask covering [0, scx_bpf_nr_cids()), or NULL if + * the calling program is not associated with a live cid-form scheduler or the + * mask is not allocated yet, as in ops.init_cids(). Treat the mask as read-only + * even though arena memory stays writable by the BPF scheduler. The mask + * follows the SCX hotplug notifications: a cid's bit is updated before + * ops.cid_online/offline() runs for it. The pointer is valid from ops.init() + * through ops.exit(). Root ops.init() runs with hotplug excluded. Other + * contexts can observe concurrent updates. + */ +__bpf_kfunc const void *scx_bpf_online_cmask(const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + struct scx_cmask *online; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return NULL; + online = sch->online_cmask; + if (unlikely(!online)) + return NULL; + + /* BPF rebases by the low 32 bits, like __arena callback args */ + return (void *)((unsigned long)online - sch->arena_kern_base); +} + /** * scx_bpf_this_cid - Return the cid of the CPU this program is running on * @@ -10708,6 +10770,7 @@ BTF_ID_FLAGS(func, scx_bpf_nr_node_ids) BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids) BTF_ID_FLAGS(func, scx_bpf_nr_cids) BTF_ID_FLAGS(func, scx_bpf_nr_online_cids) +BTF_ID_FLAGS(func, scx_bpf_online_cmask, KF_IMPLICIT_ARGS | KF_ARENA_RET) BTF_ID_FLAGS(func, scx_bpf_this_cid) BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 0adaf649d5e0..3464e0f113c1 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1561,6 +1561,7 @@ struct scx_sched { * and passes it to the callback's __arena argument. */ struct scx_cmask * __percpu *set_cmask_scratch; + struct scx_cmask *online_cmask; DECLARE_BITMAP(has_op, SCX_OPI_END); @@ -2087,7 +2088,7 @@ void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p); void scx_cgroup_lock(void); void scx_cgroup_unlock(void); #endif -s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch); +s32 scx_alloc_kern_arena_objs(struct scx_sched *sch); void scx_disable_bypass_dsp(struct scx_sched *sch); void scx_bypass(struct scx_sched *sch, bool bypass); s32 scx_link_sched(struct scx_sched *sch); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 385302d19914..f7aeb1488566 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1805,6 +1805,12 @@ void scx_sub_enable_workfn(struct kthread_work *work) goto err_disable; } + scoped_guard(cpus_read_lock) { + ret = scx_alloc_kern_arena_objs(sch); + if (ret) + goto err_disable; + } + if (sch->ops.init) { ret = SCX_CALL_OP_RET(sch, init, NULL); if (ret) { @@ -1815,10 +1821,6 @@ void scx_sub_enable_workfn(struct kthread_work *work) sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; } - ret = scx_set_cmask_scratch_alloc(sch); - if (ret) - goto err_disable; - struct scx_sub_attach_args sub_attach_args = { .ops = &sch->ops, .cgroup_path = sch->cgrp_path, -- cgit v1.2.3 From 7de9a6fb44eae4f05e68c805d58b9c618815adfa Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 15 Sep 2026 12:34:44 -1000 Subject: sched_ext: Wait for SCX_OPSS_DISPATCHING before reenqueueing a task ebf1ccff79c4 ("sched_ext: Fix ops.dequeue() semantics") moved the final ops_state store in scx_dispatch_enqueue() after the DSQ unlock so that the custody update and ops.dequeue() precede it. A task can thus be found on a DSQ while still SCX_OPSS_DISPATCHING. The dequeue and core-sched pick paths wait for the state to clear in ops_dequeue() but the reenqueue paths don't. A reenqueue in that window runs ops.enqueue() and sets SCX_OPSS_QUEUED before the dispatch has completed. The dispatcher's final store then overwrites it with SCX_OPSS_NONE and finish_dispatch() drops every later dispatch of the task. Wait for SCX_OPSS_DISPATCHING to clear before dequeueing a task for reenqueue, the same way ops_dequeue() does. Fixes: ebf1ccff79c4 ("sched_ext: Fix ops.dequeue() semantics") Cc: stable@vger.kernel.org # v7.1+ Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 13 +++++++++++++ kernel/sched/ext/internal.h | 1 + kernel/sched/ext/sub.c | 1 + 3 files changed, 15 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 70b711c4de6e..725b890fcab9 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4404,6 +4404,17 @@ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, return *reenq_flags & SCX_REENQ_ANY; } +/* + * The dispatcher stores the final ops_state after dropping the DSQ lock, so @p + * can be found on a DSQ while still %SCX_OPSS_DISPATCHING. Reenqueueing @p + * before that store lands would have it clobber the new %SCX_OPSS_QUEUED. + */ +void scx_reenq_wait_dispatching(struct task_struct *p) +{ + if (unlikely(atomic_long_read_acquire(&p->scx.ops_state) == SCX_OPSS_DISPATCHING)) + wait_ops_state(p, SCX_OPSS_DISPATCHING); +} + static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) { LIST_HEAD(tasks); @@ -4447,6 +4458,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) if (!local_task_should_reenq(rq, p, &reenq_flags, &reason)) continue; + scx_reenq_wait_dispatching(p); scx_dispatch_dequeue(rq, p); if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) @@ -4570,6 +4582,7 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag } /* @p is on @dsq, its rq and @dsq are locked */ + scx_reenq_wait_dispatching(p); dispatch_dequeue_locked(p, dsq); raw_spin_unlock(&dsq->lock); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 3464e0f113c1..115c96fbf932 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -2078,6 +2078,7 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); u64 __scx_bpf_now(struct rq *rq); void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, u64 reenq_flags, struct rq *locked_rq); +void scx_reenq_wait_dispatching(struct task_struct *p); int __scx_init_task(struct scx_sched *sch, struct task_struct *p, struct cgroup *cgrp, bool fork); void scx_enable_task(struct scx_sched *sch, struct task_struct *p); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index f7aeb1488566..a30c965b175d 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -801,6 +801,7 @@ void scx_reenq_reject(struct rq *rq) if (WARN_ON_ONCE(p->migration_pending)) continue; + scx_reenq_wait_dispatching(p); scx_dispatch_dequeue(rq, p); if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) -- cgit v1.2.3 From b61b6f95d6722ddbbbd09e689fa41b55fd36f9a5 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 11 Sep 2026 11:04:47 +0200 Subject: futex: Also allocate private hash on vfork() As Jann demonstrated, it is entirely feasible to access the mm through vfork(). Therefore we need to allocate a private hash on vfork() as well as any other CLONE_VM user. Specifically, it must be avoided to have (private) futex waiters before allocating the private hash. Fixes: ee9dce44362b ("futex: Drop CLONE_THREAD requirement for private default hash alloc") Reported-by: Jann Horn Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260911090447.GT788244@noisy.programming.kicks-ass.net --- kernel/fork.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index a5934a317634..5ef413368912 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1996,9 +1996,9 @@ static bool need_futex_hash_allocate_default(u64 clone_flags) { /* * Allocate a default futex hash for any sibling that will - * share the parent's mm, except vfork. + * share the parent's mm. */ - return (clone_flags & (CLONE_VM | CLONE_VFORK)) == CLONE_VM; + return clone_flags & CLONE_VM; } /* -- cgit v1.2.3 From d2710c8d938ae6a825a6463158e6e6f31eac792a Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 11 Sep 2026 11:09:12 +0200 Subject: signal: Prevent exec() race Hyunwoo debugged the following KASAN UAF splat: BUG: KASAN: slab-use-after-free in __send_signal_locked+0xb27/0xba0 Write of size 8 at addr ffff888007ed80c8 by task poc/79 ... Call Trace: __send_signal_locked+0xb27/0xba0 do_send_sig_info+0xa7/0x160 do_send_specific+0x76/0xa0 __x64_sys_tgkill+0x193/0x270 ... Allocated by task 80: do_timer_create+0x1a4/0x1030 __x64_sys_timer_create+0x145/0x190 ... Freed by task 12: kmem_cache_free_bulk+0x1f8/0x4a0 kvfree_rcu_bulk+0x14f/0x1c0 kfree_rcu_work+0x128/0x1a0 ... Last potentially related work creation: kvfree_call_rcu+0x39/0x390 __flush_itimer_signals+0x211/0x320 flush_itimer_signals+0x47/0x90 begin_new_exec+0xa6b/0x28c0 It turned out that this happens with a non-leader exec() as Hyunwoo explained: de_thread() calls exchange_tids() before release_task(leader), so the struct pid held by a SIGEV_THREAD_ID timer created against the leader's tid now points to the thread which called execve(). pid_task() returns that thread and lock_task_sighand() on it succeeds. If the timer signal is blocked, its sigqueue stays queued on the leader's task::pending. The next expiry of that timer can then run while release_task() flushes the queue. posixtimer_send_sigqueue() checks whether the sigqueue is already queued with a plain list_empty(), which only reads list_head::next. list_del_init() is not atomic and INIT_LIST_HEAD() stores list_head::next before list_head::prev, so the check can pass in between. list_add_tail() queues the entry on the task::pending of the live thread, and the list_head::prev store from the flush then overwrites the list_head::prev link that list_add_tail() has just set. __flush_itimer_signals() does not undo that either. With list_head::prev pointing at the entry itself, its list_del_init() only stores the same values again, so the entry is not removed from the list. It is still there after the last reference is dropped and the timer is freed by RCU, and the list_add_tail() of a later tgkill() follows that list_head::prev into the freed timer. This problem surfaced with the recent commit which moved the sigqueue flush out of the sighand lock held region. Hyonwoo proposed to fix this by using list_del_init_careful(), but that just papers over the problem. After some disucssions and various attempts to solve it, Eric pointed out that there is no reason to flush task::pending late in release_task() and it should be done in exit_signals() already. As nothing can collect and deliver signals which are queued in a dying task's pending queue, there is no reason to delay it further. But it has to be ensured that no signals can be queued into it after that point. exit_signals() sets PF_EXITING in task::flags, which can be used as an indicator for this. Cure it by: - Preventing signal queueing for task private signals (PIDTYPE_PID) when the task has PF_EXITING set in __send_signal_locked() and in posixtimer_send_sigqueue(). - Protecting the unlocked setting of PF_EXITING in exit_signals() for the task group empty and the group exit case with sighand lock - Flushing task::pending signals right there. Optimize that by moving the whole pending list to an on-stack list head under sighand lock and free the signals without the lock held. There has been quite some discussion about the lockless flush and the non-leader exec case on weakly ordered systems. The problem is that a third party which tries to send a posix timer signal relies on the PID lookup to find the target task and that lookup might result in the new leader when the signal was originaly directed to the old leader. In case that the signal was queued on the old leader then the lockless flush raised a concern over the following situation: old_leader new_leader third party A: flush_list() // list_del_init() stores to sigqueue LOCK (tasklist) old_leader->exit_state = EXIT_ZOMBIE; B: UNLOCK (tasklist) C: LOCK (tasklist) if (old_leader->exit_state) transfer_tids() D: store PID posix_timer_send_sigqueue() // Observes #D so t = new_leader E: t = get_target() F: LOCK (sighand) G: if (list_empty(sigqueue)) list_add(sigqueue) The concern was that the third party might observe #D but not observe #A and therefore would proceed to #G while the list_del() stores (#A) in flush_list() are not visible yet, which could result in list corruption. That would be possible if looking at it solely from a RELEASE+ACQUIRE ordering point of view, but B-C is a UNLOCK+LOCK hand-over, which is not the same as RELEASE+ACQUIRE: RELEASE+ACQUIRE: RCpc, only the CPUs involved agree on the ordering UNLOCK+LOCK: RCtso, the hand-over is store-ordering As B-C is UNLOCK+LOCK, which is RCtso and that does impose store order, A stores must happen before the D store. Combine with E-F, which has a data dependency from the LOAD to the LOCK and thereby constraints later LOADs, those sigqueue loads in G that come after F must in fact observe the A stores. Fixes: fb3bbcfe344e ("exit: change the release_task() paths to call flush_sigqueue() lockless") Reported-by: Hyunwoo Kim Debugged-by: Hyunwoo Kim Suggested-by: "Eric W. Biederman" Signed-off-by: Thomas Gleixner Tested-by: Kijo Park Reviewed-by: Oleg Nesterov Reviewed-by: Frederic Weisbecker Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260911090541.572536604@kernel.org Closes: https://patch.msgid.link/aok1rdkBgZsynHZB@v4bel --- kernel/exit.c | 11 +++--- kernel/signal.c | 115 +++++++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 91 insertions(+), 35 deletions(-) (limited to 'kernel') diff --git a/kernel/exit.c b/kernel/exit.c index 4e028f157597..424c44a42a4d 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -302,12 +302,13 @@ repeat: free_pids(post.pids); release_thread(p); /* - * This task was already removed from the process/thread/pid lists - * and lock_task_sighand(p) can't succeed. Nobody else can touch - * ->pending or, if group dead, signal->shared_pending. We can call - * flush_sigqueue() lockless. + * This task was already removed from the process/thread/pid lists and + * lock_task_sighand(p) can't succeed. If it's the group leader then + * flush tsk->signal->shared_pending. tsk->pending has been flushed + * already in exit_signals(). Nothing else can touch + * signal->shared_pending anymore, so flush_sigqueue() can be invoked + * lockless. */ - flush_sigqueue(&p->pending); if (thread_group_leader(p)) flush_sigqueue(&p->signal->shared_pending); diff --git a/kernel/signal.c b/kernel/signal.c index ec30550951ec..d31ebcb6ed4d 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -457,18 +457,42 @@ static void __sigqueue_free(struct sigqueue *q) kmem_cache_free(sigqueue_cachep, q); } -void flush_sigqueue(struct sigpending *queue) +/* + * flush_sigqueue_list() can only be invoked without holding sighand::siglock in + * the following cases: + * + * 1) When flushing task::pending _after_ setting task::flags PF_EXITING + * + * All functions which try to send a signal to @task will observe PF_EXITING + * and drop the signal. + * + * 2) When flushing task::signal::shared_pending _after_ the last task in a + * thread group was unhashed and task::sighand is NULL. + * + * Nothing can queue a signal anymore because sighand is NULL. + */ +static void flush_sigqueue_list(struct list_head *head) { - struct sigqueue *q; + struct sigqueue *q, *tmp; - sigemptyset(&queue->signal); - while (!list_empty(&queue->list)) { - q = list_entry(queue->list.next, struct sigqueue , list); + list_for_each_entry_safe(q, tmp, head, list) { list_del_init(&q->list); __sigqueue_free(q); } } +void flush_sigqueue(struct sigpending *queue) +{ + sigemptyset(&queue->signal); + flush_sigqueue_list(&queue->list); +} + +static void sigqueue_dequeue_pending(struct sigpending *queue, struct list_head *head) +{ + sigemptyset(&queue->signal); + list_splice_init(&queue->list, head); +} + /* * Flush all pending signals for this kthread. */ @@ -1019,6 +1043,21 @@ static inline bool legacy_queue(struct sigpending *signals, int sig) return (sig < SIGRTMIN) && sigismember(&signals->signal, sig); } +/* + * When PF_EXITING is set the task is on the way out and has t::pending + * flushed already. Prevent queueing of PIDTYPE_PID signals as they would + * be leaked. + */ +static inline bool task_can_queue_signal(struct task_struct *t, enum pid_type type) +{ + lockdep_assert_held(&t->sighand->siglock); + + if (!(t->flags & PF_EXITING)) + return true; + + return type != PIDTYPE_PID; +} + static int __send_signal_locked(int sig, struct kernel_siginfo *info, struct task_struct *t, enum pid_type type, bool force) { @@ -1030,6 +1069,10 @@ static int __send_signal_locked(int sig, struct kernel_siginfo *info, lockdep_assert_held(&t->sighand->siglock); result = TRACE_SIGNAL_IGNORED; + + if (!task_can_queue_signal(t, type)) + goto ret; + if (!prepare_signal(sig, t, force)) goto ret; @@ -1980,11 +2023,25 @@ static inline struct task_struct *posixtimer_get_target(struct k_itimer *tmr) struct task_struct *t = pid_task(tmr->it_pid, tmr->it_pid_type); if (t && tmr->it_pid_type != PIDTYPE_PID && - same_thread_group(t, current) && !current->exit_state) + same_thread_group(t, current) && !(current->flags & PF_EXITING)) t = current; return t; } +/* + * Find the target task for the POSIX timer signal and prevent that a + * PIDTYPE_PID signal is queued on a task which has PF_EXITING set. + */ +static inline struct task_struct *posixtimer_get_unignore_target(struct k_itimer *tmr) +{ + struct task_struct *t = posixtimer_get_target(tmr); + + if (t && task_can_queue_signal(t, tmr->it_pid_type)) + return t; + + return NULL; +} + void posixtimer_send_sigqueue(struct k_itimer *tmr) { struct sigqueue *q = &tmr->sigq; @@ -2002,6 +2059,9 @@ void posixtimer_send_sigqueue(struct k_itimer *tmr) if (!likely(lock_task_sighand(t, &flags))) return; + if (!task_can_queue_signal(t, tmr->it_pid_type)) + goto unlock; + /* * Update @tmr::sigqueue_seq for posix timer signals with sighand * locked to prevent a race against dequeue_signal(). @@ -2093,6 +2153,7 @@ void posixtimer_send_sigqueue(struct k_itimer *tmr) result = TRACE_SIGNAL_DELIVERED; out: trace_signal_generate(sig, &q->info, t, tmr->it_pid_type != PIDTYPE_PID, result); +unlock: unlock_task_sighand(t, &flags); } @@ -2148,7 +2209,7 @@ static void posixtimer_sig_unignore(struct task_struct *tsk, int sig) * has exited by now, drop the reference count. */ guard(rcu)(); - target = posixtimer_get_target(tmr); + target = posixtimer_get_unignore_target(tmr); if (target) posixtimer_queue_sigqueue(&tmr->sigq, target, tmr->it_pid_type); else @@ -3132,42 +3193,36 @@ static void retarget_shared_pending(struct task_struct *tsk, sigset_t *which) void exit_signals(struct task_struct *tsk) { + LIST_HEAD(sigq_list); int group_stop = 0; - sigset_t unblocked; /* * @tsk is about to have PF_EXITING set - lock out users which - * expect stable threadgroup. + * expect a stable threadgroup. */ cgroup_threadgroup_change_begin(tsk); - if (thread_group_empty(tsk) || (tsk->signal->flags & SIGNAL_GROUP_EXIT)) { + scoped_guard(spinlock_irq, &tsk->sighand->siglock) { tsk->flags |= PF_EXITING; - cgroup_threadgroup_change_end(tsk); - return; - } - spin_lock_irq(&tsk->sighand->siglock); - /* - * From now this task is not visible for group-wide signals, - * see wants_signal(), do_signal_stop(). - */ - tsk->flags |= PF_EXITING; + sigqueue_dequeue_pending(&tsk->pending, &sigq_list); - cgroup_threadgroup_change_end(tsk); + if (task_sigpending(tsk) && !thread_group_empty(tsk) && + !(tsk->signal->flags & SIGNAL_GROUP_EXIT)) { + sigset_t unblocked = tsk->blocked; - if (!task_sigpending(tsk)) - goto out; + signotset(&unblocked); + retarget_shared_pending(tsk, &unblocked); - unblocked = tsk->blocked; - signotset(&unblocked); - retarget_shared_pending(tsk, &unblocked); + if (unlikely(tsk->jobctl & JOBCTL_STOP_PENDING) && + task_participate_group_stop(tsk)) + group_stop = CLD_STOPPED; + } + } - if (unlikely(tsk->jobctl & JOBCTL_STOP_PENDING) && - task_participate_group_stop(tsk)) - group_stop = CLD_STOPPED; -out: - spin_unlock_irq(&tsk->sighand->siglock); + cgroup_threadgroup_change_end(tsk); + + flush_sigqueue_list(&sigq_list); /* * If group stop has completed, deliver the notification. This -- cgit v1.2.3 From df5cdc2c832ca4e8a6d774596b9005558761a403 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 16 Sep 2026 12:00:24 -1000 Subject: sched_ext: Derive SCX_RQ_IN_WAKEUP from the core enqueue flags schedule_deferred_locked() skips scheduling a deferred action while SCX_RQ_IN_WAKEUP is set and relies on the task_woken_scx() call that follows a wakeup enqueue to run it. enqueue_task_scx() sets the flag from the merged enqueue flags, which include the flags stashed for a remote activation. move_remote_task_to_local_dsq() thus sets SCX_RQ_IN_WAKEUP on the destination rq when the moved task was woken up, although no task_woken_scx() follows that activation. An IMMED insert into a busy destination requests a local reenqueue during that enqueue. The request gets linked but not scheduled and stays pending until an unrelated wakeup or preemption on that CPU runs the deferred actions. The IMMED task sits behind the running task in the meantime. If nothing runs them before the scheduler is disabled, the request outlives the scheduler and points into its freed per-cpu area, which the next scheduler dereferences from run_deferred(). Test the core enqueue flags for the wakeup bit. Only the core's wakeup path is followed by task_woken_scx(). Fixes: 57ccf5ccdc56 ("sched_ext: Fix enqueue_task_scx() truncation of upper enqueue flags") Cc: stable@vger.kernel.org # v7.1+ Reported-by: Andrea Righi Link: https://lore.kernel.org/all/20260916145807.3250167-1-arighi@nvidia.com/ Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 725b890fcab9..e207ccd17164 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2141,7 +2141,12 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ int sticky_cpu = p->scx.sticky_cpu; u64 enq_flags = core_enq_flags | rq->scx.remote_activate_enq_flags; - if (enq_flags & ENQUEUE_WAKEUP) + /* + * SCX_RQ_IN_WAKEUP promises a task_woken_scx() call once this enqueue + * returns. Only the core's wakeup path delivers one. The flags stashed + * for a remote activation may carry the wakeup bit without it. + */ + if (core_enq_flags & ENQUEUE_WAKEUP) rq->scx.flags |= SCX_RQ_IN_WAKEUP; /* -- cgit v1.2.3 From cb86607ada73185f321baa7f9d94a08a3a40bbf5 Mon Sep 17 00:00:00 2001 From: fangqiurong Date: Thu, 17 Sep 2026 15:52:41 +0800 Subject: sched_ext: Don't run ops.dequeue() with a DSQ lock held ops.dequeue() is invoked with the source user DSQ's lock still held on the consume and move paths (scx_consume_dispatch_q(), move_task_between_dsqs()). A BPF scheduler which locks the source user DSQ from ops.dequeue() - e.g. by iterating it with bpf_iter_scx_dsq - self-deadlocks. ops.dequeue() can only call the "any" kfuncs and none of them can lock a builtin DSQ, so the global and bypass paths can't deadlock; however, all DSQ locks share one lockdep class, so iterating any user DSQ from ops.dequeue() on those paths trips the recursion check. Move the invocation after the DSQ unlock on all three paths. SCX_TASK_IN_CUSTODY is cleared under the lock serializing the transfer so that the callback is invoked exactly once. Fixes: ebf1ccff79c4 ("sched_ext: Fix ops.dequeue() semantics") Cc: stable@vger.kernel.org # v7.1+ Acked-by: Andrea Righi Signed-off-by: fangqiurong Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 44 ++++++++++++++++++++++---------------------- kernel/sched/ext/internal.h | 3 +-- kernel/sched/ext/sub.c | 10 +++++----- 3 files changed, 28 insertions(+), 29 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index e207ccd17164..f568fd9973f6 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1499,27 +1499,22 @@ static inline bool task_scx_migrating(struct task_struct *p) return p->scx.sticky_cpu >= 0; } -/* - * Call ops.dequeue() if the task is in BPF custody and not migrating. - * Clears %SCX_TASK_IN_CUSTODY when the callback is invoked. - */ -static void call_task_dequeue(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 deq_flags) +/* Must be called under the lock serializing @p's custody transfers. */ +static bool task_leave_custody(struct task_struct *p) { if (!(p->scx.flags & SCX_TASK_IN_CUSTODY) || task_scx_migrating(p)) - return; - - if (SCX_HAS_OP(sch, dequeue)) - SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags); + return false; p->scx.flags &= ~SCX_TASK_IN_CUSTODY; + return true; } static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) { - call_task_dequeue(sch, rq, p, 0); + if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue)) + SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0); /* * Only local inserts get the wakeup treatment below. Rejects kick the @@ -1705,20 +1700,28 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, if (is_rq_owned) { rq_owned_post_enq(sch, rq, dsq, p, enq_flags); } else { + bool call_dequeue = false; + /* * Global and bypass DSQs are terminal - the task leaves the - * scheduler's custody, so ops.dequeue() fires here. It can run + * scheduler's custody, so ops.dequeue() fires. It can run * without @p's rq lock (finish_dispatch() passes the dispatch * rq); that's safe because dequeue_task_scx() waits on * SCX_OPSS_DISPATCHING (see the ops_state note above) and so * can't race it. A non-terminal DSQ keeps the task in custody. + * The custody transfer happens under @dsq->lock so that later + * consumers see the flag clear; the callback runs after + * @dsq->lock is dropped because it may lock a DSQ itself. */ if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS) - call_task_dequeue(sch, rq, p, 0); + call_dequeue = task_leave_custody(p); else p->scx.flags |= SCX_TASK_IN_CUSTODY; raw_spin_unlock(&dsq->lock); + + if (call_dequeue && SCX_HAS_OP(sch, dequeue)) + SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0); } /* @@ -2215,7 +2218,7 @@ retry: /* * A queued task must always be in BPF scheduler's custody. If * SCX_TASK_IN_CUSTODY is clear, finish_dispatch() on another - * CPU has already passed call_task_dequeue() (which clears the + * CPU has already passed task_leave_custody() (which clears the * flag), but has not yet written SCX_OPSS_NONE. That final * store does not require this rq's lock, so retrying with * cpu_relax() is bounded: we will observe NONE (or DISPATCHING, @@ -2263,7 +2266,8 @@ retry: * NONE but the task may still have %SCX_TASK_IN_CUSTODY set until * it is enqueued on the destination. */ - call_task_dequeue(sch, rq, p, deq_flags); + if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue)) + SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags); } static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_flags) @@ -2379,14 +2383,10 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl } void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p, - u64 enq_flags, struct scx_dispatch_q *src_dsq, - struct rq *dst_rq) + u64 enq_flags, struct rq *dst_rq) { struct scx_dispatch_q *dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags); - /* @p is on @dst_rq, an rq-owned @src_dsq is covered by the rq lock */ - if (!dsq_is_rq_owned(src_dsq)) - lockdep_assert_held(&src_dsq->lock); lockdep_assert_rq_held(dst_rq); WARN_ON_ONCE(p->scx.holding_cpu >= 0); @@ -2634,8 +2634,8 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, /* @p is going from a non-local DSQ to a local DSQ */ if (src_rq == dst_rq) { scx_task_unlink_from_dsq(p, src_dsq); - scx_move_local_task_to_local_dsq(sch, p, enq_flags, src_dsq, dst_rq); raw_spin_unlock(&src_dsq->lock); + scx_move_local_task_to_local_dsq(sch, p, enq_flags, dst_rq); } else { raw_spin_unlock(&src_dsq->lock); move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); @@ -2685,8 +2685,8 @@ retry: if (rq == task_rq) { scx_task_unlink_from_dsq(p, dsq); - scx_move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq); raw_spin_unlock(&dsq->lock); + scx_move_local_task_to_local_dsq(sch, p, enq_flags, rq); return true; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 115c96fbf932..d150de10a5c9 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -2065,8 +2065,7 @@ void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p); void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, int sticky_cpu); void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p, - u64 enq_flags, struct scx_dispatch_q *src_dsq, - struct rq *dst_rq); + u64 enq_flags, struct rq *dst_rq); bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, u64 enq_flags); bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index a30c965b175d..3036190b0662 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -555,8 +555,8 @@ static void scx_rescue_timerfn(struct timer_list *timer) scx.dsq_list.node); scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq); scx_rescue_admit(rq, p, slice); - scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, - &rq->scx.rescue.dsq, rq); + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, + SCX_ENQ_IGNORE_CAPS, rq); if (sched_class_above(&ext_sched_class, rq->curr->sched_class)) resched_curr(rq); } else if (p->scx.dsq && rq->scx.rescue.budget > 2 * scx_rescue_quantum_ns) { @@ -572,7 +572,7 @@ static void scx_rescue_timerfn(struct timer_list *timer) scx_task_unlink_from_dsq(p, &rq->scx.local_dsq); scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_IGNORE_CAPS, - &rq->scx.local_dsq, rq); + rq); } out_arm: scx_rescue_timer_arm(rq); @@ -596,8 +596,8 @@ void scx_rescue_flush(struct rq *rq) /* and flush out all pending ones */ list_for_each_entry_safe(p, n, &rq->scx.rescue.dsq.list, scx.dsq_list.node) { scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq); - scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, - &rq->scx.rescue.dsq, rq); + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, + SCX_ENQ_IGNORE_CAPS, rq); } timer_delete(&rq->scx.rescue.timer); -- cgit v1.2.3 From 40c2096961b4e8f48d1fc17406a90c5a36ac0a8d Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 14 Sep 2026 15:19:20 +0200 Subject: bpf: Verify global subprogs in each sleepability context Global subprograms are verified independently with a fresh verifier root. do_check_common() currently seeds that root's in_sleepable state from the program, even though a global subprogram can also run from callbacks whose execution context differs from the program's main entry point. In particular, workqueue and task-work callbacks are sleepable even when the containing program is not. A global subprogram of that program is therefore verified as non-sleepable, making in_rcu_cs() true and allowing loads of RCU-protected kptrs to produce trusted MEM_RCU pointers. The same subprogram can then be called from a sleepable callback without a classic RCU reader. It can retain such a pointer while the object is freed and use it after free. The verifier's execution-context predicates are complementary. A state is sleepable only when in_sleepable is set and no RCU, preemption, IRQ, or lock region is active. Each condition which prevents sleeping also provides RCU protection, while in_rcu_cs() treats a non-sleepable state as implicitly protected. Use this relationship to represent a global subprogram caller with only the result of in_sleepable_context(). A protected sleepable caller is normalized to in_sleepable=false at the independent verification root. This both prevents sleepable operations and makes in_rcu_cs() true without copying caller-owned lock state. Track only the contexts in which each global subprogram is actually reached. Verify it once if all reachable calls use the same context, and twice only if both sleepable and non-sleepable calls reach it. Calls found while verifying globals or asynchronous callbacks mark further contexts for checking. Repeat the existing subprogram walk until all called contexts have been verified; unreachable global calls remain unchecked. Accumulate instruction counts over those verification passes. Preserve the total recorded before each pass, since path accounting has already added this pass's synchronous instructions and its root total must also include asynchronous subprograms. This makes an unprotected callback verify the global subprogram as sleepable, turning its RCU-protected kptr load into an untrusted pointer. Protected callers and global subprograms which do not depend on implicit RCU protection remain valid. Fixes: 81f1d7a583fa ("bpf: wq: add bpf_wq_set_callback_impl") Fixes: 38aa7003e369 ("bpf: task work scheduling kfuncs") Reported-by: Nicholas Carlini Suggested-by: Nicholas Carlini Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260914131923.2544250-2-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/verifier.c | 71 +++++++++++++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 34 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ddba53eaa333..5d7080c260d8 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -9931,6 +9931,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err == -EFAULT) return err; if (bpf_subprog_is_global(env, subprog)) { + struct bpf_func_info_aux *sub_aux = subprog_aux(env, subprog); const char *sub_name = bpf_subprog_name(env, subprog); const char *operation; bool returns_void; @@ -9962,11 +9963,10 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (env->log.level & BPF_LOG_LEVEL) verbose(env, "Func#%d ('%s') is global and assumed valid.\n", subprog, sub_name); + sub_aux->called[in_sleepable_context(env)] = true; returns_void = subprog_returns_void(env, subprog); if (env->subprog_info[subprog].changes_pkt_data) clear_all_pkt_pointers(env); - /* mark global subprog for verifying after main prog */ - subprog_aux(env, subprog)->called = true; if (returns_void) bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); else @@ -10784,11 +10784,7 @@ int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, /* Check if we're in a sleepable context. */ static inline bool in_sleepable_context(struct bpf_verifier_env *env) { - return !env->cur_state->active_rcu_locks && - !env->cur_state->active_preempt_locks && - !env->cur_state->active_locks && - !env->cur_state->active_irq_id && - in_sleepable(env); + return !in_rcu_cs(env); } static const char *non_sleepable_context_description(struct bpf_verifier_env *env) @@ -19447,13 +19443,14 @@ static void free_states(struct bpf_verifier_env *env) } } -static int do_check_common(struct bpf_verifier_env *env, int subprog) +static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_sleepable) { bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); struct bpf_subprog_info *sub = subprog_info(env, subprog); struct bpf_prog_aux *aux = env->prog->aux; struct bpf_verifier_state *state; struct bpf_reg_state *regs; + u32 old_insns_total = sub->insns_total; u32 insn_processed = env->insn_processed; int ret, i; @@ -19466,7 +19463,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) state->curframe = 0; state->speculative = false; state->branches = 1; - state->in_sleepable = env->prog->sleepable; + state->in_sleepable = is_sleepable; state->frame[0] = kzalloc_obj(struct bpf_func_state, GFP_KERNEL_ACCOUNT); if (!state->frame[0]) { kfree(state); @@ -19607,8 +19604,10 @@ out: * not accounted as callees by account_current_path(). * Accumulate their total counts as total counts of the main or * global subprog hosting the async call. + * Start from the saved total of earlier contexts: adding to the current + * total would count this pass's synchronous paths twice. */ - env->subprog_info[subprog].insns_total = env->insn_processed - insn_processed; + sub->insns_total = old_insns_total + (env->insn_processed - insn_processed); return ret; } @@ -19636,14 +19635,19 @@ static int do_check_subprogs(struct bpf_verifier_env *env) { struct bpf_prog_aux *aux = env->prog->aux; struct bpf_func_info_aux *sub_aux; - int i, ret, new_cnt; + int context, i, ret, new_cnt; if (!aux->func_info) return 0; - /* exception callback is presumed to be always called */ - if (env->exception_callback_subprog) - subprog_aux(env, env->exception_callback_subprog)->called = true; + /* + * Callbacks cannot throw, so the exception callback always runs in the + * main program's context. It is presumed to be always called. + */ + if (env->exception_callback_subprog) { + sub_aux = subprog_aux(env, env->exception_callback_subprog); + sub_aux->called[env->prog->sleepable] = true; + } again: new_cnt = 0; @@ -19652,29 +19656,28 @@ again: continue; sub_aux = subprog_aux(env, i); - if (!sub_aux->called || sub_aux->verified) - continue; + for (context = 0; context < ARRAY_SIZE(sub_aux->called); context++) { + if (!sub_aux->called[context] || sub_aux->verified[context]) + continue; - env->insn_idx = env->subprog_info[i].start; - WARN_ON_ONCE(env->insn_idx == 0); - ret = do_check_common(env, i); - if (ret) { - return ret; - } else if (env->log.level & BPF_LOG_LEVEL) { - verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n", - i, bpf_subprog_name(env, i)); - } + env->insn_idx = env->subprog_info[i].start; + WARN_ON_ONCE(env->insn_idx == 0); + ret = do_check_common(env, i, context); + if (ret) + return ret; + if (env->log.level & BPF_LOG_LEVEL) + verbose(env, "Func#%d ('%s') is safe for any args " + "that match its prototype\n", + i, bpf_subprog_name(env, i)); - /* We verified new global subprog, it might have called some - * more global subprogs that we haven't verified yet, so we - * need to do another pass over subprogs to verify those. - */ - sub_aux->verified = true; - new_cnt++; + sub_aux->verified[context] = true; + new_cnt++; + } } - /* We can't loop forever as we verify at least one global subprog on - * each pass. + /* + * We can't loop forever as each pass verifies at least one new context, + * and there are only two contexts per global subprog. */ if (new_cnt) goto again; @@ -19687,7 +19690,7 @@ static int do_check_main(struct bpf_verifier_env *env) int ret; env->insn_idx = 0; - ret = do_check_common(env, 0); + ret = do_check_common(env, 0, env->prog->sleepable); if (!ret) env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; return ret; -- cgit v1.2.3 From 50e80e2bb5e2be8515205b9c496b9640ddefa434 Mon Sep 17 00:00:00 2001 From: Weiming Shi Date: Tue, 15 Sep 2026 01:02:07 +0800 Subject: bpf: Skip unsettled links in link iterator bpf_link_prime() inserts a link into link_idr before anon_inode_getfile() succeeds and before bpf_link_settle() publishes the ID in link->id. bpf_link_by_id() treats such an ID-zero link as unsettled, but the link iterator takes a reference without this check. If anon_inode_getfile() then fails, the creator removes the ID and frees its still-private link directly. The iterator is left with a dangling reference and its next bpf_link_put() accesses freed memory. Treat ID-zero entries as transient in bpf_link_get_curr_or_next(), just as bpf_link_by_id() does. BUG: KASAN: slab-use-after-free in bpf_link_put Write of size 8 by task exp/384 Call Trace: bpf_link_put kernel/bpf/syscall.c:3372 bpf_link_seq_next kernel/bpf/link_iter.c:33 bpf_seq_read kernel/bpf/bpf_iter.c:158 vfs_read fs/read_write.c:572 ksys_read fs/read_write.c:716 do_syscall_64 arch/x86/entry/syscall_64.c:84 entry_SYSCALL_64_after_hwframe arch/x86/entry/entry_64.S:121 Kernel panic - not syncing: KASAN: panic_on_warn set ... Fixes: 9f8836127308 ("bpf: Add bpf_link iterator") Reported-by: Xiang Mei Signed-off-by: Weiming Shi Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260914170206.170723-2-bestswngs@gmail.com --- kernel/bpf/syscall.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 853b47f81384..244a939b9d2d 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -6042,7 +6042,10 @@ struct bpf_link *bpf_link_get_curr_or_next(u32 *id) again: link = idr_get_next(&link_idr, id); if (link) { - link = bpf_link_inc_not_zero(link); + if (link->id) + link = bpf_link_inc_not_zero(link); + else + link = ERR_PTR(-EAGAIN); if (IS_ERR(link)) { (*id)++; goto again; -- cgit v1.2.3 From 1d653a183973f5283a3db5a38cd5e195eb152244 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Thu, 17 Sep 2026 22:24:07 +0100 Subject: fprobe: Terminate the fgraph_data list when the reservation is not filled fprobe_fgraph_entry() reserves shadow stack space for every fprobe with an exit handler, but only fills it for those whose entry handler returns 0. fgraph_reserve_data() does not clear the area, so fprobe_return() parses the unused tail as headers left over from an earlier call, and an exit handler can run twice or despite its entry handler asking to skip it. Write a zero word after the last entry to terminate the walk. A zeroed slot does not decode to a NULL fprobe on the arches that encode the header into one unsigned long, since arch_decode_fprobe_header_fp() ORs in FPROBE_HEADER_MSB_PATTERN, so make read_fprobe_header() return NULL for a zeroed slot. Link: https://lore.kernel.org/all/20260917212407.384468-1-devnexen@gmail.com/ Fixes: e0a384434ae1 ("tracing: fprobe: do not zero out unused fgraph_data") Cc: stable@vger.kernel.org Suggested-by: Masami Hiramatsu (Google) Signed-off-by: David Carlier Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/fprobe.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'kernel') diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index 1e9b00997ff2..9f2d98181779 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -171,6 +171,11 @@ static inline bool write_fprobe_header(unsigned long *stack, static inline void read_fprobe_header(unsigned long *stack, struct fprobe **fp, unsigned int *size_words) { + if (!*stack) { + *fp = NULL; + *size_words = 0; + return; + } *fp = arch_decode_fprobe_header_fp(*stack); *size_words = arch_decode_fprobe_header_size(*stack); } @@ -203,6 +208,12 @@ static inline void read_fprobe_header(unsigned long *stack, { struct __fprobe_header *fph = (struct __fprobe_header *)stack; + if (!*stack) { + *fp = NULL; + *size_words = 0; + return; + } + *fp = fph->fp; *size_words = fph->size_words; } @@ -635,6 +646,10 @@ static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops } } + /* Terminate the list, fgraph_reserve_data() does not clear it. */ + if (used && used < reserved_words) + fgraph_data[used] = 0; + /* If any exit_handler is set, data must be used. */ return used != 0; } -- cgit v1.2.3 From 261b61d3735b042ae25634f795c4540be0fc140c Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Fri, 18 Sep 2026 01:32:09 +0200 Subject: bpf: Make post-verification instruction rewrites killable After do_check() returns, the verifier runs several instruction rewrite passes. Some of them patch or remove one instruction at a time. Each operation moves the remaining instruction and auxiliary-data arrays and adjusts all branch offsets, making the overall work quadratic in the program length. A privileged loader can submit 131072 unconditional jumps by zero followed by a valid return. Verification finishes quickly, but bpf_opt_remove_nops() then spends a long time removing each jump separately. Since this post-verification work neither checks for signals nor reschedules, a pending SIGKILL cannot terminate the task until the rewrite finishes. Make bpf_patch_insn_data() and verifier_remove_insns() common cancellation and rescheduling points. These helpers run from BPF_PROG_LOAD process context, and bpf_patch_insn_data() can already sleep while reallocating auxiliary data. Report interrupted constant blinding as -EINTR and propagate it through both JIT paths, including kernels that permit interpreter fallback. Other blinding failures retain the existing fallback behavior. This does not reduce the quadratic cost of the rewrite passes, but it makes the work preemptible and allows a killed loader to be torn down promptly. Fixes: 52875a04f4b2 ("bpf: verifier: remove dead code") Reported-by: Nicholas Carlini Suggested-by: Nicholas Carlini Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260917233222.2542500-2-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/core.c | 21 +++++++++++++++++---- kernel/bpf/fixups.c | 24 ++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 8b294dfc1ad4..2e3bf8113ae9 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -1619,6 +1620,8 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp * fix it up here on error. */ bpf_jit_prog_release_other(prog, clone); + if (env && fatal_signal_pending(current)) + return ERR_PTR(-EINTR); return IS_ERR(tmp) ? tmp : ERR_PTR(-ENOMEM); } @@ -2636,11 +2639,14 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc orig_prog = prog; prog = bpf_jit_blind_constants(env, prog); /* - * If blinding was requested and we failed during blinding, we must fall - * back to the interpreter. + * Fall back to the interpreter after blinding failures, except when + * the loader was killed. */ - if (IS_ERR(prog)) + if (IS_ERR(prog)) { + if (PTR_ERR(prog) == -EINTR) + return prog; goto out_restore; + } prog = bpf_int_jit_compile(env, prog); if (prog->jited) { @@ -2659,6 +2665,8 @@ out_restore: struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp, int *err) { + struct bpf_prog *jit_prog; + /* In case of BPF to BPF calls, verifier did all the prep * work with regards to JITing, etc. */ @@ -2681,7 +2689,12 @@ struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct if (*err) return fp; - fp = bpf_prog_jit_compile(env, fp); + jit_prog = bpf_prog_jit_compile(env, fp); + if (IS_ERR(jit_prog)) { + *err = PTR_ERR(jit_prog); + return fp; + } + fp = jit_prog; bpf_prog_jit_attempt_done(fp); if (!fp->jited && jit_needed) { *err = -ENOTSUPP; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 52d3cec33672..d6f83521fc78 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "disasm.h" @@ -306,12 +307,28 @@ static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) } } +/* + * Some post-verification instruction rewriting passes require an + * O(prog->len) operation per instruction. Keep their shared primitives + * killable and preemptible. + */ +static bool bpf_rewrite_must_abort(void) +{ + if (fatal_signal_pending(current)) + return true; + cond_resched(); + return false; +} + struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, const struct bpf_insn *patch, u32 len) { struct bpf_prog *new_prog; struct bpf_insn_aux_data *new_data = NULL; + if (bpf_rewrite_must_abort()) + return NULL; + if (len > 1) { new_data = vrealloc(env->insn_aux_data, array_size(env->prog->len + len - 1, @@ -523,6 +540,9 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) unsigned int orig_prog_len = env->prog->len; int err; + if (bpf_rewrite_must_abort()) + return -EINTR; + if (bpf_prog_is_offloaded(env->prog->aux)) bpf_prog_offload_remove_insns(env, off, cnt); @@ -1356,7 +1376,7 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) } prog = bpf_jit_blind_constants(env, prog); if (IS_ERR(prog)) { - err = -ENOMEM; + err = PTR_ERR(prog); prog = orig_prog; goto out_restore; } @@ -1433,7 +1453,7 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) err = bpf_jit_subprogs(env); if (err == 0) return 0; - if (err == -EFAULT) + if (err == -EFAULT || err == -EINTR) return err; } #ifndef CONFIG_BPF_JIT_ALWAYS_ON -- cgit v1.2.3 From fd16449a9b3b31a8f18944c2f0e29e4e218ca2cf Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Fri, 18 Sep 2026 01:32:10 +0200 Subject: bpf: Preserve packet pointer class displacement in regsafe() regsafe() maps packet pointer IDs between states and checks that each current register range is a subset of the corresponding explored register range. It does not, however, preserve the displacement between registers that share a packet pointer ID. This is unsound because packet range is shared by ID. A bounds check on one class member updates every member, and a later access can consume the range through another member. Commit 022ac0750883 ("bpf: use reg->var_off instead of reg->off for pointers") folded the fixed pointer offset into r64 and removed the old off equality check, so two individually narrower registers can prune even when their displacement has changed. The explored path can then license an out-of-bounds packet access on the pruned path. Require matching range bases for packet pointers with an ID. Together with the existing ID mapping, this preserves the displacement between members of each packet-pointer class without adding per-ID state. Packet pointers without an ID remain unaffected. Fixes: 022ac0750883 ("bpf: use reg->var_off instead of reg->off for pointers") Reported-by: Nicholas Carlini Suggested-by: Nicholas Carlini Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260917233222.2542500-3-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/states.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 66fb11b6c6a7..6c88ad95b63b 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -635,6 +635,9 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, /* id relations must be preserved */ if (!check_ids(rold->id, rcur->id, idmap)) return false; + /* Preserve displacements between pointers sharing an ID. */ + if (rold->id && rold->r64.base != rcur->r64.base) + return false; /* new val must satisfy old val knowledge */ return range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off); -- cgit v1.2.3 From c26e97721b172163042b98572fead234797f2c3c Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Fri, 18 Sep 2026 01:32:12 +0200 Subject: bpf: Apply CO-RE relocations before subprogram validation check_subprogs() verifies that each subprogram ends in an exit or an unconditional jump before in-kernel CO-RE relocations are applied. An unresolved relocation can then replace that terminal instruction with an invalid helper call. The resulting fall-through into another subprogram breaks the CFG invariant used by postorder and stack liveness analysis, which can write past their per-subprogram arrays. Apply CO-RE relocations immediately after preparing the program BTF, before subprogram discovery and validation. Keep func_info and line_info validation after subprogram discovery because those records depend on the complete subprogram layout. Reject an ldimm64 first slot at the end of the instruction stream before CO-RE can inspect its missing second slot. check_subprogs() previously rejected this form before relocation processing because it is not a valid subprogram terminator. Moving CO-RE ahead of check_subprogs() removes that implicit protection, so perform an explicit check before applying relocations. Include core_relo_cnt when deciding whether to prepare program BTF. A load that supplied only CO-RE relocation metadata previously skipped both BTF setup and relocation processing. Fixes: fbd94c7afcf9 ("bpf: Pass a set of bpf_core_relo-s to prog_load command.") Suggested-by: Andrii Nakryiko Suggested-by: Alexei Starovoitov Suggested-by: Eduard Zingerman Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260917233222.2542500-5-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/check_btf.c | 12 ++++-------- kernel/bpf/verifier.c | 12 +++++++++++- 2 files changed, 15 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/check_btf.c b/kernel/bpf/check_btf.c index 0e8b3ccc7a5b..4c1ed842f661 100644 --- a/kernel/bpf/check_btf.c +++ b/kernel/bpf/check_btf.c @@ -338,9 +338,9 @@ err_free: #define MIN_CORE_RELO_SIZE sizeof(struct bpf_core_relo) #define MAX_CORE_RELO_SIZE MAX_FUNCINFO_REC_SIZE -static int check_core_relo(struct bpf_verifier_env *env, - const union bpf_attr *attr, - bpfptr_t uattr) +int bpf_check_core_relo(struct bpf_verifier_env *env, + const union bpf_attr *attr, + bpfptr_t uattr) { u32 i, nr_core_relo, ncopy, expected_size, rec_size; struct bpf_core_relo core_relo = {}; @@ -414,7 +414,7 @@ int bpf_prepare_btf_info(struct bpf_verifier_env *env, struct btf *btf; int err; - if (!attr->func_info_cnt && !attr->line_info_cnt) { + if (!attr->func_info_cnt && !attr->line_info_cnt && !attr->core_relo_cnt) { if (check_abnormal_return(env)) return -EINVAL; return 0; @@ -455,9 +455,5 @@ int bpf_check_btf_info(struct bpf_verifier_env *env, if (err) return err; - err = check_core_relo(env, attr, uattr); - if (err) - return err; - return 0; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5d7080c260d8..33161dc64568 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -21199,6 +21199,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, ret = bpf_diag_init(env); if (ret) goto err_prep; + if (env->prog->insnsi[env->prog->len - 1].code == (BPF_LD | BPF_IMM | BPF_DW)) { + verbose(env, "invalid bpf_ld_imm64 insn\n"); + ret = -EINVAL; + goto err_prep; + } if (env->signature) { ret = bpf_prog_calc_tag(env->prog); if (ret < 0) @@ -21274,6 +21279,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + /* Apply CO-RE before validating the program's instruction layout. */ + ret = bpf_check_core_relo(env, attr, uattr); + if (ret < 0) + goto skip_full_check; + /* Discover all subprograms before validating their layout and BTF. */ ret = add_subprogs(env); if (ret < 0) @@ -21283,7 +21293,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; - /* Validate BTF against the complete subprogram layout and apply CO-RE. */ + /* Validate BTF against the complete subprogram layout. */ ret = bpf_check_btf_info(env, attr, uattr); if (ret < 0) goto skip_full_check; -- cgit v1.2.3 From 71919742c83c32afcccf06a7a28e28f3f55f21a2 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Fri, 18 Sep 2026 01:32:16 +0200 Subject: bpf: Assign lock identity to callback map values A nested bpf_for_each_map_elem() callback can unlock a different element of the same map: static long inner(void *map, int *key, struct value *v, struct value **outer_value) { bpf_spin_lock(&v->lock); bpf_spin_unlock(&(*outer_value)->lock); return 0; } static long outer(void *map, int *key, struct value *v, void *ctx) { bpf_for_each_map_elem(map, inner, &v, 0); return 0; } Both callback values currently have ID zero and the same map_ptr. process_spin_lock() compares those two fields, so it accepts the unlock even though the two callbacks can receive different map elements. Assign a fresh ID to every callback map value in the for-each, timer/workqueue, and task-work constructors. Copies of one callback argument retain its ID, so locking and unlocking through that argument continues to work. Distinct callbacks also get distinct IDs for single-element arrays, including inner arrays sharing inner_map_meta. Preserve map_uid for every inner-map lookup and compare it through check_ids() during state pruning. This preserves relationships between maps, keys, and values while allowing equivalent states with different lookup IDs to match. It avoids field-specific rules for when an inner map needs an identity. Move map_uid out of the metadata union and next to the other IDs, so register comparisons can use the existing memcmp() ranges and remap the IDs separately. Clear it when resetting a register or converting a map lookup result to a socket pointer. Shrink frameno to u8, which is enough for MAX_CALL_FRAMES, to make room without growing bpf_reg_state. Fixes: d0d78c1df9b1 ("bpf: Allow locking bpf_spin_lock global variables") Reported-by: Nicholas Carlini Suggested-by: Nicholas Carlini Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260917233222.2542500-9-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/states.c | 6 ++++-- kernel/bpf/verifier.c | 13 +++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 6c88ad95b63b..e4ec007f7fa6 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -491,7 +491,8 @@ static bool regs_exact(const struct bpf_reg_state *rold, { return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && check_ids(rold->id, rcur->id, idmap) && - check_ids(rold->parent_id, rcur->parent_id, idmap); + check_ids(rold->parent_id, rcur->parent_id, idmap) && + check_ids(rold->map_uid, rcur->map_uid, idmap); } enum exact_level { @@ -616,7 +617,8 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off) && check_ids(rold->id, rcur->id, idmap) && - check_ids(rold->parent_id, rcur->parent_id, idmap); + check_ids(rold->parent_id, rcur->parent_id, idmap) && + check_ids(rold->map_uid, rcur->map_uid, idmap); case PTR_TO_PACKET_META: case PTR_TO_PACKET: /* We must have at least as much range as the old ptr diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 33161dc64568..02be326f235c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1864,6 +1864,7 @@ static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); reg->id = 0; reg->parent_id = 0; + reg->map_uid = 0; ___mark_reg_known(reg, imm); } @@ -1925,17 +1926,18 @@ static void refine_map_lookup_value(struct bpf_reg_state *reg) if (map->inner_map_meta) { reg->type = CONST_PTR_TO_MAP | maybe_null; reg->map_ptr = map->inner_map_meta; - /* transfer reg's id which is unique for every map_lookup_elem + /* + * transfer reg's id which is unique for every map_lookup_elem * as UID of the inner map. */ - if (btf_record_has_field(map->inner_map_meta->record, - BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) - reg->map_uid = reg->id; + reg->map_uid = reg->id; } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { reg->type = PTR_TO_XDP_SOCK | maybe_null; + reg->map_uid = 0; } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || map->map_type == BPF_MAP_TYPE_SOCKHASH) { reg->type = PTR_TO_SOCKET | maybe_null; + reg->map_uid = 0; } } @@ -10048,6 +10050,7 @@ int map_set_for_each_callback_args(struct bpf_verifier_env *env, __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr; callee->regs[BPF_REG_3].map_uid = caller->regs[BPF_REG_1].map_uid; + callee->regs[BPF_REG_3].id = ++env->id_gen; /* pointer to stack or null */ callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3]; @@ -10144,6 +10147,7 @@ static int set_timer_callback_state(struct bpf_verifier_env *env, __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = map_ptr; callee->regs[BPF_REG_3].map_uid = map_uid; + callee->regs[BPF_REG_3].id = ++env->id_gen; /* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); @@ -10262,6 +10266,7 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = map_ptr; callee->regs[BPF_REG_3].map_uid = map_uid; + callee->regs[BPF_REG_3].id = ++env->id_gen; /* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); -- cgit v1.2.3 From 88aed0422f39b22406f35f1e758cea25e7bbcfb5 Mon Sep 17 00:00:00 2001 From: Vinay Belgaumkar Date: Fri, 4 Sep 2026 11:16:24 -0700 Subject: perf: Fix null pointer access in is_include_guest_event() A typical module unload occurring event when there is an active perf connection leads to freeing of the pmu pointer. The call log is something like: .. __pmu_detach_event pmu_detach_event pmu_detach_events perf_pmu_unregister .. __pmu_detach_event() sets event->pmu to null. When the perf connection finally is closed, the following stack trace is observed: Oops: general protection fault, kernel NULL pointer dereference ... RIP: 0010:_free_event+0x3e/0x370 ... Call Trace: ... perf_event_release_kernel+0x260/0x2d0 perf_release+0x12/0x20 A call to mediated_pmu_unaccount_event() inside _free_event() is the root cause of this crash. Adding a check inside is_include_guest_event() ensures we don't accidentally access a null pmu ptr. In addition to this, we will now call mediated_pmu_unaccount_event() before clearing the pmu ptr so that nr_include_guest_events counts are maintained correctly. Fixes: eff95e170275 ("perf: Add APIs to create/release mediated guest vPMUs") Assisted-by: Claude:Claude-Sonnet-5 Signed-off-by: Vinay Belgaumkar Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20260904181625.1394082-1-vinay.belgaumkar@intel.com --- kernel/events/core.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index fe33fe15689d..db7b76d6b68a 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -6350,6 +6350,9 @@ static DEFINE_MUTEX(perf_mediated_pmu_mutex); /* !exclude_guest event of PMU with PERF_PMU_CAP_MEDIATED_VPMU */ static inline bool is_include_guest_event(struct perf_event *event) { + if (!event->pmu) + return false; + if ((event->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU) && !event->attr.exclude_guest) return true; @@ -13002,6 +13005,7 @@ static void __pmu_detach_event(struct pmu *pmu, struct perf_event *event, exclusive_event_destroy(event); module_put(pmu->module); + mediated_pmu_unaccount_event(event); event->pmu = NULL; /* force fault instead of UAF */ } -- cgit v1.2.3 From fe3c73d7bc769e7afc252f867a3421fe168b898d Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 15 Sep 2026 20:41:01 +0200 Subject: sched/core: Avoid false migration warning for proxy donors Proxy execution can move a blocked donor's scheduling context to the lock owner's CPU even when the donor is migration-disabled. The donor does not execute there, and its original execution CPU remains recorded in wake_cpu. set_task_cpu() warns unconditionally for migration-disabled tasks, so a subsequent proxy migration or the wakeup path returning the donor home triggers a false positive: moving a blocked scheduling context does not violate the migration-disabled execution context. For example, creating a mutex owner on CPU1 and a migration-disabled waiter on CPU0 can trigger the following warning: proxy_migrate_repro: donor blocking on CPU0 with migration disabled proxy_migrate_repro: donor moved from CPU0 to CPU1 WARNING: kernel/sched/core.c:3389 at set_task_cpu+0x1d3/0x280 ... Call Trace: try_to_wake_up+0x43f/0x780 __mutex_unlock_slowpath+0x330/0x540 owner_fn+0x9f/0xc0 [proxy_migrate_repro] ... proxy_migrate_repro: donor woke on CPU0, task_cpu=0 proxy_migrate_repro: completed Exclude blocked proxy donors from the warning. The proxy wakeup path restores an executable placement before clearing the blocked state. Fixes: b049b81bdff6 ("sched: Handle blocked-waiter migration (and return migration)") Signed-off-by: Andrea Righi Signed-off-by: Peter Zijlstra (Intel) Acked-by: John Stultz Link: https://patch.msgid.link/20260915184101.2621252-1-arighi@nvidia.com --- kernel/sched/core.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 7885ff76e69f..0b846a13c628 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3351,6 +3351,8 @@ void relax_compatible_cpus_allowed_ptr(struct task_struct *p) void set_task_cpu(struct task_struct *p, unsigned int new_cpu) { unsigned int state = READ_ONCE(p->__state); + bool proxy_migrated = sched_proxy_exec() && p->is_blocked && + task_cpu(p) != p->wake_cpu; /* * We should never call set_task_cpu() on a blocked task, @@ -3386,7 +3388,12 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) */ WARN_ON_ONCE(!cpu_online(new_cpu)); - WARN_ON_ONCE(is_migration_disabled(p)); + /* + * Proxy execution can move a blocked task's scheduling context to any + * CPU without moving its migration-disabled execution context. The + * wakeup path will return the task to a CPU where it can execute. + */ + WARN_ON_ONCE(is_migration_disabled(p) && !proxy_migrated); trace_sched_migrate_task(p, new_cpu); -- cgit v1.2.3 From 8901cee9316d53a5a97398f026c2d59a3043159d Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 19 Sep 2026 03:42:10 +0200 Subject: bpf: Compare stack frames in regs_exact() regs_exact() compares the register state up to id, followed by the ID mappings, but does not compare frameno. The PTR_TO_STACK case in regsafe() checks frameno separately, which is bypassed when exact comparison is requested. Consequently, infinite-loop detection can treat pointers to different stack frames as the same pointer and reject a finite loop. For example, initialize fp-8 to zero in the caller and to one in the callee, then pass the caller's fp-8 to the callee as r1: loop: r0 = *(u64 *)(r1 + 0); if r0 != 0 goto done; r1 = r10; r1 += -8; goto loop; done: exit; The loop terminates after reading the callee's slot on its second iteration. At the loop header, however, the only relevant difference is r1's frameno, so exact comparison incorrectly reports an infinite loop. The same problem occurs when the pointer is spilled to the stack. Move frameno into the type-specific metadata union, ahead of id, so the existing prefix comparison in regs_exact() covers it. Ordinary stack pointers do not use another union member. Iterator and IRQ stack-slot states use their dedicated union views and do not need a frame lookup. This also keeps bpf_reg_state at 80 bytes. Since frameno now shares storage with other pointer metadata, it is only meaningful for PTR_TO_STACK registers. Return NULL from bpf_func() for other register types. process_iter_arg(), get_constant_map_key() and is_dynptr_reg_valid_init() look up the frame before checking the register type and would otherwise index frame[] with a byte of the register's map or BTF pointer. They dereference the frame only after their type check. Move the states_maybe_looping() boundary from frameno to precise after the field relocation. Its prefix comparison continues to cover the complete value state and now includes frameno. Continue to ignore precise. Precision marks control whether pruning may ignore scalar ranges; they do not change the represented values, and exact comparison already compares those ranges unconditionally. Marks can also change through backtracking while an ancestor state is still being explored. Fixes: d5b892fd607a ("bpf: make infinite loop detection in is_state_visited() exact") Reported-by: Eduard Zingerman Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260919014213.1840880-2-memxor@gmail.com --- kernel/bpf/states.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index e4ec007f7fa6..012b82513a3b 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -644,10 +644,7 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, return range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off); case PTR_TO_STACK: - /* two stack pointers are equal only if they're pointing to - * the same stack frame, since fp-8 in foo != fp-8 in bar - */ - return regs_exact(rold, rcur, idmap) && rold->frameno == rcur->frameno; + return regs_exact(rold, rcur, idmap); case PTR_TO_ARENA: return true; case PTR_TO_INSN: @@ -1126,7 +1123,7 @@ static bool states_maybe_looping(struct bpf_verifier_state *old, fcur = cur->frame[fr]; for (i = 0; i < MAX_BPF_REG; i++) if (memcmp(&fold->regs[i], &fcur->regs[i], - offsetof(struct bpf_reg_state, frameno))) + offsetof(struct bpf_reg_state, precise))) return false; return true; } -- cgit v1.2.3 From bfc888f04588f591851e95c974954cfca58e6c19 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 14 Sep 2026 15:24:42 +0200 Subject: bpf: Bound ownership depth through local kptrs and graph roots Program-allocated objects can own other local objects through referenced kptrs. bpf_obj_free_fields() follows those pointers through __bpf_obj_drop_impl() synchronously, before the object storage is freed through RCU. A self-referential local kptr type therefore permits arbitrarily deep object chains, and dropping the head can exhaust the kernel stack. Long acyclic type chains have the same problem. btf_check_and_fixup_fields() still assumes referenced kptrs only point to kernel types and checks ownership through list and rbtree roots only. Its existing rule is sufficient for graph-only cycles: the target of each graph edge must contain a node, so every type in a cycle has both a root and a node. The rule rejects such a type owning another root, breaking every cycle. It also limits graph-only chains to three types, or two if the first type contains a node, and conservatively rejects longer acyclic chains. The missing local-kptr edges, rather than a missed graph-only cycle, are the bug introduced by support for bpf_kptr_xchg() into local kptrs. Replace that restriction with one bounded ownership walk covering graph roots and local referenced kptrs. Run it after all BTF records have been fixed up, reject cycles and paths deeper than eight record-bearing types, and cache each type's suffix depth while checking it against the remaining budget. This also permits the longer acyclic graph-only layouts rejected by the old rule; update their existing BTF tests accordingly. Keep the bound independent of MAX_CALL_FRAMES because recursive destruction can run below a BPF call chain. A plain local pointee without special-field metadata adds only a final non-recursing drop. Non-owning kptrs and kernel-BTF kptrs do not recurse through local records and remain outside the walk. Include local percpu-kptr edges too, although allocation of percpu objects with special fields is currently forbidden, so that relaxing that restriction cannot bypass the ownership bound. btf_check_and_fixup_fields() continues to initialize graph_root.value_rec, including for separately allocated map records. The ownership relationships belong to immutable program BTF and only need validation at BTF load time. Fixes: b0966c724584 ("bpf: Support bpf_kptr_xchg into local kptr") Reported-by: Nicholas Carlini Suggested-by: Nicholas Carlini Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260914132444.2564218-2-memxor@gmail.com --- kernel/bpf/btf.c | 134 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 86 insertions(+), 48 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index d67a169ba2bb..d870bc5e50bc 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -4268,13 +4268,10 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec) { int i; - /* There are three types that signify ownership of some other type: - * kptr_ref, bpf_list_head, bpf_rb_root. - * kptr_ref only supports storing kernel types, which can't store - * references to program allocated local types. - * - * Hence we only need to ensure that bpf_{list_head,rb_root} ownership - * does not form cycles. + /* + * Check fields which require the complete BTF and initialize runtime + * metadata. Ownership relationships are validated after every record has + * been fixed up. */ if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & (BPF_GRAPH_ROOT | BPF_UPTR))) return 0; @@ -4305,51 +4302,88 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec) if (!meta) return -EFAULT; rec->fields[i].graph_root.value_rec = meta->record; + } + return 0; +} - /* We need to set value_rec for all root types, but no need - * to check ownership cycle for a type unless it's also a - * node type. - */ - if (!(rec->field_mask & BPF_GRAPH_NODE)) +static int btf_owned_type_idx(const struct btf *btf, struct btf_struct_metas *tab, + const struct btf_field *field) +{ + struct btf_struct_meta *meta; + u32 btf_id; + + if (field->type & BPF_GRAPH_ROOT) { + btf_id = field->graph_root.value_btf_id; + } else if (field->type == BPF_KPTR_REF || field->type == BPF_KPTR_PERCPU) { + if (btf_is_kernel(field->kptr.btf)) + return -ENOENT; + btf_id = field->kptr.btf_id; + } else { + return -ENOENT; + } + + meta = btf_find_struct_meta(btf, btf_id); + if (!meta) + return field->type & BPF_GRAPH_ROOT ? -EFAULT : -ENOENT; + return meta - tab->types; +} + +/* + * Each ownership edge adds kernel frames through bpf_obj_free_fields() and + * __bpf_obj_drop_impl(). Keep the bound deliberately small because object + * destruction can itself run below a BPF call chain. A final pointee without + * special fields is not present in the struct metadata table and adds only a + * non-recursing drop. + */ +#define BTF_MAX_OWNERSHIP_DEPTH 8 + +static int btf_ownership_depth(const struct btf *btf, + struct btf_struct_metas *tab, u8 *depth, + int idx, int depth_left) +{ + const struct btf_record *rec = tab->types[idx].record; + int i, ret, max_depth = 0; + + if (!depth_left) + return -ELOOP; + if (depth[idx]) + goto done; + + for (i = 0; i < rec->cnt; i++) { + ret = btf_owned_type_idx(btf, tab, &rec->fields[i]); + if (ret == -ENOENT) continue; + if (ret < 0) + return ret; + ret = btf_ownership_depth(btf, tab, depth, ret, depth_left - 1); + if (ret < 0) + return ret; + max_depth = max(max_depth, ret); + } + depth[idx] = max_depth + 1; +done: + return depth[idx] > depth_left ? -ELOOP : depth[idx]; +} - /* We need to ensure ownership acyclicity among all types. The - * proper way to do it would be to topologically sort all BTF - * IDs based on the ownership edges, since there can be multiple - * bpf_{list_head,rb_node} in a type. Instead, we use the - * following resaoning: - * - * - A type can only be owned by another type in user BTF if it - * has a bpf_{list,rb}_node. Let's call these node types. - * - A type can only _own_ another type in user BTF if it has a - * bpf_{list_head,rb_root}. Let's call these root types. - * - * We ensure that if a type is both a root and node, its - * element types cannot be root types. - * - * To ensure acyclicity: - * - * When A is an root type but not a node, its ownership - * chain can be: - * A -> B -> C - * Where: - * - A is an root, e.g. has bpf_rb_root. - * - B is both a root and node, e.g. has bpf_rb_node and - * bpf_list_head. - * - C is only an root, e.g. has bpf_list_node - * - * When A is both a root and node, some other type already - * owns it in the BTF domain, hence it can not own - * another root type through any of the ownership edges. - * A -> B - * Where: - * - A is both an root and node. - * - B is only an node. - */ - if (meta->record->field_mask & BPF_GRAPH_ROOT) - return -ELOOP; +static int btf_check_ownership_depth(const struct btf *btf, + struct btf_struct_metas *tab) +{ + u8 *depth; + int i, ret = 0; + + depth = kvcalloc(tab->cnt, sizeof(*depth), GFP_KERNEL | __GFP_NOWARN); + if (!depth) + return -ENOMEM; + + for (i = 0; i < tab->cnt; i++) { + ret = btf_ownership_depth(btf, tab, depth, i, + BTF_MAX_OWNERSHIP_DEPTH); + if (ret < 0) + break; + ret = 0; } - return 0; + kvfree(depth); + return ret; } static void __btf_struct_show(const struct btf *btf, const struct btf_type *t, @@ -6044,6 +6078,10 @@ static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, if (err < 0) goto errout_meta; } + + err = btf_check_ownership_depth(btf, struct_meta_tab); + if (err < 0) + goto errout_meta; } err = bpf_log_attr_finalize(attr_log, &env->log); -- cgit v1.2.3 From 3bd46666cfe51e2bd333fb46a0234694ca594230 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 18 Sep 2026 14:28:37 -1000 Subject: sched_ext: Pass the initial cmask to cid-form ops.enable() The cid-form API has an obvious hole. A task's cid mask is only visible through ops.set_cmask(), which fires on affinity changes and class switches but not when a task enters a scheduler through fork, sub-sched enable or re-home, and there is no p->cpus_ptr equivalent to fall back on. Schedulers work around it by seeding the mask in ops.init_task() from p->cpus_ptr cid by cid, which is subtly wrong: on sub-sched enable and re-home, an affinity change between init_task() and enable() is delivered to the sched the task is still on, and nothing corrects the new sched's copy afterwards. Fix it by adding struct scx_enable_args to cid-form ops.enable() carrying the task's cmask, built in the per-cpu scratch under the rq lock as the task enters the scheduler, and calling set_cmask() with the same mask right after enable(), ahead of set_weight(). A scheduler can then track affinity in set_cmask() alone, and scx_qmap drops its init_task() seed. set_cmask() no longer fires for a cid-form task before it is enabled, and the class-switch republish in switching_to_scx() is limited to the cpu form. This changes the cid-form ops.enable() signature, which is fine as the cid-form API is still considered unreleased. An args struct rather than a bare cmask argument leaves room for more initial state without another signature change, and the cmask travels as a plain arena address because BTF can't type arena struct members yet. v2: The cmask travels as a u64 arena address, cmask_arena_addr, instead of a kernel-typed pointer, with the typing limitation and the planned typed alias documented (Sashiko review). v3: The initial set_cmask() is delivered before set_weight() so that the mask is in place when weight-dependent state is derived (Andrea Righi). Selftest added. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 92 ++++++++++++++++++++++++++++++--------------- kernel/sched/ext/internal.h | 56 +++++++++++++++++++++++---- 2 files changed, 110 insertions(+), 38 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index f568fd9973f6..ad391a8cbd05 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -447,37 +447,45 @@ static void switch_rq_lock(struct rq *from, struct rq *to) DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type); /** - * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task + * scx_fill_cmask_scratch - Build this cpu's arena cmask from @cpumask + * @sch: scx_sched whose scratch to fill + * @cpumask: cpus to translate into cids + * + * The scratch lives in BPF-writable arena memory and its header can't be + * trusted, so it is rewritten from kernel geometry rather than read. Caller + * must hold an rq lock so this cpu is the sole kernel writer for as long as the + * returned address is in use. + */ +static struct scx_cmask *scx_fill_cmask_scratch(struct scx_sched *sch, + const struct cpumask *cpumask) +{ + struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); + struct scx_cmask_ref ref; + + scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref); + scx_cmask_ref_from_cpumask(&ref, cpumask); + return kern_va; +} + +/** + * scx_call_op_set_cpumask - Invoke the set_cpumask or set_cmask op for @task * @sch: scx_sched being invoked * @rq: rq to update as the currently-locked rq, or NULL * @task: task whose affinity is changing * @cpumask: new cpumask * - * For cid-form schedulers, translate @cpumask to a cmask via the per-cpu - * scratch in cid.c and dispatch through the ops_cid union view. Caller - * must hold @rq's rq lock so this_cpu_ptr is stable across the call. + * For cid-form schedulers, translate @cpumask to a cmask in the per-cpu scratch + * and dispatch through the ops_cid union view. Caller must hold @rq's rq lock. */ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, struct task_struct *task, const struct cpumask *cpumask) { - if (scx_is_cid_type()) { - struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); - struct scx_cmask_ref ref; - - /* - * Build the per-cpu arena cmask from kernel geometry via @ref, - * never reading its BPF-writable header. set_cmask()'s __arena - * argument takes the kernel address and the struct_ops - * trampoline rebases it into BPF's arena pointer form. The rq - * lock makes this cpu the sole kernel writer. - */ - scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref); - scx_cmask_ref_from_cpumask(&ref, cpumask); - SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, kern_va); - } else { + if (scx_is_cid_type()) + SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, + scx_fill_cmask_scratch(sch, cpumask)); + else SCX_CALL_OP_TASK(sch, set_cpumask, rq, task, cpumask); - } } enum scx_dsq_iter_flags { @@ -3634,8 +3642,12 @@ static void set_cpus_allowed_scx(struct task_struct *p, * * Fine-grained memory write control is enforced by BPF making the const * designation pointless. Cast it away when calling the operation. + * + * The cid form receives the initial mask when the task is enabled and + * hears about changes only afterwards, see struct scx_enable_args. */ - if (SCX_HAS_OP(sch, set_cpumask)) + if (SCX_HAS_OP(sch, set_cpumask) && + (!scx_is_cid_type() || scx_get_task_state(p) == SCX_TASK_ENABLED)) scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr); } @@ -3944,8 +3956,27 @@ static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p) p->scx.weight = sched_weight_to_cgroup(weight); - if (SCX_HAS_OP(sch, enable)) - SCX_CALL_OP_TASK(sch, enable, rq, p); + if (SCX_HAS_OP(sch, enable)) { + if (scx_is_cid_type()) { + struct scx_cmask *cmask = scx_fill_cmask_scratch(sch, p->cpus_ptr); + struct scx_enable_args args = { + .cmask_arena_addr = scx_kaddr_to_arena(sch, cmask), + }; + + SCX_CALL_CID_OP_TASK(sch, enable, rq, p, &args); + } else { + SCX_CALL_OP_TASK(sch, enable, rq, p); + } + } + + /* + * The initial mask also goes out through set_cmask() so a scheduler can + * track affinity there alone, and before set_weight() so that the mask + * is in place when weight-dependent state is derived, see struct + * scx_enable_args. + */ + if (scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask)) + scx_call_op_set_cpumask(sch, rq, p, p->cpus_ptr); if (SCX_HAS_OP(sch, set_weight)) SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); @@ -4288,9 +4319,10 @@ static void switching_to_scx(struct rq *rq, struct task_struct *p) /* * set_cpus_allowed_scx() is not called while @p is associated with a - * different scheduler class. Keep the BPF scheduler up-to-date. + * different scheduler class. Keep the BPF scheduler up-to-date. The cid + * form gets its mask from scx_enable_task(). */ - if (SCX_HAS_OP(sch, set_cpumask)) + if (!scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask)) scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr); } @@ -8374,10 +8406,11 @@ static struct bpf_struct_ops bpf_sched_ext_ops = { /* * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types * identical, only param names differ across structs) are reused. Some need - * fresh stubs, set_cmask due to an argument type difference and the sub-sched - * notifiers because no cpu-form stub exists to reuse. + * fresh stubs, set_cmask and enable due to argument differences and the + * sub-sched notifiers because no cpu-form stub exists to reuse. */ static void sched_ext_ops_cid__set_cmask(struct task_struct *p, const struct scx_cmask *cmask__arena) {} +static void sched_ext_ops_cid__enable(struct task_struct *p, struct scx_enable_args *args) {} static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask__arena, u64 caps) {} static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {} @@ -8398,7 +8431,7 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .update_idle = sched_ext_ops__update_idle, .init_task = sched_ext_ops__init_task, .exit_task = sched_ext_ops__exit_task, - .enable = sched_ext_ops__enable, + .enable = sched_ext_ops_cid__enable, .disable = sched_ext_ops__disable, #ifdef CONFIG_EXT_GROUP_SCHED .cpuctl_init = sched_ext_ops__cgroup_init, @@ -10421,8 +10454,7 @@ __bpf_kfunc const void *scx_bpf_online_cmask(const struct bpf_prog_aux *aux) if (unlikely(!online)) return NULL; - /* BPF rebases by the low 32 bits, like __arena callback args */ - return (void *)((unsigned long)online - sch->arena_kern_base); + return (void *)scx_kaddr_to_arena(sch, online); } /** diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index d150de10a5c9..1df8f583b0ec 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -250,6 +250,31 @@ struct scx_exit_task_args { bool cancelled; }; +/** + * struct scx_enable_args - Argument container for cid-form ops.enable() + * @cmask_arena_addr: BPF arena address of the cmask of cids the task may run on + * + * @cmask_arena_addr is the task's affinity as it enters the scheduler. + * set_cmask() delivers the same mask right after enable(), before set_weight() + * and the first enqueue, then every affinity change afterwards, and is never + * called before enable(). A scheduler may therefore track affinity in + * set_cmask() alone. + * + * The kernel builds the mask in the scheduler arena from its own geometry, so + * the header is valid regardless of what the scheduler last wrote there. The + * memory is per-cpu scratch reused once the callback returns: copy the bits + * out, don't keep the address. The set_cmask() argument follows the same rules. + * + * The address is a plain value rather than a typed pointer because BTF can't + * mark a struct member as an arena pointer yet and a pointer member would reach + * the program typed as a kernel pointer. Cast it to struct scx_cmask __arena * + * before use. Once arena members can be typed, a typed alias will join this + * field in an anonymous union at the same offset. + */ +struct scx_enable_args { + u64 cmask_arena_addr; +}; + /* argument container for ops.cgroup_init() */ struct scx_cgroup_init_args { /* the weight of the cgroup [1..10000] */ @@ -1037,6 +1062,7 @@ struct sched_ext_ops { * - dispatch -> dispatch (cpu arg is now cid) * - update_idle -> update_idle (cpu arg is now cid) * - set_cpumask -> set_cmask (cmask instead of cpumask) + * - enable -> enable (takes struct scx_enable_args) * - cpu_online -> cid_online * - cpu_offline -> cid_offline * - dump_cpu -> dump_cid @@ -1070,7 +1096,7 @@ struct sched_ext_ops_cid { struct scx_init_task_args *args); void (*exit_task)(struct task_struct *p, struct scx_exit_task_args *args); - void (*enable)(struct task_struct *p); + void (*enable)(struct task_struct *p, struct scx_enable_args *args); void (*disable)(struct task_struct *p); void (*dump)(struct scx_dump_ctx *ctx); void (*dump_cid)(struct scx_dump_ctx *ctx, s32 cid, bool idle); @@ -1533,7 +1559,8 @@ struct scx_sched { * by BUILD_BUG_ON in scx_init()). The anonymous union lets the kernel * access either view of the same storage without function-pointer * casts: use .ops for cpu-form and shared fields, .ops_cid for the - * cid-renamed callbacks (set_cmask, select_cid, cid_online, ...). + * callbacks whose cid-form signature differs (set_cmask, enable, + * select_cid, cid_online, ...). */ union { struct sched_ext_ops ops; @@ -1556,9 +1583,9 @@ struct scx_sched { uintptr_t arena_kern_base; /* - * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask - * to ops_cid.set_cmask(). The kernel writes through the stored kern_va - * and passes it to the callback's __arena argument. + * Per-CPU arena cmask the kernel fills from a task's cpumask and hands + * to ops_cid.enable() and ops_cid.set_cmask(). The stored pointers are + * the kernel addresses. */ struct scx_cmask * __percpu *set_cmask_scratch; struct scx_cmask *online_cmask; @@ -1669,6 +1696,19 @@ static inline void *scx_arena_to_kaddr(struct scx_sched *sch, const void *bpf_pt return (void *)(sch->arena_kern_base + (u32)(uintptr_t)bpf_ptr); } +/** + * scx_kaddr_to_arena - Translate a kernel arena address to the BPF form + * @sch: scheduler whose arena hosts @kaddr + * @kaddr: kernel address inside @sch's arena + * + * __arena callback arguments need no translation. Addresses handed to BPF any + * other way, such as struct fields and kfunc return values, go through this. + */ +static inline uintptr_t scx_kaddr_to_arena(struct scx_sched *sch, const void *kaddr) +{ + return (uintptr_t)kaddr - sch->arena_kern_base; +} + enum scx_wake_flags { /* expose select WF_* flags as enums */ SCX_WAKE_FORK = WF_FORK, @@ -2302,9 +2342,9 @@ do { \ } while (0) /* - * Dispatch a task op through the cid-form ops_cid table. Only set_cmask() needs - * this: it takes an arena cmask address instead of a cpumask, so it cannot be - * invoked via its cpu-form set_cpumask() slot. + * Dispatch a task op through the cid-form ops_cid table, for the ops whose + * cid-form signature differs from the cpu-form slot: set_cmask() takes an arena + * cmask instead of a cpumask and enable() takes scx_enable_args. */ #define SCX_CALL_CID_OP_TASK(sch, op, locked_rq, task, args...) \ __SCX_CALL_OP_TASK(sch, ops_cid, op, locked_rq, task, ##args) -- cgit v1.2.3 From c21eaa72f02fc6e85621cbe09d303d8fb8bd39cd Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Sep 2026 20:48:30 +0200 Subject: posix-cpu-timers: Prevent freeing a timer which is queued on the expiry list Kijo analyzed another race in the POSIX CPU timer code: Commit bf635681c906 converted cpu_timer::firing from a tristate value to a boolean. This lost the distinction between "not owned by the firing list" and "still owned, but delivery was canceled". The resulting race is: expiry handler timer_settime() timer_delete() -------------- --------------- -------------- collect timer onto private firing list firing = true observes firing = true firing = false return TIMER_RETRY wait for handler observes firing = false finish deletion unhash and free timer resume list traversal read freed elist.next -> UAF The firing bit is clearly the wrong indicator since that commit. Check whether the timer is queued on the expiry list or not instead. If it is queued clear the firing bit to prevent signal delivery as before and return TIMER_RETRY so the caller unlocks the timer which allows the expiry code to make progress and remove it from the list. Fixes: bf635681c906 ("posix-cpu-timers: Cleanup the firing logic") Reported-by: Kijo Park Debugged-by: Kijo Park Signed-off-by: Thomas Gleixner Tested-by: Kijo Park Reviewed-by: Frederic Weisbecker Cc: stable@vger.kernel.org --- kernel/time/posix-cpu-timers.c | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) (limited to 'kernel') diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index d73d31c7994f..0bf4fcd969c8 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -408,6 +408,7 @@ static int posix_cpu_timer_create(struct k_itimer *new_timer) new_timer->kclock = &clock_posix_cpu; timerqueue_init(&new_timer->it.cpu.node); + INIT_LIST_HEAD(&new_timer->it.cpu.elist); new_timer->it.cpu.pid = get_pid(pid); rcu_read_unlock(); return 0; @@ -565,6 +566,24 @@ static struct task_struct *timer_lock_sighand(struct k_itimer *timer, unsigned l return NULL; } +/* + * If the timer is queued on the expiry list, then it cannot be dequeued because + * the firing list is not protected by sighand->lock. The delivery path is + * waiting for the timer lock. So go back, unlock and retry. + */ +static bool posix_cpu_timer_on_expiry_list(struct k_itimer *timer) +{ + if (list_empty(&timer->it.cpu.elist)) + return false; + + /* + * Prevent signal delivery as there is no point in delivering a signal + * which is made obsolete right away. + */ + timer->it.cpu.firing = false; + return true; +} + /* * Clean up a CPU-clock timer that is about to be destroyed. * This is called from timer deletion with the timer already locked. @@ -580,18 +599,10 @@ static int posix_cpu_timer_del(struct k_itimer *timer) p = timer_lock_sighand(timer, &flags); if (likely(p)) { - if (timer->it.cpu.firing) { - /* - * Prevent signal delivery. The timer cannot be dequeued - * because it is on the firing list which is not protected - * by sighand->lock. The delivery path is waiting for - * the timer lock. So go back, unlock and retry. - */ - timer->it.cpu.firing = false; + if (posix_cpu_timer_on_expiry_list(timer)) ret = TIMER_RETRY; - } else { + else disarm_timer(timer, p); - } unlock_task_sighand(p, &flags); } @@ -731,14 +742,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, /* Retrieve the current expiry time before disarming the timer */ old_expires = cpu_timer_getexpires(ctmr); - if (unlikely(timer->it.cpu.firing)) { - /* - * Prevent signal delivery. The timer cannot be dequeued - * because it is on the firing list which is not protected - * by sighand->lock. The delivery path is waiting for - * the timer lock. So go back, unlock and retry. - */ - timer->it.cpu.firing = false; + if (posix_cpu_timer_on_expiry_list(timer)) { ret = TIMER_RETRY; } else { cpu_timer_dequeue(ctmr); -- cgit v1.2.3 From a11212910cf09b2fe8db9afa41ef60c4f81879c5 Mon Sep 17 00:00:00 2001 From: Yuqi Xu Date: Sat, 19 Sep 2026 16:45:02 +0800 Subject: bpf: Check params size before reading reserved fields bpf_crypto_ctx_create() is a kfunc whose second argument is declared with the __sz annotation, so the verifier only guarantees that params__sz bytes of params are valid. The function nevertheless reads params->reserved[0] and params->reserved[1] (offsets 14 and 15) before comparing params__sz against the size of struct bpf_crypto_params, so a BPF program can pass a shorter buffer and have the kernel read past the region that was validated for it. Move the size check in front of the reserved field reads. Fixes: 3e1c6f35409f ("bpf: make common crypto API for TC/XDP programs") Reported-by: Vega Signed-off-by: Yuqi Xu Signed-off-by: Alexei Starovoitov Reviewed-by: Ren Wei Link: https://patch.msgid.link/4f3ab4b03e79017e215521743996555439bf0bb3.1789802413.git.xuyuqiabc@gmail.com --- kernel/bpf/crypto.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/crypto.c b/kernel/bpf/crypto.c index 51f89cecefb4..3f3fe2450fc6 100644 --- a/kernel/bpf/crypto.c +++ b/kernel/bpf/crypto.c @@ -149,8 +149,9 @@ bpf_crypto_ctx_create(const struct bpf_crypto_params *params, u32 params__sz, const struct bpf_crypto_type *type; struct bpf_crypto_ctx *ctx; - if (!params || params->reserved[0] || params->reserved[1] || - params__sz != sizeof(struct bpf_crypto_params)) { + if (!params || + params__sz != sizeof(struct bpf_crypto_params) || + params->reserved[0] || params->reserved[1]) { *err = -EINVAL; return NULL; } -- cgit v1.2.3 From 79a9172f3ab4ad8392c5e5c8944b9b7710ade620 Mon Sep 17 00:00:00 2001 From: Xu Yunxiang Date: Mon, 21 Sep 2026 05:04:21 +0800 Subject: bpf: Reject non-negative offsets in stack_slot_obj_get_spi() bpf_get_spi() computes (-off - 1) / BPF_REG_SIZE using C division, which truncates toward zero. For off == 0, this produces spi 0, the same index used by the valid stack slot at fp-8. stack_slot_obj_get_spi() currently checks alignment and the resulting spi bounds, but does not reject the non-negative offset itself. It can therefore validate a PTR_TO_STACK register holding fp+0 against an iterator stored at fp-8 even though the runtime receives the actual fp+0 pointer. An effectful iterator kfunc can then interpret memory outside the BPF stack as iterator state. Reject non-negative offsets before converting the offset to an spi. All valid stack objects begin at a negative offset from the frame pointer. Fixes: 06accc8779c1 ("bpf: add support for open-coded iterator loops") Signed-off-by: Xu Yunxiang Signed-off-by: Andrii Nakryiko Reviewed-by: Sun Jian Link: https://lore.kernel.org/bpf/20260920210423.345636-2-xyx2021@mail.ustc.edu.cn --- kernel/bpf/verifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 02be326f235c..dd8bb179d39e 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -567,7 +567,7 @@ static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_s } off = reg->var_off.value; - if (off % BPF_REG_SIZE) { + if (off >= 0 || off % BPF_REG_SIZE) { verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off); return -EINVAL; } -- cgit v1.2.3 From 0d6526f82c3cdefcca47f73f5fc08dc6f335eac6 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Mon, 21 Sep 2026 17:37:22 -0700 Subject: sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling bug alb_break_llc() decides whether to break LLC preference during active load balance. It does so by testing that every runnable fair task on the source rq prefers its LLC: env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable But the two counters cover different sets. nr_pref_llc_running is updated in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued, so it follows queued tasks. h_nr_runnable is updated in set_delayed()/ clear_delayed() and drops delay-dequeued tasks. So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks, alb_break_llc() returns false, and active balance is free to pull a task off its preferred LLC. Active balance only moves runnable tasks, and this is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB skips the per-task test in can_migrate_task(). The runnable set is the one we want. Fix it on the counter side. A task should be counted in nr_pref_llc_running exactly while it is both queued on its preferred LLC (pref_llc_queued) and runnable (!sched_delayed). Define that membership once in task_pref_llc_runnable(), and adjust the counter only through pref_llc_running_inc()/pref_llc_running_dec() from the four sites that change either input: account_llc_enqueue(), account_llc_dequeue(), set_delayed() and clear_delayed(). Gating every update on the same predicate keeps the delay, wake and dequeue paths from double-counting or underflowing; see the comments at those sites for the ordering. nr_llc_running and sd->llc_counts are not touched and stay on queued semantics. Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC") Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/ Reported-by: Zhan Xusheng Suggested-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Kayra Cizmeci Cc: # v7.2.x Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 7455a83a6a99..de3d589fa8eb 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1546,6 +1546,28 @@ static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, (scale * per_cpu(sd_llc_size, cpu))); } +/* + * A task counts in nr_pref_llc_running while it is queued on its preferred + * LLC (pref_llc_queued) and runnable (!sched_delayed), keeping the counter in + * the runnable domain so alb_break_llc() can compare it with h_nr_runnable. + */ +static bool task_pref_llc_runnable(struct task_struct *p) +{ + return p->pref_llc_queued && !p->se.sched_delayed; +} + +static void pref_llc_running_inc(struct rq *rq, struct task_struct *p) +{ + if (task_pref_llc_runnable(p)) + rq->nr_pref_llc_running++; +} + +static void pref_llc_running_dec(struct rq *rq, struct task_struct *p) +{ + if (task_pref_llc_runnable(p)) + rq->nr_pref_llc_running--; +} + static void account_llc_enqueue(struct rq *rq, struct task_struct *p) { int pref_llc, pref_llc_queued; @@ -1557,7 +1579,6 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p) pref_llc_queued = (pref_llc == task_llc(p)); rq->nr_llc_running++; - rq->nr_pref_llc_running += pref_llc_queued; /* * Record whether p is enqueued on its preferred @@ -1575,6 +1596,9 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p) */ p->pref_llc_queued = pref_llc_queued; + /* Skipped while delayed; clear_delayed() adds it back on wake. */ + pref_llc_running_inc(rq, p); + sd = rcu_dereference_all(rq->sd); if (sd && (unsigned int)pref_llc < sd->llc_max) sd->llc_counts[pref_llc]++; @@ -1591,7 +1615,12 @@ static void account_llc_dequeue(struct rq *rq, struct task_struct *p) rq->nr_llc_running--; if (p->pref_llc_queued) { - rq->nr_pref_llc_running--; + /* + * Skipped if still delayed (set_delayed() already removed it); + * clearing pref_llc_queued below also stops clear_delayed() + * from re-adding it. + */ + pref_llc_running_dec(rq, p); /* * Update the status in case * other logic might query @@ -1995,6 +2024,7 @@ void init_sched_mm(struct task_struct *p) * polluting account_llc_enqueue(). */ p->preferred_llc = -1; + p->pref_llc_queued = 0; } #else /* CONFIG_SCHED_CACHE */ @@ -2016,6 +2046,10 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p) {} static void account_llc_dequeue(struct rq *rq, struct task_struct *p) {} +static void pref_llc_running_inc(struct rq *rq, struct task_struct *p) {} + +static void pref_llc_running_dec(struct rq *rq, struct task_struct *p) {} + #endif /* CONFIG_SCHED_CACHE */ /* @@ -6390,15 +6424,27 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); static void set_delayed(struct sched_entity *se) { - se->sched_delayed = 1; - /* * Delayed se of cfs_rq have no tasks queued on them. * Do not adjust h_nr_runnable since __dequeue_task() * will account it for blocked tasks. + * + * This check can be removed because when flat pick + * patches get merged as only task can get delayed, + * same for clear_delayed(). */ - if (!entity_is_task(se)) + if (!entity_is_task(se)) { + se->sched_delayed = 1; return; + } + + /* + * Drop a task leaving the runnable set. + * Needs to be called before sched_delayed is set. + * clear_delayed() mirrors this after clearing the flag. + */ + pref_llc_running_dec(rq_of(cfs_rq_of(se)), task_of(se)); + se->sched_delayed = 1; for_each_sched_entity(se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); @@ -6420,6 +6466,13 @@ static void clear_delayed(struct sched_entity *se) if (!entity_is_task(se)) return; + /* + * Re-add on wake, after sched_delayed is cleared. On a final delayed + * dequeue account_llc_dequeue() already cleared pref_llc_queued, so + * this does nothing. + */ + pref_llc_running_inc(rq_of(cfs_rq_of(se)), task_of(se)); + for_each_sched_entity(se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); -- cgit v1.2.3 From d6013e2465d98d524b030a81c1223882a1bb7e4c Mon Sep 17 00:00:00 2001 From: Lu Wang Date: Mon, 21 Sep 2026 17:37:23 -0700 Subject: sched/cache: Honor migrate_llc_task semantics in active load balance, to fix LLC mis-scheduling bug Cache aware scheduling introduced the migrate_llc_task migration type to direct tasks toward their preferred LLC, but its semantics can be lost when passive load balance falls back to active load balance (ALB). This may allow ALB to select a candidate whose preferred LLC does not match the destination, moving it away from its preferred LLC. Example scenario: src_rq has two runnable tasks, p1 and p2. p1 prefers dst_rq (dst_llc), while p2 prefers src_rq (src_llc). In this case, migrate_llc_task is set because src_rq has at least one task, p1, that wants to migrate to dst_rq. In ALB, can_migrate_task() finds p2 and returns true for it, thus moving p2 out of its preferred LLC. Solution: The CPU stopper in ALB constructs a fresh lb_env that does not inherit migration_type from the passive load-balance pass. Two approaches are possible: (a) Add a new member to struct rq so ALB can inherit migrate_llc_task from the passive LB that triggered it. (b) Define a new flag LBF_ACTIVE_LB_LLC and select the stopper callback at kick time to preserve the migration semantics across the asynchronous boundary. We choose (b) because it avoids passing migration_type through the stopper, which would affect the meaning of migration_type for delayed-dequeue tasks. Fixes: e4c9a4cb244a ("sched/cache: Add migrate_llc_task migration type for cache-aware balancing") Suggested-by: Chen Yu Signed-off-by: Lu Wang Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Tim Chen Reviewed-by: Chen Yu Cc: # v7.2.x Link: https://patch.msgid.link/cb39f64a17fc2b76097264aaec74a2d6dfff4315.1790035273.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index de3d589fa8eb..514bd54ccd56 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -10448,6 +10448,7 @@ enum migration_type { #define LBF_SOME_PINNED 0x08 #define LBF_ACTIVE_LB 0x10 #define LBF_LLC_PINNED 0x20 +#define LBF_ACTIVE_LB_LLC 0x40 struct lb_env { struct sched_domain *sd; @@ -10866,6 +10867,21 @@ alb_break_llc(struct lb_env *env) return false; } +/* + * Returns true if p's preferred LLC does not match the destination CPU + * under migrate_llc_task semantics. Passive LB passes migrate_llc_task + * in env->migration_type, while active LB carries LBF_ACTIVE_LB_LLC in + * env->flags to avoid overwriting env->migration_type. + */ +static inline bool +migrate_llc_task_wrong_dst(struct task_struct *p, struct lb_env *env) +{ + return sched_cache_enabled() && + (env->migration_type == migrate_llc_task || + env->flags & LBF_ACTIVE_LB_LLC) && + READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu); +} + /* * Check if migrating task p from env->src_cpu to * env->dst_cpu breaks LLC localiy. @@ -10894,8 +10910,7 @@ static bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env) * run on env->dst_cpu, skip the tasks do not prefer * env->dst_cpu, and find the one that prefers. */ - if (env->migration_type == migrate_llc_task && - READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu)) + if (migrate_llc_task_wrong_dst(p, env)) return true; if (can_migrate_llc_task(env, p) != mig_forbid) @@ -10917,6 +10932,12 @@ alb_break_llc(struct lb_env *env) return false; } +static inline bool +migrate_llc_task_wrong_dst(struct task_struct *p, struct lb_env *env) +{ + return false; +} + static inline bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env) { @@ -11016,7 +11037,7 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) * 4) too many balance attempts have failed. */ if (env->flags & LBF_ACTIVE_LB) - return 1; + return !migrate_llc_task_wrong_dst(p, env); degrades = migrate_degrades_locality(p, env); if (!degrades) { @@ -13415,6 +13436,20 @@ static int need_active_balance(struct lb_env *env) } static int active_load_balance_cpu_stop(void *data); +static int active_load_balance_llc_cpu_stop(void *data); + +/* + * migration_type is checked elsewhere to decide migration policy, so + * it shouldn't be repurposed just to flag an LLC-directed active + * balance across the stopper. Pick the callback here instead. + */ +static inline cpu_stop_fn_t alb_stop_fn(struct lb_env *env) +{ + if (env->migration_type == migrate_llc_task) + return active_load_balance_llc_cpu_stop; + + return active_load_balance_cpu_stop; +} static int should_we_balance(struct lb_env *env) { @@ -13760,7 +13795,7 @@ more_balance: } if (active_balance) { stop_one_cpu_nowait(cpu_of(busiest), - active_load_balance_cpu_stop, busiest, + alb_stop_fn(&env), busiest, &busiest->active_balance_work); } preempt_enable(); @@ -13865,7 +13900,7 @@ update_next_balance(struct sched_domain *sd, unsigned long *next_balance) * least 1 task to be running on each physical CPU where possible, and * avoids physical / logical imbalances. */ -static int active_load_balance_cpu_stop(void *data) +static int __active_load_balance_cpu_stop(void *data, unsigned int lb_flags) { struct rq *busiest_rq = data; int busiest_cpu = cpu_of(busiest_rq); @@ -13915,7 +13950,7 @@ static int active_load_balance_cpu_stop(void *data) .src_cpu = busiest_rq->cpu, .src_rq = busiest_rq, .idle = CPU_IDLE, - .flags = LBF_ACTIVE_LB, + .flags = LBF_ACTIVE_LB | lb_flags, }; schedstat_inc(sd->alb_count); @@ -13943,6 +13978,16 @@ out_unlock: return 0; } +static int active_load_balance_cpu_stop(void *data) +{ + return __active_load_balance_cpu_stop(data, 0); +} + +static int active_load_balance_llc_cpu_stop(void *data) +{ + return __active_load_balance_cpu_stop(data, LBF_ACTIVE_LB_LLC); +} + /* * Scale the max sched_balance_rq interval with the number of CPUs in the system. * This trades load-balance latency on larger machines for less cross talk. -- cgit v1.2.3 From 28f9c0e0a0b94c5d3e1b634db545f6e1f94858c5 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Mon, 21 Sep 2026 17:37:24 -0700 Subject: sched/cache: Decouple sched_cache_group from mm to fix UAF Currently the sched cache grouping is by mm and the scheduling statistics sched_cache_stat lives in the mm structure. This ties the life cycle of scheduling stats with mm. In account_mm_sched(), the scheduling stats are accessed by task->mm->sc_stat. However, a task may be switching mm on one CPU when another CPU is running account_mm_sched(), and possibly accessing the old mm that was freed. This problem was found when running tests with KASAN by Hyunwoo: https://lore.kernel.org/lkml/apPb-Dr4nPYuHQOK@v4bel/ Instead of serializing the mm access by introducing extra acquisition of rq lock in the mm free path, extract sched_cache_stat from mm_struct, rename it as sched_cache_group and manage its life cycle apart from mm_struct with its own ref counting. This allows us in the next patch access sched_cache_group directly from task, and add a refcount on sched_cache_group when a task links to it. This prevents the use after free issue when accessing stale and released old mm and its sched cache stat a task switches to a new mm while account_mm_sched() is done elsewhere. The other benefit of this restructure is in the future, the grouping of tasks to a LLC would have the flexibility to be associated with a user defined grouping, or cgroup, cookie group, numa_group or others instead of just with a single mm address space. Rename sched_cache_stat to sched_cache_group and turn it into a refcounted object allocated from mm_struct. The mm_struct now holds a pointer (sched_cache_grp) to this object instead of embedding it. Fixes: df0d98475954 ("sched/cache: Introduce infrastructure for cache-aware load balancing") Closes: https://lore.kernel.org/lkml/apPb-Dr4nPYuHQOK@v4bel/ Closes: https://lore.kernel.org/all/343a7e07-7fad-4979-9c9b-82ec038c293c@linux.dev/ Reported-by: Hyunwoo Kim Reported-by: Zenghui Yu (Huawei) Co-developed-by: Chen Yu Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: #7.2.x Link: https://patch.msgid.link/91fd1e3266707c865bc9abecfb3e17bc676712df.1790035273.git.tim.c.chen@linux.intel.com --- kernel/exit.c | 11 +++- kernel/sched/fair.c | 173 ++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 135 insertions(+), 49 deletions(-) (limited to 'kernel') diff --git a/kernel/exit.c b/kernel/exit.c index 424c44a42a4d..024350e9b48c 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -558,18 +558,23 @@ void mm_update_next_owner(struct mm_struct *mm) */ static void exit_mm_sched_cache(struct mm_struct *mm) { + struct sched_cache_group *grp; unsigned long fp, sub; if (!current->total_numa_faults) return; /* * No lock protection due to performance considerations. - * Make sure mm->sc_stat.footprint does not become + * Make sure the group footprint does not become * negative. */ - fp = READ_ONCE(mm->sc_stat.footprint); + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp) + return; + + fp = READ_ONCE(grp->footprint); sub = min(fp, current->total_numa_faults); - WRITE_ONCE(mm->sc_stat.footprint, fp - sub); + WRITE_ONCE(grp->footprint, fp - sub); } #else static inline void exit_mm_sched_cache(struct mm_struct *mm) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 514bd54ccd56..f0a9586332fd 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1492,12 +1492,17 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) return true; if (static_branch_likely(&sched_numa_balancing)) { + struct sched_cache_group *grp = READ_ONCE(mm->sched_cache_grp); + + if (!grp) + return true; + /* * TBD: RDT exclusive LLC ways reserved should be * excluded. */ llc = sd->llc_bytes; - footprint = READ_ONCE(mm->sc_stat.footprint); + footprint = READ_ONCE(grp->footprint); /* * Scale the LLC size by 256*llc_aggr_tolerance @@ -1529,6 +1534,7 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, int cpu) { + struct sched_cache_group *grp; int scale; if (get_nr_threads(p) <= 1) @@ -1542,7 +1548,11 @@ static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, if (scale == INT_MAX) return false; - return !fits_capacity((mm->sc_stat.nr_running_avg * cpu_smt_num_threads), + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp) + return true; + + return !fits_capacity((READ_ONCE(grp->nr_running_avg) * cpu_smt_num_threads), (scale * per_cpu(sd_llc_size, cpu))); } @@ -1648,12 +1658,20 @@ static void account_llc_dequeue(struct rq *rq, struct task_struct *p) } } -void mm_init_sched(struct mm_struct *mm, - struct sched_cache_time __percpu *_pcpu_sched) +int mm_init_sched(struct mm_struct *mm, + struct sched_cache_time __percpu *_pcpu_sched) { + struct sched_cache_group *grp; unsigned long epoch = 0; int i; + grp = kzalloc_obj(*grp); + if (!grp) { + free_percpu(_pcpu_sched); + mm->sched_cache_grp = NULL; + return -ENOMEM; + } + for_each_possible_cpu(i) { struct sched_cache_time *pcpu_sched = per_cpu_ptr(_pcpu_sched, i); struct rq *rq = cpu_rq(i); @@ -1664,18 +1682,51 @@ void mm_init_sched(struct mm_struct *mm, epoch = rq->cpu_epoch; } - raw_spin_lock_init(&mm->sc_stat.lock); - mm->sc_stat.epoch = epoch; - mm->sc_stat.cpu = -1; - mm->sc_stat.next_scan = jiffies; - mm->sc_stat.nr_running_avg = 0; - mm->sc_stat.footprint = 0; + raw_spin_lock_init(&grp->lock); + grp->epoch = epoch; + grp->cpu = -1; + grp->next_scan = jiffies; + grp->nr_running_avg = 0; + grp->footprint = 0; + refcount_set(&grp->refcnt, 1); /* - * The update to mm->sc_stat should not be reordered - * before initialization to mm's other fields, in case + * The update to grp->pcpu_sched should not be reordered + * before initialization to grp's other fields, in case * the readers may get invalid mm_sched_epoch, etc. */ - smp_store_release(&mm->sc_stat.pcpu_sched, _pcpu_sched); + smp_store_release(&grp->pcpu_sched, _pcpu_sched); + /* + * Publish the group last. Not every reader qualifies it by + * grp->pcpu_sched - can_migrate_llc_task() only checks that the + * pointer is non-NULL before reading grp->footprint and + * grp->nr_running_avg - so a reachable group must already be + * fully initialized. + */ + smp_store_release(&mm->sched_cache_grp, grp); + return 0; +} + +static void sched_cache_group_free_rcu(struct rcu_head *rcu) +{ + struct sched_cache_group *grp = + container_of(rcu, struct sched_cache_group, rcu); + + free_percpu(grp->pcpu_sched); + kfree(grp); +} + +static void sched_cache_group_put(struct sched_cache_group *grp) +{ + if (!grp || !refcount_dec_and_test(&grp->refcnt)) + return; + + call_rcu(&grp->rcu, sched_cache_group_free_rcu); +} + +void mm_destroy_sched(struct mm_struct *mm) +{ + sched_cache_group_put(mm->sched_cache_grp); + mm->sched_cache_grp = NULL; } /* because why would C be fully specified */ @@ -1729,11 +1780,16 @@ static unsigned long fraction_mm_sched(struct rq *rq, static int get_pref_llc(struct task_struct *p, struct mm_struct *mm) { int mm_sched_llc = -1, mm_sched_cpu; + struct sched_cache_group *grp; if (!mm) return -1; - mm_sched_cpu = READ_ONCE(mm->sc_stat.cpu); + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp) + return -1; + + mm_sched_cpu = READ_ONCE(grp->cpu); if (mm_sched_cpu != -1) { mm_sched_llc = llc_id(mm_sched_cpu); @@ -1764,6 +1820,7 @@ static inline void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) { struct sched_cache_time *pcpu_sched; + struct sched_cache_group *grp; struct mm_struct *mm = p->mm; int mm_sched_llc = -1; unsigned long epoch; @@ -1777,10 +1834,14 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) * init_task, kthreads and user thread created * by user_mode_thread() don't have mm. */ - if (!mm || !mm->sc_stat.pcpu_sched) + if (!mm) + return; + + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp || !grp->pcpu_sched) return; - pcpu_sched = per_cpu_ptr(mm->sc_stat.pcpu_sched, cpu_of(rq)); + pcpu_sched = per_cpu_ptr(grp->pcpu_sched, cpu_of(rq)); scoped_guard (raw_spinlock, &rq->cpu_epoch_lock) { __update_mm_sched(rq, pcpu_sched); @@ -1793,11 +1854,11 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) * If this process hasn't hit task_cache_work() for a while invalidate * its preferred state. */ - if ((long)(epoch - READ_ONCE(mm->sc_stat.epoch)) > llc_epoch_affinity_timeout || + if ((long)(epoch - READ_ONCE(grp->epoch)) > llc_epoch_affinity_timeout || invalid_llc_nr(mm, p, cpu_of(rq)) || exceed_llc_capacity(mm, cpu_of(rq))) { - if (READ_ONCE(mm->sc_stat.cpu) != -1) - WRITE_ONCE(mm->sc_stat.cpu, -1); + if (READ_ONCE(grp->cpu) != -1) + WRITE_ONCE(grp->cpu, -1); } mm_sched_llc = get_pref_llc(p, mm); @@ -1814,30 +1875,35 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) static void task_tick_cache(struct rq *rq, struct task_struct *p) { struct callback_head *work = &p->cache_work; + struct sched_cache_group *grp; struct mm_struct *mm = p->mm; unsigned long epoch; if (!sched_cache_enabled()) return; - if (!mm || p->flags & PF_KTHREAD || - !mm->sc_stat.pcpu_sched) + if (!mm || p->flags & PF_KTHREAD) + return; + + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp || !grp->pcpu_sched) return; epoch = rq->cpu_epoch; /* avoid moving backwards */ - if (time_after_eq(mm->sc_stat.epoch, epoch)) + if (time_after_eq(grp->epoch, epoch)) return; - guard(raw_spinlock)(&mm->sc_stat.lock); + guard(raw_spinlock)(&grp->lock); if (work->next == work) { task_work_add(p, work, TWA_RESUME); - WRITE_ONCE(mm->sc_stat.epoch, epoch); + WRITE_ONCE(grp->epoch, epoch); } } -static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p) +static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p, + struct sched_cache_group *grp) { #ifdef CONFIG_NUMA_BALANCING int cpu, curr_cpu, nid, pref_nid; @@ -1845,7 +1911,7 @@ static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p) if (!static_branch_likely(&sched_numa_balancing)) goto out; - cpu = READ_ONCE(p->mm->sc_stat.cpu); + cpu = READ_ONCE(grp->cpu); if (cpu != -1) nid = cpu_to_node(cpu); curr_cpu = task_cpu(p); @@ -1906,6 +1972,7 @@ static void task_cache_work(struct callback_head *work) unsigned long next_scan, now = jiffies; struct task_struct *p = current, *cur; unsigned long curr_m_a_occ = 0; + struct sched_cache_group *grp; struct mm_struct *mm = p->mm; unsigned long m_a_occ = 0; cpumask_var_t cpus; @@ -1917,12 +1984,16 @@ static void task_cache_work(struct callback_head *work) if (p->flags & PF_EXITING) return; - next_scan = READ_ONCE(mm->sc_stat.next_scan); + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp) + return; + + next_scan = READ_ONCE(grp->next_scan); if (time_before(now, next_scan)) return; /* only 1 thread is allowed to scan */ - if (!try_cmpxchg(&mm->sc_stat.next_scan, &next_scan, + if (!try_cmpxchg(&grp->next_scan, &next_scan, now + max_t(unsigned long, READ_ONCE(llc_epoch_period), 1))) return; @@ -1930,8 +2001,8 @@ static void task_cache_work(struct callback_head *work) curr_cpu = task_cpu(p); if (invalid_llc_nr(mm, p, curr_cpu) || exceed_llc_capacity(mm, curr_cpu)) { - if (READ_ONCE(mm->sc_stat.cpu) != -1) - WRITE_ONCE(mm->sc_stat.cpu, -1); + if (READ_ONCE(grp->cpu) != -1) + WRITE_ONCE(grp->cpu, -1); return; } @@ -1942,7 +2013,7 @@ static void task_cache_work(struct callback_head *work) scoped_guard (cpus_read_lock) { guard(rcu)(); - get_scan_cpumasks(cpus, p); + get_scan_cpumasks(cpus, p, grp); for_each_cpu(cpu, cpus) { /* XXX sched_cluster_active */ @@ -1955,7 +2026,7 @@ static void task_cache_work(struct callback_head *work) for_each_cpu(i, sched_domain_span(sd)) { occ = fraction_mm_sched(cpu_rq(i), - per_cpu_ptr(mm->sc_stat.pcpu_sched, i)); + per_cpu_ptr(grp->pcpu_sched, i)); a_occ += occ; if (occ > m_occ) { m_occ = occ; @@ -1988,7 +2059,7 @@ static void task_cache_work(struct callback_head *work) m_a_cpu = m_cpu; } - if (llc_id(cpu) == llc_id(READ_ONCE(mm->sc_stat.cpu))) + if (llc_id(cpu) == llc_id(READ_ONCE(grp->cpu))) curr_m_a_occ = a_occ; cpumask_andnot(cpus, cpus, sched_domain_span(sd)); @@ -1997,7 +2068,7 @@ static void task_cache_work(struct callback_head *work) if (m_a_occ > (2 * curr_m_a_occ)) { /* - * Avoid switching sc_stat.cpu too fast. + * Avoid switching sched_cache_grp->cpu too fast. * The reason to choose 2X is because: * 1. It is better to keep the preferred LLC stable, * rather than changing it frequently and cause migrations @@ -2006,10 +2077,10 @@ static void task_cache_work(struct callback_head *work) * 3. 2X is chosen based on test results, as it delivers * the optimal performance gain so far. */ - WRITE_ONCE(mm->sc_stat.cpu, m_a_cpu); + WRITE_ONCE(grp->cpu, m_a_cpu); } - update_avg_scale(&mm->sc_stat.nr_running_avg, nr_running); + update_avg_scale(&grp->nr_running_avg, nr_running); free_cpumask_var(cpus); } @@ -3726,6 +3797,7 @@ static int preferred_group_nid(struct task_struct *p, int nid) static void task_numa_placement(struct task_struct *p) __context_unsafe(/* conditional locking */) { + struct sched_cache_group __maybe_unused *grp; int seq, nid, max_nid = NUMA_NO_NODE; unsigned long max_faults = 0; unsigned long fault_types[2] = { 0, 0 }; @@ -3818,19 +3890,23 @@ static void task_numa_placement(struct task_struct *p) * heuristic and occasional lost updates are tolerable. * * If a task exits, its corresponding footprint must - * be subtracted from the mm->sc_stat.footprint, otherwise - * the mm->sc_stat.footprint will not converge: - * the exiting thread's footprint remains unchanged/undecayed - * in mm->sc_stat.footprint. See exit_mm(). + * be subtracted from the mm->sched_cache_grp->footprint, + * otherwise the mm->sched_cache_grp->footprint will not + * converge: the exiting thread's footprint remains + * unchanged/undecayed in mm->sched_cache_grp->footprint. + * See exit_mm(). * * Lost updates and unsynchronized subtraction * in exit_mm() can cause footprint + diff to * go negative. Clamp to zero to prevent the * unsigned footprint from wrapping. */ - new_fp = (long)READ_ONCE(p->mm->sc_stat.footprint) + diff; - WRITE_ONCE(p->mm->sc_stat.footprint, - max(new_fp, 0L)); + grp = READ_ONCE(p->mm->sched_cache_grp); + if (!grp) + continue; + + new_fp = (long)READ_ONCE(grp->footprint) + diff; + WRITE_ONCE(grp->footprint, max(new_fp, 0L)); #endif } @@ -10778,6 +10854,7 @@ static inline bool task_misfits_asym_cpu(struct lb_env *env, struct task_struct static enum llc_mig can_migrate_llc_task(struct lb_env *env, struct task_struct *p) { + struct sched_cache_group *grp; struct mm_struct *mm; bool to_pref; int cpu, src_cpu, dst_cpu; @@ -10791,15 +10868,19 @@ static enum llc_mig can_migrate_llc_task(struct lb_env *env, if (!mm) return mig_unrestricted; - cpu = READ_ONCE(mm->sc_stat.cpu); + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp) + return mig_unrestricted; + + cpu = READ_ONCE(grp->cpu); if (cpu < 0 || cpus_share_cache(src_cpu, dst_cpu)) return mig_unrestricted; /* skip cache aware load balance for too many threads */ if (invalid_llc_nr(mm, p, dst_cpu) || exceed_llc_capacity(mm, dst_cpu)) { - if (READ_ONCE(mm->sc_stat.cpu) != -1) - WRITE_ONCE(mm->sc_stat.cpu, -1); + if (READ_ONCE(grp->cpu) != -1) + WRITE_ONCE(grp->cpu, -1); return mig_unrestricted; } -- cgit v1.2.3 From b636fef85bda7d1bab9c0a45067ab1508d79d946 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Mon, 21 Sep 2026 17:37:25 -0700 Subject: sched/cache: Introduce task_struct->sched_cache_grp to fix UAF Add a sched_cache_grp pointer to task_struct so that scheduler code can access the cache group directly via the task, without going through mm->sched_cache_grp. This decouples the scheduler's hot-path accesses from the mm_struct. Each task holds its own refcount on the sched_cache_group, separate from the reference held by its mm_struct. The reference is acquired in copy_mm() (fork) and exec_mmap() (exec), and released in exit_mm(). This fixes the use-after-free when account_mm_sched() reaches the group through a task whose mm is being switched, as reported by Hyunwoo: https://lore.kernel.org/lkml/apPb-Dr4nPYuHQOK@v4bel/ Convert all scheduler code in fair.c and exit.c to use p->sched_cache_grp instead of p->mm->sched_cache_grp. Keep the fork/exec/exit reference management out of the generic mm paths: add sched_cache_fork(), sched_cache_fork_cleanup(), sched_cache_exec_mmap() and sched_cache_exit_mm() in kernel/sched/cache_sched.c (with empty stubs for !CONFIG_SCHED_CACHE), so fs/exec.c, kernel/fork.c and kernel/exit.c each call one helper instead of open-coding the refcounting under #ifdef. Also add sched_cache_group_get() and task_cache_group_get(). Fixes: df0d98475954 ("sched/cache: Introduce infrastructure for cache-aware load balancing") Closes: https://lore.kernel.org/lkml/apPb-Dr4nPYuHQOK@v4bel/ Closes: https://lore.kernel.org/all/343a7e07-7fad-4979-9c9b-82ec038c293c@linux.dev/ Reported-by: Hyunwoo Kim Reported-by: Zenghui Yu (Huawei) Co-developed-by: Chen Yu Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: #7.2.x Link: https://patch.msgid.link/ae7081dc54736bf115215f9867abb2711a7403fb.1790035273.git.tim.c.chen@linux.intel.com --- kernel/exit.c | 33 +-------- kernel/fork.c | 2 + kernel/sched/fair.c | 196 ++++++++++++++++++++++++++++++++++++---------------- 3 files changed, 139 insertions(+), 92 deletions(-) (limited to 'kernel') diff --git a/kernel/exit.c b/kernel/exit.c index 024350e9b48c..282328d2b4cf 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -551,37 +551,6 @@ void mm_update_next_owner(struct mm_struct *mm) } #endif /* CONFIG_MEMCG */ -#if defined(CONFIG_SCHED_CACHE) && defined(CONFIG_NUMA_BALANCING) -/* - * Subtract the memory footprint of the current task from - * mm. - */ -static void exit_mm_sched_cache(struct mm_struct *mm) -{ - struct sched_cache_group *grp; - unsigned long fp, sub; - - if (!current->total_numa_faults) - return; - /* - * No lock protection due to performance considerations. - * Make sure the group footprint does not become - * negative. - */ - grp = READ_ONCE(mm->sched_cache_grp); - if (!grp) - return; - - fp = READ_ONCE(grp->footprint); - sub = min(fp, current->total_numa_faults); - WRITE_ONCE(grp->footprint, fp - sub); -} -#else -static inline void exit_mm_sched_cache(struct mm_struct *mm) -{ -} -#endif /* CONFIG_SCHED_CACHE CONFIG_NUMA_BALANCING */ - /* * Turn us into a lazy TLB process if we * aren't already.. @@ -594,7 +563,7 @@ static void exit_mm(void) if (!mm) return; - exit_mm_sched_cache(mm); + sched_cache_exit_mm(current); mmap_read_lock(mm); mmgrab_lazy_tlb(mm); diff --git a/kernel/fork.c b/kernel/fork.c index 5ef413368912..10f2d05d816a 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1599,6 +1599,7 @@ static int copy_mm(u64 clone_flags, struct task_struct *tsk) tsk->mm = mm; tsk->active_mm = mm; + sched_cache_fork(tsk); return 0; } @@ -2602,6 +2603,7 @@ bad_fork_cleanup_io: bad_fork_cleanup_namespaces: exit_nsproxy_namespaces(p); bad_fork_cleanup_mm: + sched_cache_fork_cleanup(p); if (p->mm) { mm_clear_owner(p->mm, p); mmput(p->mm); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index f0a9586332fd..974a7dfe3215 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1478,7 +1478,7 @@ static inline int get_sched_cache_scale(int mul) return (1 + (tol - 1) * mul); } -static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) +static bool exceed_llc_capacity(struct sched_cache_group *grp, int cpu) { #ifdef CONFIG_NUMA_BALANCING unsigned long llc, footprint; @@ -1492,11 +1492,6 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) return true; if (static_branch_likely(&sched_numa_balancing)) { - struct sched_cache_group *grp = READ_ONCE(mm->sched_cache_grp); - - if (!grp) - return true; - /* * TBD: RDT exclusive LLC ways reserved should be * excluded. @@ -1531,10 +1526,9 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) return false; } -static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, +static bool invalid_llc_nr(struct sched_cache_group *grp, struct task_struct *p, int cpu) { - struct sched_cache_group *grp; int scale; if (get_nr_threads(p) <= 1) @@ -1548,10 +1542,6 @@ static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, if (scale == INT_MAX) return false; - grp = READ_ONCE(mm->sched_cache_grp); - if (!grp) - return true; - return !fits_capacity((READ_ONCE(grp->nr_running_avg) * cpu_smt_num_threads), (scale * per_cpu(sd_llc_size, cpu))); } @@ -1723,6 +1713,96 @@ static void sched_cache_group_put(struct sched_cache_group *grp) call_rcu(&grp->rcu, sched_cache_group_free_rcu); } +DEFINE_FREE(sched_cache_group_put, struct sched_cache_group *, + sched_cache_group_put(_T)); + +#define rcu_deref_sched_cache_grp(tsk) \ + rcu_dereference_check((tsk)->sched_cache_grp, (tsk) == current) + +static struct sched_cache_group *sched_cache_replace_grp(struct task_struct *p, + struct sched_cache_group *new) +{ + struct sched_cache_group *old; + + old = rcu_deref_sched_cache_grp(p); + rcu_assign_pointer(p->sched_cache_grp, new); + + return old; +} + +struct sched_cache_group *sched_cache_group_get(struct sched_cache_group *grp) +{ + /* + * refcount_inc_not_zero() is the acquire primitive for lockless + * (RCU) lookups; plain refcount_inc() would scribble the count if + * it already reached zero. Return NULL in that case. + */ + if (grp && !refcount_inc_not_zero(&grp->refcnt)) + grp = NULL; + + return grp; +} + +struct sched_cache_group *task_cache_group_get(struct task_struct *p) +{ + guard(rcu)(); + return sched_cache_group_get(rcu_dereference(p->sched_cache_grp)); +} + +void sched_cache_fork(struct task_struct *p) +{ + /* + * The child takes its own reference on the mm's cache group, separate + * from the reference held by the mm. @p is not yet visible to readers, + * so a plain initializing store is enough. + */ + RCU_INIT_POINTER(p->sched_cache_grp, + sched_cache_group_get(p->mm->sched_cache_grp)); +} + +void sched_cache_fork_cleanup(struct task_struct *p) +{ + /* + * A fork that fails after sched_cache_fork() never reaches exit_mm(), + * so drop the reference here. @p never became visible, so there are no + * concurrent readers and the reference we hold keeps the group alive. + */ + sched_cache_group_put(rcu_access_pointer(p->sched_cache_grp)); + RCU_INIT_POINTER(p->sched_cache_grp, NULL); +} + +void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm) +{ + struct sched_cache_group *old; + + /* + * Acquire the new reference before publishing the pointer, then drop + * the old one. @p is current and the only writer of its own pointer. + */ + old = sched_cache_replace_grp(p, sched_cache_group_get(mm->sched_cache_grp)); + sched_cache_group_put(old); +} + +void sched_cache_exit_mm(struct task_struct *p) +{ + struct sched_cache_group *grp = sched_cache_replace_grp(p, NULL); + +#ifdef CONFIG_NUMA_BALANCING + /* + * Subtract this task's footprint from the group before dropping the + * reference, so the group footprint converges as its threads exit. + * Unlocked for performance; clamp to avoid underflow. + */ + if (grp && p->total_numa_faults) { + unsigned long fp = READ_ONCE(grp->footprint); + unsigned long sub = min(fp, p->total_numa_faults); + + WRITE_ONCE(grp->footprint, fp - sub); + } +#endif + sched_cache_group_put(grp); +} + void mm_destroy_sched(struct mm_struct *mm) { sched_cache_group_put(mm->sched_cache_grp); @@ -1777,15 +1857,10 @@ static unsigned long fraction_mm_sched(struct rq *rq, return div64_u64(NICE_0_LOAD * pcpu_sched->runtime, rq->cpu_runtime + 1); } -static int get_pref_llc(struct task_struct *p, struct mm_struct *mm) +static int get_pref_llc(struct task_struct *p, struct sched_cache_group *grp) { int mm_sched_llc = -1, mm_sched_cpu; - struct sched_cache_group *grp; - if (!mm) - return -1; - - grp = READ_ONCE(mm->sched_cache_grp); if (!grp) return -1; @@ -1819,9 +1894,8 @@ static unsigned int task_running_on_cpu(int cpu, struct task_struct *p); static inline void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) { + struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp); struct sched_cache_time *pcpu_sched; - struct sched_cache_group *grp; - struct mm_struct *mm = p->mm; int mm_sched_llc = -1; unsigned long epoch; @@ -1832,12 +1906,8 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) return; /* * init_task, kthreads and user thread created - * by user_mode_thread() don't have mm. + * by user_mode_thread() don't have a cache group. */ - if (!mm) - return; - - grp = READ_ONCE(mm->sched_cache_grp); if (!grp || !grp->pcpu_sched) return; @@ -1855,13 +1925,13 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) * its preferred state. */ if ((long)(epoch - READ_ONCE(grp->epoch)) > llc_epoch_affinity_timeout || - invalid_llc_nr(mm, p, cpu_of(rq)) || - exceed_llc_capacity(mm, cpu_of(rq))) { + invalid_llc_nr(grp, p, cpu_of(rq)) || + exceed_llc_capacity(grp, cpu_of(rq))) { if (READ_ONCE(grp->cpu) != -1) WRITE_ONCE(grp->cpu, -1); } - mm_sched_llc = get_pref_llc(p, mm); + mm_sched_llc = get_pref_llc(p, grp); /* task not on rq accounted later in account_entity_enqueue() */ if (task_running_on_cpu(rq->cpu, p) && @@ -1874,19 +1944,15 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) static void task_tick_cache(struct rq *rq, struct task_struct *p) { + struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp); struct callback_head *work = &p->cache_work; - struct sched_cache_group *grp; - struct mm_struct *mm = p->mm; unsigned long epoch; if (!sched_cache_enabled()) return; - if (!mm || p->flags & PF_KTHREAD) - return; - - grp = READ_ONCE(mm->sched_cache_grp); - if (!grp || !grp->pcpu_sched) + if (!grp || p->flags & PF_KTHREAD || + !grp->pcpu_sched) return; epoch = rq->cpu_epoch; @@ -1968,14 +2034,13 @@ static inline void update_avg_scale(u64 *avg, u64 sample) static void task_cache_work(struct callback_head *work) { + struct sched_cache_group *grp __free(sched_cache_group_put) = NULL; + cpumask_var_t cpus __free(free_cpumask_var) = CPUMASK_VAR_NULL; int cpu, m_a_cpu = -1, nr_running = 0, curr_cpu; unsigned long next_scan, now = jiffies; struct task_struct *p = current, *cur; unsigned long curr_m_a_occ = 0; - struct sched_cache_group *grp; - struct mm_struct *mm = p->mm; unsigned long m_a_occ = 0; - cpumask_var_t cpus; WARN_ON_ONCE(work != &p->cache_work); @@ -1984,7 +2049,12 @@ static void task_cache_work(struct callback_head *work) if (p->flags & PF_EXITING) return; - grp = READ_ONCE(mm->sched_cache_grp); + /* + * A reference makes sure grp is not released by others. The rcu + * lock can not be held till after zalloc_cpumask_var() below, + * because the latter might sleep. + */ + grp = task_cache_group_get(p); if (!grp) return; @@ -1999,8 +2069,8 @@ static void task_cache_work(struct callback_head *work) return; curr_cpu = task_cpu(p); - if (invalid_llc_nr(mm, p, curr_cpu) || - exceed_llc_capacity(mm, curr_cpu)) { + if (invalid_llc_nr(grp, p, curr_cpu) || + exceed_llc_capacity(grp, curr_cpu)) { if (READ_ONCE(grp->cpu) != -1) WRITE_ONCE(grp->cpu, -1); @@ -2033,9 +2103,13 @@ static void task_cache_work(struct callback_head *work) m_cpu = i; } + /* + * rcu_access_pointer() is used because the + * pointer is only compared, never dereferenced. + */ cur = rcu_dereference_all(cpu_rq(i)->curr); if (cur && !(cur->flags & (PF_EXITING | PF_KTHREAD)) && - cur->mm == mm) + rcu_access_pointer(cur->sched_cache_grp) == grp) nr_running++; } @@ -2081,7 +2155,6 @@ static void task_cache_work(struct callback_head *work) } update_avg_scale(&grp->nr_running_avg, nr_running); - free_cpumask_var(cpus); } void init_sched_mm(struct task_struct *p) @@ -2090,6 +2163,13 @@ void init_sched_mm(struct task_struct *p) init_task_work(work, task_cache_work); work->next = work; + /* + * dup_task_struct() copies the parent's task_struct, including its + * sched_cache_grp, for which the child holds no reference. Clear it + * here - before copy_mm() runs - so the child never carries a + * borrowed pointer that the fork error path would put. + */ + RCU_INIT_POINTER(p->sched_cache_grp, NULL); /* * Reset new task's preference to avoid * polluting account_llc_enqueue(). @@ -3890,10 +3970,9 @@ static void task_numa_placement(struct task_struct *p) * heuristic and occasional lost updates are tolerable. * * If a task exits, its corresponding footprint must - * be subtracted from the mm->sched_cache_grp->footprint, - * otherwise the mm->sched_cache_grp->footprint will not - * converge: the exiting thread's footprint remains - * unchanged/undecayed in mm->sched_cache_grp->footprint. + * be subtracted from p->sched_cache_grp->footprint, + * otherwise the footprint will not converge: the + * exiting thread's footprint remains unchanged/undecayed. * See exit_mm(). * * Lost updates and unsynchronized subtraction @@ -3901,12 +3980,14 @@ static void task_numa_placement(struct task_struct *p) * go negative. Clamp to zero to prevent the * unsigned footprint from wrapping. */ - grp = READ_ONCE(p->mm->sched_cache_grp); - if (!grp) - continue; + scoped_guard(rcu) { + grp = rcu_dereference(p->sched_cache_grp); - new_fp = (long)READ_ONCE(grp->footprint) + diff; - WRITE_ONCE(grp->footprint, max(new_fp, 0L)); + if (grp) { + new_fp = (long)READ_ONCE(grp->footprint) + diff; + WRITE_ONCE(grp->footprint, max(new_fp, 0L)); + } + } #endif } @@ -10855,7 +10936,6 @@ static enum llc_mig can_migrate_llc_task(struct lb_env *env, struct task_struct *p) { struct sched_cache_group *grp; - struct mm_struct *mm; bool to_pref; int cpu, src_cpu, dst_cpu; @@ -10864,11 +10944,7 @@ static enum llc_mig can_migrate_llc_task(struct lb_env *env, src_cpu = env->src_cpu; dst_cpu = env->dst_cpu; - mm = p->mm; - if (!mm) - return mig_unrestricted; - - grp = READ_ONCE(mm->sched_cache_grp); + grp = rcu_dereference_all(p->sched_cache_grp); if (!grp) return mig_unrestricted; @@ -10877,8 +10953,8 @@ static enum llc_mig can_migrate_llc_task(struct lb_env *env, return mig_unrestricted; /* skip cache aware load balance for too many threads */ - if (invalid_llc_nr(mm, p, dst_cpu) || - exceed_llc_capacity(mm, dst_cpu)) { + if (invalid_llc_nr(grp, p, dst_cpu) || + exceed_llc_capacity(grp, dst_cpu)) { if (READ_ONCE(grp->cpu) != -1) WRITE_ONCE(grp->cpu, -1); return mig_unrestricted; -- cgit v1.2.3 From 65efcccddc83d6a19e8a2e2a6117811e39e589bf Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Mon, 21 Sep 2026 17:37:26 -0700 Subject: sched/cache: Skip kernel threads for cache aware scheduling to rubustify the code Kernel thread should not be covered by cache aware scheduling as it borrows the statistics from the user space thread. Filter the kernel thread in account_mm_sched(). In theory a kernel thread does not have any valid cache group, so !grp should gate the kernel thread. Add the PF_KTHREAD check explicitly here for safety reasons, to guard against future modifications and to pair with task_tick_cache(). Fixes: df0d98475954 ("sched/cache: Introduce infrastructure for cache-aware load balancing") Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: # 7.2.x Link: https://patch.msgid.link/058f0c6ea7b991c177a17de347fa3157f25489a7.1790035273.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 974a7dfe3215..57360f5cdde4 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1907,8 +1907,14 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) /* * init_task, kthreads and user thread created * by user_mode_thread() don't have a cache group. - */ - if (!grp || !grp->pcpu_sched) + * In theory a kernel thread does not have any valid + * cache group, because sched_cache_fork() is not + * invoked for a kernel thread - !grp should gate the + * kernel thread. Use the PF_KTHREAD check explicitly + * here for safety reasons, to guard against future + * modifications and to pair with task_tick_cache(). + */ + if (p->flags & PF_KTHREAD || !grp || !grp->pcpu_sched) return; pcpu_sched = per_cpu_ptr(grp->pcpu_sched, cpu_of(rq)); -- cgit v1.2.3 From 3cb0243767fd033bdce95f4f1b5882172a2f8119 Mon Sep 17 00:00:00 2001 From: Davi Chaves Azevedo Date: Mon, 21 Sep 2026 17:37:27 -0700 Subject: sched/cache: Refresh LLC capacity across CPU hotplug, to fix capacity underestimation bug The scheduler scales LLC capacity by the fraction of cache-sharing CPUs covered by a domain: llc_bytes = cache_size * span_weight / shared_weight During CPU teardown, sched_cpu_deactivate() rebuilds scheduler domains before cacheinfo_cpu_pre_down() removes the CPU from shared_cpu_map. The new domains therefore use the old sharing weight. The later call to sched_update_llc_bytes() looks up the departing CPU's sd_llc, which has already been detached, and returns without correcting the surviving CPUs. On a Ryzen 5 7535U with twelve logical CPUs sharing a 16 MiB LLC, offlining one SMT sibling left the remaining CPUs with: llc_bytes = floor(16777216 * 11 / 12) = 15379114 bytes The correct capacity is still 16777216 bytes. On systems with active cache-aware scheduling, an underestimated capacity can cause exceed_llc_capacity() to reject aggregation for a process whose footprint would fit. Unchanged cpuset partitions sharing the physical cache can also retain stale capacity when a CPU comes online in another partition. Pass the cache-sharing mask already retained by cacheinfo to the scheduler update. Refresh every surviving CPU using its own LLC domain so that each partition receives the correct share. This also preserves the correction needed as cache-sharing maps grow during boot. Keep the existing CPU-hotplug and scheduler-domain synchronization. The update remains on the hotplug path; no steady-state scheduling operation or persistent allocation is added. Fixes: 7030513a0877 ("sched/cache: Calculate the LLC size and store it in sched_domain") Signed-off-by: Davi Chaves Azevedo Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Chen Yu Reviewed-by: Tim Chen Reviewed-by: K Prateek Nayak Tested-by: Chen Yu Tested-by: K Prateek Nayak Cc: # v7.2.x Link: https://patch.msgid.link/6751d93e15889e624796c74db0bfe66603d60b1b.1790035273.git.tim.c.chen@linux.intel.com --- kernel/sched/topology.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 0248227d983a..3dab0253976f 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -985,8 +985,8 @@ void sched_cache_active_set(void) } /* - * Update the bottom sched_domain's llc_bytes for @cpu and all its - * LLC siblings. Called from cacheinfo_cpu_online() or + * Update the bottom sched_domain's llc_bytes for @cpus sharing a physical + * LLC. Called from cacheinfo_cpu_online() or * cacheinfo_cpu_pre_down() with cpu hotplug lock held. * * Note: get_effective_llc_bytes() returns 0 on PowerPC. @@ -996,17 +996,13 @@ void sched_cache_active_set(void) * and does not populates the per-CPU struct cpu_cacheinfo array * that get_cpu_cacheinfo_llc() reads. */ -void sched_update_llc_bytes(unsigned int cpu) +void sched_update_llc_bytes(const struct cpumask *cpus) { struct sched_domain *sd, *sdp; unsigned int i; sched_domains_mutex_lock(); - sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, cpu)); - if (!sdp) - goto unlock; - /* * ci->shared_cpu_map is built incrementally as CPUs come * online, so the first CPU in an LLC initially sees @@ -1014,14 +1010,22 @@ void sched_update_llc_bytes(unsigned int cpu) * get_effective_llc_bytes(). Re-evaluating every LLC * sibling on each online event corrects this once the full * shared_cpu_map is known. + * + * The departing CPU's domains have already been detached when + * cacheinfo removes it. Use the surviving cache siblings instead. + * They may belong to different cpuset partitions, so use each CPU's + * own LLC domain to scale its share of the physical cache. */ - for_each_cpu(i, sched_domain_span(sdp)) { + for_each_cpu(i, cpus) { + sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, i)); + if (!sdp) + continue; + sd = rcu_dereference_sched_domain(cpu_rq(i)->sd); if (sd) sd->llc_bytes = get_effective_llc_bytes(i, sdp); } -unlock: sched_domains_mutex_unlock(); } -- cgit v1.2.3 From 41112a787f9182c7f2d27122817861e3f22ef928 Mon Sep 17 00:00:00 2001 From: Florian Schmaus Date: Sun, 20 Sep 2026 16:35:25 +0200 Subject: PM: hibernate: Freeze kernel threads after image preallocation Commit 783c81098445 ("PM: hibernate: call preallocate_image() after freeze prepare") moved hibernate_preallocate_memory() after dpm_prepare() so that device drivers have the opportunity to release pinned/unswappable memory during their ->prepare() callback before memory is preallocated for the snapshot image. However, that commit also placed hibernate_preallocate_memory() after freeze_kernel_threads(). While it was assumed during review that swap I/O submitted via submit_bio() is synchronous and would not depend on frozen kernel threads, this does not hold in practice. Calling hibernate_preallocate_memory() with kernel threads frozen leads to intermittent deadlocks during hibernation. Inside hibernate_preallocate_memory(), shrink_all_memory() is invoked with .may_writepage = 1 and .may_swap = 1 to aggressively reclaim and swap out pages. Any writeback or swap I/O that relies on freezable kernel threads, block device helpers, or WQ_FREEZABLE workqueues (such as those in storage drivers, device mapper, or filesystems) deadlocks waiting on tasks that are stuck in the refrigerator. Fix this by reordering hibernation_snapshot(): 1. Call dpm_prepare(PMSG_FREEZE) first, allowing device drivers to release pinned resources while kernel threads are still active. 2. Call hibernate_preallocate_memory() second, performing page reclaim and swapout while storage layers, workqueues, and kernel threads are alive. 3. Call freeze_kernel_threads() third, only after all memory preallocation and swap I/O have completed. Additionally, restore the call to swsusp_free() in the cleanup path so that preallocated image memory is properly freed if freeze_kernel_threads() fails or if TEST_FREEZER is enabled. Fixes: 783c81098445 ("PM: hibernate: call preallocate_image() after freeze prepare") Signed-off-by: Florian Schmaus Reviewed-by: Mario Limonciello (AMD) Tested-by: Matthew Leach Reviewed-by: Matthew Leach Link: https://patch.msgid.link/20260920-fix-hibernation-v1-1-f9940c2d7d7f@geekplace.eu Signed-off-by: Rafael J. Wysocki --- kernel/power/hibernate.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index d2479c69d71a..c13f68ab7f6e 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -408,9 +408,18 @@ int hibernation_snapshot(int platform_mode) if (error) goto Close; + error = dpm_prepare(PMSG_FREEZE); + if (error) + goto Complete; + + /* Preallocate image memory before freezing kernel threads and shutting down devices. */ + error = hibernate_preallocate_memory(); + if (error) + goto Complete; + error = freeze_kernel_threads(); if (error) - goto Close; + goto Cleanup; if (hibernation_test(TEST_FREEZER)) { @@ -422,15 +431,6 @@ int hibernation_snapshot(int platform_mode) goto Thaw; } - error = dpm_prepare(PMSG_FREEZE); - if (error) - goto Complete; - - /* Preallocate image memory before shutting down devices. */ - error = hibernate_preallocate_memory(); - if (error) - goto Complete; - console_suspend_all(); pm_restrict_gfp_mask(); @@ -464,10 +464,12 @@ int hibernation_snapshot(int platform_mode) platform_end(platform_mode); return error; - Complete: - dpm_complete(PMSG_RECOVER); Thaw: thaw_kernel_threads(); + Cleanup: + swsusp_free(); + Complete: + dpm_complete(PMSG_RECOVER); goto Close; } -- cgit v1.2.3 From a6c1edfbe240e4377038a0e1d233ad81fde9a21b Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Tue, 22 Sep 2026 17:20:22 +0000 Subject: bpf: Reject pkt arguments in mutating subprogs The verifier tracks changes in how PTR_TO_PACKET registers' bounds are modified across subprog boundaries. PTR_TO_PACKET registers are actually passed as PTR_TO_MEM, which is assumed valid for the entire call. This is not the case with packet memory, where a pskb_* call may invalidate its memory region. Reject BPF code that passes PTR_TO_PACKET pointers to subprogs that may mutate a packet. We cannot pass the pointer as a true PTR_TO_PACKET because we would also need to somehow pass the PTR_TO_PACKET_META or PTR_TO_PACKET_END to the subprog. Since we cannot avoid representing the pointer in the subprog as PTR_TO_MEM, only permit it if the subprog is guaranteed not to mutate the packet. Fixes: 80f281664f5a ("bpf: Support pointers in global func args") Reported-by: Nicholas Carlini Suggested-by: Nicholas Carlini Signed-off-by: Emil Tsalapatis Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260922172028.6269-6-emil@etsalapatis.com --- kernel/bpf/verifier.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index dd8bb179d39e..7ffbb804184f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -9750,6 +9750,16 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, if (check_mem_reg(env, reg, argno, arg->mem_size, BPF_READ | BPF_WRITE, NULL, NULL)) return -EINVAL; + /* + * PTR_TO_PACKET get passed as PTR_TO_MEM, preventing + * us from adjusting bounds tracking info. + */ + if ((reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) && + sub->changes_pkt_data) { + bpf_log(log, "%s is a packet pointer, but func#%d may change packet data\n", + reg_arg_name(env, argno), subprog); + return -EINVAL; + } if (!(arg->arg_type & PTR_MAYBE_NULL) && (type_may_be_null(reg->type) || bpf_register_is_null(reg))) { bpf_log(log, "%s is expected to be non-NULL\n", -- cgit v1.2.3 From f85f5917aa2fbd861c72ea71ecb14a2fa915c3f6 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Tue, 22 Sep 2026 17:20:24 +0000 Subject: bpf: Prevent variable arena/non-arena register contents The verifier marks ALU instructions that include at least one arena operand with needs_zext: These instructions are fixed up after verification to be ALU32 instructions to ensure that the result is a valid offset into an arena. However, different code paths may provide two non-arena 64-bit arguments to the same instruction. The result of the operation in that code path is wrong, since it is now unexpectedly truncated to 32 bits and zero-extended. Add logic to the verifier to ensure every instruction either always has at least one PTR_TO_ARENA argument, or never does. Since needs_zext already tracks the first scenario, add a prevent_zext field in bpf_insn_aux to track the latter. Reject instructions that use arena arguments and have prevent_zext set, or do not have arena arguments and have needs_zext set. Fixes: 6082b6c328b5 ("bpf: Recognize addr_space_cast instruction in the verifier.") Reported-by: Nicholas Carlini Suggested-by: Nicholas Carlini Signed-off-by: Emil Tsalapatis Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260922172028.6269-8-emil@etsalapatis.com --- kernel/bpf/verifier.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7ffbb804184f..41b49c56e123 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -15741,6 +15741,7 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); + struct bpf_insn_aux_data *aux = cur_aux(env); u8 opcode = BPF_OP(insn->code); int err; @@ -15752,12 +15753,23 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, /* Case where at least one operand is an arena. */ if (dst_reg->type == PTR_TO_ARENA || (src_reg && src_reg->type == PTR_TO_ARENA)) { - struct bpf_insn_aux_data *aux = cur_aux(env); if (dst_reg->type != PTR_TO_ARENA) *dst_reg = *src_reg; if (BPF_CLASS(insn->code) == BPF_ALU64) { + /* + * Only arena pointers set needs_zext, but doing so + * modifies the instruction at fixup time to an ALU32 + * and makes it unsuitable for 64-bit scalar args. We + * prevent zext from being set if the instruction has + * been previously called with non-arena registers. + */ + if (aux->prevent_zext) { + verbose(env, "same insn cannot be used with and without arena pointer\n"); + return -EINVAL; + } + /* * 32-bit operations zero upper bits automatically. * 64-bit operations need to be converted to 32. @@ -15770,6 +15782,16 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, return 0; } + /* Prevent the instruction from being used with arena pointers (see above). */ + if (env->prog->aux->arena && BPF_CLASS(insn->code) == BPF_ALU64) { + if (aux->needs_zext) { + verbose(env, "same insn cannot be used with and without arena pointer\n"); + return -EINVAL; + } + + aux->prevent_zext = true; + } + if (dst_reg->type != SCALAR_VALUE) ptr_reg = dst_reg; -- cgit v1.2.3 From 6db1ce73e9853f533eb7f413f14ba00f8ec6f80d Mon Sep 17 00:00:00 2001 From: Weiming Shi Date: Sun, 20 Sep 2026 21:23:04 +0800 Subject: bpf: Reject dev-bound-only programs on other devices __bpf_offload_dev_match() falls back to comparing offdev pointers after an exact netdev mismatch. Bound-only programs normally have NULL offdevs, so unrelated netdevs compare equal. A bound-only program on an offload-registered netdev can instead inherit a real offdev and match a sibling port. With CAP_BPF and CAP_NET_ADMIN, a caller can use bpf(BPF_LINK_CREATE) with a different target ifindex to run metadata kfuncs specialized for the bound driver on the target driver's xdp_buff. Running a veth-bound program on tun reads beyond tun's bare stack xdp_buff as a veth_xdp_buff. Oops: general protection fault, probably for non-canonical address KASAN: null-ptr-deref in range [0x0000000000000010-0x0000000000000017] RIP: 0010:veth_xdp_rx_timestamp (drivers/net/veth.c:1673) Call Trace: ... tun_build_skb (drivers/net/tun.c:1739) tun_get_user (drivers/net/tun.c:1856) tun_chr_write_iter (drivers/net/tun.c:2091) vfs_write (fs/read_write.c:595 fs/read_write.c:687) ksys_write (fs/read_write.c:739) do_syscall_64 (arch/x86/entry/syscall_64.c:84) entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:121) Kernel panic - not syncing: Fatal exception in interrupt Restrict non-offloaded programs to exact netdev matches and retain the shared-offdev fallback only for genuinely offloaded multi-port programs. Fixes: 2b3486bc2d23 ("bpf: Introduce device-bound XDP programs") Reported-by: Signed-off-by: Weiming Shi Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20260917161335.1020405-2-bestswngs@gmail.com/ Link: https://patch.msgid.link/20260920132303.4109240-3-bestswngs@gmail.com --- kernel/bpf/offload.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c index 0d6f5569588c..d855399812ee 100644 --- a/kernel/bpf/offload.c +++ b/kernel/bpf/offload.c @@ -698,6 +698,8 @@ static bool __bpf_offload_dev_match(struct bpf_prog *prog, return false; if (offload->netdev == netdev) return true; + if (!bpf_prog_is_offloaded(prog->aux)) + return false; ondev1 = bpf_offload_find_netdev(offload->netdev); ondev2 = bpf_offload_find_netdev(netdev); -- cgit v1.2.3 From cca4980630b3c7a85f53cb43c6018184ce5d4e37 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Sun, 20 Sep 2026 16:16:39 -0700 Subject: perf/core: Fix a refcount leak in attach_perf_ctx_data() The attach_perf_ctx_data() can race on global and !global cases. The global case is protected by global_ctx_data_rwsem and shares a single reference count using perf_ctx_data.global field. But when it races with !global case, it may miss to set the global field and result in a reference count leak. CPU1 CPU2 ---------------------------------------------------------------- attach_task_ctx_data(.global=1) attach_task_ctx_data(.global=0) cd1 = alloc_perf_ctx_data(); cd2 = alloc_perf_ctx_data(); // { .global = 0, .refcount = 1 }; try_cmpxchg(); // success, // task->perf_ctx_data = cd2 try_cmpxhg(); // fail; old = cd2 refcount_inc_not_zero(&old->refcount); // success // old.refcount = 2 free_perf_ctx_data(cd1); Then later detach_global_ctx_data() will see the data but it's not marked as global, so it won't call detach_task_ctx_data(). Fixes: 506e64e710ff ("perf: attach/detach PMU specific data") Assisted-by: Sashiko.dev:Gemini-3.1-pro Signed-off-by: Namhyung Kim Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260920231639.11910-1-namhyung@kernel.org --- kernel/events/core.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index db7b76d6b68a..e180134bad5e 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -5454,6 +5454,8 @@ attach_task_ctx_data(struct task_struct *task, struct kmem_cache *ctx_cache, } if (refcount_inc_not_zero(&old->refcount)) { + if (global) + old->global = true; free_perf_ctx_data(cd); /* unused */ return 0; } -- cgit v1.2.3 From 36bb85cf36cab15fb611cb44b78a5df06e4e69a2 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Mon, 10 Aug 2026 06:35:34 -0700 Subject: perf/core: Fix NULL pmu_ctx passed to pmu->sched_task() perf_pmu_sched_task() returns early when cpuctx->task_ctx is set, and cpc->task_epc is only non-NULL while a task context is scheduled in on this CPU. __perf_pmu_sched_task() therefore always passes NULL: Unable to handle kernel NULL pointer dereference at virtual address 00 pc : armv8pmu_sched_task+0x14/0x50 Call trace: armv8pmu_sched_task+0x14/0x50 (P) perf_pmu_sched_task+0xac/0x108 __perf_event_task_sched_out+0x6c/0xe0 Pass &cpc->epc instead, the CPU-wide context for this PMU, which the function already dereferences a few lines up to find pmu. armv8pmu_sched_task() is the only in-tree implementation that dereferences the argument, and it only reads ->pmu, so the oops needs BRBE, added in v6.17. Fixes: bd2756811766 ("perf: Rewrite core context handling") Signed-off-by: Puranjay Mohan Signed-off-by: Peter Zijlstra (Intel) Tested-by: Yifan Wu Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260810133540.1947118-2-puranjay@kernel.org --- kernel/events/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index e180134bad5e..7ce72f366f6c 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -3914,7 +3914,7 @@ static void __perf_pmu_sched_task(struct perf_cpu_pmu_context *cpc, perf_ctx_lock(cpuctx, cpuctx->task_ctx); perf_pmu_disable(pmu); - pmu->sched_task(cpc->task_epc, task, sched_in); + pmu->sched_task(&cpc->epc, task, sched_in); perf_pmu_enable(pmu); perf_ctx_unlock(cpuctx, cpuctx->task_ctx); -- cgit v1.2.3 From 3d8d74100954a3b17e5c5e37adfe14e16b1db103 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Mon, 10 Aug 2026 06:35:35 -0700 Subject: perf/core: Run sched_task() for PMUs with only CPU-wide events perf_pmu_sched_task() returns early when cpuctx->task_ctx is set and leaves the work to perf_ctx_sched_task_cb(), which only walks ctx->pmu_ctx_list. A PMU whose events are all CPU-wide is not on that list, so nothing calls its sched_task(). With perf record -b -e cycles -a -- ls armv8pmu_sched_task() is skipped on every switch to a task that has a perf context but no event on that PMU, and BRBE records leak across the task boundary. intel_pmu_lbr_add() calls perf_sched_cb_inc() unconditionally too, so LBR records leak the same way on x86. Drop the early return and skip only the CPCs that perf_ctx_sched_task_cb() handles. That one needs a gate of its own to make the split exact: it tests cpc->sched_cb_usage, which perf_sched_cb_inc() sets per CPU for every branch stack user, so a task with an event for that PMU pinned to another CPU would be handled twice. On x86 the second __intel_pmu_lbr_restore() finds lbr_stack_state == LBR_NONE and calls intel_pmu_lbr_reset(), throwing away the callstack the first one restored. cpc->task_epc is set only while a task context is scheduled in, and there is one epc per PMU on ctx->pmu_ctx_list, so the two gates are inverses. For the CPCs perf_pmu_sched_task() picks up, the callback now runs outside the perf_ctx_disable() and perf_ctx_enable() pair in perf_event_context_sched_in(). __perf_pmu_sched_task() disables the PMU around the call itself. Fixes: bd2756811766 ("perf: Rewrite core context handling") Signed-off-by: Puranjay Mohan Signed-off-by: Peter Zijlstra (Intel) Tested-by: Yifan Wu Link: https://patch.msgid.link/20260810133540.1947118-3-puranjay@kernel.org Cc: stable@vger.kernel.org --- kernel/events/core.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index 7ce72f366f6c..634d2ccbab82 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -3764,6 +3764,9 @@ static void perf_ctx_sched_task_cb(struct perf_event_context *ctx, list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) { cpc = this_cpc(pmu_ctx->pmu); + if (cpc->task_epc != pmu_ctx) + continue; + if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task) pmu_ctx->pmu->sched_task(pmu_ctx, task, sched_in); } @@ -3924,15 +3927,17 @@ static void perf_pmu_sched_task(struct task_struct *prev, struct task_struct *next, bool sched_in) { - struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); struct perf_cpu_pmu_context *cpc, *cpc2; - /* cpuctx->task_ctx will be handled in perf_event_context_sched_in/out */ - if (prev == next || cpuctx->task_ctx) + if (prev == next) return; - list_for_each_entry_safe(cpc, cpc2, this_cpu_ptr(&sched_cb_list), sched_cb_entry) + list_for_each_entry_safe(cpc, cpc2, this_cpu_ptr(&sched_cb_list), sched_cb_entry) { + if (cpc->task_epc) + continue; + __perf_pmu_sched_task(cpc, sched_in ? next : prev, sched_in); + } } static void perf_event_switch(struct task_struct *task, -- cgit v1.2.3 From 4409a85735cdca5c4c400c0dad1feee091872eee Mon Sep 17 00:00:00 2001 From: Liang Luo Date: Wed, 23 Sep 2026 19:07:37 +0800 Subject: sched_ext: Count SCX_EV_SUB_BYPASS_DISPATCH in the dispatch fallback When a descendant scheduler enters bypass mode, its tasks are parked in the bypass DSQs of the nearest non-bypassing ancestor, which is then responsible for running them. On behalf of such a non-bypassing host, scx_dispatch_sched() consumes those bypass DSQs from two places: the attempt made every SCX_BYPASS_HOST_NTH dispatches, and the end-of-dispatch fallback that keeps the CPU from going idle while bypassed descendants still have tasks queued. The former increments SCX_EV_SUB_BYPASS_DISPATCH but the latter does not, even though both perform the same scx_consume_dispatch_q() on the same bypass DSQ. The descendant bypass dispatches done by the fallback are therefore missing from the counter exposed via sysfs, scx_dump_state() and the scx_bpf_events() kfunc, which under-reports the actual number of such dispatches. Add the missing __scx_add_event() so the fallback counts them too. When @sch itself is bypassing, scx_dispatch_sched() takes the earlier self-bypass branch and returns before reaching these host paths; that mode is accounted for by SCX_EV_BYPASS_DISPATCH at enqueue time and is intentionally left unchanged. Fixes: 025b1bd41965 ("sched_ext: Implement hierarchical bypass mode") Signed-off-by: Liang Luo Signed-off-by: Tejun Heo --- kernel/sched/ext/inlines.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/ext/inlines.h b/kernel/sched/ext/inlines.h index ed423bcc26b8..2ff5479334cb 100644 --- a/kernel/sched/ext/inlines.h +++ b/kernel/sched/ext/inlines.h @@ -129,8 +129,10 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, * scheduler's ops.dispatch() doesn't yield any tasks. */ if (scx_bypass_dsp_enabled(sch) && - scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) + scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { + __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); return SCX_DSP_LOCAL; + } return SCX_DSP_NONE; } -- cgit v1.2.3 From a0bb6fac53fa7cf1cadb487b43d4c9276a6b82e3 Mon Sep 17 00:00:00 2001 From: Zhan Xusheng Date: Fri, 18 Sep 2026 21:29:15 +0800 Subject: sched/core: Account PSI IRQ time to the execution context, not the scheduling context psi_account_irqtime() has two callers which share rq->psi_irq_time, and they disagree about the context: __schedule() passes the outgoing rq->curr, sched_tick() passes rq->donor. Under proxy execution the donor is blocked on a mutex while rq->curr burns the CPU. The tick charges PSI_IRQ_FULL to the donor's cgroup and advances the timestamp, so the call from __schedule() then finds delta <= 0 and charges nothing. The delta is not counted twice, it lands on the wrong cgroup. Pass rq->curr, which is what the call read before commit af0c8b2bf67b ("sched: Split scheduler and execution contexts") renamed 'curr' to 'donor' across sched_tick(). Without CONFIG_SCHED_PROXY_EXEC the two rq members are a union, so this only changes anything where that option is set, and it depends on EXPERT. Fixes: af0c8b2bf67b ("sched: Split scheduler and execution contexts") Signed-off-by: Zhan Xusheng Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://patch.msgid.link/20260918132915.1236312-1-zhanxusheng@xiaomi.com --- kernel/sched/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 0b846a13c628..1fe40de6ebe3 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5798,7 +5798,7 @@ void sched_tick(void) curr = rq->curr; donor = rq->donor; - psi_account_irqtime(rq, donor, NULL); + psi_account_irqtime(rq, curr, NULL); update_rq_clock(rq); hw_pressure = arch_scale_hw_pressure(cpu_of(rq)); -- cgit v1.2.3 From c3a66e5f5bab3912e9f84223c5a982bf4333d5a1 Mon Sep 17 00:00:00 2001 From: Donggeun Yoo Date: Thu, 24 Sep 2026 19:23:20 +0900 Subject: bpf: Zero-fill other CPUs when BPF_F_CPU creates a per-cpu hash element pcpu_init_value() initializes the per-cpu area of a newly created [lru_]percpu_hash element. The area is recycled, so when the value comes from a BPF program (onallcpus == false) it writes the running CPU's slot and zeroes the rest. bpf_percpu_hash_update() passes onallcpus == true, which delegates to pcpu_copy_value(). pcpu_copy_value() writes only the CPU named in map_flags when BPF_F_CPU is set, so on the create path the other slots keep the recycled element's values: update(k1, 0xdeadc0de, BPF_F_ALL_CPUS) every CPU holds 0xdeadc0de delete(k1) element back on the freelist update(k2, 0xc0ffee, BPF_F_CPU | 0) creates, writes CPU 0 only lookup(k2) CPU 0 0xc0ffee, rest 0xdeadc0de Zero-fill the other CPUs on that arm too. Fixes: c6936161fd55 ("bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu_hash and lru_percpu_hash maps") Signed-off-by: Donggeun Yoo Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924102321.2120434-2-donggeunyoo.kernel@gmail.com --- kernel/bpf/hashtab.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 4f495dcbf670..f9464e566f10 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -1054,14 +1054,17 @@ static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr, /* When not setting the initial value on all cpus, zero-fill element * values for other cpus. Otherwise, bpf program has no way to ensure * known initial values for cpus other than current one - * (onallcpus=false always when coming from bpf prog). + * (onallcpus=false always when coming from bpf prog, + * map_flags & BPF_F_CPU when coming from syscall but setting + * only one cpu). */ - if (!onallcpus) { - int current_cpu = raw_smp_processor_id(); + if (!onallcpus || (map_flags & BPF_F_CPU)) { + int init_cpu = (map_flags & BPF_F_CPU) ? map_flags >> 32 : + raw_smp_processor_id(); int cpu; for_each_possible_cpu(cpu) { - if (cpu == current_cpu) + if (cpu == init_cpu) copy_map_value(&htab->map, per_cpu_ptr(pptr, cpu), value); else /* Since elem is preallocated, we cannot touch special fields */ zero_map_value(&htab->map, per_cpu_ptr(pptr, cpu)); -- cgit v1.2.3 From 1765a153d985c231357145e26798f9408db10e42 Mon Sep 17 00:00:00 2001 From: Guopeng Zhang Date: Thu, 24 Sep 2026 17:15:36 +0800 Subject: cgroup/pids: Restore pids.events notifications in local mode A fork rejected by the pids controller increments the counter reported by pids.events. When local event accounting is selected, however, pids_event() returns after notifying only events_local_file, leaving pids.events pollers asleep. On legacy hierarchies, pids.events.local does not exist. With pids_localevents, pids.events reports the same local counter. In both cases, pids.events changes without generating a notification. This can be reproduced with a pids_localevents mount: mkdir /tmp/test mount -t cgroup2 -o pids_localevents none /tmp/test mkdir /tmp/test/t echo 1 > /tmp/test/t/pids.max cat /tmp/test/t/pids.events # max 0 timeout 3 inotifywait -e modify /tmp/test/t/pids.events & sh -c 'echo $$ > /tmp/test/t/cgroup.procs; (true &)' 2>/dev/null wait cat /tmp/test/t/pids.events # max 1 Without this patch, inotifywait times out without reporting an event. Notify pids.events before returning from the local event path. Fixes: 3f26a885a068 ("cgroup/pids: Add pids.events.local") Cc: stable@vger.kernel.org # v6.11+ Signed-off-by: Guopeng Zhang Signed-off-by: Tejun Heo --- kernel/cgroup/pids.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'kernel') diff --git a/kernel/cgroup/pids.c b/kernel/cgroup/pids.c index ecbb839d2acb..78cdc0558d0c 100644 --- a/kernel/cgroup/pids.c +++ b/kernel/cgroup/pids.c @@ -253,6 +253,11 @@ static void pids_event(struct pids_cgroup *pids_forking, } if (!cgroup_subsys_on_dfl(pids_cgrp_subsys) || cgrp_dfl_root.flags & CGRP_ROOT_PIDS_LOCAL_EVENTS) { + /* + * pids.events reports the local counter on legacy hierarchies + * and when pids_localevents is enabled. + */ + cgroup_file_notify(&p->events_file); cgroup_file_notify(&p->events_local_file); return; } -- cgit v1.2.3 From 5bfa9f1a9dcb6ecb607adbc1c0226605c972935b Mon Sep 17 00:00:00 2001 From: Andrea Parri Date: Thu, 24 Sep 2026 11:21:39 +0200 Subject: kprobes: Fix permanent hang when flushing the kprobe optimizer Writing 0 to /proc/sys/debug/kprobes-optimization while a kprobe is jump-optimized never returns. The writer sleeps in D state forever with kprobe_sysctl_mutex held, so any later read or write of that sysctl hangs as well. For example, with vfs_read+9 as an optimizable address in this build: # cd /sys/kernel/tracing # echo 'p:myprobe vfs_read+9' >> kprobe_events # echo 1 > events/kprobes/myprobe/enable # # wait until /sys/kernel/debug/kprobes/list shows [OPTIMIZED] # echo 0 > /proc/sys/debug/kprobes-optimization INFO: task sh:246 blocked for more than 10 seconds. Call Trace: __schedule+0x1176/0x4f70 schedule+0xdc/0x2c0 schedule_timeout+0x17b/0x260 wait_for_completion+0x173/0x3c0 wait_for_kprobe_optimizer_locked+0xbc/0x130 proc_kprobes_optimization_handler+0x156/0x1b0 proc_sys_call_handler+0x324/0x490 vfs_write+0x52d/0xfe0 ksys_write+0xff/0x200 do_syscall_64+0x106/0x630 entry_SYSCALL_64_after_hwframe+0x77/0x7f ... INFO: task cat:265 is blocked on a mutex likely owned by task sh:246. wait_for_kprobe_optimizer_locked() reinitializes optimizer_completion, asks the optimizer thread to flush and sleeps in wait_for_completion(). The thread drains the (un)optimizing lists, but calls complete() only if completion_done() is true, i.e. if the completion is already done, which never happens while someone waits. disarm_all_kprobes() and kprobe_trace_self_tests_init() wait the same way. Calling complete() unconditionally would not be enough: the waiter drops kprobe_mutex while it sleeps, and nothing else serializes the sysctl handler against the debugfs "enabled" file. A second flusher that still finds the lists non-empty, e.g. because a disabled probe is queued for unoptimizing, reinitializes the completion under the first: sysctl write debugfs "enabled" write unoptimize_all_kprobes() wait_for_kprobe_optimizer_locked() init_completion(c) mutex_unlock(&kprobe_mutex) wait_for_completion(c) disarm_all_kprobes() wait_for_kprobe_optimizer_locked() init_completion(c) // c->wait is reset, the first // waiter is off the queue mutex_unlock(&kprobe_mutex) wait_for_completion(c) kprobe_optimizer() complete(c) // wakes the debugfs writer only where c is &optimizer_completion. Lining up the two writes during an optimizer pass loses the sysctl writer this way. Replace the completion with a counter of optimizer passes, bumped at the end of each pass and signalled with wake_up_var_locked(), both under kprobe_mutex. A flusher samples the count and waits with wait_var_event_mutex(), which drops kprobe_mutex only while sleeping, so a new count means a whole pass ran in the meantime. Nothing is reinitialized, so several flushers can sleep in the wait at once. Link: https://lore.kernel.org/all/20260924092142.199198-1-parri.andrea@gmail.com/ Fixes: 73c12f209462 ("kprobes: Use dedicated kthread for kprobe optimizer") Cc: stable@vger.kernel.org Assisted-by: LLM Signed-off-by: Andrea Parri Signed-off-by: Masami Hiramatsu (Google) --- kernel/kprobes.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 6337da5cab9e..4edd8ca5c657 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include @@ -526,7 +527,8 @@ enum { OPTIMIZER_ST_FLUSHING = 2, }; -static DECLARE_COMPLETION(optimizer_completion); +/* Bumped at the end of each kprobe_optimizer() pass, under 'kprobe_mutex' */ +static unsigned long optimizer_passes; #define OPTIMIZE_DELAY 5 @@ -654,9 +656,9 @@ static void kprobe_optimizer(void) do_free_cleaned_kprobes(); } - /* Step 5: Kick optimizer again if needed. But if there is a flush requested, */ - if (completion_done(&optimizer_completion)) - complete(&optimizer_completion); + /* Step 5: Wake up flushers, and kick optimizer again if needed. */ + optimizer_passes++; + wake_up_var_locked(&optimizer_passes, &kprobe_mutex); if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) kick_kprobe_optimizer(); /*normal kick*/ @@ -708,7 +710,8 @@ static void wait_for_kprobe_optimizer_locked(void) lockdep_assert_held(&kprobe_mutex); while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) { - init_completion(&optimizer_completion); + unsigned long passes = optimizer_passes; + /* * Set state to OPTIMIZER_ST_FLUSHING and wake up the thread if it's * idle. If it's already kicked, it will see the state change. @@ -717,9 +720,12 @@ static void wait_for_kprobe_optimizer_locked(void) OPTIMIZER_ST_FLUSHING) != OPTIMIZER_ST_FLUSHING) wake_up(&kprobe_optimizer_wait); - mutex_unlock(&kprobe_mutex); - wait_for_completion(&optimizer_completion); - mutex_lock(&kprobe_mutex); + /* + * kprobe_optimizer() holds 'kprobe_mutex' for a whole pass, which + * this drops while sleeping, so a new count means a full pass ran. + */ + wait_var_event_mutex(&optimizer_passes, + optimizer_passes != passes, &kprobe_mutex); } } -- cgit v1.2.3 From 31c88350b7dd1522792f726f79607f31bb55c50f Mon Sep 17 00:00:00 2001 From: Hui Peng Date: Thu, 24 Sep 2026 04:27:27 +0000 Subject: cgroup/cpuset: Return PERR_NOCPUS in remote_partition_enable() on subpartitions_cpus conflict When a remote partition is created underneath an existing local partition via a non-partition (PRS_MEMBER) intermediate cgroup, update_prstate() sees parent->partition_root_state == PRS_MEMBER and calls remote_partition_enable(). Commit 86888c7bd117 ("cgroup/cpuset: Add warnings to catch inconsistency in exclusive CPUs") replaced the cpumask_intersects(tmp->new_cpus, subpartitions_cpus) error check in remote_partition_enable() with WARN_ON_ONCE(). As a result, remote_partition_enable() emits a warning and proceeds to enable the remote partition on CPUs that are already owned by the ancestor local partition in subpartitions_cpus. This can be reproduced on Linux 7.3.0-rc3 with: mkdir -p /tmp/cg1 mount -t cgroup2 none /tmp/cg1 echo "+cpuset" > /tmp/cg1/cgroup.subtree_control mkdir /tmp/cg1/A echo 1 > /tmp/cg1/A/cpuset.cpus echo 1 > /tmp/cg1/A/cpuset.cpus.exclusive echo root > /tmp/cg1/A/cpuset.cpus.partition echo "+cpuset" > /tmp/cg1/A/cgroup.subtree_control mkdir /tmp/cg1/A/B echo 1 > /tmp/cg1/A/B/cpuset.cpus echo 1 > /tmp/cg1/A/B/cpuset.cpus.exclusive echo "+cpuset" > /tmp/cg1/A/B/cgroup.subtree_control mkdir /tmp/cg1/A/B/D echo 1 > /tmp/cg1/A/B/D/cpuset.cpus echo 1 > /tmp/cg1/A/B/D/cpuset.cpus.exclusive echo root > /tmp/cg1/A/B/D/cpuset.cpus.partition which triggers: WARNING: kernel/cgroup/cpuset.c:1594 at remote_partition_enable+0x1c1/0x300 and leaves both /tmp/cg1/A and /tmp/cg1/A/B/D as active root partitions claiming exclusive CPU 1. Fix this by returning PERR_NOCPUS when tmp->new_cpus intersects subpartitions_cpus in remote_partition_enable(), matching the error code used by remote_cpus_update() for the same subpartitions_cpus conflict, and add a regression test case to tools/testing/selftests/cgroup/test_cpuset_prs.sh. Tested in QEMU on Linux 7.3.0-rc3 using the reproducer above and tools/testing/selftests/cgroup/test_cpuset_prs.sh. Fixes: 86888c7bd117 ("cgroup/cpuset: Add warnings to catch inconsistency in exclusive CPUs") Suggested-by: Guopeng Zhang Cc: stable@vger.kernel.org Assisted-by: LLM Signed-off-by: Hui Peng Reviewed-by: Waiman Long Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 8f24171b6055..1fcec89a28b9 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1591,10 +1591,11 @@ static int remote_partition_enable(struct cpuset *cs, int new_prs, * above it or remote partition root underneath it is not allowed. */ compute_excpus(cs, tmp->new_cpus); - WARN_ON_ONCE(cpumask_intersects(tmp->new_cpus, subpartitions_cpus)); if (!cpumask_intersects(tmp->new_cpus, cpu_active_mask) || cpumask_subset(top_cpuset.effective_cpus, tmp->new_cpus)) return PERR_INVCPUS; + if (cpumask_intersects(tmp->new_cpus, subpartitions_cpus)) + return PERR_NOCPUS; if (((new_prs == PRS_ISOLATED) && !isolated_cpus_can_update(tmp->new_cpus, NULL)) || prstate_housekeeping_conflict(new_prs, tmp->new_cpus)) -- cgit v1.2.3 From 94480606a677deb68d5622cf0ded88626514b3f2 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 26 Sep 2026 09:23:02 -1000 Subject: sched_ext: Add a size argument to scx_bpf_cid_topo() so struct scx_cid_topo can grow scx_bpf_cid_topo() copies struct scx_cid_topo into a buffer the BPF program sized from its own vmlinux.h while the verifier sizes the write from the running kernel's BTF. The struct may grow and each growth then breaks every scheduler built against the older layout, rejected at load or written past its buffer. This is the usual hole for a struct handed to BPF, closed elsewhere with a size argument, and it was missed here. Take the buffer size, copy the smaller of it and the kernel's struct and set the rest to -1. Accesses to the copy are CO-RE relocated, so the struct can grow by appending fields, which its comment now states. The kfunc changes in place: the cid interface is still being finalized and no released scheduler uses the current form. Fixes: e9b55af47edf ("sched_ext: Add topological CPU IDs (cids)") Cc: stable@vger.kernel.org # v7.2+ Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/cid.c | 26 ++++++++++++++++---------- kernel/sched/ext/types.h | 4 ++++ 2 files changed, 20 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index 39f88deb94bc..d4d5f537832a 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -912,30 +912,36 @@ bool scx_cmask_empty(const struct scx_cmask *m) /** * scx_bpf_cid_topo - Copy out per-cid topology info * @cid: cid to look up - * @out__uninit: where to copy the topology info; fully written by this call + * @out: where to copy the topology info + * @out__sz: size of @out, the program's sizeof(struct scx_cid_topo) * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * - * Fill @out__uninit with the topology info for @cid. Trigger scx_error() if - * @cid is out of range. If @cid is valid but in the no-topo section, all fields - * are set to -1. All fields are also set to -1 when no cid tables have been - * published yet, which a program may observe while racing the root enable. + * Fill @out with the topology info for @cid. Trigger scx_error() if @cid is out + * of range. If @cid is valid but in the no-topo section, all fields are set to + * -1. All fields are also set to -1 when no cid tables have been published yet, + * which a program may observe while racing the root enable. + * + * The program's struct may be older or newer than the kernel's. The smaller of + * @out__sz and the kernel's size is copied and the rest of @out is set to -1. */ -__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out__uninit, +__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out, size_t out__sz, const struct bpf_prog_aux *aux) { + size_t len = min(out__sz, sizeof(*out)); struct scx_cid_topo *topo; struct scx_sched *sch; + /* the error cases and fields the kernel lacks read as -1 */ + memset(out, 0xff, out__sz); + guard(rcu)(); sch = scx_prog_sched(aux); topo = rcu_dereference(scx_cid_topo); - if (unlikely(!sch) || !cid_valid(sch, cid) || unlikely(!topo)) { - *out__uninit = SCX_CID_TOPO_NEG; + if (unlikely(!sch) || !cid_valid(sch, cid) || unlikely(!topo)) return; - } - *out__uninit = topo[cid]; + memcpy(out, &topo[cid], len); } __bpf_kfunc_end_defs(); diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index 943d8d429a2c..139176cf9fc6 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -70,6 +70,10 @@ enum scx_consts { * smaller shards if the LLC exceeds the target size. No-topo cids are packed * into their own max-sized shards. * + * New fields are appended, never inserted: scx_bpf_cid_topo() copies this + * struct out sized by the program's own layout, and an older program's copy + * must stay a prefix of the kernel's. + * * @core_cid: first cid of this cid's core (smt-sibling group) * @core_idx: global index of that core, in [0, nr_cores_at_init) * @llc_cid: first cid of this cid's LLC -- cgit v1.2.3 From db6365ced4d5855e321f772b240c0e473bcfcdd5 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 25 Sep 2026 15:25:12 +0530 Subject: workqueue: Fix NULL current_pwq deref in flush dependency check check_flush_dependency() uses current_wq_worker() to determine whether the caller is a workqueue worker and then dereferences worker->current_pwq to test whether the current workqueue is WQ_MEM_RECLAIM. current_wq_worker() only means that %current has PF_WQ_WORKER set. A kworker can reach check_flush_dependency() while it is not executing a work item. One such path is worker_thread() acting as the pool manager, where create_worker() does GFP_KERNEL allocation and the allocation path invokes the OOM notifier. In that state worker->current_pwq is NULL because current_pwq is set only by process_one_work() and cleared again after the work function returns. [ 416.760634][ T375] Call trace: [ 416.760638][ T375] check_flush_dependency+0x80/0x120 (P) [ 416.760648][ T375] __flush_work+0x98/0x224 [ 416.760657][ T375] flush_work+0x30/0x44 [ 416.760665][ T375] ... [ 416.760710][ T375] blocking_notifier_call_chain+0x58/0xa0 [ 416.760719][ T375] out_of_memory+0xb4/0x458 [ 416.760730][ T375] __alloc_pages_may_oom+0x11c/0x1a8 [ 416.760739][ T375] __alloc_pages_slowpath+0x314/0x46c [ 416.760746][ T375] __alloc_frozen_pages_noprof+0x110/0x1a4 [ 416.760753][ T375] new_slab+0x12c/0x484 [ 416.760759][ T375] ___slab_alloc+0x7a8/0xc7c [ 416.760765][ T375] __slab_alloc+0x74/0xd8 [ 416.760772][ T375] __kmalloc_cache_node_noprof+0x2ac/0x304 [ 416.760779][ T375] alloc_worker+0x28/0x60 [ 416.760785][ T375] create_worker+0x4c/0x20c [ 416.760790][ T375] worker_thread+0xe8/0x2b8 [ 416.760796][ T375] kthread+0x1a8/0x200 [ 416.760805][ T375] ret_from_fork+0x10/0x20 Guard the WQ_MEM_RECLAIM-worker warning with worker->current_pwq. If the kworker is not currently executing a work item, there is no current workqueue to diagnose with that warning. The PF_MEMALLOC warning is left unchanged so explicit reclaim context flushing a !WQ_MEM_RECLAIM target is still reported. Fixes: fca839c00a12 ("workqueue: warn if memory reclaim tries to flush !WQ_MEM_RECLAIM workqueue") Cc: stable@vger.kernel.org Assisted-by: LLM Signed-off-by: Pavankumar Kondeti Signed-off-by: Tejun Heo --- kernel/workqueue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index b8bec1689b7a..4a73d305d88a 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -3918,7 +3918,7 @@ static void check_flush_dependency(struct workqueue_struct *target_wq, WARN_ONCE(current->flags & PF_MEMALLOC, "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps", current->pid, current->comm, target_wq->name, target_func); - WARN_ONCE(worker && ((worker->current_pwq->wq->flags & + WARN_ONCE(worker && worker->current_pwq && ((worker->current_pwq->wq->flags & (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM), "workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps", worker->current_pwq->wq->name, worker->current_func, -- cgit v1.2.3