From 93d88ac4a448f200d18d1d71de0074b8e716bbee Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Tue, 25 Aug 2026 10:35:56 +0800 Subject: sched_ext: Pass the initial cpu.idle state in scx_cgroup_init_args scx_cgroup_init_args carries the initial weight and bandwidth control parameters of a cgroup to ops.cgroup_init(), but not its cpu.idle state. A cgroup that was already configured idle before the scheduler was loaded (or before it was onlined under it) is presented as non-idle, and the BPF scheduler only learns about it if cpu.idle is written again later. Add the sched_idle state to scx_cgroup_init_args and fill it in all four places that build the args: scx_tg_online() for cgroups onlined under the scheduler, scx_cgroup_init() for cgroups that already exist when the scheduler is loaded, and the sub-scheduler handover paths scx_cgroup_claim_subtree() and scx_cgroup_return_subtree(). Verified in a VM with a probe scheduler printing the init args: a cgroup configured cpu.idle=1 before loading shows sched_idle=1 in ops.cgroup_init(), the default shows 0, and later cpu.idle writes still come through ops.cgroup_set_idle(). The sub-scheduler paths are compile-tested only. Fixes: 347ed2d566da ("sched/ext: Implement cgroup_set_idle() callback") Signed-off-by: Tao Cui Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 4 +++- kernel/sched/ext/internal.h | 3 +++ kernel/sched/ext/sub.c | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 713aa26b2828..901fb0f8b976 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4766,7 +4766,8 @@ int scx_tg_online(struct task_group *tg) { .weight = tg->scx.weight, .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, - .bw_burst_us = tg->scx.bw_burst_us }; + .bw_burst_us = tg->scx.bw_burst_us, + .sched_idle = tg->scx.idle }; ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, tg->css.cgroup, &args); @@ -5187,6 +5188,7 @@ static int scx_cgroup_init(struct scx_sched *sch) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, + .sched_idle = tg->scx.idle, }; ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 0967b99a4948..076a351bb3f2 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -259,6 +259,9 @@ struct scx_cgroup_init_args { u64 bw_period_us; u64 bw_quota_us; u64 bw_burst_us; + + /* whether the cgroup is configured SCHED_IDLE via cpu.idle */ + bool sched_idle; }; enum scx_cpu_preempt_reason { diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 0554448835bd..385302d19914 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1361,6 +1361,7 @@ static s32 scx_cgroup_claim_subtree(struct scx_sched *sch) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, + .sched_idle = tg->scx.idle, }; if (tg->scx.sched != parent || @@ -1464,6 +1465,7 @@ static void scx_cgroup_return_subtree(struct scx_sched *sch) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, + .sched_idle = tg->scx.idle, }; /* the first pass must have transferred everything */ -- cgit v1.2.3 From 29871903f3a38e75e896f9a644713859490cb85f Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Tue, 1 Sep 2026 20:43:47 +0800 Subject: sched_ext: Don't deliver duplicate ops.cgroup_set_idle() for same value ops.cgroup_set_idle() is documented to be invoked when a cgroup transitions between idle and non-idle states, and scx_group_set_weight() already skips value-preserving writes. scx_group_set_idle() delivers every write unconditionally, so rewriting an already-correct cpu.idle value feeds the BPF scheduler a transition callback each time, which toggle- or accounting-based schedulers miscount. Mirror the weight guard and only deliver on an actual change. Verified with a probe scheduler printing each callback: rewriting cpu.idle=1 twice on an already-idle cgroup delivered two callbacks before and none after. Fixes: 347ed2d566da ("sched/ext: Implement cgroup_set_idle() callback") Link: https://lore.kernel.org/r/b53c61a1-4d7d-4232-941f-d48b0563d4ed@linux.dev Signed-off-by: Tao Cui Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 901fb0f8b976..5f242ab69cf4 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4933,7 +4933,8 @@ void scx_group_set_idle(struct task_group *tg, bool idle) percpu_down_read(&scx_cgroup_ops_rwsem); sch = scx_tg_knob_sched(tg); - if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle)) + if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle) && + tg->scx.idle != idle) SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); /* Update the task group's idle state */ -- cgit v1.2.3 From 0a85182723b65ad8bee8131bc38fcf0347d6679b Mon Sep 17 00:00:00 2001 From: Wanwu Li Date: Thu, 3 Sep 2026 14:06:26 +0800 Subject: sched_ext: Fix NULL sched deref in kfunc sub-sched error paths When the root scheduler has sub-scheds attached, the COMPAT kfunc wrappers scx_bpf_select_cpu_and() and scx_bpf_dsq_insert_vtime() refuse the call and report to @p's scheduler: scx_error(scx_task_sched(p), "... must be used"); The wrappers are reachable with tasks that have no scheduler. scx_bpf_select_cpu_and() is in the select_cpu kfunc group, which scx_kfunc_context_filter() opens to BPF_PROG_TYPE_SYSCALL programs; scx_bpf_dsq_insert_vtime() is in the enqueue_dispatch group, which ops.enqueue() and ops.dispatch() may call with any KF_RCU task -- the group has no kf_tasks validation, and scx_dsq_insert_preamble() checks task ownership with scx_task_on_sched() precisely because @p may be an arbitrary task. scx_task_sched(p) is p->scx.sched, which is NULL for tasks past sched_ext_dead() -- which clears it via scx_disable_and_exit_task() on exit -- and for idle tasks, which the enable paths skip as they are never scheduled through SCX. It is also an rcu_dereference_protected() that expects @p's pi_lock or rq lock, which neither wrapper holds. Passing NULL to scx_error() reaches scx_vexit(), which dereferences sch->exit_info, oopsing the kernel. One concrete trigger exercised while developing the fix: a BPF_PROG_TYPE_SYSCALL program calling the select_cpu_and wrapper on an exited-but-not-reaped task while a sub-scheduler was attached (its pid stays findable while the zombie is unreaped; faulting instruction is the scx_vexit() prologue "mov r15,[rdi+0x398]" with RDI=NULL and 0x398 the offset of sch->exit_info): sched_ext: BPF scheduler "kfunc_subsched_null" enabled sched_ext: BPF sub-scheduler "kfunc_subsched_null" enabled sched_ext: Unassociated program run_select_cpu_ (id 76) BUG: kernel NULL pointer dereference, address: 0000000000000398 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page Oops: Oops: 0000 [#1] SMP NOPTI CPU: 7 UID: 0 PID: 8201 Comm: kfunc_test_runn Tainted: G W RIP: 0010:scx_vexit+0x25/0xa0 Code: ... <4c> 8b bf 98 03 00 00 ... CR2: 0000000000000398 Call Trace: __scx_exit+0x4f/0x70 scx_bpf_select_cpu_and+0xab/0xb0 bpf_prog_430ed61a7b66e03a_run_select_cpu_and+0x9c/0xe7 ? __x64_sys_bpf+0x2c/0x40 bpf_prog_test_run_syscall+0x130/0x2f0 __sys_bpf+0x930/0x10d0 ? __x64_sys_bpf+0x2c/0x40 __x64_sys_bpf+0x2c/0x40 do_syscall_64+0xbc/0x460 entry_SYSCALL_64_after_hwframe+0x76/0x7e Read @p's scheduler under RCU instead, which the wrappers can do from their guard(rcu)(): fault it when it can be determined, and when it can't be determined -- @p is a task past sched_ext_dead() or an idle task -- there is nothing obviously wrong to report, so just refuse the call as before without faulting any scheduler. These COMPAT wrappers are scheduled for eventual removal once the deprecation grace period elapses, but until then -- and regardless of their removal timeline -- they must not oops the kernel on a task they are handed. Cc: stable@vger.kernel.org # v7.1+ Fixes: a5fa0708cbfd ("sched_ext: Enforce scheduling authority in dispatch and select_cpu operations") Suggested-by: Andrea Righi Signed-off-by: Wanwu Li Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 11 +++++++++-- kernel/sched/ext/idle.c | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 5f242ab69cf4..7e414a7c53fc 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -8949,10 +8949,17 @@ __bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, #ifdef CONFIG_EXT_SUB_SCHED /* * Disallow if any sub-scheds are attached. There is no way to tell - * which scheduler called us, just error out @p's scheduler. + * which scheduler called us, so error out @p's scheduler -- read it + * under RCU as @p's locks aren't necessarily held here. @p may be a + * task past sched_ext_dead() or an idle task, in which case its + * scheduler can't be determined and there is nothing obviously wrong + * to report; just refuse the call. */ if (unlikely(!list_empty(&sch->children))) { - scx_error(scx_task_sched(p), "__scx_bpf_dsq_insert_vtime() must be used"); + struct scx_sched *tsch = scx_task_sched_rcu(p); + + if (tsch) + scx_error(tsch, "__scx_bpf_dsq_insert_vtime() must be used"); return; } #endif diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index d2973fb3af6d..aa9fb6de0ad6 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -1142,10 +1142,17 @@ __bpf_kfunc s32 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64 #ifdef CONFIG_EXT_SUB_SCHED /* * Disallow if any sub-scheds are attached. There is no way to tell - * which scheduler called us, just error out @p's scheduler. + * which scheduler called us, so error out @p's scheduler -- read it + * under RCU as @p's locks aren't necessarily held here. @p may be a + * task past sched_ext_dead() or an idle task, in which case its + * scheduler can't be determined and there is nothing obviously wrong + * to report; just refuse the call. */ if (unlikely(!list_empty(&sch->children))) { - scx_error(scx_task_sched(p), "__scx_bpf_select_cpu_and() must be used"); + struct scx_sched *tsch = scx_task_sched_rcu(p); + + if (tsch) + scx_error(tsch, "__scx_bpf_select_cpu_and() must be used"); return -EINVAL; } #endif -- cgit v1.2.3 From 3265ef0b670180b0b946d73ee9825d6d91e98a08 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 5 Sep 2026 06:09:55 -1000 Subject: sched_ext: Rename sch to root_sch in dispatch_one() dispatch_one() uses the root scheduler for everything it does, including the two decisions to keep running @prev, which are wrong when @prev belongs to a sub-scheduler. The function has to deal with @prev's scheduler too. Rename the root's local from sch to root_sch for clarity and to make room for it. No functional change. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 7e414a7c53fc..120540cdda74 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2919,7 +2919,7 @@ static inline void maybe_queue_balance_callback(struct rq *rq) static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev) { - struct scx_sched *sch = scx_root_protected_live(); + struct scx_sched *root_sch = scx_root_protected_live(); enum scx_dsp_verdict verdict; s32 cpu = cpu_of(rq); @@ -2928,7 +2928,7 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev scx_process_sync_ecaps(rq, prev); - if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && + if ((root_sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && unlikely(rq->scx.cpu_released)) { /* * If the previous sched_class for the current CPU was not SCX, @@ -2936,8 +2936,8 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev * core. This callback complements ->cpu_release(), which is * emitted in switch_class(). */ - if (sch->ops.cpu_acquire) - SCX_CALL_OP(sch, cpu_acquire, rq, cpu, NULL); + if (root_sch->ops.cpu_acquire) + SCX_CALL_OP(root_sch, cpu_acquire, rq, cpu, NULL); rq->scx.cpu_released = false; } @@ -2955,7 +2955,7 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev * test. */ if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice && - !scx_bypassing(sch, cpu)) { + !scx_bypassing(root_sch, cpu)) { verdict = SCX_DSP_PREV; goto has_tasks; } @@ -2967,7 +2967,7 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev goto has_tasks; } - verdict = scx_dispatch_sched(sch, rq, prev, false); + verdict = scx_dispatch_sched(root_sch, rq, prev, false); if (verdict != SCX_DSP_NONE) goto has_tasks; @@ -2976,9 +2976,9 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev * %SCX_OPS_ENQ_LAST is in effect. */ if ((prev->scx.flags & SCX_TASK_QUEUED) && - (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu)) && + (!(root_sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(root_sch, cpu)) && scx_task_can_stay_on_cpu(rq, prev)) { - __scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1); + __scx_add_event(root_sch, SCX_EV_DISPATCH_KEEP_LAST, 1); verdict = SCX_DSP_PREV; goto has_tasks; } -- cgit v1.2.3 From 90f19b2816f5d243a8daf36ca83d9d9d04f00e4c Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 5 Sep 2026 06:09:56 -1000 Subject: sched_ext: Use @prev's scheduler for the keep decisions in dispatch_one() dispatch_one() tests ops flags and bypass state against the root scheduler in both places where it decides to keep running @prev: the early keep of a @prev with slice left tests the root's bypass state, and the keep-last at the end tests the root's SCX_OPS_ENQ_LAST and bypass state. Both are properties of the scheduler @prev belongs to, and put_prev_task_scx(), which acts on the outcome, reads them from that scheduler. When @prev belongs to a sub-scheduler the two sides disagree. The keep-last case is visible. The root set SCX_OPS_ENQ_LAST, so a lone @prev of a sub-scheduler is not kept and is enqueued with SCX_ENQ_LAST to a sub-scheduler that never opted in. This trips the WARN_ON_ONCE in put_prev_task_scx() for the missing flag, and the sub-scheduler queues the task like any other and triggers no follow-up scheduling event, which can lead to stalls. Test SCX_OPS_ENQ_LAST and bypass state on @prev's sched in both places and charge SCX_EV_DISPATCH_KEEP_LAST to it. Read the sched at each decision, as the dispatch in between can drop the rq lock. Fixes: 88234b075c3f ("sched_ext: Introduce scx_task_sched[_rcu]()") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 120540cdda74..adf5993fa597 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2955,7 +2955,7 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev * test. */ if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice && - !scx_bypassing(root_sch, cpu)) { + !scx_bypassing(scx_task_sched(prev), cpu)) { verdict = SCX_DSP_PREV; goto has_tasks; } @@ -2972,15 +2972,20 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev goto has_tasks; /* - * Didn't find another task to run. Keep running @prev unless - * %SCX_OPS_ENQ_LAST is in effect. + * Didn't find another task to run. Keep running @prev unless its own + * scheduler set %SCX_OPS_ENQ_LAST and takes the enqueue instead, see + * put_prev_task_scx(). Read the scheduler here as the dispatch above + * may have dropped the rq lock while @prev changed class or scheduler. */ - if ((prev->scx.flags & SCX_TASK_QUEUED) && - (!(root_sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(root_sch, cpu)) && - scx_task_can_stay_on_cpu(rq, prev)) { - __scx_add_event(root_sch, SCX_EV_DISPATCH_KEEP_LAST, 1); - verdict = SCX_DSP_PREV; - goto has_tasks; + if (prev->scx.flags & SCX_TASK_QUEUED) { + struct scx_sched *prev_sch = scx_task_sched(prev); + + if ((!(prev_sch->ops.flags & SCX_OPS_ENQ_LAST) || + scx_bypassing(prev_sch, cpu)) && scx_task_can_stay_on_cpu(rq, prev)) { + __scx_add_event(prev_sch, SCX_EV_DISPATCH_KEEP_LAST, 1); + verdict = SCX_DSP_PREV; + goto has_tasks; + } } rq->scx.flags &= ~SCX_RQ_IN_DISPATCH; return SCX_DSP_NONE; -- cgit v1.2.3 From a0d356696f87700c8c2934e3881277b0d37f0b71 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 5 Sep 2026 06:09:57 -1000 Subject: sched_ext: scx_qmap: Do not add IMMED to rescue inserts qmap's stranded fallback forces a task that can run on none of its self cids onto its first allowed cid with SCX_ENQ_RESCUE, and adds SCX_ENQ_IMMED when that cid is a time-share it holds. On such a cid the insert stops being a rescue request: 1. A task is enqueued while none of its allowed cids is in self_cids. At attach self_cids is still empty. 2. qmap inserts it into cid 0's local DSQ with SCX_ENQ_RESCUE | SCX_ENQ_IMMED. 3. The kernel finds ENQ_IMMED held on cid 0, admits the insert and skips the rescue diversion. 4. cid 0's cpu is busy, so the IMMED task is bounced back to qmap with SCX_ENQ_REENQ. 5. qmap's enqueue sees the same inputs and repeats step 2. Nothing runs in between. 6. The reenqueue limit ejects qmap with SCX_EXIT_ERROR_REENQ. The caps granted during the parent's ops.sub_attach() are delivered after the sub already holds its tasks, while the per-cid effective caps that mark the time-shares are delivered from the first dispatch after bypass lifts, so every attach that receives a time-share on a task's first allowed cid starts the loop. Drop IMMED from the rescue inserts so that step 3 diverts to the rescue path. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- tools/sched_ext/scx_qmap.bpf.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 9f6e61d7ca07..e4e51303bd29 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -358,8 +358,8 @@ s32 BPF_STRUCT_OPS(qmap_select_cid, struct task_struct *p, } /* - * A received time-shared cid is held ENQ_IMMED-only, so inserts must set - * SCX_ENQ_IMMED. + * A received time-shared cid is held ENQ_IMMED-only, so inserts meant to run + * there must set SCX_ENQ_IMMED. */ static u64 needs_immed(s32 cid) { @@ -444,9 +444,11 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) * didn't grant them or we delegated them to children - would starve in * SHARED/FIFO since we only pull from those on self cids. * - * Force it onto its first allowed cid's local DSQ. If we hold that cid - * it runs. Otherwise the insert carries SCX_ENQ_RESCUE and the kernel - * diverts the task to its rescue path. + * Force it onto its first allowed cid's local DSQ with SCX_ENQ_RESCUE. + * If we hold ENQ on that cid it runs. Otherwise the kernel diverts the + * task to its rescue path. IMMED would turn the insert into a legal + * placement on a time-shared cid and the kernel would bounce it back + * here instead of rescuing it. */ if (!cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) { s32 c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); @@ -455,7 +457,7 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) taskc->force_local = false; __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | c, slice_ns, - enq_flags | needs_immed(c) | SCX_ENQ_RESCUE); + enq_flags | SCX_ENQ_RESCUE); return; } } @@ -618,7 +620,7 @@ static bool scan_shared_dsq(bool from_timer) if (c >= 0 && c < scx_bpf_nr_cids()) { __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c, - needs_immed(c) | SCX_ENQ_RESCUE); + SCX_ENQ_RESCUE); } continue; } @@ -659,7 +661,7 @@ static bool scan_shared_dsq(bool from_timer) if (c >= 0 && c < nr_cids) { __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c, - needs_immed(c) | SCX_ENQ_RESCUE); + SCX_ENQ_RESCUE); } continue; } -- cgit v1.2.3 From 63b4ff622244483e7c530e97d787a3d6c2c38a33 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 5 Sep 2026 06:09:58 -1000 Subject: sched_ext: scx_qmap: Place only on cids whose caps are in effect qmap decides placements from self_cids, which redistribute() derives from the caps view at ops.sub_caps_updated() time. That view runs ahead of the cpus: a granted cid can be in self_cids before its cpu has reported the caps in effect through ops.sub_ecaps_updated(). ops.update_idle() only comes once BASE is in effect, so the idle-gated placements reach such a cid only through an idle bit left over from an earlier hold. The highpri scan has no gate at all: parent cpu Y, qmap cpu X grants ENQ on X to qmap sub_caps_updated() adds X to self_cids highpri scan moves a task to X with PREEMPT caps not in effect, move denied, task bounced with REENQ_CAP reject drain, enqueue the scan moves it to X again denied again dispatch syncs ecaps, sub_ecaps_updated(X) Every highpri move to X in that window is denied and bounced. The two callbacks are meant to split the roles: ops.sub_caps_updated() tracks what the node holds and drives what it delegates to its children, while ops.sub_ecaps_updated() says whether a task can run on a cpu now. qmap used the first for both. Track the caps in effect from ops.sub_ecaps_updated() as avail_cids and place only on self_cids & avail_cids, so that self_cids stays the delegation split and avail_cids gates the placement. The stranded tests keep self_cids, as they ask whether the split gives the task anywhere at all. A highpri task whose self_cids lack caps in effect waits for them instead of being moved and bounced. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- tools/sched_ext/scx_qmap.bpf.c | 83 +++++++++++++++++++++++++++++++++--------- tools/sched_ext/scx_qmap.h | 3 ++ 2 files changed, 68 insertions(+), 18 deletions(-) diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index e4e51303bd29..062bb22ee65c 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -24,6 +24,9 @@ * time-share that stays self-local. * self - The excl cpus the node kept for itself, plus all of held_shared. * owner - Who holds a cid - a child slot, CID_SELF, or CID_NONE. + * avail - Cpus whose caps are in effect, per ops.sub_ecaps_updated(). + * usable - self AND avail. Placement decisions use this: self is the + * delegation split and can run ahead of what the cpus honor. * * The scheduler splits its held-excl cpus among self and the children in * proportion to each node's cpu.weight, handing each the floor of its share as @@ -208,8 +211,8 @@ static int qmap_spin_lock(struct bpf_res_spin_lock *lock) } /* - * Try prev_cid, then scan cpus_allowed AND idle_cids AND self_cids round-robin - * from prev_cid + 1. Atomic claim retries on race; bounded by + * Try prev_cid, then scan cpus_allowed AND idle_cids AND usable_cids + * round-robin from prev_cid + 1. Atomic claim retries on race; bounded by * IDLE_PICK_RETRIES to keep the verifier's insn budget in check. */ #define IDLE_PICK_RETRIES 16 @@ -221,7 +224,7 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid, s32 cid; u32 i; - if (cmask_test(prev_cid, &qa.self_cids.mask) && + if (cmask_test(prev_cid, &qa.usable_cids.mask) && cmask_test_and_clear(prev_cid, &qa.idle_cids.mask)) return prev_cid; @@ -229,7 +232,7 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid, bpf_for(i, 0, IDLE_PICK_RETRIES) { cid = cmask_next_and2_set_wrap(&taskc->cpus_allowed, &qa.idle_cids.mask, - &qa.self_cids.mask, cid + 1); + &qa.usable_cids.mask, cid + 1); barrier_var(cid); if (cid >= nr_cids) return -1; @@ -542,7 +545,7 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) scx_bpf_dsq_insert(p, SHARED_DSQ, 0, enq_flags); cid = cmask_next_and2_set_wrap(&taskc->cpus_allowed, &qa.idle_cids.mask, - &qa.self_cids.mask, 0); + &qa.usable_cids.mask, 0); if (cid < scx_bpf_nr_cids()) scx_bpf_kick_cid(cid, SCX_KICK_IDLE); return; @@ -646,18 +649,23 @@ static bool scan_shared_dsq(bool from_timer) if (!(taskc = lookup_task_ctx(p))) return false; - /* only run highpri tasks on cids this node holds, not delegated ones */ + /* only run highpri tasks on cids this node can use right now */ if (cmask_test(this_cid, &taskc->cpus_allowed) && - cmask_test(this_cid, &qa.self_cids.mask)) + cmask_test(this_cid, &qa.usable_cids.mask)) cid = this_cid; else cid = cmask_next_and_set_wrap(&taskc->cpus_allowed, - &qa.self_cids.mask, + &qa.usable_cids.mask, this_cid + 1); if (cid >= nr_cids) { - /* stranded after the cull - rescue it from here */ - s32 c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); + s32 c; + + /* self cids lack caps in effect yet, leave it queued */ + if (cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) + continue; + /* stranded after the cull - rescue it from here */ + c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); if (c >= 0 && c < nr_cids) { __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c, @@ -1115,7 +1123,7 @@ void BPF_STRUCT_OPS(qmap_update_idle, s32 cid, bool idle) /* * The kernel delivers update_idle() for every cid this node holds * SCX_CAP_BASE on. Track every cid's idle state regardless of - * delegation: the direct-dispatch pick masks idle_cids with self_cids + * delegation: the direct-dispatch pick masks idle_cids with usable_cids * at selection, so a cid already idle when it returns to self needs no * reseed here. */ @@ -1539,6 +1547,19 @@ static __noinline void account_alloc(void) } } +/* + * usable_cids = self_cids & avail_cids. The inputs have separate writers, + * apply_partition() and qmap_sub_ecaps_updated(), so the result is rebuilt in + * full under the partition guard, in scratch first so that readers never see + * self_cids alone. + */ +static void refresh_usable(void) +{ + cmask_copy(&qa.usable_scratch.mask, &qa.self_cids.mask); + cmask_and(&qa.usable_scratch.mask, &qa.avail_cids.mask); + cmask_copy(&qa.usable_cids.mask, &qa.usable_scratch.mask); +} + /* * apply_partition - execute the plan compute_partition() built * @@ -1561,6 +1582,7 @@ __noinline void apply_partition(void) /* no excl cpu: run own tasks on the held shares, evict children */ if (!qa.part.nr_excl) { cmask_copy(&qa.self_cids.mask, &qa.held_shared.mask); + refresh_usable(); bpf_for(i, 0, MAX_SUB_SCHEDS) if (qa.sub_sched_ctxs[i].cgroup_id) scx_bpf_sub_kill(qa.sub_sched_ctxs[i].cgroup_id, @@ -1598,6 +1620,7 @@ __noinline void apply_partition(void) else if (o == CID_SELF) cmask_set(cid, &qa.self_cids.mask); } + refresh_usable(); /* * Apply each child's exclusive cids as a delta against its previous @@ -1839,8 +1862,11 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init) cmask_init(&qa.rr_cids.mask, 0, nr_cids); cmask_init(&qa.prev_rr_cids.mask, 0, nr_cids); cmask_init(&qa.self_cids.mask, 0, nr_cids); + cmask_init(&qa.avail_cids.mask, 0, nr_cids); + cmask_init(&qa.usable_cids.mask, 0, nr_cids); cmask_init(&qa.to_revoke_cids.mask, 0, nr_cids); cmask_init(&qa.to_grant_cids.mask, 0, nr_cids); + cmask_init(&qa.usable_scratch.mask, 0, nr_cids); cmask_init(&qa.held_excl.mask, 0, nr_cids); cmask_init(&qa.held_shared.mask, 0, nr_cids); @@ -1854,14 +1880,16 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init) } /* - * The root starts holding every cid. qmap_sub_ecaps_updated() maintains - * per-cid shared state as effective caps settle, and redistribute() - * rebuilds owner and self from held caps. A non-root node starts with - * nothing. + * The root starts holding every cid and gets no ecaps notifications, so + * its avail set is fixed here. qmap_sub_ecaps_updated() maintains the + * per-cid state as effective caps settle, and redistribute() rebuilds + * owner and self from held caps. A non-root node starts with nothing. */ bpf_for(i, 0, nr_cids) { if (!sub_cgroup_id) { cmask_set(i, &qa.self_cids.mask); + cmask_set(i, &qa.avail_cids.mask); + cmask_set(i, &qa.usable_cids.mask); qa.part.cid_owner[i] = CID_SELF; } else { qa.part.cid_owner[i] = CID_NONE; @@ -2002,12 +2030,31 @@ void BPF_STRUCT_OPS(qmap_sub_ecaps_updated, s32 cid, u64 before, u64 after) { /* * Effective caps updated. Track which cids hold shared caps so a self - * task placed there enqueues IMMED. + * task placed there enqueues IMMED, and which cids have ENQ_IMMED in + * effect at all (avail, see the header comment). */ - if (after & SCX_CAP_ENQ_IMMED) + if (after & SCX_CAP_ENQ_IMMED) { qa.cid_shared[cid] = (after & SCX_CAP_ENQ) ? 0 : 1; - else + cmask_set(cid, &qa.avail_cids.mask); + } else { qa.cid_shared[cid] = 0; + cmask_clear(cid, &qa.avail_cids.mask); + } + + /* + * When another runner holds the partition guard, set part_pending: + * redistribute() drains it before releasing and rr_advance() checks it + * after, so the deferred refresh lands by the next rr tick. A + * repartition that lost the guard to us runs here. + */ + if (part_try_start()) { + refresh_usable(); + part_end(); + if (__sync_fetch_and_or(&part_pending, 0)) + redistribute(); + } else { + __sync_fetch_and_or(&part_pending, 1); + } } SCX_OPS_CID_DEFINE(qmap_ops, diff --git a/tools/sched_ext/scx_qmap.h b/tools/sched_ext/scx_qmap.h index c78d61806b39..e95fffcf7b23 100644 --- a/tools/sched_ext/scx_qmap.h +++ b/tools/sched_ext/scx_qmap.h @@ -165,12 +165,15 @@ struct qmap_arena { /* bpf-internal cmasks (embedded, see struct qmap_cmask) */ struct qmap_cmask self_cids; /* cids this node runs its own tasks on */ + struct qmap_cmask avail_cids; /* cids with caps in effect on the cpu */ + struct qmap_cmask usable_cids; /* self_cids & avail_cids, placeable right now */ struct qmap_cmask idle_cids; /* idle state of all cids regardless of delegation */ struct qmap_cmask rr_cids; /* the shared pool, as a mask for grant/revoke */ /* scratch cmasks */ struct qmap_cmask to_revoke_cids; /* delta cids to revoke */ struct qmap_cmask to_grant_cids; /* delta cids to grant */ + struct qmap_cmask usable_scratch; /* refresh_usable() build area */ struct qmap_cmask prev_rr_cids; /* previous shared pool, to clear stale grants */ struct qmap_cmask held_excl; /* cids held excl (ENQ): delegatable */ struct qmap_cmask held_shared; /* cids held shared (ENQ_IMMED only): self-local */ -- cgit v1.2.3 From 89ff16f0713917303210c560eec5cd0c13bd651f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 5 Sep 2026 12:53:44 -1000 Subject: sched_ext: scx_qmap: Fix pending partition work handoff qmap can leave partition work pending with no runner. The effective-cap callback publishes its request after failing to acquire part_busy, while redistribute() checks for pending work before releasing it. Either ordering can miss a request arriving as the current runner finishes, delaying the update until the round-robin timer runs. Publish requests before trying to become the runner and release part_busy before checking for more work. Have all holders drain pending requests after releasing it, including the stats flush. Distinguish mask refreshes from repartitions so an effective-cap update only rebuilds the partition when a repartition was also requested. Fixes: e9151ed5c944 ("tools/sched_ext: scx_qmap - Expand hierarchical sub-scheduling") Reported-by: Andrea Righi Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- tools/sched_ext/scx_qmap.bpf.c | 75 ++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 062bb22ee65c..bda56c37acb5 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -1295,11 +1295,16 @@ struct { __type(value, struct round_robin_timer); } round_robin_timer SEC(".maps"); +enum part_pending_flags { + PART_REFRESH = BIT_U64(0), + PART_REDISTRIBUTE = BIT_U64(1), +}; + /* * Partition update synchronization. qa.part can be written from concurrent * contexts. This single-runner guard admits one writer at a time without * holding a lock across the grant/revoke kfuncs. part_pending coalesces - * repartition requests that arrive while it is held. + * refresh and repartition requests that arrive while it is held. * * They live in .bss, not the arena: rr_advance() runs from a bpf_timer * callback, where the verifier rejects atomic ops on arena memory. @@ -1668,33 +1673,46 @@ __noinline void apply_partition(void) } } -/* - * Recompute the split off the node's held caps and apply it. The contexts this - * runs from (the sub-sched and cgroup callbacks, the rr timer) are not - * serialized by the kernel, so a single runner does the work. A caller that - * finds the guard held leaves part_pending set; the holder drains it before - * releasing, with the rr timer as a backstop. +/** + * execute_partition - Run pending partition updates + * + * The rr timer is the backstop if the loop reaches its iteration limit. */ -static void redistribute(void) +static void execute_partition(void) { + u64 pending; s32 i; - __sync_fetch_and_or(&part_pending, 1); + bpf_for(i, 0, 1024) { + if (!part_try_start()) + break; - if (!part_try_start()) - return; + pending = __sync_fetch_and_and(&part_pending, 0); + if (pending & PART_REDISTRIBUTE) { + /* charge elapsed time before repartitioning */ + account_alloc(); + compute_partition(); + apply_partition(); + } else if (pending & PART_REFRESH) { + refresh_usable(); + } - bpf_for(i, 0, 1024) { - __sync_fetch_and_and(&part_pending, 0); - /* charge elapsed time to the current partition before rebuilding it */ - account_alloc(); - compute_partition(); - apply_partition(); + /* + * Requests are published before trying the guard. Releasing it + * before checking pending work ensures a racing request is + * either observed here or handled by a caller that acquires the + * guard. + */ + part_end(); if (!__sync_fetch_and_or(&part_pending, 0)) break; } +} - part_end(); +static void redistribute(void) +{ + __sync_fetch_and_or(&part_pending, PART_REDISTRIBUTE); + execute_partition(); } /* @@ -1708,6 +1726,7 @@ int flush_alloc(void *ctx) if (part_try_start()) { account_alloc(); part_end(); + execute_partition(); } return 0; } @@ -1765,9 +1784,7 @@ static void rr_advance(void) part_end(); - /* a resplit queued while we held the guard supersedes this rotation */ - if (__sync_fetch_and_or(&part_pending, 0)) - redistribute(); + execute_partition(); } /* advance the time-shared cid pool every round_robin_ns */ @@ -2041,20 +2058,8 @@ void BPF_STRUCT_OPS(qmap_sub_ecaps_updated, s32 cid, u64 before, u64 after) cmask_clear(cid, &qa.avail_cids.mask); } - /* - * When another runner holds the partition guard, set part_pending: - * redistribute() drains it before releasing and rr_advance() checks it - * after, so the deferred refresh lands by the next rr tick. A - * repartition that lost the guard to us runs here. - */ - if (part_try_start()) { - refresh_usable(); - part_end(); - if (__sync_fetch_and_or(&part_pending, 0)) - redistribute(); - } else { - __sync_fetch_and_or(&part_pending, 1); - } + __sync_fetch_and_or(&part_pending, PART_REFRESH); + execute_partition(); } SCX_OPS_CID_DEFINE(qmap_ops, -- cgit v1.2.3 From c7a1c6e8004ab12a9c9bfdcb603f60f9bf4a3cee Mon Sep 17 00:00:00 2001 From: fangqiurong Date: Sat, 12 Sep 2026 21:15:18 +0800 Subject: sched_ext: Close the pre-enable ops error claim window scx_alloc_and_add_sched() publishes ops->priv before scx_root_enable_workfn() switches the state to SCX_ENABLING. An error claimed via scx_bpf_error_bstr() from an associated BPF program in that window is consumed by scx_disable_workfn(), which takes the pre-enable shortcut in scx_root_disable(). The shortcut returns without any teardown and restores SCX_DISABLED with an unconditional scx_set_enable_state() xchg racing the enable workfn's own transition. The enable then completes with the claim consumed: the scheduler stays up but can never be disabled again, and bpf_scx_unreg() frees it while still in use, resulting in a use-after-free. Both WARN_ON_ONCE()s fire back to back: WARNING: kernel/sched/ext/ext.c:7522 at scx_root_enable_workfn+0xeec/0x1be0, CPU#3: scx_enable_help/276 WARNING: kernel/sched/ext/ext.c:6398 at scx_root_disable+0xb50/0xdb8, CPU#0: sched_ext_helpe/664 scx_root_enable_workfn() switches to SCX_ENABLING before the scheduler allocation, so ops->priv is never visible while SCX_DISABLED. The allocation failure path restores SCX_DISABLED. Fixes: 105dcd005be2 ("sched_ext: Introduce scx_prog_sched()") Cc: stable@vger.kernel.org Signed-off-by: fangqiurong Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index adf5993fa597..83999203a63a 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -7516,22 +7516,24 @@ static void scx_root_enable_workfn(struct kthread_work *work) #ifdef CONFIG_EXT_SUB_SCHED cgroup_get(cgrp); #endif + /* + * Transition to ENABLING to arm the disable path. Allocation failure + * still unwinds locally. Full disabling on failure applies only after + * scx_alloc_and_add_sched() succeeds. + */ + WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED); + WARN_ON_ONCE(scx_root); + sch = scx_alloc_and_add_sched(cmd, cgrp, NULL); if (IS_ERR(sch)) { ret = PTR_ERR(sch); + WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_ENABLING); goto err_free_tid_hash; } if (sch->is_cid_type) static_branch_enable(&__scx_is_cid_type); - /* - * Transition to ENABLING and clear exit info to arm the disable path. - * Failure triggers full disabling from here on. - */ - WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED); - WARN_ON_ONCE(scx_root); - atomic_long_set(&scx_nr_rejected, 0); for_each_possible_cpu(cpu) { -- cgit v1.2.3 From 9a0b159ff18c8f6fcf982bb81e15a9ceb14db43a Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 14 Sep 2026 22:12:34 -1000 Subject: sched_ext: scx_qmap: Restore unused idle claims from ops.dispatch() scx_qmap tracks idle cids itself. pick_direct_dispatch_cid() claims a cid by clearing its bit and the task is inserted into that cid's local DSQ, which kicks the CPU. When the task does not arrive, for example because the insert fell back to the global DSQ after an affinity change, the CPU wakes, finds nothing and picks idle again. That is not an idle transition, so ops.update_idle() is not called and the cid stays marked busy until an unrelated task runs on it. Restore the claim from ops.dispatch(). The kick guarantees a dispatch on the kicked CPU, and when it finds nothing to run with a NULL @prev, the CPU is going back to idle. Document the pattern in ops.update_idle(), which reports only actual transitions. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi Cc: Andrea Righi --- kernel/sched/ext/internal.h | 6 ++++++ tools/sched_ext/scx_qmap.bpf.c | 14 ++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 076a351bb3f2..0adaf649d5e0 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -572,6 +572,12 @@ struct sched_ext_ops { * * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle * tracking. + * + * Only actual transitions are reported. A CPU that is claimed with an + * idle pick and kicked but dispatches no task returns to idle without a + * transition. A scheduler tracking idle CPUs itself must restore the + * idle state from ops.dispatch() when it returns without the next task + * to run. */ void (*update_idle)(s32 cpu, bool idle); diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index bda56c37acb5..67b7c01cae55 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -818,10 +818,10 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev) batch--; cpuc->dsp_cnt--; if (!batch || !scx_bpf_dispatch_nr_slots()) { - if (scan_shared_dsq(false)) + if (scan_shared_dsq(false) || + scx_bpf_dsq_move_to_local(SHARED_DSQ, needs_immed(cid))) return; - scx_bpf_dsq_move_to_local(SHARED_DSQ, needs_immed(cid)); - return; + goto prev; } if (!cpuc->dsp_cnt) break; @@ -832,10 +832,14 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev) if (scan_shared_dsq(false)) return; - +prev: /* * No other tasks. @prev will keep running. Update its core_sched_seq as * if the task were enqueued and dispatched immediately. + * + * No @prev to keep running means the CPU goes idle. If its claim was + * never used, that is not a transition and ops.update_idle() stays + * silent. Restore the claim here. */ if (prev) { taskc = lookup_task_ctx(prev); @@ -844,6 +848,8 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev) taskc->core_sched_seq = qa.core_sched_tail_seqs[weight_to_idx(prev->scx.weight)]++; + } else { + cmask_set(cid, &qa.idle_cids.mask); } } -- cgit v1.2.3 From a9e3760b0838299649c0d57cca44daaf40ba3c33 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 14 Sep 2026 22:12:34 -1000 Subject: sched_ext: Maintain an online cid mask in the scheduler arena Schedulers on the default cid mapping treat [0, nr_online_cids) as the online set and restart on hotplug. Schedulers that install their own mapping with scx_bpf_cid_override() have no way to learn which cids are online: the count no longer identifies members and the CPU-form cpumask is unusable from cid programs. This is an obvious hole in the cid API. Add scx_bpf_online_cmask(), a kernel-maintained cmask in the scheduler's arena, allocated alongside the per-CPU scratch masks and populated after the cid mapping is finalized and before ops.init(), for child schedulers too. The pointer stays valid through ops.exit() with no reference to take. It is the arena offset as a void pointer, the same form struct_ops arena arguments arrive in. The verifier types the void return as a scalar for the program's arena cast. The mask follows the SCX hotplug notifications: seeded from cpu_active_mask and updated before ops.cid_online/offline() runs, so it lags cpu_online_mask only inside a hotplug transition. Updates walk the scheduler list under the lock that also serializes unlinking. Reads are live, not atomic snapshots. Root initialization excludes hotplug. v2: Reworded the getter kerneldoc (Andrea Righi). Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 75 +++++++++++++++++++++++++++++--- kernel/sched/ext/internal.h | 3 +- kernel/sched/ext/sub.c | 10 +++-- tools/sched_ext/include/scx/common.bpf.h | 1 + 4 files changed, 78 insertions(+), 11 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 83999203a63a..70b711c4de6e 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3670,8 +3670,20 @@ static void handle_hotplug(struct rq *rq, bool online) s16 *tbl = rcu_dereference_check(scx_cpu_to_cid_tbl, lockdep_is_cpus_held()); - if (tbl) + if (tbl) { + struct scx_sched *pos; + cpu_or_cid = tbl[cpu]; + + guard(raw_spinlock_irqsave)(&scx_sched_lock); + list_for_each_entry(pos, &scx_sched_all, all) { + struct scx_cmask *mask = pos->online_cmask; + + if (mask) + __assign_bit(cpu_or_cid, (unsigned long *)mask->bits, + online); + } + } } if (online && SCX_HAS_OP(sch, cpu_online)) @@ -5280,12 +5292,17 @@ static void free_exit_info(struct scx_exit_info *ei); static const char *scx_exit_reason(enum scx_exit_kind kind); static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind); -s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) +s32 scx_alloc_kern_arena_objs(struct scx_sched *sch) { size_t size = struct_size_t(struct scx_cmask, bits, SCX_CMASK_NR_WORDS(num_possible_cpus())); + struct scx_cmask *online; + struct scx_cmask_ref ref; int cpu; + /* hotplug stays excluded until the online mask is published */ + lockdep_assert_cpus_held(); + if (!sch->is_cid_type || !sch->arena_pool) return 0; @@ -5301,15 +5318,28 @@ s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) return -ENOMEM; scx_cmask_init(*slot, 0, num_possible_cpus()); } + + /* pack the online mask alongside the scratch masks */ + online = scx_arena_alloc(sch, size); + if (!online) + return -ENOMEM; + + scoped_guard(rcu) { + scx_cmask_ref_init_kern(sch, online, 0, num_possible_cpus(), &ref); + scx_cmask_ref_from_cpumask(&ref, cpu_active_mask); + } + sch->online_cmask = online; + return 0; } -static void scx_set_cmask_scratch_free(struct scx_sched *sch) +static void scx_free_kern_arena_objs(struct scx_sched *sch) { size_t size = struct_size_t(struct scx_cmask, bits, SCX_CMASK_NR_WORDS(num_possible_cpus())); int cpu; + scx_arena_free(sch, sch->online_cmask, size); if (!sch->set_cmask_scratch) return; @@ -5396,7 +5426,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); free_exit_info(sch->exit_info); - scx_set_cmask_scratch_free(sch); + scx_free_kern_arena_objs(sch); scx_arena_pool_destroy(sch); if (sch->arena_map) bpf_map_put(sch->arena_map); @@ -7601,7 +7631,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) goto err_disable; } - ret = scx_set_cmask_scratch_alloc(sch); + ret = scx_alloc_kern_arena_objs(sch); if (ret) { cpus_read_unlock(); goto err_disable; @@ -10338,13 +10368,45 @@ __bpf_kfunc u32 scx_bpf_nr_cids(void) * hotplug, which lets schedulers treat [0, nr_online_cids) as the online * range. Schedulers that prefer to handle hotplug without a restart should * install a custom mapping via scx_bpf_cid_override() and track onlining - * through the ops.cid_online / ops.cid_offline callbacks. + * through the ops.cid_online / ops.cid_offline callbacks, starting from the + * mask scx_bpf_online_cmask() returns. */ __bpf_kfunc u32 scx_bpf_nr_online_cids(void) { return num_online_cpus(); } +/** + * scx_bpf_online_cmask - Return the online cid mask in the scheduler arena + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return a kernel-maintained cmask covering [0, scx_bpf_nr_cids()), or NULL if + * the calling program is not associated with a live cid-form scheduler or the + * mask is not allocated yet, as in ops.init_cids(). Treat the mask as read-only + * even though arena memory stays writable by the BPF scheduler. The mask + * follows the SCX hotplug notifications: a cid's bit is updated before + * ops.cid_online/offline() runs for it. The pointer is valid from ops.init() + * through ops.exit(). Root ops.init() runs with hotplug excluded. Other + * contexts can observe concurrent updates. + */ +__bpf_kfunc const void *scx_bpf_online_cmask(const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + struct scx_cmask *online; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return NULL; + online = sch->online_cmask; + if (unlikely(!online)) + return NULL; + + /* BPF rebases by the low 32 bits, like __arena callback args */ + return (void *)((unsigned long)online - sch->arena_kern_base); +} + /** * scx_bpf_this_cid - Return the cid of the CPU this program is running on * @@ -10708,6 +10770,7 @@ BTF_ID_FLAGS(func, scx_bpf_nr_node_ids) BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids) BTF_ID_FLAGS(func, scx_bpf_nr_cids) BTF_ID_FLAGS(func, scx_bpf_nr_online_cids) +BTF_ID_FLAGS(func, scx_bpf_online_cmask, KF_IMPLICIT_ARGS | KF_ARENA_RET) BTF_ID_FLAGS(func, scx_bpf_this_cid) BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 0adaf649d5e0..3464e0f113c1 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1561,6 +1561,7 @@ struct scx_sched { * and passes it to the callback's __arena argument. */ struct scx_cmask * __percpu *set_cmask_scratch; + struct scx_cmask *online_cmask; DECLARE_BITMAP(has_op, SCX_OPI_END); @@ -2087,7 +2088,7 @@ void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p); void scx_cgroup_lock(void); void scx_cgroup_unlock(void); #endif -s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch); +s32 scx_alloc_kern_arena_objs(struct scx_sched *sch); void scx_disable_bypass_dsp(struct scx_sched *sch); void scx_bypass(struct scx_sched *sch, bool bypass); s32 scx_link_sched(struct scx_sched *sch); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 385302d19914..f7aeb1488566 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1805,6 +1805,12 @@ void scx_sub_enable_workfn(struct kthread_work *work) goto err_disable; } + scoped_guard(cpus_read_lock) { + ret = scx_alloc_kern_arena_objs(sch); + if (ret) + goto err_disable; + } + if (sch->ops.init) { ret = SCX_CALL_OP_RET(sch, init, NULL); if (ret) { @@ -1815,10 +1821,6 @@ void scx_sub_enable_workfn(struct kthread_work *work) sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; } - ret = scx_set_cmask_scratch_alloc(sch); - if (ret) - goto err_disable; - struct scx_sub_attach_args sub_attach_args = { .ops = &sch->ops, .cgroup_path = sch->cgrp_path, diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 76f5e025e107..2ddb01a059fd 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -113,6 +113,7 @@ s32 scx_bpf_this_cid(void) __ksym __weak; struct task_struct *scx_bpf_cid_curr(s32 cid) __ksym __weak; u32 scx_bpf_nr_cids(void) __ksym __weak; u32 scx_bpf_nr_online_cids(void) __ksym __weak; +const void __arena *scx_bpf_online_cmask(void) __ksym __weak; u32 scx_bpf_cidperf_cap(s32 cid) __ksym __weak; u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak; s32 scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak; -- cgit v1.2.3