From 14eba6d98cabe625218fae0bfed8959d97226bc3 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Tue, 25 Aug 2026 10:35:57 +0800 Subject: sched_ext: Rename tg->scx.idle to tg->scx.sched_idle A bare "idle" reads as CPU idle state in sched_ext (ops.update_idle(), idle cpumasks, scx_bpf_pick_idle_cpu()). cpu.idle is the cgroup analog of the SCHED_IDLE policy, so name the task_group field sched_idle to match scx_cgroup_init_args.sched_idle. Pure rename, no behavior change. Signed-off-by: Tao Cui Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 8 ++++---- kernel/sched/ext/sub.c | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 901fb0f8b976..8041c87a3562 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4684,7 +4684,7 @@ void scx_tg_init(struct task_group *tg) tg->scx.weight = CGROUP_WEIGHT_DFL; tg->scx.bw_period_us = default_bw_period_us(); tg->scx.bw_quota_us = RUNTIME_INF; - tg->scx.idle = false; + tg->scx.sched_idle = false; } /** @@ -4767,7 +4767,7 @@ int scx_tg_online(struct task_group *tg) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, - .sched_idle = tg->scx.idle }; + .sched_idle = tg->scx.sched_idle }; ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, tg->css.cgroup, &args); @@ -4937,7 +4937,7 @@ void scx_group_set_idle(struct task_group *tg, bool idle) SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); /* Update the task group's idle state */ - tg->scx.idle = idle; + tg->scx.sched_idle = idle; percpu_up_read(&scx_cgroup_ops_rwsem); } @@ -5188,7 +5188,7 @@ static int scx_cgroup_init(struct scx_sched *sch) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, - .sched_idle = tg->scx.idle, + .sched_idle = tg->scx.sched_idle, }; ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 385302d19914..a17d84db93bd 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1361,7 +1361,7 @@ static s32 scx_cgroup_claim_subtree(struct scx_sched *sch) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, - .sched_idle = tg->scx.idle, + .sched_idle = tg->scx.sched_idle, }; if (tg->scx.sched != parent || @@ -1465,7 +1465,7 @@ static void scx_cgroup_return_subtree(struct scx_sched *sch) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, - .sched_idle = tg->scx.idle, + .sched_idle = tg->scx.sched_idle, }; /* the first pass must have transferred everything */ -- cgit v1.2.3 From 92344fad5b54d1c20ea19f882a4f086ca68c11c0 Mon Sep 17 00:00:00 2001 From: fangqiurong Date: Tue, 1 Sep 2026 09:32:58 +0800 Subject: sched_ext: Skip per-CPU data allocation for built-in DSQs Only user DSQs use the per-DSQ deferred reenqueue state: schedule_dsq_reenq() serves the local DSQ through sch->pcpu->deferred_reenq_local and rejects all other built-in DSQ ids. Every DSQ, built-in or not, still allocates a struct scx_dsq_pcpu per possible CPU. As the local, reject, rescue and bypass DSQs are themselves per-CPU, the wasted memory grows with the square of the CPU count, most of it at boot before any scheduler is loaded. Skip the allocation for built-in DSQ ids. While at it, rename ->pcpu to ->pcpu_user so that the per-CPU data reads as user-DSQ only. Signed-off-by: fangqiurong Reviewed-by: Zhan Xusheng Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 8041c87a3562..0e8ddc307ecb 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1127,7 +1127,7 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, } else if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) { rq = this_rq(); - struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu, cpu_of(rq)); + struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu_user, cpu_of(rq)); struct scx_deferred_reenq_user *dru = &dsq_pcpu->deferred_reenq_user; /* @@ -5051,12 +5051,16 @@ s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch) dsq->id = dsq_id; dsq->sched = sch; - dsq->pcpu = alloc_percpu(struct scx_dsq_pcpu); - if (!dsq->pcpu) + /* per-DSQ deferred reenq state is only needed for user DSQs */ + if (dsq_id & SCX_DSQ_FLAG_BUILTIN) + return 0; + + dsq->pcpu_user = alloc_percpu(struct scx_dsq_pcpu); + if (!dsq->pcpu_user) return -ENOMEM; for_each_possible_cpu(cpu) { - struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); + struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu); pcpu->dsq = dsq; INIT_LIST_HEAD(&pcpu->deferred_reenq_user.node); @@ -5069,8 +5073,11 @@ static void exit_dsq(struct scx_dispatch_q *dsq) { s32 cpu; + if (!dsq->pcpu_user) + return; + for_each_possible_cpu(cpu) { - struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); + struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu); struct scx_deferred_reenq_user *dru = &pcpu->deferred_reenq_user; struct rq *rq = cpu_rq(cpu); @@ -5084,7 +5091,7 @@ static void exit_dsq(struct scx_dispatch_q *dsq) } } - free_percpu(dsq->pcpu); + free_percpu(dsq->pcpu_user); } static void free_dsq_rcufn(struct rcu_head *rcu) -- cgit v1.2.3 From 00d977d84226356fe766977d48bfbdf72169a281 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Wed, 2 Sep 2026 10:48:11 +0800 Subject: sched_ext: Document the rolling-cursor requirement for dsq_vtime scx_bpf_dsq_insert_vtime() orders tasks by time_before64(), which is only meaningful when the values within a given DSQ stay less than 2^63 apart. This is implicit in how a vtime scheduler works but not spelled out anywhere. Document it so BPF scheduler authors know the constraint. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 0e8ddc307ecb..6a5c83aea047 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -8910,7 +8910,9 @@ struct scx_bpf_dsq_insert_vtime_args { * * @args->vtime ordering is according to time_before64() which considers * wrapping. A numerically larger vtime may indicate an earlier position in the - * ordering and vice-versa. + * ordering and vice-versa. vtime is a rolling cursor and values used for + * ordering within a given DSQ should stay less than 2^63 apart for + * time_before64() ordering to remain well-defined. * * A DSQ can only be used as a FIFO or priority queue at any given time and this * function must not be called on a DSQ which already has one or more FIFO tasks -- cgit v1.2.3 From c659e506f9a77d7a0e100d3555bccd5a44babf0e Mon Sep 17 00:00:00 2001 From: Wanwu Li Date: Wed, 2 Sep 2026 17:36:11 +0800 Subject: sched_ext: Reject NMI calls to lock-taking kfuncs commit e06ece82d7b0 ("sched_ext: Report NMI kicks with scx_error()") made scx_bpf_kick_cpu() reject NMI calls, and its cover letter describes the reachability: sched_ext kfuncs in the "any" category "are callable from tracing progs that can attach to functions running in NMI", and an unlucky call from there "could deadlock the machine". The fix in that series made the error/exit path lock-free so scx_error() is safe to call from NMI. That closes the *error* path of every kfunc, but not a kfunc's own business-logic lock acquisition on its success path. The remaining lock-taking kfuncs that scx_kfunc_context_filter() exposes to BPF_PROG_TYPE_TRACING have the same hazard: if an NMI lands on a CPU whose interrupted context already holds the lock, the kfunc's raw spinlock acquisition spins forever and hard-locks the CPU: - scx_bpf_destroy_dsq() -> dsq->lock - scx_bpf_dsq_reenq() -> rq's deferred_reenq_lock - scx_bpf_cpuperf_set() / scx_bpf_cidperf_set() -> rq->lock - scx_bpf_sub_grant() / scx_bpf_sub_revoke() -> pshard lock (via the shared sub_cap_preamble()) - bpf_iter_scx_dsq_next() / bpf_iter_scx_dsq_destroy() -> dsq->lock (the reject is placed in bpf_iter_scx_dsq_new(), which is otherwise lockless, so next() and destroy() are no-ops while it leaves kit->dsq NULL) As things stand, there is no scenario for reenqueueing, iterating a DSQ, setting a performance target or granting sub-caps from NMI. The guards defend against a buggy or malicious BPF program turning an "any"-category kfunc into a machine-wide hard-lockup through the door that scx_kfunc_context_filter() already opens. This matches the intent of scx_bpf_kick_cpu()'s NMI check, which the commit cited above added not to enable an NMI use case but to surface such a bug as a clean abort. Route all of them through a new scx_kf_allowed_ctx() helper and reuse scx_bpf_kick_cpu()'s existing in_nmi() check - now shared with its cid equivalent scx_bpf_kick_cid() through scx_kick_cpu() - so the rule is stated once and the coverage is auditable from one place. scx_error() is already NMI-safe (commit f883dbb64ca5 ("sched_ext: Make exit claiming lock-free")), so the reject-abort cannot deadlock the lock acquisition. Kfuncs with an error return report -EDEADLK, the situation being avoided. Read-only members of the reachable sets (dsq_peek, dsq_nr_queued, cpuperf_cur/cap, sub_caps, the idle cpumask helpers and the cid lookups) take no scheduler lock on the path a tracing program reaches them, and were audited to that effect; they are correctly left unguarded. The select_cpu kfuncs do take pi_lock, but scx_kfunc_context_filter() only exposes the any/idle/cid sets to BPF_PROG_TYPE_TRACING, and struct_ops programs never run in NMI, so no lock-taking path here is reachable from NMI. Link: https://lore.kernel.org/r/20260901095652.1009104-1-liwanwu@kylinos.cn Signed-off-by: Wanwu Li Acked-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 30 +++++++++++++++++++++--------- kernel/sched/ext/internal.h | 16 ++++++++++++++++ kernel/sched/ext/sub.c | 3 +++ 3 files changed, 40 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 6a5c83aea047..387953b92668 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -9537,14 +9537,8 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) struct rq *this_rq; unsigned long irq_flags; - /* - * The per-cpu kick list is guarded only by local_irq_save(), which does - * not mask NMIs, so kicking from NMI could corrupt it and is unsupported. - */ - if (unlikely(in_nmi())) { - scx_error(sch, "scx_bpf_kick_cpu() called from NMI"); + if (!scx_kf_allowed_ctx(sch)) return; - } local_irq_save(irq_flags); @@ -9712,8 +9706,13 @@ __bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux) guard(rcu)(); sch = scx_prog_sched(aux); - if (sch) - destroy_dsq(sch, dsq_id); + if (unlikely(!sch)) + return; + + if (!scx_kf_allowed_ctx(sch)) + return; + + destroy_dsq(sch, dsq_id); } /** @@ -9750,6 +9749,9 @@ __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, if (unlikely(!sch)) return -ENODEV; + if (!scx_kf_allowed_ctx(sch)) + return -EDEADLK; + if (flags & ~__SCX_DSQ_ITER_USER_FLAGS) return -EINVAL; @@ -9876,6 +9878,9 @@ __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, return; } + if (!scx_kf_allowed_ctx(sch)) + return; + /* not specifying any filter bits is the same as %SCX_REENQ_ANY */ if (!(reenq_flags & __SCX_REENQ_FILTER_MASK)) reenq_flags |= SCX_REENQ_ANY; @@ -10263,6 +10268,9 @@ __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_au if (unlikely(!sch)) return; + if (!scx_kf_allowed_ctx(sch)) + return; + scx_cpuperf_set(sch, cpu, perf); } @@ -10288,6 +10296,10 @@ __bpf_kfunc s32 scx_bpf_cidperf_set(s32 cid, u32 perf, sch = scx_prog_sched(aux); if (unlikely(!sch)) return -ENODEV; + + if (!scx_kf_allowed_ctx(sch)) + return -EDEADLK; + cpu = scx_cid_to_cpu(sch, cid); if (cpu < 0) return cpu; diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 076a351bb3f2..4f876e8f0655 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -2115,6 +2115,22 @@ extern struct scx_sched *scx_enabling_sub_sched; #define scx_error(sch, fmt, args...) \ scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) +/* + * Tracing progs can call kfuncs from NMI. Kfuncs that take scheduler locks or + * touch the kick lists, which are only protected by irq masking, can't run + * there, so abort the scheduler instead. scx_error() is NMI-safe. + */ +static __always_inline bool __scx_kf_allowed_ctx(struct scx_sched *sch, const char *who) +{ + if (unlikely(in_nmi())) { + scx_error(sch, "%s called from NMI", who); + return false; + } + return true; +} + +#define scx_kf_allowed_ctx(sch) __scx_kf_allowed_ctx((sch), __func__) + /** * scx_root_protected_live - Root sched for paths that only run while live * diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index a17d84db93bd..380a5653dc52 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -2267,6 +2267,9 @@ static s32 sub_cap_preamble(u64 cgroup_id, u64 caps, const struct bpf_prog_aux * if (unlikely(!parent)) return -ENODEV; + if (!scx_kf_allowed_ctx(parent)) + return -EDEADLK; + if (!scx_is_cid_type()) { scx_error(parent, "sub-cap kfuncs require a cid-form scheduler"); return -EOPNOTSUPP; -- cgit v1.2.3 From 8848333264b73cb6617fc04e11c245ab1bd22b28 Mon Sep 17 00:00:00 2001 From: Wanwu Li Date: Thu, 3 Sep 2026 11:57:19 +0800 Subject: sched_ext: Make scx_locked_rq() return NULL from NMI scx_locked_rq() reads the per-CPU scx_locked_rq_state, which tracks the rq locked by the context running on the CPU. Tracing progs can call kfuncs from NMI, and an NMI interrupts - rather than replaces - the context that set scx_locked_rq_state, so a non-NULL read from NMI falsely tells the caller that it holds the interrupted context's rq lock. Three "any"-category kfuncs read scx_locked_rq() on their success path and take an unsafe fast path on a non-NULL return: - scx_bpf_task_set_slice() writes p->scx.slice directly, racing update_curr_scx()'s non-atomic read-modify-write of the same field. - scx_bpf_dsq_nr_queued() resolves %SCX_DSQ_LOCAL to (scx_locked_rq() ?: this_rq()) and can report the interrupted context's local DSQ length instead of the caller's. - scx_bpf_locked_rq() hands the interrupted context's rq to the BPF program, which may then operate on it as if it owned the rq lock. Make scx_locked_rq() return NULL from NMI so that all three take their unlocked paths: scx_bpf_task_set_slice() stashes the request into the atomic p->scx.slice_oob for application under the rq lock, scx_bpf_dsq_nr_queued() falls back to this_rq(), and scx_bpf_locked_rq() reports an error and aborts the scheduler through the NMI-safe exit path. The kfuncs that take scheduler locks reject NMI calls through scx_kf_allowed_ctx() before reaching scx_locked_rq(), and the internal callers only run from struct_ops callbacks, which never run in NMI, so no other caller is affected. Suggested-by: Tejun Heo Link: https://lore.kernel.org/r/d84b31727f04e1ed0d40042ba1c09e61@kernel.org Signed-off-by: Wanwu Li Signed-off-by: Tejun Heo --- kernel/sched/ext/internal.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 4f876e8f0655..e1eb3a0d456c 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -2221,6 +2221,15 @@ static inline void scx_schedule_reenq_local(struct rq *rq, u64 reenq_flags) */ static inline struct rq *scx_locked_rq(void) { + /* + * Tracing progs can call kfuncs from NMI. scx_locked_rq_state tracks + * the rq locked by the interrupted context, so a non-NULL read from + * NMI would falsely claim its lock. Return NULL from NMI so that + * callers take their unlocked paths. + */ + if (unlikely(in_nmi())) + return NULL; + return __this_cpu_read(scx_locked_rq_state); } -- cgit v1.2.3 From 8b9b3698796a6b99f43fd09de23bfa5431e0f07e Mon Sep 17 00:00:00 2001 From: Wanwu Li Date: Thu, 3 Sep 2026 11:57:54 +0800 Subject: sched_ext: Protect the idle-search scratch nodemask with irqsave pick_idle_cpu_from_online_nodes() uses the per-CPU per_cpu_unvisited nodemask as scratch while walking the online nodes, protected only by preempt_disable(). preempt_disable() does not mask IRQs and the idle kfuncs are callable from IRQ-enabled contexts, so a nested invocation on the same CPU can overwrite the mask with nodes_copy() while the interrupted invocation is still iterating it, leading to a wrong node traversal and a wrong idle CPU pick. Switch to irqsave so a nested invocation can't run on the same CPU. A stack-allocated nodemask would also close the race, but that would enlarge the diff to fix a race that is already rare (per-node idle, CONFIG_NUMA and a cross-node search all at once); irqsave is the minimal fix for the context that actually triggers it. The NMI case is deliberately not addressed: there is no legitimate reason to call pick_idle from NMI and doing so poses no crash risk, so such a caller is on its own. Suggested-by: Tejun Heo Link: https://lore.kernel.org/r/d84b31727f04e1ed0d40042ba1c09e61@kernel.org Signed-off-by: Wanwu Li Signed-off-by: Tejun Heo --- kernel/sched/ext/idle.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index d2973fb3af6d..93e2e0b2d1f8 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -153,7 +153,18 @@ static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, i nodemask_t *unvisited; s32 cpu = -EBUSY; - preempt_disable(); + /* + * @per_cpu_unvisited is per-CPU scratch and the idle kfuncs can be + * called from IRQ-enabled contexts, so mask IRQs to keep a nested + * invocation from clobbering the mask an outer invocation is still + * iterating. + * + * NMI nesting is not handled: there is no legitimate reason to call + * pick_idle from NMI and doing so poses no crash risk, so such a + * caller is on its own. + */ + guard(irqsave)(); + unvisited = this_cpu_ptr(&per_cpu_unvisited); /* @@ -183,7 +194,6 @@ static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, i if (cpu >= 0) break; } - preempt_enable(); return cpu; } -- cgit v1.2.3 From 117c658adb0fb9276966a86fd4c2ed112640a0ee Mon Sep 17 00:00:00 2001 From: Hemanth Selam Date: Mon, 7 Sep 2026 12:19:42 +0530 Subject: sched_ext: Fix typo "upto" in comments Correct "upto" to "up to", reported by scripts/checkpatch.pl using the misspelling list in scripts/spelling.txt. Only touches comments, no code changes. Assisted-by: Cursor:claude-opus-5 Signed-off-by: Hemanth Selam Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 387953b92668..4080902a528c 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -5849,7 +5849,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) /* * We don't want CPUs to have more than $nr_donor_target tasks and - * balancing to fill donee CPUs upto $nr_target. Once targets are + * balancing to fill donee CPUs up to $nr_target. Once targets are * calculated, find the donee CPUs. */ nr_target = DIV_ROUND_UP(nr_tasks, nr_cpus); @@ -8825,7 +8825,7 @@ __bpf_kfunc_start_defs(); * task is inserted. * * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id - * and this function can be called upto ops.dispatch_max_batch times to insert + * and this function can be called up to ops.dispatch_max_batch times to insert * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the * remaining slots. scx_bpf_dsq_move_to_local() flushes the batch and resets the * counter. -- cgit v1.2.3 From dec231d15736a22d4500628d9993aed4f3502c15 Mon Sep 17 00:00:00 2001 From: Cheng-Yang Chou Date: Thu, 10 Sep 2026 01:27:18 +0800 Subject: sched_ext: Merge adjacent ifdefs in ext.h 6a1cda143c23 ("sched_ext: Always use SMP versions in kernel/sched/ext.h") dropped the CONFIG_SMP guard from scx_update_idle(), leaving two adjacent CONFIG_SCHED_CLASS_EXT blocks. Merge the declaration and stub into the primary ifdef block. No functional change. Signed-off-by: Cheng-Yang Chou Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.h | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.h b/kernel/sched/ext/ext.h index 0b7fc46aee08..0012f708a550 100644 --- a/kernel/sched/ext/ext.h +++ b/kernel/sched/ext/ext.h @@ -21,6 +21,13 @@ int scx_check_setscheduler(struct task_struct *p, int policy); bool task_should_scx(int policy); bool scx_allow_ttwu_queue(const struct task_struct *p); void init_sched_ext_class(void); +void __scx_update_idle(struct rq *rq, bool idle, bool do_notify); + +static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) +{ + if (scx_enabled()) + __scx_update_idle(rq, idle, do_notify); +} static inline u32 scx_cpuperf_target(s32 cpu) { @@ -55,21 +62,10 @@ static inline int scx_check_setscheduler(struct task_struct *p, int policy) { re static inline bool task_on_scx(const struct task_struct *p) { return false; } static inline bool scx_allow_ttwu_queue(const struct task_struct *p) { return true; } static inline void init_sched_ext_class(void) {} +static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) {} #endif /* CONFIG_SCHED_CLASS_EXT */ -#ifdef CONFIG_SCHED_CLASS_EXT -void __scx_update_idle(struct rq *rq, bool idle, bool do_notify); - -static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) -{ - if (scx_enabled()) - __scx_update_idle(rq, idle, do_notify); -} -#else -static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) {} -#endif - #ifdef CONFIG_CGROUP_SCHED #ifdef CONFIG_EXT_GROUP_SCHED void scx_tg_init(struct task_group *tg); -- cgit v1.2.3 From f8e5a4e3f3bea6b389ae8dbeafe50fdb8ab543b2 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Fri, 18 Sep 2026 19:13:27 +0200 Subject: sched_ext: Add lazy preemption support The fair scheduling class can request lazy rescheduling, deferring an in-kernel scheduling boundary until returning to user space or until the next scheduler tick. sched_ext only exposes immediate preemption, preventing BPF schedulers from making the same trade-off. Add SCX_ENQ_PREEMPT_LAZY and SCX_KICK_PREEMPT_LAZY. Both expire the current sched_ext task slice but request lazy rescheduling. Immediate preemption, WAIT and plain kicks take precedence when requests are combined, while a lazy enqueue to a non-local DSQ retains the head-insertion semantics of SCX_ENQ_PREEMPT. Add SCX_OPS_LAZY_RESCHED as the default slice-expiry policy for newly enabled tasks and initialize it before ops.enable(). Add scx_bpf_task_set_lazy_resched() so the owning scheduler can override the persistent policy per task from any callback while preserving sub-scheduler task ownership boundaries. Bypass continues to force immediate rescheduling. For an infinite-slice task whose tick is stopped, set the lazy reschedule request before restoring the scheduler tick dependency. This lets the dependency update's IPI serve the request on return to user space and guarantees that the next tick promotes it otherwise. Use pinned rq locking for remote kick delivery so RQCF_UPDATED reflects clock updates since lock acquisition. Accumulate kick requests independently and resolve precedence while holding the target rq lock. Reject invalid SCX_KICK_IDLE combinations and unknown kick flags, which were previously silently ignored. Tested-by: Tao Cui Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 151 ++++++++++++++++++++++++++++++++++++-------- kernel/sched/ext/internal.h | 44 +++++++++++-- kernel/sched/ext/sub.c | 15 +++-- 3 files changed, 174 insertions(+), 36 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 4080902a528c..fcba7c86fe4b 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -383,11 +383,11 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) return true; /* - * %SCX_ENQ_PREEMPT clears $curr's slice if on SCX and kicks dispatch, - * so allow it to avoid spuriously triggering reenq on a combined + * The preemption flags clear $curr's slice if on SCX and kick dispatch, + * so allow them to avoid spuriously triggering reenq on a combined * PREEMPT|IMMED insertion. */ - if (enq_flags & SCX_ENQ_PREEMPT) { + if (enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) { struct task_struct *curr = rq->curr; /* @@ -1515,6 +1515,26 @@ static void call_task_dequeue(struct scx_sched *sch, struct rq *rq, p->scx.flags &= ~SCX_TASK_IN_CUSTODY; } +/* + * A task with an infinite slice may be running with its tick stopped. Lazy + * rescheduling doesn't send an IPI, so restore the tick dependency to guarantee + * that the lazy request is promoted by a real scheduler tick. Set the lazy + * request first so that the dependency update's IPI can also serve it on return + * to user space. + */ +static void scx_resched_curr_lazy(struct rq *rq) +{ + resched_curr_lazy(rq); + + if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) { + rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK; + if (rq->clock_update_flags < RQCF_UPDATED) + update_rq_clock(rq); + update_other_load_avgs(rq); + sched_update_tick_dependency(rq); + } +} + static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) @@ -1577,12 +1597,16 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, if (rq->scx.flags & SCX_RQ_IN_DISPATCH) return; - if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && + if ((enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) && p != rq->curr && rq->curr->sched_class == &ext_sched_class) { - if (likely(scx_set_task_slice(rq->curr, 0))) - resched_curr(rq); - else + if (likely(scx_set_task_slice(rq->curr, 0))) { + if (enq_flags & SCX_ENQ_PREEMPT) + resched_curr(rq); + else + scx_resched_curr_lazy(rq); + } else { __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); + } } } @@ -1672,7 +1696,7 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, scx_error(sch, "DSQ ID 0x%016llx already had PRIQ-enqueued tasks", dsq->id); - if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) { + if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) { /* new task inserted at head - use fastpath */ if (dsq_insert_head(dsq, p) && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) rcu_assign_pointer(dsq->first_task, p); @@ -2386,7 +2410,7 @@ void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct WARN_ON_ONCE(p->scx.holding_cpu >= 0); - if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) + if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) dsq_insert_head(dst_dsq, p); else list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list); @@ -3804,8 +3828,13 @@ static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) else if (SCX_HAS_OP(sch, tick)) SCX_CALL_OP_TASK(sch, tick, rq, curr); - if (!curr->scx.slice) - resched_curr(rq); + if (!curr->scx.slice) { + /* the slice can't be trusted while bypassing */ + if (READ_ONCE(curr->scx.lazy_resched) && !scx_bypassing(sch, cpu_of(rq))) + scx_resched_curr_lazy(rq); + else + resched_curr(rq); + } } #ifdef CONFIG_EXT_GROUP_SCHED @@ -3921,6 +3950,7 @@ static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p) weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; p->scx.weight = sched_weight_to_cgroup(weight); + p->scx.lazy_resched = sch->ops.flags & SCX_OPS_LAZY_RESCHED; if (SCX_HAS_OP(sch, enable)) SCX_CALL_OP_TASK(sch, enable, rq, p); @@ -5371,6 +5401,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) free_cpumask_var(pcpu->cpus_to_kick); free_cpumask_var(pcpu->cpus_to_kick_if_idle); free_cpumask_var(pcpu->cpus_to_preempt); + free_cpumask_var(pcpu->cpus_to_preempt_lazy); free_cpumask_var(pcpu->cpus_to_wait); exit_dsq(scx_bypass_dsq(sch, cpu)); @@ -6883,6 +6914,9 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, if (!cpumask_empty(pcpu->cpus_to_preempt)) scx_dump_line(&ns, " cpus_to_preempt: %*pb", cpumask_pr_args(pcpu->cpus_to_preempt)); + if (!cpumask_empty(pcpu->cpus_to_preempt_lazy)) + scx_dump_line(&ns, " preempt_lazy : %*pb", + cpumask_pr_args(pcpu->cpus_to_preempt_lazy)); if (!cpumask_empty(pcpu->cpus_to_wait)) scx_dump_line(&ns, " cpus_to_wait : %*pb", cpumask_pr_args(pcpu->cpus_to_wait)); @@ -7200,6 +7234,7 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, if (!zalloc_cpumask_var_node(&pcpu->cpus_to_kick, GFP_KERNEL, node) || !zalloc_cpumask_var_node(&pcpu->cpus_to_kick_if_idle, GFP_KERNEL, node) || !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt, GFP_KERNEL, node) || + !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt_lazy, GFP_KERNEL, node) || !zalloc_cpumask_var_node(&pcpu->cpus_to_wait, GFP_KERNEL, node)) { ret = -ENOMEM; goto err_free_pcpu; @@ -7335,6 +7370,7 @@ err_free_pcpu: free_cpumask_var(pcpu->cpus_to_kick); free_cpumask_var(pcpu->cpus_to_kick_if_idle); free_cpumask_var(pcpu->cpus_to_preempt); + free_cpumask_var(pcpu->cpus_to_preempt_lazy); free_cpumask_var(pcpu->cpus_to_wait); } for_each_possible_cpu(cpu) { @@ -8459,13 +8495,23 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r { struct rq *rq = cpu_rq(cpu); struct scx_rq *this_scx = &this_rq->scx; + struct rq_flags rf; const struct sched_class *cur_class; bool should_wait = false; bool kickable; - unsigned long flags; + bool preempt, preempt_lazy, wait, immediate; - raw_spin_rq_lock_irqsave(rq, flags); + rq_lock_irqsave(rq, &rf); cur_class = rq->curr->sched_class; + preempt = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt); + preempt_lazy = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt_lazy); + wait = cpumask_test_cpu(cpu, pcpu->cpus_to_wait); + /* + * Immediate preemption, waiting and a plain kick take precedence over + * lazy preemption. The lazy request still clears the slice, so all + * accumulated requests are served. + */ + immediate = preempt || wait || cpumask_test_cpu(cpu, pcpu->cpus_to_kick); /* * During CPU hotplug, a CPU may depend on kicking itself to make @@ -8479,19 +8525,23 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r !sched_class_above(cur_class, &ext_sched_class); if (kickable && !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) { - if (cpumask_test_cpu(cpu, pcpu->cpus_to_preempt)) { + if (preempt || preempt_lazy) { if (cur_class == &ext_sched_class) { u64 caps = scx_caps_for_preempt(pcpu->sch, rq, 0); - if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps))) + if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps))) { __scx_add_event(pcpu->sch, SCX_EV_SUB_PREEMPT_DENIED, 1); - else if (unlikely(!scx_set_task_slice(rq->curr, 0))) + /* degrade to a plain, immediate kick */ + immediate = true; + } else if (unlikely(!scx_set_task_slice(rq->curr, 0))) { __scx_add_event(pcpu->sch, SCX_EV_SLICE_DENIED, 1); + } } cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); + cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt_lazy); } - if (cpumask_test_cpu(cpu, pcpu->cpus_to_wait)) { + if (wait) { if (cur_class == &ext_sched_class) { cpumask_set_cpu(cpu, this_scx->cpus_to_sync); ksyncs[cpu] = rq->scx.kick_sync; @@ -8500,17 +8550,21 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r cpumask_clear_cpu(cpu, pcpu->cpus_to_wait); } - resched_curr(rq); + if (immediate) + resched_curr(rq); + else + scx_resched_curr_lazy(rq); } else { /* a kickable cpu was skipped solely for the missing caps */ if (kickable) __scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1); cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); + cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt_lazy); cpumask_clear_cpu(cpu, pcpu->cpus_to_wait); } scx_rq_lock_drop(rq); - raw_spin_rq_unlock_irqrestore(rq, flags); + rq_unlock_irqrestore(rq, &rf); return should_wait; } @@ -8560,7 +8614,7 @@ static void kick_cpus_irq_workfn(struct irq_work *irq_work) list_for_each_entry_safe(pcpu, tmp, &this_scx->sched_pcpus_to_kick, to_kick_node) { list_del_init(&pcpu->to_kick_node); - for_each_cpu(cpu, pcpu->cpus_to_kick) { + for_each_cpu_or(cpu, pcpu->cpus_to_kick, pcpu->cpus_to_preempt_lazy) { should_wait |= kick_one_cpu(cpu, pcpu, this_rq, ksyncs); cpumask_clear_cpu(cpu, pcpu->cpus_to_kick); cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle); @@ -9531,6 +9585,41 @@ __bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime, return true; } +/** + * scx_bpf_task_set_lazy_resched - Set task's tick reschedule mode + * @p: task of interest + * @lazy: whether slice expiry should request lazy rescheduling + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Choose whether depletion of @p's slice at the scheduler tick requests lazy or + * immediate rescheduling. This is a persistent per-task policy and does not + * itself clear @p's slice or request rescheduling, unlike the + * %SCX_ENQ_PREEMPT_LAZY and %SCX_KICK_PREEMPT_LAZY one-shot operations. + * + * With lazy rescheduling enabled, a task interrupted in user space still + * reschedules before returning to user space. A task interrupted in the kernel + * may continue until its next return to user space or the next scheduler tick, + * which promotes the request. If lazy preemption is disabled at runtime, the + * request behaves like an immediate reschedule. + * + * @p must be on the calling scheduler. + * + * Return %true on success, %false if @p is not on the calling scheduler. + */ +__bpf_kfunc bool scx_bpf_task_set_lazy_resched(struct task_struct *p, bool lazy, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + sch = scx_prog_sched(aux); + if (unlikely(!sch || !scx_task_on_sched(sch, p))) + return false; + + WRITE_ONCE(p->scx.lazy_resched, lazy); + return true; +} + void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) { struct scx_sched_pcpu *pcpu; @@ -9539,6 +9628,10 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) if (!scx_kf_allowed_ctx(sch)) return; + if (unlikely(flags & ~SCX_KICK_ALL_FLAGS)) { + scx_error(sch, "invalid kick flags 0x%llx", flags); + return; + } local_irq_save(irq_flags); @@ -9564,7 +9657,8 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) if (flags & SCX_KICK_IDLE) { struct rq *target_rq = cpu_rq(cpu); - if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_WAIT))) + if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_PREEMPT_LAZY | + SCX_KICK_WAIT))) scx_error(sch, "PREEMPT/WAIT cannot be used with SCX_KICK_IDLE"); if (raw_spin_rq_trylock(target_rq)) { @@ -9578,12 +9672,17 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) } cpumask_set_cpu(cpu, pcpu->cpus_to_kick_if_idle); } else { - cpumask_set_cpu(cpu, pcpu->cpus_to_kick); - + /* + * Accumulate requests and resolve their precedence at delivery. + */ if (flags & SCX_KICK_PREEMPT) cpumask_set_cpu(cpu, pcpu->cpus_to_preempt); + if (flags & SCX_KICK_PREEMPT_LAZY) + cpumask_set_cpu(cpu, pcpu->cpus_to_preempt_lazy); if (flags & SCX_KICK_WAIT) cpumask_set_cpu(cpu, pcpu->cpus_to_wait); + if (!(flags & SCX_KICK_PREEMPT_LAZY)) + cpumask_set_cpu(cpu, pcpu->cpus_to_kick); } if (list_empty(&pcpu->to_kick_node)) @@ -9623,8 +9722,9 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux * cid-addressed equivalent of scx_bpf_kick_cpu(). An invalid @cid aborts the * scheduler via scx_cid_to_cpu(). Caps are enforced on the delivery path: a * kick is dropped if the caller lacks baseline access on @cid, and a - * %SCX_KICK_PREEMPT degrades to a plain reschedule if the caller lacks - * %SCX_CAP_PREEMPT for a task outside its subtree. + * %SCX_KICK_PREEMPT or %SCX_KICK_PREEMPT_LAZY request degrades to a plain + * reschedule if the caller lacks %SCX_CAP_PREEMPT for a task outside its + * subtree. */ __bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) { @@ -10691,6 +10791,7 @@ __bpf_kfunc_end_defs(); BTF_KFUNCS_START(scx_kfunc_ids_any) BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS | KF_RCU); BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS | KF_RCU); +BTF_ID_FLAGS(func, scx_bpf_task_set_lazy_resched, KF_IMPLICIT_ARGS | KF_RCU); BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_kick_cid, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS) diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index e1eb3a0d456c..4f170cfb7986 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -215,6 +215,21 @@ enum scx_ops_flags { */ SCX_OPS_TID_TO_TASK = 1LLU << 8, + /* + * If set, tasks default to requesting lazy rescheduling when their + * slice runs out at the tick, the way fair.c expires a slice from + * update_curr(). The default is copied to p->scx.lazy_resched + * immediately before ops.enable(), after which + * scx_bpf_task_set_lazy_resched() may override it per task. + * + * A task in user space still reschedules on the way back from the tick; + * a task in the kernel runs on to its next return to user space or to + * the next tick, which promotes the request. When lazy preemption is + * disabled at runtime, the request behaves like an immediate + * reschedule. Rescheduling while bypassing stays immediate. + */ + SCX_OPS_LAZY_RESCHED = 1LLU << 9, + SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | SCX_OPS_ENQ_LAST | SCX_OPS_ENQ_EXITING | @@ -223,7 +238,8 @@ enum scx_ops_flags { SCX_OPS_SWITCH_PARTIAL | SCX_OPS_BUILTIN_IDLE_PER_NODE | SCX_OPS_ALWAYS_ENQ_IMMED | - SCX_OPS_TID_TO_TASK, + SCX_OPS_TID_TO_TASK | + SCX_OPS_LAZY_RESCHED, /* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */ __SCX_OPS_INTERNAL_MASK = 0xffLLU << 56, @@ -1327,6 +1343,7 @@ struct scx_sched_pcpu { cpumask_var_t cpus_to_kick; cpumask_var_t cpus_to_kick_if_idle; cpumask_var_t cpus_to_preempt; + cpumask_var_t cpus_to_preempt_lazy; cpumask_var_t cpus_to_wait; struct list_head to_kick_node; @@ -1407,15 +1424,16 @@ struct scx_sched_pnode { * the allocation pattern. * * ENQ_IMMED insert an IMMED task onto the cid's local DSQ - * - kick the cid's cpu (except SCX_KICK_PREEMPT) + * - kick the cid's cpu (except SCX_KICK_PREEMPT and + * SCX_KICK_PREEMPT_LAZY) * * ENQ insert any task onto the cid's local DSQ (implies ENQ_IMMED) * * PREEMPT preempt any task running on the cid regardless of the owning * sched (implies ENQ). Preempting a task in the sched's own subtree * doesn't require any cap. - * - SCX_ENQ_PREEMPT inserts - * - SCX_KICK_PREEMPT kicks + * - SCX_ENQ_PREEMPT and SCX_ENQ_PREEMPT_LAZY insert + * - SCX_KICK_PREEMPT and SCX_KICK_PREEMPT_LAZY kick * * PERF control the cid's cpu power/perf management state, currently the * cpufreq target set through scx_bpf_cidperf_set(). Hardware @@ -1685,6 +1703,15 @@ enum scx_enq_flags { */ SCX_ENQ_PREEMPT = 1LLU << 32, + /* + * Like %SCX_ENQ_PREEMPT, but request lazy rescheduling. The current + * task's slice is still cleared immediately so that the next scheduling + * boundary observes the new ordering. %SCX_ENQ_PREEMPT takes precedence + * if both are specified. Implies %SCX_ENQ_HEAD, which is all it means + * on a non-local DSQ, as with %SCX_ENQ_PREEMPT. + */ + SCX_ENQ_PREEMPT_LAZY = 1LLU << 35, + /* * Only allowed on local DSQs. Guarantees that the task either gets * on the CPU immediately and stays on it, or gets reenqueued back @@ -1811,6 +1838,15 @@ enum scx_kick_flags { * is not on SCX. */ SCX_KICK_WAIT = 1LLU << 2, + + /* + * Like %SCX_KICK_PREEMPT, but request lazy rescheduling. If combined + * with %SCX_KICK_PREEMPT or %SCX_KICK_WAIT, rescheduling is immediate. + */ + SCX_KICK_PREEMPT_LAZY = 1LLU << 3, + + SCX_KICK_ALL_FLAGS = SCX_KICK_IDLE | SCX_KICK_PREEMPT | + SCX_KICK_WAIT | SCX_KICK_PREEMPT_LAZY, }; enum scx_tg_flags { diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 380a5653dc52..1bd1b461ff30 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -686,9 +686,10 @@ void scx_rescue_init(struct rq *rq) * rescue is enabled, or @rq's reject DSQ after recording the reenq reason on * @p. * - * %SCX_ENQ_IMMED, %SCX_ENQ_PREEMPT and %SCX_ENQ_HEAD are cleared when diverting - * to rescue or reject. %SCX_ENQ_PREEMPT is also cleared on a fallback - * migration-disabled admission. + * %SCX_ENQ_IMMED, %SCX_ENQ_PREEMPT, %SCX_ENQ_PREEMPT_LAZY and %SCX_ENQ_HEAD are + * cleared when diverting to rescue or reject. %SCX_ENQ_PREEMPT and + * %SCX_ENQ_PREEMPT_LAZY are also cleared on a fallback migration-disabled + * admission. * * Bypass doesn't need special-casing as a bypassing sched's tasks are enqueued * to and run by its nearest non-bypassing ancestor. If root is bypassing, it @@ -709,7 +710,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r * On a remote activation the scheduling sched (@asch) differs from * @p's owner (@sch). Check caps against the scheduling sched. */ - if (*enq_flags & SCX_ENQ_PREEMPT) + if (*enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) needed |= scx_caps_for_preempt(asch, rq, *enq_flags); missing = scx_missing_caps(asch, cpu_of(rq), needed); @@ -726,7 +727,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r if (unlikely(!scx_rq_online(rq) || is_migration_disabled(p) || p->migration_pending)) { __scx_add_event(sch, SCX_EV_SUB_FORCED_ADMIT, 1); - *enq_flags &= ~SCX_ENQ_PREEMPT; + *enq_flags &= ~(SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY); return &rq->scx.local_dsq; } @@ -735,8 +736,8 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r * or HEAD - a diversion has no priority and IMMED is not allowed on * non-local DSQs. Strip the enq and task flags along with the slice. */ - *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | SCX_ENQ_HEAD | - SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL); + *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY | + SCX_ENQ_HEAD | SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL); p->scx.flags &= ~SCX_TASK_IMMED; /* the enqueuer opted for rescue instead of rejection and reenqueue */ -- cgit v1.2.3 From e4c5ba819d3ce8c146a3a2d68f507e1845fdaafb Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 21 Sep 2026 10:19:28 -0700 Subject: sched_ext: Specialize the DSQ hashtable compare rhashtable is a generic container: it does not know what a key is, so it carries the key's shape as data. dsq_hash_params declares key_len, key_offset and head_offset and nothing else, so lookups fall back to rhashtable_compare(), which reads both the offset and the length back out of ht->p at runtime: memcmp(ptr + ht->p.key_offset, arg->key, ht->p.key_len) The key is one naturally aligned u64, but neither the offset nor the length is a compile-time constant there, so the compiler cannot narrow the call and emits an out-of-line memcmp() for every element walked - two loads, a call and a length dispatch to compare eight bytes. The interpretation costs more than the comparison it is interpreting. find_user_dsq() sits on the __schedule() path, and scx_layered calls scx_bpf_dsq_nr_queued() once per layer per dispatch decision. On Meta's fleet the lookup has a significant cost and shows up in fleet wide profile. Let's supply an obj_cmpfn. dsq_hash_params is a const object passed by value into the __always_inline __rhashtable_lookup(), so params.obj_cmpfn is a compile-time constant, the ternary that selects it folds away and the callback inlines: the compare becomes a single cmp against dsq->id. Every consumer of the result only tests it against zero, so the ordering memcmp() also carries is never observed. No functional change intended. An in-kernel A/B over the DSQ ids scx_layered creates, both parameter sets compiled into the same kernel, makes the lookup 2.9x faster. Signed-off-by: Usama Arif Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index fcba7c86fe4b..fe7e07d72a69 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -189,10 +189,20 @@ static DEFINE_PER_CPU(struct scx_tid_alloc, scx_tid_alloc); */ static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); +static __always_inline int dsq_cmpfn(struct rhashtable_compare_arg *arg, const void *ptr) +{ + const struct scx_dispatch_q *dsq = ptr; + + BUILD_BUG_ON(sizeof_field(struct scx_dispatch_q, id) != sizeof(u64)); + + return dsq->id != *(const u64 *)arg->key; +} + static const struct rhashtable_params dsq_hash_params = { .key_len = sizeof_field(struct scx_dispatch_q, id), .key_offset = offsetof(struct scx_dispatch_q, id), .head_offset = offsetof(struct scx_dispatch_q, hash_node), + .obj_cmpfn = dsq_cmpfn, }; static LLIST_HEAD(dsqs_to_free); -- cgit v1.2.3 From daaab0f42e2eec54a8fe585b2fa3f5408a9b3585 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 21 Sep 2026 11:59:11 -0700 Subject: sched_ext: Specialize the TID hashtable compare scx_tid_hash_params does not provide an object comparison function, so rhashtable falls back to rhashtable_compare(). Although the key is one naturally aligned u64, the generic comparison reads the key offset and length from the table parameters at runtime and emits an out-of-line memcmp() for each element walked. scx_bpf_tid_to_task() can sit on hot scheduling paths. Supply an obj_cmpfn so the const parameters passed into the inlined rhashtable lookup specialize the comparison to a single equality test against scx->tid. The result is only tested against zero, so the ordering provided by memcmp() is not observable. No functional change intended. Suggested-by: Tejun Heo Signed-off-by: Usama Arif Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index fe7e07d72a69..f5887542b502 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -52,10 +52,20 @@ struct rhashtable scx_sched_hash; #endif /* see SCX_OPS_TID_TO_TASK */ +static __always_inline int scx_tid_cmpfn(struct rhashtable_compare_arg *arg, const void *ptr) +{ + const struct sched_ext_entity *scx = ptr; + + BUILD_BUG_ON(sizeof_field(struct sched_ext_entity, tid) != sizeof(u64)); + + return scx->tid != *(const u64 *)arg->key; +} + static const struct rhashtable_params scx_tid_hash_params = { .key_len = sizeof_field(struct sched_ext_entity, tid), .key_offset = offsetof(struct sched_ext_entity, tid), .head_offset = offsetof(struct sched_ext_entity, tid_hash_node), + .obj_cmpfn = scx_tid_cmpfn, .insecure_elasticity = true, /* inserted/removed under scx_tasks_lock */ }; static struct rhashtable scx_tid_hash; -- cgit v1.2.3 From b9e602f30bea2847a0c0319c5a6e0bb73d96cd7b Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 21 Sep 2026 11:59:12 -0700 Subject: sched_ext: Specialize the scheduler hashtable compare scx_sched_hash_params does not provide an object comparison function, so rhashtable falls back to rhashtable_compare(). Although the sub-cgroup ID key is one naturally aligned u64, the generic comparison reads the key offset and length from the table parameters at runtime and emits an out-of-line memcmp() for each element walked. Supply an obj_cmpfn so lookups specialize to a single equality test against ops.sub_cgroup_id. ext.c and sub.c are included in the same build_policy.c translation unit, so the compiler can fold the const callback into scx_find_sub_sched() even though its source is in sub.c. The result is only tested against zero, so the ordering provided by memcmp() is not observable. No functional change intended. Suggested-by: Tejun Heo Signed-off-by: Usama Arif Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index f5887542b502..f60894dbf062 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -41,10 +41,21 @@ struct scx_sched __rcu *scx_root; LIST_HEAD(scx_sched_all); #ifdef CONFIG_EXT_SUB_SCHED +static __always_inline int scx_sched_cmpfn(struct rhashtable_compare_arg *arg, + const void *ptr) +{ + const struct scx_sched *sch = ptr; + + BUILD_BUG_ON(sizeof_field(struct scx_sched, ops.sub_cgroup_id) != sizeof(u64)); + + return sch->ops.sub_cgroup_id != *(const u64 *)arg->key; +} + const struct rhashtable_params scx_sched_hash_params = { .key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id), .key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id), .head_offset = offsetof(struct scx_sched, hash_node), + .obj_cmpfn = scx_sched_cmpfn, .insecure_elasticity = true, /* inserted under scx_sched_lock */ }; -- cgit v1.2.3 From efa7bc7d719558483b14e1c44bcce11532a44581 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Wed, 23 Sep 2026 07:07:50 -0700 Subject: sched_ext: Avoid relocking DSQ during remote consumption A remote consume unlinks a task from a non-local DSQ before switching to the task's rq. Leaving p->scx.dsq set makes the following deactivate_task() reacquire the DSQ lock in scx_dispatch_dequeue() to clear the transfer state. Keep p->scx.dsq set while switching rq locks so that a racing dequeue can find the DSQ and cancel the transfer. After acquiring the source rq and confirming ownership, clear p->scx.dsq. The subsequent dequeue then takes its !dsq path without reacquiring the DSQ lock. Fleet profiles show that roughly 70% of the cycles attributed to scx_dispatch_dequeue() are spent in queued_spin_lock_slowpath() while taking dsq->lock. Signed-off-by: Usama Arif Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index f60894dbf062..03c85ad0ead4 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2599,7 +2599,7 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, * points to this CPU. See scx_dispatch_dequeue() for the counterpart. * * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is - * still valid. %false if lost to dequeue. + * still valid and detached from @dsq. %false if lost to dequeue. */ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, struct scx_dispatch_q *dsq, @@ -2619,8 +2619,18 @@ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, switch_rq_lock(locked_rq, src_rq); /* task_rq couldn't have changed if we're still the holding cpu */ - return likely(p->scx.holding_cpu == cpu) && - !WARN_ON_ONCE(src_rq != task_rq(p)); + if (likely(p->scx.holding_cpu == cpu) && !WARN_ON_ONCE(src_rq != task_rq(p))) { + /* + * Keep ->dsq set until we own @src_rq so that a racing dequeue + * can find @dsq and clear holding_cpu. Once ownership is + * confirmed, clear it under @src_rq so that deactivate_task() + * takes the !dsq path instead of retaking @dsq->lock. + */ + p->scx.dsq = NULL; + return true; + } + + return false; } static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq, -- cgit v1.2.3 From 84a0d325b0b709962772e66a804677ceaca417e2 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Wed, 23 Sep 2026 14:34:44 -0700 Subject: sched_ext: Avoid relocking DSQ during remote DSQ moves When scx_bpf_dsq_move*() moves a task from a non-local DSQ to a remote local DSQ, move_task_between_dsqs() drops src_dsq->lock while leaving p->scx.dsq set. The following deactivate_task() then reacquires src_dsq->lock in scx_dispatch_dequeue() to unlink the task and clear p->scx.dsq. Both the source rq and src_dsq are already locked before the source DSQ lock is dropped. Call dispatch_dequeue_locked() there so that deactivate_task() takes the !dsq path and avoids the redundant lock acquisition. Suggested-by: Tejun Heo Signed-off-by: Usama Arif Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 03c85ad0ead4..911bb6d433ba 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2697,6 +2697,7 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, scx_move_local_task_to_local_dsq(sch, p, enq_flags, src_dsq, dst_rq); raw_spin_unlock(&src_dsq->lock); } else { + dispatch_dequeue_locked(p, src_dsq); raw_spin_unlock(&src_dsq->lock); move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); } -- cgit v1.2.3 From e9b9ad3c138020dffcaddede86d91c984682806b Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Thu, 24 Sep 2026 06:05:03 -0700 Subject: sched_ext: Update scx_dispatch_dequeue() comments Remote consumption now reaches the !dsq branch with holding_cpu set, just like dispatch_to_local_dsq(). Clearing holding_cpu in this branch also tells dispatch_to_local_dsq() that it lost to a dequeue. The dsq-present branch races with unlink_dsq_and_switch_rq_lock(), not dispatch_to_local_dsq(). Update both comments to describe the current transfer paths and race semantics. No functional change. Suggested-by: Tejun Heo Signed-off-by: Usama Arif Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 911bb6d433ba..d01f44839da3 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1821,10 +1821,10 @@ void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) list_del_init(&p->scx.dsq_list.node); /* - * When dispatching directly from the BPF scheduler to a local - * DSQ, the task isn't associated with any DSQ but - * @p->scx.holding_cpu may be set under the protection of - * %SCX_OPSS_DISPATCHING. + * When dispatch_to_local_dsq() or remote consumption moves a + * task to a local DSQ, the task isn't associated with any DSQ + * but @p->scx.holding_cpu may be set. Clearing holding_cpu + * tells dispatch_to_local_dsq() that it lost to a dequeue. */ if (p->scx.holding_cpu >= 0) p->scx.holding_cpu = -1; @@ -1844,10 +1844,10 @@ void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) scx_task_unlink_from_dsq(p, dsq); } else { /* - * We're racing against dispatch_to_local_dsq() which already - * removed @p from @dsq and set @p->scx.holding_cpu. Clear the - * holding_cpu which tells dispatch_to_local_dsq() that it lost - * the race. + * We're racing against unlink_dsq_and_switch_rq_lock(), + * which already removed @p from @dsq and set + * @p->scx.holding_cpu. Clear holding_cpu to tell + * unlink_dsq_and_switch_rq_lock() that it lost the race. */ WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node)); p->scx.holding_cpu = -1; -- cgit v1.2.3 From 7a919c7f86de8a0bbfd5da4eacdb67833dfe3e5a Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Thu, 24 Sep 2026 13:27:11 -0700 Subject: sched_ext: Test scx_has_subs() inline before calling sub-sched hooks scx_reenq_reject(), scx_process_sync_ecaps() and scx_resolve_local_dsq() return early if there are no sub-schedulers, but clang emits the full prologue before the static branch. Without sub-schedulers, we still pay for a call and for saving and restoring up to six registers, on every wakeup, dispatch and local DSQ insert. Let's move the scx_has_subs() test into inline wrappers, so that these calls go away completely. The wrappers keep lockdep_assert_rq_held(). In a pinned perf bench sched pipe test on x86-64 with clang 22, this reduced kernel instructions by 52-54 per context switch with scx_simple, scx_lavd and scx_layered. No functional change intended. Signed-off-by: Usama Arif Signed-off-by: Tejun Heo --- kernel/sched/ext/sub.c | 25 +++++++++---------------- kernel/sched/ext/sub.h | 35 +++++++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 20 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 1bd1b461ff30..3ef4147c2d0a 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -675,7 +675,7 @@ void scx_rescue_init(struct rq *rq) } /** - * scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert + * __scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert * @sch: enqueuing sub-sched * @rq: rq whose local DSQ @p targets * @p: task being inserted @@ -695,12 +695,9 @@ void scx_rescue_init(struct rq *rq) * to and run by its nearest non-bypassing ancestor. If root is bypassing, it * always holds all caps. */ -struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 *enq_flags) +struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags) { - if (!scx_has_subs()) - return &rq->scx.local_dsq; - s32 cid = __scx_cpu_to_cid(cpu_of(rq)); struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch; u64 needed = scx_caps_for_enq(*enq_flags); @@ -783,14 +780,12 @@ bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) * scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT. * Rejection can't happen for root. */ -void scx_reenq_reject(struct rq *rq) +void __scx_reenq_reject(struct rq *rq) { LIST_HEAD(tasks); struct task_struct *p, *n; - lockdep_assert_rq_held(rq); - - if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list)) + if (list_empty(&rq->scx.reject_dsq.list)) return; /* @@ -925,7 +920,7 @@ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid) struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); /* - * Pairs with smp_mb() in scx_process_sync_ecaps(). Either the check + * Pairs with smp_mb() in __scx_process_sync_ecaps(). Either the check * below sees the node off the list and queues it, or the in-flight sync * sees the caps[] update made before this call. */ @@ -950,7 +945,7 @@ static void discard_queued_syncs(struct rq *rq) } /** - * scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] + * __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] * @rq: the cid's cpu rq * @prev: @rq's previous task from the in-progress dispatch * @@ -962,16 +957,14 @@ static void discard_queued_syncs(struct rq *rq) * learns the cid's idle state. Such a gain arms the per-rq * %SCX_RQ_SUB_IDLE_RENOTIFY gate so the next idle pick delivers it. */ -void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) +void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) { s32 cpu = cpu_of(rq); s32 cid, shard; struct llist_node *batch, *pos, *tmp; u64 lost_all = 0; - lockdep_assert_rq_held(rq); - - if (!scx_has_subs() || likely(llist_empty(&rq->scx.ecaps_to_sync))) + if (likely(llist_empty(&rq->scx.ecaps_to_sync))) return; /* diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index f7bcdfda8dd8..6a7076b9a1dd 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -28,16 +28,16 @@ bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux); void scx_free_pshards(struct scx_sched *sch); s32 scx_alloc_pshards(struct scx_sched *sch); void scx_init_root_caps(struct scx_sched *sch); -void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); +void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch); void scx_online_ecaps(struct rq *rq); void scx_offline_ecaps(struct rq *rq); void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu); void scx_discard_stale_ecaps_syncs(void); -struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 *enq_flags); +struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags); bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p); -void scx_reenq_reject(struct rq *rq); +void __scx_reenq_reject(struct rq *rq); void scx_rescue_charge(struct rq *rq, s64 delta_exec); void scx_rescue_end(struct rq *rq); bool scx_rescue_keep(struct rq *rq, struct task_struct *p); @@ -72,6 +72,33 @@ static inline void scx_dec_has_subs(struct scx_sched *sch) static_branch_dec(&__scx_has_subs); } +/* hot-path hooks, gated inline so that a root-only system skips the calls */ +static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) +{ + lockdep_assert_rq_held(rq); + + if (scx_has_subs()) + __scx_process_sync_ecaps(rq, prev); +} + +static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, + struct rq *rq, + struct task_struct *p, + u64 *enq_flags) +{ + if (!scx_has_subs()) + return &rq->scx.local_dsq; + return __scx_resolve_local_dsq(sch, rq, p, enq_flags); +} + +static inline void scx_reenq_reject(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + if (scx_has_subs()) + __scx_reenq_reject(rq); +} + #else /* CONFIG_EXT_SUB_SCHED */ static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } -- cgit v1.2.3 From 5cadf7545cc6528cc41d9d8dac52faf331cbc878 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Fri, 25 Sep 2026 16:46:04 +0200 Subject: sched_ext: Add a CID NUMA node lookup kfunc CID-form schedulers may need the physical NUMA node for a CID, for example to allocate arena memory on that node. The CID topology's node index is not a physical NUMA node ID, and scx_bpf_cpu_node() is restricted to CPU-form schedulers. Add scx_bpf_cid_node() to return the physical NUMA node for a CID and provide a compatibility helper that returns NUMA_NO_NODE on kernels without this kfunc. Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/cid.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index 39f88deb94bc..932f640ccc2d 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -640,6 +640,29 @@ __bpf_kfunc s32 scx_bpf_cid_to_cpu(s32 cid, const struct bpf_prog_aux *aux) return scx_cid_to_cpu(sch, cid); } +/** + * scx_bpf_cid_node - Return the NUMA node the given @cid belongs to + * @cid: cid to look up + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return NUMA_NO_NODE if @cid is invalid or there is no scheduler. Unlike + * scx_bpf_cpu_node(), this kfunc is available only with CID support, so BPF + * schedulers can use its presence to detect NUMA-aware CID lookups. + */ +__bpf_kfunc s32 scx_bpf_cid_node(s32 cid, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return NUMA_NO_NODE; + cpu = scx_cid_to_cpu(sch, cid); + return cpu < 0 ? NUMA_NO_NODE : cpu_to_node(cpu); +} + /** * scx_bpf_cpu_to_cid - Return the cid for @cpu * @cpu: cpu to look up @@ -952,6 +975,7 @@ static const struct btf_kfunc_id_set scx_kfunc_set_init_cids = { BTF_KFUNCS_START(scx_kfunc_ids_cid) BTF_ID_FLAGS(func, scx_bpf_cid_to_cpu, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cid_node, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cpu_to_cid, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cid_topo, KF_IMPLICIT_ARGS) BTF_KFUNCS_END(scx_kfunc_ids_cid) -- cgit v1.2.3 From 80b97ad2dfd8c812596ca86821cbe68f496da13d Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Sun, 27 Sep 2026 15:37:19 +0200 Subject: sched_ext: cid: Represent clusters explicitly The sched_ext CID topology gives each core, LLC and NUMA node a contiguous CID range. Without cluster entries, schedulers cannot obtain their CID range or global index. Walk cores by cache-sharing cluster and add cluster_cid and cluster_idx to scx_cid_topo. Treat the level inclusively: a core without a wider cluster forms one of its own, while an LLC-wide cache-sharing group forms one cluster. This gives every cluster a contiguous CID range and a dense index. The walk now takes cores in cluster order rather than LLC order. Shards are still cut on core boundaries, so a shard doesn't need to contain a whole cluster; only core, cluster, LLC and node are nested, in that order. The level follows whatever the arch reports through topology_cluster_cpumask(), which on x86 is the L2 sharing mask and is populated independently of CONFIG_SCHED_CLUSTER. Tested on a 13th Gen Intel Core i7-13800H: six SMT P-cores occupy CPUs 0-11, eight E-cores occupy CPUs 12-19 in two four-core L2 modules, and all 20 CPUs share one LLC; the P-core pairs form two-CID clusters, the E-core modules form four-CID clusters, and cluster_idx is dense across all eight clusters. Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/cid.c | 43 ++++++++++++++++++++++++++++++++++--------- kernel/sched/ext/cid.h | 11 ++++++----- kernel/sched/ext/types.h | 28 +++++++++++++++++++++++----- 3 files changed, 63 insertions(+), 19 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index dc670975c5bf..9fc219257004 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -31,6 +31,7 @@ static struct scx_cid_tables *scx_cid_tables; /* used only during alloc/free */ #define SCX_CID_TOPO_NEG (struct scx_cid_topo) { \ .core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1, \ .node_cid = -1, .node_idx = -1, .shard_cid = -1, .shard_idx = -1, \ + .cluster_cid = -1, .cluster_idx = -1, \ } /* @@ -182,12 +183,14 @@ s32 scx_cid_init(struct scx_sched *sch) cpumask_var_t to_walk __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t node_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t llc_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t cluster_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t core_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL; struct scx_cid_tables *tbls; u32 next_cid = 0; s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0; + s32 next_cluster_idx = 0; s32 next_shard_idx = 0; u32 shard_size, max_cids; u32 notopo_in_shard; @@ -215,6 +218,7 @@ s32 scx_cid_init(struct scx_sched *sch) if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) || !zalloc_cpumask_var(&node_scratch, GFP_KERNEL) || !zalloc_cpumask_var(&llc_scratch, GFP_KERNEL) || + !zalloc_cpumask_var(&cluster_scratch, GFP_KERNEL) || !zalloc_cpumask_var(&core_scratch, GFP_KERNEL) || !zalloc_cpumask_var(&llc_fallback, GFP_KERNEL) || !zalloc_cpumask_var(&online_no_topo, GFP_KERNEL)) @@ -256,6 +260,7 @@ s32 scx_cid_init(struct scx_sched *sch) u32 cores_per_shard, nr_large; u32 shard_local = 0, cores_in_shard = 0, cids_in_shard = 0; s32 shard_cid, shard_idx; + s32 cluster_cid = -1, cluster_idx = -1; /* llc_scratch = node_scratch & this llc */ cpumask_and(llc_scratch, node_scratch, llc_mask); @@ -268,15 +273,32 @@ s32 scx_cid_init(struct scx_sched *sch) tbls->shard_node[shard_idx] = nid; while (!cpumask_empty(llc_scratch)) { - s32 lcpu = cpumask_first(llc_scratch); - const struct cpumask *sib = topology_sibling_cpumask(lcpu); - s32 core_cid = next_cid; - s32 core_idx = next_core_idx++; - s32 ccpu; + const struct cpumask *sib; + s32 core_cid, core_idx, lcpu, ccpu; u32 max_cores, cids_in_core; - /* core_scratch = llc_scratch & this core */ - cpumask_and(core_scratch, llc_scratch, sib); + /* + * Take the cores of one cluster before moving + * on, so that a cluster is a contiguous cid + * range like the core, LLC and node levels. + */ + if (cpumask_empty(cluster_scratch)) { + s32 xcpu = cpumask_first(llc_scratch); + + cpumask_or(cluster_scratch, topology_cluster_cpumask(xcpu), + topology_sibling_cpumask(xcpu)); + cpumask_and(cluster_scratch, cluster_scratch, llc_scratch); + cluster_cid = next_cid; + cluster_idx = next_cluster_idx++; + } + + lcpu = cpumask_first(cluster_scratch); + sib = topology_sibling_cpumask(lcpu); + core_cid = next_cid; + core_idx = next_core_idx++; + + /* core_scratch = cluster_scratch & this core */ + cpumask_and(core_scratch, cluster_scratch, sib); if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch))) return -EINVAL; @@ -316,8 +338,11 @@ s32 scx_cid_init(struct scx_sched *sch) .node_idx = node_idx, .shard_cid = shard_cid, .shard_idx = shard_idx, + .cluster_cid = cluster_cid, + .cluster_idx = cluster_idx, }; + cpumask_clear_cpu(ccpu, cluster_scratch); cpumask_clear_cpu(ccpu, llc_scratch); cpumask_clear_cpu(ccpu, node_scratch); cpumask_clear_cpu(ccpu, to_walk); @@ -461,8 +486,8 @@ __bpf_kfunc_start_defs(); * starts must be strictly increasing with the first entry 0 and all values < * num_possible_cpus(). The last shard extends to num_possible_cpus() and no * shard may span more than SCX_CID_SHARD_MAX_CPUS cids. Topo info - * (core/LLC/node) is cleared and the shard layout is set from the input. On - * invalid input, abort the scheduler. + * (core/cluster/LLC/node) is cleared and the shard layout is set from the + * input. On invalid input, abort the scheduler. */ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid__arena, u32 cpu_to_cid_cnt, const s32 *shard_start__arena, u32 shard_start_cnt, diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h index 2fe2311a0f99..709afbfb97c6 100644 --- a/kernel/sched/ext/cid.h +++ b/kernel/sched/ext/cid.h @@ -13,11 +13,12 @@ * kernel type sized for the maximum NR_CPUS (4k), with verbose helper sequences * for every op. * - * cids give every cpu a dense, topology-ordered id. CPUs sharing a core, LLC or - * NUMA node get contiguous cid ranges, so a topology unit becomes a (start, - * length) slice of cid space. Communication can pass a slice instead of a - * cpumask, and BPF code can process, for example, a u64 word's worth of cids at - * a time. + * cids give every cpu a dense, topology-ordered id. CPUs in each core, + * cluster, LLC or NUMA node get contiguous cid ranges, so a topology unit + * becomes a (start, length) slice of cid space. A core without a wider cluster + * level forms a cluster of its own. Communication can pass a slice + * instead of a cpumask, and BPF code can process, for example, a u64 word's + * worth of cids at a time. * * The mapping is built once at root scheduler enable time by walking the * topology of online cpus only. Going by online cpus is out of necessity: diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index 139176cf9fc6..38514d6bf149 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -59,17 +59,31 @@ enum scx_consts { }; /* - * Per-cid topology info. For each topology level (core, LLC, node) and shard, - * records the first cid in the unit and its global index. Global indices are - * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges over - * [0, nr_cores_at_init) with no gaps. No-topo cids have core/LLC/node fields - * set to -1 but always have valid shard assignments. + * Per-cid topology info. For each topology level (core, cluster, LLC, node) and + * shard, records the first cid in the unit and its global index. Global indices + * are consecutive integers assigned in cid-walk order, so e.g. core_idx ranges + * over [0, nr_cores_at_init) with no gaps. No-topo cids have core/cluster/LLC/ + * node fields set to -1 but always have valid shard assignments. + * + * A cluster is the cache-sharing level between the core and the LLC. Where + * this level is absent, each core forms a cluster of its own. Where the cache + * level spans the LLC, the whole LLC forms one cluster. Each cluster has a + * unique cluster_cid and a dense cluster_idx, and its cids form a contiguous + * range. + * + * Every cid therefore has a cluster, so cluster_cid alone does not say whether + * the machine has the level: the first cluster of an LLC starts at the LLC's + * own base cid and a core-wide one at the core's. * * Shards are contiguous CID ranges used as scalable locking/work domains for * sub-scheduler operations. By default each LLC becomes one shard, split into * smaller shards if the LLC exceeds the target size. No-topo cids are packed * into their own max-sized shards. * + * Shards are cut on core boundaries only, so a shard doesn't need to contain a + * whole cluster: a cluster may straddle two shards. Only core, cluster, LLC + * and node are nested, in that order. + * * New fields are appended, never inserted: scx_bpf_cid_topo() copies this * struct out sized by the program's own layout, and an older program's copy * must stay a prefix of the kernel's. @@ -82,6 +96,8 @@ enum scx_consts { * @node_idx: global index of that node, in [0, nr_nodes_at_init) * @shard_cid: first cid of this cid's shard * @shard_idx: global index of that shard, in [0, scx_nr_cid_shards) + * @cluster_cid: first cid of this cid's cluster + * @cluster_idx: global index of that cluster, in [0, nr_clusters_at_init) */ struct scx_cid_topo { s32 core_cid; @@ -92,6 +108,8 @@ struct scx_cid_topo { s32 node_idx; s32 shard_cid; s32 shard_idx; + s32 cluster_cid; + s32 cluster_idx; }; enum scx_cid_consts { -- cgit v1.2.3 From be8be8acb26819cbd3992d8ce007d54899232b31 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:45 +0200 Subject: sched_ext: Block proxy donors before taking control Proxy execution retains mutex-blocked donors on the runqueue so their scheduling context can execute a lock owner. sched_ext cannot safely inherit such donors unless the BPF scheduler explicitly participates in their admission and ordering. Make sched_ext reject retained donors by default. Implement scx_allow_proxy_exec() to force blocked EXT tasks through the regular block path in schedule(). A donor retained by another scheduling class may already be queued when sched_ext is enabled or when a task switches to SCHED_EXT. Fully block the donor before sched_change_begin() records its queued state. This prevents sched_ext from importing a proxy session which the BPF scheduler did not admit, while leaving ordinary FAIR, RT and DL class transitions untouched. This is a preparatory change to support proxy execution with sched_ext. Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 37 +++++++++++++++++++++++++++++++++++-- kernel/sched/ext/ext.h | 2 ++ kernel/sched/syscalls.c | 2 ++ 3 files changed, 39 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 90cd68c3767f..bddea43fa6a1 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -26,7 +26,22 @@ DEFINE_RAW_SPINLOCK(scx_sched_lock); bool scx_allow_proxy_exec(const struct task_struct *p) { - return true; + return p->sched_class != &ext_sched_class; +} + +/* + * End retained proxy execution before sched_ext takes ownership of @p. + * Called with @p's pi and rq locks held immediately before + * sched_change_begin(). The caller must pass DEQUEUE_NOCLOCK so the rq clock + * is updated only once. + */ +static void scx_prepare_task_sched_change(struct task_struct *p) +{ + lockdep_assert_held(&p->pi_lock); + lockdep_assert_rq_held(task_rq(p)); + + update_rq_clock(task_rq(p)); + sched_proxy_block_task(task_rq(p), p); } /* @@ -4438,7 +4453,7 @@ int scx_check_setscheduler(struct task_struct *p, int policy) { lockdep_assert_rq_held(task_rq(p)); - /* if disallow, reject transitioning into SCX */ + /* If disallow, reject transitioning into SCX. */ if (scx_enabled() && READ_ONCE(p->scx.disallow) && p->policy != policy && policy == SCHED_EXT) return -EACCES; @@ -4446,6 +4461,20 @@ int scx_check_setscheduler(struct task_struct *p, int policy) return 0; } +/* + * Don't carry a donor retained by another class into sched_ext. The caller + * has updated the rq clock and invokes this immediately before + * sched_change_begin() records the task's queued state. + */ +void scx_prepare_setscheduler(struct task_struct *p, int policy) +{ + lockdep_assert_held(&p->pi_lock); + lockdep_assert_rq_held(task_rq(p)); + + if (scx_enabled() && p->policy != policy && policy == SCHED_EXT) + sched_proxy_block_task(task_rq(p), p); +} + static void process_ddsp_deferred_locals(struct rq *rq) { struct task_struct *p; @@ -7994,6 +8023,10 @@ static void scx_root_enable_workfn(struct kthread_work *work) if (old_class != new_class) queue_flags |= DEQUEUE_CLASS; + if (new_class == &ext_sched_class) { + scx_prepare_task_sched_change(p); + queue_flags |= DEQUEUE_NOCLOCK; + } scoped_guard (sched_change, p, queue_flags) { scx_set_task_slice(p, READ_ONCE(sch->slice_dfl)); diff --git a/kernel/sched/ext/ext.h b/kernel/sched/ext/ext.h index cca3f7c97b78..8348d6008651 100644 --- a/kernel/sched/ext/ext.h +++ b/kernel/sched/ext/ext.h @@ -18,6 +18,7 @@ bool scx_can_stop_tick(struct rq *rq); void scx_rq_activate(struct rq *rq); void scx_rq_deactivate(struct rq *rq); int scx_check_setscheduler(struct task_struct *p, int policy); +void scx_prepare_setscheduler(struct task_struct *p, int policy); bool task_should_scx(int policy); bool scx_allow_ttwu_queue(const struct task_struct *p); bool scx_allow_proxy_exec(const struct task_struct *p); @@ -62,6 +63,7 @@ static inline bool scx_can_stop_tick(struct rq *rq) { return true; } static inline void scx_rq_activate(struct rq *rq) {} static inline void scx_rq_deactivate(struct rq *rq) {} static inline int scx_check_setscheduler(struct task_struct *p, int policy) { return 0; } +static inline void scx_prepare_setscheduler(struct task_struct *p, int policy) {} static inline bool task_on_scx(const struct task_struct *p) { return false; } static inline bool scx_allow_ttwu_queue(const struct task_struct *p) { return true; } static inline bool scx_allow_proxy_exec(const struct task_struct *p) { return true; } diff --git a/kernel/sched/syscalls.c b/kernel/sched/syscalls.c index b215b0ead9a6..45a127deeadd 100644 --- a/kernel/sched/syscalls.c +++ b/kernel/sched/syscalls.c @@ -678,6 +678,8 @@ change: if (prev_class != next_class) queue_flags |= DEQUEUE_CLASS; + scx_prepare_setscheduler(p, policy); + scoped_guard (sched_change, p, queue_flags) { if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { -- cgit v1.2.3 From 5f882051efabaf19f9b4dbe4b50cee1d250ac19e Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:46 +0200 Subject: sched_ext: Fix ops.running/stopping() pairing for proxy-exec donors With proxy execution, pick_next_task() can select a blocked task as the scheduling context before find_proxy_task() resolves the execution context. From the BPF scheduler perspective, that donor is running while its scheduling context drives the lock owner; ops.tick() and other accounting must therefore remain enclosed by a matching ops.running()/ops.stopping() session. In this scenario, the session boundaries do not always match physical task switches. Keep the "running" session open when the same donor continues on the same CPU. When proxy execution migrates a donor's scheduling context to another CPU, end its running session on the source CPU and start a new session on the destination CPU only after proxy resolution succeeds. This prevents a failed resolution from exposing a provisional ops.running() event. Track these sessions with a new SCX_TASK_RUN_TRACKED flag. The explicit running-state tracking is also required by later donor-based accounting: it prevents an EXT owner executing for a non-EXT donor from being treated as the active EXT scheduling context when it is dequeued. This is a preparatory change for enabling proxy execution together with sched_ext. Acked-by: John Stultz Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 72 +++++++++++++++++++++++++++++++++++---------- kernel/sched/ext/internal.h | 6 ++++ 2 files changed, 62 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index bddea43fa6a1..63cbd3977876 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2377,10 +2377,10 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ ops_dequeue(rq, p, deq_flags); /* - * A currently running task which is going off @rq first gets dequeued - * and then stops running. As we want running <-> stopping transitions - * to be contained within runnable <-> quiescent transitions, trigger - * ->stopping() early here instead of in put_prev_task_scx(). + * A current scheduling context which is going off @rq first gets + * dequeued and then stops running. As we want running <-> stopping + * transitions to be contained within runnable <-> quiescent transitions, + * trigger ->stopping() early here instead of in put_prev_task_scx(). * * @p may go through multiple stopping <-> running transitions between * here and put_prev_task_scx() if task attribute changes occur while @@ -2388,11 +2388,13 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ * information meaningful to the BPF scheduler and can be suppressed by * skipping the callbacks if the task is !QUEUED. */ - if (task_current(rq, p) && - (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq)))) { - update_curr_scx(rq); - if (SCX_HAS_OP(sch, stopping)) - SCX_CALL_OP_TASK(sch, stopping, rq, p, false); + if (task_current_donor(rq, p) && (p->scx.flags & SCX_TASK_RUN_TRACKED)) { + if (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq))) { + update_curr_scx(rq); + if (SCX_HAS_OP(sch, stopping)) + SCX_CALL_OP_TASK(sch, stopping, rq, p, false); + } + p->scx.flags &= ~SCX_TASK_RUN_TRACKED; } if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p)) @@ -3111,10 +3113,23 @@ has_tasks: return verdict; } -static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e type) +static void scx_start_task_running(struct rq *rq, struct task_struct *p) { struct scx_sched *sch = scx_task_sched(p); + if (p->scx.flags & SCX_TASK_RUN_TRACKED) + return; + + if (SCX_HAS_OP(sch, running)) + SCX_CALL_OP_TASK(sch, running, rq, p); + + p->scx.flags |= SCX_TASK_RUN_TRACKED; +} + +static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e type) +{ + bool first = type == SNT_PICK; + if (type == SNT_REPICK) return; @@ -3129,9 +3144,20 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t p->se.exec_start = rq_clock_task(rq); - /* see dequeue_task_scx() on why we skip when !QUEUED */ - if (SCX_HAS_OP(sch, running) && (p->scx.flags & SCX_TASK_QUEUED)) - SCX_CALL_OP_TASK(sch, running, rq, p); + /* + * See dequeue_task_scx() for why we skip when !QUEUED. + * + * During a normal switch (@first), a blocked task is only a provisional + * donor. Proxy resolution may fail or migrate the donor to another CPU, + * so defer ops.running() until scx_proxy_donor_start() confirms that + * resolution succeeded. + * + * !@first denotes restoration after a SAVE/RESTORE cycle. The matching + * dequeue already issued ops.stopping(), so restart the session here + * regardless of the donor state. + */ + if ((p->scx.flags & SCX_TASK_QUEUED) && !(p->is_blocked && first)) + scx_start_task_running(rq, p); clr_task_runnable(p, true); @@ -3177,6 +3203,12 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t void scx_proxy_donor_start(struct rq *rq) { + struct task_struct *donor = rq->donor; + + lockdep_assert_rq_held(rq); + + if (donor->sched_class == &ext_sched_class && (donor->scx.flags & SCX_TASK_QUEUED)) + scx_start_task_running(rq, donor); } static enum scx_cpu_preempt_reason @@ -3252,9 +3284,17 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, scx_task_slice_ended(rq, p); } - /* see dequeue_task_scx() on why we skip when !QUEUED */ - if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED)) - SCX_CALL_OP_TASK(sch, stopping, rq, p, true); + /* + * Preserve the running session when proxy execution refreshes the same + * donor around an execution-context switch on this rq. + */ + if (next != p && (p->scx.flags & SCX_TASK_QUEUED) && + (p->scx.flags & SCX_TASK_RUN_TRACKED)) { + if (SCX_HAS_OP(sch, stopping)) + SCX_CALL_OP_TASK(sch, stopping, rq, p, true); + + p->scx.flags &= ~SCX_TASK_RUN_TRACKED; + } if (p->scx.flags & SCX_TASK_QUEUED) { set_task_runnable(rq, p); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 0bcceab612de..8439cfa36a78 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -493,6 +493,12 @@ struct sched_ext_ops { * Therefore, always use scx_bpf_task_cpu(@p) to determine the * target CPU the task is going to use. * + * Under proxy execution, the BPF scheduler continues to observe the + * donor as the current scheduling context. A blocked donor enters a + * ->running()/->stopping() session while its scheduling context drives + * the lock owner. The lock owner executing on its behalf is intentionally + * not reported through these callbacks. + * * See ->runnable() for explanation on the task state notifiers. */ void (*running)(struct task_struct *p); -- cgit v1.2.3 From cea8b60e4aee0435c23d343d31972664c65aa7c3 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:47 +0200 Subject: sched_ext: Move reject DSQ draining into core The reject DSQ drain will be generalized for placement failures outside sub-scheduler capability rejection. Move the drain into the core sched_ext implementation first so the subsequent behavioral changes can be reviewed separately. No functional change. Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ kernel/sched/ext/sub.c | 44 -------------------------------------------- kernel/sched/ext/sub.h | 10 ---------- 3 files changed, 50 insertions(+), 54 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 63cbd3977876..a4b685d554c2 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4849,6 +4849,56 @@ static void process_deferred_reenq_users(struct rq *rq) } } +#ifdef CONFIG_EXT_SUB_SCHED +/* + * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides + * from p->scx.reenq_reason_*. + * + * A task can be re-rejected repeatedly. The reenqueue is bounded per task in + * scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT. + * Rejection can't happen for root. + */ +static void scx_reenq_reject(struct rq *rq) +{ + LIST_HEAD(tasks); + struct task_struct *p, *n; + + lockdep_assert_rq_held(rq); + + if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list)) + return; + + /* + * Move to a private list so a task re-rejected by the + * scx_do_enqueue_task() below isn't revisited this round. + */ + list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) { + /* migration_pending tasks should have bypassed to local DSQ */ + if (WARN_ON_ONCE(p->migration_pending)) + continue; + + scx_reenq_wait_dispatching(p); + scx_dispatch_dequeue(rq, p); + + if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + p->scx.flags |= SCX_TASK_REENQ_CAP; + + list_add_tail(&p->scx.dsq_list.node, &tasks); + } + + list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { + list_del_init(&p->scx.dsq_list.node); + + scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); + + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + } +} +#else +static void scx_reenq_reject(struct rq *rq) {} +#endif + static void run_deferred(struct rq *rq) { process_ddsp_deferred_locals(rq); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 10567196be96..abe10a3529b9 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -772,50 +772,6 @@ bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) return true; } -/* - * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides - * from p->scx.reenq_reason_*. - * - * A task can be re-rejected repeatedly. The reenqueue is bounded per task in - * scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT. - * Rejection can't happen for root. - */ -void __scx_reenq_reject(struct rq *rq) -{ - LIST_HEAD(tasks); - struct task_struct *p, *n; - - if (list_empty(&rq->scx.reject_dsq.list)) - return; - - /* - * Move to a private list so a task re-rejected by the - * scx_do_enqueue_task() below isn't revisited this round. - */ - list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) { - /* migration_pending tasks should have bypassed to local DSQ */ - if (WARN_ON_ONCE(p->migration_pending)) - continue; - - scx_reenq_wait_dispatching(p); - scx_dispatch_dequeue(rq, p); - - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - p->scx.flags |= SCX_TASK_REENQ_CAP; - - list_add_tail(&p->scx.dsq_list.node, &tasks); - } - - list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { - list_del_init(&p->scx.dsq_list.node); - - scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - } -} - /* record a caps change, see struct scx_caps_updated */ static void caps_updated_record(struct scx_pshard *ps, const struct scx_cmask *cids, u64 caps, struct list_head *to_deliver) diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 6a7076b9a1dd..7621c28c7e8c 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -37,7 +37,6 @@ void scx_discard_stale_ecaps_syncs(void); struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags); bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p); -void __scx_reenq_reject(struct rq *rq); void scx_rescue_charge(struct rq *rq, s64 delta_exec); void scx_rescue_end(struct rq *rq); bool scx_rescue_keep(struct rq *rq, struct task_struct *p); @@ -91,14 +90,6 @@ static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch return __scx_resolve_local_dsq(sch, rq, p, enq_flags); } -static inline void scx_reenq_reject(struct rq *rq) -{ - lockdep_assert_rq_held(rq); - - if (scx_has_subs()) - __scx_reenq_reject(rq); -} - #else /* CONFIG_EXT_SUB_SCHED */ static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } @@ -120,7 +111,6 @@ static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcp static inline void scx_discard_stale_ecaps_syncs(void) {} static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; } static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; } -static inline void scx_reenq_reject(struct rq *rq) {} static inline void scx_rescue_charge(struct rq *rq, s64 delta_exec) {} static inline void scx_rescue_end(struct rq *rq) {} static inline bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { return false; } -- cgit v1.2.3 From 4eeabe0a3ef7069ba00d450b8b21fd5e0fe76e13 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:48 +0200 Subject: sched_ext: Generalize the reject DSQ reenqueue path The reject DSQ is currently specific to sub-scheduler cap failures. Its storage and initialization depend on CONFIG_EXT_SUB_SCHED, and the drain path assumes every rejected task was rejected for SCX_TASK_REENQ_CAP. Other transient placement failures need the same ability to park a task on its source rq and return it to the owning BPF scheduler. Make the reject DSQ unconditional and carry the reenqueue reason directly in p->scx.flags from the rejection site. A reenqueue reason remains valid while ops.enqueue() runs so that BPF can inspect it. Clear it immediately after the callback returns and before resolving a direct dispatch. Paths which bypass ops.enqueue() clear the reason before the kernel-selected placement. A rejection from the new placement can therefore install its reason into a clear field. Route all paths through a common cleanup point to enforce this ordering. This is a preparatory change to support proxy execution with sched_ext. Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 91 +++++++++++++++++++++++++++----------------------- kernel/sched/ext/sub.c | 2 ++ kernel/sched/sched.h | 2 +- 3 files changed, 53 insertions(+), 42 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index a4b685d554c2..9a4a5afee211 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1672,12 +1672,10 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, u64 slice, u64 vtime, u64 enq_flags) { - bool is_rq_owned = false; + bool is_rq_owned = dsq_is_rq_owned(dsq); - if (dsq->id == SCX_DSQ_LOCAL) { + if (dsq->id == SCX_DSQ_LOCAL) dsq = scx_resolve_local_dsq(sch, rq, p, &enq_flags); - is_rq_owned = true; - } WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || @@ -2065,6 +2063,12 @@ bool scx_rq_online(struct rq *rq) void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, int sticky_cpu) { + enum { + ENQ_ACTION_NONE, + ENQ_ACTION_DIRECT, + ENQ_ACTION_LOCAL_NOREFILL, + ENQ_ACTION_ENQUEUE, + } action = ENQ_ACTION_NONE; struct scx_sched *sch = scx_task_sched(p); struct task_struct **ddsp_taskp; struct scx_dispatch_q *dsq; @@ -2098,7 +2102,7 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, __scx_exit(sch, SCX_EXIT_ERROR_REENQ, 0, cpu_of(rq), "%s[%d] reenqueued %u times without running", p->comm, p->pid, p->scx.reenq_cnt); - return; + goto out; } } @@ -2163,14 +2167,14 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, * dequeue may be waiting. The store_release matches their load_acquire. */ atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq); - return; + goto out; direct: - direct_dispatch(sch, p, enq_flags); - return; + action = ENQ_ACTION_DIRECT; + goto out; local_norefill: - scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, enq_flags); - return; + action = ENQ_ACTION_LOCAL_NOREFILL; + goto out; local: dsq = &rq->scx.local_dsq; goto enqueue; @@ -2182,9 +2186,27 @@ bypass: goto enqueue; enqueue: - refill_task_slice_dfl(sch, p); - clear_direct_dispatch(p); - scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); + action = ENQ_ACTION_ENQUEUE; +out: + /* The reason is input to ops.enqueue(), not to the resulting placement. */ + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + + switch (action) { + case ENQ_ACTION_DIRECT: + direct_dispatch(sch, p, enq_flags); + break; + case ENQ_ACTION_LOCAL_NOREFILL: + scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, + enq_flags); + break; + case ENQ_ACTION_ENQUEUE: + refill_task_slice_dfl(sch, p); + clear_direct_dispatch(p); + scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); + break; + case ENQ_ACTION_NONE: + break; + } } static bool task_runnable(const struct task_struct *p) @@ -2410,6 +2432,7 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ sub_nr_running(rq, 1); scx_dispatch_dequeue(rq, p); + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; /* see scx_task_slice_ended() for the save/restore exception */ if (!((deq_flags & DEQUEUE_SAVE) && task_current(rq, p))) @@ -3140,6 +3163,7 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t */ ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC); scx_dispatch_dequeue(rq, p); + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } p->se.exec_start = rq_clock_task(rq); @@ -3314,7 +3338,6 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, if (p->scx.flags & SCX_TASK_IMMED) { p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } else { u64 enq_flags = 0; @@ -4655,8 +4678,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) scx_reenq_wait_dispatching(p); scx_dispatch_dequeue(rq, p); - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); p->scx.flags |= reason; list_add_tail(&p->scx.dsq_list.node, &tasks); @@ -4667,7 +4689,6 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; nr_enqueued++; } @@ -4780,14 +4801,11 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag dispatch_dequeue_locked(p, dsq); raw_spin_unlock(&dsq->lock); - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); p->scx.flags |= reason; scx_do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1); - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) { scx_rq_lock_drop(locked_rq); raw_spin_rq_unlock(locked_rq); @@ -4849,14 +4867,14 @@ static void process_deferred_reenq_users(struct rq *rq) } } -#ifdef CONFIG_EXT_SUB_SCHED /* - * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides - * from p->scx.reenq_reason_*. + * Drain @rq->scx.reject_dsq and reenqueue each task so that its owning BPF + * scheduler chooses placement again. * - * A task can be re-rejected repeatedly. The reenqueue is bounded per task in - * scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT. - * Rejection can't happen for root. + * A task can be re-rejected repeatedly. Reenqueues are bounded per task by + * SCX_REENQ_MAX_REPEAT in scx_do_enqueue_task(), which ejects the owning + * scheduler. The private list below prevents a task from being revisited in + * the same round. */ static void scx_reenq_reject(struct rq *rq) { @@ -4865,25 +4883,21 @@ static void scx_reenq_reject(struct rq *rq) lockdep_assert_rq_held(rq); - if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list)) + if (list_empty(&rq->scx.reject_dsq.list)) return; /* - * Move to a private list so a task re-rejected by the + * Move tasks to a private list so a task re-rejected by * scx_do_enqueue_task() below isn't revisited this round. */ list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) { /* migration_pending tasks should have bypassed to local DSQ */ - if (WARN_ON_ONCE(p->migration_pending)) - continue; + WARN_ON_ONCE(p->migration_pending); + WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)); scx_reenq_wait_dispatching(p); scx_dispatch_dequeue(rq, p); - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - p->scx.flags |= SCX_TASK_REENQ_CAP; - list_add_tail(&p->scx.dsq_list.node, &tasks); } @@ -4891,13 +4905,8 @@ static void scx_reenq_reject(struct rq *rq) list_del_init(&p->scx.dsq_list.node); scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } } -#else -static void scx_reenq_reject(struct rq *rq) {} -#endif static void run_deferred(struct rq *rq) { @@ -9015,8 +9024,8 @@ void __init init_sched_ext_class(void) /* local_dsq's sch will be set during scx_root_enable() */ BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); -#ifdef CONFIG_EXT_SUB_SCHED BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL)); +#ifdef CONFIG_EXT_SUB_SCHED scx_rescue_init(rq); #endif diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index abe10a3529b9..b5989aaffcc6 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -750,6 +750,8 @@ struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq p->scx.reenq_reason_caps = missing; p->scx.reenq_reason_cid = cid; + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); + p->scx.flags |= SCX_TASK_REENQ_CAP; return &rq->scx.reject_dsq; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index d41d23a5632b..1df76a3f38b9 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -810,8 +810,8 @@ struct scx_rq_rescue { struct scx_rq { struct scx_dispatch_q local_dsq; + struct scx_dispatch_q reject_dsq; /* staging for rejected tasks */ #ifdef CONFIG_EXT_SUB_SCHED - struct scx_dispatch_q reject_dsq; /* staging for cap-rejected tasks */ struct scx_rq_rescue rescue; #endif struct list_head runnable_list; /* runnable tasks on this rq */ -- cgit v1.2.3 From 13ae184e4b8279dcc7b84bf58e987a60d9aad69a Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:49 +0200 Subject: sched_ext: Handle proxy-exec races in remote DSQ transfers Without proxy execution, the DSQ lock and holding_cpu handshake ensure that a task cannot be dequeued or start running during an rq-lock handoff without clearing holding_cpu. Proxy execution is an exception: a task can start physically executing as a lock owner while its scheduling context remains on a DSQ; its on-CPU or migration-disabled state can therefore change without clearing holding_cpu. Recheck these states after acquiring the source rq lock. If the transfer can no longer proceed, park the task on the source rq's reject DSQ and reenqueue it through its owning scheduler. This preserves the BPF scheduler's placement policy and keeps descendant tasks within their sub-scheduler's cap grants. Queue a deferred drain for every rejection. If a drain finds a proxy-rejected task still running or donating, leave it parked and record a retry request on the rq. scx_proxy_reenqueue_retry() consumes the request and schedules another drain when proxy state changes. Without this change and proxy execution enabled, stress-ng --pipeherd can trigger this race and migrate an active execution context, leading to sleeping-while-atomic warnings and subsequent lockdep corruption. This is a preparatory change to support proxy execution with sched_ext. Suggested-by: Tejun Heo Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 143 ++++++++++++++++++++++++++++++++++++++------ kernel/sched/ext/internal.h | 9 +++ kernel/sched/ext/sub.c | 4 +- kernel/sched/sched.h | 1 + 4 files changed, 137 insertions(+), 20 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 9a4a5afee211..7ae4e6d0e2e4 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1138,8 +1138,17 @@ static void schedule_deferred_locked(struct rq *rq) schedule_deferred(rq); } +/* + * Retry proxy-rejected tasks which couldn't be reenqueued by an earlier drain. + */ void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next) { + lockdep_assert_rq_held(rq); + + if (rq->scx.flags & SCX_RQ_PROXY_RETRY) { + rq->scx.flags &= ~SCX_RQ_PROXY_RETRY; + schedule_deferred_locked(rq); + } } void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, @@ -1600,8 +1609,10 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0); /* - * Only local inserts get the wakeup treatment below. Rejects kick the - * deferred reenq and rescue parks are paced by the rescue timer. + * Only local inserts get the wakeup treatment below. Rejects kick a + * deferred reenq and rescue parks are paced by the rescue timer. Proxy + * rejects which aren't ready when drained request a later retry from + * scx_proxy_reenqueue_retry(). */ if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { if (dsq->id == SCX_DSQ_REJECT) @@ -2573,8 +2584,10 @@ static void move_remote_task_to_local_dsq(struct scx_sched *sch, * - The BPF scheduler is bypassed while the rq is offline and we can always say * no to the BPF scheduler initiated migrations while offline. * - * The caller must ensure that @p and @rq are on different CPUs. - * If enforce == true, caller must hold @p's rq lock. + * The caller must ensure that @p and @rq are on different CPUs. If @enforce is + * true, report violations attributable to BPF-directed migrations. The caller + * must hold @p's rq lock to avoid reporting a transient race as a scheduler + * error. */ static bool task_can_run_on_remote_rq(struct scx_sched *sch, struct task_struct *p, struct rq *rq, @@ -2582,11 +2595,6 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, { s32 cpu = cpu_of(rq); - /* - * To prevent races with @p still running on its old CPU while switching - * out, make sure we're holding @p's rq lock so as not to risk - * erroneously killing the BPF scheduler. - */ if (enforce) lockdep_assert_rq_held(task_rq(p)); @@ -2633,6 +2641,64 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, return true; } +/* + * Proxy execution can change @p's execution and migration-disabled state + * without touching its DSQ entry or clearing holding_cpu. Check those states + * with @p's rq locked. Without proxy execution, the holding_cpu handshake is + * sufficient and this must not affect the existing migration path. + * + * A BPF-directed transfer to a remote local DSQ performs a normal task + * migration and thus cannot move a migration-disabled task. In contrast, + * proxy_migrate_task() moves only a blocked donor's scheduling context towards + * the mutex owner and preserves its execution home in wake_cpu. The latter is + * therefore allowed even when the donor is migration-disabled. + */ +static bool task_proxy_running_or_donating(struct task_struct *p) +{ + struct rq *src_rq = task_rq(p); + + lockdep_assert_rq_held(src_rq); + + if (!sched_proxy_exec()) + return false; + + /* @p may be rq->curr under another task's scheduling context. */ + if (task_on_cpu(src_rq, p)) + return true; + + /* Don't move an active scheduling context off its source rq. */ + if (task_current_donor(src_rq, p)) + return true; + + return false; +} + +static bool task_proxy_unsafe_to_move(struct task_struct *p) +{ + if (!sched_proxy_exec()) + return false; + + return task_proxy_running_or_donating(p) || is_migration_disabled(p); +} + +/* + * Park a task whose remote transfer raced with proxy execution. Reenqueueing + * from the source rq makes the task's owning scheduler choose its placement + * again and preserves sub-scheduler containment. + */ +static void scx_proxy_reject_task(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 enq_flags) +{ + lockdep_assert_rq_held(rq); + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); + + p->scx.holding_cpu = -1; + p->scx.flags |= SCX_TASK_REENQ_PROXY; + scx_divert_strip_flags(p, &enq_flags); + + scx_dispatch_enqueue(sch, rq, &rq->scx.reject_dsq, p, 0, 0, enq_flags); +} + /** * unlink_dsq_and_switch_rq_lock() - Unlink task and switch to its rq lock * @p: target task @@ -2700,6 +2766,19 @@ static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq, struct scx_dispatch_q *dsq, struct rq *src_rq) { if (unlink_dsq_and_switch_rq_lock(p, dsq, this_rq, src_rq)) { + /* + * Proxy execution may have changed @p's running or + * migration-disabled state while switching rq locks without + * clearing holding_cpu. Park it on the source rq and let its + * owning scheduler choose its placement again. + */ + if (unlikely(task_proxy_unsafe_to_move(p))) { + p->scx.dsq = NULL; + scx_proxy_reject_task(sch, src_rq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); + switch_rq_lock(src_rq, this_rq); + return false; + } + move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, this_rq); return true; } else { @@ -2737,6 +2816,19 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, if (dst_dsq->id == SCX_DSQ_LOCAL) { dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); + /* + * Unlike the rq-lock handoff paths, @src_rq has been locked + * throughout this operation. Only active proxy state can race the + * move here; let the enforcing check below diagnose an ordinary + * migration-disabled task. + */ + if (src_rq != dst_rq && + unlikely(task_proxy_running_or_donating(p))) { + dispatch_dequeue_locked(p, src_dsq); + raw_spin_unlock(&src_dsq->lock); + scx_proxy_reject_task(sch, src_rq, p, enq_flags); + return src_rq; + } if (src_rq != dst_rq && unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { dst_dsq = find_global_dsq(sch, task_cpu(p)); @@ -2894,6 +2986,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, if (likely(p->scx.holding_cpu == raw_smp_processor_id()) && !WARN_ON_ONCE(src_rq != task_rq(p))) { bool fallback = false; + /* * If @p is staying on the same rq, there's no need to go * through the full deactivate/activate cycle. Optimize by @@ -2903,6 +2996,9 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, p->scx.holding_cpu = -1; scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, slice, vtime, enq_flags | SCX_ENQ_APPLY_SLICE); + } else if (unlikely(task_proxy_unsafe_to_move(p))) { + fallback = true; + scx_proxy_reject_task(sch, src_rq, p, enq_flags); } else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { p->scx.holding_cpu = -1; fallback = true; @@ -4868,13 +4964,13 @@ static void process_deferred_reenq_users(struct rq *rq) } /* - * Drain @rq->scx.reject_dsq and reenqueue each task so that its owning BPF - * scheduler chooses placement again. + * Drain ready tasks from @rq->scx.reject_dsq and reenqueue them so that their + * owning BPF schedulers choose placement again. Proxy-active tasks remain + * parked and rearm the retry notification for a later proxy resolution. * * A task can be re-rejected repeatedly. Reenqueues are bounded per task by * SCX_REENQ_MAX_REPEAT in scx_do_enqueue_task(), which ejects the owning - * scheduler. The private list below prevents a task from being revisited in - * the same round. + * scheduler. */ static void scx_reenq_reject(struct rq *rq) { @@ -4887,13 +4983,26 @@ static void scx_reenq_reject(struct rq *rq) return; /* - * Move tasks to a private list so a task re-rejected by + * Move ready tasks to a private list so a task re-rejected by * scx_do_enqueue_task() below isn't revisited this round. */ list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) { - /* migration_pending tasks should have bypassed to local DSQ */ - WARN_ON_ONCE(p->migration_pending); - WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)); + u32 reason = p->scx.flags & SCX_TASK_REENQ_REASON_MASK; + + WARN_ON_ONCE(!reason); + + if (sched_proxy_exec() && reason == SCX_TASK_REENQ_PROXY) { + /* Affinity machinery will dequeue and reactivate @p. */ + if (p->migration_pending) + continue; + + if (task_on_cpu(rq, p) || task_current_donor(rq, p)) { + rq->scx.flags |= SCX_RQ_PROXY_RETRY; + continue; + } + } else { + WARN_ON_ONCE(p->migration_pending); + } scx_reenq_wait_dispatching(p); scx_dispatch_dequeue(rq, p); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 8439cfa36a78..deb1a2b8ab9b 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1824,6 +1824,15 @@ enum scx_enq_flags { SCX_ENQ_SLICE_DFL = 1LLU << 62, /* carried slice is a default refill */ }; +/* Strip priority and carried slice state when diverting from a local DSQ. */ +static inline void scx_divert_strip_flags(struct task_struct *p, u64 *enq_flags) +{ + *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | + SCX_ENQ_PREEMPT_LAZY | SCX_ENQ_HEAD | + SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL); + p->scx.flags &= ~SCX_TASK_IMMED; +} + enum scx_deq_flags { /* expose select DEQUEUE_* flags as enums */ SCX_DEQ_SLEEP = DEQUEUE_SLEEP, diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index b5989aaffcc6..08599bc57801 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -733,9 +733,7 @@ struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq * or HEAD - a diversion has no priority and IMMED is not allowed on * non-local DSQs. Strip the enq and task flags along with the slice. */ - *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY | - SCX_ENQ_HEAD | SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL); - p->scx.flags &= ~SCX_TASK_IMMED; + scx_divert_strip_flags(p, enq_flags); /* the enqueuer opted for rescue instead of rejection and reenqueue */ if ((*enq_flags & SCX_ENQ_RESCUE) && likely(scx_rescue_bw_1024)) { diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 1df76a3f38b9..25e20f82788a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -791,6 +791,7 @@ enum scx_rq_flags { SCX_RQ_BAL_CB_PENDING = 1 << 6, /* must queue a cb after dispatching */ SCX_RQ_SUB_IDLE_RENOTIFY = 1 << 7, /* sub-scheds are owed update_idle() */ SCX_RQ_ROOT_IDLE_RENOTIFY = 1 << 8, /* the root is owed update_idle() */ + SCX_RQ_PROXY_RETRY = 1 << 9, /* proxy-rejected tasks need retry */ SCX_RQ_IN_WAKEUP = 1 << 16, SCX_RQ_IN_DISPATCH = 1 << 17, -- cgit v1.2.3 From 3d6577ea93c9633d95430c82c1cfe189d0f8fd47 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:50 +0200 Subject: sched_ext: Split curr|donor references properly With proxy execution, the task selected by the scheduler and the task physically executing can differ. A blocked mutex waiter donates its scheduling context to the lock owner: D -----------------> M -------------> O ----------------> T [donor] blocked on [mutex] owned by [owner] preempted by [task] \_________________________________^ donates scheduling context where: D = blocked donor M = mutex O = mutex owner T = competing runnable task During a proxy execution switch, D supplies the scheduling class, priority, and runtime budget, while O supplies the execution context: O is the task whose code physically executes. T is a competing runnable task which may preempt the D/O proxy execution. Consider FAIR and EXT tasks with sched_ext running in partial mode. FAIR can be replaced with a higher scheduling class such as RT or deadline without changing the class interaction described here. An active proxy pair can only have a donor in the same scheduling class as the owner or in a higher-priority class. If the owner belongs to a higher-priority class, pick_next_task() selects the owner directly before considering the donor, so find_proxy_task() is not entered for that donor. The blocked-on relationship may still exist, but it does not result in proxy execution. The relevant combinations are: 1. D is EXT, O is EXT, T is EXT D can interrupt T according to BPF scheduling policy. O executes with D's EXT priority and runtime budget, while T waits in EXT. 2. D is EXT, O is EXT, T is FAIR D is visible to the BPF scheduler, but cannot preempt T because EXT is below FAIR. Once T stops, BPF can dispatch D and O executes with D's EXT priority and runtime budget. If T becomes runnable again, it preempts the D/O proxy execution. 3. D is FAIR, O is EXT, T is EXT D preempts T as the higher-class scheduling context. O executes with D's FAIR priority and runtime budget, while T waits in EXT. D is not visible to the BPF scheduler. 4. D is FAIR, O is EXT, T is FAIR D competes with T according to its FAIR deadline. When D is selected, O executes with D's FAIR priority and runtime budget. D is not visible to the BPF scheduler. 5. D is FAIR, O is FAIR, T is EXT The D/O proxy execution has a FAIR scheduling context, so T cannot preempt it from the lower EXT class. 6. D is FAIR, O is FAIR, T is FAIR O, T, and D all have FAIR scheduling contexts. D remains runnable as a blocked proxy donor. When CFS selects D, O executes using D's FAIR scheduling context. When CFS selects O, O executes using its own FAIR context, and when CFS selects T, T executes normally. D is not visible to the BPF scheduler. Thus, sched_ext policy and accounting must generally use rq->donor, the scheduler-selected task which supplies the scheduling context, rather than rq->curr, the task whose code physically executes. Without proxy execution they are the same task. On nohz_full CPUs, keep the tick running whenever the selected donor is blocked. A selected blocked donor is necessarily awaiting proxy execution, so this avoids relying on the transient relationship between rq->curr and rq->donor during context switches. It also leaves ordinary sched_ext context switches and non-nohz_full CPUs untouched. Allowing proxy execution to run tickless remains a future improvement. Moreover, extend scx_dump_state() to report both contexts. Each CPU record now includes a donor= line. If an EXT donor differs from rq->curr, also emit its detailed task record. The existing '*' marker continues to identify rq->curr, while the donor= line identifies the otherwise unmarked donor record. Note that at this point in the series, CONFIG_SCHED_PROXY_EXEC still depends on !CONFIG_SCHED_CLASS_EXT, so proxy execution and sched_ext cannot be enabled together. The scheduling changes are therefore preparatory. A later patch removes this restriction. Co-developed-by: John Stultz Signed-off-by: John Stultz Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 132 +++++++++++++++++++++++++++++++------------------ kernel/sched/ext/sub.c | 5 +- kernel/sched/ext/sub.h | 11 +++-- 3 files changed, 93 insertions(+), 55 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 7ae4e6d0e2e4..9d41ec82f3bf 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -425,28 +425,28 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) */ /* - * If we're in the dispatch path holding rq lock, $curr may or may not + * If we're in the dispatch path holding rq lock, $donor may or may not * be ready depending on whether the on-going dispatch decides to extend - * $curr's slice. We say yes here and resolve it at the end of dispatch. + * $donor's slice. We say yes here and resolve it at the end of dispatch. * See dispatch_one(). */ if (rq->scx.flags & SCX_RQ_IN_DISPATCH) return true; /* - * The preemption flags clear $curr's slice if on SCX and kick dispatch, - * so allow them to avoid spuriously triggering reenq on a combined + * The preemption flags clear $donor's slice if on SCX and kick dispatch, + * so allow it to avoid spuriously triggering reenq on a combined * PREEMPT|IMMED insertion. */ if (enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) { - struct task_struct *curr = rq->curr; + struct task_struct *donor = rq->donor; /* * A protected slice refuses the preemption and the cpu stays * occupied. See rq_owned_post_enq(). */ - return curr->sched_class != &ext_sched_class || - likely(!(curr->scx.flags & SCX_TASK_PROTECTED)); + return donor->sched_class != &ext_sched_class || + likely(!(donor->scx.flags & SCX_TASK_PROTECTED)); } /* @@ -1462,20 +1462,27 @@ static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 e static void update_curr_scx(struct rq *rq) { - struct task_struct *curr = rq->curr; + struct task_struct *donor; s64 delta_exec; + /* + * update_curr_scx() is selected through rq->donor->sched_class, not + * rq->curr->sched_class, so @donor is always an EXT task here. If an EXT + * owner executes for a FAIR donor, FAIR's update_curr() runs instead. + */ + donor = rq->donor; + /* apply even on 0 delta_exec, callers may still act on the slice */ - apply_task_slice_oob(rq, curr); + apply_task_slice_oob(rq, donor); delta_exec = update_curr_common(rq); if (unlikely(delta_exec <= 0)) return; - if (curr->scx.slice != SCX_SLICE_INF) - curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); + if (donor->scx.slice != SCX_SLICE_INF) + donor->scx.slice -= min_t(u64, donor->scx.slice, delta_exec); - if (unlikely(curr == scx_rescuee(rq))) + if (unlikely(donor == scx_rescuee(rq))) scx_rescue_charge(rq, delta_exec); dl_server_update(&rq->ext_server, delta_exec); @@ -1666,9 +1673,9 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, if (rq->scx.flags & SCX_RQ_IN_DISPATCH) return; - if ((enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) && p != rq->curr && - rq->curr->sched_class == &ext_sched_class) { - if (likely(scx_set_task_slice(rq->curr, 0))) { + if ((enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) && + p != rq->donor && rq->donor->sched_class == &ext_sched_class) { + if (likely(scx_set_task_slice(rq->donor, 0))) { if (enq_flags & SCX_ENQ_PREEMPT) resched_curr(rq); else @@ -2273,13 +2280,14 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ rq->scx.flags |= SCX_RQ_IN_WAKEUP; /* - * Restoring a running task will be immediately followed by - * set_next_task_scx() which expects the task to not be on the BPF + * Restoring the current scheduling context will be immediately followed + * by set_next_task_scx() which expects the task to not be on the BPF * scheduler as tasks can only start running through local DSQs. Force * direct-dispatch into the local DSQ by setting the sticky_cpu. Mark * IGNORE_CAPS to force entry into the local DSQ. */ - if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) { + if (unlikely(enq_flags & ENQUEUE_RESTORE) && + task_current_donor(rq, p)) { sticky_cpu = cpu_of(rq); enq_flags |= SCX_ENQ_IGNORE_CAPS; } @@ -2446,7 +2454,7 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; /* see scx_task_slice_ended() for the save/restore exception */ - if (!((deq_flags & DEQUEUE_SAVE) && task_current(rq, p))) + if (!((deq_flags & DEQUEUE_SAVE) && task_current_donor(rq, p))) scx_task_slice_ended(rq, p); clear_direct_dispatch(p); @@ -3014,7 +3022,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, } /* if the destination CPU is idle, wake it up */ - if (!fallback && sched_class_above(p->sched_class, dst_rq->curr->sched_class)) + if (!fallback && sched_class_above(p->sched_class, + dst_rq->donor->sched_class)) resched_curr(dst_rq); } @@ -3248,6 +3257,7 @@ static void scx_start_task_running(struct rq *rq, struct task_struct *p) static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e type) { bool first = type == SNT_PICK; + bool can_stop_tick; if (type == SNT_REPICK) return; @@ -3283,6 +3293,7 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t /* apply any pending out-of-band slice request before the tick decision */ apply_task_slice_oob(rq, p); + can_stop_tick = p->scx.slice == SCX_SLICE_INF && !p->is_blocked; /* * @p is getting newly scheduled or got kicked after someone updated its @@ -3293,7 +3304,7 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t * nohz. In the future, we might want to add a mechanism to update * load_avgs periodically on tick-stopped CPUs. */ - if (p->scx.slice == SCX_SLICE_INF) { + if (can_stop_tick) { if (!(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) { /* * Bypass mode always assigns finite slices, so @p @@ -3314,7 +3325,8 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t /* * @rq still references the outgoing scheduling context. A finite - * slice is sufficient by itself to require the tick. + * slice or a blocked proxy donor is sufficient by itself to require + * the tick. */ if (tick_nohz_full_cpu(cpu_of(rq))) tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED); @@ -3633,7 +3645,7 @@ static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *r static struct task_struct * do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) { - struct task_struct *prev = rq->curr; + struct task_struct *prev = rq->donor; enum scx_dsp_verdict verdict; struct task_struct *p; @@ -4079,9 +4091,9 @@ void scx_tick(struct rq *rq) update_other_load_avgs(rq); } -static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) +static void task_tick_scx(struct rq *rq, struct task_struct *donor, int queued) { - struct scx_sched *sch = scx_task_sched(curr); + struct scx_sched *sch = scx_task_sched(donor); update_curr_scx(rq); @@ -4090,13 +4102,14 @@ static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) * management. */ if (scx_bypassing(sch, cpu_of(rq))) - scx_set_task_slice(curr, 0); + scx_set_task_slice(donor, 0); else if (SCX_HAS_OP(sch, tick)) - SCX_CALL_OP_TASK(sch, tick, rq, curr); + SCX_CALL_OP_TASK(sch, tick, rq, donor); - if (!curr->scx.slice) { + if (!donor->scx.slice) { /* the slice can't be trusted while bypassing */ - if (READ_ONCE(curr->scx.lazy_resched) && !scx_bypassing(sch, cpu_of(rq))) + if (READ_ONCE(donor->scx.lazy_resched) && + !scx_bypassing(sch, cpu_of(rq))) scx_resched_curr_lazy(rq); else resched_curr(rq); @@ -4789,16 +4802,16 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) } /* - * The revoke that scheduled this scan may have raced the pick: curr + * The revoke that scheduled this scan may have raced the pick: donor * may be a now-capless task, either one that kept running or one * promoted off the local DSQ between the ecaps sync and this scan. * Zero the slice to evict it. The enqueue gate blocks new capless * inserts, so no later pick can slip through after the scan. */ if ((reenq_flags & SCX_REENQ_CAP_REVOKE) && - rq->curr->sched_class == &ext_sched_class && - scx_task_reenq_on_cap_revoke(rq, rq->curr)) { - scx_set_task_slice(rq->curr, 0); + rq->donor->sched_class == &ext_sched_class && + scx_task_reenq_on_cap_revoke(rq, rq->donor)) { + scx_set_task_slice(rq->donor, 0); resched_curr(rq); } @@ -5033,14 +5046,18 @@ static void run_deferred(struct rq *rq) #ifdef CONFIG_NO_HZ_FULL bool scx_can_stop_tick(struct rq *rq) { - struct task_struct *p = rq->curr; + struct task_struct *p = rq->donor; struct scx_sched *sch = scx_task_sched(p); + /* Keep the tick running while a blocked proxy donor is selected. */ + if (p->is_blocked) + return false; + if (p->sched_class != &ext_sched_class) return true; /* - * @rq->curr may still reference an outgoing EXT task after it has been + * @rq->donor may still reference an outgoing EXT task after it has been * dequeued. If no EXT tasks are accounted on @rq, ignore its stale * slice state. If another task is dispatched from a DSQ, * set_next_task_scx() will update the dependency for the incoming task. @@ -5061,7 +5078,8 @@ bool scx_can_stop_tick(struct rq *rq) /* * @rq can dispatch from different DSQs, so we can't tell whether it * needs the tick or not by looking at nr_running. Allow stopping ticks - * iff the BPF scheduler indicated so. See set_next_task_scx(). + * iff set_next_task_scx() determined that the selected scheduling context + * can run tickless. */ return rq->scx.flags & SCX_RQ_CAN_STOP_TICK; } @@ -6576,9 +6594,9 @@ void scx_bypass(struct scx_sched *sch, bool bypass) /* * Bypass trumps protection. Cycling clears for queued - * tasks but current task needs explicit stripping. + * tasks but the current donor needs explicit stripping. */ - if (bypass && task_current(rq, p)) + if (bypass && task_current_donor(rq, p)) scx_task_slice_ended(rq, p); /* cycling deq/enq is enough, see the function comment */ @@ -7286,6 +7304,8 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, scx_rescue_dump(&ns, rq); scx_dump_line(&ns, " curr=%s[%d] class=%ps", rq->curr->comm, rq->curr->pid, rq->curr->sched_class); + scx_dump_line(&ns, " donor=%s[%d] class=%ps", + rq->donor->comm, rq->donor->pid, rq->donor->sched_class); if (!cpumask_empty(pcpu->cpus_to_kick)) scx_dump_line(&ns, " cpus_to_kick : %*pb", cpumask_pr_args(pcpu->cpus_to_kick)); @@ -7332,6 +7352,10 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, if (rq->curr->sched_class == &ext_sched_class && (dump_all_tasks || scx_task_on_sched(sch, rq->curr))) scx_dump_task(sch, s, dctx, rq, rq->curr, '*'); + if (rq->donor != rq->curr && + rq->donor->sched_class == &ext_sched_class && + (dump_all_tasks || scx_task_on_sched(sch, rq->donor))) + scx_dump_task(sch, s, dctx, rq, rq->donor, ' '); list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) if (dump_all_tasks || scx_task_on_sched(sch, p)) @@ -8890,7 +8914,7 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r bool preempt, preempt_lazy, wait, immediate; rq_lock_irqsave(rq, &rf); - cur_class = rq->curr->sched_class; + cur_class = rq->donor->sched_class; preempt = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt); preempt_lazy = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt_lazy); wait = cpumask_test_cpu(cpu, pcpu->cpus_to_wait); @@ -8921,7 +8945,7 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r __scx_add_event(pcpu->sch, SCX_EV_SUB_PREEMPT_DENIED, 1); /* degrade to a plain, immediate kick */ immediate = true; - } else if (unlikely(!scx_set_task_slice(rq->curr, 0))) { + } else if (unlikely(!scx_set_task_slice(rq->donor, 0))) { __scx_add_event(pcpu->sch, SCX_EV_SLICE_DENIED, 1); } } @@ -9924,8 +9948,10 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, return false; /* - * Directly write only when we hold the lock of the rq @p is queued or - * running on. See the write rules above. + * Directly write only when we hold the lock of the rq @p is queued on or + * provides the current scheduling context for. Under proxy execution, + * rq->donor owns and consumes the slice while rq->curr executes on its + * behalf. See the slice write rules above. * * While @p is queued on a user DSQ or in the BPF scheduler, * synchronization is the scheduler's responsibility. This write can @@ -9939,7 +9965,7 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, locked_rq = scx_locked_rq(); if (!locked_rq || (READ_ONCE(p->scx.runnable_cpu) != cpu_of(locked_rq) && - !task_current(locked_rq, p))) { + !task_current_donor(locked_rq, p))) { set_task_slice_oob(sch, p, slice); return true; } @@ -10927,12 +10953,17 @@ __bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask) } /** - * scx_bpf_task_running - Is task currently running? + * scx_bpf_task_running - Is task the current scheduling context? * @p: task of interest + * + * Under proxy execution, this reports the donor rather than the task whose + * code is physically executing. The physical execution context is intentionally + * not exposed to the BPF scheduler, which continues to observe the donor as the + * running scheduling context. */ __bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p) { - return task_rq(p)->curr == p; + return rcu_access_pointer(task_rq(p)->donor) == p; } /** @@ -10993,10 +11024,15 @@ __bpf_kfunc struct rq *scx_bpf_locked_rq(const struct bpf_prog_aux *aux) } /** - * scx_bpf_cpu_curr - Return remote CPU's curr task + * scx_bpf_cpu_curr - Return remote CPU's current scheduling context * @cpu: CPU of interest * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * + * Under proxy execution, this returns the donor, which supplies the scheduling + * policy and runtime budget, rather than the task whose code is physically + * executing. The physical execution context is intentionally not exposed to + * the BPF scheduler. + * * Callers must hold RCU read lock (KF_RCU). */ __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_aux *aux) @@ -11012,7 +11048,7 @@ __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_ if (!scx_cpu_valid(sch, cpu, NULL)) return NULL; - return rcu_dereference(cpu_rq(cpu)->curr); + return rcu_dereference(cpu_rq(cpu)->donor); } /** @@ -11036,7 +11072,7 @@ __bpf_kfunc struct task_struct *scx_bpf_cid_curr(s32 cid, const struct bpf_prog_ cpu = scx_cid_to_cpu(sch, cid); if (cpu < 0) return NULL; - return rcu_dereference(cpu_rq(cpu)->curr); + return rcu_dereference(cpu_rq(cpu)->donor); } /** diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 08599bc57801..4541ef8eec95 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -285,7 +285,8 @@ void scx_rescue_charge(struct rq *rq, s64 delta_exec) rq->scx.rescue.budget -= delta_exec; /* per-cpu usage average feeds the overload victim pick */ - pcpu = per_cpu_ptr(scx_task_sched(rq->curr)->pcpu, cpu_of(rq)); + pcpu = per_cpu_ptr(scx_task_sched(rq->scx.rescue.curr)->pcpu, + cpu_of(rq)); pcpu->rescue_avg = scx_rescue_decay_avg(pcpu) + delta_exec; if (!scx_rescue_slice_remaining(rq)) @@ -557,7 +558,7 @@ static void scx_rescue_timerfn(struct timer_list *timer) scx_rescue_admit(rq, p, slice); scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, rq); - if (sched_class_above(&ext_sched_class, rq->curr->sched_class)) + if (sched_class_above(&ext_sched_class, rq->donor->sched_class)) resched_curr(rq); } else if (p->scx.dsq && rq->scx.rescue.budget > 2 * scx_rescue_quantum_ns) { /* diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 7621c28c7e8c..c0174e523f2a 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -186,17 +186,18 @@ static inline u64 scx_caps_for_task(struct task_struct *p) return SCX_CAP_ENQ; } -/* the cap @sch needs to preempt @rq's current task, 0 if none */ -static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) +/* the cap @sch needs to preempt @rq's current scheduling context, 0 if none */ +static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, + u64 enq_flags) { - struct task_struct *curr = rq->curr; + struct task_struct *donor = rq->donor; /* a kernel-forced placement preempts regardless of caps */ if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS)) return 0; /* a non-ext task can't be preempted by ext, own-subtree needs no cap */ - if (curr->sched_class != &ext_sched_class || - scx_is_descendant(scx_task_sched(curr), sch)) + if (donor->sched_class != &ext_sched_class || + scx_is_descendant(scx_task_sched(donor), sch)) return 0; return SCX_CAP_PREEMPT; } -- cgit v1.2.3 From 74d9e0401539692198691d799a3b1ef809049528 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:51 +0200 Subject: sched_ext: Track proxy execution for NOHZ_FULL scx_can_stop_tick() currently treats any blocked rq->donor as an active proxy session. However, rq->donor can temporarily still identify an outgoing blocked task while the scheduler selects an ordinary execution context. A dependency update in that window can keep TICK_DEP_BIT_SCHED set after the context switch. Track whether proxy resolution actually selected a different execution context. Keep the tick active while that state is set instead of deriving it from a potentially stale donor. When proxy execution ends, clear the state and queue a balance callback to reevaluate the tick dependency after context_switch() updates rq->curr. This lets sched_can_stop_tick() observe the complete donor and execution selection without adding work outside the existing proxy-execution branch. Conservatively re-enable the periodic scheduler tick for the duration of proxy execution. Allowing proxy execution itself to run tickless would require making the remote NOHZ scheduler tick aware of the split between rq->curr and rq->donor, which is left as a future improvement. Perform the tick update from scx_proxy_reenqueue_retry(), which already runs after proxy resolution, so scheduler core needs no additional hook. Compile the additional runqueue state and callback out when CONFIG_NO_HZ_FULL is disabled. Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 44 +++++++++++++++++++++++++++++++++++++++++--- kernel/sched/sched.h | 4 ++++ 2 files changed, 45 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 9d41ec82f3bf..a7f23d881849 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1138,13 +1138,51 @@ static void schedule_deferred_locked(struct rq *rq) schedule_deferred(rq); } +#ifdef CONFIG_NO_HZ_FULL +static void scx_proxy_tick_bal_cb(struct rq *rq) +{ + sched_update_tick_dependency(rq); +} + +static void scx_proxy_update_tick(struct rq *rq, struct task_struct *next) +{ + bool proxy = next != rq->donor; + bool was_proxy = rq->scx.flags & SCX_RQ_PROXY_TICK; + + if (likely(proxy == was_proxy)) + return; + + if (proxy) { + /* Keep proxy execution tick-driven for now. */ + rq->scx.flags |= SCX_RQ_PROXY_TICK; + tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED); + } else { + rq->scx.flags &= ~SCX_RQ_PROXY_TICK; + /* + * The selected donor is already visible, but rq->curr still + * identifies the outgoing execution context. Reevaluate after + * context_switch() updates rq->curr so sched_can_stop_tick() sees + * the complete selection. + */ + queue_balance_callback(rq, &rq->scx.proxy_tick_bal_cb, + scx_proxy_tick_bal_cb); + } +} +#endif + /* - * Retry proxy-rejected tasks which couldn't be reenqueued by an earlier drain. + * Complete sched_ext bookkeeping after proxy resolution and retry tasks which + * couldn't be reenqueued by an earlier reject DSQ drain. */ void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next) { lockdep_assert_rq_held(rq); +#ifdef CONFIG_NO_HZ_FULL + if (scx_enabled() && tick_nohz_full_cpu(cpu_of(rq))) + scx_proxy_update_tick(rq, next); +#endif + if (rq->scx.flags & SCX_RQ_PROXY_RETRY) { rq->scx.flags &= ~SCX_RQ_PROXY_RETRY; schedule_deferred_locked(rq); @@ -5049,8 +5087,8 @@ bool scx_can_stop_tick(struct rq *rq) struct task_struct *p = rq->donor; struct scx_sched *sch = scx_task_sched(p); - /* Keep the tick running while a blocked proxy donor is selected. */ - if (p->is_blocked) + /* Proxy execution is conservatively tick-driven for now. */ + if (rq->scx.flags & SCX_RQ_PROXY_TICK) return false; if (p->sched_class != &ext_sched_class) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 25e20f82788a..64a4ccc6e0d0 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -792,6 +792,7 @@ enum scx_rq_flags { SCX_RQ_SUB_IDLE_RENOTIFY = 1 << 7, /* sub-scheds are owed update_idle() */ SCX_RQ_ROOT_IDLE_RENOTIFY = 1 << 8, /* the root is owed update_idle() */ SCX_RQ_PROXY_RETRY = 1 << 9, /* proxy-rejected tasks need retry */ + SCX_RQ_PROXY_TICK = 1 << 10, /* proxy execution requires the tick */ SCX_RQ_IN_WAKEUP = 1 << 16, SCX_RQ_IN_DISPATCH = 1 << 17, @@ -846,6 +847,9 @@ struct scx_rq { struct list_head deferred_reenq_users; /* user DSQs requesting reenq */ struct balance_callback deferred_bal_cb; struct balance_callback kick_sync_bal_cb; +#ifdef CONFIG_NO_HZ_FULL + struct balance_callback proxy_tick_bal_cb; +#endif struct irq_work deferred_irq_work; struct irq_work kick_cpus_irq_work; }; -- cgit v1.2.3 From ee172227d0dc9b169d41f5db154ed91db20073f7 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:52 +0200 Subject: sched_ext: Delegate proxy donor admission to BPF schedulers Proxy execution keeps a mutex-blocked donor runnable so that its scheduling context can execute the mutex owner. Define SCX_OPS_ENQ_BLOCKED as the admission contract for retained donors. Schedulers without the flag block EXT donors normally. Schedulers with the flag continue to own blocked donors and receive them through ops.enqueue() with SCX_ENQ_BLOCKED, allowing BPF to choose their DSQ, CPU and ordering. After BPF dispatches a donor, proxy execution may move its context to the rq of the mutex owner. This move is distinct from BPF placement: proxy_set_task_cpu() changes task_cpu() while preserving the donor's pre-proxy CPU in wake_cpu as the starting point for normal wakeup placement. A proxy-migrated donor returns through the full wakeup activation path when the mutex is released. Do not carry a retained proxy session across BPF scheduler ownership changes. Before root activation, parent-to-child takeover, rehome or punt, fully deactivate a retained donor unconditionally. The incoming scheduler then starts with clean task state and applies its admission policy the next time the task blocks. A retained donor can wake through ttwu_runnable() without another enqueue_task_scx() call. Use WF_TTWU_RQ to identify this path, where the wakeup skipped activation and enqueueing. wakeup_preempt_scx() requests dispatch reconsideration after is_blocked is cleared. A full wakeup activation has already enqueued the task and does not need another reschedule. Blocked-donor enqueueing takes precedence over the exiting and migration-disabled local-DSQ fallbacks, so that an opted-in scheduler sees every eligible donor request. Co-developed-by: John Stultz Signed-off-by: John Stultz Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 104 ++++++++++++++++++++++++++++++++++++-------- kernel/sched/ext/ext.h | 10 ++++- kernel/sched/ext/internal.h | 23 +++++++++- kernel/sched/ext/sub.c | 12 +++-- 4 files changed, 125 insertions(+), 24 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index a7f23d881849..e48aac45afea 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -24,18 +24,24 @@ DEFINE_RAW_SPINLOCK(scx_sched_lock); -bool scx_allow_proxy_exec(const struct task_struct *p) +bool __scx_allow_proxy_exec(const struct task_struct *p) { - return p->sched_class != &ext_sched_class; + struct scx_sched *sch; + + if (p->sched_class != &ext_sched_class) + return true; + + sch = scx_task_sched(p); + return !sch || (sch->ops.flags & SCX_OPS_ENQ_BLOCKED); } /* - * End retained proxy execution before sched_ext takes ownership of @p. + * End retained proxy execution before changing @p's BPF scheduler ownership. * Called with @p's pi and rq locks held immediately before * sched_change_begin(). The caller must pass DEQUEUE_NOCLOCK so the rq clock * is updated only once. */ -static void scx_prepare_task_sched_change(struct task_struct *p) +void scx_prepare_task_sched_change(struct task_struct *p) { lockdep_assert_held(&p->pi_lock); lockdep_assert_rq_held(task_rq(p)); @@ -2178,19 +2184,28 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) goto direct; - /* see %SCX_OPS_ENQ_EXITING */ - if (!(sch->ops.flags & SCX_OPS_ENQ_EXITING) && - unlikely(p->flags & PF_EXITING)) { - __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); - enq_flags |= SCX_ENQ_RESCUE; /* avoid looping on cap rejection */ - goto local; - } + /* + * A full wakeup enqueue happens before is_blocked is cleared. Don't + * report it as another blocked-donor admission. + */ + if ((sch->ops.flags & SCX_OPS_ENQ_BLOCKED) && p->is_blocked && + !(enq_flags & SCX_ENQ_WAKEUP)) { + enq_flags |= SCX_ENQ_BLOCKED; + } else { + /* see %SCX_OPS_ENQ_EXITING */ + if (unlikely(p->flags & PF_EXITING) && + !(sch->ops.flags & SCX_OPS_ENQ_EXITING)) { + __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); + enq_flags |= SCX_ENQ_RESCUE; /* avoid looping on cap rejection */ + goto local; + } - /* see %SCX_OPS_ENQ_MIGRATION_DISABLED */ - if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) && - is_migration_disabled(p)) { - __scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1); - goto local; + /* see %SCX_OPS_ENQ_MIGRATION_DISABLED */ + if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) && + is_migration_disabled(p)) { + __scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1); + goto local; + } } if (unlikely(!SCX_HAS_OP(sch, enqueue))) @@ -2532,10 +2547,23 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl /* * Preemption between SCX tasks is implemented by resetting the victim * task's slice to 0 and triggering reschedule on the target CPU. - * Nothing to do. - */ - if (p->sched_class == &ext_sched_class) + * + * A retained proxy donor can wake through ttwu_runnable() without another + * ops.enqueue(). WF_TTWU_RQ identifies this path. Request rescheduling so + * that ops.dispatch() can reconsider the task after ttwu_runnable() clears + * is_blocked. A full wakeup activation has already enqueued the task and + * doesn't need the additional reschedule. + */ + if (p->sched_class == &ext_sched_class) { + if (sched_proxy_exec() && (wake_flags & WF_TTWU_RQ) && + p->is_blocked) { + struct scx_sched *sch = scx_task_sched(p); + + if (sch && (sch->ops.flags & SCX_OPS_ENQ_BLOCKED)) + resched_curr(rq); + } return; + } /* * Getting preempted by a higher-priority class. Reenqueue IMMED tasks. @@ -2646,6 +2674,21 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, WARN_ON_ONCE(task_cpu(p) == cpu); + /* + * proxy_set_task_cpu() preserves wake_cpu when moving a donor's scheduling + * context to its lock owner's rq. For example, if BPF places donor D on + * CPU0 while its lock owner runs on CPU1, proxy execution moves D to CPU1 + * but leaves D->wake_cpu pointing to CPU0. If D is later put on a shared + * DSQ, CPU0 could otherwise consume it and move it back, only for proxy + * execution to return it to CPU1 again. + * + * When task_cpu() == wake_cpu, the donor has not been proxy-migrated and + * BPF may still choose its placement. An actively donating task is + * rejected separately by task_proxy_running_or_donating(). + */ + if (sched_proxy_exec() && p->is_blocked && task_cpu(p) != p->wake_cpu) + return false; + /* * If @p has migration disabled, @p->cpus_ptr is updated to contain only * the pinned CPU in migrate_disable_switch() while @p is being switched @@ -3469,6 +3512,24 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, if (p->scx.flags & SCX_TASK_QUEUED) { set_task_runnable(rq, p); + /* Delegate retained donor admission to its owning BPF scheduler. */ + if (p->is_blocked) { + /* + * If the donor is the same and only the mutex owner + * changes, avoid triggering another ops.enqueue(): the + * BPF scheduler has already admitted the donor, so it + * can continue running. + */ + if (next == p) + goto switch_class; + + if (WARN_ON_ONCE(!sch)) + goto switch_class; + WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_BLOCKED)); + scx_do_enqueue_task(rq, p, 0, -1); + goto switch_class; + } + /* * If @p has slice left and is being put, @p is getting * preempted by a higher priority scheduler class or core-sched @@ -7904,6 +7965,11 @@ int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) return -EINVAL; } + if ((ops->flags & SCX_OPS_ENQ_BLOCKED) && !ops->enqueue) { + scx_error(sch, "SCX_OPS_ENQ_BLOCKED requires ops.enqueue() to be implemented"); + return -EINVAL; + } + /* * SCX_OPS_TID_TO_TASK is enabled by the root scheduler. A sub-sched * may set it to declare a dependency; reject if the root hasn't diff --git a/kernel/sched/ext/ext.h b/kernel/sched/ext/ext.h index 8348d6008651..d32bc01071b0 100644 --- a/kernel/sched/ext/ext.h +++ b/kernel/sched/ext/ext.h @@ -21,12 +21,20 @@ int scx_check_setscheduler(struct task_struct *p, int policy); void scx_prepare_setscheduler(struct task_struct *p, int policy); bool task_should_scx(int policy); bool scx_allow_ttwu_queue(const struct task_struct *p); -bool scx_allow_proxy_exec(const struct task_struct *p); +bool __scx_allow_proxy_exec(const struct task_struct *p); void scx_proxy_donor_start(struct rq *rq); void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next); void init_sched_ext_class(void); void __scx_update_idle(struct rq *rq, bool idle, bool do_notify); +static inline bool scx_allow_proxy_exec(const struct task_struct *p) +{ + if (scx_enabled()) + return __scx_allow_proxy_exec(p); + + return true; +} + static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) { if (scx_enabled()) diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index deb1a2b8ab9b..d65fec631bdf 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -230,6 +230,19 @@ enum scx_ops_flags { */ SCX_OPS_LAZY_RESCHED = 1LLU << 9, + /* + * If set, mutex-blocked tasks remain runnable as proxy donors and are + * passed to ops.enqueue() with %SCX_ENQ_BLOCKED. The BPF scheduler controls + * when donors are dispatched and whether they should preempt other work. + * + * If clear, mutex-blocked tasks are removed from the runqueue normally + * and cannot donate their scheduling context through proxy execution. + * + * For blocked donors, this flag takes precedence over + * %SCX_OPS_ENQ_EXITING and %SCX_OPS_ENQ_MIGRATION_DISABLED. + */ + SCX_OPS_ENQ_BLOCKED = 1LLU << 10, + SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | SCX_OPS_ENQ_LAST | SCX_OPS_ENQ_EXITING | @@ -239,7 +252,8 @@ enum scx_ops_flags { SCX_OPS_BUILTIN_IDLE_PER_NODE | SCX_OPS_ALWAYS_ENQ_IMMED | SCX_OPS_TID_TO_TASK | - SCX_OPS_LAZY_RESCHED, + SCX_OPS_LAZY_RESCHED | + SCX_OPS_ENQ_BLOCKED, /* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */ __SCX_OPS_INTERNAL_MASK = 0xffLLU << 56, @@ -1812,6 +1826,12 @@ enum scx_enq_flags { */ SCX_ENQ_LAST = 1LLU << 41, + /* + * The task is blocked on a mutex and is being kept runnable as a proxy + * donor. Only passed to ops.enqueue() when %SCX_OPS_ENQ_BLOCKED is set. + */ + SCX_ENQ_BLOCKED = 1LLU << 42, + /* high 8 bits are internal */ __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, @@ -2152,6 +2172,7 @@ struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter); bool scx_set_task_slice(struct task_struct *p, u64 slice); void scx_task_slice_ended(struct rq *rq, struct task_struct *p); void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq); +void scx_prepare_task_sched_change(struct task_struct *p); void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p); void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, int sticky_cpu); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 4541ef8eec95..9af09807951d 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1207,7 +1207,9 @@ static void scx_rehome_task(struct scx_sched *to, struct task_struct *p) lockdep_assert_held(&p->pi_lock); lockdep_assert_rq_held(task_rq(p)); - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_prepare_task_sched_change(p); + scoped_guard (sched_change, p, + DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { scx_disable_and_exit_task(scx_task_sched(p), p); scx_set_task_state(p, SCX_TASK_INIT_BEGIN); scx_set_task_state(p, SCX_TASK_INIT); @@ -1237,7 +1239,9 @@ static void scx_punt_task(struct scx_sched *to, struct task_struct *p) lockdep_assert_rq_held(task_rq(p)); WARN_ON_ONCE(!READ_ONCE(to->bypass_depth)); - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_prepare_task_sched_change(p); + scoped_guard (sched_change, p, + DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { scx_disable_and_exit_task(scx_task_sched(p), p); scx_set_task_sched(p, to); } @@ -1895,7 +1899,9 @@ void scx_sub_enable_workfn(struct kthread_work *work) if (!(p->scx.flags & SCX_TASK_SUB_INIT)) continue; - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_prepare_task_sched_change(p); + scoped_guard (sched_change, p, + DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { /* * $p must be either READY or ENABLED. If ENABLED, * __scx_disabled_and_exit_task() first disables and -- cgit v1.2.3 From f2f095c87603b8dfb6752e130d902970371c3852 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 29 Sep 2026 07:32:26 -1000 Subject: sched_ext: Work around pahole 1.32 dropping scx_bpf_task_set_lazy_resched() from BTF With clang and pahole 1.32, x86-64 vmlinux BTF lacks scx_bpf_task_set_lazy_resched() and sched_ext fails to initialize: sched_ext: Failed to register kfunc sets (-22) clang describes the lazy argument only after the prologue moves it into a callee-saved register, which pahole 1.32 rejects as not being the argument register. Released pahole is affected whatever the eventual fix. Keep the argument on the stack with barrier_data() so that its location names no register. OPTIMIZER_HIDE_VAR() would fix x86-64 but break arm64. Fixes: f8e5a4e3f3be ("sched_ext: Add lazy preemption support") Link: https://lore.kernel.org/r/20260925213619.2187751-1-yonghong.song@linux.dev Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index e48aac45afea..405d0d1038f8 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -10136,6 +10136,11 @@ __bpf_kfunc bool scx_bpf_task_set_lazy_resched(struct task_struct *p, bool lazy, { struct scx_sched *sch; + /* + * clang's register locations for @lazy make pahole 1.32 drop this kfunc + * from BTF. Keep @lazy in memory so its location names no register. + */ + barrier_data(&lazy); guard(rcu)(); sch = scx_prog_sched(aux); if (unlikely(!sch || !scx_task_on_sched(sch, p))) -- cgit v1.2.3 From e28807c846081ab04ba5eb77e588878ee3811961 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Wed, 30 Sep 2026 11:00:18 +0200 Subject: sched_ext: Introduce scx_bpf_cgroup_nr_cpus() Hierarchical BPF schedulers may need to know how many CPUs a cgroup can run on when distributing group-wide weights. For example, fair's group share calculation scales a group's weight by the smaller of its estimated runnable task count and the number of CPUs in its effective cpuset. Without that count, a BPF scheduler can only use the machine-wide CPU count and may give a group confined to a few CPUs excessive shares. A task's effective affinity can include per-task restrictions, so it cannot reliably represent the group's cpuset. Reading the cpuset directly from BPF would require CO-RE access to private cgroup and cpuset structures, including inherited cpuset resolution, making the approach fragile across kernel versions. Add scx_bpf_cgroup_nr_cpus(), which returns the same count fair uses through cpuset_num_cpus(). The count is independent of CPU or CID numbering, so the kfunc is available to both cpu-form and cid-form schedulers from any context. Also add a compat wrapper which falls back to nr_cpu_ids on older kernels. Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 0ef48c2ba6c8..96c904b39601 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -11377,6 +11377,23 @@ out: cgroup_get(cgrp); return cgrp; } + +/** + * scx_bpf_cgroup_nr_cpus - Return the number of CPUs in a cgroup's cpuset + * @cgrp: cgroup of interest + * + * Return the number of CPUs in @cgrp's effective cpuset, which is inherited + * from the nearest ancestor with the cpuset controller enabled. This matches + * the count used by fair's group share calculation and lets hierarchical BPF + * schedulers bound a group's weight by the CPUs it can actually run on. + * + * The value is a snapshot and may change at any time through cpuset updates or + * CPU hotplug. Schedulers should re-read it when recomputing group state. + */ +__bpf_kfunc u32 scx_bpf_cgroup_nr_cpus(struct cgroup *cgrp) +{ + return cpuset_num_cpus(cgrp); +} #endif /* CONFIG_CGROUP_SCHED */ __bpf_kfunc_end_defs(); @@ -11424,6 +11441,7 @@ BTF_ID_FLAGS(func, scx_bpf_now) BTF_ID_FLAGS(func, scx_bpf_events, KF_IMPLICIT_ARGS) #ifdef CONFIG_CGROUP_SCHED BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS | KF_RCU | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_cgroup_nr_cpus, KF_RCU) #endif BTF_ID_FLAGS(func, scx_bpf_sub_grant, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_sub_revoke, KF_IMPLICIT_ARGS) -- cgit v1.2.3 From a546831dff38b624240a90a6ff6d2b43d9226d51 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 6 Oct 2026 10:29:05 -1000 Subject: sched_ext: Clear a sub-scheduler's caps before ops.sub_detach() A sub-scheduler can leave standing state behind on the cids delegated to it, cpuperf targets being the obvious case, and the kernel does not track who wrote it. Restoring it is the parent's job and ops.sub_detach() is the place for it. Today the parent can't rely on that: the dying sub-scheduler keeps every cap until it is marked dead after ops.exit(), so its ops.exit() or a late timer can write after the parent has cleaned up. Clear every cap of the sub-scheduler before ops.sub_detach(). The pshard caps go first as a pending sync recomputes ecaps from them, and ecaps are then zeroed directly. A grant racing the clear is refused under the child's pshard lock, so nothing hands the caps back. Fixes: 86094b95efcf ("sched_ext: Add per-shard cap delegation for sub-schedulers") Signed-off-by: Tejun Heo --- kernel/sched/ext/internal.h | 7 +++++++ kernel/sched/ext/sub.c | 49 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 52 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index d65fec631bdf..b5dc3c159d45 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -864,6 +864,10 @@ struct sched_ext_ops { /** * @sub_detach: Detach a sub-scheduler * @args: argument container, see the struct definition + * + * The sub-scheduler holds no caps by this point and can no longer + * affect any cid. Whatever was delegated to it, e.g. cpuperf targets, + * is the parent's to restore here. */ void (*sub_detach)(struct scx_sub_detach_args *args); @@ -947,6 +951,9 @@ struct sched_ext_ops { * ops.exit() is also called on ops.init() failure, which is a bit * unusual. This is to allow rich reporting through @info on how * ops.init() failed. + * + * A sub-scheduler holds no caps by the time ops.exit() runs. Its + * cap-gated kfuncs are denied. */ void (*exit)(struct scx_exit_info *info); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 48e17aeb1cb6..0d2fb7387cd3 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1104,6 +1104,34 @@ void scx_offline_ecaps(struct rq *rq) } } +/* + * Clear every cap @sch holds. The pshard caps go first as they are the source a + * pending sync recomputes ecaps from. ecaps are then zeroed directly for the + * cap checks. + */ +static void clear_all_caps(struct scx_sched *sch) +{ + s32 si, cpu; + u32 cap_bit; + + /* enable may have failed before scx_alloc_pshards() */ + if (!sch->pshard) + return; + + for (si = 0; si < sch->nr_pshards; si++) { + struct scx_pshard *ps = sch->pshard[si]; + + guard(raw_spinlock_irqsave)(&ps->lock); + for (cap_bit = 0; cap_bit < __SCX_NR_CAPS; cap_bit++) + scx_cmask_clear(&ps->caps[cap_bit].cmask); + } + + for_each_possible_cpu(cpu) { + guard(rq_lock_irqsave)(cpu_rq(cpu)); + WRITE_ONCE(per_cpu_ptr(sch->pcpu, cpu)->ecaps, 0); + } +} + /* * @pcpu's sched was unhashed before the grace period, so nothing re-queues its * sync node. Remove the node from @rq's pending list so the pcpu can be freed. @@ -1583,6 +1611,12 @@ dump: scx_unlink_sched(sch); + /* + * The parent restores what it delegated in ops.sub_detach(). @sch must + * hold no caps by then. + */ + clear_all_caps(sch); + mutex_unlock(&scx_enable_mutex); /* @@ -2266,8 +2300,8 @@ static s32 sub_cap_preamble(u64 cgroup_id, u64 caps, const struct bpf_prog_aux * * All-or-nothing keeps the caller-visible result binary per cid, so the denied * mask is one mask to interpret rather than a per-cap matrix. * - * Return 0 on full success, -EPERM if any cid was refused, or a negative - * errno on other failures. + * Return 0 on full success, -EPERM if any cid was refused, -ENODEV if the child + * is gone or being disabled, or a negative errno on other failures. */ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask *cmask__arena, @@ -2321,6 +2355,12 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, scoped_guard (raw_spinlock, &pps->lock) { guard(raw_spinlock_nested)(&cps->lock); + /* the child is being disabled, see clear_all_caps() */ + if (unlikely(READ_ONCE(child->aborting))) { + ret = -ENODEV; + goto out; + } + /* * Narrow granted_cids to cids the parent holds every * requested cap on. All-or-nothing per cid. @@ -2373,9 +2413,10 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, } } + ret = any_denied ? -EPERM : 0; +out: caps_updated_deliver(&to_deliver); - - return any_denied ? -EPERM : 0; + return ret; } /** -- cgit v1.2.3 From 457fff158adcd2a60f2a908957e193cb78649dbf Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 5 Oct 2026 07:52:12 -1000 Subject: sched_ext: Add ops.sub_cid_sched_updated() to report the sched running on a cid A parent sched that delegates cpus to sub-scheds has no way of telling how much each delegatee is using them. A parent that sizes its delegations by demand cannot measure it, and one that wants to hand a cid from one sub to another cannot see which sub it would take the cid from. Add ops.sub_cid_sched_updated(), invoked on a sched when what it sees running on a cid changes: NONE when no task of its subtree runs there, SELF for its own task, or a direct child's cgroup id. Changes inside a child's subtree are not reported. The op costs one pointer compare per context switch when the sched does not change, a notification reaches only the scheds that see a change, and the whole mechanism sits behind a static key that is on only while a loaded sched implements the op. After a sched's tasks are gone, its disable closes every rq session still pointing at it, so nothing refers to it once it is freed. The op is cid-form only like the other sub ops and runs with the rq lock held on the context switch path. Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 20 +++++- kernel/sched/ext/internal.h | 29 ++++++++ kernel/sched/ext/sub.c | 166 ++++++++++++++++++++++++++++++++++++++++++++ kernel/sched/ext/sub.h | 6 ++ kernel/sched/sched.h | 2 + 5 files changed, 222 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 96c904b39601..248b39d09ce3 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3333,6 +3333,8 @@ static void scx_start_task_running(struct rq *rq, struct task_struct *p) if (p->scx.flags & SCX_TASK_RUN_TRACKED) return; + scx_cid_sched_update(rq, sch); + if (SCX_HAS_OP(sch, running)) SCX_CALL_OP_TASK(sch, running, rq, p); @@ -3593,8 +3595,10 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, } switch_class: - if (next && next->sched_class != &ext_sched_class) + if (next && next->sched_class != &ext_sched_class) { + scx_cid_sched_update(rq, NULL); switch_class(rq, next); + } } static void kick_sync_wait_bal_cb(struct rq *rq) @@ -4724,6 +4728,13 @@ static void switching_to_scx(struct rq *rq, struct task_struct *p) static void switched_from_scx(struct rq *rq, struct task_struct *p) { + /* + * A class change of the running task: sched_change_begin() put @p with + * no successor, which leaves rq->scx.sched set. + */ + if (task_current_donor(rq, p)) + scx_cid_sched_update(rq, NULL); + if (task_dead_and_done(p)) return; @@ -7047,6 +7058,8 @@ static void scx_root_disable(struct scx_sched *sch) } } + scx_ops_cid_sched_updated_disable(sch); + /* no task is on scx, turn off all the switches and flush in-progress calls */ static_branch_disable(&__scx_enabled); static_branch_disable(&__scx_is_cid_type); @@ -8255,6 +8268,8 @@ static void scx_root_enable_workfn(struct kthread_work *work) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op); + scx_ops_cid_sched_updated_enable(sch); + if (sch->ops.cpu_acquire || sch->ops.cpu_release) sch->ops.flags |= SCX_OPS_HAS_CPU_PREEMPT; @@ -8905,6 +8920,7 @@ static void sched_ext_ops_cid__set_cmask(struct task_struct *p, const struct scx static void sched_ext_ops_cid__enable(struct task_struct *p, struct scx_enable_args *args) {} static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask__arena, u64 caps) {} static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {} +static void sched_ext_ops__sub_cid_sched_updated(s32 cid, u64 sched) {} static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .select_cid = sched_ext_ops__select_cpu, @@ -8939,6 +8955,7 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .sub_detach = sched_ext_ops__sub_detach, .sub_caps_updated = sched_ext_ops__sub_caps_updated, .sub_ecaps_updated = sched_ext_ops__sub_ecaps_updated, + .sub_cid_sched_updated = sched_ext_ops__sub_cid_sched_updated, .cid_online = sched_ext_ops__cpu_online, .cid_offline = sched_ext_ops__cpu_offline, .init_cids = sched_ext_ops__init_cids, @@ -11678,6 +11695,7 @@ static int __init scx_init(void) CID_OFFSET_MATCH(sub_detach, sub_detach); CID_OFFSET_MATCH(sub_caps_updated, sub_caps_updated); CID_OFFSET_MATCH(sub_ecaps_updated, sub_ecaps_updated); + CID_OFFSET_MATCH(sub_cid_sched_updated, sub_cid_sched_updated); CID_OFFSET_MATCH(init_cids, init_cids); CID_OFFSET_MATCH(init, init); CID_OFFSET_MATCH(exit, exit); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index b5dc3c159d45..8dcab02a38ab 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -366,6 +366,18 @@ struct scx_sub_detach_args { char *cgroup_path; }; +/* + * What a sched sees running on a cid, as reported by + * ops.sub_cid_sched_updated(): NONE when the cid is idle or runs a task outside + * the sched's subtree, SELF for the sched itself, or a direct child by its + * cgroup id. The sentinels lie outside the cgroup id space, whose low 32 bits + * are an idr value of at most INT_MAX. + */ +enum scx_cid_sched_consts { + SCX_CID_SCHED_NONE = (u64)-2, + SCX_CID_SCHED_SELF = (u64)-1, +}; + /** * struct sched_ext_ops - Operation table for BPF scheduler implementation * @@ -904,6 +916,22 @@ struct sched_ext_ops { */ void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); + /** + * @sub_cid_sched_updated: The sched running on a cid changed + * @cid: cid whose running sched changed + * @sched: what this sched now sees running there + * + * Invoked when the sched owning the task running on @cid changes. + * @sched is %SCX_CID_SCHED_NONE when no task of this sched's subtree + * runs there, %SCX_CID_SCHED_SELF for its own task, or the cgroup id of + * the direct child whose subtree the task belongs to. Changes inside a + * child's subtree, whether across tasks or nested sub-scheds, are not + * reported. Every sched for which @sched changes is notified. + * + * Runs with the rq lock held on the context switch path. + */ + void (*sub_cid_sched_updated)(s32 cid, u64 sched); + /* * All online ops must come before ops.cpu_online(). */ @@ -1161,6 +1189,7 @@ struct sched_ext_ops_cid { void (*sub_detach)(struct scx_sub_detach_args *args); void (*sub_caps_updated)(const struct scx_cmask *cmask__arena, u64 caps); void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); + void (*sub_cid_sched_updated)(s32 cid, u64 sched); void (*cid_online)(s32 cid); void (*cid_offline)(s32 cid); s32 (*init_cids)(void); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 0d2fb7387cd3..51a53467cf98 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -28,6 +28,13 @@ */ DEFINE_STATIC_KEY_FALSE(__scx_has_subs); +/* + * On while a loaded sched implements ops.sub_cid_sched_updated(), see + * scx_ops_cid_sched_updated_enable(). + */ +static DEFINE_STATIC_KEY_FALSE(__scx_ops_cid_sched_updated_enabled); +static s32 scx_nr_ops_cid_sched_updated; /* such scheds, under scx_enable_mutex */ + /* latched at root enable before any rescue runs */ static s32 scx_rescue_bw_1024; static s64 scx_rescue_quantum_ns; @@ -92,6 +99,161 @@ struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sche return scx_skip_subtree_pre(pos, root); } +/** + * scx_cid_sched_id_from - Resolve what @from sees when @sch runs on a cid + * @sch: sched whose task runs there, %NULL for none + * @from: sched asking + * + * Return %SCX_CID_SCHED_NONE when @sch is %NULL or outside @from's subtree, + * %SCX_CID_SCHED_SELF for @from itself, or else the cgroup id of the direct + * child of @from on the way down to @sch. + */ +static u64 scx_cid_sched_id_from(struct scx_sched *sch, struct scx_sched *from) +{ + if (!sch || !scx_is_descendant(sch, from)) + return SCX_CID_SCHED_NONE; + if (sch == from) + return SCX_CID_SCHED_SELF; + return sch->ancestors[from->level + 1]->ops.sub_cgroup_id; +} + +/* notify @from of what it now sees running on @rq's cid */ +static void scx_cid_notify_sched_updated(struct rq *rq, struct scx_sched *sch, + struct scx_sched *from) +{ + if (SCX_HAS_OP(from, sub_cid_sched_updated)) + SCX_CALL_OP(from, sub_cid_sched_updated, rq, __scx_cpu_to_cid(cpu_of(rq)), + scx_cid_sched_id_from(sch, from)); +} + +/** + * scx_cid_sched_update - Update rq->scx.sched and notify the scheds affected + * @rq: rq whose running session opens or closes + * @sch: sched starting a session, %NULL to close the open one + * + * Every sched whose scx_cid_sched_id_from() value changes is notified of the + * new one. Above the deepest common ancestor of the two scheds, both map to the + * same child, so the walk stops there. With root R, children A and B, and A's + * child A1, a switch from an A1 task to a B task reports: + * + * R R: A -> B + * / \ + * A B A: A1 -> NONE B: NONE -> SELF + * | + * A1 A1: SELF -> NONE + * + * When the task on one side is not an ext task, there is no common ancestor and + * only the other side is walked. + */ +void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch) +{ + struct scx_sched *prev = rq->scx.sched; + s32 level, common = -1; + + lockdep_assert_rq_held(rq); + + if (!static_branch_unlikely(&__scx_ops_cid_sched_updated_enabled) || prev == sch) + return; + + rq->scx.sched = sch; + + if (prev && sch) { + for (level = min(prev->level, sch->level); level >= 0; level--) + if (prev->ancestors[level] == sch->ancestors[level]) + break; + common = level; + scx_cid_notify_sched_updated(rq, sch, prev->ancestors[common]); + } + if (prev) + for (level = common + 1; level <= prev->level; level++) + scx_cid_notify_sched_updated(rq, sch, prev->ancestors[level]); + if (sch) + for (level = common + 1; level <= sch->level; level++) + scx_cid_notify_sched_updated(rq, sch, sch->ancestors[level]); +} + +/** + * scx_ops_cid_sched_updated_enable - Maintain rq->scx.sched if @sch has the op + * @sch: sched being enabled, its has_op filled in + * + * rq->scx.sched is maintained only while a loaded sched implements + * ops.sub_cid_sched_updated(), so that everyone else pays one static branch per + * session open or close. The first such sched turns the key on and then + * initializes every rq from the task running there, in that order so that no + * session change in between goes unrecorded. While the key is off every + * rq->scx.sched is NULL, see scx_ops_cid_sched_updated_disable(). + */ +void scx_ops_cid_sched_updated_enable(struct scx_sched *sch) +{ + s32 cpu; + + lockdep_assert_held(&scx_enable_mutex); + + if (!SCX_HAS_OP(sch, sub_cid_sched_updated) || scx_nr_ops_cid_sched_updated++) + return; + + static_branch_enable(&__scx_ops_cid_sched_updated_enabled); + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + struct task_struct *p; + + guard(rq_lock_irqsave)(rq); + p = rq->donor; + if (p->sched_class == &ext_sched_class && + (p->scx.flags & SCX_TASK_RUN_TRACKED)) + rq->scx.sched = scx_task_sched(p); + } +} + +/** + * scx_ops_cid_sched_updated_disable - Stop maintaining rq->scx.sched for @sch + * @sch: sched being disabled, its tasks all gone + * + * Close the session on every rq that still points at @sch, before ops.exit(), + * so that no sched is notified of @sch once it is gone. The last sched with the + * op turns the key off first and clears every rq, so while the key is off every + * rq->scx.sched is NULL. A sched whose enable failed before its has_op fill was + * never counted, so has_op gates the decrement. + */ +void scx_ops_cid_sched_updated_disable(struct scx_sched *sch) +{ + bool last; + s32 cpu; + + lockdep_assert_held(&scx_enable_mutex); + + last = SCX_HAS_OP(sch, sub_cid_sched_updated) && !--scx_nr_ops_cid_sched_updated; + if (last) + static_branch_disable(&__scx_ops_cid_sched_updated_enabled); + + /* + * Re-homing restarts a running task's session under its new sched. A + * task that blocked while its cpu's dispatch had the rq lock dropped is + * put and set next again without a restart, so rq->scx.sched keeps @sch + * until the pick completes: + * + * cpu 0, __schedule() cpu 1, scx_sub_disable(@sch) + * T of @sch blocks, stops running + * dispatch drops the rq lock + * re-home T, nothing to restart + * this sweep + * pick N, N's session opens + * + * Closing the session here reports NONE to @sch and its ancestors now + * and lets the pick report N's sched from NONE instead of from @sch. + */ + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + guard(rq_lock_irqsave)(rq); + if (rq->scx.sched == sch) + scx_cid_sched_update(rq, NULL); + if (last) + rq->scx.sched = NULL; + } +} + static struct scx_sched *scx_find_sub_sched(u64 cgroup_id) { return rhashtable_lookup(&scx_sched_hash, &cgroup_id, @@ -1599,6 +1761,8 @@ dump: scx_cgroup_unlock(); percpu_up_write(&scx_fork_rwsem); + scx_ops_cid_sched_updated_disable(sch); + /* * All tasks are moved off of @sch but there may still be on-going * operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use @@ -1827,6 +1991,8 @@ void scx_sub_enable_workfn(struct kthread_work *work) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op); + scx_ops_cid_sched_updated_enable(sch); + percpu_down_write(&scx_fork_rwsem); scx_cgroup_lock(); diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index c0174e523f2a..3cd31b24e649 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -16,6 +16,9 @@ struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root); struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root); +void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch); +void scx_ops_cid_sched_updated_enable(struct scx_sched *sch); +void scx_ops_cid_sched_updated_disable(struct scx_sched *sch); void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch); struct cgroup *sch_cgroup(struct scx_sched *sch); void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch); @@ -94,6 +97,9 @@ static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } static inline struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root) { return NULL; } +static inline void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch) {} +static inline void scx_ops_cid_sched_updated_enable(struct scx_sched *sch) {} +static inline void scx_ops_cid_sched_updated_disable(struct scx_sched *sch) {} static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } static inline const char *sch_cgrp_path(struct scx_sched *sch) { return "/"; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 0a34f0da1ec3..fc65296f5c74 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -833,6 +833,8 @@ struct scx_rq { #endif u64 clock; /* current per-rq clock -- see scx_bpf_now() */ #ifdef CONFIG_EXT_SUB_SCHED + /* sched of the open running session, see scx_cid_sched_update() */ + struct scx_sched *sched; struct llist_head ecaps_to_sync; /* pending ecaps syncs */ struct task_struct *sub_dispatch_prev; #endif -- cgit v1.2.3 From da3bad759ef2df9db38ad851a061d719091d74dd Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 7 Oct 2026 23:15:39 -1000 Subject: sched_ext: Add ops.sub_child_ecaps_updated() to report a child's effective cap changes A grant or revoke only records the target caps. They take effect on a cid at its next dispatch, and nothing tells the parent when. Without that the parent cannot act on a revoke: a child that ran a cid at a low cpuperf target leaves the target there when PERF is revoked, as the kernel resets targets only at root enable. Nor can the parent tell when it may schedule on the cid again. Add ops.sub_child_ecaps_updated(), delivered to the direct parent right after the child's own ops.sub_ecaps_updated() with the child's cgroup id and the same before and after caps, in the same dispatch context. Both deliveries are suppressed while the child is bypassing and replayed together afterwards. A disabled child reports nothing: ops.sub_detach() is where the parent restores what it had delegated. A sub is now bypassed before it is linked, so that the grants queued for it from ops.sub_attach() are consumed while it is bypassed and replayed to it and the parent once it is enabled. Otherwise a sync consumed before the sub's ops are registered would reach the parent but never the sub. v2: Drop the disable report with its sleep lock and nested-dispatch gate, the parent restores from ops.sub_detach() instead. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 4 +++ kernel/sched/ext/internal.h | 30 ++++++++++++++++++++--- kernel/sched/ext/sub.c | 59 +++++++++++++++++++++++++++++++-------------- 3 files changed, 72 insertions(+), 21 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 248b39d09ce3..2095b78b6bf6 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -8921,6 +8921,7 @@ static void sched_ext_ops_cid__enable(struct task_struct *p, struct scx_enable_a static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask__arena, u64 caps) {} static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {} static void sched_ext_ops__sub_cid_sched_updated(s32 cid, u64 sched) {} +static void sched_ext_ops__sub_child_ecaps_updated(u64 cgroup_id, s32 cid, u64 before, u64 after) {} static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .select_cid = sched_ext_ops__select_cpu, @@ -8956,6 +8957,7 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .sub_caps_updated = sched_ext_ops__sub_caps_updated, .sub_ecaps_updated = sched_ext_ops__sub_ecaps_updated, .sub_cid_sched_updated = sched_ext_ops__sub_cid_sched_updated, + .sub_child_ecaps_updated = sched_ext_ops__sub_child_ecaps_updated, .cid_online = sched_ext_ops__cpu_online, .cid_offline = sched_ext_ops__cpu_offline, .init_cids = sched_ext_ops__init_cids, @@ -11552,6 +11554,7 @@ static const u32 scx_kf_allow_flags[] = { [SCX_OP_IDX(sub_attach)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(sub_detach)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(sub_ecaps_updated)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH, + [SCX_OP_IDX(sub_child_ecaps_updated)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH, [SCX_OP_IDX(cpu_online)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(cpu_offline)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(init_cids)] = SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT_CIDS, @@ -11696,6 +11699,7 @@ static int __init scx_init(void) CID_OFFSET_MATCH(sub_caps_updated, sub_caps_updated); CID_OFFSET_MATCH(sub_ecaps_updated, sub_ecaps_updated); CID_OFFSET_MATCH(sub_cid_sched_updated, sub_cid_sched_updated); + CID_OFFSET_MATCH(sub_child_ecaps_updated, sub_child_ecaps_updated); CID_OFFSET_MATCH(init_cids, init_cids); CID_OFFSET_MATCH(init, init); CID_OFFSET_MATCH(exit, exit); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 8dcab02a38ab..2b637973244c 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -878,8 +878,9 @@ struct sched_ext_ops { * @args: argument container, see the struct definition * * The sub-scheduler holds no caps by this point and can no longer - * affect any cid. Whatever was delegated to it, e.g. cpuperf targets, - * is the parent's to restore here. + * affect any cid. Its caps went away without a + * sub_child_ecaps_updated() report, so whatever was delegated to it, + * e.g. cpuperf targets, is the parent's to restore here. */ void (*sub_detach)(struct scx_sub_detach_args *args); @@ -932,6 +933,25 @@ struct sched_ext_ops { */ void (*sub_cid_sched_updated)(s32 cid, u64 sched); + /** + * @sub_child_ecaps_updated: A child's effective caps on a cid changed + * @cgroup_id: cgroup id of the direct child + * @cid: the cid whose effective caps changed + * @before: the child's effective caps as of the last delivery + * @after: the child's effective caps now + * + * Invoked right after the child's sub_ecaps_updated(), once a grant or + * revoke is in effect on the cpu. From here on the parent can act on a + * cap the child lost, e.g. reset the cpuperf target or schedule on the + * cid. A detaching child's caps go away without a report, see + * sub_detach(). A cpu going offline drops caps without a report. + * + * Runs in dispatch context with rq lock held, and can perform all + * operations allowed in ops.dispatch() including inserting/moving + * tasks. + */ + void (*sub_child_ecaps_updated)(u64 cgroup_id, s32 cid, u64 before, u64 after); + /* * All online ops must come before ops.cpu_online(). */ @@ -1190,6 +1210,7 @@ struct sched_ext_ops_cid { void (*sub_caps_updated)(const struct scx_cmask *cmask__arena, u64 caps); void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); void (*sub_cid_sched_updated)(s32 cid, u64 sched); + void (*sub_child_ecaps_updated)(u64 cgroup_id, s32 cid, u64 before, u64 after); void (*cid_online)(s32 cid); void (*cid_offline)(s32 cid); s32 (*init_cids)(void); @@ -1453,7 +1474,10 @@ struct scx_sched_pcpu { struct llist_node ecaps_to_sync_node; /* owed a forced update_idle() re-notify on this cpu */ bool idle_renotify; - /* effective caps as of the last sub_ecaps_updated() delivery */ + /* + * ecaps as of the last sub_ecaps_updated() and + * sub_child_ecaps_updated() delivery. See __scx_process_sync_ecaps(). + */ u64 reported_ecaps; /* diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 51a53467cf98..c7c95661280b 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1121,27 +1121,43 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) lost_all |= lost; /* - * Tell the sched its effective caps on this cid changed. The - * invocation is equivalent to the dispatch path and may drop - * and re-acquire the rq lock temporarily while the rest of - * @batch is held privately, see scx_discard_ecaps_to_sync(). - * The dispatch kfuncs resolve their context on the executing - * cpu, which under core scheduling can differ from @rq's cpu, - * so the context is set up there. The rq recorded in it keeps - * the dispatches targeting @rq. + * Tell the sched and its parent that the sched's effective caps + * on this cid changed. The invocations are equivalent to the + * dispatch path and may drop and re-acquire the rq lock + * temporarily while the rest of @batch is held privately, see + * scx_discard_ecaps_to_sync(). The dispatch kfuncs resolve + * their context on the executing cpu, which under core + * scheduling can differ from @rq's cpu, so the context is set + * up there. The rq recorded in it keeps the dispatches + * targeting @rq. + * + * Bypass propagates down the hierarchy, so a sched that isn't + * bypassing has no bypassing parent. The child's bypass state + * gates both deliveries. Both report the same before value, so + * one reported_ecaps covers them. */ - if (ecaps != pcpu->reported_ecaps && - SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) && - !scx_bypassing(pcpu->sch, cpu)) { - struct scx_dsp_ctx *dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx; + if (ecaps != pcpu->reported_ecaps && !scx_bypassing(pcpu->sch, cpu)) { + struct scx_sched *parent = scx_parent(pcpu->sch); + struct scx_dsp_ctx *dspc; - dspc->rq = rq; /* stash @prev so nested dispatches can access it */ rq->scx.sub_dispatch_prev = prev; - SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, scx_cpu_arg(cpu), - pcpu->reported_ecaps, ecaps); + if (SCX_HAS_OP(pcpu->sch, sub_ecaps_updated)) { + dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx; + dspc->rq = rq; + SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, + scx_cpu_arg(cpu), pcpu->reported_ecaps, ecaps); + scx_flush_dispatch_buf(pcpu->sch, rq); + } + if (SCX_HAS_OP(parent, sub_child_ecaps_updated)) { + dspc = &this_cpu_ptr(parent->pcpu)->dsp_ctx; + dspc->rq = rq; + SCX_CALL_OP(parent, sub_child_ecaps_updated, rq, + pcpu->sch->ops.sub_cgroup_id, scx_cpu_arg(cpu), + pcpu->reported_ecaps, ecaps); + scx_flush_dispatch_buf(parent, rq); + } rq->scx.sub_dispatch_prev = NULL; - scx_flush_dispatch_buf(pcpu->sch, rq); pcpu->reported_ecaps = ecaps; } @@ -1940,6 +1956,15 @@ void scx_sub_enable_workfn(struct kthread_work *work) if (ret) goto err_disable; + /* + * Bypass before @sch is linked and grants can reach it. The parent's + * delivery advances reported_ecaps, so a sync consumed before @sch's + * ops are registered would reach the parent and never be replayed to + * @sch. While bypassed, the syncs are consumed without a delivery and + * replayed to both at unbypass. + */ + scx_bypass(sch, true); + ret = scx_link_sched(sch); if (ret) goto err_disable; @@ -1985,8 +2010,6 @@ void scx_sub_enable_workfn(struct kthread_work *work) } sch->sub_attached = true; - scx_bypass(sch, true); - for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op); -- cgit v1.2.3 From 67d0d8d018f35beaa58a5e62a1d9c83414e570cd Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Sat, 10 Oct 2026 11:06:21 +0200 Subject: sched_ext: Keep proxy donors with slice left on the local DSQ Commit ee172227d0dc ("sched_ext: Delegate proxy donor admission to BPF schedulers") makes put_prev_task_scx() pass a retained proxy donor to ops.enqueue() with SCX_ENQ_BLOCKED. Some of these puts are only proxy bookkeeping; proxy_resched_idle() drops the rq's donor reference before switching to idle when: - find_proxy_task() cannot run the mutex owner yet and retries through idle (e.g., the owner is on a remote CPU), - proxy_migrate_task() detaches the donor before migration, - proxy_deactivate() blocks a task in the chain whose owner cannot run. In the first case, BPF has already selected a donor with slice left. Returning it to ops.enqueue() forces BPF to dispatch it again before proxy resolution can continue. In the other two cases, the donor is typically deactivated right after ops.enqueue(), undoing any placement BPF makes. As also discussed at the sched_ext microconference at Linux Plumbers 2026, these ops.enqueue/dequeue() invocations do not represent any meaningful scheduling events, so we should avoid triggering them. A more reasonable semantic is to keep a donor with slice left at the head of the local DSQ, so the next pick can resolve its owner, or deactivation can remove it without an unnecessary BPF handoff. An IMMED donor can stay there only for a bookkeeping put during proxy resolution. Mark a blocked pick as awaiting resolution with SCX_RQ_PROXY_PENDING to distinguish that put from a real preemption and re-insert the donor with SCX_ENQ_IMMED, so that it only needs SCX_CAP_ENQ_IMMED rather than SCX_CAP_ENQ. A preempted IMMED donor still returns to BPF. If a higher priority class takes the CPU once proxy resolution completes, schedule a local reenqueue, so that a parked IMMED donor returns to BPF instead of lingering on the local DSQ. Update the SCX_ENQ_IMMED and SCX_ENQ_BLOCKED documentation accordingly. Moreover, a donor that has run out of slice now follows the regular put path, so it can receive SCX_ENQ_LAST when the CPU is about to go idle and the scheduler needs to arrange a follow-up scheduling event. This does not apply to bookkeeping puts, where the switch to idle is only temporary. The blocked-donor specific sanity checks go away together with the special case. When a retained donor is removed ahead of a scheduler ownership or policy change, mark the proxy reset with SCX_RQ_PROXY_BLOCKING and skip reenqueueing the active donor: sched_proxy_block_task() will dequeue it next. Report it as not runnable to ops.stopping(), as a regular dequeue of the current donor does. This avoids a transient ops.enqueue/dequeue() pair and a false ENQ_LAST warning. With this, the sched_ext core absorbs the transient proxy-bookkeeping puts and the BPF scheduler only sees a blocked donor when there is an actual placement decision to make. Fixes: ee172227d0dc ("sched_ext: Delegate proxy donor admission to BPF schedulers") Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 84 +++++++++++++++++++++++++++++++++------------ kernel/sched/ext/internal.h | 10 ++++-- kernel/sched/sched.h | 2 ++ 3 files changed, 72 insertions(+), 24 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index a2e0382ce3dd..dc5d91a051ae 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -24,6 +24,17 @@ DEFINE_RAW_SPINLOCK(scx_sched_lock); +/* + * Block a retained proxy donor, telling put_prev_task_scx() that the donor is + * about to be dequeued and must not be reenqueued. + */ +static void scx_block_proxy_donor(struct rq *rq, struct task_struct *p) +{ + rq->scx.flags |= SCX_RQ_PROXY_BLOCKING; + sched_proxy_block_task(rq, p); + rq->scx.flags &= ~SCX_RQ_PROXY_BLOCKING; +} + bool __scx_allow_proxy_exec(const struct task_struct *p) { struct scx_sched *sch; @@ -47,7 +58,7 @@ void scx_prepare_task_sched_change(struct task_struct *p) lockdep_assert_rq_held(task_rq(p)); update_rq_clock(task_rq(p)); - sched_proxy_block_task(task_rq(p), p); + scx_block_proxy_donor(task_rq(p), p); } /* @@ -1189,6 +1200,14 @@ void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next) scx_proxy_update_tick(rq, next); #endif + /* + * A higher priority class may wake up while @rq is transiently idle + * for proxy resolution. wakeup_preempt_scx() isn't invoked in that + * case, so reenqueue IMMED donors parked on the local DSQ here. + */ + if (rq->scx.nr_immed && sched_class_above(rq->next_class, &ext_sched_class)) + scx_schedule_reenq_local(rq, 0); + if (rq->scx.flags & SCX_RQ_PROXY_RETRY) { rq->scx.flags &= ~SCX_RQ_PROXY_RETRY; schedule_deferred_locked(rq); @@ -3362,6 +3381,16 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t bool first = type == SNT_PICK; bool can_stop_tick; + /* + * A blocked pick is provisional until proxy resolution completes, see + * put_prev_task_scx(). A retry can repick the same donor, so + * SNT_REPICK must set it again. + */ + rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; + if (sched_proxy_exec() && p->is_blocked && + (type == SNT_PICK || type == SNT_REPICK)) + rq->scx.flags |= SCX_RQ_PROXY_PENDING; + if (type == SNT_REPICK) return; @@ -3441,6 +3470,7 @@ void scx_proxy_donor_start(struct rq *rq) struct task_struct *donor = rq->donor; lockdep_assert_rq_held(rq); + rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; if (donor->sched_class == &ext_sched_class && (donor->scx.flags & SCX_TASK_QUEUED)) scx_start_task_running(rq, donor); @@ -3501,8 +3531,13 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, struct task_struct *next) { struct scx_sched *sch = scx_task_sched(p); + bool proxy_put = p->is_blocked && next == rq->idle && + (rq->scx.flags & SCX_RQ_PROXY_PENDING); + bool proxy_block = rq->scx.flags & SCX_RQ_PROXY_BLOCKING; bool rescue_keep = false; + rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; + /* see kick_sync_wait_bal_cb() */ smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); @@ -3526,31 +3561,24 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, if (next != p && (p->scx.flags & SCX_TASK_QUEUED) && (p->scx.flags & SCX_TASK_RUN_TRACKED)) { if (SCX_HAS_OP(sch, stopping)) - SCX_CALL_OP_TASK(sch, stopping, rq, p, true); + SCX_CALL_OP_TASK(sch, stopping, rq, p, !proxy_block); p->scx.flags &= ~SCX_TASK_RUN_TRACKED; } if (p->scx.flags & SCX_TASK_QUEUED) { - set_task_runnable(rq, p); + if (proxy_block) + goto switch_class; - /* Delegate retained donor admission to its owning BPF scheduler. */ - if (p->is_blocked) { - /* - * If the donor is the same and only the mutex owner - * changes, avoid triggering another ops.enqueue(): the - * BPF scheduler has already admitted the donor, so it - * can continue running. - */ - if (next == p) - goto switch_class; + set_task_runnable(rq, p); - if (WARN_ON_ONCE(!sch)) - goto switch_class; - WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_BLOCKED)); - scx_do_enqueue_task(rq, p, 0, -1); + /* + * If the donor is the same and only the mutex owner changes, + * avoid triggering another ops.enqueue(): the BPF scheduler has + * already admitted the donor, so it can continue running. + */ + if (p->is_blocked && next == p) goto switch_class; - } /* * If @p has slice left and is being put, @p is getting @@ -3559,12 +3587,16 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * DSQ unless it was an IMMED task. IMMED tasks should not * linger on a busy CPU, reenqueue them to the BPF scheduler. * + * The exception is an IMMED donor put by proxy_resched_idle(): + * the CPU isn't busy, proxy resolution is only retrying through + * idle, so keep the donor where it can be picked again. + * * An open rescue must keep @p on the local DSQ even if the * scheduler zeroed the slice in ops.stopping() above. */ if ((p->scx.slice || unlikely(p == scx_rescuee(rq))) && !scx_bypassing(sch, cpu_of(rq))) { - if (p->scx.flags & SCX_TASK_IMMED) { + if ((p->scx.flags & SCX_TASK_IMMED) && !proxy_put) { p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); } else { @@ -3581,6 +3613,9 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, enq_flags |= SCX_ENQ_HEAD; } else { enq_flags |= SCX_ENQ_HEAD; + /* only require SCX_CAP_ENQ_IMMED, see scx_caps_for_enq() */ + if (proxy_put && (p->scx.flags & SCX_TASK_IMMED)) + enq_flags |= SCX_ENQ_IMMED; } scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, @@ -3593,13 +3628,15 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * If @p is runnable but we're about to enter a lower * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell * ops.enqueue() that @p is the only one available for this cpu. - * This doesn't apply if the baseline access on the CPU is lost. + * This doesn't apply if the baseline access on the CPU is lost or + * proxy resolution temporarily switches to idle. * * Under core scheduling, a pick dispatches only when nothing is * locally runnable and can legitimately go idle with @p still * runnable (see do_pick_task_scx()). */ - if (next && sched_class_above(&ext_sched_class, next->sched_class) && + if (!proxy_put && next && + sched_class_above(&ext_sched_class, next->sched_class) && scx_task_can_stay_on_cpu(rq, p)) { WARN_ON_ONCE(!sched_core_enabled(rq) && !(sch->ops.flags & SCX_OPS_ENQ_LAST)); @@ -3781,6 +3818,9 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) enum scx_dsp_verdict verdict; struct task_struct *p; + /* A retry can abandon a provisional blocked-donor pick. */ + rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; + /* see kick_sync_wait_bal_cb() */ smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); @@ -4801,7 +4841,7 @@ void scx_prepare_setscheduler(struct task_struct *p, int policy) lockdep_assert_rq_held(task_rq(p)); if (scx_enabled() && p->policy != policy && policy == SCHED_EXT) - sched_proxy_block_task(task_rq(p), p); + scx_block_proxy_donor(task_rq(p), p); } static void process_ddsp_deferred_locals(struct rq *rq) diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index e1afcd4e043f..f6a2ee3ca172 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1842,8 +1842,8 @@ enum scx_enq_flags { /* * Only allowed on local DSQs. Guarantees that the task either gets * on the CPU immediately and stays on it, or gets reenqueued back - * to the BPF scheduler. It will never linger on a local DSQ or be - * silently put back after preemption. + * to the BPF scheduler. A blocked proxy donor may be put back on the + * local DSQ temporarily while proxy execution resolves its mutex owner. * * The protection persists until the next fresh enqueue - it * survives SAVE/RESTORE cycles, slice extensions and preemption. @@ -1890,6 +1890,12 @@ enum scx_enq_flags { /* * The task is blocked on a mutex and is being kept runnable as a proxy * donor. Only passed to ops.enqueue() when %SCX_OPS_ENQ_BLOCKED is set. + * + * Blocking on the mutex does not enqueue the task by itself. A donor put + * with slice left stays at the head of the local DSQ, except when an + * IMMED donor is preempted or cannot stay local. It is passed to + * ops.enqueue() when its slice runs out, the IMMED placement cannot be + * kept, or proxy execution moves it to the owner's CPU. */ SCX_ENQ_BLOCKED = 1LLU << 42, diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 7a5739acba5f..37d4808d7f19 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -793,6 +793,8 @@ enum scx_rq_flags { SCX_RQ_ROOT_IDLE_RENOTIFY = 1 << 8, /* the root is owed update_idle() */ SCX_RQ_PROXY_RETRY = 1 << 9, /* proxy-rejected tasks need retry */ SCX_RQ_PROXY_TICK = 1 << 10, /* proxy execution requires the tick */ + SCX_RQ_PROXY_PENDING = 1 << 11, /* blocked donor awaits proxy resolution */ + SCX_RQ_PROXY_BLOCKING = 1 << 12, /* donor is being removed for sched change */ SCX_RQ_IN_WAKEUP = 1 << 16, SCX_RQ_IN_DISPATCH = 1 << 17, -- cgit v1.2.3 From 43e1ef14a47257ebf7e3bb3ab9a6df9c6f817c5f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 15 Oct 2026 07:18:05 -1000 Subject: sched_ext: Fix use-after-free of a local reenq node left on an inactive cpu A local DSQ reenq is queued on the rq through a per-sched node and run from deferred work. When the reenq is requested from dispatch, the work is a balance callback queued once dispatch is done. A cpu going down installs balance push early in its teardown, and queue_balance_callback() refuses new callbacks from then on. ops.dispatch() keeps running on that cpu until the rq goes offline after an RCU grace period, so a reenq it requests in that window may never run. The IMMED handling in dispatch_one() requests one the same way. A reenq requested from elsewhere for a cpu that is already offline goes through irq_work_queue_on(), which warns, and its work doesn't run until the cpu comes back. Either way the node stays on the rq's list. When the scheduler is freed, scx_sched_free_rcu_work() warns and frees the pcpu with the node still linked, and the list is walked again once the cpu comes back. An inactive cpu's local DSQ holds only tasks that are bound to it or are being moved off, so the reenq is pointless. Ignore reenqs of an inactive cpu's local DSQ. The test is lockless. The hotplug thread of a cpu going down runs on that cpu, and it marks the cpu inactive before it installs balance push. A dispatch on the cpu runs to completion before that thread can run there, so a dispatch that saw the cpu active has queued its callback by then. Requests from outside dispatch run from the wakeup path or through irq_work, which balance push doesn't affect. The grace period after the cpu goes inactive waits for any requester that saw it active, so that irq_work is queued while the cpu is online. Fixes: 238eba8c210d ("sched_ext: Use schedule_deferred_locked() in schedule_dsq_reenq()") Fixes: 76edc2761ab8 ("sched_ext: Use irq_work_queue_on() in schedule_deferred()") Cc: stable@vger.kernel.org # v7.1+ Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi Tested-by: Andrea Righi --- kernel/sched/ext/ext.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 91120fc17d30..2c213153bdab 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1102,6 +1102,18 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, if (dsq->id == SCX_DSQ_LOCAL) { rq = container_of(dsq, struct rq, scx.local_dsq); + /* + * An inactive cpu's local DSQ holds only tasks that are bound + * to it or are being moved off, so a reenq of it is pointless. + * Once balance push is installed, queue_balance_callback() also + * drops the callback that would run a reenq requested from + * dispatch, and irq_work_queue_on() can't reach an offline cpu. + * The reenq's node would then stay on this rq's list after + * @sch's pcpu is freed. + */ + if (unlikely(!cpu_active(cpu_of(rq)))) + return; + /* * A sub-sched lacking baseline access on the target cid has no * business triggering IPIs. The lockless test is fine: slipping @@ -10020,7 +10032,8 @@ __bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id, * - Local DSQs (%SCX_DSQ_LOCAL or %SCX_DSQ_LOCAL_ON | $cpu) * - User DSQs * - * Re-enqueues are performed asynchronously. Can be called from anywhere. + * Re-enqueues are performed asynchronously. Can be called from anywhere. A + * re-enqueue of an inactive cpu's local DSQ is ignored. * * %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler * operation is locked to - e.g. the rq being dispatched for in ops.dispatch() - -- cgit v1.2.3 From 30fb2b1e4d41a641a059f295a717ffa0e24307e7 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 16 Oct 2026 19:37:16 -1000 Subject: sched_ext: Fix the running task keeping its protection into bypass Protection ends when the scheduler enters bypass, so that bypass scheduling can cut every task's slice. scx_bypass() clears it for the queued tasks by cycling them, and strips the running task's in the same walk over rq->scx.runnable_list. The running task is never on that list. set_next_task_scx() takes it off and put_prev_task_scx() puts it back, so the strip never runs. A protected running task then keeps its slice into bypass, where the tick's attempts to cut it are refused. Strip the running task's protection outside the walk. Fixes: 9cfc6ab34a31 ("sched_ext: Add SCX_TASK_PROTECTED") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 2c213153bdab..d6689bf8e769 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -6318,6 +6318,16 @@ void scx_bypass(struct scx_sched *sch, bool bypass) continue; } + /* + * Bypass trumps protection. Cycling the queued tasks below + * clears theirs. The current task isn't on the runnable list, + * so strip its protection here. + */ + p = rq->curr; + if (bypass && p->sched_class == &ext_sched_class && + scx_is_descendant(scx_task_sched(p), sch)) + scx_task_slice_ended(rq, p); + /* * The use of list_for_each_entry_safe_reverse() is required * because each task is going to be removed from and added back @@ -6330,13 +6340,6 @@ void scx_bypass(struct scx_sched *sch, bool bypass) if (!scx_is_descendant(scx_task_sched(p), sch)) continue; - /* - * Bypass trumps protection. Cycling clears for queued - * tasks but current task needs explicit stripping. - */ - if (bypass && task_current(rq, p)) - scx_task_slice_ended(rq, p); - /* cycling deq/enq is enough, see the function comment */ scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { /* nothing */ ; -- cgit v1.2.3 From 5809f749fbdbaf9c6b34ecbe6d2b413e56eec6e3 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 16 Oct 2026 19:37:17 -1000 Subject: sched_ext: Fix a bypassed task stalling on a cpu its scheduler has no caps on A bypassing sub-scheduler's tasks are run by its nearest ancestor that isn't bypassing. The ancestor puts them on the local DSQ under its own caps. Whether such a task may stay on the cpu and whether a cap revoke reenqueues it are still checked against the caps of the task's own scheduler. The bypass load balancer moves tasks between cpus without looking at caps, so a bypassed task can run on a cpu where its own scheduler holds none. Once its slice runs out, it can't stay. It goes to the ancestor's bypass DSQ without a kick, and the cpu can go idle with it queued until something else reschedules the cpu. Make both checks against the caps of the scheduler running the task. When bypass lifts, the running task keeps its slice without a cap check, so evict it then if it can't stay on the cpu. Fixes: 75a8c8202c91 ("sched_ext: Add reject DSQ for cap-rejected dispatches") Fixes: 46a85ae6fe5b ("sched_ext: Tie cpu occupancy to SCX_CAP_BASE through the task slice") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 21 ++++++++++++--------- kernel/sched/ext/internal.h | 17 +++++++++++++++++ kernel/sched/ext/sub.c | 7 +++++-- kernel/sched/ext/sub.h | 6 +++++- 4 files changed, 39 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index d6689bf8e769..6b472506944e 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -322,7 +322,6 @@ static const struct sched_class *scx_setscheduler_class(struct task_struct *p) static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu) { -#ifdef CONFIG_EXT_SUB_SCHED /* * If @sch is a sub-sched which is bypassing, its tasks should go into * the bypass DSQs of the nearest ancestor which is not bypassing. The @@ -334,11 +333,7 @@ static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 c * are re-enqueued after scx_bypassing() is turned on, guaranteeing that * all tasks are transferred to the right DSQs. */ - while (scx_parent(sch) && scx_bypassing(sch, cpu)) - sch = scx_parent(sch); -#endif /* CONFIG_EXT_SUB_SCHED */ - - return scx_bypass_dsq(sch, cpu); + return scx_bypass_dsq(scx_exec_sched(sch, cpu), cpu); } /** @@ -6322,11 +6317,19 @@ void scx_bypass(struct scx_sched *sch, bool bypass) * Bypass trumps protection. Cycling the queued tasks below * clears theirs. The current task isn't on the runnable list, * so strip its protection here. + * + * When bypass lifts, the current task keeps running on its + * slice without a cap check. Evict it if it can't stay on the + * cpu. */ p = rq->curr; - if (bypass && p->sched_class == &ext_sched_class && - scx_is_descendant(scx_task_sched(p), sch)) - scx_task_slice_ended(rq, p); + if (p->sched_class == &ext_sched_class && + scx_is_descendant(scx_task_sched(p), sch)) { + if (bypass) + scx_task_slice_ended(rq, p); + else if (!scx_task_can_stay_on_cpu(rq, p)) + scx_set_task_slice(p, 0); + } /* * The use of list_for_each_entry_safe_reverse() is required diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 352547aedbb7..03f03204d158 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -2539,4 +2539,21 @@ static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; #endif /* CONFIG_EXT_SUB_SCHED */ +/** + * scx_exec_sched - Find the sched running @sch's tasks on @cpu + * @sch: sched owning the tasks + * @cpu: cpu of interest + * + * A bypassing sched's tasks are run by its nearest ancestor that isn't + * bypassing. Return @sch if it isn't bypassing on @cpu, otherwise that + * ancestor, or the root if every ancestor is bypassing. Hold @cpu's rq lock for + * a stable answer. + */ +static inline struct scx_sched *scx_exec_sched(struct scx_sched *sch, s32 cpu) +{ + while (scx_parent(sch) && scx_bypassing(sch, cpu)) + sch = scx_parent(sch); + return sch; +} + #endif /* _KERNEL_SCHED_EXT_INTERNAL_H */ diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index fee75fcdb59d..90430493bb72 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -755,18 +755,21 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r /* @p lost the caps needed to stay on @rq's local DSQ? Record reason if so. */ bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { + s32 cpu = cpu_of(rq); u64 missing; /* migration-disabled tasks and the rescuee are admitted capless */ if (is_migration_disabled(p) || p == scx_rescuee(rq)) return false; - missing = scx_missing_caps(scx_task_sched(p), cpu_of(rq), scx_caps_for_task(p)); + /* check caps against the sched running a bypassed task */ + missing = scx_missing_caps(scx_exec_sched(scx_task_sched(p), cpu), cpu, + scx_caps_for_task(p)); if (likely(!missing)) return false; p->scx.reenq_reason_caps = missing; - p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu_of(rq)); + p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu); return true; } diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index f7bcdfda8dd8..0ac7b8e59205 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -199,6 +199,8 @@ static inline u64 scx_caps_implied(u64 cap) /* may @p keep running on @rq's cpu? requires baseline cpu access */ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { + s32 cpu = cpu_of(rq); + if (!scx_has_subs()) return true; @@ -206,7 +208,9 @@ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p if (unlikely(is_migration_disabled(p))) return true; - return likely(!scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE)); + /* check caps against the sched running a bypassed task */ + return likely(!scx_missing_caps(scx_exec_sched(scx_task_sched(p), cpu), cpu, + SCX_CAP_BASE)); } /* the task admitted for rescue on @rq, NULL if none */ -- cgit v1.2.3 From ff0545e0c263727b200f60af47d0364e4d041bde Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Sat, 17 Oct 2026 17:22:26 +0200 Subject: sched_ext: Don't leave balance callbacks queued across a pick retry The dispatch path can request deferred work to be run from a balance callback and pick_task_scx() queues rq->scx.deferred_bal_cb right after dispatching. However, a balance callback must be consumed before the rq lock is released and pick_task_scx() is not necessarily the end of the pick: - it returns RETRY_TASK when a higher priority class task has been enqueued while the dispatch had the rq lock released; - the dl_server pick can return NULL, in which case the lower classes are picked next. In both cases the callback stays queued while the core keeps picking, and the rq lock can be released again, either by prev_balance() or by another dispatch. For example, with an RT task as prev: __schedule() __pick_next_task() pick_task_scx() dispatch_pick() maybe_queue_balance_callback() <- deferred_bal_cb queued return RETRY_TASK prev_balance() balance_rt() pull_rt_task() double_lock_balance() <- rq lock released Any CPU acquiring the rq lock in that window finds a pending balance callback and triggers the following warning in rq_pin_lock(): WARNING: kernel/sched/sched.h:1919 at rq_lock+0x7f/0xe0, CPU#10: stress-ng-sleep/533 CPU: 10 UID: 0 PID: 533 Comm: stress-ng-sleep Not tainted 7.2.0-rc6-virtme #61 PREEMPT(full) Sched_ext: qmap (enabled+all) Call Trace: try_to_wake_up+0x2c1/0x5c0 hrtimer_wakeup+0x22/0x30 __hrtimer_run_queues+0x1d5/0x350 hrtimer_interrupt+0x106/0x1e0 __sysvec_apic_timer_interrupt+0x6f/0x210 sysvec_apic_timer_interrupt+0x71/0x90 This can be reproduced running scx_qmap -I together with a SCHED_FIFO workload that frequently sleeps, e.g.: $ vng -v --cpus 16 --memory 4G -- \ 'tools/sched_ext/build/bin/scx_qmap -I >/dev/null & \ stress-ng --sleep 4 --sched fifo --sched-prio 1 -t 30 -q' Fix this by queueing the balance callbacks only when the pick is final. Otherwise, defer the pending work to the irq_work path, which doesn't depend on the current rq lock section, and leave the kick sync wait pending for the next pick. Fixes: 4c95380701f5 ("sched/ext: Fold balance_scx() into pick_task_scx()") Cc: stable@vger.kernel.org # v6.19+ Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 73 +++++++++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 28 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 6b472506944e..2cd24a50f06c 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3330,7 +3330,8 @@ static struct task_struct *first_local_task(struct rq *rq) } /* - * Run dispatch and queue the follow-up work for a pick. + * Run dispatch for a pick. The follow-up work is queued by do_pick_task_scx() + * once the pick is final. */ static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf, struct task_struct *prev) @@ -3340,18 +3341,6 @@ static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf, rq_unpin_lock(rq, rf); verdict = dispatch_one(rq, prev); rq_repin_lock(rq, rf); - maybe_queue_balance_callback(rq); - - /* - * Defer to a balance callback which can drop rq lock and enable IRQs. - * Waiting directly in the pick path would deadlock against CPUs sending - * us IPIs (e.g. TLB flushes) while we wait for them. - */ - if (unlikely(rq->scx.kick_sync_pending)) { - rq->scx.kick_sync_pending = false; - queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, - kick_sync_wait_bal_cb); - } return verdict; } @@ -3378,16 +3367,8 @@ static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *r verdict = dispatch_one(rq, prev); - if (cpu_of(rq) == smp_processor_id()) { - maybe_queue_balance_callback(rq); - - /* see dispatch_pick() */ - if (unlikely(rq->scx.kick_sync_pending)) { - rq->scx.kick_sync_pending = false; - queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, - kick_sync_wait_bal_cb); - } - } else if (unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { + if (cpu_of(rq) != smp_processor_id() && + unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { /* * Balance callbacks must run in the context that queued them, * so they can't be queued on another CPU's rq. Run the deferred @@ -3430,8 +3411,10 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) else verdict = dispatch_pick(rq, rf, prev); - if (verdict == SCX_DSP_RETRY) - return RETRY_TASK; + if (verdict == SCX_DSP_RETRY) { + p = RETRY_TASK; + goto out; + } /* * If any higher-priority sched class enqueued a runnable task on this @@ -3441,8 +3424,10 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) * If @force_scx is true, always try to pick a SCHED_EXT task, * regardless of any higher-priority sched classes activity. */ - if (!force_scx && rq_modified_above(rq, &ext_sched_class)) - return RETRY_TASK; + if (!force_scx && rq_modified_above(rq, &ext_sched_class)) { + p = RETRY_TASK; + goto out; + } /* * If we're keeping @prev, replenish slice if necessary and keep running @@ -3458,7 +3443,7 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) } else { p = first_local_task(rq); if (!p) - return NULL; + goto out; if (unlikely(!p->scx.slice) && scx_task_can_stay_on_cpu(rq, p)) { struct scx_sched *sch = scx_task_sched(p); @@ -3473,6 +3458,38 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) } } +out: + if (cpu_of(rq) != smp_processor_id()) + return p; + + /* + * Balance callbacks must be consumed before the rq lock is released, so + * queue them only when the pick is final. On RETRY_TASK, or after a + * NULL pick from the dl_server, the core continues picking and can + * release the rq lock (e.g., prev_balance() or another dispatch). In + * that case, run the deferred work from irq_work, which doesn't depend + * on the current rq lock section, and leave the kick sync wait pending + * for the next pick. + */ + if (p != RETRY_TASK && (p || !force_scx)) { + maybe_queue_balance_callback(rq); + + /* + * Defer to a balance callback which can drop rq lock and enable + * IRQs. Waiting directly in the pick path would deadlock against + * CPUs sending us IPIs (e.g., TLB flushes) while we wait for + * them. + */ + if (unlikely(rq->scx.kick_sync_pending)) { + rq->scx.kick_sync_pending = false; + queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, + kick_sync_wait_bal_cb); + } + } else if (rq->scx.flags & SCX_RQ_BAL_CB_PENDING) { + rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; + schedule_deferred(rq); + } + return p; } -- cgit v1.2.3 From 680edc7f23ad71e0ecce8a2364af3ae7f30dd11a Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Sat, 17 Oct 2026 22:05:14 +0200 Subject: sched_ext: Don't queue balance callbacks from a core-sched pick Commit ff0545e0c263 ("sched_ext: Don't leave balance callbacks queued across a pick retry") queues the deferred work as a balance callback only when do_pick_task_scx() returns a final pick. However, under core scheduling, a pick that returns a task isn't final either: pick_next_task() goes on to pick the SMT siblings, and their dispatch can release the shared rq lock with the callback still queued, triggering the rq_pin_lock() warning. Moreover, dispatch_core_pick() returns early when another dispatch is in flight on the rq, and do_pick_task_scx() then consumes %SCX_RQ_BAL_CB_PENDING set by that dispatch while it's still running. This can be reproduced running scx_qmap -I on SMT siblings with coresched-tagged workloads that frequently sleep, mixed with untagged CPU-bound tasks. Fix this by always deferring the pending work to the irq_work path from dispatch_core_pick() under core scheduling, for both the local and the sibling picks, and by leaving %SCX_RQ_BAL_CB_PENDING alone in do_pick_task_scx(), so that it's handled by the dispatch that set it. Sample the core scheduling state once at the start of the pick, since core scheduling can get enabled while a non-core dispatch has the rq lock released, and the follow-up work must match the dispatch path that ran. Fixes: 4c95380701f5 ("sched/ext: Fold balance_scx() into pick_task_scx()") Cc: stable@vger.kernel.org # v6.19+ Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 2cd24a50f06c..e59e937534e9 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3367,15 +3367,20 @@ static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *r verdict = dispatch_one(rq, prev); - if (cpu_of(rq) != smp_processor_id() && - unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { - /* - * Balance callbacks must run in the context that queued them, - * so they can't be queued on another CPU's rq. Run the deferred - * work directly instead. - */ + /* + * A pick that returns a task isn't final under core scheduling: the + * selection goes on to pick the siblings, and their dispatch can + * release the rq lock, which can't happen with balance callbacks + * queued. Don't queue the deferred work as a balance callback, for this + * CPU or a sibling, and run it from irq_work instead. + * + * do_pick_task_scx() leaves %SCX_RQ_BAL_CB_PENDING alone under core + * scheduling, so that the early return above leaves it to the dispatch + * in flight, which handles it here. + */ + if (unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; - run_deferred(rq); + schedule_deferred(rq); } rq_repin_lock(rq, rf); @@ -3398,6 +3403,7 @@ static struct task_struct * do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) { struct task_struct *prev = rq->curr; + bool core_sched = sched_core_enabled(rq); enum scx_dsp_verdict verdict; struct task_struct *p; @@ -3406,7 +3412,12 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) rq_modified_begin(rq, &ext_sched_class); - if (sched_core_enabled(rq)) + /* + * Core scheduling can get enabled while dispatch_pick() has the rq lock + * released. Stick to @core_sched, so that the follow-up work below + * matches the dispatch that ran. + */ + if (core_sched) verdict = dispatch_core_pick(rq, rf, prev); else verdict = dispatch_pick(rq, rf, prev); @@ -3472,7 +3483,9 @@ out: * for the next pick. */ if (p != RETRY_TASK && (p || !force_scx)) { - maybe_queue_balance_callback(rq); + /* under core scheduling, see dispatch_core_pick() */ + if (!core_sched) + maybe_queue_balance_callback(rq); /* * Defer to a balance callback which can drop rq lock and enable @@ -3485,7 +3498,7 @@ out: queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, kick_sync_wait_bal_cb); } - } else if (rq->scx.flags & SCX_RQ_BAL_CB_PENDING) { + } else if (!core_sched && (rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; schedule_deferred(rq); } -- cgit v1.2.3 From 5547b250676c1924090133e6f3f30195f54cdbe1 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 17 Oct 2026 12:29:39 -1000 Subject: sched_ext: Factor out the rq lock switch of scx_dsq_move() Factor the rq lock switch on entry to scx_dsq_move() and its undo on exit out into kf_switch_rq_lock() and kf_restore_rq_lock() so that other kfuncs operating on a task from a DSQ iteration can share them. No functional change. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 63 ++++++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 27 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index b9f80fb29255..3e707b00e78b 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -9745,6 +9745,40 @@ static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { .filter = scx_kfunc_context_filter, }; +/* + * Lock @rq from a kfunc called either from ops.dispatch(), which holds the + * dispatched rq's lock, or from a context holding no rq lock. Under core + * scheduling, dispatch can run for a sibling rq, so the locked rq is not + * necessarily this CPU's. Return the rq locked on entry, NULL if none. + */ +static struct rq *kf_switch_rq_lock(struct rq *rq, unsigned long *flags) +{ + struct rq *entry_rq; + + local_irq_save(*flags); + + entry_rq = scx_locked_rq(); + if (entry_rq) { + if (entry_rq != rq) + switch_rq_lock(entry_rq, rq); + } else { + raw_spin_rq_lock(rq); + } + return entry_rq; +} + +/* undo kf_switch_rq_lock(), @rq is the rq locked now */ +static void kf_restore_rq_lock(struct rq *rq, struct rq *entry_rq, unsigned long flags) +{ + if (entry_rq) { + if (entry_rq != rq) + switch_rq_lock(rq, entry_rq); + } else { + scx_rq_lock_drop(rq); + raw_spin_rq_unlock_irqrestore(rq, flags); + } +} + static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, struct task_struct *p, u64 dsq_id, u64 enq_flags, bool priq) @@ -9779,27 +9813,8 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, if (unlikely(READ_ONCE(sch->aborting))) return false; - /* - * Can be called from either ops.dispatch() holding the dispatched rq's - * lock or any context where no rq lock is held. If latter, lock @p's - * task_rq which we'll likely need anyway. - */ src_rq = task_rq(p); - - local_irq_save(flags); - - /* - * Under core scheduling, dispatch can run for a sibling rq, so the - * locked rq is not necessarily this CPU's. - */ - locked_rq = scx_locked_rq(); - - if (locked_rq) { - if (locked_rq != src_rq) - switch_rq_lock(locked_rq, src_rq); - } else { - raw_spin_rq_lock(src_rq); - } + locked_rq = kf_switch_rq_lock(src_rq, &flags); p_rq = src_rq; raw_spin_lock(&src_dsq->lock); @@ -9838,13 +9853,7 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, p_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); dispatched = true; out: - if (locked_rq) { - if (locked_rq != p_rq) - switch_rq_lock(p_rq, locked_rq); - } else { - scx_rq_lock_drop(p_rq); - raw_spin_rq_unlock_irqrestore(p_rq, flags); - } + kf_restore_rq_lock(p_rq, locked_rq, flags); kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE | __SCX_DSQ_ITER_HAS_VTIME); -- cgit v1.2.3 From b3768d058c98d4e8114a246d3267d7dae1c123fb Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 17 Oct 2026 13:09:28 -1000 Subject: sched_ext: Add scx_bpf_dsq_reenq_task() to re-enqueue single tasks from a DSQ iteration scx_bpf_dsq_reenq() re-enqueues a whole DSQ, but a scheduler that loses the cpus serving a DSQ, such as a sub-scheduler after a cap revoke, needs only the stranded tasks placed again. Add scx_bpf_dsq_reenq_task(), which marks a task from a DSQ iteration. When the iteration ends, the DSQ's deferred re-enqueue walk is scheduled once and re-enqueues only the marked tasks. The kfunc locks the task's rq and the DSQ like scx_bpf_dsq_move() and is callable from the same contexts. The mark is cleared when the task leaves the DSQ, so a task consumed before the walk is not re-enqueued. An iteration's DSQ can be destroyed before the iteration ends, so schedule_dsq_reenq() now skips a destroyed DSQ instead of aborting the scheduler. scx_bpf_dsq_reenq() racing scx_bpf_destroy_dsq() takes the same path. Declare the kfunc for the tools and add a compat probe. v2: Note in the kerneldoc that another iteration's re-enqueue may take a marked task early, and that a task on another scheduler returns false (Andrea). Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 82 +++++++++++++++++++++++++++++++++++++++++++-- kernel/sched/ext/internal.h | 2 ++ 2 files changed, 81 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 3e707b00e78b..14fe63428036 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -555,11 +555,13 @@ enum scx_dsq_iter_flags { /* iterate in the reverse dispatch order */ SCX_DSQ_ITER_REV = 1U << 16, + __SCX_DSQ_ITER_HAS_REENQ = 1U << 29, __SCX_DSQ_ITER_HAS_SLICE = 1U << 30, __SCX_DSQ_ITER_HAS_VTIME = 1U << 31, __SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV, __SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS | + __SCX_DSQ_ITER_HAS_REENQ | __SCX_DSQ_ITER_HAS_SLICE | __SCX_DSQ_ITER_HAS_VTIME, }; @@ -1292,7 +1294,9 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, WRITE_ONCE(dru->flags, dru->flags | reenq_flags); } } else { - scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id); + /* destroyed since the caller resolved it, nothing to reenq */ + if (READ_ONCE(dsq->id) != SCX_DSQ_INVALID) + scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id); return; } @@ -1918,6 +1922,7 @@ void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq) RB_CLEAR_NODE(&p->scx.dsq_priq); p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; } + p->scx.dsq_flags &= ~SCX_TASK_DSQ_REENQ; list_del_init(&p->scx.dsq_list.node); dsq_dec_nr(dsq, p); @@ -5095,7 +5100,9 @@ static void process_deferred_reenq_locals(struct rq *rq) static bool user_task_should_reenq(struct task_struct *p, u64 reenq_flags, u32 *reason) { *reason = SCX_TASK_REENQ_KFUNC; - return reenq_flags & SCX_REENQ_ANY; + if (reenq_flags & SCX_REENQ_ANY) + return true; + return (reenq_flags & SCX_REENQ_MARKED) && (p->scx.dsq_flags & SCX_TASK_DSQ_REENQ); } static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flags) @@ -10087,6 +10094,8 @@ BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) +/* also in scx_kfunc_ids_unlocked, same contexts as scx_bpf_dsq_move() */ +BTF_ID_FLAGS(func, scx_bpf_dsq_reenq_task, KF_RCU) #ifdef CONFIG_EXT_SUB_SCHED BTF_ID_FLAGS(func, scx_bpf_sub_dispatch, KF_IMPLICIT_ARGS) #endif @@ -10202,6 +10211,7 @@ BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_reenq_task, KF_RCU) /* also in scx_kfunc_ids_select_cpu: also callable from ops.select_cpu()/ops.enqueue() */ BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) @@ -10610,7 +10620,8 @@ __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *i * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator * @it: iterator to destroy * - * Undo bpf_iter_scx_dsq_new(). + * Undo bpf_iter_scx_dsq_new(). If the iteration marked tasks with + * scx_bpf_dsq_reenq_task(), schedule their re-enqueue. */ __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) { @@ -10626,6 +10637,13 @@ __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) list_del_init(&kit->cursor.node); raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); } + + if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_REENQ) { + guard(preempt)(); + schedule_dsq_reenq(kit->dsq->sched, kit->dsq, SCX_REENQ_MARKED, + scx_locked_rq()); + } + kit->dsq = NULL; } @@ -10714,6 +10732,64 @@ __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, schedule_dsq_reenq(sch, dsq, reenq_flags, locked_rq); } +/** + * scx_bpf_dsq_reenq_task - Re-enqueue a task from a DSQ iteration + * @it__iter: DSQ iterator in progress + * @p: task to re-enqueue + * + * Mark @p, which is on the DSQ iterated by @it__iter and was queued before the + * iteration started, for re-enqueue. When the iteration ends, the DSQ's marked + * tasks go through ops.enqueue() with %SCX_ENQ_REENQ in DSQ order and the other + * tasks on the DSQ keep their positions. A marked task that leaves the DSQ in + * the meantime is not re-enqueued. The mark belongs to @p, so a re-enqueue + * scheduled by another iteration of the DSQ may re-enqueue @p before this + * iteration ends. @p must be on the DSQ's scheduler. + * + * Re-enqueues are performed asynchronously. Can be called from ops.dispatch() + * or any BPF context which doesn't hold a rq lock (e.g. BPF timers or SYSCALL + * programs). + * + * Return %true if @p was marked, %false if @p left the DSQ or was queued after + * the iteration started, @p is on another scheduler, or the iterator holds no + * DSQ. + */ +__bpf_kfunc bool scx_bpf_dsq_reenq_task(struct bpf_iter_scx_dsq *it__iter, + struct task_struct *p) +{ + struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; + struct scx_dispatch_q *dsq = kit->dsq; + struct rq *src_rq, *entry_rq; + unsigned long flags; + bool marked = false; + + /* NULL after a failed bpf_iter_scx_dsq_new(), see scx_dsq_move() */ + if (unlikely(!dsq)) + return false; + + /* the cursor check needs @p's rq locked */ + src_rq = task_rq(p); + entry_rq = kf_switch_rq_lock(src_rq, &flags); + + scoped_guard (raw_spinlock, &dsq->lock) { + if (nldsq_cursor_lost_task(&kit->cursor, src_rq, dsq, p)) + break; + + if (unlikely(!scx_task_on_sched(dsq->sched, p))) { + scx_error(dsq->sched, "scx_bpf_dsq_reenq_task() on %s[%d] but the task belongs to a different scheduler", + p->comm, p->pid); + break; + } + + p->scx.dsq_flags |= SCX_TASK_DSQ_REENQ; + /* the walk is scheduled once, by bpf_iter_scx_dsq_destroy() */ + kit->cursor.flags |= __SCX_DSQ_ITER_HAS_REENQ; + marked = true; + } + + kf_restore_rq_lock(src_rq, entry_rq, flags); + return marked; +} + /** * scx_bpf_reenqueue_local___v2 - Re-enqueue tasks on a local DSQ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index b408993af794..29e69a83d876 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1946,6 +1946,8 @@ enum scx_reenq_flags { /* internal: kernel-issued on cap revoke, not accepted from BPF */ SCX_REENQ_CAP_REVOKE = 1LLU << 1, + /* internal: tasks marked by scx_bpf_dsq_reenq_task() */ + SCX_REENQ_MARKED = 1LLU << 2, __SCX_REENQ_FILTER_MASK = 0xffffLLU, -- cgit v1.2.3 From 1f5dde349ac2fca1f119ae8de3f0d2f1c2a8d338 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 17 Oct 2026 13:56:04 -1000 Subject: sched_ext: Drop the locks periodically while walking a user DSQ for re-enqueue reenq_user() drops dsq->lock around each re-enqueue and its rq lock every SCX_TASK_ITER_BATCH re-enqueues. A walk filtered by SCX_REENQ_MARKED skips unmarked tasks without dropping either, so a long run of unmarked tasks is walked in one hold of both locks. Count the visited tasks instead of the re-enqueued ones and drop both locks every SCX_TASK_ITER_BATCH visits. An unfiltered walk drops the locks as often as before. Suggested-by: Andrea Righi Link: https://lore.kernel.org/all/atP-LIKpxPhR-Nuc@gpd4/ Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 14fe63428036..07857039aee6 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -5111,7 +5111,7 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag struct scx_sched *sch = dsq->sched; struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, dsq, 0); struct task_struct *p; - s32 nr_enqueued = 0; + s32 nr_visited = 0; lockdep_assert_rq_held(rq); @@ -5121,6 +5121,21 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag struct rq *task_rq; u32 reason; + /* + * Count skipped tasks too so that a long walk doesn't hold the + * locks throughout. The cursor keeps the position. + */ + if (!(++nr_visited % SCX_TASK_ITER_BATCH)) { + raw_spin_unlock(&dsq->lock); + if (locked_rq) { + scx_rq_lock_drop(locked_rq); + raw_spin_rq_unlock(locked_rq); + locked_rq = NULL; + } + cpu_relax(); + raw_spin_lock(&dsq->lock); + } + p = nldsq_cursor_next_task(&cursor, dsq); if (!p) break; @@ -5157,13 +5172,6 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag scx_do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1); - if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) { - scx_rq_lock_drop(locked_rq); - raw_spin_rq_unlock(locked_rq); - locked_rq = NULL; - cpu_relax(); - } - raw_spin_lock(&dsq->lock); } -- cgit v1.2.3