From 5f01293930d18f8473681b378bb483b7087fc0dd Mon Sep 17 00:00:00 2001 From: Changwoo Min Date: Wed, 19 Aug 2026 01:04:29 +0900 Subject: sched_ext: Allow ops.cgroup_set_bandwidth() to be sleepable ops.cgroup_set_bandwidth() is delivered from scx_group_set_bandwidth(), which runs from the cpu.max cgroup interface write path (tg_set_bandwidth()) in process context. scx_group_set_bandwidth() holds percpu_down_read(&scx_cgroup_ops_rwsem), whose read side may sleep. The call site is therefore sleepable, like ops.cgroup_init(). bpf_scx_check_member() rejects a sleepable program on any member not on its allow-list, so a BPF scheduler cannot allocate -- which is sleepable -- when a cgroup gains a cpu.max limit at runtime; it must instead pre-reserve memory for a callback that cannot allocate. Add cgroup_set_bandwidth() to the allow-list so the callback can allocate on demand, and document that it may block. A scheduler must decide at load time whether to mark the callback sleepable, but the allow-list entry is a verifier property with no symbol to probe. Add a compatibility marker whose presence in the kernel's BTF lets userspace detect this support: DEFINE_SCX_COMPAT_MARKER() emits an empty, callerless function, here scx_compat_marker_cgroup_set_bandwidth_may_sleep(). It is __used __retain so neither the compiler nor the linker (under CONFIG_LD_DEAD_CODE_DATA_ELIMINATION) drops it. The markers share the scx_compat_marker_ prefix and are collected near the end of ext.c so more can be added as further capabilities appear. Signed-off-by: Changwoo Min Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 14 ++++++++++++++ kernel/sched/ext/internal.h | 23 ++++++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 10af28a9f2c0..b646711a45fe 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -8079,6 +8079,7 @@ static int bpf_scx_check_member(const struct btf_type *t, case offsetof(struct sched_ext_ops, cgroup_init): case offsetof(struct sched_ext_ops, cgroup_exit): case offsetof(struct sched_ext_ops, cgroup_prep_move): + case offsetof(struct sched_ext_ops, cgroup_set_bandwidth): #endif case offsetof(struct sched_ext_ops, cpu_online): case offsetof(struct sched_ext_ops, cpu_offline): @@ -11041,3 +11042,16 @@ static int __init scx_init(void) return 0; } __initcall(scx_init); + +/* + * Compatibility markers for userspace. Existence of a marker function + * represents that the kernel supports that sched-ext feature. + */ + +/* + * scx_compat_marker_cgroup_set_bandwidth_may_sleep: advertises that + * ops.cgroup_set_bandwidth() may be implemented as a sleepable callback. + */ +#ifdef CONFIG_EXT_GROUP_SCHED +DEFINE_SCX_COMPAT_MARKER(cgroup_set_bandwidth_may_sleep); +#endif /* CONFIG_EXT_GROUP_SCHED */ diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 27bbf5e04d90..53e136a47924 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -753,7 +753,7 @@ struct sched_ext_ops { * @burst_us: bandwidth control burst * * Update @cgrp's bandwidth control parameters. This is from the cpu.max - * cgroup interface. + * cgroup interface. This operation may block. * * @quota_us / @period_us determines the CPU bandwidth @cgrp is entitled * to. For example, if @period_us is 1_000_000 and @quota_us is @@ -2001,6 +2001,27 @@ struct scx_bstr_buf { char line[SCX_EXIT_MSG_LEN]; }; +/* Internal helper for DEFINE_SCX_COMPAT_MARKER(). */ +#define DECLARE_SCX_COMPAT_MARKER(func) \ + extern void scx_compat_marker_##func(void) + +/** + * DEFINE_SCX_COMPAT_MARKER() - define a userspace capability marker + * @func: marker suffix; the defined symbol is scx_compat_marker_@func + * + * Emit an empty, callerless function that is retained in the kernel's BTF. + * Its presence is part of the kernel<->userspace contract: userspace probes + * scx_compat_marker_@func (e.g. via BTF) to detect that this kernel supports + * the corresponding feature. + * + * The leading declaration suppresses the missing-prototype warning; the + * trailing declaration consumes the semicolon at the use site. + */ +#define DEFINE_SCX_COMPAT_MARKER(func) \ + DECLARE_SCX_COMPAT_MARKER(func); \ + __used __retain void scx_compat_marker_##func(void) {} \ + DECLARE_SCX_COMPAT_MARKER(func) + extern struct scx_sched __rcu *scx_root; DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); -- cgit v1.2.3 From cca061dccf563907061766191b2ce3f66b7c285a Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 21 Aug 2026 09:05:52 -1000 Subject: sched_ext: Fix spurious aborts in scx_bpf_dsq_move() on ownership change races scx_dsq_move() verifies that the task belongs to the calling scheduler before taking any locks and aborts the scheduler on mismatch. The task can lose the sched association at any point: It can run and fully exit, which clears the association, or get rehomed to a different sub-sched. Both are benign races, but the early ownership check escalates them into scheduler aborts. Move the ownership check below the cursor-lost check. Every ownership change dequeues the task first, so a task that is still on the iterated DSQ under the lock while owned elsewhere indicates a genuine violation and should abort. Also fix two stale comments still referencing sched_ext_free(), which has been renamed to sched_ext_dead(). Fixes: bb4d9fd55158 ("sched_ext: scx_dsq_move() should validate the task belongs to the right scheduler") Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index b646711a45fe..c539d15cda63 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -7694,7 +7694,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) /* * Enable ops for every task. Fork is excluded by scx_fork_rwsem * preventing new tasks from being added. No need to exclude tasks - * leaving as sched_ext_free() can handle both prepped and enabled + * leaving as sched_ext_dead() can handle both prepped and enabled * tasks. Prep all tasks first and then enable them with preemption * disabled. * @@ -7786,7 +7786,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) /* * We're fully committed and can't fail. The task READY -> ENABLED - * transitions here are synchronized against sched_ext_free() through + * transitions here are synchronized against sched_ext_dead() through * scx_tasks_lock. */ percpu_down_write(&scx_fork_rwsem); @@ -9004,12 +9004,6 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, if (unlikely(READ_ONCE(sch->aborting))) return false; - if (unlikely(!scx_task_on_sched(sch, p))) { - scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler", - p->comm, p->pid); - return false; - } - /* * Can be called from either ops.dispatch() holding the dispatched rq's * lock or any context where no rq lock is held. If latter, lock @p's @@ -9041,6 +9035,17 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, goto out; } + /* + * @p has been on $src_dsq and can't move anymore. If @p is not on @sch, + * the caller didn't have authority over @p at the time of the call. + */ + if (unlikely(!scx_task_on_sched(sch, p))) { + scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler", + p->comm, p->pid); + raw_spin_unlock(&src_dsq->lock); + goto out; + } + /* @p is still on $src_dsq and stable, determine the destination */ dst_dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, task_cpu(p)); -- cgit v1.2.3 From 4881a13521886076e8d6d677f274320dcf66fea4 Mon Sep 17 00:00:00 2001 From: Wanwu Li Date: Thu, 27 Aug 2026 17:14:11 +0800 Subject: sched_ext: Fix several comment issues Fix several comment issues found during review: __setschduler_prio() -> __setscheduler_class() scx_iter_scx_dsq_new() -> bpf_iter_scx_dsq_new() scx_next_task_scx() -> set_next_task_scx() Signed-off-by: Wanwu Li Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 10 +++++----- kernel/sched/ext/internal.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index c539d15cda63..76a3f4ea237c 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -876,9 +876,9 @@ struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) * unloading. The init_tasks ("swappers") should be excluded * from the iteration because: * - * - It's unsafe to use __setschduler_prio() on an init_task to - * determine the sched_class to use as it won't preserve its - * idle_sched_class. + * - It's unsafe to use __setscheduler_class() on an init_task + * to determine the sched_class to use as it won't preserve + * its idle_sched_class. * * - ops.init/exit_task() can easily be confused if called with * init_tasks as they, e.g., share PID 0. @@ -5514,7 +5514,7 @@ static const struct kset_uevent_ops scx_uevent_ops = { }; /* - * Used by sched_fork() and __setscheduler_prio() to pick the matching + * Used by sched_fork() and __setscheduler_class() to pick the matching * sched_class. dl/rt are already handled. */ bool task_should_scx(int policy) @@ -9771,7 +9771,7 @@ __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *i * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator * @it: iterator to destroy * - * Undo scx_iter_scx_dsq_new(). + * Undo bpf_iter_scx_dsq_new(). */ __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) { diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 53e136a47924..0967b99a4948 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -442,7 +442,7 @@ struct sched_ext_ops { * * Note that this callback may be called from a CPU other than the * one the task is going to run on. This can happen when a task - * property is changed (i.e., affinity), since scx_next_task_scx(), + * property is changed (i.e., affinity), since set_next_task_scx(), * which triggers this callback, may run on a CPU different from * the task's assigned CPU. * -- cgit v1.2.3 From 068e5a0bc57e57d24cbf38def29cc5fb4db9a0df Mon Sep 17 00:00:00 2001 From: Liang Luo Date: Tue, 25 Aug 2026 13:50:53 +0800 Subject: sched_ext: Fix missing @slice and @vtime descriptions in finish_dispatch() kernel-doc Commit 13f1eae3b662 ("sched_ext: Synchronize slice and dsq_vtime writes") added the slice and vtime parameters to finish_dispatch() but did not update its kernel-doc, which produces warnings: Warning: function parameter 'slice' not described in 'finish_dispatch' Warning: function parameter 'vtime' not described in 'finish_dispatch' Describe both parameters using the same wording as dispatch_to_local_dsq(), which receives the same values. Signed-off-by: Liang Luo Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 76a3f4ea237c..713aa26b2828 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2806,6 +2806,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, * @p: task to finish dispatching * @qseq_at_dispatch: qseq when @p started getting dispatched * @dsq_id: destination DSQ ID + * @slice: slice carried by the insert verdict, 0 keeps the current value + * @vtime: vtime carried by the insert verdict, committed on PRIQ inserts * @enq_flags: %SCX_ENQ_* * * Dispatching to local DSQs may need to wait for queueing to complete or -- cgit v1.2.3