summaryrefslogtreecommitdiff
path: root/kernel
diff options
context:
space:
mode:
authorUsama Arif <usama.arif@linux.dev>2026-09-23 07:07:50 -0700
committerTejun Heo <tj@kernel.org>2026-09-23 08:53:31 -1000
commitefa7bc7d719558483b14e1c44bcce11532a44581 (patch)
tree7d1a8680f99e8d66d549dd234250b03150159cf8 /kernel
parent6b8d2bc44fbb6d232382141ae70d4ecc28a9dc36 (diff)
sched_ext: Avoid relocking DSQ during remote consumption
A remote consume unlinks a task from a non-local DSQ before switching to the task's rq. Leaving p->scx.dsq set makes the following deactivate_task() reacquire the DSQ lock in scx_dispatch_dequeue() to clear the transfer state. Keep p->scx.dsq set while switching rq locks so that a racing dequeue can find the DSQ and cancel the transfer. After acquiring the source rq and confirming ownership, clear p->scx.dsq. The subsequent dequeue then takes its !dsq path without reacquiring the DSQ lock. Fleet profiles show that roughly 70% of the cycles attributed to scx_dispatch_dequeue() are spent in queued_spin_lock_slowpath() while taking dsq->lock. Signed-off-by: Usama Arif <usama.arif@linux.dev> Signed-off-by: Tejun Heo <tj@kernel.org>
Diffstat (limited to 'kernel')
-rw-r--r--kernel/sched/ext/ext.c16
1 files changed, 13 insertions, 3 deletions
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index f60894dbf062..03c85ad0ead4 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -2599,7 +2599,7 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch,
* points to this CPU. See scx_dispatch_dequeue() for the counterpart.
*
* On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is
- * still valid. %false if lost to dequeue.
+ * still valid and detached from @dsq. %false if lost to dequeue.
*/
static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p,
struct scx_dispatch_q *dsq,
@@ -2619,8 +2619,18 @@ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p,
switch_rq_lock(locked_rq, src_rq);
/* task_rq couldn't have changed if we're still the holding cpu */
- return likely(p->scx.holding_cpu == cpu) &&
- !WARN_ON_ONCE(src_rq != task_rq(p));
+ if (likely(p->scx.holding_cpu == cpu) && !WARN_ON_ONCE(src_rq != task_rq(p))) {
+ /*
+ * Keep ->dsq set until we own @src_rq so that a racing dequeue
+ * can find @dsq and clear holding_cpu. Once ownership is
+ * confirmed, clear it under @src_rq so that deactivate_task()
+ * takes the !dsq path instead of retaking @dsq->lock.
+ */
+ p->scx.dsq = NULL;
+ return true;
+ }
+
+ return false;
}
static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq,