diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/sched/ext/ext.c | 64 |
1 files changed, 39 insertions, 25 deletions
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index cddfcc61c454..dfbfa6455629 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -6624,7 +6624,7 @@ bool scx_hardlockup(int cpu) * @sch: host sched whose bypass DSQs are balanced * @donor_rq: rq of the cpu whose bypass DSQ gives up @p, locked * @donor_dsq: @sch's bypass DSQ on @donor_rq's cpu, locked - * @p: task on @donor_dsq, homed on @donor_rq + * @p: task on @donor_dsq, homed anywhere * @donee: cpu whose bypass DSQ receives @p * * Take @p off @donor_dsq and insert it into the bypass DSQ of @donee. The @@ -6632,28 +6632,45 @@ bool scx_hardlockup(int cpu) * can still refuse the move under @p's rq lock. @p then goes back to the head * of @donor_dsq. * + * The insert runs under @p's rq lock. A task homed on @donor_rq is dequeued in + * place. unlink_dsq_and_switch_rq_lock() takes one homed elsewhere off and + * switches the rq lock from @donor_rq to its own. The switch can lose to a + * dequeue, and @p then belongs to the dequeuer. + * * @donor_rq and @donor_dsq are locked on entry and on return. @donor_dsq is - * unlocked across the insert, so the caller's walk must find its place in it - * again after the call. + * unlocked in between, and @donor_rq too while the lock is switched, so the + * caller's walk must find its place in @donor_dsq again after the call. * - * Return the cpu that received @p, or -1 if @p went back onto @donor_dsq. + * Return the cpu that received @p, or -1 if @p went back onto @donor_dsq or was + * lost to a dequeue. */ static s32 bypass_lb_move(struct scx_sched *sch, struct rq *donor_rq, struct scx_dispatch_q *donor_dsq, struct task_struct *p, s32 donee) { struct scx_dispatch_q *dst_dsq; u64 enq_flags; + struct rq *rq = task_rq(p); s32 ret; lockdep_assert_rq_held(donor_rq); lockdep_assert_held(&donor_dsq->lock); - scx_wait_dispatching(p); - dispatch_dequeue_locked(p, donor_dsq); - raw_spin_unlock(&donor_dsq->lock); + if (rq == donor_rq) { + scx_wait_dispatching(p); + dispatch_dequeue_locked(p, donor_dsq); + raw_spin_unlock(&donor_dsq->lock); + } else if (!unlink_dsq_and_switch_rq_lock(p, donor_dsq, donor_rq, rq)) { + /* lost to a dequeue, @p is the dequeuer's now */ + ret = -1; + goto out; + } - /* @p's affinity is stable under its rq lock */ - if (task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false)) { + /* + * @p's affinity is stable under its rq lock. A task whose donee is its + * own cpu can always run there, and task_can_run_on_remote_rq() warns + * when asked about a task's own cpu. + */ + if (donee == task_cpu(p) || task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false)) { dst_dsq = scx_bypass_dsq(sch, donee); enq_flags = 0; ret = donee; @@ -6664,8 +6681,10 @@ static s32 bypass_lb_move(struct scx_sched *sch, struct rq *donor_rq, ret = -1; } - scx_dispatch_enqueue(sch, donor_rq, dst_dsq, p, 0, 0, enq_flags); - + scx_dispatch_enqueue(sch, rq, dst_dsq, p, 0, 0, enq_flags); +out: + if (rq != donor_rq) + switch_rq_lock(rq, donor_rq); raw_spin_lock(&donor_dsq->lock); return ret; } @@ -6718,22 +6737,16 @@ resume: if (u32_before(cursor.priv, p->scx.dsq_seq)) continue; - /* - * If an earlier pass placed @p on @donor_dsq from a different - * CPU and the donee hasn't consumed it yet, @p is still on the - * previous CPU and task_rq(@p) != @donor_rq. @p can't be moved - * without its rq locked. Skip. - */ - if (task_rq(p) != donor_rq) - continue; - donee = cpumask_any_and_distribute(donee_mask, p->cpus_ptr); if (donee >= nr_cpu_ids) continue; /* - * The move drops @donor_dsq's lock. Park the cursor behind @p - * and resume from it afterwards. + * The move drops @donor_dsq's lock, and the rq lock too when + * @p is homed elsewhere. The donor cpu's dispatch and its + * enqueuers can reach @donor_dsq then. Park the cursor behind + * @p and resume from it afterwards. The seq test above keeps a + * task enqueued meanwhile out of the pass. */ list_move(&cursor.node, &p->scx.dsq_list.node); cpu = bypass_lb_move(sch, donor_rq, donor_dsq, p, donee); @@ -6752,9 +6765,10 @@ resume: * The rq lock is held across the walk, not cycled per * move. @donor_dsq can be very long and contended, and * the rq lock is what keeps its consumer and enqueuers - * away while the walk cycles the DSQ lock. The walk can - * still take a while, so let the cpu breathe every so - * often. + * away while the walk cycles the DSQ lock. The rare + * move of a task homed elsewhere does switch it away, + * see the cursor comment above. The walk can still take + * a while, so let the cpu breathe every so often. */ if (!(++nr_balanced % SCX_BYPASS_LB_BATCH)) { raw_spin_unlock(&donor_dsq->lock); |