diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/sched/ext/ext.c | 34 |
1 files changed, 19 insertions, 15 deletions
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 3edb51bdd0ef..7303c2c45964 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -6099,15 +6099,12 @@ static void enable_bypass_dsp(struct scx_sched *sch) /* * When a sub-sched bypasses, its tasks are queued on the bypass DSQs of - * the nearest non-bypassing ancestor or root. As enable_bypass_dsp() is - * called iff @sch is not already bypassed due to an ancestor bypassing, - * we can assume that the parent is not bypassing and thus will be the - * host of the bypass DSQs. - * - * While the situation may change in the future, the following - * guarantees that the nearest non-bypassing ancestor or root has bypass - * dispatch enabled while a descendant is bypassing, which is all that's - * required. + * the nearest non-bypassing ancestor or root. The parent hosts them + * unless it is bypassing too. Then they sit further up and the parent + * consumes only its own bypass DSQs, empty until it leaves bypass and + * becomes the host. Either way the nearest non-bypassing ancestor or + * root has bypass dispatch enabled while a descendant is bypassing, + * which is all that's required. * * scx_bypass_dsp_enabled() test is used to determine whether to enter * the bypass dispatch handling path from both bypassing and hosting @@ -6232,16 +6229,19 @@ void scx_bypass(struct scx_sched *sch, bool bypass) * Bypass state is propagated to all descendants - an scx_sched bypasses * if itself or any of its ancestors are in bypass mode. A descendant's * depth is the sum of its own bypasses and its ancestors', so every - * call propagates whether or not @sch changed state. + * call propagates whether or not @sch changed state. The descendants' + * claims are released below with @sch's, after their tasks have moved. */ raw_spin_lock(&scx_sched_lock); scx_for_each_descendant_pre(pos, sch) { if (pos == sch) continue; - if (bypass) - inc_bypass_depth(pos); - else + if (bypass) { + if (inc_bypass_depth(pos)) + enable_bypass_dsp(pos); + } else { dec_bypass_depth(pos); + } } raw_spin_unlock(&scx_sched_lock); @@ -6330,8 +6330,12 @@ void scx_bypass(struct scx_sched *sch, bool bypass) } /* disarming must come after moving all tasks out of the bypass DSQs */ - if (!bypass) - scx_disable_bypass_dsp(sch); + if (!bypass) { + guard(raw_spinlock)(&scx_sched_lock); + scx_for_each_descendant_pre(pos, sch) + if (!pos->bypass_depth) + scx_disable_bypass_dsp(pos); + } unlock: raw_spin_unlock_irqrestore(&scx_bypass_lock, flags); } |