summaryrefslogtreecommitdiff
path: root/kernel/sched
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/sched')
-rw-r--r--kernel/sched/core.c485
-rw-r--r--kernel/sched/cpufreq_schedutil.c14
-rw-r--r--kernel/sched/cputime.c63
-rw-r--r--kernel/sched/deadline.c9
-rw-r--r--kernel/sched/debug.c21
-rw-r--r--kernel/sched/ext/cid.c93
-rw-r--r--kernel/sched/ext/cid.h11
-rw-r--r--kernel/sched/ext/ext.c1594
-rw-r--r--kernel/sched/ext/ext.h37
-rw-r--r--kernel/sched/ext/idle.c64
-rw-r--r--kernel/sched/ext/inlines.h12
-rw-r--r--kernel/sched/ext/internal.h266
-rw-r--r--kernel/sched/ext/sub.c400
-rw-r--r--kernel/sched/ext/sub.h50
-rw-r--r--kernel/sched/ext/types.h32
-rw-r--r--kernel/sched/fair.c712
-rw-r--r--kernel/sched/idle.c5
-rw-r--r--kernel/sched/psi.c25
-rw-r--r--kernel/sched/rt.c7
-rw-r--r--kernel/sched/sched.h97
-rw-r--r--kernel/sched/stop_task.c5
-rw-r--r--kernel/sched/syscalls.c2
-rw-r--r--kernel/sched/topology.c40
-rw-r--r--kernel/sched/wait.c25
24 files changed, 2979 insertions, 1090 deletions
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 0b846a13c628..0ab15f03cb94 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -805,7 +805,7 @@ struct rq *_task_rq_lock(struct task_struct *p, struct rq_flags *rf)
/* Use CONFIG_PARAVIRT as this will avoid more #ifdef in arch code. */
#ifdef CONFIG_PARAVIRT
-struct static_key paravirt_steal_rq_enabled;
+DEFINE_STATIC_KEY_FALSE(paravirt_steal_rq_enabled);
#endif
static void update_rq_clock_task(struct rq *rq, s64 delta)
@@ -844,7 +844,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta)
}
#endif
#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
- if (static_key_false((&paravirt_steal_rq_enabled))) {
+ if (static_branch_unlikely(&paravirt_steal_rq_enabled)) {
u64 prev_steal;
steal = prev_steal = paravirt_steal_clock(cpu_of(rq));
@@ -1196,6 +1196,7 @@ static void __resched_curr(struct rq *rq, int tif)
{
struct task_struct *curr = rq->curr;
struct thread_info *cti = task_thread_info(curr);
+ bool need_ipi;
int cpu;
lockdep_assert_rq_held(rq);
@@ -1212,15 +1213,17 @@ static void __resched_curr(struct rq *rq, int tif)
cpu = cpu_of(rq);
- trace_sched_set_need_resched_tp(curr, cpu, tif);
if (cpu == smp_processor_id()) {
set_ti_thread_flag(cti, tif);
if (tif == TIF_NEED_RESCHED)
set_preempt_need_resched();
+ trace_sched_set_need_resched_tp(curr, cpu, tif);
return;
}
- if (set_nr_and_not_polling(cti, tif)) {
+ need_ipi = set_nr_and_not_polling(cti, tif);
+ trace_sched_set_need_resched_tp(curr, cpu, tif);
+ if (need_ipi) {
if (tif == TIF_NEED_RESCHED)
smp_send_reschedule(cpu);
} else {
@@ -2252,7 +2255,8 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
dequeue_task(rq, p, flags);
}
-static void block_task(struct rq *rq, struct task_struct *p, unsigned long task_state)
+static bool dequeue_block_task(struct rq *rq, struct task_struct *p,
+ unsigned long task_state)
{
int flags = DEQUEUE_NOCLOCK;
@@ -2273,9 +2277,15 @@ static void block_task(struct rq *rq, struct task_struct *p, unsigned long task_
*
* Where __schedule() and ttwu() have matching control dependencies.
*
- * After this, schedule() must not care about p->state any more.
+ * Once the caller invokes __block_task(), schedule() must not care about
+ * p->state any more.
*/
- if (dequeue_task(rq, p, DEQUEUE_SLEEP | flags))
+ return dequeue_task(rq, p, DEQUEUE_SLEEP | flags);
+}
+
+static void block_task(struct rq *rq, struct task_struct *p, unsigned long task_state)
+{
+ if (dequeue_block_task(rq, p, task_state))
__block_task(rq, p);
}
@@ -2504,6 +2514,24 @@ static inline bool rq_has_pinned_tasks(struct rq *rq)
return rq->nr_pinned;
}
+static inline bool task_can_migrate_to_preferred(struct task_struct *p, int cpu)
+{
+ /* No need to migrate from a preferred CPU */
+ if (cpu_preferred(cpu))
+ return false;
+
+ /* Only FAIR tasks honor preferred CPU state */
+ if (unlikely(p->sched_class != &fair_sched_class))
+ return false;
+
+ /* Ignore preferred state if task affinity is changing */
+ if (unlikely(!cpumask_test_cpu(task_cpu(p), p->cpus_ptr)))
+ return false;
+
+ return cpumask_intersects_and(p->cpus_ptr, cpu_preferred_mask,
+ task_cpu_possible_mask(p));
+}
+
/*
* Per-CPU kthreads are allowed to run on !active && online CPUs, see
* __set_cpus_allowed_ptr() and select_fallback_rq().
@@ -2519,8 +2547,12 @@ static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
return cpu_online(cpu);
/* Non kernel threads are not allowed during either online or offline. */
- if (!(p->flags & PF_KTHREAD))
+ if (!(p->flags & PF_KTHREAD)) {
+ /* Try to use preferred CPU if task's affinity allows */
+ if (task_can_migrate_to_preferred(p, cpu))
+ return false;
return cpu_active(cpu);
+ }
/* KTHREAD_IS_PER_CPU is always allowed. */
if (kthread_is_per_cpu(p))
@@ -2530,7 +2562,11 @@ static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
if (cpu_dying(cpu))
return false;
- /* But are allowed during online. */
+ /* Try to keep unbound kthreads on a preferred CPU if possible. */
+ if (task_can_migrate_to_preferred(p, cpu))
+ return false;
+
+ /* Otherwise, they are allowed to run on online CPU. */
return cpu_online(cpu);
}
@@ -3773,6 +3809,7 @@ static inline void proxy_reset_donor(struct rq *rq)
WARN_ON_ONCE(rq->donor == rq->curr);
put_prev_set_next_task(rq, rq->donor, rq->curr);
+ rq->next_class = rq->curr->sched_class;
rq_set_donor(rq, rq->curr);
zap_balance_callbacks(rq);
resched_curr(rq);
@@ -3787,6 +3824,8 @@ static inline void proxy_reset_donor(struct rq *rq)
*/
static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p)
{
+ bool dequeued;
+
/*
* Typically per __set_task_cpu(), task_cpu(p) == p->wake_cpu.
*
@@ -3809,12 +3848,23 @@ static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p)
/* If already current, don't need to return migrate */
if (task_current(rq, p))
return false;
-
- /* If we're return migrating the rq->donor, switch it out for idle */
- if (task_current_donor(rq, p))
- proxy_reset_donor(rq);
}
- block_task(rq, p, TASK_WAKING);
+
+ dequeued = dequeue_block_task(rq, p, TASK_WAKING);
+
+ /*
+ * Dequeue @p from its scheduling class before resetting rq->donor.
+ * In particular, sched_ext needs to end the donor's running session
+ * and clear SCX_TASK_QUEUED before put_prev_task_scx() is called by
+ * proxy_reset_donor(); otherwise it would reenqueue the blocked donor.
+ *
+ * Keep on_rq set until all donor references have been replaced.
+ */
+ if (task_current_donor(rq, p))
+ proxy_reset_donor(rq);
+
+ if (dequeued)
+ __block_task(rq, p);
return true;
}
#else /* !CONFIG_SCHED_PROXY_EXEC */
@@ -3905,7 +3955,7 @@ static int ttwu_runnable(struct task_struct *p, int wake_flags)
* When on_rq && !on_cpu the task is preempted, see if
* it should preempt the task that is current now.
*/
- wakeup_preempt(rq, p, wake_flags);
+ wakeup_preempt(rq, p, wake_flags | WF_TTWU_RQ);
}
ttwu_do_wakeup(p);
return 1;
@@ -5149,7 +5199,7 @@ static void do_balance_callbacks(struct rq *rq, struct balance_callback *head)
lockdep_assert_rq_held(rq);
while (head) {
- func = (void (*)(struct rq *))head->func;
+ func = head->func;
next = head->next;
head->next = NULL;
head = next;
@@ -5588,15 +5638,12 @@ unsigned long long nr_context_switches(void)
}
/*
- * Consumers of these two interfaces, like for example the cpuidle menu
- * governor, are using nonsensical data. Preferring shallow idle state selection
- * for a CPU that has IO-wait which might not even end up running the task when
- * it does become runnable.
+ * Return whether any tasks are accounted as I/O waiters on this CPU.
+ * They may wake up on a different CPU.
*/
-
-unsigned int nr_iowait_cpu(int cpu)
+bool cpu_has_iowait(int cpu)
{
- return atomic_read(&cpu_rq(cpu)->nr_iowait);
+ return atomic_read(&cpu_rq(cpu)->nr_iowait) != 0;
}
/*
@@ -5634,7 +5681,7 @@ unsigned int nr_iowait(void)
unsigned int i, sum = 0;
for_each_possible_cpu(i)
- sum += nr_iowait_cpu(i);
+ sum += atomic_read(&cpu_rq(i)->nr_iowait);
return sum;
}
@@ -5789,6 +5836,9 @@ void sched_tick(void)
unsigned long hw_pressure;
u64 resched_latency;
+ if (!cpu_preferred(cpu))
+ sched_push_current_non_preferred_cpu(rq);
+
if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE))
arch_scale_freq_tick();
@@ -5798,7 +5848,7 @@ void sched_tick(void)
curr = rq->curr;
donor = rq->donor;
- psi_account_irqtime(rq, donor, NULL);
+ psi_account_irqtime(rq, curr, NULL);
update_rq_clock(rq);
hw_pressure = arch_scale_hw_pressure(cpu_of(rq));
@@ -6283,10 +6333,7 @@ pick_next_task(struct rq *rq, struct rq_flags *rf)
* selection. In this case, do a core-wide selection.
*/
if (rq->core->core_pick_seq == rq->core->core_task_seq &&
- rq->core->core_pick_seq != rq->core_sched_seq &&
rq->core_pick) {
- WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq);
-
next = rq->core_pick;
rq->dl_server = rq->core_dl_server;
rq->core_pick = NULL;
@@ -6318,11 +6365,13 @@ restart:
}
/*
- * core->core_task_seq, core->core_pick_seq, rq->core_sched_seq
+ * core->core_task_seq, core->core_pick_seq
*
* @task_seq guards the task state ({en,de}queues)
* @pick_seq is the @task_seq we did a selection on
- * @sched_seq is the @pick_seq we scheduled
+ *
+ * Once a core-wide selection is committed, a non-NULL core_pick denotes
+ * a pick which still needs to be consumed on this CPU.
*
* However, preemptions can cause multiple picks on the same task set.
* 'Fix' this by also increasing @task_seq for every pick.
@@ -6429,7 +6478,6 @@ restart:
rq->core->core_pick_seq = rq->core->core_task_seq;
next = rq->core_pick;
- rq->core_sched_seq = rq->core->core_pick_seq;
/* Something should have been selected for current CPU */
WARN_ON_ONCE(!next);
@@ -6517,7 +6565,10 @@ static bool try_steal_cookie(int this, int that)
return false;
do {
- if (p == src->core_pick || p == src->curr)
+ if (p == src->core_pick || p == src->curr || p == src->donor)
+ goto next;
+
+ if (task_is_blocked(p))
goto next;
if (!is_cpu_allowed(p, this))
@@ -6820,6 +6871,34 @@ static void proxy_deactivate(struct rq *rq, struct task_struct *donor)
block_task(rq, donor, state);
}
+/*
+ * Remove a retained proxy donor before changing its scheduler ownership.
+ * The caller holds p->pi_lock, so p cannot wake and migrate if block_task()
+ * drops it from the runqueue. If DELAY_DEQUEUE keeps a blocked fair task
+ * queued, switching_from_fair() completes the dequeue in the immediately
+ * following sched_change_begin().
+ */
+void sched_proxy_block_task(struct rq *rq, struct task_struct *p)
+{
+ unsigned long state = READ_ONCE(p->__state);
+
+ lockdep_assert_held(&p->pi_lock);
+ lockdep_assert_rq_held(rq);
+
+ if (!p->is_blocked || !task_on_rq_queued(p))
+ return;
+ if (WARN_ON_ONCE(state == TASK_RUNNING))
+ return;
+
+ if (task_current_donor(rq, p))
+ proxy_reset_donor(rq);
+
+ if (!p->se.sched_delayed)
+ block_task(rq, p, state);
+
+ WARN_ON_ONCE(task_on_rq_queued(p) && !p->se.sched_delayed);
+}
+
static inline void proxy_release_rq_lock(struct rq *rq, struct rq_flags *rf)
__releases(__rq_lockp(rq))
{
@@ -6865,9 +6944,9 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf,
__must_hold(__rq_lockp(rq))
{
struct rq *target_rq = cpu_rq(target_cpu);
+ LIST_HEAD(migrate_list);
lockdep_assert_rq_held(rq);
- WARN_ON(p == rq->curr);
/*
* Since we are migrating a blocked donor, it could be rq->donor,
* and we want to make sure there aren't any references from this
@@ -6880,13 +6959,20 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf,
* before we release the lock.
*/
proxy_resched_idle(rq);
-
- deactivate_task(rq, p, DEQUEUE_NOCLOCK);
- proxy_set_task_cpu(p, target_cpu);
-
+ for (; p; p = p->blocked_donor) {
+ WARN_ON(p == rq->curr);
+ deactivate_task(rq, p, DEQUEUE_NOCLOCK);
+ proxy_set_task_cpu(p, target_cpu);
+ /*
+ * We can re-use se.group_node to migrate the thing,
+ * because @p is deactivated (won't be balanced) and
+ * we hold the rq_lock.
+ */
+ list_add(&p->se.group_node, &migrate_list);
+ }
proxy_release_rq_lock(rq, rf);
- attach_one_task(target_rq, p);
+ __attach_tasks(target_rq, &migrate_list);
proxy_reacquire_rq_lock(rq, rf);
}
@@ -6979,7 +7065,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
if (!READ_ONCE(owner->on_rq) || owner->se.sched_delayed) {
/* XXX Don't handle blocked owners/delayed dequeue yet */
if (curr_in_chain)
- return proxy_resched_idle(rq);
+ goto resched_idle;
__clear_task_blocked_on(p, NULL);
goto deactivate;
}
@@ -6991,7 +7077,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
* and leave that CPU to sort things out.
*/
if (curr_in_chain)
- return proxy_resched_idle(rq);
+ goto resched_idle;
goto migrate_task;
}
@@ -7004,7 +7090,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
* case we should end up back in find_proxy_task(), this time
* hopefully with all relevant tasks already enqueued.
*/
- return proxy_resched_idle(rq);
+ goto resched_idle;
}
/*
@@ -7041,7 +7127,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
* So schedule rq->idle so that ttwu_runnable() can get the rq
* lock and mark owner as running.
*/
- return proxy_resched_idle(rq);
+ goto resched_idle;
}
/*
* OK, now we're absolutely sure @owner is on this
@@ -7051,8 +7137,18 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
owner->blocked_donor = p;
}
WARN_ON_ONCE(owner && !owner->on_rq);
+
+ if (owner && !sched_cpu_cookie_match(rq, owner)) {
+ if (curr_in_chain)
+ return proxy_resched_idle(rq);
+ p = donor; /* Deactivate the donor, not the runnable owner */
+ clear_task_blocked_on(p, NULL);
+ goto deactivate;
+ }
return owner;
+resched_idle:
+ return proxy_resched_idle(rq);
deactivate:
proxy_deactivate(rq, p);
return NULL;
@@ -7184,13 +7280,12 @@ static void __sched notrace __schedule(int sched_mode)
}
} else if (!preempt && prev_state) {
/*
- * We pass task_is_blocked() as the should_block arg
- * in order to keep mutex-blocked tasks on the runqueue
- * for slection with proxy-exec (without proxy-exec
- * task_is_blocked() will always be false).
+ * Keep mutex-blocked tasks on the runqueue for proxy execution
+ * only when their scheduling class allows it. Without proxy
+ * execution, task_is_blocked() always returns false.
*/
try_to_block_task(rq, prev, &prev_state,
- !task_is_blocked(prev));
+ !task_is_blocked(prev) || !scx_allow_proxy_exec(prev));
switch_count = &prev->nvcsw;
}
@@ -7211,6 +7306,7 @@ pick_again:
}
if (next == rq->idle) {
zap_balance_callbacks(rq);
+ scx_proxy_reenqueue_retry(rq, next);
goto keep_resched;
}
}
@@ -7229,8 +7325,10 @@ pick_again:
* on_cpu.
*/
donor->sched_class->put_prev_task(rq, donor, donor);
- donor->sched_class->set_next_task(rq, donor, true);
+ donor->sched_class->set_next_task(rq, donor, SNT_PICK);
}
+ scx_proxy_donor_start(rq);
+ scx_proxy_reenqueue_retry(rq, next);
} else {
rq_set_donor(rq, next);
}
@@ -7489,27 +7587,6 @@ asmlinkage __visible void __sched notrace preempt_schedule(void)
NOKPROBE_SYMBOL(preempt_schedule);
EXPORT_SYMBOL(preempt_schedule);
-#ifdef CONFIG_PREEMPT_DYNAMIC
-# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL
-# ifndef preempt_schedule_dynamic_enabled
-# define preempt_schedule_dynamic_enabled preempt_schedule
-# define preempt_schedule_dynamic_disabled NULL
-# endif
-DEFINE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled);
-EXPORT_STATIC_CALL_TRAMP(preempt_schedule);
-# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
-static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule);
-void __sched notrace dynamic_preempt_schedule(void)
-{
- if (!static_branch_unlikely(&sk_dynamic_preempt_schedule))
- return;
- preempt_schedule();
-}
-NOKPROBE_SYMBOL(dynamic_preempt_schedule);
-EXPORT_SYMBOL(dynamic_preempt_schedule);
-# endif
-#endif /* CONFIG_PREEMPT_DYNAMIC */
-
/**
* preempt_schedule_notrace - preempt_schedule called by tracing
*
@@ -7562,27 +7639,6 @@ asmlinkage __visible void __sched notrace preempt_schedule_notrace(void)
}
EXPORT_SYMBOL_GPL(preempt_schedule_notrace);
-#ifdef CONFIG_PREEMPT_DYNAMIC
-# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
-# ifndef preempt_schedule_notrace_dynamic_enabled
-# define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace
-# define preempt_schedule_notrace_dynamic_disabled NULL
-# endif
-DEFINE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled);
-EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace);
-# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
-static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule_notrace);
-void __sched notrace dynamic_preempt_schedule_notrace(void)
-{
- if (!static_branch_unlikely(&sk_dynamic_preempt_schedule_notrace))
- return;
- preempt_schedule_notrace();
-}
-NOKPROBE_SYMBOL(dynamic_preempt_schedule_notrace);
-EXPORT_SYMBOL(dynamic_preempt_schedule_notrace);
-# endif
-#endif
-
#endif /* CONFIG_PREEMPTION */
/*
@@ -7799,7 +7855,7 @@ out_unlock:
}
#endif /* CONFIG_RT_MUTEXES */
-#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC)
+#if !defined(CONFIG_PREEMPTION)
int __sched __cond_resched(void)
{
if (should_resched(0) && !irqs_disabled()) {
@@ -7827,38 +7883,6 @@ int __sched __cond_resched(void)
EXPORT_SYMBOL(__cond_resched);
#endif
-#ifdef CONFIG_PREEMPT_DYNAMIC
-# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL
-# define cond_resched_dynamic_enabled __cond_resched
-# define cond_resched_dynamic_disabled ((void *)&__static_call_return0)
-DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched);
-EXPORT_STATIC_CALL_TRAMP(cond_resched);
-
-# define might_resched_dynamic_enabled __cond_resched
-# define might_resched_dynamic_disabled ((void *)&__static_call_return0)
-DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched);
-EXPORT_STATIC_CALL_TRAMP(might_resched);
-# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
-static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched);
-int __sched dynamic_cond_resched(void)
-{
- if (!static_branch_unlikely(&sk_dynamic_cond_resched))
- return 0;
- return __cond_resched();
-}
-EXPORT_SYMBOL(dynamic_cond_resched);
-
-static DEFINE_STATIC_KEY_FALSE(sk_dynamic_might_resched);
-int __sched dynamic_might_resched(void)
-{
- if (!static_branch_unlikely(&sk_dynamic_might_resched))
- return 0;
- return __cond_resched();
-}
-EXPORT_SYMBOL(dynamic_might_resched);
-# endif
-#endif /* CONFIG_PREEMPT_DYNAMIC */
-
/*
* __cond_resched_lock() - if a reschedule is pending, drop the given lock,
* call schedule, and on return reacquire the lock.
@@ -7928,50 +7952,21 @@ EXPORT_SYMBOL(__cond_resched_rwlock_write);
# endif
/*
- * SC:cond_resched
- * SC:might_resched
- * SC:preempt_schedule
- * SC:preempt_schedule_notrace
- * SC:irqentry_exit_cond_resched
- *
- *
* NONE:
- * cond_resched <- __cond_resched
- * might_resched <- RET0
- * preempt_schedule <- NOP
- * preempt_schedule_notrace <- NOP
- * irqentry_exit_cond_resched <- NOP
- * dynamic_preempt_lazy <- false
+ * (unselectable)
*
* VOLUNTARY:
- * cond_resched <- __cond_resched
- * might_resched <- __cond_resched
- * preempt_schedule <- NOP
- * preempt_schedule_notrace <- NOP
- * irqentry_exit_cond_resched <- NOP
- * dynamic_preempt_lazy <- false
+ * (unselectable)
*
* FULL:
- * cond_resched <- RET0
- * might_resched <- RET0
- * preempt_schedule <- preempt_schedule
- * preempt_schedule_notrace <- preempt_schedule_notrace
- * irqentry_exit_cond_resched <- irqentry_exit_cond_resched
* dynamic_preempt_lazy <- false
*
* LAZY:
- * cond_resched <- RET0
- * might_resched <- RET0
- * preempt_schedule <- preempt_schedule
- * preempt_schedule_notrace <- preempt_schedule_notrace
- * irqentry_exit_cond_resched <- irqentry_exit_cond_resched
* dynamic_preempt_lazy <- true
*/
enum {
preempt_dynamic_undefined = -1,
- preempt_dynamic_none,
- preempt_dynamic_voluntary,
preempt_dynamic_full,
preempt_dynamic_lazy,
};
@@ -7980,21 +7975,11 @@ int preempt_dynamic_mode = preempt_dynamic_undefined;
int sched_dynamic_mode(const char *str)
{
-# if !(defined(CONFIG_PREEMPT_RT) || defined(CONFIG_ARCH_HAS_PREEMPT_LAZY))
- if (!strcmp(str, "none"))
- return preempt_dynamic_none;
-
- if (!strcmp(str, "voluntary"))
- return preempt_dynamic_voluntary;
-# endif
-
if (!strcmp(str, "full"))
return preempt_dynamic_full;
-# ifdef CONFIG_ARCH_HAS_PREEMPT_LAZY
if (!strcmp(str, "lazy"))
return preempt_dynamic_lazy;
-# endif
return -EINVAL;
}
@@ -8002,71 +7987,18 @@ int sched_dynamic_mode(const char *str)
# define preempt_dynamic_key_enable(f) static_key_enable(&sk_dynamic_##f.key)
# define preempt_dynamic_key_disable(f) static_key_disable(&sk_dynamic_##f.key)
-# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
-# define preempt_dynamic_enable(f) static_call_update(f, f##_dynamic_enabled)
-# define preempt_dynamic_disable(f) static_call_update(f, f##_dynamic_disabled)
-# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
-# define preempt_dynamic_enable(f) preempt_dynamic_key_enable(f)
-# define preempt_dynamic_disable(f) preempt_dynamic_key_disable(f)
-# else
-# error "Unsupported PREEMPT_DYNAMIC mechanism"
-# endif
-
static DEFINE_MUTEX(sched_dynamic_mutex);
static void __sched_dynamic_update(int mode)
{
- /*
- * Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in
- * the ZERO state, which is invalid.
- */
- preempt_dynamic_enable(cond_resched);
- preempt_dynamic_enable(might_resched);
- preempt_dynamic_enable(preempt_schedule);
- preempt_dynamic_enable(preempt_schedule_notrace);
- preempt_dynamic_enable(irqentry_exit_cond_resched);
- preempt_dynamic_key_disable(preempt_lazy);
-
switch (mode) {
- case preempt_dynamic_none:
- preempt_dynamic_enable(cond_resched);
- preempt_dynamic_disable(might_resched);
- preempt_dynamic_disable(preempt_schedule);
- preempt_dynamic_disable(preempt_schedule_notrace);
- preempt_dynamic_disable(irqentry_exit_cond_resched);
- preempt_dynamic_key_disable(preempt_lazy);
- if (mode != preempt_dynamic_mode)
- pr_info("Dynamic Preempt: none\n");
- break;
-
- case preempt_dynamic_voluntary:
- preempt_dynamic_enable(cond_resched);
- preempt_dynamic_enable(might_resched);
- preempt_dynamic_disable(preempt_schedule);
- preempt_dynamic_disable(preempt_schedule_notrace);
- preempt_dynamic_disable(irqentry_exit_cond_resched);
- preempt_dynamic_key_disable(preempt_lazy);
- if (mode != preempt_dynamic_mode)
- pr_info("Dynamic Preempt: voluntary\n");
- break;
-
case preempt_dynamic_full:
- preempt_dynamic_disable(cond_resched);
- preempt_dynamic_disable(might_resched);
- preempt_dynamic_enable(preempt_schedule);
- preempt_dynamic_enable(preempt_schedule_notrace);
- preempt_dynamic_enable(irqentry_exit_cond_resched);
preempt_dynamic_key_disable(preempt_lazy);
if (mode != preempt_dynamic_mode)
pr_info("Dynamic Preempt: full\n");
break;
case preempt_dynamic_lazy:
- preempt_dynamic_disable(cond_resched);
- preempt_dynamic_disable(might_resched);
- preempt_dynamic_enable(preempt_schedule);
- preempt_dynamic_enable(preempt_schedule_notrace);
- preempt_dynamic_enable(irqentry_exit_cond_resched);
preempt_dynamic_key_enable(preempt_lazy);
if (mode != preempt_dynamic_mode)
pr_info("Dynamic Preempt: lazy\n");
@@ -8099,11 +8031,7 @@ __setup("preempt=", setup_preempt_mode);
static void __init preempt_dynamic_init(void)
{
if (preempt_dynamic_mode == preempt_dynamic_undefined) {
- if (IS_ENABLED(CONFIG_PREEMPT_NONE)) {
- sched_dynamic_update(preempt_dynamic_none);
- } else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) {
- sched_dynamic_update(preempt_dynamic_voluntary);
- } else if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) {
+ if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) {
sched_dynamic_update(preempt_dynamic_lazy);
} else {
/* Default static call setting, nothing to do */
@@ -8123,8 +8051,6 @@ static void __init preempt_dynamic_init(void)
} \
EXPORT_SYMBOL_GPL(preempt_model_##mode)
-PREEMPT_MODEL_ACCESSOR(none);
-PREEMPT_MODEL_ACCESSOR(voluntary);
PREEMPT_MODEL_ACCESSOR(full);
PREEMPT_MODEL_ACCESSOR(lazy);
@@ -8137,7 +8063,7 @@ static inline void preempt_dynamic_init(void) { }
#endif /* CONFIG_PREEMPT_DYNAMIC */
const char *preempt_modes[] = {
- "none", "voluntary", "full", "lazy", NULL,
+ "full", "lazy", NULL,
};
const char *preempt_model_str(void)
@@ -8759,6 +8685,9 @@ int sched_cpu_activate(unsigned int cpu)
*/
sched_set_rq_online(rq, cpu);
+ /* preferred is subset of active and follows its state */
+ set_cpu_preferred(cpu, true);
+
return 0;
}
@@ -8772,6 +8701,8 @@ int sched_cpu_deactivate(unsigned int cpu)
if (ret)
return ret;
+ set_cpu_preferred(cpu, false);
+
/*
* Remove CPU from nohz.idle_cpus_mask to prevent participating in
* load balancing when not active
@@ -11327,18 +11258,17 @@ void sched_change_end(struct sched_change_ctx *ctx)
if (p->sched_class->switched_to)
p->sched_class->switched_to(rq, p);
+ /*
+ * Check if the task can preempt the current in case the class
+ * was upgraded. Note that none of the switch*_from() methods
+ * know the new class and none of the switch*_to() methods know
+ * the old class so we must resched here.
+ */
+ if (ctx->queued)
+ wakeup_preempt(rq, p, 0);
+
if (ctx->running) {
/*
- * If this was a class promotion; let the old class
- * know it got preempted. Note that none of the
- * switch*_from() methods know the new class and none
- * of the switch*_to() methods know the old class.
- */
- if (sched_class_above(p->sched_class, ctx->class)) {
- rq->next_class->wakeup_preempt(rq, p, 0);
- rq->next_class = p->sched_class;
- }
- /*
* If this was a degradation in class; make sure to
* reschedule.
*/
@@ -11349,3 +11279,88 @@ void sched_change_end(struct sched_change_ctx *ctx)
p->sched_class->prio_changed(rq, p, ctx->prio);
}
}
+
+#ifdef CONFIG_PREFERRED_CPU
+static DEFINE_PER_CPU(struct cpu_stop_work, npc_push_task_work);
+
+static int sched_non_preferred_cpu_push_stop(void *arg)
+{
+ struct task_struct *p = arg;
+ struct rq *rq = this_rq();
+ struct rq_flags rf;
+ int cpu;
+
+ if (cpu_preferred(rq->cpu)) {
+ scoped_guard(rq_lock_irqsave, rq)
+ rq->npc_push_work_pending = false;
+ put_task_struct(p);
+ return 0;
+ }
+
+ scoped_guard (raw_spinlock_irq, &p->pi_lock) {
+ /*
+ * select_fallback_rq() may acquire the rq lock in case of
+ * fallback. So call it before grabbing rq lock. If the task
+ * migrates to another CPU before the rq lock is acquired,
+ * subsequent validation of task's current rq will help to
+ * safely bail out.
+ */
+ cpu = select_fallback_rq(rq->cpu, p);
+ context_unsafe_alias(rq);
+ rq_lock(rq, &rf);
+ rq->npc_push_work_pending = false;
+ update_rq_clock(rq);
+
+ if (task_rq(p) == rq && task_on_rq_queued(p)) {
+ struct rq *dest_rq = __migrate_task(rq, &rf, p, cpu);
+
+ if (rq != dest_rq)
+ schedstat_inc(p->stats.nr_migrations_cpu_non_preferred);
+ rq = dest_rq;
+ }
+ rq_unlock(rq, &rf);
+ }
+
+ put_task_struct(p);
+ return 0;
+}
+
+/*
+ * Push the current task running on non-preferred CPU(npc).
+ * Using this non preferred CPU will lead to more contention
+ * in the host. So it is better not to use this CPU.
+ *
+ * Since task is running, call a stopper to push the task out. This is
+ * similar to how task moves during hotplug. In select_fallback_rq() a
+ * preferred CPU will be chosen and henceforth task shouldn't come back to
+ * this CPU again.
+ *
+ * Works for FAIR class only.
+ *
+ * If task is affined only on non-preferred CPUs, no point in moving it out.
+ */
+void sched_push_current_non_preferred_cpu(struct rq *rq)
+{
+ struct task_struct *push_task = rq->curr;
+
+ scoped_guard(rq_lock, rq) {
+ /* Push the task if its explicit affinity allows */
+ if (!task_can_migrate_to_preferred(push_task, rq->cpu))
+ return;
+
+ /* There is already a stopper thread. Don't race with it. */
+ if (rq->npc_push_work_pending)
+ return;
+
+ if (is_migration_disabled(push_task))
+ return;
+
+ rq->npc_push_work_pending = true;
+ }
+
+ /* sched_tick runs with interrupts disabled. */
+ get_task_struct(push_task);
+ stop_one_cpu_nowait(rq->cpu, sched_non_preferred_cpu_push_stop,
+ push_task, this_cpu_ptr(&npc_push_task_work));
+}
+#endif
diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c
index 49ccd6f1c185..14405a614991 100644
--- a/kernel/sched/cpufreq_schedutil.c
+++ b/kernel/sched/cpufreq_schedutil.c
@@ -73,6 +73,18 @@ static void sugov_update_rate_limit_us(struct sugov_policy *sg_policy)
(s64)sg_policy->tunables->rate_limit_us * NSEC_PER_USEC;
}
+static void sugov_update_rate_limit_us_sync(struct sugov_policy *sg_policy)
+{
+ /*
+ * Serialize with rate_limit_us_store() to prevent this path from using
+ * a stale rate limit tunable value to set the limit for the given
+ * policy.
+ */
+ guard(mutex)(&sg_policy->tunables->attr_set.update_lock);
+
+ sugov_update_rate_limit_us(sg_policy);
+}
+
static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time)
{
s64 delta_ns;
@@ -861,7 +873,7 @@ static int sugov_start(struct cpufreq_policy *policy)
void (*uu)(struct update_util_data *data, u64 time, unsigned int flags);
unsigned int cpu;
- sugov_update_rate_limit_us(sg_policy);
+ sugov_update_rate_limit_us_sync(sg_policy);
sg_policy->last_freq_update_time = 0;
sg_policy->next_freq = 0;
sg_policy->work_in_progress = false;
diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c
index 06bddaa738e5..9e0fb8633d82 100644
--- a/kernel/sched/cputime.c
+++ b/kernel/sched/cputime.c
@@ -226,9 +226,8 @@ void account_steal_time(u64 cputime)
void account_idle_time(u64 cputime)
{
u64 *cpustat = kcpustat_this_cpu->cpustat;
- struct rq *rq = this_rq();
- if (atomic_read(&rq->nr_iowait) > 0)
+ if (cpu_has_iowait(smp_processor_id()))
cpustat[CPUTIME_IOWAIT] += cputime;
else
cpustat[CPUTIME_IDLE] += cputime;
@@ -255,7 +254,7 @@ void __account_forceidle_time(struct task_struct *p, u64 delta)
* occasion account more time than the calling functions think elapsed.
*/
#ifdef CONFIG_PARAVIRT
-struct static_key paravirt_steal_enabled;
+DEFINE_STATIC_KEY_FALSE(paravirt_steal_enabled);
#ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN
static u64 native_steal_clock(int cpu)
@@ -270,7 +269,7 @@ DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock);
static __always_inline u64 steal_account_process_time(u64 maxtime)
{
#ifdef CONFIG_PARAVIRT
- if (static_key_false(&paravirt_steal_enabled)) {
+ if (static_branch_unlikely(&paravirt_steal_enabled)) {
u64 steal;
steal = paravirt_steal_clock(smp_processor_id());
@@ -381,9 +380,9 @@ void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times)
* softirq as those do not count in task exec_runtime any more.
*/
static void irqtime_account_process_tick(struct task_struct *p, int user_tick,
- int ticks)
+ u64 cputime)
{
- u64 other, cputime = TICK_NSEC * ticks;
+ u64 other;
/*
* When returning from idle, many ticks can get accounted at
@@ -418,7 +417,7 @@ static void irqtime_account_process_tick(struct task_struct *p, int user_tick,
#else /* !CONFIG_IRQ_TIME_ACCOUNTING: */
static inline void irqtime_account_process_tick(struct task_struct *p, int user_tick,
- int nr_ticks) { }
+ u64 cputime) { }
#endif /* !CONFIG_IRQ_TIME_ACCOUNTING */
#if defined(CONFIG_NO_HZ_COMMON) && !defined(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE)
@@ -431,7 +430,7 @@ static void kcpustat_idle_stop(struct kernel_cpustat *kc, u64 now)
if (!kc->idle_elapse)
return;
- iowait = nr_iowait_cpu(smp_processor_id()) > 0;
+ iowait = cpu_has_iowait(smp_processor_id());
delta = now - kc->idle_entrytime;
steal = steal_account_process_time(delta);
@@ -468,18 +467,38 @@ static void kcpustat_idle_start(struct kernel_cpustat *kc, u64 now)
write_seqcount_end(&kc->idle_sleeptime_seq);
}
-void kcpustat_dyntick_stop(u64 now)
+void kcpustat_dyntick_stop(u64 now, u64 tick_start)
{
struct kernel_cpustat *kc = kcpustat_this_cpu;
if (!vtime_generic_enabled_this_cpu()) {
WARN_ON_ONCE(!kc->idle_dyntick);
+ if (kc->idle_tick_period != tick_start) {
+ kc->idle_tick_period = tick_start;
+ kc->idle_tick_overlap = 0;
+ }
+ tick_start = max(tick_start, kc->idle_dyntick_entry);
+ if (!WARN_ON_ONCE(now < tick_start))
+ kc->idle_tick_overlap += now - tick_start;
kcpustat_idle_stop(kc, now);
kc->idle_dyntick = false;
vtime_dyntick_stop();
}
}
+/*
+ * The first tick after dyntick-idle covers a period that the dyntick-idle
+ * accounting may already have accounted in part.
+ */
+static inline u64 kcpustat_tick_overlap(void)
+{
+ struct kernel_cpustat *kc = kcpustat_this_cpu;
+ u64 overlap = kc->idle_tick_overlap;
+
+ kc->idle_tick_overlap = 0;
+ return overlap;
+}
+
void kcpustat_dyntick_start(u64 now)
{
struct kernel_cpustat *kc = kcpustat_this_cpu;
@@ -487,6 +506,7 @@ void kcpustat_dyntick_start(u64 now)
if (!vtime_generic_enabled_this_cpu()) {
vtime_dyntick_start();
kc->idle_dyntick = true;
+ kc->idle_dyntick_entry = now;
kcpustat_idle_start(kc, now);
}
}
@@ -544,17 +564,22 @@ static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx,
u64 kcpustat_field_idle(int cpu)
{
return kcpustat_field_dyntick(cpu, CPUTIME_IDLE,
- !nr_iowait_cpu(cpu), ktime_get());
+ !cpu_has_iowait(cpu), ktime_get());
}
EXPORT_SYMBOL_GPL(kcpustat_field_idle);
u64 kcpustat_field_iowait(int cpu)
{
return kcpustat_field_dyntick(cpu, CPUTIME_IOWAIT,
- nr_iowait_cpu(cpu), ktime_get());
+ cpu_has_iowait(cpu), ktime_get());
}
EXPORT_SYMBOL_GPL(kcpustat_field_iowait);
#else
+static inline u64 kcpustat_tick_overlap(void)
+{
+ return 0;
+}
+
static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx,
bool compute_delta, ktime_t now)
{
@@ -601,7 +626,7 @@ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx,
u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
{
return get_cpu_sleep_time_us(cpu, CPUTIME_IDLE,
- !nr_iowait_cpu(cpu), last_update_time);
+ !cpu_has_iowait(cpu), last_update_time);
}
EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
@@ -625,7 +650,7 @@ EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
{
return get_cpu_sleep_time_us(cpu, CPUTIME_IOWAIT,
- nr_iowait_cpu(cpu), last_update_time);
+ cpu_has_iowait(cpu), last_update_time);
}
EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
@@ -695,12 +720,13 @@ void account_process_tick(struct task_struct *p, int user_tick)
if (kcpustat_idle_dyntick())
return;
+ cputime = TICK_NSEC - kcpustat_tick_overlap();
+
if (irqtime_enabled()) {
- irqtime_account_process_tick(p, user_tick, 1);
+ irqtime_account_process_tick(p, user_tick, cputime);
return;
}
- cputime = TICK_NSEC;
steal = steal_account_process_time(ULONG_MAX);
if (steal >= cputime)
@@ -1109,7 +1135,6 @@ static int kcpustat_field_vtime(u64 *cpustat,
int cpu, u64 *val)
{
struct vtime *vtime = &tsk->vtime;
- struct rq *rq = cpu_rq(cpu);
unsigned int seq;
do {
@@ -1152,11 +1177,11 @@ static int kcpustat_field_vtime(u64 *cpustat,
*val += vtime->gtime + vtime_delta(vtime);
break;
case CPUTIME_IDLE:
- if (state == VTIME_IDLE && !atomic_read(&rq->nr_iowait))
+ if (state == VTIME_IDLE && !cpu_has_iowait(cpu))
*val += vtime_delta(vtime);
break;
case CPUTIME_IOWAIT:
- if (state == VTIME_IDLE && atomic_read(&rq->nr_iowait) > 0)
+ if (state == VTIME_IDLE && cpu_has_iowait(cpu))
*val += vtime_delta(vtime);
break;
default:
@@ -1251,7 +1276,7 @@ static int kcpustat_cpu_fetch_vtime(struct kernel_cpustat *dst,
}
break;
case VTIME_IDLE:
- if (atomic_read(&cpu_rq(cpu)->nr_iowait) > 0)
+ if (cpu_has_iowait(cpu))
cpustat[CPUTIME_IOWAIT] += delta;
else
cpustat[CPUTIME_IDLE] += delta;
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 0663c00c41c0..c0ebdcde5fe5 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -1097,7 +1097,7 @@ static int start_dl_timer(struct sched_dl_entity *dl_se)
* chosen as the deadline is too small, don't even try to
* start the timer in the past!
*/
- if (ktime_us_delta(act, now) < 0)
+ if (ktime_before(act, now))
return 0;
/*
@@ -2773,11 +2773,14 @@ static void start_hrtick_dl(struct rq *rq, struct sched_dl_entity *dl_se)
* DL keeps current in tree, because ->deadline is not typically changed while
* a task is runnable.
*/
-static void set_next_task_dl(struct rq *rq, struct task_struct *p, bool first)
+static void set_next_task_dl(struct rq *rq, struct task_struct *p, enum snt_e type)
{
struct sched_dl_entity *dl_se = &p->dl;
struct dl_rq *dl_rq = &rq->dl;
+ if (type == SNT_REPICK)
+ return;
+
p->se.exec_start = rq_clock_task(rq);
if (on_dl_rq(&p->dl))
update_stats_wait_end_dl(dl_rq, dl_se);
@@ -2788,7 +2791,7 @@ static void set_next_task_dl(struct rq *rq, struct task_struct *p, bool first)
WARN_ON_ONCE(dl_rq->curr);
dl_rq->curr = dl_se;
- if (!first)
+ if (type != SNT_PICK)
return;
if (rq->donor->sched_class != &dl_sched_class)
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 72236db67983..e6a3b516c703 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -73,13 +73,13 @@ static int sched_feat_show(struct seq_file *m, void *v)
#ifdef CONFIG_JUMP_LABEL
-#define jump_label_key__true STATIC_KEY_INIT_TRUE
-#define jump_label_key__false STATIC_KEY_INIT_FALSE
+#define jump_label_key__true { .key_true = STATIC_KEY_TRUE_INIT }
+#define jump_label_key__false { .key_false = STATIC_KEY_FALSE_INIT }
#define SCHED_FEAT(name, enabled) \
jump_label_key__##enabled ,
-struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
+union sched_feat_key sched_feat_keys[__SCHED_FEAT_NR] = {
#include "features.h"
};
@@ -87,12 +87,12 @@ struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
static void sched_feat_disable(int i)
{
- static_key_disable_cpuslocked(&sched_feat_keys[i]);
+ static_branch_disable_cpuslocked(&sched_feat_keys[i].key_true);
}
static void sched_feat_enable(int i)
{
- static_key_enable_cpuslocked(&sched_feat_keys[i]);
+ static_branch_enable_cpuslocked(&sched_feat_keys[i].key_false);
}
#else /* !CONFIG_JUMP_LABEL: */
static void sched_feat_disable(int i) { };
@@ -280,16 +280,10 @@ static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf,
static int sched_dynamic_show(struct seq_file *m, void *v)
{
- int i = (IS_ENABLED(CONFIG_PREEMPT_RT) || IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY)) * 2;
int mode = READ_ONCE(preempt_dynamic_mode);
- int j;
- /* Count entries in NULL terminated preempt_modes */
- for (j = 0; preempt_modes[j]; j++)
- ;
- j -= !IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY);
-
- for (; i < j; i++) {
+ /* Stop at NULL terminator */
+ for (int i = 0; preempt_modes[i]; i++) {
if (mode == i)
seq_puts(m, "(");
seq_puts(m, preempt_modes[i]);
@@ -1446,6 +1440,7 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
P_SCHEDSTAT(nr_failed_migrations_running);
P_SCHEDSTAT(nr_failed_migrations_hot);
P_SCHEDSTAT(nr_forced_migrations);
+ P_SCHEDSTAT(nr_migrations_cpu_non_preferred);
P_SCHEDSTAT(nr_wakeups);
P_SCHEDSTAT(nr_wakeups_sync);
P_SCHEDSTAT(nr_wakeups_migrate);
diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c
index bc4eee5bb4cb..a5330b760ecd 100644
--- a/kernel/sched/ext/cid.c
+++ b/kernel/sched/ext/cid.c
@@ -31,6 +31,7 @@ static struct scx_cid_tables *scx_cid_tables; /* used only during alloc/free */
#define SCX_CID_TOPO_NEG (struct scx_cid_topo) { \
.core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1, \
.node_cid = -1, .node_idx = -1, .shard_cid = -1, .shard_idx = -1, \
+ .cluster_cid = -1, .cluster_idx = -1, \
}
/*
@@ -182,12 +183,14 @@ s32 scx_cid_init(struct scx_sched *sch)
cpumask_var_t to_walk __free(free_cpumask_var) = CPUMASK_VAR_NULL;
cpumask_var_t node_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
cpumask_var_t llc_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
+ cpumask_var_t cluster_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
cpumask_var_t core_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL;
cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL;
struct scx_cid_tables *tbls;
u32 next_cid = 0;
s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0;
+ s32 next_cluster_idx = 0;
s32 next_shard_idx = 0;
u32 shard_size, max_cids;
u32 notopo_in_shard;
@@ -215,6 +218,7 @@ s32 scx_cid_init(struct scx_sched *sch)
if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) ||
!zalloc_cpumask_var(&node_scratch, GFP_KERNEL) ||
!zalloc_cpumask_var(&llc_scratch, GFP_KERNEL) ||
+ !zalloc_cpumask_var(&cluster_scratch, GFP_KERNEL) ||
!zalloc_cpumask_var(&core_scratch, GFP_KERNEL) ||
!zalloc_cpumask_var(&llc_fallback, GFP_KERNEL) ||
!zalloc_cpumask_var(&online_no_topo, GFP_KERNEL))
@@ -256,6 +260,7 @@ s32 scx_cid_init(struct scx_sched *sch)
u32 cores_per_shard, nr_large;
u32 shard_local = 0, cores_in_shard = 0, cids_in_shard = 0;
s32 shard_cid, shard_idx;
+ s32 cluster_cid = -1, cluster_idx = -1;
/* llc_scratch = node_scratch & this llc */
cpumask_and(llc_scratch, node_scratch, llc_mask);
@@ -268,15 +273,32 @@ s32 scx_cid_init(struct scx_sched *sch)
tbls->shard_node[shard_idx] = nid;
while (!cpumask_empty(llc_scratch)) {
- s32 lcpu = cpumask_first(llc_scratch);
- const struct cpumask *sib = topology_sibling_cpumask(lcpu);
- s32 core_cid = next_cid;
- s32 core_idx = next_core_idx++;
- s32 ccpu;
+ const struct cpumask *sib;
+ s32 core_cid, core_idx, lcpu, ccpu;
u32 max_cores, cids_in_core;
- /* core_scratch = llc_scratch & this core */
- cpumask_and(core_scratch, llc_scratch, sib);
+ /*
+ * Take the cores of one cluster before moving
+ * on, so that a cluster is a contiguous cid
+ * range like the core, LLC and node levels.
+ */
+ if (cpumask_empty(cluster_scratch)) {
+ s32 xcpu = cpumask_first(llc_scratch);
+
+ cpumask_or(cluster_scratch, topology_cluster_cpumask(xcpu),
+ topology_sibling_cpumask(xcpu));
+ cpumask_and(cluster_scratch, cluster_scratch, llc_scratch);
+ cluster_cid = next_cid;
+ cluster_idx = next_cluster_idx++;
+ }
+
+ lcpu = cpumask_first(cluster_scratch);
+ sib = topology_sibling_cpumask(lcpu);
+ core_cid = next_cid;
+ core_idx = next_core_idx++;
+
+ /* core_scratch = cluster_scratch & this core */
+ cpumask_and(core_scratch, cluster_scratch, sib);
if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch)))
return -EINVAL;
@@ -316,8 +338,11 @@ s32 scx_cid_init(struct scx_sched *sch)
.node_idx = node_idx,
.shard_cid = shard_cid,
.shard_idx = shard_idx,
+ .cluster_cid = cluster_cid,
+ .cluster_idx = cluster_idx,
};
+ cpumask_clear_cpu(ccpu, cluster_scratch);
cpumask_clear_cpu(ccpu, llc_scratch);
cpumask_clear_cpu(ccpu, node_scratch);
cpumask_clear_cpu(ccpu, to_walk);
@@ -461,8 +486,8 @@ __bpf_kfunc_start_defs();
* starts must be strictly increasing with the first entry 0 and all values <
* num_possible_cpus(). The last shard extends to num_possible_cpus() and no
* shard may span more than SCX_CID_SHARD_MAX_CPUS cids. Topo info
- * (core/LLC/node) is cleared and the shard layout is set from the input. On
- * invalid input, abort the scheduler.
+ * (core/cluster/LLC/node) is cleared and the shard layout is set from the
+ * input. On invalid input, abort the scheduler.
*/
__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid__arena, u32 cpu_to_cid_cnt,
const s32 *shard_start__arena, u32 shard_start_cnt,
@@ -641,6 +666,29 @@ __bpf_kfunc s32 scx_bpf_cid_to_cpu(s32 cid, const struct bpf_prog_aux *aux)
}
/**
+ * scx_bpf_cid_node - Return the NUMA node the given @cid belongs to
+ * @cid: cid to look up
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * Return NUMA_NO_NODE if @cid is invalid or there is no scheduler. Unlike
+ * scx_bpf_cpu_node(), this kfunc is available only with CID support, so BPF
+ * schedulers can use its presence to detect NUMA-aware CID lookups.
+ */
+__bpf_kfunc s32 scx_bpf_cid_node(s32 cid, const struct bpf_prog_aux *aux)
+{
+ struct scx_sched *sch;
+ s32 cpu;
+
+ guard(rcu)();
+
+ sch = scx_prog_sched(aux);
+ if (unlikely(!sch))
+ return NUMA_NO_NODE;
+ cpu = scx_cid_to_cpu(sch, cid);
+ return cpu < 0 ? NUMA_NO_NODE : cpu_to_node(cpu);
+}
+
+/**
* scx_bpf_cpu_to_cid - Return the cid for @cpu
* @cpu: cpu to look up
* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
@@ -912,30 +960,36 @@ bool scx_cmask_empty(const struct scx_cmask *m)
/**
* scx_bpf_cid_topo - Copy out per-cid topology info
* @cid: cid to look up
- * @out__uninit: where to copy the topology info; fully written by this call
+ * @out: where to copy the topology info
+ * @out__sz: size of @out, the program's sizeof(struct scx_cid_topo)
* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
*
- * Fill @out__uninit with the topology info for @cid. Trigger scx_error() if
- * @cid is out of range. If @cid is valid but in the no-topo section, all fields
- * are set to -1. All fields are also set to -1 when no cid tables have been
- * published yet, which a program may observe while racing the root enable.
+ * Fill @out with the topology info for @cid. Trigger scx_error() if @cid is out
+ * of range. If @cid is valid but in the no-topo section, all fields are set to
+ * -1. All fields are also set to -1 when no cid tables have been published yet,
+ * which a program may observe while racing the root enable.
+ *
+ * The program's struct may be older or newer than the kernel's. The smaller of
+ * @out__sz and the kernel's size is copied and the rest of @out is set to -1.
*/
-__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out__uninit,
+__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out, size_t out__sz,
const struct bpf_prog_aux *aux)
{
+ size_t len = min(out__sz, sizeof(*out));
struct scx_cid_topo *topo;
struct scx_sched *sch;
+ /* the error cases and fields the kernel lacks read as -1 */
+ memset(out, 0xff, out__sz);
+
guard(rcu)();
sch = scx_prog_sched(aux);
topo = rcu_dereference(scx_cid_topo);
- if (unlikely(!sch) || !cid_valid(sch, cid) || unlikely(!topo)) {
- *out__uninit = SCX_CID_TOPO_NEG;
+ if (unlikely(!sch) || !cid_valid(sch, cid) || unlikely(!topo))
return;
- }
- *out__uninit = topo[cid];
+ memcpy(out, &topo[cid], len);
}
__bpf_kfunc_end_defs();
@@ -952,6 +1006,7 @@ static const struct btf_kfunc_id_set scx_kfunc_set_init_cids = {
BTF_KFUNCS_START(scx_kfunc_ids_cid)
BTF_ID_FLAGS(func, scx_bpf_cid_to_cpu, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_cid_node, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, scx_bpf_cpu_to_cid, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, scx_bpf_cid_topo, KF_IMPLICIT_ARGS)
BTF_KFUNCS_END(scx_kfunc_ids_cid)
diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h
index 2fe2311a0f99..709afbfb97c6 100644
--- a/kernel/sched/ext/cid.h
+++ b/kernel/sched/ext/cid.h
@@ -13,11 +13,12 @@
* kernel type sized for the maximum NR_CPUS (4k), with verbose helper sequences
* for every op.
*
- * cids give every cpu a dense, topology-ordered id. CPUs sharing a core, LLC or
- * NUMA node get contiguous cid ranges, so a topology unit becomes a (start,
- * length) slice of cid space. Communication can pass a slice instead of a
- * cpumask, and BPF code can process, for example, a u64 word's worth of cids at
- * a time.
+ * cids give every cpu a dense, topology-ordered id. CPUs in each core,
+ * cluster, LLC or NUMA node get contiguous cid ranges, so a topology unit
+ * becomes a (start, length) slice of cid space. A core without a wider cluster
+ * level forms a cluster of its own. Communication can pass a slice
+ * instead of a cpumask, and BPF code can process, for example, a u64 word's
+ * worth of cids at a time.
*
* The mapping is built once at root scheduler enable time by walking the
* topology of online cpus only. Going by online cpus is out of necessity:
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 3219f0da0fe4..9bae826d69f9 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -25,6 +25,43 @@
DEFINE_RAW_SPINLOCK(scx_sched_lock);
/*
+ * Block a retained proxy donor, telling put_prev_task_scx() that the donor is
+ * about to be dequeued and must not be reenqueued.
+ */
+static void scx_block_proxy_donor(struct rq *rq, struct task_struct *p)
+{
+ rq->scx.flags |= SCX_RQ_PROXY_BLOCKING;
+ sched_proxy_block_task(rq, p);
+ rq->scx.flags &= ~SCX_RQ_PROXY_BLOCKING;
+}
+
+bool __scx_allow_proxy_exec(const struct task_struct *p)
+{
+ struct scx_sched *sch;
+
+ if (p->sched_class != &ext_sched_class)
+ return true;
+
+ sch = scx_task_sched(p);
+ return !sch || (sch->ops.flags & SCX_OPS_ENQ_BLOCKED);
+}
+
+/*
+ * End retained proxy execution before changing @p's BPF scheduler ownership.
+ * Called with @p's pi and rq locks held immediately before
+ * sched_change_begin(). The caller must pass DEQUEUE_NOCLOCK so the rq clock
+ * is updated only once.
+ */
+void scx_prepare_task_sched_change(struct task_struct *p)
+{
+ lockdep_assert_held(&p->pi_lock);
+ lockdep_assert_rq_held(task_rq(p));
+
+ update_rq_clock(task_rq(p));
+ scx_block_proxy_donor(task_rq(p), p);
+}
+
+/*
* NOTE: sched_ext is in the process of growing multiple scheduler support and
* scx_root usage is in a transitional state. Naked dereferences are safe if the
* caller is one of the tasks attached to SCX and explicit RCU dereference is
@@ -41,10 +78,21 @@ struct scx_sched __rcu *scx_root;
LIST_HEAD(scx_sched_all);
#ifdef CONFIG_EXT_SUB_SCHED
+static __always_inline int scx_sched_cmpfn(struct rhashtable_compare_arg *arg,
+ const void *ptr)
+{
+ const struct scx_sched *sch = ptr;
+
+ BUILD_BUG_ON(sizeof_field(struct scx_sched, ops.sub_cgroup_id) != sizeof(u64));
+
+ return sch->ops.sub_cgroup_id != *(const u64 *)arg->key;
+}
+
const struct rhashtable_params scx_sched_hash_params = {
.key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id),
.key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id),
.head_offset = offsetof(struct scx_sched, hash_node),
+ .obj_cmpfn = scx_sched_cmpfn,
.insecure_elasticity = true, /* inserted under scx_sched_lock */
};
@@ -52,10 +100,20 @@ struct rhashtable scx_sched_hash;
#endif
/* see SCX_OPS_TID_TO_TASK */
+static __always_inline int scx_tid_cmpfn(struct rhashtable_compare_arg *arg, const void *ptr)
+{
+ const struct sched_ext_entity *scx = ptr;
+
+ BUILD_BUG_ON(sizeof_field(struct sched_ext_entity, tid) != sizeof(u64));
+
+ return scx->tid != *(const u64 *)arg->key;
+}
+
static const struct rhashtable_params scx_tid_hash_params = {
.key_len = sizeof_field(struct sched_ext_entity, tid),
.key_offset = offsetof(struct sched_ext_entity, tid),
.head_offset = offsetof(struct sched_ext_entity, tid_hash_node),
+ .obj_cmpfn = scx_tid_cmpfn,
.insecure_elasticity = true, /* inserted/removed under scx_tasks_lock */
};
static struct rhashtable scx_tid_hash;
@@ -189,10 +247,20 @@ static DEFINE_PER_CPU(struct scx_tid_alloc, scx_tid_alloc);
*/
static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task);
+static __always_inline int dsq_cmpfn(struct rhashtable_compare_arg *arg, const void *ptr)
+{
+ const struct scx_dispatch_q *dsq = ptr;
+
+ BUILD_BUG_ON(sizeof_field(struct scx_dispatch_q, id) != sizeof(u64));
+
+ return dsq->id != *(const u64 *)arg->key;
+}
+
static const struct rhashtable_params dsq_hash_params = {
.key_len = sizeof_field(struct scx_dispatch_q, id),
.key_offset = offsetof(struct scx_dispatch_q, id),
.head_offset = offsetof(struct scx_dispatch_q, hash_node),
+ .obj_cmpfn = dsq_cmpfn,
};
static LLIST_HEAD(dsqs_to_free);
@@ -322,7 +390,6 @@ static const struct sched_class *scx_setscheduler_class(struct task_struct *p)
static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu)
{
-#ifdef CONFIG_EXT_SUB_SCHED
/*
* If @sch is a sub-sched which is bypassing, its tasks should go into
* the bypass DSQs of the nearest ancestor which is not bypassing. The
@@ -334,11 +401,7 @@ static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 c
* are re-enqueued after scx_bypassing() is turned on, guaranteeing that
* all tasks are transferred to the right DSQs.
*/
- while (scx_parent(sch) && scx_bypassing(sch, cpu))
- sch = scx_parent(sch);
-#endif /* CONFIG_EXT_SUB_SCHED */
-
- return scx_bypass_dsq(sch, cpu);
+ return scx_bypass_dsq(scx_exec_sched(sch, cpu), cpu);
}
/**
@@ -374,28 +437,28 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags)
*/
/*
- * If we're in the dispatch path holding rq lock, $curr may or may not
+ * If we're in the dispatch path holding rq lock, $donor may or may not
* be ready depending on whether the on-going dispatch decides to extend
- * $curr's slice. We say yes here and resolve it at the end of dispatch.
+ * $donor's slice. We say yes here and resolve it at the end of dispatch.
* See dispatch_one().
*/
if (rq->scx.flags & SCX_RQ_IN_DISPATCH)
return true;
/*
- * %SCX_ENQ_PREEMPT clears $curr's slice if on SCX and kicks dispatch,
+ * The preemption flags clear $donor's slice if on SCX and kick dispatch,
* so allow it to avoid spuriously triggering reenq on a combined
* PREEMPT|IMMED insertion.
*/
- if (enq_flags & SCX_ENQ_PREEMPT) {
- struct task_struct *curr = rq->curr;
+ if (enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) {
+ struct task_struct *donor = rq->donor;
/*
* A protected slice refuses the preemption and the cpu stays
* occupied. See rq_owned_post_enq().
*/
- return curr->sched_class != &ext_sched_class ||
- likely(!(curr->scx.flags & SCX_TASK_PROTECTED));
+ return donor->sched_class != &ext_sched_class ||
+ likely(!(donor->scx.flags & SCX_TASK_PROTECTED));
}
/*
@@ -447,48 +510,58 @@ static void switch_rq_lock(struct rq *from, struct rq *to)
DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type);
/**
- * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task
+ * scx_fill_cmask_scratch - Build this cpu's arena cmask from @cpumask
+ * @sch: scx_sched whose scratch to fill
+ * @cpumask: cpus to translate into cids
+ *
+ * The scratch lives in BPF-writable arena memory and its header can't be
+ * trusted, so it is rewritten from kernel geometry rather than read. Caller
+ * must hold an rq lock so this cpu is the sole kernel writer for as long as the
+ * returned address is in use.
+ */
+static struct scx_cmask *scx_fill_cmask_scratch(struct scx_sched *sch,
+ const struct cpumask *cpumask)
+{
+ struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch);
+ struct scx_cmask_ref ref;
+
+ scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref);
+ scx_cmask_ref_from_cpumask(&ref, cpumask);
+ return kern_va;
+}
+
+/**
+ * scx_call_op_set_cpumask - Invoke the set_cpumask or set_cmask op for @task
* @sch: scx_sched being invoked
* @rq: rq to update as the currently-locked rq, or NULL
* @task: task whose affinity is changing
* @cpumask: new cpumask
*
- * For cid-form schedulers, translate @cpumask to a cmask via the per-cpu
- * scratch in cid.c and dispatch through the ops_cid union view. Caller
- * must hold @rq's rq lock so this_cpu_ptr is stable across the call.
+ * For cid-form schedulers, translate @cpumask to a cmask in the per-cpu scratch
+ * and dispatch through the ops_cid union view. Caller must hold @rq's rq lock.
*/
static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq,
struct task_struct *task,
const struct cpumask *cpumask)
{
- if (scx_is_cid_type()) {
- struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch);
- struct scx_cmask_ref ref;
-
- /*
- * Build the per-cpu arena cmask from kernel geometry via @ref,
- * never reading its BPF-writable header. set_cmask()'s __arena
- * argument takes the kernel address and the struct_ops
- * trampoline rebases it into BPF's arena pointer form. The rq
- * lock makes this cpu the sole kernel writer.
- */
- scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref);
- scx_cmask_ref_from_cpumask(&ref, cpumask);
- SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, kern_va);
- } else {
+ if (scx_is_cid_type())
+ SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task,
+ scx_fill_cmask_scratch(sch, cpumask));
+ else
SCX_CALL_OP_TASK(sch, set_cpumask, rq, task, cpumask);
- }
}
enum scx_dsq_iter_flags {
/* iterate in the reverse dispatch order */
SCX_DSQ_ITER_REV = 1U << 16,
+ __SCX_DSQ_ITER_HAS_REENQ = 1U << 29,
__SCX_DSQ_ITER_HAS_SLICE = 1U << 30,
__SCX_DSQ_ITER_HAS_VTIME = 1U << 31,
__SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV,
__SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS |
+ __SCX_DSQ_ITER_HAS_REENQ |
__SCX_DSQ_ITER_HAS_SLICE |
__SCX_DSQ_ITER_HAS_VTIME,
};
@@ -1079,6 +1152,65 @@ static void schedule_deferred_locked(struct rq *rq)
schedule_deferred(rq);
}
+#ifdef CONFIG_NO_HZ_FULL
+static void scx_proxy_tick_bal_cb(struct rq *rq)
+{
+ sched_update_tick_dependency(rq);
+}
+
+static void scx_proxy_update_tick(struct rq *rq, struct task_struct *next)
+{
+ bool proxy = next != rq->donor;
+ bool was_proxy = rq->scx.flags & SCX_RQ_PROXY_TICK;
+
+ if (likely(proxy == was_proxy))
+ return;
+
+ if (proxy) {
+ /* Keep proxy execution tick-driven for now. */
+ rq->scx.flags |= SCX_RQ_PROXY_TICK;
+ tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED);
+ } else {
+ rq->scx.flags &= ~SCX_RQ_PROXY_TICK;
+ /*
+ * The selected donor is already visible, but rq->curr still
+ * identifies the outgoing execution context. Reevaluate after
+ * context_switch() updates rq->curr so sched_can_stop_tick() sees
+ * the complete selection.
+ */
+ queue_balance_callback(rq, &rq->scx.proxy_tick_bal_cb,
+ scx_proxy_tick_bal_cb);
+ }
+}
+#endif
+
+/*
+ * Complete sched_ext bookkeeping after proxy resolution and retry tasks which
+ * couldn't be reenqueued by an earlier reject DSQ drain.
+ */
+void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next)
+{
+ lockdep_assert_rq_held(rq);
+
+#ifdef CONFIG_NO_HZ_FULL
+ if (scx_enabled() && tick_nohz_full_cpu(cpu_of(rq)))
+ scx_proxy_update_tick(rq, next);
+#endif
+
+ /*
+ * A higher priority class may wake up while @rq is transiently idle
+ * for proxy resolution. wakeup_preempt_scx() isn't invoked in that
+ * case, so reenqueue IMMED donors parked on the local DSQ here.
+ */
+ if (rq->scx.nr_immed && sched_class_above(rq->next_class, &ext_sched_class))
+ scx_schedule_reenq_local(rq, 0);
+
+ if (rq->scx.flags & SCX_RQ_PROXY_RETRY) {
+ rq->scx.flags &= ~SCX_RQ_PROXY_RETRY;
+ schedule_deferred_locked(rq);
+ }
+}
+
void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq,
u64 reenq_flags, struct rq *locked_rq)
{
@@ -1095,6 +1227,18 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq,
rq = container_of(dsq, struct rq, scx.local_dsq);
/*
+ * An inactive cpu's local DSQ holds only tasks that are bound
+ * to it or are being moved off, so a reenq of it is pointless.
+ * Once balance push is installed, queue_balance_callback() also
+ * drops the callback that would run a reenq requested from
+ * dispatch, and irq_work_queue_on() can't reach an offline cpu.
+ * The reenq's node would then stay on this rq's list after
+ * @sch's pcpu is freed.
+ */
+ if (unlikely(!cpu_active(cpu_of(rq))))
+ return;
+
+ /*
* A sub-sched lacking baseline access on the target cid has no
* business triggering IPIs. The lockless test is fine: slipping
* through right after a revoke is harmless and a wrong denial
@@ -1127,7 +1271,7 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq,
} else if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) {
rq = this_rq();
- struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu, cpu_of(rq));
+ struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu_user, cpu_of(rq));
struct scx_deferred_reenq_user *dru = &dsq_pcpu->deferred_reenq_user;
/*
@@ -1141,12 +1285,18 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq,
guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
+ /* @dsq is being destroyed, see free_dsq_irq_workfn() */
+ if (unlikely(READ_ONCE(dsq->id) == SCX_DSQ_INVALID))
+ return;
+
if (list_empty(&dru->node))
list_move_tail(&dru->node, &rq->scx.deferred_reenq_users);
WRITE_ONCE(dru->flags, dru->flags | reenq_flags);
}
} else {
- scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id);
+ /* destroyed since the caller resolved it, nothing to reenq */
+ if (READ_ONCE(dsq->id) != SCX_DSQ_INVALID)
+ scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id);
return;
}
@@ -1390,20 +1540,27 @@ static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 e
static void update_curr_scx(struct rq *rq)
{
- struct task_struct *curr = rq->curr;
+ struct task_struct *donor;
s64 delta_exec;
+ /*
+ * update_curr_scx() is selected through rq->donor->sched_class, not
+ * rq->curr->sched_class, so @donor is always an EXT task here. If an EXT
+ * owner executes for a FAIR donor, FAIR's update_curr() runs instead.
+ */
+ donor = rq->donor;
+
/* apply even on 0 delta_exec, callers may still act on the slice */
- apply_task_slice_oob(rq, curr);
+ apply_task_slice_oob(rq, donor);
delta_exec = update_curr_common(rq);
if (unlikely(delta_exec <= 0))
return;
- if (curr->scx.slice != SCX_SLICE_INF)
- curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec);
+ if (donor->scx.slice != SCX_SLICE_INF)
+ donor->scx.slice -= min_t(u64, donor->scx.slice, delta_exec);
- if (unlikely(curr == scx_rescuee(rq)))
+ if (unlikely(donor == scx_rescuee(rq)))
scx_rescue_charge(rq, delta_exec);
dl_server_update(&rq->ext_server, delta_exec);
@@ -1499,31 +1656,48 @@ static inline bool task_scx_migrating(struct task_struct *p)
return p->scx.sticky_cpu >= 0;
}
-/*
- * Call ops.dequeue() if the task is in BPF custody and not migrating.
- * Clears %SCX_TASK_IN_CUSTODY when the callback is invoked.
- */
-static void call_task_dequeue(struct scx_sched *sch, struct rq *rq,
- struct task_struct *p, u64 deq_flags)
+/* Must be called under the lock serializing @p's custody transfers. */
+static bool task_leave_custody(struct task_struct *p)
{
if (!(p->scx.flags & SCX_TASK_IN_CUSTODY) || task_scx_migrating(p))
- return;
-
- if (SCX_HAS_OP(sch, dequeue))
- SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags);
+ return false;
p->scx.flags &= ~SCX_TASK_IN_CUSTODY;
+ return true;
+}
+
+/*
+ * A task with an infinite slice may be running with its tick stopped. Lazy
+ * rescheduling doesn't send an IPI, so restore the tick dependency to guarantee
+ * that the lazy request is promoted by a real scheduler tick. Set the lazy
+ * request first so that the dependency update's IPI can also serve it on return
+ * to user space.
+ */
+static void scx_resched_curr_lazy(struct rq *rq)
+{
+ resched_curr_lazy(rq);
+
+ if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) {
+ rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK;
+ if (rq->clock_update_flags < RQCF_UPDATED)
+ update_rq_clock(rq);
+ update_other_load_avgs(rq);
+ sched_update_tick_dependency(rq);
+ }
}
static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq,
struct scx_dispatch_q *dsq, struct task_struct *p,
u64 enq_flags)
{
- call_task_dequeue(sch, rq, p, 0);
+ if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue))
+ SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0);
/*
- * Only local inserts get the wakeup treatment below. Rejects kick the
- * deferred reenq and rescue parks are paced by the rescue timer.
+ * Only local inserts get the wakeup treatment below. Rejects kick a
+ * deferred reenq and rescue parks are paced by the rescue timer. Proxy
+ * rejects which aren't ready when drained request a later retry from
+ * scx_proxy_reenqueue_retry().
*/
if (unlikely(dsq->id != SCX_DSQ_LOCAL)) {
if (dsq->id == SCX_DSQ_REJECT)
@@ -1577,12 +1751,16 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq,
if (rq->scx.flags & SCX_RQ_IN_DISPATCH)
return;
- if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&
- rq->curr->sched_class == &ext_sched_class) {
- if (likely(scx_set_task_slice(rq->curr, 0)))
- resched_curr(rq);
- else
+ if ((enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) &&
+ p != rq->donor && rq->donor->sched_class == &ext_sched_class) {
+ if (likely(scx_set_task_slice(rq->donor, 0))) {
+ if (enq_flags & SCX_ENQ_PREEMPT)
+ resched_curr(rq);
+ else
+ scx_resched_curr_lazy(rq);
+ } else {
__scx_add_event(sch, SCX_EV_SLICE_DENIED, 1);
+ }
}
}
@@ -1590,12 +1768,10 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
struct scx_dispatch_q *dsq, struct task_struct *p,
u64 slice, u64 vtime, u64 enq_flags)
{
- bool is_rq_owned = false;
+ bool is_rq_owned = dsq_is_rq_owned(dsq);
- if (dsq->id == SCX_DSQ_LOCAL) {
+ if (dsq->id == SCX_DSQ_LOCAL)
dsq = scx_resolve_local_dsq(sch, rq, p, &enq_flags);
- is_rq_owned = true;
- }
WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node));
WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) ||
@@ -1672,7 +1848,7 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
scx_error(sch, "DSQ ID 0x%016llx already had PRIQ-enqueued tasks",
dsq->id);
- if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) {
+ if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) {
/* new task inserted at head - use fastpath */
if (dsq_insert_head(dsq, p) && !(dsq->id & SCX_DSQ_FLAG_BUILTIN))
rcu_assign_pointer(dsq->first_task, p);
@@ -1705,20 +1881,28 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
if (is_rq_owned) {
rq_owned_post_enq(sch, rq, dsq, p, enq_flags);
} else {
+ bool call_dequeue = false;
+
/*
* Global and bypass DSQs are terminal - the task leaves the
- * scheduler's custody, so ops.dequeue() fires here. It can run
+ * scheduler's custody, so ops.dequeue() fires. It can run
* without @p's rq lock (finish_dispatch() passes the dispatch
* rq); that's safe because dequeue_task_scx() waits on
* SCX_OPSS_DISPATCHING (see the ops_state note above) and so
* can't race it. A non-terminal DSQ keeps the task in custody.
+ * The custody transfer happens under @dsq->lock so that later
+ * consumers see the flag clear; the callback runs after
+ * @dsq->lock is dropped because it may lock a DSQ itself.
*/
if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS)
- call_task_dequeue(sch, rq, p, 0);
+ call_dequeue = task_leave_custody(p);
else
p->scx.flags |= SCX_TASK_IN_CUSTODY;
raw_spin_unlock(&dsq->lock);
+
+ if (call_dequeue && SCX_HAS_OP(sch, dequeue))
+ SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0);
}
/*
@@ -1738,6 +1922,7 @@ void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq)
RB_CLEAR_NODE(&p->scx.dsq_priq);
p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ;
}
+ p->scx.dsq_flags &= ~SCX_TASK_DSQ_REENQ;
list_del_init(&p->scx.dsq_list.node);
dsq_dec_nr(dsq, p);
@@ -1766,10 +1951,10 @@ void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p)
list_del_init(&p->scx.dsq_list.node);
/*
- * When dispatching directly from the BPF scheduler to a local
- * DSQ, the task isn't associated with any DSQ but
- * @p->scx.holding_cpu may be set under the protection of
- * %SCX_OPSS_DISPATCHING.
+ * When dispatch_to_local_dsq() or remote consumption moves a
+ * task to a local DSQ, the task isn't associated with any DSQ
+ * but @p->scx.holding_cpu may be set. Clearing holding_cpu
+ * tells dispatch_to_local_dsq() that it lost to a dequeue.
*/
if (p->scx.holding_cpu >= 0)
p->scx.holding_cpu = -1;
@@ -1789,10 +1974,10 @@ void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p)
scx_task_unlink_from_dsq(p, dsq);
} else {
/*
- * We're racing against dispatch_to_local_dsq() which already
- * removed @p from @dsq and set @p->scx.holding_cpu. Clear the
- * holding_cpu which tells dispatch_to_local_dsq() that it lost
- * the race.
+ * We're racing against unlink_dsq_and_switch_rq_lock(),
+ * which already removed @p from @dsq and set
+ * @p->scx.holding_cpu. Clear holding_cpu to tell
+ * unlink_dsq_and_switch_rq_lock() that it lost the race.
*/
WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node));
p->scx.holding_cpu = -1;
@@ -1975,6 +2160,12 @@ bool scx_rq_online(struct rq *rq)
void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
int sticky_cpu)
{
+ enum {
+ ENQ_ACTION_NONE,
+ ENQ_ACTION_DIRECT,
+ ENQ_ACTION_LOCAL_NOREFILL,
+ ENQ_ACTION_ENQUEUE,
+ } action = ENQ_ACTION_NONE;
struct scx_sched *sch = scx_task_sched(p);
struct task_struct **ddsp_taskp;
struct scx_dispatch_q *dsq;
@@ -2008,7 +2199,7 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
__scx_exit(sch, SCX_EXIT_ERROR_REENQ, 0, cpu_of(rq),
"%s[%d] reenqueued %u times without running",
p->comm, p->pid, p->scx.reenq_cnt);
- return;
+ goto out;
}
}
@@ -2028,26 +2219,41 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID)
goto direct;
- /* see %SCX_OPS_ENQ_EXITING */
- if (!(sch->ops.flags & SCX_OPS_ENQ_EXITING) &&
- unlikely(p->flags & PF_EXITING)) {
- __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1);
- enq_flags |= SCX_ENQ_RESCUE; /* avoid looping on cap rejection */
- goto local;
- }
+ /*
+ * A full wakeup enqueue happens before is_blocked is cleared. Don't
+ * report it as another blocked-donor admission.
+ */
+ if ((sch->ops.flags & SCX_OPS_ENQ_BLOCKED) && p->is_blocked &&
+ !(enq_flags & SCX_ENQ_WAKEUP)) {
+ enq_flags |= SCX_ENQ_BLOCKED;
+ } else {
+ /* see %SCX_OPS_ENQ_EXITING */
+ if (unlikely(p->flags & PF_EXITING) &&
+ !(sch->ops.flags & SCX_OPS_ENQ_EXITING)) {
+ __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1);
+ enq_flags |= SCX_ENQ_RESCUE; /* avoid looping on cap rejection */
+ goto local;
+ }
- /* see %SCX_OPS_ENQ_MIGRATION_DISABLED */
- if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) &&
- is_migration_disabled(p)) {
- __scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1);
- goto local;
+ /* see %SCX_OPS_ENQ_MIGRATION_DISABLED */
+ if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) &&
+ is_migration_disabled(p)) {
+ __scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1);
+ goto local;
+ }
}
if (unlikely(!SCX_HAS_OP(sch, enqueue)))
goto global;
- /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */
- qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT;
+ /*
+ * DSQ bypass didn't trigger, enqueue on the BPF scheduler. Wrap the
+ * per-task qseq counter where the QSEQ field wraps and skip 0, which is
+ * what scx_bpf_dsq_insert() records for a task in NONE or DISPATCHING.
+ */
+ p->scx.ops_qseq = ((p->scx.ops_qseq + 1) &
+ (SCX_OPSS_QSEQ_MASK >> SCX_OPSS_QSEQ_SHIFT)) ?: 1;
+ qseq = (unsigned long)p->scx.ops_qseq << SCX_OPSS_QSEQ_SHIFT;
WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE);
atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq);
@@ -2073,14 +2279,14 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
* dequeue may be waiting. The store_release matches their load_acquire.
*/
atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq);
- return;
+ goto out;
direct:
- direct_dispatch(sch, p, enq_flags);
- return;
+ action = ENQ_ACTION_DIRECT;
+ goto out;
local_norefill:
- scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, enq_flags);
- return;
+ action = ENQ_ACTION_LOCAL_NOREFILL;
+ goto out;
local:
dsq = &rq->scx.local_dsq;
goto enqueue;
@@ -2092,9 +2298,27 @@ bypass:
goto enqueue;
enqueue:
- refill_task_slice_dfl(sch, p);
- clear_direct_dispatch(p);
- scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags);
+ action = ENQ_ACTION_ENQUEUE;
+out:
+ /* The reason is input to ops.enqueue(), not to the resulting placement. */
+ p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
+
+ switch (action) {
+ case ENQ_ACTION_DIRECT:
+ direct_dispatch(sch, p, enq_flags);
+ break;
+ case ENQ_ACTION_LOCAL_NOREFILL:
+ scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0,
+ enq_flags);
+ break;
+ case ENQ_ACTION_ENQUEUE:
+ refill_task_slice_dfl(sch, p);
+ clear_direct_dispatch(p);
+ scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags);
+ break;
+ case ENQ_ACTION_NONE:
+ break;
+ }
}
static bool task_runnable(const struct task_struct *p)
@@ -2112,8 +2336,8 @@ static void set_task_runnable(struct rq *rq, struct task_struct *p)
}
/*
- * list_add_tail() must be used. scx_bypass() depends on tasks being
- * appended to the runnable_list.
+ * list_add_tail() must be used. scx_bypass() and rq_offline_scx()
+ * depend on tasks being appended to the runnable_list.
*/
list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list);
@@ -2141,17 +2365,30 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_
int sticky_cpu = p->scx.sticky_cpu;
u64 enq_flags = core_enq_flags | rq->scx.remote_activate_enq_flags;
- if (enq_flags & ENQUEUE_WAKEUP)
+ /*
+ * An SCX-internal migration ends on arrival. Clear sticky_cpu so @p can
+ * leave custody when inserted into the destination DSQ.
+ */
+ if (sticky_cpu >= 0)
+ p->scx.sticky_cpu = -1;
+
+ /*
+ * SCX_RQ_IN_WAKEUP promises a task_woken_scx() call once this enqueue
+ * returns. Only the core's wakeup path delivers one. The flags stashed
+ * for a remote activation may carry the wakeup bit without it.
+ */
+ if (core_enq_flags & ENQUEUE_WAKEUP)
rq->scx.flags |= SCX_RQ_IN_WAKEUP;
/*
- * Restoring a running task will be immediately followed by
- * set_next_task_scx() which expects the task to not be on the BPF
+ * Restoring the current scheduling context will be immediately followed
+ * by set_next_task_scx() which expects the task to not be on the BPF
* scheduler as tasks can only start running through local DSQs. Force
* direct-dispatch into the local DSQ by setting the sticky_cpu. Mark
* IGNORE_CAPS to force entry into the local DSQ.
*/
- if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) {
+ if (unlikely(enq_flags & ENQUEUE_RESTORE) &&
+ task_current_donor(rq, p)) {
sticky_cpu = cpu_of(rq);
enq_flags |= SCX_ENQ_IGNORE_CAPS;
}
@@ -2174,9 +2411,6 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_
dl_server_start(&rq->ext_server);
scx_do_enqueue_task(rq, p, enq_flags, sticky_cpu);
-
- if (sticky_cpu >= 0)
- p->scx.sticky_cpu = -1;
out:
rq->scx.flags &= ~SCX_RQ_IN_WAKEUP;
@@ -2210,7 +2444,7 @@ retry:
/*
* A queued task must always be in BPF scheduler's custody. If
* SCX_TASK_IN_CUSTODY is clear, finish_dispatch() on another
- * CPU has already passed call_task_dequeue() (which clears the
+ * CPU has already passed task_leave_custody() (which clears the
* flag), but has not yet written SCX_OPSS_NONE. That final
* store does not require this rq's lock, so retrying with
* cpu_relax() is bounded: we will observe NONE (or DISPATCHING,
@@ -2258,7 +2492,8 @@ retry:
* NONE but the task may still have %SCX_TASK_IN_CUSTODY set until
* it is enqueued on the destination.
*/
- call_task_dequeue(sch, rq, p, deq_flags);
+ if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue))
+ SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags);
}
static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_flags)
@@ -2281,10 +2516,10 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_
ops_dequeue(rq, p, deq_flags);
/*
- * A currently running task which is going off @rq first gets dequeued
- * and then stops running. As we want running <-> stopping transitions
- * to be contained within runnable <-> quiescent transitions, trigger
- * ->stopping() early here instead of in put_prev_task_scx().
+ * A current scheduling context which is going off @rq first gets
+ * dequeued and then stops running. As we want running <-> stopping
+ * transitions to be contained within runnable <-> quiescent transitions,
+ * trigger ->stopping() early here instead of in put_prev_task_scx().
*
* @p may go through multiple stopping <-> running transitions between
* here and put_prev_task_scx() if task attribute changes occur while
@@ -2292,11 +2527,13 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_
* information meaningful to the BPF scheduler and can be suppressed by
* skipping the callbacks if the task is !QUEUED.
*/
- if (task_current(rq, p) &&
- (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq)))) {
- update_curr_scx(rq);
- if (SCX_HAS_OP(sch, stopping))
- SCX_CALL_OP_TASK(sch, stopping, rq, p, false);
+ if (task_current_donor(rq, p) && (p->scx.flags & SCX_TASK_RUN_TRACKED)) {
+ if (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq))) {
+ update_curr_scx(rq);
+ if (SCX_HAS_OP(sch, stopping))
+ SCX_CALL_OP_TASK(sch, stopping, rq, p, false);
+ }
+ p->scx.flags &= ~SCX_TASK_RUN_TRACKED;
}
if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p))
@@ -2312,9 +2549,10 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_
sub_nr_running(rq, 1);
scx_dispatch_dequeue(rq, p);
+ p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
/* see scx_task_slice_ended() for the save/restore exception */
- if (!((deq_flags & DEQUEUE_SAVE) && task_current(rq, p)))
+ if (!((deq_flags & DEQUEUE_SAVE) && task_current_donor(rq, p)))
scx_task_slice_ended(rq, p);
clear_direct_dispatch(p);
@@ -2354,10 +2592,23 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl
/*
* Preemption between SCX tasks is implemented by resetting the victim
* task's slice to 0 and triggering reschedule on the target CPU.
- * Nothing to do.
- */
- if (p->sched_class == &ext_sched_class)
+ *
+ * A retained proxy donor can wake through ttwu_runnable() without another
+ * ops.enqueue(). WF_TTWU_RQ identifies this path. Request rescheduling so
+ * that ops.dispatch() can reconsider the task after ttwu_runnable() clears
+ * is_blocked. A full wakeup activation has already enqueued the task and
+ * doesn't need the additional reschedule.
+ */
+ if (p->sched_class == &ext_sched_class) {
+ if (sched_proxy_exec() && (wake_flags & WF_TTWU_RQ) &&
+ p->is_blocked) {
+ struct scx_sched *sch = scx_task_sched(p);
+
+ if (sch && (sch->ops.flags & SCX_OPS_ENQ_BLOCKED))
+ resched_curr(rq);
+ }
return;
+ }
/*
* Getting preempted by a higher-priority class. Reenqueue IMMED tasks.
@@ -2374,19 +2625,18 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl
}
void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p,
- u64 enq_flags, struct scx_dispatch_q *src_dsq,
- struct rq *dst_rq)
+ u64 enq_flags, struct rq *dst_rq)
{
- struct scx_dispatch_q *dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags);
+ struct scx_dispatch_q *dst_dsq;
- /* @p is on @dst_rq, an rq-owned @src_dsq is covered by the rq lock */
- if (!dsq_is_rq_owned(src_dsq))
- lockdep_assert_held(&src_dsq->lock);
lockdep_assert_rq_held(dst_rq);
WARN_ON_ONCE(p->scx.holding_cpu >= 0);
+ scx_wait_dispatching(p);
+
+ dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags);
- if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT))
+ if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY))
dsq_insert_head(dst_dsq, p);
else
list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list);
@@ -2456,8 +2706,10 @@ static void move_remote_task_to_local_dsq(struct scx_sched *sch,
* - The BPF scheduler is bypassed while the rq is offline and we can always say
* no to the BPF scheduler initiated migrations while offline.
*
- * The caller must ensure that @p and @rq are on different CPUs.
- * If enforce == true, caller must hold @p's rq lock.
+ * The caller must ensure that @p and @rq are on different CPUs. If @enforce is
+ * true, report violations attributable to BPF-directed migrations. The caller
+ * must hold @p's rq lock to avoid reporting a transient race as a scheduler
+ * error.
*/
static bool task_can_run_on_remote_rq(struct scx_sched *sch,
struct task_struct *p, struct rq *rq,
@@ -2465,17 +2717,27 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch,
{
s32 cpu = cpu_of(rq);
- /*
- * To prevent races with @p still running on its old CPU while switching
- * out, make sure we're holding @p's rq lock so as not to risk
- * erroneously killing the BPF scheduler.
- */
if (enforce)
lockdep_assert_rq_held(task_rq(p));
WARN_ON_ONCE(task_cpu(p) == cpu);
/*
+ * proxy_set_task_cpu() preserves wake_cpu when moving a donor's scheduling
+ * context to its lock owner's rq. For example, if BPF places donor D on
+ * CPU0 while its lock owner runs on CPU1, proxy execution moves D to CPU1
+ * but leaves D->wake_cpu pointing to CPU0. If D is later put on a shared
+ * DSQ, CPU0 could otherwise consume it and move it back, only for proxy
+ * execution to return it to CPU1 again.
+ *
+ * When task_cpu() == wake_cpu, the donor has not been proxy-migrated and
+ * BPF may still choose its placement. An actively donating task is
+ * rejected separately by task_proxy_running_or_donating().
+ */
+ if (sched_proxy_exec() && p->is_blocked && task_cpu(p) != p->wake_cpu)
+ return false;
+
+ /*
* If @p has migration disabled, @p->cpus_ptr is updated to contain only
* the pinned CPU in migrate_disable_switch() while @p is being switched
* out. However, put_prev_task_scx() is called before @p->cpus_ptr is
@@ -2516,6 +2778,64 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch,
return true;
}
+/*
+ * Proxy execution can change @p's execution and migration-disabled state
+ * without touching its DSQ entry or clearing holding_cpu. Check those states
+ * with @p's rq locked. Without proxy execution, the holding_cpu handshake is
+ * sufficient and this must not affect the existing migration path.
+ *
+ * A BPF-directed transfer to a remote local DSQ performs a normal task
+ * migration and thus cannot move a migration-disabled task. In contrast,
+ * proxy_migrate_task() moves only a blocked donor's scheduling context towards
+ * the mutex owner and preserves its execution home in wake_cpu. The latter is
+ * therefore allowed even when the donor is migration-disabled.
+ */
+static bool task_proxy_running_or_donating(struct task_struct *p)
+{
+ struct rq *src_rq = task_rq(p);
+
+ lockdep_assert_rq_held(src_rq);
+
+ if (!sched_proxy_exec())
+ return false;
+
+ /* @p may be rq->curr under another task's scheduling context. */
+ if (task_on_cpu(src_rq, p))
+ return true;
+
+ /* Don't move an active scheduling context off its source rq. */
+ if (task_current_donor(src_rq, p))
+ return true;
+
+ return false;
+}
+
+static bool task_proxy_unsafe_to_move(struct task_struct *p)
+{
+ if (!sched_proxy_exec())
+ return false;
+
+ return task_proxy_running_or_donating(p) || is_migration_disabled(p);
+}
+
+/*
+ * Park a task whose remote transfer raced with proxy execution. Reenqueueing
+ * from the source rq makes the task's owning scheduler choose its placement
+ * again and preserves sub-scheduler containment.
+ */
+static void scx_proxy_reject_task(struct scx_sched *sch, struct rq *rq,
+ struct task_struct *p, u64 enq_flags)
+{
+ lockdep_assert_rq_held(rq);
+ WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK);
+
+ p->scx.holding_cpu = -1;
+ p->scx.flags |= SCX_TASK_REENQ_PROXY;
+ scx_divert_strip_flags(p, &enq_flags);
+
+ scx_dispatch_enqueue(sch, rq, &rq->scx.reject_dsq, p, 0, 0, enq_flags);
+}
+
/**
* unlink_dsq_and_switch_rq_lock() - Unlink task and switch to its rq lock
* @p: target task
@@ -2544,7 +2864,7 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch,
* points to this CPU. See scx_dispatch_dequeue() for the counterpart.
*
* On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is
- * still valid. %false if lost to dequeue.
+ * still valid and detached from @dsq. %false if lost to dequeue.
*/
static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p,
struct scx_dispatch_q *dsq,
@@ -2564,8 +2884,19 @@ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p,
switch_rq_lock(locked_rq, src_rq);
/* task_rq couldn't have changed if we're still the holding cpu */
- return likely(p->scx.holding_cpu == cpu) &&
- !WARN_ON_ONCE(src_rq != task_rq(p));
+ if (likely(p->scx.holding_cpu == cpu) && !WARN_ON_ONCE(src_rq != task_rq(p))) {
+ /*
+ * Keep ->dsq set until we own @src_rq so that a racing dequeue
+ * can find @dsq and clear holding_cpu. Once ownership is
+ * confirmed, clear it under @src_rq so that deactivate_task()
+ * takes the !dsq path instead of retaking @dsq->lock.
+ */
+ p->scx.dsq = NULL;
+ scx_wait_dispatching(p);
+ return true;
+ }
+
+ return false;
}
static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq,
@@ -2573,6 +2904,19 @@ static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq,
struct scx_dispatch_q *dsq, struct rq *src_rq)
{
if (unlink_dsq_and_switch_rq_lock(p, dsq, this_rq, src_rq)) {
+ /*
+ * Proxy execution may have changed @p's running or
+ * migration-disabled state while switching rq locks without
+ * clearing holding_cpu. Park it on the source rq and let its
+ * owning scheduler choose its placement again.
+ */
+ if (unlikely(task_proxy_unsafe_to_move(p))) {
+ p->scx.dsq = NULL;
+ scx_proxy_reject_task(sch, src_rq, p, enq_flags | SCX_ENQ_CLEAR_OPSS);
+ switch_rq_lock(src_rq, this_rq);
+ return false;
+ }
+
move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, this_rq);
return true;
} else {
@@ -2608,8 +2952,23 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch,
lockdep_assert_held(&src_dsq->lock);
lockdep_assert_rq_held(src_rq);
+ scx_wait_dispatching(p);
+
if (dst_dsq->id == SCX_DSQ_LOCAL) {
dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq);
+ /*
+ * Unlike the rq-lock handoff paths, @src_rq has been locked
+ * throughout this operation. Only active proxy state can race the
+ * move here; let the enforcing check below diagnose an ordinary
+ * migration-disabled task.
+ */
+ if (src_rq != dst_rq &&
+ unlikely(task_proxy_running_or_donating(p))) {
+ dispatch_dequeue_locked(p, src_dsq);
+ raw_spin_unlock(&src_dsq->lock);
+ scx_proxy_reject_task(sch, src_rq, p, enq_flags);
+ return src_rq;
+ }
if (src_rq != dst_rq &&
unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
dst_dsq = find_global_dsq(sch, task_cpu(p));
@@ -2629,9 +2988,10 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch,
/* @p is going from a non-local DSQ to a local DSQ */
if (src_rq == dst_rq) {
scx_task_unlink_from_dsq(p, src_dsq);
- scx_move_local_task_to_local_dsq(sch, p, enq_flags, src_dsq, dst_rq);
raw_spin_unlock(&src_dsq->lock);
+ scx_move_local_task_to_local_dsq(sch, p, enq_flags, dst_rq);
} else {
+ dispatch_dequeue_locked(p, src_dsq);
raw_spin_unlock(&src_dsq->lock);
move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq);
}
@@ -2680,8 +3040,8 @@ retry:
if (rq == task_rq) {
scx_task_unlink_from_dsq(p, dsq);
- scx_move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq);
raw_spin_unlock(&dsq->lock);
+ scx_move_local_task_to_local_dsq(sch, p, enq_flags, rq);
return true;
}
@@ -2766,6 +3126,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
if (likely(p->scx.holding_cpu == raw_smp_processor_id()) &&
!WARN_ON_ONCE(src_rq != task_rq(p))) {
bool fallback = false;
+
/*
* If @p is staying on the same rq, there's no need to go
* through the full deactivate/activate cycle. Optimize by
@@ -2775,6 +3136,9 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
p->scx.holding_cpu = -1;
scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p,
slice, vtime, enq_flags | SCX_ENQ_APPLY_SLICE);
+ } else if (unlikely(task_proxy_unsafe_to_move(p))) {
+ fallback = true;
+ scx_proxy_reject_task(sch, src_rq, p, enq_flags);
} else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
p->scx.holding_cpu = -1;
fallback = true;
@@ -2790,7 +3154,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
}
/* if the destination CPU is idle, wake it up */
- if (!fallback && sched_class_above(p->sched_class, dst_rq->curr->sched_class))
+ if (!fallback && sched_class_above(p->sched_class,
+ dst_rq->donor->sched_class))
resched_curr(dst_rq);
}
@@ -3008,10 +3373,39 @@ has_tasks:
return verdict;
}
-static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
+static void scx_start_task_running(struct rq *rq, struct task_struct *p)
{
struct scx_sched *sch = scx_task_sched(p);
+ if (p->scx.flags & SCX_TASK_RUN_TRACKED)
+ return;
+
+ scx_cid_sched_update(rq, sch);
+
+ if (SCX_HAS_OP(sch, running))
+ SCX_CALL_OP_TASK(sch, running, rq, p);
+
+ p->scx.flags |= SCX_TASK_RUN_TRACKED;
+}
+
+static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e type)
+{
+ bool first = type == SNT_PICK;
+ bool can_stop_tick;
+
+ /*
+ * A blocked pick is provisional until proxy resolution completes, see
+ * put_prev_task_scx(). A retry can repick the same donor, so
+ * SNT_REPICK must set it again.
+ */
+ rq->scx.flags &= ~SCX_RQ_PROXY_PENDING;
+ if (sched_proxy_exec() && p->is_blocked &&
+ (type == SNT_PICK || type == SNT_REPICK))
+ rq->scx.flags |= SCX_RQ_PROXY_PENDING;
+
+ if (type == SNT_REPICK)
+ return;
+
if (p->scx.flags & SCX_TASK_QUEUED) {
/*
* Core-sched might decide to execute @p before it is
@@ -3019,18 +3413,31 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
*/
ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC);
scx_dispatch_dequeue(rq, p);
+ p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
}
p->se.exec_start = rq_clock_task(rq);
- /* see dequeue_task_scx() on why we skip when !QUEUED */
- if (SCX_HAS_OP(sch, running) && (p->scx.flags & SCX_TASK_QUEUED))
- SCX_CALL_OP_TASK(sch, running, rq, p);
+ /*
+ * See dequeue_task_scx() for why we skip when !QUEUED.
+ *
+ * During a normal switch (@first), a blocked task is only a provisional
+ * donor. Proxy resolution may fail or migrate the donor to another CPU,
+ * so defer ops.running() until scx_proxy_donor_start() confirms that
+ * resolution succeeded.
+ *
+ * !@first denotes restoration after a SAVE/RESTORE cycle. The matching
+ * dequeue already issued ops.stopping(), so restart the session here
+ * regardless of the donor state.
+ */
+ if ((p->scx.flags & SCX_TASK_QUEUED) && !(p->is_blocked && first))
+ scx_start_task_running(rq, p);
clr_task_runnable(p, true);
/* apply any pending out-of-band slice request before the tick decision */
apply_task_slice_oob(rq, p);
+ can_stop_tick = p->scx.slice == SCX_SLICE_INF && !p->is_blocked;
/*
* @p is getting newly scheduled or got kicked after someone updated its
@@ -3041,7 +3448,7 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
* nohz. In the future, we might want to add a mechanism to update
* load_avgs periodically on tick-stopped CPUs.
*/
- if (p->scx.slice == SCX_SLICE_INF) {
+ if (can_stop_tick) {
if (!(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) {
/*
* Bypass mode always assigns finite slices, so @p
@@ -3062,13 +3469,25 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
/*
* @rq still references the outgoing scheduling context. A finite
- * slice is sufficient by itself to require the tick.
+ * slice or a blocked proxy donor is sufficient by itself to require
+ * the tick.
*/
if (tick_nohz_full_cpu(cpu_of(rq)))
tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED);
}
}
+void scx_proxy_donor_start(struct rq *rq)
+{
+ struct task_struct *donor = rq->donor;
+
+ lockdep_assert_rq_held(rq);
+ rq->scx.flags &= ~SCX_RQ_PROXY_PENDING;
+
+ if (donor->sched_class == &ext_sched_class && (donor->scx.flags & SCX_TASK_QUEUED))
+ scx_start_task_running(rq, donor);
+}
+
static enum scx_cpu_preempt_reason
preempt_reason_from_class(const struct sched_class *class)
{
@@ -3124,8 +3543,13 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
struct task_struct *next)
{
struct scx_sched *sch = scx_task_sched(p);
+ bool proxy_put = p->is_blocked && next == rq->idle &&
+ (rq->scx.flags & SCX_RQ_PROXY_PENDING);
+ bool proxy_block = rq->scx.flags & SCX_RQ_PROXY_BLOCKING;
bool rescue_keep = false;
+ rq->scx.flags &= ~SCX_RQ_PROXY_PENDING;
+
/* see kick_sync_wait_bal_cb() */
smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1);
@@ -3142,29 +3566,51 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
scx_task_slice_ended(rq, p);
}
- /* see dequeue_task_scx() on why we skip when !QUEUED */
- if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED))
- SCX_CALL_OP_TASK(sch, stopping, rq, p, true);
+ /*
+ * Preserve the running session when proxy execution refreshes the same
+ * donor around an execution-context switch on this rq.
+ */
+ if (next != p && (p->scx.flags & SCX_TASK_QUEUED) &&
+ (p->scx.flags & SCX_TASK_RUN_TRACKED)) {
+ if (SCX_HAS_OP(sch, stopping))
+ SCX_CALL_OP_TASK(sch, stopping, rq, p, !proxy_block);
+
+ p->scx.flags &= ~SCX_TASK_RUN_TRACKED;
+ }
if (p->scx.flags & SCX_TASK_QUEUED) {
+ if (proxy_block)
+ goto switch_class;
+
set_task_runnable(rq, p);
/*
+ * If the donor is the same and only the mutex owner changes,
+ * avoid triggering another ops.enqueue(): the BPF scheduler has
+ * already admitted the donor, so it can continue running.
+ */
+ if (p->is_blocked && next == p)
+ goto switch_class;
+
+ /*
* If @p has slice left and is being put, @p is getting
* preempted by a higher priority scheduler class or core-sched
* forcing a different task. Leave it at the head of the local
* DSQ unless it was an IMMED task. IMMED tasks should not
* linger on a busy CPU, reenqueue them to the BPF scheduler.
*
+ * The exception is an IMMED donor put by proxy_resched_idle():
+ * the CPU isn't busy, proxy resolution is only retrying through
+ * idle, so keep the donor where it can be picked again.
+ *
* An open rescue must keep @p on the local DSQ even if the
* scheduler zeroed the slice in ops.stopping() above.
*/
if ((p->scx.slice || unlikely(p == scx_rescuee(rq))) &&
!scx_bypassing(sch, cpu_of(rq))) {
- if (p->scx.flags & SCX_TASK_IMMED) {
+ if ((p->scx.flags & SCX_TASK_IMMED) && !proxy_put) {
p->scx.flags |= SCX_TASK_REENQ_PREEMPTED;
scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
- p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
} else {
u64 enq_flags = 0;
@@ -3179,6 +3625,9 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
enq_flags |= SCX_ENQ_HEAD;
} else {
enq_flags |= SCX_ENQ_HEAD;
+ /* only require SCX_CAP_ENQ_IMMED, see scx_caps_for_enq() */
+ if (proxy_put && (p->scx.flags & SCX_TASK_IMMED))
+ enq_flags |= SCX_ENQ_IMMED;
}
scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0,
@@ -3190,27 +3639,40 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
/*
* If @p is runnable but we're about to enter a lower
* sched_class, %SCX_OPS_ENQ_LAST must be set. Tell
- * ops.enqueue() that @p is the only one available for this cpu,
- * which should trigger an explicit follow-up scheduling event.
- * This doesn't apply if the baseline access on the CPU is lost.
+ * ops.enqueue() that @p is the only one available for this cpu.
+ * This doesn't apply if the baseline access on the CPU is lost or
+ * proxy resolution temporarily switches to idle.
*
* Under core scheduling, a pick dispatches only when nothing is
* locally runnable and can legitimately go idle with @p still
* runnable (see do_pick_task_scx()).
*/
- if (next && sched_class_above(&ext_sched_class, next->sched_class) &&
+ if (!proxy_put && next &&
+ sched_class_above(&ext_sched_class, next->sched_class) &&
scx_task_can_stay_on_cpu(rq, p)) {
WARN_ON_ONCE(!sched_core_enabled(rq) &&
!(sch->ops.flags & SCX_OPS_ENQ_LAST));
scx_do_enqueue_task(rq, p, SCX_ENQ_LAST, -1);
+
+ /*
+ * A task put on the local DSQ runs, as anywhere else.
+ * Here the insert can't reschedule on its own: the pick
+ * has settled on @next and @p is still curr, so
+ * resched_curr() would flag @p and __schedule() clears
+ * that right after the pick. Flag @next instead.
+ */
+ if (p->scx.dsq == &rq->scx.local_dsq)
+ set_tsk_need_resched(next);
} else {
scx_do_enqueue_task(rq, p, 0, -1);
}
}
switch_class:
- if (next && next->sched_class != &ext_sched_class)
+ if (next && next->sched_class != &ext_sched_class) {
+ scx_cid_sched_update(rq, NULL);
switch_class(rq, next);
+ }
}
static void kick_sync_wait_bal_cb(struct rq *rq)
@@ -3278,7 +3740,8 @@ static struct task_struct *first_local_task(struct rq *rq)
}
/*
- * Run dispatch and queue the follow-up work for a pick.
+ * Run dispatch for a pick. The follow-up work is queued by do_pick_task_scx()
+ * once the pick is final.
*/
static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf,
struct task_struct *prev)
@@ -3288,18 +3751,6 @@ static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf,
rq_unpin_lock(rq, rf);
verdict = dispatch_one(rq, prev);
rq_repin_lock(rq, rf);
- maybe_queue_balance_callback(rq);
-
- /*
- * Defer to a balance callback which can drop rq lock and enable IRQs.
- * Waiting directly in the pick path would deadlock against CPUs sending
- * us IPIs (e.g. TLB flushes) while we wait for them.
- */
- if (unlikely(rq->scx.kick_sync_pending)) {
- rq->scx.kick_sync_pending = false;
- queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb,
- kick_sync_wait_bal_cb);
- }
return verdict;
}
@@ -3326,23 +3777,20 @@ static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *r
verdict = dispatch_one(rq, prev);
- if (cpu_of(rq) == smp_processor_id()) {
- maybe_queue_balance_callback(rq);
-
- /* see dispatch_pick() */
- if (unlikely(rq->scx.kick_sync_pending)) {
- rq->scx.kick_sync_pending = false;
- queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb,
- kick_sync_wait_bal_cb);
- }
- } else if (unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) {
- /*
- * Balance callbacks must run in the context that queued them,
- * so they can't be queued on another CPU's rq. Run the deferred
- * work directly instead.
- */
+ /*
+ * A pick that returns a task isn't final under core scheduling: the
+ * selection goes on to pick the siblings, and their dispatch can
+ * release the rq lock, which can't happen with balance callbacks
+ * queued. Don't queue the deferred work as a balance callback, for this
+ * CPU or a sibling, and run it from irq_work instead.
+ *
+ * do_pick_task_scx() leaves %SCX_RQ_BAL_CB_PENDING alone under core
+ * scheduling, so that the early return above leaves it to the dispatch
+ * in flight, which handles it here.
+ */
+ if (unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) {
rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING;
- run_deferred(rq);
+ schedule_deferred(rq);
}
rq_repin_lock(rq, rf);
@@ -3364,22 +3812,33 @@ static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *r
static struct task_struct *
do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx)
{
- struct task_struct *prev = rq->curr;
+ struct task_struct *prev = rq->donor;
+ bool core_sched = sched_core_enabled(rq);
enum scx_dsp_verdict verdict;
struct task_struct *p;
+ /* A retry can abandon a provisional blocked-donor pick. */
+ rq->scx.flags &= ~SCX_RQ_PROXY_PENDING;
+
/* see kick_sync_wait_bal_cb() */
smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1);
rq_modified_begin(rq, &ext_sched_class);
- if (sched_core_enabled(rq))
+ /*
+ * Core scheduling can get enabled while dispatch_pick() has the rq lock
+ * released. Stick to @core_sched, so that the follow-up work below
+ * matches the dispatch that ran.
+ */
+ if (core_sched)
verdict = dispatch_core_pick(rq, rf, prev);
else
verdict = dispatch_pick(rq, rf, prev);
- if (verdict == SCX_DSP_RETRY)
- return RETRY_TASK;
+ if (verdict == SCX_DSP_RETRY) {
+ p = RETRY_TASK;
+ goto out;
+ }
/*
* If any higher-priority sched class enqueued a runnable task on this
@@ -3389,8 +3848,10 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx)
* If @force_scx is true, always try to pick a SCHED_EXT task,
* regardless of any higher-priority sched classes activity.
*/
- if (!force_scx && rq_modified_above(rq, &ext_sched_class))
- return RETRY_TASK;
+ if (!force_scx && rq_modified_above(rq, &ext_sched_class)) {
+ p = RETRY_TASK;
+ goto out;
+ }
/*
* If we're keeping @prev, replenish slice if necessary and keep running
@@ -3406,7 +3867,7 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx)
} else {
p = first_local_task(rq);
if (!p)
- return NULL;
+ goto out;
if (unlikely(!p->scx.slice) && scx_task_can_stay_on_cpu(rq, p)) {
struct scx_sched *sch = scx_task_sched(p);
@@ -3421,6 +3882,40 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx)
}
}
+out:
+ if (cpu_of(rq) != smp_processor_id())
+ return p;
+
+ /*
+ * Balance callbacks must be consumed before the rq lock is released, so
+ * queue them only when the pick is final. On RETRY_TASK, or after a
+ * NULL pick from the dl_server, the core continues picking and can
+ * release the rq lock (e.g., prev_balance() or another dispatch). In
+ * that case, run the deferred work from irq_work, which doesn't depend
+ * on the current rq lock section, and leave the kick sync wait pending
+ * for the next pick.
+ */
+ if (p != RETRY_TASK && (p || !force_scx)) {
+ /* under core scheduling, see dispatch_core_pick() */
+ if (!core_sched)
+ maybe_queue_balance_callback(rq);
+
+ /*
+ * Defer to a balance callback which can drop rq lock and enable
+ * IRQs. Waiting directly in the pick path would deadlock against
+ * CPUs sending us IPIs (e.g., TLB flushes) while we wait for
+ * them.
+ */
+ if (unlikely(rq->scx.kick_sync_pending)) {
+ rq->scx.kick_sync_pending = false;
+ queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb,
+ kick_sync_wait_bal_cb);
+ }
+ } else if (!core_sched && (rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) {
+ rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING;
+ schedule_deferred(rq);
+ }
+
return p;
}
@@ -3629,8 +4124,12 @@ static void set_cpus_allowed_scx(struct task_struct *p,
*
* Fine-grained memory write control is enforced by BPF making the const
* designation pointless. Cast it away when calling the operation.
+ *
+ * The cid form receives the initial mask when the task is enabled and
+ * hears about changes only afterwards, see struct scx_enable_args.
*/
- if (SCX_HAS_OP(sch, set_cpumask))
+ if (SCX_HAS_OP(sch, set_cpumask) &&
+ (!scx_is_cid_type() || scx_get_task_state(p) == SCX_TASK_ENABLED))
scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr);
}
@@ -3714,8 +4213,26 @@ static void rq_online_scx(struct rq *rq)
static void rq_offline_scx(struct rq *rq)
{
+ struct task_struct *p, *n;
+
rq->scx.flags &= ~SCX_RQ_ONLINE;
+
+ /* sched domain rebuilds call rq_offline with the CPU staying alive */
+ if (cpu_active(cpu_of(rq)))
+ return;
+
scx_rescue_flush(rq);
+
+ /*
+ * An offline CPU no longer calls ops.dispatch(). Re-enqueue its tasks
+ * onto the local DSQ so that they run here and balance_push() moves
+ * them off.
+ */
+ list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, scx.runnable_node) {
+ if (p->scx.dsq == &rq->scx.local_dsq)
+ continue;
+ guard(sched_change)(p, DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK);
+ }
}
static bool check_rq_for_timeouts(struct rq *rq)
@@ -3806,9 +4323,9 @@ void scx_tick(struct rq *rq)
update_other_load_avgs(rq);
}
-static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued)
+static void task_tick_scx(struct rq *rq, struct task_struct *donor, int queued)
{
- struct scx_sched *sch = scx_task_sched(curr);
+ struct scx_sched *sch = scx_task_sched(donor);
update_curr_scx(rq);
@@ -3817,12 +4334,18 @@ static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued)
* management.
*/
if (scx_bypassing(sch, cpu_of(rq)))
- scx_set_task_slice(curr, 0);
+ scx_set_task_slice(donor, 0);
else if (SCX_HAS_OP(sch, tick))
- SCX_CALL_OP_TASK(sch, tick, rq, curr);
+ SCX_CALL_OP_TASK(sch, tick, rq, donor);
- if (!curr->scx.slice)
- resched_curr(rq);
+ if (!donor->scx.slice) {
+ /* the slice can't be trusted while bypassing */
+ if (READ_ONCE(donor->scx.lazy_resched) &&
+ !scx_bypassing(sch, cpu_of(rq)))
+ scx_resched_curr_lazy(rq);
+ else
+ resched_curr(rq);
+ }
}
#ifdef CONFIG_EXT_GROUP_SCHED
@@ -3938,9 +4461,29 @@ static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p)
weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO];
p->scx.weight = sched_weight_to_cgroup(weight);
+ p->scx.lazy_resched = sch->ops.flags & SCX_OPS_LAZY_RESCHED;
+
+ if (SCX_HAS_OP(sch, enable)) {
+ if (scx_is_cid_type()) {
+ struct scx_cmask *cmask = scx_fill_cmask_scratch(sch, p->cpus_ptr);
+ struct scx_enable_args args = {
+ .cmask_arena_addr = scx_kaddr_to_arena(sch, cmask),
+ };
+
+ SCX_CALL_CID_OP_TASK(sch, enable, rq, p, &args);
+ } else {
+ SCX_CALL_OP_TASK(sch, enable, rq, p);
+ }
+ }
- if (SCX_HAS_OP(sch, enable))
- SCX_CALL_OP_TASK(sch, enable, rq, p);
+ /*
+ * The initial mask also goes out through set_cmask() so a scheduler can
+ * track affinity there alone, and before set_weight() so that the mask
+ * is in place when weight-dependent state is derived, see struct
+ * scx_enable_args.
+ */
+ if (scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask))
+ scx_call_op_set_cpumask(sch, rq, p, p->cpus_ptr);
if (SCX_HAS_OP(sch, set_weight))
SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight);
@@ -4283,14 +4826,22 @@ static void switching_to_scx(struct rq *rq, struct task_struct *p)
/*
* set_cpus_allowed_scx() is not called while @p is associated with a
- * different scheduler class. Keep the BPF scheduler up-to-date.
+ * different scheduler class. Keep the BPF scheduler up-to-date. The cid
+ * form gets its mask from scx_enable_task().
*/
- if (SCX_HAS_OP(sch, set_cpumask))
+ if (!scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask))
scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr);
}
static void switched_from_scx(struct rq *rq, struct task_struct *p)
{
+ /*
+ * A class change of the running task: sched_change_begin() put @p with
+ * no successor, which leaves rq->scx.sched set.
+ */
+ if (task_current_donor(rq, p))
+ scx_cid_sched_update(rq, NULL);
+
if (task_dead_and_done(p))
return;
@@ -4313,7 +4864,7 @@ int scx_check_setscheduler(struct task_struct *p, int policy)
{
lockdep_assert_rq_held(task_rq(p));
- /* if disallow, reject transitioning into SCX */
+ /* If disallow, reject transitioning into SCX. */
if (scx_enabled() && READ_ONCE(p->scx.disallow) &&
p->policy != policy && policy == SCHED_EXT)
return -EACCES;
@@ -4321,6 +4872,20 @@ int scx_check_setscheduler(struct task_struct *p, int policy)
return 0;
}
+/*
+ * Don't carry a donor retained by another class into sched_ext. The caller
+ * has updated the rq clock and invokes this immediately before
+ * sched_change_begin() records the task's queued state.
+ */
+void scx_prepare_setscheduler(struct task_struct *p, int policy)
+{
+ lockdep_assert_held(&p->pi_lock);
+ lockdep_assert_rq_held(task_rq(p));
+
+ if (scx_enabled() && p->policy != policy && policy == SCHED_EXT)
+ scx_block_proxy_donor(task_rq(p), p);
+}
+
static void process_ddsp_deferred_locals(struct rq *rq)
{
struct task_struct *p;
@@ -4404,6 +4969,18 @@ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p,
return *reenq_flags & SCX_REENQ_ANY;
}
+/*
+ * The dispatcher runs ops.dequeue() and stores the final ops_state after
+ * dropping the DSQ lock, so @p can be found on a DSQ while still
+ * %SCX_OPSS_DISPATCHING. Wait for the dispatch to finish before operating on
+ * @p.
+ */
+void scx_wait_dispatching(struct task_struct *p)
+{
+ if (unlikely(atomic_long_read_acquire(&p->scx.ops_state) == SCX_OPSS_DISPATCHING))
+ wait_ops_state(p, SCX_OPSS_DISPATCHING);
+}
+
static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)
{
LIST_HEAD(tasks);
@@ -4447,10 +5024,10 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)
if (!local_task_should_reenq(rq, p, &reenq_flags, &reason))
continue;
+ scx_wait_dispatching(p);
scx_dispatch_dequeue(rq, p);
- if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
- p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
+ WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK);
p->scx.flags |= reason;
list_add_tail(&p->scx.dsq_list.node, &tasks);
@@ -4461,21 +5038,20 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)
scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
- p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
nr_enqueued++;
}
/*
- * The revoke that scheduled this scan may have raced the pick: curr
+ * The revoke that scheduled this scan may have raced the pick: donor
* may be a now-capless task, either one that kept running or one
* promoted off the local DSQ between the ecaps sync and this scan.
* Zero the slice to evict it. The enqueue gate blocks new capless
* inserts, so no later pick can slip through after the scan.
*/
if ((reenq_flags & SCX_REENQ_CAP_REVOKE) &&
- rq->curr->sched_class == &ext_sched_class &&
- scx_task_reenq_on_cap_revoke(rq, rq->curr)) {
- scx_set_task_slice(rq->curr, 0);
+ rq->donor->sched_class == &ext_sched_class &&
+ scx_task_reenq_on_cap_revoke(rq, rq->donor)) {
+ scx_set_task_slice(rq->donor, 0);
resched_curr(rq);
}
@@ -4524,7 +5100,9 @@ static void process_deferred_reenq_locals(struct rq *rq)
static bool user_task_should_reenq(struct task_struct *p, u64 reenq_flags, u32 *reason)
{
*reason = SCX_TASK_REENQ_KFUNC;
- return reenq_flags & SCX_REENQ_ANY;
+ if (reenq_flags & SCX_REENQ_ANY)
+ return true;
+ return (reenq_flags & SCX_REENQ_MARKED) && (p->scx.dsq_flags & SCX_TASK_DSQ_REENQ);
}
static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flags)
@@ -4533,7 +5111,7 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag
struct scx_sched *sch = dsq->sched;
struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, dsq, 0);
struct task_struct *p;
- s32 nr_enqueued = 0;
+ s32 nr_visited = 0;
lockdep_assert_rq_held(rq);
@@ -4543,6 +5121,21 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag
struct rq *task_rq;
u32 reason;
+ /*
+ * Count skipped tasks too so that a long walk doesn't hold the
+ * locks throughout. The cursor keeps the position.
+ */
+ if (!(++nr_visited % SCX_TASK_ITER_BATCH)) {
+ raw_spin_unlock(&dsq->lock);
+ if (locked_rq) {
+ scx_rq_lock_drop(locked_rq);
+ raw_spin_rq_unlock(locked_rq);
+ locked_rq = NULL;
+ }
+ cpu_relax();
+ raw_spin_lock(&dsq->lock);
+ }
+
p = nldsq_cursor_next_task(&cursor, dsq);
if (!p)
break;
@@ -4570,24 +5163,15 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag
}
/* @p is on @dsq, its rq and @dsq are locked */
+ scx_wait_dispatching(p);
dispatch_dequeue_locked(p, dsq);
raw_spin_unlock(&dsq->lock);
- if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
- p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
+ WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK);
p->scx.flags |= reason;
scx_do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1);
- p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
-
- if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) {
- scx_rq_lock_drop(locked_rq);
- raw_spin_rq_unlock(locked_rq);
- locked_rq = NULL;
- cpu_relax();
- }
-
raw_spin_lock(&dsq->lock);
}
@@ -4642,6 +5226,60 @@ static void process_deferred_reenq_users(struct rq *rq)
}
}
+/*
+ * Drain ready tasks from @rq->scx.reject_dsq and reenqueue them so that their
+ * owning BPF schedulers choose placement again. Proxy-active tasks remain
+ * parked and rearm the retry notification for a later proxy resolution.
+ *
+ * A task can be re-rejected repeatedly. Reenqueues are bounded per task by
+ * SCX_REENQ_MAX_REPEAT in scx_do_enqueue_task(), which ejects the owning
+ * scheduler.
+ */
+static void scx_reenq_reject(struct rq *rq)
+{
+ LIST_HEAD(tasks);
+ struct task_struct *p, *n;
+
+ lockdep_assert_rq_held(rq);
+
+ if (list_empty(&rq->scx.reject_dsq.list))
+ return;
+
+ /*
+ * Move ready tasks to a private list so a task re-rejected by
+ * scx_do_enqueue_task() below isn't revisited this round.
+ */
+ list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) {
+ u32 reason = p->scx.flags & SCX_TASK_REENQ_REASON_MASK;
+
+ WARN_ON_ONCE(!reason);
+
+ if (sched_proxy_exec() && reason == SCX_TASK_REENQ_PROXY) {
+ /* Affinity machinery will dequeue and reactivate @p. */
+ if (p->migration_pending)
+ continue;
+
+ if (task_on_cpu(rq, p) || task_current_donor(rq, p)) {
+ rq->scx.flags |= SCX_RQ_PROXY_RETRY;
+ continue;
+ }
+ } else {
+ WARN_ON_ONCE(p->migration_pending);
+ }
+
+ scx_wait_dispatching(p);
+ scx_dispatch_dequeue(rq, p);
+
+ list_add_tail(&p->scx.dsq_list.node, &tasks);
+ }
+
+ list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {
+ list_del_init(&p->scx.dsq_list.node);
+
+ scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
+ }
+}
+
static void run_deferred(struct rq *rq)
{
process_ddsp_deferred_locals(rq);
@@ -4658,14 +5296,18 @@ static void run_deferred(struct rq *rq)
#ifdef CONFIG_NO_HZ_FULL
bool scx_can_stop_tick(struct rq *rq)
{
- struct task_struct *p = rq->curr;
+ struct task_struct *p = rq->donor;
struct scx_sched *sch = scx_task_sched(p);
+ /* Proxy execution is conservatively tick-driven for now. */
+ if (rq->scx.flags & SCX_RQ_PROXY_TICK)
+ return false;
+
if (p->sched_class != &ext_sched_class)
return true;
/*
- * @rq->curr may still reference an outgoing EXT task after it has been
+ * @rq->donor may still reference an outgoing EXT task after it has been
* dequeued. If no EXT tasks are accounted on @rq, ignore its stale
* slice state. If another task is dispatched from a DSQ,
* set_next_task_scx() will update the dependency for the incoming task.
@@ -4686,7 +5328,8 @@ bool scx_can_stop_tick(struct rq *rq)
/*
* @rq can dispatch from different DSQs, so we can't tell whether it
* needs the tick or not by looking at nr_running. Allow stopping ticks
- * iff the BPF scheduler indicated so. See set_next_task_scx().
+ * iff set_next_task_scx() determined that the selected scheduling context
+ * can run tickless.
*/
return rq->scx.flags & SCX_RQ_CAN_STOP_TICK;
}
@@ -4701,7 +5344,7 @@ void scx_tg_init(struct task_group *tg)
tg->scx.weight = CGROUP_WEIGHT_DFL;
tg->scx.bw_period_us = default_bw_period_us();
tg->scx.bw_quota_us = RUNTIME_INF;
- tg->scx.idle = false;
+ tg->scx.sched_idle = false;
}
/**
@@ -4784,7 +5427,7 @@ int scx_tg_online(struct task_group *tg)
.bw_period_us = tg->scx.bw_period_us,
.bw_quota_us = tg->scx.bw_quota_us,
.bw_burst_us = tg->scx.bw_burst_us,
- .sched_idle = tg->scx.idle };
+ .sched_idle = tg->scx.sched_idle };
ret = SCX_CALL_OP_RET(sch, cgroup_init,
NULL, tg->css.cgroup, &args);
@@ -4951,11 +5594,11 @@ void scx_group_set_idle(struct task_group *tg, bool idle)
sch = scx_tg_knob_sched(tg);
if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle) &&
- tg->scx.idle != idle)
+ tg->scx.sched_idle != idle)
SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle);
/* Update the task group's idle state */
- tg->scx.idle = idle;
+ tg->scx.sched_idle = idle;
percpu_up_read(&scx_cgroup_ops_rwsem);
}
@@ -5069,12 +5712,16 @@ s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch)
dsq->id = dsq_id;
dsq->sched = sch;
- dsq->pcpu = alloc_percpu(struct scx_dsq_pcpu);
- if (!dsq->pcpu)
+ /* per-DSQ deferred reenq state is only needed for user DSQs */
+ if (dsq_id & SCX_DSQ_FLAG_BUILTIN)
+ return 0;
+
+ dsq->pcpu_user = alloc_percpu(struct scx_dsq_pcpu);
+ if (!dsq->pcpu_user)
return -ENOMEM;
for_each_possible_cpu(cpu) {
- struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu);
+ struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu);
pcpu->dsq = dsq;
INIT_LIST_HEAD(&pcpu->deferred_reenq_user.node);
@@ -5087,14 +5734,17 @@ static void exit_dsq(struct scx_dispatch_q *dsq)
{
s32 cpu;
+ if (!dsq->pcpu_user)
+ return;
+
for_each_possible_cpu(cpu) {
- struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu);
+ struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu);
struct scx_deferred_reenq_user *dru = &pcpu->deferred_reenq_user;
struct rq *rq = cpu_rq(cpu);
/*
- * There must have been a RCU grace period since the last
- * insertion and @dsq should be off the deferred list by now.
+ * free_dsq_irq_workfn() cancelled the reenqs before the grace
+ * period and schedule_dsq_reenq() queues nothing after that.
*/
if (WARN_ON_ONCE(!list_empty(&dru->node))) {
guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
@@ -5102,7 +5752,7 @@ static void exit_dsq(struct scx_dispatch_q *dsq)
}
}
- free_percpu(dsq->pcpu);
+ free_percpu(dsq->pcpu_user);
}
static void free_dsq_rcufn(struct rcu_head *rcu)
@@ -5118,8 +5768,26 @@ static void free_dsq_irq_workfn(struct irq_work *irq_work)
struct llist_node *to_free = llist_del_all(&dsqs_to_free);
struct scx_dispatch_q *dsq, *tmp_dsq;
- llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node)
+ llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) {
+ s32 cpu;
+
+ /*
+ * Cancel pending reenqs. Only a run_deferred() that started
+ * before the grace period can hold one, and it keeps IRQs off,
+ * so the grace period waits for it. After this sweep,
+ * schedule_dsq_reenq() sees the invalid id under the same lock
+ * and queues nothing.
+ */
+ for_each_possible_cpu(cpu) {
+ struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu);
+ struct rq *rq = cpu_rq(cpu);
+
+ guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
+ list_del_init(&pcpu->deferred_reenq_user.node);
+ }
+
call_rcu(&dsq->rcu, free_dsq_rcufn);
+ }
}
static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn);
@@ -5206,7 +5874,7 @@ static int scx_cgroup_init(struct scx_sched *sch)
.bw_period_us = tg->scx.bw_period_us,
.bw_quota_us = tg->scx.bw_quota_us,
.bw_burst_us = tg->scx.bw_burst_us,
- .sched_idle = tg->scx.idle,
+ .sched_idle = tg->scx.sched_idle,
};
ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args);
@@ -5400,6 +6068,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
free_cpumask_var(pcpu->cpus_to_kick);
free_cpumask_var(pcpu->cpus_to_kick_if_idle);
free_cpumask_var(pcpu->cpus_to_preempt);
+ free_cpumask_var(pcpu->cpus_to_preempt_lazy);
free_cpumask_var(pcpu->cpus_to_wait);
exit_dsq(scx_bypass_dsq(sch, cpu));
@@ -5878,7 +6547,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node)
/*
* We don't want CPUs to have more than $nr_donor_target tasks and
- * balancing to fill donee CPUs upto $nr_target. Once targets are
+ * balancing to fill donee CPUs up to $nr_target. Once targets are
* calculated, find the donee CPUs.
*/
nr_target = DIV_ROUND_UP(nr_tasks, nr_cpus);
@@ -5990,15 +6659,12 @@ static void enable_bypass_dsp(struct scx_sched *sch)
/*
* When a sub-sched bypasses, its tasks are queued on the bypass DSQs of
- * the nearest non-bypassing ancestor or root. As enable_bypass_dsp() is
- * called iff @sch is not already bypassed due to an ancestor bypassing,
- * we can assume that the parent is not bypassing and thus will be the
- * host of the bypass DSQs.
- *
- * While the situation may change in the future, the following
- * guarantees that the nearest non-bypassing ancestor or root has bypass
- * dispatch enabled while a descendant is bypassing, which is all that's
- * required.
+ * the nearest non-bypassing ancestor or root. The parent hosts them
+ * unless it is bypassing too. Then they sit further up and the parent
+ * consumes only its own bypass DSQs, empty until it leaves bypass and
+ * becomes the host. Either way the nearest non-bypassing ancestor or
+ * root has bypass dispatch enabled while a descendant is bypassing,
+ * which is all that's required.
*
* scx_bypass_dsp_enabled() test is used to determine whether to enter
* the bypass dispatch handling path from both bypassing and hosting
@@ -6107,35 +6773,46 @@ void scx_bypass(struct scx_sched *sch, bool bypass)
struct scx_sched *pos;
unsigned long flags;
int cpu;
+ bool changed;
raw_spin_lock_irqsave(&scx_bypass_lock, flags);
if (bypass) {
- if (!inc_bypass_depth(sch))
- goto unlock;
-
- enable_bypass_dsp(sch);
+ changed = inc_bypass_depth(sch);
+ if (changed)
+ enable_bypass_dsp(sch);
} else {
- if (!dec_bypass_depth(sch))
- goto unlock;
+ changed = dec_bypass_depth(sch);
}
/*
* Bypass state is propagated to all descendants - an scx_sched bypasses
- * if itself or any of its ancestors are in bypass mode.
+ * if itself or any of its ancestors are in bypass mode. A descendant's
+ * depth is the sum of its own bypasses and its ancestors', so every
+ * call propagates whether or not @sch changed state. The descendants'
+ * claims are released below with @sch's, after their tasks have moved.
*/
raw_spin_lock(&scx_sched_lock);
scx_for_each_descendant_pre(pos, sch) {
if (pos == sch)
continue;
- if (bypass)
- inc_bypass_depth(pos);
- else
+ if (bypass) {
+ if (inc_bypass_depth(pos))
+ enable_bypass_dsp(pos);
+ } else {
dec_bypass_depth(pos);
+ }
}
raw_spin_unlock(&scx_sched_lock);
/*
+ * A descendant's depth is at least @sch's, so no bypass state changed
+ * unless @sch's did.
+ */
+ if (!changed)
+ goto unlock;
+
+ /*
* No task property is changing. We just need to make sure all currently
* queued tasks are re-queued according to the new scx_bypassing()
* state. As an optimization, walk each rq's runnable_list instead of
@@ -6180,6 +6857,24 @@ void scx_bypass(struct scx_sched *sch, bool bypass)
}
/*
+ * Bypass trumps protection. Cycling the queued tasks below
+ * clears theirs. The current donor isn't on the runnable list,
+ * so strip its protection here.
+ *
+ * When bypass lifts, the current donor keeps running on its
+ * slice without a cap check. Evict it if it can't stay on the
+ * cpu.
+ */
+ p = rq->donor;
+ if (p->sched_class == &ext_sched_class &&
+ scx_is_descendant(scx_task_sched(p), sch)) {
+ if (bypass)
+ scx_task_slice_ended(rq, p);
+ else if (!scx_task_can_stay_on_cpu(rq, p))
+ scx_set_task_slice(p, 0);
+ }
+
+ /*
* The use of list_for_each_entry_safe_reverse() is required
* because each task is going to be removed from and added back
* to the runnable_list during iteration. Because they're added
@@ -6191,13 +6886,6 @@ void scx_bypass(struct scx_sched *sch, bool bypass)
if (!scx_is_descendant(scx_task_sched(p), sch))
continue;
- /*
- * Bypass trumps protection. Cycling clears for queued
- * tasks but current task needs explicit stripping.
- */
- if (bypass && task_current(rq, p))
- scx_task_slice_ended(rq, p);
-
/* cycling deq/enq is enough, see the function comment */
scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
/* nothing */ ;
@@ -6213,8 +6901,12 @@ void scx_bypass(struct scx_sched *sch, bool bypass)
}
/* disarming must come after moving all tasks out of the bypass DSQs */
- if (!bypass)
- scx_disable_bypass_dsp(sch);
+ if (!bypass) {
+ guard(raw_spinlock)(&scx_sched_lock);
+ scx_for_each_descendant_pre(pos, sch)
+ if (!pos->bypass_depth)
+ scx_disable_bypass_dsp(pos);
+ }
unlock:
raw_spin_unlock_irqrestore(&scx_bypass_lock, flags);
}
@@ -6525,6 +7217,8 @@ static void scx_root_disable(struct scx_sched *sch)
}
}
+ scx_ops_cid_sched_updated_disable(sch);
+
/* no task is on scx, turn off all the switches and flush in-progress calls */
static_branch_disable(&__scx_enabled);
static_branch_disable(&__scx_is_cid_type);
@@ -6897,12 +7591,14 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s,
seq_buf_init(&ns, buf, avail);
dump_newline(&ns);
- scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu",
+ scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ksync=%lu",
cpu, rq->scx.nr_running, rq->scx.flags, rq->scx.cpu_released,
- rq->scx.ops_qseq, rq->scx.kick_sync);
+ rq->scx.kick_sync);
scx_rescue_dump(&ns, rq);
scx_dump_line(&ns, " curr=%s[%d] class=%ps",
rq->curr->comm, rq->curr->pid, rq->curr->sched_class);
+ scx_dump_line(&ns, " donor=%s[%d] class=%ps",
+ rq->donor->comm, rq->donor->pid, rq->donor->sched_class);
if (!cpumask_empty(pcpu->cpus_to_kick))
scx_dump_line(&ns, " cpus_to_kick : %*pb",
cpumask_pr_args(pcpu->cpus_to_kick));
@@ -6912,6 +7608,9 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s,
if (!cpumask_empty(pcpu->cpus_to_preempt))
scx_dump_line(&ns, " cpus_to_preempt: %*pb",
cpumask_pr_args(pcpu->cpus_to_preempt));
+ if (!cpumask_empty(pcpu->cpus_to_preempt_lazy))
+ scx_dump_line(&ns, " preempt_lazy : %*pb",
+ cpumask_pr_args(pcpu->cpus_to_preempt_lazy));
if (!cpumask_empty(pcpu->cpus_to_wait))
scx_dump_line(&ns, " cpus_to_wait : %*pb",
cpumask_pr_args(pcpu->cpus_to_wait));
@@ -6946,6 +7645,10 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s,
if (rq->curr->sched_class == &ext_sched_class &&
(dump_all_tasks || scx_task_on_sched(sch, rq->curr)))
scx_dump_task(sch, s, dctx, rq, rq->curr, '*');
+ if (rq->donor != rq->curr &&
+ rq->donor->sched_class == &ext_sched_class &&
+ (dump_all_tasks || scx_task_on_sched(sch, rq->donor)))
+ scx_dump_task(sch, s, dctx, rq, rq->donor, ' ');
list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node)
if (dump_all_tasks || scx_task_on_sched(sch, p))
@@ -7229,6 +7932,7 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
if (!zalloc_cpumask_var_node(&pcpu->cpus_to_kick, GFP_KERNEL, node) ||
!zalloc_cpumask_var_node(&pcpu->cpus_to_kick_if_idle, GFP_KERNEL, node) ||
!zalloc_cpumask_var_node(&pcpu->cpus_to_preempt, GFP_KERNEL, node) ||
+ !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt_lazy, GFP_KERNEL, node) ||
!zalloc_cpumask_var_node(&pcpu->cpus_to_wait, GFP_KERNEL, node)) {
ret = -ENOMEM;
goto err_free_pcpu;
@@ -7364,6 +8068,7 @@ err_free_pcpu:
free_cpumask_var(pcpu->cpus_to_kick);
free_cpumask_var(pcpu->cpus_to_kick_if_idle);
free_cpumask_var(pcpu->cpus_to_preempt);
+ free_cpumask_var(pcpu->cpus_to_preempt_lazy);
free_cpumask_var(pcpu->cpus_to_wait);
}
for_each_possible_cpu(cpu) {
@@ -7454,6 +8159,11 @@ int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops)
return -EINVAL;
}
+ if ((ops->flags & SCX_OPS_ENQ_BLOCKED) && !ops->enqueue) {
+ scx_error(sch, "SCX_OPS_ENQ_BLOCKED requires ops.enqueue() to be implemented");
+ return -EINVAL;
+ }
+
/*
* SCX_OPS_TID_TO_TASK is enabled by the root scheduler. A sub-sched
* may set it to declare a dependency; reject if the root hasn't
@@ -7717,6 +8427,8 @@ static void scx_root_enable_workfn(struct kthread_work *work)
if (((void (**)(void))ops)[i])
set_bit(i, sch->has_op);
+ scx_ops_cid_sched_updated_enable(sch);
+
if (sch->ops.cpu_acquire || sch->ops.cpu_release)
sch->ops.flags |= SCX_OPS_HAS_CPU_PREEMPT;
@@ -7843,6 +8555,10 @@ static void scx_root_enable_workfn(struct kthread_work *work)
if (old_class != new_class)
queue_flags |= DEQUEUE_CLASS;
+ if (new_class == &ext_sched_class) {
+ scx_prepare_task_sched_change(p);
+ queue_flags |= DEQUEUE_NOCLOCK;
+ }
scoped_guard (sched_change, p, queue_flags) {
scx_set_task_slice(p, READ_ONCE(sch->slice_dfl));
@@ -8356,12 +9072,15 @@ static struct bpf_struct_ops bpf_sched_ext_ops = {
/*
* cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types
* identical, only param names differ across structs) are reused. Some need
- * fresh stubs, set_cmask due to an argument type difference and the sub-sched
- * notifiers because no cpu-form stub exists to reuse.
+ * fresh stubs, set_cmask and enable due to argument differences and the
+ * sub-sched notifiers because no cpu-form stub exists to reuse.
*/
static void sched_ext_ops_cid__set_cmask(struct task_struct *p, const struct scx_cmask *cmask__arena) {}
+static void sched_ext_ops_cid__enable(struct task_struct *p, struct scx_enable_args *args) {}
static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask__arena, u64 caps) {}
static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {}
+static void sched_ext_ops__sub_cid_sched_updated(s32 cid, u64 sched) {}
+static void sched_ext_ops__sub_child_ecaps_updated(u64 cgroup_id, s32 cid, u64 before, u64 after) {}
static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = {
.select_cid = sched_ext_ops__select_cpu,
@@ -8380,7 +9099,7 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = {
.update_idle = sched_ext_ops__update_idle,
.init_task = sched_ext_ops__init_task,
.exit_task = sched_ext_ops__exit_task,
- .enable = sched_ext_ops__enable,
+ .enable = sched_ext_ops_cid__enable,
.disable = sched_ext_ops__disable,
#ifdef CONFIG_EXT_GROUP_SCHED
.cpuctl_init = sched_ext_ops__cgroup_init,
@@ -8396,6 +9115,8 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = {
.sub_detach = sched_ext_ops__sub_detach,
.sub_caps_updated = sched_ext_ops__sub_caps_updated,
.sub_ecaps_updated = sched_ext_ops__sub_ecaps_updated,
+ .sub_cid_sched_updated = sched_ext_ops__sub_cid_sched_updated,
+ .sub_child_ecaps_updated = sched_ext_ops__sub_child_ecaps_updated,
.cid_online = sched_ext_ops__cpu_online,
.cid_offline = sched_ext_ops__cpu_offline,
.init_cids = sched_ext_ops__init_cids,
@@ -8490,13 +9211,23 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r
{
struct rq *rq = cpu_rq(cpu);
struct scx_rq *this_scx = &this_rq->scx;
+ struct rq_flags rf;
const struct sched_class *cur_class;
bool should_wait = false;
bool kickable;
- unsigned long flags;
+ bool preempt, preempt_lazy, wait, immediate;
- raw_spin_rq_lock_irqsave(rq, flags);
- cur_class = rq->curr->sched_class;
+ rq_lock_irqsave(rq, &rf);
+ cur_class = rq->donor->sched_class;
+ preempt = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt);
+ preempt_lazy = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt_lazy);
+ wait = cpumask_test_cpu(cpu, pcpu->cpus_to_wait);
+ /*
+ * Immediate preemption, waiting and a plain kick take precedence over
+ * lazy preemption. The lazy request still clears the slice, so all
+ * accumulated requests are served.
+ */
+ immediate = preempt || wait || cpumask_test_cpu(cpu, pcpu->cpus_to_kick);
/*
* During CPU hotplug, a CPU may depend on kicking itself to make
@@ -8510,19 +9241,23 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r
!sched_class_above(cur_class, &ext_sched_class);
if (kickable && !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) {
- if (cpumask_test_cpu(cpu, pcpu->cpus_to_preempt)) {
+ if (preempt || preempt_lazy) {
if (cur_class == &ext_sched_class) {
u64 caps = scx_caps_for_preempt(pcpu->sch, rq, 0);
- if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps)))
+ if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps))) {
__scx_add_event(pcpu->sch, SCX_EV_SUB_PREEMPT_DENIED, 1);
- else if (unlikely(!scx_set_task_slice(rq->curr, 0)))
+ /* degrade to a plain, immediate kick */
+ immediate = true;
+ } else if (unlikely(!scx_set_task_slice(rq->donor, 0))) {
__scx_add_event(pcpu->sch, SCX_EV_SLICE_DENIED, 1);
+ }
}
cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt);
+ cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt_lazy);
}
- if (cpumask_test_cpu(cpu, pcpu->cpus_to_wait)) {
+ if (wait) {
if (cur_class == &ext_sched_class) {
cpumask_set_cpu(cpu, this_scx->cpus_to_sync);
ksyncs[cpu] = rq->scx.kick_sync;
@@ -8531,17 +9266,21 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r
cpumask_clear_cpu(cpu, pcpu->cpus_to_wait);
}
- resched_curr(rq);
+ if (immediate)
+ resched_curr(rq);
+ else
+ scx_resched_curr_lazy(rq);
} else {
/* a kickable cpu was skipped solely for the missing caps */
if (kickable)
__scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1);
cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt);
+ cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt_lazy);
cpumask_clear_cpu(cpu, pcpu->cpus_to_wait);
}
scx_rq_lock_drop(rq);
- raw_spin_rq_unlock_irqrestore(rq, flags);
+ rq_unlock_irqrestore(rq, &rf);
return should_wait;
}
@@ -8591,7 +9330,7 @@ static void kick_cpus_irq_workfn(struct irq_work *irq_work)
list_for_each_entry_safe(pcpu, tmp, &this_scx->sched_pcpus_to_kick, to_kick_node) {
list_del_init(&pcpu->to_kick_node);
- for_each_cpu(cpu, pcpu->cpus_to_kick) {
+ for_each_cpu_or(cpu, pcpu->cpus_to_kick, pcpu->cpus_to_preempt_lazy) {
should_wait |= kick_one_cpu(cpu, pcpu, this_rq, ksyncs);
cpumask_clear_cpu(cpu, pcpu->cpus_to_kick);
cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle);
@@ -8722,8 +9461,8 @@ void __init init_sched_ext_class(void)
/* local_dsq's sch will be set during scx_root_enable() */
BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL));
-#ifdef CONFIG_EXT_SUB_SCHED
BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL));
+#ifdef CONFIG_EXT_SUB_SCHED
scx_rescue_init(rq);
#endif
@@ -8856,7 +9595,7 @@ __bpf_kfunc_start_defs();
* task is inserted.
*
* When called from ops.dispatch(), there are no restrictions on @p or @dsq_id
- * and this function can be called upto ops.dispatch_max_batch times to insert
+ * and this function can be called up to ops.dispatch_max_batch times to insert
* multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the
* remaining slots. scx_bpf_dsq_move_to_local() flushes the batch and resets the
* counter.
@@ -8941,7 +9680,9 @@ struct scx_bpf_dsq_insert_vtime_args {
*
* @args->vtime ordering is according to time_before64() which considers
* wrapping. A numerically larger vtime may indicate an earlier position in the
- * ordering and vice-versa.
+ * ordering and vice-versa. vtime is a rolling cursor and values used for
+ * ordering within a given DSQ should stay less than 2^63 apart for
+ * time_before64() ordering to remain well-defined.
*
* A DSQ can only be used as a FIFO or priority queue at any given time and this
* function must not be called on a DSQ which already has one or more FIFO tasks
@@ -9019,6 +9760,40 @@ static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = {
.filter = scx_kfunc_context_filter,
};
+/*
+ * Lock @rq from a kfunc called either from ops.dispatch(), which holds the
+ * dispatched rq's lock, or from a context holding no rq lock. Under core
+ * scheduling, dispatch can run for a sibling rq, so the locked rq is not
+ * necessarily this CPU's. Return the rq locked on entry, NULL if none.
+ */
+static struct rq *kf_switch_rq_lock(struct rq *rq, unsigned long *flags)
+{
+ struct rq *entry_rq;
+
+ local_irq_save(*flags);
+
+ entry_rq = scx_locked_rq();
+ if (entry_rq) {
+ if (entry_rq != rq)
+ switch_rq_lock(entry_rq, rq);
+ } else {
+ raw_spin_rq_lock(rq);
+ }
+ return entry_rq;
+}
+
+/* undo kf_switch_rq_lock(), @rq is the rq locked now */
+static void kf_restore_rq_lock(struct rq *rq, struct rq *entry_rq, unsigned long flags)
+{
+ if (entry_rq) {
+ if (entry_rq != rq)
+ switch_rq_lock(rq, entry_rq);
+ } else {
+ scx_rq_lock_drop(rq);
+ raw_spin_rq_unlock_irqrestore(rq, flags);
+ }
+}
+
static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
struct task_struct *p, u64 dsq_id, u64 enq_flags,
bool priq)
@@ -9053,27 +9828,8 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
if (unlikely(READ_ONCE(sch->aborting)))
return false;
- /*
- * Can be called from either ops.dispatch() holding the dispatched rq's
- * lock or any context where no rq lock is held. If latter, lock @p's
- * task_rq which we'll likely need anyway.
- */
src_rq = task_rq(p);
-
- local_irq_save(flags);
-
- /*
- * Under core scheduling, dispatch can run for a sibling rq, so the
- * locked rq is not necessarily this CPU's.
- */
- locked_rq = scx_locked_rq();
-
- if (locked_rq) {
- if (locked_rq != src_rq)
- switch_rq_lock(locked_rq, src_rq);
- } else {
- raw_spin_rq_lock(src_rq);
- }
+ locked_rq = kf_switch_rq_lock(src_rq, &flags);
p_rq = src_rq;
raw_spin_lock(&src_dsq->lock);
@@ -9112,13 +9868,7 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
p_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq);
dispatched = true;
out:
- if (locked_rq) {
- if (locked_rq != p_rq)
- switch_rq_lock(p_rq, locked_rq);
- } else {
- scx_rq_lock_drop(p_rq);
- raw_spin_rq_unlock_irqrestore(p_rq, flags);
- }
+ kf_restore_rq_lock(p_rq, locked_rq, flags);
kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE |
__SCX_DSQ_ITER_HAS_VTIME);
@@ -9352,6 +10102,8 @@ BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU)
BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU)
BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU)
BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU)
+/* also in scx_kfunc_ids_unlocked, same contexts as scx_bpf_dsq_move() */
+BTF_ID_FLAGS(func, scx_bpf_dsq_reenq_task, KF_RCU)
#ifdef CONFIG_EXT_SUB_SCHED
BTF_ID_FLAGS(func, scx_bpf_sub_dispatch, KF_IMPLICIT_ARGS)
#endif
@@ -9467,6 +10219,7 @@ BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU)
BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU)
BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU)
BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU)
+BTF_ID_FLAGS(func, scx_bpf_dsq_reenq_task, KF_RCU)
/* also in scx_kfunc_ids_select_cpu: also callable from ops.select_cpu()/ops.enqueue() */
BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU)
BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU)
@@ -9511,8 +10264,10 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice,
return false;
/*
- * Directly write only when we hold the lock of the rq @p is queued or
- * running on. See the write rules above.
+ * Directly write only when we hold the lock of the rq @p is queued on or
+ * provides the current scheduling context for. Under proxy execution,
+ * rq->donor owns and consumes the slice while rq->curr executes on its
+ * behalf. See the slice write rules above.
*
* While @p is queued on a user DSQ or in the BPF scheduler,
* synchronization is the scheduler's responsibility. This write can
@@ -9526,7 +10281,7 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice,
locked_rq = scx_locked_rq();
if (!locked_rq ||
(READ_ONCE(p->scx.runnable_cpu) != cpu_of(locked_rq) &&
- !task_current(locked_rq, p))) {
+ !task_current_donor(locked_rq, p))) {
set_task_slice_oob(sch, p, slice);
return true;
}
@@ -9567,18 +10322,56 @@ __bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime,
return true;
}
+/**
+ * scx_bpf_task_set_lazy_resched - Set task's tick reschedule mode
+ * @p: task of interest
+ * @lazy: whether slice expiry should request lazy rescheduling
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * Choose whether depletion of @p's slice at the scheduler tick requests lazy or
+ * immediate rescheduling. This is a persistent per-task policy and does not
+ * itself clear @p's slice or request rescheduling, unlike the
+ * %SCX_ENQ_PREEMPT_LAZY and %SCX_KICK_PREEMPT_LAZY one-shot operations.
+ *
+ * With lazy rescheduling enabled, a task interrupted in user space still
+ * reschedules before returning to user space. A task interrupted in the kernel
+ * may continue until its next return to user space or the next scheduler tick,
+ * which promotes the request. If lazy preemption is disabled at runtime, the
+ * request behaves like an immediate reschedule.
+ *
+ * @p must be on the calling scheduler.
+ *
+ * Return %true on success, %false if @p is not on the calling scheduler.
+ */
+__bpf_kfunc bool scx_bpf_task_set_lazy_resched(struct task_struct *p, bool lazy,
+ const struct bpf_prog_aux *aux)
+{
+ struct scx_sched *sch;
+
+ /*
+ * clang's register locations for @lazy make pahole 1.32 drop this kfunc
+ * from BTF. Keep @lazy in memory so its location names no register.
+ */
+ barrier_data(&lazy);
+ guard(rcu)();
+ sch = scx_prog_sched(aux);
+ if (unlikely(!sch || !scx_task_on_sched(sch, p)))
+ return false;
+
+ WRITE_ONCE(p->scx.lazy_resched, lazy);
+ return true;
+}
+
void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags)
{
struct scx_sched_pcpu *pcpu;
struct rq *this_rq;
unsigned long irq_flags;
- /*
- * The per-cpu kick list is guarded only by local_irq_save(), which does
- * not mask NMIs, so kicking from NMI could corrupt it and is unsupported.
- */
- if (unlikely(in_nmi())) {
- scx_error(sch, "scx_bpf_kick_cpu() called from NMI");
+ if (!scx_kf_allowed_ctx(sch))
+ return;
+ if (unlikely(flags & ~SCX_KICK_ALL_FLAGS)) {
+ scx_error(sch, "invalid kick flags 0x%llx", flags);
return;
}
@@ -9606,7 +10399,8 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags)
if (flags & SCX_KICK_IDLE) {
struct rq *target_rq = cpu_rq(cpu);
- if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_WAIT)))
+ if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_PREEMPT_LAZY |
+ SCX_KICK_WAIT)))
scx_error(sch, "PREEMPT/WAIT cannot be used with SCX_KICK_IDLE");
if (raw_spin_rq_trylock(target_rq)) {
@@ -9620,12 +10414,17 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags)
}
cpumask_set_cpu(cpu, pcpu->cpus_to_kick_if_idle);
} else {
- cpumask_set_cpu(cpu, pcpu->cpus_to_kick);
-
+ /*
+ * Accumulate requests and resolve their precedence at delivery.
+ */
if (flags & SCX_KICK_PREEMPT)
cpumask_set_cpu(cpu, pcpu->cpus_to_preempt);
+ if (flags & SCX_KICK_PREEMPT_LAZY)
+ cpumask_set_cpu(cpu, pcpu->cpus_to_preempt_lazy);
if (flags & SCX_KICK_WAIT)
cpumask_set_cpu(cpu, pcpu->cpus_to_wait);
+ if (!(flags & SCX_KICK_PREEMPT_LAZY))
+ cpumask_set_cpu(cpu, pcpu->cpus_to_kick);
}
if (list_empty(&pcpu->to_kick_node))
@@ -9665,8 +10464,9 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux
* cid-addressed equivalent of scx_bpf_kick_cpu(). An invalid @cid aborts the
* scheduler via scx_cid_to_cpu(). Caps are enforced on the delivery path: a
* kick is dropped if the caller lacks baseline access on @cid, and a
- * %SCX_KICK_PREEMPT degrades to a plain reschedule if the caller lacks
- * %SCX_CAP_PREEMPT for a task outside its subtree.
+ * %SCX_KICK_PREEMPT or %SCX_KICK_PREEMPT_LAZY request degrades to a plain
+ * reschedule if the caller lacks %SCX_CAP_PREEMPT for a task outside its
+ * subtree.
*/
__bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux)
{
@@ -9748,8 +10548,13 @@ __bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux)
guard(rcu)();
sch = scx_prog_sched(aux);
- if (sch)
- destroy_dsq(sch, dsq_id);
+ if (unlikely(!sch))
+ return;
+
+ if (!scx_kf_allowed_ctx(sch))
+ return;
+
+ destroy_dsq(sch, dsq_id);
}
/**
@@ -9786,6 +10591,9 @@ __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id,
if (unlikely(!sch))
return -ENODEV;
+ if (!scx_kf_allowed_ctx(sch))
+ return -EDEADLK;
+
if (flags & ~__SCX_DSQ_ITER_USER_FLAGS)
return -EINVAL;
@@ -9820,7 +10628,8 @@ __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *i
* bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator
* @it: iterator to destroy
*
- * Undo bpf_iter_scx_dsq_new().
+ * Undo bpf_iter_scx_dsq_new(). If the iteration marked tasks with
+ * scx_bpf_dsq_reenq_task(), schedule their re-enqueue.
*/
__bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it)
{
@@ -9836,6 +10645,13 @@ __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it)
list_del_init(&kit->cursor.node);
raw_spin_unlock_irqrestore(&kit->dsq->lock, flags);
}
+
+ if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_REENQ) {
+ guard(preempt)();
+ schedule_dsq_reenq(kit->dsq->sched, kit->dsq, SCX_REENQ_MARKED,
+ scx_locked_rq());
+ }
+
kit->dsq = NULL;
}
@@ -9888,7 +10704,8 @@ __bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id,
* - Local DSQs (%SCX_DSQ_LOCAL or %SCX_DSQ_LOCAL_ON | $cpu)
* - User DSQs
*
- * Re-enqueues are performed asynchronously. Can be called from anywhere.
+ * Re-enqueues are performed asynchronously. Can be called from anywhere. A
+ * re-enqueue of an inactive cpu's local DSQ is ignored.
*
* %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler
* operation is locked to - e.g. the rq being dispatched for in ops.dispatch() -
@@ -9912,6 +10729,9 @@ __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags,
return;
}
+ if (!scx_kf_allowed_ctx(sch))
+ return;
+
/* not specifying any filter bits is the same as %SCX_REENQ_ANY */
if (!(reenq_flags & __SCX_REENQ_FILTER_MASK))
reenq_flags |= SCX_REENQ_ANY;
@@ -9921,6 +10741,64 @@ __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags,
}
/**
+ * scx_bpf_dsq_reenq_task - Re-enqueue a task from a DSQ iteration
+ * @it__iter: DSQ iterator in progress
+ * @p: task to re-enqueue
+ *
+ * Mark @p, which is on the DSQ iterated by @it__iter and was queued before the
+ * iteration started, for re-enqueue. When the iteration ends, the DSQ's marked
+ * tasks go through ops.enqueue() with %SCX_ENQ_REENQ in DSQ order and the other
+ * tasks on the DSQ keep their positions. A marked task that leaves the DSQ in
+ * the meantime is not re-enqueued. The mark belongs to @p, so a re-enqueue
+ * scheduled by another iteration of the DSQ may re-enqueue @p before this
+ * iteration ends. @p must be on the DSQ's scheduler.
+ *
+ * Re-enqueues are performed asynchronously. Can be called from ops.dispatch()
+ * or any BPF context which doesn't hold a rq lock (e.g. BPF timers or SYSCALL
+ * programs).
+ *
+ * Return %true if @p was marked, %false if @p left the DSQ or was queued after
+ * the iteration started, @p is on another scheduler, or the iterator holds no
+ * DSQ.
+ */
+__bpf_kfunc bool scx_bpf_dsq_reenq_task(struct bpf_iter_scx_dsq *it__iter,
+ struct task_struct *p)
+{
+ struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter;
+ struct scx_dispatch_q *dsq = kit->dsq;
+ struct rq *src_rq, *entry_rq;
+ unsigned long flags;
+ bool marked = false;
+
+ /* NULL after a failed bpf_iter_scx_dsq_new(), see scx_dsq_move() */
+ if (unlikely(!dsq))
+ return false;
+
+ /* the cursor check needs @p's rq locked */
+ src_rq = task_rq(p);
+ entry_rq = kf_switch_rq_lock(src_rq, &flags);
+
+ scoped_guard (raw_spinlock, &dsq->lock) {
+ if (nldsq_cursor_lost_task(&kit->cursor, src_rq, dsq, p))
+ break;
+
+ if (unlikely(!scx_task_on_sched(dsq->sched, p))) {
+ scx_error(dsq->sched, "scx_bpf_dsq_reenq_task() on %s[%d] but the task belongs to a different scheduler",
+ p->comm, p->pid);
+ break;
+ }
+
+ p->scx.dsq_flags |= SCX_TASK_DSQ_REENQ;
+ /* the walk is scheduled once, by bpf_iter_scx_dsq_destroy() */
+ kit->cursor.flags |= __SCX_DSQ_ITER_HAS_REENQ;
+ marked = true;
+ }
+
+ kf_restore_rq_lock(src_rq, entry_rq, flags);
+ return marked;
+}
+
+/**
* scx_bpf_reenqueue_local___v2 - Re-enqueue tasks on a local DSQ
* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
*
@@ -10299,6 +11177,9 @@ __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_au
if (unlikely(!sch))
return;
+ if (!scx_kf_allowed_ctx(sch))
+ return;
+
scx_cpuperf_set(sch, cpu, perf);
}
@@ -10324,6 +11205,10 @@ __bpf_kfunc s32 scx_bpf_cidperf_set(s32 cid, u32 perf,
sch = scx_prog_sched(aux);
if (unlikely(!sch))
return -ENODEV;
+
+ if (!scx_kf_allowed_ctx(sch))
+ return -EDEADLK;
+
cpu = scx_cid_to_cpu(sch, cid);
if (cpu < 0)
return cpu;
@@ -10403,8 +11288,7 @@ __bpf_kfunc const void *scx_bpf_online_cmask(const struct bpf_prog_aux *aux)
if (unlikely(!online))
return NULL;
- /* BPF rebases by the low 32 bits, like __arena callback args */
- return (void *)((unsigned long)online - sch->arena_kern_base);
+ return (void *)scx_kaddr_to_arena(sch, online);
}
/**
@@ -10457,12 +11341,17 @@ __bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask)
}
/**
- * scx_bpf_task_running - Is task currently running?
+ * scx_bpf_task_running - Is task the current scheduling context?
* @p: task of interest
+ *
+ * Under proxy execution, this reports the donor rather than the task whose
+ * code is physically executing. The physical execution context is intentionally
+ * not exposed to the BPF scheduler, which continues to observe the donor as the
+ * running scheduling context.
*/
__bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p)
{
- return task_rq(p)->curr == p;
+ return rcu_access_pointer(task_rq(p)->donor) == p;
}
/**
@@ -10523,10 +11412,15 @@ __bpf_kfunc struct rq *scx_bpf_locked_rq(const struct bpf_prog_aux *aux)
}
/**
- * scx_bpf_cpu_curr - Return remote CPU's curr task
+ * scx_bpf_cpu_curr - Return remote CPU's current scheduling context
* @cpu: CPU of interest
* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
*
+ * Under proxy execution, this returns the donor, which supplies the scheduling
+ * policy and runtime budget, rather than the task whose code is physically
+ * executing. The physical execution context is intentionally not exposed to
+ * the BPF scheduler.
+ *
* Callers must hold RCU read lock (KF_RCU).
*/
__bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_aux *aux)
@@ -10542,7 +11436,7 @@ __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_
if (!scx_cpu_valid(sch, cpu, NULL))
return NULL;
- return rcu_dereference(cpu_rq(cpu)->curr);
+ return rcu_dereference(cpu_rq(cpu)->donor);
}
/**
@@ -10566,7 +11460,7 @@ __bpf_kfunc struct task_struct *scx_bpf_cid_curr(s32 cid, const struct bpf_prog_
cpu = scx_cid_to_cpu(sch, cid);
if (cpu < 0)
return NULL;
- return rcu_dereference(cpu_rq(cpu)->curr);
+ return rcu_dereference(cpu_rq(cpu)->donor);
}
/**
@@ -10740,6 +11634,23 @@ out:
cgroup_get(cgrp);
return cgrp;
}
+
+/**
+ * scx_bpf_cgroup_nr_cpus - Return the number of CPUs in a cgroup's cpuset
+ * @cgrp: cgroup of interest
+ *
+ * Return the number of CPUs in @cgrp's effective cpuset, which is inherited
+ * from the nearest ancestor with the cpuset controller enabled. This matches
+ * the count used by fair's group share calculation and lets hierarchical BPF
+ * schedulers bound a group's weight by the CPUs it can actually run on.
+ *
+ * The value is a snapshot and may change at any time through cpuset updates or
+ * CPU hotplug. Schedulers should re-read it when recomputing group state.
+ */
+__bpf_kfunc u32 scx_bpf_cgroup_nr_cpus(struct cgroup *cgrp)
+{
+ return cpuset_num_cpus(cgrp);
+}
#endif /* CONFIG_CGROUP_SCHED */
__bpf_kfunc_end_defs();
@@ -10747,6 +11658,7 @@ __bpf_kfunc_end_defs();
BTF_KFUNCS_START(scx_kfunc_ids_any)
BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS | KF_RCU);
BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS | KF_RCU);
+BTF_ID_FLAGS(func, scx_bpf_task_set_lazy_resched, KF_IMPLICIT_ARGS | KF_RCU);
BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, scx_bpf_kick_cid, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS)
@@ -10786,6 +11698,7 @@ BTF_ID_FLAGS(func, scx_bpf_now)
BTF_ID_FLAGS(func, scx_bpf_events, KF_IMPLICIT_ARGS)
#ifdef CONFIG_CGROUP_SCHED
BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS | KF_RCU | KF_ACQUIRE)
+BTF_ID_FLAGS(func, scx_bpf_cgroup_nr_cpus, KF_RCU)
#endif
BTF_ID_FLAGS(func, scx_bpf_sub_grant, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, scx_bpf_sub_revoke, KF_IMPLICIT_ARGS)
@@ -10879,6 +11792,7 @@ static const u32 scx_kf_allow_flags[] = {
[SCX_OP_IDX(sub_attach)] = SCX_KF_ALLOW_UNLOCKED,
[SCX_OP_IDX(sub_detach)] = SCX_KF_ALLOW_UNLOCKED,
[SCX_OP_IDX(sub_ecaps_updated)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH,
+ [SCX_OP_IDX(sub_child_ecaps_updated)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH,
[SCX_OP_IDX(cpu_online)] = SCX_KF_ALLOW_UNLOCKED,
[SCX_OP_IDX(cpu_offline)] = SCX_KF_ALLOW_UNLOCKED,
[SCX_OP_IDX(init_cids)] = SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT_CIDS,
@@ -11022,6 +11936,8 @@ static int __init scx_init(void)
CID_OFFSET_MATCH(sub_detach, sub_detach);
CID_OFFSET_MATCH(sub_caps_updated, sub_caps_updated);
CID_OFFSET_MATCH(sub_ecaps_updated, sub_ecaps_updated);
+ CID_OFFSET_MATCH(sub_cid_sched_updated, sub_cid_sched_updated);
+ CID_OFFSET_MATCH(sub_child_ecaps_updated, sub_child_ecaps_updated);
CID_OFFSET_MATCH(init_cids, init_cids);
CID_OFFSET_MATCH(init, init);
CID_OFFSET_MATCH(exit, exit);
diff --git a/kernel/sched/ext/ext.h b/kernel/sched/ext/ext.h
index 0b7fc46aee08..d32bc01071b0 100644
--- a/kernel/sched/ext/ext.h
+++ b/kernel/sched/ext/ext.h
@@ -18,9 +18,28 @@ bool scx_can_stop_tick(struct rq *rq);
void scx_rq_activate(struct rq *rq);
void scx_rq_deactivate(struct rq *rq);
int scx_check_setscheduler(struct task_struct *p, int policy);
+void scx_prepare_setscheduler(struct task_struct *p, int policy);
bool task_should_scx(int policy);
bool scx_allow_ttwu_queue(const struct task_struct *p);
+bool __scx_allow_proxy_exec(const struct task_struct *p);
+void scx_proxy_donor_start(struct rq *rq);
+void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next);
void init_sched_ext_class(void);
+void __scx_update_idle(struct rq *rq, bool idle, bool do_notify);
+
+static inline bool scx_allow_proxy_exec(const struct task_struct *p)
+{
+ if (scx_enabled())
+ return __scx_allow_proxy_exec(p);
+
+ return true;
+}
+
+static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify)
+{
+ if (scx_enabled())
+ __scx_update_idle(rq, idle, do_notify);
+}
static inline u32 scx_cpuperf_target(s32 cpu)
{
@@ -52,24 +71,18 @@ static inline bool scx_can_stop_tick(struct rq *rq) { return true; }
static inline void scx_rq_activate(struct rq *rq) {}
static inline void scx_rq_deactivate(struct rq *rq) {}
static inline int scx_check_setscheduler(struct task_struct *p, int policy) { return 0; }
+static inline void scx_prepare_setscheduler(struct task_struct *p, int policy) {}
static inline bool task_on_scx(const struct task_struct *p) { return false; }
static inline bool scx_allow_ttwu_queue(const struct task_struct *p) { return true; }
+static inline bool scx_allow_proxy_exec(const struct task_struct *p) { return true; }
+static inline void scx_proxy_donor_start(struct rq *rq) {}
+static inline void scx_proxy_reenqueue_retry(struct rq *rq,
+ struct task_struct *next) {}
static inline void init_sched_ext_class(void) {}
+static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) {}
#endif /* CONFIG_SCHED_CLASS_EXT */
-#ifdef CONFIG_SCHED_CLASS_EXT
-void __scx_update_idle(struct rq *rq, bool idle, bool do_notify);
-
-static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify)
-{
- if (scx_enabled())
- __scx_update_idle(rq, idle, do_notify);
-}
-#else
-static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) {}
-#endif
-
#ifdef CONFIG_CGROUP_SCHED
#ifdef CONFIG_EXT_GROUP_SCHED
void scx_tg_init(struct task_group *tg);
diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c
index aa9fb6de0ad6..922fbc3065cc 100644
--- a/kernel/sched/ext/idle.c
+++ b/kernel/sched/ext/idle.c
@@ -153,7 +153,18 @@ static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, i
nodemask_t *unvisited;
s32 cpu = -EBUSY;
- preempt_disable();
+ /*
+ * @per_cpu_unvisited is per-CPU scratch and the idle kfuncs can be
+ * called from IRQ-enabled contexts, so mask IRQs to keep a nested
+ * invocation from clobbering the mask an outer invocation is still
+ * iterating.
+ *
+ * NMI nesting is not handled: there is no legitimate reason to call
+ * pick_idle from NMI and doing so poses no crash risk, so such a
+ * caller is on its own.
+ */
+ guard(irqsave)();
+
unvisited = this_cpu_ptr(&per_cpu_unvisited);
/*
@@ -183,7 +194,6 @@ static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, i
if (cpu >= 0)
break;
}
- preempt_enable();
return cpu;
}
@@ -952,12 +962,43 @@ static bool is_bpf_migration_disabled(const struct task_struct *p)
return p->migration_disabled;
}
+static s32 select_cpu_restricted(struct task_struct *p, s32 prev_cpu,
+ const struct cpumask *allowed, u64 flags,
+ bool migration_disabled)
+{
+ s32 cpu;
+
+ if (migration_disabled)
+ cpu = task_cpu(p);
+ else
+ cpu = cpumask_first(p->cpus_ptr);
+
+ if (!cpumask_test_cpu(cpu, p->cpus_ptr))
+ return -EBUSY;
+
+ if (allowed && !cpumask_test_cpu(cpu, allowed))
+ return -EBUSY;
+
+ if (flags & SCX_PICK_IDLE_IN_NODE) {
+ int node = scx_cpu_node_if_enabled(prev_cpu);
+
+ if (scx_cpu_node_if_enabled(cpu) != node)
+ return -EBUSY;
+ }
+
+ if (!scx_idle_test_and_clear_cpu(cpu))
+ return -EBUSY;
+
+ return cpu;
+}
+
static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p,
s32 prev_cpu, u64 wake_flags,
const struct cpumask *allowed, u64 flags)
{
unsigned long irq_flags;
bool we_locked = false;
+ bool migration_disabled;
s32 cpu;
if (!scx_cpu_valid(sch, prev_cpu, NULL))
@@ -994,20 +1035,17 @@ static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p,
/*
* This may also be called from ops.enqueue(), so we need to handle
- * per-CPU tasks as well. For these tasks, we can skip all idle CPU
- * selection optimizations and simply check whether the previously
- * used CPU is idle and within the allowed cpumask.
+ * per-CPU tasks as well. For these tasks, skip general idle CPU
+ * selection and validate the task's only usable CPU directly.
*/
- if (p->nr_cpus_allowed == 1 || is_bpf_migration_disabled(p)) {
- if (cpumask_test_cpu(prev_cpu, allowed ?: p->cpus_ptr) &&
- scx_idle_test_and_clear_cpu(prev_cpu))
- cpu = prev_cpu;
- else
- cpu = -EBUSY;
- } else {
+ migration_disabled = is_bpf_migration_disabled(p);
+
+ if (p->nr_cpus_allowed == 1 || migration_disabled)
+ cpu = select_cpu_restricted(p, prev_cpu, allowed, flags,
+ migration_disabled);
+ else
cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags,
allowed ?: p->cpus_ptr, flags);
- }
if (we_locked)
raw_spin_unlock_irqrestore(&p->pi_lock, irq_flags);
diff --git a/kernel/sched/ext/inlines.h b/kernel/sched/ext/inlines.h
index ed423bcc26b8..8de9d8fb4b25 100644
--- a/kernel/sched/ext/inlines.h
+++ b/kernel/sched/ext/inlines.h
@@ -70,7 +70,13 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
#endif /* CONFIG_EXT_SUB_SCHED */
}
- if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq))
+ /*
+ * scx_rq_online() can't be used. Its cpu_active() test goes false
+ * before CPU hotplug waits for an RCU grace period, and
+ * rq_offline_scx() moves this CPU's tasks to the local DSQ only after
+ * the wait. The grace period can depend on those tasks running.
+ */
+ if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !(rq->scx.flags & SCX_RQ_ONLINE))
return SCX_DSP_NONE;
dspc->rq = rq;
@@ -129,8 +135,10 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
* scheduler's ops.dispatch() doesn't yield any tasks.
*/
if (scx_bypass_dsp_enabled(sch) &&
- scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0))
+ scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) {
+ __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1);
return SCX_DSP_LOCAL;
+ }
return SCX_DSP_NONE;
}
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 3464e0f113c1..29e69a83d876 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -215,6 +215,34 @@ enum scx_ops_flags {
*/
SCX_OPS_TID_TO_TASK = 1LLU << 8,
+ /*
+ * If set, tasks default to requesting lazy rescheduling when their
+ * slice runs out at the tick, the way fair.c expires a slice from
+ * update_curr(). The default is copied to p->scx.lazy_resched
+ * immediately before ops.enable(), after which
+ * scx_bpf_task_set_lazy_resched() may override it per task.
+ *
+ * A task in user space still reschedules on the way back from the tick;
+ * a task in the kernel runs on to its next return to user space or to
+ * the next tick, which promotes the request. When lazy preemption is
+ * disabled at runtime, the request behaves like an immediate
+ * reschedule. Rescheduling while bypassing stays immediate.
+ */
+ SCX_OPS_LAZY_RESCHED = 1LLU << 9,
+
+ /*
+ * If set, mutex-blocked tasks remain runnable as proxy donors and are
+ * passed to ops.enqueue() with %SCX_ENQ_BLOCKED. The BPF scheduler controls
+ * when donors are dispatched and whether they should preempt other work.
+ *
+ * If clear, mutex-blocked tasks are removed from the runqueue normally
+ * and cannot donate their scheduling context through proxy execution.
+ *
+ * For blocked donors, this flag takes precedence over
+ * %SCX_OPS_ENQ_EXITING and %SCX_OPS_ENQ_MIGRATION_DISABLED.
+ */
+ SCX_OPS_ENQ_BLOCKED = 1LLU << 10,
+
SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE |
SCX_OPS_ENQ_LAST |
SCX_OPS_ENQ_EXITING |
@@ -223,7 +251,9 @@ enum scx_ops_flags {
SCX_OPS_SWITCH_PARTIAL |
SCX_OPS_BUILTIN_IDLE_PER_NODE |
SCX_OPS_ALWAYS_ENQ_IMMED |
- SCX_OPS_TID_TO_TASK,
+ SCX_OPS_TID_TO_TASK |
+ SCX_OPS_LAZY_RESCHED |
+ SCX_OPS_ENQ_BLOCKED,
/* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */
__SCX_OPS_INTERNAL_MASK = 0xffLLU << 56,
@@ -250,6 +280,31 @@ struct scx_exit_task_args {
bool cancelled;
};
+/**
+ * struct scx_enable_args - Argument container for cid-form ops.enable()
+ * @cmask_arena_addr: BPF arena address of the cmask of cids the task may run on
+ *
+ * @cmask_arena_addr is the task's affinity as it enters the scheduler.
+ * set_cmask() delivers the same mask right after enable(), before set_weight()
+ * and the first enqueue, then every affinity change afterwards, and is never
+ * called before enable(). A scheduler may therefore track affinity in
+ * set_cmask() alone.
+ *
+ * The kernel builds the mask in the scheduler arena from its own geometry, so
+ * the header is valid regardless of what the scheduler last wrote there. The
+ * memory is per-cpu scratch reused once the callback returns: copy the bits
+ * out, don't keep the address. The set_cmask() argument follows the same rules.
+ *
+ * The address is a plain value rather than a typed pointer because BTF can't
+ * mark a struct member as an arena pointer yet and a pointer member would reach
+ * the program typed as a kernel pointer. Cast it to struct scx_cmask __arena *
+ * before use. Once arena members can be typed, a typed alias will join this
+ * field in an anonymous union at the same offset.
+ */
+struct scx_enable_args {
+ u64 cmask_arena_addr;
+};
+
/* argument container for ops.cgroup_init() */
struct scx_cgroup_init_args {
/* the weight of the cgroup [1..10000] */
@@ -311,6 +366,18 @@ struct scx_sub_detach_args {
char *cgroup_path;
};
+/*
+ * What a sched sees running on a cid, as reported by
+ * ops.sub_cid_sched_updated(): NONE when the cid is idle or runs a task outside
+ * the sched's subtree, SELF for the sched itself, or a direct child by its
+ * cgroup id. The sentinels lie outside the cgroup id space, whose low 32 bits
+ * are an idr value of at most INT_MAX.
+ */
+enum scx_cid_sched_consts {
+ SCX_CID_SCHED_NONE = (u64)-2,
+ SCX_CID_SCHED_SELF = (u64)-1,
+};
+
/**
* struct sched_ext_ops - Operation table for BPF scheduler implementation
*
@@ -452,6 +519,12 @@ struct sched_ext_ops {
* Therefore, always use scx_bpf_task_cpu(@p) to determine the
* target CPU the task is going to use.
*
+ * Under proxy execution, the BPF scheduler continues to observe the
+ * donor as the current scheduling context. A blocked donor enters a
+ * ->running()/->stopping() session while its scheduling context drives
+ * the lock owner. The lock owner executing on its behalf is intentionally
+ * not reported through these callbacks.
+ *
* See ->runnable() for explanation on the task state notifiers.
*/
void (*running)(struct task_struct *p);
@@ -803,6 +876,11 @@ struct sched_ext_ops {
/**
* @sub_detach: Detach a sub-scheduler
* @args: argument container, see the struct definition
+ *
+ * The sub-scheduler holds no caps by this point and can no longer
+ * affect any cid. Its caps went away without a
+ * sub_child_ecaps_updated() report, so whatever was delegated to it,
+ * e.g. cpuperf targets, is the parent's to restore here.
*/
void (*sub_detach)(struct scx_sub_detach_args *args);
@@ -839,6 +917,41 @@ struct sched_ext_ops {
*/
void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after);
+ /**
+ * @sub_cid_sched_updated: The sched running on a cid changed
+ * @cid: cid whose running sched changed
+ * @sched: what this sched now sees running there
+ *
+ * Invoked when the sched owning the task running on @cid changes.
+ * @sched is %SCX_CID_SCHED_NONE when no task of this sched's subtree
+ * runs there, %SCX_CID_SCHED_SELF for its own task, or the cgroup id of
+ * the direct child whose subtree the task belongs to. Changes inside a
+ * child's subtree, whether across tasks or nested sub-scheds, are not
+ * reported. Every sched for which @sched changes is notified.
+ *
+ * Runs with the rq lock held on the context switch path.
+ */
+ void (*sub_cid_sched_updated)(s32 cid, u64 sched);
+
+ /**
+ * @sub_child_ecaps_updated: A child's effective caps on a cid changed
+ * @cgroup_id: cgroup id of the direct child
+ * @cid: the cid whose effective caps changed
+ * @before: the child's effective caps as of the last delivery
+ * @after: the child's effective caps now
+ *
+ * Invoked right after the child's sub_ecaps_updated(), once a grant or
+ * revoke is in effect on the cpu. From here on the parent can act on a
+ * cap the child lost, e.g. reset the cpuperf target or schedule on the
+ * cid. A detaching child's caps go away without a report, see
+ * sub_detach(). A cpu going offline drops caps without a report.
+ *
+ * Runs in dispatch context with rq lock held, and can perform all
+ * operations allowed in ops.dispatch() including inserting/moving
+ * tasks.
+ */
+ void (*sub_child_ecaps_updated)(u64 cgroup_id, s32 cid, u64 before, u64 after);
+
/*
* All online ops must come before ops.cpu_online().
*/
@@ -886,6 +999,9 @@ struct sched_ext_ops {
* ops.exit() is also called on ops.init() failure, which is a bit
* unusual. This is to allow rich reporting through @info on how
* ops.init() failed.
+ *
+ * A sub-scheduler holds no caps by the time ops.exit() runs. Its
+ * cap-gated kfuncs are denied.
*/
void (*exit)(struct scx_exit_info *info);
@@ -1037,6 +1153,7 @@ struct sched_ext_ops {
* - dispatch -> dispatch (cpu arg is now cid)
* - update_idle -> update_idle (cpu arg is now cid)
* - set_cpumask -> set_cmask (cmask instead of cpumask)
+ * - enable -> enable (takes struct scx_enable_args)
* - cpu_online -> cid_online
* - cpu_offline -> cid_offline
* - dump_cpu -> dump_cid
@@ -1070,7 +1187,7 @@ struct sched_ext_ops_cid {
struct scx_init_task_args *args);
void (*exit_task)(struct task_struct *p,
struct scx_exit_task_args *args);
- void (*enable)(struct task_struct *p);
+ void (*enable)(struct task_struct *p, struct scx_enable_args *args);
void (*disable)(struct task_struct *p);
void (*dump)(struct scx_dump_ctx *ctx);
void (*dump_cid)(struct scx_dump_ctx *ctx, s32 cid, bool idle);
@@ -1092,6 +1209,8 @@ struct sched_ext_ops_cid {
void (*sub_detach)(struct scx_sub_detach_args *args);
void (*sub_caps_updated)(const struct scx_cmask *cmask__arena, u64 caps);
void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after);
+ void (*sub_cid_sched_updated)(s32 cid, u64 sched);
+ void (*sub_child_ecaps_updated)(u64 cgroup_id, s32 cid, u64 before, u64 after);
void (*cid_online)(s32 cid);
void (*cid_offline)(s32 cid);
s32 (*init_cids)(void);
@@ -1333,6 +1452,7 @@ struct scx_sched_pcpu {
cpumask_var_t cpus_to_kick;
cpumask_var_t cpus_to_kick_if_idle;
cpumask_var_t cpus_to_preempt;
+ cpumask_var_t cpus_to_preempt_lazy;
cpumask_var_t cpus_to_wait;
struct list_head to_kick_node;
@@ -1354,7 +1474,10 @@ struct scx_sched_pcpu {
struct llist_node ecaps_to_sync_node;
/* owed a forced update_idle() re-notify on this cpu */
bool idle_renotify;
- /* effective caps as of the last sub_ecaps_updated() delivery */
+ /*
+ * ecaps as of the last sub_ecaps_updated() and
+ * sub_child_ecaps_updated() delivery. See __scx_process_sync_ecaps().
+ */
u64 reported_ecaps;
/*
@@ -1413,15 +1536,16 @@ struct scx_sched_pnode {
* the allocation pattern.
*
* ENQ_IMMED insert an IMMED task onto the cid's local DSQ
- * - kick the cid's cpu (except SCX_KICK_PREEMPT)
+ * - kick the cid's cpu (except SCX_KICK_PREEMPT and
+ * SCX_KICK_PREEMPT_LAZY)
*
* ENQ insert any task onto the cid's local DSQ (implies ENQ_IMMED)
*
* PREEMPT preempt any task running on the cid regardless of the owning
* sched (implies ENQ). Preempting a task in the sched's own subtree
* doesn't require any cap.
- * - SCX_ENQ_PREEMPT inserts
- * - SCX_KICK_PREEMPT kicks
+ * - SCX_ENQ_PREEMPT and SCX_ENQ_PREEMPT_LAZY insert
+ * - SCX_KICK_PREEMPT and SCX_KICK_PREEMPT_LAZY kick
*
* PERF control the cid's cpu power/perf management state, currently the
* cpufreq target set through scx_bpf_cidperf_set(). Hardware
@@ -1533,7 +1657,8 @@ struct scx_sched {
* by BUILD_BUG_ON in scx_init()). The anonymous union lets the kernel
* access either view of the same storage without function-pointer
* casts: use .ops for cpu-form and shared fields, .ops_cid for the
- * cid-renamed callbacks (set_cmask, select_cid, cid_online, ...).
+ * callbacks whose cid-form signature differs (set_cmask, enable,
+ * select_cid, cid_online, ...).
*/
union {
struct sched_ext_ops ops;
@@ -1556,9 +1681,9 @@ struct scx_sched {
uintptr_t arena_kern_base;
/*
- * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask
- * to ops_cid.set_cmask(). The kernel writes through the stored kern_va
- * and passes it to the callback's __arena argument.
+ * Per-CPU arena cmask the kernel fills from a task's cpumask and hands
+ * to ops_cid.enable() and ops_cid.set_cmask(). The stored pointers are
+ * the kernel addresses.
*/
struct scx_cmask * __percpu *set_cmask_scratch;
struct scx_cmask *online_cmask;
@@ -1669,6 +1794,19 @@ static inline void *scx_arena_to_kaddr(struct scx_sched *sch, const void *bpf_pt
return (void *)(sch->arena_kern_base + (u32)(uintptr_t)bpf_ptr);
}
+/**
+ * scx_kaddr_to_arena - Translate a kernel arena address to the BPF form
+ * @sch: scheduler whose arena hosts @kaddr
+ * @kaddr: kernel address inside @sch's arena
+ *
+ * __arena callback arguments need no translation. Addresses handed to BPF any
+ * other way, such as struct fields and kfunc return values, go through this.
+ */
+static inline uintptr_t scx_kaddr_to_arena(struct scx_sched *sch, const void *kaddr)
+{
+ return (uintptr_t)kaddr - sch->arena_kern_base;
+}
+
enum scx_wake_flags {
/* expose select WF_* flags as enums */
SCX_WAKE_FORK = WF_FORK,
@@ -1693,10 +1831,19 @@ enum scx_enq_flags {
SCX_ENQ_PREEMPT = 1LLU << 32,
/*
+ * Like %SCX_ENQ_PREEMPT, but request lazy rescheduling. The current
+ * task's slice is still cleared immediately so that the next scheduling
+ * boundary observes the new ordering. %SCX_ENQ_PREEMPT takes precedence
+ * if both are specified. Implies %SCX_ENQ_HEAD, which is all it means
+ * on a non-local DSQ, as with %SCX_ENQ_PREEMPT.
+ */
+ SCX_ENQ_PREEMPT_LAZY = 1LLU << 35,
+
+ /*
* Only allowed on local DSQs. Guarantees that the task either gets
* on the CPU immediately and stays on it, or gets reenqueued back
- * to the BPF scheduler. It will never linger on a local DSQ or be
- * silently put back after preemption.
+ * to the BPF scheduler. A blocked proxy donor may be put back on the
+ * local DSQ temporarily while proxy execution resolves its mutex owner.
*
* The protection persists until the next fresh enqueue - it
* survives SAVE/RESTORE cycles, slice extensions and preemption.
@@ -1734,11 +1881,24 @@ enum scx_enq_flags {
* %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with the
* %SCX_ENQ_LAST flag set.
*
- * The BPF scheduler is responsible for triggering a follow-up
- * scheduling event. Otherwise, Execution may stall.
+ * If the task is queued on the local DSQ of the CPU it was running on,
+ * it continues to run. Otherwise, the CPU goes idle. A scheduler that
+ * wants a full dispatch cycle on the CPU should kick it.
*/
SCX_ENQ_LAST = 1LLU << 41,
+ /*
+ * The task is blocked on a mutex and is being kept runnable as a proxy
+ * donor. Only passed to ops.enqueue() when %SCX_OPS_ENQ_BLOCKED is set.
+ *
+ * Blocking on the mutex does not enqueue the task by itself. A donor put
+ * with slice left stays at the head of the local DSQ, except when an
+ * IMMED donor is preempted or cannot stay local. It is passed to
+ * ops.enqueue() when its slice runs out, the IMMED placement cannot be
+ * kept, or proxy execution moves it to the owner's CPU.
+ */
+ SCX_ENQ_BLOCKED = 1LLU << 42,
+
/* high 8 bits are internal */
__SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56,
@@ -1751,6 +1911,15 @@ enum scx_enq_flags {
SCX_ENQ_SLICE_DFL = 1LLU << 62, /* carried slice is a default refill */
};
+/* Strip priority and carried slice state when diverting from a local DSQ. */
+static inline void scx_divert_strip_flags(struct task_struct *p, u64 *enq_flags)
+{
+ *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT |
+ SCX_ENQ_PREEMPT_LAZY | SCX_ENQ_HEAD |
+ SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL);
+ p->scx.flags &= ~SCX_TASK_IMMED;
+}
+
enum scx_deq_flags {
/* expose select DEQUEUE_* flags as enums */
SCX_DEQ_SLEEP = DEQUEUE_SLEEP,
@@ -1777,6 +1946,8 @@ enum scx_reenq_flags {
/* internal: kernel-issued on cap revoke, not accepted from BPF */
SCX_REENQ_CAP_REVOKE = 1LLU << 1,
+ /* internal: tasks marked by scx_bpf_dsq_reenq_task() */
+ SCX_REENQ_MARKED = 1LLU << 2,
__SCX_REENQ_FILTER_MASK = 0xffffLLU,
@@ -1818,6 +1989,15 @@ enum scx_kick_flags {
* is not on SCX.
*/
SCX_KICK_WAIT = 1LLU << 2,
+
+ /*
+ * Like %SCX_KICK_PREEMPT, but request lazy rescheduling. If combined
+ * with %SCX_KICK_PREEMPT or %SCX_KICK_WAIT, rescheduling is immediate.
+ */
+ SCX_KICK_PREEMPT_LAZY = 1LLU << 3,
+
+ SCX_KICK_ALL_FLAGS = SCX_KICK_IDLE | SCX_KICK_PREEMPT |
+ SCX_KICK_WAIT | SCX_KICK_PREEMPT_LAZY,
};
enum scx_tg_flags {
@@ -1958,6 +2138,11 @@ enum scx_ops_state {
* dequeue/requeue, the dispatcher can tell whether it still has a claim
* on the task being dispatched.
*
+ * QSEQ is generated from the per-task p->scx.ops_qseq counter so that
+ * it doesn't repeat across QUEUED instances of the same task even if
+ * the task moves between rqs. 0 is never used as a valid QSEQ since
+ * NONE and DISPATCHING map to this value.
+ *
* As some 32bit archs can't do 64bit store_release/load_acquire,
* p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on
* 32bit machines. The dispatch race window QSEQ protects is very narrow
@@ -2061,12 +2246,12 @@ struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter);
bool scx_set_task_slice(struct task_struct *p, u64 slice);
void scx_task_slice_ended(struct rq *rq, struct task_struct *p);
void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq);
+void scx_prepare_task_sched_change(struct task_struct *p);
void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p);
void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
int sticky_cpu);
void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p,
- u64 enq_flags, struct scx_dispatch_q *src_dsq,
- struct rq *dst_rq);
+ u64 enq_flags, struct rq *dst_rq);
bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq,
struct scx_dispatch_q *dsq, u64 enq_flags);
bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq);
@@ -2078,6 +2263,7 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags);
u64 __scx_bpf_now(struct rq *rq);
void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq,
u64 reenq_flags, struct rq *locked_rq);
+void scx_wait_dispatching(struct task_struct *p);
int __scx_init_task(struct scx_sched *sch, struct task_struct *p,
struct cgroup *cgrp, bool fork);
void scx_enable_task(struct scx_sched *sch, struct task_struct *p);
@@ -2122,6 +2308,22 @@ extern struct scx_sched *scx_enabling_sub_sched;
#define scx_error(sch, fmt, args...) \
scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args)
+/*
+ * Tracing progs can call kfuncs from NMI. Kfuncs that take scheduler locks or
+ * touch the kick lists, which are only protected by irq masking, can't run
+ * there, so abort the scheduler instead. scx_error() is NMI-safe.
+ */
+static __always_inline bool __scx_kf_allowed_ctx(struct scx_sched *sch, const char *who)
+{
+ if (unlikely(in_nmi())) {
+ scx_error(sch, "%s called from NMI", who);
+ return false;
+ }
+ return true;
+}
+
+#define scx_kf_allowed_ctx(sch) __scx_kf_allowed_ctx((sch), __func__)
+
/**
* scx_root_protected_live - Root sched for paths that only run while live
*
@@ -2212,6 +2414,15 @@ static inline void scx_schedule_reenq_local(struct rq *rq, u64 reenq_flags)
*/
static inline struct rq *scx_locked_rq(void)
{
+ /*
+ * Tracing progs can call kfuncs from NMI. scx_locked_rq_state tracks
+ * the rq locked by the interrupted context, so a non-NULL read from
+ * NMI would falsely claim its lock. Return NULL from NMI so that
+ * callers take their unlocked paths.
+ */
+ if (unlikely(in_nmi()))
+ return NULL;
+
return __this_cpu_read(scx_locked_rq_state);
}
@@ -2302,9 +2513,9 @@ do { \
} while (0)
/*
- * Dispatch a task op through the cid-form ops_cid table. Only set_cmask() needs
- * this: it takes an arena cmask address instead of a cpumask, so it cannot be
- * invoked via its cpu-form set_cpumask() slot.
+ * Dispatch a task op through the cid-form ops_cid table, for the ops whose
+ * cid-form signature differs from the cpu-form slot: set_cmask() takes an arena
+ * cmask instead of a cpumask and enable() takes scx_enable_args.
*/
#define SCX_CALL_CID_OP_TASK(sch, op, locked_rq, task, args...) \
__SCX_CALL_OP_TASK(sch, ops_cid, op, locked_rq, task, ##args)
@@ -2493,4 +2704,21 @@ static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL;
#endif /* CONFIG_EXT_SUB_SCHED */
+/**
+ * scx_exec_sched - Find the sched running @sch's tasks on @cpu
+ * @sch: sched owning the tasks
+ * @cpu: cpu of interest
+ *
+ * A bypassing sched's tasks are run by its nearest ancestor that isn't
+ * bypassing. Return @sch if it isn't bypassing on @cpu, otherwise that
+ * ancestor, or the root if every ancestor is bypassing. Hold @cpu's rq lock for
+ * a stable answer.
+ */
+static inline struct scx_sched *scx_exec_sched(struct scx_sched *sch, s32 cpu)
+{
+ while (scx_parent(sch) && scx_bypassing(sch, cpu))
+ sch = scx_parent(sch);
+ return sch;
+}
+
#endif /* _KERNEL_SCHED_EXT_INTERNAL_H */
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index 34e642a1a403..eb1270f7723c 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -28,6 +28,13 @@
*/
DEFINE_STATIC_KEY_FALSE(__scx_has_subs);
+/*
+ * On while a loaded sched implements ops.sub_cid_sched_updated(), see
+ * scx_ops_cid_sched_updated_enable().
+ */
+static DEFINE_STATIC_KEY_FALSE(__scx_ops_cid_sched_updated_enabled);
+static s32 scx_nr_ops_cid_sched_updated; /* such scheds, under scx_enable_mutex */
+
/* latched at root enable before any rescue runs */
static s32 scx_rescue_bw_1024;
static s64 scx_rescue_quantum_ns;
@@ -92,6 +99,161 @@ struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sche
return scx_skip_subtree_pre(pos, root);
}
+/**
+ * scx_cid_sched_id_from - Resolve what @from sees when @sch runs on a cid
+ * @sch: sched whose task runs there, %NULL for none
+ * @from: sched asking
+ *
+ * Return %SCX_CID_SCHED_NONE when @sch is %NULL or outside @from's subtree,
+ * %SCX_CID_SCHED_SELF for @from itself, or else the cgroup id of the direct
+ * child of @from on the way down to @sch.
+ */
+static u64 scx_cid_sched_id_from(struct scx_sched *sch, struct scx_sched *from)
+{
+ if (!sch || !scx_is_descendant(sch, from))
+ return SCX_CID_SCHED_NONE;
+ if (sch == from)
+ return SCX_CID_SCHED_SELF;
+ return sch->ancestors[from->level + 1]->ops.sub_cgroup_id;
+}
+
+/* notify @from of what it now sees running on @rq's cid */
+static void scx_cid_notify_sched_updated(struct rq *rq, struct scx_sched *sch,
+ struct scx_sched *from)
+{
+ if (SCX_HAS_OP(from, sub_cid_sched_updated))
+ SCX_CALL_OP(from, sub_cid_sched_updated, rq, __scx_cpu_to_cid(cpu_of(rq)),
+ scx_cid_sched_id_from(sch, from));
+}
+
+/**
+ * scx_cid_sched_update - Update rq->scx.sched and notify the scheds affected
+ * @rq: rq whose running session opens or closes
+ * @sch: sched starting a session, %NULL to close the open one
+ *
+ * Every sched whose scx_cid_sched_id_from() value changes is notified of the
+ * new one. Above the deepest common ancestor of the two scheds, both map to the
+ * same child, so the walk stops there. With root R, children A and B, and A's
+ * child A1, a switch from an A1 task to a B task reports:
+ *
+ * R R: A -> B
+ * / \
+ * A B A: A1 -> NONE B: NONE -> SELF
+ * |
+ * A1 A1: SELF -> NONE
+ *
+ * When the task on one side is not an ext task, there is no common ancestor and
+ * only the other side is walked.
+ */
+void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch)
+{
+ struct scx_sched *prev = rq->scx.sched;
+ s32 level, common = -1;
+
+ lockdep_assert_rq_held(rq);
+
+ if (!static_branch_unlikely(&__scx_ops_cid_sched_updated_enabled) || prev == sch)
+ return;
+
+ rq->scx.sched = sch;
+
+ if (prev && sch) {
+ for (level = min(prev->level, sch->level); level >= 0; level--)
+ if (prev->ancestors[level] == sch->ancestors[level])
+ break;
+ common = level;
+ scx_cid_notify_sched_updated(rq, sch, prev->ancestors[common]);
+ }
+ if (prev)
+ for (level = common + 1; level <= prev->level; level++)
+ scx_cid_notify_sched_updated(rq, sch, prev->ancestors[level]);
+ if (sch)
+ for (level = common + 1; level <= sch->level; level++)
+ scx_cid_notify_sched_updated(rq, sch, sch->ancestors[level]);
+}
+
+/**
+ * scx_ops_cid_sched_updated_enable - Maintain rq->scx.sched if @sch has the op
+ * @sch: sched being enabled, its has_op filled in
+ *
+ * rq->scx.sched is maintained only while a loaded sched implements
+ * ops.sub_cid_sched_updated(), so that everyone else pays one static branch per
+ * session open or close. The first such sched turns the key on and then
+ * initializes every rq from the task running there, in that order so that no
+ * session change in between goes unrecorded. While the key is off every
+ * rq->scx.sched is NULL, see scx_ops_cid_sched_updated_disable().
+ */
+void scx_ops_cid_sched_updated_enable(struct scx_sched *sch)
+{
+ s32 cpu;
+
+ lockdep_assert_held(&scx_enable_mutex);
+
+ if (!SCX_HAS_OP(sch, sub_cid_sched_updated) || scx_nr_ops_cid_sched_updated++)
+ return;
+
+ static_branch_enable(&__scx_ops_cid_sched_updated_enabled);
+
+ for_each_possible_cpu(cpu) {
+ struct rq *rq = cpu_rq(cpu);
+ struct task_struct *p;
+
+ guard(rq_lock_irqsave)(rq);
+ p = rq->donor;
+ if (p->sched_class == &ext_sched_class &&
+ (p->scx.flags & SCX_TASK_RUN_TRACKED))
+ rq->scx.sched = scx_task_sched(p);
+ }
+}
+
+/**
+ * scx_ops_cid_sched_updated_disable - Stop maintaining rq->scx.sched for @sch
+ * @sch: sched being disabled, its tasks all gone
+ *
+ * Close the session on every rq that still points at @sch, before ops.exit(),
+ * so that no sched is notified of @sch once it is gone. The last sched with the
+ * op turns the key off first and clears every rq, so while the key is off every
+ * rq->scx.sched is NULL. A sched whose enable failed before its has_op fill was
+ * never counted, so has_op gates the decrement.
+ */
+void scx_ops_cid_sched_updated_disable(struct scx_sched *sch)
+{
+ bool last;
+ s32 cpu;
+
+ lockdep_assert_held(&scx_enable_mutex);
+
+ last = SCX_HAS_OP(sch, sub_cid_sched_updated) && !--scx_nr_ops_cid_sched_updated;
+ if (last)
+ static_branch_disable(&__scx_ops_cid_sched_updated_enabled);
+
+ /*
+ * Re-homing restarts a running task's session under its new sched. A
+ * task that blocked while its cpu's dispatch had the rq lock dropped is
+ * put and set next again without a restart, so rq->scx.sched keeps @sch
+ * until the pick completes:
+ *
+ * cpu 0, __schedule() cpu 1, scx_sub_disable(@sch)
+ * T of @sch blocks, stops running
+ * dispatch drops the rq lock
+ * re-home T, nothing to restart
+ * this sweep
+ * pick N, N's session opens
+ *
+ * Closing the session here reports NONE to @sch and its ancestors now
+ * and lets the pick report N's sched from NONE instead of from @sch.
+ */
+ for_each_possible_cpu(cpu) {
+ struct rq *rq = cpu_rq(cpu);
+
+ guard(rq_lock_irqsave)(rq);
+ if (rq->scx.sched == sch)
+ scx_cid_sched_update(rq, NULL);
+ if (last)
+ rq->scx.sched = NULL;
+ }
+}
+
static struct scx_sched *scx_find_sub_sched(u64 cgroup_id)
{
return rhashtable_lookup(&scx_sched_hash, &cgroup_id,
@@ -285,7 +447,8 @@ void scx_rescue_charge(struct rq *rq, s64 delta_exec)
rq->scx.rescue.budget -= delta_exec;
/* per-cpu usage average feeds the overload victim pick */
- pcpu = per_cpu_ptr(scx_task_sched(rq->curr)->pcpu, cpu_of(rq));
+ pcpu = per_cpu_ptr(scx_task_sched(rq->scx.rescue.curr)->pcpu,
+ cpu_of(rq));
pcpu->rescue_avg = scx_rescue_decay_avg(pcpu) + delta_exec;
if (!scx_rescue_slice_remaining(rq))
@@ -555,9 +718,9 @@ static void scx_rescue_timerfn(struct timer_list *timer)
scx.dsq_list.node);
scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq);
scx_rescue_admit(rq, p, slice);
- scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS,
- &rq->scx.rescue.dsq, rq);
- if (sched_class_above(&ext_sched_class, rq->curr->sched_class))
+ scx_move_local_task_to_local_dsq(scx_task_sched(p), p,
+ SCX_ENQ_IGNORE_CAPS, rq);
+ if (sched_class_above(&ext_sched_class, rq->donor->sched_class))
resched_curr(rq);
} else if (p->scx.dsq && rq->scx.rescue.budget > 2 * scx_rescue_quantum_ns) {
/*
@@ -572,7 +735,7 @@ static void scx_rescue_timerfn(struct timer_list *timer)
scx_task_unlink_from_dsq(p, &rq->scx.local_dsq);
scx_move_local_task_to_local_dsq(scx_task_sched(p), p,
SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_IGNORE_CAPS,
- &rq->scx.local_dsq, rq);
+ rq);
}
out_arm:
scx_rescue_timer_arm(rq);
@@ -585,10 +748,6 @@ void scx_rescue_flush(struct rq *rq)
lockdep_assert_rq_held(rq);
- /* sched domain rebuilds call rq_offline with the CPU staying alive */
- if (cpu_active(cpu_of(rq)))
- return;
-
/* end the current rescue */
if (rq->scx.rescue.curr)
scx_task_slice_ended(rq, rq->scx.rescue.curr);
@@ -596,8 +755,8 @@ void scx_rescue_flush(struct rq *rq)
/* and flush out all pending ones */
list_for_each_entry_safe(p, n, &rq->scx.rescue.dsq.list, scx.dsq_list.node) {
scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq);
- scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS,
- &rq->scx.rescue.dsq, rq);
+ scx_move_local_task_to_local_dsq(scx_task_sched(p), p,
+ SCX_ENQ_IGNORE_CAPS, rq);
}
timer_delete(&rq->scx.rescue.timer);
@@ -675,7 +834,7 @@ void scx_rescue_init(struct rq *rq)
}
/**
- * scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert
+ * __scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert
* @sch: enqueuing sub-sched
* @rq: rq whose local DSQ @p targets
* @p: task being inserted
@@ -686,20 +845,18 @@ void scx_rescue_init(struct rq *rq)
* rescue is enabled, or @rq's reject DSQ after recording the reenq reason on
* @p.
*
- * %SCX_ENQ_IMMED, %SCX_ENQ_PREEMPT and %SCX_ENQ_HEAD are cleared when diverting
- * to rescue or reject. %SCX_ENQ_PREEMPT is also cleared on a fallback
- * migration-disabled admission.
+ * %SCX_ENQ_IMMED, %SCX_ENQ_PREEMPT, %SCX_ENQ_PREEMPT_LAZY and %SCX_ENQ_HEAD are
+ * cleared when diverting to rescue or reject. %SCX_ENQ_PREEMPT and
+ * %SCX_ENQ_PREEMPT_LAZY are also cleared on a fallback migration-disabled
+ * admission.
*
* Bypass doesn't need special-casing as a bypassing sched's tasks are enqueued
* to and run by its nearest non-bypassing ancestor. If root is bypassing, it
* always holds all caps.
*/
-struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
- struct task_struct *p, u64 *enq_flags)
+struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
+ struct task_struct *p, u64 *enq_flags)
{
- if (!scx_has_subs())
- return &rq->scx.local_dsq;
-
s32 cid = __scx_cpu_to_cid(cpu_of(rq));
struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch;
u64 needed = scx_caps_for_enq(*enq_flags);
@@ -709,7 +866,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r
* On a remote activation the scheduling sched (@asch) differs from
* @p's owner (@sch). Check caps against the scheduling sched.
*/
- if (*enq_flags & SCX_ENQ_PREEMPT)
+ if (*enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY))
needed |= scx_caps_for_preempt(asch, rq, *enq_flags);
missing = scx_missing_caps(asch, cpu_of(rq), needed);
@@ -726,7 +883,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r
if (unlikely(!scx_rq_online(rq) || is_migration_disabled(p) ||
p->migration_pending)) {
__scx_add_event(sch, SCX_EV_SUB_FORCED_ADMIT, 1);
- *enq_flags &= ~SCX_ENQ_PREEMPT;
+ *enq_flags &= ~(SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY);
return &rq->scx.local_dsq;
}
@@ -735,9 +892,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r
* or HEAD - a diversion has no priority and IMMED is not allowed on
* non-local DSQs. Strip the enq and task flags along with the slice.
*/
- *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | SCX_ENQ_HEAD |
- SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL);
- p->scx.flags &= ~SCX_TASK_IMMED;
+ scx_divert_strip_flags(p, enq_flags);
/* the enqueuer opted for rescue instead of rejection and reenqueue */
if ((*enq_flags & SCX_ENQ_RESCUE) && likely(scx_rescue_bw_1024)) {
@@ -752,6 +907,8 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r
p->scx.reenq_reason_caps = missing;
p->scx.reenq_reason_cid = cid;
+ WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK);
+ p->scx.flags |= SCX_TASK_REENQ_CAP;
return &rq->scx.reject_dsq;
}
@@ -759,66 +916,24 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r
/* @p lost the caps needed to stay on @rq's local DSQ? Record reason if so. */
bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p)
{
+ s32 cpu = cpu_of(rq);
u64 missing;
/* migration-disabled tasks and the rescuee are admitted capless */
if (is_migration_disabled(p) || p == scx_rescuee(rq))
return false;
- missing = scx_missing_caps(scx_task_sched(p), cpu_of(rq), scx_caps_for_task(p));
+ /* check caps against the sched running a bypassed task */
+ missing = scx_missing_caps(scx_exec_sched(scx_task_sched(p), cpu), cpu,
+ scx_caps_for_task(p));
if (likely(!missing))
return false;
p->scx.reenq_reason_caps = missing;
- p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu_of(rq));
+ p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu);
return true;
}
-/*
- * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides
- * from p->scx.reenq_reason_*.
- *
- * A task can be re-rejected repeatedly. The reenqueue is bounded per task in
- * scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT.
- * Rejection can't happen for root.
- */
-void scx_reenq_reject(struct rq *rq)
-{
- LIST_HEAD(tasks);
- struct task_struct *p, *n;
-
- lockdep_assert_rq_held(rq);
-
- if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list))
- return;
-
- /*
- * Move to a private list so a task re-rejected by the
- * scx_do_enqueue_task() below isn't revisited this round.
- */
- list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) {
- /* migration_pending tasks should have bypassed to local DSQ */
- if (WARN_ON_ONCE(p->migration_pending))
- continue;
-
- scx_dispatch_dequeue(rq, p);
-
- if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
- p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
- p->scx.flags |= SCX_TASK_REENQ_CAP;
-
- list_add_tail(&p->scx.dsq_list.node, &tasks);
- }
-
- list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {
- list_del_init(&p->scx.dsq_list.node);
-
- scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
-
- p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
- }
-}
-
/* record a caps change, see struct scx_caps_updated */
static void caps_updated_record(struct scx_pshard *ps, const struct scx_cmask *cids, u64 caps,
struct list_head *to_deliver)
@@ -924,7 +1039,7 @@ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid)
struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
/*
- * Pairs with smp_mb() in scx_process_sync_ecaps(). Either the check
+ * Pairs with smp_mb() in __scx_process_sync_ecaps(). Either the check
* below sees the node off the list and queues it, or the in-flight sync
* sees the caps[] update made before this call.
*/
@@ -949,7 +1064,7 @@ static void discard_queued_syncs(struct rq *rq)
}
/**
- * scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[]
+ * __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[]
* @rq: the cid's cpu rq
* @prev: @rq's previous task from the in-progress dispatch
*
@@ -961,16 +1076,14 @@ static void discard_queued_syncs(struct rq *rq)
* learns the cid's idle state. Such a gain arms the per-rq
* %SCX_RQ_SUB_IDLE_RENOTIFY gate so the next idle pick delivers it.
*/
-void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
+void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
{
s32 cpu = cpu_of(rq);
s32 cid, shard;
struct llist_node *batch, *pos, *tmp;
u64 lost_all = 0;
- lockdep_assert_rq_held(rq);
-
- if (!scx_has_subs() || likely(llist_empty(&rq->scx.ecaps_to_sync)))
+ if (likely(llist_empty(&rq->scx.ecaps_to_sync)))
return;
/*
@@ -1011,27 +1124,43 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
lost_all |= lost;
/*
- * Tell the sched its effective caps on this cid changed. The
- * invocation is equivalent to the dispatch path and may drop
- * and re-acquire the rq lock temporarily while the rest of
- * @batch is held privately, see scx_discard_ecaps_to_sync().
- * The dispatch kfuncs resolve their context on the executing
- * cpu, which under core scheduling can differ from @rq's cpu,
- * so the context is set up there. The rq recorded in it keeps
- * the dispatches targeting @rq.
+ * Tell the sched and its parent that the sched's effective caps
+ * on this cid changed. The invocations are equivalent to the
+ * dispatch path and may drop and re-acquire the rq lock
+ * temporarily while the rest of @batch is held privately, see
+ * scx_discard_ecaps_to_sync(). The dispatch kfuncs resolve
+ * their context on the executing cpu, which under core
+ * scheduling can differ from @rq's cpu, so the context is set
+ * up there. The rq recorded in it keeps the dispatches
+ * targeting @rq.
+ *
+ * Bypass propagates down the hierarchy, so a sched that isn't
+ * bypassing has no bypassing parent. The child's bypass state
+ * gates both deliveries. Both report the same before value, so
+ * one reported_ecaps covers them.
*/
- if (ecaps != pcpu->reported_ecaps &&
- SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) &&
- !scx_bypassing(pcpu->sch, cpu)) {
- struct scx_dsp_ctx *dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx;
+ if (ecaps != pcpu->reported_ecaps && !scx_bypassing(pcpu->sch, cpu)) {
+ struct scx_sched *parent = scx_parent(pcpu->sch);
+ struct scx_dsp_ctx *dspc;
- dspc->rq = rq;
/* stash @prev so nested dispatches can access it */
rq->scx.sub_dispatch_prev = prev;
- SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, scx_cpu_arg(cpu),
- pcpu->reported_ecaps, ecaps);
+ if (SCX_HAS_OP(pcpu->sch, sub_ecaps_updated)) {
+ dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx;
+ dspc->rq = rq;
+ SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq,
+ scx_cpu_arg(cpu), pcpu->reported_ecaps, ecaps);
+ scx_flush_dispatch_buf(pcpu->sch, rq);
+ }
+ if (SCX_HAS_OP(parent, sub_child_ecaps_updated)) {
+ dspc = &this_cpu_ptr(parent->pcpu)->dsp_ctx;
+ dspc->rq = rq;
+ SCX_CALL_OP(parent, sub_child_ecaps_updated, rq,
+ pcpu->sch->ops.sub_cgroup_id, scx_cpu_arg(cpu),
+ pcpu->reported_ecaps, ecaps);
+ scx_flush_dispatch_buf(parent, rq);
+ }
rq->scx.sub_dispatch_prev = NULL;
- scx_flush_dispatch_buf(pcpu->sch, rq);
pcpu->reported_ecaps = ecaps;
}
@@ -1157,6 +1286,34 @@ void scx_offline_ecaps(struct rq *rq)
}
/*
+ * Clear every cap @sch holds. The pshard caps go first as they are the source a
+ * pending sync recomputes ecaps from. ecaps are then zeroed directly for the
+ * cap checks.
+ */
+static void clear_all_caps(struct scx_sched *sch)
+{
+ s32 si, cpu;
+ u32 cap_bit;
+
+ /* enable may have failed before scx_alloc_pshards() */
+ if (!sch->pshard)
+ return;
+
+ for (si = 0; si < sch->nr_pshards; si++) {
+ struct scx_pshard *ps = sch->pshard[si];
+
+ guard(raw_spinlock_irqsave)(&ps->lock);
+ for (cap_bit = 0; cap_bit < __SCX_NR_CAPS; cap_bit++)
+ scx_cmask_clear(&ps->caps[cap_bit].cmask);
+ }
+
+ for_each_possible_cpu(cpu) {
+ guard(rq_lock_irqsave)(cpu_rq(cpu));
+ WRITE_ONCE(per_cpu_ptr(sch->pcpu, cpu)->ecaps, 0);
+ }
+}
+
+/*
* @pcpu's sched was unhashed before the grace period, so nothing re-queues its
* sync node. Remove the node from @rq's pending list so the pcpu can be freed.
*/
@@ -1255,7 +1412,9 @@ static void scx_rehome_task(struct scx_sched *to, struct task_struct *p)
lockdep_assert_held(&p->pi_lock);
lockdep_assert_rq_held(task_rq(p));
- scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
+ scx_prepare_task_sched_change(p);
+ scoped_guard (sched_change, p,
+ DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) {
scx_disable_and_exit_task(scx_task_sched(p), p);
scx_set_task_state(p, SCX_TASK_INIT_BEGIN);
scx_set_task_state(p, SCX_TASK_INIT);
@@ -1285,7 +1444,9 @@ static void scx_punt_task(struct scx_sched *to, struct task_struct *p)
lockdep_assert_rq_held(task_rq(p));
WARN_ON_ONCE(!READ_ONCE(to->bypass_depth));
- scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
+ scx_prepare_task_sched_change(p);
+ scoped_guard (sched_change, p,
+ DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) {
scx_disable_and_exit_task(scx_task_sched(p), p);
scx_set_task_sched(p, to);
}
@@ -1361,7 +1522,7 @@ static s32 scx_cgroup_claim_subtree(struct scx_sched *sch)
.bw_period_us = tg->scx.bw_period_us,
.bw_quota_us = tg->scx.bw_quota_us,
.bw_burst_us = tg->scx.bw_burst_us,
- .sched_idle = tg->scx.idle,
+ .sched_idle = tg->scx.sched_idle,
};
if (tg->scx.sched != parent ||
@@ -1465,7 +1626,7 @@ static void scx_cgroup_return_subtree(struct scx_sched *sch)
.bw_period_us = tg->scx.bw_period_us,
.bw_quota_us = tg->scx.bw_quota_us,
.bw_burst_us = tg->scx.bw_burst_us,
- .sched_idle = tg->scx.idle,
+ .sched_idle = tg->scx.sched_idle,
};
/* the first pass must have transferred everything */
@@ -1619,6 +1780,8 @@ dump:
scx_cgroup_unlock();
percpu_up_write(&scx_fork_rwsem);
+ scx_ops_cid_sched_updated_disable(sch);
+
/*
* All tasks are moved off of @sch but there may still be on-going
* operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use
@@ -1631,6 +1794,12 @@ dump:
scx_unlink_sched(sch);
+ /*
+ * The parent restores what it delegated in ops.sub_detach(). @sch must
+ * hold no caps by then.
+ */
+ clear_all_caps(sch);
+
mutex_unlock(&scx_enable_mutex);
/*
@@ -1790,6 +1959,15 @@ void scx_sub_enable_workfn(struct kthread_work *work)
if (ret)
goto err_disable;
+ /*
+ * Bypass before @sch is linked and grants can reach it. The parent's
+ * delivery advances reported_ecaps, so a sync consumed before @sch's
+ * ops are registered would reach the parent and never be replayed to
+ * @sch. While bypassed, the syncs are consumed without a delivery and
+ * replayed to both at unbypass.
+ */
+ scx_bypass(sch, true);
+
ret = scx_link_sched(sch);
if (ret)
goto err_disable;
@@ -1835,12 +2013,12 @@ void scx_sub_enable_workfn(struct kthread_work *work)
}
sch->sub_attached = true;
- scx_bypass(sch, true);
-
for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++)
if (((void (**)(void))ops)[i])
set_bit(i, sch->has_op);
+ scx_ops_cid_sched_updated_enable(sch);
+
percpu_down_write(&scx_fork_rwsem);
scx_cgroup_lock();
@@ -1943,7 +2121,9 @@ void scx_sub_enable_workfn(struct kthread_work *work)
if (!(p->scx.flags & SCX_TASK_SUB_INIT))
continue;
- scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
+ scx_prepare_task_sched_change(p);
+ scoped_guard (sched_change, p,
+ DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) {
/*
* $p must be either READY or ENABLED. If ENABLED,
* __scx_disabled_and_exit_task() first disables and
@@ -2269,6 +2449,9 @@ static s32 sub_cap_preamble(u64 cgroup_id, u64 caps, const struct bpf_prog_aux *
if (unlikely(!parent))
return -ENODEV;
+ if (!scx_kf_allowed_ctx(parent))
+ return -EDEADLK;
+
if (!scx_is_cid_type()) {
scx_error(parent, "sub-cap kfuncs require a cid-form scheduler");
return -EOPNOTSUPP;
@@ -2309,8 +2492,8 @@ static s32 sub_cap_preamble(u64 cgroup_id, u64 caps, const struct bpf_prog_aux *
* All-or-nothing keeps the caller-visible result binary per cid, so the denied
* mask is one mask to interpret rather than a per-cap matrix.
*
- * Return 0 on full success, -EPERM if any cid was refused, or a negative
- * errno on other failures.
+ * Return 0 on full success, -EPERM if any cid was refused, -ENODEV if the child
+ * is gone or being disabled, or a negative errno on other failures.
*/
__bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps,
const struct scx_cmask *cmask__arena,
@@ -2364,6 +2547,12 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps,
scoped_guard (raw_spinlock, &pps->lock) {
guard(raw_spinlock_nested)(&cps->lock);
+ /* the child is being disabled, see clear_all_caps() */
+ if (unlikely(READ_ONCE(child->aborting))) {
+ ret = -ENODEV;
+ goto out;
+ }
+
/*
* Narrow granted_cids to cids the parent holds every
* requested cap on. All-or-nothing per cid.
@@ -2416,9 +2605,10 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps,
}
}
+ ret = any_denied ? -EPERM : 0;
+out:
caps_updated_deliver(&to_deliver);
-
- return any_denied ? -EPERM : 0;
+ return ret;
}
/**
diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h
index f7bcdfda8dd8..b76d750463f0 100644
--- a/kernel/sched/ext/sub.h
+++ b/kernel/sched/ext/sub.h
@@ -16,6 +16,9 @@
struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root);
struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root);
+void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch);
+void scx_ops_cid_sched_updated_enable(struct scx_sched *sch);
+void scx_ops_cid_sched_updated_disable(struct scx_sched *sch);
void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch);
struct cgroup *sch_cgroup(struct scx_sched *sch);
void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch);
@@ -28,16 +31,15 @@ bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux);
void scx_free_pshards(struct scx_sched *sch);
s32 scx_alloc_pshards(struct scx_sched *sch);
void scx_init_root_caps(struct scx_sched *sch);
-void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev);
+void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev);
void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch);
void scx_online_ecaps(struct rq *rq);
void scx_offline_ecaps(struct rq *rq);
void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu);
void scx_discard_stale_ecaps_syncs(void);
-struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
- struct task_struct *p, u64 *enq_flags);
+struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
+ struct task_struct *p, u64 *enq_flags);
bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p);
-void scx_reenq_reject(struct rq *rq);
void scx_rescue_charge(struct rq *rq, s64 delta_exec);
void scx_rescue_end(struct rq *rq);
bool scx_rescue_keep(struct rq *rq, struct task_struct *p);
@@ -72,10 +74,32 @@ static inline void scx_dec_has_subs(struct scx_sched *sch)
static_branch_dec(&__scx_has_subs);
}
+/* hot-path hooks, gated inline so that a root-only system skips the calls */
+static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
+{
+ lockdep_assert_rq_held(rq);
+
+ if (scx_has_subs())
+ __scx_process_sync_ecaps(rq, prev);
+}
+
+static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch,
+ struct rq *rq,
+ struct task_struct *p,
+ u64 *enq_flags)
+{
+ if (!scx_has_subs())
+ return &rq->scx.local_dsq;
+ return __scx_resolve_local_dsq(sch, rq, p, enq_flags);
+}
+
#else /* CONFIG_EXT_SUB_SCHED */
static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }
static inline struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root) { return NULL; }
+static inline void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch) {}
+static inline void scx_ops_cid_sched_updated_enable(struct scx_sched *sch) {}
+static inline void scx_ops_cid_sched_updated_disable(struct scx_sched *sch) {}
static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {}
static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; }
static inline const char *sch_cgrp_path(struct scx_sched *sch) { return "/"; }
@@ -93,7 +117,6 @@ static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcp
static inline void scx_discard_stale_ecaps_syncs(void) {}
static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; }
static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; }
-static inline void scx_reenq_reject(struct rq *rq) {}
static inline void scx_rescue_charge(struct rq *rq, s64 delta_exec) {}
static inline void scx_rescue_end(struct rq *rq) {}
static inline bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { return false; }
@@ -169,17 +192,18 @@ static inline u64 scx_caps_for_task(struct task_struct *p)
return SCX_CAP_ENQ;
}
-/* the cap @sch needs to preempt @rq's current task, 0 if none */
-static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags)
+/* the cap @sch needs to preempt @rq's current scheduling context, 0 if none */
+static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq,
+ u64 enq_flags)
{
- struct task_struct *curr = rq->curr;
+ struct task_struct *donor = rq->donor;
/* a kernel-forced placement preempts regardless of caps */
if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS))
return 0;
/* a non-ext task can't be preempted by ext, own-subtree needs no cap */
- if (curr->sched_class != &ext_sched_class ||
- scx_is_descendant(scx_task_sched(curr), sch))
+ if (donor->sched_class != &ext_sched_class ||
+ scx_is_descendant(scx_task_sched(donor), sch))
return 0;
return SCX_CAP_PREEMPT;
}
@@ -199,6 +223,8 @@ static inline u64 scx_caps_implied(u64 cap)
/* may @p keep running on @rq's cpu? requires baseline cpu access */
static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p)
{
+ s32 cpu = cpu_of(rq);
+
if (!scx_has_subs())
return true;
@@ -206,7 +232,9 @@ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p
if (unlikely(is_migration_disabled(p)))
return true;
- return likely(!scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE));
+ /* check caps against the sched running a bypassed task */
+ return likely(!scx_missing_caps(scx_exec_sched(scx_task_sched(p), cpu), cpu,
+ SCX_CAP_BASE));
}
/* the task admitted for rescue on @rq, NULL if none */
diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h
index 943d8d429a2c..38514d6bf149 100644
--- a/kernel/sched/ext/types.h
+++ b/kernel/sched/ext/types.h
@@ -59,17 +59,35 @@ enum scx_consts {
};
/*
- * Per-cid topology info. For each topology level (core, LLC, node) and shard,
- * records the first cid in the unit and its global index. Global indices are
- * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges over
- * [0, nr_cores_at_init) with no gaps. No-topo cids have core/LLC/node fields
- * set to -1 but always have valid shard assignments.
+ * Per-cid topology info. For each topology level (core, cluster, LLC, node) and
+ * shard, records the first cid in the unit and its global index. Global indices
+ * are consecutive integers assigned in cid-walk order, so e.g. core_idx ranges
+ * over [0, nr_cores_at_init) with no gaps. No-topo cids have core/cluster/LLC/
+ * node fields set to -1 but always have valid shard assignments.
+ *
+ * A cluster is the cache-sharing level between the core and the LLC. Where
+ * this level is absent, each core forms a cluster of its own. Where the cache
+ * level spans the LLC, the whole LLC forms one cluster. Each cluster has a
+ * unique cluster_cid and a dense cluster_idx, and its cids form a contiguous
+ * range.
+ *
+ * Every cid therefore has a cluster, so cluster_cid alone does not say whether
+ * the machine has the level: the first cluster of an LLC starts at the LLC's
+ * own base cid and a core-wide one at the core's.
*
* Shards are contiguous CID ranges used as scalable locking/work domains for
* sub-scheduler operations. By default each LLC becomes one shard, split into
* smaller shards if the LLC exceeds the target size. No-topo cids are packed
* into their own max-sized shards.
*
+ * Shards are cut on core boundaries only, so a shard doesn't need to contain a
+ * whole cluster: a cluster may straddle two shards. Only core, cluster, LLC
+ * and node are nested, in that order.
+ *
+ * New fields are appended, never inserted: scx_bpf_cid_topo() copies this
+ * struct out sized by the program's own layout, and an older program's copy
+ * must stay a prefix of the kernel's.
+ *
* @core_cid: first cid of this cid's core (smt-sibling group)
* @core_idx: global index of that core, in [0, nr_cores_at_init)
* @llc_cid: first cid of this cid's LLC
@@ -78,6 +96,8 @@ enum scx_consts {
* @node_idx: global index of that node, in [0, nr_nodes_at_init)
* @shard_cid: first cid of this cid's shard
* @shard_idx: global index of that shard, in [0, scx_nr_cid_shards)
+ * @cluster_cid: first cid of this cid's cluster
+ * @cluster_idx: global index of that cluster, in [0, nr_clusters_at_init)
*/
struct scx_cid_topo {
s32 core_cid;
@@ -88,6 +108,8 @@ struct scx_cid_topo {
s32 node_idx;
s32 shard_cid;
s32 shard_idx;
+ s32 cluster_cid;
+ s32 cluster_idx;
};
enum scx_cid_consts {
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 7455a83a6a99..b5546cd27dba 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -24,6 +24,7 @@
#include <linux/mmap_lock.h>
#include <linux/hugetlb_inline.h>
#include <linux/jiffies.h>
+#include <linux/math.h>
#include <linux/mm_api.h>
#include <linux/highmem.h>
#include <linux/hrtimer.h>
@@ -820,12 +821,6 @@ static u64 ineligible_vruntime(struct cfs_rq *cfs_rq)
if (curr && !curr->on_rq)
curr = NULL;
- /*
- * This is called from set_next_task_fair(.first=true) /
- * set_protect_slice() so curr had better be set and on_rq.
- */
- WARN_ON_ONCE(!curr);
-
if (weight) {
s64 runtime = cfs_rq->sum_w_vruntime;
@@ -1004,71 +999,17 @@ static inline bool __entity_less(struct rb_node *a, const struct rb_node *b)
return entity_before(__node_2_se(a), __node_2_se(b));
}
-static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node *node)
-{
- if (node) {
- struct sched_entity *rse = __node_2_se(node);
-
- if (vruntime_cmp(se->min_vruntime, ">", rse->min_vruntime))
- se->min_vruntime = rse->min_vruntime;
- }
-}
-
-static inline void __min_slice_update(struct sched_entity *se, struct rb_node *node)
+/* min() for wrapping vruntimes */
+static inline u64 __min_vruntime(u64 a, u64 b)
{
- if (node) {
- struct sched_entity *rse = __node_2_se(node);
- if (rse->min_slice < se->min_slice)
- se->min_slice = rse->min_slice;
- }
-}
-
-static inline void __max_slice_update(struct sched_entity *se, struct rb_node *node)
-{
- if (node) {
- struct sched_entity *rse = __node_2_se(node);
- if (rse->max_slice > se->max_slice)
- se->max_slice = rse->max_slice;
- }
+ return vruntime_cmp(a, "<", b) ? a : b;
}
-static inline void min_vruntime_copy(struct sched_entity *new, struct sched_entity *old)
-{
- new->min_vruntime = old->min_vruntime;
- new->min_slice = old->min_slice;
- new->max_slice = old->max_slice;
-}
-
-/*
- * se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime)
- */
-static inline bool min_vruntime_update(struct sched_entity *se, bool exit)
-{
- u64 old_min_vruntime = se->min_vruntime;
- u64 old_min_slice = se->min_slice;
- u64 old_max_slice = se->max_slice;
- struct rb_node *node = &se->run_node;
-
- se->min_vruntime = se->vruntime;
- __min_vruntime_update(se, node->rb_right);
- __min_vruntime_update(se, node->rb_left);
-
- se->min_slice = se->slice;
- __min_slice_update(se, node->rb_right);
- __min_slice_update(se, node->rb_left);
-
- se->max_slice = se->slice;
- __max_slice_update(se, node->rb_right);
- __max_slice_update(se, node->rb_left);
-
- return se->min_vruntime == old_min_vruntime &&
- se->min_slice == old_min_slice &&
- se->max_slice == old_max_slice;
-}
-
-
-RB_DECLARE_CALLBACKS_MULTI(static, min_vruntime_cb, struct sched_entity,
- run_node, min_vruntime_copy, min_vruntime_update);
+RB_DECLARE_CALLBACKS(static, min_vruntime_cb,
+ struct sched_entity, run_node,
+ RB_AUG(vruntime, min_vruntime, __min_vruntime),
+ RB_AUG(slice, min_slice, min),
+ RB_AUG(slice, max_slice, max));
/*
* Enqueue an entity into the rb-tree:
@@ -1137,10 +1078,9 @@ static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity
/* If there are shorter slices than se's one */
if (slice != se->slice) {
+ vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se));
if (sched_feat(PREEMPT_SHORT))
vprot = min_vruntime(vprot, ineligible_vruntime(cfs_rq));
- else
- vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se));
}
se->vprot = vprot;
@@ -1148,10 +1088,14 @@ static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity
static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
{
- u64 slice = cfs_rq_min_slice(cfs_rq);
u64 vruntime = min_vruntime(se->vruntime, avg_vruntime(cfs_rq));
+ u64 slice = cfs_rq_min_slice(cfs_rq);
+ u64 vprot = min_vruntime(se->vprot, vruntime + calc_delta_fair(slice, se));
+
+ if (sched_feat(PREEMPT_SHORT) && slice != se->slice)
+ vprot = min_vruntime(vprot, ineligible_vruntime(cfs_rq));
- se->vprot = min_vruntime(se->vprot, vruntime + calc_delta_fair(slice, se));
+ se->vprot = vprot;
}
static inline bool protect_slice(struct sched_entity *se)
@@ -1478,7 +1422,7 @@ static inline int get_sched_cache_scale(int mul)
return (1 + (tol - 1) * mul);
}
-static bool exceed_llc_capacity(struct mm_struct *mm, int cpu)
+static bool exceed_llc_capacity(struct sched_cache_group *grp, int cpu)
{
#ifdef CONFIG_NUMA_BALANCING
unsigned long llc, footprint;
@@ -1497,7 +1441,7 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu)
* excluded.
*/
llc = sd->llc_bytes;
- footprint = READ_ONCE(mm->sc_stat.footprint);
+ footprint = READ_ONCE(grp->footprint);
/*
* Scale the LLC size by 256*llc_aggr_tolerance
@@ -1526,7 +1470,7 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu)
return false;
}
-static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p,
+static bool invalid_llc_nr(struct sched_cache_group *grp, struct task_struct *p,
int cpu)
{
int scale;
@@ -1542,10 +1486,32 @@ static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p,
if (scale == INT_MAX)
return false;
- return !fits_capacity((mm->sc_stat.nr_running_avg * cpu_smt_num_threads),
+ return !fits_capacity((READ_ONCE(grp->nr_running_avg) * cpu_smt_num_threads),
(scale * per_cpu(sd_llc_size, cpu)));
}
+/*
+ * A task counts in nr_pref_llc_running while it is queued on its preferred
+ * LLC (pref_llc_queued) and runnable (!sched_delayed), keeping the counter in
+ * the runnable domain so alb_break_llc() can compare it with h_nr_runnable.
+ */
+static bool task_pref_llc_runnable(struct task_struct *p)
+{
+ return p->pref_llc_queued && !p->se.sched_delayed;
+}
+
+static void pref_llc_running_inc(struct rq *rq, struct task_struct *p)
+{
+ if (task_pref_llc_runnable(p))
+ rq->nr_pref_llc_running++;
+}
+
+static void pref_llc_running_dec(struct rq *rq, struct task_struct *p)
+{
+ if (task_pref_llc_runnable(p))
+ rq->nr_pref_llc_running--;
+}
+
static void account_llc_enqueue(struct rq *rq, struct task_struct *p)
{
int pref_llc, pref_llc_queued;
@@ -1557,7 +1523,6 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p)
pref_llc_queued = (pref_llc == task_llc(p));
rq->nr_llc_running++;
- rq->nr_pref_llc_running += pref_llc_queued;
/*
* Record whether p is enqueued on its preferred
@@ -1575,6 +1540,9 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p)
*/
p->pref_llc_queued = pref_llc_queued;
+ /* Skipped while delayed; clear_delayed() adds it back on wake. */
+ pref_llc_running_inc(rq, p);
+
sd = rcu_dereference_all(rq->sd);
if (sd && (unsigned int)pref_llc < sd->llc_max)
sd->llc_counts[pref_llc]++;
@@ -1591,7 +1559,12 @@ static void account_llc_dequeue(struct rq *rq, struct task_struct *p)
rq->nr_llc_running--;
if (p->pref_llc_queued) {
- rq->nr_pref_llc_running--;
+ /*
+ * Skipped if still delayed (set_delayed() already removed it);
+ * clearing pref_llc_queued below also stops clear_delayed()
+ * from re-adding it.
+ */
+ pref_llc_running_dec(rq, p);
/*
* Update the status in case
* other logic might query
@@ -1619,12 +1592,20 @@ static void account_llc_dequeue(struct rq *rq, struct task_struct *p)
}
}
-void mm_init_sched(struct mm_struct *mm,
- struct sched_cache_time __percpu *_pcpu_sched)
+int mm_init_sched(struct mm_struct *mm,
+ struct sched_cache_time __percpu *_pcpu_sched)
{
+ struct sched_cache_group *grp;
unsigned long epoch = 0;
int i;
+ grp = kzalloc_obj(*grp);
+ if (!grp) {
+ free_percpu(_pcpu_sched);
+ mm->sched_cache_grp = NULL;
+ return -ENOMEM;
+ }
+
for_each_possible_cpu(i) {
struct sched_cache_time *pcpu_sched = per_cpu_ptr(_pcpu_sched, i);
struct rq *rq = cpu_rq(i);
@@ -1635,18 +1616,141 @@ void mm_init_sched(struct mm_struct *mm,
epoch = rq->cpu_epoch;
}
- raw_spin_lock_init(&mm->sc_stat.lock);
- mm->sc_stat.epoch = epoch;
- mm->sc_stat.cpu = -1;
- mm->sc_stat.next_scan = jiffies;
- mm->sc_stat.nr_running_avg = 0;
- mm->sc_stat.footprint = 0;
+ raw_spin_lock_init(&grp->lock);
+ grp->epoch = epoch;
+ grp->cpu = -1;
+ grp->next_scan = jiffies;
+ grp->nr_running_avg = 0;
+ grp->footprint = 0;
+ refcount_set(&grp->refcnt, 1);
/*
- * The update to mm->sc_stat should not be reordered
- * before initialization to mm's other fields, in case
+ * The update to grp->pcpu_sched should not be reordered
+ * before initialization to grp's other fields, in case
* the readers may get invalid mm_sched_epoch, etc.
*/
- smp_store_release(&mm->sc_stat.pcpu_sched, _pcpu_sched);
+ smp_store_release(&grp->pcpu_sched, _pcpu_sched);
+ /*
+ * Publish the group last. Not every reader qualifies it by
+ * grp->pcpu_sched - can_migrate_llc_task() only checks that the
+ * pointer is non-NULL before reading grp->footprint and
+ * grp->nr_running_avg - so a reachable group must already be
+ * fully initialized.
+ */
+ smp_store_release(&mm->sched_cache_grp, grp);
+ return 0;
+}
+
+static void sched_cache_group_free_rcu(struct rcu_head *rcu)
+{
+ struct sched_cache_group *grp =
+ container_of(rcu, struct sched_cache_group, rcu);
+
+ free_percpu(grp->pcpu_sched);
+ kfree(grp);
+}
+
+static void sched_cache_group_put(struct sched_cache_group *grp)
+{
+ if (!grp || !refcount_dec_and_test(&grp->refcnt))
+ return;
+
+ call_rcu(&grp->rcu, sched_cache_group_free_rcu);
+}
+
+DEFINE_FREE(sched_cache_group_put, struct sched_cache_group *,
+ sched_cache_group_put(_T));
+
+#define rcu_deref_sched_cache_grp(tsk) \
+ rcu_dereference_check((tsk)->sched_cache_grp, (tsk) == current)
+
+static struct sched_cache_group *sched_cache_replace_grp(struct task_struct *p,
+ struct sched_cache_group *new)
+{
+ struct sched_cache_group *old;
+
+ old = rcu_deref_sched_cache_grp(p);
+ rcu_assign_pointer(p->sched_cache_grp, new);
+
+ return old;
+}
+
+struct sched_cache_group *sched_cache_group_get(struct sched_cache_group *grp)
+{
+ /*
+ * refcount_inc_not_zero() is the acquire primitive for lockless
+ * (RCU) lookups; plain refcount_inc() would scribble the count if
+ * it already reached zero. Return NULL in that case.
+ */
+ if (grp && !refcount_inc_not_zero(&grp->refcnt))
+ grp = NULL;
+
+ return grp;
+}
+
+struct sched_cache_group *task_cache_group_get(struct task_struct *p)
+{
+ guard(rcu)();
+ return sched_cache_group_get(rcu_dereference(p->sched_cache_grp));
+}
+
+void sched_cache_fork(struct task_struct *p)
+{
+ /*
+ * The child takes its own reference on the mm's cache group, separate
+ * from the reference held by the mm. @p is not yet visible to readers,
+ * so a plain initializing store is enough.
+ */
+ RCU_INIT_POINTER(p->sched_cache_grp,
+ sched_cache_group_get(p->mm->sched_cache_grp));
+}
+
+void sched_cache_fork_cleanup(struct task_struct *p)
+{
+ /*
+ * A fork that fails after sched_cache_fork() never reaches exit_mm(),
+ * so drop the reference here. @p never became visible, so there are no
+ * concurrent readers and the reference we hold keeps the group alive.
+ */
+ sched_cache_group_put(rcu_access_pointer(p->sched_cache_grp));
+ RCU_INIT_POINTER(p->sched_cache_grp, NULL);
+}
+
+void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm)
+{
+ struct sched_cache_group *old;
+
+ /*
+ * Acquire the new reference before publishing the pointer, then drop
+ * the old one. @p is current and the only writer of its own pointer.
+ */
+ old = sched_cache_replace_grp(p, sched_cache_group_get(mm->sched_cache_grp));
+ sched_cache_group_put(old);
+}
+
+void sched_cache_exit_mm(struct task_struct *p)
+{
+ struct sched_cache_group *grp = sched_cache_replace_grp(p, NULL);
+
+#ifdef CONFIG_NUMA_BALANCING
+ /*
+ * Subtract this task's footprint from the group before dropping the
+ * reference, so the group footprint converges as its threads exit.
+ * Unlocked for performance; clamp to avoid underflow.
+ */
+ if (grp && p->total_numa_faults) {
+ unsigned long fp = READ_ONCE(grp->footprint);
+ unsigned long sub = min(fp, p->total_numa_faults);
+
+ WRITE_ONCE(grp->footprint, fp - sub);
+ }
+#endif
+ sched_cache_group_put(grp);
+}
+
+void mm_destroy_sched(struct mm_struct *mm)
+{
+ sched_cache_group_put(mm->sched_cache_grp);
+ mm->sched_cache_grp = NULL;
}
/* because why would C be fully specified */
@@ -1697,14 +1801,14 @@ static unsigned long fraction_mm_sched(struct rq *rq,
return div64_u64(NICE_0_LOAD * pcpu_sched->runtime, rq->cpu_runtime + 1);
}
-static int get_pref_llc(struct task_struct *p, struct mm_struct *mm)
+static int get_pref_llc(struct task_struct *p, struct sched_cache_group *grp)
{
int mm_sched_llc = -1, mm_sched_cpu;
- if (!mm)
+ if (!grp)
return -1;
- mm_sched_cpu = READ_ONCE(mm->sc_stat.cpu);
+ mm_sched_cpu = READ_ONCE(grp->cpu);
if (mm_sched_cpu != -1) {
mm_sched_llc = llc_id(mm_sched_cpu);
@@ -1734,8 +1838,8 @@ static unsigned int task_running_on_cpu(int cpu, struct task_struct *p);
static inline
void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
{
+ struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp);
struct sched_cache_time *pcpu_sched;
- struct mm_struct *mm = p->mm;
int mm_sched_llc = -1;
unsigned long epoch;
@@ -1746,12 +1850,18 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
return;
/*
* init_task, kthreads and user thread created
- * by user_mode_thread() don't have mm.
+ * by user_mode_thread() don't have a cache group.
+ * In theory a kernel thread does not have any valid
+ * cache group, because sched_cache_fork() is not
+ * invoked for a kernel thread - !grp should gate the
+ * kernel thread. Use the PF_KTHREAD check explicitly
+ * here for safety reasons, to guard against future
+ * modifications and to pair with task_tick_cache().
*/
- if (!mm || !mm->sc_stat.pcpu_sched)
+ if (p->flags & PF_KTHREAD || !grp || !grp->pcpu_sched)
return;
- pcpu_sched = per_cpu_ptr(mm->sc_stat.pcpu_sched, cpu_of(rq));
+ pcpu_sched = per_cpu_ptr(grp->pcpu_sched, cpu_of(rq));
scoped_guard (raw_spinlock, &rq->cpu_epoch_lock) {
__update_mm_sched(rq, pcpu_sched);
@@ -1764,14 +1874,14 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
* If this process hasn't hit task_cache_work() for a while invalidate
* its preferred state.
*/
- if ((long)(epoch - READ_ONCE(mm->sc_stat.epoch)) > llc_epoch_affinity_timeout ||
- invalid_llc_nr(mm, p, cpu_of(rq)) ||
- exceed_llc_capacity(mm, cpu_of(rq))) {
- if (READ_ONCE(mm->sc_stat.cpu) != -1)
- WRITE_ONCE(mm->sc_stat.cpu, -1);
+ if ((long)(epoch - READ_ONCE(grp->epoch)) > llc_epoch_affinity_timeout ||
+ invalid_llc_nr(grp, p, cpu_of(rq)) ||
+ exceed_llc_capacity(grp, cpu_of(rq))) {
+ if (READ_ONCE(grp->cpu) != -1)
+ WRITE_ONCE(grp->cpu, -1);
}
- mm_sched_llc = get_pref_llc(p, mm);
+ mm_sched_llc = get_pref_llc(p, grp);
/* task not on rq accounted later in account_entity_enqueue() */
if (task_running_on_cpu(rq->cpu, p) &&
@@ -1784,31 +1894,32 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
static void task_tick_cache(struct rq *rq, struct task_struct *p)
{
+ struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp);
struct callback_head *work = &p->cache_work;
- struct mm_struct *mm = p->mm;
unsigned long epoch;
if (!sched_cache_enabled())
return;
- if (!mm || p->flags & PF_KTHREAD ||
- !mm->sc_stat.pcpu_sched)
+ if (!grp || p->flags & PF_KTHREAD ||
+ !grp->pcpu_sched)
return;
epoch = rq->cpu_epoch;
/* avoid moving backwards */
- if (time_after_eq(mm->sc_stat.epoch, epoch))
+ if (time_after_eq(grp->epoch, epoch))
return;
- guard(raw_spinlock)(&mm->sc_stat.lock);
+ guard(raw_spinlock)(&grp->lock);
if (work->next == work) {
task_work_add(p, work, TWA_RESUME);
- WRITE_ONCE(mm->sc_stat.epoch, epoch);
+ WRITE_ONCE(grp->epoch, epoch);
}
}
-static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p)
+static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p,
+ struct sched_cache_group *grp)
{
#ifdef CONFIG_NUMA_BALANCING
int cpu, curr_cpu, nid, pref_nid;
@@ -1816,7 +1927,7 @@ static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p)
if (!static_branch_likely(&sched_numa_balancing))
goto out;
- cpu = READ_ONCE(p->mm->sc_stat.cpu);
+ cpu = READ_ONCE(grp->cpu);
if (cpu != -1)
nid = cpu_to_node(cpu);
curr_cpu = task_cpu(p);
@@ -1873,13 +1984,13 @@ static inline void update_avg_scale(u64 *avg, u64 sample)
static void task_cache_work(struct callback_head *work)
{
+ struct sched_cache_group *grp __free(sched_cache_group_put) = NULL;
+ cpumask_var_t cpus __free(free_cpumask_var) = CPUMASK_VAR_NULL;
int cpu, m_a_cpu = -1, nr_running = 0, curr_cpu;
unsigned long next_scan, now = jiffies;
struct task_struct *p = current, *cur;
unsigned long curr_m_a_occ = 0;
- struct mm_struct *mm = p->mm;
unsigned long m_a_occ = 0;
- cpumask_var_t cpus;
WARN_ON_ONCE(work != &p->cache_work);
@@ -1888,21 +1999,30 @@ static void task_cache_work(struct callback_head *work)
if (p->flags & PF_EXITING)
return;
- next_scan = READ_ONCE(mm->sc_stat.next_scan);
+ /*
+ * A reference makes sure grp is not released by others. The rcu
+ * lock can not be held till after zalloc_cpumask_var() below,
+ * because the latter might sleep.
+ */
+ grp = task_cache_group_get(p);
+ if (!grp)
+ return;
+
+ next_scan = READ_ONCE(grp->next_scan);
if (time_before(now, next_scan))
return;
/* only 1 thread is allowed to scan */
- if (!try_cmpxchg(&mm->sc_stat.next_scan, &next_scan,
+ if (!try_cmpxchg(&grp->next_scan, &next_scan,
now + max_t(unsigned long,
READ_ONCE(llc_epoch_period), 1)))
return;
curr_cpu = task_cpu(p);
- if (invalid_llc_nr(mm, p, curr_cpu) ||
- exceed_llc_capacity(mm, curr_cpu)) {
- if (READ_ONCE(mm->sc_stat.cpu) != -1)
- WRITE_ONCE(mm->sc_stat.cpu, -1);
+ if (invalid_llc_nr(grp, p, curr_cpu) ||
+ exceed_llc_capacity(grp, curr_cpu)) {
+ if (READ_ONCE(grp->cpu) != -1)
+ WRITE_ONCE(grp->cpu, -1);
return;
}
@@ -1913,7 +2033,7 @@ static void task_cache_work(struct callback_head *work)
scoped_guard (cpus_read_lock) {
guard(rcu)();
- get_scan_cpumasks(cpus, p);
+ get_scan_cpumasks(cpus, p, grp);
for_each_cpu(cpu, cpus) {
/* XXX sched_cluster_active */
@@ -1926,16 +2046,20 @@ static void task_cache_work(struct callback_head *work)
for_each_cpu(i, sched_domain_span(sd)) {
occ = fraction_mm_sched(cpu_rq(i),
- per_cpu_ptr(mm->sc_stat.pcpu_sched, i));
+ per_cpu_ptr(grp->pcpu_sched, i));
a_occ += occ;
if (occ > m_occ) {
m_occ = occ;
m_cpu = i;
}
+ /*
+ * rcu_access_pointer() is used because the
+ * pointer is only compared, never dereferenced.
+ */
cur = rcu_dereference_all(cpu_rq(i)->curr);
if (cur && !(cur->flags & (PF_EXITING | PF_KTHREAD)) &&
- cur->mm == mm)
+ rcu_access_pointer(cur->sched_cache_grp) == grp)
nr_running++;
}
@@ -1959,7 +2083,7 @@ static void task_cache_work(struct callback_head *work)
m_a_cpu = m_cpu;
}
- if (llc_id(cpu) == llc_id(READ_ONCE(mm->sc_stat.cpu)))
+ if (llc_id(cpu) == llc_id(READ_ONCE(grp->cpu)))
curr_m_a_occ = a_occ;
cpumask_andnot(cpus, cpus, sched_domain_span(sd));
@@ -1968,7 +2092,7 @@ static void task_cache_work(struct callback_head *work)
if (m_a_occ > (2 * curr_m_a_occ)) {
/*
- * Avoid switching sc_stat.cpu too fast.
+ * Avoid switching sched_cache_grp->cpu too fast.
* The reason to choose 2X is because:
* 1. It is better to keep the preferred LLC stable,
* rather than changing it frequently and cause migrations
@@ -1977,11 +2101,10 @@ static void task_cache_work(struct callback_head *work)
* 3. 2X is chosen based on test results, as it delivers
* the optimal performance gain so far.
*/
- WRITE_ONCE(mm->sc_stat.cpu, m_a_cpu);
+ WRITE_ONCE(grp->cpu, m_a_cpu);
}
- update_avg_scale(&mm->sc_stat.nr_running_avg, nr_running);
- free_cpumask_var(cpus);
+ update_avg_scale(&grp->nr_running_avg, nr_running);
}
void init_sched_mm(struct task_struct *p)
@@ -1991,10 +2114,18 @@ void init_sched_mm(struct task_struct *p)
init_task_work(work, task_cache_work);
work->next = work;
/*
+ * dup_task_struct() copies the parent's task_struct, including its
+ * sched_cache_grp, for which the child holds no reference. Clear it
+ * here - before copy_mm() runs - so the child never carries a
+ * borrowed pointer that the fork error path would put.
+ */
+ RCU_INIT_POINTER(p->sched_cache_grp, NULL);
+ /*
* Reset new task's preference to avoid
* polluting account_llc_enqueue().
*/
p->preferred_llc = -1;
+ p->pref_llc_queued = 0;
}
#else /* CONFIG_SCHED_CACHE */
@@ -2016,6 +2147,10 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p) {}
static void account_llc_dequeue(struct rq *rq, struct task_struct *p) {}
+static void pref_llc_running_inc(struct rq *rq, struct task_struct *p) {}
+
+static void pref_llc_running_dec(struct rq *rq, struct task_struct *p) {}
+
#endif /* CONFIG_SCHED_CACHE */
/*
@@ -3522,7 +3657,7 @@ static void update_task_scan_period(struct task_struct *p,
p->mm->numa_next_scan = jiffies +
msecs_to_jiffies(p->numa_scan_period);
- return;
+ goto out;
}
/*
@@ -3566,7 +3701,10 @@ static void update_task_scan_period(struct task_struct *p,
p->numa_scan_period = clamp(p->numa_scan_period + diff,
task_scan_min(p), task_scan_max(p));
- memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
+
+out:
+ memset(p->numa_faults_locality, 0,
+ sizeof(p->numa_faults_locality));
}
/*
@@ -3692,6 +3830,7 @@ static int preferred_group_nid(struct task_struct *p, int nid)
static void task_numa_placement(struct task_struct *p)
__context_unsafe(/* conditional locking */)
{
+ struct sched_cache_group __maybe_unused *grp;
int seq, nid, max_nid = NUMA_NO_NODE;
unsigned long max_faults = 0;
unsigned long fault_types[2] = { 0, 0 };
@@ -3784,19 +3923,24 @@ static void task_numa_placement(struct task_struct *p)
* heuristic and occasional lost updates are tolerable.
*
* If a task exits, its corresponding footprint must
- * be subtracted from the mm->sc_stat.footprint, otherwise
- * the mm->sc_stat.footprint will not converge:
- * the exiting thread's footprint remains unchanged/undecayed
- * in mm->sc_stat.footprint. See exit_mm().
+ * be subtracted from p->sched_cache_grp->footprint,
+ * otherwise the footprint will not converge: the
+ * exiting thread's footprint remains unchanged/undecayed.
+ * See exit_mm().
*
* Lost updates and unsynchronized subtraction
* in exit_mm() can cause footprint + diff to
* go negative. Clamp to zero to prevent the
* unsigned footprint from wrapping.
*/
- new_fp = (long)READ_ONCE(p->mm->sc_stat.footprint) + diff;
- WRITE_ONCE(p->mm->sc_stat.footprint,
- max(new_fp, 0L));
+ scoped_guard(rcu) {
+ grp = rcu_dereference(p->sched_cache_grp);
+
+ if (grp) {
+ new_fp = (long)READ_ONCE(grp->footprint) + diff;
+ WRITE_ONCE(grp->footprint, max(new_fp, 0L));
+ }
+ }
#endif
}
@@ -6266,12 +6410,20 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
load = cfs_rq->sum_weight;
if (curr && curr->on_rq)
load += avg_vruntime_weight(cfs_rq, curr->h_load.weight);
-
- weight = avg_vruntime_weight(cfs_rq, se->h_load.weight);
- lag *= load + weight;
if (WARN_ON_ONCE(!load))
load = 1;
- lag = div64_long(lag, load);
+
+ weight = avg_vruntime_weight(cfs_rq, se->h_load.weight);
+
+ /*
+ * Rewrite, because the product (W + w_i)*vl'_i might
+ * overflow s64:
+ *
+ * vl_i = (W + w_i)*vl'_i / W
+ * = (W*vl'_i + w_i*vl'_i) / W
+ * = vl'_i + w_i*vl'_i/W
+ */
+ lag += div64_long(lag * weight, load);
/*
* A heavy entity (relative to the tree) will pull the
@@ -6390,15 +6542,27 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
static void set_delayed(struct sched_entity *se)
{
- se->sched_delayed = 1;
-
/*
* Delayed se of cfs_rq have no tasks queued on them.
* Do not adjust h_nr_runnable since __dequeue_task()
* will account it for blocked tasks.
+ *
+ * This check can be removed because when flat pick
+ * patches get merged as only task can get delayed,
+ * same for clear_delayed().
*/
- if (!entity_is_task(se))
+ if (!entity_is_task(se)) {
+ se->sched_delayed = 1;
return;
+ }
+
+ /*
+ * Drop a task leaving the runnable set.
+ * Needs to be called before sched_delayed is set.
+ * clear_delayed() mirrors this after clearing the flag.
+ */
+ pref_llc_running_dec(rq_of(cfs_rq_of(se)), task_of(se));
+ se->sched_delayed = 1;
for_each_sched_entity(se) {
struct cfs_rq *cfs_rq = cfs_rq_of(se);
@@ -6420,6 +6584,13 @@ static void clear_delayed(struct sched_entity *se)
if (!entity_is_task(se))
return;
+ /*
+ * Re-add on wake, after sched_delayed is cleared. On a final delayed
+ * dequeue account_llc_dequeue() already cleared pref_llc_queued, so
+ * this does nothing.
+ */
+ pref_llc_running_inc(rq_of(cfs_rq_of(se)), task_of(se));
+
for_each_sched_entity(se) {
struct cfs_rq *cfs_rq = cfs_rq_of(se);
@@ -8022,10 +8193,12 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
if (p->in_iowait)
cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT);
+ curr = (cfs_rq->curr == se);
/*
- * XXX comment on the curr thing
+ * This can actually happen with CFS_BANDWIDTH, a task can be throttled
+ * and dequeued, unthrottled from another CPU and then enqueued again
+ * before schedule().
*/
- curr = (cfs_rq->curr == se);
if (curr)
place_entity(cfs_rq, se, flags);
@@ -8458,8 +8631,8 @@ static int
sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
{
unsigned long load, min_load = ULONG_MAX;
- unsigned int min_exit_latency = UINT_MAX;
- u64 latest_idle_timestamp = 0;
+ u64 min_exit_latency = U64_MAX;
+ unsigned int nr_candidates = 0;
int least_loaded_cpu = this_cpu;
int shallowest_idle_cpu = -1;
int i;
@@ -8480,24 +8653,16 @@ sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *
if (available_idle_cpu(i)) {
struct cpuidle_state *idle = idle_get_state(rq);
- if (idle && idle->exit_latency < min_exit_latency) {
- /*
- * We give priority to a CPU whose idle state
- * has the smallest exit latency irrespective
- * of any idle timestamp.
- */
- min_exit_latency = idle->exit_latency;
- latest_idle_timestamp = rq->idle_stamp;
- shallowest_idle_cpu = i;
- } else if ((!idle || idle->exit_latency == min_exit_latency) &&
- rq->idle_stamp > latest_idle_timestamp) {
- /*
- * If equal or no active idle state, then
- * the most recently idled CPU might have
- * a warmer cache.
- */
- latest_idle_timestamp = rq->idle_stamp;
+ u64 exit_latency = idle ? idle->exit_latency : U64_MAX;
+
+ if (shallowest_idle_cpu == -1 || exit_latency < min_exit_latency) {
+ min_exit_latency = exit_latency;
shallowest_idle_cpu = i;
+ nr_candidates = 1;
+ } else if (exit_latency == min_exit_latency) {
+ nr_candidates++;
+ if (!reciprocal_scale(sched_rng(), nr_candidates))
+ shallowest_idle_cpu = i;
}
} else if (shallowest_idle_cpu == -1) {
load = cpu_load(cpu_rq(i));
@@ -8596,6 +8761,35 @@ static inline bool test_idle_cores(int cpu)
}
/*
+ * Redirect a CPU to a higher-priority available sibling in its SMT domain,
+ * subject to task affinity.
+ */
+static inline int select_idle_smt_cpu(struct task_struct *p, int cpu)
+{
+ struct sched_domain *sd;
+ int best = cpu;
+ int sibling;
+
+ if (!sched_smt_active())
+ return cpu;
+
+ sd = rcu_dereference_all(cpu_rq(cpu)->sd);
+ if (!sd || !(sd->flags & SD_SHARE_CPUCAPACITY) ||
+ !(sd->flags & SD_ASYM_PACKING))
+ return cpu;
+
+ for_each_cpu_and(sibling, sched_domain_span(sd), p->cpus_ptr) {
+ if (sibling == best || !choose_idle_cpu(sibling, p))
+ continue;
+
+ if (sched_asym_prefer(sibling, best))
+ best = sibling;
+ }
+
+ return best;
+}
+
+/*
* Scans the local SMT mask to see if the entire core is idle, and records this
* information in sd_balance_shared->has_idle_cores.
*
@@ -8979,7 +9173,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
if (choose_idle_cpu(target, p) &&
asym_fits_cpu(task_util, util_min, util_max, target))
- return target;
+ goto select_smt_priority;
/*
* If the previous CPU is cache affine and idle, don't be stupid:
@@ -8989,8 +9183,10 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
asym_fits_cpu(task_util, util_min, util_max, prev)) {
if (!static_branch_unlikely(&sched_cluster_active) ||
- cpus_share_resources(prev, target))
- return prev;
+ cpus_share_resources(prev, target)) {
+ target = prev;
+ goto select_smt_priority;
+ }
prev_aff = prev;
}
@@ -9008,7 +9204,8 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
prev == smp_processor_id() &&
this_rq()->nr_running <= 1 &&
asym_fits_cpu(task_util, util_min, util_max, prev)) {
- return prev;
+ target = prev;
+ goto select_smt_priority;
}
/* Check a recently used CPU as a potential idle candidate: */
@@ -9022,8 +9219,10 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
asym_fits_cpu(task_util, util_min, util_max, recent_used_cpu)) {
if (!static_branch_unlikely(&sched_cluster_active) ||
- cpus_share_resources(recent_used_cpu, target))
- return recent_used_cpu;
+ cpus_share_resources(recent_used_cpu, target)) {
+ target = recent_used_cpu;
+ goto select_smt_priority;
+ }
} else {
recent_used_cpu = -1;
@@ -9045,7 +9244,11 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
*/
if (sd) {
i = select_idle_capacity(p, sd, target);
- return ((unsigned)i < nr_cpumask_bits) ? i : target;
+ if ((unsigned int)i < nr_cpumask_bits) {
+ target = i;
+ goto select_smt_priority;
+ }
+ return target;
}
}
@@ -9058,14 +9261,18 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
if (!has_idle_core && cpus_share_cache(prev, target)) {
i = select_idle_smt(p, sd, prev);
- if ((unsigned int)i < nr_cpumask_bits)
- return i;
+ if ((unsigned int)i < nr_cpumask_bits) {
+ target = i;
+ goto select_smt_priority;
+ }
}
}
i = select_idle_cpu(p, sd, has_idle_core, target);
- if ((unsigned)i < nr_cpumask_bits)
- return i;
+ if ((unsigned int)i < nr_cpumask_bits) {
+ target = i;
+ goto select_smt_priority;
+ }
/*
* For cluster machines which have lower sharing cache like L2 or
@@ -9073,12 +9280,19 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
* first. But prev_cpu or recent_used_cpu may also be a good candidate,
* use them if possible when no idle CPU found in select_idle_cpu().
*/
- if ((unsigned int)prev_aff < nr_cpumask_bits)
- return prev_aff;
- if ((unsigned int)recent_used_cpu < nr_cpumask_bits)
- return recent_used_cpu;
+ if ((unsigned int)prev_aff < nr_cpumask_bits) {
+ target = prev_aff;
+ goto select_smt_priority;
+ }
+ if ((unsigned int)recent_used_cpu < nr_cpumask_bits) {
+ target = recent_used_cpu;
+ goto select_smt_priority;
+ }
return target;
+
+select_smt_priority:
+ return select_idle_smt_cpu(p, target);
}
/**
@@ -9755,8 +9969,10 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags)
}
/* Slow path */
- if (unlikely(sd))
- return sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag);
+ if (unlikely(sd)) {
+ new_cpu = sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag);
+ return select_idle_smt_cpu(p, new_cpu);
+ }
/* Fast path */
if (wake_flags & WF_TTWU)
@@ -9856,8 +10072,14 @@ static inline bool set_preempt_buddy(struct cfs_rq *cfs_rq, struct sched_entity
static inline bool set_short_buddy(struct cfs_rq *cfs_rq, struct sched_entity *pse)
{
- if (cfs_rq->next && cfs_rq->next->slice < pse->slice)
- return false;
+ if (cfs_rq->next) {
+ if (cfs_rq->next->slice < pse->slice)
+ return false;
+
+ if (cfs_rq->next->slice == pse->slice &&
+ entity_before(cfs_rq->next, pse))
+ return false;
+ }
set_next_buddy(cfs_rq, pse);
return true;
@@ -10395,6 +10617,7 @@ enum migration_type {
#define LBF_SOME_PINNED 0x08
#define LBF_ACTIVE_LB 0x10
#define LBF_LLC_PINNED 0x20
+#define LBF_ACTIVE_LB_LLC 0x40
struct lb_env {
struct sched_domain *sd;
@@ -10724,7 +10947,7 @@ static inline bool task_misfits_asym_cpu(struct lb_env *env, struct task_struct
static enum llc_mig can_migrate_llc_task(struct lb_env *env,
struct task_struct *p)
{
- struct mm_struct *mm;
+ struct sched_cache_group *grp;
bool to_pref;
int cpu, src_cpu, dst_cpu;
@@ -10733,19 +10956,19 @@ static enum llc_mig can_migrate_llc_task(struct lb_env *env,
src_cpu = env->src_cpu;
dst_cpu = env->dst_cpu;
- mm = p->mm;
- if (!mm)
+ grp = rcu_dereference_all(p->sched_cache_grp);
+ if (!grp)
return mig_unrestricted;
- cpu = READ_ONCE(mm->sc_stat.cpu);
+ cpu = READ_ONCE(grp->cpu);
if (cpu < 0 || cpus_share_cache(src_cpu, dst_cpu))
return mig_unrestricted;
/* skip cache aware load balance for too many threads */
- if (invalid_llc_nr(mm, p, dst_cpu) ||
- exceed_llc_capacity(mm, dst_cpu)) {
- if (READ_ONCE(mm->sc_stat.cpu) != -1)
- WRITE_ONCE(mm->sc_stat.cpu, -1);
+ if (invalid_llc_nr(grp, p, dst_cpu) ||
+ exceed_llc_capacity(grp, dst_cpu)) {
+ if (READ_ONCE(grp->cpu) != -1)
+ WRITE_ONCE(grp->cpu, -1);
return mig_unrestricted;
}
@@ -10814,6 +11037,21 @@ alb_break_llc(struct lb_env *env)
}
/*
+ * Returns true if p's preferred LLC does not match the destination CPU
+ * under migrate_llc_task semantics. Passive LB passes migrate_llc_task
+ * in env->migration_type, while active LB carries LBF_ACTIVE_LB_LLC in
+ * env->flags to avoid overwriting env->migration_type.
+ */
+static inline bool
+migrate_llc_task_wrong_dst(struct task_struct *p, struct lb_env *env)
+{
+ return sched_cache_enabled() &&
+ (env->migration_type == migrate_llc_task ||
+ env->flags & LBF_ACTIVE_LB_LLC) &&
+ READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu);
+}
+
+/*
* Check if migrating task p from env->src_cpu to
* env->dst_cpu breaks LLC localiy.
*/
@@ -10841,8 +11079,7 @@ static bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env)
* run on env->dst_cpu, skip the tasks do not prefer
* env->dst_cpu, and find the one that prefers.
*/
- if (env->migration_type == migrate_llc_task &&
- READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu))
+ if (migrate_llc_task_wrong_dst(p, env))
return true;
if (can_migrate_llc_task(env, p) != mig_forbid)
@@ -10865,6 +11102,12 @@ alb_break_llc(struct lb_env *env)
}
static inline bool
+migrate_llc_task_wrong_dst(struct task_struct *p, struct lb_env *env)
+{
+ return false;
+}
+
+static inline bool
migrate_degrades_llc(struct task_struct *p, struct lb_env *env)
{
return false;
@@ -10963,7 +11206,7 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
* 4) too many balance attempts have failed.
*/
if (env->flags & LBF_ACTIVE_LB)
- return 1;
+ return !migrate_llc_task_wrong_dst(p, env);
degrades = migrate_degrades_locality(p, env);
if (!degrades) {
@@ -11202,21 +11445,7 @@ next:
*/
static void attach_tasks(struct lb_env *env)
{
- struct list_head *tasks = &env->tasks;
- struct task_struct *p;
- struct rq_flags rf;
-
- rq_lock(env->dst_rq, &rf);
- update_rq_clock(env->dst_rq);
-
- while (!list_empty(tasks)) {
- p = list_first_entry(tasks, struct task_struct, se.group_node);
- list_del_init(&p->se.group_node);
-
- attach_task(env->dst_rq, p);
- }
-
- rq_unlock(env->dst_rq, &rf);
+ __attach_tasks(env->dst_rq, &env->tasks);
}
#ifdef CONFIG_NO_HZ_COMMON
@@ -11875,6 +12104,14 @@ static void record_sg_llc_stats(struct lb_env *env,
return;
/*
+ * A redo repeats stats published on the first pass. Hotplug can
+ * still leave env->cpus partial, so util would not match capacity.
+ */
+ if ((env->flags & LBF_ALL_PINNED) ||
+ !cpumask_subset(sched_group_span(group), env->cpus))
+ return;
+
+ /*
* At this point we know this group spans a LLC domain.
* Record the statistic of this group in its corresponding
* shared LLC domain.
@@ -13362,6 +13599,20 @@ static int need_active_balance(struct lb_env *env)
}
static int active_load_balance_cpu_stop(void *data);
+static int active_load_balance_llc_cpu_stop(void *data);
+
+/*
+ * migration_type is checked elsewhere to decide migration policy, so
+ * it shouldn't be repurposed just to flag an LLC-directed active
+ * balance across the stopper. Pick the callback here instead.
+ */
+static inline cpu_stop_fn_t alb_stop_fn(struct lb_env *env)
+{
+ if (env->migration_type == migrate_llc_task)
+ return active_load_balance_llc_cpu_stop;
+
+ return active_load_balance_cpu_stop;
+}
static int should_we_balance(struct lb_env *env)
{
@@ -13495,7 +13746,7 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq,
};
bool need_unlock = false;
- cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask);
+ cpumask_and(cpus, sched_domain_span(sd), cpu_preferred_mask);
schedstat_inc(sd->lb_count[idle]);
@@ -13707,7 +13958,7 @@ more_balance:
}
if (active_balance) {
stop_one_cpu_nowait(cpu_of(busiest),
- active_load_balance_cpu_stop, busiest,
+ alb_stop_fn(&env), busiest,
&busiest->active_balance_work);
}
preempt_enable();
@@ -13812,7 +14063,7 @@ update_next_balance(struct sched_domain *sd, unsigned long *next_balance)
* least 1 task to be running on each physical CPU where possible, and
* avoids physical / logical imbalances.
*/
-static int active_load_balance_cpu_stop(void *data)
+static int __active_load_balance_cpu_stop(void *data, unsigned int lb_flags)
{
struct rq *busiest_rq = data;
int busiest_cpu = cpu_of(busiest_rq);
@@ -13862,7 +14113,7 @@ static int active_load_balance_cpu_stop(void *data)
.src_cpu = busiest_rq->cpu,
.src_rq = busiest_rq,
.idle = CPU_IDLE,
- .flags = LBF_ACTIVE_LB,
+ .flags = LBF_ACTIVE_LB | lb_flags,
};
schedstat_inc(sd->alb_count);
@@ -13890,6 +14141,16 @@ out_unlock:
return 0;
}
+static int active_load_balance_cpu_stop(void *data)
+{
+ return __active_load_balance_cpu_stop(data, 0);
+}
+
+static int active_load_balance_llc_cpu_stop(void *data)
+{
+ return __active_load_balance_cpu_stop(data, LBF_ACTIVE_LB_LLC);
+}
+
/*
* Scale the max sched_balance_rq interval with the number of CPUs in the system.
* This trades load-balance latency on larger machines for less cross talk.
@@ -14610,10 +14871,8 @@ static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf)
*/
this_rq->idle_stamp = rq_clock(this_rq);
- /*
- * Do not pull tasks towards !active CPUs...
- */
- if (!cpu_active(this_cpu))
+ /* Do not pull tasks towards !preferred CPUs */
+ if (!cpu_preferred(this_cpu))
return 0;
/*
@@ -15253,14 +15512,18 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p)
}
}
-static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)
+static void set_next_task_fair(struct rq *rq, struct task_struct *p, enum snt_e type)
{
struct sched_entity *se = &p->se;
- bool throttled = false;
struct cfs_rq *cfs_rq = &rq->cfs;
unsigned long weight = NICE_0_LOAD;
+ bool first = type == SNT_PICK;
+ bool throttled = false;
bool on_rq = se->on_rq;
+ if (type == SNT_REPICK)
+ goto repick;
+
clear_buddies(cfs_rq, se);
if (on_rq)
@@ -15304,11 +15567,18 @@ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)
WARN_ON_ONCE(se->sched_delayed);
- if (hrtick_enabled_fair(rq))
- hrtick_start_fair(rq, p);
-
update_misfit_status(p, rq);
sched_fair_update_stop_tick(rq, p);
+
+repick:
+ /*
+ * A same-task repick skips put_prev_task_fair(), but
+ * pick_task_fair() refreshed the entity hrtick_start_fair() reads
+ * before selecting it again. rq->cfs.curr identifies that entity,
+ * including with group scheduling.
+ */
+ if (hrtick_enabled_fair(rq))
+ hrtick_start_fair(rq, p);
}
void init_cfs_rq(struct cfs_rq *cfs_rq)
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index eb73b65ce6c4..76f3c84ca684 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -487,8 +487,11 @@ static void put_prev_task_idle(struct rq *rq, struct task_struct *prev, struct t
update_rq_avg_idle(rq);
}
-static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first)
+static void set_next_task_idle(struct rq *rq, struct task_struct *next, enum snt_e type)
{
+ if (type == SNT_REPICK)
+ return;
+
update_idle_core(rq);
scx_update_idle(rq, true, true);
schedstat_inc(rq->sched_goidle);
diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c
index 4e152410653d..7c5381423334 100644
--- a/kernel/sched/psi.c
+++ b/kernel/sched/psi.c
@@ -626,8 +626,6 @@ static void init_rtpoll_triggers(struct psi_group *group, u64 now)
static void psi_schedule_rtpoll_work(struct psi_group *group, unsigned long delay,
bool force)
{
- struct task_struct *task;
-
/*
* atomic_xchg should be called even when !force to provide a
* full memory barrier (see the comment inside psi_rtpoll_work).
@@ -635,19 +633,16 @@ static void psi_schedule_rtpoll_work(struct psi_group *group, unsigned long dela
if (atomic_xchg(&group->rtpoll_scheduled, 1) && !force)
return;
- rcu_read_lock();
-
- task = rcu_dereference(group->rtpoll_task);
/*
- * kworker might be NULL in case psi_trigger_destroy races with
- * psi_task_change (hotpath) which can't use locks
+ * Only test whether a worker is installed; do not dereference the task.
+ * A racing trigger destruction may leave the timer armed.
+ * psi_cgroup_free() shuts it down before freeing the group.
+ * The system PSI group is static.
*/
- if (likely(task))
+ if (likely(rcu_access_pointer(group->rtpoll_task)))
mod_timer(&group->rtpoll_timer, jiffies + delay);
else
atomic_set(&group->rtpoll_scheduled, 0);
-
- rcu_read_unlock();
}
static void psi_rtpoll_work(struct psi_group *group)
@@ -1489,21 +1484,11 @@ void psi_trigger_destroy(struct psi_trigger *t)
}
/*
- * Wait for psi_schedule_rtpoll_work RCU to complete its read-side
- * critical section before destroying the trigger and optionally the
- * rtpoll_task.
- */
- synchronize_rcu();
- /*
* Stop kthread 'psimon' after releasing rtpoll_trigger_lock to prevent
* a deadlock while waiting for psi_rtpoll_work to acquire
* rtpoll_trigger_lock
*/
if (task_to_destroy) {
- /*
- * After the RCU grace period has expired, the worker
- * can no longer be found through group->rtpoll_task.
- */
kthread_stop(task_to_destroy);
atomic_set(&group->rtpoll_scheduled, 0);
}
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 85303add726d..1535046a23ff 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -1654,11 +1654,14 @@ static void wakeup_preempt_rt(struct rq *rq, struct task_struct *p, int flags)
check_preempt_equal_prio(rq, p);
}
-static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool first)
+static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, enum snt_e type)
{
struct sched_rt_entity *rt_se = &p->rt;
struct rt_rq *rt_rq = &rq->rt;
+ if (type == SNT_REPICK)
+ return;
+
p->se.exec_start = rq_clock_task(rq);
if (on_rt_rq(&p->rt))
update_stats_wait_end_rt(rt_rq, rt_se);
@@ -1666,7 +1669,7 @@ static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool f
/* The running task is never eligible for pushing */
dequeue_pushable_task(rq, p);
- if (!first)
+ if (type != SNT_PICK)
return;
/*
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index e656c7059bf8..87c5335add37 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -791,6 +791,10 @@ enum scx_rq_flags {
SCX_RQ_BAL_CB_PENDING = 1 << 6, /* must queue a cb after dispatching */
SCX_RQ_SUB_IDLE_RENOTIFY = 1 << 7, /* sub-scheds are owed update_idle() */
SCX_RQ_ROOT_IDLE_RENOTIFY = 1 << 8, /* the root is owed update_idle() */
+ SCX_RQ_PROXY_RETRY = 1 << 9, /* proxy-rejected tasks need retry */
+ SCX_RQ_PROXY_TICK = 1 << 10, /* proxy execution requires the tick */
+ SCX_RQ_PROXY_PENDING = 1 << 11, /* blocked donor awaits proxy resolution */
+ SCX_RQ_PROXY_BLOCKING = 1 << 12, /* donor is being removed for sched change */
SCX_RQ_IN_WAKEUP = 1 << 16,
SCX_RQ_IN_DISPATCH = 1 << 17,
@@ -810,13 +814,12 @@ struct scx_rq_rescue {
struct scx_rq {
struct scx_dispatch_q local_dsq;
+ struct scx_dispatch_q reject_dsq; /* staging for rejected tasks */
#ifdef CONFIG_EXT_SUB_SCHED
- struct scx_dispatch_q reject_dsq; /* staging for cap-rejected tasks */
struct scx_rq_rescue rescue;
#endif
struct list_head runnable_list; /* runnable tasks on this rq */
struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */
- unsigned long ops_qseq;
/* both stashed across the activate_task() in move_remote_task_to_local_dsq() */
u64 remote_activate_enq_flags;
struct scx_sched *remote_activate_sch;
@@ -831,6 +834,8 @@ struct scx_rq {
#endif
u64 clock; /* current per-rq clock -- see scx_bpf_now() */
#ifdef CONFIG_EXT_SUB_SCHED
+ /* sched of the open running session, see scx_cid_sched_update() */
+ struct scx_sched *sched;
struct llist_head ecaps_to_sync; /* pending ecaps syncs */
struct task_struct *sub_dispatch_prev;
#endif
@@ -845,6 +850,9 @@ struct scx_rq {
struct list_head deferred_reenq_users; /* user DSQs requesting reenq */
struct balance_callback deferred_bal_cb;
struct balance_callback kick_sync_bal_cb;
+#ifdef CONFIG_NO_HZ_FULL
+ struct balance_callback proxy_tick_bal_cb;
+#endif
struct irq_work deferred_irq_work;
struct irq_work kick_cpus_irq_work;
};
@@ -1326,6 +1334,9 @@ struct rq {
#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
u64 prev_steal_time_rq;
#endif
+#ifdef CONFIG_PREFERRED_CPU
+ bool npc_push_work_pending;
+#endif
/* calc_load related fields */
unsigned long calc_load_update;
@@ -1371,7 +1382,6 @@ struct rq {
struct task_struct *core_pick;
struct sched_dl_entity *core_dl_server;
unsigned int core_enabled;
- unsigned int core_sched_seq;
struct rb_root core_tree;
/* shared state -- careful with sched_core_cpu_deactivate() */
@@ -2447,16 +2457,25 @@ extern __read_mostly unsigned int sysctl_sched_features;
#ifdef CONFIG_JUMP_LABEL
-#define SCHED_FEAT(name, enabled) \
-static __always_inline bool static_branch_##name(struct static_key *key) \
-{ \
- return static_key_##enabled(key); \
+union sched_feat_key {
+ struct static_key_true key_true;
+ struct static_key_false key_false;
+};
+
+#define sched_feat_branch_true(key) static_branch_likely(&(key)->key_true)
+#define sched_feat_branch_false(key) static_branch_unlikely(&(key)->key_false)
+
+#define SCHED_FEAT(name, enabled) \
+static __always_inline bool \
+static_branch_##name(union sched_feat_key *key) \
+{ \
+ return sched_feat_branch_##enabled(key); \
}
#include "features.h"
#undef SCHED_FEAT
-extern struct static_key sched_feat_keys[__SCHED_FEAT_NR];
+extern union sched_feat_key sched_feat_keys[__SCHED_FEAT_NR];
#define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x]))
#else /* !CONFIG_JUMP_LABEL: */
@@ -2508,6 +2527,12 @@ static inline bool task_is_blocked(struct task_struct *p)
return !!p->blocked_on;
}
+#ifdef CONFIG_SCHED_PROXY_EXEC
+void sched_proxy_block_task(struct rq *rq, struct task_struct *p);
+#else
+static inline void sched_proxy_block_task(struct rq *rq, struct task_struct *p) {}
+#endif
+
static inline int task_on_cpu(struct rq *rq, struct task_struct *p)
{
return p->on_cpu;
@@ -2527,11 +2552,17 @@ static inline int task_on_rq_migrating(struct task_struct *p)
#define WF_EXEC 0x02 /* Wakeup after exec; maps to SD_BALANCE_EXEC */
#define WF_FORK 0x04 /* Wakeup after fork; maps to SD_BALANCE_FORK */
#define WF_TTWU 0x08 /* Wakeup; maps to SD_BALANCE_WAKE */
-
-#define WF_SYNC 0x10 /* Waker goes to sleep after wakeup */
+/*
+ * Hint that the caller expects the waker to sleep soon.
+ * Scheduler classes may use it for placement or preemption.
+ * Callers must not rely on it to prevent migration,
+ * preserve CPU locality or make the wakee run next.
+ */
+#define WF_SYNC 0x10
#define WF_MIGRATED 0x20 /* Internal use, task got migrated */
#define WF_CURRENT_CPU 0x40 /* Prefer to move the wakee to the current CPU. */
#define WF_RQ_SELECTED 0x80 /* ->select_task_rq() was called */
+#define WF_TTWU_RQ 0x100 /* Wakeup completed through ttwu_runnable() */
static_assert(WF_EXEC == SD_BALANCE_EXEC);
static_assert(WF_FORK == SD_BALANCE_FORK);
@@ -2621,6 +2652,12 @@ struct affinity_context {
extern s64 update_curr_common(struct rq *rq);
+enum snt_e {
+ SNT_NORMAL, /* set_next_task() */
+ SNT_PICK, /* put_prev_set_next_task(): prev != next */
+ SNT_REPICK, /* put_prev_set_next_task(): prev == next */
+};
+
struct sched_class {
#ifdef CONFIG_UCLAMP_TASK
@@ -2678,7 +2715,7 @@ struct sched_class {
* __schedule: rq->lock
*/
void (*put_prev_task)(struct rq *rq, struct task_struct *p, struct task_struct *next);
- void (*set_next_task)(struct rq *rq, struct task_struct *p, bool first);
+ void (*set_next_task)(struct rq *rq, struct task_struct *p, enum snt_e type);
/*
* select_task_rq: p->pi_lock
@@ -2781,7 +2818,7 @@ static inline void put_prev_task(struct rq *rq, struct task_struct *prev)
static inline void set_next_task(struct rq *rq, struct task_struct *next)
{
- next->sched_class->set_next_task(rq, next, false);
+ next->sched_class->set_next_task(rq, next, SNT_NORMAL);
}
static inline void
@@ -2802,11 +2839,13 @@ static inline void put_prev_set_next_task(struct rq *rq,
__put_prev_set_next_dl_server(rq, prev, next);
- if (next == prev)
+ if (next == prev) {
+ next->sched_class->set_next_task(rq, next, SNT_REPICK);
return;
+ }
prev->sched_class->put_prev_task(rq, prev, next);
- next->sched_class->set_next_task(rq, next, true);
+ next->sched_class->set_next_task(rq, next, SNT_PICK);
}
/*
@@ -3139,6 +3178,25 @@ static inline void attach_one_task(struct rq *rq, struct task_struct *p)
attach_task(rq, p);
}
+/*
+ * __attach_tasks() - attaches a list of tasks (using se.group_node) to
+ * the new rq
+ */
+static inline void __attach_tasks(struct rq *rq, struct list_head *tasks)
+{
+ guard(rq_lock)(rq);
+ update_rq_clock(rq);
+
+ while (!list_empty(tasks)) {
+ struct task_struct *p;
+
+ p = list_first_entry(tasks, struct task_struct, se.group_node);
+ list_del_init(&p->se.group_node);
+
+ attach_task(rq, p);
+ }
+}
+
#ifdef CONFIG_PREEMPT_RT
# define SCHED_NR_MIGRATE_BREAK 8
#else
@@ -4222,8 +4280,9 @@ extern void balance_callbacks(struct rq *rq, struct balance_callback *head);
* after which it is enqueued again.
*
* Typically this must be called while holding task_rq_lock, since most/all
- * properties are serialized under those locks. There is currently one
- * exception to this rule in sched/ext which only holds rq->lock.
+ * properties are serialized under those locks. There are currently two
+ * exceptions to this rule in sched/ext which only hold rq->lock: scx_bypass()
+ * and rq_offline_scx().
*/
/*
@@ -4252,4 +4311,10 @@ DEFINE_CLASS_IS_UNCONDITIONAL(sched_change)
#include "ext/ext.h"
+#ifdef CONFIG_PREFERRED_CPU
+void sched_push_current_non_preferred_cpu(struct rq *rq);
+#else /* !CONFIG_PREFERRED_CPU */
+static inline void sched_push_current_non_preferred_cpu(struct rq *rq) { }
+#endif
+
#endif /* _KERNEL_SCHED_SCHED_H */
diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c
index c909ca0d8c87..1e0109ec36b3 100644
--- a/kernel/sched/stop_task.c
+++ b/kernel/sched/stop_task.c
@@ -27,8 +27,11 @@ wakeup_preempt_stop(struct rq *rq, struct task_struct *p, int flags)
/* we're never preempted */
}
-static void set_next_task_stop(struct rq *rq, struct task_struct *stop, bool first)
+static void set_next_task_stop(struct rq *rq, struct task_struct *stop, enum snt_e type)
{
+ if (type == SNT_REPICK)
+ return;
+
stop->se.exec_start = rq_clock_task(rq);
}
diff --git a/kernel/sched/syscalls.c b/kernel/sched/syscalls.c
index b215b0ead9a6..45a127deeadd 100644
--- a/kernel/sched/syscalls.c
+++ b/kernel/sched/syscalls.c
@@ -678,6 +678,8 @@ change:
if (prev_class != next_class)
queue_flags |= DEQUEUE_CLASS;
+ scx_prepare_setscheduler(p, policy);
+
scoped_guard (sched_change, p, queue_flags) {
if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) {
diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index 0248227d983a..919d0fb00bd9 100644
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -32,6 +32,20 @@ static int __init sched_debug_setup(char *str)
}
early_param("sched_verbose", sched_debug_setup);
+#ifdef CONFIG_SCHED_SMT
+static bool sched_smt_asym_packing __read_mostly;
+
+static int __init setup_sched_smt_asym_packing(char *str)
+{
+ if (strcmp(str, "on"))
+ return 0;
+
+ sched_smt_asym_packing = true;
+ return 1;
+}
+__setup("sched_smt_asym_packing=", setup_sched_smt_asym_packing);
+#endif
+
static inline bool sched_debug(void)
{
return sched_debug_verbose;
@@ -985,8 +999,8 @@ void sched_cache_active_set(void)
}
/*
- * Update the bottom sched_domain's llc_bytes for @cpu and all its
- * LLC siblings. Called from cacheinfo_cpu_online() or
+ * Update the bottom sched_domain's llc_bytes for @cpus sharing a physical
+ * LLC. Called from cacheinfo_cpu_online() or
* cacheinfo_cpu_pre_down() with cpu hotplug lock held.
*
* Note: get_effective_llc_bytes() returns 0 on PowerPC.
@@ -996,17 +1010,13 @@ void sched_cache_active_set(void)
* and does not populates the per-CPU struct cpu_cacheinfo array
* that get_cpu_cacheinfo_llc() reads.
*/
-void sched_update_llc_bytes(unsigned int cpu)
+void sched_update_llc_bytes(const struct cpumask *cpus)
{
struct sched_domain *sd, *sdp;
unsigned int i;
sched_domains_mutex_lock();
- sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, cpu));
- if (!sdp)
- goto unlock;
-
/*
* ci->shared_cpu_map is built incrementally as CPUs come
* online, so the first CPU in an LLC initially sees
@@ -1014,14 +1024,22 @@ void sched_update_llc_bytes(unsigned int cpu)
* get_effective_llc_bytes(). Re-evaluating every LLC
* sibling on each online event corrects this once the full
* shared_cpu_map is known.
+ *
+ * The departing CPU's domains have already been detached when
+ * cacheinfo removes it. Use the surviving cache siblings instead.
+ * They may belong to different cpuset partitions, so use each CPU's
+ * own LLC domain to scale its share of the physical cache.
*/
- for_each_cpu(i, sched_domain_span(sdp)) {
+ for_each_cpu(i, cpus) {
+ sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, i));
+ if (!sdp)
+ continue;
+
sd = rcu_dereference_sched_domain(cpu_rq(i)->sd);
if (sd)
sd->llc_bytes = get_effective_llc_bytes(i, sdp);
}
-unlock:
sched_domains_mutex_unlock();
}
@@ -1950,6 +1968,10 @@ sd_init(struct sched_domain_topology_level *tl,
if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS,
"wrong sd_flags in topology description\n"))
sd_flags &= TOPOLOGY_SD_FLAGS;
+#ifdef CONFIG_SCHED_SMT
+ if (sched_smt_asym_packing && (sd_flags & SD_SHARE_CPUCAPACITY))
+ sd_flags |= SD_ASYM_PACKING;
+#endif
sd_flags |= asym_cpu_capacity_classify(sd_span, cpu_map);
*sd = (struct sched_domain){
diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c
index d033f600f48c..f8d6d8b9d03e 100644
--- a/kernel/sched/wait.c
+++ b/kernel/sched/wait.c
@@ -54,14 +54,13 @@ int add_wait_queue_priority_exclusive(struct wait_queue_head *wq_head,
{
struct list_head *head = &wq_head->head;
- wq_entry->flags |= WQ_FLAG_EXCLUSIVE | WQ_FLAG_PRIORITY;
-
guard(spinlock_irqsave)(&wq_head->lock);
if (!list_empty(head) &&
(list_first_entry(head, typeof(*wq_entry), entry)->flags & WQ_FLAG_PRIORITY))
return -EBUSY;
+ wq_entry->flags |= WQ_FLAG_EXCLUSIVE | WQ_FLAG_PRIORITY;
list_add(&wq_entry->entry, head);
return 0;
}
@@ -174,15 +173,11 @@ EXPORT_SYMBOL_GPL(__wake_up_locked_key);
* @mode: which threads
* @key: opaque value to be passed to wakeup targets
*
- * The sync wakeup differs that the waker knows that it will schedule
- * away soon, so while the target thread will be woken up, it will not
- * be migrated to another CPU - ie. the two threads are 'synchronized'
- * with each other. This can prevent needless bouncing between CPUs.
- *
- * On UP it can prevent extra preemption.
+ * Passes WF_SYNC to waitqueue wake functions. The default wake function
+ * forwards it to the scheduler; see WF_SYNC for the hint's semantics.
*
- * If this function wakes up a task, it executes a full memory barrier before
- * accessing the task state.
+ * If this function wakes up a task, it executes a full memory barrier
+ * before accessing the task state.
*/
void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode,
void *key)
@@ -200,15 +195,7 @@ EXPORT_SYMBOL_GPL(__wake_up_sync_key);
* @mode: which threads
* @key: opaque value to be passed to wakeup targets
*
- * The sync wakeup differs in that the waker knows that it will schedule
- * away soon, so while the target thread will be woken up, it will not
- * be migrated to another CPU - ie. the two threads are 'synchronized'
- * with each other. This can prevent needless bouncing between CPUs.
- *
- * On UP it can prevent extra preemption.
- *
- * If this function wakes up a task, it executes a full memory barrier before
- * accessing the task state.
+ * Same as __wake_up_sync_key(), but called with @wq_head->lock held.
*/
void __wake_up_locked_sync_key(struct wait_queue_head *wq_head,
unsigned int mode, void *key)
generated by cgit v1.2.3 (git 2.0.4) at 2026-10-24 22:56:19 +0000