summaryrefslogtreecommitdiff
path: root/kernel/sched/ext
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/sched/ext')
-rw-r--r--kernel/sched/ext/ext.c142
-rw-r--r--kernel/sched/ext/idle.c11
-rw-r--r--kernel/sched/ext/internal.h12
-rw-r--r--kernel/sched/ext/sub.c12
4 files changed, 139 insertions, 38 deletions
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 51de1d8b72a1..3219f0da0fe4 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -2919,7 +2919,7 @@ static inline void maybe_queue_balance_callback(struct rq *rq)
static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev)
{
- struct scx_sched *sch = scx_root_protected_live();
+ struct scx_sched *root_sch = scx_root_protected_live();
enum scx_dsp_verdict verdict;
s32 cpu = cpu_of(rq);
@@ -2928,7 +2928,7 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev
scx_process_sync_ecaps(rq, prev);
- if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) &&
+ if ((root_sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) &&
unlikely(rq->scx.cpu_released)) {
/*
* If the previous sched_class for the current CPU was not SCX,
@@ -2936,8 +2936,8 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev
* core. This callback complements ->cpu_release(), which is
* emitted in switch_class().
*/
- if (sch->ops.cpu_acquire)
- SCX_CALL_OP(sch, cpu_acquire, rq, cpu, NULL);
+ if (root_sch->ops.cpu_acquire)
+ SCX_CALL_OP(root_sch, cpu_acquire, rq, cpu, NULL);
rq->scx.cpu_released = false;
}
@@ -2955,7 +2955,7 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev
* test.
*/
if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice &&
- !scx_bypassing(sch, cpu)) {
+ !scx_bypassing(scx_task_sched(prev), cpu)) {
verdict = SCX_DSP_PREV;
goto has_tasks;
}
@@ -2967,20 +2967,25 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev
goto has_tasks;
}
- verdict = scx_dispatch_sched(sch, rq, prev, false);
+ verdict = scx_dispatch_sched(root_sch, rq, prev, false);
if (verdict != SCX_DSP_NONE)
goto has_tasks;
/*
- * Didn't find another task to run. Keep running @prev unless
- * %SCX_OPS_ENQ_LAST is in effect.
+ * Didn't find another task to run. Keep running @prev unless its own
+ * scheduler set %SCX_OPS_ENQ_LAST and takes the enqueue instead, see
+ * put_prev_task_scx(). Read the scheduler here as the dispatch above
+ * may have dropped the rq lock while @prev changed class or scheduler.
*/
- if ((prev->scx.flags & SCX_TASK_QUEUED) &&
- (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu)) &&
- scx_task_can_stay_on_cpu(rq, prev)) {
- __scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1);
- verdict = SCX_DSP_PREV;
- goto has_tasks;
+ if (prev->scx.flags & SCX_TASK_QUEUED) {
+ struct scx_sched *prev_sch = scx_task_sched(prev);
+
+ if ((!(prev_sch->ops.flags & SCX_OPS_ENQ_LAST) ||
+ scx_bypassing(prev_sch, cpu)) && scx_task_can_stay_on_cpu(rq, prev)) {
+ __scx_add_event(prev_sch, SCX_EV_DISPATCH_KEEP_LAST, 1);
+ verdict = SCX_DSP_PREV;
+ goto has_tasks;
+ }
}
rq->scx.flags &= ~SCX_RQ_IN_DISPATCH;
return SCX_DSP_NONE;
@@ -3665,8 +3670,20 @@ static void handle_hotplug(struct rq *rq, bool online)
s16 *tbl = rcu_dereference_check(scx_cpu_to_cid_tbl,
lockdep_is_cpus_held());
- if (tbl)
+ if (tbl) {
+ struct scx_sched *pos;
+
cpu_or_cid = tbl[cpu];
+
+ guard(raw_spinlock_irqsave)(&scx_sched_lock);
+ list_for_each_entry(pos, &scx_sched_all, all) {
+ struct scx_cmask *mask = pos->online_cmask;
+
+ if (mask)
+ __assign_bit(cpu_or_cid, (unsigned long *)mask->bits,
+ online);
+ }
+ }
}
if (online && SCX_HAS_OP(sch, cpu_online))
@@ -4766,7 +4783,8 @@ int scx_tg_online(struct task_group *tg)
{ .weight = tg->scx.weight,
.bw_period_us = tg->scx.bw_period_us,
.bw_quota_us = tg->scx.bw_quota_us,
- .bw_burst_us = tg->scx.bw_burst_us };
+ .bw_burst_us = tg->scx.bw_burst_us,
+ .sched_idle = tg->scx.idle };
ret = SCX_CALL_OP_RET(sch, cgroup_init,
NULL, tg->css.cgroup, &args);
@@ -4932,7 +4950,8 @@ void scx_group_set_idle(struct task_group *tg, bool idle)
percpu_down_read(&scx_cgroup_ops_rwsem);
sch = scx_tg_knob_sched(tg);
- if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle))
+ if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle) &&
+ tg->scx.idle != idle)
SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle);
/* Update the task group's idle state */
@@ -5187,6 +5206,7 @@ static int scx_cgroup_init(struct scx_sched *sch)
.bw_period_us = tg->scx.bw_period_us,
.bw_quota_us = tg->scx.bw_quota_us,
.bw_burst_us = tg->scx.bw_burst_us,
+ .sched_idle = tg->scx.idle,
};
ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args);
@@ -5272,12 +5292,17 @@ static void free_exit_info(struct scx_exit_info *ei);
static const char *scx_exit_reason(enum scx_exit_kind kind);
static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind);
-s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch)
+s32 scx_alloc_kern_arena_objs(struct scx_sched *sch)
{
size_t size = struct_size_t(struct scx_cmask, bits,
SCX_CMASK_NR_WORDS(num_possible_cpus()));
+ struct scx_cmask *online;
+ struct scx_cmask_ref ref;
int cpu;
+ /* hotplug stays excluded until the online mask is published */
+ lockdep_assert_cpus_held();
+
if (!sch->is_cid_type || !sch->arena_pool)
return 0;
@@ -5293,15 +5318,28 @@ s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch)
return -ENOMEM;
scx_cmask_init(*slot, 0, num_possible_cpus());
}
+
+ /* pack the online mask alongside the scratch masks */
+ online = scx_arena_alloc(sch, size);
+ if (!online)
+ return -ENOMEM;
+
+ scoped_guard(rcu) {
+ scx_cmask_ref_init_kern(sch, online, 0, num_possible_cpus(), &ref);
+ scx_cmask_ref_from_cpumask(&ref, cpu_active_mask);
+ }
+ sch->online_cmask = online;
+
return 0;
}
-static void scx_set_cmask_scratch_free(struct scx_sched *sch)
+static void scx_free_kern_arena_objs(struct scx_sched *sch)
{
size_t size = struct_size_t(struct scx_cmask, bits,
SCX_CMASK_NR_WORDS(num_possible_cpus()));
int cpu;
+ scx_arena_free(sch, sch->online_cmask, size);
if (!sch->set_cmask_scratch)
return;
@@ -5388,7 +5426,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL);
free_exit_info(sch->exit_info);
- scx_set_cmask_scratch_free(sch);
+ scx_free_kern_arena_objs(sch);
scx_arena_pool_destroy(sch);
if (sch->arena_map)
bpf_map_put(sch->arena_map);
@@ -7508,22 +7546,24 @@ static void scx_root_enable_workfn(struct kthread_work *work)
#ifdef CONFIG_EXT_SUB_SCHED
cgroup_get(cgrp);
#endif
+ /*
+ * Transition to ENABLING to arm the disable path. Allocation failure
+ * still unwinds locally. Full disabling on failure applies only after
+ * scx_alloc_and_add_sched() succeeds.
+ */
+ WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED);
+ WARN_ON_ONCE(scx_root);
+
sch = scx_alloc_and_add_sched(cmd, cgrp, NULL);
if (IS_ERR(sch)) {
ret = PTR_ERR(sch);
+ WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_ENABLING);
goto err_free_tid_hash;
}
if (sch->is_cid_type)
static_branch_enable(&__scx_is_cid_type);
- /*
- * Transition to ENABLING and clear exit info to arm the disable path.
- * Failure triggers full disabling from here on.
- */
- WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED);
- WARN_ON_ONCE(scx_root);
-
atomic_long_set(&scx_nr_rejected, 0);
for_each_possible_cpu(cpu) {
@@ -7591,7 +7631,7 @@ static void scx_root_enable_workfn(struct kthread_work *work)
goto err_disable;
}
- ret = scx_set_cmask_scratch_alloc(sch);
+ ret = scx_alloc_kern_arena_objs(sch);
if (ret) {
cpus_read_unlock();
goto err_disable;
@@ -8946,10 +8986,17 @@ __bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id,
#ifdef CONFIG_EXT_SUB_SCHED
/*
* Disallow if any sub-scheds are attached. There is no way to tell
- * which scheduler called us, just error out @p's scheduler.
+ * which scheduler called us, so error out @p's scheduler -- read it
+ * under RCU as @p's locks aren't necessarily held here. @p may be a
+ * task past sched_ext_dead() or an idle task, in which case its
+ * scheduler can't be determined and there is nothing obviously wrong
+ * to report; just refuse the call.
*/
if (unlikely(!list_empty(&sch->children))) {
- scx_error(scx_task_sched(p), "__scx_bpf_dsq_insert_vtime() must be used");
+ struct scx_sched *tsch = scx_task_sched_rcu(p);
+
+ if (tsch)
+ scx_error(tsch, "__scx_bpf_dsq_insert_vtime() must be used");
return;
}
#endif
@@ -10321,7 +10368,8 @@ __bpf_kfunc u32 scx_bpf_nr_cids(void)
* hotplug, which lets schedulers treat [0, nr_online_cids) as the online
* range. Schedulers that prefer to handle hotplug without a restart should
* install a custom mapping via scx_bpf_cid_override() and track onlining
- * through the ops.cid_online / ops.cid_offline callbacks.
+ * through the ops.cid_online / ops.cid_offline callbacks, starting from the
+ * mask scx_bpf_online_cmask() returns.
*/
__bpf_kfunc u32 scx_bpf_nr_online_cids(void)
{
@@ -10329,6 +10377,37 @@ __bpf_kfunc u32 scx_bpf_nr_online_cids(void)
}
/**
+ * scx_bpf_online_cmask - Return the online cid mask in the scheduler arena
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * Return a kernel-maintained cmask covering [0, scx_bpf_nr_cids()), or NULL if
+ * the calling program is not associated with a live cid-form scheduler or the
+ * mask is not allocated yet, as in ops.init_cids(). Treat the mask as read-only
+ * even though arena memory stays writable by the BPF scheduler. The mask
+ * follows the SCX hotplug notifications: a cid's bit is updated before
+ * ops.cid_online/offline() runs for it. The pointer is valid from ops.init()
+ * through ops.exit(). Root ops.init() runs with hotplug excluded. Other
+ * contexts can observe concurrent updates.
+ */
+__bpf_kfunc const void *scx_bpf_online_cmask(const struct bpf_prog_aux *aux)
+{
+ struct scx_sched *sch;
+ struct scx_cmask *online;
+
+ guard(rcu)();
+
+ sch = scx_prog_sched(aux);
+ if (unlikely(!sch))
+ return NULL;
+ online = sch->online_cmask;
+ if (unlikely(!online))
+ return NULL;
+
+ /* BPF rebases by the low 32 bits, like __arena callback args */
+ return (void *)((unsigned long)online - sch->arena_kern_base);
+}
+
+/**
* scx_bpf_this_cid - Return the cid of the CPU this program is running on
*
* cid-addressed equivalent of bpf_get_smp_processor_id() for scx programs.
@@ -10691,6 +10770,7 @@ BTF_ID_FLAGS(func, scx_bpf_nr_node_ids)
BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids)
BTF_ID_FLAGS(func, scx_bpf_nr_cids)
BTF_ID_FLAGS(func, scx_bpf_nr_online_cids)
+BTF_ID_FLAGS(func, scx_bpf_online_cmask, KF_IMPLICIT_ARGS | KF_ARENA_RET)
BTF_ID_FLAGS(func, scx_bpf_this_cid)
BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE)
BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE)
diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c
index d2973fb3af6d..aa9fb6de0ad6 100644
--- a/kernel/sched/ext/idle.c
+++ b/kernel/sched/ext/idle.c
@@ -1142,10 +1142,17 @@ __bpf_kfunc s32 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64
#ifdef CONFIG_EXT_SUB_SCHED
/*
* Disallow if any sub-scheds are attached. There is no way to tell
- * which scheduler called us, just error out @p's scheduler.
+ * which scheduler called us, so error out @p's scheduler -- read it
+ * under RCU as @p's locks aren't necessarily held here. @p may be a
+ * task past sched_ext_dead() or an idle task, in which case its
+ * scheduler can't be determined and there is nothing obviously wrong
+ * to report; just refuse the call.
*/
if (unlikely(!list_empty(&sch->children))) {
- scx_error(scx_task_sched(p), "__scx_bpf_select_cpu_and() must be used");
+ struct scx_sched *tsch = scx_task_sched_rcu(p);
+
+ if (tsch)
+ scx_error(tsch, "__scx_bpf_select_cpu_and() must be used");
return -EINVAL;
}
#endif
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 0967b99a4948..3464e0f113c1 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -259,6 +259,9 @@ struct scx_cgroup_init_args {
u64 bw_period_us;
u64 bw_quota_us;
u64 bw_burst_us;
+
+ /* whether the cgroup is configured SCHED_IDLE via cpu.idle */
+ bool sched_idle;
};
enum scx_cpu_preempt_reason {
@@ -569,6 +572,12 @@ struct sched_ext_ops {
*
* Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle
* tracking.
+ *
+ * Only actual transitions are reported. A CPU that is claimed with an
+ * idle pick and kicked but dispatches no task returns to idle without a
+ * transition. A scheduler tracking idle CPUs itself must restore the
+ * idle state from ops.dispatch() when it returns without the next task
+ * to run.
*/
void (*update_idle)(s32 cpu, bool idle);
@@ -1552,6 +1561,7 @@ struct scx_sched {
* and passes it to the callback's __arena argument.
*/
struct scx_cmask * __percpu *set_cmask_scratch;
+ struct scx_cmask *online_cmask;
DECLARE_BITMAP(has_op, SCX_OPI_END);
@@ -2078,7 +2088,7 @@ void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p);
void scx_cgroup_lock(void);
void scx_cgroup_unlock(void);
#endif
-s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch);
+s32 scx_alloc_kern_arena_objs(struct scx_sched *sch);
void scx_disable_bypass_dsp(struct scx_sched *sch);
void scx_bypass(struct scx_sched *sch, bool bypass);
s32 scx_link_sched(struct scx_sched *sch);
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index 9e7040482bde..34e642a1a403 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -1361,6 +1361,7 @@ static s32 scx_cgroup_claim_subtree(struct scx_sched *sch)
.bw_period_us = tg->scx.bw_period_us,
.bw_quota_us = tg->scx.bw_quota_us,
.bw_burst_us = tg->scx.bw_burst_us,
+ .sched_idle = tg->scx.idle,
};
if (tg->scx.sched != parent ||
@@ -1464,6 +1465,7 @@ static void scx_cgroup_return_subtree(struct scx_sched *sch)
.bw_period_us = tg->scx.bw_period_us,
.bw_quota_us = tg->scx.bw_quota_us,
.bw_burst_us = tg->scx.bw_burst_us,
+ .sched_idle = tg->scx.idle,
};
/* the first pass must have transferred everything */
@@ -1803,6 +1805,12 @@ void scx_sub_enable_workfn(struct kthread_work *work)
goto err_disable;
}
+ scoped_guard(cpus_read_lock) {
+ ret = scx_alloc_kern_arena_objs(sch);
+ if (ret)
+ goto err_disable;
+ }
+
if (sch->ops.init) {
ret = SCX_CALL_OP_RET(sch, init, NULL);
if (ret) {
@@ -1813,10 +1821,6 @@ void scx_sub_enable_workfn(struct kthread_work *work)
sch->exit_info->flags |= SCX_EFLAG_INITIALIZED;
}
- ret = scx_set_cmask_scratch_alloc(sch);
- if (ret)
- goto err_disable;
-
struct scx_sub_attach_args sub_attach_args = {
.ops = &sch->ops,
.cgroup_path = sch->cgrp_path,
generated by cgit v1.2.3 (git 2.0.4) at 2026-09-16 19:18:25 +0000