diff options
65 files changed, 6166 insertions, 719 deletions
diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst index 794ae80b3ba3..13ba2cb7831c 100644 --- a/Documentation/scheduler/sched-ext.rst +++ b/Documentation/scheduler/sched-ext.rst @@ -503,6 +503,19 @@ and edge cases, to name a few examples: class, in which case it will exit the tick-dispatch loop even though it is runnable and has a non-zero slice. +* Under proxy execution, sched_ext continues to observe the donor as the + current scheduling context. Accordingly, ``ops.running()`` and + ``ops.stopping()`` report when the donor's scheduling context becomes active + and inactive, even when the donor is blocked and a lock owner executes on its + behalf. The physical execution context is intentionally not reported through + these callbacks. + + A blocked donor enters a running session only after proxy resolution finds + an execution context. The session remains active if only the physical + execution context changes while the donor remains the same. Running sessions + are tracked so that ``ops.running()`` and ``ops.stopping()`` remain paired + and are not emitted recursively. + See the "Scheduling Cycle" section for a more detailed description of how a freshly woken up task gets on a CPU. diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 36c04797436c..efd635a99309 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -91,7 +91,7 @@ struct scx_dispatch_q { struct rhash_head hash_node; struct llist_node free_node; struct scx_sched *sched; - struct scx_dsq_pcpu __percpu *pcpu; + struct scx_dsq_pcpu __percpu *pcpu_user; struct rcu_head rcu; }; @@ -104,6 +104,7 @@ enum scx_ent_flags { SCX_TASK_SUB_INIT = 1 << 4, /* task being initialized for a sub sched */ SCX_TASK_IMMED = 1 << 5, /* task is on local DSQ with %SCX_ENQ_IMMED */ SCX_TASK_PROTECTED = 1 << 6, /* slice and DSQ head position protected */ + SCX_TASK_RUN_TRACKED = 1 << 7, /* task is in an ops.running()/stopping() session */ /* * Bits 8 to 10 are used to carry task state: @@ -136,6 +137,7 @@ enum scx_ent_flags { * IMMED reenqueued due to failed ENQ_IMMED * PREEMPTED preempted while running * CAP sub-sched cap miss, see p->scx.reenq_reason_* + * PROXY proxy state prevented a remote DSQ transfer */ SCX_TASK_REENQ_REASON_SHIFT = 12, SCX_TASK_REENQ_REASON_BITS = 3, @@ -146,6 +148,7 @@ enum scx_ent_flags { SCX_TASK_REENQ_IMMED = 2 << SCX_TASK_REENQ_REASON_SHIFT, SCX_TASK_REENQ_PREEMPTED = 3 << SCX_TASK_REENQ_REASON_SHIFT, SCX_TASK_REENQ_CAP = 4 << SCX_TASK_REENQ_REASON_SHIFT, + SCX_TASK_REENQ_PROXY = 5 << SCX_TASK_REENQ_REASON_SHIFT, /* iteration cursor, not a task */ SCX_TASK_CURSOR = 1 << 31, @@ -154,6 +157,7 @@ enum scx_ent_flags { /* scx_entity.dsq_flags */ enum scx_ent_dsq_flags { SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ + SCX_TASK_DSQ_REENQ = 1 << 1, /* marked by scx_bpf_dsq_reenq_task() */ }; enum scx_dsq_lnode_flags { @@ -197,9 +201,19 @@ struct sched_ext_entity { u64 ddsp_slice; u64 ddsp_vtime; struct scx_dsq_list_node dsq_list; /* dispatch order */ - struct rb_node dsq_priq; /* p->scx.dsq_vtime order */ u32 dsq_seq; u32 dsq_flags; /* protected by DSQ lock */ + + /* + * Used to order tasks when dispatching to the vtime-ordered priority + * queue of a dsq. This is usually set through + * scx_bpf_dsq_insert_vtime() but can also be modified directly by the + * BPF scheduler. Modifying it while a task is queued on a dsq may + * mangle the ordering and is not recommended. Kept next to @dsq_priq + * as rbtree insertion reads both on every visited node. + */ + u64 dsq_vtime; + struct rb_node dsq_priq; /* p->scx.dsq_vtime order */ u32 flags; /* protected by rq lock */ u32 weight; u32 reenq_cnt; /* reenqueues since last run */ @@ -226,7 +240,7 @@ struct sched_ext_entity { u64 tid; struct rhash_head tid_hash_node; /* see SCX_OPS_TID_TO_TASK */ - /* BPF scheduler modifiable fields */ + /* BPF scheduler modifiable fields, along with @dsq_vtime above */ /* * Runtime budget in nsecs - how long the task may hold its cpu. Owned @@ -242,15 +256,6 @@ struct sched_ext_entity { u64 slice; /* - * Used to order tasks when dispatching to the vtime-ordered priority - * queue of a dsq. This is usually set through - * scx_bpf_dsq_insert_vtime() but can also be modified directly by the - * BPF scheduler. Modifying it while a task is queued on a dsq may - * mangle the ordering and is not recommended. - */ - u64 dsq_vtime; - - /* * Out-of-band slice request from scx_bpf_task_set_slice() when the * caller does not hold the rq lock, applied under the rq lock at the * next slice consideration. One atomic64 packs the pending flag, the @@ -279,6 +284,14 @@ struct sched_ext_entity { */ bool disallow; /* reject switching into SCX */ + /* + * If set, depletion of this task's slice at the scheduler tick requests + * lazy instead of immediate rescheduling. Initialized from + * %SCX_OPS_LAZY_RESCHED immediately before ops.enable() and may be + * modified afterwards with scx_bpf_task_set_lazy_resched(). + */ + bool lazy_resched; + /* cold fields */ #ifdef CONFIG_EXT_GROUP_SCHED struct cgroup *cgrp_moving_from; @@ -324,7 +337,7 @@ struct scx_task_group { u64 bw_period_us; u64 bw_quota_us; u64 bw_burst_us; - bool idle; + bool sched_idle; #endif }; diff --git a/init/Kconfig b/init/Kconfig index 046ae0bf1186..b31aa298c4fe 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -743,8 +743,6 @@ config SCHED_PROXY_EXEC bool "Proxy Execution" # Avoid some build failures w/ PREEMPT_RT until it can be fixed depends on !PREEMPT_RT - # Need to investigate how to inform sched_ext of split contexts - depends on !SCHED_CLASS_EXT # Not particularly useful until we get to multi-rq proxying depends on EXPERT help diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index 4b08866d75f6..a5330b760ecd 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -31,6 +31,7 @@ static struct scx_cid_tables *scx_cid_tables; /* used only during alloc/free */ #define SCX_CID_TOPO_NEG (struct scx_cid_topo) { \ .core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1, \ .node_cid = -1, .node_idx = -1, .shard_cid = -1, .shard_idx = -1, \ + .cluster_cid = -1, .cluster_idx = -1, \ } /* @@ -182,12 +183,14 @@ s32 scx_cid_init(struct scx_sched *sch) cpumask_var_t to_walk __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t node_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t llc_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t cluster_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t core_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL; struct scx_cid_tables *tbls; u32 next_cid = 0; s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0; + s32 next_cluster_idx = 0; s32 next_shard_idx = 0; u32 shard_size, max_cids; u32 notopo_in_shard; @@ -215,6 +218,7 @@ s32 scx_cid_init(struct scx_sched *sch) if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) || !zalloc_cpumask_var(&node_scratch, GFP_KERNEL) || !zalloc_cpumask_var(&llc_scratch, GFP_KERNEL) || + !zalloc_cpumask_var(&cluster_scratch, GFP_KERNEL) || !zalloc_cpumask_var(&core_scratch, GFP_KERNEL) || !zalloc_cpumask_var(&llc_fallback, GFP_KERNEL) || !zalloc_cpumask_var(&online_no_topo, GFP_KERNEL)) @@ -256,6 +260,7 @@ s32 scx_cid_init(struct scx_sched *sch) u32 cores_per_shard, nr_large; u32 shard_local = 0, cores_in_shard = 0, cids_in_shard = 0; s32 shard_cid, shard_idx; + s32 cluster_cid = -1, cluster_idx = -1; /* llc_scratch = node_scratch & this llc */ cpumask_and(llc_scratch, node_scratch, llc_mask); @@ -268,15 +273,32 @@ s32 scx_cid_init(struct scx_sched *sch) tbls->shard_node[shard_idx] = nid; while (!cpumask_empty(llc_scratch)) { - s32 lcpu = cpumask_first(llc_scratch); - const struct cpumask *sib = topology_sibling_cpumask(lcpu); - s32 core_cid = next_cid; - s32 core_idx = next_core_idx++; - s32 ccpu; + const struct cpumask *sib; + s32 core_cid, core_idx, lcpu, ccpu; u32 max_cores, cids_in_core; - /* core_scratch = llc_scratch & this core */ - cpumask_and(core_scratch, llc_scratch, sib); + /* + * Take the cores of one cluster before moving + * on, so that a cluster is a contiguous cid + * range like the core, LLC and node levels. + */ + if (cpumask_empty(cluster_scratch)) { + s32 xcpu = cpumask_first(llc_scratch); + + cpumask_or(cluster_scratch, topology_cluster_cpumask(xcpu), + topology_sibling_cpumask(xcpu)); + cpumask_and(cluster_scratch, cluster_scratch, llc_scratch); + cluster_cid = next_cid; + cluster_idx = next_cluster_idx++; + } + + lcpu = cpumask_first(cluster_scratch); + sib = topology_sibling_cpumask(lcpu); + core_cid = next_cid; + core_idx = next_core_idx++; + + /* core_scratch = cluster_scratch & this core */ + cpumask_and(core_scratch, cluster_scratch, sib); if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch))) return -EINVAL; @@ -316,8 +338,11 @@ s32 scx_cid_init(struct scx_sched *sch) .node_idx = node_idx, .shard_cid = shard_cid, .shard_idx = shard_idx, + .cluster_cid = cluster_cid, + .cluster_idx = cluster_idx, }; + cpumask_clear_cpu(ccpu, cluster_scratch); cpumask_clear_cpu(ccpu, llc_scratch); cpumask_clear_cpu(ccpu, node_scratch); cpumask_clear_cpu(ccpu, to_walk); @@ -461,8 +486,8 @@ __bpf_kfunc_start_defs(); * starts must be strictly increasing with the first entry 0 and all values < * num_possible_cpus(). The last shard extends to num_possible_cpus() and no * shard may span more than SCX_CID_SHARD_MAX_CPUS cids. Topo info - * (core/LLC/node) is cleared and the shard layout is set from the input. On - * invalid input, abort the scheduler. + * (core/cluster/LLC/node) is cleared and the shard layout is set from the + * input. On invalid input, abort the scheduler. */ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid__arena, u32 cpu_to_cid_cnt, const s32 *shard_start__arena, u32 shard_start_cnt, @@ -641,6 +666,29 @@ __bpf_kfunc s32 scx_bpf_cid_to_cpu(s32 cid, const struct bpf_prog_aux *aux) } /** + * scx_bpf_cid_node - Return the NUMA node the given @cid belongs to + * @cid: cid to look up + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return NUMA_NO_NODE if @cid is invalid or there is no scheduler. Unlike + * scx_bpf_cpu_node(), this kfunc is available only with CID support, so BPF + * schedulers can use its presence to detect NUMA-aware CID lookups. + */ +__bpf_kfunc s32 scx_bpf_cid_node(s32 cid, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return NUMA_NO_NODE; + cpu = scx_cid_to_cpu(sch, cid); + return cpu < 0 ? NUMA_NO_NODE : cpu_to_node(cpu); +} + +/** * scx_bpf_cpu_to_cid - Return the cid for @cpu * @cpu: cpu to look up * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs @@ -958,6 +1006,7 @@ static const struct btf_kfunc_id_set scx_kfunc_set_init_cids = { BTF_KFUNCS_START(scx_kfunc_ids_cid) BTF_ID_FLAGS(func, scx_bpf_cid_to_cpu, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cid_node, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cpu_to_cid, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cid_topo, KF_IMPLICIT_ARGS) BTF_KFUNCS_END(scx_kfunc_ids_cid) diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h index 2fe2311a0f99..709afbfb97c6 100644 --- a/kernel/sched/ext/cid.h +++ b/kernel/sched/ext/cid.h @@ -13,11 +13,12 @@ * kernel type sized for the maximum NR_CPUS (4k), with verbose helper sequences * for every op. * - * cids give every cpu a dense, topology-ordered id. CPUs sharing a core, LLC or - * NUMA node get contiguous cid ranges, so a topology unit becomes a (start, - * length) slice of cid space. Communication can pass a slice instead of a - * cpumask, and BPF code can process, for example, a u64 word's worth of cids at - * a time. + * cids give every cpu a dense, topology-ordered id. CPUs in each core, + * cluster, LLC or NUMA node get contiguous cid ranges, so a topology unit + * becomes a (start, length) slice of cid space. A core without a wider cluster + * level forms a cluster of its own. Communication can pass a slice + * instead of a cpumask, and BPF code can process, for example, a u64 word's + * worth of cids at a time. * * The mapping is built once at root scheduler enable time by walking the * topology of online cpus only. Going by online cpus is out of necessity: diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 2b6239763b23..9bae826d69f9 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -24,9 +24,41 @@ DEFINE_RAW_SPINLOCK(scx_sched_lock); -bool scx_allow_proxy_exec(const struct task_struct *p) +/* + * Block a retained proxy donor, telling put_prev_task_scx() that the donor is + * about to be dequeued and must not be reenqueued. + */ +static void scx_block_proxy_donor(struct rq *rq, struct task_struct *p) { - return true; + rq->scx.flags |= SCX_RQ_PROXY_BLOCKING; + sched_proxy_block_task(rq, p); + rq->scx.flags &= ~SCX_RQ_PROXY_BLOCKING; +} + +bool __scx_allow_proxy_exec(const struct task_struct *p) +{ + struct scx_sched *sch; + + if (p->sched_class != &ext_sched_class) + return true; + + sch = scx_task_sched(p); + return !sch || (sch->ops.flags & SCX_OPS_ENQ_BLOCKED); +} + +/* + * End retained proxy execution before changing @p's BPF scheduler ownership. + * Called with @p's pi and rq locks held immediately before + * sched_change_begin(). The caller must pass DEQUEUE_NOCLOCK so the rq clock + * is updated only once. + */ +void scx_prepare_task_sched_change(struct task_struct *p) +{ + lockdep_assert_held(&p->pi_lock); + lockdep_assert_rq_held(task_rq(p)); + + update_rq_clock(task_rq(p)); + scx_block_proxy_donor(task_rq(p), p); } /* @@ -46,10 +78,21 @@ struct scx_sched __rcu *scx_root; LIST_HEAD(scx_sched_all); #ifdef CONFIG_EXT_SUB_SCHED +static __always_inline int scx_sched_cmpfn(struct rhashtable_compare_arg *arg, + const void *ptr) +{ + const struct scx_sched *sch = ptr; + + BUILD_BUG_ON(sizeof_field(struct scx_sched, ops.sub_cgroup_id) != sizeof(u64)); + + return sch->ops.sub_cgroup_id != *(const u64 *)arg->key; +} + const struct rhashtable_params scx_sched_hash_params = { .key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id), .key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id), .head_offset = offsetof(struct scx_sched, hash_node), + .obj_cmpfn = scx_sched_cmpfn, .insecure_elasticity = true, /* inserted under scx_sched_lock */ }; @@ -57,10 +100,20 @@ struct rhashtable scx_sched_hash; #endif /* see SCX_OPS_TID_TO_TASK */ +static __always_inline int scx_tid_cmpfn(struct rhashtable_compare_arg *arg, const void *ptr) +{ + const struct sched_ext_entity *scx = ptr; + + BUILD_BUG_ON(sizeof_field(struct sched_ext_entity, tid) != sizeof(u64)); + + return scx->tid != *(const u64 *)arg->key; +} + static const struct rhashtable_params scx_tid_hash_params = { .key_len = sizeof_field(struct sched_ext_entity, tid), .key_offset = offsetof(struct sched_ext_entity, tid), .head_offset = offsetof(struct sched_ext_entity, tid_hash_node), + .obj_cmpfn = scx_tid_cmpfn, .insecure_elasticity = true, /* inserted/removed under scx_tasks_lock */ }; static struct rhashtable scx_tid_hash; @@ -194,10 +247,20 @@ static DEFINE_PER_CPU(struct scx_tid_alloc, scx_tid_alloc); */ static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); +static __always_inline int dsq_cmpfn(struct rhashtable_compare_arg *arg, const void *ptr) +{ + const struct scx_dispatch_q *dsq = ptr; + + BUILD_BUG_ON(sizeof_field(struct scx_dispatch_q, id) != sizeof(u64)); + + return dsq->id != *(const u64 *)arg->key; +} + static const struct rhashtable_params dsq_hash_params = { .key_len = sizeof_field(struct scx_dispatch_q, id), .key_offset = offsetof(struct scx_dispatch_q, id), .head_offset = offsetof(struct scx_dispatch_q, hash_node), + .obj_cmpfn = dsq_cmpfn, }; static LLIST_HEAD(dsqs_to_free); @@ -327,7 +390,6 @@ static const struct sched_class *scx_setscheduler_class(struct task_struct *p) static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu) { -#ifdef CONFIG_EXT_SUB_SCHED /* * If @sch is a sub-sched which is bypassing, its tasks should go into * the bypass DSQs of the nearest ancestor which is not bypassing. The @@ -339,11 +401,7 @@ static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 c * are re-enqueued after scx_bypassing() is turned on, guaranteeing that * all tasks are transferred to the right DSQs. */ - while (scx_parent(sch) && scx_bypassing(sch, cpu)) - sch = scx_parent(sch); -#endif /* CONFIG_EXT_SUB_SCHED */ - - return scx_bypass_dsq(sch, cpu); + return scx_bypass_dsq(scx_exec_sched(sch, cpu), cpu); } /** @@ -379,28 +437,28 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) */ /* - * If we're in the dispatch path holding rq lock, $curr may or may not + * If we're in the dispatch path holding rq lock, $donor may or may not * be ready depending on whether the on-going dispatch decides to extend - * $curr's slice. We say yes here and resolve it at the end of dispatch. + * $donor's slice. We say yes here and resolve it at the end of dispatch. * See dispatch_one(). */ if (rq->scx.flags & SCX_RQ_IN_DISPATCH) return true; /* - * %SCX_ENQ_PREEMPT clears $curr's slice if on SCX and kicks dispatch, + * The preemption flags clear $donor's slice if on SCX and kick dispatch, * so allow it to avoid spuriously triggering reenq on a combined * PREEMPT|IMMED insertion. */ - if (enq_flags & SCX_ENQ_PREEMPT) { - struct task_struct *curr = rq->curr; + if (enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) { + struct task_struct *donor = rq->donor; /* * A protected slice refuses the preemption and the cpu stays * occupied. See rq_owned_post_enq(). */ - return curr->sched_class != &ext_sched_class || - likely(!(curr->scx.flags & SCX_TASK_PROTECTED)); + return donor->sched_class != &ext_sched_class || + likely(!(donor->scx.flags & SCX_TASK_PROTECTED)); } /* @@ -497,11 +555,13 @@ enum scx_dsq_iter_flags { /* iterate in the reverse dispatch order */ SCX_DSQ_ITER_REV = 1U << 16, + __SCX_DSQ_ITER_HAS_REENQ = 1U << 29, __SCX_DSQ_ITER_HAS_SLICE = 1U << 30, __SCX_DSQ_ITER_HAS_VTIME = 1U << 31, __SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV, __SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS | + __SCX_DSQ_ITER_HAS_REENQ | __SCX_DSQ_ITER_HAS_SLICE | __SCX_DSQ_ITER_HAS_VTIME, }; @@ -1092,8 +1152,63 @@ static void schedule_deferred_locked(struct rq *rq) schedule_deferred(rq); } +#ifdef CONFIG_NO_HZ_FULL +static void scx_proxy_tick_bal_cb(struct rq *rq) +{ + sched_update_tick_dependency(rq); +} + +static void scx_proxy_update_tick(struct rq *rq, struct task_struct *next) +{ + bool proxy = next != rq->donor; + bool was_proxy = rq->scx.flags & SCX_RQ_PROXY_TICK; + + if (likely(proxy == was_proxy)) + return; + + if (proxy) { + /* Keep proxy execution tick-driven for now. */ + rq->scx.flags |= SCX_RQ_PROXY_TICK; + tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED); + } else { + rq->scx.flags &= ~SCX_RQ_PROXY_TICK; + /* + * The selected donor is already visible, but rq->curr still + * identifies the outgoing execution context. Reevaluate after + * context_switch() updates rq->curr so sched_can_stop_tick() sees + * the complete selection. + */ + queue_balance_callback(rq, &rq->scx.proxy_tick_bal_cb, + scx_proxy_tick_bal_cb); + } +} +#endif + +/* + * Complete sched_ext bookkeeping after proxy resolution and retry tasks which + * couldn't be reenqueued by an earlier reject DSQ drain. + */ void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next) { + lockdep_assert_rq_held(rq); + +#ifdef CONFIG_NO_HZ_FULL + if (scx_enabled() && tick_nohz_full_cpu(cpu_of(rq))) + scx_proxy_update_tick(rq, next); +#endif + + /* + * A higher priority class may wake up while @rq is transiently idle + * for proxy resolution. wakeup_preempt_scx() isn't invoked in that + * case, so reenqueue IMMED donors parked on the local DSQ here. + */ + if (rq->scx.nr_immed && sched_class_above(rq->next_class, &ext_sched_class)) + scx_schedule_reenq_local(rq, 0); + + if (rq->scx.flags & SCX_RQ_PROXY_RETRY) { + rq->scx.flags &= ~SCX_RQ_PROXY_RETRY; + schedule_deferred_locked(rq); + } } void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, @@ -1112,6 +1227,18 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, rq = container_of(dsq, struct rq, scx.local_dsq); /* + * An inactive cpu's local DSQ holds only tasks that are bound + * to it or are being moved off, so a reenq of it is pointless. + * Once balance push is installed, queue_balance_callback() also + * drops the callback that would run a reenq requested from + * dispatch, and irq_work_queue_on() can't reach an offline cpu. + * The reenq's node would then stay on this rq's list after + * @sch's pcpu is freed. + */ + if (unlikely(!cpu_active(cpu_of(rq)))) + return; + + /* * A sub-sched lacking baseline access on the target cid has no * business triggering IPIs. The lockless test is fine: slipping * through right after a revoke is harmless and a wrong denial @@ -1144,7 +1271,7 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, } else if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) { rq = this_rq(); - struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu, cpu_of(rq)); + struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu_user, cpu_of(rq)); struct scx_deferred_reenq_user *dru = &dsq_pcpu->deferred_reenq_user; /* @@ -1167,7 +1294,9 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, WRITE_ONCE(dru->flags, dru->flags | reenq_flags); } } else { - scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id); + /* destroyed since the caller resolved it, nothing to reenq */ + if (READ_ONCE(dsq->id) != SCX_DSQ_INVALID) + scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id); return; } @@ -1411,20 +1540,27 @@ static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 e static void update_curr_scx(struct rq *rq) { - struct task_struct *curr = rq->curr; + struct task_struct *donor; s64 delta_exec; + /* + * update_curr_scx() is selected through rq->donor->sched_class, not + * rq->curr->sched_class, so @donor is always an EXT task here. If an EXT + * owner executes for a FAIR donor, FAIR's update_curr() runs instead. + */ + donor = rq->donor; + /* apply even on 0 delta_exec, callers may still act on the slice */ - apply_task_slice_oob(rq, curr); + apply_task_slice_oob(rq, donor); delta_exec = update_curr_common(rq); if (unlikely(delta_exec <= 0)) return; - if (curr->scx.slice != SCX_SLICE_INF) - curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); + if (donor->scx.slice != SCX_SLICE_INF) + donor->scx.slice -= min_t(u64, donor->scx.slice, delta_exec); - if (unlikely(curr == scx_rescuee(rq))) + if (unlikely(donor == scx_rescuee(rq))) scx_rescue_charge(rq, delta_exec); dl_server_update(&rq->ext_server, delta_exec); @@ -1530,6 +1666,26 @@ static bool task_leave_custody(struct task_struct *p) return true; } +/* + * A task with an infinite slice may be running with its tick stopped. Lazy + * rescheduling doesn't send an IPI, so restore the tick dependency to guarantee + * that the lazy request is promoted by a real scheduler tick. Set the lazy + * request first so that the dependency update's IPI can also serve it on return + * to user space. + */ +static void scx_resched_curr_lazy(struct rq *rq) +{ + resched_curr_lazy(rq); + + if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) { + rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK; + if (rq->clock_update_flags < RQCF_UPDATED) + update_rq_clock(rq); + update_other_load_avgs(rq); + sched_update_tick_dependency(rq); + } +} + static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) @@ -1538,8 +1694,10 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0); /* - * Only local inserts get the wakeup treatment below. Rejects kick the - * deferred reenq and rescue parks are paced by the rescue timer. + * Only local inserts get the wakeup treatment below. Rejects kick a + * deferred reenq and rescue parks are paced by the rescue timer. Proxy + * rejects which aren't ready when drained request a later retry from + * scx_proxy_reenqueue_retry(). */ if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { if (dsq->id == SCX_DSQ_REJECT) @@ -1593,12 +1751,16 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, if (rq->scx.flags & SCX_RQ_IN_DISPATCH) return; - if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && - rq->curr->sched_class == &ext_sched_class) { - if (likely(scx_set_task_slice(rq->curr, 0))) - resched_curr(rq); - else + if ((enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) && + p != rq->donor && rq->donor->sched_class == &ext_sched_class) { + if (likely(scx_set_task_slice(rq->donor, 0))) { + if (enq_flags & SCX_ENQ_PREEMPT) + resched_curr(rq); + else + scx_resched_curr_lazy(rq); + } else { __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); + } } } @@ -1606,12 +1768,10 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, u64 slice, u64 vtime, u64 enq_flags) { - bool is_rq_owned = false; + bool is_rq_owned = dsq_is_rq_owned(dsq); - if (dsq->id == SCX_DSQ_LOCAL) { + if (dsq->id == SCX_DSQ_LOCAL) dsq = scx_resolve_local_dsq(sch, rq, p, &enq_flags); - is_rq_owned = true; - } WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || @@ -1688,7 +1848,7 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, scx_error(sch, "DSQ ID 0x%016llx already had PRIQ-enqueued tasks", dsq->id); - if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) { + if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) { /* new task inserted at head - use fastpath */ if (dsq_insert_head(dsq, p) && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) rcu_assign_pointer(dsq->first_task, p); @@ -1762,6 +1922,7 @@ void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq) RB_CLEAR_NODE(&p->scx.dsq_priq); p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; } + p->scx.dsq_flags &= ~SCX_TASK_DSQ_REENQ; list_del_init(&p->scx.dsq_list.node); dsq_dec_nr(dsq, p); @@ -1790,10 +1951,10 @@ void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) list_del_init(&p->scx.dsq_list.node); /* - * When dispatching directly from the BPF scheduler to a local - * DSQ, the task isn't associated with any DSQ but - * @p->scx.holding_cpu may be set under the protection of - * %SCX_OPSS_DISPATCHING. + * When dispatch_to_local_dsq() or remote consumption moves a + * task to a local DSQ, the task isn't associated with any DSQ + * but @p->scx.holding_cpu may be set. Clearing holding_cpu + * tells dispatch_to_local_dsq() that it lost to a dequeue. */ if (p->scx.holding_cpu >= 0) p->scx.holding_cpu = -1; @@ -1813,10 +1974,10 @@ void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) scx_task_unlink_from_dsq(p, dsq); } else { /* - * We're racing against dispatch_to_local_dsq() which already - * removed @p from @dsq and set @p->scx.holding_cpu. Clear the - * holding_cpu which tells dispatch_to_local_dsq() that it lost - * the race. + * We're racing against unlink_dsq_and_switch_rq_lock(), + * which already removed @p from @dsq and set + * @p->scx.holding_cpu. Clear holding_cpu to tell + * unlink_dsq_and_switch_rq_lock() that it lost the race. */ WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node)); p->scx.holding_cpu = -1; @@ -1999,6 +2160,12 @@ bool scx_rq_online(struct rq *rq) void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, int sticky_cpu) { + enum { + ENQ_ACTION_NONE, + ENQ_ACTION_DIRECT, + ENQ_ACTION_LOCAL_NOREFILL, + ENQ_ACTION_ENQUEUE, + } action = ENQ_ACTION_NONE; struct scx_sched *sch = scx_task_sched(p); struct task_struct **ddsp_taskp; struct scx_dispatch_q *dsq; @@ -2032,7 +2199,7 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, __scx_exit(sch, SCX_EXIT_ERROR_REENQ, 0, cpu_of(rq), "%s[%d] reenqueued %u times without running", p->comm, p->pid, p->scx.reenq_cnt); - return; + goto out; } } @@ -2052,19 +2219,28 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) goto direct; - /* see %SCX_OPS_ENQ_EXITING */ - if (!(sch->ops.flags & SCX_OPS_ENQ_EXITING) && - unlikely(p->flags & PF_EXITING)) { - __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); - enq_flags |= SCX_ENQ_RESCUE; /* avoid looping on cap rejection */ - goto local; - } + /* + * A full wakeup enqueue happens before is_blocked is cleared. Don't + * report it as another blocked-donor admission. + */ + if ((sch->ops.flags & SCX_OPS_ENQ_BLOCKED) && p->is_blocked && + !(enq_flags & SCX_ENQ_WAKEUP)) { + enq_flags |= SCX_ENQ_BLOCKED; + } else { + /* see %SCX_OPS_ENQ_EXITING */ + if (unlikely(p->flags & PF_EXITING) && + !(sch->ops.flags & SCX_OPS_ENQ_EXITING)) { + __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); + enq_flags |= SCX_ENQ_RESCUE; /* avoid looping on cap rejection */ + goto local; + } - /* see %SCX_OPS_ENQ_MIGRATION_DISABLED */ - if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) && - is_migration_disabled(p)) { - __scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1); - goto local; + /* see %SCX_OPS_ENQ_MIGRATION_DISABLED */ + if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) && + is_migration_disabled(p)) { + __scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1); + goto local; + } } if (unlikely(!SCX_HAS_OP(sch, enqueue))) @@ -2103,14 +2279,14 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, * dequeue may be waiting. The store_release matches their load_acquire. */ atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq); - return; + goto out; direct: - direct_dispatch(sch, p, enq_flags); - return; + action = ENQ_ACTION_DIRECT; + goto out; local_norefill: - scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, enq_flags); - return; + action = ENQ_ACTION_LOCAL_NOREFILL; + goto out; local: dsq = &rq->scx.local_dsq; goto enqueue; @@ -2122,9 +2298,27 @@ bypass: goto enqueue; enqueue: - refill_task_slice_dfl(sch, p); - clear_direct_dispatch(p); - scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); + action = ENQ_ACTION_ENQUEUE; +out: + /* The reason is input to ops.enqueue(), not to the resulting placement. */ + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + + switch (action) { + case ENQ_ACTION_DIRECT: + direct_dispatch(sch, p, enq_flags); + break; + case ENQ_ACTION_LOCAL_NOREFILL: + scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, + enq_flags); + break; + case ENQ_ACTION_ENQUEUE: + refill_task_slice_dfl(sch, p); + clear_direct_dispatch(p); + scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); + break; + case ENQ_ACTION_NONE: + break; + } } static bool task_runnable(const struct task_struct *p) @@ -2187,13 +2381,14 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ rq->scx.flags |= SCX_RQ_IN_WAKEUP; /* - * Restoring a running task will be immediately followed by - * set_next_task_scx() which expects the task to not be on the BPF + * Restoring the current scheduling context will be immediately followed + * by set_next_task_scx() which expects the task to not be on the BPF * scheduler as tasks can only start running through local DSQs. Force * direct-dispatch into the local DSQ by setting the sticky_cpu. Mark * IGNORE_CAPS to force entry into the local DSQ. */ - if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) { + if (unlikely(enq_flags & ENQUEUE_RESTORE) && + task_current_donor(rq, p)) { sticky_cpu = cpu_of(rq); enq_flags |= SCX_ENQ_IGNORE_CAPS; } @@ -2321,10 +2516,10 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ ops_dequeue(rq, p, deq_flags); /* - * A currently running task which is going off @rq first gets dequeued - * and then stops running. As we want running <-> stopping transitions - * to be contained within runnable <-> quiescent transitions, trigger - * ->stopping() early here instead of in put_prev_task_scx(). + * A current scheduling context which is going off @rq first gets + * dequeued and then stops running. As we want running <-> stopping + * transitions to be contained within runnable <-> quiescent transitions, + * trigger ->stopping() early here instead of in put_prev_task_scx(). * * @p may go through multiple stopping <-> running transitions between * here and put_prev_task_scx() if task attribute changes occur while @@ -2332,11 +2527,13 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ * information meaningful to the BPF scheduler and can be suppressed by * skipping the callbacks if the task is !QUEUED. */ - if (task_current(rq, p) && - (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq)))) { - update_curr_scx(rq); - if (SCX_HAS_OP(sch, stopping)) - SCX_CALL_OP_TASK(sch, stopping, rq, p, false); + if (task_current_donor(rq, p) && (p->scx.flags & SCX_TASK_RUN_TRACKED)) { + if (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq))) { + update_curr_scx(rq); + if (SCX_HAS_OP(sch, stopping)) + SCX_CALL_OP_TASK(sch, stopping, rq, p, false); + } + p->scx.flags &= ~SCX_TASK_RUN_TRACKED; } if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p)) @@ -2352,9 +2549,10 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ sub_nr_running(rq, 1); scx_dispatch_dequeue(rq, p); + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; /* see scx_task_slice_ended() for the save/restore exception */ - if (!((deq_flags & DEQUEUE_SAVE) && task_current(rq, p))) + if (!((deq_flags & DEQUEUE_SAVE) && task_current_donor(rq, p))) scx_task_slice_ended(rq, p); clear_direct_dispatch(p); @@ -2394,10 +2592,23 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl /* * Preemption between SCX tasks is implemented by resetting the victim * task's slice to 0 and triggering reschedule on the target CPU. - * Nothing to do. - */ - if (p->sched_class == &ext_sched_class) + * + * A retained proxy donor can wake through ttwu_runnable() without another + * ops.enqueue(). WF_TTWU_RQ identifies this path. Request rescheduling so + * that ops.dispatch() can reconsider the task after ttwu_runnable() clears + * is_blocked. A full wakeup activation has already enqueued the task and + * doesn't need the additional reschedule. + */ + if (p->sched_class == &ext_sched_class) { + if (sched_proxy_exec() && (wake_flags & WF_TTWU_RQ) && + p->is_blocked) { + struct scx_sched *sch = scx_task_sched(p); + + if (sch && (sch->ops.flags & SCX_OPS_ENQ_BLOCKED)) + resched_curr(rq); + } return; + } /* * Getting preempted by a higher-priority class. Reenqueue IMMED tasks. @@ -2425,7 +2636,7 @@ void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags); - if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) + if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) dsq_insert_head(dst_dsq, p); else list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list); @@ -2495,8 +2706,10 @@ static void move_remote_task_to_local_dsq(struct scx_sched *sch, * - The BPF scheduler is bypassed while the rq is offline and we can always say * no to the BPF scheduler initiated migrations while offline. * - * The caller must ensure that @p and @rq are on different CPUs. - * If enforce == true, caller must hold @p's rq lock. + * The caller must ensure that @p and @rq are on different CPUs. If @enforce is + * true, report violations attributable to BPF-directed migrations. The caller + * must hold @p's rq lock to avoid reporting a transient race as a scheduler + * error. */ static bool task_can_run_on_remote_rq(struct scx_sched *sch, struct task_struct *p, struct rq *rq, @@ -2504,17 +2717,27 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, { s32 cpu = cpu_of(rq); - /* - * To prevent races with @p still running on its old CPU while switching - * out, make sure we're holding @p's rq lock so as not to risk - * erroneously killing the BPF scheduler. - */ if (enforce) lockdep_assert_rq_held(task_rq(p)); WARN_ON_ONCE(task_cpu(p) == cpu); /* + * proxy_set_task_cpu() preserves wake_cpu when moving a donor's scheduling + * context to its lock owner's rq. For example, if BPF places donor D on + * CPU0 while its lock owner runs on CPU1, proxy execution moves D to CPU1 + * but leaves D->wake_cpu pointing to CPU0. If D is later put on a shared + * DSQ, CPU0 could otherwise consume it and move it back, only for proxy + * execution to return it to CPU1 again. + * + * When task_cpu() == wake_cpu, the donor has not been proxy-migrated and + * BPF may still choose its placement. An actively donating task is + * rejected separately by task_proxy_running_or_donating(). + */ + if (sched_proxy_exec() && p->is_blocked && task_cpu(p) != p->wake_cpu) + return false; + + /* * If @p has migration disabled, @p->cpus_ptr is updated to contain only * the pinned CPU in migrate_disable_switch() while @p is being switched * out. However, put_prev_task_scx() is called before @p->cpus_ptr is @@ -2555,6 +2778,64 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, return true; } +/* + * Proxy execution can change @p's execution and migration-disabled state + * without touching its DSQ entry or clearing holding_cpu. Check those states + * with @p's rq locked. Without proxy execution, the holding_cpu handshake is + * sufficient and this must not affect the existing migration path. + * + * A BPF-directed transfer to a remote local DSQ performs a normal task + * migration and thus cannot move a migration-disabled task. In contrast, + * proxy_migrate_task() moves only a blocked donor's scheduling context towards + * the mutex owner and preserves its execution home in wake_cpu. The latter is + * therefore allowed even when the donor is migration-disabled. + */ +static bool task_proxy_running_or_donating(struct task_struct *p) +{ + struct rq *src_rq = task_rq(p); + + lockdep_assert_rq_held(src_rq); + + if (!sched_proxy_exec()) + return false; + + /* @p may be rq->curr under another task's scheduling context. */ + if (task_on_cpu(src_rq, p)) + return true; + + /* Don't move an active scheduling context off its source rq. */ + if (task_current_donor(src_rq, p)) + return true; + + return false; +} + +static bool task_proxy_unsafe_to_move(struct task_struct *p) +{ + if (!sched_proxy_exec()) + return false; + + return task_proxy_running_or_donating(p) || is_migration_disabled(p); +} + +/* + * Park a task whose remote transfer raced with proxy execution. Reenqueueing + * from the source rq makes the task's owning scheduler choose its placement + * again and preserves sub-scheduler containment. + */ +static void scx_proxy_reject_task(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 enq_flags) +{ + lockdep_assert_rq_held(rq); + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); + + p->scx.holding_cpu = -1; + p->scx.flags |= SCX_TASK_REENQ_PROXY; + scx_divert_strip_flags(p, &enq_flags); + + scx_dispatch_enqueue(sch, rq, &rq->scx.reject_dsq, p, 0, 0, enq_flags); +} + /** * unlink_dsq_and_switch_rq_lock() - Unlink task and switch to its rq lock * @p: target task @@ -2583,7 +2864,7 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, * points to this CPU. See scx_dispatch_dequeue() for the counterpart. * * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is - * still valid. %false if lost to dequeue. + * still valid and detached from @dsq. %false if lost to dequeue. */ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, struct scx_dispatch_q *dsq, @@ -2604,6 +2885,13 @@ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, /* task_rq couldn't have changed if we're still the holding cpu */ if (likely(p->scx.holding_cpu == cpu) && !WARN_ON_ONCE(src_rq != task_rq(p))) { + /* + * Keep ->dsq set until we own @src_rq so that a racing dequeue + * can find @dsq and clear holding_cpu. Once ownership is + * confirmed, clear it under @src_rq so that deactivate_task() + * takes the !dsq path instead of retaking @dsq->lock. + */ + p->scx.dsq = NULL; scx_wait_dispatching(p); return true; } @@ -2616,6 +2904,19 @@ static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq, struct scx_dispatch_q *dsq, struct rq *src_rq) { if (unlink_dsq_and_switch_rq_lock(p, dsq, this_rq, src_rq)) { + /* + * Proxy execution may have changed @p's running or + * migration-disabled state while switching rq locks without + * clearing holding_cpu. Park it on the source rq and let its + * owning scheduler choose its placement again. + */ + if (unlikely(task_proxy_unsafe_to_move(p))) { + p->scx.dsq = NULL; + scx_proxy_reject_task(sch, src_rq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); + switch_rq_lock(src_rq, this_rq); + return false; + } + move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, this_rq); return true; } else { @@ -2655,6 +2956,19 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, if (dst_dsq->id == SCX_DSQ_LOCAL) { dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); + /* + * Unlike the rq-lock handoff paths, @src_rq has been locked + * throughout this operation. Only active proxy state can race the + * move here; let the enforcing check below diagnose an ordinary + * migration-disabled task. + */ + if (src_rq != dst_rq && + unlikely(task_proxy_running_or_donating(p))) { + dispatch_dequeue_locked(p, src_dsq); + raw_spin_unlock(&src_dsq->lock); + scx_proxy_reject_task(sch, src_rq, p, enq_flags); + return src_rq; + } if (src_rq != dst_rq && unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { dst_dsq = find_global_dsq(sch, task_cpu(p)); @@ -2677,6 +2991,7 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, raw_spin_unlock(&src_dsq->lock); scx_move_local_task_to_local_dsq(sch, p, enq_flags, dst_rq); } else { + dispatch_dequeue_locked(p, src_dsq); raw_spin_unlock(&src_dsq->lock); move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); } @@ -2811,6 +3126,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, if (likely(p->scx.holding_cpu == raw_smp_processor_id()) && !WARN_ON_ONCE(src_rq != task_rq(p))) { bool fallback = false; + /* * If @p is staying on the same rq, there's no need to go * through the full deactivate/activate cycle. Optimize by @@ -2820,6 +3136,9 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, p->scx.holding_cpu = -1; scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, slice, vtime, enq_flags | SCX_ENQ_APPLY_SLICE); + } else if (unlikely(task_proxy_unsafe_to_move(p))) { + fallback = true; + scx_proxy_reject_task(sch, src_rq, p, enq_flags); } else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { p->scx.holding_cpu = -1; fallback = true; @@ -2835,7 +3154,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, } /* if the destination CPU is idle, wake it up */ - if (!fallback && sched_class_above(p->sched_class, dst_rq->curr->sched_class)) + if (!fallback && sched_class_above(p->sched_class, + dst_rq->donor->sched_class)) resched_curr(dst_rq); } @@ -3053,10 +3373,36 @@ has_tasks: return verdict; } -static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e type) +static void scx_start_task_running(struct rq *rq, struct task_struct *p) { struct scx_sched *sch = scx_task_sched(p); + if (p->scx.flags & SCX_TASK_RUN_TRACKED) + return; + + scx_cid_sched_update(rq, sch); + + if (SCX_HAS_OP(sch, running)) + SCX_CALL_OP_TASK(sch, running, rq, p); + + p->scx.flags |= SCX_TASK_RUN_TRACKED; +} + +static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e type) +{ + bool first = type == SNT_PICK; + bool can_stop_tick; + + /* + * A blocked pick is provisional until proxy resolution completes, see + * put_prev_task_scx(). A retry can repick the same donor, so + * SNT_REPICK must set it again. + */ + rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; + if (sched_proxy_exec() && p->is_blocked && + (type == SNT_PICK || type == SNT_REPICK)) + rq->scx.flags |= SCX_RQ_PROXY_PENDING; + if (type == SNT_REPICK) return; @@ -3067,18 +3413,31 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t */ ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC); scx_dispatch_dequeue(rq, p); + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } p->se.exec_start = rq_clock_task(rq); - /* see dequeue_task_scx() on why we skip when !QUEUED */ - if (SCX_HAS_OP(sch, running) && (p->scx.flags & SCX_TASK_QUEUED)) - SCX_CALL_OP_TASK(sch, running, rq, p); + /* + * See dequeue_task_scx() for why we skip when !QUEUED. + * + * During a normal switch (@first), a blocked task is only a provisional + * donor. Proxy resolution may fail or migrate the donor to another CPU, + * so defer ops.running() until scx_proxy_donor_start() confirms that + * resolution succeeded. + * + * !@first denotes restoration after a SAVE/RESTORE cycle. The matching + * dequeue already issued ops.stopping(), so restart the session here + * regardless of the donor state. + */ + if ((p->scx.flags & SCX_TASK_QUEUED) && !(p->is_blocked && first)) + scx_start_task_running(rq, p); clr_task_runnable(p, true); /* apply any pending out-of-band slice request before the tick decision */ apply_task_slice_oob(rq, p); + can_stop_tick = p->scx.slice == SCX_SLICE_INF && !p->is_blocked; /* * @p is getting newly scheduled or got kicked after someone updated its @@ -3089,7 +3448,7 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t * nohz. In the future, we might want to add a mechanism to update * load_avgs periodically on tick-stopped CPUs. */ - if (p->scx.slice == SCX_SLICE_INF) { + if (can_stop_tick) { if (!(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) { /* * Bypass mode always assigns finite slices, so @p @@ -3110,7 +3469,8 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t /* * @rq still references the outgoing scheduling context. A finite - * slice is sufficient by itself to require the tick. + * slice or a blocked proxy donor is sufficient by itself to require + * the tick. */ if (tick_nohz_full_cpu(cpu_of(rq))) tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED); @@ -3119,6 +3479,13 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t void scx_proxy_donor_start(struct rq *rq) { + struct task_struct *donor = rq->donor; + + lockdep_assert_rq_held(rq); + rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; + + if (donor->sched_class == &ext_sched_class && (donor->scx.flags & SCX_TASK_QUEUED)) + scx_start_task_running(rq, donor); } static enum scx_cpu_preempt_reason @@ -3176,8 +3543,13 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, struct task_struct *next) { struct scx_sched *sch = scx_task_sched(p); + bool proxy_put = p->is_blocked && next == rq->idle && + (rq->scx.flags & SCX_RQ_PROXY_PENDING); + bool proxy_block = rq->scx.flags & SCX_RQ_PROXY_BLOCKING; bool rescue_keep = false; + rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; + /* see kick_sync_wait_bal_cb() */ smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); @@ -3194,29 +3566,51 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, scx_task_slice_ended(rq, p); } - /* see dequeue_task_scx() on why we skip when !QUEUED */ - if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED)) - SCX_CALL_OP_TASK(sch, stopping, rq, p, true); + /* + * Preserve the running session when proxy execution refreshes the same + * donor around an execution-context switch on this rq. + */ + if (next != p && (p->scx.flags & SCX_TASK_QUEUED) && + (p->scx.flags & SCX_TASK_RUN_TRACKED)) { + if (SCX_HAS_OP(sch, stopping)) + SCX_CALL_OP_TASK(sch, stopping, rq, p, !proxy_block); + + p->scx.flags &= ~SCX_TASK_RUN_TRACKED; + } if (p->scx.flags & SCX_TASK_QUEUED) { + if (proxy_block) + goto switch_class; + set_task_runnable(rq, p); /* + * If the donor is the same and only the mutex owner changes, + * avoid triggering another ops.enqueue(): the BPF scheduler has + * already admitted the donor, so it can continue running. + */ + if (p->is_blocked && next == p) + goto switch_class; + + /* * If @p has slice left and is being put, @p is getting * preempted by a higher priority scheduler class or core-sched * forcing a different task. Leave it at the head of the local * DSQ unless it was an IMMED task. IMMED tasks should not * linger on a busy CPU, reenqueue them to the BPF scheduler. * + * The exception is an IMMED donor put by proxy_resched_idle(): + * the CPU isn't busy, proxy resolution is only retrying through + * idle, so keep the donor where it can be picked again. + * * An open rescue must keep @p on the local DSQ even if the * scheduler zeroed the slice in ops.stopping() above. */ if ((p->scx.slice || unlikely(p == scx_rescuee(rq))) && !scx_bypassing(sch, cpu_of(rq))) { - if (p->scx.flags & SCX_TASK_IMMED) { + if ((p->scx.flags & SCX_TASK_IMMED) && !proxy_put) { p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } else { u64 enq_flags = 0; @@ -3231,6 +3625,9 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, enq_flags |= SCX_ENQ_HEAD; } else { enq_flags |= SCX_ENQ_HEAD; + /* only require SCX_CAP_ENQ_IMMED, see scx_caps_for_enq() */ + if (proxy_put && (p->scx.flags & SCX_TASK_IMMED)) + enq_flags |= SCX_ENQ_IMMED; } scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, @@ -3243,13 +3640,15 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * If @p is runnable but we're about to enter a lower * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell * ops.enqueue() that @p is the only one available for this cpu. - * This doesn't apply if the baseline access on the CPU is lost. + * This doesn't apply if the baseline access on the CPU is lost or + * proxy resolution temporarily switches to idle. * * Under core scheduling, a pick dispatches only when nothing is * locally runnable and can legitimately go idle with @p still * runnable (see do_pick_task_scx()). */ - if (next && sched_class_above(&ext_sched_class, next->sched_class) && + if (!proxy_put && next && + sched_class_above(&ext_sched_class, next->sched_class) && scx_task_can_stay_on_cpu(rq, p)) { WARN_ON_ONCE(!sched_core_enabled(rq) && !(sch->ops.flags & SCX_OPS_ENQ_LAST)); @@ -3270,8 +3669,10 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, } switch_class: - if (next && next->sched_class != &ext_sched_class) + if (next && next->sched_class != &ext_sched_class) { + scx_cid_sched_update(rq, NULL); switch_class(rq, next); + } } static void kick_sync_wait_bal_cb(struct rq *rq) @@ -3339,7 +3740,8 @@ static struct task_struct *first_local_task(struct rq *rq) } /* - * Run dispatch and queue the follow-up work for a pick. + * Run dispatch for a pick. The follow-up work is queued by do_pick_task_scx() + * once the pick is final. */ static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf, struct task_struct *prev) @@ -3349,18 +3751,6 @@ static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf, rq_unpin_lock(rq, rf); verdict = dispatch_one(rq, prev); rq_repin_lock(rq, rf); - maybe_queue_balance_callback(rq); - - /* - * Defer to a balance callback which can drop rq lock and enable IRQs. - * Waiting directly in the pick path would deadlock against CPUs sending - * us IPIs (e.g. TLB flushes) while we wait for them. - */ - if (unlikely(rq->scx.kick_sync_pending)) { - rq->scx.kick_sync_pending = false; - queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, - kick_sync_wait_bal_cb); - } return verdict; } @@ -3387,23 +3777,20 @@ static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *r verdict = dispatch_one(rq, prev); - if (cpu_of(rq) == smp_processor_id()) { - maybe_queue_balance_callback(rq); - - /* see dispatch_pick() */ - if (unlikely(rq->scx.kick_sync_pending)) { - rq->scx.kick_sync_pending = false; - queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, - kick_sync_wait_bal_cb); - } - } else if (unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { - /* - * Balance callbacks must run in the context that queued them, - * so they can't be queued on another CPU's rq. Run the deferred - * work directly instead. - */ + /* + * A pick that returns a task isn't final under core scheduling: the + * selection goes on to pick the siblings, and their dispatch can + * release the rq lock, which can't happen with balance callbacks + * queued. Don't queue the deferred work as a balance callback, for this + * CPU or a sibling, and run it from irq_work instead. + * + * do_pick_task_scx() leaves %SCX_RQ_BAL_CB_PENDING alone under core + * scheduling, so that the early return above leaves it to the dispatch + * in flight, which handles it here. + */ + if (unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; - run_deferred(rq); + schedule_deferred(rq); } rq_repin_lock(rq, rf); @@ -3425,22 +3812,33 @@ static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *r static struct task_struct * do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) { - struct task_struct *prev = rq->curr; + struct task_struct *prev = rq->donor; + bool core_sched = sched_core_enabled(rq); enum scx_dsp_verdict verdict; struct task_struct *p; + /* A retry can abandon a provisional blocked-donor pick. */ + rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; + /* see kick_sync_wait_bal_cb() */ smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); rq_modified_begin(rq, &ext_sched_class); - if (sched_core_enabled(rq)) + /* + * Core scheduling can get enabled while dispatch_pick() has the rq lock + * released. Stick to @core_sched, so that the follow-up work below + * matches the dispatch that ran. + */ + if (core_sched) verdict = dispatch_core_pick(rq, rf, prev); else verdict = dispatch_pick(rq, rf, prev); - if (verdict == SCX_DSP_RETRY) - return RETRY_TASK; + if (verdict == SCX_DSP_RETRY) { + p = RETRY_TASK; + goto out; + } /* * If any higher-priority sched class enqueued a runnable task on this @@ -3450,8 +3848,10 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) * If @force_scx is true, always try to pick a SCHED_EXT task, * regardless of any higher-priority sched classes activity. */ - if (!force_scx && rq_modified_above(rq, &ext_sched_class)) - return RETRY_TASK; + if (!force_scx && rq_modified_above(rq, &ext_sched_class)) { + p = RETRY_TASK; + goto out; + } /* * If we're keeping @prev, replenish slice if necessary and keep running @@ -3467,7 +3867,7 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) } else { p = first_local_task(rq); if (!p) - return NULL; + goto out; if (unlikely(!p->scx.slice) && scx_task_can_stay_on_cpu(rq, p)) { struct scx_sched *sch = scx_task_sched(p); @@ -3482,6 +3882,40 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) } } +out: + if (cpu_of(rq) != smp_processor_id()) + return p; + + /* + * Balance callbacks must be consumed before the rq lock is released, so + * queue them only when the pick is final. On RETRY_TASK, or after a + * NULL pick from the dl_server, the core continues picking and can + * release the rq lock (e.g., prev_balance() or another dispatch). In + * that case, run the deferred work from irq_work, which doesn't depend + * on the current rq lock section, and leave the kick sync wait pending + * for the next pick. + */ + if (p != RETRY_TASK && (p || !force_scx)) { + /* under core scheduling, see dispatch_core_pick() */ + if (!core_sched) + maybe_queue_balance_callback(rq); + + /* + * Defer to a balance callback which can drop rq lock and enable + * IRQs. Waiting directly in the pick path would deadlock against + * CPUs sending us IPIs (e.g., TLB flushes) while we wait for + * them. + */ + if (unlikely(rq->scx.kick_sync_pending)) { + rq->scx.kick_sync_pending = false; + queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, + kick_sync_wait_bal_cb); + } + } else if (!core_sched && (rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { + rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; + schedule_deferred(rq); + } + return p; } @@ -3889,9 +4323,9 @@ void scx_tick(struct rq *rq) update_other_load_avgs(rq); } -static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) +static void task_tick_scx(struct rq *rq, struct task_struct *donor, int queued) { - struct scx_sched *sch = scx_task_sched(curr); + struct scx_sched *sch = scx_task_sched(donor); update_curr_scx(rq); @@ -3900,12 +4334,18 @@ static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) * management. */ if (scx_bypassing(sch, cpu_of(rq))) - scx_set_task_slice(curr, 0); + scx_set_task_slice(donor, 0); else if (SCX_HAS_OP(sch, tick)) - SCX_CALL_OP_TASK(sch, tick, rq, curr); + SCX_CALL_OP_TASK(sch, tick, rq, donor); - if (!curr->scx.slice) - resched_curr(rq); + if (!donor->scx.slice) { + /* the slice can't be trusted while bypassing */ + if (READ_ONCE(donor->scx.lazy_resched) && + !scx_bypassing(sch, cpu_of(rq))) + scx_resched_curr_lazy(rq); + else + resched_curr(rq); + } } #ifdef CONFIG_EXT_GROUP_SCHED @@ -4021,6 +4461,7 @@ static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p) weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; p->scx.weight = sched_weight_to_cgroup(weight); + p->scx.lazy_resched = sch->ops.flags & SCX_OPS_LAZY_RESCHED; if (SCX_HAS_OP(sch, enable)) { if (scx_is_cid_type()) { @@ -4394,6 +4835,13 @@ static void switching_to_scx(struct rq *rq, struct task_struct *p) static void switched_from_scx(struct rq *rq, struct task_struct *p) { + /* + * A class change of the running task: sched_change_begin() put @p with + * no successor, which leaves rq->scx.sched set. + */ + if (task_current_donor(rq, p)) + scx_cid_sched_update(rq, NULL); + if (task_dead_and_done(p)) return; @@ -4416,7 +4864,7 @@ int scx_check_setscheduler(struct task_struct *p, int policy) { lockdep_assert_rq_held(task_rq(p)); - /* if disallow, reject transitioning into SCX */ + /* If disallow, reject transitioning into SCX. */ if (scx_enabled() && READ_ONCE(p->scx.disallow) && p->policy != policy && policy == SCHED_EXT) return -EACCES; @@ -4424,6 +4872,20 @@ int scx_check_setscheduler(struct task_struct *p, int policy) return 0; } +/* + * Don't carry a donor retained by another class into sched_ext. The caller + * has updated the rq clock and invokes this immediately before + * sched_change_begin() records the task's queued state. + */ +void scx_prepare_setscheduler(struct task_struct *p, int policy) +{ + lockdep_assert_held(&p->pi_lock); + lockdep_assert_rq_held(task_rq(p)); + + if (scx_enabled() && p->policy != policy && policy == SCHED_EXT) + scx_block_proxy_donor(task_rq(p), p); +} + static void process_ddsp_deferred_locals(struct rq *rq) { struct task_struct *p; @@ -4565,8 +5027,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) scx_wait_dispatching(p); scx_dispatch_dequeue(rq, p); - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); p->scx.flags |= reason; list_add_tail(&p->scx.dsq_list.node, &tasks); @@ -4577,21 +5038,20 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; nr_enqueued++; } /* - * The revoke that scheduled this scan may have raced the pick: curr + * The revoke that scheduled this scan may have raced the pick: donor * may be a now-capless task, either one that kept running or one * promoted off the local DSQ between the ecaps sync and this scan. * Zero the slice to evict it. The enqueue gate blocks new capless * inserts, so no later pick can slip through after the scan. */ if ((reenq_flags & SCX_REENQ_CAP_REVOKE) && - rq->curr->sched_class == &ext_sched_class && - scx_task_reenq_on_cap_revoke(rq, rq->curr)) { - scx_set_task_slice(rq->curr, 0); + rq->donor->sched_class == &ext_sched_class && + scx_task_reenq_on_cap_revoke(rq, rq->donor)) { + scx_set_task_slice(rq->donor, 0); resched_curr(rq); } @@ -4640,7 +5100,9 @@ static void process_deferred_reenq_locals(struct rq *rq) static bool user_task_should_reenq(struct task_struct *p, u64 reenq_flags, u32 *reason) { *reason = SCX_TASK_REENQ_KFUNC; - return reenq_flags & SCX_REENQ_ANY; + if (reenq_flags & SCX_REENQ_ANY) + return true; + return (reenq_flags & SCX_REENQ_MARKED) && (p->scx.dsq_flags & SCX_TASK_DSQ_REENQ); } static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flags) @@ -4649,7 +5111,7 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag struct scx_sched *sch = dsq->sched; struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, dsq, 0); struct task_struct *p; - s32 nr_enqueued = 0; + s32 nr_visited = 0; lockdep_assert_rq_held(rq); @@ -4659,6 +5121,21 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag struct rq *task_rq; u32 reason; + /* + * Count skipped tasks too so that a long walk doesn't hold the + * locks throughout. The cursor keeps the position. + */ + if (!(++nr_visited % SCX_TASK_ITER_BATCH)) { + raw_spin_unlock(&dsq->lock); + if (locked_rq) { + scx_rq_lock_drop(locked_rq); + raw_spin_rq_unlock(locked_rq); + locked_rq = NULL; + } + cpu_relax(); + raw_spin_lock(&dsq->lock); + } + p = nldsq_cursor_next_task(&cursor, dsq); if (!p) break; @@ -4690,21 +5167,11 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag dispatch_dequeue_locked(p, dsq); raw_spin_unlock(&dsq->lock); - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); p->scx.flags |= reason; scx_do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1); - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - - if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) { - scx_rq_lock_drop(locked_rq); - raw_spin_rq_unlock(locked_rq); - locked_rq = NULL; - cpu_relax(); - } - raw_spin_lock(&dsq->lock); } @@ -4759,6 +5226,60 @@ static void process_deferred_reenq_users(struct rq *rq) } } +/* + * Drain ready tasks from @rq->scx.reject_dsq and reenqueue them so that their + * owning BPF schedulers choose placement again. Proxy-active tasks remain + * parked and rearm the retry notification for a later proxy resolution. + * + * A task can be re-rejected repeatedly. Reenqueues are bounded per task by + * SCX_REENQ_MAX_REPEAT in scx_do_enqueue_task(), which ejects the owning + * scheduler. + */ +static void scx_reenq_reject(struct rq *rq) +{ + LIST_HEAD(tasks); + struct task_struct *p, *n; + + lockdep_assert_rq_held(rq); + + if (list_empty(&rq->scx.reject_dsq.list)) + return; + + /* + * Move ready tasks to a private list so a task re-rejected by + * scx_do_enqueue_task() below isn't revisited this round. + */ + list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) { + u32 reason = p->scx.flags & SCX_TASK_REENQ_REASON_MASK; + + WARN_ON_ONCE(!reason); + + if (sched_proxy_exec() && reason == SCX_TASK_REENQ_PROXY) { + /* Affinity machinery will dequeue and reactivate @p. */ + if (p->migration_pending) + continue; + + if (task_on_cpu(rq, p) || task_current_donor(rq, p)) { + rq->scx.flags |= SCX_RQ_PROXY_RETRY; + continue; + } + } else { + WARN_ON_ONCE(p->migration_pending); + } + + scx_wait_dispatching(p); + scx_dispatch_dequeue(rq, p); + + list_add_tail(&p->scx.dsq_list.node, &tasks); + } + + list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { + list_del_init(&p->scx.dsq_list.node); + + scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); + } +} + static void run_deferred(struct rq *rq) { process_ddsp_deferred_locals(rq); @@ -4775,14 +5296,18 @@ static void run_deferred(struct rq *rq) #ifdef CONFIG_NO_HZ_FULL bool scx_can_stop_tick(struct rq *rq) { - struct task_struct *p = rq->curr; + struct task_struct *p = rq->donor; struct scx_sched *sch = scx_task_sched(p); + /* Proxy execution is conservatively tick-driven for now. */ + if (rq->scx.flags & SCX_RQ_PROXY_TICK) + return false; + if (p->sched_class != &ext_sched_class) return true; /* - * @rq->curr may still reference an outgoing EXT task after it has been + * @rq->donor may still reference an outgoing EXT task after it has been * dequeued. If no EXT tasks are accounted on @rq, ignore its stale * slice state. If another task is dispatched from a DSQ, * set_next_task_scx() will update the dependency for the incoming task. @@ -4803,7 +5328,8 @@ bool scx_can_stop_tick(struct rq *rq) /* * @rq can dispatch from different DSQs, so we can't tell whether it * needs the tick or not by looking at nr_running. Allow stopping ticks - * iff the BPF scheduler indicated so. See set_next_task_scx(). + * iff set_next_task_scx() determined that the selected scheduling context + * can run tickless. */ return rq->scx.flags & SCX_RQ_CAN_STOP_TICK; } @@ -4818,7 +5344,7 @@ void scx_tg_init(struct task_group *tg) tg->scx.weight = CGROUP_WEIGHT_DFL; tg->scx.bw_period_us = default_bw_period_us(); tg->scx.bw_quota_us = RUNTIME_INF; - tg->scx.idle = false; + tg->scx.sched_idle = false; } /** @@ -4901,7 +5427,7 @@ int scx_tg_online(struct task_group *tg) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, - .sched_idle = tg->scx.idle }; + .sched_idle = tg->scx.sched_idle }; ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, tg->css.cgroup, &args); @@ -5068,11 +5594,11 @@ void scx_group_set_idle(struct task_group *tg, bool idle) sch = scx_tg_knob_sched(tg); if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle) && - tg->scx.idle != idle) + tg->scx.sched_idle != idle) SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); /* Update the task group's idle state */ - tg->scx.idle = idle; + tg->scx.sched_idle = idle; percpu_up_read(&scx_cgroup_ops_rwsem); } @@ -5186,12 +5712,16 @@ s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch) dsq->id = dsq_id; dsq->sched = sch; - dsq->pcpu = alloc_percpu(struct scx_dsq_pcpu); - if (!dsq->pcpu) + /* per-DSQ deferred reenq state is only needed for user DSQs */ + if (dsq_id & SCX_DSQ_FLAG_BUILTIN) + return 0; + + dsq->pcpu_user = alloc_percpu(struct scx_dsq_pcpu); + if (!dsq->pcpu_user) return -ENOMEM; for_each_possible_cpu(cpu) { - struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); + struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu); pcpu->dsq = dsq; INIT_LIST_HEAD(&pcpu->deferred_reenq_user.node); @@ -5204,8 +5734,11 @@ static void exit_dsq(struct scx_dispatch_q *dsq) { s32 cpu; + if (!dsq->pcpu_user) + return; + for_each_possible_cpu(cpu) { - struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); + struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu); struct scx_deferred_reenq_user *dru = &pcpu->deferred_reenq_user; struct rq *rq = cpu_rq(cpu); @@ -5219,7 +5752,7 @@ static void exit_dsq(struct scx_dispatch_q *dsq) } } - free_percpu(dsq->pcpu); + free_percpu(dsq->pcpu_user); } static void free_dsq_rcufn(struct rcu_head *rcu) @@ -5246,7 +5779,7 @@ static void free_dsq_irq_workfn(struct irq_work *irq_work) * and queues nothing. */ for_each_possible_cpu(cpu) { - struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); + struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu); struct rq *rq = cpu_rq(cpu); guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); @@ -5341,7 +5874,7 @@ static int scx_cgroup_init(struct scx_sched *sch) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, - .sched_idle = tg->scx.idle, + .sched_idle = tg->scx.sched_idle, }; ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args); @@ -5535,6 +6068,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) free_cpumask_var(pcpu->cpus_to_kick); free_cpumask_var(pcpu->cpus_to_kick_if_idle); free_cpumask_var(pcpu->cpus_to_preempt); + free_cpumask_var(pcpu->cpus_to_preempt_lazy); free_cpumask_var(pcpu->cpus_to_wait); exit_dsq(scx_bypass_dsq(sch, cpu)); @@ -6013,7 +6547,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) /* * We don't want CPUs to have more than $nr_donor_target tasks and - * balancing to fill donee CPUs upto $nr_target. Once targets are + * balancing to fill donee CPUs up to $nr_target. Once targets are * calculated, find the donee CPUs. */ nr_target = DIV_ROUND_UP(nr_tasks, nr_cpus); @@ -6323,6 +6857,24 @@ void scx_bypass(struct scx_sched *sch, bool bypass) } /* + * Bypass trumps protection. Cycling the queued tasks below + * clears theirs. The current donor isn't on the runnable list, + * so strip its protection here. + * + * When bypass lifts, the current donor keeps running on its + * slice without a cap check. Evict it if it can't stay on the + * cpu. + */ + p = rq->donor; + if (p->sched_class == &ext_sched_class && + scx_is_descendant(scx_task_sched(p), sch)) { + if (bypass) + scx_task_slice_ended(rq, p); + else if (!scx_task_can_stay_on_cpu(rq, p)) + scx_set_task_slice(p, 0); + } + + /* * The use of list_for_each_entry_safe_reverse() is required * because each task is going to be removed from and added back * to the runnable_list during iteration. Because they're added @@ -6334,13 +6886,6 @@ void scx_bypass(struct scx_sched *sch, bool bypass) if (!scx_is_descendant(scx_task_sched(p), sch)) continue; - /* - * Bypass trumps protection. Cycling clears for queued - * tasks but current task needs explicit stripping. - */ - if (bypass && task_current(rq, p)) - scx_task_slice_ended(rq, p); - /* cycling deq/enq is enough, see the function comment */ scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { /* nothing */ ; @@ -6672,6 +7217,8 @@ static void scx_root_disable(struct scx_sched *sch) } } + scx_ops_cid_sched_updated_disable(sch); + /* no task is on scx, turn off all the switches and flush in-progress calls */ static_branch_disable(&__scx_enabled); static_branch_disable(&__scx_is_cid_type); @@ -7050,6 +7597,8 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, scx_rescue_dump(&ns, rq); scx_dump_line(&ns, " curr=%s[%d] class=%ps", rq->curr->comm, rq->curr->pid, rq->curr->sched_class); + scx_dump_line(&ns, " donor=%s[%d] class=%ps", + rq->donor->comm, rq->donor->pid, rq->donor->sched_class); if (!cpumask_empty(pcpu->cpus_to_kick)) scx_dump_line(&ns, " cpus_to_kick : %*pb", cpumask_pr_args(pcpu->cpus_to_kick)); @@ -7059,6 +7608,9 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, if (!cpumask_empty(pcpu->cpus_to_preempt)) scx_dump_line(&ns, " cpus_to_preempt: %*pb", cpumask_pr_args(pcpu->cpus_to_preempt)); + if (!cpumask_empty(pcpu->cpus_to_preempt_lazy)) + scx_dump_line(&ns, " preempt_lazy : %*pb", + cpumask_pr_args(pcpu->cpus_to_preempt_lazy)); if (!cpumask_empty(pcpu->cpus_to_wait)) scx_dump_line(&ns, " cpus_to_wait : %*pb", cpumask_pr_args(pcpu->cpus_to_wait)); @@ -7093,6 +7645,10 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, if (rq->curr->sched_class == &ext_sched_class && (dump_all_tasks || scx_task_on_sched(sch, rq->curr))) scx_dump_task(sch, s, dctx, rq, rq->curr, '*'); + if (rq->donor != rq->curr && + rq->donor->sched_class == &ext_sched_class && + (dump_all_tasks || scx_task_on_sched(sch, rq->donor))) + scx_dump_task(sch, s, dctx, rq, rq->donor, ' '); list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) if (dump_all_tasks || scx_task_on_sched(sch, p)) @@ -7376,6 +7932,7 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, if (!zalloc_cpumask_var_node(&pcpu->cpus_to_kick, GFP_KERNEL, node) || !zalloc_cpumask_var_node(&pcpu->cpus_to_kick_if_idle, GFP_KERNEL, node) || !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt, GFP_KERNEL, node) || + !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt_lazy, GFP_KERNEL, node) || !zalloc_cpumask_var_node(&pcpu->cpus_to_wait, GFP_KERNEL, node)) { ret = -ENOMEM; goto err_free_pcpu; @@ -7511,6 +8068,7 @@ err_free_pcpu: free_cpumask_var(pcpu->cpus_to_kick); free_cpumask_var(pcpu->cpus_to_kick_if_idle); free_cpumask_var(pcpu->cpus_to_preempt); + free_cpumask_var(pcpu->cpus_to_preempt_lazy); free_cpumask_var(pcpu->cpus_to_wait); } for_each_possible_cpu(cpu) { @@ -7601,6 +8159,11 @@ int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) return -EINVAL; } + if ((ops->flags & SCX_OPS_ENQ_BLOCKED) && !ops->enqueue) { + scx_error(sch, "SCX_OPS_ENQ_BLOCKED requires ops.enqueue() to be implemented"); + return -EINVAL; + } + /* * SCX_OPS_TID_TO_TASK is enabled by the root scheduler. A sub-sched * may set it to declare a dependency; reject if the root hasn't @@ -7864,6 +8427,8 @@ static void scx_root_enable_workfn(struct kthread_work *work) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op); + scx_ops_cid_sched_updated_enable(sch); + if (sch->ops.cpu_acquire || sch->ops.cpu_release) sch->ops.flags |= SCX_OPS_HAS_CPU_PREEMPT; @@ -7990,6 +8555,10 @@ static void scx_root_enable_workfn(struct kthread_work *work) if (old_class != new_class) queue_flags |= DEQUEUE_CLASS; + if (new_class == &ext_sched_class) { + scx_prepare_task_sched_change(p); + queue_flags |= DEQUEUE_NOCLOCK; + } scoped_guard (sched_change, p, queue_flags) { scx_set_task_slice(p, READ_ONCE(sch->slice_dfl)); @@ -8510,6 +9079,8 @@ static void sched_ext_ops_cid__set_cmask(struct task_struct *p, const struct scx static void sched_ext_ops_cid__enable(struct task_struct *p, struct scx_enable_args *args) {} static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask__arena, u64 caps) {} static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {} +static void sched_ext_ops__sub_cid_sched_updated(s32 cid, u64 sched) {} +static void sched_ext_ops__sub_child_ecaps_updated(u64 cgroup_id, s32 cid, u64 before, u64 after) {} static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .select_cid = sched_ext_ops__select_cpu, @@ -8544,6 +9115,8 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .sub_detach = sched_ext_ops__sub_detach, .sub_caps_updated = sched_ext_ops__sub_caps_updated, .sub_ecaps_updated = sched_ext_ops__sub_ecaps_updated, + .sub_cid_sched_updated = sched_ext_ops__sub_cid_sched_updated, + .sub_child_ecaps_updated = sched_ext_ops__sub_child_ecaps_updated, .cid_online = sched_ext_ops__cpu_online, .cid_offline = sched_ext_ops__cpu_offline, .init_cids = sched_ext_ops__init_cids, @@ -8638,13 +9211,23 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r { struct rq *rq = cpu_rq(cpu); struct scx_rq *this_scx = &this_rq->scx; + struct rq_flags rf; const struct sched_class *cur_class; bool should_wait = false; bool kickable; - unsigned long flags; + bool preempt, preempt_lazy, wait, immediate; - raw_spin_rq_lock_irqsave(rq, flags); - cur_class = rq->curr->sched_class; + rq_lock_irqsave(rq, &rf); + cur_class = rq->donor->sched_class; + preempt = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt); + preempt_lazy = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt_lazy); + wait = cpumask_test_cpu(cpu, pcpu->cpus_to_wait); + /* + * Immediate preemption, waiting and a plain kick take precedence over + * lazy preemption. The lazy request still clears the slice, so all + * accumulated requests are served. + */ + immediate = preempt || wait || cpumask_test_cpu(cpu, pcpu->cpus_to_kick); /* * During CPU hotplug, a CPU may depend on kicking itself to make @@ -8658,19 +9241,23 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r !sched_class_above(cur_class, &ext_sched_class); if (kickable && !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) { - if (cpumask_test_cpu(cpu, pcpu->cpus_to_preempt)) { + if (preempt || preempt_lazy) { if (cur_class == &ext_sched_class) { u64 caps = scx_caps_for_preempt(pcpu->sch, rq, 0); - if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps))) + if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps))) { __scx_add_event(pcpu->sch, SCX_EV_SUB_PREEMPT_DENIED, 1); - else if (unlikely(!scx_set_task_slice(rq->curr, 0))) + /* degrade to a plain, immediate kick */ + immediate = true; + } else if (unlikely(!scx_set_task_slice(rq->donor, 0))) { __scx_add_event(pcpu->sch, SCX_EV_SLICE_DENIED, 1); + } } cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); + cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt_lazy); } - if (cpumask_test_cpu(cpu, pcpu->cpus_to_wait)) { + if (wait) { if (cur_class == &ext_sched_class) { cpumask_set_cpu(cpu, this_scx->cpus_to_sync); ksyncs[cpu] = rq->scx.kick_sync; @@ -8679,17 +9266,21 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r cpumask_clear_cpu(cpu, pcpu->cpus_to_wait); } - resched_curr(rq); + if (immediate) + resched_curr(rq); + else + scx_resched_curr_lazy(rq); } else { /* a kickable cpu was skipped solely for the missing caps */ if (kickable) __scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1); cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); + cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt_lazy); cpumask_clear_cpu(cpu, pcpu->cpus_to_wait); } scx_rq_lock_drop(rq); - raw_spin_rq_unlock_irqrestore(rq, flags); + rq_unlock_irqrestore(rq, &rf); return should_wait; } @@ -8739,7 +9330,7 @@ static void kick_cpus_irq_workfn(struct irq_work *irq_work) list_for_each_entry_safe(pcpu, tmp, &this_scx->sched_pcpus_to_kick, to_kick_node) { list_del_init(&pcpu->to_kick_node); - for_each_cpu(cpu, pcpu->cpus_to_kick) { + for_each_cpu_or(cpu, pcpu->cpus_to_kick, pcpu->cpus_to_preempt_lazy) { should_wait |= kick_one_cpu(cpu, pcpu, this_rq, ksyncs); cpumask_clear_cpu(cpu, pcpu->cpus_to_kick); cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle); @@ -8870,8 +9461,8 @@ void __init init_sched_ext_class(void) /* local_dsq's sch will be set during scx_root_enable() */ BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); -#ifdef CONFIG_EXT_SUB_SCHED BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL)); +#ifdef CONFIG_EXT_SUB_SCHED scx_rescue_init(rq); #endif @@ -9004,7 +9595,7 @@ __bpf_kfunc_start_defs(); * task is inserted. * * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id - * and this function can be called upto ops.dispatch_max_batch times to insert + * and this function can be called up to ops.dispatch_max_batch times to insert * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the * remaining slots. scx_bpf_dsq_move_to_local() flushes the batch and resets the * counter. @@ -9089,7 +9680,9 @@ struct scx_bpf_dsq_insert_vtime_args { * * @args->vtime ordering is according to time_before64() which considers * wrapping. A numerically larger vtime may indicate an earlier position in the - * ordering and vice-versa. + * ordering and vice-versa. vtime is a rolling cursor and values used for + * ordering within a given DSQ should stay less than 2^63 apart for + * time_before64() ordering to remain well-defined. * * A DSQ can only be used as a FIFO or priority queue at any given time and this * function must not be called on a DSQ which already has one or more FIFO tasks @@ -9167,6 +9760,40 @@ static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { .filter = scx_kfunc_context_filter, }; +/* + * Lock @rq from a kfunc called either from ops.dispatch(), which holds the + * dispatched rq's lock, or from a context holding no rq lock. Under core + * scheduling, dispatch can run for a sibling rq, so the locked rq is not + * necessarily this CPU's. Return the rq locked on entry, NULL if none. + */ +static struct rq *kf_switch_rq_lock(struct rq *rq, unsigned long *flags) +{ + struct rq *entry_rq; + + local_irq_save(*flags); + + entry_rq = scx_locked_rq(); + if (entry_rq) { + if (entry_rq != rq) + switch_rq_lock(entry_rq, rq); + } else { + raw_spin_rq_lock(rq); + } + return entry_rq; +} + +/* undo kf_switch_rq_lock(), @rq is the rq locked now */ +static void kf_restore_rq_lock(struct rq *rq, struct rq *entry_rq, unsigned long flags) +{ + if (entry_rq) { + if (entry_rq != rq) + switch_rq_lock(rq, entry_rq); + } else { + scx_rq_lock_drop(rq); + raw_spin_rq_unlock_irqrestore(rq, flags); + } +} + static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, struct task_struct *p, u64 dsq_id, u64 enq_flags, bool priq) @@ -9201,27 +9828,8 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, if (unlikely(READ_ONCE(sch->aborting))) return false; - /* - * Can be called from either ops.dispatch() holding the dispatched rq's - * lock or any context where no rq lock is held. If latter, lock @p's - * task_rq which we'll likely need anyway. - */ src_rq = task_rq(p); - - local_irq_save(flags); - - /* - * Under core scheduling, dispatch can run for a sibling rq, so the - * locked rq is not necessarily this CPU's. - */ - locked_rq = scx_locked_rq(); - - if (locked_rq) { - if (locked_rq != src_rq) - switch_rq_lock(locked_rq, src_rq); - } else { - raw_spin_rq_lock(src_rq); - } + locked_rq = kf_switch_rq_lock(src_rq, &flags); p_rq = src_rq; raw_spin_lock(&src_dsq->lock); @@ -9260,13 +9868,7 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, p_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); dispatched = true; out: - if (locked_rq) { - if (locked_rq != p_rq) - switch_rq_lock(p_rq, locked_rq); - } else { - scx_rq_lock_drop(p_rq); - raw_spin_rq_unlock_irqrestore(p_rq, flags); - } + kf_restore_rq_lock(p_rq, locked_rq, flags); kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE | __SCX_DSQ_ITER_HAS_VTIME); @@ -9500,6 +10102,8 @@ BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) +/* also in scx_kfunc_ids_unlocked, same contexts as scx_bpf_dsq_move() */ +BTF_ID_FLAGS(func, scx_bpf_dsq_reenq_task, KF_RCU) #ifdef CONFIG_EXT_SUB_SCHED BTF_ID_FLAGS(func, scx_bpf_sub_dispatch, KF_IMPLICIT_ARGS) #endif @@ -9615,6 +10219,7 @@ BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_reenq_task, KF_RCU) /* also in scx_kfunc_ids_select_cpu: also callable from ops.select_cpu()/ops.enqueue() */ BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) @@ -9659,8 +10264,10 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, return false; /* - * Directly write only when we hold the lock of the rq @p is queued or - * running on. See the write rules above. + * Directly write only when we hold the lock of the rq @p is queued on or + * provides the current scheduling context for. Under proxy execution, + * rq->donor owns and consumes the slice while rq->curr executes on its + * behalf. See the slice write rules above. * * While @p is queued on a user DSQ or in the BPF scheduler, * synchronization is the scheduler's responsibility. This write can @@ -9674,7 +10281,7 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, locked_rq = scx_locked_rq(); if (!locked_rq || (READ_ONCE(p->scx.runnable_cpu) != cpu_of(locked_rq) && - !task_current(locked_rq, p))) { + !task_current_donor(locked_rq, p))) { set_task_slice_oob(sch, p, slice); return true; } @@ -9715,18 +10322,56 @@ __bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime, return true; } +/** + * scx_bpf_task_set_lazy_resched - Set task's tick reschedule mode + * @p: task of interest + * @lazy: whether slice expiry should request lazy rescheduling + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Choose whether depletion of @p's slice at the scheduler tick requests lazy or + * immediate rescheduling. This is a persistent per-task policy and does not + * itself clear @p's slice or request rescheduling, unlike the + * %SCX_ENQ_PREEMPT_LAZY and %SCX_KICK_PREEMPT_LAZY one-shot operations. + * + * With lazy rescheduling enabled, a task interrupted in user space still + * reschedules before returning to user space. A task interrupted in the kernel + * may continue until its next return to user space or the next scheduler tick, + * which promotes the request. If lazy preemption is disabled at runtime, the + * request behaves like an immediate reschedule. + * + * @p must be on the calling scheduler. + * + * Return %true on success, %false if @p is not on the calling scheduler. + */ +__bpf_kfunc bool scx_bpf_task_set_lazy_resched(struct task_struct *p, bool lazy, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + /* + * clang's register locations for @lazy make pahole 1.32 drop this kfunc + * from BTF. Keep @lazy in memory so its location names no register. + */ + barrier_data(&lazy); + guard(rcu)(); + sch = scx_prog_sched(aux); + if (unlikely(!sch || !scx_task_on_sched(sch, p))) + return false; + + WRITE_ONCE(p->scx.lazy_resched, lazy); + return true; +} + void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) { struct scx_sched_pcpu *pcpu; struct rq *this_rq; unsigned long irq_flags; - /* - * The per-cpu kick list is guarded only by local_irq_save(), which does - * not mask NMIs, so kicking from NMI could corrupt it and is unsupported. - */ - if (unlikely(in_nmi())) { - scx_error(sch, "scx_bpf_kick_cpu() called from NMI"); + if (!scx_kf_allowed_ctx(sch)) + return; + if (unlikely(flags & ~SCX_KICK_ALL_FLAGS)) { + scx_error(sch, "invalid kick flags 0x%llx", flags); return; } @@ -9754,7 +10399,8 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) if (flags & SCX_KICK_IDLE) { struct rq *target_rq = cpu_rq(cpu); - if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_WAIT))) + if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_PREEMPT_LAZY | + SCX_KICK_WAIT))) scx_error(sch, "PREEMPT/WAIT cannot be used with SCX_KICK_IDLE"); if (raw_spin_rq_trylock(target_rq)) { @@ -9768,12 +10414,17 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) } cpumask_set_cpu(cpu, pcpu->cpus_to_kick_if_idle); } else { - cpumask_set_cpu(cpu, pcpu->cpus_to_kick); - + /* + * Accumulate requests and resolve their precedence at delivery. + */ if (flags & SCX_KICK_PREEMPT) cpumask_set_cpu(cpu, pcpu->cpus_to_preempt); + if (flags & SCX_KICK_PREEMPT_LAZY) + cpumask_set_cpu(cpu, pcpu->cpus_to_preempt_lazy); if (flags & SCX_KICK_WAIT) cpumask_set_cpu(cpu, pcpu->cpus_to_wait); + if (!(flags & SCX_KICK_PREEMPT_LAZY)) + cpumask_set_cpu(cpu, pcpu->cpus_to_kick); } if (list_empty(&pcpu->to_kick_node)) @@ -9813,8 +10464,9 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux * cid-addressed equivalent of scx_bpf_kick_cpu(). An invalid @cid aborts the * scheduler via scx_cid_to_cpu(). Caps are enforced on the delivery path: a * kick is dropped if the caller lacks baseline access on @cid, and a - * %SCX_KICK_PREEMPT degrades to a plain reschedule if the caller lacks - * %SCX_CAP_PREEMPT for a task outside its subtree. + * %SCX_KICK_PREEMPT or %SCX_KICK_PREEMPT_LAZY request degrades to a plain + * reschedule if the caller lacks %SCX_CAP_PREEMPT for a task outside its + * subtree. */ __bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) { @@ -9896,8 +10548,13 @@ __bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux) guard(rcu)(); sch = scx_prog_sched(aux); - if (sch) - destroy_dsq(sch, dsq_id); + if (unlikely(!sch)) + return; + + if (!scx_kf_allowed_ctx(sch)) + return; + + destroy_dsq(sch, dsq_id); } /** @@ -9934,6 +10591,9 @@ __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, if (unlikely(!sch)) return -ENODEV; + if (!scx_kf_allowed_ctx(sch)) + return -EDEADLK; + if (flags & ~__SCX_DSQ_ITER_USER_FLAGS) return -EINVAL; @@ -9968,7 +10628,8 @@ __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *i * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator * @it: iterator to destroy * - * Undo bpf_iter_scx_dsq_new(). + * Undo bpf_iter_scx_dsq_new(). If the iteration marked tasks with + * scx_bpf_dsq_reenq_task(), schedule their re-enqueue. */ __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) { @@ -9984,6 +10645,13 @@ __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) list_del_init(&kit->cursor.node); raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); } + + if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_REENQ) { + guard(preempt)(); + schedule_dsq_reenq(kit->dsq->sched, kit->dsq, SCX_REENQ_MARKED, + scx_locked_rq()); + } + kit->dsq = NULL; } @@ -10036,7 +10704,8 @@ __bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id, * - Local DSQs (%SCX_DSQ_LOCAL or %SCX_DSQ_LOCAL_ON | $cpu) * - User DSQs * - * Re-enqueues are performed asynchronously. Can be called from anywhere. + * Re-enqueues are performed asynchronously. Can be called from anywhere. A + * re-enqueue of an inactive cpu's local DSQ is ignored. * * %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler * operation is locked to - e.g. the rq being dispatched for in ops.dispatch() - @@ -10060,6 +10729,9 @@ __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, return; } + if (!scx_kf_allowed_ctx(sch)) + return; + /* not specifying any filter bits is the same as %SCX_REENQ_ANY */ if (!(reenq_flags & __SCX_REENQ_FILTER_MASK)) reenq_flags |= SCX_REENQ_ANY; @@ -10069,6 +10741,64 @@ __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, } /** + * scx_bpf_dsq_reenq_task - Re-enqueue a task from a DSQ iteration + * @it__iter: DSQ iterator in progress + * @p: task to re-enqueue + * + * Mark @p, which is on the DSQ iterated by @it__iter and was queued before the + * iteration started, for re-enqueue. When the iteration ends, the DSQ's marked + * tasks go through ops.enqueue() with %SCX_ENQ_REENQ in DSQ order and the other + * tasks on the DSQ keep their positions. A marked task that leaves the DSQ in + * the meantime is not re-enqueued. The mark belongs to @p, so a re-enqueue + * scheduled by another iteration of the DSQ may re-enqueue @p before this + * iteration ends. @p must be on the DSQ's scheduler. + * + * Re-enqueues are performed asynchronously. Can be called from ops.dispatch() + * or any BPF context which doesn't hold a rq lock (e.g. BPF timers or SYSCALL + * programs). + * + * Return %true if @p was marked, %false if @p left the DSQ or was queued after + * the iteration started, @p is on another scheduler, or the iterator holds no + * DSQ. + */ +__bpf_kfunc bool scx_bpf_dsq_reenq_task(struct bpf_iter_scx_dsq *it__iter, + struct task_struct *p) +{ + struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; + struct scx_dispatch_q *dsq = kit->dsq; + struct rq *src_rq, *entry_rq; + unsigned long flags; + bool marked = false; + + /* NULL after a failed bpf_iter_scx_dsq_new(), see scx_dsq_move() */ + if (unlikely(!dsq)) + return false; + + /* the cursor check needs @p's rq locked */ + src_rq = task_rq(p); + entry_rq = kf_switch_rq_lock(src_rq, &flags); + + scoped_guard (raw_spinlock, &dsq->lock) { + if (nldsq_cursor_lost_task(&kit->cursor, src_rq, dsq, p)) + break; + + if (unlikely(!scx_task_on_sched(dsq->sched, p))) { + scx_error(dsq->sched, "scx_bpf_dsq_reenq_task() on %s[%d] but the task belongs to a different scheduler", + p->comm, p->pid); + break; + } + + p->scx.dsq_flags |= SCX_TASK_DSQ_REENQ; + /* the walk is scheduled once, by bpf_iter_scx_dsq_destroy() */ + kit->cursor.flags |= __SCX_DSQ_ITER_HAS_REENQ; + marked = true; + } + + kf_restore_rq_lock(src_rq, entry_rq, flags); + return marked; +} + +/** * scx_bpf_reenqueue_local___v2 - Re-enqueue tasks on a local DSQ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * @@ -10447,6 +11177,9 @@ __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_au if (unlikely(!sch)) return; + if (!scx_kf_allowed_ctx(sch)) + return; + scx_cpuperf_set(sch, cpu, perf); } @@ -10472,6 +11205,10 @@ __bpf_kfunc s32 scx_bpf_cidperf_set(s32 cid, u32 perf, sch = scx_prog_sched(aux); if (unlikely(!sch)) return -ENODEV; + + if (!scx_kf_allowed_ctx(sch)) + return -EDEADLK; + cpu = scx_cid_to_cpu(sch, cid); if (cpu < 0) return cpu; @@ -10604,12 +11341,17 @@ __bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask) } /** - * scx_bpf_task_running - Is task currently running? + * scx_bpf_task_running - Is task the current scheduling context? * @p: task of interest + * + * Under proxy execution, this reports the donor rather than the task whose + * code is physically executing. The physical execution context is intentionally + * not exposed to the BPF scheduler, which continues to observe the donor as the + * running scheduling context. */ __bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p) { - return task_rq(p)->curr == p; + return rcu_access_pointer(task_rq(p)->donor) == p; } /** @@ -10670,10 +11412,15 @@ __bpf_kfunc struct rq *scx_bpf_locked_rq(const struct bpf_prog_aux *aux) } /** - * scx_bpf_cpu_curr - Return remote CPU's curr task + * scx_bpf_cpu_curr - Return remote CPU's current scheduling context * @cpu: CPU of interest * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * + * Under proxy execution, this returns the donor, which supplies the scheduling + * policy and runtime budget, rather than the task whose code is physically + * executing. The physical execution context is intentionally not exposed to + * the BPF scheduler. + * * Callers must hold RCU read lock (KF_RCU). */ __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_aux *aux) @@ -10689,7 +11436,7 @@ __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_ if (!scx_cpu_valid(sch, cpu, NULL)) return NULL; - return rcu_dereference(cpu_rq(cpu)->curr); + return rcu_dereference(cpu_rq(cpu)->donor); } /** @@ -10713,7 +11460,7 @@ __bpf_kfunc struct task_struct *scx_bpf_cid_curr(s32 cid, const struct bpf_prog_ cpu = scx_cid_to_cpu(sch, cid); if (cpu < 0) return NULL; - return rcu_dereference(cpu_rq(cpu)->curr); + return rcu_dereference(cpu_rq(cpu)->donor); } /** @@ -10887,6 +11634,23 @@ out: cgroup_get(cgrp); return cgrp; } + +/** + * scx_bpf_cgroup_nr_cpus - Return the number of CPUs in a cgroup's cpuset + * @cgrp: cgroup of interest + * + * Return the number of CPUs in @cgrp's effective cpuset, which is inherited + * from the nearest ancestor with the cpuset controller enabled. This matches + * the count used by fair's group share calculation and lets hierarchical BPF + * schedulers bound a group's weight by the CPUs it can actually run on. + * + * The value is a snapshot and may change at any time through cpuset updates or + * CPU hotplug. Schedulers should re-read it when recomputing group state. + */ +__bpf_kfunc u32 scx_bpf_cgroup_nr_cpus(struct cgroup *cgrp) +{ + return cpuset_num_cpus(cgrp); +} #endif /* CONFIG_CGROUP_SCHED */ __bpf_kfunc_end_defs(); @@ -10894,6 +11658,7 @@ __bpf_kfunc_end_defs(); BTF_KFUNCS_START(scx_kfunc_ids_any) BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS | KF_RCU); BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS | KF_RCU); +BTF_ID_FLAGS(func, scx_bpf_task_set_lazy_resched, KF_IMPLICIT_ARGS | KF_RCU); BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_kick_cid, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS) @@ -10933,6 +11698,7 @@ BTF_ID_FLAGS(func, scx_bpf_now) BTF_ID_FLAGS(func, scx_bpf_events, KF_IMPLICIT_ARGS) #ifdef CONFIG_CGROUP_SCHED BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS | KF_RCU | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_cgroup_nr_cpus, KF_RCU) #endif BTF_ID_FLAGS(func, scx_bpf_sub_grant, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_sub_revoke, KF_IMPLICIT_ARGS) @@ -11026,6 +11792,7 @@ static const u32 scx_kf_allow_flags[] = { [SCX_OP_IDX(sub_attach)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(sub_detach)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(sub_ecaps_updated)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH, + [SCX_OP_IDX(sub_child_ecaps_updated)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH, [SCX_OP_IDX(cpu_online)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(cpu_offline)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(init_cids)] = SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT_CIDS, @@ -11169,6 +11936,8 @@ static int __init scx_init(void) CID_OFFSET_MATCH(sub_detach, sub_detach); CID_OFFSET_MATCH(sub_caps_updated, sub_caps_updated); CID_OFFSET_MATCH(sub_ecaps_updated, sub_ecaps_updated); + CID_OFFSET_MATCH(sub_cid_sched_updated, sub_cid_sched_updated); + CID_OFFSET_MATCH(sub_child_ecaps_updated, sub_child_ecaps_updated); CID_OFFSET_MATCH(init_cids, init_cids); CID_OFFSET_MATCH(init, init); CID_OFFSET_MATCH(exit, exit); diff --git a/kernel/sched/ext/ext.h b/kernel/sched/ext/ext.h index 3cfbfeb1bf9d..d32bc01071b0 100644 --- a/kernel/sched/ext/ext.h +++ b/kernel/sched/ext/ext.h @@ -18,12 +18,28 @@ bool scx_can_stop_tick(struct rq *rq); void scx_rq_activate(struct rq *rq); void scx_rq_deactivate(struct rq *rq); int scx_check_setscheduler(struct task_struct *p, int policy); +void scx_prepare_setscheduler(struct task_struct *p, int policy); bool task_should_scx(int policy); bool scx_allow_ttwu_queue(const struct task_struct *p); -bool scx_allow_proxy_exec(const struct task_struct *p); +bool __scx_allow_proxy_exec(const struct task_struct *p); void scx_proxy_donor_start(struct rq *rq); void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next); void init_sched_ext_class(void); +void __scx_update_idle(struct rq *rq, bool idle, bool do_notify); + +static inline bool scx_allow_proxy_exec(const struct task_struct *p) +{ + if (scx_enabled()) + return __scx_allow_proxy_exec(p); + + return true; +} + +static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) +{ + if (scx_enabled()) + __scx_update_idle(rq, idle, do_notify); +} static inline u32 scx_cpuperf_target(s32 cpu) { @@ -55,6 +71,7 @@ static inline bool scx_can_stop_tick(struct rq *rq) { return true; } static inline void scx_rq_activate(struct rq *rq) {} static inline void scx_rq_deactivate(struct rq *rq) {} static inline int scx_check_setscheduler(struct task_struct *p, int policy) { return 0; } +static inline void scx_prepare_setscheduler(struct task_struct *p, int policy) {} static inline bool task_on_scx(const struct task_struct *p) { return false; } static inline bool scx_allow_ttwu_queue(const struct task_struct *p) { return true; } static inline bool scx_allow_proxy_exec(const struct task_struct *p) { return true; } @@ -62,21 +79,10 @@ static inline void scx_proxy_donor_start(struct rq *rq) {} static inline void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next) {} static inline void init_sched_ext_class(void) {} +static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) {} #endif /* CONFIG_SCHED_CLASS_EXT */ -#ifdef CONFIG_SCHED_CLASS_EXT -void __scx_update_idle(struct rq *rq, bool idle, bool do_notify); - -static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) -{ - if (scx_enabled()) - __scx_update_idle(rq, idle, do_notify); -} -#else -static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) {} -#endif - #ifdef CONFIG_CGROUP_SCHED #ifdef CONFIG_EXT_GROUP_SCHED void scx_tg_init(struct task_group *tg); diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index 28ed9e327085..922fbc3065cc 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -153,7 +153,18 @@ static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, i nodemask_t *unvisited; s32 cpu = -EBUSY; - preempt_disable(); + /* + * @per_cpu_unvisited is per-CPU scratch and the idle kfuncs can be + * called from IRQ-enabled contexts, so mask IRQs to keep a nested + * invocation from clobbering the mask an outer invocation is still + * iterating. + * + * NMI nesting is not handled: there is no legitimate reason to call + * pick_idle from NMI and doing so poses no crash risk, so such a + * caller is on its own. + */ + guard(irqsave)(); + unvisited = this_cpu_ptr(&per_cpu_unvisited); /* @@ -183,7 +194,6 @@ static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, i if (cpu >= 0) break; } - preempt_enable(); return cpu; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 352547aedbb7..29e69a83d876 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -215,6 +215,34 @@ enum scx_ops_flags { */ SCX_OPS_TID_TO_TASK = 1LLU << 8, + /* + * If set, tasks default to requesting lazy rescheduling when their + * slice runs out at the tick, the way fair.c expires a slice from + * update_curr(). The default is copied to p->scx.lazy_resched + * immediately before ops.enable(), after which + * scx_bpf_task_set_lazy_resched() may override it per task. + * + * A task in user space still reschedules on the way back from the tick; + * a task in the kernel runs on to its next return to user space or to + * the next tick, which promotes the request. When lazy preemption is + * disabled at runtime, the request behaves like an immediate + * reschedule. Rescheduling while bypassing stays immediate. + */ + SCX_OPS_LAZY_RESCHED = 1LLU << 9, + + /* + * If set, mutex-blocked tasks remain runnable as proxy donors and are + * passed to ops.enqueue() with %SCX_ENQ_BLOCKED. The BPF scheduler controls + * when donors are dispatched and whether they should preempt other work. + * + * If clear, mutex-blocked tasks are removed from the runqueue normally + * and cannot donate their scheduling context through proxy execution. + * + * For blocked donors, this flag takes precedence over + * %SCX_OPS_ENQ_EXITING and %SCX_OPS_ENQ_MIGRATION_DISABLED. + */ + SCX_OPS_ENQ_BLOCKED = 1LLU << 10, + SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | SCX_OPS_ENQ_LAST | SCX_OPS_ENQ_EXITING | @@ -223,7 +251,9 @@ enum scx_ops_flags { SCX_OPS_SWITCH_PARTIAL | SCX_OPS_BUILTIN_IDLE_PER_NODE | SCX_OPS_ALWAYS_ENQ_IMMED | - SCX_OPS_TID_TO_TASK, + SCX_OPS_TID_TO_TASK | + SCX_OPS_LAZY_RESCHED | + SCX_OPS_ENQ_BLOCKED, /* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */ __SCX_OPS_INTERNAL_MASK = 0xffLLU << 56, @@ -336,6 +366,18 @@ struct scx_sub_detach_args { char *cgroup_path; }; +/* + * What a sched sees running on a cid, as reported by + * ops.sub_cid_sched_updated(): NONE when the cid is idle or runs a task outside + * the sched's subtree, SELF for the sched itself, or a direct child by its + * cgroup id. The sentinels lie outside the cgroup id space, whose low 32 bits + * are an idr value of at most INT_MAX. + */ +enum scx_cid_sched_consts { + SCX_CID_SCHED_NONE = (u64)-2, + SCX_CID_SCHED_SELF = (u64)-1, +}; + /** * struct sched_ext_ops - Operation table for BPF scheduler implementation * @@ -477,6 +519,12 @@ struct sched_ext_ops { * Therefore, always use scx_bpf_task_cpu(@p) to determine the * target CPU the task is going to use. * + * Under proxy execution, the BPF scheduler continues to observe the + * donor as the current scheduling context. A blocked donor enters a + * ->running()/->stopping() session while its scheduling context drives + * the lock owner. The lock owner executing on its behalf is intentionally + * not reported through these callbacks. + * * See ->runnable() for explanation on the task state notifiers. */ void (*running)(struct task_struct *p); @@ -828,6 +876,11 @@ struct sched_ext_ops { /** * @sub_detach: Detach a sub-scheduler * @args: argument container, see the struct definition + * + * The sub-scheduler holds no caps by this point and can no longer + * affect any cid. Its caps went away without a + * sub_child_ecaps_updated() report, so whatever was delegated to it, + * e.g. cpuperf targets, is the parent's to restore here. */ void (*sub_detach)(struct scx_sub_detach_args *args); @@ -864,6 +917,41 @@ struct sched_ext_ops { */ void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); + /** + * @sub_cid_sched_updated: The sched running on a cid changed + * @cid: cid whose running sched changed + * @sched: what this sched now sees running there + * + * Invoked when the sched owning the task running on @cid changes. + * @sched is %SCX_CID_SCHED_NONE when no task of this sched's subtree + * runs there, %SCX_CID_SCHED_SELF for its own task, or the cgroup id of + * the direct child whose subtree the task belongs to. Changes inside a + * child's subtree, whether across tasks or nested sub-scheds, are not + * reported. Every sched for which @sched changes is notified. + * + * Runs with the rq lock held on the context switch path. + */ + void (*sub_cid_sched_updated)(s32 cid, u64 sched); + + /** + * @sub_child_ecaps_updated: A child's effective caps on a cid changed + * @cgroup_id: cgroup id of the direct child + * @cid: the cid whose effective caps changed + * @before: the child's effective caps as of the last delivery + * @after: the child's effective caps now + * + * Invoked right after the child's sub_ecaps_updated(), once a grant or + * revoke is in effect on the cpu. From here on the parent can act on a + * cap the child lost, e.g. reset the cpuperf target or schedule on the + * cid. A detaching child's caps go away without a report, see + * sub_detach(). A cpu going offline drops caps without a report. + * + * Runs in dispatch context with rq lock held, and can perform all + * operations allowed in ops.dispatch() including inserting/moving + * tasks. + */ + void (*sub_child_ecaps_updated)(u64 cgroup_id, s32 cid, u64 before, u64 after); + /* * All online ops must come before ops.cpu_online(). */ @@ -911,6 +999,9 @@ struct sched_ext_ops { * ops.exit() is also called on ops.init() failure, which is a bit * unusual. This is to allow rich reporting through @info on how * ops.init() failed. + * + * A sub-scheduler holds no caps by the time ops.exit() runs. Its + * cap-gated kfuncs are denied. */ void (*exit)(struct scx_exit_info *info); @@ -1118,6 +1209,8 @@ struct sched_ext_ops_cid { void (*sub_detach)(struct scx_sub_detach_args *args); void (*sub_caps_updated)(const struct scx_cmask *cmask__arena, u64 caps); void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); + void (*sub_cid_sched_updated)(s32 cid, u64 sched); + void (*sub_child_ecaps_updated)(u64 cgroup_id, s32 cid, u64 before, u64 after); void (*cid_online)(s32 cid); void (*cid_offline)(s32 cid); s32 (*init_cids)(void); @@ -1359,6 +1452,7 @@ struct scx_sched_pcpu { cpumask_var_t cpus_to_kick; cpumask_var_t cpus_to_kick_if_idle; cpumask_var_t cpus_to_preempt; + cpumask_var_t cpus_to_preempt_lazy; cpumask_var_t cpus_to_wait; struct list_head to_kick_node; @@ -1380,7 +1474,10 @@ struct scx_sched_pcpu { struct llist_node ecaps_to_sync_node; /* owed a forced update_idle() re-notify on this cpu */ bool idle_renotify; - /* effective caps as of the last sub_ecaps_updated() delivery */ + /* + * ecaps as of the last sub_ecaps_updated() and + * sub_child_ecaps_updated() delivery. See __scx_process_sync_ecaps(). + */ u64 reported_ecaps; /* @@ -1439,15 +1536,16 @@ struct scx_sched_pnode { * the allocation pattern. * * ENQ_IMMED insert an IMMED task onto the cid's local DSQ - * - kick the cid's cpu (except SCX_KICK_PREEMPT) + * - kick the cid's cpu (except SCX_KICK_PREEMPT and + * SCX_KICK_PREEMPT_LAZY) * * ENQ insert any task onto the cid's local DSQ (implies ENQ_IMMED) * * PREEMPT preempt any task running on the cid regardless of the owning * sched (implies ENQ). Preempting a task in the sched's own subtree * doesn't require any cap. - * - SCX_ENQ_PREEMPT inserts - * - SCX_KICK_PREEMPT kicks + * - SCX_ENQ_PREEMPT and SCX_ENQ_PREEMPT_LAZY insert + * - SCX_KICK_PREEMPT and SCX_KICK_PREEMPT_LAZY kick * * PERF control the cid's cpu power/perf management state, currently the * cpufreq target set through scx_bpf_cidperf_set(). Hardware @@ -1733,10 +1831,19 @@ enum scx_enq_flags { SCX_ENQ_PREEMPT = 1LLU << 32, /* + * Like %SCX_ENQ_PREEMPT, but request lazy rescheduling. The current + * task's slice is still cleared immediately so that the next scheduling + * boundary observes the new ordering. %SCX_ENQ_PREEMPT takes precedence + * if both are specified. Implies %SCX_ENQ_HEAD, which is all it means + * on a non-local DSQ, as with %SCX_ENQ_PREEMPT. + */ + SCX_ENQ_PREEMPT_LAZY = 1LLU << 35, + + /* * Only allowed on local DSQs. Guarantees that the task either gets * on the CPU immediately and stays on it, or gets reenqueued back - * to the BPF scheduler. It will never linger on a local DSQ or be - * silently put back after preemption. + * to the BPF scheduler. A blocked proxy donor may be put back on the + * local DSQ temporarily while proxy execution resolves its mutex owner. * * The protection persists until the next fresh enqueue - it * survives SAVE/RESTORE cycles, slice extensions and preemption. @@ -1780,6 +1887,18 @@ enum scx_enq_flags { */ SCX_ENQ_LAST = 1LLU << 41, + /* + * The task is blocked on a mutex and is being kept runnable as a proxy + * donor. Only passed to ops.enqueue() when %SCX_OPS_ENQ_BLOCKED is set. + * + * Blocking on the mutex does not enqueue the task by itself. A donor put + * with slice left stays at the head of the local DSQ, except when an + * IMMED donor is preempted or cannot stay local. It is passed to + * ops.enqueue() when its slice runs out, the IMMED placement cannot be + * kept, or proxy execution moves it to the owner's CPU. + */ + SCX_ENQ_BLOCKED = 1LLU << 42, + /* high 8 bits are internal */ __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, @@ -1792,6 +1911,15 @@ enum scx_enq_flags { SCX_ENQ_SLICE_DFL = 1LLU << 62, /* carried slice is a default refill */ }; +/* Strip priority and carried slice state when diverting from a local DSQ. */ +static inline void scx_divert_strip_flags(struct task_struct *p, u64 *enq_flags) +{ + *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | + SCX_ENQ_PREEMPT_LAZY | SCX_ENQ_HEAD | + SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL); + p->scx.flags &= ~SCX_TASK_IMMED; +} + enum scx_deq_flags { /* expose select DEQUEUE_* flags as enums */ SCX_DEQ_SLEEP = DEQUEUE_SLEEP, @@ -1818,6 +1946,8 @@ enum scx_reenq_flags { /* internal: kernel-issued on cap revoke, not accepted from BPF */ SCX_REENQ_CAP_REVOKE = 1LLU << 1, + /* internal: tasks marked by scx_bpf_dsq_reenq_task() */ + SCX_REENQ_MARKED = 1LLU << 2, __SCX_REENQ_FILTER_MASK = 0xffffLLU, @@ -1859,6 +1989,15 @@ enum scx_kick_flags { * is not on SCX. */ SCX_KICK_WAIT = 1LLU << 2, + + /* + * Like %SCX_KICK_PREEMPT, but request lazy rescheduling. If combined + * with %SCX_KICK_PREEMPT or %SCX_KICK_WAIT, rescheduling is immediate. + */ + SCX_KICK_PREEMPT_LAZY = 1LLU << 3, + + SCX_KICK_ALL_FLAGS = SCX_KICK_IDLE | SCX_KICK_PREEMPT | + SCX_KICK_WAIT | SCX_KICK_PREEMPT_LAZY, }; enum scx_tg_flags { @@ -2107,6 +2246,7 @@ struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter); bool scx_set_task_slice(struct task_struct *p, u64 slice); void scx_task_slice_ended(struct rq *rq, struct task_struct *p); void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq); +void scx_prepare_task_sched_change(struct task_struct *p); void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p); void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, int sticky_cpu); @@ -2168,6 +2308,22 @@ extern struct scx_sched *scx_enabling_sub_sched; #define scx_error(sch, fmt, args...) \ scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) +/* + * Tracing progs can call kfuncs from NMI. Kfuncs that take scheduler locks or + * touch the kick lists, which are only protected by irq masking, can't run + * there, so abort the scheduler instead. scx_error() is NMI-safe. + */ +static __always_inline bool __scx_kf_allowed_ctx(struct scx_sched *sch, const char *who) +{ + if (unlikely(in_nmi())) { + scx_error(sch, "%s called from NMI", who); + return false; + } + return true; +} + +#define scx_kf_allowed_ctx(sch) __scx_kf_allowed_ctx((sch), __func__) + /** * scx_root_protected_live - Root sched for paths that only run while live * @@ -2258,6 +2414,15 @@ static inline void scx_schedule_reenq_local(struct rq *rq, u64 reenq_flags) */ static inline struct rq *scx_locked_rq(void) { + /* + * Tracing progs can call kfuncs from NMI. scx_locked_rq_state tracks + * the rq locked by the interrupted context, so a non-NULL read from + * NMI would falsely claim its lock. Return NULL from NMI so that + * callers take their unlocked paths. + */ + if (unlikely(in_nmi())) + return NULL; + return __this_cpu_read(scx_locked_rq_state); } @@ -2539,4 +2704,21 @@ static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; #endif /* CONFIG_EXT_SUB_SCHED */ +/** + * scx_exec_sched - Find the sched running @sch's tasks on @cpu + * @sch: sched owning the tasks + * @cpu: cpu of interest + * + * A bypassing sched's tasks are run by its nearest ancestor that isn't + * bypassing. Return @sch if it isn't bypassing on @cpu, otherwise that + * ancestor, or the root if every ancestor is bypassing. Hold @cpu's rq lock for + * a stable answer. + */ +static inline struct scx_sched *scx_exec_sched(struct scx_sched *sch, s32 cpu) +{ + while (scx_parent(sch) && scx_bypassing(sch, cpu)) + sch = scx_parent(sch); + return sch; +} + #endif /* _KERNEL_SCHED_EXT_INTERNAL_H */ diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 212de1cb8502..eb1270f7723c 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -28,6 +28,13 @@ */ DEFINE_STATIC_KEY_FALSE(__scx_has_subs); +/* + * On while a loaded sched implements ops.sub_cid_sched_updated(), see + * scx_ops_cid_sched_updated_enable(). + */ +static DEFINE_STATIC_KEY_FALSE(__scx_ops_cid_sched_updated_enabled); +static s32 scx_nr_ops_cid_sched_updated; /* such scheds, under scx_enable_mutex */ + /* latched at root enable before any rescue runs */ static s32 scx_rescue_bw_1024; static s64 scx_rescue_quantum_ns; @@ -92,6 +99,161 @@ struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sche return scx_skip_subtree_pre(pos, root); } +/** + * scx_cid_sched_id_from - Resolve what @from sees when @sch runs on a cid + * @sch: sched whose task runs there, %NULL for none + * @from: sched asking + * + * Return %SCX_CID_SCHED_NONE when @sch is %NULL or outside @from's subtree, + * %SCX_CID_SCHED_SELF for @from itself, or else the cgroup id of the direct + * child of @from on the way down to @sch. + */ +static u64 scx_cid_sched_id_from(struct scx_sched *sch, struct scx_sched *from) +{ + if (!sch || !scx_is_descendant(sch, from)) + return SCX_CID_SCHED_NONE; + if (sch == from) + return SCX_CID_SCHED_SELF; + return sch->ancestors[from->level + 1]->ops.sub_cgroup_id; +} + +/* notify @from of what it now sees running on @rq's cid */ +static void scx_cid_notify_sched_updated(struct rq *rq, struct scx_sched *sch, + struct scx_sched *from) +{ + if (SCX_HAS_OP(from, sub_cid_sched_updated)) + SCX_CALL_OP(from, sub_cid_sched_updated, rq, __scx_cpu_to_cid(cpu_of(rq)), + scx_cid_sched_id_from(sch, from)); +} + +/** + * scx_cid_sched_update - Update rq->scx.sched and notify the scheds affected + * @rq: rq whose running session opens or closes + * @sch: sched starting a session, %NULL to close the open one + * + * Every sched whose scx_cid_sched_id_from() value changes is notified of the + * new one. Above the deepest common ancestor of the two scheds, both map to the + * same child, so the walk stops there. With root R, children A and B, and A's + * child A1, a switch from an A1 task to a B task reports: + * + * R R: A -> B + * / \ + * A B A: A1 -> NONE B: NONE -> SELF + * | + * A1 A1: SELF -> NONE + * + * When the task on one side is not an ext task, there is no common ancestor and + * only the other side is walked. + */ +void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch) +{ + struct scx_sched *prev = rq->scx.sched; + s32 level, common = -1; + + lockdep_assert_rq_held(rq); + + if (!static_branch_unlikely(&__scx_ops_cid_sched_updated_enabled) || prev == sch) + return; + + rq->scx.sched = sch; + + if (prev && sch) { + for (level = min(prev->level, sch->level); level >= 0; level--) + if (prev->ancestors[level] == sch->ancestors[level]) + break; + common = level; + scx_cid_notify_sched_updated(rq, sch, prev->ancestors[common]); + } + if (prev) + for (level = common + 1; level <= prev->level; level++) + scx_cid_notify_sched_updated(rq, sch, prev->ancestors[level]); + if (sch) + for (level = common + 1; level <= sch->level; level++) + scx_cid_notify_sched_updated(rq, sch, sch->ancestors[level]); +} + +/** + * scx_ops_cid_sched_updated_enable - Maintain rq->scx.sched if @sch has the op + * @sch: sched being enabled, its has_op filled in + * + * rq->scx.sched is maintained only while a loaded sched implements + * ops.sub_cid_sched_updated(), so that everyone else pays one static branch per + * session open or close. The first such sched turns the key on and then + * initializes every rq from the task running there, in that order so that no + * session change in between goes unrecorded. While the key is off every + * rq->scx.sched is NULL, see scx_ops_cid_sched_updated_disable(). + */ +void scx_ops_cid_sched_updated_enable(struct scx_sched *sch) +{ + s32 cpu; + + lockdep_assert_held(&scx_enable_mutex); + + if (!SCX_HAS_OP(sch, sub_cid_sched_updated) || scx_nr_ops_cid_sched_updated++) + return; + + static_branch_enable(&__scx_ops_cid_sched_updated_enabled); + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + struct task_struct *p; + + guard(rq_lock_irqsave)(rq); + p = rq->donor; + if (p->sched_class == &ext_sched_class && + (p->scx.flags & SCX_TASK_RUN_TRACKED)) + rq->scx.sched = scx_task_sched(p); + } +} + +/** + * scx_ops_cid_sched_updated_disable - Stop maintaining rq->scx.sched for @sch + * @sch: sched being disabled, its tasks all gone + * + * Close the session on every rq that still points at @sch, before ops.exit(), + * so that no sched is notified of @sch once it is gone. The last sched with the + * op turns the key off first and clears every rq, so while the key is off every + * rq->scx.sched is NULL. A sched whose enable failed before its has_op fill was + * never counted, so has_op gates the decrement. + */ +void scx_ops_cid_sched_updated_disable(struct scx_sched *sch) +{ + bool last; + s32 cpu; + + lockdep_assert_held(&scx_enable_mutex); + + last = SCX_HAS_OP(sch, sub_cid_sched_updated) && !--scx_nr_ops_cid_sched_updated; + if (last) + static_branch_disable(&__scx_ops_cid_sched_updated_enabled); + + /* + * Re-homing restarts a running task's session under its new sched. A + * task that blocked while its cpu's dispatch had the rq lock dropped is + * put and set next again without a restart, so rq->scx.sched keeps @sch + * until the pick completes: + * + * cpu 0, __schedule() cpu 1, scx_sub_disable(@sch) + * T of @sch blocks, stops running + * dispatch drops the rq lock + * re-home T, nothing to restart + * this sweep + * pick N, N's session opens + * + * Closing the session here reports NONE to @sch and its ancestors now + * and lets the pick report N's sched from NONE instead of from @sch. + */ + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + guard(rq_lock_irqsave)(rq); + if (rq->scx.sched == sch) + scx_cid_sched_update(rq, NULL); + if (last) + rq->scx.sched = NULL; + } +} + static struct scx_sched *scx_find_sub_sched(u64 cgroup_id) { return rhashtable_lookup(&scx_sched_hash, &cgroup_id, @@ -285,7 +447,8 @@ void scx_rescue_charge(struct rq *rq, s64 delta_exec) rq->scx.rescue.budget -= delta_exec; /* per-cpu usage average feeds the overload victim pick */ - pcpu = per_cpu_ptr(scx_task_sched(rq->curr)->pcpu, cpu_of(rq)); + pcpu = per_cpu_ptr(scx_task_sched(rq->scx.rescue.curr)->pcpu, + cpu_of(rq)); pcpu->rescue_avg = scx_rescue_decay_avg(pcpu) + delta_exec; if (!scx_rescue_slice_remaining(rq)) @@ -557,7 +720,7 @@ static void scx_rescue_timerfn(struct timer_list *timer) scx_rescue_admit(rq, p, slice); scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, rq); - if (sched_class_above(&ext_sched_class, rq->curr->sched_class)) + if (sched_class_above(&ext_sched_class, rq->donor->sched_class)) resched_curr(rq); } else if (p->scx.dsq && rq->scx.rescue.budget > 2 * scx_rescue_quantum_ns) { /* @@ -671,7 +834,7 @@ void scx_rescue_init(struct rq *rq) } /** - * scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert + * __scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert * @sch: enqueuing sub-sched * @rq: rq whose local DSQ @p targets * @p: task being inserted @@ -682,20 +845,18 @@ void scx_rescue_init(struct rq *rq) * rescue is enabled, or @rq's reject DSQ after recording the reenq reason on * @p. * - * %SCX_ENQ_IMMED, %SCX_ENQ_PREEMPT and %SCX_ENQ_HEAD are cleared when diverting - * to rescue or reject. %SCX_ENQ_PREEMPT is also cleared on a fallback - * migration-disabled admission. + * %SCX_ENQ_IMMED, %SCX_ENQ_PREEMPT, %SCX_ENQ_PREEMPT_LAZY and %SCX_ENQ_HEAD are + * cleared when diverting to rescue or reject. %SCX_ENQ_PREEMPT and + * %SCX_ENQ_PREEMPT_LAZY are also cleared on a fallback migration-disabled + * admission. * * Bypass doesn't need special-casing as a bypassing sched's tasks are enqueued * to and run by its nearest non-bypassing ancestor. If root is bypassing, it * always holds all caps. */ -struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 *enq_flags) +struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags) { - if (!scx_has_subs()) - return &rq->scx.local_dsq; - s32 cid = __scx_cpu_to_cid(cpu_of(rq)); struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch; u64 needed = scx_caps_for_enq(*enq_flags); @@ -705,7 +866,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r * On a remote activation the scheduling sched (@asch) differs from * @p's owner (@sch). Check caps against the scheduling sched. */ - if (*enq_flags & SCX_ENQ_PREEMPT) + if (*enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) needed |= scx_caps_for_preempt(asch, rq, *enq_flags); missing = scx_missing_caps(asch, cpu_of(rq), needed); @@ -722,7 +883,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r if (unlikely(!scx_rq_online(rq) || is_migration_disabled(p) || p->migration_pending)) { __scx_add_event(sch, SCX_EV_SUB_FORCED_ADMIT, 1); - *enq_flags &= ~SCX_ENQ_PREEMPT; + *enq_flags &= ~(SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY); return &rq->scx.local_dsq; } @@ -731,9 +892,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r * or HEAD - a diversion has no priority and IMMED is not allowed on * non-local DSQs. Strip the enq and task flags along with the slice. */ - *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | SCX_ENQ_HEAD | - SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL); - p->scx.flags &= ~SCX_TASK_IMMED; + scx_divert_strip_flags(p, enq_flags); /* the enqueuer opted for rescue instead of rejection and reenqueue */ if ((*enq_flags & SCX_ENQ_RESCUE) && likely(scx_rescue_bw_1024)) { @@ -748,6 +907,8 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r p->scx.reenq_reason_caps = missing; p->scx.reenq_reason_cid = cid; + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); + p->scx.flags |= SCX_TASK_REENQ_CAP; return &rq->scx.reject_dsq; } @@ -755,67 +916,24 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r /* @p lost the caps needed to stay on @rq's local DSQ? Record reason if so. */ bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { + s32 cpu = cpu_of(rq); u64 missing; /* migration-disabled tasks and the rescuee are admitted capless */ if (is_migration_disabled(p) || p == scx_rescuee(rq)) return false; - missing = scx_missing_caps(scx_task_sched(p), cpu_of(rq), scx_caps_for_task(p)); + /* check caps against the sched running a bypassed task */ + missing = scx_missing_caps(scx_exec_sched(scx_task_sched(p), cpu), cpu, + scx_caps_for_task(p)); if (likely(!missing)) return false; p->scx.reenq_reason_caps = missing; - p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu_of(rq)); + p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu); return true; } -/* - * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides - * from p->scx.reenq_reason_*. - * - * A task can be re-rejected repeatedly. The reenqueue is bounded per task in - * scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT. - * Rejection can't happen for root. - */ -void scx_reenq_reject(struct rq *rq) -{ - LIST_HEAD(tasks); - struct task_struct *p, *n; - - lockdep_assert_rq_held(rq); - - if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list)) - return; - - /* - * Move to a private list so a task re-rejected by the - * scx_do_enqueue_task() below isn't revisited this round. - */ - list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) { - /* migration_pending tasks should have bypassed to local DSQ */ - if (WARN_ON_ONCE(p->migration_pending)) - continue; - - scx_wait_dispatching(p); - scx_dispatch_dequeue(rq, p); - - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - p->scx.flags |= SCX_TASK_REENQ_CAP; - - list_add_tail(&p->scx.dsq_list.node, &tasks); - } - - list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { - list_del_init(&p->scx.dsq_list.node); - - scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - } -} - /* record a caps change, see struct scx_caps_updated */ static void caps_updated_record(struct scx_pshard *ps, const struct scx_cmask *cids, u64 caps, struct list_head *to_deliver) @@ -921,7 +1039,7 @@ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid) struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); /* - * Pairs with smp_mb() in scx_process_sync_ecaps(). Either the check + * Pairs with smp_mb() in __scx_process_sync_ecaps(). Either the check * below sees the node off the list and queues it, or the in-flight sync * sees the caps[] update made before this call. */ @@ -946,7 +1064,7 @@ static void discard_queued_syncs(struct rq *rq) } /** - * scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] + * __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] * @rq: the cid's cpu rq * @prev: @rq's previous task from the in-progress dispatch * @@ -958,16 +1076,14 @@ static void discard_queued_syncs(struct rq *rq) * learns the cid's idle state. Such a gain arms the per-rq * %SCX_RQ_SUB_IDLE_RENOTIFY gate so the next idle pick delivers it. */ -void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) +void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) { s32 cpu = cpu_of(rq); s32 cid, shard; struct llist_node *batch, *pos, *tmp; u64 lost_all = 0; - lockdep_assert_rq_held(rq); - - if (!scx_has_subs() || likely(llist_empty(&rq->scx.ecaps_to_sync))) + if (likely(llist_empty(&rq->scx.ecaps_to_sync))) return; /* @@ -1008,27 +1124,43 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) lost_all |= lost; /* - * Tell the sched its effective caps on this cid changed. The - * invocation is equivalent to the dispatch path and may drop - * and re-acquire the rq lock temporarily while the rest of - * @batch is held privately, see scx_discard_ecaps_to_sync(). - * The dispatch kfuncs resolve their context on the executing - * cpu, which under core scheduling can differ from @rq's cpu, - * so the context is set up there. The rq recorded in it keeps - * the dispatches targeting @rq. + * Tell the sched and its parent that the sched's effective caps + * on this cid changed. The invocations are equivalent to the + * dispatch path and may drop and re-acquire the rq lock + * temporarily while the rest of @batch is held privately, see + * scx_discard_ecaps_to_sync(). The dispatch kfuncs resolve + * their context on the executing cpu, which under core + * scheduling can differ from @rq's cpu, so the context is set + * up there. The rq recorded in it keeps the dispatches + * targeting @rq. + * + * Bypass propagates down the hierarchy, so a sched that isn't + * bypassing has no bypassing parent. The child's bypass state + * gates both deliveries. Both report the same before value, so + * one reported_ecaps covers them. */ - if (ecaps != pcpu->reported_ecaps && - SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) && - !scx_bypassing(pcpu->sch, cpu)) { - struct scx_dsp_ctx *dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx; + if (ecaps != pcpu->reported_ecaps && !scx_bypassing(pcpu->sch, cpu)) { + struct scx_sched *parent = scx_parent(pcpu->sch); + struct scx_dsp_ctx *dspc; - dspc->rq = rq; /* stash @prev so nested dispatches can access it */ rq->scx.sub_dispatch_prev = prev; - SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, scx_cpu_arg(cpu), - pcpu->reported_ecaps, ecaps); + if (SCX_HAS_OP(pcpu->sch, sub_ecaps_updated)) { + dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx; + dspc->rq = rq; + SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, + scx_cpu_arg(cpu), pcpu->reported_ecaps, ecaps); + scx_flush_dispatch_buf(pcpu->sch, rq); + } + if (SCX_HAS_OP(parent, sub_child_ecaps_updated)) { + dspc = &this_cpu_ptr(parent->pcpu)->dsp_ctx; + dspc->rq = rq; + SCX_CALL_OP(parent, sub_child_ecaps_updated, rq, + pcpu->sch->ops.sub_cgroup_id, scx_cpu_arg(cpu), + pcpu->reported_ecaps, ecaps); + scx_flush_dispatch_buf(parent, rq); + } rq->scx.sub_dispatch_prev = NULL; - scx_flush_dispatch_buf(pcpu->sch, rq); pcpu->reported_ecaps = ecaps; } @@ -1154,6 +1286,34 @@ void scx_offline_ecaps(struct rq *rq) } /* + * Clear every cap @sch holds. The pshard caps go first as they are the source a + * pending sync recomputes ecaps from. ecaps are then zeroed directly for the + * cap checks. + */ +static void clear_all_caps(struct scx_sched *sch) +{ + s32 si, cpu; + u32 cap_bit; + + /* enable may have failed before scx_alloc_pshards() */ + if (!sch->pshard) + return; + + for (si = 0; si < sch->nr_pshards; si++) { + struct scx_pshard *ps = sch->pshard[si]; + + guard(raw_spinlock_irqsave)(&ps->lock); + for (cap_bit = 0; cap_bit < __SCX_NR_CAPS; cap_bit++) + scx_cmask_clear(&ps->caps[cap_bit].cmask); + } + + for_each_possible_cpu(cpu) { + guard(rq_lock_irqsave)(cpu_rq(cpu)); + WRITE_ONCE(per_cpu_ptr(sch->pcpu, cpu)->ecaps, 0); + } +} + +/* * @pcpu's sched was unhashed before the grace period, so nothing re-queues its * sync node. Remove the node from @rq's pending list so the pcpu can be freed. */ @@ -1252,7 +1412,9 @@ static void scx_rehome_task(struct scx_sched *to, struct task_struct *p) lockdep_assert_held(&p->pi_lock); lockdep_assert_rq_held(task_rq(p)); - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_prepare_task_sched_change(p); + scoped_guard (sched_change, p, + DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { scx_disable_and_exit_task(scx_task_sched(p), p); scx_set_task_state(p, SCX_TASK_INIT_BEGIN); scx_set_task_state(p, SCX_TASK_INIT); @@ -1282,7 +1444,9 @@ static void scx_punt_task(struct scx_sched *to, struct task_struct *p) lockdep_assert_rq_held(task_rq(p)); WARN_ON_ONCE(!READ_ONCE(to->bypass_depth)); - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_prepare_task_sched_change(p); + scoped_guard (sched_change, p, + DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { scx_disable_and_exit_task(scx_task_sched(p), p); scx_set_task_sched(p, to); } @@ -1358,7 +1522,7 @@ static s32 scx_cgroup_claim_subtree(struct scx_sched *sch) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, - .sched_idle = tg->scx.idle, + .sched_idle = tg->scx.sched_idle, }; if (tg->scx.sched != parent || @@ -1462,7 +1626,7 @@ static void scx_cgroup_return_subtree(struct scx_sched *sch) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, - .sched_idle = tg->scx.idle, + .sched_idle = tg->scx.sched_idle, }; /* the first pass must have transferred everything */ @@ -1616,6 +1780,8 @@ dump: scx_cgroup_unlock(); percpu_up_write(&scx_fork_rwsem); + scx_ops_cid_sched_updated_disable(sch); + /* * All tasks are moved off of @sch but there may still be on-going * operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use @@ -1628,6 +1794,12 @@ dump: scx_unlink_sched(sch); + /* + * The parent restores what it delegated in ops.sub_detach(). @sch must + * hold no caps by then. + */ + clear_all_caps(sch); + mutex_unlock(&scx_enable_mutex); /* @@ -1787,6 +1959,15 @@ void scx_sub_enable_workfn(struct kthread_work *work) if (ret) goto err_disable; + /* + * Bypass before @sch is linked and grants can reach it. The parent's + * delivery advances reported_ecaps, so a sync consumed before @sch's + * ops are registered would reach the parent and never be replayed to + * @sch. While bypassed, the syncs are consumed without a delivery and + * replayed to both at unbypass. + */ + scx_bypass(sch, true); + ret = scx_link_sched(sch); if (ret) goto err_disable; @@ -1832,12 +2013,12 @@ void scx_sub_enable_workfn(struct kthread_work *work) } sch->sub_attached = true; - scx_bypass(sch, true); - for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op); + scx_ops_cid_sched_updated_enable(sch); + percpu_down_write(&scx_fork_rwsem); scx_cgroup_lock(); @@ -1940,7 +2121,9 @@ void scx_sub_enable_workfn(struct kthread_work *work) if (!(p->scx.flags & SCX_TASK_SUB_INIT)) continue; - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_prepare_task_sched_change(p); + scoped_guard (sched_change, p, + DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { /* * $p must be either READY or ENABLED. If ENABLED, * __scx_disabled_and_exit_task() first disables and @@ -2266,6 +2449,9 @@ static s32 sub_cap_preamble(u64 cgroup_id, u64 caps, const struct bpf_prog_aux * if (unlikely(!parent)) return -ENODEV; + if (!scx_kf_allowed_ctx(parent)) + return -EDEADLK; + if (!scx_is_cid_type()) { scx_error(parent, "sub-cap kfuncs require a cid-form scheduler"); return -EOPNOTSUPP; @@ -2306,8 +2492,8 @@ static s32 sub_cap_preamble(u64 cgroup_id, u64 caps, const struct bpf_prog_aux * * All-or-nothing keeps the caller-visible result binary per cid, so the denied * mask is one mask to interpret rather than a per-cap matrix. * - * Return 0 on full success, -EPERM if any cid was refused, or a negative - * errno on other failures. + * Return 0 on full success, -EPERM if any cid was refused, -ENODEV if the child + * is gone or being disabled, or a negative errno on other failures. */ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask *cmask__arena, @@ -2361,6 +2547,12 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, scoped_guard (raw_spinlock, &pps->lock) { guard(raw_spinlock_nested)(&cps->lock); + /* the child is being disabled, see clear_all_caps() */ + if (unlikely(READ_ONCE(child->aborting))) { + ret = -ENODEV; + goto out; + } + /* * Narrow granted_cids to cids the parent holds every * requested cap on. All-or-nothing per cid. @@ -2413,9 +2605,10 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, } } + ret = any_denied ? -EPERM : 0; +out: caps_updated_deliver(&to_deliver); - - return any_denied ? -EPERM : 0; + return ret; } /** diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index f7bcdfda8dd8..b76d750463f0 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -16,6 +16,9 @@ struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root); struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root); +void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch); +void scx_ops_cid_sched_updated_enable(struct scx_sched *sch); +void scx_ops_cid_sched_updated_disable(struct scx_sched *sch); void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch); struct cgroup *sch_cgroup(struct scx_sched *sch); void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch); @@ -28,16 +31,15 @@ bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux); void scx_free_pshards(struct scx_sched *sch); s32 scx_alloc_pshards(struct scx_sched *sch); void scx_init_root_caps(struct scx_sched *sch); -void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); +void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch); void scx_online_ecaps(struct rq *rq); void scx_offline_ecaps(struct rq *rq); void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu); void scx_discard_stale_ecaps_syncs(void); -struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 *enq_flags); +struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags); bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p); -void scx_reenq_reject(struct rq *rq); void scx_rescue_charge(struct rq *rq, s64 delta_exec); void scx_rescue_end(struct rq *rq); bool scx_rescue_keep(struct rq *rq, struct task_struct *p); @@ -72,10 +74,32 @@ static inline void scx_dec_has_subs(struct scx_sched *sch) static_branch_dec(&__scx_has_subs); } +/* hot-path hooks, gated inline so that a root-only system skips the calls */ +static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) +{ + lockdep_assert_rq_held(rq); + + if (scx_has_subs()) + __scx_process_sync_ecaps(rq, prev); +} + +static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, + struct rq *rq, + struct task_struct *p, + u64 *enq_flags) +{ + if (!scx_has_subs()) + return &rq->scx.local_dsq; + return __scx_resolve_local_dsq(sch, rq, p, enq_flags); +} + #else /* CONFIG_EXT_SUB_SCHED */ static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } static inline struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root) { return NULL; } +static inline void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch) {} +static inline void scx_ops_cid_sched_updated_enable(struct scx_sched *sch) {} +static inline void scx_ops_cid_sched_updated_disable(struct scx_sched *sch) {} static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } static inline const char *sch_cgrp_path(struct scx_sched *sch) { return "/"; } @@ -93,7 +117,6 @@ static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcp static inline void scx_discard_stale_ecaps_syncs(void) {} static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; } static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; } -static inline void scx_reenq_reject(struct rq *rq) {} static inline void scx_rescue_charge(struct rq *rq, s64 delta_exec) {} static inline void scx_rescue_end(struct rq *rq) {} static inline bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { return false; } @@ -169,17 +192,18 @@ static inline u64 scx_caps_for_task(struct task_struct *p) return SCX_CAP_ENQ; } -/* the cap @sch needs to preempt @rq's current task, 0 if none */ -static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) +/* the cap @sch needs to preempt @rq's current scheduling context, 0 if none */ +static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, + u64 enq_flags) { - struct task_struct *curr = rq->curr; + struct task_struct *donor = rq->donor; /* a kernel-forced placement preempts regardless of caps */ if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS)) return 0; /* a non-ext task can't be preempted by ext, own-subtree needs no cap */ - if (curr->sched_class != &ext_sched_class || - scx_is_descendant(scx_task_sched(curr), sch)) + if (donor->sched_class != &ext_sched_class || + scx_is_descendant(scx_task_sched(donor), sch)) return 0; return SCX_CAP_PREEMPT; } @@ -199,6 +223,8 @@ static inline u64 scx_caps_implied(u64 cap) /* may @p keep running on @rq's cpu? requires baseline cpu access */ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { + s32 cpu = cpu_of(rq); + if (!scx_has_subs()) return true; @@ -206,7 +232,9 @@ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p if (unlikely(is_migration_disabled(p))) return true; - return likely(!scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE)); + /* check caps against the sched running a bypassed task */ + return likely(!scx_missing_caps(scx_exec_sched(scx_task_sched(p), cpu), cpu, + SCX_CAP_BASE)); } /* the task admitted for rescue on @rq, NULL if none */ diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index 139176cf9fc6..38514d6bf149 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -59,17 +59,31 @@ enum scx_consts { }; /* - * Per-cid topology info. For each topology level (core, LLC, node) and shard, - * records the first cid in the unit and its global index. Global indices are - * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges over - * [0, nr_cores_at_init) with no gaps. No-topo cids have core/LLC/node fields - * set to -1 but always have valid shard assignments. + * Per-cid topology info. For each topology level (core, cluster, LLC, node) and + * shard, records the first cid in the unit and its global index. Global indices + * are consecutive integers assigned in cid-walk order, so e.g. core_idx ranges + * over [0, nr_cores_at_init) with no gaps. No-topo cids have core/cluster/LLC/ + * node fields set to -1 but always have valid shard assignments. + * + * A cluster is the cache-sharing level between the core and the LLC. Where + * this level is absent, each core forms a cluster of its own. Where the cache + * level spans the LLC, the whole LLC forms one cluster. Each cluster has a + * unique cluster_cid and a dense cluster_idx, and its cids form a contiguous + * range. + * + * Every cid therefore has a cluster, so cluster_cid alone does not say whether + * the machine has the level: the first cluster of an LLC starts at the LLC's + * own base cid and a core-wide one at the core's. * * Shards are contiguous CID ranges used as scalable locking/work domains for * sub-scheduler operations. By default each LLC becomes one shard, split into * smaller shards if the LLC exceeds the target size. No-topo cids are packed * into their own max-sized shards. * + * Shards are cut on core boundaries only, so a shard doesn't need to contain a + * whole cluster: a cluster may straddle two shards. Only core, cluster, LLC + * and node are nested, in that order. + * * New fields are appended, never inserted: scx_bpf_cid_topo() copies this * struct out sized by the program's own layout, and an older program's copy * must stay a prefix of the kernel's. @@ -82,6 +96,8 @@ enum scx_consts { * @node_idx: global index of that node, in [0, nr_nodes_at_init) * @shard_cid: first cid of this cid's shard * @shard_idx: global index of that shard, in [0, scx_nr_cid_shards) + * @cluster_cid: first cid of this cid's cluster + * @cluster_idx: global index of that cluster, in [0, nr_clusters_at_init) */ struct scx_cid_topo { s32 core_cid; @@ -92,6 +108,8 @@ struct scx_cid_topo { s32 node_idx; s32 shard_cid; s32 shard_idx; + s32 cluster_cid; + s32 cluster_idx; }; enum scx_cid_consts { diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 499a5649582e..87c5335add37 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -791,6 +791,10 @@ enum scx_rq_flags { SCX_RQ_BAL_CB_PENDING = 1 << 6, /* must queue a cb after dispatching */ SCX_RQ_SUB_IDLE_RENOTIFY = 1 << 7, /* sub-scheds are owed update_idle() */ SCX_RQ_ROOT_IDLE_RENOTIFY = 1 << 8, /* the root is owed update_idle() */ + SCX_RQ_PROXY_RETRY = 1 << 9, /* proxy-rejected tasks need retry */ + SCX_RQ_PROXY_TICK = 1 << 10, /* proxy execution requires the tick */ + SCX_RQ_PROXY_PENDING = 1 << 11, /* blocked donor awaits proxy resolution */ + SCX_RQ_PROXY_BLOCKING = 1 << 12, /* donor is being removed for sched change */ SCX_RQ_IN_WAKEUP = 1 << 16, SCX_RQ_IN_DISPATCH = 1 << 17, @@ -810,8 +814,8 @@ struct scx_rq_rescue { struct scx_rq { struct scx_dispatch_q local_dsq; + struct scx_dispatch_q reject_dsq; /* staging for rejected tasks */ #ifdef CONFIG_EXT_SUB_SCHED - struct scx_dispatch_q reject_dsq; /* staging for cap-rejected tasks */ struct scx_rq_rescue rescue; #endif struct list_head runnable_list; /* runnable tasks on this rq */ @@ -830,6 +834,8 @@ struct scx_rq { #endif u64 clock; /* current per-rq clock -- see scx_bpf_now() */ #ifdef CONFIG_EXT_SUB_SCHED + /* sched of the open running session, see scx_cid_sched_update() */ + struct scx_sched *sched; struct llist_head ecaps_to_sync; /* pending ecaps syncs */ struct task_struct *sub_dispatch_prev; #endif @@ -844,6 +850,9 @@ struct scx_rq { struct list_head deferred_reenq_users; /* user DSQs requesting reenq */ struct balance_callback deferred_bal_cb; struct balance_callback kick_sync_bal_cb; +#ifdef CONFIG_NO_HZ_FULL + struct balance_callback proxy_tick_bal_cb; +#endif struct irq_work deferred_irq_work; struct irq_work kick_cpus_irq_work; }; diff --git a/kernel/sched/syscalls.c b/kernel/sched/syscalls.c index b215b0ead9a6..45a127deeadd 100644 --- a/kernel/sched/syscalls.c +++ b/kernel/sched/syscalls.c @@ -678,6 +678,8 @@ change: if (prev_class != next_class) queue_flags |= DEQUEUE_CLASS; + scx_prepare_setscheduler(p, policy); + scoped_guard (sched_change, p, queue_flags) { if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile index 21554f089692..70d2503b6b33 100644 --- a/tools/sched_ext/Makefile +++ b/tools/sched_ext/Makefile @@ -89,7 +89,7 @@ ifneq ($(wildcard $(GENHDR)),) GENFLAGS := -DHAVE_GENHDR endif -CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ +CFLAGS += -g -O2 -pthread -Wall -Werror $(GENFLAGS) \ -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -I$(TOOLSINCDIR) -I$(APIDIR) -I$(CURDIR)/include diff --git a/tools/sched_ext/include/scx/cid.bpf.h b/tools/sched_ext/include/scx/cid.bpf.h index 69fb4e97bc77..d1e6ad1432d3 100644 --- a/tools/sched_ext/include/scx/cid.bpf.h +++ b/tools/sched_ext/include/scx/cid.bpf.h @@ -3,8 +3,8 @@ * BPF-side helpers for cids and cmasks. See kernel/sched/ext/cid.h for the * authoritative layout and semantics. The BPF-side helpers use the cmask_* * naming (no scx_ prefix); cmask is the SCX bitmap type so the prefix is - * redundant in BPF code. Atomics use __sync_val_compare_and_swap and every - * helper is inline (no .c counterpart). + * redundant in BPF code. Every helper is inline except the binary operations, + * which are weak global functions defined here, so there is no .c counterpart. * * Included by scx/common.bpf.h; don't include directly. * @@ -16,6 +16,11 @@ #include "bpf_arena_common.bpf.h" +/* libbpf's bpf_helpers.h defines it from 1.4 on */ +#ifndef __arg_arena +#define __arg_arena __attribute((btf_decl_tag("arg:arena"))) +#endif + #ifndef BIT_U64 #define BIT_U64(nr) (1ULL << (nr)) #endif @@ -76,7 +81,7 @@ static __always_inline void __cmask_init(struct scx_cmask __arena *m, u32 base, m->nr_cids = nr_cids; m->alloc_words = alloc_words; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { if (i >= alloc_words) break; m->bits[i] = 0; @@ -122,24 +127,115 @@ static __always_inline void cmask_reframe(struct scx_cmask __arena *m, u32 base, m->nr_cids = nr_cids; } +/** + * cmask_nr_words - Words of bits[] that @m's active range spans + * @m: cmask to measure + * + * @m->base need not be word aligned: bits[0] covers the whole word @m->base + * falls in, so the count is taken from that word, not from @m->base. + */ +static __always_inline u32 cmask_nr_words(const struct scx_cmask __arena *m) +{ + u32 wbase = m->base / 64; + + return m->nr_cids ? (m->base + m->nr_cids - 1) / 64 - wbase + 1 : 0; +} + +/** + * cmask_word - Read word @k of @m + * @m: cmask to read + * @k: word index, counted from the word @m->base falls in + * + * A word past the active range reads as 0, so a scan that looks one word + * ahead (at an SMT sibling that falls in the next word, say) needs no bound + * of its own. + */ +static __always_inline u64 cmask_word(const struct scx_cmask __arena *m, u32 k) +{ + if (k >= cmask_nr_words(m)) + return 0; + return m->bits[k]; +} + +/** + * cmask_range_word - Bits of word @k of @m that fall in [@start, @start + @nr) + * @m: cmask the word belongs to + * @k: word index, counted from the word @m->base falls in + * @start: first cid of the range + * @nr: number of cids in the range + * + * Scoping a scan to a domain that is contiguous in cid space, an LLC or a + * node, is then one AND per word, rather than a test of the ends of the + * range at every bit or a mask kept per domain. + */ +static __always_inline u64 cmask_range_word(const struct scx_cmask __arena *m, + u32 k, u32 start, u32 nr) +{ + u64 wlo = (u64)(m->base / 64 + k) * 64, whi = wlo + 64; + u64 lo = start, hi = (u64)start + nr; + + if (lo < wlo) + lo = wlo; + if (hi > whi) + hi = whi; + if (lo >= hi) + return 0; + + return GENMASK_U64(hi - wlo - 1, lo - wlo); +} + +/** + * __cmask_test - Test a cid without checking it against the active range + * @cid: cid to test + * @m: cmask to test + * + * The caller must already know that @cid lies within + * [@m->base, @m->base + @m->nr_cids), or the read runs off @m->bits. + * + * For hot scans that have already bounded @cid, where the per-call range check + * is repeated work on every candidate. + */ +static __always_inline bool __cmask_test(u32 cid, const struct scx_cmask __arena *m) +{ + return *__cmask_word(cid, m) & BIT_U64(cid & 63); +} + static __always_inline bool cmask_test(u32 cid, const struct scx_cmask __arena *m) { if (!__cmask_contains(cid, m)) return false; - return *__cmask_word(cid, m) & BIT_U64(cid & 63); + return __cmask_test(cid, m); } /* - * x86 BPF JIT rejects BPF_OR | BPF_FETCH and BPF_AND | BPF_FETCH on arena - * pointers (see bpf_jit_supports_insn() in arch/x86/net/bpf_jit_comp.c). Only - * BPF_CMPXCHG / BPF_XCHG / BPF_ADD with FETCH are allowed. Implement - * test_and_{set,clear} and the atomic set/clear via a cmpxchg loop. + * The bit helpers use the fetching bitwise builtins, which order like the + * cmpxchg they replace, and not every JIT accepts those on arena pointers. The + * loader sets SCX_LIB_FEAT_ARENA_FETCH_BITOPS when a probe with the fetching + * form loads, and the helpers run a cmpxchg loop otherwise. The fetched value + * is consumed through barrier_var() because clang 19 lowers a builtin whose + * result is unused to the non-fetching form, which is relaxed on arm64. * * CMASK_CAS_TRIES is sized so exhausting it means seconds of real spinning - * on one word - past any plausible contention. Abort hard. + * on one word, past any plausible contention. Abort hard. */ #define CMASK_CAS_TRIES (1U << 23) +static __always_inline u64 __cmask_fetch_or(u64 __arena *w, u64 mask) +{ + u64 old = __atomic_fetch_or(w, mask, __ATOMIC_ACQ_REL); + + barrier_var(old); + return old; +} + +static __always_inline u64 __cmask_fetch_andnot(u64 __arena *w, u64 mask) +{ + u64 old = __atomic_fetch_and(w, ~mask, __ATOMIC_ACQ_REL); + + barrier_var(old); + return old; +} + static __always_inline void cmask_set(u32 cid, struct scx_cmask __arena *m) { u64 __arena *w; @@ -150,7 +246,11 @@ static __always_inline void cmask_set(u32 cid, struct scx_cmask __arena *m) return; w = __cmask_word(cid, m); bit = BIT_U64(cid & 63); - bpf_for(i, 0, CMASK_CAS_TRIES) { + if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) { + __cmask_fetch_or(w, bit); + return; + } + bpf_arena_for(i, 0, CMASK_CAS_TRIES) { old = *w; if (old & bit) return; @@ -171,7 +271,11 @@ static __always_inline void cmask_clear(u32 cid, struct scx_cmask __arena *m) return; w = __cmask_word(cid, m); bit = BIT_U64(cid & 63); - bpf_for(i, 0, CMASK_CAS_TRIES) { + if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) { + __cmask_fetch_andnot(w, bit); + return; + } + bpf_arena_for(i, 0, CMASK_CAS_TRIES) { old = *w; if (!(old & bit)) return; @@ -192,7 +296,9 @@ static __always_inline bool cmask_test_and_set(u32 cid, struct scx_cmask __arena return false; w = __cmask_word(cid, m); bit = BIT_U64(cid & 63); - bpf_for(i, 0, CMASK_CAS_TRIES) { + if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) + return __cmask_fetch_or(w, bit) & bit; + bpf_arena_for(i, 0, CMASK_CAS_TRIES) { old = *w; if (old & bit) return true; @@ -214,7 +320,9 @@ static __always_inline bool cmask_test_and_clear(u32 cid, struct scx_cmask __are return false; w = __cmask_word(cid, m); bit = BIT_U64(cid & 63); - bpf_for(i, 0, CMASK_CAS_TRIES) { + if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) + return __cmask_fetch_andnot(w, bit) & bit; + bpf_arena_for(i, 0, CMASK_CAS_TRIES) { old = *w; if (!(old & bit)) return false; @@ -268,17 +376,242 @@ static __always_inline bool __cmask_test_and_clear(u32 cid, struct scx_cmask __a return prev; } +/* atomically or @mask into *@w */ +static __always_inline void __cmask_word_or(u64 __arena *w, u64 mask) +{ + u64 old, new; + u32 i; + + if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) { + __cmask_fetch_or(w, mask); + return; + } + bpf_arena_for(i, 0, CMASK_CAS_TRIES) { + old = *w; + if ((old & mask) == mask) + return; + new = old | mask; + if (__sync_val_compare_and_swap(w, old, new) == old) + return; + } + scx_bpf_error("__cmask_word_or CAS exhausted"); +} + +/* atomically clear @mask bits in *@w */ +static __always_inline void __cmask_word_andnot(u64 __arena *w, u64 mask) +{ + u64 old, new; + u32 i; + + if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) { + __cmask_fetch_andnot(w, mask); + return; + } + bpf_arena_for(i, 0, CMASK_CAS_TRIES) { + old = *w; + if (!(old & mask)) + return; + new = old & ~mask; + if (__sync_val_compare_and_swap(w, old, new) == old) + return; + } + scx_bpf_error("__cmask_word_andnot CAS exhausted"); +} + +/** + * cmask_full_range - Test whether every cid in [@start, @start + @nr) is set + * @m: cmask to test + * @start: first cid of the range + * @nr: number of cids in the range + * + * The range is clamped to @m's active range first. True when no clamped bit is + * clear, including when the clamped range is empty. + */ +static __always_inline bool cmask_full_range(const struct scx_cmask __arena *m, u32 start, + u32 nr) +{ + u64 end = (u64)start + nr; + u32 wbase = m->base / 64; + u32 first_wi, last_wi, first_bit, last_bit, last, i; + + if (start < m->base) + start = m->base; + if (end > m->base + m->nr_cids) + end = m->base + m->nr_cids; + if (start >= end) + return true; + last = end - 1; + + first_wi = start / 64 - wbase; + last_wi = last / 64 - wbase; + first_bit = start & 63; + last_bit = last & 63; + + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { + u32 wi = first_wi + i; + u64 mask = ~0ULL; + + if (wi > last_wi) + break; + if (wi == first_wi) + mask &= GENMASK_U64(63, first_bit); + if (wi == last_wi) + mask &= GENMASK_U64(last_bit, 0); + if ((m->bits[wi] & mask) != mask) + return false; + } + return true; +} + +/** + * cmask_set_range - Set every cid in [@start, @start + @nr) + * @m: cmask to modify + * @start: first cid of the range + * @nr: number of cids in the range + * + * The range is clamped to @m's active range first. Words are updated atomically + * so concurrent updates of other bits sharing a word are never lost. The range + * as a whole does not transition atomically. + */ +static __always_inline void cmask_set_range(struct scx_cmask __arena *m, u32 start, u32 nr) +{ + u64 end = (u64)start + nr; + u32 wbase = m->base / 64; + u32 first_wi, last_wi, first_bit, last_bit, last, i; + + if (start < m->base) + start = m->base; + if (end > m->base + m->nr_cids) + end = m->base + m->nr_cids; + if (start >= end) + return; + last = end - 1; + + first_wi = start / 64 - wbase; + last_wi = last / 64 - wbase; + first_bit = start & 63; + last_bit = last & 63; + + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { + u32 wi = first_wi + i; + u64 mask = ~0ULL; + + if (wi > last_wi) + break; + if (wi == first_wi) + mask &= GENMASK_U64(63, first_bit); + if (wi == last_wi) + mask &= GENMASK_U64(last_bit, 0); + __cmask_word_or(&m->bits[wi], mask); + } +} + +/** + * cmask_clear_range - Clear every cid in [@start, @start + @nr) + * @m: cmask to modify + * @start: first cid of the range + * @nr: number of cids in the range + * + * The range is clamped to @m's active range first. Words are updated atomically + * so concurrent updates of other bits sharing a word are never lost. The range + * as a whole does not transition atomically. + */ +static __always_inline void cmask_clear_range(struct scx_cmask __arena *m, u32 start, u32 nr) +{ + u64 end = (u64)start + nr; + u32 wbase = m->base / 64; + u32 first_wi, last_wi, first_bit, last_bit, last, i; + + if (start < m->base) + start = m->base; + if (end > m->base + m->nr_cids) + end = m->base + m->nr_cids; + if (start >= end) + return; + last = end - 1; + + first_wi = start / 64 - wbase; + last_wi = last / 64 - wbase; + first_bit = start & 63; + last_bit = last & 63; + + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { + u32 wi = first_wi + i; + u64 mask = ~0ULL; + + if (wi > last_wi) + break; + if (wi == first_wi) + mask &= GENMASK_U64(63, first_bit); + if (wi == last_wi) + mask &= GENMASK_U64(last_bit, 0); + __cmask_word_andnot(&m->bits[wi], mask); + } +} + static __always_inline void cmask_zero(struct scx_cmask __arena *m) { u32 nr_words = CMASK_NR_WORDS(m->nr_cids), i; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { if (i >= nr_words) break; m->bits[i] = 0; } } +/** + * cmask_fill - Set every cid in @m's active range + * @m: cmask to fill + * + * Counterpart to cmask_zero(). Storage past the active range is left as is, + * matching the kernel-side scx_cmask_fill(). + */ +static __always_inline void cmask_fill(struct scx_cmask __arena *m) +{ + u32 nr_words, head_bits, tail_bits, i; + + if (!m->nr_cids) + return; + nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1; + + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { + if (i >= nr_words) + break; + m->bits[i] = ~0LLU; + } + + /* clear word-0 bits below base */ + head_bits = m->base & 63; + if (head_bits) + m->bits[0] &= ~((1LLU << head_bits) - 1); + + /* clear last-word bits at or past base + nr_cids */ + tail_bits = (m->base + m->nr_cids) & 63; + if (tail_bits) + m->bits[nr_words - 1] &= (1LLU << tail_bits) - 1; +} + +/** + * cmask_empty - Test whether no cid is set in @m's active range + * @m: cmask to test + * + * Scans the words the active range spans, whose bits outside the range every + * mutator keeps zero. + */ +static __always_inline bool cmask_empty(const struct scx_cmask __arena *m) +{ + u32 nr_words = cmask_nr_words(m), i; + + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { + if (i >= nr_words) + break; + if (m->bits[i]) + return false; + } + return true; +} + /* * BPF_-prefixed to avoid colliding with the kernel's anonymous CMASK_OP_* * enum in ext/cid.c, which is exported via BTF and reachable through @@ -291,92 +624,136 @@ enum { BPF_CMASK_OP_ANDNOT, }; -static __always_inline void cmask_op_word(struct scx_cmask __arena *dst, - const struct scx_cmask __arena *src, - u32 di, u32 si, u64 mask, int op) +/* range bits of word @k of @m, which spans @nr words: 0 past the span */ +static __always_inline u64 __cmask_word_range(const struct scx_cmask __arena *m, u32 k, + u32 nr) { - u64 dv = dst->bits[di]; - u64 sv = src->bits[si]; - u64 rv; + u64 bits = ~0ULL; - if (op == BPF_CMASK_OP_AND) - rv = dv & sv; - else if (op == BPF_CMASK_OP_OR) - rv = dv | sv; - else if (op == BPF_CMASK_OP_ANDNOT) - rv = dv & ~sv; - else - rv = sv; - - dst->bits[di] = (dv & ~mask) | (rv & mask); + if (k >= nr) + return 0; + if (k == 0) + bits &= GENMASK_U64(63, m->base & 63); + if (k == nr - 1) + bits &= GENMASK_U64((m->base + m->nr_cids - 1) & 63, 0); + return bits; } -static __always_inline void cmask_op(struct scx_cmask __arena *dst, - const struct scx_cmask __arena *src, int op) +/* + * The binary operations combine cmasks of any active ranges. A cmask holds bits + * only for its own range and reads as the operation's identity outside it, all + * ones for AND and all zeros for OR, so an operand alters only the bits it + * covers: ANDing a shard's mask into a global one edits that shard's window and + * leaves the rest of the global mask alone. ANDNOT is AND with @src2 inverted + * over @src2's range, so where only @src2 has range the result is ~@src2. + * + * @dst is written on its range intersected with the union of the sources' + * ranges and untouched elsewhere, and its padding bits outside its own range + * stay zero, the invariant every cmask mutator keeps. @dst may alias either + * source, and the two sources may be one mask. Returns whether @dst has a bit + * set afterwards, over all of @dst's range. + * + * The operations are global functions, verified once per program. Inlined, + * every data-dependent branch in this loop would multiply the states the + * verifier explores per iteration at every call site. + */ +static __always_inline bool __cmask_op(struct scx_cmask __arena *dst, + const struct scx_cmask __arena *src1, + const struct scx_cmask __arena *src2, int op) { - u32 d_end = dst->base + dst->nr_cids; - u32 s_end = src->base + src->nr_cids; - u32 lo = dst->base > src->base ? dst->base : src->base; - u32 hi = d_end < s_end ? d_end : s_end; - u32 d_base = dst->base / 64; - u32 s_base = src->base / 64; - u32 lo_word, hi_word, i; - u64 head_mask, tail_mask; - - if (lo >= hi) - return; - - lo_word = lo / 64; - hi_word = (hi - 1) / 64; - head_mask = GENMASK_U64(63, lo & 63); - tail_mask = GENMASK_U64((hi - 1) & 63, 0); - - bpf_for(i, 0, CMASK_MAX_WORDS) { - u32 w = lo_word + i; - u64 m; - - if (w > hi_word) - break; - - m = GENMASK_U64(63, 0); - if (w == lo_word) - m &= head_mask; - if (w == hi_word) - m &= tail_mask; + u32 d_first = dst->base / 64, d_nr = cmask_nr_words(dst); + u32 s1_first = src1->base / 64, s1_nr = cmask_nr_words(src1); + u32 s2_first = src2->base / 64, s2_nr = cmask_nr_words(src2); + u64 any = bpf_arena_loop_zero; + u32 i; - cmask_op_word(dst, src, w - d_base, w - s_base, m, op); + bpf_arena_for(i, 0, d_nr) { + u32 k1 = d_first + i - s1_first, k2 = d_first + i - s2_first; + u64 m1 = __cmask_word_range(src1, k1, s1_nr); + u64 m2 = op == BPF_CMASK_OP_COPY ? m1 : __cmask_word_range(src2, k2, s2_nr); + u64 um = __cmask_word_range(dst, i, d_nr) & (m1 | m2); + u64 dv = dst->bits[i], v1, v2, rv; + + if (um) { + /* a word in range holds zeros outside the range */ + v1 = m1 ? src1->bits[k1] : 0; + v2 = m2 ? src2->bits[k2] : 0; + if (op == BPF_CMASK_OP_AND) + rv = (v1 | ~m1) & (v2 | ~m2); + else if (op == BPF_CMASK_OP_OR) + rv = v1 | v2; + else if (op == BPF_CMASK_OP_ANDNOT) + rv = (v1 | ~m1) & ~v2; + else + rv = v1; + dv = (dv & ~um) | (rv & um); + dst->bits[i] = dv; + } + any |= dv; } + return any != 0; } -/* - * cmask_and/or/copy only modify @dst bits that lie in the intersection of - * [@dst->base, @dst->base + @dst->nr_cids) and [@src->base, - * @src->base + @src->nr_cids). Bits in @dst outside that window - * keep their prior values - in particular, cmask_copy() does NOT zero @dst - * bits that lie outside @src's range. +/** + * cmask_and - Store @src1 AND @src2 in @dst + * @dst: destination, may be @src1 or @src2 + * @src1: first operand + * @src2: second operand + * + * See __cmask_op() for how the ranges combine. Return whether @dst has a bit + * set afterwards. */ -static __always_inline void cmask_and(struct scx_cmask __arena *dst, - const struct scx_cmask __arena *src) +__weak bool cmask_and(struct scx_cmask __arena __arg_arena *dst, + const struct scx_cmask __arena __arg_arena *src1, + const struct scx_cmask __arena __arg_arena *src2) { - cmask_op(dst, src, BPF_CMASK_OP_AND); + return __cmask_op(dst, src1, src2, BPF_CMASK_OP_AND); } -static __always_inline void cmask_or(struct scx_cmask __arena *dst, - const struct scx_cmask __arena *src) +/** + * cmask_or - Store @src1 OR @src2 in @dst + * @dst: destination, may be @src1 or @src2 + * @src1: first operand + * @src2: second operand + * + * See __cmask_op() for how the ranges combine. Return whether @dst has a bit + * set afterwards. + */ +__weak bool cmask_or(struct scx_cmask __arena __arg_arena *dst, + const struct scx_cmask __arena __arg_arena *src1, + const struct scx_cmask __arena __arg_arena *src2) { - cmask_op(dst, src, BPF_CMASK_OP_OR); + return __cmask_op(dst, src1, src2, BPF_CMASK_OP_OR); } -static __always_inline void cmask_copy(struct scx_cmask __arena *dst, - const struct scx_cmask __arena *src) +/** + * cmask_andnot - Store @src1 AND NOT @src2 in @dst + * @dst: destination, may be @src1 or @src2 + * @src1: operand to remove bits from + * @src2: bits to remove, inverted over its own range + * + * See __cmask_op() for how the ranges combine. Return whether @dst has a bit + * set afterwards. + */ +__weak bool cmask_andnot(struct scx_cmask __arena __arg_arena *dst, + const struct scx_cmask __arena __arg_arena *src1, + const struct scx_cmask __arena __arg_arena *src2) { - cmask_op(dst, src, BPF_CMASK_OP_COPY); + return __cmask_op(dst, src1, src2, BPF_CMASK_OP_ANDNOT); } -static __always_inline void cmask_andnot(struct scx_cmask __arena *dst, - const struct scx_cmask __arena *src) +/** + * cmask_copy - Copy @src into @dst + * @dst: destination + * @src: source + * + * The single-source case of __cmask_op(): @dst is written over @src's range and + * untouched elsewhere. + */ +__weak void cmask_copy(struct scx_cmask __arena __arg_arena *dst, + const struct scx_cmask __arena __arg_arena *src) { - cmask_op(dst, src, BPF_CMASK_OP_ANDNOT); + __cmask_op(dst, src, src, BPF_CMASK_OP_COPY); } /* @@ -395,7 +772,7 @@ static __always_inline bool cmask_equal(const struct scx_cmask __arena *a, return true; nr_words = (a->base + a->nr_cids - 1) / 64 - a->base / 64 + 1; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { if (i >= nr_words) break; if (a->bits[i] != b->bits[i]) @@ -428,7 +805,7 @@ static __always_inline u32 cmask_next_set(const struct scx_cmask __arena *m, u32 start_wi = cid / 64 - base; start_bit = cid & 63; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { u32 wi = start_wi + i; u64 word; u32 found; @@ -457,7 +834,7 @@ static __always_inline u32 cmask_first_set(const struct scx_cmask __arena *m) #define cmask_for_each(cid, m) \ for ((cid) = cmask_first_set(m); \ - (cid) < (m)->base + (m)->nr_cids; \ + (cid) < (m)->base + (m)->nr_cids && can_loop; \ (cid) = cmask_next_set((m), (cid) + 1)) /* @@ -495,7 +872,7 @@ static __always_inline bool cmask_subset(const struct scx_cmask __arena *a, lo_word = lo / 64; hi_word = (hi - 1) / 64; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { u32 w = lo_word + i; if (w > hi_word) @@ -514,13 +891,14 @@ static __always_inline bool cmask_subset(const struct scx_cmask __arena *a, static __always_inline u32 cmask_weight(const struct scx_cmask __arena *m) { u32 nr_words, i; - u32 count = 0; + /* callers compare the sum, see bpf_arena_loop_zero */ + u32 count = bpf_arena_loop_zero; if (!m->nr_cids) return 0; nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { if (i >= nr_words) break; count += __builtin_popcountll(m->bits[i]); @@ -528,10 +906,7 @@ static __always_inline u32 cmask_weight(const struct scx_cmask __arena *m) return count; } -/* - * True if @a and @b share any set bit. Walk only the intersection of their - * ranges, matching the semantics of cmask_and(). - */ +/* true if @a and @b share any set bit, over the intersection of their ranges */ static __always_inline bool cmask_intersects(const struct scx_cmask __arena *a, const struct scx_cmask __arena *b) { @@ -552,7 +927,7 @@ static __always_inline bool cmask_intersects(const struct scx_cmask __arena *a, head_mask = GENMASK_U64(63, lo & 63); tail_mask = GENMASK_U64((hi - 1) & 63, 0); - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { u32 w = lo_word + i; u64 mask, av, bv; @@ -603,7 +978,7 @@ static __always_inline u32 cmask_next_and_set(const struct scx_cmask __arena *a, start_wi = start / 64; start_bit = start & 63; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { u32 abs_wi = start_wi + i; u64 word; u32 found; @@ -646,6 +1021,17 @@ static __always_inline u32 cmask_next_set_wrap(const struct scx_cmask __arena *m return found < start ? found : end; } +/** + * cmask_end - One past the last cid of @m's active range + * @m: cmask of interest + * + * The scan helpers return this value when no set cid is found. + */ +static __always_inline u32 cmask_end(const struct scx_cmask __arena *m) +{ + return m->base + m->nr_cids; +} + /* * Find the next cid set in both @a and @b at or after @start, wrapping to * @a->base if none found in the forward half. Return a->base + a->nr_cids @@ -669,6 +1055,121 @@ static __always_inline u32 cmask_next_and_set_wrap(const struct scx_cmask __aren } /* + * The distribute helpers rotate through one slot per cid, a cacheline each so + * picks on one CPU do not bounce the line of another. A missing allocation is a + * setup bug and aborts the scheduler. On a CPU outside the scheduler's cid + * space the helpers fall back to the first matching cid. + * + * TODO: The slots are one allocation without node placement. Move them to a + * per-cid arena allocator once the library has one, for node-local slots + * without the cacheline padding. + */ +struct cmask_rotor_slot { + u32 cid; +} __attribute__((aligned(SCX_CACHELINE_SIZE))); + +struct cmask_rotor_slot __arena *cmask_distribute_rotors __weak; + +/** + * cmask_distribute_init - Allocate the distribute rotor from @map + * @map: the scheduler's arena map + * + * The shared arena init calls this once. A scheduler with its own arena calls + * it before the first distribute pick. On a kernel without the cid kfuncs + * nothing can pick, so the call allocates nothing and succeeds. Return 0 on + * success, -ENOMEM if the allocation fails. + */ +static __always_inline int cmask_distribute_init(void *map) +{ + u64 size; + u32 pages; + + /* no cid kfuncs means no picks and nothing to allocate */ + if (!bpf_ksym_exists(scx_bpf_nr_cids)) + return 0; + + size = scx_bpf_nr_cids() * sizeof(*cmask_distribute_rotors); + pages = (size + PAGE_SIZE - 1) / PAGE_SIZE; + cmask_distribute_rotors = bpf_arena_alloc_pages(map, NULL, pages, NUMA_NO_NODE, 0); + return cmask_distribute_rotors ? 0 : -ENOMEM; +} + +static __always_inline u32 __arena *__cmask_distribute_rotor(void) +{ + s32 cid = scx_bpf_this_cid(); + + if (unlikely(!cmask_distribute_rotors)) { + scx_bpf_error("cmask distribute rotor not allocated"); + return NULL; + } + if (cid < 0) + return NULL; + return &cmask_distribute_rotors[cid].cid; +} + +/** + * cmask_any_distribute - Pick a set cid, spreading successive picks + * @m: cmask to pick from + * + * Counterpart of bpf_cpumask_any_distribute(): a per-cid rotor makes successive + * picks rotate through the set cids instead of repeating the first one. Returns + * cmask_end(@m) if @m is empty. + */ +static __always_inline u32 cmask_any_distribute(const struct scx_cmask __arena *m) +{ + u32 __arena *rotor = __cmask_distribute_rotor(); + u32 pick; + + if (unlikely(!rotor)) + return cmask_first_set(m); + + pick = cmask_next_set_wrap(m, *rotor + 1); + if (pick < cmask_end(m)) + *rotor = pick; + return pick; +} + +/** + * cmask_any_and_distribute - Pick a cid set in both masks, spreading picks + * @a: first cmask, the scan is bounded by its range + * @b: second cmask + * + * Counterpart of bpf_cpumask_any_and_distribute(). Shares the rotor with + * cmask_any_distribute() the same way the kernel counterparts share theirs. + * Returns cmask_end(@a) if the intersection is empty. + */ +static __always_inline u32 cmask_any_and_distribute(const struct scx_cmask __arena *a, + const struct scx_cmask __arena *b) +{ + u32 __arena *rotor = __cmask_distribute_rotor(); + u32 pick; + + if (unlikely(!rotor)) + return cmask_next_and_set(a, b, a->base); + + pick = cmask_next_and_set_wrap(a, b, *rotor + 1); + if (pick < cmask_end(a)) + *rotor = pick; + return pick; +} + +/** + * cmask_distribute_rotor_pos - Last cid picked by the distribute helpers + * + * For callers that anchor scans of their own on the shared rotor. Returns 0 + * when nothing has been picked on this cid yet or the CPU is outside the cid + * space. + */ +static __always_inline u32 cmask_distribute_rotor_pos(void) +{ + u32 __arena *rotor = __cmask_distribute_rotor(); + + if (unlikely(!rotor)) + return 0; + return *rotor; +} + +/* * Like cmask_next_and_set() but over the intersection of THREE masks. Return * a->base + a->nr_cids if no cid is set in all three at or after @start. */ @@ -701,7 +1202,7 @@ static __always_inline u32 cmask_next_and2_set(const struct scx_cmask __arena *a start_wi = start / 64; start_bit = start & 63; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { u32 abs_wi = start_wi + i; u64 word; u32 found; @@ -763,7 +1264,7 @@ static __always_inline void cmask_from_cpumask(struct scx_cmask __arena *m, s32 cpu; cmask_zero(m); - bpf_for(cpu, 0, nr_cpu_ids) { + bpf_arena_for(cpu, 0, nr_cpu_ids) { s32 cid; if (!bpf_cpumask_test_cpu(cpu, cpumask)) diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 22f24ebef8a9..9376d5704a47 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -27,6 +27,7 @@ #include "user_exit_info.bpf.h" #include "enum_defs.autogen.h" #include "bpf_arena_common.bpf.h" +#include "const-defs.h" #define PF_IDLE 0x00000002 /* I am an IDLE thread */ #define PF_IO_WORKER 0x00000010 /* Task is an IO worker */ @@ -74,6 +75,7 @@ struct task_struct *scx_bpf_dsq_peek(u64 dsq_id) __ksym __weak; int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, u64 flags) __ksym __weak; struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) __ksym __weak; void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) __ksym __weak; +bool scx_bpf_dsq_reenq_task(struct bpf_iter_scx_dsq *it, struct task_struct *p) __ksym __weak; void scx_bpf_exit_bstr(s64 exit_code, char *fmt, unsigned long long *data, u32 data__sz) __ksym __weak; void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym __weak; @@ -104,10 +106,11 @@ struct task_struct *scx_bpf_cpu_curr(s32 cpu) __ksym __weak; struct task_struct *scx_bpf_tid_to_task(u64 tid) __ksym __weak; u64 scx_bpf_now(void) __ksym __weak; void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __weak; +u32 scx_bpf_cgroup_nr_cpus(struct cgroup *cgrp) __ksym __weak; s32 scx_bpf_cpu_to_cid(s32 cpu) __ksym __weak; s32 scx_bpf_cid_to_cpu(s32 cid) __ksym __weak; -void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out, size_t out__sz) __ksym __weak; -void scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak; +s32 scx_bpf_cid_node(s32 cid) __ksym __weak; +/* scx_bpf_cid_topo() and scx_bpf_kick_cid() are declared in compat.bpf.h */ s32 scx_bpf_task_cid(const struct task_struct *p) __ksym __weak; s32 scx_bpf_this_cid(void) __ksym __weak; struct task_struct *scx_bpf_cid_curr(s32 cid) __ksym __weak; @@ -116,7 +119,7 @@ u32 scx_bpf_nr_online_cids(void) __ksym __weak; const void __arena *scx_bpf_online_cmask(void) __ksym __weak; u32 scx_bpf_cidperf_cap(s32 cid) __ksym __weak; u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak; -s32 scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak; +/* scx_bpf_cidperf_set() is declared in compat.bpf.h */ /* sub-scheduler cap control, scx_bpf_sub_caps() cgroup_id 0 == self */ s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask __arena *cmask__arena, struct scx_cmask __arena *denied_out__arena__nullable) __ksym __weak; @@ -307,7 +310,7 @@ BPF_PROG(name, ##args) * Similar to MEMBER_VPTR() but is intended for use with arrays where the * element count needs to be explicit. * It can be used in cases where a global array is defined with an initial - * size but is intended to be be resized before loading the BPF program. + * size but is intended to be resized before loading the BPF program. * Without this version of the macro, MEMBER_VPTR() will use the compile time * size of the array to compute the max, which will result in rejection by * the verifier. @@ -533,14 +536,14 @@ static __always_inline const struct cpumask *cast_mask(struct bpf_cpumask *mask) /* * True if the non-sleepable BPF trampoline prolog (__bpf_prog_enter) calls * migrate_disable() for the current task. Recorded once by - * scx_lib_init_probe, an fentry program on bpf_scx_reg() that fires during - * the natural scheduler-attach call chain (auto-attached by scx_ops_attach!). + * scx_lib_init_probe, an fentry program that fires during the natural + * scheduler-attach call chain (auto-attached by scx_ops_attach!). * - * Defaults to true (conservative). Over-reporting in is_migration_disabled() + * Defaults to false (conservative). Over-reporting in is_migration_disabled() * causes local-only dispatch, which is safe. Under-reporting can crash the * scheduler, so we err high if the probe somehow fails to run. */ -bool __scx_prolog_disables_migration __weak = true; +bool __scx_prolog_disables_migration __weak = false; /* * scx_lib_init_probe - non-sleepable prolog probe. @@ -551,13 +554,20 @@ bool __scx_prolog_disables_migration __weak = true; * ops.init() fires. Its address is taken in the vtable, so the symbol * is non-inlinable and has been stable since introduction. * + * A cid-form scheduler registers through bpf_scx_reg_cid(), the .reg + * callback of bpf_sched_ext_ops_cid, and so never enters bpf_scx_reg(). + * The cid-form open paths (SCX_OPS_CID_OPEN(), scx_ops_cid_open!()) + * therefore repoint this program at bpf_scx_reg_cid(). Repointing rather + * than adding a second program keeps the object loadable on kernels + * predating the cid form, where bpf_scx_reg_cid() has no BTF entry. + * * Entering via fentry runs us through __bpf_prog_enter -- the * non-sleepable prolog that consumers of is_migration_disabled() live * under. * * Loud warning: the prolog adds at most 1 to migration_disabled. * Reading > 1 means something upstream in the - * bpf_struct_ops_link_create -> bpf_scx_reg path disabled migration + * bpf_struct_ops_link_create -> .reg path disabled migration * before the prolog ran, invalidating the probe; audit and adjust. */ SEC("fentry/bpf_scx_reg") __weak @@ -1185,8 +1195,75 @@ static inline u64 scx_clock_irq(u32 cpu) #define struct_size_t(type, member, count) \ struct_size((type *)NULL, member, count) +/* + * <linux/stddef.h>'s TRAILING_OVERLAP(): embed a struct that ends in a flexible + * array member together with its storage. @NAME is a complete @TYPE and the + * @MEMBERS declared after the array reserve the space the array grows into. + * Unlike the kernel's, the padding is named after @NAME so that one struct can + * embed several, and it is sized with __builtin_offsetof() because + * bpf_helpers.h redefines offsetof() as a pointer cast, which is not a constant + * expression. + */ +#define __TRAILING_OVERLAP(TYPE, NAME, FAM, ATTRS, MEMBERS) \ + union { \ + TYPE NAME; \ + struct { \ + unsigned char __offset_to_##NAME[__builtin_offsetof(TYPE, FAM)]; \ + MEMBERS \ + } ATTRS; \ + } + +#define TRAILING_OVERLAP(TYPE, NAME, FAM, MEMBERS) \ + __TRAILING_OVERLAP(TYPE, NAME, FAM, /* no attrs */, MEMBERS) + +/* + * Loop counters the verifier cannot see through. + * + * A may_goto loop converges when the state at its head is within the state + * of a previous iteration. A counter tracked as a precise constant prevents + * that convergence: every iteration creates a new state, and the verifier + * unrolls the loop until it runs out of budget whenever the counter feeds an + * operation that requires precision. + * + * Loading a writable global gives the verifier an unknown scalar while its + * runtime value remains one. Using it as the step makes the counter unknown + * after the first iteration, and the load on every increment keeps the loop + * body from making it precise again. volatile is required to keep the compiler + * from hoisting or eliminating the loads, and the value must remain non-const + * so the verifier cannot resolve it. The variable is weak so the objects linked + * into a scheduler share one copy. Its runtime value must never be changed from + * one. + * + * bpf_for() avoids this verifier behavior too, but calls bpf_iter_num_next() + * on every iteration. bpf_arena_for() is intended for hot scheduler walks + * where that cost matters. @var must be no wider than u32, and @start and + * @end must be representable as u32. + */ +volatile u32 bpf_arena_loop_one __weak = 1; + +/* + * A loop-carried accumulator that a caller later compares is kept precise, and + * a precise scalar whose range grows every iteration keeps a may_goto loop from + * converging. Initializing it from this zero makes it unknown from the first + * iteration, so every pass through the loop head looks the same. + */ +volatile u32 bpf_arena_loop_zero __weak = 0; + +#define __bpf_arena_loop_start(var, start) \ + ({ \ + _Static_assert(sizeof(var) <= sizeof(u32), \ + "bpf_arena_for() index must fit in u32"); \ + (start); \ + }) + +#define bpf_arena_for(var, start, end) \ + for (var = __bpf_arena_loop_start(var, start); \ + var < (end) && can_loop; \ + var += bpf_arena_loop_one) + #include "compat.bpf.h" #include "enums.bpf.h" +#include "features.bpf.h" #include "cid.bpf.h" #endif /* __SCX_COMMON_BPF_H */ diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h index 6944221f96cc..254882bd8ce6 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -143,6 +143,69 @@ static inline void scx_bpf_cid_override(const s32 __arena *cpu_to_cid, u32 cpu_t shard_start, shard_start_cnt); } +/* + * v7.3: b38332be61a8 ("sched_ext: Make scx_bpf_kick_cid() return void") flipped + * the return type from s32 to void without renaming the kfunc, so the v7.2 and + * v7.3 kfuncs share a name but have incompatible prototypes. libbpf leaves a + * weak kfunc unresolved when the prototype doesn't match, so declare both + * flavors: the ___vN suffix is stripped on the BPF side, both resolve against + * the same kernel symbol and only the matching one gets bound. Drop the + * wrapper and move the decl back to common.bpf.h after v7.6. + */ +void scx_bpf_kick_cid___v2(s32 cid, u64 flags) __ksym __weak; +s32 scx_bpf_kick_cid___v1(s32 cid, u64 flags) __ksym __weak; + +static inline void scx_bpf_kick_cid(s32 cid, u64 flags) +{ + if (bpf_ksym_exists(scx_bpf_kick_cid___v2)) + scx_bpf_kick_cid___v2(cid, flags); + else if (bpf_ksym_exists(scx_bpf_kick_cid___v1)) + scx_bpf_kick_cid___v1(cid, flags); +} + +/* + * v7.3: 94480606a677 ("sched_ext: Add a size argument to scx_bpf_cid_topo() so + * struct scx_cid_topo can grow") added out__sz without renaming the kfunc, so + * the v7.2 and v7.3 kfuncs share a name with incompatible prototypes. Same + * two-flavor trick as scx_bpf_kick_cid() above. The size is always the output + * buffer's, so a macro supplies it and callers keep the two-argument form. + * Drop the wrapper and move the decl back to common.bpf.h after v7.6. + */ +void scx_bpf_cid_topo___v2(s32 cid, struct scx_cid_topo *out, size_t out__sz) __ksym __weak; +void scx_bpf_cid_topo___v1(s32 cid, struct scx_cid_topo *out) __ksym __weak; + +static inline void __scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out, size_t out__sz) +{ + if (bpf_ksym_exists(scx_bpf_cid_topo___v2)) + scx_bpf_cid_topo___v2(cid, out, out__sz); + else if (bpf_ksym_exists(scx_bpf_cid_topo___v1)) + scx_bpf_cid_topo___v1(cid, out); +} + +#define scx_bpf_cid_topo(cid, out) __scx_bpf_cid_topo((cid), (out), sizeof(*(out))) + +/* + * v7.3: 3a21e34eb258 ("sched_ext: Gate scx_bpf_cidperf_set() behind a new + * SCX_CAP_PERF") flipped the return type the other way, from void to s32, again + * without renaming the kfunc. Same two-flavor trick as scx_bpf_kick_cid() + * above; the v7.2 kfunc can't fail, so report success for it. Drop the wrapper + * and move the decl back to common.bpf.h after v7.6. + */ +s32 scx_bpf_cidperf_set___v2(s32 cid, u32 perf) __ksym __weak; +void scx_bpf_cidperf_set___v1(s32 cid, u32 perf) __ksym __weak; + +static inline s32 scx_bpf_cidperf_set(s32 cid, u32 perf) +{ + if (bpf_ksym_exists(scx_bpf_cidperf_set___v2)) { + return scx_bpf_cidperf_set___v2(cid, perf); + } else if (bpf_ksym_exists(scx_bpf_cidperf_set___v1)) { + scx_bpf_cidperf_set___v1(cid, perf); + return 0; + } else { + return -EOPNOTSUPP; + } +} + /** * __COMPAT_is_enq_cpu_selected - Test if SCX_ENQ_CPU_SELECTED is on * in a compatible way. We will preserve this __COMPAT helper until v6.16. @@ -243,6 +306,18 @@ static inline bool __COMPAT_is_enq_cpu_selected(u64 enq_flags) scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) : \ scx_bpf_pick_any_cpu(cpus_allowed, flags)) +/* v7.4: Add scx_bpf_cid_node(). */ +#define __COMPAT_scx_bpf_cid_node(cid) \ + (bpf_ksym_exists(scx_bpf_cid_node) ? \ + scx_bpf_cid_node(cid) : NUMA_NO_NODE) + +/* + * v7.4: Add scx_bpf_cgroup_nr_cpus(). + */ +#define __COMPAT_scx_bpf_cgroup_nr_cpus(cgrp) \ + (bpf_ksym_exists(scx_bpf_cgroup_nr_cpus) ? \ + scx_bpf_cgroup_nr_cpus(cgrp) : scx_bpf_nr_cpu_ids()) + /* * v6.18: Add a helper to retrieve the current task running on a CPU. * @@ -403,6 +478,15 @@ static inline void scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime) p->scx.dsq_vtime = vtime; } +/* v7.4: Add scx_bpf_task_set_lazy_resched(). */ +bool scx_bpf_task_set_lazy_resched___new(struct task_struct *p, bool lazy) __ksym __weak; + +static inline void scx_bpf_task_set_lazy_resched(struct task_struct *p, bool lazy) +{ + if (bpf_ksym_exists(scx_bpf_task_set_lazy_resched___new)) + scx_bpf_task_set_lazy_resched___new(p, lazy); +} + /* * v7.1: New scx_bpf_dsq_reenq() that allows re-enqueues on more DSQs. This * will eventually deprecate scx_bpf_reenqueue_local(). @@ -414,6 +498,12 @@ static inline bool __COMPAT_has_generic_reenq(void) return bpf_ksym_exists(scx_bpf_dsq_reenq___compat); } +/* v7.4: Add scx_bpf_dsq_reenq_task(). */ +static inline bool __COMPAT_has_dsq_reenq_task(void) +{ + return bpf_ksym_exists(scx_bpf_dsq_reenq_task); +} + /* * v6.19: The new void variant can be called from anywhere while the older v1 * variant can only be called from ops.cpu_release(). The double ___ prefixes on diff --git a/tools/sched_ext/include/scx/compat.h b/tools/sched_ext/include/scx/compat.h index 7c12df45fdba..07fb55e63c57 100644 --- a/tools/sched_ext/include/scx/compat.h +++ b/tools/sched_ext/include/scx/compat.h @@ -214,6 +214,7 @@ static inline bool __COMPAT_struct_has_field(const char *type, const char *field #define SCX_OPS_ALLOW_QUEUED_WAKEUP SCX_OPS_FLAG(SCX_OPS_ALLOW_QUEUED_WAKEUP) #define SCX_OPS_BUILTIN_IDLE_PER_NODE SCX_OPS_FLAG(SCX_OPS_BUILTIN_IDLE_PER_NODE) #define SCX_OPS_ALWAYS_ENQ_IMMED SCX_OPS_FLAG(SCX_OPS_ALWAYS_ENQ_IMMED) +#define SCX_OPS_ENQ_BLOCKED SCX_OPS_FLAG(SCX_OPS_ENQ_BLOCKED) #define SCX_PICK_IDLE_FLAG(name) __COMPAT_ENUM_OR_ZERO("scx_pick_idle_cpu_flags", #name) @@ -274,23 +275,23 @@ static inline long scx_hotplug_seq(void) * - v7.1: ops.sub_attach(), ops.sub_detach(), ops.sub_cgroup_id * - v7.3: ops.rescue_bandwidth_ppt, ops.rescue_quantum_us */ -#define __SCX_OPS_OPEN(__ops_name, __scx_name, __ops_struct) ({ \ +#define __SCX_OPS_OPEN(__ops_name, __scx_name, __ops_struct, __opts) ({ \ struct __scx_name *__oskel; \ \ SCX_BUG_ON(!__COMPAT_struct_has_field(__ops_struct, "dump"), \ __ops_struct ".dump() missing, kernel too old?"); \ \ - __oskel = __scx_name##__open(); \ + __oskel = __scx_name##__open_opts(__opts); \ SCX_BUG_ON(!__oskel, "Could not open " #__scx_name); \ __oskel->struct_ops.__ops_name->hotplug_seq = scx_hotplug_seq(); \ SCX_ENUM_INIT(__oskel); \ __oskel; \ }) -#define SCX_OPS_OPEN(__ops_name, __scx_name) ({ \ +#define SCX_OPS_OPEN_OPTS(__ops_name, __scx_name, __opts) ({ \ struct __scx_name *__skel; \ \ - __skel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops"); \ + __skel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops", __opts); \ if (__skel->struct_ops.__ops_name->cgroup_set_bandwidth && \ !__COMPAT_struct_has_field("sched_ext_ops", "cgroup_set_bandwidth")) { \ fprintf(stderr, "WARNING: kernel doesn't support ops.cgroup_set_bandwidth()\n"); \ @@ -329,12 +330,29 @@ static inline long scx_hotplug_seq(void) __skel; \ }) +#define SCX_OPS_OPEN(__ops_name, __scx_name) \ + SCX_OPS_OPEN_OPTS(__ops_name, __scx_name, 0) + /* * Open a cid-form (struct sched_ext_ops_cid) skeleton. The cid form postdates * every op the load-time fix-ups above handle, so none of them apply. + * + * A cid-form scheduler registers through bpf_scx_reg_cid() rather than + * bpf_scx_reg(), so repoint common.bpf.h's prolog probe at it. Both the cid + * form and bpf_scx_reg_cid() appeared in v7.2, so a kernel that accepts this + * skeleton always has the symbol. */ +#define SCX_OPS_CID_OPEN_OPTS(__ops_name, __scx_name, __opts) ({ \ + struct __scx_name *__cskel; \ + \ + __cskel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops_cid", __opts); \ + bpf_program__set_attach_target(__cskel->progs.scx_lib_init_probe, 0, \ + "bpf_scx_reg_cid"); \ + __cskel; \ +}) + #define SCX_OPS_CID_OPEN(__ops_name, __scx_name) \ - __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops_cid") + SCX_OPS_CID_OPEN_OPTS(__ops_name, __scx_name, 0) /* * Associate non-struct_ops BPF programs with the scheduler's struct_ops map so diff --git a/tools/sched_ext/include/scx/const-defs.h b/tools/sched_ext/include/scx/const-defs.h new file mode 100644 index 000000000000..4c8f45756b51 --- /dev/null +++ b/tools/sched_ext/include/scx/const-defs.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 Tejun Heo <tj@kernel.org> */ +#pragma once + +/* + * Constants shared across the BPF headers, the lib and the schedulers. + * Freestanding so that scheduler interface headers consumed by bindgen can + * include it. Rust code cannot, so rust/scx_arena/scx_arena/src/lib.rs mirrors + * the values. Keep them in sync. + */ +enum scx_const_defs { + /* mirrors the kernel's per-arch L1_CACHE_SHIFT */ +#if defined(__TARGET_ARCH_s390) || defined(__s390x__) + SCX_CACHELINE_SIZE = 256, +#elif defined(__TARGET_ARCH_powerpc) || defined(__powerpc64__) + SCX_CACHELINE_SIZE = 128, +#else + SCX_CACHELINE_SIZE = 64, +#endif +}; diff --git a/tools/sched_ext/include/scx/enum_defs.autogen.h b/tools/sched_ext/include/scx/enum_defs.autogen.h index 63b6b14b19bd..4fa82033b99c 100644 --- a/tools/sched_ext/include/scx/enum_defs.autogen.h +++ b/tools/sched_ext/include/scx/enum_defs.autogen.h @@ -85,10 +85,12 @@ #define HAVE_SCX_ENQ_HEAD #define HAVE_SCX_ENQ_CPU_SELECTED #define HAVE_SCX_ENQ_PREEMPT +#define HAVE_SCX_ENQ_PREEMPT_LAZY #define HAVE_SCX_ENQ_IMMED #define HAVE_SCX_ENQ_RESCUE #define HAVE_SCX_ENQ_REENQ #define HAVE_SCX_ENQ_LAST +#define HAVE_SCX_ENQ_BLOCKED #define HAVE___SCX_ENQ_INTERNAL_MASK #define HAVE_SCX_ENQ_CLEAR_OPSS #define HAVE_SCX_ENQ_DSQ_PRIQ @@ -105,6 +107,7 @@ #define HAVE_SCX_TASK_SUB_INIT #define HAVE_SCX_TASK_IMMED #define HAVE_SCX_TASK_PROTECTED +#define HAVE_SCX_TASK_RUN_TRACKED #define HAVE_SCX_TASK_STATE_SHIFT #define HAVE_SCX_TASK_STATE_BITS #define HAVE_SCX_TASK_STATE_MASK @@ -122,6 +125,7 @@ #define HAVE_SCX_TASK_REENQ_IMMED #define HAVE_SCX_TASK_REENQ_PREEMPTED #define HAVE_SCX_TASK_REENQ_CAP +#define HAVE_SCX_TASK_REENQ_PROXY #define HAVE_SCX_TASK_CURSOR #define HAVE_SCX_ECODE_RSN_HOTPLUG #define HAVE_SCX_ECODE_RSN_CGROUP_OFFLINE @@ -149,6 +153,8 @@ #define HAVE_SCX_KICK_IDLE #define HAVE_SCX_KICK_PREEMPT #define HAVE_SCX_KICK_WAIT +#define HAVE_SCX_KICK_PREEMPT_LAZY +#define HAVE_SCX_KICK_ALL_FLAGS #define HAVE_SCX_OPI_BEGIN #define HAVE_SCX_OPI_NORMAL_BEGIN #define HAVE_SCX_OPI_NORMAL_END @@ -164,6 +170,8 @@ #define HAVE_SCX_OPS_BUILTIN_IDLE_PER_NODE #define HAVE_SCX_OPS_ALWAYS_ENQ_IMMED #define HAVE_SCX_OPS_TID_TO_TASK +#define HAVE_SCX_OPS_LAZY_RESCHED +#define HAVE_SCX_OPS_ENQ_BLOCKED #define HAVE_SCX_OPS_ALL_FLAGS #define HAVE___SCX_OPS_INTERNAL_MASK #define HAVE_SCX_OPS_HAS_CPU_PREEMPT @@ -191,6 +199,8 @@ #define HAVE_SCX_RQ_BAL_CB_PENDING #define HAVE_SCX_RQ_SUB_IDLE_RENOTIFY #define HAVE_SCX_RQ_ROOT_IDLE_RENOTIFY +#define HAVE_SCX_RQ_PROXY_RETRY +#define HAVE_SCX_RQ_PROXY_TICK #define HAVE_SCX_RQ_IN_WAKEUP #define HAVE_SCX_RQ_IN_DISPATCH #define HAVE_SCX_SCHED_PCPU_BYPASSING diff --git a/tools/sched_ext/include/scx/enums.autogen.bpf.h b/tools/sched_ext/include/scx/enums.autogen.bpf.h index 7268131010de..405a3e6b8f33 100644 --- a/tools/sched_ext/include/scx/enums.autogen.bpf.h +++ b/tools/sched_ext/include/scx/enums.autogen.bpf.h @@ -1,9 +1,9 @@ /* - * WARNING: This file is autogenerated from scripts/gen_enums.py. If you would - * like to access an enum that is currently missing, add it to the script - * and run it from the root directory to update this file. + * Definitions for SCX constants. Keep this table in sync + * with rust/scx_utils/src/enums.rs. */ + const volatile u64 __SCX_OPS_NAME_LEN __weak; #define SCX_OPS_NAME_LEN __SCX_OPS_NAME_LEN @@ -22,6 +22,9 @@ const volatile u64 __SCX_RQ_CAN_STOP_TICK __weak; const volatile u64 __SCX_RQ_BAL_PENDING __weak; #define SCX_RQ_BAL_PENDING __SCX_RQ_BAL_PENDING +const volatile u64 __SCX_RQ_BAL_KEEP __weak; +#define SCX_RQ_BAL_KEEP __SCX_RQ_BAL_KEEP + const volatile u64 __SCX_RQ_BYPASSING __weak; #define SCX_RQ_BYPASSING __SCX_RQ_BYPASSING @@ -112,6 +115,12 @@ const volatile u64 __SCX_KICK_PREEMPT __weak; const volatile u64 __SCX_KICK_WAIT __weak; #define SCX_KICK_WAIT __SCX_KICK_WAIT +const volatile u64 __SCX_KICK_PREEMPT_LAZY __weak; +#define SCX_KICK_PREEMPT_LAZY __SCX_KICK_PREEMPT_LAZY + +const volatile u64 __SCX_KICK_ALL_FLAGS __weak; +#define SCX_KICK_ALL_FLAGS __SCX_KICK_ALL_FLAGS + const volatile u64 __SCX_ENQ_WAKEUP __weak; #define SCX_ENQ_WAKEUP __SCX_ENQ_WAKEUP @@ -121,12 +130,18 @@ const volatile u64 __SCX_ENQ_HEAD __weak; const volatile u64 __SCX_ENQ_PREEMPT __weak; #define SCX_ENQ_PREEMPT __SCX_ENQ_PREEMPT +const volatile u64 __SCX_ENQ_PREEMPT_LAZY __weak; +#define SCX_ENQ_PREEMPT_LAZY __SCX_ENQ_PREEMPT_LAZY + const volatile u64 __SCX_ENQ_IMMED __weak; #define SCX_ENQ_IMMED __SCX_ENQ_IMMED const volatile u64 __SCX_ENQ_RESCUE __weak; #define SCX_ENQ_RESCUE __SCX_ENQ_RESCUE +const volatile u64 __SCX_ENQ_BLOCKED __weak; +#define SCX_ENQ_BLOCKED __SCX_ENQ_BLOCKED + const volatile u64 __SCX_ENQ_REENQ __weak; #define SCX_ENQ_REENQ __SCX_ENQ_REENQ diff --git a/tools/sched_ext/include/scx/enums.autogen.h b/tools/sched_ext/include/scx/enums.autogen.h index e61632654517..7ec2e5e41cb2 100644 --- a/tools/sched_ext/include/scx/enums.autogen.h +++ b/tools/sched_ext/include/scx/enums.autogen.h @@ -1,7 +1,8 @@ +#pragma once + /* - * WARNING: This file is autogenerated from scripts/gen_enums.py. If you would - * like to access an enum that is currently missing, add it to the script - * and run it from the root directory to update this file. + * BTF enum lookups for SCX constants. Keep this table in sync + * with rust/scx_utils/src/enums.rs. */ #define SCX_ENUM_INIT(skel) do { \ @@ -11,6 +12,7 @@ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_ONLINE); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CAN_STOP_TICK); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BAL_PENDING); \ + SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BAL_KEEP); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BYPASSING); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CLK_VALID); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_IN_WAKEUP); \ @@ -32,20 +34,24 @@ SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_STATE_BITS); \ SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_STATE_MASK); \ SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_CURSOR); \ - SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_NONE); \ - SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_INIT); \ - SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_READY); \ - SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_ENABLED); \ - SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_NR_STATES); \ + SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_NONE); \ + SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_INIT); \ + SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_READY); \ + SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_ENABLED); \ + SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_NR_STATES); \ SCX_ENUM_SET(skel, scx_ent_dsq_flags, SCX_TASK_DSQ_ON_PRIQ); \ SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_IDLE); \ SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_PREEMPT); \ SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_WAIT); \ + SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_PREEMPT_LAZY); \ + SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_ALL_FLAGS); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_WAKEUP); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_HEAD); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_PREEMPT); \ + SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_PREEMPT_LAZY); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_IMMED); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_RESCUE); \ + SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_BLOCKED); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_REENQ); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_LAST); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_CLEAR_OPSS); \ diff --git a/tools/sched_ext/include/scx/enums_abi.autogen.h b/tools/sched_ext/include/scx/enums_abi.autogen.h index d53899764f5a..1186daa9cb14 100644 --- a/tools/sched_ext/include/scx/enums_abi.autogen.h +++ b/tools/sched_ext/include/scx/enums_abi.autogen.h @@ -97,10 +97,12 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_enq_flags", "SCX_ENQ_HEAD", 0x10000LLU }, { "scx_enq_flags", "SCX_ENQ_CPU_SELECTED", 0x100000LLU }, { "scx_enq_flags", "SCX_ENQ_PREEMPT", 0x100000000LLU }, + { "scx_enq_flags", "SCX_ENQ_PREEMPT_LAZY", 0x800000000LLU }, { "scx_enq_flags", "SCX_ENQ_IMMED", 0x200000000LLU }, { "scx_enq_flags", "SCX_ENQ_RESCUE", 0x400000000LLU }, { "scx_enq_flags", "SCX_ENQ_REENQ", 0x10000000000LLU }, { "scx_enq_flags", "SCX_ENQ_LAST", 0x20000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_BLOCKED", 0x40000000000LLU }, { "scx_enq_flags", "__SCX_ENQ_INTERNAL_MASK", 0xff00000000000000LLU }, { "scx_enq_flags", "SCX_ENQ_CLEAR_OPSS", 0x100000000000000LLU }, { "scx_enq_flags", "SCX_ENQ_DSQ_PRIQ", 0x200000000000000LLU }, @@ -117,6 +119,7 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_ent_flags", "SCX_TASK_SUB_INIT", 0x10LLU }, { "scx_ent_flags", "SCX_TASK_IMMED", 0x20LLU }, { "scx_ent_flags", "SCX_TASK_PROTECTED", 0x40LLU }, + { "scx_ent_flags", "SCX_TASK_RUN_TRACKED", 0x80LLU }, { "scx_ent_flags", "SCX_TASK_STATE_SHIFT", 0x8LLU }, { "scx_ent_flags", "SCX_TASK_STATE_BITS", 0x3LLU }, { "scx_ent_flags", "SCX_TASK_STATE_MASK", 0x700LLU }, @@ -134,6 +137,7 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_ent_flags", "SCX_TASK_REENQ_IMMED", 0x2000LLU }, { "scx_ent_flags", "SCX_TASK_REENQ_PREEMPTED", 0x3000LLU }, { "scx_ent_flags", "SCX_TASK_REENQ_CAP", 0x4000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_PROXY", 0x5000LLU }, { "scx_ent_flags", "SCX_TASK_CURSOR", 0xffffffff80000000LLU }, { "scx_exit_code", "SCX_ECODE_RSN_HOTPLUG", 0x100000000LLU }, { "scx_exit_code", "SCX_ECODE_RSN_CGROUP_OFFLINE", 0x200000000LLU }, @@ -161,6 +165,8 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_kick_flags", "SCX_KICK_IDLE", 0x1LLU }, { "scx_kick_flags", "SCX_KICK_PREEMPT", 0x2LLU }, { "scx_kick_flags", "SCX_KICK_WAIT", 0x4LLU }, + { "scx_kick_flags", "SCX_KICK_PREEMPT_LAZY", 0x8LLU }, + { "scx_kick_flags", "SCX_KICK_ALL_FLAGS", 0xfLLU }, { "scx_opi", "SCX_OPI_BEGIN", 0x0LLU }, { "scx_opi", "SCX_OPI_NORMAL_BEGIN", 0x0LLU }, { "scx_opi", "SCX_OPI_NORMAL_END", 0x21LLU }, @@ -176,7 +182,9 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_ops_flags", "SCX_OPS_BUILTIN_IDLE_PER_NODE", 0x40LLU }, { "scx_ops_flags", "SCX_OPS_ALWAYS_ENQ_IMMED", 0x80LLU }, { "scx_ops_flags", "SCX_OPS_TID_TO_TASK", 0x100LLU }, - { "scx_ops_flags", "SCX_OPS_ALL_FLAGS", 0x1ffLLU }, + { "scx_ops_flags", "SCX_OPS_LAZY_RESCHED", 0x200LLU }, + { "scx_ops_flags", "SCX_OPS_ENQ_BLOCKED", 0x400LLU }, + { "scx_ops_flags", "SCX_OPS_ALL_FLAGS", 0x7ffLLU }, { "scx_ops_flags", "__SCX_OPS_INTERNAL_MASK", 0xff00000000000000LLU }, { "scx_ops_flags", "SCX_OPS_HAS_CPU_PREEMPT", 0x100000000000000LLU }, { "scx_ops_state", "SCX_OPSS_NONE", 0x0LLU }, @@ -203,6 +211,8 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_rq_flags", "SCX_RQ_BAL_CB_PENDING", 0x40LLU }, { "scx_rq_flags", "SCX_RQ_SUB_IDLE_RENOTIFY", 0x80LLU }, { "scx_rq_flags", "SCX_RQ_ROOT_IDLE_RENOTIFY", 0x100LLU }, + { "scx_rq_flags", "SCX_RQ_PROXY_RETRY", 0x200LLU }, + { "scx_rq_flags", "SCX_RQ_PROXY_TICK", 0x400LLU }, { "scx_rq_flags", "SCX_RQ_IN_WAKEUP", 0x10000LLU }, { "scx_rq_flags", "SCX_RQ_IN_DISPATCH", 0x20000LLU }, { "scx_sched_pcpu_flags", "SCX_SCHED_PCPU_BYPASSING", 0x1LLU }, diff --git a/tools/sched_ext/include/scx/features.bpf.h b/tools/sched_ext/include/scx/features.bpf.h new file mode 100644 index 000000000000..992f4a74b277 --- /dev/null +++ b/tools/sched_ext/include/scx/features.bpf.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Kernel features the loader probes before load and hands to the BPF side in + * one rodata word, so that a program can carry both the path a feature enables + * and its fallback while the verifier prunes the one the running kernel cannot + * take. scx_utils sets the word in the ops open macros. The bits are mirrored + * in rust/scx_utils/src/compat.rs. + * + * Included by scx/common.bpf.h; don't include directly. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo <tj@kernel.org> + */ +#ifndef __SCX_FEATURES_BPF_H +#define __SCX_FEATURES_BPF_H + +enum scx_lib_feature { + /* the JIT lowers fetching AND, OR and XOR on arena pointers */ + SCX_LIB_FEAT_ARENA_FETCH_BITOPS = 1ULL << 0, +}; + +const volatile u64 scx_lib_features __weak; + +static __always_inline bool scx_lib_has(u64 feat) +{ + return scx_lib_features & feat; +} + +#endif /* __SCX_FEATURES_BPF_H */ diff --git a/tools/sched_ext/include/scx/user_exit_info.h b/tools/sched_ext/include/scx/user_exit_info.h index 56a02b549aef..cf27ac2447ef 100644 --- a/tools/sched_ext/include/scx/user_exit_info.h +++ b/tools/sched_ext/include/scx/user_exit_info.h @@ -52,6 +52,7 @@ enum scx_exit_code { /* Reasons */ SCX_ECODE_RSN_HOTPLUG = 1LLU << 32, + SCX_ECODE_RSN_CGROUP_OFFLINE = 2LLU << 32, /* Actions */ SCX_ECODE_ACT_RESTART = 1LLU << 48, diff --git a/tools/sched_ext/scx_flatcg.bpf.c b/tools/sched_ext/scx_flatcg.bpf.c index 454ebb820c5e..5e6abd0bccb3 100644 --- a/tools/sched_ext/scx_flatcg.bpf.c +++ b/tools/sched_ext/scx_flatcg.bpf.c @@ -144,7 +144,7 @@ static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) cgc_a = container_of(a, struct cgv_node, rb_node); cgc_b = container_of(b, struct cgv_node, rb_node); - return cgc_a->cvtime < cgc_b->cvtime; + return time_before(cgc_a->cvtime, cgc_b->cvtime); } static struct fcg_cpu_ctx *find_cpu_ctx(void) diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 2f3653199842..e8e451be862d 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -369,6 +369,22 @@ static u64 needs_immed(s32 cid) return qa.cid_shared[cid] ? SCX_ENQ_IMMED : 0; } +static void dispatch_to_rescue(struct task_struct *p, task_ctx_t *taskc, + u64 enq_flags) +{ + u32 cid = cmask_next_set_wrap(&taskc->cpus_allowed, 0); + + if (cid >= scx_bpf_nr_cids()) { + scx_bpf_error("task %d has no allowed cid", p->pid); + return; + } + + taskc->force_local = false; + __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); + scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | cid, slice_ns, + enq_flags | SCX_ENQ_RESCUE); +} + /* first cid this node does NOT hold for fault injection, -1 if none */ static s32 first_unavail_cid(void) { @@ -442,6 +458,9 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) */ taskc->core_sched_seq = qa.core_sched_tail_seqs[idx]++; + if (enq_flags & SCX_ENQ_BLOCKED) + __sync_fetch_and_add(&qa.nr_enq_blocked, 1); + /* * A task of ours that can run on none of our self cids - the parent * didn't grant them or we delegated them to children - would starve in @@ -451,18 +470,64 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) * If we hold ENQ on that cid it runs. Otherwise the kernel diverts the * task to its rescue path. IMMED would turn the insert into a legal * placement on a time-shared cid and the kernel would bounce it back - * here instead of rescuing it. + * here instead of rescuing it. Do this before the blocked-donor fast + * paths, which also require an eligible self cid to make progress. */ if (!cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) { - s32 c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); + dispatch_to_rescue(p, taskc, enq_flags); + return; + } - if (c >= 0 && c < scx_bpf_nr_cids()) { - taskc->force_local = false; - __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); - scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | c, slice_ns, - enq_flags | SCX_ENQ_RESCUE); + /* + * SCX_OPS_ALWAYS_ENQ_IMMED makes the local insertion below implicitly + * carry SCX_ENQ_IMMED. If the CPU can't run the blocked donor immediately, + * the core returns it through ops.enqueue() with SCX_ENQ_REENQ. Inserting + * it into the same local DSQ would repeat the IMMED handback until the + * scheduler is ejected. Move reenqueued blocked donors to the shared DSQ, + * which doesn't carry SCX_ENQ_IMMED, so another CPU can consume them. + */ + if ((enq_flags & (SCX_ENQ_BLOCKED | SCX_ENQ_REENQ)) == + (SCX_ENQ_BLOCKED | SCX_ENQ_REENQ)) { + taskc->force_local = false; + scx_bpf_dsq_insert(p, SHARED_DSQ, 0, enq_flags); + cid = cmask_next_and2_set_wrap(&taskc->cpus_allowed, + &qa.idle_cids.mask, + &qa.self_cids.mask, 0); + if (cid < scx_bpf_nr_cids()) + scx_bpf_kick_cid(cid, SCX_KICK_IDLE); + return; + } + + /* + * Insert a blocked mutex donor at the head of an eligible local DSQ with + * a fresh slice and %SCX_ENQ_PREEMPT, requesting an immediate reschedule. + * The test above guarantees that cpus_allowed intersects self_cids, but + * the donor's current cid may have been delegated to a child. Search the + * intersection starting at the current cid, preserving it when qmap still + * holds it and wrapping to another eligible self cid otherwise. + * + * A self cid may be held exclusively with SCX_CAP_ENQ or time-shared with + * only SCX_CAP_ENQ_IMMED. Add needs_immed() so either kind can accept the + * local insertion instead of rejecting and reenqueuing the donor for a + * capability miss. Once selected, the core proxy-exec path can run the + * mutex owner using the donor's scheduling context. + * + * This policy is intentionally unfair and can strongly prioritize tasks + * using contended mutexes; scx_qmap is a demonstration scheduler and + * this behavior makes proxy-exec support easy to observe. + */ + if (enq_flags & SCX_ENQ_BLOCKED) { + cid = cmask_next_and_set_wrap(&taskc->cpus_allowed, + &qa.self_cids.mask, + scx_bpf_task_cid(p)); + if (cid >= scx_bpf_nr_cids()) { + /* self_cids may have changed since the intersection test */ + dispatch_to_rescue(p, taskc, enq_flags); return; } + scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | cid, slice_ns, + enq_flags | needs_immed(cid) | SCX_ENQ_PREEMPT); + return; } /* @@ -587,9 +652,34 @@ static void update_core_sched_head_seq(struct task_struct *p) } /* + * Send @p, stranded on the DSQ @it iterates, to the kernel rescue path. To + * exercise both kfuncs, move tasks with an even pid there with + * scx_bpf_dsq_move() and re-enqueue the others with scx_bpf_dsq_reenq_task(), + * which takes them through qmap_enqueue()'s rescue. The choice is per task so + * that a marked task isn't moved by another scan before the re-enqueue reaches + * it. + */ +static __always_inline void rescue_stranded(struct bpf_iter_scx_dsq *it, + struct task_struct *p, task_ctx_t *taskc) +{ + s32 c; + + if (p->pid & 1) { + scx_bpf_dsq_reenq_task(it, p); + return; + } + + c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); + if (c >= 0 && c < scx_bpf_nr_cids()) { + __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); + scx_bpf_dsq_move(it, p, SCX_DSQ_LOCAL_ON | c, SCX_ENQ_RESCUE); + } +} + +/* * One pass over SHARED_DSQ: rescue stranded tasks and boost highpri ones. A * task whose cids were lost while it was queued in the fifos would strand on - * SHARED_DSQ, which is consumed only on self cids it can't run on - move it to + * SHARED_DSQ, which is consumed only on self cids it can't run on - send it to * the kernel rescue path. One whose cids were lost after the highpri cull is * likewise rescued out of HIGHPRI_DSQ below. * @@ -598,9 +688,9 @@ static void update_core_sched_head_seq(struct task_struct *p) * consuming them first. This makes minor difference only when dsp_batch is * larger than 1. * - * scx_bpf_dsq_move[_vtime]() are allowed both from ops.dispatch() and - * non-rq-lock holding BPF programs. As demonstration, this function is called - * from qmap_dispatch() and monitor_timerfn(). + * scx_bpf_dsq_move[_vtime]() and scx_bpf_dsq_reenq_task() are allowed both from + * ops.dispatch() and non-rq-lock holding BPF programs. As demonstration, this + * function is called from qmap_dispatch() and monitor_timerfn(). */ static bool scan_shared_dsq(bool from_timer) { @@ -612,19 +702,13 @@ static bool scan_shared_dsq(bool from_timer) bpf_for_each(scx_dsq, p, SHARED_DSQ, 0) { static u64 highpri_seq; task_ctx_t *taskc; - s32 c; if (!(taskc = lookup_task_ctx(p))) return false; /* stranded? rescue - it can't be dispatched here either way */ if (!cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) { - c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); - if (c >= 0 && c < scx_bpf_nr_cids()) { - __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); - scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c, - SCX_ENQ_RESCUE); - } + rescue_stranded(BPF_FOR_EACH_ITER, p, taskc); continue; } @@ -658,19 +742,12 @@ static bool scan_shared_dsq(bool from_timer) &qa.usable_cids.mask, this_cid + 1); if (cid >= nr_cids) { - s32 c; - /* self cids lack caps in effect yet, leave it queued */ if (cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) continue; /* stranded after the cull - rescue it from here */ - c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); - if (c >= 0 && c < nr_cids) { - __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); - scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c, - SCX_ENQ_RESCUE); - } + rescue_stranded(BPF_FOR_EACH_ITER, p, taskc); continue; } @@ -1408,7 +1485,8 @@ __noinline void compute_partition(void) /* find out the cids we hold */ scx_bpf_sub_caps(0, SCX_CAP_ENQ, &qa.held_excl.mask); scx_bpf_sub_caps(0, SCX_CAP_ENQ_IMMED, &qa.held_shared.mask); - cmask_andnot(&qa.held_shared.mask, &qa.held_excl.mask); /* held only as ENQ_IMMED */ + /* held only as ENQ_IMMED */ + cmask_andnot(&qa.held_shared.mask, &qa.held_shared.mask, &qa.held_excl.mask); qa.part.nr_shared = 0; qa.part.nr_rr = 0; @@ -1563,8 +1641,7 @@ static __noinline void account_alloc(void) */ static void refresh_usable(void) { - cmask_copy(&qa.usable_scratch.mask, &qa.self_cids.mask); - cmask_and(&qa.usable_scratch.mask, &qa.avail_cids.mask); + cmask_and(&qa.usable_scratch.mask, &qa.self_cids.mask, &qa.avail_cids.mask); cmask_copy(&qa.usable_cids.mask, &qa.usable_scratch.mask); } @@ -1643,10 +1720,8 @@ __noinline void apply_partition(void) if (!cgid) continue; - cmask_copy(&qa.to_revoke_cids.mask, &ssc->prev_granted.mask); - cmask_andnot(&qa.to_revoke_cids.mask, &ssc->granted_cids.mask); - cmask_copy(&qa.to_grant_cids.mask, &ssc->granted_cids.mask); - cmask_andnot(&qa.to_grant_cids.mask, &ssc->prev_granted.mask); + cmask_andnot(&qa.to_revoke_cids.mask, &ssc->prev_granted.mask, &ssc->granted_cids.mask); + cmask_andnot(&qa.to_grant_cids.mask, &ssc->granted_cids.mask, &ssc->prev_granted.mask); scx_bpf_sub_revoke(cgid, SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, &qa.prev_rr_cids.mask); @@ -1719,13 +1794,90 @@ static void redistribute(void) } /* - * Userspace pokes this (PROG_RUN) to bring alloc_ns[] current before reading - * it for the stats display. Skipping when the partition guard is held is - * fine - alloc_ts is untouched, so the elapsed time is charged next time. + * Owner id for a @sched value of ops.sub_cid_sched_updated(). A child is + * attached before its first task runs and its tasks are re-homed before it + * detaches, so a child's cgroup id always has its slot. + */ +static s32 cid_sched_owner(u64 sched) +{ + s32 i; + + if (sched == SCX_CID_SCHED_NONE) + return CID_NONE; + if (sched == SCX_CID_SCHED_SELF) + return CID_SELF; + bpf_for(i, 0, MAX_SUB_SCHEDS) + if (qa.sub_sched_ctxs[i].cgroup_id == sched) + return i; + return CID_NONE; +} + +/* used_ns[] is summed from every cpu, hence the atomic adds */ +static void cid_sched_charge(s32 cid, u64 now) +{ + s32 owner = qa.cid_sched[cid]; + u64 delta = now - qa.cid_sched_since[cid]; + + if (owner >= 0 && owner < MAX_SUB_SCHEDS) + __sync_fetch_and_add(&qa.used_ns[owner], delta); + else if (owner == CID_SELF) + __sync_fetch_and_add(&qa.self_used_ns, delta); + qa.cid_sched_since[cid] = now; +} + +void BPF_STRUCT_OPS(qmap_sub_cid_sched_updated, s32 cid, u64 sched) +{ + if (cid < 0 || cid >= SCX_QMAP_MAX_CPUS) + return; + + cid_sched_charge(cid, bpf_ktime_get_ns()); + qa.cid_sched[cid] = cid_sched_owner(sched); +} + +/* + * Snapshot the used time for the stats display: the closed intervals plus the + * ones still open. The reads race the notifications on other cpus, so an + * interval closing in between can be missing from one snapshot or counted in + * two. The next snapshot evens it out and the display floors a negative + * difference at zero. + */ +static void snapshot_used(void) +{ + u64 now = bpf_ktime_get_ns(); + s32 nr_cids = qa.nr_cids; + s32 cid, i; + + if (nr_cids < 0 || nr_cids > SCX_QMAP_MAX_CPUS) + return; + + bpf_for(i, 0, MAX_SUB_SCHEDS) + qa.used_snap_ns[i] = qa.used_ns[i]; + qa.self_used_snap_ns = qa.self_used_ns; + + bpf_for(cid, 0, nr_cids) { + s32 owner = qa.cid_sched[cid]; + u64 since = qa.cid_sched_since[cid]; + + /* restarted after @now by a notification on another cpu */ + if (since > now) + continue; + if (owner >= 0 && owner < MAX_SUB_SCHEDS) + qa.used_snap_ns[owner] += now - since; + else if (owner == CID_SELF) + qa.self_used_snap_ns += now - since; + } +} + +/* + * Userspace pokes this (PROG_RUN) to bring alloc_ns[] and the used snapshot + * current before reading them for the stats display. Skipping the alloc part + * when the partition guard is held is fine - alloc_ts is untouched, so the + * elapsed time is charged next time. */ SEC("syscall") int flush_alloc(void *ctx) { + snapshot_used(); if (part_try_start()) { account_alloc(); part_end(); @@ -1877,6 +2029,9 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init) /* cache the cid count, trusted to be <= SCX_QMAP_MAX_CPUS hereafter */ qa.nr_cids = nr_cids; + bpf_for(i, 0, nr_cids) + qa.cid_sched[i] = CID_NONE; + /* cmasks are embedded in qa, so they only need initializing */ cmask_init(&qa.idle_cids.mask, 0, nr_cids); cmask_init(&qa.rr_cids.mask, 0, nr_cids); @@ -1892,7 +2047,7 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init) scx_bpf_sub_caps(0, SCX_CAP_ENQ, &qa.held_excl.mask); scx_bpf_sub_caps(0, SCX_CAP_ENQ_IMMED, &qa.held_shared.mask); - cmask_andnot(&qa.held_shared.mask, &qa.held_excl.mask); + cmask_andnot(&qa.held_shared.mask, &qa.held_shared.mask, &qa.held_excl.mask); bpf_for(i, 0, MAX_SUB_SCHEDS) { cmask_init(&qa.sub_sched_ctxs[i].granted_cids.mask, 0, nr_cids); @@ -2024,12 +2179,36 @@ s32 BPF_STRUCT_OPS(qmap_sub_attach, struct scx_sub_attach_args *args) void BPF_STRUCT_OPS(qmap_sub_detach, struct scx_sub_detach_args *args) { - s32 i; + u64 cgid = args->ops->sub_cgroup_id; + s32 nr_cids = qa.nr_cids; + s32 pos = qa.part.rr_pos; + u64 holder_cgid; + bool reset_pool; + s32 i, cid; + + if (nr_cids < 0 || nr_cids > SCX_QMAP_MAX_CPUS || pos < 0 || pos >= MAX_PARTS) { + scx_bpf_error("-ERANGE"); + return; + } + + /* + * The child's caps are gone without a report, see ops.sub_detach(). + * cpuperf targets persist until the next write. Reset the child's excl + * cids, and the pool unless another child holds it: that child's revoke + * report or its own detach resets it then. + */ + holder_cgid = qa.part.rr_slots[pos]; + reset_pool = !holder_cgid || holder_cgid == cgid; for (i = 0; i < MAX_SUB_SCHEDS; i++) { - if (qa.sub_sched_ctxs[i].cgroup_id != args->ops->sub_cgroup_id) + if (qa.sub_sched_ctxs[i].cgroup_id != cgid) continue; + bpf_arena_for(cid, 0, nr_cids) + if (cmask_test(cid, &qa.sub_sched_ctxs[i].granted_cids.mask) || + (reset_pool && cmask_test(cid, &qa.rr_cids.mask))) + scx_bpf_cidperf_set(cid, SCX_CPUPERF_ONE); + qa.sub_sched_ctxs[i].cgroup_id = 0; qa.sub_sched_ctxs[i].weight = 100; cmask_init(&qa.sub_sched_ctxs[i].granted_cids.mask, 0, qa.nr_cids); @@ -2065,6 +2244,16 @@ void BPF_STRUCT_OPS(qmap_sub_ecaps_updated, s32 cid, u64 before, u64 after) execute_partition(); } +void BPF_STRUCT_OPS(qmap_sub_child_ecaps_updated, u64 cgroup_id, s32 cid, u64 before, + u64 after) +{ + __sync_fetch_and_add(&qa.nr_child_ecaps, 1); + + /* a child's last target must not outlive its PERF cap */ + if ((before & ~after) & SCX_CAP_PERF) + scx_bpf_cidperf_set(cid, SCX_CPUPERF_ONE); +} + SCX_OPS_CID_DEFINE(qmap_ops, .flags = SCX_OPS_ENQ_EXITING | SCX_OPS_TID_TO_TASK, .select_cid = (void *)qmap_select_cid, @@ -2088,6 +2277,8 @@ SCX_OPS_CID_DEFINE(qmap_ops, .sub_detach = (void *)qmap_sub_detach, .sub_caps_updated = (void *)qmap_sub_caps_updated, .sub_ecaps_updated = (void *)qmap_sub_ecaps_updated, + .sub_cid_sched_updated = (void *)qmap_sub_cid_sched_updated, + .sub_child_ecaps_updated = (void *)qmap_sub_child_ecaps_updated, .init_cids = (void *)qmap_init_cids, .init = (void *)qmap_init, .exit = (void *)qmap_exit, diff --git a/tools/sched_ext/scx_qmap.c b/tools/sched_ext/scx_qmap.c index 5bb5f687e579..17cf584a2a3d 100644 --- a/tools/sched_ext/scx_qmap.c +++ b/tools/sched_ext/scx_qmap.c @@ -46,7 +46,7 @@ const char help_fmt[] = "See the top-of-file comment in .bpf.c for the design.\n" "\n" "Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-b COUNT]\n" -" [-N COUNT] [-P] [-M] [-H] [-c CG_PATH] [-d PID] [-D LEN] [-S] [-p] [-I]\n" +" [-N COUNT] [-P] [-M] [-H] [-c CG_PATH] [-d PID] [-D LEN] [-S] [-p] [-I] [-X]\n" " [-F COUNT] [-i SEC] [-R MS] [-J MODE] [-v]\n" "\n" " -s SLICE_US Override slice duration\n" @@ -65,6 +65,7 @@ const char help_fmt[] = " -S Suppress qmap-specific debug dump\n" " -p Switch only tasks on SCHED_EXT policy instead of all\n" " -I Turn on SCX_OPS_ALWAYS_ENQ_IMMED\n" +" -X Turn on SCX_OPS_ENQ_BLOCKED\n" " -F COUNT IMMED stress: force every COUNT'th enqueue to a busy local DSQ (use with -I)\n" " -C MODE cid-override test (shuffle|bad-dup|bad-range|bad-mono)\n" " -i SEC Stats interval, seconds (default 5)\n" @@ -104,11 +105,15 @@ struct hier_prev { u64 alloc_ns[MAX_SUB_SCHEDS]; u64 self_alloc_ns; u64 alloc_window_ns; + u64 used_snap_ns[MAX_SUB_SCHEDS]; + u64 self_used_snap_ns; u64 nr_dsps[MAX_SUB_SCHEDS]; u64 nr_reenq_cap; u64 nr_reenq_immed; + u64 nr_enq_blocked; u64 nr_inject_attempts; u64 nr_rescue_dsp; + u64 nr_child_ecaps; }; /* current wall-clock time as "HH:MM:SS" for the startup and interval headers */ @@ -157,10 +162,24 @@ static void format_cid_ranges(struct qmap_arena *qa, s32 owner, char *buf, size_ strcpy(buf, "-"); } +/* + * Delta of a cumulative ns counter over the interval, as a fraction of the + * interval. The used snapshot can briefly run behind the previous one (see + * snapshot_used()), hence the floor. + */ +static double delta_ratio(u64 cur, u64 prev, double secs) +{ + s64 delta = cur - prev; + + return secs > 0 && delta > 0 ? delta / (secs * 1e9) : 0.0; +} + /* partition summary + one row per sched: weight, cpus, dispatch rate, cids */ static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cgid) { - char ranges[128], who[16]; + /* worst case, no ranges: a comma and up to six digits per cid */ + static char ranges[SCX_QMAP_MAX_CPUS * 7]; + char who[16]; const char *rr = "-"; double secs; u32 i; @@ -190,26 +209,39 @@ static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cg } format_cid_ranges(qa, CID_SHARED, ranges, sizeof(ranges)); - printf("hier : nsub=%llu excl=%u shared=%s rr=%s reenq cap/immed +%llu/+%llu inj=+%llu rescue=+%llu\n", + printf("hier : nsub=%llu excl=%u shared=%s rr=%s reenq cap/immed +%llu/+%llu blocked=+%llu inj=+%llu rescue=+%llu child_ecaps=+%llu\n", (unsigned long long)qa->nr_sub_scheds, qa->part.nr_excl, ranges, rr, (unsigned long long)(qa->nr_reenq_cap - prev->nr_reenq_cap), (unsigned long long)(qa->nr_reenq_immed - prev->nr_reenq_immed), + (unsigned long long)(qa->nr_enq_blocked - prev->nr_enq_blocked), (unsigned long long)(qa->nr_inject_attempts - prev->nr_inject_attempts), - (unsigned long long)(qa->nr_rescue_dsp - prev->nr_rescue_dsp)); + (unsigned long long)(qa->nr_rescue_dsp - prev->nr_rescue_dsp), + (unsigned long long)(qa->nr_child_ecaps - prev->nr_child_ecaps)); prev->nr_reenq_cap = qa->nr_reenq_cap; prev->nr_reenq_immed = qa->nr_reenq_immed; + prev->nr_enq_blocked = qa->nr_enq_blocked; prev->nr_inject_attempts = qa->nr_inject_attempts; prev->nr_rescue_dsp = qa->nr_rescue_dsp; + prev->nr_child_ecaps = qa->nr_child_ecaps; - printf("hier : %-4s %10s %4s %6s %8s %s\n", - "", "cgroup", "w", "alloc", "disp/s", "cids"); + /* + * alloc is the cid-time the partition handed each participant, and used + * is the cid-time its tasks actually ran, per + * ops.sub_cid_sched_updated(). Both are in cpus over the window. + */ + printf("hier : %-4s %10s %4s %6s %6s %8s %s\n", + "", "cgroup", "w", "alloc", "used", "disp/s", "cids"); format_cid_ranges(qa, CID_SELF, ranges, sizeof(ranges)); - printf("hier : %-4s %10llu %4u %6.2f %8s %s\n", "self", + printf("hier : %-4s %10llu %4u %6.2f %6.2f %8s %s\n", "self", (unsigned long long)own_cgid, 100, - secs > 0 ? (qa->self_alloc_ns - prev->self_alloc_ns) / (secs * 1e9) : 0.0, + delta_ratio(qa->self_alloc_ns, prev->self_alloc_ns, secs), + delta_ratio(qa->self_used_snap_ns, prev->self_used_snap_ns, secs), "-", ranges); prev->self_alloc_ns = qa->self_alloc_ns; + /* used accrues without a window, so prev moves only with the window */ + if (secs > 0) + prev->self_used_snap_ns = qa->self_used_snap_ns; for (i = 0; i < MAX_SUB_SCHEDS; i++) { struct sub_sched_ctx *sc = &qa->sub_sched_ctxs[i]; @@ -219,12 +251,15 @@ static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cg snprintf(who, sizeof(who), "sub%u", i); format_cid_ranges(qa, i, ranges, sizeof(ranges)); - printf("hier : %-4s %10llu %4u %6.2f %8.1f %s\n", who, + printf("hier : %-4s %10llu %4u %6.2f %6.2f %8.1f %s\n", who, (unsigned long long)sc->cgroup_id, sc->weight, - secs > 0 ? (qa->alloc_ns[i] - prev->alloc_ns[i]) / (secs * 1e9) : 0.0, + delta_ratio(qa->alloc_ns[i], prev->alloc_ns[i], secs), + delta_ratio(qa->used_snap_ns[i], prev->used_snap_ns[i], secs), secs > 0 ? (sc->nr_dsps - prev->nr_dsps[i]) / secs : 0.0, ranges); prev->alloc_ns[i] = qa->alloc_ns[i]; + if (secs > 0) + prev->used_snap_ns[i] = qa->used_snap_ns[i]; prev->nr_dsps[i] = sc->nr_dsps; } } @@ -263,7 +298,7 @@ restart: skel->rodata->max_tasks = 16384; while ((opt = getopt(argc, argv, - "s:e:t:T:l:b:N:PMHc:d:D:SpIF:C:i:R:J:B:q:vh")) != -1) { + "s:e:t:T:l:b:N:PMHc:d:D:SpIXF:C:i:R:J:B:q:vh")) != -1) { switch (opt) { case 's': skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; @@ -324,6 +359,9 @@ restart: case 'I': skel->struct_ops.qmap_ops->flags |= SCX_OPS_ALWAYS_ENQ_IMMED; break; + case 'X': + skel->struct_ops.qmap_ops->flags |= SCX_OPS_ENQ_BLOCKED; + break; case 'F': skel->rodata->immed_stress_nth = strtoul(optarg, NULL, 0); break; diff --git a/tools/sched_ext/scx_qmap.h b/tools/sched_ext/scx_qmap.h index e95fffcf7b23..9e6eefb39735 100644 --- a/tools/sched_ext/scx_qmap.h +++ b/tools/sched_ext/scx_qmap.h @@ -163,6 +163,17 @@ struct qmap_arena { u64 alloc_ts; /* last accounting timestamp */ u64 alloc_window_ns; /* total accounted time, the alloc denominator */ + /* + * The per-cid fields are written only by that cid's notifications and + * read by flush_alloc() for the snapshot userspace displays. + */ + s32 cid_sched[SCX_QMAP_MAX_CPUS]; /* per cid: owner id of the sched running there */ + u64 cid_sched_since[SCX_QMAP_MAX_CPUS]; /* when that last changed */ + u64 used_ns[MAX_SUB_SCHEDS]; /* per child slot, closed intervals */ + u64 self_used_ns; + u64 used_snap_ns[MAX_SUB_SCHEDS]; /* used_ns[] plus the intervals open at the flush */ + u64 self_used_snap_ns; + /* bpf-internal cmasks (embedded, see struct qmap_cmask) */ struct qmap_cmask self_cids; /* cids this node runs its own tasks on */ struct qmap_cmask avail_cids; /* cids with caps in effect on the cpu */ @@ -181,8 +192,10 @@ struct qmap_arena { /* bpf -> userspace: stats */ u64 nr_reenq_cap; /* SCX_TASK_REENQ_CAP bounces */ u64 nr_reenq_immed; /* SCX_TASK_REENQ_IMMED bounces */ + u64 nr_enq_blocked; /* SCX_ENQ_BLOCKED dispatches */ u64 nr_inject_attempts; /* fault-injection: dispatches to an unheld cid */ u64 nr_rescue_dsp; /* SCX_ENQ_RESCUE dispatch attempts */ + u64 nr_child_ecaps; /* ops.sub_child_ecaps_updated() deliveries */ u32 inject_mode; /* fault-injection mode (QMAP_INJ_*) */ }; diff --git a/tools/testing/selftests/sched_ext/.gitignore b/tools/testing/selftests/sched_ext/.gitignore index ae5491a114c0..54a1fd2af713 100644 --- a/tools/testing/selftests/sched_ext/.gitignore +++ b/tools/testing/selftests/sched_ext/.gitignore @@ -4,3 +4,7 @@ !Makefile !.gitignore !config +!test_modules/ +!test_modules/scx_enq_blocked_test.c +!test_modules/Makefile +test_modules/*.mod.c diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile index af919a56a4f4..c5d3a2eaea7d 100644 --- a/tools/testing/selftests/sched_ext/Makefile +++ b/tools/testing/selftests/sched_ext/Makefile @@ -5,10 +5,12 @@ include ../../../scripts/Makefile.arch include ../../../scripts/Makefile.include TEST_GEN_PROGS := runner +TEST_GEN_MODS_DIR := test_modules # override lib.mk's default rules OVERRIDE_TARGETS := 1 include ../lib.mk +include ../cgroup/lib/libcgroup.mk CURDIR := $(abspath .) REPOROOT := $(abspath ../../../..) @@ -53,7 +55,7 @@ ifneq ($(wildcard $(GENHDR)),) GENFLAGS := -DHAVE_GENHDR endif -CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ +CFLAGS += -g -O2 -pthread -Wall -Werror $(GENFLAGS) \ -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -I$(TOOLSINCDIR) -I$(APIDIR) -I$(CURDIR)/include -I$(SCXTOOLSINCDIR) @@ -62,7 +64,7 @@ ifneq ($(LLVM),) CFLAGS += -Wno-unused-command-line-argument endif -LDFLAGS = -lelf -lz -lpthread -lzstd +LDFLAGS += -lelf -lz -lpthread -lzstd IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - </dev/null | \ grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__') @@ -153,7 +155,7 @@ $(INCLUDE_DIR)/%.bpf.skel.h: $(SCXOBJ_DIR)/%.bpf.o $(INCLUDE_DIR)/vmlinux.h $(BP override define CLEAN rm -rf $(OUTPUT_DIR) - rm -f $(TEST_GEN_PROGS) + rm -f $(TEST_GEN_PROGS) $(EXTRA_CLEAN) endef # Every testcase takes all of the BPF progs are dependencies by default. This @@ -162,10 +164,12 @@ endef all_test_bpfprogs := $(foreach prog,$(wildcard *.bpf.c),$(INCLUDE_DIR)/$(patsubst %.c,%.skel.h,$(prog))) auto-test-targets := \ + cgroup_nr_cpus \ create_dsq \ dequeue \ dequeue_iter \ dequeue_remote \ + enq_blocked \ enq_last_no_enq_fails \ ddsp_bogus_dsq_fail \ ddsp_vtimelocal_fail \ @@ -175,6 +179,7 @@ auto-test-targets := \ exit \ hotplug \ init_enable_count \ + kick \ maximal \ maybe_null \ minimal \ @@ -214,7 +219,7 @@ $(testcase-targets): $(SCXOBJ_DIR)/%.o: %.c $(SCXOBJ_DIR)/runner.o $(all_test_bp $(SCXOBJ_DIR)/util.o: util.c | $(SCXOBJ_DIR) $(CC) $(CFLAGS) -c $< -o $@ -$(OUTPUT)/runner: $(SCXOBJ_DIR)/runner.o $(SCXOBJ_DIR)/util.o $(BPFOBJ) $(testcase-targets) +$(OUTPUT)/runner: $(SCXOBJ_DIR)/runner.o $(SCXOBJ_DIR)/util.o $(BPFOBJ) $(LIBCGROUP_O) $(testcase-targets) @echo "$(testcase-targets)" $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) diff --git a/tools/testing/selftests/sched_ext/allowed_cpus.bpf.c b/tools/testing/selftests/sched_ext/allowed_cpus.bpf.c index 9dd72d0da29b..f14d7e5bef9c 100644 --- a/tools/testing/selftests/sched_ext/allowed_cpus.bpf.c +++ b/tools/testing/selftests/sched_ext/allowed_cpus.bpf.c @@ -147,23 +147,41 @@ void BPF_STRUCT_OPS(allowed_cpus_exit, struct scx_exit_info *ei) } struct task_cpu_arg { - pid_t pid; + u64 pid; + s64 custom_cpu; }; SEC("syscall") int select_cpu_from_user(struct task_cpu_arg *input) { struct task_struct *p; - int cpu; + struct bpf_cpumask *mask; + s32 cpu; p = bpf_task_from_pid(input->pid); if (!p) return -EINVAL; + mask = bpf_cpumask_create(); + if (!mask) { + bpf_task_release(p); + return -ENOMEM; + } + + /* A negative custom_cpu leaves the custom mask empty. */ + if (input->custom_cpu >= 0) + bpf_cpumask_set_cpu(input->custom_cpu, mask); + bpf_rcu_read_lock(); - cpu = scx_bpf_select_cpu_and(p, bpf_get_smp_processor_id(), 0, p->cpus_ptr, 0); + cpu = scx_bpf_select_cpu_and(p, bpf_get_smp_processor_id(), 0, + cast_mask(mask), 0); + if (cpu >= 0 && + (!bpf_cpumask_test_cpu(cpu, cast_mask(mask)) || + !bpf_cpumask_test_cpu(cpu, &p->cpus_mask))) + cpu = -ERANGE; bpf_rcu_read_unlock(); + bpf_cpumask_release(mask); bpf_task_release(p); return cpu; diff --git a/tools/testing/selftests/sched_ext/allowed_cpus.c b/tools/testing/selftests/sched_ext/allowed_cpus.c index 093f285ab4ba..76311728a6b1 100644 --- a/tools/testing/selftests/sched_ext/allowed_cpus.c +++ b/tools/testing/selftests/sched_ext/allowed_cpus.c @@ -2,7 +2,10 @@ /* * Copyright (c) 2025 Andrea Righi <arighi@nvidia.com> */ +#define _GNU_SOURCE #include <bpf/bpf.h> +#include <limits.h> +#include <sched.h> #include <scx/common.h> #include <sys/wait.h> #include <unistd.h> @@ -23,17 +26,19 @@ static enum scx_test_status setup(void **ctx) return SCX_TEST_PASS; } -static int test_select_cpu_from_user(const struct allowed_cpus *skel) +static int test_select_cpu_from_user(const struct allowed_cpus *skel, + const char *name, int custom_cpu, + bool expect_busy) { int fd, ret; - __u64 args[1]; + __s32 cpu; + __u64 args[] = { getpid(), (__u64)(__s64)custom_cpu }; LIBBPF_OPTS(bpf_test_run_opts, attr, .ctx_in = args, .ctx_size_in = sizeof(args), ); - args[0] = getpid(); fd = bpf_program__fd(skel->progs.select_cpu_from_user); if (fd < 0) return fd; @@ -42,29 +47,123 @@ static int test_select_cpu_from_user(const struct allowed_cpus *skel) if (ret < 0) return ret; - fprintf(stderr, "%s: CPU %d\n", __func__, attr.retval); + /* test_run returns the signed BPF result through an unsigned field. */ + cpu = (__s32)attr.retval; + if ((expect_busy && cpu != -EBUSY) || + (!expect_busy && cpu != -EBUSY && cpu != custom_cpu)) { + SCX_ERR("%s: unexpected CPU selection result %d", name, cpu); + return -EINVAL; + } return 0; } +/* Grow until the mask covers the kernel's CPU range, including offline CPUs. */ +static int alloc_affinity(cpu_set_t **mask, size_t *size) +{ + int nr_cpus = CPU_SETSIZE; + cpu_set_t *cpus; + int err; + + for (;;) { + *size = CPU_ALLOC_SIZE(nr_cpus); + cpus = CPU_ALLOC(nr_cpus); + if (!cpus) + return -ENOMEM; + CPU_ZERO_S(*size, cpus); + if (!sched_getaffinity(0, *size, cpus)) { + *mask = cpus; + return nr_cpus; + } + err = errno; + CPU_FREE(cpus); + if (err != EINVAL) + return -err; + if (nr_cpus > INT_MAX / 2) + return -EOVERFLOW; + nr_cpus *= 2; + } +} + static enum scx_test_status run(void *ctx) { struct allowed_cpus *skel = ctx; - struct bpf_link *link; + enum scx_test_status status = SCX_TEST_FAIL; + cpu_set_t *original = NULL, *pinned = NULL; + bool affinity_changed = false; + size_t size; + int first = -1, second = -1, cpu, nr_cpus; + struct bpf_link *link = NULL; + + nr_cpus = alloc_affinity(&original, &size); + if (nr_cpus < 0) { + SCX_ERR("Failed to get affinity (%d)", -nr_cpus); + goto out; + } + pinned = CPU_ALLOC(nr_cpus); + if (!pinned) { + SCX_ERR("Failed to allocate affinity mask"); + goto out; + } + for (cpu = 0; cpu < nr_cpus; cpu++) { + if (!CPU_ISSET_S(cpu, size, original)) + continue; + if (first < 0) { + first = cpu; + } else { + second = cpu; + break; + } + } + if (first < 0) { + SCX_ERR("No CPU in affinity mask"); + goto out; + } link = bpf_map__attach_struct_ops(skel->maps.allowed_cpus_ops); - SCX_FAIL_IF(!link, "Failed to attach scheduler"); - - /* Pick an idle CPU from user-space */ - SCX_FAIL_IF(test_select_cpu_from_user(skel), "Failed to pick idle CPU"); - - /* Just sleeping is fine, plenty of scheduling events happening */ + if (!link) { + SCX_ERR("Failed to attach scheduler"); + goto out; + } + + if (test_select_cpu_from_user(skel, "empty mask", -1, true)) + goto out; + + /* A legal candidate may be busy; selection need not succeed. */ + if (test_select_cpu_from_user(skel, "legal candidate", first, false)) + goto out; + + if (second >= 0) { + CPU_ZERO_S(size, pinned); + CPU_SET_S(first, size, pinned); + if (sched_setaffinity(0, size, pinned)) { + SCX_ERR("Failed to pin task (%d)", errno); + goto out; + } + affinity_changed = true; + if (test_select_cpu_from_user(skel, "disjoint masks", second, true)) + goto out; + } else { + fprintf(stderr, "Skipping disjoint masks: need two allowed CPUs\n"); + } + + /* Just sleeping is fine, plenty of scheduling events happening. */ sleep(1); - - SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_NONE)); + if (UEI_EXITED(skel, uei)) { + SCX_ERR("Scheduler exited unexpectedly"); + goto out; + } + status = SCX_TEST_PASS; + +out: + if (affinity_changed && sched_setaffinity(0, size, original)) { + SCX_ERR("Failed to restore affinity (%d)", errno); + status = SCX_TEST_FAIL; + } bpf_link__destroy(link); - - return SCX_TEST_PASS; + CPU_FREE(pinned); + CPU_FREE(original); + return status; } static void cleanup(void *ctx) diff --git a/tools/testing/selftests/sched_ext/cgroup_nr_cpus.bpf.c b/tools/testing/selftests/sched_ext/cgroup_nr_cpus.bpf.c new file mode 100644 index 000000000000..841c83abb41b --- /dev/null +++ b/tools/testing/selftests/sched_ext/cgroup_nr_cpus.bpf.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Validate scx_bpf_cgroup_nr_cpus() from both BPF_PROG_TYPE_SYSCALL and + * struct_ops contexts. + * + * Copyright (c) 2026 NVIDIA Corporation. + */ + +#include <scx/common.bpf.h> + +char _license[] SEC("license") = "GPL"; + +UEI_DEFINE(uei); + +/* input to cgroup_nr_cpus_read() */ +u64 query_cgid; +/* output of cgroup_nr_cpus_read(), -1 if @query_cgid couldn't be resolved */ +s64 query_nr_cpus = -1; + +/* recorded by ops.cgroup_init() for @init_cgid */ +u64 init_cgid; +s64 init_nr_cpus = -1; + +SEC("syscall") +int cgroup_nr_cpus_read(void *ctx) +{ + struct cgroup *cgrp; + + query_nr_cpus = -1; + + cgrp = bpf_cgroup_from_id(query_cgid); + if (!cgrp) + return -ENOENT; + + query_nr_cpus = scx_bpf_cgroup_nr_cpus(cgrp); + bpf_cgroup_release(cgrp); + + return 0; +} + +s32 BPF_STRUCT_OPS(cgroup_nr_cpus_cgroup_init, struct cgroup *cgrp, + struct scx_cgroup_init_args *args) +{ + if (cgrp->kn->id == init_cgid) + init_nr_cpus = scx_bpf_cgroup_nr_cpus(cgrp); + + return 0; +} + +void BPF_STRUCT_OPS(cgroup_nr_cpus_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + +SEC(".struct_ops.link") +struct sched_ext_ops cgroup_nr_cpus_ops = { + .cgroup_init = (void *)cgroup_nr_cpus_cgroup_init, + .exit = (void *)cgroup_nr_cpus_exit, + .name = "cgroup_nr_cpus", +}; diff --git a/tools/testing/selftests/sched_ext/cgroup_nr_cpus.c b/tools/testing/selftests/sched_ext/cgroup_nr_cpus.c new file mode 100644 index 000000000000..1475a8b3f05f --- /dev/null +++ b/tools/testing/selftests/sched_ext/cgroup_nr_cpus.c @@ -0,0 +1,419 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Verify that scx_bpf_cgroup_nr_cpus() reports the number of CPUs in a + * cgroup's effective cpuset, including inherited and updated cpusets. + * + * Copyright (c) 2026 NVIDIA Corporation. + */ + +#define _GNU_SOURCE +#include <bpf/bpf.h> +#include <errno.h> +#include <fcntl.h> +#include <limits.h> +#include <linux/limits.h> +#include <scx/common.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#include "cgroup_nr_cpus.bpf.skel.h" +#include "cgroup_util.h" +#include "scx_test.h" + +/* + * Hierarchy under the cgroup2 root, all with the cpu controller enabled so + * that ops.cgroup_init() runs for each of them: + * + * parent cpuset enabled by the root, enables cpuset for its children + * parent/child owns a cpuset + * parent/child/leaf no cpuset of its own, inherits child's + */ +struct cgroup_nr_cpus_ctx { + struct cgroup_nr_cpus *skel; + struct bpf_link *link; + char root[PATH_MAX]; + char parent[PATH_MAX]; + char child[PATH_MAX]; + char leaf[PATH_MAX]; + bool parent_created; + bool child_created; + bool leaf_created; +}; + +static int join_path(char *dst, size_t dst_size, const char *parent, const char *name) +{ + int ret; + + ret = snprintf(dst, dst_size, "%s/%s", parent, name); + if (ret < 0 || (size_t)ret >= dst_size) + return -ENAMETOOLONG; + return 0; +} + +static u64 cgroup_id(const char *path) +{ + union { + u64 id; + unsigned char bytes[8]; + } id = {}; + struct file_handle *handle; + int mount_id, ret; + + handle = calloc(1, sizeof(*handle) + sizeof(id)); + if (!handle) + return 0; + handle->handle_bytes = sizeof(id); + ret = name_to_handle_at(AT_FDCWD, path, handle, &mount_id, 0); + if (!ret && handle->handle_bytes == sizeof(id)) + memcpy(id.bytes, handle->f_handle, sizeof(id)); + free(handle); + + return ret ? 0 : id.id; +} + +/* + * Parse a cpulist such as "0-3,8,10-11". Return the number of CPUs and the + * lowest and highest CPU in @first and @last, or -errno on failure. + */ +static int parse_cpulist(const char *cpulist, u32 *first, u32 *last) +{ + const char *p = cpulist; + u32 lowest = UINT_MAX, highest = 0; + int count = 0; + + while (*p && *p != '\n') { + unsigned long start, end_cpu; + char *end; + + errno = 0; + start = strtoul(p, &end, 10); + if (errno || end == p || start > INT_MAX) + return -EINVAL; + end_cpu = start; + p = end; + if (*p == '-') { + end_cpu = strtoul(p + 1, &end, 10); + if (errno || end == p + 1 || end_cpu > INT_MAX || end_cpu < start) + return -EINVAL; + p = end; + } + if (end_cpu - start + 1 > (unsigned long)(INT_MAX - count)) + return -EOVERFLOW; + if (start < lowest) + lowest = start; + if (end_cpu > highest) + highest = end_cpu; + count += end_cpu - start + 1; + if (*p == ',') + p++; + else if (*p && *p != '\n') + return -EINVAL; + } + + if (first) + *first = lowest; + if (last) + *last = highest; + return count; +} + +/* + * Number of CPUs in @cgroup's cpuset.cpus.effective, or -errno. + * + * A sparse cpulist can exceed a page on large systems. cg_read() does a single + * bounded read, so size the buffer for the worst case of NR_CPUS=8192 and + * reject a read that fills it rather than parsing a truncated list. + */ +static int effective_nr_cpus(const char *cgroup, u32 *first, u32 *last) +{ + static char buf[65536]; + + if (cg_read(cgroup, "cpuset.cpus.effective", buf, sizeof(buf))) + return -EIO; + if (strlen(buf) >= sizeof(buf) - 1) + return -EOVERFLOW; + return parse_cpulist(buf, first, last); +} + +/* Run the SYSCALL program to sample scx_bpf_cgroup_nr_cpus() for @path. */ +static int kfunc_nr_cpus(struct cgroup_nr_cpus_ctx *ctx, const char *path, s64 *nr_cpus) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + u64 cgid; + int err; + + cgid = cgroup_id(path); + if (!cgid) { + SCX_ERR("Failed to read cgroup ID of %s", path); + return -ENOENT; + } + + ctx->skel->bss->query_cgid = cgid; + err = bpf_prog_test_run_opts(bpf_program__fd(ctx->skel->progs.cgroup_nr_cpus_read), + &topts); + if (err || topts.retval) { + SCX_ERR("BPF_PROG_RUN failed for %s (err=%d retval=%d)", + path, err, (int)topts.retval); + return err ?: -EIO; + } + + *nr_cpus = ctx->skel->data->query_nr_cpus; + return 0; +} + +static bool check_nr_cpus(struct cgroup_nr_cpus_ctx *ctx, const char *path, int expected, + const char *what) +{ + s64 nr_cpus; + + if (kfunc_nr_cpus(ctx, path, &nr_cpus)) + return false; + if (nr_cpus != expected) { + SCX_ERR("%s: expected %d CPUs, got %lld", what, expected, + (long long)nr_cpus); + return false; + } + return true; +} + +/* + * Like check_nr_cpus() but tolerate a transient mismatch. A cpuset css being + * disabled stays attached to its cgroup until it's asynchronously offlined, and + * cpuset_num_cpus() keeps reporting its stale mask until then. + */ +static bool wait_nr_cpus(struct cgroup_nr_cpus_ctx *ctx, const char *path, int expected, + const char *what) +{ + s64 nr_cpus = -1; + int i; + + for (i = 0; i < 1000; i++) { + if (kfunc_nr_cpus(ctx, path, &nr_cpus)) + return false; + if (nr_cpus == expected) + return true; + usleep(1000); + } + SCX_ERR("%s: expected %d CPUs, got %lld", what, expected, (long long)nr_cpus); + return false; +} + +static bool controller_enabled(const char *cgroup, const char *file, const char *controller) +{ + char buf[4096], *saveptr, *token; + + if (cg_read(cgroup, file, buf, sizeof(buf))) + return false; + for (token = strtok_r(buf, "\n ", &saveptr); token; + token = strtok_r(NULL, "\n ", &saveptr)) + if (!strcmp(token, controller)) + return true; + return false; +} + +static void cleanup_ctx(struct cgroup_nr_cpus_ctx *ctx) +{ + bpf_link__destroy(ctx->link); + cgroup_nr_cpus__destroy(ctx->skel); + if (ctx->leaf_created) + cg_destroy(ctx->leaf); + if (ctx->child_created) + cg_destroy(ctx->child); + if (ctx->parent_created) + cg_destroy(ctx->parent); +} + +/* + * Enable @controller in the root's subtree_control if needed. Like the cgroup + * selftests, leave it enabled afterwards: the root is shared, and another + * manager may start relying on the controller while the test runs. + */ +static enum scx_test_status enable_controller(const char *root, const char *controller) +{ + char value[32]; + + if (controller_enabled(root, "cgroup.subtree_control", controller)) + return SCX_TEST_PASS; + if (!controller_enabled(root, "cgroup.controllers", controller)) + return SCX_TEST_SKIP; + + snprintf(value, sizeof(value), "+%s", controller); + if (cg_write(root, "cgroup.subtree_control", value)) + return SCX_TEST_SKIP; + return SCX_TEST_PASS; +} + +static enum scx_test_status setup_cgroups(struct cgroup_nr_cpus_ctx *ctx) +{ + enum scx_test_status status; + char name[64]; + + if (cg_find_unified_root(ctx->root, sizeof(ctx->root), NULL)) + return SCX_TEST_SKIP; + + status = enable_controller(ctx->root, "cpu"); + if (status != SCX_TEST_PASS) + return status; + status = enable_controller(ctx->root, "cpuset"); + if (status != SCX_TEST_PASS) + return status; + + snprintf(name, sizeof(name), "scx_nr_cpus_%d", getpid()); + if (join_path(ctx->parent, sizeof(ctx->parent), ctx->root, name) || + join_path(ctx->child, sizeof(ctx->child), ctx->parent, "child") || + join_path(ctx->leaf, sizeof(ctx->leaf), ctx->child, "leaf")) { + SCX_ERR("Cgroup path is too long"); + return SCX_TEST_FAIL; + } + + if (cg_create(ctx->parent)) { + SCX_ERR("Failed to create cgroup %s", ctx->parent); + return SCX_TEST_FAIL; + } + ctx->parent_created = true; + if (cg_write(ctx->parent, "cgroup.subtree_control", "+cpu +cpuset")) { + SCX_ERR("Failed to enable controllers in %s", ctx->parent); + return SCX_TEST_FAIL; + } + if (cg_create(ctx->child)) { + SCX_ERR("Failed to create cgroup %s", ctx->child); + return SCX_TEST_FAIL; + } + ctx->child_created = true; + if (cg_write(ctx->child, "cgroup.subtree_control", "+cpu")) { + SCX_ERR("Failed to enable cpu in %s", ctx->child); + return SCX_TEST_FAIL; + } + if (cg_create(ctx->leaf)) { + SCX_ERR("Failed to create cgroup %s", ctx->leaf); + return SCX_TEST_FAIL; + } + ctx->leaf_created = true; + + return SCX_TEST_PASS; +} + +static enum scx_test_status run(void *arg) +{ + struct cgroup_nr_cpus_ctx ctx = {}; + enum scx_test_status status; + char value[32]; + u32 first, last; + int nr_root, nr_child; + + (void)arg; + + /* + * SCX_ENUM_INIT() exits the process if vmlinux BTF can't be loaded, so + * run it before creating any cgroups that would then be left behind. + */ + ctx.skel = cgroup_nr_cpus__open(); + if (!ctx.skel) { + SCX_ERR("Failed to open skel"); + return SCX_TEST_FAIL; + } + SCX_ENUM_INIT(ctx.skel); + + status = setup_cgroups(&ctx); + if (status != SCX_TEST_PASS) + goto out; + status = SCX_TEST_FAIL; + + nr_root = effective_nr_cpus(ctx.root, NULL, NULL); + nr_child = effective_nr_cpus(ctx.child, &first, &last); + if (nr_root < 0 || nr_child < 0) { + SCX_ERR("Failed to read effective cpusets"); + goto out; + } + /* The effective cpuset can be empty, e.g. under a partition root. */ + if (nr_child < 2) { + status = SCX_TEST_SKIP; + goto out; + } + + ctx.skel->bss->init_cgid = cgroup_id(ctx.leaf); + if (!ctx.skel->bss->init_cgid) { + SCX_ERR("Failed to read cgroup ID of %s", ctx.leaf); + goto out; + } + if (cgroup_nr_cpus__load(ctx.skel)) { + SCX_ERR("Failed to load skel"); + goto out; + } + + /* The kfunc must be callable without a scheduler attached. */ + if (!check_nr_cpus(&ctx, ctx.root, nr_root, "root") || + !check_nr_cpus(&ctx, ctx.child, nr_child, "child") || + !check_nr_cpus(&ctx, ctx.leaf, nr_child, "inherited leaf")) + goto out; + + /* ops.cgroup_init() runs for existing cgroups when attaching. */ + ctx.link = bpf_map__attach_struct_ops(ctx.skel->maps.cgroup_nr_cpus_ops); + if (!ctx.link) { + SCX_ERR("Failed to attach scheduler"); + goto out; + } + if (ctx.skel->data->init_nr_cpus != nr_child) { + SCX_ERR("ops.cgroup_init(): expected %d CPUs, got %lld", nr_child, + (long long)ctx.skel->data->init_nr_cpus); + goto out; + } + + /* Non-contiguous cpuset, observed by the owner and by the inheritor. */ + if (nr_child > 2) { + snprintf(value, sizeof(value), "%u,%u", first, last); + if (cg_write(ctx.child, "cpuset.cpus", value)) { + SCX_ERR("Failed to set cpuset.cpus=%s for %s", value, ctx.child); + goto out; + } + if (!check_nr_cpus(&ctx, ctx.child, 2, "sparse child") || + !check_nr_cpus(&ctx, ctx.leaf, 2, "sparse inherited leaf")) + goto out; + } + + snprintf(value, sizeof(value), "%u", first); + if (cg_write(ctx.child, "cpuset.cpus", value)) { + SCX_ERR("Failed to set cpuset.cpus=%s for %s", value, ctx.child); + goto out; + } + if (!check_nr_cpus(&ctx, ctx.child, 1, "single-CPU child") || + !check_nr_cpus(&ctx, ctx.leaf, 1, "single-CPU inherited leaf")) + goto out; + + /* + * Disabling cpuset below @parent makes @child and @leaf inherit + * @parent's effective cpuset, which spans all of the root's CPUs. + */ + if (cg_write(ctx.parent, "cgroup.subtree_control", "-cpuset")) { + SCX_ERR("Failed to disable cpuset in %s", ctx.parent); + goto out; + } + nr_child = effective_nr_cpus(ctx.parent, NULL, NULL); + if (nr_child < 0) { + SCX_ERR("Failed to read effective cpuset of %s", ctx.parent); + goto out; + } + if (!wait_nr_cpus(&ctx, ctx.child, nr_child, "child after cpuset disable") || + !wait_nr_cpus(&ctx, ctx.leaf, nr_child, "leaf after cpuset disable")) + goto out; + + if (UEI_EXITED(ctx.skel, uei)) { + SCX_ERR("Scheduler exited unexpectedly"); + goto out; + } + + status = SCX_TEST_PASS; +out: + cleanup_ctx(&ctx); + return status; +} + +struct scx_test cgroup_nr_cpus = { + .name = "cgroup_nr_cpus", + .description = "Verify scx_bpf_cgroup_nr_cpus() reports effective cpuset CPU counts", + .run = run, +}; +REGISTER_SCX_TEST(&cgroup_nr_cpus) diff --git a/tools/testing/selftests/sched_ext/config b/tools/testing/selftests/sched_ext/config index aa901b05c8ad..8173f9170ffe 100644 --- a/tools/testing/selftests/sched_ext/config +++ b/tools/testing/selftests/sched_ext/config @@ -1,8 +1,11 @@ CONFIG_SCHED_CLASS_EXT=y CONFIG_CGROUPS=y CONFIG_CGROUP_SCHED=y +CONFIG_CPUSETS=y CONFIG_EXT_GROUP_SCHED=y CONFIG_BPF=y CONFIG_BPF_SYSCALL=y CONFIG_DEBUG_INFO=y CONFIG_DEBUG_INFO_BTF=y +CONFIG_EXPERT=y +CONFIG_SCHED_PROXY_EXEC=y diff --git a/tools/testing/selftests/sched_ext/create_dsq.bpf.c b/tools/testing/selftests/sched_ext/create_dsq.bpf.c index 2cfc4ffd60e2..24dacd1aab0c 100644 --- a/tools/testing/selftests/sched_ext/create_dsq.bpf.c +++ b/tools/testing/selftests/sched_ext/create_dsq.bpf.c @@ -10,6 +10,10 @@ char _license[] SEC("license") = "GPL"; +u32 nr_lifecycle_tests; + +UEI_DEFINE(uei); + void BPF_STRUCT_OPS(create_dsq_exit_task, struct task_struct *p, struct scx_exit_task_args *args) { @@ -43,16 +47,55 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(create_dsq_init) } bpf_for(i, 0, 1024) { + err = scx_bpf_create_dsq(i, -1); + if (err != -EEXIST) { + scx_bpf_error("Duplicate DSQ %d creation returned %d", i, err); + return -EINVAL; + } + + /* A rejected duplicate must leave the original DSQ accessible. */ + err = scx_bpf_dsq_nr_queued(i); + if (err) { + scx_bpf_error("Original DSQ %d queue count is %d", i, err); + return -EINVAL; + } + scx_bpf_destroy_dsq(i); + err = scx_bpf_dsq_nr_queued(i); + if (err != -ENOENT) { + scx_bpf_error("Destroyed DSQ %d queue count is %d", i, err); + return -EINVAL; + } + + err = scx_bpf_create_dsq(i, -1); + if (err) { + scx_bpf_error("Failed to recreate DSQ %d: %d", i, err); + return err; + } + + err = scx_bpf_dsq_nr_queued(i); + if (err) { + scx_bpf_error("Recreated DSQ %d queue count is %d", i, err); + return -EINVAL; + } + + scx_bpf_destroy_dsq(i); + nr_lifecycle_tests++; } return 0; } +void BPF_STRUCT_OPS(create_dsq_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + SEC(".struct_ops.link") struct sched_ext_ops create_dsq_ops = { .init_task = (void *) create_dsq_init_task, .exit_task = (void *) create_dsq_exit_task, .init = (void *) create_dsq_init, + .exit = (void *) create_dsq_exit, .name = "create_dsq", }; diff --git a/tools/testing/selftests/sched_ext/create_dsq.c b/tools/testing/selftests/sched_ext/create_dsq.c index d67431f57ac6..84143d5710f4 100644 --- a/tools/testing/selftests/sched_ext/create_dsq.c +++ b/tools/testing/selftests/sched_ext/create_dsq.c @@ -35,8 +35,12 @@ static enum scx_test_status run(void *ctx) return SCX_TEST_FAIL; } + SCX_ASSERT_ALIVE(skel, uei); + bpf_link__destroy(link); + SCX_EQ(skel->bss->nr_lifecycle_tests, 1024); + return SCX_TEST_PASS; } @@ -49,7 +53,7 @@ static void cleanup(void *ctx) struct scx_test create_dsq = { .name = "create_dsq", - .description = "Create and destroy a dsq in a loop", + .description = "Create, reject duplicates, destroy and recreate DSQs", .setup = setup, .run = run, .cleanup = cleanup, diff --git a/tools/testing/selftests/sched_ext/cyclic_kick_wait.c b/tools/testing/selftests/sched_ext/cyclic_kick_wait.c index c2e5aa9de715..178d1d75f332 100644 --- a/tools/testing/selftests/sched_ext/cyclic_kick_wait.c +++ b/tools/testing/selftests/sched_ext/cyclic_kick_wait.c @@ -152,7 +152,7 @@ static enum scx_test_status run(void *ctx) sleep(5); - if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) { + if (UEI_EXITED(skel, uei)) { SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)", (unsigned long long)skel->data->uei.kind, (long long)skel->data->uei.exit_code); diff --git a/tools/testing/selftests/sched_ext/dequeue_remote.c b/tools/testing/selftests/sched_ext/dequeue_remote.c index f2c1901030de..af344581d5f1 100644 --- a/tools/testing/selftests/sched_ext/dequeue_remote.c +++ b/tools/testing/selftests/sched_ext/dequeue_remote.c @@ -43,6 +43,7 @@ static void worker_fn(int id) for (j = 0; j < 20000 + id * 1000; j++) sum += j; + asm volatile("" : : "r"(sum)); if (id & 1) usleep(100); else diff --git a/tools/testing/selftests/sched_ext/enq_blocked.bpf.c b/tools/testing/selftests/sched_ext/enq_blocked.bpf.c new file mode 100644 index 000000000000..212690bf4e07 --- /dev/null +++ b/tools/testing/selftests/sched_ext/enq_blocked.bpf.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + * + * Verify that SCX_OPS_ENQ_BLOCKED passes blocked proxy donors through + * ops.enqueue() and record whether callbacks occur on the donor or owner CPU. + */ + +#include <scx/common.bpf.h> + +#define SHARED_DSQ 0 + +char _license[] SEC("license") = "GPL"; + +s32 donor_pid; +s32 donor_cpu = -1; +s32 owner_cpu = -1; +u64 nr_blocked_enqueues; +u64 nr_blocked_enqueues_donor_cpu; +u64 nr_blocked_enqueues_owner_cpu; +u64 nr_blocked_enqueues_other_cpu; +u64 nr_blocked_wakeups; +static u64 vtime_now; + +UEI_DEFINE(uei); + +s32 BPF_STRUCT_OPS(enq_blocked_select_cpu, + struct task_struct *p, s32 prev_cpu, u64 wake_flags) +{ + return prev_cpu; +} + +void BPF_STRUCT_OPS(enq_blocked_enqueue, struct task_struct *p, u64 enq_flags) +{ + u64 vtime = p->scx.dsq_vtime; + + if (enq_flags & SCX_ENQ_BLOCKED) { + int cpu = scx_bpf_task_cpu(p); + + if (enq_flags & SCX_ENQ_WAKEUP) + __sync_fetch_and_add(&nr_blocked_wakeups, 1); + + if (p->pid == donor_pid) { + __sync_fetch_and_add(&nr_blocked_enqueues, 1); + if (cpu == donor_cpu) + __sync_fetch_and_add(&nr_blocked_enqueues_donor_cpu, 1); + else if (cpu == owner_cpu) + __sync_fetch_and_add(&nr_blocked_enqueues_owner_cpu, 1); + else + __sync_fetch_and_add(&nr_blocked_enqueues_other_cpu, 1); + } + } + + /* Limit the amount of budget an idling task can accumulate. */ + if (time_before(vtime, vtime_now - SCX_SLICE_DFL)) + vtime = vtime_now - SCX_SLICE_DFL; + + scx_bpf_dsq_insert_vtime(p, SHARED_DSQ, SCX_SLICE_DFL, vtime, + enq_flags); + scx_bpf_kick_cpu(scx_bpf_task_cpu(p), SCX_KICK_IDLE); +} + +void BPF_STRUCT_OPS(enq_blocked_dispatch, s32 cpu, struct task_struct *prev) +{ + scx_bpf_dsq_move_to_local(SHARED_DSQ, 0); +} + +void BPF_STRUCT_OPS(enq_blocked_running, struct task_struct *p) +{ + if (time_before(vtime_now, p->scx.dsq_vtime)) + vtime_now = p->scx.dsq_vtime; +} + +void BPF_STRUCT_OPS(enq_blocked_stopping, struct task_struct *p, bool runnable) +{ + u64 delta = scale_by_task_weight_inverse(p, + SCX_SLICE_DFL - p->scx.slice); + + scx_bpf_task_set_dsq_vtime(p, p->scx.dsq_vtime + delta); +} + +void BPF_STRUCT_OPS(enq_blocked_enable, struct task_struct *p) +{ + scx_bpf_task_set_dsq_vtime(p, vtime_now); +} + +s32 BPF_STRUCT_OPS_SLEEPABLE(enq_blocked_init) +{ + int ret; + + ret = scx_bpf_create_dsq(SHARED_DSQ, -1); + if (ret) { + scx_bpf_error("failed to create DSQ %d (%d)", SHARED_DSQ, ret); + return ret; + } + + return 0; +} + +void BPF_STRUCT_OPS(enq_blocked_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + +SEC(".struct_ops.link") +struct sched_ext_ops enq_blocked_ops = { + .select_cpu = (void *)enq_blocked_select_cpu, + .enqueue = (void *)enq_blocked_enqueue, + .dispatch = (void *)enq_blocked_dispatch, + .running = (void *)enq_blocked_running, + .stopping = (void *)enq_blocked_stopping, + .enable = (void *)enq_blocked_enable, + .init = (void *)enq_blocked_init, + .exit = (void *)enq_blocked_exit, + .name = "enq_blocked", +}; diff --git a/tools/testing/selftests/sched_ext/enq_blocked.c b/tools/testing/selftests/sched_ext/enq_blocked.c new file mode 100644 index 000000000000..1d5e090b54bc --- /dev/null +++ b/tools/testing/selftests/sched_ext/enq_blocked.c @@ -0,0 +1,917 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + * + * Exercise a priority inversion with the owner and donor first pinned to the + * same CPU, then with each on a different CPU. A high-priority donor blocks on + * a mutex held by a low-priority owner while one medium-priority contender per + * available CPU keeps the system busy. A weighted-vruntime BPF scheduler runs + * both CPU placement configurations with SCX_OPS_ENQ_BLOCKED first disabled + * and then enabled. The test validates blocked-donor admission and reports the + * average mutex hold and wait times, plus their enabled-minus-disabled deltas, + * for each configuration. The timing data is informational. + * + * CONFIG_SCHED_PROXY_EXEC=y is required to exercise the proxy-execution paths. + */ +#define _GNU_SOURCE + +#include <bpf/bpf.h> +#include <errno.h> +#include <fcntl.h> +#include <limits.h> +#include <pthread.h> +#include <sched.h> +#include <scx/common.h> +#include <stdatomic.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/ioctl.h> +#include <sys/resource.h> +#include <sys/syscall.h> +#include <time.h> +#include <unistd.h> + +#include "enq_blocked.bpf.skel.h" +#include "enq_blocked.h" +#include "scx_test.h" + +#define MODULE_NAME "scx_enq_blocked_test" +#define MODULE_FILE "test_modules/" MODULE_NAME ".ko" +#define DEVICE_PATH "/dev/scx_enq_blocked" +#define WAIT_STEP_US 1000 +#define WAIT_TIMEOUT_MS 2000 +#define NR_WARMUP_TRIALS 1 +#define NR_MEASURED_TRIALS 10 +#define NR_TRIALS (NR_WARMUP_TRIALS + NR_MEASURED_TRIALS) +#define JOIN_TIMEOUT_MS ((NR_TRIALS + 1) * WAIT_TIMEOUT_MS) +#define OWNER_NICE 19 +#define DONOR_NICE -20 +#define CONTENDER_NICE 0 + +struct thread_ctx { + atomic_bool start_donor; + atomic_bool abort; + atomic_bool stop_contender; + atomic_bool measurement_ready; + atomic_int donor_pid; + atomic_int donor_completed; + int fd; + int donor_cpu; + int owner_cpu; +}; + +struct contender_ctx { + struct thread_ctx *thread_ctx; + atomic_int status; + int cpu; +}; + +struct run_result { + struct enq_blocked_stats stats; + u64 nr_blocked_enqueues; + u64 nr_blocked_enqueues_donor_cpu; + u64 nr_blocked_enqueues_owner_cpu; + u64 nr_blocked_enqueues_other_cpu; + u64 nr_blocked_wakeups; +}; + +static bool parse_bool(const char *value, bool *result) +{ + if (!strcasecmp(value, "1") || !strcasecmp(value, "y") || + !strcasecmp(value, "yes") || !strcasecmp(value, "on") || + !strcasecmp(value, "true")) { + *result = true; + return true; + } + + if (!strcasecmp(value, "0") || !strcasecmp(value, "n") || + !strcasecmp(value, "no") || !strcasecmp(value, "off") || + !strcasecmp(value, "false")) { + *result = false; + return true; + } + + return false; +} + +static bool cmdline_bool(const char *name, bool default_value) +{ + char cmdline[4096], *newline, *saveptr = NULL, *token; + size_t name_len = strlen(name); + bool value = default_value; + FILE *file; + + file = fopen("/proc/cmdline", "r"); + if (!file) + return default_value; + + if (!fgets(cmdline, sizeof(cmdline), file)) { + fclose(file); + return default_value; + } + fclose(file); + newline = strchr(cmdline, '\n'); + if (newline) + *newline = '\0'; + + for (token = strtok_r(cmdline, " ", &saveptr); token; + token = strtok_r(NULL, " ", &saveptr)) { + bool parsed; + + if (strncmp(token, name, name_len) || token[name_len] != '=') + continue; + if (parse_bool(token + name_len + 1, &parsed)) + value = parsed; + } + + return value; +} + +static int module_path(char *path, size_t size) +{ + ssize_t len; + char *slash; + + len = readlink("/proc/self/exe", path, size - 1); + if (len < 0) + return -errno; + path[len] = '\0'; + + slash = strrchr(path, '/'); + if (!slash) + return -EINVAL; + *slash = '\0'; + + if (snprintf(slash, size - (slash - path), "/%s", MODULE_FILE) >= + size - (slash - path)) + return -ENAMETOOLONG; + + return 0; +} + +static int load_test_module(bool *loaded_here) +{ + char path[PATH_MAX]; + int fd, err; + + err = module_path(path, sizeof(path)); + if (err) + return err; + + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -errno; + + if (syscall(SYS_finit_module, fd, "", 0)) { + err = errno; + close(fd); + if (err == EEXIST) + return 0; + return -err; + } + + close(fd); + *loaded_here = true; + return 0; +} + +static void unload_test_module(bool loaded_here) +{ + if (loaded_here && syscall(SYS_delete_module, MODULE_NAME, O_NONBLOCK)) + SCX_ERR("Failed to unload %s (%d)", MODULE_NAME, errno); +} + +static int pin_to_cpu(int cpu) +{ + cpu_set_t mask; + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); + return sched_setaffinity(0, sizeof(mask), &mask) ? errno : 0; +} + +static int select_test_cpus(bool cross_cpu, cpu_set_t *mask, int *donor_cpu, + int *owner_cpu) +{ + int cpu, first = -1; + + if (sched_getaffinity(0, sizeof(*mask), mask)) + return -errno; + + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) { + if (!CPU_ISSET(cpu, mask)) + continue; + if (first < 0) { + first = cpu; + if (!cross_cpu) + break; + } else { + *donor_cpu = first; + *owner_cpu = cpu; + return 0; + } + } + + if (first < 0) + return -ENODEV; + if (cross_cpu) + return -EAGAIN; + + *donor_cpu = first; + *owner_cpu = first; + return 0; +} + +static int set_nice(int nice) +{ + return setpriority(PRIO_PROCESS, 0, nice) ? errno : 0; +} + +static bool wait_for_pid(atomic_int *pid) +{ + int waited_ms; + + for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) { + if (atomic_load_explicit(pid, memory_order_acquire) > 0) + return true; + usleep(WAIT_STEP_US); + } + + return false; +} + +static int wait_for_contenders(struct contender_ctx *contenders, + size_t nr_contenders) +{ + size_t i, nr_ready; + int status, waited_ms; + + for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) { + nr_ready = 0; + for (i = 0; i < nr_contenders; i++) { + status = atomic_load_explicit(&contenders[i].status, + memory_order_acquire); + if (status < 0) + return status; + if (status > 0) + nr_ready++; + } + if (nr_ready == nr_contenders) + return 1; + usleep(WAIT_STEP_US); + } + + return -ETIMEDOUT; +} + +static int wait_for_donor_state(struct thread_ctx *ctx, int expected) +{ + int state, waited_ms; + + for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) { + state = ioctl(ctx->fd, ENQ_BLOCKED_IOCTL_DONOR_STATE); + if (state == expected) + return state; + if (state < 0 && errno != ENOENT) + return -errno; + usleep(WAIT_STEP_US); + } + + return -ETIMEDOUT; +} + +static bool wait_for_donor(struct thread_ctx *ctx, int trial) +{ + int waited_ms; + + for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) { + if (atomic_load_explicit(&ctx->donor_completed, + memory_order_acquire) >= trial) + return true; + if (atomic_load_explicit(&ctx->abort, memory_order_relaxed)) + return false; + usleep(WAIT_STEP_US); + } + + return false; +} + +static bool wait_for_measurement(struct thread_ctx *ctx) +{ + while (!atomic_load_explicit(&ctx->measurement_ready, + memory_order_acquire) && + !atomic_load_explicit(&ctx->abort, memory_order_relaxed)) + sched_yield(); + + return !atomic_load_explicit(&ctx->abort, memory_order_relaxed); +} + +static void *contender_fn(void *arg) +{ + struct contender_ctx *contender = arg; + struct thread_ctx *ctx = contender->thread_ctx; + int err; + + err = pin_to_cpu(contender->cpu); + if (!err) + err = set_nice(CONTENDER_NICE); + atomic_store_explicit(&contender->status, err ? -err : 1, + memory_order_release); + if (err) + return (void *)(uintptr_t)err; + + while (!atomic_load_explicit(&ctx->stop_contender, + memory_order_relaxed)) + ; + + return NULL; +} + +static void *owner_fn(void *arg) +{ + struct thread_ctx *ctx = arg; + int err, i; + + err = pin_to_cpu(ctx->owner_cpu); + if (err) + return (void *)(uintptr_t)err; + err = set_nice(OWNER_NICE); + if (err) + return (void *)(uintptr_t)err; + + for (i = 0; i < NR_TRIALS; i++) { + if (ioctl(ctx->fd, ENQ_BLOCKED_IOCTL_OWNER)) + return (void *)(uintptr_t)errno; + if (!wait_for_donor(ctx, i + 1)) + return (void *)(uintptr_t)ETIMEDOUT; + + if (i + 1 == NR_WARMUP_TRIALS && !wait_for_measurement(ctx)) + return NULL; + } + + return NULL; +} + +static int run_donor_trial(struct thread_ctx *ctx) +{ + int waited_ms; + + for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) { + if (!ioctl(ctx->fd, ENQ_BLOCKED_IOCTL_DONOR)) + return 0; + if (errno != EAGAIN) + return -errno; + usleep(WAIT_STEP_US); + } + + return -ETIMEDOUT; +} + +static void *donor_fn(void *arg) +{ + struct thread_ctx *ctx = arg; + int err, i; + + err = pin_to_cpu(ctx->donor_cpu); + if (err) + return (void *)(uintptr_t)err; + err = set_nice(DONOR_NICE); + if (err) + return (void *)(uintptr_t)err; + + atomic_store_explicit(&ctx->donor_pid, syscall(SYS_gettid), + memory_order_release); + while (!atomic_load_explicit(&ctx->start_donor, memory_order_acquire) && + !atomic_load_explicit(&ctx->abort, memory_order_relaxed)) + sched_yield(); + + if (atomic_load_explicit(&ctx->abort, memory_order_relaxed)) + return NULL; + + for (i = 0; i < NR_TRIALS; i++) { + err = run_donor_trial(ctx); + if (err) + return (void *)(uintptr_t)-err; + atomic_store_explicit(&ctx->donor_completed, i + 1, + memory_order_release); + } + + return NULL; +} + +static void print_avg_time(const char *name, u64 total_ns, u64 samples) +{ + u64 avg_ns = samples ? total_ns / samples : 0; + + printf(" %s_avg_ns=%llu (%llu.%03llu ms, samples=%llu)\n", name, + (unsigned long long)avg_ns, + (unsigned long long)(avg_ns / 1000000), + (unsigned long long)((avg_ns / 1000) % 1000), + (unsigned long long)samples); +} + +static void print_avg_delta(const char *name, u64 disabled_total, + u64 disabled_samples, u64 enabled_total, + u64 enabled_samples) +{ + u64 disabled_avg, enabled_avg; + s64 delta_ns; + double delta_pct; + + if (!disabled_samples || !enabled_samples) + return; + + disabled_avg = disabled_total / disabled_samples; + enabled_avg = enabled_total / enabled_samples; + delta_ns = (s64)enabled_avg - (s64)disabled_avg; + delta_pct = disabled_avg ? 100.0 * delta_ns / disabled_avg : 0.0; + + printf(" %s_delta_ns=%+lld (%+.2f%%)\n", name, + (long long)delta_ns, delta_pct); +} + +static int join_thread(pthread_t thread, const struct timespec *deadline, + int *thread_err) +{ + void *result; + int err; + + err = pthread_timedjoin_np(thread, &result, deadline); + if (err) + return err; + + *thread_err = (int)(uintptr_t)result; + return 0; +} + +static void set_join_deadline(struct timespec *deadline) +{ + clock_gettime(CLOCK_REALTIME, deadline); + deadline->tv_sec += JOIN_TIMEOUT_MS / 1000; + deadline->tv_nsec += (JOIN_TIMEOUT_MS % 1000) * 1000000; + if (deadline->tv_nsec >= 1000000000) { + deadline->tv_sec++; + deadline->tv_nsec -= 1000000000; + } +} + +static enum scx_test_status setup(void **ctx) +{ + struct enq_blocked *skel; + u64 flag; + + skel = enq_blocked__open(); + SCX_FAIL_IF(!skel, "Failed to open skel"); + SCX_ENUM_INIT(skel); + + flag = SCX_OPS_ENQ_BLOCKED; + if (!flag) { + enq_blocked__destroy(skel); + fprintf(stderr, "SKIP: SCX_OPS_ENQ_BLOCKED is unavailable\n"); + return SCX_TEST_SKIP; + } + + enq_blocked__destroy(skel); + *ctx = NULL; + return SCX_TEST_PASS; +} + +static enum scx_test_status run_one(bool enq_blocked, bool cross_cpu, + struct run_result *result) +{ + struct enq_blocked *skel; + struct thread_ctx thread_ctx = {}; + struct contender_ctx *contender_ctxs = NULL; + struct bpf_link *link = NULL; + pthread_t owner, donor, *contenders = NULL; + struct timespec join_deadline; + cpu_set_t allowed_cpus; + bool module_loaded = false; + bool owner_started = false, donor_started = false; + bool join_timed_out = false; + bool proxy_enabled; + enum scx_test_status status = SCX_TEST_PASS; + int cpu, donor_pid, donor_state, err, thread_err; + size_t i, nr_contenders, nr_contenders_started = 0; + size_t nr_contenders_joined = 0; + u64 nr_blocked, nr_blocked_donor_cpu, nr_blocked_owner_cpu; + u64 nr_blocked_other_cpu, nr_blocked_wakeups; + struct enq_blocked_stats stats; + + err = select_test_cpus(cross_cpu, &allowed_cpus, &thread_ctx.donor_cpu, + &thread_ctx.owner_cpu); + if (err == -EAGAIN) { + fprintf(stderr, "SKIP: cross-CPU case requires two allowed CPUs\n"); + return SCX_TEST_SKIP; + } + if (err) { + SCX_ERR("Failed to select test CPUs (%d)", -err); + return SCX_TEST_FAIL; + } + nr_contenders = CPU_COUNT(&allowed_cpus); + contenders = calloc(nr_contenders, sizeof(*contenders)); + contender_ctxs = calloc(nr_contenders, sizeof(*contender_ctxs)); + if (!contenders || !contender_ctxs) { + SCX_ERR("Failed to allocate %zu contender threads", nr_contenders); + status = SCX_TEST_FAIL; + goto out_contenders; + } + + skel = enq_blocked__open(); + if (!skel) { + SCX_ERR("Failed to open skel"); + status = SCX_TEST_FAIL; + goto out_contenders; + } + SCX_ENUM_INIT(skel); + skel->struct_ops.enq_blocked_ops->flags = + SCX_OPS_ENQ_LAST | + (enq_blocked ? SCX_OPS_ENQ_BLOCKED : 0); + if (enq_blocked__load(skel)) { + SCX_ERR("Failed to load skel"); + status = SCX_TEST_FAIL; + goto out_skel; + } + + err = load_test_module(&module_loaded); + if (err == -EPERM || err == -ENOENT) { + fprintf(stderr, "SKIP: cannot load mutex fixture (%d)\n", -err); + status = SCX_TEST_SKIP; + goto out_skel; + } + if (err) { + SCX_ERR("Failed to load mutex fixture (%d)", -err); + status = SCX_TEST_FAIL; + goto out_skel; + } + + thread_ctx.fd = open(DEVICE_PATH, O_RDONLY | O_CLOEXEC); + if (thread_ctx.fd < 0) { + SCX_ERR("Failed to open %s (%d)", DEVICE_PATH, errno); + status = SCX_TEST_FAIL; + goto out_module; + } + err = ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_PROXY_SUPPORTED); + if (err < 0) { + SCX_ERR("Failed to query proxy-exec support (%d)", errno); + status = SCX_TEST_FAIL; + goto out_fd; + } + proxy_enabled = err && cmdline_bool("sched_proxy_exec", true); + if (!proxy_enabled) { + fprintf(stderr, "SKIP: proxy execution is not enabled\n"); + status = SCX_TEST_SKIP; + goto out_fd; + } + if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_RESET_STATS)) { + SCX_ERR("Failed to reset mutex statistics (%d)", errno); + status = SCX_TEST_FAIL; + goto out_fd; + } + + if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_PREP_ATTACH)) { + SCX_ERR("Failed to prepare scheduler attachment (%d)", errno); + status = SCX_TEST_FAIL; + goto out_fd; + } + + err = pthread_create(&owner, NULL, owner_fn, &thread_ctx); + if (err) { + SCX_ERR("Failed to create owner thread (%d)", err); + status = SCX_TEST_FAIL; + goto out; + } + owner_started = true; + + err = pthread_create(&donor, NULL, donor_fn, &thread_ctx); + if (err) { + SCX_ERR("Failed to create donor thread (%d)", err); + status = SCX_TEST_FAIL; + goto out; + } + donor_started = true; + + if (!wait_for_pid(&thread_ctx.donor_pid)) { + SCX_ERR("Timed out waiting for donor thread"); + status = SCX_TEST_FAIL; + goto out; + } + + donor_pid = atomic_load_explicit(&thread_ctx.donor_pid, + memory_order_acquire); + skel->bss->donor_pid = donor_pid; + skel->data->donor_cpu = thread_ctx.donor_cpu; + skel->data->owner_cpu = thread_ctx.owner_cpu; + atomic_store_explicit(&thread_ctx.start_donor, true, + memory_order_release); + + donor_state = ENQ_BLOCKED_DONOR_SLEEPING; + if (proxy_enabled) + donor_state |= ENQ_BLOCKED_DONOR_ON_RQ; + err = wait_for_donor_state(&thread_ctx, donor_state); + if (err < 0) { + SCX_ERR("Donor did not block before scheduler attachment (%d)", -err); + status = SCX_TEST_FAIL; + goto out; + } + + link = bpf_map__attach_struct_ops(skel->maps.enq_blocked_ops); + if (!link) { + SCX_ERR("Failed to attach scheduler"); + status = SCX_TEST_FAIL; + goto out; + } + + /* Scheduler ownership changes start from a fully blocked donor. */ + donor_state = ENQ_BLOCKED_DONOR_SLEEPING; + err = wait_for_donor_state(&thread_ctx, donor_state); + if (err < 0) { + SCX_ERR("Unexpected donor state after scheduler attachment (%d)", + -err); + status = SCX_TEST_FAIL; + goto out; + } + + if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_ATTACH_DONE)) { + SCX_ERR("Failed to complete scheduler attachment (%d)", errno); + status = SCX_TEST_FAIL; + goto out; + } + + i = 0; + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) { + if (!CPU_ISSET(cpu, &allowed_cpus)) + continue; + + contender_ctxs[i].thread_ctx = &thread_ctx; + contender_ctxs[i].cpu = cpu; + atomic_init(&contender_ctxs[i].status, 0); + err = pthread_create(&contenders[i], NULL, contender_fn, + &contender_ctxs[i]); + if (err) { + SCX_ERR("Failed to create contender for CPU %d (%d)", + cpu, err); + status = SCX_TEST_FAIL; + goto out; + } + nr_contenders_started++; + i++; + } + + err = wait_for_contenders(contender_ctxs, nr_contenders); + if (err != 1) { + SCX_ERR("Contender threads failed (%d)", -err); + status = SCX_TEST_FAIL; + goto out; + } + + /* + * The first trial spans scheduler attachment and validates the state + * transition, but including it would skew scheduling latency. Exclude it + * from both the mutex and BPF enqueue measurements. + */ + if (!wait_for_donor(&thread_ctx, NR_WARMUP_TRIALS)) { + SCX_ERR("Timed out waiting for warm-up trial"); + status = SCX_TEST_FAIL; + goto out; + } + if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_RESET_STATS)) { + SCX_ERR("Failed to reset mutex statistics after warm-up (%d)", + errno); + status = SCX_TEST_FAIL; + goto out; + } + skel->bss->nr_blocked_enqueues = 0; + skel->bss->nr_blocked_enqueues_donor_cpu = 0; + skel->bss->nr_blocked_enqueues_owner_cpu = 0; + skel->bss->nr_blocked_enqueues_other_cpu = 0; + skel->bss->nr_blocked_wakeups = 0; + atomic_store_explicit(&thread_ctx.measurement_ready, true, + memory_order_release); + +out: + ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_ATTACH_DONE); + if (status != SCX_TEST_PASS) { + atomic_store_explicit(&thread_ctx.abort, true, memory_order_release); + atomic_store_explicit(&thread_ctx.start_donor, true, + memory_order_release); + atomic_store_explicit(&thread_ctx.measurement_ready, true, + memory_order_release); + } + + set_join_deadline(&join_deadline); + if (donor_started) { + err = join_thread(donor, &join_deadline, &thread_err); + if (err == ETIMEDOUT) { + SCX_ERR("Timed out waiting for donor thread"); + join_timed_out = true; + status = SCX_TEST_FAIL; + } else if (err) { + SCX_ERR("Failed to join donor thread (%d)", err); + status = SCX_TEST_FAIL; + } else { + donor_started = false; + if (thread_err) { + SCX_ERR("Donor thread failed (%d)", thread_err); + status = SCX_TEST_FAIL; + } + } + } + if (!join_timed_out && owner_started) { + err = join_thread(owner, &join_deadline, &thread_err); + if (err == ETIMEDOUT) { + SCX_ERR("Timed out waiting for owner thread"); + join_timed_out = true; + status = SCX_TEST_FAIL; + } else if (err) { + SCX_ERR("Failed to join owner thread (%d)", err); + status = SCX_TEST_FAIL; + } else { + owner_started = false; + if (thread_err) { + SCX_ERR("Owner thread failed (%d)", thread_err); + status = SCX_TEST_FAIL; + } + } + } + atomic_store_explicit(&thread_ctx.stop_contender, true, + memory_order_release); + for (i = 0; !join_timed_out && i < nr_contenders_started; i++) { + err = join_thread(contenders[i], &join_deadline, &thread_err); + if (err == ETIMEDOUT) { + SCX_ERR("Timed out waiting for contender on CPU %d", + contender_ctxs[i].cpu); + join_timed_out = true; + status = SCX_TEST_FAIL; + } else if (err) { + SCX_ERR("Failed to join contender on CPU %d (%d)", + contender_ctxs[i].cpu, err); + status = SCX_TEST_FAIL; + } else { + nr_contenders_joined++; + if (thread_err) { + SCX_ERR("Contender on CPU %d failed (%d)", + contender_ctxs[i].cpu, thread_err); + status = SCX_TEST_FAIL; + } + } + } + + /* Restore the fair scheduler before waiting for any stranded thread. */ + if (join_timed_out) { + atomic_store_explicit(&thread_ctx.abort, true, + memory_order_release); + if (link) { + bpf_link__destroy(link); + link = NULL; + } + if (donor_started) + pthread_join(donor, NULL); + if (owner_started) + pthread_join(owner, NULL); + for (i = nr_contenders_joined; + i < nr_contenders_started; i++) + pthread_join(contenders[i], NULL); + } + + if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_GET_STATS, &stats)) { + SCX_ERR("Failed to read mutex statistics (%d)", errno); + status = SCX_TEST_FAIL; + } else { + result->stats = stats; + printf("\n[topology=%s SCX_OPS_ENQ_BLOCKED=%s]\n", + cross_cpu ? "cross-cpu" : "same-cpu", + enq_blocked ? "enabled" : "disabled"); + printf(" proxy_exec=%s\n", + proxy_enabled ? "enabled" : "disabled"); + printf(" donor_cpu=%d\n", thread_ctx.donor_cpu); + printf(" owner_cpu=%d\n", thread_ctx.owner_cpu); + printf(" nr_contenders=%zu\n", nr_contenders); + printf(" measured_trials=%d\n", NR_MEASURED_TRIALS); + printf(" owner_nice=%d\n", OWNER_NICE); + printf(" donor_nice=%d\n", DONOR_NICE); + printf(" contender_nice=%d\n", CONTENDER_NICE); + print_avg_time("mutex_hold", stats.hold_time_ns, stats.nr_holds); + print_avg_time("mutex_wait", stats.wait_time_ns, stats.nr_waits); + if (stats.nr_holds != NR_MEASURED_TRIALS || + stats.nr_waits != NR_MEASURED_TRIALS) { + SCX_ERR("Expected %d measured trials, got %llu holds and %llu waits", + NR_MEASURED_TRIALS, + (unsigned long long)stats.nr_holds, + (unsigned long long)stats.nr_waits); + status = SCX_TEST_FAIL; + } + } + + nr_blocked = skel->bss->nr_blocked_enqueues; + nr_blocked_donor_cpu = skel->bss->nr_blocked_enqueues_donor_cpu; + nr_blocked_owner_cpu = skel->bss->nr_blocked_enqueues_owner_cpu; + nr_blocked_other_cpu = skel->bss->nr_blocked_enqueues_other_cpu; + nr_blocked_wakeups = skel->bss->nr_blocked_wakeups; + result->nr_blocked_enqueues = nr_blocked; + result->nr_blocked_enqueues_donor_cpu = nr_blocked_donor_cpu; + result->nr_blocked_enqueues_owner_cpu = nr_blocked_owner_cpu; + result->nr_blocked_enqueues_other_cpu = nr_blocked_other_cpu; + result->nr_blocked_wakeups = nr_blocked_wakeups; + printf(" nr_blocked_enqueues=%llu\n", + (unsigned long long)nr_blocked); + printf(" nr_blocked_enqueues_donor_cpu=%llu\n", + (unsigned long long)nr_blocked_donor_cpu); + printf(" nr_blocked_enqueues_owner_cpu=%llu\n", + (unsigned long long)nr_blocked_owner_cpu); + printf(" nr_blocked_enqueues_other_cpu=%llu\n", + (unsigned long long)nr_blocked_other_cpu); + printf(" nr_blocked_wakeups=%llu\n", + (unsigned long long)nr_blocked_wakeups); + if (status == SCX_TEST_PASS) { + if (enq_blocked && proxy_enabled && !nr_blocked) { + SCX_ERR("ops.enqueue() did not receive the blocked donor"); + status = SCX_TEST_FAIL; + } else if ((!enq_blocked || !proxy_enabled) && nr_blocked) { + SCX_ERR("ops.enqueue() unexpectedly received %llu blocked donors", + (unsigned long long)nr_blocked); + status = SCX_TEST_FAIL; + } else if (nr_blocked_wakeups) { + SCX_ERR("Ordinary wakeups received %llu blocked enqueue flags", + (unsigned long long)nr_blocked_wakeups); + status = SCX_TEST_FAIL; + } else if (nr_blocked_other_cpu) { + SCX_ERR("Blocked donor had %llu enqueues on unexpected CPUs", + (unsigned long long)nr_blocked_other_cpu); + status = SCX_TEST_FAIL; + } + } + + if (UEI_EXITED(skel, uei)) { + SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)", + (unsigned long long)skel->data->uei.kind, + (long long)skel->data->uei.exit_code); + status = SCX_TEST_FAIL; + } + + if (link) + bpf_link__destroy(link); +out_fd: + close(thread_ctx.fd); +out_module: + unload_test_module(module_loaded); +out_skel: + enq_blocked__destroy(skel); +out_contenders: + free(contender_ctxs); + free(contenders); + return status; +} + +static enum scx_test_status run_topology(bool cross_cpu) +{ + struct run_result disabled = {}, enabled = {}; + enum scx_test_status status; + + status = run_one(false, cross_cpu, &disabled); + if (status != SCX_TEST_PASS) + return status; + + status = run_one(true, cross_cpu, &enabled); + if (status != SCX_TEST_PASS) + return status; + + printf("\n[topology=%s delta: enabled - disabled]\n", + cross_cpu ? "cross-cpu" : "same-cpu"); + print_avg_delta("mutex_hold", disabled.stats.hold_time_ns, + disabled.stats.nr_holds, enabled.stats.hold_time_ns, + enabled.stats.nr_holds); + print_avg_delta("mutex_wait", disabled.stats.wait_time_ns, + disabled.stats.nr_waits, enabled.stats.wait_time_ns, + enabled.stats.nr_waits); + + return SCX_TEST_PASS; +} + +static enum scx_test_status run(void *ctx) +{ + enum scx_test_status status; + + (void)ctx; + + status = run_topology(false); + if (status != SCX_TEST_PASS) + return status; + + status = run_topology(true); + if (status == SCX_TEST_SKIP) + return SCX_TEST_PASS; + + return status; +} + +struct scx_test enq_blocked = { + .name = "enq_blocked", + .description = "Verify proxy donor admission under CPU-wide contention", + .setup = setup, + .run = run, +}; + +REGISTER_SCX_TEST(&enq_blocked) diff --git a/tools/testing/selftests/sched_ext/enq_blocked.h b/tools/testing/selftests/sched_ext/enq_blocked.h new file mode 100644 index 000000000000..ef1eb97feebe --- /dev/null +++ b/tools/testing/selftests/sched_ext/enq_blocked.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES */ +#ifndef __ENQ_BLOCKED_H +#define __ENQ_BLOCKED_H + +#include <linux/ioctl.h> +#include <linux/types.h> + +struct enq_blocked_stats { + __u64 hold_time_ns; + __u64 wait_time_ns; + __u64 nr_holds; + __u64 nr_waits; +}; + +#define ENQ_BLOCKED_IOCTL_OWNER _IO('s', 1) +#define ENQ_BLOCKED_IOCTL_DONOR _IO('s', 2) +#define ENQ_BLOCKED_IOCTL_RESET_STATS _IO('s', 3) +#define ENQ_BLOCKED_IOCTL_GET_STATS _IOR('s', 4, struct enq_blocked_stats) +#define ENQ_BLOCKED_IOCTL_PREP_ATTACH _IO('s', 5) +#define ENQ_BLOCKED_IOCTL_ATTACH_DONE _IO('s', 6) +#define ENQ_BLOCKED_IOCTL_DONOR_STATE _IO('s', 7) +#define ENQ_BLOCKED_IOCTL_PROXY_SUPPORTED _IO('s', 8) + +#define ENQ_BLOCKED_DONOR_SLEEPING (1U << 0) +#define ENQ_BLOCKED_DONOR_ON_RQ (1U << 1) + +#endif /* __ENQ_BLOCKED_H */ diff --git a/tools/testing/selftests/sched_ext/exit.c b/tools/testing/selftests/sched_ext/exit.c index 01b17092d5c8..fd6f1c9ab2db 100644 --- a/tools/testing/selftests/sched_ext/exit.c +++ b/tools/testing/selftests/sched_ext/exit.c @@ -43,8 +43,12 @@ static enum scx_test_status run(void *ctx) } /* Assumes uei.kind is written last */ - while (skel->data->uei.kind == EXIT_KIND(SCX_EXIT_NONE)) - sched_yield(); + if (!SCX_TIMED_WAIT_UEI(skel, uei, 10000)) { + SCX_ERR("Timed out waiting for scheduler to exit"); + bpf_link__destroy(link); + exit__destroy(skel); + return SCX_TEST_FAIL; + } SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG_BPF)); SCX_EQ(skel->data->uei.exit_code, tc); diff --git a/tools/testing/selftests/sched_ext/hotplug.c b/tools/testing/selftests/sched_ext/hotplug.c index 0cfbb111a2d0..3c07686a93c8 100644 --- a/tools/testing/selftests/sched_ext/hotplug.c +++ b/tools/testing/selftests/sched_ext/hotplug.c @@ -21,15 +21,17 @@ static bool is_cpu_online(void) return file_read_long(online_path) > 0; } -static void toggle_online_status(bool online) +static int toggle_online_status(bool online) { long val = online ? 1 : 0; int ret; ret = file_write_long(online_path, val); if (ret != 0) - fprintf(stderr, "Failed to bring CPU %s (%s)", + fprintf(stderr, "Failed to bring CPU %s (%s)\n", online ? "online" : "offline", strerror(errno)); + + return ret; } static enum scx_test_status setup(void **ctx) @@ -44,6 +46,7 @@ static enum scx_test_status test_hotplug(bool onlining, bool cbs_defined) { struct hotplug *skel; struct bpf_link *link; + enum scx_test_status status = SCX_TEST_FAIL; long kind, code; SCX_ASSERT(is_cpu_online()); @@ -54,8 +57,8 @@ static enum scx_test_status test_hotplug(bool onlining, bool cbs_defined) SCX_FAIL_IF(hotplug__load(skel), "Failed to load skel"); /* Testing the offline -> online path, so go offline before starting */ - if (onlining) - toggle_online_status(0); + if (onlining && toggle_online_status(0)) + goto out_destroy_skel; if (cbs_defined) { kind = SCX_KIND_VAL(SCX_EXIT_UNREG_BPF); @@ -79,28 +82,34 @@ static enum scx_test_status test_hotplug(bool onlining, bool cbs_defined) return SCX_TEST_FAIL; } - toggle_online_status(onlining ? 1 : 0); + if (toggle_online_status(onlining ? 1 : 0)) + goto out_destroy_link; - while (!UEI_EXITED(skel, uei)) - sched_yield(); + if (!SCX_TIMED_WAIT_UEI(skel, uei, 10000)) { + SCX_ERR("Timed out waiting for scheduler to exit"); + goto out_destroy_link; + } SCX_EQ(skel->data->uei.kind, kind); SCX_EQ(UEI_REPORT(skel, uei), code); - if (!onlining) - toggle_online_status(1); + if (!onlining && toggle_online_status(1)) + goto out_destroy_link; + status = SCX_TEST_PASS; +out_destroy_link: bpf_link__destroy(link); +out_destroy_skel: hotplug__destroy(skel); - return SCX_TEST_PASS; + return status; } static enum scx_test_status test_hotplug_attach(void) { struct hotplug *skel; struct bpf_link *link; - enum scx_test_status status = SCX_TEST_PASS; + enum scx_test_status status = SCX_TEST_FAIL; long kind, code; SCX_ASSERT(is_cpu_online()); @@ -115,14 +124,18 @@ static enum scx_test_status test_hotplug_attach(void) * Take the CPU offline to increment the global hotplug seq, which * should cause attach to fail due to us setting the hotplug seq above */ - toggle_online_status(0); + if (toggle_online_status(0)) + goto out_destroy_skel; link = bpf_map__attach_struct_ops(skel->maps.hotplug_nocb_ops); - toggle_online_status(1); + if (toggle_online_status(1)) + goto out_destroy_link; SCX_ASSERT(link); - while (!UEI_EXITED(skel, uei)) - sched_yield(); + if (!SCX_TIMED_WAIT_UEI(skel, uei, 10000)) { + SCX_ERR("Timed out waiting for scheduler to exit"); + goto out_destroy_link; + } kind = SCX_KIND_VAL(SCX_EXIT_UNREG_KERN); code = SCX_ECODE_VAL(SCX_ECODE_ACT_RESTART) | @@ -130,7 +143,10 @@ static enum scx_test_status test_hotplug_attach(void) SCX_EQ(skel->data->uei.kind, kind); SCX_EQ(UEI_REPORT(skel, uei), code); + status = SCX_TEST_PASS; +out_destroy_link: bpf_link__destroy(link); +out_destroy_skel: hotplug__destroy(skel); return status; diff --git a/tools/testing/selftests/sched_ext/init_enable_count.bpf.c b/tools/testing/selftests/sched_ext/init_enable_count.bpf.c index 5eb9edb1837d..c0a5dd8fe183 100644 --- a/tools/testing/selftests/sched_ext/init_enable_count.bpf.c +++ b/tools/testing/selftests/sched_ext/init_enable_count.bpf.c @@ -15,6 +15,8 @@ char _license[] SEC("license") = "GPL"; u64 init_task_cnt, exit_task_cnt, enable_cnt, disable_cnt; u64 init_fork_cnt, init_transition_cnt; +UEI_DEFINE(uei); + s32 BPF_STRUCT_OPS_SLEEPABLE(cnt_init_task, struct task_struct *p, struct scx_init_task_args *args) { @@ -43,11 +45,17 @@ void BPF_STRUCT_OPS(cnt_disable, struct task_struct *p) __sync_fetch_and_add(&disable_cnt, 1); } +void BPF_STRUCT_OPS(cnt_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + SEC(".struct_ops.link") struct sched_ext_ops init_enable_count_ops = { .init_task = (void *) cnt_init_task, .exit_task = (void *) cnt_exit_task, .enable = (void *) cnt_enable, .disable = (void *) cnt_disable, + .exit = (void *) cnt_exit, .name = "init_enable_count", }; diff --git a/tools/testing/selftests/sched_ext/init_enable_count.c b/tools/testing/selftests/sched_ext/init_enable_count.c index 44577e30e764..c4a57038a1d6 100644 --- a/tools/testing/selftests/sched_ext/init_enable_count.c +++ b/tools/testing/selftests/sched_ext/init_enable_count.c @@ -5,6 +5,7 @@ * Copyright (c) 2023 Tejun Heo <tj@kernel.org> */ #include <signal.h> +#include <string.h> #include <stdio.h> #include <unistd.h> #include <sched.h> @@ -72,7 +73,17 @@ static enum scx_test_status run_test(bool global) close(pipe_fds[1]); signal(SIGCHLD, SIG_DFL); + SCX_ASSERT_ALIVE(skel, uei); + bpf_link__destroy(link); + + /* + * The clean unregister above is recorded in uei and would + * otherwise trip the liveness check of the second cycle, which + * reuses this skeleton. + */ + memset(&skel->data->uei, 0, sizeof(skel->data->uei)); + SCX_GE(skel->bss->init_task_cnt, num_pre_forks); SCX_GE(skel->bss->exit_task_cnt, num_pre_forks); @@ -124,6 +135,8 @@ static enum scx_test_status run_test(bool global) status); } + SCX_ASSERT_ALIVE(skel, uei); + bpf_link__destroy(link); SCX_GE(skel->bss->init_task_cnt, 2 * num_children); diff --git a/tools/testing/selftests/sched_ext/kick.bpf.c b/tools/testing/selftests/sched_ext/kick.bpf.c new file mode 100644 index 000000000000..bd9605d1573d --- /dev/null +++ b/tools/testing/selftests/sched_ext/kick.bpf.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + */ +#include <scx/common.bpf.h> + +#include "kick_test.h" + +char _license[] SEC("license") = "GPL"; + +const volatile u32 scenario; +const volatile s32 victim_pid; +const volatile s32 challenger_pid; +const volatile s32 target_cpu; +const volatile s32 lazy_resched_override = -1; + +u32 state; +u64 slice_before; +u64 slice_at_resched; +s32 resched_tif; +u64 nr_wait_callbacks; +bool enq_preempt_issued; + +UEI_DEFINE(uei); + +static bool is_trace_scenario(void) +{ + return scenario <= TICK_EXPIRY; +} + +void BPF_STRUCT_OPS(kick_enqueue, struct task_struct *p, u64 enq_flags) +{ + switch (scenario) { + case ENQ_BOTH: + if (p->pid == victim_pid) { + scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_INF, enq_flags); + } else if (p->pid == challenger_pid && state == KICK_STATE_QUEUED) { + enq_preempt_issued = true; + scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, + enq_flags | SCX_ENQ_PREEMPT | + SCX_ENQ_PREEMPT_LAZY); + } else { + scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); + } + goto kick_last; + case INVALID_KICK_IDLE: + scx_bpf_kick_cpu(scx_bpf_task_cpu(p), SCX_KICK_PREEMPT_LAZY | SCX_KICK_IDLE); + break; + case INVALID_KICK_UNKNOWN: + scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 1LLU << 63); + break; + } + + scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); +kick_last: + if (enq_flags & SCX_ENQ_LAST) + scx_bpf_kick_cpu(target_cpu, SCX_KICK_IDLE); +} + +static void set_lazy_resched_override(struct task_struct *p) +{ + if (p->pid == victim_pid && lazy_resched_override >= 0) + scx_bpf_task_set_lazy_resched(p, lazy_resched_override); +} + +void BPF_STRUCT_OPS(kick_running, struct task_struct *p) +{ + if (!is_trace_scenario() || p->pid != victim_pid) + return; + set_lazy_resched_override(p); + if (__sync_val_compare_and_swap(&state, KICK_STATE_ARMED, KICK_STATE_QUEUED) != + KICK_STATE_ARMED) + return; + + slice_before = p->scx.slice; + + switch (scenario) { + case KICK_IMMEDIATE: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT); + break; + case KICK_LAZY: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY); + break; + case KICK_LAZY_THEN_IMMEDIATE: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY); + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT); + break; + case KICK_IMMEDIATE_THEN_LAZY: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT); + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY); + break; + case KICK_PLAIN_THEN_LAZY: + scx_bpf_kick_cpu(target_cpu, 0); + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY); + break; + case KICK_LAZY_THEN_PLAIN: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY); + scx_bpf_kick_cpu(target_cpu, 0); + break; + case KICK_BOTH: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT | SCX_KICK_PREEMPT_LAZY); + break; + case KICK_LAZY_WAIT: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY | SCX_KICK_WAIT); + break; + case ENQ_BOTH: + /* The userspace controller wakes a competing task. */ + break; + case TICK_EXPIRY: + /* no kick: the slice runs out at the tick */ + break; + } +} + +SEC("fexit/__resched_curr") +int BPF_PROG(kick_need_resched, struct rq *rq, int tif) +{ + struct task_struct *task = BPF_CORE_READ(rq, curr); + u64 slice; + + if (!task || BPF_CORE_READ(task, pid) != victim_pid || + BPF_CORE_READ(rq, cpu) != target_cpu || state != KICK_STATE_QUEUED) + return 0; + if (scenario == ENQ_BOTH && !enq_preempt_issued) + return 0; + + slice = BPF_CORE_READ(task, scx.slice); + if (slice) + return 0; + + slice_at_resched = slice; + resched_tif = tif; + state = KICK_STATE_RESCHED; + return 0; +} + +SEC("fentry/kick_sync_wait_bal_cb") +int BPF_PROG(kick_wait_callback, struct rq *rq) +{ + if (scenario == KICK_LAZY_WAIT && BPF_CORE_READ(rq, cpu) == target_cpu) + __sync_fetch_and_add(&nr_wait_callbacks, 1); + return 0; +} + +void BPF_STRUCT_OPS(kick_stopping, struct task_struct *p, bool runnable) +{ + if (p->pid == victim_pid && state == KICK_STATE_RESCHED) + state = KICK_STATE_DONE; +} + +void BPF_STRUCT_OPS(kick_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + +SEC(".struct_ops.link") +struct sched_ext_ops kick_ops = { + .enqueue = (void *)kick_enqueue, + .running = (void *)kick_running, + .stopping = (void *)kick_stopping, + .exit = (void *)kick_exit, + .flags = SCX_OPS_ENQ_LAST, + .name = "kick", +}; diff --git a/tools/testing/selftests/sched_ext/kick.c b/tools/testing/selftests/sched_ext/kick.c new file mode 100644 index 000000000000..40c91d54c1eb --- /dev/null +++ b/tools/testing/selftests/sched_ext/kick.c @@ -0,0 +1,521 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + */ +#define _GNU_SOURCE +#include <bpf/bpf.h> +#include <errno.h> +#include <linux/sched.h> +#include <sched.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/prctl.h> +#include <sys/wait.h> +#include <unistd.h> +#include <scx/common.h> + +#include "kick.bpf.skel.h" +#include "kick_test.h" +#include "scx_test.h" +#include "util.h" + +#define WAIT_LOOPS 3000 + +struct observation { + u64 slice_before; + u64 slice_at_resched; + u64 nr_wait_callbacks; + s32 resched_tif; +}; + +struct kick_ctx { + cpu_set_t original_mask; + int target_cpu; +}; + +static bool enum_supported(const char *type, const char *name) +{ + u64 value; + + return __COMPAT_read_enum(type, name, &value); +} + +static enum scx_test_status setup_controller(void **ctx_ptr) +{ + struct kick_ctx *ctx; + cpu_set_t controller_mask; + int cpu, controller_cpu = -1; + + ctx = calloc(1, sizeof(*ctx)); + SCX_FAIL_IF(!ctx, "Failed to allocate context"); + if (sched_getaffinity(0, sizeof(ctx->original_mask), &ctx->original_mask)) { + free(ctx); + SCX_FAIL("Failed to get affinity (%d)", errno); + } + + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) { + if (!CPU_ISSET(cpu, &ctx->original_mask)) + continue; + if (controller_cpu < 0) { + controller_cpu = cpu; + } else { + ctx->target_cpu = cpu; + break; + } + } + if (cpu == CPU_SETSIZE) { + printf("SKIP: two allowed CPUs are required\n"); + free(ctx); + return SCX_TEST_SKIP; + } + + CPU_ZERO(&controller_mask); + CPU_SET(controller_cpu, &controller_mask); + if (sched_setaffinity(0, sizeof(controller_mask), &controller_mask)) { + free(ctx); + SCX_FAIL("Failed to pin controller to CPU %d (%d)", controller_cpu, errno); + } + + *ctx_ptr = ctx; + return SCX_TEST_PASS; +} + +static void cleanup_controller(void *ctx_ptr) +{ + struct kick_ctx *ctx = ctx_ptr; + + sched_setaffinity(0, sizeof(ctx->original_mask), &ctx->original_mask); + free(ctx); +} + +static bool wait_for_state(struct kick *skel, u32 wanted) +{ + int i; + + for (i = 0; i < WAIT_LOOPS; i++) { + if (__atomic_load_n(&skel->bss->state, __ATOMIC_ACQUIRE) == wanted) + return true; + if (UEI_EXITED(skel, uei)) + return false; + usleep(1000); + } + return false; +} + +static bool wait_for_counter(struct kick *skel, const u64 *counter, u64 wanted) +{ + int i; + + for (i = 0; i < WAIT_LOOPS; i++) { + if (__atomic_load_n(counter, __ATOMIC_ACQUIRE) >= wanted) + return true; + if (UEI_EXITED(skel, uei)) + return false; + usleep(1000); + } + return false; +} + +static enum scx_test_status trace_one(struct kick_ctx *ctx, u32 scenario, u64 ops_flags, + s32 lazy_resched_override, struct observation *obs) +{ + struct bpf_link *ops_link = NULL; + struct kick *skel = NULL; + struct scx_test_gated_worker challenger = { .pid = -1, .start_fd = -1 }; + struct scx_test_gated_worker victim; + enum scx_test_status ret = SCX_TEST_FAIL; + int cpu = ctx->target_cpu; + + victim = scx_test_spawn_gated_worker(cpu, false); + if (victim.pid < 0) { + SCX_ERR("Failed to spawn victim"); + return SCX_TEST_FAIL; + } + if (scenario == ENQ_BOTH) { + challenger = scx_test_spawn_gated_worker(cpu, false); + if (challenger.pid < 0) { + SCX_ERR("Failed to spawn enqueue challenger"); + goto out; + } + } + + skel = kick__open(); + if (!skel) { + SCX_ERR("Failed to open scenario %u", scenario); + goto out; + } + SCX_ENUM_INIT(skel); + skel->rodata->scenario = scenario; + skel->rodata->victim_pid = victim.pid; + skel->rodata->challenger_pid = challenger.pid; + skel->rodata->target_cpu = cpu; + skel->rodata->lazy_resched_override = lazy_resched_override; + skel->struct_ops.kick_ops->flags |= ops_flags; + if (scenario != KICK_LAZY_WAIT) + bpf_program__set_autoload(skel->progs.kick_wait_callback, false); + if (kick__load(skel)) { + SCX_ERR("Failed to load scenario %u", scenario); + goto out; + } + + bpf_map__set_autoattach(skel->maps.kick_ops, false); + if (kick__attach(skel)) { + SCX_ERR("Failed to attach __resched_curr tracer"); + goto out; + } + skel->bss->state = KICK_STATE_ARMED; + ops_link = bpf_map__attach_struct_ops(skel->maps.kick_ops); + if (!ops_link) { + SCX_ERR("Failed to attach scenario %u", scenario); + goto out; + } + if (!scx_test_start_gated_worker(&victim)) { + SCX_ERR("Failed to start victim"); + goto out; + } + if (scenario == ENQ_BOTH) { + if (!wait_for_state(skel, KICK_STATE_QUEUED)) { + SCX_ERR("Enqueue scenario did not arm"); + goto out; + } + if (!scx_test_start_gated_worker(&challenger)) { + SCX_ERR("Failed to start enqueue challenger"); + goto out; + } + } + if (!wait_for_state(skel, KICK_STATE_DONE)) { + SCX_ERR("Scenario %u stopped in state %u, exit kind %d", scenario, skel->bss->state, + skel->data->uei.kind); + goto out; + } + if (scenario == KICK_LAZY_WAIT && + !wait_for_counter(skel, &skel->bss->nr_wait_callbacks, 1)) { + SCX_ERR("WAIT callback did not run"); + goto out; + } + + obs->slice_before = skel->bss->slice_before; + obs->slice_at_resched = skel->bss->slice_at_resched; + obs->nr_wait_callbacks = skel->bss->nr_wait_callbacks; + obs->resched_tif = skel->bss->resched_tif; + ret = SCX_TEST_PASS; +out: + scx_test_stop_gated_worker(&victim); + scx_test_stop_gated_worker(&challenger); + if (ops_link) + bpf_link__destroy(ops_link); + if (skel) + kick__destroy(skel); + return ret; +} + +static bool observation_valid(const struct observation *obs) +{ + return obs->slice_before > 0 && !obs->slice_at_resched; +} + +static bool trace_target_supported(const char *name, enum bpf_attach_type attach_type) +{ + return libbpf_find_vmlinux_btf_id(name, attach_type) > 0; +} + +static enum scx_test_status setup_immediate(void **ctx) +{ + if (!enum_supported("scx_kick_flags", "SCX_KICK_PREEMPT")) { + printf("SKIP: SCX_KICK_PREEMPT is not supported\n"); + return SCX_TEST_SKIP; + } + if (!trace_target_supported("__resched_curr", BPF_TRACE_FEXIT)) { + printf("SKIP: __resched_curr is not available in BTF\n"); + return SCX_TEST_SKIP; + } + return setup_controller(ctx); +} + +static enum scx_test_status setup_lazy(void **ctx) +{ + if (!enum_supported("scx_kick_flags", "SCX_KICK_PREEMPT_LAZY")) { + printf("SKIP: SCX_KICK_PREEMPT_LAZY is not supported\n"); + return SCX_TEST_SKIP; + } + return setup_immediate(ctx); +} + +static enum scx_test_status setup_tick(void **ctx) +{ + if (!enum_supported("scx_ops_flags", "SCX_OPS_LAZY_RESCHED")) { + printf("SKIP: SCX_OPS_LAZY_RESCHED is not supported\n"); + return SCX_TEST_SKIP; + } + return setup_lazy(ctx); +} + +static enum scx_test_status setup_coalesce(void **ctx) +{ + if (!enum_supported("scx_enq_flags", "SCX_ENQ_PREEMPT_LAZY")) { + printf("SKIP: SCX_ENQ_PREEMPT_LAZY is not supported\n"); + return SCX_TEST_SKIP; + } + if (!trace_target_supported("kick_sync_wait_bal_cb", BPF_TRACE_FENTRY)) { + printf("SKIP: kick_sync_wait_bal_cb is not available in BTF\n"); + return SCX_TEST_SKIP; + } + return setup_lazy(ctx); +} + +static enum scx_test_status setup_invalid(void **ctx) +{ + if (!enum_supported("scx_kick_flags", "SCX_KICK_PREEMPT_LAZY")) { + printf("SKIP: SCX_KICK_PREEMPT_LAZY is not supported\n"); + return SCX_TEST_SKIP; + } + return setup_controller(ctx); +} + +static enum scx_test_status run_immediate(void *ctx) +{ + struct observation obs; + enum scx_test_status status; + + status = trace_one(ctx, KICK_IMMEDIATE, 0, -1, &obs); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&obs)); + return SCX_TEST_PASS; +} + +static enum scx_test_status run_lazy(void *ctx) +{ + struct observation immediate, lazy; + enum scx_test_status status; + int lazy_mode = scx_test_preempt_lazy_mode(); + + if (lazy_mode < 0) { + printf("SKIP: running kernel preemption mode is unavailable\n"); + return SCX_TEST_SKIP; + } + + status = trace_one(ctx, KICK_IMMEDIATE, 0, -1, &immediate); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&immediate)); + status = trace_one(ctx, KICK_LAZY, 0, -1, &lazy); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&lazy)); + + if (lazy_mode > 0) + SCX_FAIL_IF(lazy.resched_tif == immediate.resched_tif, + "Lazy mode used immediate TIF %d", lazy.resched_tif); + else if (!lazy_mode) + SCX_EQ(lazy.resched_tif, immediate.resched_tif); + + return SCX_TEST_PASS; +} + +static enum scx_test_status run_coalesce(void *ctx) +{ + struct observation immediate, lazy_first, immediate_first; + struct observation plain_first, lazy_then_plain; + struct observation both, lazy_wait, enq_both; + enum scx_test_status status; + + if (scx_test_preempt_lazy_mode() != 1) { + printf("SKIP: kick coalescing requires active lazy preemption\n"); + return SCX_TEST_SKIP; + } + + status = trace_one(ctx, KICK_IMMEDIATE, 0, -1, &immediate); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&immediate)); + status = trace_one(ctx, KICK_LAZY_THEN_IMMEDIATE, 0, -1, &lazy_first); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&lazy_first)); + status = trace_one(ctx, KICK_IMMEDIATE_THEN_LAZY, 0, -1, &immediate_first); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&immediate_first)); + SCX_EQ(lazy_first.resched_tif, immediate.resched_tif); + SCX_EQ(immediate_first.resched_tif, immediate.resched_tif); + + /* + * A plain kick in the same batch doesn't clear the slice by itself. The + * lazy preemption must still expire it, and the plain kick must still + * reschedule immediately, whichever came first. + */ + status = trace_one(ctx, KICK_PLAIN_THEN_LAZY, 0, -1, &plain_first); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&plain_first)); + status = trace_one(ctx, KICK_LAZY_THEN_PLAIN, 0, -1, &lazy_then_plain); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&lazy_then_plain)); + SCX_EQ(plain_first.resched_tif, immediate.resched_tif); + SCX_EQ(lazy_then_plain.resched_tif, immediate.resched_tif); + + /* Immediate kick and WAIT both take precedence over lazy preemption. */ + status = trace_one(ctx, KICK_BOTH, 0, -1, &both); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&both)); + status = trace_one(ctx, KICK_LAZY_WAIT, 0, -1, &lazy_wait); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&lazy_wait)); + SCX_EQ(both.resched_tif, immediate.resched_tif); + SCX_EQ(lazy_wait.resched_tif, immediate.resched_tif); + SCX_GT(lazy_wait.nr_wait_callbacks, 0); + + /* Immediate enqueue preemption likewise takes precedence over lazy. */ + status = trace_one(ctx, ENQ_BOTH, 0, -1, &enq_both); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&enq_both)); + SCX_EQ(enq_both.resched_tif, immediate.resched_tif); + return SCX_TEST_PASS; +} + +/* + * A slice running out at the tick reschedules immediately by default and lazily + * with SCX_OPS_LAZY_RESCHED, the way fair.c expires a slice. + */ +static enum scx_test_status run_tick(void *ctx) +{ + struct observation immediate, lazy, force_lazy, force_immediate; + enum scx_test_status status; + u64 lazy_flag; + int lazy_mode = scx_test_preempt_lazy_mode(); + bool found; + + if (lazy_mode < 0) { + printf("SKIP: running kernel preemption mode is unavailable\n"); + return SCX_TEST_SKIP; + } + + found = __COMPAT_read_enum("scx_ops_flags", "SCX_OPS_LAZY_RESCHED", &lazy_flag); + SCX_ASSERT(found); + status = trace_one(ctx, TICK_EXPIRY, 0, -1, &immediate); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&immediate)); + status = trace_one(ctx, TICK_EXPIRY, lazy_flag, -1, &lazy); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&lazy)); + if (lazy_mode > 0) { + SCX_FAIL_IF(lazy.resched_tif == immediate.resched_tif, + "Lazy slice expiry used immediate TIF %d", lazy.resched_tif); + + status = trace_one(ctx, TICK_EXPIRY, 0, 1, &force_lazy); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&force_lazy)); + status = trace_one(ctx, TICK_EXPIRY, lazy_flag, 0, &force_immediate); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&force_immediate)); + SCX_EQ(force_lazy.resched_tif, lazy.resched_tif); + SCX_EQ(force_immediate.resched_tif, immediate.resched_tif); + } else { + SCX_EQ(lazy.resched_tif, immediate.resched_tif); + } + + return SCX_TEST_PASS; +} + +static enum scx_test_status invalid_one(struct kick_ctx *ctx, u32 scenario) +{ + struct bpf_link *ops_link = NULL; + struct kick *skel = NULL; + struct scx_test_gated_worker victim; + enum scx_test_status ret = SCX_TEST_FAIL; + int cpu = ctx->target_cpu; + int i; + u64 error_kind; + + error_kind = EXIT_KIND(SCX_EXIT_ERROR); + victim = scx_test_spawn_gated_worker(cpu, false); + if (victim.pid < 0) + return SCX_TEST_FAIL; + + skel = kick__open(); + if (!skel) + goto out; + SCX_ENUM_INIT(skel); + skel->rodata->scenario = scenario; + bpf_program__set_autoload(skel->progs.kick_need_resched, false); + bpf_program__set_autoload(skel->progs.kick_wait_callback, false); + if (kick__load(skel)) + goto out; + ops_link = bpf_map__attach_struct_ops(skel->maps.kick_ops); + if (!ops_link || !scx_test_start_gated_worker(&victim)) + goto out; + + for (i = 0; i < WAIT_LOOPS; i++) { + if (UEI_EXITED(skel, uei) == error_kind) { + ret = SCX_TEST_PASS; + break; + } + usleep(1000); + } +out: + scx_test_stop_gated_worker(&victim); + if (ops_link) + bpf_link__destroy(ops_link); + if (skel) + kick__destroy(skel); + return ret; +} + +static enum scx_test_status run_invalid(void *ctx) +{ + enum scx_test_status status; + u32 scenario; + + for (scenario = INVALID_KICK_IDLE; + scenario <= INVALID_KICK_UNKNOWN; scenario++) { + status = invalid_one(ctx, scenario); + SCX_EQ(status, SCX_TEST_PASS); + } + return SCX_TEST_PASS; +} + +static struct scx_test kick_immediate = { + .name = "kick_immediate", + .description = "Trace immediate kick slice expiration and rescheduling", + .setup = setup_immediate, + .run = run_immediate, + .cleanup = cleanup_controller, +}; + +static struct scx_test kick_lazy = { + .name = "kick_lazy", + .description = "Trace lazy kick slice expiration and rescheduling", + .setup = setup_lazy, + .run = run_lazy, + .cleanup = cleanup_controller, +}; + +static struct scx_test kick_coalesce = { + .name = "kick_coalesce", + .description = "Verify lazy preemption coalesces with immediate requests", + .setup = setup_coalesce, + .run = run_coalesce, + .cleanup = cleanup_controller, +}; + +static struct scx_test kick_tick = { + .name = "kick_tick", + .description = "Trace slice expiry at the tick, immediate and lazy", + .setup = setup_tick, + .run = run_tick, + .cleanup = cleanup_controller, +}; + +static struct scx_test kick_invalid = { + .name = "kick_invalid", + .description = "Verify invalid kick flag combinations fail", + .setup = setup_invalid, + .run = run_invalid, + .cleanup = cleanup_controller, +}; + +__attribute__((constructor)) +static void register_kick_tests(void) +{ + scx_test_register(&kick_immediate); + scx_test_register(&kick_lazy); + scx_test_register(&kick_coalesce); + scx_test_register(&kick_tick); + scx_test_register(&kick_invalid); +} diff --git a/tools/testing/selftests/sched_ext/kick_test.h b/tools/testing/selftests/sched_ext/kick_test.h new file mode 100644 index 000000000000..7d3c608c6b63 --- /dev/null +++ b/tools/testing/selftests/sched_ext/kick_test.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES */ +#ifndef __KICK_TEST_H__ +#define __KICK_TEST_H__ + +enum kick_scenario { + KICK_IMMEDIATE, + KICK_LAZY, + KICK_LAZY_THEN_IMMEDIATE, + KICK_IMMEDIATE_THEN_LAZY, + KICK_PLAIN_THEN_LAZY, + KICK_LAZY_THEN_PLAIN, + KICK_BOTH, + KICK_LAZY_WAIT, + ENQ_BOTH, + TICK_EXPIRY, + INVALID_KICK_IDLE, + INVALID_KICK_UNKNOWN, +}; + +enum kick_state { + KICK_STATE_IDLE, + KICK_STATE_ARMED, + KICK_STATE_QUEUED, + KICK_STATE_RESCHED, + KICK_STATE_DONE, +}; + +#endif /* __KICK_TEST_H__ */ diff --git a/tools/testing/selftests/sched_ext/maximal.bpf.c b/tools/testing/selftests/sched_ext/maximal.bpf.c index 04a369078aac..9ea664e2ff70 100644 --- a/tools/testing/selftests/sched_ext/maximal.bpf.c +++ b/tools/testing/selftests/sched_ext/maximal.bpf.c @@ -14,6 +14,8 @@ char _license[] SEC("license") = "GPL"; #define DSQ_ID 0 +UEI_DEFINE(uei); + s32 BPF_STRUCT_OPS(maximal_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags) { @@ -132,7 +134,9 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(maximal_init) } void BPF_STRUCT_OPS(maximal_exit, struct scx_exit_info *info) -{} +{ + UEI_RECORD(uei, info); +} SEC(".struct_ops.link") struct sched_ext_ops maximal_ops = { diff --git a/tools/testing/selftests/sched_ext/maximal.c b/tools/testing/selftests/sched_ext/maximal.c index 1dc369224670..6c6b80e0446e 100644 --- a/tools/testing/selftests/sched_ext/maximal.c +++ b/tools/testing/selftests/sched_ext/maximal.c @@ -35,6 +35,8 @@ static enum scx_test_status run(void *ctx) link = bpf_map__attach_struct_ops(skel->maps.maximal_ops); SCX_FAIL_IF(!link, "Failed to attach scheduler"); + SCX_ASSERT_ALIVE(skel, uei); + bpf_link__destroy(link); return SCX_TEST_PASS; diff --git a/tools/testing/selftests/sched_ext/nohz_tick.bpf.c b/tools/testing/selftests/sched_ext/nohz_tick.bpf.c index 6998c5dd6bcb..5fec5673fbf2 100644 --- a/tools/testing/selftests/sched_ext/nohz_tick.bpf.c +++ b/tools/testing/selftests/sched_ext/nohz_tick.bpf.c @@ -6,13 +6,22 @@ */ #include <scx/common.bpf.h> +#include "nohz_tick_test.h" + char _license[] SEC("license") = "GPL"; const volatile s32 test_cpu; -bool finite_phase; +u32 phase; +s32 victim_pid; +s32 challenger_pid; +s32 trigger_pid; u64 nr_inf_running; u64 nr_finite_running; u64 nr_finite_ticks; +u64 nr_lazy_victim_running; +u64 nr_lazy_enq_running; +u64 nr_lazy_kick_running; +u64 nr_lazy_ticks; UEI_DEFINE(uei); @@ -24,9 +33,31 @@ s32 BPF_STRUCT_OPS(nohz_tick_select_cpu, struct task_struct *p, s32 prev_cpu, void BPF_STRUCT_OPS(nohz_tick_enqueue, struct task_struct *p, u64 enq_flags) { - u64 slice = finite_phase ? 1000000ULL : SCX_SLICE_INF; + u64 slice; + u64 dsq_id = SCX_DSQ_GLOBAL; + + switch (phase) { + case NOHZ_PHASE_INF: + slice = SCX_SLICE_INF; + break; + case NOHZ_PHASE_FINITE: + slice = 1000000ULL; + break; + case NOHZ_PHASE_LAZY_ENQ: + case NOHZ_PHASE_LAZY_KICK: + dsq_id = SCX_DSQ_LOCAL; + slice = p->pid == victim_pid ? SCX_SLICE_INF : SCX_SLICE_DFL; + if (phase == NOHZ_PHASE_LAZY_ENQ && p->pid == challenger_pid) + enq_flags |= SCX_ENQ_PREEMPT_LAZY; + break; + default: + slice = SCX_SLICE_DFL; + break; + } - scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, slice, enq_flags); + scx_bpf_dsq_insert(p, dsq_id, slice, enq_flags); + if (phase == NOHZ_PHASE_LAZY_KICK && p->pid == trigger_pid) + scx_bpf_kick_cpu(test_cpu, SCX_KICK_PREEMPT_LAZY); if (enq_flags & SCX_ENQ_LAST) scx_bpf_kick_cpu(test_cpu, SCX_KICK_IDLE); } @@ -36,16 +67,28 @@ void BPF_STRUCT_OPS(nohz_tick_running, struct task_struct *p) if (bpf_get_smp_processor_id() != test_cpu) return; - if (finite_phase) + if (phase == NOHZ_PHASE_FINITE) __sync_fetch_and_add(&nr_finite_running, 1); - else + else if (phase == NOHZ_PHASE_INF) __sync_fetch_and_add(&nr_inf_running, 1); + else if (p->pid == victim_pid) + __sync_fetch_and_add(&nr_lazy_victim_running, 1); + else if (phase == NOHZ_PHASE_LAZY_ENQ && p->pid == challenger_pid) + __sync_fetch_and_add(&nr_lazy_enq_running, 1); + else if (phase == NOHZ_PHASE_LAZY_KICK && p->pid == challenger_pid) + __sync_fetch_and_add(&nr_lazy_kick_running, 1); } void BPF_STRUCT_OPS(nohz_tick_tick, struct task_struct *p) { - if (bpf_get_smp_processor_id() == test_cpu && finite_phase) + if (bpf_get_smp_processor_id() != test_cpu) + return; + + if (phase == NOHZ_PHASE_FINITE) __sync_fetch_and_add(&nr_finite_ticks, 1); + else if ((phase == NOHZ_PHASE_LAZY_ENQ || + phase == NOHZ_PHASE_LAZY_KICK) && p->pid == victim_pid) + __sync_fetch_and_add(&nr_lazy_ticks, 1); } void BPF_STRUCT_OPS(nohz_tick_exit, struct scx_exit_info *ei) @@ -61,5 +104,5 @@ struct sched_ext_ops nohz_tick_ops = { .tick = (void *)nohz_tick_tick, .exit = (void *)nohz_tick_exit, .name = "nohz_tick", - .timeout_ms = 1000U, + .timeout_ms = 5000U, }; diff --git a/tools/testing/selftests/sched_ext/nohz_tick.c b/tools/testing/selftests/sched_ext/nohz_tick.c index 028f54391c2c..e19a97034b7f 100644 --- a/tools/testing/selftests/sched_ext/nohz_tick.c +++ b/tools/testing/selftests/sched_ext/nohz_tick.c @@ -21,19 +21,24 @@ #include <scx/common.h> #include "nohz_tick.bpf.skel.h" +#include "nohz_tick_test.h" #include "scx_test.h" +#include "util.h" #ifndef SCHED_EXT #define SCHED_EXT 7 #endif #define MIN_FINITE_TICKS 3 -#define PHASE_TIMEOUT_MS 1000 +#define PHASE_TIMEOUT_MS 5000 +#define TICK_STOP_STABLE_MS 100 struct nohz_tick_ctx { struct nohz_tick *skel; cpu_set_t original_mask; int test_cpu; + int housekeeping_cpu; + bool test_lazy; }; static int first_allowed_cpu(const cpu_set_t *mask, int first, int last) @@ -47,20 +52,21 @@ static int first_allowed_cpu(const cpu_set_t *mask, int first, int last) return -1; } -static int find_nohz_full_cpu(const cpu_set_t *allowed) +static int read_nohz_full_mask(cpu_set_t *mask) { char buf[4096], *cur, *end; FILE *file; + int ret = 0; file = fopen("/sys/devices/system/cpu/nohz_full", "r"); if (!file) - return -1; + return -errno; if (!fgets(buf, sizeof(buf), file)) { - fclose(file); - return -1; + ret = ferror(file) ? -errno : -EINVAL; + goto out; } - fclose(file); + CPU_ZERO(mask); cur = buf; while (*cur) { long first, last; @@ -73,25 +79,30 @@ static int find_nohz_full_cpu(const cpu_set_t *allowed) errno = 0; first = strtol(cur, &end, 10); - if (errno || end == cur || first < 0 || first >= CPU_SETSIZE) - return -1; + if (errno || end == cur || first < 0 || first >= CPU_SETSIZE) { + ret = -EINVAL; + goto out; + } cur = end; last = first; if (*cur == '-') { cur++; errno = 0; last = strtol(cur, &end, 10); - if (errno || end == cur || last < first) - return -1; + if (errno || end == cur || last < first) { + ret = -EINVAL; + goto out; + } cur = end; } - cpu = first_allowed_cpu(allowed, first, last); - if (cpu >= 0) - return cpu; + for (cpu = first; cpu <= last && cpu < CPU_SETSIZE; cpu++) + CPU_SET(cpu, mask); } - return -1; +out: + fclose(file); + return ret; } static pid_t start_worker(int cpu) @@ -146,14 +157,41 @@ static int pause_worker(pid_t pid) return 0; } -static bool wait_for_counter(const u64 *counter, u64 value, int timeout_ms) +static bool wait_for_counter(struct nohz_tick *skel, const u64 *counter, u64 value, + int timeout_ms) { int elapsed; for (elapsed = 0; elapsed < timeout_ms; elapsed++) { if (__atomic_load_n(counter, __ATOMIC_RELAXED) >= value) return true; + if (UEI_EXITED(skel, uei)) + return false; + usleep(1000); + } + + return false; +} + +static bool wait_for_tick_stop(struct nohz_tick *skel, const u64 *counter, int timeout_ms) +{ + u64 prev = __atomic_load_n(counter, __ATOMIC_RELAXED); + int elapsed, stable = 0; + + for (elapsed = 0; elapsed < timeout_ms; elapsed++) { + u64 curr; + usleep(1000); + if (UEI_EXITED(skel, uei)) + return false; + curr = __atomic_load_n(counter, __ATOMIC_RELAXED); + if (curr == prev) { + if (++stable >= TICK_STOP_STABLE_MS) + return true; + } else { + prev = curr; + stable = 0; + } } return false; @@ -162,8 +200,10 @@ static bool wait_for_counter(const u64 *counter, u64 value, int timeout_ms) static enum scx_test_status setup(void **ctx_ptr) { struct nohz_tick_ctx *ctx; - cpu_set_t controller_mask; - int cpu; + cpu_set_t controller_mask, nohz_full_mask; + bool lazy_supported; + u64 enum_value, lazy_ops_flag; + int cpu, i, lazy_mode, ret; ctx = calloc(1, sizeof(*ctx)); SCX_FAIL_IF(!ctx, "Failed to allocate context"); @@ -173,15 +213,26 @@ static enum scx_test_status setup(void **ctx_ptr) SCX_FAIL("Failed to get affinity (%d)", errno); } - cpu = find_nohz_full_cpu(&ctx->original_mask); - if (cpu < 0) { + ret = read_nohz_full_mask(&nohz_full_mask); + if (ret) { + fprintf(stderr, "SKIP: failed to read NOHZ_FULL mask (%d)\n", ret); + free(ctx); + return SCX_TEST_SKIP; + } + + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) + if (CPU_ISSET(cpu, &ctx->original_mask) && CPU_ISSET(cpu, &nohz_full_mask)) + break; + if (cpu == CPU_SETSIZE) { fprintf(stderr, "SKIP: no allowed NOHZ_FULL CPU\n"); free(ctx); return SCX_TEST_SKIP; } controller_mask = ctx->original_mask; - CPU_CLR(cpu, &controller_mask); + for (i = 0; i < CPU_SETSIZE; i++) + if (CPU_ISSET(i, &nohz_full_mask)) + CPU_CLR(i, &controller_mask); if (CPU_COUNT(&controller_mask) == 0) { fprintf(stderr, "SKIP: no housekeeping CPU available\n"); free(ctx); @@ -189,6 +240,21 @@ static enum scx_test_status setup(void **ctx_ptr) } ctx->test_cpu = cpu; + ctx->housekeeping_cpu = first_allowed_cpu(&controller_mask, 0, CPU_SETSIZE - 1); + lazy_supported = __COMPAT_read_enum("scx_enq_flags", "SCX_ENQ_PREEMPT_LAZY", + &enum_value) && + __COMPAT_read_enum("scx_kick_flags", "SCX_KICK_PREEMPT_LAZY", + &enum_value) && + __COMPAT_read_enum("scx_ops_flags", "SCX_OPS_LAZY_RESCHED", + &lazy_ops_flag); + lazy_mode = scx_test_preempt_lazy_mode(); + ctx->test_lazy = lazy_supported && lazy_mode > 0; + if (lazy_supported && lazy_mode < 0) + fprintf(stderr, + "SKIP: kernel preemption mode unavailable; skipping lazy NOHZ_FULL phases\n"); + else if (lazy_supported && !lazy_mode) + fprintf(stderr, + "SKIP: lazy preemption inactive; skipping lazy NOHZ_FULL phases\n"); ctx->skel = nohz_tick__open(); if (!ctx->skel) { free(ctx); @@ -199,6 +265,8 @@ static enum scx_test_status setup(void **ctx_ptr) ctx->skel->rodata->test_cpu = cpu; ctx->skel->struct_ops.nohz_tick_ops->flags |= SCX_OPS_SWITCH_PARTIAL | SCX_OPS_ENQ_LAST; + if (lazy_supported) + ctx->skel->struct_ops.nohz_tick_ops->flags |= lazy_ops_flag; if (nohz_tick__load(ctx->skel)) { nohz_tick__destroy(ctx->skel); free(ctx); @@ -220,11 +288,15 @@ static enum scx_test_status run(void *ctx_ptr) struct nohz_tick_ctx *ctx = ctx_ptr; struct nohz_tick *skel = ctx->skel; struct bpf_link *link = NULL; + struct scx_test_gated_worker victim = { .pid = -1, .start_fd = -1 }; + struct scx_test_gated_worker challenger = { .pid = -1, .start_fd = -1 }; + struct scx_test_gated_worker trigger = { .pid = -1, .start_fd = -1 }; enum scx_test_status status = SCX_TEST_FAIL; pid_t finite_worker = -1; pid_t inf_worker = -1; u64 finite_running; u64 finite_ticks; + u64 victim_running; int ret; link = bpf_map__attach_struct_ops(skel->maps.nohz_tick_ops); @@ -241,7 +313,7 @@ static enum scx_test_status run(void *ctx_ptr) SCX_ERR("Failed to start infinite-slice worker (%d)", errno); goto out; } - if (!wait_for_counter(&skel->bss->nr_inf_running, 1, + if (!wait_for_counter(skel, &skel->bss->nr_inf_running, 1, PHASE_TIMEOUT_MS)) { SCX_ERR("Infinite-slice worker was not scheduled"); goto out; @@ -260,18 +332,18 @@ static enum scx_test_status run(void *ctx_ptr) /* * The next EXT task receives a finite slice and must restart the tick. */ - __atomic_store_n(&skel->bss->finite_phase, true, __ATOMIC_RELEASE); + __atomic_store_n(&skel->bss->phase, NOHZ_PHASE_FINITE, __ATOMIC_RELEASE); finite_worker = start_worker(ctx->test_cpu); if (finite_worker < 0) { SCX_ERR("Failed to start finite-slice worker (%d)", errno); goto out; } - if (!wait_for_counter(&skel->bss->nr_finite_running, 1, + if (!wait_for_counter(skel, &skel->bss->nr_finite_running, 1, PHASE_TIMEOUT_MS)) { SCX_ERR("Finite-slice worker was not scheduled"); goto out; } - if (!wait_for_counter(&skel->bss->nr_finite_ticks, MIN_FINITE_TICKS, + if (!wait_for_counter(skel, &skel->bss->nr_finite_ticks, MIN_FINITE_TICKS, PHASE_TIMEOUT_MS)) { SCX_ERR("Finite-slice worker received only %llu scheduler ticks", (unsigned long long)skel->bss->nr_finite_ticks); @@ -295,12 +367,12 @@ static enum scx_test_status run(void *ctx_ptr) SCX_ERR("Failed to start second finite-slice worker (%d)", errno); goto out; } - if (!wait_for_counter(&skel->bss->nr_finite_running, + if (!wait_for_counter(skel, &skel->bss->nr_finite_running, finite_running + 1, PHASE_TIMEOUT_MS)) { SCX_ERR("Second finite-slice worker was not scheduled"); goto out; } - if (!wait_for_counter(&skel->bss->nr_finite_ticks, + if (!wait_for_counter(skel, &skel->bss->nr_finite_ticks, finite_ticks + MIN_FINITE_TICKS, PHASE_TIMEOUT_MS)) { SCX_ERR("Second finite-slice worker received only %llu scheduler ticks", @@ -308,8 +380,88 @@ static enum scx_test_status run(void *ctx_ptr) finite_ticks)); goto out; } + stop_worker(finite_worker); + finite_worker = -1; + stop_worker(inf_worker); + inf_worker = -1; + if (!ctx->test_lazy) + goto check_exit; + + /* + * A lazy local enqueue must restart the tick after clearing the slice + * of an infinite-slice task on a full-dynticks CPU. + */ + __atomic_store_n(&skel->bss->phase, NOHZ_PHASE_LAZY_ENQ, __ATOMIC_RELEASE); + victim = scx_test_spawn_gated_worker(ctx->test_cpu, true); + challenger = scx_test_spawn_gated_worker(ctx->test_cpu, true); + if (victim.pid < 0 || challenger.pid < 0) { + SCX_ERR("Failed to spawn lazy-enqueue workers"); + goto out; + } + skel->bss->victim_pid = victim.pid; + skel->bss->challenger_pid = challenger.pid; + if (!scx_test_start_gated_worker(&victim) || + !wait_for_counter(skel, &skel->bss->nr_lazy_victim_running, 1, + PHASE_TIMEOUT_MS)) { + SCX_ERR("Lazy-enqueue victim was not scheduled"); + goto out; + } + if (!wait_for_tick_stop(skel, &skel->bss->nr_lazy_ticks, PHASE_TIMEOUT_MS)) { + SCX_ERR("Tick did not stop before lazy enqueue"); + goto out; + } + + if (!scx_test_start_gated_worker(&challenger) || + !wait_for_counter(skel, &skel->bss->nr_lazy_enq_running, 1, PHASE_TIMEOUT_MS)) { + SCX_ERR("Lazy enqueue made no progress on CPU %d", ctx->test_cpu); + goto out; + } + scx_test_stop_gated_worker(&victim); + scx_test_stop_gated_worker(&challenger); + + /* Repeat with a lazy kick delivered from a housekeeping CPU. */ + __atomic_store_n(&skel->bss->phase, NOHZ_PHASE_LAZY_KICK, __ATOMIC_RELEASE); + victim = scx_test_spawn_gated_worker(ctx->test_cpu, true); + challenger = scx_test_spawn_gated_worker(ctx->test_cpu, true); + trigger = scx_test_spawn_gated_worker(ctx->housekeeping_cpu, true); + if (victim.pid < 0 || challenger.pid < 0 || trigger.pid < 0) { + SCX_ERR("Failed to spawn lazy-kick workers"); + goto out; + } + skel->bss->victim_pid = victim.pid; + skel->bss->challenger_pid = challenger.pid; + skel->bss->trigger_pid = trigger.pid; + victim_running = __atomic_load_n(&skel->bss->nr_lazy_victim_running, + __ATOMIC_RELAXED); + if (!scx_test_start_gated_worker(&victim) || + !wait_for_counter(skel, &skel->bss->nr_lazy_victim_running, victim_running + 1, + PHASE_TIMEOUT_MS)) { + SCX_ERR("Lazy-kick victim was not scheduled"); + goto out; + } + if (!scx_test_start_gated_worker(&challenger)) { + SCX_ERR("Failed to start lazy-kick challenger"); + goto out; + } + if (!wait_for_tick_stop(skel, &skel->bss->nr_lazy_ticks, PHASE_TIMEOUT_MS)) { + SCX_ERR("Tick did not stop before lazy kick"); + goto out; + } + if (__atomic_load_n(&skel->bss->nr_lazy_kick_running, __ATOMIC_RELAXED)) { + SCX_ERR("Lazy-kick challenger ran before the kick"); + goto out; + } + if (!scx_test_start_gated_worker(&trigger) || + !wait_for_counter(skel, &skel->bss->nr_lazy_kick_running, 1, PHASE_TIMEOUT_MS)) { + SCX_ERR("Lazy kick made no progress on CPU %d", ctx->test_cpu); + goto out; + } + scx_test_stop_gated_worker(&trigger); + scx_test_stop_gated_worker(&victim); + scx_test_stop_gated_worker(&challenger); - if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) { +check_exit: + if (UEI_EXITED(skel, uei)) { SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)", (unsigned long long)skel->data->uei.kind, (long long)skel->data->uei.exit_code); @@ -321,6 +473,9 @@ static enum scx_test_status run(void *ctx_ptr) (unsigned long long)skel->bss->nr_finite_ticks); status = SCX_TEST_PASS; out: + scx_test_stop_gated_worker(&trigger); + scx_test_stop_gated_worker(&victim); + scx_test_stop_gated_worker(&challenger); stop_worker(finite_worker); stop_worker(inf_worker); if (link) diff --git a/tools/testing/selftests/sched_ext/nohz_tick_test.h b/tools/testing/selftests/sched_ext/nohz_tick_test.h new file mode 100644 index 000000000000..b122d723cd8b --- /dev/null +++ b/tools/testing/selftests/sched_ext/nohz_tick_test.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES */ +#ifndef __NOHZ_TICK_TEST_H__ +#define __NOHZ_TICK_TEST_H__ + +enum nohz_phase { + NOHZ_PHASE_INF, + NOHZ_PHASE_FINITE, + NOHZ_PHASE_LAZY_ENQ, + NOHZ_PHASE_LAZY_KICK, +}; + +#endif /* __NOHZ_TICK_TEST_H__ */ diff --git a/tools/testing/selftests/sched_ext/prog_run.c b/tools/testing/selftests/sched_ext/prog_run.c index 1129ec2aaddc..369c744ac06e 100644 --- a/tools/testing/selftests/sched_ext/prog_run.c +++ b/tools/testing/selftests/sched_ext/prog_run.c @@ -55,8 +55,11 @@ static enum scx_test_status run(void *ctx) } /* Assumes uei.kind is written last */ - while (skel->data->uei.kind == EXIT_KIND(SCX_EXIT_NONE)) - sched_yield(); + if (!SCX_TIMED_WAIT_UEI(skel, uei, 10000)) { + SCX_ERR("Timed out waiting for scheduler to exit"); + status = SCX_TEST_FAIL; + goto out; + } if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_UNREG_BPF)) { SCX_ERR("Unexpected exit kind: %llu", diff --git a/tools/testing/selftests/sched_ext/reload_loop.c b/tools/testing/selftests/sched_ext/reload_loop.c index 49297b83d748..7d8880fa1a03 100644 --- a/tools/testing/selftests/sched_ext/reload_loop.c +++ b/tools/testing/selftests/sched_ext/reload_loop.c @@ -4,6 +4,7 @@ * Copyright (c) 2024 David Vernet <dvernet@meta.com> */ #include <bpf/bpf.h> +#include <errno.h> #include <pthread.h> #include <scx/common.h> #include <sys/wait.h> @@ -11,8 +12,14 @@ #include "maximal.bpf.skel.h" #include "scx_test.h" +enum { + RELOAD_ITERS = 1024, + RELOAD_THREADS = 2, +}; + static struct maximal *skel; -static pthread_t threads[2]; +static pthread_t threads[RELOAD_THREADS]; +static int unexpected_errno; bool force_exit = false; @@ -31,32 +38,58 @@ static enum scx_test_status setup(void **ctx) static void *do_reload_loop(void *arg) { - u32 i; + u32 i, failures = 0; - for (i = 0; i < 1024 && !force_exit; i++) { + for (i = 0; i < RELOAD_ITERS && !force_exit; i++) { struct bpf_link *link; + errno = 0; link = bpf_map__attach_struct_ops(skel->maps.maximal_ops); - if (link) - bpf_link__destroy(link); + if (!link) { + int e = errno; + + if (e != EBUSY) + __atomic_store_n(&unexpected_errno, e, + __ATOMIC_RELAXED); + failures++; + continue; + } + + bpf_link__destroy(link); } - return NULL; + return (void *)(unsigned long)failures; } static enum scx_test_status run(void *ctx) { - int err; + unsigned long failures[RELOAD_THREADS] = {}; + int err, i; void *ret; err = pthread_create(&threads[0], NULL, do_reload_loop, NULL); SCX_FAIL_IF(err, "Failed to create thread 0"); err = pthread_create(&threads[1], NULL, do_reload_loop, NULL); - SCX_FAIL_IF(err, "Failed to create thread 1"); + if (err) { + force_exit = true; + pthread_join(threads[0], NULL); + SCX_FAIL("Failed to create thread 1"); + } SCX_FAIL_IF(pthread_join(threads[0], &ret), "thread 0 failed"); + failures[0] = (unsigned long)ret; + SCX_FAIL_IF(pthread_join(threads[1], &ret), "thread 1 failed"); + failures[1] = (unsigned long)ret; + + SCX_FAIL_IF(unexpected_errno, "unexpected attach errno %d", + unexpected_errno); + + for (i = 0; i < RELOAD_THREADS; i++) + SCX_FAIL_IF(failures[i] == RELOAD_ITERS, + "thread %d: every attach failed (%lu EBUSY)", + i, failures[i]); return SCX_TEST_PASS; } diff --git a/tools/testing/selftests/sched_ext/rt_stall.c b/tools/testing/selftests/sched_ext/rt_stall.c index a5041fc2e44f..a1552b38a6e8 100644 --- a/tools/testing/selftests/sched_ext/rt_stall.c +++ b/tools/testing/selftests/sched_ext/rt_stall.c @@ -17,7 +17,6 @@ #include <scx/common.h> #include "rt_stall.bpf.skel.h" #include "scx_test.h" -#include "../kselftest.h" #define CORE_ID 0 /* CPU to pin tasks to */ #define RUN_TIME 5 /* How long to run the test in seconds */ @@ -35,15 +34,17 @@ static void signal_ready(int fd) } /* Wait for a child to signal readiness via a pipe */ -static void wait_ready(int fd) +static bool wait_ready(int fd) { + bool ready; char c; - if (read(fd, &c, 1) != 1) { + ready = read(fd, &c, 1) == 1; + if (!ready) perror("read from ready pipe"); - exit(EXIT_FAILURE); - } close(fd); + + return ready; } /* Simple busy-wait function for test tasks */ @@ -151,13 +152,11 @@ static bool sched_stress_test(bool is_ext) float ext_runtime, rt_runtime, actual_ratio; int ext_pid, rt_pid; int ext_ready[2], rt_ready[2]; - - ksft_print_header(); - ksft_set_plan(1); + bool ret = false; if (pipe(ext_ready) || pipe(rt_ready)) { perror("pipe"); - ksft_exit_fail(); + return false; } /* Create and set up a EXT task */ @@ -172,7 +171,7 @@ static bool sched_stress_test(bool is_ext) exit(0); } else if (ext_pid < 0) { perror("fork task"); - ksft_exit_fail(); + return false; } /* Create an RT task */ @@ -188,7 +187,7 @@ static bool sched_stress_test(bool is_ext) exit(0); } else if (rt_pid < 0) { perror("fork for RT task"); - ksft_exit_fail(); + goto out_kill_ext; } /* @@ -199,45 +198,47 @@ static bool sched_stress_test(bool is_ext) */ close(ext_ready[1]); close(rt_ready[1]); - wait_ready(ext_ready[0]); - wait_ready(rt_ready[0]); + if (!wait_ready(ext_ready[0]) || !wait_ready(rt_ready[0])) + goto out_kill; /* Let the processes run for the specified time */ sleep(RUN_TIME); /* Get runtime for the EXT task */ ext_runtime = get_process_runtime(ext_pid); - if (ext_runtime == -1) - ksft_exit_fail_msg("Error getting runtime for %s task (PID %d)\n", - class_str, ext_pid); - ksft_print_msg("Runtime of %s task (PID %d) is %f seconds\n", - class_str, ext_pid, ext_runtime); + if (ext_runtime == -1) { + fprintf(stderr, "Failed to read %s task runtime\n", class_str); + goto out_kill; + } /* Get runtime for the RT task */ rt_runtime = get_process_runtime(rt_pid); - if (rt_runtime == -1) - ksft_exit_fail_msg("Error getting runtime for RT task (PID %d)\n", rt_pid); - ksft_print_msg("Runtime of RT task (PID %d) is %f seconds\n", rt_pid, rt_runtime); - - /* Kill the processes */ - kill(ext_pid, SIGKILL); - kill(rt_pid, SIGKILL); - waitpid(ext_pid, NULL, 0); - waitpid(rt_pid, NULL, 0); + if (rt_runtime == -1) { + fprintf(stderr, "Failed to read RT task runtime\n"); + goto out_kill; + } /* Verify that the scx task got enough runtime */ actual_ratio = ext_runtime / (ext_runtime + rt_runtime); - ksft_print_msg("%s task got %.2f%% of total runtime\n", - class_str, actual_ratio * 100); + fprintf(stderr, "%s task ran %.3fs, RT task ran %.3fs (%.2f%% of runtime)\n", + class_str, ext_runtime, rt_runtime, actual_ratio * 100); - if (actual_ratio >= expected_min_ratio) { - ksft_test_result_pass("PASS: %s task got more than %.2f%% of runtime\n", - class_str, expected_min_ratio * 100); - return true; + if (actual_ratio < expected_min_ratio) { + fprintf(stderr, "%s task got less than %.2f%% of runtime\n", + class_str, expected_min_ratio * 100); + goto out_kill; } - ksft_test_result_fail("FAIL: %s task got less than %.2f%% of runtime\n", - class_str, expected_min_ratio * 100); - return false; + + ret = true; + +out_kill: + kill(rt_pid, SIGKILL); + waitpid(rt_pid, NULL, 0); +out_kill_ext: + kill(ext_pid, SIGKILL); + waitpid(ext_pid, NULL, 0); + + return ret; } static enum scx_test_status run(void *ctx) @@ -263,14 +264,17 @@ static enum scx_test_status run(void *ctx) link = bpf_map__attach_struct_ops(skel->maps.rt_stall_ops); SCX_FAIL_IF(!link, "Failed to attach scheduler"); } + res = sched_stress_test(is_ext); + if (is_ext) { - SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_NONE)); + int exit_kind = skel->data->uei.kind; bpf_link__destroy(link); + SCX_EQ(exit_kind, EXIT_KIND(SCX_EXIT_NONE)); } if (!res) - ksft_exit_fail(); + return SCX_TEST_FAIL; } return SCX_TEST_PASS; diff --git a/tools/testing/selftests/sched_ext/runner.c b/tools/testing/selftests/sched_ext/runner.c index c264807caa91..57dda75384f1 100644 --- a/tools/testing/selftests/sched_ext/runner.c +++ b/tools/testing/selftests/sched_ext/runner.c @@ -243,7 +243,7 @@ int main(int argc, char **argv) printf(" - %s\n", failed_tests[i]); } - return failed > 0 ? 1 : 0; + return failed > 0 || exit_req ? 1 : 0; } void scx_test_register(struct scx_test *test) diff --git a/tools/testing/selftests/sched_ext/scx_test.h b/tools/testing/selftests/sched_ext/scx_test.h index 90b8d6915bb7..8490b01fff74 100644 --- a/tools/testing/selftests/sched_ext/scx_test.h +++ b/tools/testing/selftests/sched_ext/scx_test.h @@ -18,8 +18,6 @@ enum scx_test_status { SCX_TEST_FAIL, }; -#define EXIT_KIND(__ent) __COMPAT_ENUM_OR_ZERO("scx_exit_kind", #__ent) - struct scx_test { /** * name - The name of the testcase. @@ -110,6 +108,41 @@ void scx_test_register(struct scx_test *test); #define SCX_ASSERT(_x) SCX_FAIL_IF(!(_x), "Expected %s to be true (%lu)", \ #_x, (u64)(_x)) +/* + * SCX_ASSERT_ALIVE - Fail the test if the scheduler under test exited + * on its own between attach and this check, e.g. from a runtime error. + * A scheduler that dies mid-test freezes whatever state the test is + * about to assert on, which can make the assertions pass vacuously. + * uei.msg carries the actual error text, so print it alongside the + * recorded kind and exit code. + */ +#define SCX_ASSERT_ALIVE(__skel, __uei) \ + do { \ + if (UEI_EXITED(__skel, __uei)) \ + SCX_FAIL("Scheduler exited unexpectedly" \ + " (kind=%llu code=%lld msg='%s')", \ + (unsigned long long)(__skel)->data->__uei.kind, \ + (long long)(__skel)->data->__uei.exit_code, \ + (__skel)->data->__uei.msg); \ + } while (0) + +/* + * SCX_TIMED_WAIT_UEI - Wait for the scheduler under test to exit, i.e. + * its uei to be filled in, for up to @__ms iterations at ~1ms each. + * Returns true if the scheduler exited and false on timeout, so a + * scheduler that never exits fails the test instead of hanging it. + * The bound is iteration-based rather than wall-clock and stretches + * on a loaded system instead of failing early. + */ +#define SCX_TIMED_WAIT_UEI(__skel, __uei, __ms) \ + ({ \ + s64 __left = (__ms); \ + \ + while (__left-- > 0 && !UEI_EXITED(__skel, __uei)) \ + usleep(1000); \ + UEI_EXITED(__skel, __uei) != 0; \ + }) + #define SCX_ECODE_VAL(__ecode) ({ \ u64 __val = 0; \ bool __found = false; \ @@ -119,13 +152,25 @@ void scx_test_register(struct scx_test *test); (s64)__val; \ }) +/* + * SCX_KIND_VAL - Resolve an scx_exit_kind enumerator from kernel BTF. + * + * Unlike __COMPAT_ENUM_OR_ZERO(), a missing enumerator fails the test + * loudly instead of silently resolving to 0 (== SCX_EXIT_NONE), which + * would make the comparison vacuous. + */ #define SCX_KIND_VAL(__kind) ({ \ - u64 __val = 0; \ - bool __found = false; \ + u64 __val = 0; \ \ - __found = __COMPAT_read_enum("scx_exit_kind", #__kind, &__val); \ - SCX_ASSERT(__found); \ + if (!__COMPAT_read_enum("scx_exit_kind", #__kind, &__val)) \ + SCX_FAIL("scx_exit_kind.%s not found in kernel BTF", #__kind); \ __val; \ }) +/* + * EXIT_KIND - Alias of SCX_KIND_VAL() for resolving scx_exit_kind + * enumerator values, kept for readability at the EXIT_KIND() call sites. + */ +#define EXIT_KIND(__ent) SCX_KIND_VAL(__ent) + #endif // # __SCX_TEST_H__ diff --git a/tools/testing/selftests/sched_ext/test_modules/Makefile b/tools/testing/selftests/sched_ext/test_modules/Makefile new file mode 100644 index 000000000000..a0e9e9401ead --- /dev/null +++ b/tools/testing/selftests/sched_ext/test_modules/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + +TESTMODS_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))) +KDIR ?= $(if $(O),$(O),$(realpath ../../../../..)) + +obj-m += scx_enq_blocked_test.o + +all: + +$(Q)$(MAKE) -C $(KDIR) M=$(TESTMODS_DIR) modules + +clean: + +$(Q)$(MAKE) -C $(KDIR) M=$(TESTMODS_DIR) clean diff --git a/tools/testing/selftests/sched_ext/test_modules/scx_enq_blocked_test.c b/tools/testing/selftests/sched_ext/test_modules/scx_enq_blocked_test.c new file mode 100644 index 000000000000..908689ed5578 --- /dev/null +++ b/tools/testing/selftests/sched_ext/test_modules/scx_enq_blocked_test.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + * + * Kernel mutex fixture for the sched_ext SCX_OPS_ENQ_BLOCKED selftest. + */ + +#include <linux/atomic.h> +#include <linux/fs.h> +#include <linux/jiffies.h> +#include <linux/ktime.h> +#include <linux/miscdevice.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/sched.h> +#include <linux/uaccess.h> + +#include "../enq_blocked.h" + +#define DONOR_WAIT_TIMEOUT msecs_to_jiffies(2000) +#define ATTACH_WAIT_TIMEOUT msecs_to_jiffies(10000) +#define MUTEX_HOLD_TIME msecs_to_jiffies(200) + +static DEFINE_MUTEX(test_mutex); +static DEFINE_SPINLOCK(donor_lock); +static struct task_struct *donor_task; +static atomic_t owner_ready = ATOMIC_INIT(0); +static atomic_t donor_started = ATOMIC_INIT(0); +static atomic_t attach_pending = ATOMIC_INIT(0); +static atomic_t attach_done = ATOMIC_INIT(0); +static atomic64_t hold_time_ns = ATOMIC64_INIT(0); +static atomic64_t wait_time_ns = ATOMIC64_INIT(0); +static atomic64_t nr_holds = ATOMIC64_INIT(0); +static atomic64_t nr_waits = ATOMIC64_INIT(0); + +static long run_owner(void) +{ + unsigned long timeout; + u64 start_ns; + long ret = 0; + + atomic_set(&donor_started, 0); + mutex_lock(&test_mutex); + start_ns = ktime_get_ns(); + atomic_set(&owner_ready, 1); + + timeout = jiffies + DONOR_WAIT_TIMEOUT; + while (!atomic_read(&donor_started)) { + if (time_after(jiffies, timeout)) { + ret = -ETIMEDOUT; + goto out; + } + cond_resched(); + } + if (atomic_xchg(&attach_pending, 0)) { + timeout = jiffies + ATTACH_WAIT_TIMEOUT; + while (!atomic_read(&attach_done)) { + if (time_after(jiffies, timeout)) { + ret = -ETIMEDOUT; + goto out; + } + cond_resched(); + } + } + + /* Keep yielding while the donor blocks on test_mutex. */ + timeout = jiffies + MUTEX_HOLD_TIME; + while (time_before(jiffies, timeout)) + cond_resched(); + +out: + atomic_set(&owner_ready, 0); + atomic64_add(ktime_get_ns() - start_ns, &hold_time_ns); + atomic64_inc(&nr_holds); + mutex_unlock(&test_mutex); + return ret; +} + +static long run_donor(void) +{ + unsigned long flags; + u64 start_ns; + + if (!atomic_read(&owner_ready)) + return -EAGAIN; + + get_task_struct(current); + spin_lock_irqsave(&donor_lock, flags); + WARN_ON_ONCE(donor_task); + donor_task = current; + spin_unlock_irqrestore(&donor_lock, flags); + + atomic_set(&donor_started, 1); + start_ns = ktime_get_ns(); + mutex_lock(&test_mutex); + + spin_lock_irqsave(&donor_lock, flags); + donor_task = NULL; + spin_unlock_irqrestore(&donor_lock, flags); + put_task_struct(current); + + atomic64_add(ktime_get_ns() - start_ns, &wait_time_ns); + atomic64_inc(&nr_waits); + mutex_unlock(&test_mutex); + return 0; +} + +static long get_donor_state(void) +{ + struct task_struct *task; + unsigned long flags; + long state = 0; + + spin_lock_irqsave(&donor_lock, flags); + task = donor_task; + if (task) + get_task_struct(task); + spin_unlock_irqrestore(&donor_lock, flags); + if (!task) + return -ENOENT; + + if (READ_ONCE(task->__state) != TASK_RUNNING) + state |= ENQ_BLOCKED_DONOR_SLEEPING; + if (READ_ONCE(task->on_rq)) + state |= ENQ_BLOCKED_DONOR_ON_RQ; + put_task_struct(task); + return state; +} + +static void reset_stats(void) +{ + atomic64_set(&hold_time_ns, 0); + atomic64_set(&wait_time_ns, 0); + atomic64_set(&nr_holds, 0); + atomic64_set(&nr_waits, 0); +} + +static long get_stats(unsigned long arg) +{ + struct enq_blocked_stats stats = { + .hold_time_ns = atomic64_read(&hold_time_ns), + .wait_time_ns = atomic64_read(&wait_time_ns), + .nr_holds = atomic64_read(&nr_holds), + .nr_waits = atomic64_read(&nr_waits), + }; + + return copy_to_user((void __user *)arg, &stats, sizeof(stats)) ? + -EFAULT : 0; +} + +static long enq_blocked_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + switch (cmd) { + case ENQ_BLOCKED_IOCTL_OWNER: + return run_owner(); + case ENQ_BLOCKED_IOCTL_DONOR: + return run_donor(); + case ENQ_BLOCKED_IOCTL_RESET_STATS: + reset_stats(); + return 0; + case ENQ_BLOCKED_IOCTL_GET_STATS: + return get_stats(arg); + case ENQ_BLOCKED_IOCTL_PREP_ATTACH: + atomic_set(&attach_done, 0); + atomic_set(&attach_pending, 1); + return 0; + case ENQ_BLOCKED_IOCTL_ATTACH_DONE: + atomic_set(&attach_done, 1); + return 0; + case ENQ_BLOCKED_IOCTL_DONOR_STATE: + return get_donor_state(); + case ENQ_BLOCKED_IOCTL_PROXY_SUPPORTED: + return IS_ENABLED(CONFIG_SCHED_PROXY_EXEC); + default: + return -EINVAL; + } +} + +static const struct file_operations enq_blocked_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = enq_blocked_ioctl, +}; + +static struct miscdevice enq_blocked_device = { + .minor = MISC_DYNAMIC_MINOR, + .name = "scx_enq_blocked", + .fops = &enq_blocked_fops, + .mode = 0600, +}; + +module_misc_device(enq_blocked_device); +MODULE_AUTHOR("Andrea Righi <arighi@nvidia.com>"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("sched_ext blocked donor test module"); diff --git a/tools/testing/selftests/sched_ext/util.c b/tools/testing/selftests/sched_ext/util.c index 2111329ed289..c93fc5df552d 100644 --- a/tools/testing/selftests/sched_ext/util.c +++ b/tools/testing/selftests/sched_ext/util.c @@ -5,10 +5,21 @@ */ #include <errno.h> #include <fcntl.h> +#include <sched.h> +#include <signal.h> #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <sys/prctl.h> +#include <sys/wait.h> #include <unistd.h> +#include <zlib.h> + +#include "util.h" + +#ifndef SCHED_EXT +#define SCHED_EXT 7 +#endif /* Returns read len on success, or -errno on failure. */ static ssize_t read_text(const char *path, char *buf, size_t max_len) @@ -69,3 +80,135 @@ int file_write_long(const char *path, long val) return 0; } + +struct scx_test_gated_worker scx_test_spawn_gated_worker(int cpu, bool set_sched_ext) +{ + struct scx_test_gated_worker worker = { .pid = -1, .start_fd = -1 }; + int ready[2], start[2]; + pid_t parent = getpid(); + char byte = 1; + + if (pipe(ready)) + return worker; + if (pipe(start)) { + close(ready[0]); + close(ready[1]); + return worker; + } + + worker.pid = fork(); + if (!worker.pid) { + struct sched_param param = {}; + cpu_set_t mask; + + close(ready[0]); + close(start[1]); + if (prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() != parent) + _exit(1); + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); + if (sched_setaffinity(0, sizeof(mask), &mask)) + _exit(1); + if (set_sched_ext && sched_setscheduler(0, SCHED_EXT, ¶m)) + _exit(1); + if (write(ready[1], &byte, 1) != 1) + _exit(1); + close(ready[1]); + if (read(start[0], &byte, 1) != 1) + _exit(1); + close(start[0]); + for (;;) + asm volatile("" ::: "memory"); + } + if (worker.pid < 0) { + close(ready[0]); + close(ready[1]); + close(start[0]); + close(start[1]); + return worker; + } + + close(ready[1]); + close(start[0]); + if (read(ready[0], &byte, 1) != 1) { + close(ready[0]); + close(start[1]); + kill(worker.pid, SIGKILL); + waitpid(worker.pid, NULL, 0); + worker.pid = -1; + return worker; + } + close(ready[0]); + worker.start_fd = start[1]; + return worker; +} + +bool scx_test_start_gated_worker(struct scx_test_gated_worker *worker) +{ + char byte = 1; + + if (write(worker->start_fd, &byte, 1) != 1) + return false; + close(worker->start_fd); + worker->start_fd = -1; + return true; +} + +void scx_test_stop_gated_worker(struct scx_test_gated_worker *worker) +{ + if (worker->start_fd >= 0) + close(worker->start_fd); + if (worker->pid > 0) { + kill(worker->pid, SIGKILL); + waitpid(worker->pid, NULL, 0); + } + worker->pid = -1; + worker->start_fd = -1; +} + +static int config_preempt_lazy_mode(void) +{ + bool dynamic = false, lazy = false, immediate = false; + char buf[128]; + gzFile file; + + file = gzopen("/proc/config.gz", "r"); + if (!file) + return -1; + + while (gzgets(file, buf, sizeof(buf))) { + if (!strcmp(buf, "CONFIG_PREEMPT_DYNAMIC=y\n")) + dynamic = true; + else if (!strcmp(buf, "CONFIG_PREEMPT_LAZY=y\n")) + lazy = true; + else if (!strcmp(buf, "CONFIG_PREEMPT_NONE=y\n") || + !strcmp(buf, "CONFIG_PREEMPT_VOLUNTARY=y\n") || + !strcmp(buf, "CONFIG_PREEMPT=y\n") || + !strcmp(buf, "CONFIG_PREEMPT_RT=y\n")) + immediate = true; + } + gzclose(file); + + if (dynamic) + return -1; + if (lazy) + return 1; + if (immediate) + return 0; + return -1; +} + +int scx_test_preempt_lazy_mode(void) +{ + char buf[128]; + + if (read_text("/sys/kernel/debug/sched/preempt", buf, sizeof(buf)) > 0) { + if (strstr(buf, "(lazy)")) + return 1; + if (strstr(buf, "(none)") || strstr(buf, "(voluntary)") || + strstr(buf, "(full)")) + return 0; + } + + return config_preempt_lazy_mode(); +} diff --git a/tools/testing/selftests/sched_ext/util.h b/tools/testing/selftests/sched_ext/util.h index 681cec04b439..aac34fc4ef71 100644 --- a/tools/testing/selftests/sched_ext/util.h +++ b/tools/testing/selftests/sched_ext/util.h @@ -7,7 +7,19 @@ #ifndef __SCX_TEST_UTIL_H__ #define __SCX_TEST_UTIL_H__ +#include <stdbool.h> +#include <sys/types.h> + +struct scx_test_gated_worker { + pid_t pid; + int start_fd; +}; + long file_read_long(const char *path); int file_write_long(const char *path, long val); +struct scx_test_gated_worker scx_test_spawn_gated_worker(int cpu, bool set_sched_ext); +bool scx_test_start_gated_worker(struct scx_test_gated_worker *worker); +void scx_test_stop_gated_worker(struct scx_test_gated_worker *worker); +int scx_test_preempt_lazy_mode(void); #endif // __SCX_TEST_UTIL_H__ |