diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2026-10-22 10:37:35 -1000 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2026-10-22 10:37:35 -1000 |
| commit | 98aec86f8c6bf2e1091fefb99eaba8b910e8eef1 (patch) | |
| tree | 262edefdccfcca1df2cd15ffef6bfc7563ad48ef /tools | |
| parent | 77f77fb7923feab3edd0accf793235623ef4b342 (diff) | |
| parent | 1f5dde349ac2fca1f119ae8de3f0d2f1c2a8d338 (diff) | |
Merge tag 'sched_ext-for-7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext
Pull sched_ext updates from Tejun Heo:
- Proxy execution can now be enabled together with sched_ext. A BPF
scheduler can opt in to receive mutex-blocked tasks that donate their
scheduling context to the lock owner and place them itself.
- Lazy preemption: a BPF scheduler can preempt or kick a CPU lazily,
deferring the switch to the next return to user space or tick, as the
fair class can.
- Sub-schedulers: a parent now learns which child runs on each CPU it
delegated and when a cap change takes effect. A detaching child loses
its caps before the parent restores the CPUs.
- New kfuncs re-enqueue single tasks from a dispatch queue iteration
and report a cgroup's CPU count and a CID's NUMA node. The CID
topology now includes clusters.
- Hot path optimizations: dispatch queue and task lookups compare keys
inline, remote task moves no longer take a dispatch queue lock twice,
and sub-scheduler hooks cost nothing without sub-schedulers.
- Kfuncs callable from tracing programs are hardened against NMI and
IRQ nesting, which could hard-lock the CPU or pick a wrong idle CPU.
- Built-in dispatch queues no longer allocate per-CPU state that grew
with the square of the CPU count.
- Header syncs with the scx repo, a BTF workaround for pahole 1.32, and
selftest fixes and coverage.
* tag 'sched_ext-for-7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext: (69 commits)
sched_ext: Drop the locks periodically while walking a user DSQ for re-enqueue
sched_ext: scx_qmap: Also re-enqueue stranded tasks with scx_bpf_dsq_reenq_task()
sched_ext: Add scx_bpf_dsq_reenq_task() to re-enqueue single tasks from a DSQ iteration
sched_ext: Factor out the rq lock switch of scx_dsq_move()
sched_ext: Don't queue balance callbacks from a core-sched pick
sched_ext: Don't leave balance callbacks queued across a pick retry
sched_ext: Fix a bypassed task stalling on a cpu its scheduler has no caps on
sched_ext: Fix the running task keeping its protection into bypass
sched_ext: Fix use-after-free of a local reenq node left on an inactive cpu
sched_ext: Keep proxy donors with slice left on the local DSQ
selftests/sched_ext: Report reload_loop attach failures
selftests/sched_ext: Bound exit waits
selftests/sched_ext: Add SCX_ASSERT_ALIVE and adopt it
selftests/sched_ext: Fail loudly when EXIT_KIND is missing from BTF
sched_ext: Add SCX_ECODE_RSN_CGROUP_OFFLINE to user_exit_info.h
selftests/sched_ext: Fix reload_loop error-path use-after-free
selftests/sched_ext: Fix build error in dequeue_remote selftest
sched_ext: scx_qmap: Reset a child's cpuperf targets once its PERF revoke takes effect
sched_ext: Add ops.sub_child_ecaps_updated() to report a child's effective cap changes
sched_ext: scx_qmap: Show actual cid use per participant
...
Diffstat (limited to 'tools')
51 files changed, 4459 insertions, 303 deletions
diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile index 21554f089692..70d2503b6b33 100644 --- a/tools/sched_ext/Makefile +++ b/tools/sched_ext/Makefile @@ -89,7 +89,7 @@ ifneq ($(wildcard $(GENHDR)),) GENFLAGS := -DHAVE_GENHDR endif -CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ +CFLAGS += -g -O2 -pthread -Wall -Werror $(GENFLAGS) \ -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -I$(TOOLSINCDIR) -I$(APIDIR) -I$(CURDIR)/include diff --git a/tools/sched_ext/include/scx/cid.bpf.h b/tools/sched_ext/include/scx/cid.bpf.h index 69fb4e97bc77..d1e6ad1432d3 100644 --- a/tools/sched_ext/include/scx/cid.bpf.h +++ b/tools/sched_ext/include/scx/cid.bpf.h @@ -3,8 +3,8 @@ * BPF-side helpers for cids and cmasks. See kernel/sched/ext/cid.h for the * authoritative layout and semantics. The BPF-side helpers use the cmask_* * naming (no scx_ prefix); cmask is the SCX bitmap type so the prefix is - * redundant in BPF code. Atomics use __sync_val_compare_and_swap and every - * helper is inline (no .c counterpart). + * redundant in BPF code. Every helper is inline except the binary operations, + * which are weak global functions defined here, so there is no .c counterpart. * * Included by scx/common.bpf.h; don't include directly. * @@ -16,6 +16,11 @@ #include "bpf_arena_common.bpf.h" +/* libbpf's bpf_helpers.h defines it from 1.4 on */ +#ifndef __arg_arena +#define __arg_arena __attribute((btf_decl_tag("arg:arena"))) +#endif + #ifndef BIT_U64 #define BIT_U64(nr) (1ULL << (nr)) #endif @@ -76,7 +81,7 @@ static __always_inline void __cmask_init(struct scx_cmask __arena *m, u32 base, m->nr_cids = nr_cids; m->alloc_words = alloc_words; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { if (i >= alloc_words) break; m->bits[i] = 0; @@ -122,24 +127,115 @@ static __always_inline void cmask_reframe(struct scx_cmask __arena *m, u32 base, m->nr_cids = nr_cids; } +/** + * cmask_nr_words - Words of bits[] that @m's active range spans + * @m: cmask to measure + * + * @m->base need not be word aligned: bits[0] covers the whole word @m->base + * falls in, so the count is taken from that word, not from @m->base. + */ +static __always_inline u32 cmask_nr_words(const struct scx_cmask __arena *m) +{ + u32 wbase = m->base / 64; + + return m->nr_cids ? (m->base + m->nr_cids - 1) / 64 - wbase + 1 : 0; +} + +/** + * cmask_word - Read word @k of @m + * @m: cmask to read + * @k: word index, counted from the word @m->base falls in + * + * A word past the active range reads as 0, so a scan that looks one word + * ahead (at an SMT sibling that falls in the next word, say) needs no bound + * of its own. + */ +static __always_inline u64 cmask_word(const struct scx_cmask __arena *m, u32 k) +{ + if (k >= cmask_nr_words(m)) + return 0; + return m->bits[k]; +} + +/** + * cmask_range_word - Bits of word @k of @m that fall in [@start, @start + @nr) + * @m: cmask the word belongs to + * @k: word index, counted from the word @m->base falls in + * @start: first cid of the range + * @nr: number of cids in the range + * + * Scoping a scan to a domain that is contiguous in cid space, an LLC or a + * node, is then one AND per word, rather than a test of the ends of the + * range at every bit or a mask kept per domain. + */ +static __always_inline u64 cmask_range_word(const struct scx_cmask __arena *m, + u32 k, u32 start, u32 nr) +{ + u64 wlo = (u64)(m->base / 64 + k) * 64, whi = wlo + 64; + u64 lo = start, hi = (u64)start + nr; + + if (lo < wlo) + lo = wlo; + if (hi > whi) + hi = whi; + if (lo >= hi) + return 0; + + return GENMASK_U64(hi - wlo - 1, lo - wlo); +} + +/** + * __cmask_test - Test a cid without checking it against the active range + * @cid: cid to test + * @m: cmask to test + * + * The caller must already know that @cid lies within + * [@m->base, @m->base + @m->nr_cids), or the read runs off @m->bits. + * + * For hot scans that have already bounded @cid, where the per-call range check + * is repeated work on every candidate. + */ +static __always_inline bool __cmask_test(u32 cid, const struct scx_cmask __arena *m) +{ + return *__cmask_word(cid, m) & BIT_U64(cid & 63); +} + static __always_inline bool cmask_test(u32 cid, const struct scx_cmask __arena *m) { if (!__cmask_contains(cid, m)) return false; - return *__cmask_word(cid, m) & BIT_U64(cid & 63); + return __cmask_test(cid, m); } /* - * x86 BPF JIT rejects BPF_OR | BPF_FETCH and BPF_AND | BPF_FETCH on arena - * pointers (see bpf_jit_supports_insn() in arch/x86/net/bpf_jit_comp.c). Only - * BPF_CMPXCHG / BPF_XCHG / BPF_ADD with FETCH are allowed. Implement - * test_and_{set,clear} and the atomic set/clear via a cmpxchg loop. + * The bit helpers use the fetching bitwise builtins, which order like the + * cmpxchg they replace, and not every JIT accepts those on arena pointers. The + * loader sets SCX_LIB_FEAT_ARENA_FETCH_BITOPS when a probe with the fetching + * form loads, and the helpers run a cmpxchg loop otherwise. The fetched value + * is consumed through barrier_var() because clang 19 lowers a builtin whose + * result is unused to the non-fetching form, which is relaxed on arm64. * * CMASK_CAS_TRIES is sized so exhausting it means seconds of real spinning - * on one word - past any plausible contention. Abort hard. + * on one word, past any plausible contention. Abort hard. */ #define CMASK_CAS_TRIES (1U << 23) +static __always_inline u64 __cmask_fetch_or(u64 __arena *w, u64 mask) +{ + u64 old = __atomic_fetch_or(w, mask, __ATOMIC_ACQ_REL); + + barrier_var(old); + return old; +} + +static __always_inline u64 __cmask_fetch_andnot(u64 __arena *w, u64 mask) +{ + u64 old = __atomic_fetch_and(w, ~mask, __ATOMIC_ACQ_REL); + + barrier_var(old); + return old; +} + static __always_inline void cmask_set(u32 cid, struct scx_cmask __arena *m) { u64 __arena *w; @@ -150,7 +246,11 @@ static __always_inline void cmask_set(u32 cid, struct scx_cmask __arena *m) return; w = __cmask_word(cid, m); bit = BIT_U64(cid & 63); - bpf_for(i, 0, CMASK_CAS_TRIES) { + if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) { + __cmask_fetch_or(w, bit); + return; + } + bpf_arena_for(i, 0, CMASK_CAS_TRIES) { old = *w; if (old & bit) return; @@ -171,7 +271,11 @@ static __always_inline void cmask_clear(u32 cid, struct scx_cmask __arena *m) return; w = __cmask_word(cid, m); bit = BIT_U64(cid & 63); - bpf_for(i, 0, CMASK_CAS_TRIES) { + if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) { + __cmask_fetch_andnot(w, bit); + return; + } + bpf_arena_for(i, 0, CMASK_CAS_TRIES) { old = *w; if (!(old & bit)) return; @@ -192,7 +296,9 @@ static __always_inline bool cmask_test_and_set(u32 cid, struct scx_cmask __arena return false; w = __cmask_word(cid, m); bit = BIT_U64(cid & 63); - bpf_for(i, 0, CMASK_CAS_TRIES) { + if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) + return __cmask_fetch_or(w, bit) & bit; + bpf_arena_for(i, 0, CMASK_CAS_TRIES) { old = *w; if (old & bit) return true; @@ -214,7 +320,9 @@ static __always_inline bool cmask_test_and_clear(u32 cid, struct scx_cmask __are return false; w = __cmask_word(cid, m); bit = BIT_U64(cid & 63); - bpf_for(i, 0, CMASK_CAS_TRIES) { + if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) + return __cmask_fetch_andnot(w, bit) & bit; + bpf_arena_for(i, 0, CMASK_CAS_TRIES) { old = *w; if (!(old & bit)) return false; @@ -268,17 +376,242 @@ static __always_inline bool __cmask_test_and_clear(u32 cid, struct scx_cmask __a return prev; } +/* atomically or @mask into *@w */ +static __always_inline void __cmask_word_or(u64 __arena *w, u64 mask) +{ + u64 old, new; + u32 i; + + if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) { + __cmask_fetch_or(w, mask); + return; + } + bpf_arena_for(i, 0, CMASK_CAS_TRIES) { + old = *w; + if ((old & mask) == mask) + return; + new = old | mask; + if (__sync_val_compare_and_swap(w, old, new) == old) + return; + } + scx_bpf_error("__cmask_word_or CAS exhausted"); +} + +/* atomically clear @mask bits in *@w */ +static __always_inline void __cmask_word_andnot(u64 __arena *w, u64 mask) +{ + u64 old, new; + u32 i; + + if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) { + __cmask_fetch_andnot(w, mask); + return; + } + bpf_arena_for(i, 0, CMASK_CAS_TRIES) { + old = *w; + if (!(old & mask)) + return; + new = old & ~mask; + if (__sync_val_compare_and_swap(w, old, new) == old) + return; + } + scx_bpf_error("__cmask_word_andnot CAS exhausted"); +} + +/** + * cmask_full_range - Test whether every cid in [@start, @start + @nr) is set + * @m: cmask to test + * @start: first cid of the range + * @nr: number of cids in the range + * + * The range is clamped to @m's active range first. True when no clamped bit is + * clear, including when the clamped range is empty. + */ +static __always_inline bool cmask_full_range(const struct scx_cmask __arena *m, u32 start, + u32 nr) +{ + u64 end = (u64)start + nr; + u32 wbase = m->base / 64; + u32 first_wi, last_wi, first_bit, last_bit, last, i; + + if (start < m->base) + start = m->base; + if (end > m->base + m->nr_cids) + end = m->base + m->nr_cids; + if (start >= end) + return true; + last = end - 1; + + first_wi = start / 64 - wbase; + last_wi = last / 64 - wbase; + first_bit = start & 63; + last_bit = last & 63; + + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { + u32 wi = first_wi + i; + u64 mask = ~0ULL; + + if (wi > last_wi) + break; + if (wi == first_wi) + mask &= GENMASK_U64(63, first_bit); + if (wi == last_wi) + mask &= GENMASK_U64(last_bit, 0); + if ((m->bits[wi] & mask) != mask) + return false; + } + return true; +} + +/** + * cmask_set_range - Set every cid in [@start, @start + @nr) + * @m: cmask to modify + * @start: first cid of the range + * @nr: number of cids in the range + * + * The range is clamped to @m's active range first. Words are updated atomically + * so concurrent updates of other bits sharing a word are never lost. The range + * as a whole does not transition atomically. + */ +static __always_inline void cmask_set_range(struct scx_cmask __arena *m, u32 start, u32 nr) +{ + u64 end = (u64)start + nr; + u32 wbase = m->base / 64; + u32 first_wi, last_wi, first_bit, last_bit, last, i; + + if (start < m->base) + start = m->base; + if (end > m->base + m->nr_cids) + end = m->base + m->nr_cids; + if (start >= end) + return; + last = end - 1; + + first_wi = start / 64 - wbase; + last_wi = last / 64 - wbase; + first_bit = start & 63; + last_bit = last & 63; + + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { + u32 wi = first_wi + i; + u64 mask = ~0ULL; + + if (wi > last_wi) + break; + if (wi == first_wi) + mask &= GENMASK_U64(63, first_bit); + if (wi == last_wi) + mask &= GENMASK_U64(last_bit, 0); + __cmask_word_or(&m->bits[wi], mask); + } +} + +/** + * cmask_clear_range - Clear every cid in [@start, @start + @nr) + * @m: cmask to modify + * @start: first cid of the range + * @nr: number of cids in the range + * + * The range is clamped to @m's active range first. Words are updated atomically + * so concurrent updates of other bits sharing a word are never lost. The range + * as a whole does not transition atomically. + */ +static __always_inline void cmask_clear_range(struct scx_cmask __arena *m, u32 start, u32 nr) +{ + u64 end = (u64)start + nr; + u32 wbase = m->base / 64; + u32 first_wi, last_wi, first_bit, last_bit, last, i; + + if (start < m->base) + start = m->base; + if (end > m->base + m->nr_cids) + end = m->base + m->nr_cids; + if (start >= end) + return; + last = end - 1; + + first_wi = start / 64 - wbase; + last_wi = last / 64 - wbase; + first_bit = start & 63; + last_bit = last & 63; + + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { + u32 wi = first_wi + i; + u64 mask = ~0ULL; + + if (wi > last_wi) + break; + if (wi == first_wi) + mask &= GENMASK_U64(63, first_bit); + if (wi == last_wi) + mask &= GENMASK_U64(last_bit, 0); + __cmask_word_andnot(&m->bits[wi], mask); + } +} + static __always_inline void cmask_zero(struct scx_cmask __arena *m) { u32 nr_words = CMASK_NR_WORDS(m->nr_cids), i; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { if (i >= nr_words) break; m->bits[i] = 0; } } +/** + * cmask_fill - Set every cid in @m's active range + * @m: cmask to fill + * + * Counterpart to cmask_zero(). Storage past the active range is left as is, + * matching the kernel-side scx_cmask_fill(). + */ +static __always_inline void cmask_fill(struct scx_cmask __arena *m) +{ + u32 nr_words, head_bits, tail_bits, i; + + if (!m->nr_cids) + return; + nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1; + + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { + if (i >= nr_words) + break; + m->bits[i] = ~0LLU; + } + + /* clear word-0 bits below base */ + head_bits = m->base & 63; + if (head_bits) + m->bits[0] &= ~((1LLU << head_bits) - 1); + + /* clear last-word bits at or past base + nr_cids */ + tail_bits = (m->base + m->nr_cids) & 63; + if (tail_bits) + m->bits[nr_words - 1] &= (1LLU << tail_bits) - 1; +} + +/** + * cmask_empty - Test whether no cid is set in @m's active range + * @m: cmask to test + * + * Scans the words the active range spans, whose bits outside the range every + * mutator keeps zero. + */ +static __always_inline bool cmask_empty(const struct scx_cmask __arena *m) +{ + u32 nr_words = cmask_nr_words(m), i; + + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { + if (i >= nr_words) + break; + if (m->bits[i]) + return false; + } + return true; +} + /* * BPF_-prefixed to avoid colliding with the kernel's anonymous CMASK_OP_* * enum in ext/cid.c, which is exported via BTF and reachable through @@ -291,92 +624,136 @@ enum { BPF_CMASK_OP_ANDNOT, }; -static __always_inline void cmask_op_word(struct scx_cmask __arena *dst, - const struct scx_cmask __arena *src, - u32 di, u32 si, u64 mask, int op) +/* range bits of word @k of @m, which spans @nr words: 0 past the span */ +static __always_inline u64 __cmask_word_range(const struct scx_cmask __arena *m, u32 k, + u32 nr) { - u64 dv = dst->bits[di]; - u64 sv = src->bits[si]; - u64 rv; + u64 bits = ~0ULL; - if (op == BPF_CMASK_OP_AND) - rv = dv & sv; - else if (op == BPF_CMASK_OP_OR) - rv = dv | sv; - else if (op == BPF_CMASK_OP_ANDNOT) - rv = dv & ~sv; - else - rv = sv; - - dst->bits[di] = (dv & ~mask) | (rv & mask); + if (k >= nr) + return 0; + if (k == 0) + bits &= GENMASK_U64(63, m->base & 63); + if (k == nr - 1) + bits &= GENMASK_U64((m->base + m->nr_cids - 1) & 63, 0); + return bits; } -static __always_inline void cmask_op(struct scx_cmask __arena *dst, - const struct scx_cmask __arena *src, int op) +/* + * The binary operations combine cmasks of any active ranges. A cmask holds bits + * only for its own range and reads as the operation's identity outside it, all + * ones for AND and all zeros for OR, so an operand alters only the bits it + * covers: ANDing a shard's mask into a global one edits that shard's window and + * leaves the rest of the global mask alone. ANDNOT is AND with @src2 inverted + * over @src2's range, so where only @src2 has range the result is ~@src2. + * + * @dst is written on its range intersected with the union of the sources' + * ranges and untouched elsewhere, and its padding bits outside its own range + * stay zero, the invariant every cmask mutator keeps. @dst may alias either + * source, and the two sources may be one mask. Returns whether @dst has a bit + * set afterwards, over all of @dst's range. + * + * The operations are global functions, verified once per program. Inlined, + * every data-dependent branch in this loop would multiply the states the + * verifier explores per iteration at every call site. + */ +static __always_inline bool __cmask_op(struct scx_cmask __arena *dst, + const struct scx_cmask __arena *src1, + const struct scx_cmask __arena *src2, int op) { - u32 d_end = dst->base + dst->nr_cids; - u32 s_end = src->base + src->nr_cids; - u32 lo = dst->base > src->base ? dst->base : src->base; - u32 hi = d_end < s_end ? d_end : s_end; - u32 d_base = dst->base / 64; - u32 s_base = src->base / 64; - u32 lo_word, hi_word, i; - u64 head_mask, tail_mask; - - if (lo >= hi) - return; - - lo_word = lo / 64; - hi_word = (hi - 1) / 64; - head_mask = GENMASK_U64(63, lo & 63); - tail_mask = GENMASK_U64((hi - 1) & 63, 0); - - bpf_for(i, 0, CMASK_MAX_WORDS) { - u32 w = lo_word + i; - u64 m; - - if (w > hi_word) - break; - - m = GENMASK_U64(63, 0); - if (w == lo_word) - m &= head_mask; - if (w == hi_word) - m &= tail_mask; + u32 d_first = dst->base / 64, d_nr = cmask_nr_words(dst); + u32 s1_first = src1->base / 64, s1_nr = cmask_nr_words(src1); + u32 s2_first = src2->base / 64, s2_nr = cmask_nr_words(src2); + u64 any = bpf_arena_loop_zero; + u32 i; - cmask_op_word(dst, src, w - d_base, w - s_base, m, op); + bpf_arena_for(i, 0, d_nr) { + u32 k1 = d_first + i - s1_first, k2 = d_first + i - s2_first; + u64 m1 = __cmask_word_range(src1, k1, s1_nr); + u64 m2 = op == BPF_CMASK_OP_COPY ? m1 : __cmask_word_range(src2, k2, s2_nr); + u64 um = __cmask_word_range(dst, i, d_nr) & (m1 | m2); + u64 dv = dst->bits[i], v1, v2, rv; + + if (um) { + /* a word in range holds zeros outside the range */ + v1 = m1 ? src1->bits[k1] : 0; + v2 = m2 ? src2->bits[k2] : 0; + if (op == BPF_CMASK_OP_AND) + rv = (v1 | ~m1) & (v2 | ~m2); + else if (op == BPF_CMASK_OP_OR) + rv = v1 | v2; + else if (op == BPF_CMASK_OP_ANDNOT) + rv = (v1 | ~m1) & ~v2; + else + rv = v1; + dv = (dv & ~um) | (rv & um); + dst->bits[i] = dv; + } + any |= dv; } + return any != 0; } -/* - * cmask_and/or/copy only modify @dst bits that lie in the intersection of - * [@dst->base, @dst->base + @dst->nr_cids) and [@src->base, - * @src->base + @src->nr_cids). Bits in @dst outside that window - * keep their prior values - in particular, cmask_copy() does NOT zero @dst - * bits that lie outside @src's range. +/** + * cmask_and - Store @src1 AND @src2 in @dst + * @dst: destination, may be @src1 or @src2 + * @src1: first operand + * @src2: second operand + * + * See __cmask_op() for how the ranges combine. Return whether @dst has a bit + * set afterwards. */ -static __always_inline void cmask_and(struct scx_cmask __arena *dst, - const struct scx_cmask __arena *src) +__weak bool cmask_and(struct scx_cmask __arena __arg_arena *dst, + const struct scx_cmask __arena __arg_arena *src1, + const struct scx_cmask __arena __arg_arena *src2) { - cmask_op(dst, src, BPF_CMASK_OP_AND); + return __cmask_op(dst, src1, src2, BPF_CMASK_OP_AND); } -static __always_inline void cmask_or(struct scx_cmask __arena *dst, - const struct scx_cmask __arena *src) +/** + * cmask_or - Store @src1 OR @src2 in @dst + * @dst: destination, may be @src1 or @src2 + * @src1: first operand + * @src2: second operand + * + * See __cmask_op() for how the ranges combine. Return whether @dst has a bit + * set afterwards. + */ +__weak bool cmask_or(struct scx_cmask __arena __arg_arena *dst, + const struct scx_cmask __arena __arg_arena *src1, + const struct scx_cmask __arena __arg_arena *src2) { - cmask_op(dst, src, BPF_CMASK_OP_OR); + return __cmask_op(dst, src1, src2, BPF_CMASK_OP_OR); } -static __always_inline void cmask_copy(struct scx_cmask __arena *dst, - const struct scx_cmask __arena *src) +/** + * cmask_andnot - Store @src1 AND NOT @src2 in @dst + * @dst: destination, may be @src1 or @src2 + * @src1: operand to remove bits from + * @src2: bits to remove, inverted over its own range + * + * See __cmask_op() for how the ranges combine. Return whether @dst has a bit + * set afterwards. + */ +__weak bool cmask_andnot(struct scx_cmask __arena __arg_arena *dst, + const struct scx_cmask __arena __arg_arena *src1, + const struct scx_cmask __arena __arg_arena *src2) { - cmask_op(dst, src, BPF_CMASK_OP_COPY); + return __cmask_op(dst, src1, src2, BPF_CMASK_OP_ANDNOT); } -static __always_inline void cmask_andnot(struct scx_cmask __arena *dst, - const struct scx_cmask __arena *src) +/** + * cmask_copy - Copy @src into @dst + * @dst: destination + * @src: source + * + * The single-source case of __cmask_op(): @dst is written over @src's range and + * untouched elsewhere. + */ +__weak void cmask_copy(struct scx_cmask __arena __arg_arena *dst, + const struct scx_cmask __arena __arg_arena *src) { - cmask_op(dst, src, BPF_CMASK_OP_ANDNOT); + __cmask_op(dst, src, src, BPF_CMASK_OP_COPY); } /* @@ -395,7 +772,7 @@ static __always_inline bool cmask_equal(const struct scx_cmask __arena *a, return true; nr_words = (a->base + a->nr_cids - 1) / 64 - a->base / 64 + 1; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { if (i >= nr_words) break; if (a->bits[i] != b->bits[i]) @@ -428,7 +805,7 @@ static __always_inline u32 cmask_next_set(const struct scx_cmask __arena *m, u32 start_wi = cid / 64 - base; start_bit = cid & 63; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { u32 wi = start_wi + i; u64 word; u32 found; @@ -457,7 +834,7 @@ static __always_inline u32 cmask_first_set(const struct scx_cmask __arena *m) #define cmask_for_each(cid, m) \ for ((cid) = cmask_first_set(m); \ - (cid) < (m)->base + (m)->nr_cids; \ + (cid) < (m)->base + (m)->nr_cids && can_loop; \ (cid) = cmask_next_set((m), (cid) + 1)) /* @@ -495,7 +872,7 @@ static __always_inline bool cmask_subset(const struct scx_cmask __arena *a, lo_word = lo / 64; hi_word = (hi - 1) / 64; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { u32 w = lo_word + i; if (w > hi_word) @@ -514,13 +891,14 @@ static __always_inline bool cmask_subset(const struct scx_cmask __arena *a, static __always_inline u32 cmask_weight(const struct scx_cmask __arena *m) { u32 nr_words, i; - u32 count = 0; + /* callers compare the sum, see bpf_arena_loop_zero */ + u32 count = bpf_arena_loop_zero; if (!m->nr_cids) return 0; nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { if (i >= nr_words) break; count += __builtin_popcountll(m->bits[i]); @@ -528,10 +906,7 @@ static __always_inline u32 cmask_weight(const struct scx_cmask __arena *m) return count; } -/* - * True if @a and @b share any set bit. Walk only the intersection of their - * ranges, matching the semantics of cmask_and(). - */ +/* true if @a and @b share any set bit, over the intersection of their ranges */ static __always_inline bool cmask_intersects(const struct scx_cmask __arena *a, const struct scx_cmask __arena *b) { @@ -552,7 +927,7 @@ static __always_inline bool cmask_intersects(const struct scx_cmask __arena *a, head_mask = GENMASK_U64(63, lo & 63); tail_mask = GENMASK_U64((hi - 1) & 63, 0); - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { u32 w = lo_word + i; u64 mask, av, bv; @@ -603,7 +978,7 @@ static __always_inline u32 cmask_next_and_set(const struct scx_cmask __arena *a, start_wi = start / 64; start_bit = start & 63; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { u32 abs_wi = start_wi + i; u64 word; u32 found; @@ -646,6 +1021,17 @@ static __always_inline u32 cmask_next_set_wrap(const struct scx_cmask __arena *m return found < start ? found : end; } +/** + * cmask_end - One past the last cid of @m's active range + * @m: cmask of interest + * + * The scan helpers return this value when no set cid is found. + */ +static __always_inline u32 cmask_end(const struct scx_cmask __arena *m) +{ + return m->base + m->nr_cids; +} + /* * Find the next cid set in both @a and @b at or after @start, wrapping to * @a->base if none found in the forward half. Return a->base + a->nr_cids @@ -669,6 +1055,121 @@ static __always_inline u32 cmask_next_and_set_wrap(const struct scx_cmask __aren } /* + * The distribute helpers rotate through one slot per cid, a cacheline each so + * picks on one CPU do not bounce the line of another. A missing allocation is a + * setup bug and aborts the scheduler. On a CPU outside the scheduler's cid + * space the helpers fall back to the first matching cid. + * + * TODO: The slots are one allocation without node placement. Move them to a + * per-cid arena allocator once the library has one, for node-local slots + * without the cacheline padding. + */ +struct cmask_rotor_slot { + u32 cid; +} __attribute__((aligned(SCX_CACHELINE_SIZE))); + +struct cmask_rotor_slot __arena *cmask_distribute_rotors __weak; + +/** + * cmask_distribute_init - Allocate the distribute rotor from @map + * @map: the scheduler's arena map + * + * The shared arena init calls this once. A scheduler with its own arena calls + * it before the first distribute pick. On a kernel without the cid kfuncs + * nothing can pick, so the call allocates nothing and succeeds. Return 0 on + * success, -ENOMEM if the allocation fails. + */ +static __always_inline int cmask_distribute_init(void *map) +{ + u64 size; + u32 pages; + + /* no cid kfuncs means no picks and nothing to allocate */ + if (!bpf_ksym_exists(scx_bpf_nr_cids)) + return 0; + + size = scx_bpf_nr_cids() * sizeof(*cmask_distribute_rotors); + pages = (size + PAGE_SIZE - 1) / PAGE_SIZE; + cmask_distribute_rotors = bpf_arena_alloc_pages(map, NULL, pages, NUMA_NO_NODE, 0); + return cmask_distribute_rotors ? 0 : -ENOMEM; +} + +static __always_inline u32 __arena *__cmask_distribute_rotor(void) +{ + s32 cid = scx_bpf_this_cid(); + + if (unlikely(!cmask_distribute_rotors)) { + scx_bpf_error("cmask distribute rotor not allocated"); + return NULL; + } + if (cid < 0) + return NULL; + return &cmask_distribute_rotors[cid].cid; +} + +/** + * cmask_any_distribute - Pick a set cid, spreading successive picks + * @m: cmask to pick from + * + * Counterpart of bpf_cpumask_any_distribute(): a per-cid rotor makes successive + * picks rotate through the set cids instead of repeating the first one. Returns + * cmask_end(@m) if @m is empty. + */ +static __always_inline u32 cmask_any_distribute(const struct scx_cmask __arena *m) +{ + u32 __arena *rotor = __cmask_distribute_rotor(); + u32 pick; + + if (unlikely(!rotor)) + return cmask_first_set(m); + + pick = cmask_next_set_wrap(m, *rotor + 1); + if (pick < cmask_end(m)) + *rotor = pick; + return pick; +} + +/** + * cmask_any_and_distribute - Pick a cid set in both masks, spreading picks + * @a: first cmask, the scan is bounded by its range + * @b: second cmask + * + * Counterpart of bpf_cpumask_any_and_distribute(). Shares the rotor with + * cmask_any_distribute() the same way the kernel counterparts share theirs. + * Returns cmask_end(@a) if the intersection is empty. + */ +static __always_inline u32 cmask_any_and_distribute(const struct scx_cmask __arena *a, + const struct scx_cmask __arena *b) +{ + u32 __arena *rotor = __cmask_distribute_rotor(); + u32 pick; + + if (unlikely(!rotor)) + return cmask_next_and_set(a, b, a->base); + + pick = cmask_next_and_set_wrap(a, b, *rotor + 1); + if (pick < cmask_end(a)) + *rotor = pick; + return pick; +} + +/** + * cmask_distribute_rotor_pos - Last cid picked by the distribute helpers + * + * For callers that anchor scans of their own on the shared rotor. Returns 0 + * when nothing has been picked on this cid yet or the CPU is outside the cid + * space. + */ +static __always_inline u32 cmask_distribute_rotor_pos(void) +{ + u32 __arena *rotor = __cmask_distribute_rotor(); + + if (unlikely(!rotor)) + return 0; + return *rotor; +} + +/* * Like cmask_next_and_set() but over the intersection of THREE masks. Return * a->base + a->nr_cids if no cid is set in all three at or after @start. */ @@ -701,7 +1202,7 @@ static __always_inline u32 cmask_next_and2_set(const struct scx_cmask __arena *a start_wi = start / 64; start_bit = start & 63; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { u32 abs_wi = start_wi + i; u64 word; u32 found; @@ -763,7 +1264,7 @@ static __always_inline void cmask_from_cpumask(struct scx_cmask __arena *m, s32 cpu; cmask_zero(m); - bpf_for(cpu, 0, nr_cpu_ids) { + bpf_arena_for(cpu, 0, nr_cpu_ids) { s32 cid; if (!bpf_cpumask_test_cpu(cpu, cpumask)) diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 22f24ebef8a9..9376d5704a47 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -27,6 +27,7 @@ #include "user_exit_info.bpf.h" #include "enum_defs.autogen.h" #include "bpf_arena_common.bpf.h" +#include "const-defs.h" #define PF_IDLE 0x00000002 /* I am an IDLE thread */ #define PF_IO_WORKER 0x00000010 /* Task is an IO worker */ @@ -74,6 +75,7 @@ struct task_struct *scx_bpf_dsq_peek(u64 dsq_id) __ksym __weak; int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, u64 flags) __ksym __weak; struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) __ksym __weak; void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) __ksym __weak; +bool scx_bpf_dsq_reenq_task(struct bpf_iter_scx_dsq *it, struct task_struct *p) __ksym __weak; void scx_bpf_exit_bstr(s64 exit_code, char *fmt, unsigned long long *data, u32 data__sz) __ksym __weak; void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym __weak; @@ -104,10 +106,11 @@ struct task_struct *scx_bpf_cpu_curr(s32 cpu) __ksym __weak; struct task_struct *scx_bpf_tid_to_task(u64 tid) __ksym __weak; u64 scx_bpf_now(void) __ksym __weak; void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __weak; +u32 scx_bpf_cgroup_nr_cpus(struct cgroup *cgrp) __ksym __weak; s32 scx_bpf_cpu_to_cid(s32 cpu) __ksym __weak; s32 scx_bpf_cid_to_cpu(s32 cid) __ksym __weak; -void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out, size_t out__sz) __ksym __weak; -void scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak; +s32 scx_bpf_cid_node(s32 cid) __ksym __weak; +/* scx_bpf_cid_topo() and scx_bpf_kick_cid() are declared in compat.bpf.h */ s32 scx_bpf_task_cid(const struct task_struct *p) __ksym __weak; s32 scx_bpf_this_cid(void) __ksym __weak; struct task_struct *scx_bpf_cid_curr(s32 cid) __ksym __weak; @@ -116,7 +119,7 @@ u32 scx_bpf_nr_online_cids(void) __ksym __weak; const void __arena *scx_bpf_online_cmask(void) __ksym __weak; u32 scx_bpf_cidperf_cap(s32 cid) __ksym __weak; u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak; -s32 scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak; +/* scx_bpf_cidperf_set() is declared in compat.bpf.h */ /* sub-scheduler cap control, scx_bpf_sub_caps() cgroup_id 0 == self */ s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask __arena *cmask__arena, struct scx_cmask __arena *denied_out__arena__nullable) __ksym __weak; @@ -307,7 +310,7 @@ BPF_PROG(name, ##args) * Similar to MEMBER_VPTR() but is intended for use with arrays where the * element count needs to be explicit. * It can be used in cases where a global array is defined with an initial - * size but is intended to be be resized before loading the BPF program. + * size but is intended to be resized before loading the BPF program. * Without this version of the macro, MEMBER_VPTR() will use the compile time * size of the array to compute the max, which will result in rejection by * the verifier. @@ -533,14 +536,14 @@ static __always_inline const struct cpumask *cast_mask(struct bpf_cpumask *mask) /* * True if the non-sleepable BPF trampoline prolog (__bpf_prog_enter) calls * migrate_disable() for the current task. Recorded once by - * scx_lib_init_probe, an fentry program on bpf_scx_reg() that fires during - * the natural scheduler-attach call chain (auto-attached by scx_ops_attach!). + * scx_lib_init_probe, an fentry program that fires during the natural + * scheduler-attach call chain (auto-attached by scx_ops_attach!). * - * Defaults to true (conservative). Over-reporting in is_migration_disabled() + * Defaults to false (conservative). Over-reporting in is_migration_disabled() * causes local-only dispatch, which is safe. Under-reporting can crash the * scheduler, so we err high if the probe somehow fails to run. */ -bool __scx_prolog_disables_migration __weak = true; +bool __scx_prolog_disables_migration __weak = false; /* * scx_lib_init_probe - non-sleepable prolog probe. @@ -551,13 +554,20 @@ bool __scx_prolog_disables_migration __weak = true; * ops.init() fires. Its address is taken in the vtable, so the symbol * is non-inlinable and has been stable since introduction. * + * A cid-form scheduler registers through bpf_scx_reg_cid(), the .reg + * callback of bpf_sched_ext_ops_cid, and so never enters bpf_scx_reg(). + * The cid-form open paths (SCX_OPS_CID_OPEN(), scx_ops_cid_open!()) + * therefore repoint this program at bpf_scx_reg_cid(). Repointing rather + * than adding a second program keeps the object loadable on kernels + * predating the cid form, where bpf_scx_reg_cid() has no BTF entry. + * * Entering via fentry runs us through __bpf_prog_enter -- the * non-sleepable prolog that consumers of is_migration_disabled() live * under. * * Loud warning: the prolog adds at most 1 to migration_disabled. * Reading > 1 means something upstream in the - * bpf_struct_ops_link_create -> bpf_scx_reg path disabled migration + * bpf_struct_ops_link_create -> .reg path disabled migration * before the prolog ran, invalidating the probe; audit and adjust. */ SEC("fentry/bpf_scx_reg") __weak @@ -1185,8 +1195,75 @@ static inline u64 scx_clock_irq(u32 cpu) #define struct_size_t(type, member, count) \ struct_size((type *)NULL, member, count) +/* + * <linux/stddef.h>'s TRAILING_OVERLAP(): embed a struct that ends in a flexible + * array member together with its storage. @NAME is a complete @TYPE and the + * @MEMBERS declared after the array reserve the space the array grows into. + * Unlike the kernel's, the padding is named after @NAME so that one struct can + * embed several, and it is sized with __builtin_offsetof() because + * bpf_helpers.h redefines offsetof() as a pointer cast, which is not a constant + * expression. + */ +#define __TRAILING_OVERLAP(TYPE, NAME, FAM, ATTRS, MEMBERS) \ + union { \ + TYPE NAME; \ + struct { \ + unsigned char __offset_to_##NAME[__builtin_offsetof(TYPE, FAM)]; \ + MEMBERS \ + } ATTRS; \ + } + +#define TRAILING_OVERLAP(TYPE, NAME, FAM, MEMBERS) \ + __TRAILING_OVERLAP(TYPE, NAME, FAM, /* no attrs */, MEMBERS) + +/* + * Loop counters the verifier cannot see through. + * + * A may_goto loop converges when the state at its head is within the state + * of a previous iteration. A counter tracked as a precise constant prevents + * that convergence: every iteration creates a new state, and the verifier + * unrolls the loop until it runs out of budget whenever the counter feeds an + * operation that requires precision. + * + * Loading a writable global gives the verifier an unknown scalar while its + * runtime value remains one. Using it as the step makes the counter unknown + * after the first iteration, and the load on every increment keeps the loop + * body from making it precise again. volatile is required to keep the compiler + * from hoisting or eliminating the loads, and the value must remain non-const + * so the verifier cannot resolve it. The variable is weak so the objects linked + * into a scheduler share one copy. Its runtime value must never be changed from + * one. + * + * bpf_for() avoids this verifier behavior too, but calls bpf_iter_num_next() + * on every iteration. bpf_arena_for() is intended for hot scheduler walks + * where that cost matters. @var must be no wider than u32, and @start and + * @end must be representable as u32. + */ +volatile u32 bpf_arena_loop_one __weak = 1; + +/* + * A loop-carried accumulator that a caller later compares is kept precise, and + * a precise scalar whose range grows every iteration keeps a may_goto loop from + * converging. Initializing it from this zero makes it unknown from the first + * iteration, so every pass through the loop head looks the same. + */ +volatile u32 bpf_arena_loop_zero __weak = 0; + +#define __bpf_arena_loop_start(var, start) \ + ({ \ + _Static_assert(sizeof(var) <= sizeof(u32), \ + "bpf_arena_for() index must fit in u32"); \ + (start); \ + }) + +#define bpf_arena_for(var, start, end) \ + for (var = __bpf_arena_loop_start(var, start); \ + var < (end) && can_loop; \ + var += bpf_arena_loop_one) + #include "compat.bpf.h" #include "enums.bpf.h" +#include "features.bpf.h" #include "cid.bpf.h" #endif /* __SCX_COMMON_BPF_H */ diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h index 6944221f96cc..254882bd8ce6 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -143,6 +143,69 @@ static inline void scx_bpf_cid_override(const s32 __arena *cpu_to_cid, u32 cpu_t shard_start, shard_start_cnt); } +/* + * v7.3: b38332be61a8 ("sched_ext: Make scx_bpf_kick_cid() return void") flipped + * the return type from s32 to void without renaming the kfunc, so the v7.2 and + * v7.3 kfuncs share a name but have incompatible prototypes. libbpf leaves a + * weak kfunc unresolved when the prototype doesn't match, so declare both + * flavors: the ___vN suffix is stripped on the BPF side, both resolve against + * the same kernel symbol and only the matching one gets bound. Drop the + * wrapper and move the decl back to common.bpf.h after v7.6. + */ +void scx_bpf_kick_cid___v2(s32 cid, u64 flags) __ksym __weak; +s32 scx_bpf_kick_cid___v1(s32 cid, u64 flags) __ksym __weak; + +static inline void scx_bpf_kick_cid(s32 cid, u64 flags) +{ + if (bpf_ksym_exists(scx_bpf_kick_cid___v2)) + scx_bpf_kick_cid___v2(cid, flags); + else if (bpf_ksym_exists(scx_bpf_kick_cid___v1)) + scx_bpf_kick_cid___v1(cid, flags); +} + +/* + * v7.3: 94480606a677 ("sched_ext: Add a size argument to scx_bpf_cid_topo() so + * struct scx_cid_topo can grow") added out__sz without renaming the kfunc, so + * the v7.2 and v7.3 kfuncs share a name with incompatible prototypes. Same + * two-flavor trick as scx_bpf_kick_cid() above. The size is always the output + * buffer's, so a macro supplies it and callers keep the two-argument form. + * Drop the wrapper and move the decl back to common.bpf.h after v7.6. + */ +void scx_bpf_cid_topo___v2(s32 cid, struct scx_cid_topo *out, size_t out__sz) __ksym __weak; +void scx_bpf_cid_topo___v1(s32 cid, struct scx_cid_topo *out) __ksym __weak; + +static inline void __scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out, size_t out__sz) +{ + if (bpf_ksym_exists(scx_bpf_cid_topo___v2)) + scx_bpf_cid_topo___v2(cid, out, out__sz); + else if (bpf_ksym_exists(scx_bpf_cid_topo___v1)) + scx_bpf_cid_topo___v1(cid, out); +} + +#define scx_bpf_cid_topo(cid, out) __scx_bpf_cid_topo((cid), (out), sizeof(*(out))) + +/* + * v7.3: 3a21e34eb258 ("sched_ext: Gate scx_bpf_cidperf_set() behind a new + * SCX_CAP_PERF") flipped the return type the other way, from void to s32, again + * without renaming the kfunc. Same two-flavor trick as scx_bpf_kick_cid() + * above; the v7.2 kfunc can't fail, so report success for it. Drop the wrapper + * and move the decl back to common.bpf.h after v7.6. + */ +s32 scx_bpf_cidperf_set___v2(s32 cid, u32 perf) __ksym __weak; +void scx_bpf_cidperf_set___v1(s32 cid, u32 perf) __ksym __weak; + +static inline s32 scx_bpf_cidperf_set(s32 cid, u32 perf) +{ + if (bpf_ksym_exists(scx_bpf_cidperf_set___v2)) { + return scx_bpf_cidperf_set___v2(cid, perf); + } else if (bpf_ksym_exists(scx_bpf_cidperf_set___v1)) { + scx_bpf_cidperf_set___v1(cid, perf); + return 0; + } else { + return -EOPNOTSUPP; + } +} + /** * __COMPAT_is_enq_cpu_selected - Test if SCX_ENQ_CPU_SELECTED is on * in a compatible way. We will preserve this __COMPAT helper until v6.16. @@ -243,6 +306,18 @@ static inline bool __COMPAT_is_enq_cpu_selected(u64 enq_flags) scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) : \ scx_bpf_pick_any_cpu(cpus_allowed, flags)) +/* v7.4: Add scx_bpf_cid_node(). */ +#define __COMPAT_scx_bpf_cid_node(cid) \ + (bpf_ksym_exists(scx_bpf_cid_node) ? \ + scx_bpf_cid_node(cid) : NUMA_NO_NODE) + +/* + * v7.4: Add scx_bpf_cgroup_nr_cpus(). + */ +#define __COMPAT_scx_bpf_cgroup_nr_cpus(cgrp) \ + (bpf_ksym_exists(scx_bpf_cgroup_nr_cpus) ? \ + scx_bpf_cgroup_nr_cpus(cgrp) : scx_bpf_nr_cpu_ids()) + /* * v6.18: Add a helper to retrieve the current task running on a CPU. * @@ -403,6 +478,15 @@ static inline void scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime) p->scx.dsq_vtime = vtime; } +/* v7.4: Add scx_bpf_task_set_lazy_resched(). */ +bool scx_bpf_task_set_lazy_resched___new(struct task_struct *p, bool lazy) __ksym __weak; + +static inline void scx_bpf_task_set_lazy_resched(struct task_struct *p, bool lazy) +{ + if (bpf_ksym_exists(scx_bpf_task_set_lazy_resched___new)) + scx_bpf_task_set_lazy_resched___new(p, lazy); +} + /* * v7.1: New scx_bpf_dsq_reenq() that allows re-enqueues on more DSQs. This * will eventually deprecate scx_bpf_reenqueue_local(). @@ -414,6 +498,12 @@ static inline bool __COMPAT_has_generic_reenq(void) return bpf_ksym_exists(scx_bpf_dsq_reenq___compat); } +/* v7.4: Add scx_bpf_dsq_reenq_task(). */ +static inline bool __COMPAT_has_dsq_reenq_task(void) +{ + return bpf_ksym_exists(scx_bpf_dsq_reenq_task); +} + /* * v6.19: The new void variant can be called from anywhere while the older v1 * variant can only be called from ops.cpu_release(). The double ___ prefixes on diff --git a/tools/sched_ext/include/scx/compat.h b/tools/sched_ext/include/scx/compat.h index 7c12df45fdba..07fb55e63c57 100644 --- a/tools/sched_ext/include/scx/compat.h +++ b/tools/sched_ext/include/scx/compat.h @@ -214,6 +214,7 @@ static inline bool __COMPAT_struct_has_field(const char *type, const char *field #define SCX_OPS_ALLOW_QUEUED_WAKEUP SCX_OPS_FLAG(SCX_OPS_ALLOW_QUEUED_WAKEUP) #define SCX_OPS_BUILTIN_IDLE_PER_NODE SCX_OPS_FLAG(SCX_OPS_BUILTIN_IDLE_PER_NODE) #define SCX_OPS_ALWAYS_ENQ_IMMED SCX_OPS_FLAG(SCX_OPS_ALWAYS_ENQ_IMMED) +#define SCX_OPS_ENQ_BLOCKED SCX_OPS_FLAG(SCX_OPS_ENQ_BLOCKED) #define SCX_PICK_IDLE_FLAG(name) __COMPAT_ENUM_OR_ZERO("scx_pick_idle_cpu_flags", #name) @@ -274,23 +275,23 @@ static inline long scx_hotplug_seq(void) * - v7.1: ops.sub_attach(), ops.sub_detach(), ops.sub_cgroup_id * - v7.3: ops.rescue_bandwidth_ppt, ops.rescue_quantum_us */ -#define __SCX_OPS_OPEN(__ops_name, __scx_name, __ops_struct) ({ \ +#define __SCX_OPS_OPEN(__ops_name, __scx_name, __ops_struct, __opts) ({ \ struct __scx_name *__oskel; \ \ SCX_BUG_ON(!__COMPAT_struct_has_field(__ops_struct, "dump"), \ __ops_struct ".dump() missing, kernel too old?"); \ \ - __oskel = __scx_name##__open(); \ + __oskel = __scx_name##__open_opts(__opts); \ SCX_BUG_ON(!__oskel, "Could not open " #__scx_name); \ __oskel->struct_ops.__ops_name->hotplug_seq = scx_hotplug_seq(); \ SCX_ENUM_INIT(__oskel); \ __oskel; \ }) -#define SCX_OPS_OPEN(__ops_name, __scx_name) ({ \ +#define SCX_OPS_OPEN_OPTS(__ops_name, __scx_name, __opts) ({ \ struct __scx_name *__skel; \ \ - __skel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops"); \ + __skel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops", __opts); \ if (__skel->struct_ops.__ops_name->cgroup_set_bandwidth && \ !__COMPAT_struct_has_field("sched_ext_ops", "cgroup_set_bandwidth")) { \ fprintf(stderr, "WARNING: kernel doesn't support ops.cgroup_set_bandwidth()\n"); \ @@ -329,12 +330,29 @@ static inline long scx_hotplug_seq(void) __skel; \ }) +#define SCX_OPS_OPEN(__ops_name, __scx_name) \ + SCX_OPS_OPEN_OPTS(__ops_name, __scx_name, 0) + /* * Open a cid-form (struct sched_ext_ops_cid) skeleton. The cid form postdates * every op the load-time fix-ups above handle, so none of them apply. + * + * A cid-form scheduler registers through bpf_scx_reg_cid() rather than + * bpf_scx_reg(), so repoint common.bpf.h's prolog probe at it. Both the cid + * form and bpf_scx_reg_cid() appeared in v7.2, so a kernel that accepts this + * skeleton always has the symbol. */ +#define SCX_OPS_CID_OPEN_OPTS(__ops_name, __scx_name, __opts) ({ \ + struct __scx_name *__cskel; \ + \ + __cskel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops_cid", __opts); \ + bpf_program__set_attach_target(__cskel->progs.scx_lib_init_probe, 0, \ + "bpf_scx_reg_cid"); \ + __cskel; \ +}) + #define SCX_OPS_CID_OPEN(__ops_name, __scx_name) \ - __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops_cid") + SCX_OPS_CID_OPEN_OPTS(__ops_name, __scx_name, 0) /* * Associate non-struct_ops BPF programs with the scheduler's struct_ops map so diff --git a/tools/sched_ext/include/scx/const-defs.h b/tools/sched_ext/include/scx/const-defs.h new file mode 100644 index 000000000000..4c8f45756b51 --- /dev/null +++ b/tools/sched_ext/include/scx/const-defs.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 Tejun Heo <tj@kernel.org> */ +#pragma once + +/* + * Constants shared across the BPF headers, the lib and the schedulers. + * Freestanding so that scheduler interface headers consumed by bindgen can + * include it. Rust code cannot, so rust/scx_arena/scx_arena/src/lib.rs mirrors + * the values. Keep them in sync. + */ +enum scx_const_defs { + /* mirrors the kernel's per-arch L1_CACHE_SHIFT */ +#if defined(__TARGET_ARCH_s390) || defined(__s390x__) + SCX_CACHELINE_SIZE = 256, +#elif defined(__TARGET_ARCH_powerpc) || defined(__powerpc64__) + SCX_CACHELINE_SIZE = 128, +#else + SCX_CACHELINE_SIZE = 64, +#endif +}; diff --git a/tools/sched_ext/include/scx/enum_defs.autogen.h b/tools/sched_ext/include/scx/enum_defs.autogen.h index 63b6b14b19bd..4fa82033b99c 100644 --- a/tools/sched_ext/include/scx/enum_defs.autogen.h +++ b/tools/sched_ext/include/scx/enum_defs.autogen.h @@ -85,10 +85,12 @@ #define HAVE_SCX_ENQ_HEAD #define HAVE_SCX_ENQ_CPU_SELECTED #define HAVE_SCX_ENQ_PREEMPT +#define HAVE_SCX_ENQ_PREEMPT_LAZY #define HAVE_SCX_ENQ_IMMED #define HAVE_SCX_ENQ_RESCUE #define HAVE_SCX_ENQ_REENQ #define HAVE_SCX_ENQ_LAST +#define HAVE_SCX_ENQ_BLOCKED #define HAVE___SCX_ENQ_INTERNAL_MASK #define HAVE_SCX_ENQ_CLEAR_OPSS #define HAVE_SCX_ENQ_DSQ_PRIQ @@ -105,6 +107,7 @@ #define HAVE_SCX_TASK_SUB_INIT #define HAVE_SCX_TASK_IMMED #define HAVE_SCX_TASK_PROTECTED +#define HAVE_SCX_TASK_RUN_TRACKED #define HAVE_SCX_TASK_STATE_SHIFT #define HAVE_SCX_TASK_STATE_BITS #define HAVE_SCX_TASK_STATE_MASK @@ -122,6 +125,7 @@ #define HAVE_SCX_TASK_REENQ_IMMED #define HAVE_SCX_TASK_REENQ_PREEMPTED #define HAVE_SCX_TASK_REENQ_CAP +#define HAVE_SCX_TASK_REENQ_PROXY #define HAVE_SCX_TASK_CURSOR #define HAVE_SCX_ECODE_RSN_HOTPLUG #define HAVE_SCX_ECODE_RSN_CGROUP_OFFLINE @@ -149,6 +153,8 @@ #define HAVE_SCX_KICK_IDLE #define HAVE_SCX_KICK_PREEMPT #define HAVE_SCX_KICK_WAIT +#define HAVE_SCX_KICK_PREEMPT_LAZY +#define HAVE_SCX_KICK_ALL_FLAGS #define HAVE_SCX_OPI_BEGIN #define HAVE_SCX_OPI_NORMAL_BEGIN #define HAVE_SCX_OPI_NORMAL_END @@ -164,6 +170,8 @@ #define HAVE_SCX_OPS_BUILTIN_IDLE_PER_NODE #define HAVE_SCX_OPS_ALWAYS_ENQ_IMMED #define HAVE_SCX_OPS_TID_TO_TASK +#define HAVE_SCX_OPS_LAZY_RESCHED +#define HAVE_SCX_OPS_ENQ_BLOCKED #define HAVE_SCX_OPS_ALL_FLAGS #define HAVE___SCX_OPS_INTERNAL_MASK #define HAVE_SCX_OPS_HAS_CPU_PREEMPT @@ -191,6 +199,8 @@ #define HAVE_SCX_RQ_BAL_CB_PENDING #define HAVE_SCX_RQ_SUB_IDLE_RENOTIFY #define HAVE_SCX_RQ_ROOT_IDLE_RENOTIFY +#define HAVE_SCX_RQ_PROXY_RETRY +#define HAVE_SCX_RQ_PROXY_TICK #define HAVE_SCX_RQ_IN_WAKEUP #define HAVE_SCX_RQ_IN_DISPATCH #define HAVE_SCX_SCHED_PCPU_BYPASSING diff --git a/tools/sched_ext/include/scx/enums.autogen.bpf.h b/tools/sched_ext/include/scx/enums.autogen.bpf.h index 7268131010de..405a3e6b8f33 100644 --- a/tools/sched_ext/include/scx/enums.autogen.bpf.h +++ b/tools/sched_ext/include/scx/enums.autogen.bpf.h @@ -1,9 +1,9 @@ /* - * WARNING: This file is autogenerated from scripts/gen_enums.py. If you would - * like to access an enum that is currently missing, add it to the script - * and run it from the root directory to update this file. + * Definitions for SCX constants. Keep this table in sync + * with rust/scx_utils/src/enums.rs. */ + const volatile u64 __SCX_OPS_NAME_LEN __weak; #define SCX_OPS_NAME_LEN __SCX_OPS_NAME_LEN @@ -22,6 +22,9 @@ const volatile u64 __SCX_RQ_CAN_STOP_TICK __weak; const volatile u64 __SCX_RQ_BAL_PENDING __weak; #define SCX_RQ_BAL_PENDING __SCX_RQ_BAL_PENDING +const volatile u64 __SCX_RQ_BAL_KEEP __weak; +#define SCX_RQ_BAL_KEEP __SCX_RQ_BAL_KEEP + const volatile u64 __SCX_RQ_BYPASSING __weak; #define SCX_RQ_BYPASSING __SCX_RQ_BYPASSING @@ -112,6 +115,12 @@ const volatile u64 __SCX_KICK_PREEMPT __weak; const volatile u64 __SCX_KICK_WAIT __weak; #define SCX_KICK_WAIT __SCX_KICK_WAIT +const volatile u64 __SCX_KICK_PREEMPT_LAZY __weak; +#define SCX_KICK_PREEMPT_LAZY __SCX_KICK_PREEMPT_LAZY + +const volatile u64 __SCX_KICK_ALL_FLAGS __weak; +#define SCX_KICK_ALL_FLAGS __SCX_KICK_ALL_FLAGS + const volatile u64 __SCX_ENQ_WAKEUP __weak; #define SCX_ENQ_WAKEUP __SCX_ENQ_WAKEUP @@ -121,12 +130,18 @@ const volatile u64 __SCX_ENQ_HEAD __weak; const volatile u64 __SCX_ENQ_PREEMPT __weak; #define SCX_ENQ_PREEMPT __SCX_ENQ_PREEMPT +const volatile u64 __SCX_ENQ_PREEMPT_LAZY __weak; +#define SCX_ENQ_PREEMPT_LAZY __SCX_ENQ_PREEMPT_LAZY + const volatile u64 __SCX_ENQ_IMMED __weak; #define SCX_ENQ_IMMED __SCX_ENQ_IMMED const volatile u64 __SCX_ENQ_RESCUE __weak; #define SCX_ENQ_RESCUE __SCX_ENQ_RESCUE +const volatile u64 __SCX_ENQ_BLOCKED __weak; +#define SCX_ENQ_BLOCKED __SCX_ENQ_BLOCKED + const volatile u64 __SCX_ENQ_REENQ __weak; #define SCX_ENQ_REENQ __SCX_ENQ_REENQ diff --git a/tools/sched_ext/include/scx/enums.autogen.h b/tools/sched_ext/include/scx/enums.autogen.h index e61632654517..7ec2e5e41cb2 100644 --- a/tools/sched_ext/include/scx/enums.autogen.h +++ b/tools/sched_ext/include/scx/enums.autogen.h @@ -1,7 +1,8 @@ +#pragma once + /* - * WARNING: This file is autogenerated from scripts/gen_enums.py. If you would - * like to access an enum that is currently missing, add it to the script - * and run it from the root directory to update this file. + * BTF enum lookups for SCX constants. Keep this table in sync + * with rust/scx_utils/src/enums.rs. */ #define SCX_ENUM_INIT(skel) do { \ @@ -11,6 +12,7 @@ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_ONLINE); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CAN_STOP_TICK); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BAL_PENDING); \ + SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BAL_KEEP); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BYPASSING); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CLK_VALID); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_IN_WAKEUP); \ @@ -32,20 +34,24 @@ SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_STATE_BITS); \ SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_STATE_MASK); \ SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_CURSOR); \ - SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_NONE); \ - SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_INIT); \ - SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_READY); \ - SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_ENABLED); \ - SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_NR_STATES); \ + SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_NONE); \ + SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_INIT); \ + SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_READY); \ + SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_ENABLED); \ + SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_NR_STATES); \ SCX_ENUM_SET(skel, scx_ent_dsq_flags, SCX_TASK_DSQ_ON_PRIQ); \ SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_IDLE); \ SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_PREEMPT); \ SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_WAIT); \ + SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_PREEMPT_LAZY); \ + SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_ALL_FLAGS); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_WAKEUP); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_HEAD); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_PREEMPT); \ + SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_PREEMPT_LAZY); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_IMMED); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_RESCUE); \ + SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_BLOCKED); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_REENQ); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_LAST); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_CLEAR_OPSS); \ diff --git a/tools/sched_ext/include/scx/enums_abi.autogen.h b/tools/sched_ext/include/scx/enums_abi.autogen.h index d53899764f5a..1186daa9cb14 100644 --- a/tools/sched_ext/include/scx/enums_abi.autogen.h +++ b/tools/sched_ext/include/scx/enums_abi.autogen.h @@ -97,10 +97,12 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_enq_flags", "SCX_ENQ_HEAD", 0x10000LLU }, { "scx_enq_flags", "SCX_ENQ_CPU_SELECTED", 0x100000LLU }, { "scx_enq_flags", "SCX_ENQ_PREEMPT", 0x100000000LLU }, + { "scx_enq_flags", "SCX_ENQ_PREEMPT_LAZY", 0x800000000LLU }, { "scx_enq_flags", "SCX_ENQ_IMMED", 0x200000000LLU }, { "scx_enq_flags", "SCX_ENQ_RESCUE", 0x400000000LLU }, { "scx_enq_flags", "SCX_ENQ_REENQ", 0x10000000000LLU }, { "scx_enq_flags", "SCX_ENQ_LAST", 0x20000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_BLOCKED", 0x40000000000LLU }, { "scx_enq_flags", "__SCX_ENQ_INTERNAL_MASK", 0xff00000000000000LLU }, { "scx_enq_flags", "SCX_ENQ_CLEAR_OPSS", 0x100000000000000LLU }, { "scx_enq_flags", "SCX_ENQ_DSQ_PRIQ", 0x200000000000000LLU }, @@ -117,6 +119,7 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_ent_flags", "SCX_TASK_SUB_INIT", 0x10LLU }, { "scx_ent_flags", "SCX_TASK_IMMED", 0x20LLU }, { "scx_ent_flags", "SCX_TASK_PROTECTED", 0x40LLU }, + { "scx_ent_flags", "SCX_TASK_RUN_TRACKED", 0x80LLU }, { "scx_ent_flags", "SCX_TASK_STATE_SHIFT", 0x8LLU }, { "scx_ent_flags", "SCX_TASK_STATE_BITS", 0x3LLU }, { "scx_ent_flags", "SCX_TASK_STATE_MASK", 0x700LLU }, @@ -134,6 +137,7 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_ent_flags", "SCX_TASK_REENQ_IMMED", 0x2000LLU }, { "scx_ent_flags", "SCX_TASK_REENQ_PREEMPTED", 0x3000LLU }, { "scx_ent_flags", "SCX_TASK_REENQ_CAP", 0x4000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_PROXY", 0x5000LLU }, { "scx_ent_flags", "SCX_TASK_CURSOR", 0xffffffff80000000LLU }, { "scx_exit_code", "SCX_ECODE_RSN_HOTPLUG", 0x100000000LLU }, { "scx_exit_code", "SCX_ECODE_RSN_CGROUP_OFFLINE", 0x200000000LLU }, @@ -161,6 +165,8 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_kick_flags", "SCX_KICK_IDLE", 0x1LLU }, { "scx_kick_flags", "SCX_KICK_PREEMPT", 0x2LLU }, { "scx_kick_flags", "SCX_KICK_WAIT", 0x4LLU }, + { "scx_kick_flags", "SCX_KICK_PREEMPT_LAZY", 0x8LLU }, + { "scx_kick_flags", "SCX_KICK_ALL_FLAGS", 0xfLLU }, { "scx_opi", "SCX_OPI_BEGIN", 0x0LLU }, { "scx_opi", "SCX_OPI_NORMAL_BEGIN", 0x0LLU }, { "scx_opi", "SCX_OPI_NORMAL_END", 0x21LLU }, @@ -176,7 +182,9 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_ops_flags", "SCX_OPS_BUILTIN_IDLE_PER_NODE", 0x40LLU }, { "scx_ops_flags", "SCX_OPS_ALWAYS_ENQ_IMMED", 0x80LLU }, { "scx_ops_flags", "SCX_OPS_TID_TO_TASK", 0x100LLU }, - { "scx_ops_flags", "SCX_OPS_ALL_FLAGS", 0x1ffLLU }, + { "scx_ops_flags", "SCX_OPS_LAZY_RESCHED", 0x200LLU }, + { "scx_ops_flags", "SCX_OPS_ENQ_BLOCKED", 0x400LLU }, + { "scx_ops_flags", "SCX_OPS_ALL_FLAGS", 0x7ffLLU }, { "scx_ops_flags", "__SCX_OPS_INTERNAL_MASK", 0xff00000000000000LLU }, { "scx_ops_flags", "SCX_OPS_HAS_CPU_PREEMPT", 0x100000000000000LLU }, { "scx_ops_state", "SCX_OPSS_NONE", 0x0LLU }, @@ -203,6 +211,8 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_rq_flags", "SCX_RQ_BAL_CB_PENDING", 0x40LLU }, { "scx_rq_flags", "SCX_RQ_SUB_IDLE_RENOTIFY", 0x80LLU }, { "scx_rq_flags", "SCX_RQ_ROOT_IDLE_RENOTIFY", 0x100LLU }, + { "scx_rq_flags", "SCX_RQ_PROXY_RETRY", 0x200LLU }, + { "scx_rq_flags", "SCX_RQ_PROXY_TICK", 0x400LLU }, { "scx_rq_flags", "SCX_RQ_IN_WAKEUP", 0x10000LLU }, { "scx_rq_flags", "SCX_RQ_IN_DISPATCH", 0x20000LLU }, { "scx_sched_pcpu_flags", "SCX_SCHED_PCPU_BYPASSING", 0x1LLU }, diff --git a/tools/sched_ext/include/scx/features.bpf.h b/tools/sched_ext/include/scx/features.bpf.h new file mode 100644 index 000000000000..992f4a74b277 --- /dev/null +++ b/tools/sched_ext/include/scx/features.bpf.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Kernel features the loader probes before load and hands to the BPF side in + * one rodata word, so that a program can carry both the path a feature enables + * and its fallback while the verifier prunes the one the running kernel cannot + * take. scx_utils sets the word in the ops open macros. The bits are mirrored + * in rust/scx_utils/src/compat.rs. + * + * Included by scx/common.bpf.h; don't include directly. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo <tj@kernel.org> + */ +#ifndef __SCX_FEATURES_BPF_H +#define __SCX_FEATURES_BPF_H + +enum scx_lib_feature { + /* the JIT lowers fetching AND, OR and XOR on arena pointers */ + SCX_LIB_FEAT_ARENA_FETCH_BITOPS = 1ULL << 0, +}; + +const volatile u64 scx_lib_features __weak; + +static __always_inline bool scx_lib_has(u64 feat) +{ + return scx_lib_features & feat; +} + +#endif /* __SCX_FEATURES_BPF_H */ diff --git a/tools/sched_ext/include/scx/user_exit_info.h b/tools/sched_ext/include/scx/user_exit_info.h index 56a02b549aef..cf27ac2447ef 100644 --- a/tools/sched_ext/include/scx/user_exit_info.h +++ b/tools/sched_ext/include/scx/user_exit_info.h @@ -52,6 +52,7 @@ enum scx_exit_code { /* Reasons */ SCX_ECODE_RSN_HOTPLUG = 1LLU << 32, + SCX_ECODE_RSN_CGROUP_OFFLINE = 2LLU << 32, /* Actions */ SCX_ECODE_ACT_RESTART = 1LLU << 48, diff --git a/tools/sched_ext/scx_flatcg.bpf.c b/tools/sched_ext/scx_flatcg.bpf.c index 454ebb820c5e..5e6abd0bccb3 100644 --- a/tools/sched_ext/scx_flatcg.bpf.c +++ b/tools/sched_ext/scx_flatcg.bpf.c @@ -144,7 +144,7 @@ static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) cgc_a = container_of(a, struct cgv_node, rb_node); cgc_b = container_of(b, struct cgv_node, rb_node); - return cgc_a->cvtime < cgc_b->cvtime; + return time_before(cgc_a->cvtime, cgc_b->cvtime); } static struct fcg_cpu_ctx *find_cpu_ctx(void) diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 2f3653199842..e8e451be862d 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -369,6 +369,22 @@ static u64 needs_immed(s32 cid) return qa.cid_shared[cid] ? SCX_ENQ_IMMED : 0; } +static void dispatch_to_rescue(struct task_struct *p, task_ctx_t *taskc, + u64 enq_flags) +{ + u32 cid = cmask_next_set_wrap(&taskc->cpus_allowed, 0); + + if (cid >= scx_bpf_nr_cids()) { + scx_bpf_error("task %d has no allowed cid", p->pid); + return; + } + + taskc->force_local = false; + __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); + scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | cid, slice_ns, + enq_flags | SCX_ENQ_RESCUE); +} + /* first cid this node does NOT hold for fault injection, -1 if none */ static s32 first_unavail_cid(void) { @@ -442,6 +458,9 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) */ taskc->core_sched_seq = qa.core_sched_tail_seqs[idx]++; + if (enq_flags & SCX_ENQ_BLOCKED) + __sync_fetch_and_add(&qa.nr_enq_blocked, 1); + /* * A task of ours that can run on none of our self cids - the parent * didn't grant them or we delegated them to children - would starve in @@ -451,18 +470,64 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) * If we hold ENQ on that cid it runs. Otherwise the kernel diverts the * task to its rescue path. IMMED would turn the insert into a legal * placement on a time-shared cid and the kernel would bounce it back - * here instead of rescuing it. + * here instead of rescuing it. Do this before the blocked-donor fast + * paths, which also require an eligible self cid to make progress. */ if (!cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) { - s32 c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); + dispatch_to_rescue(p, taskc, enq_flags); + return; + } - if (c >= 0 && c < scx_bpf_nr_cids()) { - taskc->force_local = false; - __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); - scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | c, slice_ns, - enq_flags | SCX_ENQ_RESCUE); + /* + * SCX_OPS_ALWAYS_ENQ_IMMED makes the local insertion below implicitly + * carry SCX_ENQ_IMMED. If the CPU can't run the blocked donor immediately, + * the core returns it through ops.enqueue() with SCX_ENQ_REENQ. Inserting + * it into the same local DSQ would repeat the IMMED handback until the + * scheduler is ejected. Move reenqueued blocked donors to the shared DSQ, + * which doesn't carry SCX_ENQ_IMMED, so another CPU can consume them. + */ + if ((enq_flags & (SCX_ENQ_BLOCKED | SCX_ENQ_REENQ)) == + (SCX_ENQ_BLOCKED | SCX_ENQ_REENQ)) { + taskc->force_local = false; + scx_bpf_dsq_insert(p, SHARED_DSQ, 0, enq_flags); + cid = cmask_next_and2_set_wrap(&taskc->cpus_allowed, + &qa.idle_cids.mask, + &qa.self_cids.mask, 0); + if (cid < scx_bpf_nr_cids()) + scx_bpf_kick_cid(cid, SCX_KICK_IDLE); + return; + } + + /* + * Insert a blocked mutex donor at the head of an eligible local DSQ with + * a fresh slice and %SCX_ENQ_PREEMPT, requesting an immediate reschedule. + * The test above guarantees that cpus_allowed intersects self_cids, but + * the donor's current cid may have been delegated to a child. Search the + * intersection starting at the current cid, preserving it when qmap still + * holds it and wrapping to another eligible self cid otherwise. + * + * A self cid may be held exclusively with SCX_CAP_ENQ or time-shared with + * only SCX_CAP_ENQ_IMMED. Add needs_immed() so either kind can accept the + * local insertion instead of rejecting and reenqueuing the donor for a + * capability miss. Once selected, the core proxy-exec path can run the + * mutex owner using the donor's scheduling context. + * + * This policy is intentionally unfair and can strongly prioritize tasks + * using contended mutexes; scx_qmap is a demonstration scheduler and + * this behavior makes proxy-exec support easy to observe. + */ + if (enq_flags & SCX_ENQ_BLOCKED) { + cid = cmask_next_and_set_wrap(&taskc->cpus_allowed, + &qa.self_cids.mask, + scx_bpf_task_cid(p)); + if (cid >= scx_bpf_nr_cids()) { + /* self_cids may have changed since the intersection test */ + dispatch_to_rescue(p, taskc, enq_flags); return; } + scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | cid, slice_ns, + enq_flags | needs_immed(cid) | SCX_ENQ_PREEMPT); + return; } /* @@ -587,9 +652,34 @@ static void update_core_sched_head_seq(struct task_struct *p) } /* + * Send @p, stranded on the DSQ @it iterates, to the kernel rescue path. To + * exercise both kfuncs, move tasks with an even pid there with + * scx_bpf_dsq_move() and re-enqueue the others with scx_bpf_dsq_reenq_task(), + * which takes them through qmap_enqueue()'s rescue. The choice is per task so + * that a marked task isn't moved by another scan before the re-enqueue reaches + * it. + */ +static __always_inline void rescue_stranded(struct bpf_iter_scx_dsq *it, + struct task_struct *p, task_ctx_t *taskc) +{ + s32 c; + + if (p->pid & 1) { + scx_bpf_dsq_reenq_task(it, p); + return; + } + + c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); + if (c >= 0 && c < scx_bpf_nr_cids()) { + __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); + scx_bpf_dsq_move(it, p, SCX_DSQ_LOCAL_ON | c, SCX_ENQ_RESCUE); + } +} + +/* * One pass over SHARED_DSQ: rescue stranded tasks and boost highpri ones. A * task whose cids were lost while it was queued in the fifos would strand on - * SHARED_DSQ, which is consumed only on self cids it can't run on - move it to + * SHARED_DSQ, which is consumed only on self cids it can't run on - send it to * the kernel rescue path. One whose cids were lost after the highpri cull is * likewise rescued out of HIGHPRI_DSQ below. * @@ -598,9 +688,9 @@ static void update_core_sched_head_seq(struct task_struct *p) * consuming them first. This makes minor difference only when dsp_batch is * larger than 1. * - * scx_bpf_dsq_move[_vtime]() are allowed both from ops.dispatch() and - * non-rq-lock holding BPF programs. As demonstration, this function is called - * from qmap_dispatch() and monitor_timerfn(). + * scx_bpf_dsq_move[_vtime]() and scx_bpf_dsq_reenq_task() are allowed both from + * ops.dispatch() and non-rq-lock holding BPF programs. As demonstration, this + * function is called from qmap_dispatch() and monitor_timerfn(). */ static bool scan_shared_dsq(bool from_timer) { @@ -612,19 +702,13 @@ static bool scan_shared_dsq(bool from_timer) bpf_for_each(scx_dsq, p, SHARED_DSQ, 0) { static u64 highpri_seq; task_ctx_t *taskc; - s32 c; if (!(taskc = lookup_task_ctx(p))) return false; /* stranded? rescue - it can't be dispatched here either way */ if (!cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) { - c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); - if (c >= 0 && c < scx_bpf_nr_cids()) { - __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); - scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c, - SCX_ENQ_RESCUE); - } + rescue_stranded(BPF_FOR_EACH_ITER, p, taskc); continue; } @@ -658,19 +742,12 @@ static bool scan_shared_dsq(bool from_timer) &qa.usable_cids.mask, this_cid + 1); if (cid >= nr_cids) { - s32 c; - /* self cids lack caps in effect yet, leave it queued */ if (cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) continue; /* stranded after the cull - rescue it from here */ - c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); - if (c >= 0 && c < nr_cids) { - __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); - scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c, - SCX_ENQ_RESCUE); - } + rescue_stranded(BPF_FOR_EACH_ITER, p, taskc); continue; } @@ -1408,7 +1485,8 @@ __noinline void compute_partition(void) /* find out the cids we hold */ scx_bpf_sub_caps(0, SCX_CAP_ENQ, &qa.held_excl.mask); scx_bpf_sub_caps(0, SCX_CAP_ENQ_IMMED, &qa.held_shared.mask); - cmask_andnot(&qa.held_shared.mask, &qa.held_excl.mask); /* held only as ENQ_IMMED */ + /* held only as ENQ_IMMED */ + cmask_andnot(&qa.held_shared.mask, &qa.held_shared.mask, &qa.held_excl.mask); qa.part.nr_shared = 0; qa.part.nr_rr = 0; @@ -1563,8 +1641,7 @@ static __noinline void account_alloc(void) */ static void refresh_usable(void) { - cmask_copy(&qa.usable_scratch.mask, &qa.self_cids.mask); - cmask_and(&qa.usable_scratch.mask, &qa.avail_cids.mask); + cmask_and(&qa.usable_scratch.mask, &qa.self_cids.mask, &qa.avail_cids.mask); cmask_copy(&qa.usable_cids.mask, &qa.usable_scratch.mask); } @@ -1643,10 +1720,8 @@ __noinline void apply_partition(void) if (!cgid) continue; - cmask_copy(&qa.to_revoke_cids.mask, &ssc->prev_granted.mask); - cmask_andnot(&qa.to_revoke_cids.mask, &ssc->granted_cids.mask); - cmask_copy(&qa.to_grant_cids.mask, &ssc->granted_cids.mask); - cmask_andnot(&qa.to_grant_cids.mask, &ssc->prev_granted.mask); + cmask_andnot(&qa.to_revoke_cids.mask, &ssc->prev_granted.mask, &ssc->granted_cids.mask); + cmask_andnot(&qa.to_grant_cids.mask, &ssc->granted_cids.mask, &ssc->prev_granted.mask); scx_bpf_sub_revoke(cgid, SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, &qa.prev_rr_cids.mask); @@ -1719,13 +1794,90 @@ static void redistribute(void) } /* - * Userspace pokes this (PROG_RUN) to bring alloc_ns[] current before reading - * it for the stats display. Skipping when the partition guard is held is - * fine - alloc_ts is untouched, so the elapsed time is charged next time. + * Owner id for a @sched value of ops.sub_cid_sched_updated(). A child is + * attached before its first task runs and its tasks are re-homed before it + * detaches, so a child's cgroup id always has its slot. + */ +static s32 cid_sched_owner(u64 sched) +{ + s32 i; + + if (sched == SCX_CID_SCHED_NONE) + return CID_NONE; + if (sched == SCX_CID_SCHED_SELF) + return CID_SELF; + bpf_for(i, 0, MAX_SUB_SCHEDS) + if (qa.sub_sched_ctxs[i].cgroup_id == sched) + return i; + return CID_NONE; +} + +/* used_ns[] is summed from every cpu, hence the atomic adds */ +static void cid_sched_charge(s32 cid, u64 now) +{ + s32 owner = qa.cid_sched[cid]; + u64 delta = now - qa.cid_sched_since[cid]; + + if (owner >= 0 && owner < MAX_SUB_SCHEDS) + __sync_fetch_and_add(&qa.used_ns[owner], delta); + else if (owner == CID_SELF) + __sync_fetch_and_add(&qa.self_used_ns, delta); + qa.cid_sched_since[cid] = now; +} + +void BPF_STRUCT_OPS(qmap_sub_cid_sched_updated, s32 cid, u64 sched) +{ + if (cid < 0 || cid >= SCX_QMAP_MAX_CPUS) + return; + + cid_sched_charge(cid, bpf_ktime_get_ns()); + qa.cid_sched[cid] = cid_sched_owner(sched); +} + +/* + * Snapshot the used time for the stats display: the closed intervals plus the + * ones still open. The reads race the notifications on other cpus, so an + * interval closing in between can be missing from one snapshot or counted in + * two. The next snapshot evens it out and the display floors a negative + * difference at zero. + */ +static void snapshot_used(void) +{ + u64 now = bpf_ktime_get_ns(); + s32 nr_cids = qa.nr_cids; + s32 cid, i; + + if (nr_cids < 0 || nr_cids > SCX_QMAP_MAX_CPUS) + return; + + bpf_for(i, 0, MAX_SUB_SCHEDS) + qa.used_snap_ns[i] = qa.used_ns[i]; + qa.self_used_snap_ns = qa.self_used_ns; + + bpf_for(cid, 0, nr_cids) { + s32 owner = qa.cid_sched[cid]; + u64 since = qa.cid_sched_since[cid]; + + /* restarted after @now by a notification on another cpu */ + if (since > now) + continue; + if (owner >= 0 && owner < MAX_SUB_SCHEDS) + qa.used_snap_ns[owner] += now - since; + else if (owner == CID_SELF) + qa.self_used_snap_ns += now - since; + } +} + +/* + * Userspace pokes this (PROG_RUN) to bring alloc_ns[] and the used snapshot + * current before reading them for the stats display. Skipping the alloc part + * when the partition guard is held is fine - alloc_ts is untouched, so the + * elapsed time is charged next time. */ SEC("syscall") int flush_alloc(void *ctx) { + snapshot_used(); if (part_try_start()) { account_alloc(); part_end(); @@ -1877,6 +2029,9 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init) /* cache the cid count, trusted to be <= SCX_QMAP_MAX_CPUS hereafter */ qa.nr_cids = nr_cids; + bpf_for(i, 0, nr_cids) + qa.cid_sched[i] = CID_NONE; + /* cmasks are embedded in qa, so they only need initializing */ cmask_init(&qa.idle_cids.mask, 0, nr_cids); cmask_init(&qa.rr_cids.mask, 0, nr_cids); @@ -1892,7 +2047,7 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init) scx_bpf_sub_caps(0, SCX_CAP_ENQ, &qa.held_excl.mask); scx_bpf_sub_caps(0, SCX_CAP_ENQ_IMMED, &qa.held_shared.mask); - cmask_andnot(&qa.held_shared.mask, &qa.held_excl.mask); + cmask_andnot(&qa.held_shared.mask, &qa.held_shared.mask, &qa.held_excl.mask); bpf_for(i, 0, MAX_SUB_SCHEDS) { cmask_init(&qa.sub_sched_ctxs[i].granted_cids.mask, 0, nr_cids); @@ -2024,12 +2179,36 @@ s32 BPF_STRUCT_OPS(qmap_sub_attach, struct scx_sub_attach_args *args) void BPF_STRUCT_OPS(qmap_sub_detach, struct scx_sub_detach_args *args) { - s32 i; + u64 cgid = args->ops->sub_cgroup_id; + s32 nr_cids = qa.nr_cids; + s32 pos = qa.part.rr_pos; + u64 holder_cgid; + bool reset_pool; + s32 i, cid; + + if (nr_cids < 0 || nr_cids > SCX_QMAP_MAX_CPUS || pos < 0 || pos >= MAX_PARTS) { + scx_bpf_error("-ERANGE"); + return; + } + + /* + * The child's caps are gone without a report, see ops.sub_detach(). + * cpuperf targets persist until the next write. Reset the child's excl + * cids, and the pool unless another child holds it: that child's revoke + * report or its own detach resets it then. + */ + holder_cgid = qa.part.rr_slots[pos]; + reset_pool = !holder_cgid || holder_cgid == cgid; for (i = 0; i < MAX_SUB_SCHEDS; i++) { - if (qa.sub_sched_ctxs[i].cgroup_id != args->ops->sub_cgroup_id) + if (qa.sub_sched_ctxs[i].cgroup_id != cgid) continue; + bpf_arena_for(cid, 0, nr_cids) + if (cmask_test(cid, &qa.sub_sched_ctxs[i].granted_cids.mask) || + (reset_pool && cmask_test(cid, &qa.rr_cids.mask))) + scx_bpf_cidperf_set(cid, SCX_CPUPERF_ONE); + qa.sub_sched_ctxs[i].cgroup_id = 0; qa.sub_sched_ctxs[i].weight = 100; cmask_init(&qa.sub_sched_ctxs[i].granted_cids.mask, 0, qa.nr_cids); @@ -2065,6 +2244,16 @@ void BPF_STRUCT_OPS(qmap_sub_ecaps_updated, s32 cid, u64 before, u64 after) execute_partition(); } +void BPF_STRUCT_OPS(qmap_sub_child_ecaps_updated, u64 cgroup_id, s32 cid, u64 before, + u64 after) +{ + __sync_fetch_and_add(&qa.nr_child_ecaps, 1); + + /* a child's last target must not outlive its PERF cap */ + if ((before & ~after) & SCX_CAP_PERF) + scx_bpf_cidperf_set(cid, SCX_CPUPERF_ONE); +} + SCX_OPS_CID_DEFINE(qmap_ops, .flags = SCX_OPS_ENQ_EXITING | SCX_OPS_TID_TO_TASK, .select_cid = (void *)qmap_select_cid, @@ -2088,6 +2277,8 @@ SCX_OPS_CID_DEFINE(qmap_ops, .sub_detach = (void *)qmap_sub_detach, .sub_caps_updated = (void *)qmap_sub_caps_updated, .sub_ecaps_updated = (void *)qmap_sub_ecaps_updated, + .sub_cid_sched_updated = (void *)qmap_sub_cid_sched_updated, + .sub_child_ecaps_updated = (void *)qmap_sub_child_ecaps_updated, .init_cids = (void *)qmap_init_cids, .init = (void *)qmap_init, .exit = (void *)qmap_exit, diff --git a/tools/sched_ext/scx_qmap.c b/tools/sched_ext/scx_qmap.c index 5bb5f687e579..17cf584a2a3d 100644 --- a/tools/sched_ext/scx_qmap.c +++ b/tools/sched_ext/scx_qmap.c @@ -46,7 +46,7 @@ const char help_fmt[] = "See the top-of-file comment in .bpf.c for the design.\n" "\n" "Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-b COUNT]\n" -" [-N COUNT] [-P] [-M] [-H] [-c CG_PATH] [-d PID] [-D LEN] [-S] [-p] [-I]\n" +" [-N COUNT] [-P] [-M] [-H] [-c CG_PATH] [-d PID] [-D LEN] [-S] [-p] [-I] [-X]\n" " [-F COUNT] [-i SEC] [-R MS] [-J MODE] [-v]\n" "\n" " -s SLICE_US Override slice duration\n" @@ -65,6 +65,7 @@ const char help_fmt[] = " -S Suppress qmap-specific debug dump\n" " -p Switch only tasks on SCHED_EXT policy instead of all\n" " -I Turn on SCX_OPS_ALWAYS_ENQ_IMMED\n" +" -X Turn on SCX_OPS_ENQ_BLOCKED\n" " -F COUNT IMMED stress: force every COUNT'th enqueue to a busy local DSQ (use with -I)\n" " -C MODE cid-override test (shuffle|bad-dup|bad-range|bad-mono)\n" " -i SEC Stats interval, seconds (default 5)\n" @@ -104,11 +105,15 @@ struct hier_prev { u64 alloc_ns[MAX_SUB_SCHEDS]; u64 self_alloc_ns; u64 alloc_window_ns; + u64 used_snap_ns[MAX_SUB_SCHEDS]; + u64 self_used_snap_ns; u64 nr_dsps[MAX_SUB_SCHEDS]; u64 nr_reenq_cap; u64 nr_reenq_immed; + u64 nr_enq_blocked; u64 nr_inject_attempts; u64 nr_rescue_dsp; + u64 nr_child_ecaps; }; /* current wall-clock time as "HH:MM:SS" for the startup and interval headers */ @@ -157,10 +162,24 @@ static void format_cid_ranges(struct qmap_arena *qa, s32 owner, char *buf, size_ strcpy(buf, "-"); } +/* + * Delta of a cumulative ns counter over the interval, as a fraction of the + * interval. The used snapshot can briefly run behind the previous one (see + * snapshot_used()), hence the floor. + */ +static double delta_ratio(u64 cur, u64 prev, double secs) +{ + s64 delta = cur - prev; + + return secs > 0 && delta > 0 ? delta / (secs * 1e9) : 0.0; +} + /* partition summary + one row per sched: weight, cpus, dispatch rate, cids */ static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cgid) { - char ranges[128], who[16]; + /* worst case, no ranges: a comma and up to six digits per cid */ + static char ranges[SCX_QMAP_MAX_CPUS * 7]; + char who[16]; const char *rr = "-"; double secs; u32 i; @@ -190,26 +209,39 @@ static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cg } format_cid_ranges(qa, CID_SHARED, ranges, sizeof(ranges)); - printf("hier : nsub=%llu excl=%u shared=%s rr=%s reenq cap/immed +%llu/+%llu inj=+%llu rescue=+%llu\n", + printf("hier : nsub=%llu excl=%u shared=%s rr=%s reenq cap/immed +%llu/+%llu blocked=+%llu inj=+%llu rescue=+%llu child_ecaps=+%llu\n", (unsigned long long)qa->nr_sub_scheds, qa->part.nr_excl, ranges, rr, (unsigned long long)(qa->nr_reenq_cap - prev->nr_reenq_cap), (unsigned long long)(qa->nr_reenq_immed - prev->nr_reenq_immed), + (unsigned long long)(qa->nr_enq_blocked - prev->nr_enq_blocked), (unsigned long long)(qa->nr_inject_attempts - prev->nr_inject_attempts), - (unsigned long long)(qa->nr_rescue_dsp - prev->nr_rescue_dsp)); + (unsigned long long)(qa->nr_rescue_dsp - prev->nr_rescue_dsp), + (unsigned long long)(qa->nr_child_ecaps - prev->nr_child_ecaps)); prev->nr_reenq_cap = qa->nr_reenq_cap; prev->nr_reenq_immed = qa->nr_reenq_immed; + prev->nr_enq_blocked = qa->nr_enq_blocked; prev->nr_inject_attempts = qa->nr_inject_attempts; prev->nr_rescue_dsp = qa->nr_rescue_dsp; + prev->nr_child_ecaps = qa->nr_child_ecaps; - printf("hier : %-4s %10s %4s %6s %8s %s\n", - "", "cgroup", "w", "alloc", "disp/s", "cids"); + /* + * alloc is the cid-time the partition handed each participant, and used + * is the cid-time its tasks actually ran, per + * ops.sub_cid_sched_updated(). Both are in cpus over the window. + */ + printf("hier : %-4s %10s %4s %6s %6s %8s %s\n", + "", "cgroup", "w", "alloc", "used", "disp/s", "cids"); format_cid_ranges(qa, CID_SELF, ranges, sizeof(ranges)); - printf("hier : %-4s %10llu %4u %6.2f %8s %s\n", "self", + printf("hier : %-4s %10llu %4u %6.2f %6.2f %8s %s\n", "self", (unsigned long long)own_cgid, 100, - secs > 0 ? (qa->self_alloc_ns - prev->self_alloc_ns) / (secs * 1e9) : 0.0, + delta_ratio(qa->self_alloc_ns, prev->self_alloc_ns, secs), + delta_ratio(qa->self_used_snap_ns, prev->self_used_snap_ns, secs), "-", ranges); prev->self_alloc_ns = qa->self_alloc_ns; + /* used accrues without a window, so prev moves only with the window */ + if (secs > 0) + prev->self_used_snap_ns = qa->self_used_snap_ns; for (i = 0; i < MAX_SUB_SCHEDS; i++) { struct sub_sched_ctx *sc = &qa->sub_sched_ctxs[i]; @@ -219,12 +251,15 @@ static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cg snprintf(who, sizeof(who), "sub%u", i); format_cid_ranges(qa, i, ranges, sizeof(ranges)); - printf("hier : %-4s %10llu %4u %6.2f %8.1f %s\n", who, + printf("hier : %-4s %10llu %4u %6.2f %6.2f %8.1f %s\n", who, (unsigned long long)sc->cgroup_id, sc->weight, - secs > 0 ? (qa->alloc_ns[i] - prev->alloc_ns[i]) / (secs * 1e9) : 0.0, + delta_ratio(qa->alloc_ns[i], prev->alloc_ns[i], secs), + delta_ratio(qa->used_snap_ns[i], prev->used_snap_ns[i], secs), secs > 0 ? (sc->nr_dsps - prev->nr_dsps[i]) / secs : 0.0, ranges); prev->alloc_ns[i] = qa->alloc_ns[i]; + if (secs > 0) + prev->used_snap_ns[i] = qa->used_snap_ns[i]; prev->nr_dsps[i] = sc->nr_dsps; } } @@ -263,7 +298,7 @@ restart: skel->rodata->max_tasks = 16384; while ((opt = getopt(argc, argv, - "s:e:t:T:l:b:N:PMHc:d:D:SpIF:C:i:R:J:B:q:vh")) != -1) { + "s:e:t:T:l:b:N:PMHc:d:D:SpIXF:C:i:R:J:B:q:vh")) != -1) { switch (opt) { case 's': skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; @@ -324,6 +359,9 @@ restart: case 'I': skel->struct_ops.qmap_ops->flags |= SCX_OPS_ALWAYS_ENQ_IMMED; break; + case 'X': + skel->struct_ops.qmap_ops->flags |= SCX_OPS_ENQ_BLOCKED; + break; case 'F': skel->rodata->immed_stress_nth = strtoul(optarg, NULL, 0); break; diff --git a/tools/sched_ext/scx_qmap.h b/tools/sched_ext/scx_qmap.h index e95fffcf7b23..9e6eefb39735 100644 --- a/tools/sched_ext/scx_qmap.h +++ b/tools/sched_ext/scx_qmap.h @@ -163,6 +163,17 @@ struct qmap_arena { u64 alloc_ts; /* last accounting timestamp */ u64 alloc_window_ns; /* total accounted time, the alloc denominator */ + /* + * The per-cid fields are written only by that cid's notifications and + * read by flush_alloc() for the snapshot userspace displays. + */ + s32 cid_sched[SCX_QMAP_MAX_CPUS]; /* per cid: owner id of the sched running there */ + u64 cid_sched_since[SCX_QMAP_MAX_CPUS]; /* when that last changed */ + u64 used_ns[MAX_SUB_SCHEDS]; /* per child slot, closed intervals */ + u64 self_used_ns; + u64 used_snap_ns[MAX_SUB_SCHEDS]; /* used_ns[] plus the intervals open at the flush */ + u64 self_used_snap_ns; + /* bpf-internal cmasks (embedded, see struct qmap_cmask) */ struct qmap_cmask self_cids; /* cids this node runs its own tasks on */ struct qmap_cmask avail_cids; /* cids with caps in effect on the cpu */ @@ -181,8 +192,10 @@ struct qmap_arena { /* bpf -> userspace: stats */ u64 nr_reenq_cap; /* SCX_TASK_REENQ_CAP bounces */ u64 nr_reenq_immed; /* SCX_TASK_REENQ_IMMED bounces */ + u64 nr_enq_blocked; /* SCX_ENQ_BLOCKED dispatches */ u64 nr_inject_attempts; /* fault-injection: dispatches to an unheld cid */ u64 nr_rescue_dsp; /* SCX_ENQ_RESCUE dispatch attempts */ + u64 nr_child_ecaps; /* ops.sub_child_ecaps_updated() deliveries */ u32 inject_mode; /* fault-injection mode (QMAP_INJ_*) */ }; diff --git a/tools/testing/selftests/sched_ext/.gitignore b/tools/testing/selftests/sched_ext/.gitignore index ae5491a114c0..54a1fd2af713 100644 --- a/tools/testing/selftests/sched_ext/.gitignore +++ b/tools/testing/selftests/sched_ext/.gitignore @@ -4,3 +4,7 @@ !Makefile !.gitignore !config +!test_modules/ +!test_modules/scx_enq_blocked_test.c +!test_modules/Makefile +test_modules/*.mod.c diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile index af919a56a4f4..c5d3a2eaea7d 100644 --- a/tools/testing/selftests/sched_ext/Makefile +++ b/tools/testing/selftests/sched_ext/Makefile @@ -5,10 +5,12 @@ include ../../../scripts/Makefile.arch include ../../../scripts/Makefile.include TEST_GEN_PROGS := runner +TEST_GEN_MODS_DIR := test_modules # override lib.mk's default rules OVERRIDE_TARGETS := 1 include ../lib.mk +include ../cgroup/lib/libcgroup.mk CURDIR := $(abspath .) REPOROOT := $(abspath ../../../..) @@ -53,7 +55,7 @@ ifneq ($(wildcard $(GENHDR)),) GENFLAGS := -DHAVE_GENHDR endif -CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ +CFLAGS += -g -O2 -pthread -Wall -Werror $(GENFLAGS) \ -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -I$(TOOLSINCDIR) -I$(APIDIR) -I$(CURDIR)/include -I$(SCXTOOLSINCDIR) @@ -62,7 +64,7 @@ ifneq ($(LLVM),) CFLAGS += -Wno-unused-command-line-argument endif -LDFLAGS = -lelf -lz -lpthread -lzstd +LDFLAGS += -lelf -lz -lpthread -lzstd IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - </dev/null | \ grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__') @@ -153,7 +155,7 @@ $(INCLUDE_DIR)/%.bpf.skel.h: $(SCXOBJ_DIR)/%.bpf.o $(INCLUDE_DIR)/vmlinux.h $(BP override define CLEAN rm -rf $(OUTPUT_DIR) - rm -f $(TEST_GEN_PROGS) + rm -f $(TEST_GEN_PROGS) $(EXTRA_CLEAN) endef # Every testcase takes all of the BPF progs are dependencies by default. This @@ -162,10 +164,12 @@ endef all_test_bpfprogs := $(foreach prog,$(wildcard *.bpf.c),$(INCLUDE_DIR)/$(patsubst %.c,%.skel.h,$(prog))) auto-test-targets := \ + cgroup_nr_cpus \ create_dsq \ dequeue \ dequeue_iter \ dequeue_remote \ + enq_blocked \ enq_last_no_enq_fails \ ddsp_bogus_dsq_fail \ ddsp_vtimelocal_fail \ @@ -175,6 +179,7 @@ auto-test-targets := \ exit \ hotplug \ init_enable_count \ + kick \ maximal \ maybe_null \ minimal \ @@ -214,7 +219,7 @@ $(testcase-targets): $(SCXOBJ_DIR)/%.o: %.c $(SCXOBJ_DIR)/runner.o $(all_test_bp $(SCXOBJ_DIR)/util.o: util.c | $(SCXOBJ_DIR) $(CC) $(CFLAGS) -c $< -o $@ -$(OUTPUT)/runner: $(SCXOBJ_DIR)/runner.o $(SCXOBJ_DIR)/util.o $(BPFOBJ) $(testcase-targets) +$(OUTPUT)/runner: $(SCXOBJ_DIR)/runner.o $(SCXOBJ_DIR)/util.o $(BPFOBJ) $(LIBCGROUP_O) $(testcase-targets) @echo "$(testcase-targets)" $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) diff --git a/tools/testing/selftests/sched_ext/allowed_cpus.bpf.c b/tools/testing/selftests/sched_ext/allowed_cpus.bpf.c index 9dd72d0da29b..f14d7e5bef9c 100644 --- a/tools/testing/selftests/sched_ext/allowed_cpus.bpf.c +++ b/tools/testing/selftests/sched_ext/allowed_cpus.bpf.c @@ -147,23 +147,41 @@ void BPF_STRUCT_OPS(allowed_cpus_exit, struct scx_exit_info *ei) } struct task_cpu_arg { - pid_t pid; + u64 pid; + s64 custom_cpu; }; SEC("syscall") int select_cpu_from_user(struct task_cpu_arg *input) { struct task_struct *p; - int cpu; + struct bpf_cpumask *mask; + s32 cpu; p = bpf_task_from_pid(input->pid); if (!p) return -EINVAL; + mask = bpf_cpumask_create(); + if (!mask) { + bpf_task_release(p); + return -ENOMEM; + } + + /* A negative custom_cpu leaves the custom mask empty. */ + if (input->custom_cpu >= 0) + bpf_cpumask_set_cpu(input->custom_cpu, mask); + bpf_rcu_read_lock(); - cpu = scx_bpf_select_cpu_and(p, bpf_get_smp_processor_id(), 0, p->cpus_ptr, 0); + cpu = scx_bpf_select_cpu_and(p, bpf_get_smp_processor_id(), 0, + cast_mask(mask), 0); + if (cpu >= 0 && + (!bpf_cpumask_test_cpu(cpu, cast_mask(mask)) || + !bpf_cpumask_test_cpu(cpu, &p->cpus_mask))) + cpu = -ERANGE; bpf_rcu_read_unlock(); + bpf_cpumask_release(mask); bpf_task_release(p); return cpu; diff --git a/tools/testing/selftests/sched_ext/allowed_cpus.c b/tools/testing/selftests/sched_ext/allowed_cpus.c index 093f285ab4ba..76311728a6b1 100644 --- a/tools/testing/selftests/sched_ext/allowed_cpus.c +++ b/tools/testing/selftests/sched_ext/allowed_cpus.c @@ -2,7 +2,10 @@ /* * Copyright (c) 2025 Andrea Righi <arighi@nvidia.com> */ +#define _GNU_SOURCE #include <bpf/bpf.h> +#include <limits.h> +#include <sched.h> #include <scx/common.h> #include <sys/wait.h> #include <unistd.h> @@ -23,17 +26,19 @@ static enum scx_test_status setup(void **ctx) return SCX_TEST_PASS; } -static int test_select_cpu_from_user(const struct allowed_cpus *skel) +static int test_select_cpu_from_user(const struct allowed_cpus *skel, + const char *name, int custom_cpu, + bool expect_busy) { int fd, ret; - __u64 args[1]; + __s32 cpu; + __u64 args[] = { getpid(), (__u64)(__s64)custom_cpu }; LIBBPF_OPTS(bpf_test_run_opts, attr, .ctx_in = args, .ctx_size_in = sizeof(args), ); - args[0] = getpid(); fd = bpf_program__fd(skel->progs.select_cpu_from_user); if (fd < 0) return fd; @@ -42,29 +47,123 @@ static int test_select_cpu_from_user(const struct allowed_cpus *skel) if (ret < 0) return ret; - fprintf(stderr, "%s: CPU %d\n", __func__, attr.retval); + /* test_run returns the signed BPF result through an unsigned field. */ + cpu = (__s32)attr.retval; + if ((expect_busy && cpu != -EBUSY) || + (!expect_busy && cpu != -EBUSY && cpu != custom_cpu)) { + SCX_ERR("%s: unexpected CPU selection result %d", name, cpu); + return -EINVAL; + } return 0; } +/* Grow until the mask covers the kernel's CPU range, including offline CPUs. */ +static int alloc_affinity(cpu_set_t **mask, size_t *size) +{ + int nr_cpus = CPU_SETSIZE; + cpu_set_t *cpus; + int err; + + for (;;) { + *size = CPU_ALLOC_SIZE(nr_cpus); + cpus = CPU_ALLOC(nr_cpus); + if (!cpus) + return -ENOMEM; + CPU_ZERO_S(*size, cpus); + if (!sched_getaffinity(0, *size, cpus)) { + *mask = cpus; + return nr_cpus; + } + err = errno; + CPU_FREE(cpus); + if (err != EINVAL) + return -err; + if (nr_cpus > INT_MAX / 2) + return -EOVERFLOW; + nr_cpus *= 2; + } +} + static enum scx_test_status run(void *ctx) { struct allowed_cpus *skel = ctx; - struct bpf_link *link; + enum scx_test_status status = SCX_TEST_FAIL; + cpu_set_t *original = NULL, *pinned = NULL; + bool affinity_changed = false; + size_t size; + int first = -1, second = -1, cpu, nr_cpus; + struct bpf_link *link = NULL; + + nr_cpus = alloc_affinity(&original, &size); + if (nr_cpus < 0) { + SCX_ERR("Failed to get affinity (%d)", -nr_cpus); + goto out; + } + pinned = CPU_ALLOC(nr_cpus); + if (!pinned) { + SCX_ERR("Failed to allocate affinity mask"); + goto out; + } + for (cpu = 0; cpu < nr_cpus; cpu++) { + if (!CPU_ISSET_S(cpu, size, original)) + continue; + if (first < 0) { + first = cpu; + } else { + second = cpu; + break; + } + } + if (first < 0) { + SCX_ERR("No CPU in affinity mask"); + goto out; + } link = bpf_map__attach_struct_ops(skel->maps.allowed_cpus_ops); - SCX_FAIL_IF(!link, "Failed to attach scheduler"); - - /* Pick an idle CPU from user-space */ - SCX_FAIL_IF(test_select_cpu_from_user(skel), "Failed to pick idle CPU"); - - /* Just sleeping is fine, plenty of scheduling events happening */ + if (!link) { + SCX_ERR("Failed to attach scheduler"); + goto out; + } + + if (test_select_cpu_from_user(skel, "empty mask", -1, true)) + goto out; + + /* A legal candidate may be busy; selection need not succeed. */ + if (test_select_cpu_from_user(skel, "legal candidate", first, false)) + goto out; + + if (second >= 0) { + CPU_ZERO_S(size, pinned); + CPU_SET_S(first, size, pinned); + if (sched_setaffinity(0, size, pinned)) { + SCX_ERR("Failed to pin task (%d)", errno); + goto out; + } + affinity_changed = true; + if (test_select_cpu_from_user(skel, "disjoint masks", second, true)) + goto out; + } else { + fprintf(stderr, "Skipping disjoint masks: need two allowed CPUs\n"); + } + + /* Just sleeping is fine, plenty of scheduling events happening. */ sleep(1); - - SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_NONE)); + if (UEI_EXITED(skel, uei)) { + SCX_ERR("Scheduler exited unexpectedly"); + goto out; + } + status = SCX_TEST_PASS; + +out: + if (affinity_changed && sched_setaffinity(0, size, original)) { + SCX_ERR("Failed to restore affinity (%d)", errno); + status = SCX_TEST_FAIL; + } bpf_link__destroy(link); - - return SCX_TEST_PASS; + CPU_FREE(pinned); + CPU_FREE(original); + return status; } static void cleanup(void *ctx) diff --git a/tools/testing/selftests/sched_ext/cgroup_nr_cpus.bpf.c b/tools/testing/selftests/sched_ext/cgroup_nr_cpus.bpf.c new file mode 100644 index 000000000000..841c83abb41b --- /dev/null +++ b/tools/testing/selftests/sched_ext/cgroup_nr_cpus.bpf.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Validate scx_bpf_cgroup_nr_cpus() from both BPF_PROG_TYPE_SYSCALL and + * struct_ops contexts. + * + * Copyright (c) 2026 NVIDIA Corporation. + */ + +#include <scx/common.bpf.h> + +char _license[] SEC("license") = "GPL"; + +UEI_DEFINE(uei); + +/* input to cgroup_nr_cpus_read() */ +u64 query_cgid; +/* output of cgroup_nr_cpus_read(), -1 if @query_cgid couldn't be resolved */ +s64 query_nr_cpus = -1; + +/* recorded by ops.cgroup_init() for @init_cgid */ +u64 init_cgid; +s64 init_nr_cpus = -1; + +SEC("syscall") +int cgroup_nr_cpus_read(void *ctx) +{ + struct cgroup *cgrp; + + query_nr_cpus = -1; + + cgrp = bpf_cgroup_from_id(query_cgid); + if (!cgrp) + return -ENOENT; + + query_nr_cpus = scx_bpf_cgroup_nr_cpus(cgrp); + bpf_cgroup_release(cgrp); + + return 0; +} + +s32 BPF_STRUCT_OPS(cgroup_nr_cpus_cgroup_init, struct cgroup *cgrp, + struct scx_cgroup_init_args *args) +{ + if (cgrp->kn->id == init_cgid) + init_nr_cpus = scx_bpf_cgroup_nr_cpus(cgrp); + + return 0; +} + +void BPF_STRUCT_OPS(cgroup_nr_cpus_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + +SEC(".struct_ops.link") +struct sched_ext_ops cgroup_nr_cpus_ops = { + .cgroup_init = (void *)cgroup_nr_cpus_cgroup_init, + .exit = (void *)cgroup_nr_cpus_exit, + .name = "cgroup_nr_cpus", +}; diff --git a/tools/testing/selftests/sched_ext/cgroup_nr_cpus.c b/tools/testing/selftests/sched_ext/cgroup_nr_cpus.c new file mode 100644 index 000000000000..1475a8b3f05f --- /dev/null +++ b/tools/testing/selftests/sched_ext/cgroup_nr_cpus.c @@ -0,0 +1,419 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Verify that scx_bpf_cgroup_nr_cpus() reports the number of CPUs in a + * cgroup's effective cpuset, including inherited and updated cpusets. + * + * Copyright (c) 2026 NVIDIA Corporation. + */ + +#define _GNU_SOURCE +#include <bpf/bpf.h> +#include <errno.h> +#include <fcntl.h> +#include <limits.h> +#include <linux/limits.h> +#include <scx/common.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#include "cgroup_nr_cpus.bpf.skel.h" +#include "cgroup_util.h" +#include "scx_test.h" + +/* + * Hierarchy under the cgroup2 root, all with the cpu controller enabled so + * that ops.cgroup_init() runs for each of them: + * + * parent cpuset enabled by the root, enables cpuset for its children + * parent/child owns a cpuset + * parent/child/leaf no cpuset of its own, inherits child's + */ +struct cgroup_nr_cpus_ctx { + struct cgroup_nr_cpus *skel; + struct bpf_link *link; + char root[PATH_MAX]; + char parent[PATH_MAX]; + char child[PATH_MAX]; + char leaf[PATH_MAX]; + bool parent_created; + bool child_created; + bool leaf_created; +}; + +static int join_path(char *dst, size_t dst_size, const char *parent, const char *name) +{ + int ret; + + ret = snprintf(dst, dst_size, "%s/%s", parent, name); + if (ret < 0 || (size_t)ret >= dst_size) + return -ENAMETOOLONG; + return 0; +} + +static u64 cgroup_id(const char *path) +{ + union { + u64 id; + unsigned char bytes[8]; + } id = {}; + struct file_handle *handle; + int mount_id, ret; + + handle = calloc(1, sizeof(*handle) + sizeof(id)); + if (!handle) + return 0; + handle->handle_bytes = sizeof(id); + ret = name_to_handle_at(AT_FDCWD, path, handle, &mount_id, 0); + if (!ret && handle->handle_bytes == sizeof(id)) + memcpy(id.bytes, handle->f_handle, sizeof(id)); + free(handle); + + return ret ? 0 : id.id; +} + +/* + * Parse a cpulist such as "0-3,8,10-11". Return the number of CPUs and the + * lowest and highest CPU in @first and @last, or -errno on failure. + */ +static int parse_cpulist(const char *cpulist, u32 *first, u32 *last) +{ + const char *p = cpulist; + u32 lowest = UINT_MAX, highest = 0; + int count = 0; + + while (*p && *p != '\n') { + unsigned long start, end_cpu; + char *end; + + errno = 0; + start = strtoul(p, &end, 10); + if (errno || end == p || start > INT_MAX) + return -EINVAL; + end_cpu = start; + p = end; + if (*p == '-') { + end_cpu = strtoul(p + 1, &end, 10); + if (errno || end == p + 1 || end_cpu > INT_MAX || end_cpu < start) + return -EINVAL; + p = end; + } + if (end_cpu - start + 1 > (unsigned long)(INT_MAX - count)) + return -EOVERFLOW; + if (start < lowest) + lowest = start; + if (end_cpu > highest) + highest = end_cpu; + count += end_cpu - start + 1; + if (*p == ',') + p++; + else if (*p && *p != '\n') + return -EINVAL; + } + + if (first) + *first = lowest; + if (last) + *last = highest; + return count; +} + +/* + * Number of CPUs in @cgroup's cpuset.cpus.effective, or -errno. + * + * A sparse cpulist can exceed a page on large systems. cg_read() does a single + * bounded read, so size the buffer for the worst case of NR_CPUS=8192 and + * reject a read that fills it rather than parsing a truncated list. + */ +static int effective_nr_cpus(const char *cgroup, u32 *first, u32 *last) +{ + static char buf[65536]; + + if (cg_read(cgroup, "cpuset.cpus.effective", buf, sizeof(buf))) + return -EIO; + if (strlen(buf) >= sizeof(buf) - 1) + return -EOVERFLOW; + return parse_cpulist(buf, first, last); +} + +/* Run the SYSCALL program to sample scx_bpf_cgroup_nr_cpus() for @path. */ +static int kfunc_nr_cpus(struct cgroup_nr_cpus_ctx *ctx, const char *path, s64 *nr_cpus) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + u64 cgid; + int err; + + cgid = cgroup_id(path); + if (!cgid) { + SCX_ERR("Failed to read cgroup ID of %s", path); + return -ENOENT; + } + + ctx->skel->bss->query_cgid = cgid; + err = bpf_prog_test_run_opts(bpf_program__fd(ctx->skel->progs.cgroup_nr_cpus_read), + &topts); + if (err || topts.retval) { + SCX_ERR("BPF_PROG_RUN failed for %s (err=%d retval=%d)", + path, err, (int)topts.retval); + return err ?: -EIO; + } + + *nr_cpus = ctx->skel->data->query_nr_cpus; + return 0; +} + +static bool check_nr_cpus(struct cgroup_nr_cpus_ctx *ctx, const char *path, int expected, + const char *what) +{ + s64 nr_cpus; + + if (kfunc_nr_cpus(ctx, path, &nr_cpus)) + return false; + if (nr_cpus != expected) { + SCX_ERR("%s: expected %d CPUs, got %lld", what, expected, + (long long)nr_cpus); + return false; + } + return true; +} + +/* + * Like check_nr_cpus() but tolerate a transient mismatch. A cpuset css being + * disabled stays attached to its cgroup until it's asynchronously offlined, and + * cpuset_num_cpus() keeps reporting its stale mask until then. + */ +static bool wait_nr_cpus(struct cgroup_nr_cpus_ctx *ctx, const char *path, int expected, + const char *what) +{ + s64 nr_cpus = -1; + int i; + + for (i = 0; i < 1000; i++) { + if (kfunc_nr_cpus(ctx, path, &nr_cpus)) + return false; + if (nr_cpus == expected) + return true; + usleep(1000); + } + SCX_ERR("%s: expected %d CPUs, got %lld", what, expected, (long long)nr_cpus); + return false; +} + +static bool controller_enabled(const char *cgroup, const char *file, const char *controller) +{ + char buf[4096], *saveptr, *token; + + if (cg_read(cgroup, file, buf, sizeof(buf))) + return false; + for (token = strtok_r(buf, "\n ", &saveptr); token; + token = strtok_r(NULL, "\n ", &saveptr)) + if (!strcmp(token, controller)) + return true; + return false; +} + +static void cleanup_ctx(struct cgroup_nr_cpus_ctx *ctx) +{ + bpf_link__destroy(ctx->link); + cgroup_nr_cpus__destroy(ctx->skel); + if (ctx->leaf_created) + cg_destroy(ctx->leaf); + if (ctx->child_created) + cg_destroy(ctx->child); + if (ctx->parent_created) + cg_destroy(ctx->parent); +} + +/* + * Enable @controller in the root's subtree_control if needed. Like the cgroup + * selftests, leave it enabled afterwards: the root is shared, and another + * manager may start relying on the controller while the test runs. + */ +static enum scx_test_status enable_controller(const char *root, const char *controller) +{ + char value[32]; + + if (controller_enabled(root, "cgroup.subtree_control", controller)) + return SCX_TEST_PASS; + if (!controller_enabled(root, "cgroup.controllers", controller)) + return SCX_TEST_SKIP; + + snprintf(value, sizeof(value), "+%s", controller); + if (cg_write(root, "cgroup.subtree_control", value)) + return SCX_TEST_SKIP; + return SCX_TEST_PASS; +} + +static enum scx_test_status setup_cgroups(struct cgroup_nr_cpus_ctx *ctx) +{ + enum scx_test_status status; + char name[64]; + + if (cg_find_unified_root(ctx->root, sizeof(ctx->root), NULL)) + return SCX_TEST_SKIP; + + status = enable_controller(ctx->root, "cpu"); + if (status != SCX_TEST_PASS) + return status; + status = enable_controller(ctx->root, "cpuset"); + if (status != SCX_TEST_PASS) + return status; + + snprintf(name, sizeof(name), "scx_nr_cpus_%d", getpid()); + if (join_path(ctx->parent, sizeof(ctx->parent), ctx->root, name) || + join_path(ctx->child, sizeof(ctx->child), ctx->parent, "child") || + join_path(ctx->leaf, sizeof(ctx->leaf), ctx->child, "leaf")) { + SCX_ERR("Cgroup path is too long"); + return SCX_TEST_FAIL; + } + + if (cg_create(ctx->parent)) { + SCX_ERR("Failed to create cgroup %s", ctx->parent); + return SCX_TEST_FAIL; + } + ctx->parent_created = true; + if (cg_write(ctx->parent, "cgroup.subtree_control", "+cpu +cpuset")) { + SCX_ERR("Failed to enable controllers in %s", ctx->parent); + return SCX_TEST_FAIL; + } + if (cg_create(ctx->child)) { + SCX_ERR("Failed to create cgroup %s", ctx->child); + return SCX_TEST_FAIL; + } + ctx->child_created = true; + if (cg_write(ctx->child, "cgroup.subtree_control", "+cpu")) { + SCX_ERR("Failed to enable cpu in %s", ctx->child); + return SCX_TEST_FAIL; + } + if (cg_create(ctx->leaf)) { + SCX_ERR("Failed to create cgroup %s", ctx->leaf); + return SCX_TEST_FAIL; + } + ctx->leaf_created = true; + + return SCX_TEST_PASS; +} + +static enum scx_test_status run(void *arg) +{ + struct cgroup_nr_cpus_ctx ctx = {}; + enum scx_test_status status; + char value[32]; + u32 first, last; + int nr_root, nr_child; + + (void)arg; + + /* + * SCX_ENUM_INIT() exits the process if vmlinux BTF can't be loaded, so + * run it before creating any cgroups that would then be left behind. + */ + ctx.skel = cgroup_nr_cpus__open(); + if (!ctx.skel) { + SCX_ERR("Failed to open skel"); + return SCX_TEST_FAIL; + } + SCX_ENUM_INIT(ctx.skel); + + status = setup_cgroups(&ctx); + if (status != SCX_TEST_PASS) + goto out; + status = SCX_TEST_FAIL; + + nr_root = effective_nr_cpus(ctx.root, NULL, NULL); + nr_child = effective_nr_cpus(ctx.child, &first, &last); + if (nr_root < 0 || nr_child < 0) { + SCX_ERR("Failed to read effective cpusets"); + goto out; + } + /* The effective cpuset can be empty, e.g. under a partition root. */ + if (nr_child < 2) { + status = SCX_TEST_SKIP; + goto out; + } + + ctx.skel->bss->init_cgid = cgroup_id(ctx.leaf); + if (!ctx.skel->bss->init_cgid) { + SCX_ERR("Failed to read cgroup ID of %s", ctx.leaf); + goto out; + } + if (cgroup_nr_cpus__load(ctx.skel)) { + SCX_ERR("Failed to load skel"); + goto out; + } + + /* The kfunc must be callable without a scheduler attached. */ + if (!check_nr_cpus(&ctx, ctx.root, nr_root, "root") || + !check_nr_cpus(&ctx, ctx.child, nr_child, "child") || + !check_nr_cpus(&ctx, ctx.leaf, nr_child, "inherited leaf")) + goto out; + + /* ops.cgroup_init() runs for existing cgroups when attaching. */ + ctx.link = bpf_map__attach_struct_ops(ctx.skel->maps.cgroup_nr_cpus_ops); + if (!ctx.link) { + SCX_ERR("Failed to attach scheduler"); + goto out; + } + if (ctx.skel->data->init_nr_cpus != nr_child) { + SCX_ERR("ops.cgroup_init(): expected %d CPUs, got %lld", nr_child, + (long long)ctx.skel->data->init_nr_cpus); + goto out; + } + + /* Non-contiguous cpuset, observed by the owner and by the inheritor. */ + if (nr_child > 2) { + snprintf(value, sizeof(value), "%u,%u", first, last); + if (cg_write(ctx.child, "cpuset.cpus", value)) { + SCX_ERR("Failed to set cpuset.cpus=%s for %s", value, ctx.child); + goto out; + } + if (!check_nr_cpus(&ctx, ctx.child, 2, "sparse child") || + !check_nr_cpus(&ctx, ctx.leaf, 2, "sparse inherited leaf")) + goto out; + } + + snprintf(value, sizeof(value), "%u", first); + if (cg_write(ctx.child, "cpuset.cpus", value)) { + SCX_ERR("Failed to set cpuset.cpus=%s for %s", value, ctx.child); + goto out; + } + if (!check_nr_cpus(&ctx, ctx.child, 1, "single-CPU child") || + !check_nr_cpus(&ctx, ctx.leaf, 1, "single-CPU inherited leaf")) + goto out; + + /* + * Disabling cpuset below @parent makes @child and @leaf inherit + * @parent's effective cpuset, which spans all of the root's CPUs. + */ + if (cg_write(ctx.parent, "cgroup.subtree_control", "-cpuset")) { + SCX_ERR("Failed to disable cpuset in %s", ctx.parent); + goto out; + } + nr_child = effective_nr_cpus(ctx.parent, NULL, NULL); + if (nr_child < 0) { + SCX_ERR("Failed to read effective cpuset of %s", ctx.parent); + goto out; + } + if (!wait_nr_cpus(&ctx, ctx.child, nr_child, "child after cpuset disable") || + !wait_nr_cpus(&ctx, ctx.leaf, nr_child, "leaf after cpuset disable")) + goto out; + + if (UEI_EXITED(ctx.skel, uei)) { + SCX_ERR("Scheduler exited unexpectedly"); + goto out; + } + + status = SCX_TEST_PASS; +out: + cleanup_ctx(&ctx); + return status; +} + +struct scx_test cgroup_nr_cpus = { + .name = "cgroup_nr_cpus", + .description = "Verify scx_bpf_cgroup_nr_cpus() reports effective cpuset CPU counts", + .run = run, +}; +REGISTER_SCX_TEST(&cgroup_nr_cpus) diff --git a/tools/testing/selftests/sched_ext/config b/tools/testing/selftests/sched_ext/config index aa901b05c8ad..8173f9170ffe 100644 --- a/tools/testing/selftests/sched_ext/config +++ b/tools/testing/selftests/sched_ext/config @@ -1,8 +1,11 @@ CONFIG_SCHED_CLASS_EXT=y CONFIG_CGROUPS=y CONFIG_CGROUP_SCHED=y +CONFIG_CPUSETS=y CONFIG_EXT_GROUP_SCHED=y CONFIG_BPF=y CONFIG_BPF_SYSCALL=y CONFIG_DEBUG_INFO=y CONFIG_DEBUG_INFO_BTF=y +CONFIG_EXPERT=y +CONFIG_SCHED_PROXY_EXEC=y diff --git a/tools/testing/selftests/sched_ext/create_dsq.bpf.c b/tools/testing/selftests/sched_ext/create_dsq.bpf.c index 2cfc4ffd60e2..24dacd1aab0c 100644 --- a/tools/testing/selftests/sched_ext/create_dsq.bpf.c +++ b/tools/testing/selftests/sched_ext/create_dsq.bpf.c @@ -10,6 +10,10 @@ char _license[] SEC("license") = "GPL"; +u32 nr_lifecycle_tests; + +UEI_DEFINE(uei); + void BPF_STRUCT_OPS(create_dsq_exit_task, struct task_struct *p, struct scx_exit_task_args *args) { @@ -43,16 +47,55 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(create_dsq_init) } bpf_for(i, 0, 1024) { + err = scx_bpf_create_dsq(i, -1); + if (err != -EEXIST) { + scx_bpf_error("Duplicate DSQ %d creation returned %d", i, err); + return -EINVAL; + } + + /* A rejected duplicate must leave the original DSQ accessible. */ + err = scx_bpf_dsq_nr_queued(i); + if (err) { + scx_bpf_error("Original DSQ %d queue count is %d", i, err); + return -EINVAL; + } + scx_bpf_destroy_dsq(i); + err = scx_bpf_dsq_nr_queued(i); + if (err != -ENOENT) { + scx_bpf_error("Destroyed DSQ %d queue count is %d", i, err); + return -EINVAL; + } + + err = scx_bpf_create_dsq(i, -1); + if (err) { + scx_bpf_error("Failed to recreate DSQ %d: %d", i, err); + return err; + } + + err = scx_bpf_dsq_nr_queued(i); + if (err) { + scx_bpf_error("Recreated DSQ %d queue count is %d", i, err); + return -EINVAL; + } + + scx_bpf_destroy_dsq(i); + nr_lifecycle_tests++; } return 0; } +void BPF_STRUCT_OPS(create_dsq_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + SEC(".struct_ops.link") struct sched_ext_ops create_dsq_ops = { .init_task = (void *) create_dsq_init_task, .exit_task = (void *) create_dsq_exit_task, .init = (void *) create_dsq_init, + .exit = (void *) create_dsq_exit, .name = "create_dsq", }; diff --git a/tools/testing/selftests/sched_ext/create_dsq.c b/tools/testing/selftests/sched_ext/create_dsq.c index d67431f57ac6..84143d5710f4 100644 --- a/tools/testing/selftests/sched_ext/create_dsq.c +++ b/tools/testing/selftests/sched_ext/create_dsq.c @@ -35,8 +35,12 @@ static enum scx_test_status run(void *ctx) return SCX_TEST_FAIL; } + SCX_ASSERT_ALIVE(skel, uei); + bpf_link__destroy(link); + SCX_EQ(skel->bss->nr_lifecycle_tests, 1024); + return SCX_TEST_PASS; } @@ -49,7 +53,7 @@ static void cleanup(void *ctx) struct scx_test create_dsq = { .name = "create_dsq", - .description = "Create and destroy a dsq in a loop", + .description = "Create, reject duplicates, destroy and recreate DSQs", .setup = setup, .run = run, .cleanup = cleanup, diff --git a/tools/testing/selftests/sched_ext/cyclic_kick_wait.c b/tools/testing/selftests/sched_ext/cyclic_kick_wait.c index c2e5aa9de715..178d1d75f332 100644 --- a/tools/testing/selftests/sched_ext/cyclic_kick_wait.c +++ b/tools/testing/selftests/sched_ext/cyclic_kick_wait.c @@ -152,7 +152,7 @@ static enum scx_test_status run(void *ctx) sleep(5); - if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) { + if (UEI_EXITED(skel, uei)) { SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)", (unsigned long long)skel->data->uei.kind, (long long)skel->data->uei.exit_code); diff --git a/tools/testing/selftests/sched_ext/dequeue_remote.c b/tools/testing/selftests/sched_ext/dequeue_remote.c index f2c1901030de..af344581d5f1 100644 --- a/tools/testing/selftests/sched_ext/dequeue_remote.c +++ b/tools/testing/selftests/sched_ext/dequeue_remote.c @@ -43,6 +43,7 @@ static void worker_fn(int id) for (j = 0; j < 20000 + id * 1000; j++) sum += j; + asm volatile("" : : "r"(sum)); if (id & 1) usleep(100); else diff --git a/tools/testing/selftests/sched_ext/enq_blocked.bpf.c b/tools/testing/selftests/sched_ext/enq_blocked.bpf.c new file mode 100644 index 000000000000..212690bf4e07 --- /dev/null +++ b/tools/testing/selftests/sched_ext/enq_blocked.bpf.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + * + * Verify that SCX_OPS_ENQ_BLOCKED passes blocked proxy donors through + * ops.enqueue() and record whether callbacks occur on the donor or owner CPU. + */ + +#include <scx/common.bpf.h> + +#define SHARED_DSQ 0 + +char _license[] SEC("license") = "GPL"; + +s32 donor_pid; +s32 donor_cpu = -1; +s32 owner_cpu = -1; +u64 nr_blocked_enqueues; +u64 nr_blocked_enqueues_donor_cpu; +u64 nr_blocked_enqueues_owner_cpu; +u64 nr_blocked_enqueues_other_cpu; +u64 nr_blocked_wakeups; +static u64 vtime_now; + +UEI_DEFINE(uei); + +s32 BPF_STRUCT_OPS(enq_blocked_select_cpu, + struct task_struct *p, s32 prev_cpu, u64 wake_flags) +{ + return prev_cpu; +} + +void BPF_STRUCT_OPS(enq_blocked_enqueue, struct task_struct *p, u64 enq_flags) +{ + u64 vtime = p->scx.dsq_vtime; + + if (enq_flags & SCX_ENQ_BLOCKED) { + int cpu = scx_bpf_task_cpu(p); + + if (enq_flags & SCX_ENQ_WAKEUP) + __sync_fetch_and_add(&nr_blocked_wakeups, 1); + + if (p->pid == donor_pid) { + __sync_fetch_and_add(&nr_blocked_enqueues, 1); + if (cpu == donor_cpu) + __sync_fetch_and_add(&nr_blocked_enqueues_donor_cpu, 1); + else if (cpu == owner_cpu) + __sync_fetch_and_add(&nr_blocked_enqueues_owner_cpu, 1); + else + __sync_fetch_and_add(&nr_blocked_enqueues_other_cpu, 1); + } + } + + /* Limit the amount of budget an idling task can accumulate. */ + if (time_before(vtime, vtime_now - SCX_SLICE_DFL)) + vtime = vtime_now - SCX_SLICE_DFL; + + scx_bpf_dsq_insert_vtime(p, SHARED_DSQ, SCX_SLICE_DFL, vtime, + enq_flags); + scx_bpf_kick_cpu(scx_bpf_task_cpu(p), SCX_KICK_IDLE); +} + +void BPF_STRUCT_OPS(enq_blocked_dispatch, s32 cpu, struct task_struct *prev) +{ + scx_bpf_dsq_move_to_local(SHARED_DSQ, 0); +} + +void BPF_STRUCT_OPS(enq_blocked_running, struct task_struct *p) +{ + if (time_before(vtime_now, p->scx.dsq_vtime)) + vtime_now = p->scx.dsq_vtime; +} + +void BPF_STRUCT_OPS(enq_blocked_stopping, struct task_struct *p, bool runnable) +{ + u64 delta = scale_by_task_weight_inverse(p, + SCX_SLICE_DFL - p->scx.slice); + + scx_bpf_task_set_dsq_vtime(p, p->scx.dsq_vtime + delta); +} + +void BPF_STRUCT_OPS(enq_blocked_enable, struct task_struct *p) +{ + scx_bpf_task_set_dsq_vtime(p, vtime_now); +} + +s32 BPF_STRUCT_OPS_SLEEPABLE(enq_blocked_init) +{ + int ret; + + ret = scx_bpf_create_dsq(SHARED_DSQ, -1); + if (ret) { + scx_bpf_error("failed to create DSQ %d (%d)", SHARED_DSQ, ret); + return ret; + } + + return 0; +} + +void BPF_STRUCT_OPS(enq_blocked_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + +SEC(".struct_ops.link") +struct sched_ext_ops enq_blocked_ops = { + .select_cpu = (void *)enq_blocked_select_cpu, + .enqueue = (void *)enq_blocked_enqueue, + .dispatch = (void *)enq_blocked_dispatch, + .running = (void *)enq_blocked_running, + .stopping = (void *)enq_blocked_stopping, + .enable = (void *)enq_blocked_enable, + .init = (void *)enq_blocked_init, + .exit = (void *)enq_blocked_exit, + .name = "enq_blocked", +}; diff --git a/tools/testing/selftests/sched_ext/enq_blocked.c b/tools/testing/selftests/sched_ext/enq_blocked.c new file mode 100644 index 000000000000..1d5e090b54bc --- /dev/null +++ b/tools/testing/selftests/sched_ext/enq_blocked.c @@ -0,0 +1,917 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + * + * Exercise a priority inversion with the owner and donor first pinned to the + * same CPU, then with each on a different CPU. A high-priority donor blocks on + * a mutex held by a low-priority owner while one medium-priority contender per + * available CPU keeps the system busy. A weighted-vruntime BPF scheduler runs + * both CPU placement configurations with SCX_OPS_ENQ_BLOCKED first disabled + * and then enabled. The test validates blocked-donor admission and reports the + * average mutex hold and wait times, plus their enabled-minus-disabled deltas, + * for each configuration. The timing data is informational. + * + * CONFIG_SCHED_PROXY_EXEC=y is required to exercise the proxy-execution paths. + */ +#define _GNU_SOURCE + +#include <bpf/bpf.h> +#include <errno.h> +#include <fcntl.h> +#include <limits.h> +#include <pthread.h> +#include <sched.h> +#include <scx/common.h> +#include <stdatomic.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/ioctl.h> +#include <sys/resource.h> +#include <sys/syscall.h> +#include <time.h> +#include <unistd.h> + +#include "enq_blocked.bpf.skel.h" +#include "enq_blocked.h" +#include "scx_test.h" + +#define MODULE_NAME "scx_enq_blocked_test" +#define MODULE_FILE "test_modules/" MODULE_NAME ".ko" +#define DEVICE_PATH "/dev/scx_enq_blocked" +#define WAIT_STEP_US 1000 +#define WAIT_TIMEOUT_MS 2000 +#define NR_WARMUP_TRIALS 1 +#define NR_MEASURED_TRIALS 10 +#define NR_TRIALS (NR_WARMUP_TRIALS + NR_MEASURED_TRIALS) +#define JOIN_TIMEOUT_MS ((NR_TRIALS + 1) * WAIT_TIMEOUT_MS) +#define OWNER_NICE 19 +#define DONOR_NICE -20 +#define CONTENDER_NICE 0 + +struct thread_ctx { + atomic_bool start_donor; + atomic_bool abort; + atomic_bool stop_contender; + atomic_bool measurement_ready; + atomic_int donor_pid; + atomic_int donor_completed; + int fd; + int donor_cpu; + int owner_cpu; +}; + +struct contender_ctx { + struct thread_ctx *thread_ctx; + atomic_int status; + int cpu; +}; + +struct run_result { + struct enq_blocked_stats stats; + u64 nr_blocked_enqueues; + u64 nr_blocked_enqueues_donor_cpu; + u64 nr_blocked_enqueues_owner_cpu; + u64 nr_blocked_enqueues_other_cpu; + u64 nr_blocked_wakeups; +}; + +static bool parse_bool(const char *value, bool *result) +{ + if (!strcasecmp(value, "1") || !strcasecmp(value, "y") || + !strcasecmp(value, "yes") || !strcasecmp(value, "on") || + !strcasecmp(value, "true")) { + *result = true; + return true; + } + + if (!strcasecmp(value, "0") || !strcasecmp(value, "n") || + !strcasecmp(value, "no") || !strcasecmp(value, "off") || + !strcasecmp(value, "false")) { + *result = false; + return true; + } + + return false; +} + +static bool cmdline_bool(const char *name, bool default_value) +{ + char cmdline[4096], *newline, *saveptr = NULL, *token; + size_t name_len = strlen(name); + bool value = default_value; + FILE *file; + + file = fopen("/proc/cmdline", "r"); + if (!file) + return default_value; + + if (!fgets(cmdline, sizeof(cmdline), file)) { + fclose(file); + return default_value; + } + fclose(file); + newline = strchr(cmdline, '\n'); + if (newline) + *newline = '\0'; + + for (token = strtok_r(cmdline, " ", &saveptr); token; + token = strtok_r(NULL, " ", &saveptr)) { + bool parsed; + + if (strncmp(token, name, name_len) || token[name_len] != '=') + continue; + if (parse_bool(token + name_len + 1, &parsed)) + value = parsed; + } + + return value; +} + +static int module_path(char *path, size_t size) +{ + ssize_t len; + char *slash; + + len = readlink("/proc/self/exe", path, size - 1); + if (len < 0) + return -errno; + path[len] = '\0'; + + slash = strrchr(path, '/'); + if (!slash) + return -EINVAL; + *slash = '\0'; + + if (snprintf(slash, size - (slash - path), "/%s", MODULE_FILE) >= + size - (slash - path)) + return -ENAMETOOLONG; + + return 0; +} + +static int load_test_module(bool *loaded_here) +{ + char path[PATH_MAX]; + int fd, err; + + err = module_path(path, sizeof(path)); + if (err) + return err; + + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -errno; + + if (syscall(SYS_finit_module, fd, "", 0)) { + err = errno; + close(fd); + if (err == EEXIST) + return 0; + return -err; + } + + close(fd); + *loaded_here = true; + return 0; +} + +static void unload_test_module(bool loaded_here) +{ + if (loaded_here && syscall(SYS_delete_module, MODULE_NAME, O_NONBLOCK)) + SCX_ERR("Failed to unload %s (%d)", MODULE_NAME, errno); +} + +static int pin_to_cpu(int cpu) +{ + cpu_set_t mask; + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); + return sched_setaffinity(0, sizeof(mask), &mask) ? errno : 0; +} + +static int select_test_cpus(bool cross_cpu, cpu_set_t *mask, int *donor_cpu, + int *owner_cpu) +{ + int cpu, first = -1; + + if (sched_getaffinity(0, sizeof(*mask), mask)) + return -errno; + + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) { + if (!CPU_ISSET(cpu, mask)) + continue; + if (first < 0) { + first = cpu; + if (!cross_cpu) + break; + } else { + *donor_cpu = first; + *owner_cpu = cpu; + return 0; + } + } + + if (first < 0) + return -ENODEV; + if (cross_cpu) + return -EAGAIN; + + *donor_cpu = first; + *owner_cpu = first; + return 0; +} + +static int set_nice(int nice) +{ + return setpriority(PRIO_PROCESS, 0, nice) ? errno : 0; +} + +static bool wait_for_pid(atomic_int *pid) +{ + int waited_ms; + + for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) { + if (atomic_load_explicit(pid, memory_order_acquire) > 0) + return true; + usleep(WAIT_STEP_US); + } + + return false; +} + +static int wait_for_contenders(struct contender_ctx *contenders, + size_t nr_contenders) +{ + size_t i, nr_ready; + int status, waited_ms; + + for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) { + nr_ready = 0; + for (i = 0; i < nr_contenders; i++) { + status = atomic_load_explicit(&contenders[i].status, + memory_order_acquire); + if (status < 0) + return status; + if (status > 0) + nr_ready++; + } + if (nr_ready == nr_contenders) + return 1; + usleep(WAIT_STEP_US); + } + + return -ETIMEDOUT; +} + +static int wait_for_donor_state(struct thread_ctx *ctx, int expected) +{ + int state, waited_ms; + + for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) { + state = ioctl(ctx->fd, ENQ_BLOCKED_IOCTL_DONOR_STATE); + if (state == expected) + return state; + if (state < 0 && errno != ENOENT) + return -errno; + usleep(WAIT_STEP_US); + } + + return -ETIMEDOUT; +} + +static bool wait_for_donor(struct thread_ctx *ctx, int trial) +{ + int waited_ms; + + for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) { + if (atomic_load_explicit(&ctx->donor_completed, + memory_order_acquire) >= trial) + return true; + if (atomic_load_explicit(&ctx->abort, memory_order_relaxed)) + return false; + usleep(WAIT_STEP_US); + } + + return false; +} + +static bool wait_for_measurement(struct thread_ctx *ctx) +{ + while (!atomic_load_explicit(&ctx->measurement_ready, + memory_order_acquire) && + !atomic_load_explicit(&ctx->abort, memory_order_relaxed)) + sched_yield(); + + return !atomic_load_explicit(&ctx->abort, memory_order_relaxed); +} + +static void *contender_fn(void *arg) +{ + struct contender_ctx *contender = arg; + struct thread_ctx *ctx = contender->thread_ctx; + int err; + + err = pin_to_cpu(contender->cpu); + if (!err) + err = set_nice(CONTENDER_NICE); + atomic_store_explicit(&contender->status, err ? -err : 1, + memory_order_release); + if (err) + return (void *)(uintptr_t)err; + + while (!atomic_load_explicit(&ctx->stop_contender, + memory_order_relaxed)) + ; + + return NULL; +} + +static void *owner_fn(void *arg) +{ + struct thread_ctx *ctx = arg; + int err, i; + + err = pin_to_cpu(ctx->owner_cpu); + if (err) + return (void *)(uintptr_t)err; + err = set_nice(OWNER_NICE); + if (err) + return (void *)(uintptr_t)err; + + for (i = 0; i < NR_TRIALS; i++) { + if (ioctl(ctx->fd, ENQ_BLOCKED_IOCTL_OWNER)) + return (void *)(uintptr_t)errno; + if (!wait_for_donor(ctx, i + 1)) + return (void *)(uintptr_t)ETIMEDOUT; + + if (i + 1 == NR_WARMUP_TRIALS && !wait_for_measurement(ctx)) + return NULL; + } + + return NULL; +} + +static int run_donor_trial(struct thread_ctx *ctx) +{ + int waited_ms; + + for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) { + if (!ioctl(ctx->fd, ENQ_BLOCKED_IOCTL_DONOR)) + return 0; + if (errno != EAGAIN) + return -errno; + usleep(WAIT_STEP_US); + } + + return -ETIMEDOUT; +} + +static void *donor_fn(void *arg) +{ + struct thread_ctx *ctx = arg; + int err, i; + + err = pin_to_cpu(ctx->donor_cpu); + if (err) + return (void *)(uintptr_t)err; + err = set_nice(DONOR_NICE); + if (err) + return (void *)(uintptr_t)err; + + atomic_store_explicit(&ctx->donor_pid, syscall(SYS_gettid), + memory_order_release); + while (!atomic_load_explicit(&ctx->start_donor, memory_order_acquire) && + !atomic_load_explicit(&ctx->abort, memory_order_relaxed)) + sched_yield(); + + if (atomic_load_explicit(&ctx->abort, memory_order_relaxed)) + return NULL; + + for (i = 0; i < NR_TRIALS; i++) { + err = run_donor_trial(ctx); + if (err) + return (void *)(uintptr_t)-err; + atomic_store_explicit(&ctx->donor_completed, i + 1, + memory_order_release); + } + + return NULL; +} + +static void print_avg_time(const char *name, u64 total_ns, u64 samples) +{ + u64 avg_ns = samples ? total_ns / samples : 0; + + printf(" %s_avg_ns=%llu (%llu.%03llu ms, samples=%llu)\n", name, + (unsigned long long)avg_ns, + (unsigned long long)(avg_ns / 1000000), + (unsigned long long)((avg_ns / 1000) % 1000), + (unsigned long long)samples); +} + +static void print_avg_delta(const char *name, u64 disabled_total, + u64 disabled_samples, u64 enabled_total, + u64 enabled_samples) +{ + u64 disabled_avg, enabled_avg; + s64 delta_ns; + double delta_pct; + + if (!disabled_samples || !enabled_samples) + return; + + disabled_avg = disabled_total / disabled_samples; + enabled_avg = enabled_total / enabled_samples; + delta_ns = (s64)enabled_avg - (s64)disabled_avg; + delta_pct = disabled_avg ? 100.0 * delta_ns / disabled_avg : 0.0; + + printf(" %s_delta_ns=%+lld (%+.2f%%)\n", name, + (long long)delta_ns, delta_pct); +} + +static int join_thread(pthread_t thread, const struct timespec *deadline, + int *thread_err) +{ + void *result; + int err; + + err = pthread_timedjoin_np(thread, &result, deadline); + if (err) + return err; + + *thread_err = (int)(uintptr_t)result; + return 0; +} + +static void set_join_deadline(struct timespec *deadline) +{ + clock_gettime(CLOCK_REALTIME, deadline); + deadline->tv_sec += JOIN_TIMEOUT_MS / 1000; + deadline->tv_nsec += (JOIN_TIMEOUT_MS % 1000) * 1000000; + if (deadline->tv_nsec >= 1000000000) { + deadline->tv_sec++; + deadline->tv_nsec -= 1000000000; + } +} + +static enum scx_test_status setup(void **ctx) +{ + struct enq_blocked *skel; + u64 flag; + + skel = enq_blocked__open(); + SCX_FAIL_IF(!skel, "Failed to open skel"); + SCX_ENUM_INIT(skel); + + flag = SCX_OPS_ENQ_BLOCKED; + if (!flag) { + enq_blocked__destroy(skel); + fprintf(stderr, "SKIP: SCX_OPS_ENQ_BLOCKED is unavailable\n"); + return SCX_TEST_SKIP; + } + + enq_blocked__destroy(skel); + *ctx = NULL; + return SCX_TEST_PASS; +} + +static enum scx_test_status run_one(bool enq_blocked, bool cross_cpu, + struct run_result *result) +{ + struct enq_blocked *skel; + struct thread_ctx thread_ctx = {}; + struct contender_ctx *contender_ctxs = NULL; + struct bpf_link *link = NULL; + pthread_t owner, donor, *contenders = NULL; + struct timespec join_deadline; + cpu_set_t allowed_cpus; + bool module_loaded = false; + bool owner_started = false, donor_started = false; + bool join_timed_out = false; + bool proxy_enabled; + enum scx_test_status status = SCX_TEST_PASS; + int cpu, donor_pid, donor_state, err, thread_err; + size_t i, nr_contenders, nr_contenders_started = 0; + size_t nr_contenders_joined = 0; + u64 nr_blocked, nr_blocked_donor_cpu, nr_blocked_owner_cpu; + u64 nr_blocked_other_cpu, nr_blocked_wakeups; + struct enq_blocked_stats stats; + + err = select_test_cpus(cross_cpu, &allowed_cpus, &thread_ctx.donor_cpu, + &thread_ctx.owner_cpu); + if (err == -EAGAIN) { + fprintf(stderr, "SKIP: cross-CPU case requires two allowed CPUs\n"); + return SCX_TEST_SKIP; + } + if (err) { + SCX_ERR("Failed to select test CPUs (%d)", -err); + return SCX_TEST_FAIL; + } + nr_contenders = CPU_COUNT(&allowed_cpus); + contenders = calloc(nr_contenders, sizeof(*contenders)); + contender_ctxs = calloc(nr_contenders, sizeof(*contender_ctxs)); + if (!contenders || !contender_ctxs) { + SCX_ERR("Failed to allocate %zu contender threads", nr_contenders); + status = SCX_TEST_FAIL; + goto out_contenders; + } + + skel = enq_blocked__open(); + if (!skel) { + SCX_ERR("Failed to open skel"); + status = SCX_TEST_FAIL; + goto out_contenders; + } + SCX_ENUM_INIT(skel); + skel->struct_ops.enq_blocked_ops->flags = + SCX_OPS_ENQ_LAST | + (enq_blocked ? SCX_OPS_ENQ_BLOCKED : 0); + if (enq_blocked__load(skel)) { + SCX_ERR("Failed to load skel"); + status = SCX_TEST_FAIL; + goto out_skel; + } + + err = load_test_module(&module_loaded); + if (err == -EPERM || err == -ENOENT) { + fprintf(stderr, "SKIP: cannot load mutex fixture (%d)\n", -err); + status = SCX_TEST_SKIP; + goto out_skel; + } + if (err) { + SCX_ERR("Failed to load mutex fixture (%d)", -err); + status = SCX_TEST_FAIL; + goto out_skel; + } + + thread_ctx.fd = open(DEVICE_PATH, O_RDONLY | O_CLOEXEC); + if (thread_ctx.fd < 0) { + SCX_ERR("Failed to open %s (%d)", DEVICE_PATH, errno); + status = SCX_TEST_FAIL; + goto out_module; + } + err = ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_PROXY_SUPPORTED); + if (err < 0) { + SCX_ERR("Failed to query proxy-exec support (%d)", errno); + status = SCX_TEST_FAIL; + goto out_fd; + } + proxy_enabled = err && cmdline_bool("sched_proxy_exec", true); + if (!proxy_enabled) { + fprintf(stderr, "SKIP: proxy execution is not enabled\n"); + status = SCX_TEST_SKIP; + goto out_fd; + } + if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_RESET_STATS)) { + SCX_ERR("Failed to reset mutex statistics (%d)", errno); + status = SCX_TEST_FAIL; + goto out_fd; + } + + if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_PREP_ATTACH)) { + SCX_ERR("Failed to prepare scheduler attachment (%d)", errno); + status = SCX_TEST_FAIL; + goto out_fd; + } + + err = pthread_create(&owner, NULL, owner_fn, &thread_ctx); + if (err) { + SCX_ERR("Failed to create owner thread (%d)", err); + status = SCX_TEST_FAIL; + goto out; + } + owner_started = true; + + err = pthread_create(&donor, NULL, donor_fn, &thread_ctx); + if (err) { + SCX_ERR("Failed to create donor thread (%d)", err); + status = SCX_TEST_FAIL; + goto out; + } + donor_started = true; + + if (!wait_for_pid(&thread_ctx.donor_pid)) { + SCX_ERR("Timed out waiting for donor thread"); + status = SCX_TEST_FAIL; + goto out; + } + + donor_pid = atomic_load_explicit(&thread_ctx.donor_pid, + memory_order_acquire); + skel->bss->donor_pid = donor_pid; + skel->data->donor_cpu = thread_ctx.donor_cpu; + skel->data->owner_cpu = thread_ctx.owner_cpu; + atomic_store_explicit(&thread_ctx.start_donor, true, + memory_order_release); + + donor_state = ENQ_BLOCKED_DONOR_SLEEPING; + if (proxy_enabled) + donor_state |= ENQ_BLOCKED_DONOR_ON_RQ; + err = wait_for_donor_state(&thread_ctx, donor_state); + if (err < 0) { + SCX_ERR("Donor did not block before scheduler attachment (%d)", -err); + status = SCX_TEST_FAIL; + goto out; + } + + link = bpf_map__attach_struct_ops(skel->maps.enq_blocked_ops); + if (!link) { + SCX_ERR("Failed to attach scheduler"); + status = SCX_TEST_FAIL; + goto out; + } + + /* Scheduler ownership changes start from a fully blocked donor. */ + donor_state = ENQ_BLOCKED_DONOR_SLEEPING; + err = wait_for_donor_state(&thread_ctx, donor_state); + if (err < 0) { + SCX_ERR("Unexpected donor state after scheduler attachment (%d)", + -err); + status = SCX_TEST_FAIL; + goto out; + } + + if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_ATTACH_DONE)) { + SCX_ERR("Failed to complete scheduler attachment (%d)", errno); + status = SCX_TEST_FAIL; + goto out; + } + + i = 0; + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) { + if (!CPU_ISSET(cpu, &allowed_cpus)) + continue; + + contender_ctxs[i].thread_ctx = &thread_ctx; + contender_ctxs[i].cpu = cpu; + atomic_init(&contender_ctxs[i].status, 0); + err = pthread_create(&contenders[i], NULL, contender_fn, + &contender_ctxs[i]); + if (err) { + SCX_ERR("Failed to create contender for CPU %d (%d)", + cpu, err); + status = SCX_TEST_FAIL; + goto out; + } + nr_contenders_started++; + i++; + } + + err = wait_for_contenders(contender_ctxs, nr_contenders); + if (err != 1) { + SCX_ERR("Contender threads failed (%d)", -err); + status = SCX_TEST_FAIL; + goto out; + } + + /* + * The first trial spans scheduler attachment and validates the state + * transition, but including it would skew scheduling latency. Exclude it + * from both the mutex and BPF enqueue measurements. + */ + if (!wait_for_donor(&thread_ctx, NR_WARMUP_TRIALS)) { + SCX_ERR("Timed out waiting for warm-up trial"); + status = SCX_TEST_FAIL; + goto out; + } + if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_RESET_STATS)) { + SCX_ERR("Failed to reset mutex statistics after warm-up (%d)", + errno); + status = SCX_TEST_FAIL; + goto out; + } + skel->bss->nr_blocked_enqueues = 0; + skel->bss->nr_blocked_enqueues_donor_cpu = 0; + skel->bss->nr_blocked_enqueues_owner_cpu = 0; + skel->bss->nr_blocked_enqueues_other_cpu = 0; + skel->bss->nr_blocked_wakeups = 0; + atomic_store_explicit(&thread_ctx.measurement_ready, true, + memory_order_release); + +out: + ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_ATTACH_DONE); + if (status != SCX_TEST_PASS) { + atomic_store_explicit(&thread_ctx.abort, true, memory_order_release); + atomic_store_explicit(&thread_ctx.start_donor, true, + memory_order_release); + atomic_store_explicit(&thread_ctx.measurement_ready, true, + memory_order_release); + } + + set_join_deadline(&join_deadline); + if (donor_started) { + err = join_thread(donor, &join_deadline, &thread_err); + if (err == ETIMEDOUT) { + SCX_ERR("Timed out waiting for donor thread"); + join_timed_out = true; + status = SCX_TEST_FAIL; + } else if (err) { + SCX_ERR("Failed to join donor thread (%d)", err); + status = SCX_TEST_FAIL; + } else { + donor_started = false; + if (thread_err) { + SCX_ERR("Donor thread failed (%d)", thread_err); + status = SCX_TEST_FAIL; + } + } + } + if (!join_timed_out && owner_started) { + err = join_thread(owner, &join_deadline, &thread_err); + if (err == ETIMEDOUT) { + SCX_ERR("Timed out waiting for owner thread"); + join_timed_out = true; + status = SCX_TEST_FAIL; + } else if (err) { + SCX_ERR("Failed to join owner thread (%d)", err); + status = SCX_TEST_FAIL; + } else { + owner_started = false; + if (thread_err) { + SCX_ERR("Owner thread failed (%d)", thread_err); + status = SCX_TEST_FAIL; + } + } + } + atomic_store_explicit(&thread_ctx.stop_contender, true, + memory_order_release); + for (i = 0; !join_timed_out && i < nr_contenders_started; i++) { + err = join_thread(contenders[i], &join_deadline, &thread_err); + if (err == ETIMEDOUT) { + SCX_ERR("Timed out waiting for contender on CPU %d", + contender_ctxs[i].cpu); + join_timed_out = true; + status = SCX_TEST_FAIL; + } else if (err) { + SCX_ERR("Failed to join contender on CPU %d (%d)", + contender_ctxs[i].cpu, err); + status = SCX_TEST_FAIL; + } else { + nr_contenders_joined++; + if (thread_err) { + SCX_ERR("Contender on CPU %d failed (%d)", + contender_ctxs[i].cpu, thread_err); + status = SCX_TEST_FAIL; + } + } + } + + /* Restore the fair scheduler before waiting for any stranded thread. */ + if (join_timed_out) { + atomic_store_explicit(&thread_ctx.abort, true, + memory_order_release); + if (link) { + bpf_link__destroy(link); + link = NULL; + } + if (donor_started) + pthread_join(donor, NULL); + if (owner_started) + pthread_join(owner, NULL); + for (i = nr_contenders_joined; + i < nr_contenders_started; i++) + pthread_join(contenders[i], NULL); + } + + if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_GET_STATS, &stats)) { + SCX_ERR("Failed to read mutex statistics (%d)", errno); + status = SCX_TEST_FAIL; + } else { + result->stats = stats; + printf("\n[topology=%s SCX_OPS_ENQ_BLOCKED=%s]\n", + cross_cpu ? "cross-cpu" : "same-cpu", + enq_blocked ? "enabled" : "disabled"); + printf(" proxy_exec=%s\n", + proxy_enabled ? "enabled" : "disabled"); + printf(" donor_cpu=%d\n", thread_ctx.donor_cpu); + printf(" owner_cpu=%d\n", thread_ctx.owner_cpu); + printf(" nr_contenders=%zu\n", nr_contenders); + printf(" measured_trials=%d\n", NR_MEASURED_TRIALS); + printf(" owner_nice=%d\n", OWNER_NICE); + printf(" donor_nice=%d\n", DONOR_NICE); + printf(" contender_nice=%d\n", CONTENDER_NICE); + print_avg_time("mutex_hold", stats.hold_time_ns, stats.nr_holds); + print_avg_time("mutex_wait", stats.wait_time_ns, stats.nr_waits); + if (stats.nr_holds != NR_MEASURED_TRIALS || + stats.nr_waits != NR_MEASURED_TRIALS) { + SCX_ERR("Expected %d measured trials, got %llu holds and %llu waits", + NR_MEASURED_TRIALS, + (unsigned long long)stats.nr_holds, + (unsigned long long)stats.nr_waits); + status = SCX_TEST_FAIL; + } + } + + nr_blocked = skel->bss->nr_blocked_enqueues; + nr_blocked_donor_cpu = skel->bss->nr_blocked_enqueues_donor_cpu; + nr_blocked_owner_cpu = skel->bss->nr_blocked_enqueues_owner_cpu; + nr_blocked_other_cpu = skel->bss->nr_blocked_enqueues_other_cpu; + nr_blocked_wakeups = skel->bss->nr_blocked_wakeups; + result->nr_blocked_enqueues = nr_blocked; + result->nr_blocked_enqueues_donor_cpu = nr_blocked_donor_cpu; + result->nr_blocked_enqueues_owner_cpu = nr_blocked_owner_cpu; + result->nr_blocked_enqueues_other_cpu = nr_blocked_other_cpu; + result->nr_blocked_wakeups = nr_blocked_wakeups; + printf(" nr_blocked_enqueues=%llu\n", + (unsigned long long)nr_blocked); + printf(" nr_blocked_enqueues_donor_cpu=%llu\n", + (unsigned long long)nr_blocked_donor_cpu); + printf(" nr_blocked_enqueues_owner_cpu=%llu\n", + (unsigned long long)nr_blocked_owner_cpu); + printf(" nr_blocked_enqueues_other_cpu=%llu\n", + (unsigned long long)nr_blocked_other_cpu); + printf(" nr_blocked_wakeups=%llu\n", + (unsigned long long)nr_blocked_wakeups); + if (status == SCX_TEST_PASS) { + if (enq_blocked && proxy_enabled && !nr_blocked) { + SCX_ERR("ops.enqueue() did not receive the blocked donor"); + status = SCX_TEST_FAIL; + } else if ((!enq_blocked || !proxy_enabled) && nr_blocked) { + SCX_ERR("ops.enqueue() unexpectedly received %llu blocked donors", + (unsigned long long)nr_blocked); + status = SCX_TEST_FAIL; + } else if (nr_blocked_wakeups) { + SCX_ERR("Ordinary wakeups received %llu blocked enqueue flags", + (unsigned long long)nr_blocked_wakeups); + status = SCX_TEST_FAIL; + } else if (nr_blocked_other_cpu) { + SCX_ERR("Blocked donor had %llu enqueues on unexpected CPUs", + (unsigned long long)nr_blocked_other_cpu); + status = SCX_TEST_FAIL; + } + } + + if (UEI_EXITED(skel, uei)) { + SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)", + (unsigned long long)skel->data->uei.kind, + (long long)skel->data->uei.exit_code); + status = SCX_TEST_FAIL; + } + + if (link) + bpf_link__destroy(link); +out_fd: + close(thread_ctx.fd); +out_module: + unload_test_module(module_loaded); +out_skel: + enq_blocked__destroy(skel); +out_contenders: + free(contender_ctxs); + free(contenders); + return status; +} + +static enum scx_test_status run_topology(bool cross_cpu) +{ + struct run_result disabled = {}, enabled = {}; + enum scx_test_status status; + + status = run_one(false, cross_cpu, &disabled); + if (status != SCX_TEST_PASS) + return status; + + status = run_one(true, cross_cpu, &enabled); + if (status != SCX_TEST_PASS) + return status; + + printf("\n[topology=%s delta: enabled - disabled]\n", + cross_cpu ? "cross-cpu" : "same-cpu"); + print_avg_delta("mutex_hold", disabled.stats.hold_time_ns, + disabled.stats.nr_holds, enabled.stats.hold_time_ns, + enabled.stats.nr_holds); + print_avg_delta("mutex_wait", disabled.stats.wait_time_ns, + disabled.stats.nr_waits, enabled.stats.wait_time_ns, + enabled.stats.nr_waits); + + return SCX_TEST_PASS; +} + +static enum scx_test_status run(void *ctx) +{ + enum scx_test_status status; + + (void)ctx; + + status = run_topology(false); + if (status != SCX_TEST_PASS) + return status; + + status = run_topology(true); + if (status == SCX_TEST_SKIP) + return SCX_TEST_PASS; + + return status; +} + +struct scx_test enq_blocked = { + .name = "enq_blocked", + .description = "Verify proxy donor admission under CPU-wide contention", + .setup = setup, + .run = run, +}; + +REGISTER_SCX_TEST(&enq_blocked) diff --git a/tools/testing/selftests/sched_ext/enq_blocked.h b/tools/testing/selftests/sched_ext/enq_blocked.h new file mode 100644 index 000000000000..ef1eb97feebe --- /dev/null +++ b/tools/testing/selftests/sched_ext/enq_blocked.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES */ +#ifndef __ENQ_BLOCKED_H +#define __ENQ_BLOCKED_H + +#include <linux/ioctl.h> +#include <linux/types.h> + +struct enq_blocked_stats { + __u64 hold_time_ns; + __u64 wait_time_ns; + __u64 nr_holds; + __u64 nr_waits; +}; + +#define ENQ_BLOCKED_IOCTL_OWNER _IO('s', 1) +#define ENQ_BLOCKED_IOCTL_DONOR _IO('s', 2) +#define ENQ_BLOCKED_IOCTL_RESET_STATS _IO('s', 3) +#define ENQ_BLOCKED_IOCTL_GET_STATS _IOR('s', 4, struct enq_blocked_stats) +#define ENQ_BLOCKED_IOCTL_PREP_ATTACH _IO('s', 5) +#define ENQ_BLOCKED_IOCTL_ATTACH_DONE _IO('s', 6) +#define ENQ_BLOCKED_IOCTL_DONOR_STATE _IO('s', 7) +#define ENQ_BLOCKED_IOCTL_PROXY_SUPPORTED _IO('s', 8) + +#define ENQ_BLOCKED_DONOR_SLEEPING (1U << 0) +#define ENQ_BLOCKED_DONOR_ON_RQ (1U << 1) + +#endif /* __ENQ_BLOCKED_H */ diff --git a/tools/testing/selftests/sched_ext/exit.c b/tools/testing/selftests/sched_ext/exit.c index 01b17092d5c8..fd6f1c9ab2db 100644 --- a/tools/testing/selftests/sched_ext/exit.c +++ b/tools/testing/selftests/sched_ext/exit.c @@ -43,8 +43,12 @@ static enum scx_test_status run(void *ctx) } /* Assumes uei.kind is written last */ - while (skel->data->uei.kind == EXIT_KIND(SCX_EXIT_NONE)) - sched_yield(); + if (!SCX_TIMED_WAIT_UEI(skel, uei, 10000)) { + SCX_ERR("Timed out waiting for scheduler to exit"); + bpf_link__destroy(link); + exit__destroy(skel); + return SCX_TEST_FAIL; + } SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG_BPF)); SCX_EQ(skel->data->uei.exit_code, tc); diff --git a/tools/testing/selftests/sched_ext/hotplug.c b/tools/testing/selftests/sched_ext/hotplug.c index 0cfbb111a2d0..3c07686a93c8 100644 --- a/tools/testing/selftests/sched_ext/hotplug.c +++ b/tools/testing/selftests/sched_ext/hotplug.c @@ -21,15 +21,17 @@ static bool is_cpu_online(void) return file_read_long(online_path) > 0; } -static void toggle_online_status(bool online) +static int toggle_online_status(bool online) { long val = online ? 1 : 0; int ret; ret = file_write_long(online_path, val); if (ret != 0) - fprintf(stderr, "Failed to bring CPU %s (%s)", + fprintf(stderr, "Failed to bring CPU %s (%s)\n", online ? "online" : "offline", strerror(errno)); + + return ret; } static enum scx_test_status setup(void **ctx) @@ -44,6 +46,7 @@ static enum scx_test_status test_hotplug(bool onlining, bool cbs_defined) { struct hotplug *skel; struct bpf_link *link; + enum scx_test_status status = SCX_TEST_FAIL; long kind, code; SCX_ASSERT(is_cpu_online()); @@ -54,8 +57,8 @@ static enum scx_test_status test_hotplug(bool onlining, bool cbs_defined) SCX_FAIL_IF(hotplug__load(skel), "Failed to load skel"); /* Testing the offline -> online path, so go offline before starting */ - if (onlining) - toggle_online_status(0); + if (onlining && toggle_online_status(0)) + goto out_destroy_skel; if (cbs_defined) { kind = SCX_KIND_VAL(SCX_EXIT_UNREG_BPF); @@ -79,28 +82,34 @@ static enum scx_test_status test_hotplug(bool onlining, bool cbs_defined) return SCX_TEST_FAIL; } - toggle_online_status(onlining ? 1 : 0); + if (toggle_online_status(onlining ? 1 : 0)) + goto out_destroy_link; - while (!UEI_EXITED(skel, uei)) - sched_yield(); + if (!SCX_TIMED_WAIT_UEI(skel, uei, 10000)) { + SCX_ERR("Timed out waiting for scheduler to exit"); + goto out_destroy_link; + } SCX_EQ(skel->data->uei.kind, kind); SCX_EQ(UEI_REPORT(skel, uei), code); - if (!onlining) - toggle_online_status(1); + if (!onlining && toggle_online_status(1)) + goto out_destroy_link; + status = SCX_TEST_PASS; +out_destroy_link: bpf_link__destroy(link); +out_destroy_skel: hotplug__destroy(skel); - return SCX_TEST_PASS; + return status; } static enum scx_test_status test_hotplug_attach(void) { struct hotplug *skel; struct bpf_link *link; - enum scx_test_status status = SCX_TEST_PASS; + enum scx_test_status status = SCX_TEST_FAIL; long kind, code; SCX_ASSERT(is_cpu_online()); @@ -115,14 +124,18 @@ static enum scx_test_status test_hotplug_attach(void) * Take the CPU offline to increment the global hotplug seq, which * should cause attach to fail due to us setting the hotplug seq above */ - toggle_online_status(0); + if (toggle_online_status(0)) + goto out_destroy_skel; link = bpf_map__attach_struct_ops(skel->maps.hotplug_nocb_ops); - toggle_online_status(1); + if (toggle_online_status(1)) + goto out_destroy_link; SCX_ASSERT(link); - while (!UEI_EXITED(skel, uei)) - sched_yield(); + if (!SCX_TIMED_WAIT_UEI(skel, uei, 10000)) { + SCX_ERR("Timed out waiting for scheduler to exit"); + goto out_destroy_link; + } kind = SCX_KIND_VAL(SCX_EXIT_UNREG_KERN); code = SCX_ECODE_VAL(SCX_ECODE_ACT_RESTART) | @@ -130,7 +143,10 @@ static enum scx_test_status test_hotplug_attach(void) SCX_EQ(skel->data->uei.kind, kind); SCX_EQ(UEI_REPORT(skel, uei), code); + status = SCX_TEST_PASS; +out_destroy_link: bpf_link__destroy(link); +out_destroy_skel: hotplug__destroy(skel); return status; diff --git a/tools/testing/selftests/sched_ext/init_enable_count.bpf.c b/tools/testing/selftests/sched_ext/init_enable_count.bpf.c index 5eb9edb1837d..c0a5dd8fe183 100644 --- a/tools/testing/selftests/sched_ext/init_enable_count.bpf.c +++ b/tools/testing/selftests/sched_ext/init_enable_count.bpf.c @@ -15,6 +15,8 @@ char _license[] SEC("license") = "GPL"; u64 init_task_cnt, exit_task_cnt, enable_cnt, disable_cnt; u64 init_fork_cnt, init_transition_cnt; +UEI_DEFINE(uei); + s32 BPF_STRUCT_OPS_SLEEPABLE(cnt_init_task, struct task_struct *p, struct scx_init_task_args *args) { @@ -43,11 +45,17 @@ void BPF_STRUCT_OPS(cnt_disable, struct task_struct *p) __sync_fetch_and_add(&disable_cnt, 1); } +void BPF_STRUCT_OPS(cnt_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + SEC(".struct_ops.link") struct sched_ext_ops init_enable_count_ops = { .init_task = (void *) cnt_init_task, .exit_task = (void *) cnt_exit_task, .enable = (void *) cnt_enable, .disable = (void *) cnt_disable, + .exit = (void *) cnt_exit, .name = "init_enable_count", }; diff --git a/tools/testing/selftests/sched_ext/init_enable_count.c b/tools/testing/selftests/sched_ext/init_enable_count.c index 44577e30e764..c4a57038a1d6 100644 --- a/tools/testing/selftests/sched_ext/init_enable_count.c +++ b/tools/testing/selftests/sched_ext/init_enable_count.c @@ -5,6 +5,7 @@ * Copyright (c) 2023 Tejun Heo <tj@kernel.org> */ #include <signal.h> +#include <string.h> #include <stdio.h> #include <unistd.h> #include <sched.h> @@ -72,7 +73,17 @@ static enum scx_test_status run_test(bool global) close(pipe_fds[1]); signal(SIGCHLD, SIG_DFL); + SCX_ASSERT_ALIVE(skel, uei); + bpf_link__destroy(link); + + /* + * The clean unregister above is recorded in uei and would + * otherwise trip the liveness check of the second cycle, which + * reuses this skeleton. + */ + memset(&skel->data->uei, 0, sizeof(skel->data->uei)); + SCX_GE(skel->bss->init_task_cnt, num_pre_forks); SCX_GE(skel->bss->exit_task_cnt, num_pre_forks); @@ -124,6 +135,8 @@ static enum scx_test_status run_test(bool global) status); } + SCX_ASSERT_ALIVE(skel, uei); + bpf_link__destroy(link); SCX_GE(skel->bss->init_task_cnt, 2 * num_children); diff --git a/tools/testing/selftests/sched_ext/kick.bpf.c b/tools/testing/selftests/sched_ext/kick.bpf.c new file mode 100644 index 000000000000..bd9605d1573d --- /dev/null +++ b/tools/testing/selftests/sched_ext/kick.bpf.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + */ +#include <scx/common.bpf.h> + +#include "kick_test.h" + +char _license[] SEC("license") = "GPL"; + +const volatile u32 scenario; +const volatile s32 victim_pid; +const volatile s32 challenger_pid; +const volatile s32 target_cpu; +const volatile s32 lazy_resched_override = -1; + +u32 state; +u64 slice_before; +u64 slice_at_resched; +s32 resched_tif; +u64 nr_wait_callbacks; +bool enq_preempt_issued; + +UEI_DEFINE(uei); + +static bool is_trace_scenario(void) +{ + return scenario <= TICK_EXPIRY; +} + +void BPF_STRUCT_OPS(kick_enqueue, struct task_struct *p, u64 enq_flags) +{ + switch (scenario) { + case ENQ_BOTH: + if (p->pid == victim_pid) { + scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_INF, enq_flags); + } else if (p->pid == challenger_pid && state == KICK_STATE_QUEUED) { + enq_preempt_issued = true; + scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, + enq_flags | SCX_ENQ_PREEMPT | + SCX_ENQ_PREEMPT_LAZY); + } else { + scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); + } + goto kick_last; + case INVALID_KICK_IDLE: + scx_bpf_kick_cpu(scx_bpf_task_cpu(p), SCX_KICK_PREEMPT_LAZY | SCX_KICK_IDLE); + break; + case INVALID_KICK_UNKNOWN: + scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 1LLU << 63); + break; + } + + scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); +kick_last: + if (enq_flags & SCX_ENQ_LAST) + scx_bpf_kick_cpu(target_cpu, SCX_KICK_IDLE); +} + +static void set_lazy_resched_override(struct task_struct *p) +{ + if (p->pid == victim_pid && lazy_resched_override >= 0) + scx_bpf_task_set_lazy_resched(p, lazy_resched_override); +} + +void BPF_STRUCT_OPS(kick_running, struct task_struct *p) +{ + if (!is_trace_scenario() || p->pid != victim_pid) + return; + set_lazy_resched_override(p); + if (__sync_val_compare_and_swap(&state, KICK_STATE_ARMED, KICK_STATE_QUEUED) != + KICK_STATE_ARMED) + return; + + slice_before = p->scx.slice; + + switch (scenario) { + case KICK_IMMEDIATE: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT); + break; + case KICK_LAZY: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY); + break; + case KICK_LAZY_THEN_IMMEDIATE: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY); + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT); + break; + case KICK_IMMEDIATE_THEN_LAZY: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT); + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY); + break; + case KICK_PLAIN_THEN_LAZY: + scx_bpf_kick_cpu(target_cpu, 0); + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY); + break; + case KICK_LAZY_THEN_PLAIN: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY); + scx_bpf_kick_cpu(target_cpu, 0); + break; + case KICK_BOTH: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT | SCX_KICK_PREEMPT_LAZY); + break; + case KICK_LAZY_WAIT: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY | SCX_KICK_WAIT); + break; + case ENQ_BOTH: + /* The userspace controller wakes a competing task. */ + break; + case TICK_EXPIRY: + /* no kick: the slice runs out at the tick */ + break; + } +} + +SEC("fexit/__resched_curr") +int BPF_PROG(kick_need_resched, struct rq *rq, int tif) +{ + struct task_struct *task = BPF_CORE_READ(rq, curr); + u64 slice; + + if (!task || BPF_CORE_READ(task, pid) != victim_pid || + BPF_CORE_READ(rq, cpu) != target_cpu || state != KICK_STATE_QUEUED) + return 0; + if (scenario == ENQ_BOTH && !enq_preempt_issued) + return 0; + + slice = BPF_CORE_READ(task, scx.slice); + if (slice) + return 0; + + slice_at_resched = slice; + resched_tif = tif; + state = KICK_STATE_RESCHED; + return 0; +} + +SEC("fentry/kick_sync_wait_bal_cb") +int BPF_PROG(kick_wait_callback, struct rq *rq) +{ + if (scenario == KICK_LAZY_WAIT && BPF_CORE_READ(rq, cpu) == target_cpu) + __sync_fetch_and_add(&nr_wait_callbacks, 1); + return 0; +} + +void BPF_STRUCT_OPS(kick_stopping, struct task_struct *p, bool runnable) +{ + if (p->pid == victim_pid && state == KICK_STATE_RESCHED) + state = KICK_STATE_DONE; +} + +void BPF_STRUCT_OPS(kick_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + +SEC(".struct_ops.link") +struct sched_ext_ops kick_ops = { + .enqueue = (void *)kick_enqueue, + .running = (void *)kick_running, + .stopping = (void *)kick_stopping, + .exit = (void *)kick_exit, + .flags = SCX_OPS_ENQ_LAST, + .name = "kick", +}; diff --git a/tools/testing/selftests/sched_ext/kick.c b/tools/testing/selftests/sched_ext/kick.c new file mode 100644 index 000000000000..40c91d54c1eb --- /dev/null +++ b/tools/testing/selftests/sched_ext/kick.c @@ -0,0 +1,521 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + */ +#define _GNU_SOURCE +#include <bpf/bpf.h> +#include <errno.h> +#include <linux/sched.h> +#include <sched.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/prctl.h> +#include <sys/wait.h> +#include <unistd.h> +#include <scx/common.h> + +#include "kick.bpf.skel.h" +#include "kick_test.h" +#include "scx_test.h" +#include "util.h" + +#define WAIT_LOOPS 3000 + +struct observation { + u64 slice_before; + u64 slice_at_resched; + u64 nr_wait_callbacks; + s32 resched_tif; +}; + +struct kick_ctx { + cpu_set_t original_mask; + int target_cpu; +}; + +static bool enum_supported(const char *type, const char *name) +{ + u64 value; + + return __COMPAT_read_enum(type, name, &value); +} + +static enum scx_test_status setup_controller(void **ctx_ptr) +{ + struct kick_ctx *ctx; + cpu_set_t controller_mask; + int cpu, controller_cpu = -1; + + ctx = calloc(1, sizeof(*ctx)); + SCX_FAIL_IF(!ctx, "Failed to allocate context"); + if (sched_getaffinity(0, sizeof(ctx->original_mask), &ctx->original_mask)) { + free(ctx); + SCX_FAIL("Failed to get affinity (%d)", errno); + } + + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) { + if (!CPU_ISSET(cpu, &ctx->original_mask)) + continue; + if (controller_cpu < 0) { + controller_cpu = cpu; + } else { + ctx->target_cpu = cpu; + break; + } + } + if (cpu == CPU_SETSIZE) { + printf("SKIP: two allowed CPUs are required\n"); + free(ctx); + return SCX_TEST_SKIP; + } + + CPU_ZERO(&controller_mask); + CPU_SET(controller_cpu, &controller_mask); + if (sched_setaffinity(0, sizeof(controller_mask), &controller_mask)) { + free(ctx); + SCX_FAIL("Failed to pin controller to CPU %d (%d)", controller_cpu, errno); + } + + *ctx_ptr = ctx; + return SCX_TEST_PASS; +} + +static void cleanup_controller(void *ctx_ptr) +{ + struct kick_ctx *ctx = ctx_ptr; + + sched_setaffinity(0, sizeof(ctx->original_mask), &ctx->original_mask); + free(ctx); +} + +static bool wait_for_state(struct kick *skel, u32 wanted) +{ + int i; + + for (i = 0; i < WAIT_LOOPS; i++) { + if (__atomic_load_n(&skel->bss->state, __ATOMIC_ACQUIRE) == wanted) + return true; + if (UEI_EXITED(skel, uei)) + return false; + usleep(1000); + } + return false; +} + +static bool wait_for_counter(struct kick *skel, const u64 *counter, u64 wanted) +{ + int i; + + for (i = 0; i < WAIT_LOOPS; i++) { + if (__atomic_load_n(counter, __ATOMIC_ACQUIRE) >= wanted) + return true; + if (UEI_EXITED(skel, uei)) + return false; + usleep(1000); + } + return false; +} + +static enum scx_test_status trace_one(struct kick_ctx *ctx, u32 scenario, u64 ops_flags, + s32 lazy_resched_override, struct observation *obs) +{ + struct bpf_link *ops_link = NULL; + struct kick *skel = NULL; + struct scx_test_gated_worker challenger = { .pid = -1, .start_fd = -1 }; + struct scx_test_gated_worker victim; + enum scx_test_status ret = SCX_TEST_FAIL; + int cpu = ctx->target_cpu; + + victim = scx_test_spawn_gated_worker(cpu, false); + if (victim.pid < 0) { + SCX_ERR("Failed to spawn victim"); + return SCX_TEST_FAIL; + } + if (scenario == ENQ_BOTH) { + challenger = scx_test_spawn_gated_worker(cpu, false); + if (challenger.pid < 0) { + SCX_ERR("Failed to spawn enqueue challenger"); + goto out; + } + } + + skel = kick__open(); + if (!skel) { + SCX_ERR("Failed to open scenario %u", scenario); + goto out; + } + SCX_ENUM_INIT(skel); + skel->rodata->scenario = scenario; + skel->rodata->victim_pid = victim.pid; + skel->rodata->challenger_pid = challenger.pid; + skel->rodata->target_cpu = cpu; + skel->rodata->lazy_resched_override = lazy_resched_override; + skel->struct_ops.kick_ops->flags |= ops_flags; + if (scenario != KICK_LAZY_WAIT) + bpf_program__set_autoload(skel->progs.kick_wait_callback, false); + if (kick__load(skel)) { + SCX_ERR("Failed to load scenario %u", scenario); + goto out; + } + + bpf_map__set_autoattach(skel->maps.kick_ops, false); + if (kick__attach(skel)) { + SCX_ERR("Failed to attach __resched_curr tracer"); + goto out; + } + skel->bss->state = KICK_STATE_ARMED; + ops_link = bpf_map__attach_struct_ops(skel->maps.kick_ops); + if (!ops_link) { + SCX_ERR("Failed to attach scenario %u", scenario); + goto out; + } + if (!scx_test_start_gated_worker(&victim)) { + SCX_ERR("Failed to start victim"); + goto out; + } + if (scenario == ENQ_BOTH) { + if (!wait_for_state(skel, KICK_STATE_QUEUED)) { + SCX_ERR("Enqueue scenario did not arm"); + goto out; + } + if (!scx_test_start_gated_worker(&challenger)) { + SCX_ERR("Failed to start enqueue challenger"); + goto out; + } + } + if (!wait_for_state(skel, KICK_STATE_DONE)) { + SCX_ERR("Scenario %u stopped in state %u, exit kind %d", scenario, skel->bss->state, + skel->data->uei.kind); + goto out; + } + if (scenario == KICK_LAZY_WAIT && + !wait_for_counter(skel, &skel->bss->nr_wait_callbacks, 1)) { + SCX_ERR("WAIT callback did not run"); + goto out; + } + + obs->slice_before = skel->bss->slice_before; + obs->slice_at_resched = skel->bss->slice_at_resched; + obs->nr_wait_callbacks = skel->bss->nr_wait_callbacks; + obs->resched_tif = skel->bss->resched_tif; + ret = SCX_TEST_PASS; +out: + scx_test_stop_gated_worker(&victim); + scx_test_stop_gated_worker(&challenger); + if (ops_link) + bpf_link__destroy(ops_link); + if (skel) + kick__destroy(skel); + return ret; +} + +static bool observation_valid(const struct observation *obs) +{ + return obs->slice_before > 0 && !obs->slice_at_resched; +} + +static bool trace_target_supported(const char *name, enum bpf_attach_type attach_type) +{ + return libbpf_find_vmlinux_btf_id(name, attach_type) > 0; +} + +static enum scx_test_status setup_immediate(void **ctx) +{ + if (!enum_supported("scx_kick_flags", "SCX_KICK_PREEMPT")) { + printf("SKIP: SCX_KICK_PREEMPT is not supported\n"); + return SCX_TEST_SKIP; + } + if (!trace_target_supported("__resched_curr", BPF_TRACE_FEXIT)) { + printf("SKIP: __resched_curr is not available in BTF\n"); + return SCX_TEST_SKIP; + } + return setup_controller(ctx); +} + +static enum scx_test_status setup_lazy(void **ctx) +{ + if (!enum_supported("scx_kick_flags", "SCX_KICK_PREEMPT_LAZY")) { + printf("SKIP: SCX_KICK_PREEMPT_LAZY is not supported\n"); + return SCX_TEST_SKIP; + } + return setup_immediate(ctx); +} + +static enum scx_test_status setup_tick(void **ctx) +{ + if (!enum_supported("scx_ops_flags", "SCX_OPS_LAZY_RESCHED")) { + printf("SKIP: SCX_OPS_LAZY_RESCHED is not supported\n"); + return SCX_TEST_SKIP; + } + return setup_lazy(ctx); +} + +static enum scx_test_status setup_coalesce(void **ctx) +{ + if (!enum_supported("scx_enq_flags", "SCX_ENQ_PREEMPT_LAZY")) { + printf("SKIP: SCX_ENQ_PREEMPT_LAZY is not supported\n"); + return SCX_TEST_SKIP; + } + if (!trace_target_supported("kick_sync_wait_bal_cb", BPF_TRACE_FENTRY)) { + printf("SKIP: kick_sync_wait_bal_cb is not available in BTF\n"); + return SCX_TEST_SKIP; + } + return setup_lazy(ctx); +} + +static enum scx_test_status setup_invalid(void **ctx) +{ + if (!enum_supported("scx_kick_flags", "SCX_KICK_PREEMPT_LAZY")) { + printf("SKIP: SCX_KICK_PREEMPT_LAZY is not supported\n"); + return SCX_TEST_SKIP; + } + return setup_controller(ctx); +} + +static enum scx_test_status run_immediate(void *ctx) +{ + struct observation obs; + enum scx_test_status status; + + status = trace_one(ctx, KICK_IMMEDIATE, 0, -1, &obs); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&obs)); + return SCX_TEST_PASS; +} + +static enum scx_test_status run_lazy(void *ctx) +{ + struct observation immediate, lazy; + enum scx_test_status status; + int lazy_mode = scx_test_preempt_lazy_mode(); + + if (lazy_mode < 0) { + printf("SKIP: running kernel preemption mode is unavailable\n"); + return SCX_TEST_SKIP; + } + + status = trace_one(ctx, KICK_IMMEDIATE, 0, -1, &immediate); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&immediate)); + status = trace_one(ctx, KICK_LAZY, 0, -1, &lazy); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&lazy)); + + if (lazy_mode > 0) + SCX_FAIL_IF(lazy.resched_tif == immediate.resched_tif, + "Lazy mode used immediate TIF %d", lazy.resched_tif); + else if (!lazy_mode) + SCX_EQ(lazy.resched_tif, immediate.resched_tif); + + return SCX_TEST_PASS; +} + +static enum scx_test_status run_coalesce(void *ctx) +{ + struct observation immediate, lazy_first, immediate_first; + struct observation plain_first, lazy_then_plain; + struct observation both, lazy_wait, enq_both; + enum scx_test_status status; + + if (scx_test_preempt_lazy_mode() != 1) { + printf("SKIP: kick coalescing requires active lazy preemption\n"); + return SCX_TEST_SKIP; + } + + status = trace_one(ctx, KICK_IMMEDIATE, 0, -1, &immediate); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&immediate)); + status = trace_one(ctx, KICK_LAZY_THEN_IMMEDIATE, 0, -1, &lazy_first); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&lazy_first)); + status = trace_one(ctx, KICK_IMMEDIATE_THEN_LAZY, 0, -1, &immediate_first); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&immediate_first)); + SCX_EQ(lazy_first.resched_tif, immediate.resched_tif); + SCX_EQ(immediate_first.resched_tif, immediate.resched_tif); + + /* + * A plain kick in the same batch doesn't clear the slice by itself. The + * lazy preemption must still expire it, and the plain kick must still + * reschedule immediately, whichever came first. + */ + status = trace_one(ctx, KICK_PLAIN_THEN_LAZY, 0, -1, &plain_first); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&plain_first)); + status = trace_one(ctx, KICK_LAZY_THEN_PLAIN, 0, -1, &lazy_then_plain); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&lazy_then_plain)); + SCX_EQ(plain_first.resched_tif, immediate.resched_tif); + SCX_EQ(lazy_then_plain.resched_tif, immediate.resched_tif); + + /* Immediate kick and WAIT both take precedence over lazy preemption. */ + status = trace_one(ctx, KICK_BOTH, 0, -1, &both); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&both)); + status = trace_one(ctx, KICK_LAZY_WAIT, 0, -1, &lazy_wait); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&lazy_wait)); + SCX_EQ(both.resched_tif, immediate.resched_tif); + SCX_EQ(lazy_wait.resched_tif, immediate.resched_tif); + SCX_GT(lazy_wait.nr_wait_callbacks, 0); + + /* Immediate enqueue preemption likewise takes precedence over lazy. */ + status = trace_one(ctx, ENQ_BOTH, 0, -1, &enq_both); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&enq_both)); + SCX_EQ(enq_both.resched_tif, immediate.resched_tif); + return SCX_TEST_PASS; +} + +/* + * A slice running out at the tick reschedules immediately by default and lazily + * with SCX_OPS_LAZY_RESCHED, the way fair.c expires a slice. + */ +static enum scx_test_status run_tick(void *ctx) +{ + struct observation immediate, lazy, force_lazy, force_immediate; + enum scx_test_status status; + u64 lazy_flag; + int lazy_mode = scx_test_preempt_lazy_mode(); + bool found; + + if (lazy_mode < 0) { + printf("SKIP: running kernel preemption mode is unavailable\n"); + return SCX_TEST_SKIP; + } + + found = __COMPAT_read_enum("scx_ops_flags", "SCX_OPS_LAZY_RESCHED", &lazy_flag); + SCX_ASSERT(found); + status = trace_one(ctx, TICK_EXPIRY, 0, -1, &immediate); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&immediate)); + status = trace_one(ctx, TICK_EXPIRY, lazy_flag, -1, &lazy); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&lazy)); + if (lazy_mode > 0) { + SCX_FAIL_IF(lazy.resched_tif == immediate.resched_tif, + "Lazy slice expiry used immediate TIF %d", lazy.resched_tif); + + status = trace_one(ctx, TICK_EXPIRY, 0, 1, &force_lazy); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&force_lazy)); + status = trace_one(ctx, TICK_EXPIRY, lazy_flag, 0, &force_immediate); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&force_immediate)); + SCX_EQ(force_lazy.resched_tif, lazy.resched_tif); + SCX_EQ(force_immediate.resched_tif, immediate.resched_tif); + } else { + SCX_EQ(lazy.resched_tif, immediate.resched_tif); + } + + return SCX_TEST_PASS; +} + +static enum scx_test_status invalid_one(struct kick_ctx *ctx, u32 scenario) +{ + struct bpf_link *ops_link = NULL; + struct kick *skel = NULL; + struct scx_test_gated_worker victim; + enum scx_test_status ret = SCX_TEST_FAIL; + int cpu = ctx->target_cpu; + int i; + u64 error_kind; + + error_kind = EXIT_KIND(SCX_EXIT_ERROR); + victim = scx_test_spawn_gated_worker(cpu, false); + if (victim.pid < 0) + return SCX_TEST_FAIL; + + skel = kick__open(); + if (!skel) + goto out; + SCX_ENUM_INIT(skel); + skel->rodata->scenario = scenario; + bpf_program__set_autoload(skel->progs.kick_need_resched, false); + bpf_program__set_autoload(skel->progs.kick_wait_callback, false); + if (kick__load(skel)) + goto out; + ops_link = bpf_map__attach_struct_ops(skel->maps.kick_ops); + if (!ops_link || !scx_test_start_gated_worker(&victim)) + goto out; + + for (i = 0; i < WAIT_LOOPS; i++) { + if (UEI_EXITED(skel, uei) == error_kind) { + ret = SCX_TEST_PASS; + break; + } + usleep(1000); + } +out: + scx_test_stop_gated_worker(&victim); + if (ops_link) + bpf_link__destroy(ops_link); + if (skel) + kick__destroy(skel); + return ret; +} + +static enum scx_test_status run_invalid(void *ctx) +{ + enum scx_test_status status; + u32 scenario; + + for (scenario = INVALID_KICK_IDLE; + scenario <= INVALID_KICK_UNKNOWN; scenario++) { + status = invalid_one(ctx, scenario); + SCX_EQ(status, SCX_TEST_PASS); + } + return SCX_TEST_PASS; +} + +static struct scx_test kick_immediate = { + .name = "kick_immediate", + .description = "Trace immediate kick slice expiration and rescheduling", + .setup = setup_immediate, + .run = run_immediate, + .cleanup = cleanup_controller, +}; + +static struct scx_test kick_lazy = { + .name = "kick_lazy", + .description = "Trace lazy kick slice expiration and rescheduling", + .setup = setup_lazy, + .run = run_lazy, + .cleanup = cleanup_controller, +}; + +static struct scx_test kick_coalesce = { + .name = "kick_coalesce", + .description = "Verify lazy preemption coalesces with immediate requests", + .setup = setup_coalesce, + .run = run_coalesce, + .cleanup = cleanup_controller, +}; + +static struct scx_test kick_tick = { + .name = "kick_tick", + .description = "Trace slice expiry at the tick, immediate and lazy", + .setup = setup_tick, + .run = run_tick, + .cleanup = cleanup_controller, +}; + +static struct scx_test kick_invalid = { + .name = "kick_invalid", + .description = "Verify invalid kick flag combinations fail", + .setup = setup_invalid, + .run = run_invalid, + .cleanup = cleanup_controller, +}; + +__attribute__((constructor)) +static void register_kick_tests(void) +{ + scx_test_register(&kick_immediate); + scx_test_register(&kick_lazy); + scx_test_register(&kick_coalesce); + scx_test_register(&kick_tick); + scx_test_register(&kick_invalid); +} diff --git a/tools/testing/selftests/sched_ext/kick_test.h b/tools/testing/selftests/sched_ext/kick_test.h new file mode 100644 index 000000000000..7d3c608c6b63 --- /dev/null +++ b/tools/testing/selftests/sched_ext/kick_test.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES */ +#ifndef __KICK_TEST_H__ +#define __KICK_TEST_H__ + +enum kick_scenario { + KICK_IMMEDIATE, + KICK_LAZY, + KICK_LAZY_THEN_IMMEDIATE, + KICK_IMMEDIATE_THEN_LAZY, + KICK_PLAIN_THEN_LAZY, + KICK_LAZY_THEN_PLAIN, + KICK_BOTH, + KICK_LAZY_WAIT, + ENQ_BOTH, + TICK_EXPIRY, + INVALID_KICK_IDLE, + INVALID_KICK_UNKNOWN, +}; + +enum kick_state { + KICK_STATE_IDLE, + KICK_STATE_ARMED, + KICK_STATE_QUEUED, + KICK_STATE_RESCHED, + KICK_STATE_DONE, +}; + +#endif /* __KICK_TEST_H__ */ diff --git a/tools/testing/selftests/sched_ext/maximal.bpf.c b/tools/testing/selftests/sched_ext/maximal.bpf.c index 04a369078aac..9ea664e2ff70 100644 --- a/tools/testing/selftests/sched_ext/maximal.bpf.c +++ b/tools/testing/selftests/sched_ext/maximal.bpf.c @@ -14,6 +14,8 @@ char _license[] SEC("license") = "GPL"; #define DSQ_ID 0 +UEI_DEFINE(uei); + s32 BPF_STRUCT_OPS(maximal_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags) { @@ -132,7 +134,9 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(maximal_init) } void BPF_STRUCT_OPS(maximal_exit, struct scx_exit_info *info) -{} +{ + UEI_RECORD(uei, info); +} SEC(".struct_ops.link") struct sched_ext_ops maximal_ops = { diff --git a/tools/testing/selftests/sched_ext/maximal.c b/tools/testing/selftests/sched_ext/maximal.c index 1dc369224670..6c6b80e0446e 100644 --- a/tools/testing/selftests/sched_ext/maximal.c +++ b/tools/testing/selftests/sched_ext/maximal.c @@ -35,6 +35,8 @@ static enum scx_test_status run(void *ctx) link = bpf_map__attach_struct_ops(skel->maps.maximal_ops); SCX_FAIL_IF(!link, "Failed to attach scheduler"); + SCX_ASSERT_ALIVE(skel, uei); + bpf_link__destroy(link); return SCX_TEST_PASS; diff --git a/tools/testing/selftests/sched_ext/nohz_tick.bpf.c b/tools/testing/selftests/sched_ext/nohz_tick.bpf.c index 6998c5dd6bcb..5fec5673fbf2 100644 --- a/tools/testing/selftests/sched_ext/nohz_tick.bpf.c +++ b/tools/testing/selftests/sched_ext/nohz_tick.bpf.c @@ -6,13 +6,22 @@ */ #include <scx/common.bpf.h> +#include "nohz_tick_test.h" + char _license[] SEC("license") = "GPL"; const volatile s32 test_cpu; -bool finite_phase; +u32 phase; +s32 victim_pid; +s32 challenger_pid; +s32 trigger_pid; u64 nr_inf_running; u64 nr_finite_running; u64 nr_finite_ticks; +u64 nr_lazy_victim_running; +u64 nr_lazy_enq_running; +u64 nr_lazy_kick_running; +u64 nr_lazy_ticks; UEI_DEFINE(uei); @@ -24,9 +33,31 @@ s32 BPF_STRUCT_OPS(nohz_tick_select_cpu, struct task_struct *p, s32 prev_cpu, void BPF_STRUCT_OPS(nohz_tick_enqueue, struct task_struct *p, u64 enq_flags) { - u64 slice = finite_phase ? 1000000ULL : SCX_SLICE_INF; + u64 slice; + u64 dsq_id = SCX_DSQ_GLOBAL; + + switch (phase) { + case NOHZ_PHASE_INF: + slice = SCX_SLICE_INF; + break; + case NOHZ_PHASE_FINITE: + slice = 1000000ULL; + break; + case NOHZ_PHASE_LAZY_ENQ: + case NOHZ_PHASE_LAZY_KICK: + dsq_id = SCX_DSQ_LOCAL; + slice = p->pid == victim_pid ? SCX_SLICE_INF : SCX_SLICE_DFL; + if (phase == NOHZ_PHASE_LAZY_ENQ && p->pid == challenger_pid) + enq_flags |= SCX_ENQ_PREEMPT_LAZY; + break; + default: + slice = SCX_SLICE_DFL; + break; + } - scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, slice, enq_flags); + scx_bpf_dsq_insert(p, dsq_id, slice, enq_flags); + if (phase == NOHZ_PHASE_LAZY_KICK && p->pid == trigger_pid) + scx_bpf_kick_cpu(test_cpu, SCX_KICK_PREEMPT_LAZY); if (enq_flags & SCX_ENQ_LAST) scx_bpf_kick_cpu(test_cpu, SCX_KICK_IDLE); } @@ -36,16 +67,28 @@ void BPF_STRUCT_OPS(nohz_tick_running, struct task_struct *p) if (bpf_get_smp_processor_id() != test_cpu) return; - if (finite_phase) + if (phase == NOHZ_PHASE_FINITE) __sync_fetch_and_add(&nr_finite_running, 1); - else + else if (phase == NOHZ_PHASE_INF) __sync_fetch_and_add(&nr_inf_running, 1); + else if (p->pid == victim_pid) + __sync_fetch_and_add(&nr_lazy_victim_running, 1); + else if (phase == NOHZ_PHASE_LAZY_ENQ && p->pid == challenger_pid) + __sync_fetch_and_add(&nr_lazy_enq_running, 1); + else if (phase == NOHZ_PHASE_LAZY_KICK && p->pid == challenger_pid) + __sync_fetch_and_add(&nr_lazy_kick_running, 1); } void BPF_STRUCT_OPS(nohz_tick_tick, struct task_struct *p) { - if (bpf_get_smp_processor_id() == test_cpu && finite_phase) + if (bpf_get_smp_processor_id() != test_cpu) + return; + + if (phase == NOHZ_PHASE_FINITE) __sync_fetch_and_add(&nr_finite_ticks, 1); + else if ((phase == NOHZ_PHASE_LAZY_ENQ || + phase == NOHZ_PHASE_LAZY_KICK) && p->pid == victim_pid) + __sync_fetch_and_add(&nr_lazy_ticks, 1); } void BPF_STRUCT_OPS(nohz_tick_exit, struct scx_exit_info *ei) @@ -61,5 +104,5 @@ struct sched_ext_ops nohz_tick_ops = { .tick = (void *)nohz_tick_tick, .exit = (void *)nohz_tick_exit, .name = "nohz_tick", - .timeout_ms = 1000U, + .timeout_ms = 5000U, }; diff --git a/tools/testing/selftests/sched_ext/nohz_tick.c b/tools/testing/selftests/sched_ext/nohz_tick.c index 028f54391c2c..e19a97034b7f 100644 --- a/tools/testing/selftests/sched_ext/nohz_tick.c +++ b/tools/testing/selftests/sched_ext/nohz_tick.c @@ -21,19 +21,24 @@ #include <scx/common.h> #include "nohz_tick.bpf.skel.h" +#include "nohz_tick_test.h" #include "scx_test.h" +#include "util.h" #ifndef SCHED_EXT #define SCHED_EXT 7 #endif #define MIN_FINITE_TICKS 3 -#define PHASE_TIMEOUT_MS 1000 +#define PHASE_TIMEOUT_MS 5000 +#define TICK_STOP_STABLE_MS 100 struct nohz_tick_ctx { struct nohz_tick *skel; cpu_set_t original_mask; int test_cpu; + int housekeeping_cpu; + bool test_lazy; }; static int first_allowed_cpu(const cpu_set_t *mask, int first, int last) @@ -47,20 +52,21 @@ static int first_allowed_cpu(const cpu_set_t *mask, int first, int last) return -1; } -static int find_nohz_full_cpu(const cpu_set_t *allowed) +static int read_nohz_full_mask(cpu_set_t *mask) { char buf[4096], *cur, *end; FILE *file; + int ret = 0; file = fopen("/sys/devices/system/cpu/nohz_full", "r"); if (!file) - return -1; + return -errno; if (!fgets(buf, sizeof(buf), file)) { - fclose(file); - return -1; + ret = ferror(file) ? -errno : -EINVAL; + goto out; } - fclose(file); + CPU_ZERO(mask); cur = buf; while (*cur) { long first, last; @@ -73,25 +79,30 @@ static int find_nohz_full_cpu(const cpu_set_t *allowed) errno = 0; first = strtol(cur, &end, 10); - if (errno || end == cur || first < 0 || first >= CPU_SETSIZE) - return -1; + if (errno || end == cur || first < 0 || first >= CPU_SETSIZE) { + ret = -EINVAL; + goto out; + } cur = end; last = first; if (*cur == '-') { cur++; errno = 0; last = strtol(cur, &end, 10); - if (errno || end == cur || last < first) - return -1; + if (errno || end == cur || last < first) { + ret = -EINVAL; + goto out; + } cur = end; } - cpu = first_allowed_cpu(allowed, first, last); - if (cpu >= 0) - return cpu; + for (cpu = first; cpu <= last && cpu < CPU_SETSIZE; cpu++) + CPU_SET(cpu, mask); } - return -1; +out: + fclose(file); + return ret; } static pid_t start_worker(int cpu) @@ -146,14 +157,41 @@ static int pause_worker(pid_t pid) return 0; } -static bool wait_for_counter(const u64 *counter, u64 value, int timeout_ms) +static bool wait_for_counter(struct nohz_tick *skel, const u64 *counter, u64 value, + int timeout_ms) { int elapsed; for (elapsed = 0; elapsed < timeout_ms; elapsed++) { if (__atomic_load_n(counter, __ATOMIC_RELAXED) >= value) return true; + if (UEI_EXITED(skel, uei)) + return false; + usleep(1000); + } + + return false; +} + +static bool wait_for_tick_stop(struct nohz_tick *skel, const u64 *counter, int timeout_ms) +{ + u64 prev = __atomic_load_n(counter, __ATOMIC_RELAXED); + int elapsed, stable = 0; + + for (elapsed = 0; elapsed < timeout_ms; elapsed++) { + u64 curr; + usleep(1000); + if (UEI_EXITED(skel, uei)) + return false; + curr = __atomic_load_n(counter, __ATOMIC_RELAXED); + if (curr == prev) { + if (++stable >= TICK_STOP_STABLE_MS) + return true; + } else { + prev = curr; + stable = 0; + } } return false; @@ -162,8 +200,10 @@ static bool wait_for_counter(const u64 *counter, u64 value, int timeout_ms) static enum scx_test_status setup(void **ctx_ptr) { struct nohz_tick_ctx *ctx; - cpu_set_t controller_mask; - int cpu; + cpu_set_t controller_mask, nohz_full_mask; + bool lazy_supported; + u64 enum_value, lazy_ops_flag; + int cpu, i, lazy_mode, ret; ctx = calloc(1, sizeof(*ctx)); SCX_FAIL_IF(!ctx, "Failed to allocate context"); @@ -173,15 +213,26 @@ static enum scx_test_status setup(void **ctx_ptr) SCX_FAIL("Failed to get affinity (%d)", errno); } - cpu = find_nohz_full_cpu(&ctx->original_mask); - if (cpu < 0) { + ret = read_nohz_full_mask(&nohz_full_mask); + if (ret) { + fprintf(stderr, "SKIP: failed to read NOHZ_FULL mask (%d)\n", ret); + free(ctx); + return SCX_TEST_SKIP; + } + + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) + if (CPU_ISSET(cpu, &ctx->original_mask) && CPU_ISSET(cpu, &nohz_full_mask)) + break; + if (cpu == CPU_SETSIZE) { fprintf(stderr, "SKIP: no allowed NOHZ_FULL CPU\n"); free(ctx); return SCX_TEST_SKIP; } controller_mask = ctx->original_mask; - CPU_CLR(cpu, &controller_mask); + for (i = 0; i < CPU_SETSIZE; i++) + if (CPU_ISSET(i, &nohz_full_mask)) + CPU_CLR(i, &controller_mask); if (CPU_COUNT(&controller_mask) == 0) { fprintf(stderr, "SKIP: no housekeeping CPU available\n"); free(ctx); @@ -189,6 +240,21 @@ static enum scx_test_status setup(void **ctx_ptr) } ctx->test_cpu = cpu; + ctx->housekeeping_cpu = first_allowed_cpu(&controller_mask, 0, CPU_SETSIZE - 1); + lazy_supported = __COMPAT_read_enum("scx_enq_flags", "SCX_ENQ_PREEMPT_LAZY", + &enum_value) && + __COMPAT_read_enum("scx_kick_flags", "SCX_KICK_PREEMPT_LAZY", + &enum_value) && + __COMPAT_read_enum("scx_ops_flags", "SCX_OPS_LAZY_RESCHED", + &lazy_ops_flag); + lazy_mode = scx_test_preempt_lazy_mode(); + ctx->test_lazy = lazy_supported && lazy_mode > 0; + if (lazy_supported && lazy_mode < 0) + fprintf(stderr, + "SKIP: kernel preemption mode unavailable; skipping lazy NOHZ_FULL phases\n"); + else if (lazy_supported && !lazy_mode) + fprintf(stderr, + "SKIP: lazy preemption inactive; skipping lazy NOHZ_FULL phases\n"); ctx->skel = nohz_tick__open(); if (!ctx->skel) { free(ctx); @@ -199,6 +265,8 @@ static enum scx_test_status setup(void **ctx_ptr) ctx->skel->rodata->test_cpu = cpu; ctx->skel->struct_ops.nohz_tick_ops->flags |= SCX_OPS_SWITCH_PARTIAL | SCX_OPS_ENQ_LAST; + if (lazy_supported) + ctx->skel->struct_ops.nohz_tick_ops->flags |= lazy_ops_flag; if (nohz_tick__load(ctx->skel)) { nohz_tick__destroy(ctx->skel); free(ctx); @@ -220,11 +288,15 @@ static enum scx_test_status run(void *ctx_ptr) struct nohz_tick_ctx *ctx = ctx_ptr; struct nohz_tick *skel = ctx->skel; struct bpf_link *link = NULL; + struct scx_test_gated_worker victim = { .pid = -1, .start_fd = -1 }; + struct scx_test_gated_worker challenger = { .pid = -1, .start_fd = -1 }; + struct scx_test_gated_worker trigger = { .pid = -1, .start_fd = -1 }; enum scx_test_status status = SCX_TEST_FAIL; pid_t finite_worker = -1; pid_t inf_worker = -1; u64 finite_running; u64 finite_ticks; + u64 victim_running; int ret; link = bpf_map__attach_struct_ops(skel->maps.nohz_tick_ops); @@ -241,7 +313,7 @@ static enum scx_test_status run(void *ctx_ptr) SCX_ERR("Failed to start infinite-slice worker (%d)", errno); goto out; } - if (!wait_for_counter(&skel->bss->nr_inf_running, 1, + if (!wait_for_counter(skel, &skel->bss->nr_inf_running, 1, PHASE_TIMEOUT_MS)) { SCX_ERR("Infinite-slice worker was not scheduled"); goto out; @@ -260,18 +332,18 @@ static enum scx_test_status run(void *ctx_ptr) /* * The next EXT task receives a finite slice and must restart the tick. */ - __atomic_store_n(&skel->bss->finite_phase, true, __ATOMIC_RELEASE); + __atomic_store_n(&skel->bss->phase, NOHZ_PHASE_FINITE, __ATOMIC_RELEASE); finite_worker = start_worker(ctx->test_cpu); if (finite_worker < 0) { SCX_ERR("Failed to start finite-slice worker (%d)", errno); goto out; } - if (!wait_for_counter(&skel->bss->nr_finite_running, 1, + if (!wait_for_counter(skel, &skel->bss->nr_finite_running, 1, PHASE_TIMEOUT_MS)) { SCX_ERR("Finite-slice worker was not scheduled"); goto out; } - if (!wait_for_counter(&skel->bss->nr_finite_ticks, MIN_FINITE_TICKS, + if (!wait_for_counter(skel, &skel->bss->nr_finite_ticks, MIN_FINITE_TICKS, PHASE_TIMEOUT_MS)) { SCX_ERR("Finite-slice worker received only %llu scheduler ticks", (unsigned long long)skel->bss->nr_finite_ticks); @@ -295,12 +367,12 @@ static enum scx_test_status run(void *ctx_ptr) SCX_ERR("Failed to start second finite-slice worker (%d)", errno); goto out; } - if (!wait_for_counter(&skel->bss->nr_finite_running, + if (!wait_for_counter(skel, &skel->bss->nr_finite_running, finite_running + 1, PHASE_TIMEOUT_MS)) { SCX_ERR("Second finite-slice worker was not scheduled"); goto out; } - if (!wait_for_counter(&skel->bss->nr_finite_ticks, + if (!wait_for_counter(skel, &skel->bss->nr_finite_ticks, finite_ticks + MIN_FINITE_TICKS, PHASE_TIMEOUT_MS)) { SCX_ERR("Second finite-slice worker received only %llu scheduler ticks", @@ -308,8 +380,88 @@ static enum scx_test_status run(void *ctx_ptr) finite_ticks)); goto out; } + stop_worker(finite_worker); + finite_worker = -1; + stop_worker(inf_worker); + inf_worker = -1; + if (!ctx->test_lazy) + goto check_exit; + + /* + * A lazy local enqueue must restart the tick after clearing the slice + * of an infinite-slice task on a full-dynticks CPU. + */ + __atomic_store_n(&skel->bss->phase, NOHZ_PHASE_LAZY_ENQ, __ATOMIC_RELEASE); + victim = scx_test_spawn_gated_worker(ctx->test_cpu, true); + challenger = scx_test_spawn_gated_worker(ctx->test_cpu, true); + if (victim.pid < 0 || challenger.pid < 0) { + SCX_ERR("Failed to spawn lazy-enqueue workers"); + goto out; + } + skel->bss->victim_pid = victim.pid; + skel->bss->challenger_pid = challenger.pid; + if (!scx_test_start_gated_worker(&victim) || + !wait_for_counter(skel, &skel->bss->nr_lazy_victim_running, 1, + PHASE_TIMEOUT_MS)) { + SCX_ERR("Lazy-enqueue victim was not scheduled"); + goto out; + } + if (!wait_for_tick_stop(skel, &skel->bss->nr_lazy_ticks, PHASE_TIMEOUT_MS)) { + SCX_ERR("Tick did not stop before lazy enqueue"); + goto out; + } + + if (!scx_test_start_gated_worker(&challenger) || + !wait_for_counter(skel, &skel->bss->nr_lazy_enq_running, 1, PHASE_TIMEOUT_MS)) { + SCX_ERR("Lazy enqueue made no progress on CPU %d", ctx->test_cpu); + goto out; + } + scx_test_stop_gated_worker(&victim); + scx_test_stop_gated_worker(&challenger); + + /* Repeat with a lazy kick delivered from a housekeeping CPU. */ + __atomic_store_n(&skel->bss->phase, NOHZ_PHASE_LAZY_KICK, __ATOMIC_RELEASE); + victim = scx_test_spawn_gated_worker(ctx->test_cpu, true); + challenger = scx_test_spawn_gated_worker(ctx->test_cpu, true); + trigger = scx_test_spawn_gated_worker(ctx->housekeeping_cpu, true); + if (victim.pid < 0 || challenger.pid < 0 || trigger.pid < 0) { + SCX_ERR("Failed to spawn lazy-kick workers"); + goto out; + } + skel->bss->victim_pid = victim.pid; + skel->bss->challenger_pid = challenger.pid; + skel->bss->trigger_pid = trigger.pid; + victim_running = __atomic_load_n(&skel->bss->nr_lazy_victim_running, + __ATOMIC_RELAXED); + if (!scx_test_start_gated_worker(&victim) || + !wait_for_counter(skel, &skel->bss->nr_lazy_victim_running, victim_running + 1, + PHASE_TIMEOUT_MS)) { + SCX_ERR("Lazy-kick victim was not scheduled"); + goto out; + } + if (!scx_test_start_gated_worker(&challenger)) { + SCX_ERR("Failed to start lazy-kick challenger"); + goto out; + } + if (!wait_for_tick_stop(skel, &skel->bss->nr_lazy_ticks, PHASE_TIMEOUT_MS)) { + SCX_ERR("Tick did not stop before lazy kick"); + goto out; + } + if (__atomic_load_n(&skel->bss->nr_lazy_kick_running, __ATOMIC_RELAXED)) { + SCX_ERR("Lazy-kick challenger ran before the kick"); + goto out; + } + if (!scx_test_start_gated_worker(&trigger) || + !wait_for_counter(skel, &skel->bss->nr_lazy_kick_running, 1, PHASE_TIMEOUT_MS)) { + SCX_ERR("Lazy kick made no progress on CPU %d", ctx->test_cpu); + goto out; + } + scx_test_stop_gated_worker(&trigger); + scx_test_stop_gated_worker(&victim); + scx_test_stop_gated_worker(&challenger); - if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) { +check_exit: + if (UEI_EXITED(skel, uei)) { SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)", (unsigned long long)skel->data->uei.kind, (long long)skel->data->uei.exit_code); @@ -321,6 +473,9 @@ static enum scx_test_status run(void *ctx_ptr) (unsigned long long)skel->bss->nr_finite_ticks); status = SCX_TEST_PASS; out: + scx_test_stop_gated_worker(&trigger); + scx_test_stop_gated_worker(&victim); + scx_test_stop_gated_worker(&challenger); stop_worker(finite_worker); stop_worker(inf_worker); if (link) diff --git a/tools/testing/selftests/sched_ext/nohz_tick_test.h b/tools/testing/selftests/sched_ext/nohz_tick_test.h new file mode 100644 index 000000000000..b122d723cd8b --- /dev/null +++ b/tools/testing/selftests/sched_ext/nohz_tick_test.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES */ +#ifndef __NOHZ_TICK_TEST_H__ +#define __NOHZ_TICK_TEST_H__ + +enum nohz_phase { + NOHZ_PHASE_INF, + NOHZ_PHASE_FINITE, + NOHZ_PHASE_LAZY_ENQ, + NOHZ_PHASE_LAZY_KICK, +}; + +#endif /* __NOHZ_TICK_TEST_H__ */ diff --git a/tools/testing/selftests/sched_ext/prog_run.c b/tools/testing/selftests/sched_ext/prog_run.c index 1129ec2aaddc..369c744ac06e 100644 --- a/tools/testing/selftests/sched_ext/prog_run.c +++ b/tools/testing/selftests/sched_ext/prog_run.c @@ -55,8 +55,11 @@ static enum scx_test_status run(void *ctx) } /* Assumes uei.kind is written last */ - while (skel->data->uei.kind == EXIT_KIND(SCX_EXIT_NONE)) - sched_yield(); + if (!SCX_TIMED_WAIT_UEI(skel, uei, 10000)) { + SCX_ERR("Timed out waiting for scheduler to exit"); + status = SCX_TEST_FAIL; + goto out; + } if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_UNREG_BPF)) { SCX_ERR("Unexpected exit kind: %llu", diff --git a/tools/testing/selftests/sched_ext/reload_loop.c b/tools/testing/selftests/sched_ext/reload_loop.c index 49297b83d748..7d8880fa1a03 100644 --- a/tools/testing/selftests/sched_ext/reload_loop.c +++ b/tools/testing/selftests/sched_ext/reload_loop.c @@ -4,6 +4,7 @@ * Copyright (c) 2024 David Vernet <dvernet@meta.com> */ #include <bpf/bpf.h> +#include <errno.h> #include <pthread.h> #include <scx/common.h> #include <sys/wait.h> @@ -11,8 +12,14 @@ #include "maximal.bpf.skel.h" #include "scx_test.h" +enum { + RELOAD_ITERS = 1024, + RELOAD_THREADS = 2, +}; + static struct maximal *skel; -static pthread_t threads[2]; +static pthread_t threads[RELOAD_THREADS]; +static int unexpected_errno; bool force_exit = false; @@ -31,32 +38,58 @@ static enum scx_test_status setup(void **ctx) static void *do_reload_loop(void *arg) { - u32 i; + u32 i, failures = 0; - for (i = 0; i < 1024 && !force_exit; i++) { + for (i = 0; i < RELOAD_ITERS && !force_exit; i++) { struct bpf_link *link; + errno = 0; link = bpf_map__attach_struct_ops(skel->maps.maximal_ops); - if (link) - bpf_link__destroy(link); + if (!link) { + int e = errno; + + if (e != EBUSY) + __atomic_store_n(&unexpected_errno, e, + __ATOMIC_RELAXED); + failures++; + continue; + } + + bpf_link__destroy(link); } - return NULL; + return (void *)(unsigned long)failures; } static enum scx_test_status run(void *ctx) { - int err; + unsigned long failures[RELOAD_THREADS] = {}; + int err, i; void *ret; err = pthread_create(&threads[0], NULL, do_reload_loop, NULL); SCX_FAIL_IF(err, "Failed to create thread 0"); err = pthread_create(&threads[1], NULL, do_reload_loop, NULL); - SCX_FAIL_IF(err, "Failed to create thread 1"); + if (err) { + force_exit = true; + pthread_join(threads[0], NULL); + SCX_FAIL("Failed to create thread 1"); + } SCX_FAIL_IF(pthread_join(threads[0], &ret), "thread 0 failed"); + failures[0] = (unsigned long)ret; + SCX_FAIL_IF(pthread_join(threads[1], &ret), "thread 1 failed"); + failures[1] = (unsigned long)ret; + + SCX_FAIL_IF(unexpected_errno, "unexpected attach errno %d", + unexpected_errno); + + for (i = 0; i < RELOAD_THREADS; i++) + SCX_FAIL_IF(failures[i] == RELOAD_ITERS, + "thread %d: every attach failed (%lu EBUSY)", + i, failures[i]); return SCX_TEST_PASS; } diff --git a/tools/testing/selftests/sched_ext/rt_stall.c b/tools/testing/selftests/sched_ext/rt_stall.c index a5041fc2e44f..a1552b38a6e8 100644 --- a/tools/testing/selftests/sched_ext/rt_stall.c +++ b/tools/testing/selftests/sched_ext/rt_stall.c @@ -17,7 +17,6 @@ #include <scx/common.h> #include "rt_stall.bpf.skel.h" #include "scx_test.h" -#include "../kselftest.h" #define CORE_ID 0 /* CPU to pin tasks to */ #define RUN_TIME 5 /* How long to run the test in seconds */ @@ -35,15 +34,17 @@ static void signal_ready(int fd) } /* Wait for a child to signal readiness via a pipe */ -static void wait_ready(int fd) +static bool wait_ready(int fd) { + bool ready; char c; - if (read(fd, &c, 1) != 1) { + ready = read(fd, &c, 1) == 1; + if (!ready) perror("read from ready pipe"); - exit(EXIT_FAILURE); - } close(fd); + + return ready; } /* Simple busy-wait function for test tasks */ @@ -151,13 +152,11 @@ static bool sched_stress_test(bool is_ext) float ext_runtime, rt_runtime, actual_ratio; int ext_pid, rt_pid; int ext_ready[2], rt_ready[2]; - - ksft_print_header(); - ksft_set_plan(1); + bool ret = false; if (pipe(ext_ready) || pipe(rt_ready)) { perror("pipe"); - ksft_exit_fail(); + return false; } /* Create and set up a EXT task */ @@ -172,7 +171,7 @@ static bool sched_stress_test(bool is_ext) exit(0); } else if (ext_pid < 0) { perror("fork task"); - ksft_exit_fail(); + return false; } /* Create an RT task */ @@ -188,7 +187,7 @@ static bool sched_stress_test(bool is_ext) exit(0); } else if (rt_pid < 0) { perror("fork for RT task"); - ksft_exit_fail(); + goto out_kill_ext; } /* @@ -199,45 +198,47 @@ static bool sched_stress_test(bool is_ext) */ close(ext_ready[1]); close(rt_ready[1]); - wait_ready(ext_ready[0]); - wait_ready(rt_ready[0]); + if (!wait_ready(ext_ready[0]) || !wait_ready(rt_ready[0])) + goto out_kill; /* Let the processes run for the specified time */ sleep(RUN_TIME); /* Get runtime for the EXT task */ ext_runtime = get_process_runtime(ext_pid); - if (ext_runtime == -1) - ksft_exit_fail_msg("Error getting runtime for %s task (PID %d)\n", - class_str, ext_pid); - ksft_print_msg("Runtime of %s task (PID %d) is %f seconds\n", - class_str, ext_pid, ext_runtime); + if (ext_runtime == -1) { + fprintf(stderr, "Failed to read %s task runtime\n", class_str); + goto out_kill; + } /* Get runtime for the RT task */ rt_runtime = get_process_runtime(rt_pid); - if (rt_runtime == -1) - ksft_exit_fail_msg("Error getting runtime for RT task (PID %d)\n", rt_pid); - ksft_print_msg("Runtime of RT task (PID %d) is %f seconds\n", rt_pid, rt_runtime); - - /* Kill the processes */ - kill(ext_pid, SIGKILL); - kill(rt_pid, SIGKILL); - waitpid(ext_pid, NULL, 0); - waitpid(rt_pid, NULL, 0); + if (rt_runtime == -1) { + fprintf(stderr, "Failed to read RT task runtime\n"); + goto out_kill; + } /* Verify that the scx task got enough runtime */ actual_ratio = ext_runtime / (ext_runtime + rt_runtime); - ksft_print_msg("%s task got %.2f%% of total runtime\n", - class_str, actual_ratio * 100); + fprintf(stderr, "%s task ran %.3fs, RT task ran %.3fs (%.2f%% of runtime)\n", + class_str, ext_runtime, rt_runtime, actual_ratio * 100); - if (actual_ratio >= expected_min_ratio) { - ksft_test_result_pass("PASS: %s task got more than %.2f%% of runtime\n", - class_str, expected_min_ratio * 100); - return true; + if (actual_ratio < expected_min_ratio) { + fprintf(stderr, "%s task got less than %.2f%% of runtime\n", + class_str, expected_min_ratio * 100); + goto out_kill; } - ksft_test_result_fail("FAIL: %s task got less than %.2f%% of runtime\n", - class_str, expected_min_ratio * 100); - return false; + + ret = true; + +out_kill: + kill(rt_pid, SIGKILL); + waitpid(rt_pid, NULL, 0); +out_kill_ext: + kill(ext_pid, SIGKILL); + waitpid(ext_pid, NULL, 0); + + return ret; } static enum scx_test_status run(void *ctx) @@ -263,14 +264,17 @@ static enum scx_test_status run(void *ctx) link = bpf_map__attach_struct_ops(skel->maps.rt_stall_ops); SCX_FAIL_IF(!link, "Failed to attach scheduler"); } + res = sched_stress_test(is_ext); + if (is_ext) { - SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_NONE)); + int exit_kind = skel->data->uei.kind; bpf_link__destroy(link); + SCX_EQ(exit_kind, EXIT_KIND(SCX_EXIT_NONE)); } if (!res) - ksft_exit_fail(); + return SCX_TEST_FAIL; } return SCX_TEST_PASS; diff --git a/tools/testing/selftests/sched_ext/runner.c b/tools/testing/selftests/sched_ext/runner.c index c264807caa91..57dda75384f1 100644 --- a/tools/testing/selftests/sched_ext/runner.c +++ b/tools/testing/selftests/sched_ext/runner.c @@ -243,7 +243,7 @@ int main(int argc, char **argv) printf(" - %s\n", failed_tests[i]); } - return failed > 0 ? 1 : 0; + return failed > 0 || exit_req ? 1 : 0; } void scx_test_register(struct scx_test *test) diff --git a/tools/testing/selftests/sched_ext/scx_test.h b/tools/testing/selftests/sched_ext/scx_test.h index 90b8d6915bb7..8490b01fff74 100644 --- a/tools/testing/selftests/sched_ext/scx_test.h +++ b/tools/testing/selftests/sched_ext/scx_test.h @@ -18,8 +18,6 @@ enum scx_test_status { SCX_TEST_FAIL, }; -#define EXIT_KIND(__ent) __COMPAT_ENUM_OR_ZERO("scx_exit_kind", #__ent) - struct scx_test { /** * name - The name of the testcase. @@ -110,6 +108,41 @@ void scx_test_register(struct scx_test *test); #define SCX_ASSERT(_x) SCX_FAIL_IF(!(_x), "Expected %s to be true (%lu)", \ #_x, (u64)(_x)) +/* + * SCX_ASSERT_ALIVE - Fail the test if the scheduler under test exited + * on its own between attach and this check, e.g. from a runtime error. + * A scheduler that dies mid-test freezes whatever state the test is + * about to assert on, which can make the assertions pass vacuously. + * uei.msg carries the actual error text, so print it alongside the + * recorded kind and exit code. + */ +#define SCX_ASSERT_ALIVE(__skel, __uei) \ + do { \ + if (UEI_EXITED(__skel, __uei)) \ + SCX_FAIL("Scheduler exited unexpectedly" \ + " (kind=%llu code=%lld msg='%s')", \ + (unsigned long long)(__skel)->data->__uei.kind, \ + (long long)(__skel)->data->__uei.exit_code, \ + (__skel)->data->__uei.msg); \ + } while (0) + +/* + * SCX_TIMED_WAIT_UEI - Wait for the scheduler under test to exit, i.e. + * its uei to be filled in, for up to @__ms iterations at ~1ms each. + * Returns true if the scheduler exited and false on timeout, so a + * scheduler that never exits fails the test instead of hanging it. + * The bound is iteration-based rather than wall-clock and stretches + * on a loaded system instead of failing early. + */ +#define SCX_TIMED_WAIT_UEI(__skel, __uei, __ms) \ + ({ \ + s64 __left = (__ms); \ + \ + while (__left-- > 0 && !UEI_EXITED(__skel, __uei)) \ + usleep(1000); \ + UEI_EXITED(__skel, __uei) != 0; \ + }) + #define SCX_ECODE_VAL(__ecode) ({ \ u64 __val = 0; \ bool __found = false; \ @@ -119,13 +152,25 @@ void scx_test_register(struct scx_test *test); (s64)__val; \ }) +/* + * SCX_KIND_VAL - Resolve an scx_exit_kind enumerator from kernel BTF. + * + * Unlike __COMPAT_ENUM_OR_ZERO(), a missing enumerator fails the test + * loudly instead of silently resolving to 0 (== SCX_EXIT_NONE), which + * would make the comparison vacuous. + */ #define SCX_KIND_VAL(__kind) ({ \ - u64 __val = 0; \ - bool __found = false; \ + u64 __val = 0; \ \ - __found = __COMPAT_read_enum("scx_exit_kind", #__kind, &__val); \ - SCX_ASSERT(__found); \ + if (!__COMPAT_read_enum("scx_exit_kind", #__kind, &__val)) \ + SCX_FAIL("scx_exit_kind.%s not found in kernel BTF", #__kind); \ __val; \ }) +/* + * EXIT_KIND - Alias of SCX_KIND_VAL() for resolving scx_exit_kind + * enumerator values, kept for readability at the EXIT_KIND() call sites. + */ +#define EXIT_KIND(__ent) SCX_KIND_VAL(__ent) + #endif // # __SCX_TEST_H__ diff --git a/tools/testing/selftests/sched_ext/test_modules/Makefile b/tools/testing/selftests/sched_ext/test_modules/Makefile new file mode 100644 index 000000000000..a0e9e9401ead --- /dev/null +++ b/tools/testing/selftests/sched_ext/test_modules/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + +TESTMODS_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))) +KDIR ?= $(if $(O),$(O),$(realpath ../../../../..)) + +obj-m += scx_enq_blocked_test.o + +all: + +$(Q)$(MAKE) -C $(KDIR) M=$(TESTMODS_DIR) modules + +clean: + +$(Q)$(MAKE) -C $(KDIR) M=$(TESTMODS_DIR) clean diff --git a/tools/testing/selftests/sched_ext/test_modules/scx_enq_blocked_test.c b/tools/testing/selftests/sched_ext/test_modules/scx_enq_blocked_test.c new file mode 100644 index 000000000000..908689ed5578 --- /dev/null +++ b/tools/testing/selftests/sched_ext/test_modules/scx_enq_blocked_test.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + * + * Kernel mutex fixture for the sched_ext SCX_OPS_ENQ_BLOCKED selftest. + */ + +#include <linux/atomic.h> +#include <linux/fs.h> +#include <linux/jiffies.h> +#include <linux/ktime.h> +#include <linux/miscdevice.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/sched.h> +#include <linux/uaccess.h> + +#include "../enq_blocked.h" + +#define DONOR_WAIT_TIMEOUT msecs_to_jiffies(2000) +#define ATTACH_WAIT_TIMEOUT msecs_to_jiffies(10000) +#define MUTEX_HOLD_TIME msecs_to_jiffies(200) + +static DEFINE_MUTEX(test_mutex); +static DEFINE_SPINLOCK(donor_lock); +static struct task_struct *donor_task; +static atomic_t owner_ready = ATOMIC_INIT(0); +static atomic_t donor_started = ATOMIC_INIT(0); +static atomic_t attach_pending = ATOMIC_INIT(0); +static atomic_t attach_done = ATOMIC_INIT(0); +static atomic64_t hold_time_ns = ATOMIC64_INIT(0); +static atomic64_t wait_time_ns = ATOMIC64_INIT(0); +static atomic64_t nr_holds = ATOMIC64_INIT(0); +static atomic64_t nr_waits = ATOMIC64_INIT(0); + +static long run_owner(void) +{ + unsigned long timeout; + u64 start_ns; + long ret = 0; + + atomic_set(&donor_started, 0); + mutex_lock(&test_mutex); + start_ns = ktime_get_ns(); + atomic_set(&owner_ready, 1); + + timeout = jiffies + DONOR_WAIT_TIMEOUT; + while (!atomic_read(&donor_started)) { + if (time_after(jiffies, timeout)) { + ret = -ETIMEDOUT; + goto out; + } + cond_resched(); + } + if (atomic_xchg(&attach_pending, 0)) { + timeout = jiffies + ATTACH_WAIT_TIMEOUT; + while (!atomic_read(&attach_done)) { + if (time_after(jiffies, timeout)) { + ret = -ETIMEDOUT; + goto out; + } + cond_resched(); + } + } + + /* Keep yielding while the donor blocks on test_mutex. */ + timeout = jiffies + MUTEX_HOLD_TIME; + while (time_before(jiffies, timeout)) + cond_resched(); + +out: + atomic_set(&owner_ready, 0); + atomic64_add(ktime_get_ns() - start_ns, &hold_time_ns); + atomic64_inc(&nr_holds); + mutex_unlock(&test_mutex); + return ret; +} + +static long run_donor(void) +{ + unsigned long flags; + u64 start_ns; + + if (!atomic_read(&owner_ready)) + return -EAGAIN; + + get_task_struct(current); + spin_lock_irqsave(&donor_lock, flags); + WARN_ON_ONCE(donor_task); + donor_task = current; + spin_unlock_irqrestore(&donor_lock, flags); + + atomic_set(&donor_started, 1); + start_ns = ktime_get_ns(); + mutex_lock(&test_mutex); + + spin_lock_irqsave(&donor_lock, flags); + donor_task = NULL; + spin_unlock_irqrestore(&donor_lock, flags); + put_task_struct(current); + + atomic64_add(ktime_get_ns() - start_ns, &wait_time_ns); + atomic64_inc(&nr_waits); + mutex_unlock(&test_mutex); + return 0; +} + +static long get_donor_state(void) +{ + struct task_struct *task; + unsigned long flags; + long state = 0; + + spin_lock_irqsave(&donor_lock, flags); + task = donor_task; + if (task) + get_task_struct(task); + spin_unlock_irqrestore(&donor_lock, flags); + if (!task) + return -ENOENT; + + if (READ_ONCE(task->__state) != TASK_RUNNING) + state |= ENQ_BLOCKED_DONOR_SLEEPING; + if (READ_ONCE(task->on_rq)) + state |= ENQ_BLOCKED_DONOR_ON_RQ; + put_task_struct(task); + return state; +} + +static void reset_stats(void) +{ + atomic64_set(&hold_time_ns, 0); + atomic64_set(&wait_time_ns, 0); + atomic64_set(&nr_holds, 0); + atomic64_set(&nr_waits, 0); +} + +static long get_stats(unsigned long arg) +{ + struct enq_blocked_stats stats = { + .hold_time_ns = atomic64_read(&hold_time_ns), + .wait_time_ns = atomic64_read(&wait_time_ns), + .nr_holds = atomic64_read(&nr_holds), + .nr_waits = atomic64_read(&nr_waits), + }; + + return copy_to_user((void __user *)arg, &stats, sizeof(stats)) ? + -EFAULT : 0; +} + +static long enq_blocked_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + switch (cmd) { + case ENQ_BLOCKED_IOCTL_OWNER: + return run_owner(); + case ENQ_BLOCKED_IOCTL_DONOR: + return run_donor(); + case ENQ_BLOCKED_IOCTL_RESET_STATS: + reset_stats(); + return 0; + case ENQ_BLOCKED_IOCTL_GET_STATS: + return get_stats(arg); + case ENQ_BLOCKED_IOCTL_PREP_ATTACH: + atomic_set(&attach_done, 0); + atomic_set(&attach_pending, 1); + return 0; + case ENQ_BLOCKED_IOCTL_ATTACH_DONE: + atomic_set(&attach_done, 1); + return 0; + case ENQ_BLOCKED_IOCTL_DONOR_STATE: + return get_donor_state(); + case ENQ_BLOCKED_IOCTL_PROXY_SUPPORTED: + return IS_ENABLED(CONFIG_SCHED_PROXY_EXEC); + default: + return -EINVAL; + } +} + +static const struct file_operations enq_blocked_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = enq_blocked_ioctl, +}; + +static struct miscdevice enq_blocked_device = { + .minor = MISC_DYNAMIC_MINOR, + .name = "scx_enq_blocked", + .fops = &enq_blocked_fops, + .mode = 0600, +}; + +module_misc_device(enq_blocked_device); +MODULE_AUTHOR("Andrea Righi <arighi@nvidia.com>"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("sched_ext blocked donor test module"); diff --git a/tools/testing/selftests/sched_ext/util.c b/tools/testing/selftests/sched_ext/util.c index 2111329ed289..c93fc5df552d 100644 --- a/tools/testing/selftests/sched_ext/util.c +++ b/tools/testing/selftests/sched_ext/util.c @@ -5,10 +5,21 @@ */ #include <errno.h> #include <fcntl.h> +#include <sched.h> +#include <signal.h> #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <sys/prctl.h> +#include <sys/wait.h> #include <unistd.h> +#include <zlib.h> + +#include "util.h" + +#ifndef SCHED_EXT +#define SCHED_EXT 7 +#endif /* Returns read len on success, or -errno on failure. */ static ssize_t read_text(const char *path, char *buf, size_t max_len) @@ -69,3 +80,135 @@ int file_write_long(const char *path, long val) return 0; } + +struct scx_test_gated_worker scx_test_spawn_gated_worker(int cpu, bool set_sched_ext) +{ + struct scx_test_gated_worker worker = { .pid = -1, .start_fd = -1 }; + int ready[2], start[2]; + pid_t parent = getpid(); + char byte = 1; + + if (pipe(ready)) + return worker; + if (pipe(start)) { + close(ready[0]); + close(ready[1]); + return worker; + } + + worker.pid = fork(); + if (!worker.pid) { + struct sched_param param = {}; + cpu_set_t mask; + + close(ready[0]); + close(start[1]); + if (prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() != parent) + _exit(1); + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); + if (sched_setaffinity(0, sizeof(mask), &mask)) + _exit(1); + if (set_sched_ext && sched_setscheduler(0, SCHED_EXT, ¶m)) + _exit(1); + if (write(ready[1], &byte, 1) != 1) + _exit(1); + close(ready[1]); + if (read(start[0], &byte, 1) != 1) + _exit(1); + close(start[0]); + for (;;) + asm volatile("" ::: "memory"); + } + if (worker.pid < 0) { + close(ready[0]); + close(ready[1]); + close(start[0]); + close(start[1]); + return worker; + } + + close(ready[1]); + close(start[0]); + if (read(ready[0], &byte, 1) != 1) { + close(ready[0]); + close(start[1]); + kill(worker.pid, SIGKILL); + waitpid(worker.pid, NULL, 0); + worker.pid = -1; + return worker; + } + close(ready[0]); + worker.start_fd = start[1]; + return worker; +} + +bool scx_test_start_gated_worker(struct scx_test_gated_worker *worker) +{ + char byte = 1; + + if (write(worker->start_fd, &byte, 1) != 1) + return false; + close(worker->start_fd); + worker->start_fd = -1; + return true; +} + +void scx_test_stop_gated_worker(struct scx_test_gated_worker *worker) +{ + if (worker->start_fd >= 0) + close(worker->start_fd); + if (worker->pid > 0) { + kill(worker->pid, SIGKILL); + waitpid(worker->pid, NULL, 0); + } + worker->pid = -1; + worker->start_fd = -1; +} + +static int config_preempt_lazy_mode(void) +{ + bool dynamic = false, lazy = false, immediate = false; + char buf[128]; + gzFile file; + + file = gzopen("/proc/config.gz", "r"); + if (!file) + return -1; + + while (gzgets(file, buf, sizeof(buf))) { + if (!strcmp(buf, "CONFIG_PREEMPT_DYNAMIC=y\n")) + dynamic = true; + else if (!strcmp(buf, "CONFIG_PREEMPT_LAZY=y\n")) + lazy = true; + else if (!strcmp(buf, "CONFIG_PREEMPT_NONE=y\n") || + !strcmp(buf, "CONFIG_PREEMPT_VOLUNTARY=y\n") || + !strcmp(buf, "CONFIG_PREEMPT=y\n") || + !strcmp(buf, "CONFIG_PREEMPT_RT=y\n")) + immediate = true; + } + gzclose(file); + + if (dynamic) + return -1; + if (lazy) + return 1; + if (immediate) + return 0; + return -1; +} + +int scx_test_preempt_lazy_mode(void) +{ + char buf[128]; + + if (read_text("/sys/kernel/debug/sched/preempt", buf, sizeof(buf)) > 0) { + if (strstr(buf, "(lazy)")) + return 1; + if (strstr(buf, "(none)") || strstr(buf, "(voluntary)") || + strstr(buf, "(full)")) + return 0; + } + + return config_preempt_lazy_mode(); +} diff --git a/tools/testing/selftests/sched_ext/util.h b/tools/testing/selftests/sched_ext/util.h index 681cec04b439..aac34fc4ef71 100644 --- a/tools/testing/selftests/sched_ext/util.h +++ b/tools/testing/selftests/sched_ext/util.h @@ -7,7 +7,19 @@ #ifndef __SCX_TEST_UTIL_H__ #define __SCX_TEST_UTIL_H__ +#include <stdbool.h> +#include <sys/types.h> + +struct scx_test_gated_worker { + pid_t pid; + int start_fd; +}; + long file_read_long(const char *path); int file_write_long(const char *path, long val); +struct scx_test_gated_worker scx_test_spawn_gated_worker(int cpu, bool set_sched_ext); +bool scx_test_start_gated_worker(struct scx_test_gated_worker *worker); +void scx_test_stop_gated_worker(struct scx_test_gated_worker *worker); +int scx_test_preempt_lazy_mode(void); #endif // __SCX_TEST_UTIL_H__ |