Merge tag 'sched_ext-for-7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext

Pull sched_ext updates from Tejun Heo:

 - Proxy execution can now be enabled together with sched_ext. A BPF
   scheduler can opt in to receive mutex-blocked tasks that donate their
   scheduling context to the lock owner and place them itself.

 - Lazy preemption: a BPF scheduler can preempt or kick a CPU lazily,
   deferring the switch to the next return to user space or tick, as the
   fair class can.

 - Sub-schedulers: a parent now learns which child runs on each CPU it
   delegated and when a cap change takes effect. A detaching child loses
   its caps before the parent restores the CPUs.

 - New kfuncs re-enqueue single tasks from a dispatch queue iteration
   and report a cgroup's CPU count and a CID's NUMA node. The CID
   topology now includes clusters.

 - Hot path optimizations: dispatch queue and task lookups compare keys
   inline, remote task moves no longer take a dispatch queue lock twice,
   and sub-scheduler hooks cost nothing without sub-schedulers.

 - Kfuncs callable from tracing programs are hardened against NMI and
   IRQ nesting, which could hard-lock the CPU or pick a wrong idle CPU.

 - Built-in dispatch queues no longer allocate per-CPU state that grew
   with the square of the CPU count.

 - Header syncs with the scx repo, a BTF workaround for pahole 1.32, and
   selftest fixes and coverage.

* tag 'sched_ext-for-7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext: (69 commits)
  sched_ext: Drop the locks periodically while walking a user DSQ for re-enqueue
  sched_ext: scx_qmap: Also re-enqueue stranded tasks with scx_bpf_dsq_reenq_task()
  sched_ext: Add scx_bpf_dsq_reenq_task() to re-enqueue single tasks from a DSQ iteration
  sched_ext: Factor out the rq lock switch of scx_dsq_move()
  sched_ext: Don't queue balance callbacks from a core-sched pick
  sched_ext: Don't leave balance callbacks queued across a pick retry
  sched_ext: Fix a bypassed task stalling on a cpu its scheduler has no caps on
  sched_ext: Fix the running task keeping its protection into bypass
  sched_ext: Fix use-after-free of a local reenq node left on an inactive cpu
  sched_ext: Keep proxy donors with slice left on the local DSQ
  selftests/sched_ext: Report reload_loop attach failures
  selftests/sched_ext: Bound exit waits
  selftests/sched_ext: Add SCX_ASSERT_ALIVE and adopt it
  selftests/sched_ext: Fail loudly when EXIT_KIND is missing from BTF
  sched_ext: Add SCX_ECODE_RSN_CGROUP_OFFLINE to user_exit_info.h
  selftests/sched_ext: Fix reload_loop error-path use-after-free
  selftests/sched_ext: Fix build error in dequeue_remote selftest
  sched_ext: scx_qmap: Reset a child's cpuperf targets once its PERF revoke takes effect
  sched_ext: Add ops.sub_child_ecaps_updated() to report a child's effective cap changes
  sched_ext: scx_qmap: Show actual cid use per participant
  ...

1 files changed, 26 insertions, 13 deletions

diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h
index 36c04797436c..efd635a99309 100644
--- a/include/linux/sched/ext.h
+++ b/include/linux/sched/ext.h
@@ -91,7 +91,7 @@ struct scx_dispatch_q {
 	struct rhash_head	hash_node;
 	struct llist_node	free_node;
 	struct scx_sched	*sched;
-	struct scx_dsq_pcpu __percpu *pcpu;
+	struct scx_dsq_pcpu __percpu *pcpu_user;
 	struct rcu_head		rcu;
 };
 
@@ -104,6 +104,7 @@ enum scx_ent_flags {
 	SCX_TASK_SUB_INIT	= 1 << 4, /* task being initialized for a sub sched */
 	SCX_TASK_IMMED		= 1 << 5, /* task is on local DSQ with %SCX_ENQ_IMMED */
 	SCX_TASK_PROTECTED	= 1 << 6, /* slice and DSQ head position protected */
+	SCX_TASK_RUN_TRACKED	= 1 << 7, /* task is in an ops.running()/stopping() session */
 
 	/*
 	 * Bits 8 to 10 are used to carry task state:
@@ -136,6 +137,7 @@ enum scx_ent_flags {
 	 * IMMED	reenqueued due to failed ENQ_IMMED
 	 * PREEMPTED	preempted while running
 	 * CAP		sub-sched cap miss, see p->scx.reenq_reason_*
+	 * PROXY	proxy state prevented a remote DSQ transfer
 	 */
 	SCX_TASK_REENQ_REASON_SHIFT = 12,
 	SCX_TASK_REENQ_REASON_BITS = 3,
@@ -146,6 +148,7 @@ enum scx_ent_flags {
 	SCX_TASK_REENQ_IMMED	= 2 << SCX_TASK_REENQ_REASON_SHIFT,
 	SCX_TASK_REENQ_PREEMPTED = 3 << SCX_TASK_REENQ_REASON_SHIFT,
 	SCX_TASK_REENQ_CAP	= 4 << SCX_TASK_REENQ_REASON_SHIFT,
+	SCX_TASK_REENQ_PROXY	= 5 << SCX_TASK_REENQ_REASON_SHIFT,
 
 	/* iteration cursor, not a task */
 	SCX_TASK_CURSOR		= 1 << 31,
@@ -154,6 +157,7 @@ enum scx_ent_flags {
 /* scx_entity.dsq_flags */
 enum scx_ent_dsq_flags {
 	SCX_TASK_DSQ_ON_PRIQ	= 1 << 0, /* task is queued on the priority queue of a dsq */
+	SCX_TASK_DSQ_REENQ	= 1 << 1, /* marked by scx_bpf_dsq_reenq_task() */
 };
 
 enum scx_dsq_lnode_flags {
@@ -197,9 +201,19 @@ struct sched_ext_entity {
 	u64			ddsp_slice;
 	u64			ddsp_vtime;
 	struct scx_dsq_list_node dsq_list;	/* dispatch order */
-	struct rb_node		dsq_priq;	/* p->scx.dsq_vtime order */
 	u32			dsq_seq;
 	u32			dsq_flags;	/* protected by DSQ lock */
+
+	/*
+	 * Used to order tasks when dispatching to the vtime-ordered priority
+	 * queue of a dsq. This is usually set through
+	 * scx_bpf_dsq_insert_vtime() but can also be modified directly by the
+	 * BPF scheduler. Modifying it while a task is queued on a dsq may
+	 * mangle the ordering and is not recommended. Kept next to @dsq_priq
+	 * as rbtree insertion reads both on every visited node.
+	 */
+	u64			dsq_vtime;
+	struct rb_node		dsq_priq;	/* p->scx.dsq_vtime order */
 	u32			flags;		/* protected by rq lock */
 	u32			weight;
 	u32			reenq_cnt;	/* reenqueues since last run */
@@ -226,7 +240,7 @@ struct sched_ext_entity {
 	u64			tid;
 	struct rhash_head	tid_hash_node;	/* see SCX_OPS_TID_TO_TASK */
 
-	/* BPF scheduler modifiable fields */
+	/* BPF scheduler modifiable fields, along with @dsq_vtime above */
 
 	/*
 	 * Runtime budget in nsecs - how long the task may hold its cpu. Owned
@@ -242,15 +256,6 @@ struct sched_ext_entity {
 	u64			slice;
 
 	/*
-	 * Used to order tasks when dispatching to the vtime-ordered priority
-	 * queue of a dsq. This is usually set through
-	 * scx_bpf_dsq_insert_vtime() but can also be modified directly by the
-	 * BPF scheduler. Modifying it while a task is queued on a dsq may
-	 * mangle the ordering and is not recommended.
-	 */
-	u64			dsq_vtime;
-
-	/*
 	 * Out-of-band slice request from scx_bpf_task_set_slice() when the
 	 * caller does not hold the rq lock, applied under the rq lock at the
 	 * next slice consideration. One atomic64 packs the pending flag, the
@@ -279,6 +284,14 @@ struct sched_ext_entity {
 	 */
 	bool			disallow;	/* reject switching into SCX */
 
+	/*
+	 * If set, depletion of this task's slice at the scheduler tick requests
+	 * lazy instead of immediate rescheduling. Initialized from
+	 * %SCX_OPS_LAZY_RESCHED immediately before ops.enable() and may be
+	 * modified afterwards with scx_bpf_task_set_lazy_resched().
+	 */
+	bool			lazy_resched;
+
 	/* cold fields */
 #ifdef CONFIG_EXT_GROUP_SCHED
 	struct cgroup		*cgrp_moving_from;
@@ -324,7 +337,7 @@ struct scx_task_group {
 	u64			bw_period_us;
 	u64			bw_quota_us;
 	u64			bw_burst_us;
-	bool			idle;
+	bool			sched_idle;
 #endif
 };