summaryrefslogtreecommitdiff
path: root/include
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2026-09-27 08:24:16 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2026-09-27 08:24:16 -0700
commit673dab7eac1618b6622bbee9eb2aaa47817631e3 (patch)
treed0e914a549a5b6faa0b6ca1b11ec7a2cc2b6c952 /include
parent5ccda18d1ba2ecf436102a211baf1fbd5d81703f (diff)
parenta0bb6fac53fa7cf1cadb487b43d4c9276a6b82e3 (diff)
Merge tag 'sched-urgent-2026-09-27' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull scheduler fixes from Ingo Molnar: - Fix LLC mis-scheduling bugs (Tim Chen, Lu Wang) - Fix cache-grouping related scheduling statistics UAF bugs (Tim Chen) - Skip kernel threads for cache aware scheduling to rubustify the code (Chen Yu) - Refresh LLC capacity across CPU hotplug, to fix capacity underestimation bug (Davi Chaves Azevedo) - Account PSI IRQ time to the execution context, not the scheduling context, to fix proxy scheduling accounting bug (Zhan Xusheng) * tag 'sched-urgent-2026-09-27' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: sched/core: Account PSI IRQ time to the execution context, not the scheduling context sched/cache: Refresh LLC capacity across CPU hotplug, to fix capacity underestimation bug sched/cache: Skip kernel threads for cache aware scheduling to rubustify the code sched/cache: Introduce task_struct->sched_cache_grp to fix UAF sched/cache: Decouple sched_cache_group from mm to fix UAF sched/cache: Honor migrate_llc_task semantics in active load balance, to fix LLC mis-scheduling bug sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling bug
Diffstat (limited to 'include')
-rw-r--r--include/linux/mm_types.h15
-rw-r--r--include/linux/sched.h20
-rw-r--r--include/linux/sched/topology.h4
3 files changed, 25 insertions, 14 deletions
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 6d815f6440c9..f3e5a2fadbe5 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1226,7 +1226,7 @@ struct mm_struct {
struct mm_mm_cid mm_cid;
/* sched_cache related statistics */
- struct sched_cache_stat sc_stat;
+ struct sched_cache_group *sched_cache_grp;
#ifdef CONFIG_MMU
atomic_long_t pgtables_bytes; /* size of all page tables */
#endif
@@ -1624,8 +1624,9 @@ static inline unsigned int mm_cid_size(void)
#endif /* CONFIG_SCHED_MM_CID */
#ifdef CONFIG_SCHED_CACHE
-void mm_init_sched(struct mm_struct *mm,
- struct sched_cache_time __percpu *pcpu_sched);
+int mm_init_sched(struct mm_struct *mm,
+ struct sched_cache_time __percpu *pcpu_sched);
+void mm_destroy_sched(struct mm_struct *mm);
static inline int mm_alloc_sched_noprof(struct mm_struct *mm)
{
@@ -1635,17 +1636,11 @@ static inline int mm_alloc_sched_noprof(struct mm_struct *mm)
if (!pcpu_sched)
return -ENOMEM;
- mm_init_sched(mm, pcpu_sched);
- return 0;
+ return mm_init_sched(mm, pcpu_sched);
}
#define mm_alloc_sched(...) alloc_hooks(mm_alloc_sched_noprof(__VA_ARGS__))
-static inline void mm_destroy_sched(struct mm_struct *mm)
-{
- free_percpu(mm->sc_stat.pcpu_sched);
- mm->sc_stat.pcpu_sched = NULL;
-}
#else /* !CONFIG_SCHED_CACHE */
static inline int mm_alloc_sched(struct mm_struct *mm) { return 0; }
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 705970d07614..d35ae49a991f 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1433,6 +1433,7 @@ struct task_struct {
#ifdef CONFIG_SCHED_CACHE
struct callback_head cache_work;
+ struct sched_cache_group __rcu *sched_cache_grp;
int preferred_llc;
/* 1: task was enqueued to its preferred LLC, 0 otherwise */
int pref_llc_queued;
@@ -2405,7 +2406,7 @@ struct sched_cache_time {
unsigned long epoch;
};
-struct sched_cache_stat {
+struct sched_cache_group {
struct sched_cache_time __percpu *pcpu_sched;
raw_spinlock_t lock;
unsigned long epoch;
@@ -2413,11 +2414,26 @@ struct sched_cache_stat {
unsigned long next_scan;
unsigned long footprint;
int cpu;
+ refcount_t refcnt;
+ struct rcu_head rcu;
} ____cacheline_aligned_in_smp;
+struct sched_cache_group *sched_cache_group_get(struct sched_cache_group *grp);
+struct sched_cache_group *task_cache_group_get(struct task_struct *p);
+
+void sched_cache_fork(struct task_struct *p);
+void sched_cache_fork_cleanup(struct task_struct *p);
+void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm);
+void sched_cache_exit_mm(struct task_struct *p);
+
#else
-struct sched_cache_stat { };
+struct sched_cache_group { };
+
+static inline void sched_cache_fork(struct task_struct *p) { }
+static inline void sched_cache_fork_cleanup(struct task_struct *p) { }
+static inline void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm) { }
+static inline void sched_cache_exit_mm(struct task_struct *p) { }
#endif
diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h
index b5d9d7c2b8ad..f96812d71c51 100644
--- a/include/linux/sched/topology.h
+++ b/include/linux/sched/topology.h
@@ -281,9 +281,9 @@ static inline int task_node(const struct task_struct *p)
}
#ifdef CONFIG_SCHED_CACHE
-extern void sched_update_llc_bytes(unsigned int cpu);
+extern void sched_update_llc_bytes(const struct cpumask *cpus);
#else
-static inline void sched_update_llc_bytes(unsigned int cpu) { }
+static inline void sched_update_llc_bytes(const struct cpumask *cpus) { }
#endif
#endif /* _LINUX_SCHED_TOPOLOGY_H */