From de715325cc47b5a699019b6071620be8da5df146 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Mon, 25 Aug 2025 18:01:58 +0200 Subject: cpu: Revert "cpu/hotplug: Prevent self deadlock on CPU hot-unplug" 1) The commit: 2b8272ff4a70 ("cpu/hotplug: Prevent self deadlock on CPU hot-unplug") was added to fix an issue where the hotplug control task (BP) was throttled between CPUHP_AP_IDLE_DEAD and CPUHP_HRTIMERS_PREPARE waiting in the hrtimer blindspot for the bandwidth callback queued in the dead CPU. 2) Later on, the commit: 38685e2a0476 ("cpu/hotplug: Don't offline the last non-isolated CPU") plugged on the target selection for the workqueue offloaded CPU down process to prevent from destroying the last CPU domain. 3) Finally: 5c0930ccaad5 ("hrtimers: Push pending hrtimers away from outgoing CPU earlier") removed entirely the conditions for the race exposed and partially fixed in 1). The offloading of the CPU down process to a workqueue on another CPU then becomes unnecessary. But the last CPU belonging to scheduler domains must still remain online. Therefore revert the now obsolete commit 2b8272ff4a70b866106ae13c36be7ecbef5d5da2 and move the housekeeping check under the cpu_hotplug_lock write held. Since HK_TYPE_DOMAIN will include both isolcpus and cpuset isolated partition, the hotplug lock will synchronize against concurrent cpuset partition updates. Signed-off-by: Frederic Weisbecker Cc: Marco Crivellari Cc: Michal Hocko Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Waiman Long --- kernel/cpu.c | 37 +++++++++++-------------------------- 1 file changed, 11 insertions(+), 26 deletions(-) (limited to 'kernel') diff --git a/kernel/cpu.c b/kernel/cpu.c index 8df2d773fe3b..40b8496f47c5 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -1410,6 +1410,16 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen, cpus_write_lock(); + /* + * Keep at least one housekeeping cpu onlined to avoid generating + * an empty sched_domain span. + */ + if (cpumask_any_and(cpu_online_mask, + housekeeping_cpumask(HK_TYPE_DOMAIN)) >= nr_cpu_ids) { + ret = -EBUSY; + goto out; + } + cpuhp_tasks_frozen = tasks_frozen; prev_state = cpuhp_set_state(cpu, st, target); @@ -1456,22 +1466,8 @@ out: return ret; } -struct cpu_down_work { - unsigned int cpu; - enum cpuhp_state target; -}; - -static long __cpu_down_maps_locked(void *arg) -{ - struct cpu_down_work *work = arg; - - return _cpu_down(work->cpu, 0, work->target); -} - static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target) { - struct cpu_down_work work = { .cpu = cpu, .target = target, }; - /* * If the platform does not support hotplug, report it explicitly to * differentiate it from a transient offlining failure. @@ -1480,18 +1476,7 @@ static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target) return -EOPNOTSUPP; if (cpu_hotplug_disabled) return -EBUSY; - - /* - * Ensure that the control task does not run on the to be offlined - * CPU to prevent a deadlock against cfs_b->period_timer. - * Also keep at least one housekeeping cpu onlined to avoid generating - * an empty sched_domain span. - */ - for_each_cpu_and(cpu, cpu_online_mask, housekeeping_cpumask(HK_TYPE_DOMAIN)) { - if (cpu != work.cpu) - return work_on_cpu(cpu, __cpu_down_maps_locked, &work); - } - return -EBUSY; + return _cpu_down(cpu, 0, target); } static int cpu_down(unsigned int cpu, enum cpuhp_state target) -- cgit v1.2.3 From 4fca0e550d506e1c95504c2d9247bc92bf621bf6 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Mon, 26 May 2025 13:06:21 +0200 Subject: sched/isolation: Save boot defined domain flags HK_TYPE_DOMAIN will soon integrate not only boot defined isolcpus= CPUs but also cpuset isolated partitions. Housekeeping still needs a way to record what was initially passed to isolcpus= in order to keep these CPUs isolated after a cpuset isolated partition is modified or destroyed while containing some of them. Create a new HK_TYPE_DOMAIN_BOOT to keep track of those. Signed-off-by: Frederic Weisbecker Reviewed-by: Phil Auld Reviewed-by: Waiman Long Cc: Ingo Molnar Cc: Marco Crivellari Cc: Michal Hocko Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Vlastimil Babka Cc: Waiman Long --- kernel/sched/isolation.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index 3ad0d6df6a0a..11a623fa6320 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -11,6 +11,7 @@ #include "sched.h" enum hk_flags { + HK_FLAG_DOMAIN_BOOT = BIT(HK_TYPE_DOMAIN_BOOT), HK_FLAG_DOMAIN = BIT(HK_TYPE_DOMAIN), HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ), HK_FLAG_KERNEL_NOISE = BIT(HK_TYPE_KERNEL_NOISE), @@ -239,7 +240,7 @@ static int __init housekeeping_isolcpus_setup(char *str) if (!strncmp(str, "domain,", 7)) { str += 7; - flags |= HK_FLAG_DOMAIN; + flags |= HK_FLAG_DOMAIN | HK_FLAG_DOMAIN_BOOT; continue; } @@ -269,7 +270,7 @@ static int __init housekeeping_isolcpus_setup(char *str) /* Default behaviour for isolcpus without flags */ if (!flags) - flags |= HK_FLAG_DOMAIN; + flags |= HK_FLAG_DOMAIN | HK_FLAG_DOMAIN_BOOT; return housekeeping_setup(str, flags); } -- cgit v1.2.3 From 0f4dfdc17b3c6bf055d5d60cb0c465c09addf04e Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 28 May 2025 14:18:24 +0200 Subject: cpuset: Convert boot_hk_cpus to use HK_TYPE_DOMAIN_BOOT boot_hk_cpus is an ad-hoc copy of HK_TYPE_DOMAIN_BOOT. Remove it and use the official version. Signed-off-by: Frederic Weisbecker Reviewed-by: Phil Auld Reviewed-by: Chen Ridong Reviewed-by: Waiman Long Cc: Johannes Weiner Cc: Marco Crivellari Cc: Michal Hocko Cc: Michal Koutny Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Vlastimil Babka Cc: Waiman Long Cc: cgroups@vger.kernel.org --- kernel/cgroup/cpuset.c | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 6e6eb09b8db6..3afa72f8d579 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -88,12 +88,6 @@ static cpumask_var_t isolated_cpus; */ static bool isolated_cpus_updating; -/* - * Housekeeping (HK_TYPE_DOMAIN) CPUs at boot - */ -static cpumask_var_t boot_hk_cpus; -static bool have_boot_isolcpus; - /* * A flag to force sched domain rebuild at the end of an operation. * It can be set in @@ -1453,15 +1447,16 @@ static bool isolated_cpus_can_update(struct cpumask *add_cpus, * @new_cpus: cpu mask * Return: true if there is conflict, false otherwise * - * CPUs outside of boot_hk_cpus, if defined, can only be used in an + * CPUs outside of HK_TYPE_DOMAIN_BOOT, if defined, can only be used in an * isolated partition. */ static bool prstate_housekeeping_conflict(int prstate, struct cpumask *new_cpus) { - if (!have_boot_isolcpus) + if (!housekeeping_enabled(HK_TYPE_DOMAIN_BOOT)) return false; - if ((prstate != PRS_ISOLATED) && !cpumask_subset(new_cpus, boot_hk_cpus)) + if ((prstate != PRS_ISOLATED) && + !cpumask_subset(new_cpus, housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT))) return true; return false; @@ -3892,12 +3887,9 @@ int __init cpuset_init(void) BUG_ON(!alloc_cpumask_var(&cpus_attach, GFP_KERNEL)); - have_boot_isolcpus = housekeeping_enabled(HK_TYPE_DOMAIN); - if (have_boot_isolcpus) { - BUG_ON(!alloc_cpumask_var(&boot_hk_cpus, GFP_KERNEL)); - cpumask_copy(boot_hk_cpus, housekeeping_cpumask(HK_TYPE_DOMAIN)); - cpumask_andnot(isolated_cpus, cpu_possible_mask, boot_hk_cpus); - } + if (housekeeping_enabled(HK_TYPE_DOMAIN_BOOT)) + cpumask_andnot(isolated_cpus, cpu_possible_mask, + housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT)); return 0; } -- cgit v1.2.3 From b5de34ed87f39fc3f6eb7e7df543317e7efb94a8 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 23 Dec 2025 15:12:46 +0100 Subject: timers/migration: Prevent from lockdep false positive warning Testing housekeeping_cpu() will soon require that either the RCU "lock" is held or the cpuset mutex. When CPUs get isolated through cpuset, the change is propagated to timer migration such that isolation is also performed from the migration tree. However that propagation is done using workqueue which tests if the target is actually isolated before proceeding. Lockdep doesn't know that the workqueue caller holds cpuset mutex and that it waits for the work, making the housekeeping cpumask read safe. Shut down the future warning by removing this test. It is unecessary beyond hotplug, the workqueue is already targeted towards isolated CPUs. Signed-off-by: Frederic Weisbecker Cc: Gabriele Monaco --- kernel/time/timer_migration.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c index 18dda1aa782d..3879575a4975 100644 --- a/kernel/time/timer_migration.c +++ b/kernel/time/timer_migration.c @@ -1497,7 +1497,7 @@ static int tmigr_clear_cpu_available(unsigned int cpu) return 0; } -static int tmigr_set_cpu_available(unsigned int cpu) +static int __tmigr_set_cpu_available(unsigned int cpu) { struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu); @@ -1505,9 +1505,6 @@ static int tmigr_set_cpu_available(unsigned int cpu) if (WARN_ON_ONCE(!tmc->tmgroup)) return -EINVAL; - if (tmigr_is_isolated(cpu)) - return 0; - guard(mutex)(&tmigr_available_mutex); cpumask_set_cpu(cpu, tmigr_available_cpumask); @@ -1523,6 +1520,14 @@ static int tmigr_set_cpu_available(unsigned int cpu) return 0; } +static int tmigr_set_cpu_available(unsigned int cpu) +{ + if (tmigr_is_isolated(cpu)) + return 0; + + return __tmigr_set_cpu_available(cpu); +} + static void tmigr_cpu_isolate(struct work_struct *ignored) { tmigr_clear_cpu_available(smp_processor_id()); @@ -1530,7 +1535,12 @@ static void tmigr_cpu_isolate(struct work_struct *ignored) static void tmigr_cpu_unisolate(struct work_struct *ignored) { - tmigr_set_cpu_available(smp_processor_id()); + /* + * Don't call tmigr_is_isolated() ->housekeeping_cpu() directly because + * the cpuset mutex is correctly held by the workqueue caller but lockdep + * doesn't know that. + */ + __tmigr_set_cpu_available(smp_processor_id()); } /** -- cgit v1.2.3 From 622c508bcf4852ad56ae37bf6f08e9a4e6aa95b2 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 27 May 2025 15:33:01 +0200 Subject: cpu: Provide lockdep check for CPU hotplug lock write-held cpuset modifies partitions, including isolated, while holding the cpu hotplug lock read-held. This means that write-holding the CPU hotplug lock is safe to synchronize against housekeeping cpumask changes. Provide a lockdep check to validate that. Signed-off-by: Frederic Weisbecker Cc: Marco Crivellari Cc: Michal Hocko Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Waiman Long Cc: linux-kernel@vger.kernel.org --- kernel/cpu.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'kernel') diff --git a/kernel/cpu.c b/kernel/cpu.c index 40b8496f47c5..01968a5c4a16 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -534,6 +534,11 @@ int lockdep_is_cpus_held(void) { return percpu_rwsem_is_held(&cpu_hotplug_lock); } + +int lockdep_is_cpus_write_held(void) +{ + return percpu_rwsem_is_write_held(&cpu_hotplug_lock); +} #endif static void lockdep_acquire_cpus_lock(void) -- cgit v1.2.3 From a7e546354d9c534dc045ce0cf9ff05d81351a751 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 27 May 2025 15:35:14 +0200 Subject: cpuset: Provide lockdep check for cpuset lock held MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cpuset modifies partitions, including isolated, while holding the cpuset mutex. This means that holding the cpuset mutex is safe to synchronize against housekeeping cpumask changes. Provide a lockdep check to validate that. Signed-off-by: Frederic Weisbecker Cc: "Michal Koutný" Cc: Johannes Weiner Cc: Marco Crivellari Cc: Michal Hocko Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Vlastimil Babka Cc: Waiman Long Cc: cgroups@vger.kernel.org Cc: linux-kernel@vger.kernel.org --- kernel/cgroup/cpuset.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'kernel') diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 3afa72f8d579..5e2e3514c22e 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -283,6 +283,13 @@ void cpuset_full_unlock(void) cpus_read_unlock(); } +#ifdef CONFIG_LOCKDEP +bool lockdep_is_cpuset_held(void) +{ + return lockdep_is_held(&cpuset_mutex); +} +#endif + static DEFINE_SPINLOCK(callback_lock); void cpuset_callback_lock_irq(void) -- cgit v1.2.3 From 27c3a5967f054ab666704cb28f2aeb18ca07cab7 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 27 May 2025 15:36:03 +0200 Subject: sched/isolation: Convert housekeeping cpumasks to rcu pointers HK_TYPE_DOMAIN's cpumask will soon be made modifiable by cpuset. A synchronization mechanism is then needed to synchronize the updates with the housekeeping cpumask readers. Turn the housekeeping cpumasks into RCU pointers. Once a housekeeping cpumask will be modified, the update side will wait for an RCU grace period and propagate the change to interested subsystem when deemed necessary. Signed-off-by: Frederic Weisbecker Cc: Ingo Molnar Cc: Marco Crivellari Cc: Michal Hocko Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Vlastimil Babka Cc: Waiman Long --- kernel/sched/isolation.c | 60 +++++++++++++++++++++++++++++------------------- kernel/sched/sched.h | 1 + 2 files changed, 38 insertions(+), 23 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index 11a623fa6320..6f77289c14c3 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -21,7 +21,7 @@ DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); EXPORT_SYMBOL_GPL(housekeeping_overridden); struct housekeeping { - cpumask_var_t cpumasks[HK_TYPE_MAX]; + struct cpumask __rcu *cpumasks[HK_TYPE_MAX]; unsigned long flags; }; @@ -33,17 +33,28 @@ bool housekeeping_enabled(enum hk_type type) } EXPORT_SYMBOL_GPL(housekeeping_enabled); +const struct cpumask *housekeeping_cpumask(enum hk_type type) +{ + if (static_branch_unlikely(&housekeeping_overridden)) { + if (housekeeping.flags & BIT(type)) { + return rcu_dereference_check(housekeeping.cpumasks[type], 1); + } + } + return cpu_possible_mask; +} +EXPORT_SYMBOL_GPL(housekeeping_cpumask); + int housekeeping_any_cpu(enum hk_type type) { int cpu; if (static_branch_unlikely(&housekeeping_overridden)) { if (housekeeping.flags & BIT(type)) { - cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id()); + cpu = sched_numa_find_closest(housekeeping_cpumask(type), smp_processor_id()); if (cpu < nr_cpu_ids) return cpu; - cpu = cpumask_any_and_distribute(housekeeping.cpumasks[type], cpu_online_mask); + cpu = cpumask_any_and_distribute(housekeeping_cpumask(type), cpu_online_mask); if (likely(cpu < nr_cpu_ids)) return cpu; /* @@ -59,28 +70,18 @@ int housekeeping_any_cpu(enum hk_type type) } EXPORT_SYMBOL_GPL(housekeeping_any_cpu); -const struct cpumask *housekeeping_cpumask(enum hk_type type) -{ - if (static_branch_unlikely(&housekeeping_overridden)) - if (housekeeping.flags & BIT(type)) - return housekeeping.cpumasks[type]; - return cpu_possible_mask; -} -EXPORT_SYMBOL_GPL(housekeeping_cpumask); - void housekeeping_affine(struct task_struct *t, enum hk_type type) { if (static_branch_unlikely(&housekeeping_overridden)) if (housekeeping.flags & BIT(type)) - set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]); + set_cpus_allowed_ptr(t, housekeeping_cpumask(type)); } EXPORT_SYMBOL_GPL(housekeeping_affine); bool housekeeping_test_cpu(int cpu, enum hk_type type) { - if (static_branch_unlikely(&housekeeping_overridden)) - if (housekeeping.flags & BIT(type)) - return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]); + if (static_branch_unlikely(&housekeeping_overridden) && housekeeping.flags & BIT(type)) + return cpumask_test_cpu(cpu, housekeeping_cpumask(type)); return true; } EXPORT_SYMBOL_GPL(housekeeping_test_cpu); @@ -96,20 +97,33 @@ void __init housekeeping_init(void) if (housekeeping.flags & HK_FLAG_KERNEL_NOISE) sched_tick_offload_init(); - + /* + * Realloc with a proper allocator so that any cpumask update + * can indifferently free the old version with kfree(). + */ for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) { + struct cpumask *omask, *nmask = kmalloc(cpumask_size(), GFP_KERNEL); + + if (WARN_ON_ONCE(!nmask)) + return; + + omask = rcu_dereference(housekeeping.cpumasks[type]); + /* We need at least one CPU to handle housekeeping work */ - WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type])); + WARN_ON_ONCE(cpumask_empty(omask)); + cpumask_copy(nmask, omask); + RCU_INIT_POINTER(housekeeping.cpumasks[type], nmask); + memblock_free(omask, cpumask_size()); } } static void __init housekeeping_setup_type(enum hk_type type, cpumask_var_t housekeeping_staging) { + struct cpumask *mask = memblock_alloc_or_panic(cpumask_size(), SMP_CACHE_BYTES); - alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]); - cpumask_copy(housekeeping.cpumasks[type], - housekeeping_staging); + cpumask_copy(mask, housekeeping_staging); + RCU_INIT_POINTER(housekeeping.cpumasks[type], mask); } static int __init housekeeping_setup(char *str, unsigned long flags) @@ -162,7 +176,7 @@ static int __init housekeeping_setup(char *str, unsigned long flags) for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) { if (!cpumask_equal(housekeeping_staging, - housekeeping.cpumasks[type])) { + housekeeping_cpumask(type))) { pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); goto free_housekeeping_staging; } @@ -183,7 +197,7 @@ static int __init housekeeping_setup(char *str, unsigned long flags) iter_flags = flags & (HK_FLAG_KERNEL_NOISE | HK_FLAG_DOMAIN); first_cpu = (type == HK_TYPE_MAX || !iter_flags) ? 0 : cpumask_first_and_and(cpu_present_mask, - housekeeping_staging, housekeeping.cpumasks[type]); + housekeeping_staging, housekeeping_cpumask(type)); if (first_cpu >= min(nr_cpu_ids, setup_max_cpus)) { pr_warn("Housekeeping: must include one present CPU " "neither in nohz_full= nor in isolcpus=domain, " diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index d30cca6870f5..475bdab3b8db 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include -- cgit v1.2.3 From 03ff7351016983653bab5b38e58529d0e38a28cf Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 28 May 2025 18:05:32 +0200 Subject: cpuset: Update HK_TYPE_DOMAIN cpumask from cpuset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Until now, HK_TYPE_DOMAIN used to only include boot defined isolated CPUs passed through isolcpus= boot option. Users interested in also knowing the runtime defined isolated CPUs through cpuset must use different APIs: cpuset_cpu_is_isolated(), cpu_is_isolated(), etc... There are many drawbacks to that approach: 1) Most interested subsystems want to know about all isolated CPUs, not just those defined on boot time. 2) cpuset_cpu_is_isolated() / cpu_is_isolated() are not synchronized with concurrent cpuset changes. 3) Further cpuset modifications are not propagated to subsystems Solve 1) and 2) and centralize all isolated CPUs within the HK_TYPE_DOMAIN housekeeping cpumask. Subsystems can rely on RCU to synchronize against concurrent changes. The propagation mentioned in 3) will be handled in further patches. [Chen Ridong: Fix cpu_hotplug_lock deadlock and use correct static branch API] Signed-off-by: Frederic Weisbecker Reviewed-by: Waiman Long Reviewed-by: Chen Ridong Signed-off-by: Chen Ridong Cc: "Michal Koutný" Cc: Ingo Molnar Cc: Johannes Weiner Cc: Marco Crivellari Cc: Michal Hocko Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Vlastimil Babka Cc: Waiman Long Cc: cgroups@vger.kernel.org --- kernel/cgroup/cpuset.c | 5 ++-- kernel/sched/isolation.c | 75 ++++++++++++++++++++++++++++++++++++++++++++---- kernel/sched/sched.h | 1 + 3 files changed, 73 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 5e2e3514c22e..e146e1f34bf9 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1482,14 +1482,15 @@ static void update_isolation_cpumasks(void) if (!isolated_cpus_updating) return; - lockdep_assert_cpus_held(); - ret = workqueue_unbound_exclude_cpumask(isolated_cpus); WARN_ON_ONCE(ret < 0); ret = tmigr_isolated_exclude_cpumask(isolated_cpus); WARN_ON_ONCE(ret < 0); + ret = housekeeping_update(isolated_cpus); + WARN_ON_ONCE(ret < 0); + isolated_cpus_updating = false; } diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index 6f77289c14c3..674a02891b38 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -29,18 +29,48 @@ static struct housekeeping housekeeping; bool housekeeping_enabled(enum hk_type type) { - return !!(housekeeping.flags & BIT(type)); + return !!(READ_ONCE(housekeeping.flags) & BIT(type)); } EXPORT_SYMBOL_GPL(housekeeping_enabled); +static bool housekeeping_dereference_check(enum hk_type type) +{ + if (IS_ENABLED(CONFIG_LOCKDEP) && type == HK_TYPE_DOMAIN) { + /* Cpuset isn't even writable yet? */ + if (system_state <= SYSTEM_SCHEDULING) + return true; + + /* CPU hotplug write locked, so cpuset partition can't be overwritten */ + if (IS_ENABLED(CONFIG_HOTPLUG_CPU) && lockdep_is_cpus_write_held()) + return true; + + /* Cpuset lock held, partitions not writable */ + if (IS_ENABLED(CONFIG_CPUSETS) && lockdep_is_cpuset_held()) + return true; + + return false; + } + + return true; +} + +static inline struct cpumask *housekeeping_cpumask_dereference(enum hk_type type) +{ + return rcu_dereference_all_check(housekeeping.cpumasks[type], + housekeeping_dereference_check(type)); +} + const struct cpumask *housekeeping_cpumask(enum hk_type type) { + const struct cpumask *mask = NULL; + if (static_branch_unlikely(&housekeeping_overridden)) { - if (housekeeping.flags & BIT(type)) { - return rcu_dereference_check(housekeeping.cpumasks[type], 1); - } + if (READ_ONCE(housekeeping.flags) & BIT(type)) + mask = housekeeping_cpumask_dereference(type); } - return cpu_possible_mask; + if (!mask) + mask = cpu_possible_mask; + return mask; } EXPORT_SYMBOL_GPL(housekeeping_cpumask); @@ -80,12 +110,45 @@ EXPORT_SYMBOL_GPL(housekeeping_affine); bool housekeeping_test_cpu(int cpu, enum hk_type type) { - if (static_branch_unlikely(&housekeeping_overridden) && housekeeping.flags & BIT(type)) + if (static_branch_unlikely(&housekeeping_overridden) && + READ_ONCE(housekeeping.flags) & BIT(type)) return cpumask_test_cpu(cpu, housekeeping_cpumask(type)); return true; } EXPORT_SYMBOL_GPL(housekeeping_test_cpu); +int housekeeping_update(struct cpumask *isol_mask) +{ + struct cpumask *trial, *old = NULL; + + lockdep_assert_cpus_held(); + + trial = kmalloc(cpumask_size(), GFP_KERNEL); + if (!trial) + return -ENOMEM; + + cpumask_andnot(trial, housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT), isol_mask); + if (!cpumask_intersects(trial, cpu_online_mask)) { + kfree(trial); + return -EINVAL; + } + + if (!housekeeping.flags) + static_branch_enable_cpuslocked(&housekeeping_overridden); + + if (housekeeping.flags & HK_FLAG_DOMAIN) + old = housekeeping_cpumask_dereference(HK_TYPE_DOMAIN); + else + WRITE_ONCE(housekeeping.flags, housekeeping.flags | HK_FLAG_DOMAIN); + rcu_assign_pointer(housekeeping.cpumasks[HK_TYPE_DOMAIN], trial); + + synchronize_rcu(); + + kfree(old); + + return 0; +} + void __init housekeeping_init(void) { enum hk_type type; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 475bdab3b8db..653e898a996a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include -- cgit v1.2.3 From b7eb4edcc3b5cd7ffdcbd56fa7a12de41b39424d Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Thu, 12 Jun 2025 15:36:16 +0200 Subject: sched/isolation: Flush memcg workqueues on cpuset isolated partition change The HK_TYPE_DOMAIN housekeeping cpumask is now modifiable at runtime. In order to synchronize against memcg workqueue to make sure that no asynchronous draining is still pending or executing on a newly made isolated CPU, the housekeeping susbsystem must flush the memcg workqueues. However the memcg workqueues can't be flushed easily since they are queued to the main per-CPU workqueue pool. Solve this with creating a memcg specific pool and provide and use the appropriate flushing API. Acked-by: Shakeel Butt Signed-off-by: Frederic Weisbecker Cc: Andrew Morton Cc: Ingo Molnar Cc: Johannes Weiner Cc: Marco Crivellari Cc: Michal Hocko Cc: Muchun Song Cc: Peter Zijlstra Cc: Roman Gushchin Cc: Shakeel Butt Cc: Tejun Heo Cc: Thomas Gleixner Cc: Vlastimil Babka Cc: Waiman Long Cc: cgroups@vger.kernel.org Cc: linux-mm@kvack.org --- kernel/sched/isolation.c | 2 ++ kernel/sched/sched.h | 1 + 2 files changed, 3 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index 674a02891b38..f4053ebf4027 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -144,6 +144,8 @@ int housekeeping_update(struct cpumask *isol_mask) synchronize_rcu(); + mem_cgroup_flush_workqueue(); + kfree(old); return 0; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 653e898a996a..65dfa48e54b7 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include -- cgit v1.2.3 From ce84ad5e994aea5d41ff47135a71439ad4f54005 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 13 Jun 2025 14:48:31 +0200 Subject: sched/isolation: Flush vmstat workqueues on cpuset isolated partition change The HK_TYPE_DOMAIN housekeeping cpumask is now modifiable at runtime. In order to synchronize against vmstat workqueue to make sure that no asynchronous vmstat work is still pending or executing on a newly made isolated CPU, the housekeeping susbsystem must flush the vmstat workqueues. This involves flushing the whole mm_percpu_wq workqueue, shared with LRU drain, introducing here a welcome side effect. Signed-off-by: Frederic Weisbecker Cc: Andrew Morton Cc: Ingo Molnar Cc: Marco Crivellari Cc: Michal Hocko Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Vlastimil Babka Cc: Waiman Long Cc: linux-mm@kvack.org --- kernel/sched/isolation.c | 1 + kernel/sched/sched.h | 1 + 2 files changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index f4053ebf4027..160b3fcab209 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -145,6 +145,7 @@ int housekeeping_update(struct cpumask *isol_mask) synchronize_rcu(); mem_cgroup_flush_workqueue(); + vmstat_flush_workqueue(); kfree(old); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 65dfa48e54b7..2d0c408fca0b 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -68,6 +68,7 @@ #include #include #include +#include #include #include #include -- cgit v1.2.3 From 29b306c44eb5eefdfa02d6ba1205f479f82fb088 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 30 Sep 2025 15:21:33 +0200 Subject: PCI: Flush PCI probe workqueue on cpuset isolated partition change The HK_TYPE_DOMAIN housekeeping cpumask is now modifiable at runtime. In order to synchronize against PCI probe works and make sure that no asynchronous probing is still pending or executing on a newly isolated CPU, the housekeeping subsystem must flush the PCI probe works. However the PCI probe works can't be flushed easily since they are queued to the main per-CPU workqueue pool. Solve this with creating a PCI probe-specific pool and provide and use the appropriate flushing API. Signed-off-by: Frederic Weisbecker Acked-by: Bjorn Helgaas Cc: Marco Crivellari Cc: Michal Hocko Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Vlastimil Babka Cc: Waiman Long Cc: linux-pci@vger.kernel.org --- kernel/sched/isolation.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index 160b3fcab209..1e4c3154b0a4 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -8,6 +8,7 @@ * */ #include +#include #include "sched.h" enum hk_flags { @@ -144,6 +145,7 @@ int housekeeping_update(struct cpumask *isol_mask) synchronize_rcu(); + pci_probe_flush_workqueue(); mem_cgroup_flush_workqueue(); vmstat_flush_workqueue(); -- cgit v1.2.3 From 23f09dcc0a0fa3b4e48516bdea1c90223dfb3d6c Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 28 May 2025 18:19:23 +0200 Subject: cpuset: Propagate cpuset isolation update to workqueue through housekeeping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Until now, cpuset would propagate isolated partition changes to workqueues so that unbound workers get properly reaffined. Since housekeeping now centralizes, synchronize and propagates isolation cpumask changes, perform the work from that subsystem for consolidation and consistency purposes. For simplification purpose, the target function is adapted to take the new housekeeping mask instead of the isolated mask. Suggested-by: Tejun Heo Signed-off-by: Frederic Weisbecker Reviewed-by: Waiman Long Acked-by: Tejun Heo Cc: "Michal Koutný" Cc: Ingo Molnar Cc: Johannes Weiner Cc: Lai Jiangshan Cc: Marco Crivellari Cc: Michal Hocko Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Vlastimil Babka Cc: Waiman Long Cc: cgroups@vger.kernel.org --- kernel/cgroup/cpuset.c | 5 +---- kernel/sched/isolation.c | 3 +++ kernel/workqueue.c | 17 ++++++++++------- 3 files changed, 14 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index e146e1f34bf9..6309ec5d7b2a 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1482,15 +1482,12 @@ static void update_isolation_cpumasks(void) if (!isolated_cpus_updating) return; - ret = workqueue_unbound_exclude_cpumask(isolated_cpus); + ret = housekeeping_update(isolated_cpus); WARN_ON_ONCE(ret < 0); ret = tmigr_isolated_exclude_cpumask(isolated_cpus); WARN_ON_ONCE(ret < 0); - ret = housekeeping_update(isolated_cpus); - WARN_ON_ONCE(ret < 0); - isolated_cpus_updating = false; } diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index 1e4c3154b0a4..5bcb6d760f20 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -121,6 +121,7 @@ EXPORT_SYMBOL_GPL(housekeeping_test_cpu); int housekeeping_update(struct cpumask *isol_mask) { struct cpumask *trial, *old = NULL; + int err; lockdep_assert_cpus_held(); @@ -148,6 +149,8 @@ int housekeeping_update(struct cpumask *isol_mask) pci_probe_flush_workqueue(); mem_cgroup_flush_workqueue(); vmstat_flush_workqueue(); + err = workqueue_unbound_housekeeping_update(housekeeping_cpumask(HK_TYPE_DOMAIN)); + WARN_ON_ONCE(err < 0); kfree(old); diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 253311af47c6..eb5660013222 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -6959,13 +6959,16 @@ static int workqueue_apply_unbound_cpumask(const cpumask_var_t unbound_cpumask) } /** - * workqueue_unbound_exclude_cpumask - Exclude given CPUs from unbound cpumask - * @exclude_cpumask: the cpumask to be excluded from wq_unbound_cpumask + * workqueue_unbound_housekeeping_update - Propagate housekeeping cpumask update + * @hk: the new housekeeping cpumask * - * This function can be called from cpuset code to provide a set of isolated - * CPUs that should be excluded from wq_unbound_cpumask. + * Update the unbound workqueue cpumask on top of the new housekeeping cpumask such + * that the effective unbound affinity is the intersection of the new housekeeping + * with the requested affinity set via nohz_full=/isolcpus= or sysfs. + * + * Return: 0 on success and -errno on failure. */ -int workqueue_unbound_exclude_cpumask(cpumask_var_t exclude_cpumask) +int workqueue_unbound_housekeeping_update(const struct cpumask *hk) { cpumask_var_t cpumask; int ret = 0; @@ -6981,14 +6984,14 @@ int workqueue_unbound_exclude_cpumask(cpumask_var_t exclude_cpumask) * (HK_TYPE_WQ ∩ HK_TYPE_DOMAIN) house keeping mask and rewritten * by any subsequent write to workqueue/cpumask sysfs file. */ - if (!cpumask_andnot(cpumask, wq_requested_unbound_cpumask, exclude_cpumask)) + if (!cpumask_and(cpumask, wq_requested_unbound_cpumask, hk)) cpumask_copy(cpumask, wq_requested_unbound_cpumask); if (!cpumask_equal(cpumask, wq_unbound_cpumask)) ret = workqueue_apply_unbound_cpumask(cpumask); /* Save the current isolated cpumask & export it via sysfs */ if (!ret) - cpumask_copy(wq_isolated_cpumask, exclude_cpumask); + cpumask_andnot(wq_isolated_cpumask, cpu_possible_mask, hk); mutex_unlock(&wq_pool_mutex); free_cpumask_var(cpumask); -- cgit v1.2.3 From f5c145ae4f26c25b853f059f1aa14a46f4e9f1ab Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Mon, 22 Dec 2025 00:55:49 +0100 Subject: cpuset: Propagate cpuset isolation update to timers through housekeeping Until now, cpuset would propagate isolated partition changes to timer migration so that unbound timers don't get migrated to isolated CPUs. Since housekeeping now centralizes, synchronize and propagates isolation cpumask changes, perform the work from that subsystem for consolidation and consistency purposes. Signed-off-by: Frederic Weisbecker --- kernel/cgroup/cpuset.c | 3 --- kernel/sched/isolation.c | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 6309ec5d7b2a..080fa2fb10c7 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1485,9 +1485,6 @@ static void update_isolation_cpumasks(void) ret = housekeeping_update(isolated_cpus); WARN_ON_ONCE(ret < 0); - ret = tmigr_isolated_exclude_cpumask(isolated_cpus); - WARN_ON_ONCE(ret < 0); - isolated_cpus_updating = false; } diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index 5bcb6d760f20..a30d19b641f7 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -149,9 +149,13 @@ int housekeeping_update(struct cpumask *isol_mask) pci_probe_flush_workqueue(); mem_cgroup_flush_workqueue(); vmstat_flush_workqueue(); + err = workqueue_unbound_housekeeping_update(housekeeping_cpumask(HK_TYPE_DOMAIN)); WARN_ON_ONCE(err < 0); + err = tmigr_isolated_exclude_cpumask(isol_mask); + WARN_ON_ONCE(err < 0); + kfree(old); return 0; -- cgit v1.2.3 From 0947d018cf574b6c19d64aa3f67ecd0a5add9e31 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Mon, 22 Dec 2025 16:22:56 +0100 Subject: timers/migration: Remove superfluous cpuset isolation test Cpuset isolated partitions are now included in HK_TYPE_DOMAIN. Testing if a CPU is part of an isolated partition alone is now useless. Remove the superflous test. Signed-off-by: Frederic Weisbecker Reviewed-by: Waiman Long --- kernel/time/timer_migration.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c index 3879575a4975..6da9cd562b20 100644 --- a/kernel/time/timer_migration.c +++ b/kernel/time/timer_migration.c @@ -466,9 +466,8 @@ static inline bool tmigr_is_isolated(int cpu) { if (!static_branch_unlikely(&tmigr_exclude_isolated)) return false; - return (!housekeeping_cpu(cpu, HK_TYPE_DOMAIN) || - cpuset_cpu_is_isolated(cpu)) && - housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE); + return (!housekeeping_cpu(cpu, HK_TYPE_DOMAIN) && + housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE)); } /* -- cgit v1.2.3 From 6440966067dc078f7b46942b3e2cb5d1035fc0b9 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 28 May 2025 18:21:32 +0200 Subject: cpuset: Remove cpuset_cpu_is_isolated() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The set of cpuset isolated CPUs is now included in HK_TYPE_DOMAIN housekeeping cpumask. There is no usecase left interested in just checking what is isolated by cpuset and not by the isolcpus= kernel boot parameter. Signed-off-by: Frederic Weisbecker Reviewed-by: Waiman Long Cc: "Michal Koutný" Cc: Johannes Weiner Cc: Marco Crivellari Cc: Michal Hocko Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Vlastimil Babka Cc: Waiman Long Cc: cgroups@vger.kernel.org --- kernel/cgroup/cpuset.c | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 080fa2fb10c7..801694de82a3 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -29,7 +29,6 @@ #include #include #include -#include #include #include #include @@ -1488,17 +1487,6 @@ static void update_isolation_cpumasks(void) isolated_cpus_updating = false; } -/** - * cpuset_cpu_is_isolated - Check if the given CPU is isolated - * @cpu: the CPU number to be checked - * Return: true if CPU is used in an isolated partition, false otherwise - */ -bool cpuset_cpu_is_isolated(int cpu) -{ - return cpumask_test_cpu(cpu, isolated_cpus); -} -EXPORT_SYMBOL_GPL(cpuset_cpu_is_isolated); - /** * rm_siblings_excl_cpus - Remove exclusive CPUs that are used by sibling cpusets * @parent: Parent cpuset containing all siblings -- cgit v1.2.3 From 012fef0e4859d1ffb41f5f7d72b3c61068829e6b Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Mon, 12 May 2025 11:37:11 +0200 Subject: kthread: Refine naming of affinity related fields The kthreads preferred affinity related fields use "hotplug" as the base of their naming because the affinity management was initially deemed to deal with CPU hotplug. The scope of this role is going to broaden now and also deal with cpuset isolated partition updates. Switch the naming accordingly. Signed-off-by: Frederic Weisbecker Acked-by: Waiman Long Cc: Marco Crivellari Cc: Michal Hocko Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Vlastimil Babka Cc: Waiman Long --- kernel/kthread.c | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) (limited to 'kernel') diff --git a/kernel/kthread.c b/kernel/kthread.c index 99a3808d086f..f1e4f1f35cae 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -35,8 +35,8 @@ static DEFINE_SPINLOCK(kthread_create_lock); static LIST_HEAD(kthread_create_list); struct task_struct *kthreadd_task; -static LIST_HEAD(kthreads_hotplug); -static DEFINE_MUTEX(kthreads_hotplug_lock); +static LIST_HEAD(kthread_affinity_list); +static DEFINE_MUTEX(kthread_affinity_lock); struct kthread_create_info { @@ -69,7 +69,7 @@ struct kthread { /* To store the full name if task comm is truncated. */ char *full_name; struct task_struct *task; - struct list_head hotplug_node; + struct list_head affinity_node; struct cpumask *preferred_affinity; }; @@ -128,7 +128,7 @@ bool set_kthread_struct(struct task_struct *p) init_completion(&kthread->exited); init_completion(&kthread->parked); - INIT_LIST_HEAD(&kthread->hotplug_node); + INIT_LIST_HEAD(&kthread->affinity_node); p->vfork_done = &kthread->exited; kthread->task = p; @@ -323,10 +323,10 @@ void __noreturn kthread_exit(long result) { struct kthread *kthread = to_kthread(current); kthread->result = result; - if (!list_empty(&kthread->hotplug_node)) { - mutex_lock(&kthreads_hotplug_lock); - list_del(&kthread->hotplug_node); - mutex_unlock(&kthreads_hotplug_lock); + if (!list_empty(&kthread->affinity_node)) { + mutex_lock(&kthread_affinity_lock); + list_del(&kthread->affinity_node); + mutex_unlock(&kthread_affinity_lock); if (kthread->preferred_affinity) { kfree(kthread->preferred_affinity); @@ -390,9 +390,9 @@ static void kthread_affine_node(void) return; } - mutex_lock(&kthreads_hotplug_lock); - WARN_ON_ONCE(!list_empty(&kthread->hotplug_node)); - list_add_tail(&kthread->hotplug_node, &kthreads_hotplug); + mutex_lock(&kthread_affinity_lock); + WARN_ON_ONCE(!list_empty(&kthread->affinity_node)); + list_add_tail(&kthread->affinity_node, &kthread_affinity_list); /* * The node cpumask is racy when read from kthread() but: * - a racing CPU going down will either fail on the subsequent @@ -402,7 +402,7 @@ static void kthread_affine_node(void) */ kthread_fetch_affinity(kthread, affinity); set_cpus_allowed_ptr(current, affinity); - mutex_unlock(&kthreads_hotplug_lock); + mutex_unlock(&kthread_affinity_lock); free_cpumask_var(affinity); } @@ -873,16 +873,16 @@ int kthread_affine_preferred(struct task_struct *p, const struct cpumask *mask) goto out; } - mutex_lock(&kthreads_hotplug_lock); + mutex_lock(&kthread_affinity_lock); cpumask_copy(kthread->preferred_affinity, mask); - WARN_ON_ONCE(!list_empty(&kthread->hotplug_node)); - list_add_tail(&kthread->hotplug_node, &kthreads_hotplug); + WARN_ON_ONCE(!list_empty(&kthread->affinity_node)); + list_add_tail(&kthread->affinity_node, &kthread_affinity_list); kthread_fetch_affinity(kthread, affinity); scoped_guard (raw_spinlock_irqsave, &p->pi_lock) set_cpus_allowed_force(p, affinity); - mutex_unlock(&kthreads_hotplug_lock); + mutex_unlock(&kthread_affinity_lock); out: free_cpumask_var(affinity); @@ -903,9 +903,9 @@ static int kthreads_online_cpu(unsigned int cpu) struct kthread *k; int ret; - guard(mutex)(&kthreads_hotplug_lock); + guard(mutex)(&kthread_affinity_lock); - if (list_empty(&kthreads_hotplug)) + if (list_empty(&kthread_affinity_list)) return 0; if (!zalloc_cpumask_var(&affinity, GFP_KERNEL)) @@ -913,7 +913,7 @@ static int kthreads_online_cpu(unsigned int cpu) ret = 0; - list_for_each_entry(k, &kthreads_hotplug, hotplug_node) { + list_for_each_entry(k, &kthread_affinity_list, affinity_node) { if (WARN_ON_ONCE((k->task->flags & PF_NO_SETAFFINITY) || kthread_is_per_cpu(k->task))) { ret = -EINVAL; -- cgit v1.2.3 From 5564c12385b7a6a2991e31017d12fcef96fa830a Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 3 Jun 2025 15:30:32 +0200 Subject: kthread: Include unbound kthreads in the managed affinity list The managed affinity list currently contains only unbound kthreads that have affinity preferences. Unbound kthreads globally affine by default are outside of the list because their affinity is automatically managed by the scheduler (through the fallback housekeeping mask) and by cpuset. However in order to preserve the preferred affinity of kthreads, cpuset will delegate the isolated partition update propagation to the housekeeping and kthread code. Prepare for that with including all unbound kthreads in the managed affinity list. Signed-off-by: Frederic Weisbecker Reviewed-by: Waiman Long Cc: Marco Crivellari Cc: Michal Hocko Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Vlastimil Babka Cc: Waiman Long --- kernel/kthread.c | 68 +++++++++++++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 28 deletions(-) (limited to 'kernel') diff --git a/kernel/kthread.c b/kernel/kthread.c index f1e4f1f35cae..51c0908d3d02 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -365,9 +365,10 @@ static void kthread_fetch_affinity(struct kthread *kthread, struct cpumask *cpum if (kthread->preferred_affinity) { pref = kthread->preferred_affinity; } else { - if (WARN_ON_ONCE(kthread->node == NUMA_NO_NODE)) - return; - pref = cpumask_of_node(kthread->node); + if (kthread->node == NUMA_NO_NODE) + pref = housekeeping_cpumask(HK_TYPE_KTHREAD); + else + pref = cpumask_of_node(kthread->node); } cpumask_and(cpumask, pref, housekeeping_cpumask(HK_TYPE_KTHREAD)); @@ -380,32 +381,29 @@ static void kthread_affine_node(void) struct kthread *kthread = to_kthread(current); cpumask_var_t affinity; - WARN_ON_ONCE(kthread_is_per_cpu(current)); + if (WARN_ON_ONCE(kthread_is_per_cpu(current))) + return; - if (kthread->node == NUMA_NO_NODE) { - housekeeping_affine(current, HK_TYPE_KTHREAD); - } else { - if (!zalloc_cpumask_var(&affinity, GFP_KERNEL)) { - WARN_ON_ONCE(1); - return; - } + if (!zalloc_cpumask_var(&affinity, GFP_KERNEL)) { + WARN_ON_ONCE(1); + return; + } - mutex_lock(&kthread_affinity_lock); - WARN_ON_ONCE(!list_empty(&kthread->affinity_node)); - list_add_tail(&kthread->affinity_node, &kthread_affinity_list); - /* - * The node cpumask is racy when read from kthread() but: - * - a racing CPU going down will either fail on the subsequent - * call to set_cpus_allowed_ptr() or be migrated to housekeepers - * afterwards by the scheduler. - * - a racing CPU going up will be handled by kthreads_online_cpu() - */ - kthread_fetch_affinity(kthread, affinity); - set_cpus_allowed_ptr(current, affinity); - mutex_unlock(&kthread_affinity_lock); + mutex_lock(&kthread_affinity_lock); + WARN_ON_ONCE(!list_empty(&kthread->affinity_node)); + list_add_tail(&kthread->affinity_node, &kthread_affinity_list); + /* + * The node cpumask is racy when read from kthread() but: + * - a racing CPU going down will either fail on the subsequent + * call to set_cpus_allowed_ptr() or be migrated to housekeepers + * afterwards by the scheduler. + * - a racing CPU going up will be handled by kthreads_online_cpu() + */ + kthread_fetch_affinity(kthread, affinity); + set_cpus_allowed_ptr(current, affinity); + mutex_unlock(&kthread_affinity_lock); - free_cpumask_var(affinity); - } + free_cpumask_var(affinity); } static int kthread(void *_create) @@ -919,8 +917,22 @@ static int kthreads_online_cpu(unsigned int cpu) ret = -EINVAL; continue; } - kthread_fetch_affinity(k, affinity); - set_cpus_allowed_ptr(k->task, affinity); + + /* + * Unbound kthreads without preferred affinity are already affine + * to housekeeping, whether those CPUs are online or not. So no need + * to handle newly online CPUs for them. + * + * But kthreads with a preferred affinity or node are different: + * if none of their preferred CPUs are online and part of + * housekeeping at the same time, they must be affine to housekeeping. + * But as soon as one of their preferred CPU becomes online, they must + * be affine to them. + */ + if (k->preferred_affinity || k->node != NUMA_NO_NODE) { + kthread_fetch_affinity(k, affinity); + set_cpus_allowed_ptr(k->task, affinity); + } } free_cpumask_var(affinity); -- cgit v1.2.3 From 92a734606e901cf7af239e3bd3cfb0c3a11d7b5e Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 3 Jun 2025 16:41:56 +0200 Subject: kthread: Include kthreadd to the managed affinity list The unbound kthreads affinity management performed by cpuset is going to be imported to the kthread core code for consolidation purposes. Treat kthreadd just like any other kthread. Signed-off-by: Frederic Weisbecker Reviewed-by: Waiman Long Cc: Marco Crivellari Cc: Michal Hocko Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Vlastimil Babka Cc: Waiman Long --- kernel/kthread.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/kthread.c b/kernel/kthread.c index 51c0908d3d02..85ccf5bb17c9 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -818,12 +818,13 @@ int kthreadd(void *unused) /* Setup a clean context for our children to inherit. */ set_task_comm(tsk, comm); ignore_signals(tsk); - set_cpus_allowed_ptr(tsk, housekeeping_cpumask(HK_TYPE_KTHREAD)); set_mems_allowed(node_states[N_MEMORY]); current->flags |= PF_NOFREEZE; cgroup_init_kthreadd(); + kthread_affine_node(); + for (;;) { set_current_state(TASK_INTERRUPTIBLE); if (list_empty(&kthread_create_list)) -- cgit v1.2.3 From 041ee6f3727a5efdc497d0363ab5ffbd98a2c77c Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Mon, 2 Jun 2025 21:21:55 +0200 Subject: kthread: Rely on HK_TYPE_DOMAIN for preferred affinity management Unbound kthreads want to run neither on nohz_full CPUs nor on domain isolated CPUs. And since nohz_full implies domain isolation, checking the latter is enough to verify both. Therefore exclude kthreads from domain isolation. Signed-off-by: Frederic Weisbecker Reviewed-by: Waiman Long Cc: Marco Crivellari Cc: Michal Hocko Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Vlastimil Babka Cc: Waiman Long --- kernel/kthread.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/kthread.c b/kernel/kthread.c index 85ccf5bb17c9..968fa5868d21 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -362,18 +362,20 @@ static void kthread_fetch_affinity(struct kthread *kthread, struct cpumask *cpum { const struct cpumask *pref; + guard(rcu)(); + if (kthread->preferred_affinity) { pref = kthread->preferred_affinity; } else { if (kthread->node == NUMA_NO_NODE) - pref = housekeeping_cpumask(HK_TYPE_KTHREAD); + pref = housekeeping_cpumask(HK_TYPE_DOMAIN); else pref = cpumask_of_node(kthread->node); } - cpumask_and(cpumask, pref, housekeeping_cpumask(HK_TYPE_KTHREAD)); + cpumask_and(cpumask, pref, housekeeping_cpumask(HK_TYPE_DOMAIN)); if (cpumask_empty(cpumask)) - cpumask_copy(cpumask, housekeeping_cpumask(HK_TYPE_KTHREAD)); + cpumask_copy(cpumask, housekeeping_cpumask(HK_TYPE_DOMAIN)); } static void kthread_affine_node(void) -- cgit v1.2.3 From e894f633980804a528a2d6996c4ea651df631632 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 4 Jun 2025 14:02:40 +0200 Subject: kthread: Honour kthreads preferred affinity after cpuset changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When cpuset isolated partitions get updated, unbound kthreads get indifferently affine to all non isolated CPUs, regardless of their individual affinity preferences. For example kswapd is a per-node kthread that prefers to be affine to the node it refers to. Whenever an isolated partition is created, updated or deleted, kswapd's node affinity is going to be broken if any CPU in the related node is not isolated because kswapd will be affine globally. Fix this with letting the consolidated kthread managed affinity code do the affinity update on behalf of cpuset. Signed-off-by: Frederic Weisbecker Reviewed-by: Waiman Long Cc: Michal Koutný Cc: Ingo Molnar Cc: Johannes Weiner Cc: Marco Crivellari Cc: Michal Hocko Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Vlastimil Babka Cc: Waiman Long Cc: cgroups@vger.kernel.org --- kernel/cgroup/cpuset.c | 5 ++--- kernel/kthread.c | 41 +++++++++++++++++++++++++++++++---------- kernel/sched/isolation.c | 3 +++ 3 files changed, 36 insertions(+), 13 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 801694de82a3..9a8292b7c7f1 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1208,11 +1208,10 @@ void cpuset_update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus) if (top_cs) { /* + * PF_KTHREAD tasks are handled by housekeeping. * PF_NO_SETAFFINITY tasks are ignored. - * All per cpu kthreads should have PF_NO_SETAFFINITY - * flag set, see kthread_set_per_cpu(). */ - if (task->flags & PF_NO_SETAFFINITY) + if (task->flags & (PF_KTHREAD | PF_NO_SETAFFINITY)) continue; cpumask_andnot(new_cpus, possible_mask, subpartitions_cpus); } else { diff --git a/kernel/kthread.c b/kernel/kthread.c index 968fa5868d21..03008154249c 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -891,14 +891,7 @@ out: } EXPORT_SYMBOL_GPL(kthread_affine_preferred); -/* - * Re-affine kthreads according to their preferences - * and the newly online CPU. The CPU down part is handled - * by select_fallback_rq() which default re-affines to - * housekeepers from other nodes in case the preferred - * affinity doesn't apply anymore. - */ -static int kthreads_online_cpu(unsigned int cpu) +static int kthreads_update_affinity(bool force) { cpumask_var_t affinity; struct kthread *k; @@ -924,7 +917,8 @@ static int kthreads_online_cpu(unsigned int cpu) /* * Unbound kthreads without preferred affinity are already affine * to housekeeping, whether those CPUs are online or not. So no need - * to handle newly online CPUs for them. + * to handle newly online CPUs for them. However housekeeping changes + * have to be applied. * * But kthreads with a preferred affinity or node are different: * if none of their preferred CPUs are online and part of @@ -932,7 +926,7 @@ static int kthreads_online_cpu(unsigned int cpu) * But as soon as one of their preferred CPU becomes online, they must * be affine to them. */ - if (k->preferred_affinity || k->node != NUMA_NO_NODE) { + if (force || k->preferred_affinity || k->node != NUMA_NO_NODE) { kthread_fetch_affinity(k, affinity); set_cpus_allowed_ptr(k->task, affinity); } @@ -943,6 +937,33 @@ static int kthreads_online_cpu(unsigned int cpu) return ret; } +/** + * kthreads_update_housekeeping - Update kthreads affinity on cpuset change + * + * When cpuset changes a partition type to/from "isolated" or updates related + * cpumasks, propagate the housekeeping cpumask change to preferred kthreads + * affinity. + * + * Returns 0 if successful, -ENOMEM if temporary mask couldn't + * be allocated or -EINVAL in case of internal error. + */ +int kthreads_update_housekeeping(void) +{ + return kthreads_update_affinity(true); +} + +/* + * Re-affine kthreads according to their preferences + * and the newly online CPU. The CPU down part is handled + * by select_fallback_rq() which default re-affines to + * housekeepers from other nodes in case the preferred + * affinity doesn't apply anymore. + */ +static int kthreads_online_cpu(unsigned int cpu) +{ + return kthreads_update_affinity(false); +} + static int kthreads_init(void) { return cpuhp_setup_state(CPUHP_AP_KTHREADS_ONLINE, "kthreads:online", diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index a30d19b641f7..3b725d39c06e 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -156,6 +156,9 @@ int housekeeping_update(struct cpumask *isol_mask) err = tmigr_isolated_exclude_cpumask(isol_mask); WARN_ON_ONCE(err < 0); + err = kthreads_update_housekeeping(); + WARN_ON_ONCE(err < 0); + kfree(old); return 0; -- cgit v1.2.3 From 60ba9c38b9316efa3eb1c8a927bee6f8a4f101a5 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 18 Jun 2025 16:52:04 +0200 Subject: kthread: Comment on the purpose and placement of kthread_affine_node() call It may not appear obvious why kthread_affine_node() is not called before the kthread creation completion instead of after the first wake-up. The reason is that kthread_affine_node() applies a default affinity behaviour that only takes place if no affinity preference have already been passed by the kthread creation call site. Add a comment to clarify that. Reported-by: Peter Zijlstra Signed-off-by: Frederic Weisbecker Cc: Marco Crivellari Cc: Michal Hocko Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Vlastimil Babka Cc: Waiman Long --- kernel/kthread.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'kernel') diff --git a/kernel/kthread.c b/kernel/kthread.c index 03008154249c..51f419139dea 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -453,6 +453,10 @@ static int kthread(void *_create) self->started = 1; + /* + * Apply default node affinity if no call to kthread_bind[_mask]() nor + * kthread_affine_preferred() was issued before the first wake-up. + */ if (!(current->flags & PF_NO_SETAFFINITY) && !self->preferred_affinity) kthread_affine_node(); -- cgit v1.2.3 From d279138a2788ac22cff23710b2d4a3ebd160c09d Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 26 Aug 2025 17:16:48 +0200 Subject: kthread: Document kthread_affine_preferred() The documentation of this new API has been overlooked during its introduction. Fill the gap. Signed-off-by: Frederic Weisbecker Cc: Marco Crivellari Cc: Michal Hocko Cc: Peter Zijlstra Cc: Tejun Heo Cc: Thomas Gleixner Cc: Waiman Long --- kernel/kthread.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'kernel') diff --git a/kernel/kthread.c b/kernel/kthread.c index 51f419139dea..c50f4c0eabfe 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -856,6 +856,18 @@ int kthreadd(void *unused) return 0; } +/** + * kthread_affine_preferred - Define a kthread's preferred affinity + * @p: thread created by kthread_create(). + * @mask: preferred mask of CPUs (might not be online, must be possible) for @p + * to run on. + * + * Similar to kthread_bind_mask() except that the affinity is not a requirement + * but rather a preference that can be constrained by CPU isolation or CPU hotplug. + * Must be called before the first wakeup of the kthread. + * + * Returns 0 if the affinity has been applied. + */ int kthread_affine_preferred(struct task_struct *p, const struct cpumask *mask) { struct kthread *kthread = to_kthread(p); -- cgit v1.2.3