diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2026-10-29 07:35:19 -1000 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2026-10-29 07:35:19 -1000 |
| commit | 2152c755f76a9170218a098f79e2b081a08b1d6c (patch) | |
| tree | de8d773332f39aba7ad4ac190cdfc8e5bb1e3769 /kernel | |
| parent | c339f4010dea8b0d754f84ace70cd1b089878ef8 (diff) | |
| parent | 518cec2def9992192291f5028edc077fb2e475e9 (diff) | |
Merge tag 'cgroup-for-7.4-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup
Pull cgroup fixes from Tejun Heo:
- cgroup.kill did not kill a process whose thread group leader had
exited while other threads still ran, nor those threads once moved to
another cgroup, letting a process survive its container being killed
- A cgroup.procs write under favordynmods could migrate a newly forked
process before fork linked it to its cgroup, corrupting cgroup task
lists
- cpuset partition fixes for nested hierarchies: invalidation, revival,
shrinking and switches between root and isolated could leak isolated
CPUs, skip or miscompute the housekeeping check, or leave wrong
effective CPUs:
- A root partition can no longer be created under an isolated
parent or stay valid under one, as disabling it could leave no
housekeeping CPUs
- Enabling a partition failed when a grandchild held tasks,
although the parent had none of its own
- Separately, an allocation racing with a cpuset.mems update could
drop its mempolicy nodemask and land on the wrong node, and a
memoryless node in cpuset.mems triggered a spurious "movable
nodes only" warning
- Two no-functional-change cleanups of the effective CPU
computation
* tag 'cgroup-for-7.4-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup: (27 commits)
cgroup/cpuset: Integrate compute_partition_effective_cpumask() into compute_effective_cpumask()
cgroup/cpuset: Relocate compute_*effective_cpumask() and compute_effective_nodemask()
cgroup/cpuset: Release invalid isolated CPUs with a shrunk root partition parent
cgroup/cpuset: Release invalid isolated CPUs in compute_partition_effective_cpumask()
cgroup/cpuset: Make sure compute_partition_effective_cpumask() is only called for valid partition
cgroup/cpuset: Prevent recursive call of remote_cpus_update() via update_cpumasks_hier()
selftests/cgroup: Check for a single range nohz_full CPUs before running test matrix
cgroup/cpuset: Minor fixes to prstate_housekeeping_conflict()
cgroup/cpuset: Fix incorrect effective_cpus bug in update_cpumasks_hier()
cgroup/cpuset: Properly compute effective xcpus in update_parent_effective_cpumask()
cgroup/cpuset: Find furthest isolated ancestor for invalidated isolated partition
cgroup: don't kill migrated processes through their exiting threads
cgroup/cpuset: check effective mems for a movable-only configuration
cgroup/cpuset: Fix task checking when enabling a partition
cgroup: serialize migrations of newly forked processes
selftests/cgroup: test cgroup.kill with an exited thread group leader
cgroup: fix cgroup.kill for exited thread group leader
selftests/cgroup: Skip test_cpuset_prs.sh test if some required CPUs are offline
selftests/cgroup: Add tests for nohz_full housekeeping check
selftests/cgroup: Add tests for isolcpus housekeeping check
...
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/cgroup/cgroup.c | 30 | ||||
| -rw-r--r-- | kernel/cgroup/cpuset-internal.h | 6 | ||||
| -rw-r--r-- | kernel/cgroup/cpuset.c | 678 |
3 files changed, 440 insertions, 274 deletions
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 14634246a666..a284225f8dc9 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -4303,7 +4303,7 @@ static void __cgroup_kill(struct cgroup *cgrp) cgrp->kill_seq++; spin_unlock_irq(&css_set_lock); - css_task_iter_start(&cgrp->self, CSS_TASK_ITER_PROCS | CSS_TASK_ITER_THREADED, &it); + css_task_iter_start(&cgrp->self, CSS_TASK_ITER_THREADED, &it); while ((task = css_task_iter_next(&it))) { /* Ignore kernel threads here. */ if (task->flags & PF_KTHREAD) @@ -6880,6 +6880,31 @@ static struct cgroup *cgroup_get_from_file(struct file *f) return cgrp; } +/* + * A new threadgroup gets its own cgroup_threadgroup_rwsem. Once its PID is + * visible, a cgroup.procs writer can find the child and lock that rwsem even + * though cgroup_post_fork() has not linked the child to its css_set yet. + * + * cgroup_can_fork() already holds the global read lock, which keeps + * cgroup_enable_per_threadgroup_rwsem stable. Lock the child's rwsem as a + * nested instance until cgroup_post_fork() finishes. Threads share their + * parent's signal_struct and are covered by the existing read lock. + */ +static void cgroup_fork_child_lock(struct task_struct *child) +{ + if (cgroup_enable_per_threadgroup_rwsem && + child->signal != current->signal) + down_read_nested(&child->signal->cgroup_threadgroup_rwsem, + SINGLE_DEPTH_NESTING); +} + +static void cgroup_fork_child_unlock(struct task_struct *child) +{ + if (cgroup_enable_per_threadgroup_rwsem && + child->signal != current->signal) + up_read(&child->signal->cgroup_threadgroup_rwsem); +} + /** * cgroup_css_set_fork - find or create a css_set for a child process * @kargs: the arguments passed to create the child process @@ -7045,6 +7070,7 @@ int cgroup_can_fork(struct task_struct *child, struct kernel_clone_args *kargs) goto out_revert; } while_each_subsys_mask(); + cgroup_fork_child_lock(child); return 0; out_revert: @@ -7079,6 +7105,7 @@ void cgroup_cancel_fork(struct task_struct *child, if (ss->cancel_fork) ss->cancel_fork(child, kargs->cset); + cgroup_fork_child_unlock(child); cgroup_css_set_put_fork(kargs); } @@ -7176,6 +7203,7 @@ void cgroup_post_fork(struct task_struct *child, if (unlikely(kill)) do_send_sig_info(SIGKILL, SEND_SIG_NOINFO, child, PIDTYPE_TGID); + cgroup_fork_child_unlock(child); cgroup_css_set_put_fork(kargs); } diff --git a/kernel/cgroup/cpuset-internal.h b/kernel/cgroup/cpuset-internal.h index 5da9c91295b0..1ba94ffaff9d 100644 --- a/kernel/cgroup/cpuset-internal.h +++ b/kernel/cgroup/cpuset-internal.h @@ -35,6 +35,7 @@ enum prs_errcode { PERR_HKEEPING, PERR_ACCESS, PERR_REMOTE, + PERR_ISOPARENT, }; /* bits in struct cpuset flags field */ @@ -161,6 +162,11 @@ struct cpuset { bool remote_partition; /* + * Set to true when a remote partition is undergoing update. + */ + bool remote_partition_updating; + + /* * number of SCHED_DEADLINE tasks attached to this cpuset, so that we * know when to rebuild associated root domain bandwidth information. */ diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 0ebba2d646c3..3219f927c35e 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -60,6 +60,7 @@ static const char * const perr_strings[] = { [PERR_HKEEPING] = "partition config conflicts with housekeeping setup", [PERR_ACCESS] = "Enable partition not permitted", [PERR_REMOTE] = "Have remote partition underneath", + [PERR_ISOPARENT] = "Root partition is not allowed with isolated parent", }; /* @@ -254,15 +255,6 @@ static inline bool cs_is_member(const struct cpuset *cs) } /* - * Callers should hold callback_lock to modify partition_root_state. - */ -static inline void make_partition_invalid(struct cpuset *cs) -{ - if (cs->partition_root_state > 0) - cs->partition_root_state = -cs->partition_root_state; -} - -/* * Send notification event of whenever partition_root_state changes. */ static inline void notify_partition_change(struct cpuset *cs, int old_prs) @@ -477,10 +469,10 @@ static inline bool partition_is_populated(struct cpuset *cs, rcu_read_lock(); cpuset_for_each_descendant_pre(cp, pos_css, cs) { - if (cp == cs || cp == excluded_child) + if (cp == cs) continue; - if (is_partition_valid(cp)) { + if (cp == excluded_child || is_partition_valid(cp)) { pos_css = css_rightmost_descendant(pos_css); continue; } @@ -1143,43 +1135,6 @@ void cpuset_update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus) css_task_iter_end(&it); } -/** - * compute_effective_cpumask - Compute the effective cpumask of the cpuset - * @new_cpus: the temp variable for the new effective_cpus mask - * @cs: the cpuset the need to recompute the new effective_cpus mask - * @parent: the parent cpuset - * - * For v2, the parent's effective_cpus is inherited if cpumask is empty. - * The result is valid only if the given cpuset isn't a partition root. - */ -static void compute_effective_cpumask(struct cpumask *new_cpus, - struct cpuset *cs, struct cpuset *parent) -{ - bool has_cpus; - - has_cpus = cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus); - if (!has_cpus && is_in_v2_mode()) - cpumask_copy(new_cpus, parent->effective_cpus); -} - -/** - * compute_effective_nodemask - Compute the effective nodemask of the cpuset - * @new_mems: the temp variable for the new effective_mems mask - * @cs: the cpuset the need to recompute the new effective_mems mask - * @parent: the parent cpuset - * - * For v2, the parent's effective_mems is inherited if nodemask is empty. - */ -static void compute_effective_nodemask(nodemask_t *new_mems, - struct cpuset *cs, struct cpuset *parent) -{ - bool has_mems; - - has_mems = nodes_and(*new_mems, cs->mems_allowed, parent->effective_mems); - if (!has_mems && is_in_v2_mode()) - nodes_copy(*new_mems, parent->effective_mems); -} - /* * Commands for update_parent_effective_cpumask */ @@ -1256,6 +1211,16 @@ static void reset_partition_data(struct cpuset *cs) cpumask_copy(cs->effective_cpus, parent->effective_cpus); } +static inline bool is_remote_partition(struct cpuset *cs) +{ + return cs->remote_partition; +} + +static inline bool is_local_partition(struct cpuset *cs) +{ + return is_partition_valid(cs) && !is_remote_partition(cs); +} + /* Return true if isolated_cpus changes. */ static bool isolated_cpu_update(int new_prs, int cpu) { @@ -1299,6 +1264,45 @@ static void isolated_cpus_update(int old_prs, int new_prs, struct cpumask *xcpus } /* + * isolated_cpus_release - Remove the relevant CPUs in @xcpus from isolated_cpus + * @cs: the current cpuset + * @xcpus: the set of effective xcpus to be considered for removal + * + * This function is called when an isolated partition is invalidated or when + * some of its effective xcpus are to be removed from isolated_cpus due to + * changes in parent cpuset. The given @xcpus must be modifiable as the function + * will update its value to determine the right set of CPUs to be removed. + */ +static void isolated_cpus_release(struct cpuset *cs, struct cpumask *xcpus) +{ + struct cpuset *ancestor = parent_cs(cs); + struct cpuset *iancestor = NULL; + + /* + * Search the furthest isolated partition root ancestor available + * until top_cpuset or a remote partition root is hit. + */ + for (; ancestor; ancestor = parent_cs(ancestor)) { + if (ancestor->partition_root_state == PRS_ISOLATED) + iancestor = ancestor; + if (is_remote_partition(ancestor)) + break; + } + + /* + * If a furthest isolated partition acnestor is found, use it to + * determine which of the CPUs in @xcpus should be removed from + * isolated_cpus. Otherwise, all of them will be removed. + */ + if (iancestor && !cpumask_andnot(xcpus, xcpus, iancestor->effective_xcpus)) + return; /* No CPU should be removed from isolated_cpus */ + + spin_lock_irq(&callback_lock); + isolated_cpus_update(PRS_ISOLATED, PRS_ROOT, xcpus); + spin_unlock_irq(&callback_lock); +} + +/* * partition_xcpus_add - Add new exclusive CPUs to partition * @new_prs: new partition_root_state * @parent: parent cpuset @@ -1353,66 +1357,99 @@ static void partition_xcpus_del(int old_prs, struct cpuset *parent, } /* - * isolated_cpus_can_update - check for isolated & nohz_full conflicts - * @add_cpus: cpu mask for cpus that are going to be isolated - * @del_cpus: cpu mask for cpus that are no longer isolated, can be NULL - * Return: false if there is conflict, true otherwise - * - * If nohz_full is enabled and we have isolated CPUs, their combination must - * still leave housekeeping CPUs. + * prstate_housekeeping_conflict - check for partition & housekeeping conflicts + * @new_prs: new partition root state to be checked + * @parent_prs: Parent partition root state + * @new_cpus: the set of new effective xcpus in the current cpuset + * @del_cpus: old CPUs going to be removed from current cpuset, can be NULL + * Return: true if there is conflict, false otherwise * - * TBD: Should consider merging this function into - * prstate_housekeeping_conflict(). + * There are two different housekeeping conflicts to be checked: + * 1) If new_prs is PRS_ROOT, none of the @new_cpus can be a boot-time isolated + * CPU. IOW, the whole @new_cpus must be a subset of HK_TYPE_DOMAIN_BOOT. + * 2) If nohz_full is enabled and we have isolated CPUs, their combination must + * still leave housekeeping CPUs. This check is only needed if new_prs + * differs from parent_prs and one of them is PRS_ISOLATED. */ -static bool isolated_cpus_can_update(struct cpumask *add_cpus, - struct cpumask *del_cpus) +static bool prstate_housekeeping_conflict(int new_prs, int parent_prs, + struct cpumask *new_cpus, + struct cpumask *del_cpus) { cpumask_var_t full_hk_cpus; - int res = true; + int res; - if (!housekeeping_enabled(HK_TYPE_KERNEL_NOISE)) - return true; + WARN_ON_ONCE(parent_prs <= 0); + if (!new_prs) + return false; - if (del_cpus && cpumask_weight_and(del_cpus, - housekeeping_cpumask(HK_TYPE_KERNEL_NOISE))) + /* + * For a currently invalid partition root state, do the check assuming + * that it may be converted back to valid. + * + * There are code paths where an invalid partition may be reverted back + * to a valid partition but we still need to run the housekeeping check + * before the conversion happens to make sure the switch is valid. We + * can either do the negative to positive PRS switch individually in all + * these code paths or do it here which I think will make the code a bit + * easier to read. + */ + if (new_prs < 0) + new_prs = -new_prs; + + if (housekeeping_enabled(HK_TYPE_DOMAIN_BOOT) && (new_prs == PRS_ROOT) && + !cpumask_subset(new_cpus, housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT))) return true; - if (!alloc_cpumask_var(&full_hk_cpus, GFP_KERNEL)) + if (del_cpus && cpumask_empty(del_cpus)) + del_cpus = NULL; + + /* + * Exit 2nd check early and return false if + * 1) there is no boot time isolated nohz_full CPU + * 2) parent and child has the same partition state + * 3) new_prs is PRS_ROOT, parent_prs is PRS_ISOLATED and no CPU will + * be returned back to the parent. + */ + if (!housekeeping_enabled(HK_TYPE_KERNEL_NOISE) || + (new_prs == parent_prs) || (!del_cpus && (new_prs == PRS_ROOT))) return false; + /* + * Now make sure that @new_cpus contains new CPUs to be isolated and + * @del_cpus contains isolated CPUs to be un-isolated. + * With the guards above, @new_cpus can't be NULL after the swap. + */ + if (parent_prs == PRS_ISOLATED) + swap(new_cpus, del_cpus); + + if (!alloc_cpumask_var(&full_hk_cpus, GFP_KERNEL)) + return true; + + /* + * Check if del_cpus may contain a CPU that can become an active + * housekeeping CPU again. + */ + if (del_cpus) { + cpumask_and(full_hk_cpus, housekeeping_cpumask(HK_TYPE_KERNEL_NOISE), + housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT)); + if (cpumask_first_and_and(del_cpus, full_hk_cpus, + cpu_active_mask) < nr_cpu_ids) { + res = false; + goto out; + } + } + cpumask_and(full_hk_cpus, housekeeping_cpumask(HK_TYPE_KERNEL_NOISE), housekeeping_cpumask(HK_TYPE_DOMAIN)); cpumask_andnot(full_hk_cpus, full_hk_cpus, isolated_cpus); cpumask_and(full_hk_cpus, full_hk_cpus, cpu_active_mask); - if (!cpumask_weight_andnot(full_hk_cpus, add_cpus)) - res = false; - + res = !cpumask_weight_andnot(full_hk_cpus, new_cpus); +out: free_cpumask_var(full_hk_cpus); return res; } /* - * prstate_housekeeping_conflict - check for partition & housekeeping conflicts - * @prstate: partition root state to be checked - * @new_cpus: cpu mask - * Return: true if there is conflict, false otherwise - * - * CPUs outside of HK_TYPE_DOMAIN_BOOT, if defined, can only be used in an - * isolated partition. - */ -static bool prstate_housekeeping_conflict(int prstate, struct cpumask *new_cpus) -{ - if (!housekeeping_enabled(HK_TYPE_DOMAIN_BOOT)) - return false; - - if ((prstate != PRS_ISOLATED) && - !cpumask_subset(new_cpus, housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT))) - return true; - - return false; -} - -/* * cpuset_update_sd_hk_unlock - Rebuild sched domains, update HK & unlock * * Update housekeeping cpumasks and rebuild sched domains if necessary and @@ -1548,14 +1585,112 @@ static int compute_trialcs_excpus(struct cpuset *trialcs, struct cpuset *cs) return rm_siblings_excl_cpus(parent, cs, excpus); } -static inline bool is_remote_partition(struct cpuset *cs) +/** + * compute_partition_effective_cpumask - compute effective_cpus for partition + * @cs: partition root cpuset + * @new_ecpus: previously computed effective_cpus to be updated + * + * Compute the effective_cpus of a partition root by scanning effective_xcpus + * of child partition roots and excluding their effective_xcpus. + * + * This has the side effect of invalidating valid child partition roots, + * if necessary. Since it is called from either cpuset_hotplug_update_tasks() + * or update_cpumasks_hier() where parent and children are modified + * successively, we don't need to call update_parent_effective_cpumask() + * and the child's effective_cpus will be updated in later iterations. + */ +static void compute_partition_effective_cpumask(struct cpuset *cs, + struct cpumask *new_ecpus) { - return cs->remote_partition; + struct cgroup_subsys_state *css; + struct cpuset *child; + bool populated = partition_is_populated(cs, NULL); + + /* + * Check child partition roots to see if they should be invalidated + * when all the effective_cpus will be used up and cs is populated. + * It is better to sacrifice a child partition root than invalidating + * the whole subtree. + * + * Return early if the effective_cpus is already empty. + */ + compute_excpus(cs, new_ecpus); + if (!cpumask_and(new_ecpus, new_ecpus, cpu_active_mask)) + return; + + rcu_read_lock(); + cpuset_for_each_child(child, css, cs) { + if (!is_partition_valid(child)) + continue; + + /* + * There shouldn't be a remote partition underneath another + * partition root. + */ + WARN_ON_ONCE(is_remote_partition(child)); + if (populated && cpumask_subset(new_ecpus, child->effective_xcpus)) { + int old_prs = child->partition_root_state; + int parent_prs = cs->partition_root_state; + + /* + * Invalidate child partition and update isolated_cpus + * if necessary. + */ + spin_lock_irq(&callback_lock); + WRITE_ONCE(child->prs_err, PERR_NOCPUS); + if ((old_prs == PRS_ISOLATED) && (parent_prs == PRS_ROOT)) + isolated_cpus_update(PRS_ISOLATED, PRS_ROOT, + child->effective_xcpus); + child->partition_root_state = -old_prs; + spin_unlock_irq(&callback_lock); + notify_partition_change(child, old_prs); + continue; + } + cpumask_andnot(new_ecpus, new_ecpus, + child->effective_xcpus); + } + rcu_read_unlock(); } -static inline bool is_local_partition(struct cpuset *cs) +/** + * compute_effective_cpumask - Compute the effective cpumask of the cpuset + * @new_cpus: the temp variable for the new effective_cpus mask + * @cs: the cpuset the need to recompute the new effective_cpus mask + * @parent: the parent cpuset + * + * For v2, the parent's effective_cpus is inherited if cpumask is empty. + * The result is valid only if the given cpuset isn't a partition root. + */ +static void compute_effective_cpumask(struct cpumask *new_cpus, + struct cpuset *cs, struct cpuset *parent) { - return is_partition_valid(cs) && !is_remote_partition(cs); + bool has_cpus; + + if (is_remote_partition(cs) || + (is_partition_valid(cs) && is_partition_valid(parent))) + return compute_partition_effective_cpumask(cs, new_cpus); + + has_cpus = cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus); + if (!has_cpus && is_in_v2_mode()) + cpumask_copy(new_cpus, parent->effective_cpus); +} + +/** + * compute_effective_nodemask - Compute the effective nodemask of the cpuset + * @new_mems: the temp variable for the new effective_mems mask + * @cs: the cpuset the need to recompute the new effective_mems mask + * @parent: the parent cpuset + * + * For v2, the parent's effective_mems is inherited if nodemask is empty. + */ +static void compute_effective_nodemask(nodemask_t *new_mems, + struct cpuset *cs, struct cpuset *parent) +{ + bool has_mems; + + has_mems = nodes_and(*new_mems, cs->mems_allowed, parent->effective_mems); + if (!has_mems && is_in_v2_mode()) + nodes_copy(*new_mems, parent->effective_mems); } /* @@ -1593,9 +1728,7 @@ static int remote_partition_enable(struct cpuset *cs, int new_prs, return PERR_INVCPUS; if (cpumask_intersects(tmp->new_cpus, subpartitions_cpus)) return PERR_NOCPUS; - if (((new_prs == PRS_ISOLATED) && - !isolated_cpus_can_update(tmp->new_cpus, NULL)) || - prstate_housekeeping_conflict(new_prs, tmp->new_cpus)) + if (prstate_housekeeping_conflict(new_prs, PRS_ROOT, tmp->new_cpus, NULL)) return PERR_HKEEPING; spin_lock_irq(&callback_lock); @@ -1697,13 +1830,14 @@ static void remote_cpus_update(struct cpuset *cs, struct cpumask *xcpus, else if (cpumask_intersects(tmp->addmask, subpartitions_cpus) || cpumask_subset(top_cpuset.effective_cpus, tmp->addmask)) WRITE_ONCE(cs->prs_err, PERR_NOCPUS); - else if ((prs == PRS_ISOLATED) && - !isolated_cpus_can_update(tmp->addmask, tmp->delmask)) + else if (prstate_housekeeping_conflict(prs, PRS_ROOT, + excpus, tmp->delmask)) WRITE_ONCE(cs->prs_err, PERR_HKEEPING); if (cs->prs_err) goto invalidate; } + cs->remote_partition_updating = true; spin_lock_irq(&callback_lock); if (adding) partition_xcpus_add(prs, NULL, tmp->addmask); @@ -1725,6 +1859,7 @@ static void remote_cpus_update(struct cpuset *cs, struct cpumask *xcpus, */ cpuset_update_tasks_cpumask(&top_cpuset, tmp->new_cpus); update_sibling_cpumasks(&top_cpuset, NULL, tmp); + cs->remote_partition_updating = false; return; invalidate: @@ -1735,7 +1870,7 @@ invalidate: * update_parent_effective_cpumask - update effective_cpus mask of parent cpuset * @cs: The cpuset that requests change in partition root state * @cmd: Partition root state change command - * @newmask: Optional new cpumask for partcmd_update + * @newmask: Optional new effective xcpus for partcmd_update * @tmp: Temporary addmask and delmask * Return: 0 or a partition root state error code * @@ -1750,11 +1885,11 @@ invalidate: * given back to parent's effective_cpus. 0 will always be returned. * * For partcmd_update, if the optional newmask is specified, the cpu list is - * to be changed from effective_xcpus to newmask. Otherwise, effective_xcpus is - * assumed to remain the same. The cpuset should either be a valid or invalid - * partition root. The partition root state may change from valid to invalid - * or vice versa. An error code will be returned if transitioning from - * invalid to valid violates the exclusivity rule. + * to be changed from effective_xcpus to newmask. Otherwise, effective_xcpus + * will be computed and a valid partition will be invalidated if it is the new + * cpumask is empty. The cpuset should either be a valid or invalid partition + * root. The partition root state may change from valid to invalid or vice + * versa. * * For partcmd_invalidate, the current partition will be made invalid. * @@ -1777,7 +1912,6 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, int deleting; /* Deleting cpus from parent's effective_cpus */ int old_prs, new_prs; int part_error = PERR_NONE; /* Partition error? */ - struct cpumask *xcpus = user_xcpus(cs); int parent_prs = parent->partition_root_state; bool nocpu; @@ -1820,34 +1954,30 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, if (!newmask && xcpus_empty(cs)) return PERR_CPUSEMPTY; - nocpu = tasks_nocpu_error(parent, cs, xcpus); + /* + * Except for partcmd_disable and non-NULL newmask, use delmask + * to compute the current effective xcpus value and use it for + * tasks_nocpu_error check. + */ + if ((cmd != partcmd_disable) && !newmask) { + compute_excpus(cs, tmp->delmask); + nocpu = tasks_nocpu_error(parent, cs, tmp->delmask); + } if ((cmd == partcmd_enable) || (cmd == partcmd_enablei)) { - /* - * Need to call compute_excpus() in case - * exclusive_cpus not set. Sibling conflict should only happen - * if exclusive_cpus isn't set. - */ - xcpus = tmp->delmask; - if (compute_excpus(cs, xcpus)) - WARN_ON_ONCE(!cpumask_empty(cs->exclusive_cpus)); new_prs = (cmd == partcmd_enable) ? PRS_ROOT : PRS_ISOLATED; /* * Enabling partition root is not allowed if its * effective_xcpus is empty. */ - if (cpumask_empty(xcpus)) + if (cpumask_empty(tmp->delmask)) return PERR_INVCPUS; - if (prstate_housekeeping_conflict(new_prs, xcpus)) + if (prstate_housekeeping_conflict(new_prs, parent_prs, tmp->delmask, NULL)) return PERR_HKEEPING; - if ((new_prs == PRS_ISOLATED) && (new_prs != parent_prs) && - !isolated_cpus_can_update(xcpus, NULL)) - return PERR_HKEEPING; - - if (tasks_nocpu_error(parent, cs, xcpus)) + if (nocpu) return PERR_NOCPUS; /* @@ -1859,16 +1989,14 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, * * Warn if it is not the case. */ - cpumask_and(tmp->new_cpus, xcpus, cpu_active_mask); + cpumask_and(tmp->new_cpus, tmp->delmask, cpu_active_mask); WARN_ON_ONCE(!cpumask_subset(tmp->new_cpus, parent->effective_cpus)); - deleting = true; } else if (cmd == partcmd_disable) { /* * May need to add cpus back to parent's effective_cpus * (and maybe removed from subpartitions_cpus/isolated_cpus) - * for valid partition root. xcpus may contain CPUs that - * shouldn't be removed from the two global cpumasks. + * for valid partition root. */ if (is_partition_valid(cs)) { cpumask_copy(tmp->addmask, cs->effective_xcpus); @@ -1883,9 +2011,13 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, part_error = PERR_CPUSEMPTY; goto write_error; } + if ((parent_prs == PRS_ISOLATED) && (old_prs == -PRS_ROOT)) { + part_error = PERR_ISOPARENT; + goto write_error; + } - /* Check newmask again, whether cpus are available for parent/cs */ - nocpu |= tasks_nocpu_error(parent, cs, newmask); + /* Check newmask whether cpus are available for parent/cs */ + nocpu = tasks_nocpu_error(parent, cs, newmask); /* * partcmd_update with newmask: @@ -1917,22 +2049,6 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, } /* - * TBD: Invalidate a currently valid child root partition may - * still break isolated_cpus_can_update() rule if parent is an - * isolated partition. - */ - if (is_partition_valid(cs) && (old_prs != parent_prs)) { - if ((parent_prs == PRS_ROOT) && - /* Adding to parent means removing isolated CPUs */ - !isolated_cpus_can_update(tmp->delmask, tmp->addmask)) - part_error = PERR_HKEEPING; - if ((parent_prs == PRS_ISOLATED) && - /* Adding to parent means adding isolated CPUs */ - !isolated_cpus_can_update(tmp->addmask, tmp->delmask)) - part_error = PERR_HKEEPING; - } - - /* * The new CPUs to be removed from parent's effective CPUs * must be present. */ @@ -1956,7 +2072,7 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, /* * partcmd_update w/o newmask * - * delmask = effective_xcpus & parent->effective_cpus + * delmask = newly computed effective xcpus value * * This can be called from: * 1) update_cpumasks_hier() @@ -1967,39 +2083,72 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, * * A partition error happens when parent has tasks and all * its effective CPUs will have to be distributed out. + * + * For an isolated partition with a shrunk root partition parent + * we may need to return some or all of its isolated CPUs. */ + nocpu |= cpumask_empty(tmp->delmask); + if ((parent_prs == PRS_ROOT) && (old_prs == PRS_ISOLATED)) { + cpumask_clear(tmp->addmask); + if (nocpu) + cpumask_copy(tmp->addmask, cs->effective_xcpus); + else if (!cpumask_subset(cs->effective_xcpus, parent->effective_xcpus)) + cpumask_andnot(tmp->addmask, cs->effective_xcpus, + parent->effective_xcpus); + if (!cpumask_empty(tmp->addmask)) + isolated_cpus_release(cs, tmp->addmask); + } if (nocpu) { part_error = PERR_NOCPUS; if (is_partition_valid(cs)) adding = cpumask_and(tmp->addmask, cs->effective_xcpus, parent->effective_xcpus); - } else if (is_partition_invalid(cs) && !cpumask_empty(xcpus) && - cpumask_subset(xcpus, parent->effective_xcpus)) { + } else if ((parent_prs == PRS_ISOLATED) && + ((old_prs == PRS_ROOT) || (old_prs == -PRS_ROOT))) { + part_error = PERR_ISOPARENT; + if (old_prs == PRS_ROOT) { + cpumask_copy(tmp->addmask, cs->effective_xcpus); + adding = true; + } + } else if (is_partition_invalid(cs)) { struct cgroup_subsys_state *css; struct cpuset *child; bool exclusive = true; /* - * Convert invalid partition to valid has to - * pass the cpu exclusivity test. + * Convert invalid partition to valid has to pass the + * cpu exclusivity test when cpuset.cpus.exclusive is + * empty as well as the housekeeping check. */ - rcu_read_lock(); - cpuset_for_each_child(child, css, parent) { - if (child == cs) - continue; - if (!cpusets_are_exclusive(cs, child)) { - exclusive = false; - break; + if (prstate_housekeeping_conflict(cs->partition_root_state, + parent_prs, tmp->delmask, NULL)) { + part_error = PERR_HKEEPING; + goto write_error; + } + if (cpumask_empty(cs->exclusive_cpus)) { + rcu_read_lock(); + cpuset_for_each_child(child, css, parent) { + if (child == cs) + continue; + if (!cpusets_are_exclusive(cs, child)) { + exclusive = false; + break; + } } + rcu_read_unlock(); } - rcu_read_unlock(); if (exclusive) - deleting = cpumask_and(tmp->delmask, - xcpus, parent->effective_cpus); + deleting = !cpumask_empty(tmp->delmask); else part_error = PERR_NOTEXCL; } + /* + * When both cs and parent remain valid, parent's + * compute_partition_effective_cpumask() call would have removed + * cs's effective_xcpus from the parent's effective_cpus. So + * there is no need to adjust parent's effective_cpus here. + */ } write_error: @@ -2067,76 +2216,6 @@ write_error: return 0; } -/** - * compute_partition_effective_cpumask - compute effective_cpus for partition - * @cs: partition root cpuset - * @new_ecpus: previously computed effective_cpus to be updated - * - * Compute the effective_cpus of a partition root by scanning effective_xcpus - * of child partition roots and excluding their effective_xcpus. - * - * This has the side effect of invalidating valid child partition roots, - * if necessary. Since it is called from either cpuset_hotplug_update_tasks() - * or update_cpumasks_hier() where parent and children are modified - * successively, we don't need to call update_parent_effective_cpumask() - * and the child's effective_cpus will be updated in later iterations. - * - * Note that rcu_read_lock() is assumed to be held. - */ -static void compute_partition_effective_cpumask(struct cpuset *cs, - struct cpumask *new_ecpus) -{ - struct cgroup_subsys_state *css; - struct cpuset *child; - bool populated = partition_is_populated(cs, NULL); - - /* - * Check child partition roots to see if they should be - * invalidated when - * 1) child effective_xcpus not a subset of new - * excluisve_cpus - * 2) All the effective_cpus will be used up and cp - * has tasks - */ - compute_excpus(cs, new_ecpus); - cpumask_and(new_ecpus, new_ecpus, cpu_active_mask); - - rcu_read_lock(); - cpuset_for_each_child(child, css, cs) { - if (!is_partition_valid(child)) - continue; - - /* - * There shouldn't be a remote partition underneath another - * partition root. - */ - WARN_ON_ONCE(is_remote_partition(child)); - WRITE_ONCE(child->prs_err, 0); - if (!cpumask_subset(child->effective_xcpus, - cs->effective_xcpus)) - WRITE_ONCE(child->prs_err, PERR_INVCPUS); - else if (populated && - cpumask_subset(new_ecpus, child->effective_xcpus)) - WRITE_ONCE(child->prs_err, PERR_NOCPUS); - - if (child->prs_err) { - int old_prs = child->partition_root_state; - - /* - * Invalidate child partition - */ - spin_lock_irq(&callback_lock); - make_partition_invalid(child); - spin_unlock_irq(&callback_lock); - notify_partition_change(child, old_prs); - continue; - } - cpumask_andnot(new_ecpus, new_ecpus, - child->effective_xcpus); - } - rcu_read_unlock(); -} - /* * update_cpumasks_hier - Update effective cpumasks and tasks in the subtree * @cs: the cpuset to consider @@ -2166,16 +2245,19 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp, old_prs = new_prs = cp->partition_root_state; /* - * For child remote partition root (!= cs), we need to call - * remote_cpus_update() if effective_xcpus will be changed. - * Otherwise, we can skip the whole subtree. + * For a child remote partition with remote_partition_updating + * set, we can skip the whole subtree as the this function will + * be called eventually. We can also skip the subtree if the + * effective_xcpus isn't going to be changed. Otherwise, we + * will have to call remote_cpus_update(). * * remote_cpus_update() will reuse tmp->new_cpus only after * its value is being processed. */ if (remote && (cp != cs)) { compute_excpus(cp, tmp->new_cpus); - if (cpumask_equal(cp->effective_xcpus, tmp->new_cpus)) { + if (cp->remote_partition_updating || + cpumask_equal(cp->effective_xcpus, tmp->new_cpus)) { pos_css = css_rightmost_descendant(pos_css); continue; } @@ -2188,10 +2270,7 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp, remote = (new_prs == old_prs); } - if (remote || (is_partition_valid(parent) && is_partition_valid(cp))) - compute_partition_effective_cpumask(cp, tmp->new_cpus); - else - compute_effective_cpumask(tmp->new_cpus, cp, parent); + compute_effective_cpumask(tmp->new_cpus, cp, parent); if (remote) goto get_css; /* Ready to update cpuset data */ @@ -2229,7 +2308,9 @@ update_parent_effective: * cpuset if the parent's effective_cpus changes. */ if ((cp != cs) && old_prs) { - switch (parent->partition_root_state) { + int parent_prs = parent->partition_root_state; + + switch (parent_prs) { case PRS_ROOT: case PRS_ISOLATED: update_parent = true; @@ -2237,12 +2318,20 @@ update_parent_effective: default: /* - * When parent is not a partition root or is - * invalid, child partition roots become - * invalid too. + * When parent is not a valid partition root, + * child partition roots become invalid too. + * + * For a to be invalidated isolated partition, + * call return_isolated_cpus() to remove the + * relevant CPUs from isolated_cpus. */ if (is_partition_valid(cp)) - new_prs = -cp->partition_root_state; + new_prs = -old_prs; + + if (old_prs == PRS_ISOLATED) { + cpumask_copy(tmp->delmask, cp->effective_xcpus); + isolated_cpus_release(cp, tmp->delmask); + } WRITE_ONCE(cp->prs_err, is_partition_invalid(parent) ? PERR_INVPARENT : PERR_NOTPART); @@ -2257,10 +2346,13 @@ get_css: if (update_parent) { update_parent_effective_cpumask(cp, partcmd_update, NULL, tmp); /* - * The cpuset partition_root_state may become - * invalid. Capture it. + * Capture current partition_root_state as it may have + * changed. Recompute effective cpumask if changed. */ new_prs = cp->partition_root_state; + if (new_prs != old_prs) { + compute_effective_cpumask(tmp->new_cpus, cp, parent); + } } spin_lock_irq(&callback_lock); @@ -2391,10 +2483,6 @@ static enum prs_errcode validate_partition(struct cpuset *cs, struct cpuset *tri if (cpumask_empty(trialcs->effective_xcpus)) return PERR_INVCPUS; - if (prstate_housekeeping_conflict(trialcs->partition_root_state, - trialcs->effective_xcpus)) - return PERR_HKEEPING; - if (tasks_nocpu_error(parent, cs, trialcs->effective_xcpus)) return PERR_NOCPUS; @@ -2411,14 +2499,36 @@ static enum prs_errcode validate_partition(struct cpuset *cs, struct cpuset *tri * CPU modifications may cause a partition to be disabled or require state updates. */ static void partition_cpus_change(struct cpuset *cs, struct cpuset *trialcs, - struct tmpmasks *tmp) + struct tmpmasks *tmp) { enum prs_errcode prs_err; + int parent_prs = is_remote_partition(cs) + ? PRS_ROOT : parent_cs(cs)->partition_root_state; if (cs_is_member(cs)) return; prs_err = validate_partition(cs, trialcs); + if (!prs_err && (parent_prs > 0) && + !cpumask_empty(trialcs->effective_xcpus)) { + int prs = cs->partition_root_state; + + /* + * Check for housekeeping conflict by passing the whole + * effective_xcpus as well as those CPUs that are to be + * removed from cs as delmask. + */ + if (prs > 0) + cpumask_andnot(tmp->delmask, cs->effective_xcpus, + trialcs->effective_xcpus); + else + cpumask_clear(tmp->delmask); + + if (prstate_housekeeping_conflict(prs, parent_prs, + trialcs->effective_xcpus, tmp->delmask)) + prs_err = PERR_HKEEPING; + } + if (prs_err) { WRITE_ONCE(cs->prs_err, prs_err); trialcs->prs_err = prs_err; @@ -2774,6 +2884,8 @@ static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems) continue; rcu_read_unlock(); + check_insane_mems_config(new_mems); + spin_lock_irq(&callback_lock); cp->effective_mems = *new_mems; spin_unlock_irq(&callback_lock); @@ -2827,8 +2939,6 @@ static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs, if (retval < 0) return retval; - check_insane_mems_config(&trialcs->mems_allowed); - spin_lock_irq(&callback_lock); cs->mems_allowed = trialcs->mems_allowed; spin_unlock_irq(&callback_lock); @@ -2862,7 +2972,9 @@ static int update_prstate(struct cpuset *cs, int new_prs) int err = PERR_NONE, old_prs = cs->partition_root_state; struct cpuset *parent = parent_cs(cs); struct tmpmasks tmpmask; + int parent_prs = parent->partition_root_state; bool isolcpus_updated = false; + bool disable_partition = false; if (old_prs == new_prs) return 0; @@ -2906,26 +3018,45 @@ static int update_prstate(struct cpuset *cs, int new_prs) enum partition_cmd cmd = (new_prs == PRS_ROOT) ? partcmd_enable : partcmd_enablei; + /* + * It is not allowed to create a root partition under + * an isolated parent partition. + */ + if ((new_prs == PRS_ROOT) && (parent_prs == PRS_ISOLATED)) { + err = PERR_ISOPARENT; + goto out; + } err = update_parent_effective_cpumask(cs, cmd, NULL, &tmpmask); } else { err = remote_partition_enable(cs, new_prs, &tmpmask); } } else if (old_prs && new_prs) { + int parent_prs = is_remote_partition(cs) + ? PRS_ROOT : parent->partition_root_state; + /* * A change in load balance state only, no change in cpumasks. * Need to update isolated_cpus. */ - if (((new_prs == PRS_ISOLATED) && - !isolated_cpus_can_update(cs->effective_xcpus, NULL)) || - prstate_housekeeping_conflict(new_prs, cs->effective_xcpus)) + if (prstate_housekeeping_conflict(new_prs, parent_prs, + cs->effective_xcpus, NULL)) err = PERR_HKEEPING; - else + else if (parent_prs == PRS_ISOLATED) + err = PERR_ISOPARENT; + + if (!err) isolcpus_updated = true; + else + disable_partition = true; } else { /* * Switching back to member is always allowed even if it * disables child partitions. */ + disable_partition = true; + } + + if (disable_partition) { if (is_remote_partition(cs)) remote_partition_disable(cs, &tmpmask); else @@ -2934,7 +3065,7 @@ static int update_prstate(struct cpuset *cs, int new_prs) /* * Invalidation of child partitions will be done in - * update_cpumasks_hier(). + * update_cpumasks_hier() below. */ } out: @@ -2953,8 +3084,8 @@ out: isolated_cpus_update(old_prs, new_prs, cs->effective_xcpus); spin_unlock_irq(&callback_lock); - /* Force update if switching back to member & update effective_xcpus */ - update_cpumasks_hier(cs, &tmpmask, !new_prs); + /* Force update if partition is disabled & update effective_xcpus */ + update_cpumasks_hier(cs, &tmpmask, disable_partition); /* A newly created partition must have effective_xcpus set */ WARN_ON_ONCE(!old_prs && (new_prs > 0) @@ -3852,7 +3983,6 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp) static nodemask_t new_mems; bool cpus_updated; bool mems_updated; - bool remote; int partcmd = -1; struct cpuset *parent; @@ -3864,21 +3994,12 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp) if (!tmp || !cs->partition_root_state) goto update_tasks; - /* - * Compute effective_cpus for valid partition root, may invalidate - * child partition roots if necessary. - */ - remote = is_remote_partition(cs); - if (remote || (is_partition_valid(cs) && is_partition_valid(parent))) - compute_partition_effective_cpumask(cs, &new_cpus); - - if (remote && (cpumask_empty(subpartitions_cpus) || - (cpumask_empty(&new_cpus) && - partition_is_populated(cs, NULL)))) { + if (is_remote_partition(cs) && (cpumask_empty(subpartitions_cpus) || + (cpumask_empty(&new_cpus) && + partition_is_populated(cs, NULL)))) { WRITE_ONCE(cs->prs_err, PERR_HOTPLUG); remote_partition_disable(cs, tmp); compute_effective_cpumask(&new_cpus, cs, parent); - remote = false; } /* @@ -3904,8 +4025,8 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp) if (partcmd >= 0) { update_parent_effective_cpumask(cs, partcmd, NULL, tmp); - if ((partcmd == partcmd_invalidate) || is_partition_valid(cs)) { - compute_partition_effective_cpumask(cs, &new_cpus); + if (is_partition_valid(cs) || (partcmd == partcmd_invalidate)) { + compute_effective_cpumask(&new_cpus, cs, parent); cpuset_force_rebuild(); } } @@ -4265,10 +4386,21 @@ nodemask_t cpuset_mems_allowed(struct task_struct *tsk) * @nodemask: the nodemask to be checked * * Are any of the nodes in the nodemask allowed in current->mems_allowed? + * + * A cpuset update rebinds the task's mempolicy and changes mems_allowed in + * separate steps, so retry if an update raced with the check. */ int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask) { - return nodes_intersects(*nodemask, current->mems_allowed); + unsigned int seq; + int ret; + + do { + seq = read_mems_allowed_begin(); + ret = nodes_intersects(*nodemask, current->mems_allowed); + } while (read_mems_allowed_retry(seq)); + + return ret; } /* |
