summaryrefslogtreecommitdiff
path: root/kernel
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2026-10-29 07:35:19 -1000
committerLinus Torvalds <torvalds@linux-foundation.org>2026-10-29 07:35:19 -1000
commit2152c755f76a9170218a098f79e2b081a08b1d6c (patch)
treede8d773332f39aba7ad4ac190cdfc8e5bb1e3769 /kernel
parentc339f4010dea8b0d754f84ace70cd1b089878ef8 (diff)
parent518cec2def9992192291f5028edc077fb2e475e9 (diff)
Merge tag 'cgroup-for-7.4-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup
Pull cgroup fixes from Tejun Heo: - cgroup.kill did not kill a process whose thread group leader had exited while other threads still ran, nor those threads once moved to another cgroup, letting a process survive its container being killed - A cgroup.procs write under favordynmods could migrate a newly forked process before fork linked it to its cgroup, corrupting cgroup task lists - cpuset partition fixes for nested hierarchies: invalidation, revival, shrinking and switches between root and isolated could leak isolated CPUs, skip or miscompute the housekeeping check, or leave wrong effective CPUs: - A root partition can no longer be created under an isolated parent or stay valid under one, as disabling it could leave no housekeeping CPUs - Enabling a partition failed when a grandchild held tasks, although the parent had none of its own - Separately, an allocation racing with a cpuset.mems update could drop its mempolicy nodemask and land on the wrong node, and a memoryless node in cpuset.mems triggered a spurious "movable nodes only" warning - Two no-functional-change cleanups of the effective CPU computation * tag 'cgroup-for-7.4-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup: (27 commits) cgroup/cpuset: Integrate compute_partition_effective_cpumask() into compute_effective_cpumask() cgroup/cpuset: Relocate compute_*effective_cpumask() and compute_effective_nodemask() cgroup/cpuset: Release invalid isolated CPUs with a shrunk root partition parent cgroup/cpuset: Release invalid isolated CPUs in compute_partition_effective_cpumask() cgroup/cpuset: Make sure compute_partition_effective_cpumask() is only called for valid partition cgroup/cpuset: Prevent recursive call of remote_cpus_update() via update_cpumasks_hier() selftests/cgroup: Check for a single range nohz_full CPUs before running test matrix cgroup/cpuset: Minor fixes to prstate_housekeeping_conflict() cgroup/cpuset: Fix incorrect effective_cpus bug in update_cpumasks_hier() cgroup/cpuset: Properly compute effective xcpus in update_parent_effective_cpumask() cgroup/cpuset: Find furthest isolated ancestor for invalidated isolated partition cgroup: don't kill migrated processes through their exiting threads cgroup/cpuset: check effective mems for a movable-only configuration cgroup/cpuset: Fix task checking when enabling a partition cgroup: serialize migrations of newly forked processes selftests/cgroup: test cgroup.kill with an exited thread group leader cgroup: fix cgroup.kill for exited thread group leader selftests/cgroup: Skip test_cpuset_prs.sh test if some required CPUs are offline selftests/cgroup: Add tests for nohz_full housekeeping check selftests/cgroup: Add tests for isolcpus housekeeping check ...
Diffstat (limited to 'kernel')
-rw-r--r--kernel/cgroup/cgroup.c30
-rw-r--r--kernel/cgroup/cpuset-internal.h6
-rw-r--r--kernel/cgroup/cpuset.c678
3 files changed, 440 insertions, 274 deletions
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 14634246a666..a284225f8dc9 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -4303,7 +4303,7 @@ static void __cgroup_kill(struct cgroup *cgrp)
cgrp->kill_seq++;
spin_unlock_irq(&css_set_lock);
- css_task_iter_start(&cgrp->self, CSS_TASK_ITER_PROCS | CSS_TASK_ITER_THREADED, &it);
+ css_task_iter_start(&cgrp->self, CSS_TASK_ITER_THREADED, &it);
while ((task = css_task_iter_next(&it))) {
/* Ignore kernel threads here. */
if (task->flags & PF_KTHREAD)
@@ -6880,6 +6880,31 @@ static struct cgroup *cgroup_get_from_file(struct file *f)
return cgrp;
}
+/*
+ * A new threadgroup gets its own cgroup_threadgroup_rwsem. Once its PID is
+ * visible, a cgroup.procs writer can find the child and lock that rwsem even
+ * though cgroup_post_fork() has not linked the child to its css_set yet.
+ *
+ * cgroup_can_fork() already holds the global read lock, which keeps
+ * cgroup_enable_per_threadgroup_rwsem stable. Lock the child's rwsem as a
+ * nested instance until cgroup_post_fork() finishes. Threads share their
+ * parent's signal_struct and are covered by the existing read lock.
+ */
+static void cgroup_fork_child_lock(struct task_struct *child)
+{
+ if (cgroup_enable_per_threadgroup_rwsem &&
+ child->signal != current->signal)
+ down_read_nested(&child->signal->cgroup_threadgroup_rwsem,
+ SINGLE_DEPTH_NESTING);
+}
+
+static void cgroup_fork_child_unlock(struct task_struct *child)
+{
+ if (cgroup_enable_per_threadgroup_rwsem &&
+ child->signal != current->signal)
+ up_read(&child->signal->cgroup_threadgroup_rwsem);
+}
+
/**
* cgroup_css_set_fork - find or create a css_set for a child process
* @kargs: the arguments passed to create the child process
@@ -7045,6 +7070,7 @@ int cgroup_can_fork(struct task_struct *child, struct kernel_clone_args *kargs)
goto out_revert;
} while_each_subsys_mask();
+ cgroup_fork_child_lock(child);
return 0;
out_revert:
@@ -7079,6 +7105,7 @@ void cgroup_cancel_fork(struct task_struct *child,
if (ss->cancel_fork)
ss->cancel_fork(child, kargs->cset);
+ cgroup_fork_child_unlock(child);
cgroup_css_set_put_fork(kargs);
}
@@ -7176,6 +7203,7 @@ void cgroup_post_fork(struct task_struct *child,
if (unlikely(kill))
do_send_sig_info(SIGKILL, SEND_SIG_NOINFO, child, PIDTYPE_TGID);
+ cgroup_fork_child_unlock(child);
cgroup_css_set_put_fork(kargs);
}
diff --git a/kernel/cgroup/cpuset-internal.h b/kernel/cgroup/cpuset-internal.h
index 5da9c91295b0..1ba94ffaff9d 100644
--- a/kernel/cgroup/cpuset-internal.h
+++ b/kernel/cgroup/cpuset-internal.h
@@ -35,6 +35,7 @@ enum prs_errcode {
PERR_HKEEPING,
PERR_ACCESS,
PERR_REMOTE,
+ PERR_ISOPARENT,
};
/* bits in struct cpuset flags field */
@@ -161,6 +162,11 @@ struct cpuset {
bool remote_partition;
/*
+ * Set to true when a remote partition is undergoing update.
+ */
+ bool remote_partition_updating;
+
+ /*
* number of SCHED_DEADLINE tasks attached to this cpuset, so that we
* know when to rebuild associated root domain bandwidth information.
*/
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index 0ebba2d646c3..3219f927c35e 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -60,6 +60,7 @@ static const char * const perr_strings[] = {
[PERR_HKEEPING] = "partition config conflicts with housekeeping setup",
[PERR_ACCESS] = "Enable partition not permitted",
[PERR_REMOTE] = "Have remote partition underneath",
+ [PERR_ISOPARENT] = "Root partition is not allowed with isolated parent",
};
/*
@@ -254,15 +255,6 @@ static inline bool cs_is_member(const struct cpuset *cs)
}
/*
- * Callers should hold callback_lock to modify partition_root_state.
- */
-static inline void make_partition_invalid(struct cpuset *cs)
-{
- if (cs->partition_root_state > 0)
- cs->partition_root_state = -cs->partition_root_state;
-}
-
-/*
* Send notification event of whenever partition_root_state changes.
*/
static inline void notify_partition_change(struct cpuset *cs, int old_prs)
@@ -477,10 +469,10 @@ static inline bool partition_is_populated(struct cpuset *cs,
rcu_read_lock();
cpuset_for_each_descendant_pre(cp, pos_css, cs) {
- if (cp == cs || cp == excluded_child)
+ if (cp == cs)
continue;
- if (is_partition_valid(cp)) {
+ if (cp == excluded_child || is_partition_valid(cp)) {
pos_css = css_rightmost_descendant(pos_css);
continue;
}
@@ -1143,43 +1135,6 @@ void cpuset_update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus)
css_task_iter_end(&it);
}
-/**
- * compute_effective_cpumask - Compute the effective cpumask of the cpuset
- * @new_cpus: the temp variable for the new effective_cpus mask
- * @cs: the cpuset the need to recompute the new effective_cpus mask
- * @parent: the parent cpuset
- *
- * For v2, the parent's effective_cpus is inherited if cpumask is empty.
- * The result is valid only if the given cpuset isn't a partition root.
- */
-static void compute_effective_cpumask(struct cpumask *new_cpus,
- struct cpuset *cs, struct cpuset *parent)
-{
- bool has_cpus;
-
- has_cpus = cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus);
- if (!has_cpus && is_in_v2_mode())
- cpumask_copy(new_cpus, parent->effective_cpus);
-}
-
-/**
- * compute_effective_nodemask - Compute the effective nodemask of the cpuset
- * @new_mems: the temp variable for the new effective_mems mask
- * @cs: the cpuset the need to recompute the new effective_mems mask
- * @parent: the parent cpuset
- *
- * For v2, the parent's effective_mems is inherited if nodemask is empty.
- */
-static void compute_effective_nodemask(nodemask_t *new_mems,
- struct cpuset *cs, struct cpuset *parent)
-{
- bool has_mems;
-
- has_mems = nodes_and(*new_mems, cs->mems_allowed, parent->effective_mems);
- if (!has_mems && is_in_v2_mode())
- nodes_copy(*new_mems, parent->effective_mems);
-}
-
/*
* Commands for update_parent_effective_cpumask
*/
@@ -1256,6 +1211,16 @@ static void reset_partition_data(struct cpuset *cs)
cpumask_copy(cs->effective_cpus, parent->effective_cpus);
}
+static inline bool is_remote_partition(struct cpuset *cs)
+{
+ return cs->remote_partition;
+}
+
+static inline bool is_local_partition(struct cpuset *cs)
+{
+ return is_partition_valid(cs) && !is_remote_partition(cs);
+}
+
/* Return true if isolated_cpus changes. */
static bool isolated_cpu_update(int new_prs, int cpu)
{
@@ -1299,6 +1264,45 @@ static void isolated_cpus_update(int old_prs, int new_prs, struct cpumask *xcpus
}
/*
+ * isolated_cpus_release - Remove the relevant CPUs in @xcpus from isolated_cpus
+ * @cs: the current cpuset
+ * @xcpus: the set of effective xcpus to be considered for removal
+ *
+ * This function is called when an isolated partition is invalidated or when
+ * some of its effective xcpus are to be removed from isolated_cpus due to
+ * changes in parent cpuset. The given @xcpus must be modifiable as the function
+ * will update its value to determine the right set of CPUs to be removed.
+ */
+static void isolated_cpus_release(struct cpuset *cs, struct cpumask *xcpus)
+{
+ struct cpuset *ancestor = parent_cs(cs);
+ struct cpuset *iancestor = NULL;
+
+ /*
+ * Search the furthest isolated partition root ancestor available
+ * until top_cpuset or a remote partition root is hit.
+ */
+ for (; ancestor; ancestor = parent_cs(ancestor)) {
+ if (ancestor->partition_root_state == PRS_ISOLATED)
+ iancestor = ancestor;
+ if (is_remote_partition(ancestor))
+ break;
+ }
+
+ /*
+ * If a furthest isolated partition acnestor is found, use it to
+ * determine which of the CPUs in @xcpus should be removed from
+ * isolated_cpus. Otherwise, all of them will be removed.
+ */
+ if (iancestor && !cpumask_andnot(xcpus, xcpus, iancestor->effective_xcpus))
+ return; /* No CPU should be removed from isolated_cpus */
+
+ spin_lock_irq(&callback_lock);
+ isolated_cpus_update(PRS_ISOLATED, PRS_ROOT, xcpus);
+ spin_unlock_irq(&callback_lock);
+}
+
+/*
* partition_xcpus_add - Add new exclusive CPUs to partition
* @new_prs: new partition_root_state
* @parent: parent cpuset
@@ -1353,66 +1357,99 @@ static void partition_xcpus_del(int old_prs, struct cpuset *parent,
}
/*
- * isolated_cpus_can_update - check for isolated & nohz_full conflicts
- * @add_cpus: cpu mask for cpus that are going to be isolated
- * @del_cpus: cpu mask for cpus that are no longer isolated, can be NULL
- * Return: false if there is conflict, true otherwise
- *
- * If nohz_full is enabled and we have isolated CPUs, their combination must
- * still leave housekeeping CPUs.
+ * prstate_housekeeping_conflict - check for partition & housekeeping conflicts
+ * @new_prs: new partition root state to be checked
+ * @parent_prs: Parent partition root state
+ * @new_cpus: the set of new effective xcpus in the current cpuset
+ * @del_cpus: old CPUs going to be removed from current cpuset, can be NULL
+ * Return: true if there is conflict, false otherwise
*
- * TBD: Should consider merging this function into
- * prstate_housekeeping_conflict().
+ * There are two different housekeeping conflicts to be checked:
+ * 1) If new_prs is PRS_ROOT, none of the @new_cpus can be a boot-time isolated
+ * CPU. IOW, the whole @new_cpus must be a subset of HK_TYPE_DOMAIN_BOOT.
+ * 2) If nohz_full is enabled and we have isolated CPUs, their combination must
+ * still leave housekeeping CPUs. This check is only needed if new_prs
+ * differs from parent_prs and one of them is PRS_ISOLATED.
*/
-static bool isolated_cpus_can_update(struct cpumask *add_cpus,
- struct cpumask *del_cpus)
+static bool prstate_housekeeping_conflict(int new_prs, int parent_prs,
+ struct cpumask *new_cpus,
+ struct cpumask *del_cpus)
{
cpumask_var_t full_hk_cpus;
- int res = true;
+ int res;
- if (!housekeeping_enabled(HK_TYPE_KERNEL_NOISE))
- return true;
+ WARN_ON_ONCE(parent_prs <= 0);
+ if (!new_prs)
+ return false;
- if (del_cpus && cpumask_weight_and(del_cpus,
- housekeeping_cpumask(HK_TYPE_KERNEL_NOISE)))
+ /*
+ * For a currently invalid partition root state, do the check assuming
+ * that it may be converted back to valid.
+ *
+ * There are code paths where an invalid partition may be reverted back
+ * to a valid partition but we still need to run the housekeeping check
+ * before the conversion happens to make sure the switch is valid. We
+ * can either do the negative to positive PRS switch individually in all
+ * these code paths or do it here which I think will make the code a bit
+ * easier to read.
+ */
+ if (new_prs < 0)
+ new_prs = -new_prs;
+
+ if (housekeeping_enabled(HK_TYPE_DOMAIN_BOOT) && (new_prs == PRS_ROOT) &&
+ !cpumask_subset(new_cpus, housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT)))
return true;
- if (!alloc_cpumask_var(&full_hk_cpus, GFP_KERNEL))
+ if (del_cpus && cpumask_empty(del_cpus))
+ del_cpus = NULL;
+
+ /*
+ * Exit 2nd check early and return false if
+ * 1) there is no boot time isolated nohz_full CPU
+ * 2) parent and child has the same partition state
+ * 3) new_prs is PRS_ROOT, parent_prs is PRS_ISOLATED and no CPU will
+ * be returned back to the parent.
+ */
+ if (!housekeeping_enabled(HK_TYPE_KERNEL_NOISE) ||
+ (new_prs == parent_prs) || (!del_cpus && (new_prs == PRS_ROOT)))
return false;
+ /*
+ * Now make sure that @new_cpus contains new CPUs to be isolated and
+ * @del_cpus contains isolated CPUs to be un-isolated.
+ * With the guards above, @new_cpus can't be NULL after the swap.
+ */
+ if (parent_prs == PRS_ISOLATED)
+ swap(new_cpus, del_cpus);
+
+ if (!alloc_cpumask_var(&full_hk_cpus, GFP_KERNEL))
+ return true;
+
+ /*
+ * Check if del_cpus may contain a CPU that can become an active
+ * housekeeping CPU again.
+ */
+ if (del_cpus) {
+ cpumask_and(full_hk_cpus, housekeeping_cpumask(HK_TYPE_KERNEL_NOISE),
+ housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT));
+ if (cpumask_first_and_and(del_cpus, full_hk_cpus,
+ cpu_active_mask) < nr_cpu_ids) {
+ res = false;
+ goto out;
+ }
+ }
+
cpumask_and(full_hk_cpus, housekeeping_cpumask(HK_TYPE_KERNEL_NOISE),
housekeeping_cpumask(HK_TYPE_DOMAIN));
cpumask_andnot(full_hk_cpus, full_hk_cpus, isolated_cpus);
cpumask_and(full_hk_cpus, full_hk_cpus, cpu_active_mask);
- if (!cpumask_weight_andnot(full_hk_cpus, add_cpus))
- res = false;
-
+ res = !cpumask_weight_andnot(full_hk_cpus, new_cpus);
+out:
free_cpumask_var(full_hk_cpus);
return res;
}
/*
- * prstate_housekeeping_conflict - check for partition & housekeeping conflicts
- * @prstate: partition root state to be checked
- * @new_cpus: cpu mask
- * Return: true if there is conflict, false otherwise
- *
- * CPUs outside of HK_TYPE_DOMAIN_BOOT, if defined, can only be used in an
- * isolated partition.
- */
-static bool prstate_housekeeping_conflict(int prstate, struct cpumask *new_cpus)
-{
- if (!housekeeping_enabled(HK_TYPE_DOMAIN_BOOT))
- return false;
-
- if ((prstate != PRS_ISOLATED) &&
- !cpumask_subset(new_cpus, housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT)))
- return true;
-
- return false;
-}
-
-/*
* cpuset_update_sd_hk_unlock - Rebuild sched domains, update HK & unlock
*
* Update housekeeping cpumasks and rebuild sched domains if necessary and
@@ -1548,14 +1585,112 @@ static int compute_trialcs_excpus(struct cpuset *trialcs, struct cpuset *cs)
return rm_siblings_excl_cpus(parent, cs, excpus);
}
-static inline bool is_remote_partition(struct cpuset *cs)
+/**
+ * compute_partition_effective_cpumask - compute effective_cpus for partition
+ * @cs: partition root cpuset
+ * @new_ecpus: previously computed effective_cpus to be updated
+ *
+ * Compute the effective_cpus of a partition root by scanning effective_xcpus
+ * of child partition roots and excluding their effective_xcpus.
+ *
+ * This has the side effect of invalidating valid child partition roots,
+ * if necessary. Since it is called from either cpuset_hotplug_update_tasks()
+ * or update_cpumasks_hier() where parent and children are modified
+ * successively, we don't need to call update_parent_effective_cpumask()
+ * and the child's effective_cpus will be updated in later iterations.
+ */
+static void compute_partition_effective_cpumask(struct cpuset *cs,
+ struct cpumask *new_ecpus)
{
- return cs->remote_partition;
+ struct cgroup_subsys_state *css;
+ struct cpuset *child;
+ bool populated = partition_is_populated(cs, NULL);
+
+ /*
+ * Check child partition roots to see if they should be invalidated
+ * when all the effective_cpus will be used up and cs is populated.
+ * It is better to sacrifice a child partition root than invalidating
+ * the whole subtree.
+ *
+ * Return early if the effective_cpus is already empty.
+ */
+ compute_excpus(cs, new_ecpus);
+ if (!cpumask_and(new_ecpus, new_ecpus, cpu_active_mask))
+ return;
+
+ rcu_read_lock();
+ cpuset_for_each_child(child, css, cs) {
+ if (!is_partition_valid(child))
+ continue;
+
+ /*
+ * There shouldn't be a remote partition underneath another
+ * partition root.
+ */
+ WARN_ON_ONCE(is_remote_partition(child));
+ if (populated && cpumask_subset(new_ecpus, child->effective_xcpus)) {
+ int old_prs = child->partition_root_state;
+ int parent_prs = cs->partition_root_state;
+
+ /*
+ * Invalidate child partition and update isolated_cpus
+ * if necessary.
+ */
+ spin_lock_irq(&callback_lock);
+ WRITE_ONCE(child->prs_err, PERR_NOCPUS);
+ if ((old_prs == PRS_ISOLATED) && (parent_prs == PRS_ROOT))
+ isolated_cpus_update(PRS_ISOLATED, PRS_ROOT,
+ child->effective_xcpus);
+ child->partition_root_state = -old_prs;
+ spin_unlock_irq(&callback_lock);
+ notify_partition_change(child, old_prs);
+ continue;
+ }
+ cpumask_andnot(new_ecpus, new_ecpus,
+ child->effective_xcpus);
+ }
+ rcu_read_unlock();
}
-static inline bool is_local_partition(struct cpuset *cs)
+/**
+ * compute_effective_cpumask - Compute the effective cpumask of the cpuset
+ * @new_cpus: the temp variable for the new effective_cpus mask
+ * @cs: the cpuset the need to recompute the new effective_cpus mask
+ * @parent: the parent cpuset
+ *
+ * For v2, the parent's effective_cpus is inherited if cpumask is empty.
+ * The result is valid only if the given cpuset isn't a partition root.
+ */
+static void compute_effective_cpumask(struct cpumask *new_cpus,
+ struct cpuset *cs, struct cpuset *parent)
{
- return is_partition_valid(cs) && !is_remote_partition(cs);
+ bool has_cpus;
+
+ if (is_remote_partition(cs) ||
+ (is_partition_valid(cs) && is_partition_valid(parent)))
+ return compute_partition_effective_cpumask(cs, new_cpus);
+
+ has_cpus = cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus);
+ if (!has_cpus && is_in_v2_mode())
+ cpumask_copy(new_cpus, parent->effective_cpus);
+}
+
+/**
+ * compute_effective_nodemask - Compute the effective nodemask of the cpuset
+ * @new_mems: the temp variable for the new effective_mems mask
+ * @cs: the cpuset the need to recompute the new effective_mems mask
+ * @parent: the parent cpuset
+ *
+ * For v2, the parent's effective_mems is inherited if nodemask is empty.
+ */
+static void compute_effective_nodemask(nodemask_t *new_mems,
+ struct cpuset *cs, struct cpuset *parent)
+{
+ bool has_mems;
+
+ has_mems = nodes_and(*new_mems, cs->mems_allowed, parent->effective_mems);
+ if (!has_mems && is_in_v2_mode())
+ nodes_copy(*new_mems, parent->effective_mems);
}
/*
@@ -1593,9 +1728,7 @@ static int remote_partition_enable(struct cpuset *cs, int new_prs,
return PERR_INVCPUS;
if (cpumask_intersects(tmp->new_cpus, subpartitions_cpus))
return PERR_NOCPUS;
- if (((new_prs == PRS_ISOLATED) &&
- !isolated_cpus_can_update(tmp->new_cpus, NULL)) ||
- prstate_housekeeping_conflict(new_prs, tmp->new_cpus))
+ if (prstate_housekeeping_conflict(new_prs, PRS_ROOT, tmp->new_cpus, NULL))
return PERR_HKEEPING;
spin_lock_irq(&callback_lock);
@@ -1697,13 +1830,14 @@ static void remote_cpus_update(struct cpuset *cs, struct cpumask *xcpus,
else if (cpumask_intersects(tmp->addmask, subpartitions_cpus) ||
cpumask_subset(top_cpuset.effective_cpus, tmp->addmask))
WRITE_ONCE(cs->prs_err, PERR_NOCPUS);
- else if ((prs == PRS_ISOLATED) &&
- !isolated_cpus_can_update(tmp->addmask, tmp->delmask))
+ else if (prstate_housekeeping_conflict(prs, PRS_ROOT,
+ excpus, tmp->delmask))
WRITE_ONCE(cs->prs_err, PERR_HKEEPING);
if (cs->prs_err)
goto invalidate;
}
+ cs->remote_partition_updating = true;
spin_lock_irq(&callback_lock);
if (adding)
partition_xcpus_add(prs, NULL, tmp->addmask);
@@ -1725,6 +1859,7 @@ static void remote_cpus_update(struct cpuset *cs, struct cpumask *xcpus,
*/
cpuset_update_tasks_cpumask(&top_cpuset, tmp->new_cpus);
update_sibling_cpumasks(&top_cpuset, NULL, tmp);
+ cs->remote_partition_updating = false;
return;
invalidate:
@@ -1735,7 +1870,7 @@ invalidate:
* update_parent_effective_cpumask - update effective_cpus mask of parent cpuset
* @cs: The cpuset that requests change in partition root state
* @cmd: Partition root state change command
- * @newmask: Optional new cpumask for partcmd_update
+ * @newmask: Optional new effective xcpus for partcmd_update
* @tmp: Temporary addmask and delmask
* Return: 0 or a partition root state error code
*
@@ -1750,11 +1885,11 @@ invalidate:
* given back to parent's effective_cpus. 0 will always be returned.
*
* For partcmd_update, if the optional newmask is specified, the cpu list is
- * to be changed from effective_xcpus to newmask. Otherwise, effective_xcpus is
- * assumed to remain the same. The cpuset should either be a valid or invalid
- * partition root. The partition root state may change from valid to invalid
- * or vice versa. An error code will be returned if transitioning from
- * invalid to valid violates the exclusivity rule.
+ * to be changed from effective_xcpus to newmask. Otherwise, effective_xcpus
+ * will be computed and a valid partition will be invalidated if it is the new
+ * cpumask is empty. The cpuset should either be a valid or invalid partition
+ * root. The partition root state may change from valid to invalid or vice
+ * versa.
*
* For partcmd_invalidate, the current partition will be made invalid.
*
@@ -1777,7 +1912,6 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
int deleting; /* Deleting cpus from parent's effective_cpus */
int old_prs, new_prs;
int part_error = PERR_NONE; /* Partition error? */
- struct cpumask *xcpus = user_xcpus(cs);
int parent_prs = parent->partition_root_state;
bool nocpu;
@@ -1820,34 +1954,30 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
if (!newmask && xcpus_empty(cs))
return PERR_CPUSEMPTY;
- nocpu = tasks_nocpu_error(parent, cs, xcpus);
+ /*
+ * Except for partcmd_disable and non-NULL newmask, use delmask
+ * to compute the current effective xcpus value and use it for
+ * tasks_nocpu_error check.
+ */
+ if ((cmd != partcmd_disable) && !newmask) {
+ compute_excpus(cs, tmp->delmask);
+ nocpu = tasks_nocpu_error(parent, cs, tmp->delmask);
+ }
if ((cmd == partcmd_enable) || (cmd == partcmd_enablei)) {
- /*
- * Need to call compute_excpus() in case
- * exclusive_cpus not set. Sibling conflict should only happen
- * if exclusive_cpus isn't set.
- */
- xcpus = tmp->delmask;
- if (compute_excpus(cs, xcpus))
- WARN_ON_ONCE(!cpumask_empty(cs->exclusive_cpus));
new_prs = (cmd == partcmd_enable) ? PRS_ROOT : PRS_ISOLATED;
/*
* Enabling partition root is not allowed if its
* effective_xcpus is empty.
*/
- if (cpumask_empty(xcpus))
+ if (cpumask_empty(tmp->delmask))
return PERR_INVCPUS;
- if (prstate_housekeeping_conflict(new_prs, xcpus))
+ if (prstate_housekeeping_conflict(new_prs, parent_prs, tmp->delmask, NULL))
return PERR_HKEEPING;
- if ((new_prs == PRS_ISOLATED) && (new_prs != parent_prs) &&
- !isolated_cpus_can_update(xcpus, NULL))
- return PERR_HKEEPING;
-
- if (tasks_nocpu_error(parent, cs, xcpus))
+ if (nocpu)
return PERR_NOCPUS;
/*
@@ -1859,16 +1989,14 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
*
* Warn if it is not the case.
*/
- cpumask_and(tmp->new_cpus, xcpus, cpu_active_mask);
+ cpumask_and(tmp->new_cpus, tmp->delmask, cpu_active_mask);
WARN_ON_ONCE(!cpumask_subset(tmp->new_cpus, parent->effective_cpus));
-
deleting = true;
} else if (cmd == partcmd_disable) {
/*
* May need to add cpus back to parent's effective_cpus
* (and maybe removed from subpartitions_cpus/isolated_cpus)
- * for valid partition root. xcpus may contain CPUs that
- * shouldn't be removed from the two global cpumasks.
+ * for valid partition root.
*/
if (is_partition_valid(cs)) {
cpumask_copy(tmp->addmask, cs->effective_xcpus);
@@ -1883,9 +2011,13 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
part_error = PERR_CPUSEMPTY;
goto write_error;
}
+ if ((parent_prs == PRS_ISOLATED) && (old_prs == -PRS_ROOT)) {
+ part_error = PERR_ISOPARENT;
+ goto write_error;
+ }
- /* Check newmask again, whether cpus are available for parent/cs */
- nocpu |= tasks_nocpu_error(parent, cs, newmask);
+ /* Check newmask whether cpus are available for parent/cs */
+ nocpu = tasks_nocpu_error(parent, cs, newmask);
/*
* partcmd_update with newmask:
@@ -1917,22 +2049,6 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
}
/*
- * TBD: Invalidate a currently valid child root partition may
- * still break isolated_cpus_can_update() rule if parent is an
- * isolated partition.
- */
- if (is_partition_valid(cs) && (old_prs != parent_prs)) {
- if ((parent_prs == PRS_ROOT) &&
- /* Adding to parent means removing isolated CPUs */
- !isolated_cpus_can_update(tmp->delmask, tmp->addmask))
- part_error = PERR_HKEEPING;
- if ((parent_prs == PRS_ISOLATED) &&
- /* Adding to parent means adding isolated CPUs */
- !isolated_cpus_can_update(tmp->addmask, tmp->delmask))
- part_error = PERR_HKEEPING;
- }
-
- /*
* The new CPUs to be removed from parent's effective CPUs
* must be present.
*/
@@ -1956,7 +2072,7 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
/*
* partcmd_update w/o newmask
*
- * delmask = effective_xcpus & parent->effective_cpus
+ * delmask = newly computed effective xcpus value
*
* This can be called from:
* 1) update_cpumasks_hier()
@@ -1967,39 +2083,72 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
*
* A partition error happens when parent has tasks and all
* its effective CPUs will have to be distributed out.
+ *
+ * For an isolated partition with a shrunk root partition parent
+ * we may need to return some or all of its isolated CPUs.
*/
+ nocpu |= cpumask_empty(tmp->delmask);
+ if ((parent_prs == PRS_ROOT) && (old_prs == PRS_ISOLATED)) {
+ cpumask_clear(tmp->addmask);
+ if (nocpu)
+ cpumask_copy(tmp->addmask, cs->effective_xcpus);
+ else if (!cpumask_subset(cs->effective_xcpus, parent->effective_xcpus))
+ cpumask_andnot(tmp->addmask, cs->effective_xcpus,
+ parent->effective_xcpus);
+ if (!cpumask_empty(tmp->addmask))
+ isolated_cpus_release(cs, tmp->addmask);
+ }
if (nocpu) {
part_error = PERR_NOCPUS;
if (is_partition_valid(cs))
adding = cpumask_and(tmp->addmask,
cs->effective_xcpus,
parent->effective_xcpus);
- } else if (is_partition_invalid(cs) && !cpumask_empty(xcpus) &&
- cpumask_subset(xcpus, parent->effective_xcpus)) {
+ } else if ((parent_prs == PRS_ISOLATED) &&
+ ((old_prs == PRS_ROOT) || (old_prs == -PRS_ROOT))) {
+ part_error = PERR_ISOPARENT;
+ if (old_prs == PRS_ROOT) {
+ cpumask_copy(tmp->addmask, cs->effective_xcpus);
+ adding = true;
+ }
+ } else if (is_partition_invalid(cs)) {
struct cgroup_subsys_state *css;
struct cpuset *child;
bool exclusive = true;
/*
- * Convert invalid partition to valid has to
- * pass the cpu exclusivity test.
+ * Convert invalid partition to valid has to pass the
+ * cpu exclusivity test when cpuset.cpus.exclusive is
+ * empty as well as the housekeeping check.
*/
- rcu_read_lock();
- cpuset_for_each_child(child, css, parent) {
- if (child == cs)
- continue;
- if (!cpusets_are_exclusive(cs, child)) {
- exclusive = false;
- break;
+ if (prstate_housekeeping_conflict(cs->partition_root_state,
+ parent_prs, tmp->delmask, NULL)) {
+ part_error = PERR_HKEEPING;
+ goto write_error;
+ }
+ if (cpumask_empty(cs->exclusive_cpus)) {
+ rcu_read_lock();
+ cpuset_for_each_child(child, css, parent) {
+ if (child == cs)
+ continue;
+ if (!cpusets_are_exclusive(cs, child)) {
+ exclusive = false;
+ break;
+ }
}
+ rcu_read_unlock();
}
- rcu_read_unlock();
if (exclusive)
- deleting = cpumask_and(tmp->delmask,
- xcpus, parent->effective_cpus);
+ deleting = !cpumask_empty(tmp->delmask);
else
part_error = PERR_NOTEXCL;
}
+ /*
+ * When both cs and parent remain valid, parent's
+ * compute_partition_effective_cpumask() call would have removed
+ * cs's effective_xcpus from the parent's effective_cpus. So
+ * there is no need to adjust parent's effective_cpus here.
+ */
}
write_error:
@@ -2067,76 +2216,6 @@ write_error:
return 0;
}
-/**
- * compute_partition_effective_cpumask - compute effective_cpus for partition
- * @cs: partition root cpuset
- * @new_ecpus: previously computed effective_cpus to be updated
- *
- * Compute the effective_cpus of a partition root by scanning effective_xcpus
- * of child partition roots and excluding their effective_xcpus.
- *
- * This has the side effect of invalidating valid child partition roots,
- * if necessary. Since it is called from either cpuset_hotplug_update_tasks()
- * or update_cpumasks_hier() where parent and children are modified
- * successively, we don't need to call update_parent_effective_cpumask()
- * and the child's effective_cpus will be updated in later iterations.
- *
- * Note that rcu_read_lock() is assumed to be held.
- */
-static void compute_partition_effective_cpumask(struct cpuset *cs,
- struct cpumask *new_ecpus)
-{
- struct cgroup_subsys_state *css;
- struct cpuset *child;
- bool populated = partition_is_populated(cs, NULL);
-
- /*
- * Check child partition roots to see if they should be
- * invalidated when
- * 1) child effective_xcpus not a subset of new
- * excluisve_cpus
- * 2) All the effective_cpus will be used up and cp
- * has tasks
- */
- compute_excpus(cs, new_ecpus);
- cpumask_and(new_ecpus, new_ecpus, cpu_active_mask);
-
- rcu_read_lock();
- cpuset_for_each_child(child, css, cs) {
- if (!is_partition_valid(child))
- continue;
-
- /*
- * There shouldn't be a remote partition underneath another
- * partition root.
- */
- WARN_ON_ONCE(is_remote_partition(child));
- WRITE_ONCE(child->prs_err, 0);
- if (!cpumask_subset(child->effective_xcpus,
- cs->effective_xcpus))
- WRITE_ONCE(child->prs_err, PERR_INVCPUS);
- else if (populated &&
- cpumask_subset(new_ecpus, child->effective_xcpus))
- WRITE_ONCE(child->prs_err, PERR_NOCPUS);
-
- if (child->prs_err) {
- int old_prs = child->partition_root_state;
-
- /*
- * Invalidate child partition
- */
- spin_lock_irq(&callback_lock);
- make_partition_invalid(child);
- spin_unlock_irq(&callback_lock);
- notify_partition_change(child, old_prs);
- continue;
- }
- cpumask_andnot(new_ecpus, new_ecpus,
- child->effective_xcpus);
- }
- rcu_read_unlock();
-}
-
/*
* update_cpumasks_hier - Update effective cpumasks and tasks in the subtree
* @cs: the cpuset to consider
@@ -2166,16 +2245,19 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp,
old_prs = new_prs = cp->partition_root_state;
/*
- * For child remote partition root (!= cs), we need to call
- * remote_cpus_update() if effective_xcpus will be changed.
- * Otherwise, we can skip the whole subtree.
+ * For a child remote partition with remote_partition_updating
+ * set, we can skip the whole subtree as the this function will
+ * be called eventually. We can also skip the subtree if the
+ * effective_xcpus isn't going to be changed. Otherwise, we
+ * will have to call remote_cpus_update().
*
* remote_cpus_update() will reuse tmp->new_cpus only after
* its value is being processed.
*/
if (remote && (cp != cs)) {
compute_excpus(cp, tmp->new_cpus);
- if (cpumask_equal(cp->effective_xcpus, tmp->new_cpus)) {
+ if (cp->remote_partition_updating ||
+ cpumask_equal(cp->effective_xcpus, tmp->new_cpus)) {
pos_css = css_rightmost_descendant(pos_css);
continue;
}
@@ -2188,10 +2270,7 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp,
remote = (new_prs == old_prs);
}
- if (remote || (is_partition_valid(parent) && is_partition_valid(cp)))
- compute_partition_effective_cpumask(cp, tmp->new_cpus);
- else
- compute_effective_cpumask(tmp->new_cpus, cp, parent);
+ compute_effective_cpumask(tmp->new_cpus, cp, parent);
if (remote)
goto get_css; /* Ready to update cpuset data */
@@ -2229,7 +2308,9 @@ update_parent_effective:
* cpuset if the parent's effective_cpus changes.
*/
if ((cp != cs) && old_prs) {
- switch (parent->partition_root_state) {
+ int parent_prs = parent->partition_root_state;
+
+ switch (parent_prs) {
case PRS_ROOT:
case PRS_ISOLATED:
update_parent = true;
@@ -2237,12 +2318,20 @@ update_parent_effective:
default:
/*
- * When parent is not a partition root or is
- * invalid, child partition roots become
- * invalid too.
+ * When parent is not a valid partition root,
+ * child partition roots become invalid too.
+ *
+ * For a to be invalidated isolated partition,
+ * call return_isolated_cpus() to remove the
+ * relevant CPUs from isolated_cpus.
*/
if (is_partition_valid(cp))
- new_prs = -cp->partition_root_state;
+ new_prs = -old_prs;
+
+ if (old_prs == PRS_ISOLATED) {
+ cpumask_copy(tmp->delmask, cp->effective_xcpus);
+ isolated_cpus_release(cp, tmp->delmask);
+ }
WRITE_ONCE(cp->prs_err,
is_partition_invalid(parent)
? PERR_INVPARENT : PERR_NOTPART);
@@ -2257,10 +2346,13 @@ get_css:
if (update_parent) {
update_parent_effective_cpumask(cp, partcmd_update, NULL, tmp);
/*
- * The cpuset partition_root_state may become
- * invalid. Capture it.
+ * Capture current partition_root_state as it may have
+ * changed. Recompute effective cpumask if changed.
*/
new_prs = cp->partition_root_state;
+ if (new_prs != old_prs) {
+ compute_effective_cpumask(tmp->new_cpus, cp, parent);
+ }
}
spin_lock_irq(&callback_lock);
@@ -2391,10 +2483,6 @@ static enum prs_errcode validate_partition(struct cpuset *cs, struct cpuset *tri
if (cpumask_empty(trialcs->effective_xcpus))
return PERR_INVCPUS;
- if (prstate_housekeeping_conflict(trialcs->partition_root_state,
- trialcs->effective_xcpus))
- return PERR_HKEEPING;
-
if (tasks_nocpu_error(parent, cs, trialcs->effective_xcpus))
return PERR_NOCPUS;
@@ -2411,14 +2499,36 @@ static enum prs_errcode validate_partition(struct cpuset *cs, struct cpuset *tri
* CPU modifications may cause a partition to be disabled or require state updates.
*/
static void partition_cpus_change(struct cpuset *cs, struct cpuset *trialcs,
- struct tmpmasks *tmp)
+ struct tmpmasks *tmp)
{
enum prs_errcode prs_err;
+ int parent_prs = is_remote_partition(cs)
+ ? PRS_ROOT : parent_cs(cs)->partition_root_state;
if (cs_is_member(cs))
return;
prs_err = validate_partition(cs, trialcs);
+ if (!prs_err && (parent_prs > 0) &&
+ !cpumask_empty(trialcs->effective_xcpus)) {
+ int prs = cs->partition_root_state;
+
+ /*
+ * Check for housekeeping conflict by passing the whole
+ * effective_xcpus as well as those CPUs that are to be
+ * removed from cs as delmask.
+ */
+ if (prs > 0)
+ cpumask_andnot(tmp->delmask, cs->effective_xcpus,
+ trialcs->effective_xcpus);
+ else
+ cpumask_clear(tmp->delmask);
+
+ if (prstate_housekeeping_conflict(prs, parent_prs,
+ trialcs->effective_xcpus, tmp->delmask))
+ prs_err = PERR_HKEEPING;
+ }
+
if (prs_err) {
WRITE_ONCE(cs->prs_err, prs_err);
trialcs->prs_err = prs_err;
@@ -2774,6 +2884,8 @@ static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems)
continue;
rcu_read_unlock();
+ check_insane_mems_config(new_mems);
+
spin_lock_irq(&callback_lock);
cp->effective_mems = *new_mems;
spin_unlock_irq(&callback_lock);
@@ -2827,8 +2939,6 @@ static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs,
if (retval < 0)
return retval;
- check_insane_mems_config(&trialcs->mems_allowed);
-
spin_lock_irq(&callback_lock);
cs->mems_allowed = trialcs->mems_allowed;
spin_unlock_irq(&callback_lock);
@@ -2862,7 +2972,9 @@ static int update_prstate(struct cpuset *cs, int new_prs)
int err = PERR_NONE, old_prs = cs->partition_root_state;
struct cpuset *parent = parent_cs(cs);
struct tmpmasks tmpmask;
+ int parent_prs = parent->partition_root_state;
bool isolcpus_updated = false;
+ bool disable_partition = false;
if (old_prs == new_prs)
return 0;
@@ -2906,26 +3018,45 @@ static int update_prstate(struct cpuset *cs, int new_prs)
enum partition_cmd cmd = (new_prs == PRS_ROOT)
? partcmd_enable : partcmd_enablei;
+ /*
+ * It is not allowed to create a root partition under
+ * an isolated parent partition.
+ */
+ if ((new_prs == PRS_ROOT) && (parent_prs == PRS_ISOLATED)) {
+ err = PERR_ISOPARENT;
+ goto out;
+ }
err = update_parent_effective_cpumask(cs, cmd, NULL, &tmpmask);
} else {
err = remote_partition_enable(cs, new_prs, &tmpmask);
}
} else if (old_prs && new_prs) {
+ int parent_prs = is_remote_partition(cs)
+ ? PRS_ROOT : parent->partition_root_state;
+
/*
* A change in load balance state only, no change in cpumasks.
* Need to update isolated_cpus.
*/
- if (((new_prs == PRS_ISOLATED) &&
- !isolated_cpus_can_update(cs->effective_xcpus, NULL)) ||
- prstate_housekeeping_conflict(new_prs, cs->effective_xcpus))
+ if (prstate_housekeeping_conflict(new_prs, parent_prs,
+ cs->effective_xcpus, NULL))
err = PERR_HKEEPING;
- else
+ else if (parent_prs == PRS_ISOLATED)
+ err = PERR_ISOPARENT;
+
+ if (!err)
isolcpus_updated = true;
+ else
+ disable_partition = true;
} else {
/*
* Switching back to member is always allowed even if it
* disables child partitions.
*/
+ disable_partition = true;
+ }
+
+ if (disable_partition) {
if (is_remote_partition(cs))
remote_partition_disable(cs, &tmpmask);
else
@@ -2934,7 +3065,7 @@ static int update_prstate(struct cpuset *cs, int new_prs)
/*
* Invalidation of child partitions will be done in
- * update_cpumasks_hier().
+ * update_cpumasks_hier() below.
*/
}
out:
@@ -2953,8 +3084,8 @@ out:
isolated_cpus_update(old_prs, new_prs, cs->effective_xcpus);
spin_unlock_irq(&callback_lock);
- /* Force update if switching back to member & update effective_xcpus */
- update_cpumasks_hier(cs, &tmpmask, !new_prs);
+ /* Force update if partition is disabled & update effective_xcpus */
+ update_cpumasks_hier(cs, &tmpmask, disable_partition);
/* A newly created partition must have effective_xcpus set */
WARN_ON_ONCE(!old_prs && (new_prs > 0)
@@ -3852,7 +3983,6 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp)
static nodemask_t new_mems;
bool cpus_updated;
bool mems_updated;
- bool remote;
int partcmd = -1;
struct cpuset *parent;
@@ -3864,21 +3994,12 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp)
if (!tmp || !cs->partition_root_state)
goto update_tasks;
- /*
- * Compute effective_cpus for valid partition root, may invalidate
- * child partition roots if necessary.
- */
- remote = is_remote_partition(cs);
- if (remote || (is_partition_valid(cs) && is_partition_valid(parent)))
- compute_partition_effective_cpumask(cs, &new_cpus);
-
- if (remote && (cpumask_empty(subpartitions_cpus) ||
- (cpumask_empty(&new_cpus) &&
- partition_is_populated(cs, NULL)))) {
+ if (is_remote_partition(cs) && (cpumask_empty(subpartitions_cpus) ||
+ (cpumask_empty(&new_cpus) &&
+ partition_is_populated(cs, NULL)))) {
WRITE_ONCE(cs->prs_err, PERR_HOTPLUG);
remote_partition_disable(cs, tmp);
compute_effective_cpumask(&new_cpus, cs, parent);
- remote = false;
}
/*
@@ -3904,8 +4025,8 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp)
if (partcmd >= 0) {
update_parent_effective_cpumask(cs, partcmd, NULL, tmp);
- if ((partcmd == partcmd_invalidate) || is_partition_valid(cs)) {
- compute_partition_effective_cpumask(cs, &new_cpus);
+ if (is_partition_valid(cs) || (partcmd == partcmd_invalidate)) {
+ compute_effective_cpumask(&new_cpus, cs, parent);
cpuset_force_rebuild();
}
}
@@ -4265,10 +4386,21 @@ nodemask_t cpuset_mems_allowed(struct task_struct *tsk)
* @nodemask: the nodemask to be checked
*
* Are any of the nodes in the nodemask allowed in current->mems_allowed?
+ *
+ * A cpuset update rebinds the task's mempolicy and changes mems_allowed in
+ * separate steps, so retry if an update raced with the check.
*/
int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask)
{
- return nodes_intersects(*nodemask, current->mems_allowed);
+ unsigned int seq;
+ int ret;
+
+ do {
+ seq = read_mems_allowed_begin();
+ ret = nodes_intersects(*nodemask, current->mems_allowed);
+ } while (read_mems_allowed_retry(seq));
+
+ return ret;
}
/*