summaryrefslogtreecommitdiff
path: root/kernel/scftorture.c
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2026-08-18 15:29:53 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2026-08-18 15:29:53 -0700
commit0dd1a54f44348d9cf6bae57a2b5cb0b53826a2c7 (patch)
treecc7ca2ad4a92fb593ffdf72fe6ebc0dff75155c0 /kernel/scftorture.c
parentb844715e8aca2929c0a97a32b3a5650496b5872f (diff)
parent99b49e02f9488335156c896c24aab0785623eb67 (diff)
Merge tag 'smp-core-2026-08-17' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull SMP core updates from Thomas Gleixner: - Reduce the preemption disabled sections in smp_call_function*(). The various smp call functions keep preemption disabled accross the full operation which includes the wait for completion. Especially the latter can take some time when one of the target CPUs is not immediately responding to the IPI, which can result in large latency spikes. To improve this provide a per task CPU mask to track the CPUs to wait for. That makes the information required for the wait task local and therefore allows to reenable preemption before the wait. While this comes with moderate extra memory cost this reduces SMP function call induced latency measured in a fleet for high priority tasks from ~17ms to ~1.5ms (~90%). - Reduce the overhead of the CSD debug code by replacing the heavy memory barriers with smp_store_release()/acquire() - Remove obsolute unused hotplug states * tag 'smp-core-2026-08-17' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: scftorture: Remove preempt_disable() in scftorture_invoke_one() smp: Remove preempt_disable() from on_each_cpu_cond_mask() smp: Remove preempt_disable() from smp_call_function() smp: Enable preemption early in smp_call_function_many_cond() smp: Alloc percpu csd data in smpcfd_prepare_cpu() only once smp: Use task-local IPI cpumask in smp_call_function_many_cond() smp: Refactor remote CPU selection in smp_call_function_any() smp: Enable preemption early in smp_call_function_single() smp: Disable preemption explicitly in __csd_lock_wait() cpu/hotplug: Remove CPUHP_AP_ARM_CORESIGHT_CTI_STARTING smp: Use release stores for csd_lock_record() state
Diffstat (limited to 'kernel/scftorture.c')
-rw-r--r--kernel/scftorture.c13
1 files changed, 4 insertions, 9 deletions
diff --git a/kernel/scftorture.c b/kernel/scftorture.c
index 327c315f411c..2082f9b44370 100644
--- a/kernel/scftorture.c
+++ b/kernel/scftorture.c
@@ -348,6 +348,8 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra
int ret = 0;
struct scf_check *scfcp = NULL;
struct scf_selector *scfsp = scf_sel_rand(trsp);
+ bool is_single = (scfsp->scfs_prim == SCF_PRIM_SINGLE ||
+ scfsp->scfs_prim == SCF_PRIM_SINGLE_RPC);
if (scfsp->scfs_prim == SCF_PRIM_SINGLE || scfsp->scfs_wait) {
scfcp = kmalloc_obj(*scfcp, GFP_ATOMIC);
@@ -364,8 +366,6 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra
}
if (use_cpus_read_lock)
cpus_read_lock();
- else
- preempt_disable();
switch (scfsp->scfs_prim) {
case SCF_PRIM_RESCHED:
if (IS_BUILTIN(CONFIG_SCF_TORTURE_TEST)) {
@@ -411,13 +411,10 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra
if (!ret) {
if (use_cpus_read_lock)
cpus_read_unlock();
- else
- preempt_enable();
+
wait_for_completion(&scfcp->scfc_completion);
if (use_cpus_read_lock)
cpus_read_lock();
- else
- preempt_disable();
} else {
scfp->n_single_rpc_ofl++;
scf_add_to_free_list(scfcp);
@@ -452,7 +449,7 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra
scfcp->scfc_out = true;
}
if (scfcp && scfsp->scfs_wait) {
- if (WARN_ON_ONCE((num_online_cpus() > 1 || scfsp->scfs_prim == SCF_PRIM_SINGLE) &&
+ if (WARN_ON_ONCE(((use_cpus_read_lock && num_online_cpus() > 1) || is_single) &&
!scfcp->scfc_out)) {
pr_warn("%s: Memory-ordering failure, scfs_prim: %d.\n", __func__, scfsp->scfs_prim);
atomic_inc(&n_mb_out_errs); // Leak rather than trash!
@@ -463,8 +460,6 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra
}
if (use_cpus_read_lock)
cpus_read_unlock();
- else
- preempt_enable();
if (allocfail)
schedule_timeout_idle((1 + longwait) * HZ); // Let no-wait handlers complete.
else if (!(torture_random(trsp) & 0xfff))