Merge tag 'wq-for-7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq

Pull workqueue updates from Tejun Heo:

 - Flushing a workqueue now visits only the queues used since the last
   flush instead of one per possible CPU. Idle flushes get 10 to 100
   times faster on large machines, which XFS hits on every log force.

 - Per-cpu workqueues are built through the same attribute path as
   unbound ones and get a max_active of their own, continuing the work
   to turn them into an affinity scope of unbound workqueues.

 - New tracepoints for work items turning CPU-intensive, rescuer mayday
   and rescue events, and BH workers yielding with work pending.

 - A KCSAN annotation and small cleanups

* tag 'wq-for-7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq:
  workqueue: mark pool->nr_running as __data_racy
  workqueue: BUG_ON() a pwq release before workqueue_init()
  workqueue: Derive the standard attrs from the workqueue flags
  workqueue: Give percpu workqueues their own max_active
  workqueue: Make flush_workqueue() visit only pwqs active since the last flush
  workqueue: Maintain pwq->total_in_flight
  workqueue: Add for_each_node_with_fallback()
  workqueue: Fix typo "upto" in comment
  workqueue: Add workqueue_bh_budget_yield tracepoint
  workqueue: Add workqueue_mayday and workqueue_rescued tracepoints
  workqueue: Add workqueue_cpu_intensive tracepoint
  workqueue: rename the pwq slot helpers shared with percpu
  workqueue: build percpu pwqs through the attrs path
  workqueue: make the default pwq optional
  workqueue: resolve the backing pool in alloc_pwq()
  workqueue: move the unbound-only attrs check to apply_workqueue_attrs()

1 files changed, 396 insertions, 197 deletions

diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index c1cb328bce3c..f2e8071a6f4e 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -56,6 +56,7 @@
 #include <linux/kvm_para.h>
 #include <linux/delay.h>
 #include <linux/irq_work.h>
+#include <linux/math64.h>
 
 #include "workqueue_internal.h"
 
@@ -161,6 +162,9 @@ struct llc_shard_layout {
  * LN: pool->lock and wq_node_nr_active->lock protected for writes. Either for
  *     reads.
  *
+ * FN: wq_flush_pnode->lock protected. list_empty(&pwq->flush_node) can also be
+ *     tested with only pool->lock held.
+ *
  * K: Only modified by worker while holding pool->lock. Can be safely read by
  *    self, while holding pool->lock or from IRQ context if %current is the
  *    kworker.
@@ -208,7 +212,7 @@ struct worker_pool {
 	 * but w/ pool->lock held. The readers grab pool->lock and are
 	 * guaranteed to see if the counter reached zero.
 	 */
-	int			nr_running;
+	int __data_racy nr_running;
 
 	struct list_head	worklist;	/* L: list of pending works */
 
@@ -274,8 +278,10 @@ struct pool_workqueue {
 	int			work_color;	/* L: current color */
 	int			flush_color;	/* L: flushing color */
 	int			refcnt;		/* L: reference count */
+	int			total_in_flight; /* L: sum of nr_in_flight[] */
 	int			nr_in_flight[WORK_NR_COLORS];
 						/* L: nr of in_flight works */
+	struct list_head	flush_node;	/* FN: node on wq_flush_pnode->pwqs */
 	bool			plugged;	/* L: execution suspended */
 
 	/*
@@ -345,6 +351,21 @@ struct wq_node_nr_active {
 };
 
 /*
+ * Per-node list of pwqs active since the last flush so that flushes visit only
+ * those. One per possible node plus the NUMA_NO_NODE fallback at nr_node_ids,
+ * each allocated on its node. A pwq is added when a work item is queued to it
+ * while off the list and removed by a flush which finds it with nothing in
+ * flight. ->work_color mirrors wq->work_color. Reading it and adding the pwq in
+ * one ->lock section keeps queueing coherent against flushing: a work item is
+ * either stamped with the new color or its pwq is visible to the flusher.
+ */
+struct wq_flush_pnode {
+	raw_spinlock_t		lock;		/* nests inside pool locks */
+	int			work_color;	/* FN: mirrors wq->work_color */
+	struct list_head	pwqs;		/* FN: pwqs active since last flush */
+};
+
+/*
  * The externally visible workqueue.  It relays the issued work items to
  * the appropriate worker_pool through its pool_workqueues.
  */
@@ -359,6 +380,7 @@ struct workqueue_struct {
 	struct wq_flusher	*first_flusher;	/* WQ: first flusher */
 	struct list_head	flusher_queue;	/* WQ: flush waiters */
 	struct list_head	flusher_overflow; /* WQ: flush overflow list */
+	struct wq_flush_pnode	**flush_pnodes; /* I: per-node flush membership */
 
 	struct list_head	maydays;	/* MD: pwqs requesting rescue */
 	struct worker		*rescuer;	/* MD: rescue worker */
@@ -367,6 +389,7 @@ struct workqueue_struct {
 
 	/* See alloc_workqueue() function comment for info on min/max_active */
 	int			max_active;	/* WO: max active works */
+	int			percpu_max_active; /* WO: max active works per cpu */
 	int			min_active;	/* WO: min active works */
 	int			saved_max_active; /* WQ: saved max_active */
 	int			saved_min_active; /* WQ: saved min_active */
@@ -510,12 +533,6 @@ static DEFINE_IDR(worker_pool_idr);	/* PR: idr of all pools */
 /* PL: hash of all unbound pools keyed by pool->attrs */
 static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
 
-/* I: attributes used when instantiating standard unbound pools on demand */
-static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
-
-/* I: attributes used when instantiating ordered pools on demand */
-static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
-
 /*
  * I: kthread_worker to release pwq's. pwq release needs to be bounced to a
  * process context while holding a pool lock. Bounce to a dedicated kthread
@@ -619,6 +636,26 @@ static void show_one_worker_pool(struct worker_pool *pool);
 	list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node,		\
 				 lockdep_is_held(&(wq->mutex)))
 
+/*
+ * Per-node arrays in this file carry an extra slot at nr_node_ids serving
+ * NUMA_NO_NODE. Return the slot after all nodes.
+ */
+static int next_node_with_fallback(int node)
+{
+	if (node >= nr_node_ids)
+		return nr_node_ids + 1;
+
+	node = next_node(node, node_possible_map);
+	if (node >= nr_node_ids)
+		return nr_node_ids;
+	return node;
+}
+
+#define for_each_node_with_fallback(node)				\
+	for ((node) = first_node(node_possible_map);			\
+	     (node) <= nr_node_ids;					\
+	     (node) = next_node_with_fallback(node))
+
 #ifdef CONFIG_DEBUG_OBJECTS_WORK
 
 static const struct debug_obj_descr work_debug_descr;
@@ -739,7 +776,7 @@ static int worker_pool_assign_id(struct worker_pool *pool)
 }
 
 static struct pool_workqueue __rcu **
-unbound_pwq_slot(struct workqueue_struct *wq, int cpu)
+pwq_slot(struct workqueue_struct *wq, int cpu)
 {
        if (cpu >= 0)
                return per_cpu_ptr(wq->cpu_pwq, cpu);
@@ -748,9 +785,9 @@ unbound_pwq_slot(struct workqueue_struct *wq, int cpu)
 }
 
 /* @cpu < 0 for dfl_pwq */
-static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu)
+static struct pool_workqueue *installed_pwq(struct workqueue_struct *wq, int cpu)
 {
-	return rcu_dereference_check(*unbound_pwq_slot(wq, cpu),
+	return rcu_dereference_check(*pwq_slot(wq, cpu),
 				     lockdep_is_held(&wq_pool_mutex) ||
 				     lockdep_is_held(&wq->mutex));
 }
@@ -765,7 +802,7 @@ static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu)
  */
 static struct cpumask *unbound_effective_cpumask(struct workqueue_struct *wq)
 {
-	return unbound_pwq(wq, -1)->pool->attrs->__pod_cpumask;
+	return installed_pwq(wq, -1)->pool->attrs->__pod_cpumask;
 }
 
 static unsigned int work_color_to_flags(int color)
@@ -1532,6 +1569,7 @@ void wq_worker_tick(struct task_struct *task)
 	struct worker *worker = kthread_data(task);
 	struct pool_workqueue *pwq = worker->current_pwq;
 	struct worker_pool *pool = worker->pool;
+	u64 delta;
 
 	if (!pwq)
 		return;
@@ -1572,6 +1610,11 @@ void wq_worker_tick(struct task_struct *task)
 		pwq->stats[PWQ_STAT_CM_WAKEUP]++;
 
 	raw_spin_unlock(&pool->lock);
+
+	delta = READ_ONCE(worker->task->se.sum_exec_runtime) - worker->current_at;
+	trace_workqueue_cpu_intensive(pwq, worker->current_work,
+				      worker->current_func,
+				      div_u64(delta, NSEC_PER_USEC));
 }
 
 /**
@@ -1635,6 +1678,22 @@ static struct wq_node_nr_active *wq_node_nr_active(struct workqueue_struct *wq,
 }
 
 /**
+ * wq_flush_pnode - Determine wq_flush_pnode to use
+ * @wq: workqueue of interest
+ * @node: NUMA node, can be %NUMA_NO_NODE
+ *
+ * Return @wq's wq_flush_pnode for @node, the nr_node_ids fallback if @node is
+ * %NUMA_NO_NODE.
+ */
+static struct wq_flush_pnode *wq_flush_pnode(struct workqueue_struct *wq, int node)
+{
+	if (node == NUMA_NO_NODE)
+		node = nr_node_ids;
+
+	return wq->flush_pnodes[node];
+}
+
+/**
  * wq_update_node_max_active - Update per-node max_actives to use
  * @wq: workqueue to update
  * @off_cpu: CPU that's going down, -1 if a CPU is not going down
@@ -1712,6 +1771,14 @@ static void put_pwq(struct pool_workqueue *pwq)
 	lockdep_assert_held(&pwq->pool->lock);
 	if (likely(--pwq->refcnt))
 		return;
+
+	/*
+	 * pwq_release_worker is only created in workqueue_init(). Getting
+	 * here before that means an early workqueue allocation failed, which
+	 * is fatal anyway.
+	 */
+	BUG_ON(!pwq_release_worker);
+
 	/*
 	 * @pwq can't be released under pool->lock, bounce to a dedicated
 	 * kthread_worker to avoid A-A deadlocks.
@@ -1788,10 +1855,10 @@ static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill)
 
 	/*
 	 * A concurrency-managed per-cpu pool accounts nr_active per pwq, so
-	 * pwq->nr_active against wq->max_active is sufficient.
+	 * pwq->nr_active against wq->percpu_max_active is sufficient.
 	 */
 	if (is_percpu_pool(pool)) {
-		obtained = pwq->nr_active < READ_ONCE(wq->max_active);
+		obtained = pwq->nr_active < READ_ONCE(wq->percpu_max_active);
 		goto out;
 	}
 
@@ -2060,6 +2127,22 @@ static void pwq_dec_nr_active(struct pool_workqueue *pwq)
 }
 
 /**
+ * pwq_inc_nr_in_flight - increment pwq's nr_in_flight
+ * @pwq: pwq of interest
+ * @work_color: color of the work item being queued
+ *
+ * A work item or a barrier with @work_color is being queued to @pwq.
+ *
+ * CONTEXT:
+ * raw_spin_lock_irq(pool->lock).
+ */
+static void pwq_inc_nr_in_flight(struct pool_workqueue *pwq, int work_color)
+{
+	pwq->nr_in_flight[work_color]++;
+	pwq->total_in_flight++;
+}
+
+/**
  * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
  * @pwq: pwq of interest
  * @work_data: work_data of work which left the queue
@@ -2083,6 +2166,7 @@ static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_
 		pwq_dec_nr_active(pwq);
 
 	pwq->nr_in_flight[color]--;
+	pwq->total_in_flight--;
 
 	/* is flush in progress and are we at the flushing tip? */
 	if (likely(pwq->flush_color != color))
@@ -2432,7 +2516,21 @@ retry:
 	if (WARN_ON(!list_empty(&work->entry)))
 		goto out;
 
-	pwq->nr_in_flight[pwq->work_color]++;
+	/*
+	 * Add @pwq to its wq_flush_pnode list if off it. Syncing ->work_color
+	 * in the same fpn->lock section is what keeps a concurrent flush from
+	 * missing @work, see wq_flush_pnode.
+	 */
+	if (unlikely(list_empty(&pwq->flush_node))) {
+		struct wq_flush_pnode *fpn = wq_flush_pnode(wq, pool->node);
+
+		raw_spin_lock(&fpn->lock);
+		pwq->work_color = fpn->work_color;
+		list_add_tail(&pwq->flush_node, &fpn->pwqs);
+		raw_spin_unlock(&fpn->lock);
+	}
+
+	pwq_inc_nr_in_flight(pwq, pwq->work_color);
 	work_flags = work_color_to_flags(pwq->work_color);
 
 	/*
@@ -3117,6 +3215,7 @@ static void send_mayday(struct pool_workqueue *pwq)
 		list_add_tail(&pwq->mayday_node, &wq->maydays);
 		wake_up_process(wq->rescuer->task);
 		pwq->stats[PWQ_STAT_MAYDAY]++;
+		trace_workqueue_mayday(pwq);
 	}
 }
 
@@ -3623,6 +3722,7 @@ static bool assign_rescuer_work(struct pool_workqueue *pwq, struct worker *rescu
 	list_for_each_entry_safe_from(work, n, &pool->worklist, entry) {
 		if (get_work_pwq(work) == pwq && assign_work(work, rescuer, &n)) {
 			pwq->stats[PWQ_STAT_RESCUED]++;
+			trace_workqueue_rescued(pwq, work, work->func);
 			/* put the cursor for next search */
 			list_move_tail(&cursor->entry, &n->entry);
 			return true;
@@ -3758,6 +3858,9 @@ static void bh_worker(struct worker *worker)
 	struct worker_pool *pool = worker->pool;
 	int nr_restarts = BH_WORKER_RESTARTS;
 	unsigned long end = jiffies + BH_WORKER_JIFFIES;
+	bool budget_exhausted = false;
+	bool timeout = false;
+	int restarts = 0;
 
 	worker_lock_callback(pool);
 	raw_spin_lock_irq(&pool->lock);
@@ -3780,8 +3883,23 @@ static void bh_worker(struct worker *worker)
 
 		if (assign_work(work, worker, NULL))
 			process_scheduled_works(worker);
-	} while (keep_working(pool) &&
-		 --nr_restarts && time_before(jiffies, end));
+
+		if (!keep_working(pool))
+			break;
+
+		if (!--nr_restarts) {
+			budget_exhausted = true;
+			break;
+		}
+
+		if (!time_before(jiffies, end)) {
+			budget_exhausted = true;
+			timeout = true;
+			break;
+		}
+
+		restarts++;
+	} while (1);
 
 	worker_set_flags(worker, WORKER_PREP);
 done:
@@ -3789,6 +3907,10 @@ done:
 	kick_pool(pool);
 	raw_spin_unlock_irq(&pool->lock);
 	worker_unlock_callback(pool);
+
+	if (budget_exhausted)
+		trace_workqueue_bh_budget_yield(pool, restarts, timeout,
+						pool->attrs->nice == HIGHPRI_NICE_LEVEL);
 }
 
 /*
@@ -4008,7 +4130,15 @@ static void insert_wq_barrier(struct pool_workqueue *pwq,
 		__set_bit(WORK_STRUCT_LINKED_BIT, bits);
 	}
 
-	pwq->nr_in_flight[work_color]++;
+	/*
+	 * Flushes must wait for barriers too as a barrier pins @pwq and a
+	 * worker until it runs and destroy_workqueue()'s drain relies on that.
+	 * @target being in flight keeps @pwq on its wq_flush_pnode list, so no
+	 * need to add it here.
+	 */
+	WARN_ON_ONCE(list_empty(&pwq->flush_node));
+
+	pwq_inc_nr_in_flight(pwq, work_color);
 	work_flags |= work_color_to_flags(work_color);
 
 	insert_work(pwq, &barr->work, head, work_flags);
@@ -4020,23 +4150,23 @@ static void insert_wq_barrier(struct pool_workqueue *pwq,
  * @flush_color: new flush color, < 0 for no-op
  * @work_color: new work color, < 0 for no-op
  *
- * Prepare pwqs for workqueue flushing.
+ * Prepare pwqs for workqueue flushing. Only the pwqs on @wq's wq_flush_pnode
+ * lists, which have been active since the last flush, are visited.
  *
- * If @flush_color is non-negative, flush_color on all pwqs should be
- * -1.  If no pwq has in-flight commands at the specified color, all
- * pwq->flush_color's stay at -1 and %false is returned.  If any pwq
- * has in flight commands, its pwq->flush_color is set to
- * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
- * wakeup logic is armed and %true is returned.
+ * If @flush_color >= 0, flush_color on all visited pwqs should be -1. If no
+ * visited pwq has in-flight work items at the specified color, all
+ * pwq->flush_color's stay at -1 and return %false. If any visited pwq has
+ * in-flight work items, set its pwq->flush_color to @flush_color, update
+ * @wq->nr_pwqs_to_flush accordingly, arm pwq wakeup logic and return %true.
  *
- * The caller should have initialized @wq->first_flusher prior to
- * calling this function with non-negative @flush_color.  If
- * @flush_color is negative, no flush color update is done and %false
- * is returned.
+ * The caller should have initialized @wq->first_flusher prior to calling this
+ * function with non-negative @flush_color. If @flush_color < 0, no flush color
+ * update is done and %false is returned.
  *
- * If @work_color is non-negative, all pwqs should have the same
- * work_color which is previous to @work_color and all will be
- * advanced to @work_color.
+ * If @work_color >= 0, all visited pwqs and every wq_flush_pnode's work color
+ * mirror are advanced to @work_color. The mirrors of nodes with no active pwqs
+ * must be advanced too as a pwq added to a list later stamps the color it reads
+ * from the mirror.
  *
  * CONTEXT:
  * mutex_lock(wq->mutex).
@@ -4048,50 +4178,81 @@ static void insert_wq_barrier(struct pool_workqueue *pwq,
 static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
 				      int flush_color, int work_color)
 {
+	struct pool_workqueue *pwq, *next;
 	bool wait = false;
-	struct pool_workqueue *pwq;
-	struct worker_pool *current_pool = NULL;
+	int node;
 
 	if (flush_color >= 0) {
 		WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
 		atomic_set(&wq->nr_pwqs_to_flush, 1);
 	}
 
-	/*
-	 * For unbound workqueue, pwqs will map to only a few pools.
-	 * Most of the time, pwqs within the same pool will be linked
-	 * sequentially to wq->pwqs by cpu index. So in the majority
-	 * of pwq iters, the pool is the same, only doing lock/unlock
-	 * if the pool has changed. This can largely reduce expensive
-	 * lock operations.
-	 */
-	for_each_pwq(pwq, wq) {
-		if (current_pool != pwq->pool) {
-			if (likely(current_pool))
-				raw_spin_unlock_irq(&current_pool->lock);
-			current_pool = pwq->pool;
-			raw_spin_lock_irq(&current_pool->lock);
+	for_each_node_with_fallback(node) {
+		struct wq_flush_pnode *fpn = wq->flush_pnodes[node];
+		struct worker_pool *current_pool = NULL;
+		LIST_HEAD(to_visit);
+		LIST_HEAD(to_keep);
+
+		/*
+		 * Advance the node's color mirror and take its active pwqs.
+		 * Work items queued to off-list pwqs afterwards carry the new
+		 * color and don't need visiting.
+		 */
+		raw_spin_lock_irq(&fpn->lock);
+		if (work_color >= 0) {
+			WARN_ON_ONCE(work_color != work_next_color(fpn->work_color));
+			fpn->work_color = work_color;
 		}
+		list_splice_init(&fpn->pwqs, &to_visit);
+		raw_spin_unlock_irq(&fpn->lock);
+
+		/*
+		 * Unbound workqueues map multiple pwqs to one pool. Only cycle
+		 * pool->lock when the pool changes.
+		 */
+		list_for_each_entry_safe(pwq, next, &to_visit, flush_node) {
+			if (current_pool != pwq->pool) {
+				if (likely(current_pool))
+					raw_spin_unlock_irq(&current_pool->lock);
+				current_pool = pwq->pool;
+				raw_spin_lock_irq(&current_pool->lock);
+			}
 
-		if (flush_color >= 0) {
-			WARN_ON_ONCE(pwq->flush_color != -1);
+			if (flush_color >= 0) {
+				WARN_ON_ONCE(pwq->flush_color != -1);
 
-			if (pwq->nr_in_flight[flush_color]) {
-				pwq->flush_color = flush_color;
-				atomic_inc(&wq->nr_pwqs_to_flush);
-				wait = true;
+				if (pwq->nr_in_flight[flush_color]) {
+					pwq->flush_color = flush_color;
+					atomic_inc(&wq->nr_pwqs_to_flush);
+					wait = true;
+				}
 			}
-		}
 
-		if (work_color >= 0) {
-			WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
-			pwq->work_color = work_color;
+			if (work_color >= 0) {
+				WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
+				pwq->work_color = work_color;
+			}
+
+			/*
+			 * A cascade can arm a color retired several advances
+			 * ago, so keep a pwq listed while any color has work
+			 * items in flight.
+			 */
+			if (pwq->total_in_flight)
+				list_move_tail(&pwq->flush_node, &to_keep);
+			else
+				list_del_init(&pwq->flush_node);
 		}
 
-	}
+		if (current_pool)
+			raw_spin_unlock_irq(&current_pool->lock);
 
-	if (current_pool)
-		raw_spin_unlock_irq(&current_pool->lock);
+		if (!list_empty(&to_keep)) {
+			raw_spin_lock_irq(&fpn->lock);
+			list_splice_tail(&to_keep, &fpn->pwqs);
+			raw_spin_unlock_irq(&fpn->lock);
+		}
+	}
 
 	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
 		complete(&wq->first_flusher->done);
@@ -5049,13 +5210,10 @@ static void free_node_nr_active(struct wq_node_nr_active **nna_ar)
 {
 	int node;
 
-	for_each_node(node) {
+	for_each_node_with_fallback(node) {
 		kfree(nna_ar[node]);
 		nna_ar[node] = NULL;
 	}
-
-	kfree(nna_ar[nr_node_ids]);
-	nna_ar[nr_node_ids] = NULL;
 }
 
 static void init_node_nr_active(struct wq_node_nr_active *nna)
@@ -5075,21 +5233,15 @@ static int alloc_node_nr_active(struct wq_node_nr_active **nna_ar)
 	struct wq_node_nr_active *nna;
 	int node;
 
-	for_each_node(node) {
-		nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, node);
+	for_each_node_with_fallback(node) {
+		nna = kzalloc_node(sizeof(*nna), GFP_KERNEL,
+				   node < nr_node_ids ? node : NUMA_NO_NODE);
 		if (!nna)
 			goto err_free;
 		init_node_nr_active(nna);
 		nna_ar[node] = nna;
 	}
 
-	/* [nr_node_ids] is used as the fallback */
-	nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, NUMA_NO_NODE);
-	if (!nna)
-		goto err_free;
-	init_node_nr_active(nna);
-	nna_ar[nr_node_ids] = nna;
-
 	return 0;
 
 err_free:
@@ -5097,6 +5249,47 @@ err_free:
 	return -ENOMEM;
 }
 
+static void free_flush_pnodes(struct workqueue_struct *wq)
+{
+	int node;
+
+	if (!wq->flush_pnodes)
+		return;
+
+	for_each_node_with_fallback(node)
+		kfree(wq->flush_pnodes[node]);
+	kfree(wq->flush_pnodes);
+	wq->flush_pnodes = NULL;
+}
+
+static int alloc_flush_pnodes(struct workqueue_struct *wq)
+{
+	struct wq_flush_pnode *fpn;
+	int node;
+
+	wq->flush_pnodes = kcalloc(nr_node_ids + 1, sizeof(*wq->flush_pnodes), GFP_KERNEL);
+	if (!wq->flush_pnodes)
+		return -ENOMEM;
+
+	for_each_node_with_fallback(node) {
+		fpn = kzalloc_node(sizeof(*fpn), GFP_KERNEL,
+				   node < nr_node_ids ? node : NUMA_NO_NODE);
+		if (!fpn)
+			goto err_free;
+
+		raw_spin_lock_init(&fpn->lock);
+		fpn->work_color = wq->work_color;
+		INIT_LIST_HEAD(&fpn->pwqs);
+		wq->flush_pnodes[node] = fpn;
+	}
+
+	return 0;
+
+err_free:
+	free_flush_pnodes(wq);
+	return -ENOMEM;
+}
+
 static void rcu_free_wq(struct rcu_head *rcu)
 {
 	struct workqueue_struct *wq =
@@ -5105,6 +5298,7 @@ static void rcu_free_wq(struct rcu_head *rcu)
 	if (wq->flags & WQ_UNBOUND)
 		free_node_nr_active(wq->node_nr_active);
 
+	free_flush_pnodes(wq);
 	wq_free_lockdep(wq);
 	free_percpu(wq->cpu_pwq);
 	free_workqueue_attrs(wq->attrs);
@@ -5286,6 +5480,19 @@ static void pwq_release_workfn(struct kthread_work *work)
 		is_last = list_empty(&wq->pwqs);
 
 		/*
+		 * An idle pwq can linger on its wq_flush_pnode list until the
+		 * next flush. wq->mutex excludes flushers, which never drop it
+		 * with pwqs on their private lists.
+		 */
+		if (!list_empty(&pwq->flush_node)) {
+			struct wq_flush_pnode *fpn = wq_flush_pnode(wq, pool->node);
+
+			raw_spin_lock_irq(&fpn->lock);
+			list_del_init(&pwq->flush_node);
+			raw_spin_unlock_irq(&fpn->lock);
+		}
+
+		/*
 		 * For ordered workqueue with a plugged dfl_pwq, restart it now.
 		 */
 		if (!is_last && (wq->flags & __WQ_ORDERED))
@@ -5335,6 +5542,7 @@ static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
 	pwq->refcnt = 1;
 	INIT_LIST_HEAD(&pwq->inactive_works);
 	INIT_LIST_HEAD(&pwq->pending_node);
+	INIT_LIST_HEAD(&pwq->flush_node);
 	INIT_LIST_HEAD(&pwq->pwqs_node);
 	INIT_LIST_HEAD(&pwq->mayday_node);
 	kthread_init_work(&pwq->release_work, pwq_release_workfn);
@@ -5385,22 +5593,34 @@ static struct worker_pool *get_percpu_pool(struct workqueue_struct *wq, int cpu)
 	return &per_cpu_ptr(pools, cpu)[highpri];
 }
 
-/* obtain a pool matching @attr and create a pwq associating the pool and @wq */
+/*
+ * Obtain the pool backing @wq on @cpu and create a pwq associating the two.
+ * A WQ_PERCPU workqueue is backed by the static per-cpu pool of @cpu,
+ * everything else by a pool matching @attrs. @cpu < 0 is always unbound.
+ */
 static struct pool_workqueue *alloc_pwq(struct workqueue_struct *wq,
-					const struct workqueue_attrs *attrs)
+					const struct workqueue_attrs *attrs,
+					int cpu)
 {
 	struct worker_pool *pool;
 	struct pool_workqueue *pwq;
 
 	lockdep_assert_held(&wq_pool_mutex);
 
-	pool = get_unbound_pool(attrs);
-	if (!pool)
-		return NULL;
+	WARN_ON_ONCE((wq->flags & WQ_PERCPU) && cpu < 0);
+
+	if (cpu >= 0 && (wq->flags & WQ_PERCPU)) {
+		pool = get_percpu_pool(wq, cpu);
+	} else {
+		pool = get_unbound_pool(attrs);
+		if (!pool)
+			return NULL;
+	}
 
 	pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
 	if (!pwq) {
-		put_unbound_pool(pool);
+		if (!is_percpu_pool(pool))
+			put_unbound_pool(pool);
 		return NULL;
 	}
 
@@ -5408,6 +5628,13 @@ static struct pool_workqueue *alloc_pwq(struct workqueue_struct *wq,
 	return pwq;
 }
 
+/* create a pwq backed by an unbound pool matching @attrs */
+static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
+						const struct workqueue_attrs *attrs)
+{
+	return alloc_pwq(wq, attrs, -1);
+}
+
 /**
  * wq_calc_pod_cpumask - calculate a wq_attrs' cpumask for a pod
  * @attrs: the wq_attrs of the default pwq of the target workqueue
@@ -5437,10 +5664,10 @@ static void wq_calc_pod_cpumask(struct workqueue_attrs *attrs, int cpu)
 }
 
 /* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */
-static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq,
-					int cpu, struct pool_workqueue *pwq)
+static struct pool_workqueue *install_pwq(struct workqueue_struct *wq,
+					  int cpu, struct pool_workqueue *pwq)
 {
-	struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu);
+	struct pool_workqueue __rcu **slot = pwq_slot(wq, cpu);
 	struct pool_workqueue *old_pwq;
 
 	lockdep_assert_held(&wq_pool_mutex);
@@ -5503,15 +5730,18 @@ apply_wqattrs_prepare(struct workqueue_struct *wq,
 
 	/*
 	 * If something goes wrong during CPU up/down, we'll fall back to
-	 * the default pwq covering whole @attrs->cpumask.  Always create
-	 * it even if we don't use it immediately.
+	 * the default pwq covering whole @attrs->cpumask.  Create it even
+	 * if we don't use it immediately.  A percpu workqueue has a pwq on
+	 * every possible CPU and never falls back, so it has no default.
 	 */
 	copy_workqueue_attrs(new_attrs, attrs);
 	wqattrs_actualize_cpumask(new_attrs, unbound_cpumask);
 	cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask);
-	ctx->dfl_pwq = alloc_pwq(wq, new_attrs);
-	if (!ctx->dfl_pwq)
-		goto out_free;
+	if (!(wq->flags & WQ_PERCPU)) {
+		ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
+		if (!ctx->dfl_pwq)
+			goto out_free;
+	}
 
 	for_each_possible_cpu(cpu) {
 		if (new_attrs->ordered) {
@@ -5519,7 +5749,7 @@ apply_wqattrs_prepare(struct workqueue_struct *wq,
 			ctx->pwq_tbl[cpu] = ctx->dfl_pwq;
 		} else {
 			wq_calc_pod_cpumask(new_attrs, cpu);
-			ctx->pwq_tbl[cpu] = alloc_pwq(wq, new_attrs);
+			ctx->pwq_tbl[cpu] = alloc_pwq(wq, new_attrs, cpu);
 			if (!ctx->pwq_tbl[cpu])
 				goto out_free;
 		}
@@ -5561,9 +5791,10 @@ static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
 
 	/* save the previous pwqs and install the new ones */
 	for_each_possible_cpu(cpu)
-		ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu,
-							ctx->pwq_tbl[cpu]);
-	ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq);
+		ctx->pwq_tbl[cpu] = install_pwq(ctx->wq, cpu,
+						ctx->pwq_tbl[cpu]);
+	if (ctx->dfl_pwq)
+		ctx->dfl_pwq = install_pwq(ctx->wq, -1, ctx->dfl_pwq);
 
 	/* update node_nr_active->max, which only unbound workqueues have */
 	if (ctx->wq->flags & WQ_UNBOUND)
@@ -5577,10 +5808,6 @@ static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,
 {
 	struct apply_wqattrs_ctx *ctx;
 
-	/* only unbound workqueues can change attributes */
-	if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
-		return -EINVAL;
-
 	ctx = apply_wqattrs_prepare(wq, attrs, wq_unbound_cpumask);
 	if (IS_ERR(ctx))
 		return PTR_ERR(ctx);
@@ -5612,6 +5839,10 @@ int apply_workqueue_attrs(struct workqueue_struct *wq,
 {
 	int ret;
 
+	/* only unbound workqueues can change attributes */
+	if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
+		return -EINVAL;
+
 	mutex_lock(&wq_pool_mutex);
 	ret = apply_workqueue_attrs_locked(wq, attrs);
 	mutex_unlock(&wq_pool_mutex);
@@ -5660,11 +5891,11 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu)
 
 	/* nothing to do if the target cpumask matches the current pwq */
 	wq_calc_pod_cpumask(target_attrs, cpu);
-	if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs))
+	if (wqattrs_equal(target_attrs, installed_pwq(wq, cpu)->pool->attrs))
 		return;
 
 	/* create a new pwq */
-	pwq = alloc_pwq(wq, target_attrs);
+	pwq = alloc_unbound_pwq(wq, target_attrs);
 	if (!pwq) {
 		pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n",
 			wq->name);
@@ -5673,49 +5904,43 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu)
 
 	/* Install the new pwq. */
 	mutex_lock(&wq->mutex);
-	old_pwq = install_unbound_pwq(wq, cpu, pwq);
+	old_pwq = install_pwq(wq, cpu, pwq);
 	goto out_unlock;
 
 use_dfl_pwq:
 	mutex_lock(&wq->mutex);
-	pwq = unbound_pwq(wq, -1);
+	pwq = installed_pwq(wq, -1);
 	raw_spin_lock_irq(&pwq->pool->lock);
 	get_pwq(pwq);
 	raw_spin_unlock_irq(&pwq->pool->lock);
-	old_pwq = install_unbound_pwq(wq, cpu, pwq);
+	old_pwq = install_pwq(wq, cpu, pwq);
 out_unlock:
 	mutex_unlock(&wq->mutex);
 	put_pwq_unlocked(old_pwq);
 }
 
-static int alloc_and_link_percpu_pwqs(struct workqueue_struct *wq)
+/* the attrs @wq asked for, as spelled by its flags */
+static struct workqueue_attrs *alloc_wq_std_attrs(struct workqueue_struct *wq)
 {
-	struct pool_workqueue *pwq;
-	int cpu;
-
-	for_each_possible_cpu(cpu) {
-		struct worker_pool *pool = get_percpu_pool(wq, cpu);
-
-		pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
-		if (!pwq)
-			return -ENOMEM;
+	struct workqueue_attrs *attrs;
 
-		init_pwq(pwq, wq, pool);
+	attrs = alloc_workqueue_attrs();
+	if (!attrs)
+		return NULL;
 
-		mutex_lock(&wq->mutex);
-		link_pwq(pwq);
-		mutex_unlock(&wq->mutex);
+	if (wq->flags & WQ_HIGHPRI)
+		attrs->nice = HIGHPRI_NICE_LEVEL;
 
-		rcu_assign_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu), pwq);
-	}
+	if (wq->flags & __WQ_ORDERED)
+		attrs->ordered = true;
 
-	return 0;
+	return attrs;
 }
 
 static int alloc_and_link_pwqs(struct workqueue_struct *wq)
 {
-	bool highpri = wq->flags & WQ_HIGHPRI;
-	int cpu, ret;
+	struct workqueue_attrs *attrs;
+	int ret;
 
 	lockdep_assert_held(&wq_pool_mutex);
 
@@ -5723,47 +5948,29 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq)
 	if (!wq->cpu_pwq)
 		goto enomem;
 
-	if (!(wq->flags & WQ_UNBOUND)) {
-		ret = alloc_and_link_percpu_pwqs(wq);
-	} else if (wq->flags & __WQ_ORDERED) {
-		struct pool_workqueue *dfl_pwq;
+	attrs = alloc_wq_std_attrs(wq);
+	if (!attrs)
+		goto enomem;
+
+	ret = apply_workqueue_attrs_locked(wq, attrs);
+	free_workqueue_attrs(attrs);
+	if (ret)
+		goto enomem;
+
+	if (wq->flags & __WQ_ORDERED) {
+		struct pool_workqueue *dfl_pwq = rcu_access_pointer(wq->dfl_pwq);
 
-		ret = apply_workqueue_attrs_locked(wq, ordered_wq_attrs[highpri]);
 		/* there should only be single pwq for ordering guarantee */
-		dfl_pwq = rcu_access_pointer(wq->dfl_pwq);
-		WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node ||
-			      wq->pwqs.prev != &dfl_pwq->pwqs_node),
+		WARN(wq->pwqs.next != &dfl_pwq->pwqs_node ||
+		     wq->pwqs.prev != &dfl_pwq->pwqs_node,
 		     "ordering guarantee broken for workqueue %s\n", wq->name);
-	} else {
-		ret = apply_workqueue_attrs_locked(wq, unbound_std_wq_attrs[highpri]);
 	}
 
-	if (ret)
-		goto enomem;
 	return 0;
 
 enomem:
-	if (wq->cpu_pwq) {
-		for_each_possible_cpu(cpu) {
-			struct pool_workqueue __rcu **slot;
-			struct pool_workqueue *pwq;
-
-			slot = per_cpu_ptr(wq->cpu_pwq, cpu);
-			pwq = rcu_access_pointer(*slot);
-			if (pwq) {
-				/*
-				 * Unlink pwq from wq->pwqs since link_pwq()
-				 * may have already added it. wq->mutex is not
-				 * needed as the wq has not been published yet.
-				 */
-				if (!list_empty(&pwq->pwqs_node))
-					list_del_rcu(&pwq->pwqs_node);
-				kmem_cache_free(pwq_cache, pwq);
-			}
-		}
-		free_percpu(wq->cpu_pwq);
-		wq->cpu_pwq = NULL;
-	}
+	free_percpu(wq->cpu_pwq);
+	wq->cpu_pwq = NULL;
 	return -ENOMEM;
 }
 
@@ -5828,9 +6035,9 @@ static int init_rescuer(struct workqueue_struct *wq)
  * wq_adjust_max_active - update a wq's max_active to the current setting
  * @wq: target workqueue
  *
- * If @wq isn't freezing, set @wq->max_active to the saved_max_active and
- * activate inactive work items accordingly. If @wq is freezing, clear
- * @wq->max_active to zero.
+ * If @wq isn't freezing, set the limit that applies to @wq's backing to the
+ * saved_max_active and activate inactive work items accordingly. If @wq is
+ * freezing, clear it to zero.
  */
 static void wq_adjust_max_active(struct workqueue_struct *wq)
 {
@@ -5847,20 +6054,25 @@ static void wq_adjust_max_active(struct workqueue_struct *wq)
 		new_min = wq->saved_min_active;
 	}
 
-	if (wq->max_active == new_max && wq->min_active == new_min)
-		return;
-
 	/*
-	 * Update @wq->max/min_active and then kick inactive work items if more
-	 * active work items are allowed. This doesn't break work item ordering
+	 * Update the limit and then kick inactive work items if more active
+	 * work items are allowed. This doesn't break work item ordering
 	 * because new work items are always queued behind existing inactive
 	 * work items if there are any.
 	 */
-	WRITE_ONCE(wq->max_active, new_max);
-	WRITE_ONCE(wq->min_active, new_min);
+	if (wq->flags & WQ_UNBOUND) {
+		if (wq->max_active == new_max && wq->min_active == new_min)
+			return;
 
-	if (wq->flags & WQ_UNBOUND)
+		WRITE_ONCE(wq->max_active, new_max);
+		WRITE_ONCE(wq->min_active, new_min);
 		wq_update_node_max_active(wq, -1);
+	} else {
+		if (wq->percpu_max_active == new_max)
+			return;
+
+		WRITE_ONCE(wq->percpu_max_active, new_max);
+	}
 
 	if (new_max == 0)
 		return;
@@ -5957,10 +6169,14 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt,
 
 	/* init wq */
 	wq->flags = flags;
-	wq->max_active = max_active;
-	wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE);
-	wq->saved_max_active = wq->max_active;
-	wq->saved_min_active = wq->min_active;
+	if (flags & WQ_UNBOUND) {
+		wq->max_active = max_active;
+		wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE);
+		wq->saved_min_active = wq->min_active;
+	} else {
+		wq->percpu_max_active = max_active;
+	}
+	wq->saved_max_active = max_active;
 	mutex_init(&wq->mutex);
 	atomic_set(&wq->nr_pwqs_to_flush, 0);
 	INIT_LIST_HEAD(&wq->pwqs);
@@ -5970,6 +6186,9 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt,
 
 	INIT_LIST_HEAD(&wq->list);
 
+	if (alloc_flush_pnodes(wq) < 0)
+		goto err_free_wq;
+
 	if (flags & WQ_UNBOUND) {
 		if (alloc_node_nr_active(wq->node_nr_active) < 0)
 			goto err_free_wq;
@@ -6007,12 +6226,13 @@ err_unlock_free_node_nr_active:
 	 * flushing the pwq_release_worker ensures that the pwq_release_workfn()
 	 * completes before calling kfree(wq).
 	 */
-	if (wq->flags & WQ_UNBOUND) {
+	if (pwq_release_worker)
 		kthread_flush_worker(pwq_release_worker);
+	if (wq->flags & WQ_UNBOUND)
 		free_node_nr_active(wq->node_nr_active);
-	}
 err_free_wq:
 	free_workqueue_attrs(wq->attrs);
+	free_flush_pnodes(wq);
 	kfree(wq);
 	return NULL;
 err_unlock_destroy:
@@ -6104,11 +6324,8 @@ EXPORT_SYMBOL_GPL(alloc_workqueue_lockdep_map);
 
 static bool pwq_busy(struct pool_workqueue *pwq)
 {
-	int i;
-
-	for (i = 0; i < WORK_NR_COLORS; i++)
-		if (pwq->nr_in_flight[i])
-			return true;
+	if (pwq->total_in_flight)
+		return true;
 
 	if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1))
 		return true;
@@ -6199,12 +6416,12 @@ void destroy_workqueue(struct workqueue_struct *wq)
 	rcu_read_lock();
 
 	for_each_possible_cpu(cpu) {
-		put_pwq_unlocked(unbound_pwq(wq, cpu));
-		RCU_INIT_POINTER(*unbound_pwq_slot(wq, cpu), NULL);
+		put_pwq_unlocked(installed_pwq(wq, cpu));
+		RCU_INIT_POINTER(*pwq_slot(wq, cpu), NULL);
 	}
 
-	put_pwq_unlocked(unbound_pwq(wq, -1));
-	RCU_INIT_POINTER(*unbound_pwq_slot(wq, -1), NULL);
+	put_pwq_unlocked(installed_pwq(wq, -1));
+	RCU_INIT_POINTER(*pwq_slot(wq, -1), NULL);
 
 	rcu_read_unlock();
 }
@@ -8203,24 +8420,6 @@ void __init workqueue_init_early(void)
 			init_cpu_worker_pool(pool, cpu, std_nice[i++]);
 	}
 
-	/* create default unbound and ordered wq attrs */
-	for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
-		struct workqueue_attrs *attrs;
-
-		BUG_ON(!(attrs = alloc_workqueue_attrs()));
-		attrs->nice = std_nice[i];
-		unbound_std_wq_attrs[i] = attrs;
-
-		/*
-		 * An ordered wq should have only one pwq as ordering is
-		 * guaranteed by max_active which is enforced by pwqs.
-		 */
-		BUG_ON(!(attrs = alloc_workqueue_attrs()));
-		attrs->nice = std_nice[i];
-		attrs->ordered = true;
-		ordered_wq_attrs[i] = attrs;
-	}
-
 	system_wq = alloc_workqueue("events", WQ_PERCPU | __WQ_DEPRECATED, 0);
 	system_percpu_wq = alloc_workqueue("events", WQ_PERCPU, 0);
 	system_highpri_wq = alloc_workqueue("events_highpri",
@@ -8264,7 +8463,7 @@ static void __init wq_cpu_intensive_thresh_init(void)
 	 * lot slower CPUs including microcontrollers where the threshold is way
 	 * too low.
 	 *
-	 * Let's scale up the threshold upto 1 second if BogoMips is below 4000.
+	 * Let's scale up the threshold up to 1 second if BogoMips is below 4000.
 	 * This is by no means accurate but it doesn't have to be. The mechanism
 	 * is still useful even when the threshold is fully scaled up. Also, as
 	 * the reports would usually be applicable to everyone, some machines