diff options
Diffstat (limited to 'kernel/workqueue.c')
| -rw-r--r-- | kernel/workqueue.c | 593 |
1 files changed, 396 insertions, 197 deletions
diff --git a/kernel/workqueue.c b/kernel/workqueue.c index c1cb328bce3c..f2e8071a6f4e 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -56,6 +56,7 @@ #include <linux/kvm_para.h> #include <linux/delay.h> #include <linux/irq_work.h> +#include <linux/math64.h> #include "workqueue_internal.h" @@ -161,6 +162,9 @@ struct llc_shard_layout { * LN: pool->lock and wq_node_nr_active->lock protected for writes. Either for * reads. * + * FN: wq_flush_pnode->lock protected. list_empty(&pwq->flush_node) can also be + * tested with only pool->lock held. + * * K: Only modified by worker while holding pool->lock. Can be safely read by * self, while holding pool->lock or from IRQ context if %current is the * kworker. @@ -208,7 +212,7 @@ struct worker_pool { * but w/ pool->lock held. The readers grab pool->lock and are * guaranteed to see if the counter reached zero. */ - int nr_running; + int __data_racy nr_running; struct list_head worklist; /* L: list of pending works */ @@ -274,8 +278,10 @@ struct pool_workqueue { int work_color; /* L: current color */ int flush_color; /* L: flushing color */ int refcnt; /* L: reference count */ + int total_in_flight; /* L: sum of nr_in_flight[] */ int nr_in_flight[WORK_NR_COLORS]; /* L: nr of in_flight works */ + struct list_head flush_node; /* FN: node on wq_flush_pnode->pwqs */ bool plugged; /* L: execution suspended */ /* @@ -345,6 +351,21 @@ struct wq_node_nr_active { }; /* + * Per-node list of pwqs active since the last flush so that flushes visit only + * those. One per possible node plus the NUMA_NO_NODE fallback at nr_node_ids, + * each allocated on its node. A pwq is added when a work item is queued to it + * while off the list and removed by a flush which finds it with nothing in + * flight. ->work_color mirrors wq->work_color. Reading it and adding the pwq in + * one ->lock section keeps queueing coherent against flushing: a work item is + * either stamped with the new color or its pwq is visible to the flusher. + */ +struct wq_flush_pnode { + raw_spinlock_t lock; /* nests inside pool locks */ + int work_color; /* FN: mirrors wq->work_color */ + struct list_head pwqs; /* FN: pwqs active since last flush */ +}; + +/* * The externally visible workqueue. It relays the issued work items to * the appropriate worker_pool through its pool_workqueues. */ @@ -359,6 +380,7 @@ struct workqueue_struct { struct wq_flusher *first_flusher; /* WQ: first flusher */ struct list_head flusher_queue; /* WQ: flush waiters */ struct list_head flusher_overflow; /* WQ: flush overflow list */ + struct wq_flush_pnode **flush_pnodes; /* I: per-node flush membership */ struct list_head maydays; /* MD: pwqs requesting rescue */ struct worker *rescuer; /* MD: rescue worker */ @@ -367,6 +389,7 @@ struct workqueue_struct { /* See alloc_workqueue() function comment for info on min/max_active */ int max_active; /* WO: max active works */ + int percpu_max_active; /* WO: max active works per cpu */ int min_active; /* WO: min active works */ int saved_max_active; /* WQ: saved max_active */ int saved_min_active; /* WQ: saved min_active */ @@ -510,12 +533,6 @@ static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */ /* PL: hash of all unbound pools keyed by pool->attrs */ static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER); -/* I: attributes used when instantiating standard unbound pools on demand */ -static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS]; - -/* I: attributes used when instantiating ordered pools on demand */ -static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS]; - /* * I: kthread_worker to release pwq's. pwq release needs to be bounced to a * process context while holding a pool lock. Bounce to a dedicated kthread @@ -619,6 +636,26 @@ static void show_one_worker_pool(struct worker_pool *pool); list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \ lockdep_is_held(&(wq->mutex))) +/* + * Per-node arrays in this file carry an extra slot at nr_node_ids serving + * NUMA_NO_NODE. Return the slot after all nodes. + */ +static int next_node_with_fallback(int node) +{ + if (node >= nr_node_ids) + return nr_node_ids + 1; + + node = next_node(node, node_possible_map); + if (node >= nr_node_ids) + return nr_node_ids; + return node; +} + +#define for_each_node_with_fallback(node) \ + for ((node) = first_node(node_possible_map); \ + (node) <= nr_node_ids; \ + (node) = next_node_with_fallback(node)) + #ifdef CONFIG_DEBUG_OBJECTS_WORK static const struct debug_obj_descr work_debug_descr; @@ -739,7 +776,7 @@ static int worker_pool_assign_id(struct worker_pool *pool) } static struct pool_workqueue __rcu ** -unbound_pwq_slot(struct workqueue_struct *wq, int cpu) +pwq_slot(struct workqueue_struct *wq, int cpu) { if (cpu >= 0) return per_cpu_ptr(wq->cpu_pwq, cpu); @@ -748,9 +785,9 @@ unbound_pwq_slot(struct workqueue_struct *wq, int cpu) } /* @cpu < 0 for dfl_pwq */ -static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu) +static struct pool_workqueue *installed_pwq(struct workqueue_struct *wq, int cpu) { - return rcu_dereference_check(*unbound_pwq_slot(wq, cpu), + return rcu_dereference_check(*pwq_slot(wq, cpu), lockdep_is_held(&wq_pool_mutex) || lockdep_is_held(&wq->mutex)); } @@ -765,7 +802,7 @@ static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu) */ static struct cpumask *unbound_effective_cpumask(struct workqueue_struct *wq) { - return unbound_pwq(wq, -1)->pool->attrs->__pod_cpumask; + return installed_pwq(wq, -1)->pool->attrs->__pod_cpumask; } static unsigned int work_color_to_flags(int color) @@ -1532,6 +1569,7 @@ void wq_worker_tick(struct task_struct *task) struct worker *worker = kthread_data(task); struct pool_workqueue *pwq = worker->current_pwq; struct worker_pool *pool = worker->pool; + u64 delta; if (!pwq) return; @@ -1572,6 +1610,11 @@ void wq_worker_tick(struct task_struct *task) pwq->stats[PWQ_STAT_CM_WAKEUP]++; raw_spin_unlock(&pool->lock); + + delta = READ_ONCE(worker->task->se.sum_exec_runtime) - worker->current_at; + trace_workqueue_cpu_intensive(pwq, worker->current_work, + worker->current_func, + div_u64(delta, NSEC_PER_USEC)); } /** @@ -1635,6 +1678,22 @@ static struct wq_node_nr_active *wq_node_nr_active(struct workqueue_struct *wq, } /** + * wq_flush_pnode - Determine wq_flush_pnode to use + * @wq: workqueue of interest + * @node: NUMA node, can be %NUMA_NO_NODE + * + * Return @wq's wq_flush_pnode for @node, the nr_node_ids fallback if @node is + * %NUMA_NO_NODE. + */ +static struct wq_flush_pnode *wq_flush_pnode(struct workqueue_struct *wq, int node) +{ + if (node == NUMA_NO_NODE) + node = nr_node_ids; + + return wq->flush_pnodes[node]; +} + +/** * wq_update_node_max_active - Update per-node max_actives to use * @wq: workqueue to update * @off_cpu: CPU that's going down, -1 if a CPU is not going down @@ -1712,6 +1771,14 @@ static void put_pwq(struct pool_workqueue *pwq) lockdep_assert_held(&pwq->pool->lock); if (likely(--pwq->refcnt)) return; + + /* + * pwq_release_worker is only created in workqueue_init(). Getting + * here before that means an early workqueue allocation failed, which + * is fatal anyway. + */ + BUG_ON(!pwq_release_worker); + /* * @pwq can't be released under pool->lock, bounce to a dedicated * kthread_worker to avoid A-A deadlocks. @@ -1788,10 +1855,10 @@ static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill) /* * A concurrency-managed per-cpu pool accounts nr_active per pwq, so - * pwq->nr_active against wq->max_active is sufficient. + * pwq->nr_active against wq->percpu_max_active is sufficient. */ if (is_percpu_pool(pool)) { - obtained = pwq->nr_active < READ_ONCE(wq->max_active); + obtained = pwq->nr_active < READ_ONCE(wq->percpu_max_active); goto out; } @@ -2060,6 +2127,22 @@ static void pwq_dec_nr_active(struct pool_workqueue *pwq) } /** + * pwq_inc_nr_in_flight - increment pwq's nr_in_flight + * @pwq: pwq of interest + * @work_color: color of the work item being queued + * + * A work item or a barrier with @work_color is being queued to @pwq. + * + * CONTEXT: + * raw_spin_lock_irq(pool->lock). + */ +static void pwq_inc_nr_in_flight(struct pool_workqueue *pwq, int work_color) +{ + pwq->nr_in_flight[work_color]++; + pwq->total_in_flight++; +} + +/** * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight * @pwq: pwq of interest * @work_data: work_data of work which left the queue @@ -2083,6 +2166,7 @@ static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_ pwq_dec_nr_active(pwq); pwq->nr_in_flight[color]--; + pwq->total_in_flight--; /* is flush in progress and are we at the flushing tip? */ if (likely(pwq->flush_color != color)) @@ -2432,7 +2516,21 @@ retry: if (WARN_ON(!list_empty(&work->entry))) goto out; - pwq->nr_in_flight[pwq->work_color]++; + /* + * Add @pwq to its wq_flush_pnode list if off it. Syncing ->work_color + * in the same fpn->lock section is what keeps a concurrent flush from + * missing @work, see wq_flush_pnode. + */ + if (unlikely(list_empty(&pwq->flush_node))) { + struct wq_flush_pnode *fpn = wq_flush_pnode(wq, pool->node); + + raw_spin_lock(&fpn->lock); + pwq->work_color = fpn->work_color; + list_add_tail(&pwq->flush_node, &fpn->pwqs); + raw_spin_unlock(&fpn->lock); + } + + pwq_inc_nr_in_flight(pwq, pwq->work_color); work_flags = work_color_to_flags(pwq->work_color); /* @@ -3117,6 +3215,7 @@ static void send_mayday(struct pool_workqueue *pwq) list_add_tail(&pwq->mayday_node, &wq->maydays); wake_up_process(wq->rescuer->task); pwq->stats[PWQ_STAT_MAYDAY]++; + trace_workqueue_mayday(pwq); } } @@ -3623,6 +3722,7 @@ static bool assign_rescuer_work(struct pool_workqueue *pwq, struct worker *rescu list_for_each_entry_safe_from(work, n, &pool->worklist, entry) { if (get_work_pwq(work) == pwq && assign_work(work, rescuer, &n)) { pwq->stats[PWQ_STAT_RESCUED]++; + trace_workqueue_rescued(pwq, work, work->func); /* put the cursor for next search */ list_move_tail(&cursor->entry, &n->entry); return true; @@ -3758,6 +3858,9 @@ static void bh_worker(struct worker *worker) struct worker_pool *pool = worker->pool; int nr_restarts = BH_WORKER_RESTARTS; unsigned long end = jiffies + BH_WORKER_JIFFIES; + bool budget_exhausted = false; + bool timeout = false; + int restarts = 0; worker_lock_callback(pool); raw_spin_lock_irq(&pool->lock); @@ -3780,8 +3883,23 @@ static void bh_worker(struct worker *worker) if (assign_work(work, worker, NULL)) process_scheduled_works(worker); - } while (keep_working(pool) && - --nr_restarts && time_before(jiffies, end)); + + if (!keep_working(pool)) + break; + + if (!--nr_restarts) { + budget_exhausted = true; + break; + } + + if (!time_before(jiffies, end)) { + budget_exhausted = true; + timeout = true; + break; + } + + restarts++; + } while (1); worker_set_flags(worker, WORKER_PREP); done: @@ -3789,6 +3907,10 @@ done: kick_pool(pool); raw_spin_unlock_irq(&pool->lock); worker_unlock_callback(pool); + + if (budget_exhausted) + trace_workqueue_bh_budget_yield(pool, restarts, timeout, + pool->attrs->nice == HIGHPRI_NICE_LEVEL); } /* @@ -4008,7 +4130,15 @@ static void insert_wq_barrier(struct pool_workqueue *pwq, __set_bit(WORK_STRUCT_LINKED_BIT, bits); } - pwq->nr_in_flight[work_color]++; + /* + * Flushes must wait for barriers too as a barrier pins @pwq and a + * worker until it runs and destroy_workqueue()'s drain relies on that. + * @target being in flight keeps @pwq on its wq_flush_pnode list, so no + * need to add it here. + */ + WARN_ON_ONCE(list_empty(&pwq->flush_node)); + + pwq_inc_nr_in_flight(pwq, work_color); work_flags |= work_color_to_flags(work_color); insert_work(pwq, &barr->work, head, work_flags); @@ -4020,23 +4150,23 @@ static void insert_wq_barrier(struct pool_workqueue *pwq, * @flush_color: new flush color, < 0 for no-op * @work_color: new work color, < 0 for no-op * - * Prepare pwqs for workqueue flushing. + * Prepare pwqs for workqueue flushing. Only the pwqs on @wq's wq_flush_pnode + * lists, which have been active since the last flush, are visited. * - * If @flush_color is non-negative, flush_color on all pwqs should be - * -1. If no pwq has in-flight commands at the specified color, all - * pwq->flush_color's stay at -1 and %false is returned. If any pwq - * has in flight commands, its pwq->flush_color is set to - * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq - * wakeup logic is armed and %true is returned. + * If @flush_color >= 0, flush_color on all visited pwqs should be -1. If no + * visited pwq has in-flight work items at the specified color, all + * pwq->flush_color's stay at -1 and return %false. If any visited pwq has + * in-flight work items, set its pwq->flush_color to @flush_color, update + * @wq->nr_pwqs_to_flush accordingly, arm pwq wakeup logic and return %true. * - * The caller should have initialized @wq->first_flusher prior to - * calling this function with non-negative @flush_color. If - * @flush_color is negative, no flush color update is done and %false - * is returned. + * The caller should have initialized @wq->first_flusher prior to calling this + * function with non-negative @flush_color. If @flush_color < 0, no flush color + * update is done and %false is returned. * - * If @work_color is non-negative, all pwqs should have the same - * work_color which is previous to @work_color and all will be - * advanced to @work_color. + * If @work_color >= 0, all visited pwqs and every wq_flush_pnode's work color + * mirror are advanced to @work_color. The mirrors of nodes with no active pwqs + * must be advanced too as a pwq added to a list later stamps the color it reads + * from the mirror. * * CONTEXT: * mutex_lock(wq->mutex). @@ -4048,50 +4178,81 @@ static void insert_wq_barrier(struct pool_workqueue *pwq, static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq, int flush_color, int work_color) { + struct pool_workqueue *pwq, *next; bool wait = false; - struct pool_workqueue *pwq; - struct worker_pool *current_pool = NULL; + int node; if (flush_color >= 0) { WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush)); atomic_set(&wq->nr_pwqs_to_flush, 1); } - /* - * For unbound workqueue, pwqs will map to only a few pools. - * Most of the time, pwqs within the same pool will be linked - * sequentially to wq->pwqs by cpu index. So in the majority - * of pwq iters, the pool is the same, only doing lock/unlock - * if the pool has changed. This can largely reduce expensive - * lock operations. - */ - for_each_pwq(pwq, wq) { - if (current_pool != pwq->pool) { - if (likely(current_pool)) - raw_spin_unlock_irq(¤t_pool->lock); - current_pool = pwq->pool; - raw_spin_lock_irq(¤t_pool->lock); + for_each_node_with_fallback(node) { + struct wq_flush_pnode *fpn = wq->flush_pnodes[node]; + struct worker_pool *current_pool = NULL; + LIST_HEAD(to_visit); + LIST_HEAD(to_keep); + + /* + * Advance the node's color mirror and take its active pwqs. + * Work items queued to off-list pwqs afterwards carry the new + * color and don't need visiting. + */ + raw_spin_lock_irq(&fpn->lock); + if (work_color >= 0) { + WARN_ON_ONCE(work_color != work_next_color(fpn->work_color)); + fpn->work_color = work_color; } + list_splice_init(&fpn->pwqs, &to_visit); + raw_spin_unlock_irq(&fpn->lock); + + /* + * Unbound workqueues map multiple pwqs to one pool. Only cycle + * pool->lock when the pool changes. + */ + list_for_each_entry_safe(pwq, next, &to_visit, flush_node) { + if (current_pool != pwq->pool) { + if (likely(current_pool)) + raw_spin_unlock_irq(¤t_pool->lock); + current_pool = pwq->pool; + raw_spin_lock_irq(¤t_pool->lock); + } - if (flush_color >= 0) { - WARN_ON_ONCE(pwq->flush_color != -1); + if (flush_color >= 0) { + WARN_ON_ONCE(pwq->flush_color != -1); - if (pwq->nr_in_flight[flush_color]) { - pwq->flush_color = flush_color; - atomic_inc(&wq->nr_pwqs_to_flush); - wait = true; + if (pwq->nr_in_flight[flush_color]) { + pwq->flush_color = flush_color; + atomic_inc(&wq->nr_pwqs_to_flush); + wait = true; + } } - } - if (work_color >= 0) { - WARN_ON_ONCE(work_color != work_next_color(pwq->work_color)); - pwq->work_color = work_color; + if (work_color >= 0) { + WARN_ON_ONCE(work_color != work_next_color(pwq->work_color)); + pwq->work_color = work_color; + } + + /* + * A cascade can arm a color retired several advances + * ago, so keep a pwq listed while any color has work + * items in flight. + */ + if (pwq->total_in_flight) + list_move_tail(&pwq->flush_node, &to_keep); + else + list_del_init(&pwq->flush_node); } - } + if (current_pool) + raw_spin_unlock_irq(¤t_pool->lock); - if (current_pool) - raw_spin_unlock_irq(¤t_pool->lock); + if (!list_empty(&to_keep)) { + raw_spin_lock_irq(&fpn->lock); + list_splice_tail(&to_keep, &fpn->pwqs); + raw_spin_unlock_irq(&fpn->lock); + } + } if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush)) complete(&wq->first_flusher->done); @@ -5049,13 +5210,10 @@ static void free_node_nr_active(struct wq_node_nr_active **nna_ar) { int node; - for_each_node(node) { + for_each_node_with_fallback(node) { kfree(nna_ar[node]); nna_ar[node] = NULL; } - - kfree(nna_ar[nr_node_ids]); - nna_ar[nr_node_ids] = NULL; } static void init_node_nr_active(struct wq_node_nr_active *nna) @@ -5075,21 +5233,15 @@ static int alloc_node_nr_active(struct wq_node_nr_active **nna_ar) struct wq_node_nr_active *nna; int node; - for_each_node(node) { - nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, node); + for_each_node_with_fallback(node) { + nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, + node < nr_node_ids ? node : NUMA_NO_NODE); if (!nna) goto err_free; init_node_nr_active(nna); nna_ar[node] = nna; } - /* [nr_node_ids] is used as the fallback */ - nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, NUMA_NO_NODE); - if (!nna) - goto err_free; - init_node_nr_active(nna); - nna_ar[nr_node_ids] = nna; - return 0; err_free: @@ -5097,6 +5249,47 @@ err_free: return -ENOMEM; } +static void free_flush_pnodes(struct workqueue_struct *wq) +{ + int node; + + if (!wq->flush_pnodes) + return; + + for_each_node_with_fallback(node) + kfree(wq->flush_pnodes[node]); + kfree(wq->flush_pnodes); + wq->flush_pnodes = NULL; +} + +static int alloc_flush_pnodes(struct workqueue_struct *wq) +{ + struct wq_flush_pnode *fpn; + int node; + + wq->flush_pnodes = kcalloc(nr_node_ids + 1, sizeof(*wq->flush_pnodes), GFP_KERNEL); + if (!wq->flush_pnodes) + return -ENOMEM; + + for_each_node_with_fallback(node) { + fpn = kzalloc_node(sizeof(*fpn), GFP_KERNEL, + node < nr_node_ids ? node : NUMA_NO_NODE); + if (!fpn) + goto err_free; + + raw_spin_lock_init(&fpn->lock); + fpn->work_color = wq->work_color; + INIT_LIST_HEAD(&fpn->pwqs); + wq->flush_pnodes[node] = fpn; + } + + return 0; + +err_free: + free_flush_pnodes(wq); + return -ENOMEM; +} + static void rcu_free_wq(struct rcu_head *rcu) { struct workqueue_struct *wq = @@ -5105,6 +5298,7 @@ static void rcu_free_wq(struct rcu_head *rcu) if (wq->flags & WQ_UNBOUND) free_node_nr_active(wq->node_nr_active); + free_flush_pnodes(wq); wq_free_lockdep(wq); free_percpu(wq->cpu_pwq); free_workqueue_attrs(wq->attrs); @@ -5286,6 +5480,19 @@ static void pwq_release_workfn(struct kthread_work *work) is_last = list_empty(&wq->pwqs); /* + * An idle pwq can linger on its wq_flush_pnode list until the + * next flush. wq->mutex excludes flushers, which never drop it + * with pwqs on their private lists. + */ + if (!list_empty(&pwq->flush_node)) { + struct wq_flush_pnode *fpn = wq_flush_pnode(wq, pool->node); + + raw_spin_lock_irq(&fpn->lock); + list_del_init(&pwq->flush_node); + raw_spin_unlock_irq(&fpn->lock); + } + + /* * For ordered workqueue with a plugged dfl_pwq, restart it now. */ if (!is_last && (wq->flags & __WQ_ORDERED)) @@ -5335,6 +5542,7 @@ static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq, pwq->refcnt = 1; INIT_LIST_HEAD(&pwq->inactive_works); INIT_LIST_HEAD(&pwq->pending_node); + INIT_LIST_HEAD(&pwq->flush_node); INIT_LIST_HEAD(&pwq->pwqs_node); INIT_LIST_HEAD(&pwq->mayday_node); kthread_init_work(&pwq->release_work, pwq_release_workfn); @@ -5385,22 +5593,34 @@ static struct worker_pool *get_percpu_pool(struct workqueue_struct *wq, int cpu) return &per_cpu_ptr(pools, cpu)[highpri]; } -/* obtain a pool matching @attr and create a pwq associating the pool and @wq */ +/* + * Obtain the pool backing @wq on @cpu and create a pwq associating the two. + * A WQ_PERCPU workqueue is backed by the static per-cpu pool of @cpu, + * everything else by a pool matching @attrs. @cpu < 0 is always unbound. + */ static struct pool_workqueue *alloc_pwq(struct workqueue_struct *wq, - const struct workqueue_attrs *attrs) + const struct workqueue_attrs *attrs, + int cpu) { struct worker_pool *pool; struct pool_workqueue *pwq; lockdep_assert_held(&wq_pool_mutex); - pool = get_unbound_pool(attrs); - if (!pool) - return NULL; + WARN_ON_ONCE((wq->flags & WQ_PERCPU) && cpu < 0); + + if (cpu >= 0 && (wq->flags & WQ_PERCPU)) { + pool = get_percpu_pool(wq, cpu); + } else { + pool = get_unbound_pool(attrs); + if (!pool) + return NULL; + } pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node); if (!pwq) { - put_unbound_pool(pool); + if (!is_percpu_pool(pool)) + put_unbound_pool(pool); return NULL; } @@ -5408,6 +5628,13 @@ static struct pool_workqueue *alloc_pwq(struct workqueue_struct *wq, return pwq; } +/* create a pwq backed by an unbound pool matching @attrs */ +static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq, + const struct workqueue_attrs *attrs) +{ + return alloc_pwq(wq, attrs, -1); +} + /** * wq_calc_pod_cpumask - calculate a wq_attrs' cpumask for a pod * @attrs: the wq_attrs of the default pwq of the target workqueue @@ -5437,10 +5664,10 @@ static void wq_calc_pod_cpumask(struct workqueue_attrs *attrs, int cpu) } /* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */ -static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq, - int cpu, struct pool_workqueue *pwq) +static struct pool_workqueue *install_pwq(struct workqueue_struct *wq, + int cpu, struct pool_workqueue *pwq) { - struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu); + struct pool_workqueue __rcu **slot = pwq_slot(wq, cpu); struct pool_workqueue *old_pwq; lockdep_assert_held(&wq_pool_mutex); @@ -5503,15 +5730,18 @@ apply_wqattrs_prepare(struct workqueue_struct *wq, /* * If something goes wrong during CPU up/down, we'll fall back to - * the default pwq covering whole @attrs->cpumask. Always create - * it even if we don't use it immediately. + * the default pwq covering whole @attrs->cpumask. Create it even + * if we don't use it immediately. A percpu workqueue has a pwq on + * every possible CPU and never falls back, so it has no default. */ copy_workqueue_attrs(new_attrs, attrs); wqattrs_actualize_cpumask(new_attrs, unbound_cpumask); cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask); - ctx->dfl_pwq = alloc_pwq(wq, new_attrs); - if (!ctx->dfl_pwq) - goto out_free; + if (!(wq->flags & WQ_PERCPU)) { + ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs); + if (!ctx->dfl_pwq) + goto out_free; + } for_each_possible_cpu(cpu) { if (new_attrs->ordered) { @@ -5519,7 +5749,7 @@ apply_wqattrs_prepare(struct workqueue_struct *wq, ctx->pwq_tbl[cpu] = ctx->dfl_pwq; } else { wq_calc_pod_cpumask(new_attrs, cpu); - ctx->pwq_tbl[cpu] = alloc_pwq(wq, new_attrs); + ctx->pwq_tbl[cpu] = alloc_pwq(wq, new_attrs, cpu); if (!ctx->pwq_tbl[cpu]) goto out_free; } @@ -5561,9 +5791,10 @@ static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx) /* save the previous pwqs and install the new ones */ for_each_possible_cpu(cpu) - ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu, - ctx->pwq_tbl[cpu]); - ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq); + ctx->pwq_tbl[cpu] = install_pwq(ctx->wq, cpu, + ctx->pwq_tbl[cpu]); + if (ctx->dfl_pwq) + ctx->dfl_pwq = install_pwq(ctx->wq, -1, ctx->dfl_pwq); /* update node_nr_active->max, which only unbound workqueues have */ if (ctx->wq->flags & WQ_UNBOUND) @@ -5577,10 +5808,6 @@ static int apply_workqueue_attrs_locked(struct workqueue_struct *wq, { struct apply_wqattrs_ctx *ctx; - /* only unbound workqueues can change attributes */ - if (WARN_ON(!(wq->flags & WQ_UNBOUND))) - return -EINVAL; - ctx = apply_wqattrs_prepare(wq, attrs, wq_unbound_cpumask); if (IS_ERR(ctx)) return PTR_ERR(ctx); @@ -5612,6 +5839,10 @@ int apply_workqueue_attrs(struct workqueue_struct *wq, { int ret; + /* only unbound workqueues can change attributes */ + if (WARN_ON(!(wq->flags & WQ_UNBOUND))) + return -EINVAL; + mutex_lock(&wq_pool_mutex); ret = apply_workqueue_attrs_locked(wq, attrs); mutex_unlock(&wq_pool_mutex); @@ -5660,11 +5891,11 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) /* nothing to do if the target cpumask matches the current pwq */ wq_calc_pod_cpumask(target_attrs, cpu); - if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs)) + if (wqattrs_equal(target_attrs, installed_pwq(wq, cpu)->pool->attrs)) return; /* create a new pwq */ - pwq = alloc_pwq(wq, target_attrs); + pwq = alloc_unbound_pwq(wq, target_attrs); if (!pwq) { pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n", wq->name); @@ -5673,49 +5904,43 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) /* Install the new pwq. */ mutex_lock(&wq->mutex); - old_pwq = install_unbound_pwq(wq, cpu, pwq); + old_pwq = install_pwq(wq, cpu, pwq); goto out_unlock; use_dfl_pwq: mutex_lock(&wq->mutex); - pwq = unbound_pwq(wq, -1); + pwq = installed_pwq(wq, -1); raw_spin_lock_irq(&pwq->pool->lock); get_pwq(pwq); raw_spin_unlock_irq(&pwq->pool->lock); - old_pwq = install_unbound_pwq(wq, cpu, pwq); + old_pwq = install_pwq(wq, cpu, pwq); out_unlock: mutex_unlock(&wq->mutex); put_pwq_unlocked(old_pwq); } -static int alloc_and_link_percpu_pwqs(struct workqueue_struct *wq) +/* the attrs @wq asked for, as spelled by its flags */ +static struct workqueue_attrs *alloc_wq_std_attrs(struct workqueue_struct *wq) { - struct pool_workqueue *pwq; - int cpu; - - for_each_possible_cpu(cpu) { - struct worker_pool *pool = get_percpu_pool(wq, cpu); - - pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node); - if (!pwq) - return -ENOMEM; + struct workqueue_attrs *attrs; - init_pwq(pwq, wq, pool); + attrs = alloc_workqueue_attrs(); + if (!attrs) + return NULL; - mutex_lock(&wq->mutex); - link_pwq(pwq); - mutex_unlock(&wq->mutex); + if (wq->flags & WQ_HIGHPRI) + attrs->nice = HIGHPRI_NICE_LEVEL; - rcu_assign_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu), pwq); - } + if (wq->flags & __WQ_ORDERED) + attrs->ordered = true; - return 0; + return attrs; } static int alloc_and_link_pwqs(struct workqueue_struct *wq) { - bool highpri = wq->flags & WQ_HIGHPRI; - int cpu, ret; + struct workqueue_attrs *attrs; + int ret; lockdep_assert_held(&wq_pool_mutex); @@ -5723,47 +5948,29 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq) if (!wq->cpu_pwq) goto enomem; - if (!(wq->flags & WQ_UNBOUND)) { - ret = alloc_and_link_percpu_pwqs(wq); - } else if (wq->flags & __WQ_ORDERED) { - struct pool_workqueue *dfl_pwq; + attrs = alloc_wq_std_attrs(wq); + if (!attrs) + goto enomem; + + ret = apply_workqueue_attrs_locked(wq, attrs); + free_workqueue_attrs(attrs); + if (ret) + goto enomem; + + if (wq->flags & __WQ_ORDERED) { + struct pool_workqueue *dfl_pwq = rcu_access_pointer(wq->dfl_pwq); - ret = apply_workqueue_attrs_locked(wq, ordered_wq_attrs[highpri]); /* there should only be single pwq for ordering guarantee */ - dfl_pwq = rcu_access_pointer(wq->dfl_pwq); - WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node || - wq->pwqs.prev != &dfl_pwq->pwqs_node), + WARN(wq->pwqs.next != &dfl_pwq->pwqs_node || + wq->pwqs.prev != &dfl_pwq->pwqs_node, "ordering guarantee broken for workqueue %s\n", wq->name); - } else { - ret = apply_workqueue_attrs_locked(wq, unbound_std_wq_attrs[highpri]); } - if (ret) - goto enomem; return 0; enomem: - if (wq->cpu_pwq) { - for_each_possible_cpu(cpu) { - struct pool_workqueue __rcu **slot; - struct pool_workqueue *pwq; - - slot = per_cpu_ptr(wq->cpu_pwq, cpu); - pwq = rcu_access_pointer(*slot); - if (pwq) { - /* - * Unlink pwq from wq->pwqs since link_pwq() - * may have already added it. wq->mutex is not - * needed as the wq has not been published yet. - */ - if (!list_empty(&pwq->pwqs_node)) - list_del_rcu(&pwq->pwqs_node); - kmem_cache_free(pwq_cache, pwq); - } - } - free_percpu(wq->cpu_pwq); - wq->cpu_pwq = NULL; - } + free_percpu(wq->cpu_pwq); + wq->cpu_pwq = NULL; return -ENOMEM; } @@ -5828,9 +6035,9 @@ static int init_rescuer(struct workqueue_struct *wq) * wq_adjust_max_active - update a wq's max_active to the current setting * @wq: target workqueue * - * If @wq isn't freezing, set @wq->max_active to the saved_max_active and - * activate inactive work items accordingly. If @wq is freezing, clear - * @wq->max_active to zero. + * If @wq isn't freezing, set the limit that applies to @wq's backing to the + * saved_max_active and activate inactive work items accordingly. If @wq is + * freezing, clear it to zero. */ static void wq_adjust_max_active(struct workqueue_struct *wq) { @@ -5847,20 +6054,25 @@ static void wq_adjust_max_active(struct workqueue_struct *wq) new_min = wq->saved_min_active; } - if (wq->max_active == new_max && wq->min_active == new_min) - return; - /* - * Update @wq->max/min_active and then kick inactive work items if more - * active work items are allowed. This doesn't break work item ordering + * Update the limit and then kick inactive work items if more active + * work items are allowed. This doesn't break work item ordering * because new work items are always queued behind existing inactive * work items if there are any. */ - WRITE_ONCE(wq->max_active, new_max); - WRITE_ONCE(wq->min_active, new_min); + if (wq->flags & WQ_UNBOUND) { + if (wq->max_active == new_max && wq->min_active == new_min) + return; - if (wq->flags & WQ_UNBOUND) + WRITE_ONCE(wq->max_active, new_max); + WRITE_ONCE(wq->min_active, new_min); wq_update_node_max_active(wq, -1); + } else { + if (wq->percpu_max_active == new_max) + return; + + WRITE_ONCE(wq->percpu_max_active, new_max); + } if (new_max == 0) return; @@ -5957,10 +6169,14 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, /* init wq */ wq->flags = flags; - wq->max_active = max_active; - wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE); - wq->saved_max_active = wq->max_active; - wq->saved_min_active = wq->min_active; + if (flags & WQ_UNBOUND) { + wq->max_active = max_active; + wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE); + wq->saved_min_active = wq->min_active; + } else { + wq->percpu_max_active = max_active; + } + wq->saved_max_active = max_active; mutex_init(&wq->mutex); atomic_set(&wq->nr_pwqs_to_flush, 0); INIT_LIST_HEAD(&wq->pwqs); @@ -5970,6 +6186,9 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, INIT_LIST_HEAD(&wq->list); + if (alloc_flush_pnodes(wq) < 0) + goto err_free_wq; + if (flags & WQ_UNBOUND) { if (alloc_node_nr_active(wq->node_nr_active) < 0) goto err_free_wq; @@ -6007,12 +6226,13 @@ err_unlock_free_node_nr_active: * flushing the pwq_release_worker ensures that the pwq_release_workfn() * completes before calling kfree(wq). */ - if (wq->flags & WQ_UNBOUND) { + if (pwq_release_worker) kthread_flush_worker(pwq_release_worker); + if (wq->flags & WQ_UNBOUND) free_node_nr_active(wq->node_nr_active); - } err_free_wq: free_workqueue_attrs(wq->attrs); + free_flush_pnodes(wq); kfree(wq); return NULL; err_unlock_destroy: @@ -6104,11 +6324,8 @@ EXPORT_SYMBOL_GPL(alloc_workqueue_lockdep_map); static bool pwq_busy(struct pool_workqueue *pwq) { - int i; - - for (i = 0; i < WORK_NR_COLORS; i++) - if (pwq->nr_in_flight[i]) - return true; + if (pwq->total_in_flight) + return true; if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1)) return true; @@ -6199,12 +6416,12 @@ void destroy_workqueue(struct workqueue_struct *wq) rcu_read_lock(); for_each_possible_cpu(cpu) { - put_pwq_unlocked(unbound_pwq(wq, cpu)); - RCU_INIT_POINTER(*unbound_pwq_slot(wq, cpu), NULL); + put_pwq_unlocked(installed_pwq(wq, cpu)); + RCU_INIT_POINTER(*pwq_slot(wq, cpu), NULL); } - put_pwq_unlocked(unbound_pwq(wq, -1)); - RCU_INIT_POINTER(*unbound_pwq_slot(wq, -1), NULL); + put_pwq_unlocked(installed_pwq(wq, -1)); + RCU_INIT_POINTER(*pwq_slot(wq, -1), NULL); rcu_read_unlock(); } @@ -8203,24 +8420,6 @@ void __init workqueue_init_early(void) init_cpu_worker_pool(pool, cpu, std_nice[i++]); } - /* create default unbound and ordered wq attrs */ - for (i = 0; i < NR_STD_WORKER_POOLS; i++) { - struct workqueue_attrs *attrs; - - BUG_ON(!(attrs = alloc_workqueue_attrs())); - attrs->nice = std_nice[i]; - unbound_std_wq_attrs[i] = attrs; - - /* - * An ordered wq should have only one pwq as ordering is - * guaranteed by max_active which is enforced by pwqs. - */ - BUG_ON(!(attrs = alloc_workqueue_attrs())); - attrs->nice = std_nice[i]; - attrs->ordered = true; - ordered_wq_attrs[i] = attrs; - } - system_wq = alloc_workqueue("events", WQ_PERCPU | __WQ_DEPRECATED, 0); system_percpu_wq = alloc_workqueue("events", WQ_PERCPU, 0); system_highpri_wq = alloc_workqueue("events_highpri", @@ -8264,7 +8463,7 @@ static void __init wq_cpu_intensive_thresh_init(void) * lot slower CPUs including microcontrollers where the threshold is way * too low. * - * Let's scale up the threshold upto 1 second if BogoMips is below 4000. + * Let's scale up the threshold up to 1 second if BogoMips is below 4000. * This is by no means accurate but it doesn't have to be. The mechanism * is still useful even when the threshold is fully scaled up. Also, as * the reports would usually be applicable to everyone, some machines |