diff options
| -rw-r--r-- | Documentation/core-api/workqueue.rst | 2 | ||||
| -rw-r--r-- | kernel/workqueue.c | 323 | ||||
| -rw-r--r-- | tools/workqueue/wq_dump.py | 16 | ||||
| -rw-r--r-- | tools/workqueue/wq_monitor.py | 6 |
4 files changed, 240 insertions, 107 deletions
diff --git a/Documentation/core-api/workqueue.rst b/Documentation/core-api/workqueue.rst index 411e1b28b8de..bb770f556568 100644 --- a/Documentation/core-api/workqueue.rst +++ b/Documentation/core-api/workqueue.rst @@ -356,7 +356,7 @@ Guidelines well under the default limit. * A wq serves as a domain for forward progress guarantee - (``WQ_MEM_RECLAIM``, flush and work item attributes. Work items + (``WQ_MEM_RECLAIM``), flush and work item attributes. Work items which are not involved in memory reclaim and don't need to be flushed as a part of a group of work items, and don't require any special attribute, can use one of the system wq. There is no diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 78068ae8f28a..bfeef512f6dd 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -226,6 +226,8 @@ struct worker_pool { /* L: hash of busy workers */ struct worker *manager; /* L: purely informational */ + /* L: last worker woken by kick_pool() */ + struct worker *last_woken_worker; struct list_head workers; /* A: attached workers */ struct ida worker_ida; /* worker IDs for task name */ @@ -369,7 +371,7 @@ struct workqueue_struct { int saved_max_active; /* WQ: saved max_active */ int saved_min_active; /* WQ: saved min_active */ - struct workqueue_attrs *unbound_attrs; /* PW: only for unbound wqs */ + struct workqueue_attrs *attrs; /* PW: workqueue attributes */ struct pool_workqueue __rcu *dfl_pwq; /* PW: only for unbound wqs */ #ifdef CONFIG_SYSFS @@ -757,7 +759,7 @@ static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu) * unbound_effective_cpumask - effective cpumask of an unbound workqueue * @wq: workqueue of interest * - * @wq->unbound_attrs->cpumask contains the cpumask requested by the user which + * @wq->attrs->cpumask contains the cpumask requested by the user which * is masked with wq_unbound_cpumask to determine the effective cpumask. The * default pwq is always mapped to the pool with the current effective cpumask. */ @@ -1258,19 +1260,27 @@ static void kick_bh_pool(struct worker_pool *pool) } /** - * kick_pool - wake up an idle worker if necessary + * kick_pool_pick - select an idle worker to kick, deferring the wakeup * @pool: pool to kick + * @wakep: out-param, set to the task to wake after pool->lock is dropped * - * @pool may have pending work items. Wake up worker if necessary. Returns - * whether a worker was woken up. + * Like kick_pool() but, for a regular (non-BH) pool, returns the picked + * worker's task via @wakep instead of waking it, so the caller can issue the + * wakeup after dropping pool->lock (the wakeup takes rq->lock). Worker + * selection, wake_cpu setup and the BH kick still happen under the lock. + * Returns whether a worker was selected or kicked. + * + * Must be called with @pool->lock held. */ -static bool kick_pool(struct worker_pool *pool) +static bool kick_pool_pick(struct worker_pool *pool, struct task_struct **wakep) { struct worker *worker = first_idle_worker(pool); struct task_struct *p; lockdep_assert_held(&pool->lock); + *wakep = NULL; + if (!need_more_worker(pool) || !worker) return false; @@ -1299,21 +1309,42 @@ static bool kick_pool(struct worker_pool *pool) * its affinity scope. Repatriate. */ if (!pool->attrs->affn_strict && - !cpumask_test_cpu(p->wake_cpu, pool->attrs->__pod_cpumask)) { + !cpumask_test_cpu(READ_ONCE(p->wake_cpu), + pool->attrs->__pod_cpumask)) { struct work_struct *work = list_first_entry(&pool->worklist, struct work_struct, entry); int wake_cpu = cpumask_any_and_distribute(pool->attrs->__pod_cpumask, cpu_online_mask); if (wake_cpu < nr_cpu_ids) { - p->wake_cpu = wake_cpu; + WRITE_ONCE(p->wake_cpu, wake_cpu); get_work_pwq(work)->stats[PWQ_STAT_REPATRIATED]++; } } #endif - wake_up_process(p); + /* Track the last idle worker woken, used for stall diagnostics. */ + pool->last_woken_worker = worker; + + *wakep = p; return true; } +/** + * kick_pool - wake up an idle worker if necessary + * @pool: pool to kick + * + * @pool may have pending work items. Wake up worker if necessary. Returns + * whether a worker was woken up. + */ +static bool kick_pool(struct worker_pool *pool) +{ + struct task_struct *p; + bool kicked = kick_pool_pick(pool, &p); + + if (p) + wake_up_process(p); + return kicked; +} + #ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT /* @@ -1438,7 +1469,7 @@ void wq_worker_running(struct task_struct *task) * CPU intensive auto-detection cares about how long a work item hogged * CPU without sleeping. Reset the starting timestamp on wakeup. */ - worker->current_at = worker->task->se.sum_exec_runtime; + worker->current_at = READ_ONCE(worker->task->se.sum_exec_runtime); WRITE_ONCE(worker->sleeping, 0); } @@ -1505,7 +1536,11 @@ void wq_worker_tick(struct task_struct *task) if (!pwq) return; - pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC; + /* + * @pwq is shared across CPUs for unbound wqs and this advisory stat is + * bumped outside pool->lock, so the update is intentionally racy. + */ + data_race(pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC); if (!wq_cpu_intensive_thresh_us) return; @@ -1523,7 +1558,7 @@ void wq_worker_tick(struct task_struct *task) * We probably want to make this prettier in the future. */ if ((worker->flags & WORKER_NOT_RUNNING) || READ_ONCE(worker->sleeping) || - worker->task->se.sum_exec_runtime - worker->current_at < + READ_ONCE(worker->task->se.sum_exec_runtime) - worker->current_at < wq_cpu_intensive_thresh_us * NSEC_PER_USEC) return; @@ -1570,14 +1605,19 @@ work_func_t wq_worker_last_func(struct task_struct *task) return worker->last_func; } +/* True if @pool is a static per-cpu pool rather than an unbound one. */ +static bool is_percpu_pool(struct worker_pool *pool) +{ + return pool->cpu >= 0; +} + /** * wq_node_nr_active - Determine wq_node_nr_active to use * @wq: workqueue of interest * @node: NUMA node, can be %NUMA_NO_NODE * - * Determine wq_node_nr_active to use for @wq on @node. Returns: - * - * - %NULL for per-cpu workqueues as they don't need to use shared nr_active. + * Determine wq_node_nr_active to use for @wq on @node. @wq must be unbound. + * Returns: * * - node_nr_active[nr_node_ids] if @node is %NUMA_NO_NODE. * @@ -1586,8 +1626,7 @@ work_func_t wq_worker_last_func(struct task_struct *task) static struct wq_node_nr_active *wq_node_nr_active(struct workqueue_struct *wq, int node) { - if (!(wq->flags & WQ_UNBOUND)) - return NULL; + BUG_ON(!(wq->flags & WQ_UNBOUND)); if (node == NUMA_NO_NODE) node = nr_node_ids; @@ -1742,13 +1781,16 @@ static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill) { struct workqueue_struct *wq = pwq->wq; struct worker_pool *pool = pwq->pool; - struct wq_node_nr_active *nna = wq_node_nr_active(wq, pool->node); + struct wq_node_nr_active *nna; bool obtained = false; lockdep_assert_held(&pool->lock); - if (!nna) { - /* BH or per-cpu workqueue, pwq->nr_active is sufficient */ + /* + * A concurrency-managed per-cpu pool accounts nr_active per pwq, so + * pwq->nr_active against wq->max_active is sufficient. + */ + if (is_percpu_pool(pool)) { obtained = pwq->nr_active < READ_ONCE(wq->max_active); goto out; } @@ -1756,6 +1798,8 @@ static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill) if (unlikely(pwq->plugged)) return false; + nna = wq_node_nr_active(wq, pool->node); + /* * Unbound workqueue uses per-node shared nr_active $nna. If @pwq is * already waiting on $nna, pwq_dec_nr_active() will maintain the @@ -1973,7 +2017,7 @@ out_unlock: static void pwq_dec_nr_active(struct pool_workqueue *pwq) { struct worker_pool *pool = pwq->pool; - struct wq_node_nr_active *nna = wq_node_nr_active(pwq->wq, pool->node); + struct wq_node_nr_active *nna; lockdep_assert_held(&pool->lock); @@ -1984,14 +2028,16 @@ static void pwq_dec_nr_active(struct pool_workqueue *pwq) pwq->nr_active--; /* - * For a percpu workqueue, it's simple. Just need to kick the first + * A concurrency-managed per-cpu pool only needs to kick the first * inactive work item on @pwq itself. */ - if (!nna) { + if (is_percpu_pool(pool)) { pwq_activate_first_inactive(pwq, false); return; } + nna = wq_node_nr_active(pwq->wq, pool->node); + /* * If @pwq is for an unbound workqueue, it's more complicated because * multiple pwqs and pools may be sharing the nr_active count. When a @@ -2277,6 +2323,7 @@ static void __queue_work(int cpu, struct workqueue_struct *wq, { struct pool_workqueue *pwq; struct worker_pool *last_pool, *pool; + struct task_struct *wake_task = NULL; unsigned int work_flags; unsigned int req_cpu = cpu; @@ -2399,7 +2446,7 @@ retry: trace_workqueue_activate_work(work); insert_work(pwq, work, &pool->worklist, work_flags); - kick_pool(pool); + kick_pool_pick(pool, &wake_task); } else { work_flags |= WORK_STRUCT_INACTIVE; insert_work(pwq, work, &pwq->inactive_works, work_flags); @@ -2407,6 +2454,8 @@ retry: out: raw_spin_unlock(&pool->lock); + if (wake_task) + wake_up_process(wake_task); rcu_read_unlock(); } @@ -2716,7 +2765,7 @@ static struct worker *alloc_worker(int node) static cpumask_t *pool_allowed_cpus(struct worker_pool *pool) { - if (pool->cpu < 0 && pool->attrs->affn_strict) + if (!is_percpu_pool(pool) && pool->attrs->affn_strict) return pool->attrs->__pod_cpumask; else return pool->attrs->cpumask; @@ -2948,6 +2997,13 @@ static void set_worker_dying(struct worker *worker, struct list_head *list) pool->nr_workers--; pool->nr_idle--; + /* + * Clear last_woken_worker if it points to this worker, so that + * show_cpu_pool_busy_workers() cannot dereference a freed worker. + */ + if (pool->last_woken_worker == worker) + pool->last_woken_worker = NULL; + worker->flags |= WORKER_DIE; list_move(&worker->entry, list); @@ -3223,6 +3279,7 @@ __acquires(&pool->lock) { struct pool_workqueue *pwq = get_work_pwq(work); struct worker_pool *pool = worker->pool; + struct task_struct *wake_task = NULL; unsigned long work_data; int lockdep_start_depth, rcu_start_depth; bool bh_draining = pool->flags & POOL_BH_DRAINING; @@ -3249,7 +3306,7 @@ __acquires(&pool->lock) worker->current_func = work->func; worker->current_pwq = pwq; if (worker->task) - worker->current_at = worker->task->se.sum_exec_runtime; + worker->current_at = READ_ONCE(worker->task->se.sum_exec_runtime); worker->current_start = jiffies; work_data = *work_data_bits(work); worker->current_color = get_work_color(work_data); @@ -3276,8 +3333,11 @@ __acquires(&pool->lock) * since nr_running would always be >= 1 at this point. This is used to * chain execution of the pending work items for WORKER_NOT_RUNNING * workers such as the UNBOUND and CPU_INTENSIVE ones. + * + * Select the worker under pool->lock; the wakeup is deferred until + * after the lock is dropped, guarded by the rcu_read_lock() below. */ - kick_pool(pool); + kick_pool_pick(pool, &wake_task); /* * Record the last pool and clear PENDING which should be the last @@ -3288,7 +3348,12 @@ __acquires(&pool->lock) set_work_pool_and_clear_pending(work, pool->id, pool_offq_flags(pool)); pwq->stats[PWQ_STAT_STARTED]++; + + rcu_read_lock(); raw_spin_unlock_irq(&pool->lock); + if (wake_task) + wake_up_process(wake_task); + rcu_read_unlock(); rcu_start_depth = rcu_preempt_depth(); lockdep_start_depth = lockdep_depth(current); @@ -5033,7 +5098,7 @@ static void rcu_free_wq(struct rcu_head *rcu) wq_free_lockdep(wq); free_percpu(wq->cpu_pwq); - free_workqueue_attrs(wq->unbound_attrs); + free_workqueue_attrs(wq->attrs); kfree(wq); } @@ -5068,7 +5133,7 @@ static void put_unbound_pool(struct worker_pool *pool) return; /* sanity checks */ - if (WARN_ON(!(pool->cpu < 0)) || + if (WARN_ON(is_percpu_pool(pool)) || WARN_ON(!list_empty(&pool->worklist))) return; @@ -5220,7 +5285,7 @@ static void pwq_release_workfn(struct kthread_work *work) mutex_unlock(&wq->mutex); } - if (wq->flags & WQ_UNBOUND) { + if (!is_percpu_pool(pool)) { mutex_lock(&wq_pool_mutex); put_unbound_pool(pool); mutex_unlock(&wq_pool_mutex); @@ -5297,8 +5362,22 @@ static void link_pwq(struct pool_workqueue *pwq) list_add_tail_rcu(&pwq->pwqs_node, &wq->pwqs); } +/* Return the static per-cpu worker_pool that backs @wq on @cpu. */ +static struct worker_pool *get_percpu_pool(struct workqueue_struct *wq, int cpu) +{ + struct worker_pool __percpu *pools; + bool highpri = wq->flags & WQ_HIGHPRI; + + if (wq->flags & WQ_BH) + pools = bh_worker_pools; + else + pools = cpu_worker_pools; + + return &per_cpu_ptr(pools, cpu)[highpri]; +} + /* obtain a pool matching @attr and create a pwq associating the pool and @wq */ -static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq, +static struct pool_workqueue *alloc_pwq(struct workqueue_struct *wq, const struct workqueue_attrs *attrs) { struct worker_pool *pool; @@ -5421,7 +5500,7 @@ apply_wqattrs_prepare(struct workqueue_struct *wq, copy_workqueue_attrs(new_attrs, attrs); wqattrs_actualize_cpumask(new_attrs, unbound_cpumask); cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask); - ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs); + ctx->dfl_pwq = alloc_pwq(wq, new_attrs); if (!ctx->dfl_pwq) goto out_free; @@ -5431,7 +5510,7 @@ apply_wqattrs_prepare(struct workqueue_struct *wq, ctx->pwq_tbl[cpu] = ctx->dfl_pwq; } else { wq_calc_pod_cpumask(new_attrs, cpu); - ctx->pwq_tbl[cpu] = alloc_unbound_pwq(wq, new_attrs); + ctx->pwq_tbl[cpu] = alloc_pwq(wq, new_attrs); if (!ctx->pwq_tbl[cpu]) goto out_free; } @@ -5469,7 +5548,7 @@ static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx) /* all pwqs have been created successfully, let's install'em */ mutex_lock(&ctx->wq->mutex); - copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs); + copy_workqueue_attrs(ctx->wq->attrs, ctx->attrs); /* save the previous pwqs and install the new ones */ for_each_possible_cpu(cpu) @@ -5477,8 +5556,9 @@ static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx) ctx->pwq_tbl[cpu]); ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq); - /* update node_nr_active->max */ - wq_update_node_max_active(ctx->wq, -1); + /* update node_nr_active->max, which only unbound workqueues have */ + if (ctx->wq->flags & WQ_UNBOUND) + wq_update_node_max_active(ctx->wq, -1); mutex_unlock(&ctx->wq->mutex); } @@ -5556,7 +5636,7 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) lockdep_assert_held(&wq_pool_mutex); - if (!(wq->flags & WQ_UNBOUND) || wq->unbound_attrs->ordered) + if (!(wq->flags & WQ_UNBOUND) || wq->attrs->ordered) return; /* @@ -5566,7 +5646,7 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) */ target_attrs = unbound_wq_update_pwq_attrs_buf; - copy_workqueue_attrs(target_attrs, wq->unbound_attrs); + copy_workqueue_attrs(target_attrs, wq->attrs); wqattrs_actualize_cpumask(target_attrs, wq_unbound_cpumask); /* nothing to do if the target cpumask matches the current pwq */ @@ -5575,7 +5655,7 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) return; /* create a new pwq */ - pwq = alloc_unbound_pwq(wq, target_attrs); + pwq = alloc_pwq(wq, target_attrs); if (!pwq) { pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n", wq->name); @@ -5599,47 +5679,44 @@ out_unlock: put_pwq_unlocked(old_pwq); } -static int alloc_and_link_pwqs(struct workqueue_struct *wq) +static int alloc_and_link_percpu_pwqs(struct workqueue_struct *wq) { - bool highpri = wq->flags & WQ_HIGHPRI; - int cpu, ret; + struct pool_workqueue *pwq; + int cpu; - lockdep_assert_held(&wq_pool_mutex); + for_each_possible_cpu(cpu) { + struct worker_pool *pool = get_percpu_pool(wq, cpu); - wq->cpu_pwq = alloc_percpu(struct pool_workqueue *); - if (!wq->cpu_pwq) - goto enomem; + pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node); + if (!pwq) + return -ENOMEM; - if (!(wq->flags & WQ_UNBOUND)) { - struct worker_pool __percpu *pools; + init_pwq(pwq, wq, pool); - if (wq->flags & WQ_BH) - pools = bh_worker_pools; - else - pools = cpu_worker_pools; + mutex_lock(&wq->mutex); + link_pwq(pwq); + mutex_unlock(&wq->mutex); - for_each_possible_cpu(cpu) { - struct pool_workqueue **pwq_p; - struct worker_pool *pool; + rcu_assign_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu), pwq); + } - pool = &(per_cpu_ptr(pools, cpu)[highpri]); - pwq_p = per_cpu_ptr(wq->cpu_pwq, cpu); + return 0; +} - *pwq_p = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, - pool->node); - if (!*pwq_p) - goto enomem; +static int alloc_and_link_pwqs(struct workqueue_struct *wq) +{ + bool highpri = wq->flags & WQ_HIGHPRI; + int cpu, ret; - init_pwq(*pwq_p, wq, pool); + lockdep_assert_held(&wq_pool_mutex); - mutex_lock(&wq->mutex); - link_pwq(*pwq_p); - mutex_unlock(&wq->mutex); - } - return 0; - } + wq->cpu_pwq = alloc_percpu(struct pool_workqueue __rcu *); + if (!wq->cpu_pwq) + goto enomem; - if (wq->flags & __WQ_ORDERED) { + if (!(wq->flags & WQ_UNBOUND)) { + ret = alloc_and_link_percpu_pwqs(wq); + } else if (wq->flags & __WQ_ORDERED) { struct pool_workqueue *dfl_pwq; ret = apply_workqueue_attrs_locked(wq, ordered_wq_attrs[highpri]); @@ -5659,8 +5736,11 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq) enomem: if (wq->cpu_pwq) { for_each_possible_cpu(cpu) { - struct pool_workqueue *pwq = *per_cpu_ptr(wq->cpu_pwq, cpu); + struct pool_workqueue __rcu **slot; + struct pool_workqueue *pwq; + slot = per_cpu_ptr(wq->cpu_pwq, cpu); + pwq = rcu_access_pointer(*slot); if (pwq) { /* * Unlink pwq from wq->pwqs since link_pwq() @@ -5828,11 +5908,9 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, if (!wq) return NULL; - if (flags & WQ_UNBOUND) { - wq->unbound_attrs = alloc_workqueue_attrs_noprof(); - if (!wq->unbound_attrs) - goto err_free_wq; - } + wq->attrs = alloc_workqueue_attrs_noprof(); + if (!wq->attrs) + goto err_free_wq; name_len = vsnprintf(wq->name, sizeof(wq->name), fmt, args); @@ -5925,7 +6003,7 @@ err_unlock_free_node_nr_active: free_node_nr_active(wq->node_nr_active); } err_free_wq: - free_workqueue_attrs(wq->unbound_attrs); + free_workqueue_attrs(wq->attrs); kfree(wq); return NULL; err_unlock_destroy: @@ -6244,7 +6322,7 @@ bool workqueue_congested(int cpu, struct workqueue_struct *wq) if (cpu == WORK_CPU_UNBOUND) cpu = smp_processor_id(); - pwq = *per_cpu_ptr(wq->cpu_pwq, cpu); + pwq = rcu_dereference_sched(*per_cpu_ptr(wq->cpu_pwq, cpu)); ret = !list_empty(&pwq->inactive_works); preempt_enable(); @@ -6869,9 +6947,9 @@ int workqueue_online_cpu(unsigned int cpu) /* update pod affinity of unbound workqueues */ list_for_each_entry(wq, &workqueues, list) { - struct workqueue_attrs *attrs = wq->unbound_attrs; + struct workqueue_attrs *attrs = wq->attrs; - if (attrs) { + if (wq->flags & WQ_UNBOUND) { const struct wq_pod_type *pt = wqattrs_pod_type(attrs); int tcpu; @@ -6904,9 +6982,9 @@ int workqueue_offline_cpu(unsigned int cpu) cpumask_clear_cpu(cpu, wq_online_cpumask); list_for_each_entry(wq, &workqueues, list) { - struct workqueue_attrs *attrs = wq->unbound_attrs; + struct workqueue_attrs *attrs = wq->attrs; - if (attrs) { + if (wq->flags & WQ_UNBOUND) { const struct wq_pod_type *pt = wqattrs_pod_type(attrs); int tcpu; @@ -7084,7 +7162,7 @@ static int workqueue_apply_unbound_cpumask(const cpumask_var_t unbound_cpumask) if (!(wq->flags & WQ_UNBOUND) || (wq->flags & __WQ_DESTROYING)) continue; - ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs, unbound_cpumask); + ctx = apply_wqattrs_prepare(wq, wq->attrs, unbound_cpumask); if (IS_ERR(ctx)) { ret = PTR_ERR(ctx); break; @@ -7302,7 +7380,7 @@ static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr, int written; mutex_lock(&wq->mutex); - written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice); + written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->attrs->nice); mutex_unlock(&wq->mutex); return written; @@ -7319,7 +7397,7 @@ static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq) if (!attrs) return NULL; - copy_workqueue_attrs(attrs, wq->unbound_attrs); + copy_workqueue_attrs(attrs, wq->attrs); return attrs; } @@ -7356,7 +7434,7 @@ static ssize_t wq_cpumask_show(struct device *dev, mutex_lock(&wq->mutex); written = scnprintf(buf, PAGE_SIZE, "%*pb\n", - cpumask_pr_args(wq->unbound_attrs->cpumask)); + cpumask_pr_args(wq->attrs->cpumask)); mutex_unlock(&wq->mutex); return written; } @@ -7392,13 +7470,13 @@ static ssize_t wq_affn_scope_show(struct device *dev, int written; mutex_lock(&wq->mutex); - if (wq->unbound_attrs->affn_scope == WQ_AFFN_DFL) + if (wq->attrs->affn_scope == WQ_AFFN_DFL) written = scnprintf(buf, PAGE_SIZE, "%s (%s)\n", wq_affn_names[WQ_AFFN_DFL], wq_affn_names[wq_affn_dfl]); else written = scnprintf(buf, PAGE_SIZE, "%s\n", - wq_affn_names[wq->unbound_attrs->affn_scope]); + wq_affn_names[wq->attrs->affn_scope]); mutex_unlock(&wq->mutex); return written; @@ -7433,7 +7511,7 @@ static ssize_t wq_affinity_strict_show(struct device *dev, struct workqueue_struct *wq = dev_to_wq(dev); return scnprintf(buf, PAGE_SIZE, "%d\n", - wq->unbound_attrs->affn_strict); + wq->attrs->affn_strict); } static ssize_t wq_affinity_strict_store(struct device *dev, @@ -7606,7 +7684,7 @@ int workqueue_sysfs_register(struct workqueue_struct *wq) dev_set_name(&wq_dev->dev, "%s", wq->name); /* - * unbound_attrs are created separately. Suppress uevent until + * attrs are created separately. Suppress uevent until * everything is ready. */ dev_set_uevent_suppress(&wq_dev->dev, true); @@ -7689,20 +7767,58 @@ module_param_named(panic_on_stall_time, wq_panic_on_stall_time, uint, 0644); MODULE_PARM_DESC(panic_on_stall_time, "Panic if stall exceeds this many seconds (0=disabled)"); /* - * Show workers that might prevent the processing of pending work items. - * A busy worker that is not running on the CPU (e.g. sleeping in - * wait_event_idle() with PF_WQ_WORKER cleared) can stall the pool just as - * effectively as a CPU-bound one, so dump every in-flight worker. + * Report that a pool has no worker in running state, which is a sign that the + * pool may be stuck. Print pool info. Must be called with pool->lock held and + * inside a printk_deferred_enter/exit region. + */ +static void show_pool_no_running_worker(struct worker_pool *pool) +{ + lockdep_assert_held(&pool->lock); + + printk_deferred_enter(); + pr_info("pool %d: no worker in running state, cpu=%d is %s (nr_workers=%d nr_idle=%d)\n", + pool->id, pool->cpu, + idle_cpu(pool->cpu) ? "idle" : "busy", + pool->nr_workers, pool->nr_idle); + pr_info("The pool might have trouble waking an idle worker.\n"); + /* + * last_woken_worker and its task are valid here: set_worker_dying() + * clears it under pool->lock before setting WORKER_DIE, so if + * last_woken_worker is non-NULL the kthread has not yet exited and + * worker->task is still alive. + */ + if (pool->last_woken_worker) { + pr_info("Backtrace of last woken worker:\n"); + sched_show_task(pool->last_woken_worker->task); + } else { + pr_info("Last woken worker empty\n"); + } + printk_deferred_exit(); +} + +/* + * Show running workers that might prevent the processing of pending work items. + * If no running worker is found, the pool may be stuck waiting for an idle + * worker to be woken, so report the pool state and the last woken worker. */ static void show_cpu_pool_busy_workers(struct worker_pool *pool) { + bool found_running = false; struct worker *worker; unsigned long irq_flags; - int bkt; + int cpu, bkt; raw_spin_lock_irqsave(&pool->lock, irq_flags); + /* Snapshot cpu inside the lock to safely use it after unlock. */ + cpu = pool->cpu; + hash_for_each(pool->busy_hash, bkt, worker, hentry) { + /* Skip workers that are not actively running on the CPU. */ + if (!task_is_running(worker->task)) + continue; + + found_running = true; /* * Defer printing to avoid deadlocks in console * drivers that queue work while holding locks @@ -7716,7 +7832,24 @@ static void show_cpu_pool_busy_workers(struct worker_pool *pool) printk_deferred_exit(); } + /* + * If no running worker was found, the pool is likely stuck. Print pool + * state and the backtrace of the last woken worker, which is the prime + * suspect for the stall. + */ + if (!found_running) + show_pool_no_running_worker(pool); + raw_spin_unlock_irqrestore(&pool->lock, irq_flags); + + /* + * Trigger a backtrace on the stalled CPU to capture what it is + * currently executing. Skip an offline CPU, whose NMI is never acked + * and would make the backtrace busy-wait until it times out. Done + * after releasing the lock to avoid issues with NMI delivery. + */ + if (!found_running && cpu_online(cpu)) + trigger_single_cpu_backtrace(cpu); } static void show_cpu_pools_busy_workers(void) @@ -7832,7 +7965,7 @@ static void wq_watchdog_timer_fn(struct timer_list *unused) lockup_detected = true; stall_time = jiffies_to_msecs(now - pool_ts) / 1000; max_stall_time = max(max_stall_time, stall_time); - if (pool->cpu >= 0 && !(pool->flags & POOL_BH)) { + if (is_percpu_pool(pool) && !(pool->flags & POOL_BH)) { pool->cpu_stall = true; cpu_pool_stall = true; } diff --git a/tools/workqueue/wq_dump.py b/tools/workqueue/wq_dump.py index ce4161f52f2f..31afc24ef17b 100644 --- a/tools/workqueue/wq_dump.py +++ b/tools/workqueue/wq_dump.py @@ -46,6 +46,11 @@ each workqueue: import sys +import argparse +parser = argparse.ArgumentParser(description=desc, + formatter_class=argparse.RawTextHelpFormatter) +args = parser.parse_args() + import drgn from drgn.helpers.linux.list import list_for_each_entry,list_empty from drgn.helpers.linux.percpu import per_cpu_ptr @@ -53,11 +58,6 @@ from drgn.helpers.linux.cpumask import for_each_cpu,for_each_possible_cpu from drgn.helpers.linux.nodemask import for_each_node from drgn.helpers.linux.idr import idr_for_each -import argparse -parser = argparse.ArgumentParser(description=desc, - formatter_class=argparse.RawTextHelpFormatter) -args = parser.parse_args() - def err(s): print(s, file=sys.stderr, flush=True) sys.exit(1) @@ -85,7 +85,7 @@ def wq_type_str(wq): if wq.flags & WQ_ORDERED: return f'{"ordered":{wq_type_len}}' else: - if wq.unbound_attrs.affn_strict: + if wq.attrs.affn_strict: return f'{"unbound,S":{wq_type_len}}' else: return f'{"unbound":{wq_type_len}}' @@ -205,8 +205,8 @@ for wq in list_for_each_entry('struct workqueue_struct', workqueues.address_of_( continue print(f'{wq.name.string_().decode():{WQ_NAME_LEN}}', end='') - if wq.unbound_attrs.value_() != 0: - print(f' {cpumask_str(wq.unbound_attrs.cpumask):{ucpus_len}}', end='') + if wq.flags & WQ_UNBOUND: + print(f' {cpumask_str(wq.attrs.cpumask):{ucpus_len}}', end='') else: print(f' {"":{ucpus_len}}', end='') diff --git a/tools/workqueue/wq_monitor.py b/tools/workqueue/wq_monitor.py index 9e964c5be40c..7f47fa398e3c 100644 --- a/tools/workqueue/wq_monitor.py +++ b/tools/workqueue/wq_monitor.py @@ -37,9 +37,6 @@ import re import time import json -import drgn -from drgn.helpers.linux.list import list_for_each_entry - import argparse parser = argparse.ArgumentParser(description=desc, formatter_class=argparse.RawTextHelpFormatter) @@ -51,6 +48,9 @@ parser.add_argument('-j', '--json', action='store_true', help='Output in json') args = parser.parse_args() +import drgn +from drgn.helpers.linux.list import list_for_each_entry + workqueues = prog['workqueues'] WQ_UNBOUND = prog['WQ_UNBOUND'] |