diff options
Diffstat (limited to 'fs/resctrl')
| -rw-r--r-- | fs/resctrl/ctrlmondata.c | 139 | ||||
| -rw-r--r-- | fs/resctrl/internal.h | 5 | ||||
| -rw-r--r-- | fs/resctrl/monitor.c | 276 | ||||
| -rw-r--r-- | fs/resctrl/pseudo_lock.c | 4 | ||||
| -rw-r--r-- | fs/resctrl/rdtgroup.c | 910 |
5 files changed, 854 insertions, 480 deletions
diff --git a/fs/resctrl/ctrlmondata.c b/fs/resctrl/ctrlmondata.c index cc4237c57cbe..18ec9f564b5a 100644 --- a/fs/resctrl/ctrlmondata.c +++ b/fs/resctrl/ctrlmondata.c @@ -261,7 +261,7 @@ next: return -EINVAL; } dom = strim(dom); - list_for_each_entry(d, &r->ctrl_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { if (d->hdr.id == dom_id) { data.buf = dom; data.closid = rdtgrp->closid; @@ -312,17 +312,20 @@ ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of, char *tok, *resname; int ret = 0; - /* Valid input requires a trailing newline */ - if (nbytes == 0 || buf[nbytes - 1] != '\n') - return -EINVAL; - buf[nbytes - 1] = '\0'; - rdtgrp = rdtgroup_kn_lock_live(of->kn); if (!rdtgrp) { rdtgroup_kn_unlock(of->kn); return -ENOENT; } - rdt_last_cmd_clear(); + + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') { + rdt_last_cmd_puts("schemata: Invalid input\n"); + ret = -EINVAL; + goto out_unlock; + } + + buf[nbytes - 1] = '\0'; /* * No changes to pseudo-locked region allowed. It has to be removed @@ -331,7 +334,7 @@ ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of, if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { ret = -EINVAL; rdt_last_cmd_puts("Resource group is pseudo-locked\n"); - goto out; + goto out_unlock; } rdt_staged_configs_clear(); @@ -341,16 +344,16 @@ ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of, if (!tok) { rdt_last_cmd_puts("Missing ':'\n"); ret = -EINVAL; - goto out; + goto out_clear_staged; } if (tok[0] == '\0') { rdt_last_cmd_printf("Missing '%s' value\n", resname); ret = -EINVAL; - goto out; + goto out_clear_staged; } ret = rdtgroup_parse_resource(resname, tok, rdtgrp); if (ret) - goto out; + goto out_clear_staged; } list_for_each_entry(s, &resctrl_schema_all, list) { @@ -365,7 +368,7 @@ ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of, ret = resctrl_arch_update_domains(r, rdtgrp->closid); if (ret) - goto out; + goto out_clear_staged; } if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { @@ -378,8 +381,9 @@ ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of, ret = rdtgroup_pseudo_lock_create(rdtgrp); } -out: +out_clear_staged: rdt_staged_configs_clear(); +out_unlock: rdtgroup_kn_unlock(of->kn); return ret ?: nbytes; } @@ -397,7 +401,7 @@ static void show_doms(struct seq_file *s, struct resctrl_schema *schema, if (resource_name) seq_printf(s, "%*s:", max_name_width, resource_name); - list_for_each_entry(dom, &r->ctrl_domains, hdr.list) { + list_for_each_entry_rcu(dom, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { if (sep) seq_puts(s, ";"); @@ -429,7 +433,6 @@ int rdtgroup_schemata_show(struct kernfs_open_file *of, } } else if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { if (!rdtgrp->plr->d) { - rdt_last_cmd_clear(); rdt_last_cmd_puts("Cache domain offline\n"); ret = -ENODEV; } else { @@ -465,17 +468,20 @@ ssize_t rdtgroup_mba_mbps_event_write(struct kernfs_open_file *of, struct rdtgroup *rdtgrp; int ret = 0; - /* Valid input requires a trailing newline */ - if (nbytes == 0 || buf[nbytes - 1] != '\n') - return -EINVAL; - buf[nbytes - 1] = '\0'; - rdtgrp = rdtgroup_kn_lock_live(of->kn); if (!rdtgrp) { rdtgroup_kn_unlock(of->kn); return -ENOENT; } - rdt_last_cmd_clear(); + + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') { + rdt_last_cmd_puts("mba_MBps_event: Invalid input\n"); + ret = -EINVAL; + goto out_unlock; + } + + buf[nbytes - 1] = '\0'; if (!strcmp(buf, "mbm_local_bytes")) { if (resctrl_is_mon_event_enabled(QOS_L3_MBM_LOCAL_EVENT_ID)) @@ -494,6 +500,7 @@ ssize_t rdtgroup_mba_mbps_event_write(struct kernfs_open_file *of, if (ret) rdt_last_cmd_printf("Unsupported event id '%s'\n", buf); +out_unlock: rdtgroup_kn_unlock(of->kn); return ret ?: nbytes; @@ -535,6 +542,8 @@ struct rdt_domain_hdr *resctrl_find_domain(struct list_head *h, int id, struct rdt_domain_hdr *d; struct list_head *l; + lockdep_assert_cpus_held(); + list_for_each(l, h) { d = list_entry(l, struct rdt_domain_hdr, list); /* When id is found, return its domain. */ @@ -717,7 +726,7 @@ int rdtgroup_mondata_show(struct seq_file *m, void *arg) * struct mon_data. Search all domains in the resource for * one that matches this cache id. */ - list_for_each_entry(d, &r->mon_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { if (d->ci_id == domid) { cpu = cpumask_any(&d->hdr.cpu_mask); ci = get_cpu_cacheinfo_level(cpu, RESCTRL_L3_CACHE); @@ -769,10 +778,12 @@ out: int resctrl_io_alloc_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); - struct rdt_resource *r = s->res; + struct rdt_resource *r; - mutex_lock(&rdtgroup_mutex); + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; if (r->cache.io_alloc_capable) { if (resctrl_arch_get_io_alloc_enabled(r)) seq_puts(seq, "enabled\n"); @@ -782,7 +793,7 @@ int resctrl_io_alloc_show(struct kernfs_open_file *of, struct seq_file *seq, voi seq_puts(seq, "not supported\n"); } - mutex_unlock(&rdtgroup_mutex); + info_kn_unlock(of->kn); return 0; } @@ -817,7 +828,7 @@ static int resctrl_io_alloc_init_cbm(struct resctrl_schema *s, u32 closid) /* Keep CDP_CODE and CDP_DATA of io_alloc CLOSID's CBM in sync. */ if (resctrl_arch_get_cdp_enabled(r->rid)) { peer_type = resctrl_peer_type(s->conf_type); - list_for_each_entry(d, &s->res->ctrl_domains, hdr.list) + list_for_each_entry_rcu(d, &s->res->ctrl_domains, hdr.list, lockdep_is_cpus_held()) memcpy(&d->staged_config[peer_type], &d->staged_config[s->conf_type], sizeof(d->staged_config[0])); @@ -847,21 +858,24 @@ ssize_t resctrl_io_alloc_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); - struct rdt_resource *r = s->res; + struct rdt_resource *r; char const *grp_name; u32 io_alloc_closid; bool enable; int ret; - ret = kstrtobool(buf, &enable); - if (ret) - return ret; - - cpus_read_lock(); - mutex_lock(&rdtgroup_mutex); + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; rdt_last_cmd_clear(); + ret = kstrtobool(buf, &enable); + if (ret) { + rdt_last_cmd_puts("io_alloc: Invalid input\n"); + goto out_unlock; + } + if (!r->cache.io_alloc_capable) { rdt_last_cmd_printf("io_alloc is not supported on %s\n", s->name); ret = -ENODEV; @@ -907,8 +921,7 @@ ssize_t resctrl_io_alloc_write(struct kernfs_open_file *of, char *buf, } out_unlock: - mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); + info_kn_unlock(of->kn); return ret ?: nbytes; } @@ -916,14 +929,15 @@ out_unlock: int resctrl_io_alloc_cbm_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); - struct rdt_resource *r = s->res; + struct rdt_resource *r; int ret = 0; - cpus_read_lock(); - mutex_lock(&rdtgroup_mutex); + if (!info_kn_lock(of->kn)) + return -ENOENT; rdt_last_cmd_clear(); + r = s->res; if (!r->cache.io_alloc_capable) { rdt_last_cmd_printf("io_alloc is not supported on %s\n", s->name); ret = -ENODEV; @@ -945,8 +959,7 @@ int resctrl_io_alloc_cbm_show(struct kernfs_open_file *of, struct seq_file *seq, show_doms(seq, s, NULL, resctrl_io_alloc_closid(r)); out_unlock: - mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); + info_kn_unlock(of->kn); return ret; } @@ -954,25 +967,34 @@ static int resctrl_io_alloc_parse_line(char *line, struct rdt_resource *r, struct resctrl_schema *s, u32 closid) { enum resctrl_conf_type peer_type; + unsigned long dom_id = ULONG_MAX; struct rdt_parse_data data; struct rdt_ctrl_domain *d; + bool update_all = false; char *dom = NULL, *id; - unsigned long dom_id; next: if (!line || line[0] == '\0') return 0; + if (update_all) { + rdt_last_cmd_puts("Configurations after global '*'\n"); + return -EINVAL; + } + dom = strsep(&line, ";"); id = strsep(&dom, "="); - if (!dom || kstrtoul(id, 10, &dom_id)) { + + if (dom && !strcmp(id, "*")) { + update_all = true; + } else if (!dom || kstrtoul(id, 10, &dom_id)) { rdt_last_cmd_puts("Missing '=' or non-numeric domain\n"); return -EINVAL; } dom = strim(dom); - list_for_each_entry(d, &r->ctrl_domains, hdr.list) { - if (d->hdr.id == dom_id) { + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { + if (update_all || d->hdr.id == dom_id) { data.buf = dom; data.mode = RDT_MODE_SHAREABLE; data.closid = closid; @@ -988,10 +1010,15 @@ next: &d->staged_config[s->conf_type], sizeof(d->staged_config[0])); } - goto next; + if (!update_all) + goto next; } } + if (update_all) + goto next; + + rdt_last_cmd_printf("Invalid domain %lu\n", dom_id); return -EINVAL; } @@ -999,20 +1026,25 @@ ssize_t resctrl_io_alloc_cbm_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); - struct rdt_resource *r = s->res; + struct rdt_resource *r; u32 io_alloc_closid; int ret = 0; + if (!info_kn_lock(of->kn)) + return -ENOENT; + rdt_last_cmd_clear(); + + r = s->res; + /* Valid input requires a trailing newline */ - if (nbytes == 0 || buf[nbytes - 1] != '\n') - return -EINVAL; + if (nbytes == 0 || buf[nbytes - 1] != '\n') { + rdt_last_cmd_puts("io_alloc_cbm: Invalid input\n"); + ret = -EINVAL; + goto out_unlock; + } buf[nbytes - 1] = '\0'; - cpus_read_lock(); - mutex_lock(&rdtgroup_mutex); - rdt_last_cmd_clear(); - if (!r->cache.io_alloc_capable) { rdt_last_cmd_printf("io_alloc is not supported on %s\n", s->name); ret = -ENODEV; @@ -1037,8 +1069,7 @@ ssize_t resctrl_io_alloc_cbm_write(struct kernfs_open_file *of, char *buf, out_clear_configs: rdt_staged_configs_clear(); out_unlock: - mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); + info_kn_unlock(of->kn); return ret ?: nbytes; } diff --git a/fs/resctrl/internal.h b/fs/resctrl/internal.h index 1a9b29119f88..e62a277dee85 100644 --- a/fs/resctrl/internal.h +++ b/fs/resctrl/internal.h @@ -335,8 +335,9 @@ __printf(1, 2) void rdt_last_cmd_printf(const char *fmt, ...); struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn); - void rdtgroup_kn_unlock(struct kernfs_node *kn); +bool info_kn_lock(struct kernfs_node *kn); +void info_kn_unlock(struct kernfs_node *kn); int rdtgroup_kn_mode_restrict(struct rdtgroup *r, const char *name); @@ -408,6 +409,8 @@ void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free); void resctrl_file_fflags_init(const char *config, unsigned long fflags); +void resctrl_file_mode_init(const char *config, umode_t mode); + void rdt_staged_configs_clear(void); bool closid_allocated(unsigned int closid); diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c index 49f3f6b846b2..73413cb128ea 100644 --- a/fs/resctrl/monitor.c +++ b/fs/resctrl/monitor.c @@ -135,17 +135,17 @@ void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free) struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); u32 idx_limit = resctrl_arch_system_num_rmid_idx(); struct rmid_entry *entry; + bool rmid_dirty = true; u32 idx, cur_idx = 1; void *arch_mon_ctx; void *arch_priv; - bool rmid_dirty; u64 val = 0; arch_priv = mon_event_all[QOS_L3_OCCUP_EVENT_ID].arch_priv; arch_mon_ctx = resctrl_arch_mon_ctx_alloc(r, QOS_L3_OCCUP_EVENT_ID); if (IS_ERR(arch_mon_ctx)) { - pr_warn_ratelimited("Failed to allocate monitor context: %ld", - PTR_ERR(arch_mon_ctx)); + pr_warn_ratelimited("Failed to allocate monitor context: %pe", + arch_mon_ctx); return; } @@ -161,22 +161,27 @@ void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free) break; entry = __rmid_entry(idx); - if (resctrl_arch_rmid_read(r, &d->hdr, entry->closid, entry->rmid, - QOS_L3_OCCUP_EVENT_ID, arch_priv, &val, - arch_mon_ctx)) { - rmid_dirty = true; - } else { - rmid_dirty = (val >= resctrl_rmid_realloc_threshold); - - /* - * x86's CLOSID and RMID are independent numbers, so the entry's - * CLOSID is an empty CLOSID (X86_RESCTRL_EMPTY_CLOSID). On Arm the - * RMID (PMG) extends the CLOSID (PARTID) space with bits that aren't - * used to select the configuration. It is thus necessary to track both - * CLOSID and RMID because there may be dependencies between them - * on some architectures. - */ - trace_mon_llc_occupancy_limbo(entry->closid, entry->rmid, d->hdr.id, val); + if (!force_free) { + if (resctrl_arch_rmid_read(r, &d->hdr, entry->closid, + entry->rmid, QOS_L3_OCCUP_EVENT_ID, + arch_priv, &val, arch_mon_ctx)) { + rmid_dirty = true; + } else { + rmid_dirty = (val >= resctrl_rmid_realloc_threshold); + + /* + * x86's CLOSID and RMID are independent numbers, + * so the entry's CLOSID is an empty CLOSID + * (X86_RESCTRL_EMPTY_CLOSID). On Arm the RMID + * (PMG) extends the CLOSID (PARTID) space with + * bits that aren't used to select the configuration. + * It is thus necessary to track both CLOSID and + * RMID because there may be dependencies between + * them on some architectures. + */ + trace_mon_llc_occupancy_limbo(entry->closid, entry->rmid, + d->hdr.id, val); + } } if (force_free || !rmid_dirty) { @@ -234,6 +239,8 @@ static struct rmid_entry *resctrl_find_free_rmid(u32 closid) * * When the CLOSID and RMID are independent numbers, the first free CLOSID will * be returned. + * + * Return: Free CLOSID on success, < 0 on failure. */ int resctrl_find_cleanest_closid(void) { @@ -302,7 +309,7 @@ static void add_rmid_to_limbo(struct rmid_entry *entry) idx = resctrl_arch_rmid_idx_encode(entry->closid, entry->rmid); entry->busy = 0; - list_for_each_entry(d, &r->mon_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { /* * For the first limbo RMID in the domain, * setup up the limbo worker. @@ -451,8 +458,10 @@ static int __l3_mon_event_count(struct rdtgroup *rdtgrp, struct rmid_read *rr) } /* Reading a single domain, must be on a CPU in that domain. */ - if (!cpumask_test_cpu(cpu, &d->hdr.cpu_mask)) + if (!cpumask_test_cpu(cpu, &d->hdr.cpu_mask)) { + rr->err = -EIO; return -EINVAL; + } if (rr->is_mbm_cntr) rr->err = resctrl_arch_cntr_read(rr->r, d, closid, rmid, cntr_id, rr->evt->evtid, &tval); @@ -489,8 +498,10 @@ static int __l3_mon_event_count_sum(struct rdtgroup *rdtgrp, struct rmid_read *r } /* Summing domains that share a cache, must be on a CPU for that cache. */ - if (!cpumask_test_cpu(cpu, &rr->ci->shared_cpu_map)) + if (!cpumask_test_cpu(cpu, &rr->ci->shared_cpu_map)) { + rr->err = -EIO; return -EINVAL; + } /* * Legacy files must report the sum of an event across all @@ -500,6 +511,11 @@ static int __l3_mon_event_count_sum(struct rdtgroup *rdtgrp, struct rmid_read *r * all domains fail for any reason. */ ret = -EINVAL; + /* + * RCU list being traversed with CPU hotplug lock held. lockdep + * unable to help prove this here since this work is scheduled via + * smp_call*(). Not called from MBM overflow handler. + */ list_for_each_entry(d, &rr->r->mon_domains, hdr.list) { if (d->ci_id != rr->ci->id) continue; @@ -621,14 +637,22 @@ void mon_event_count(void *info) rr->err = 0; } -static struct rdt_ctrl_domain *get_ctrl_domain_from_cpu(int cpu, - struct rdt_resource *r) +/* + * Find the software controller's ctrl domain that contains @cpu on resource @r. + * + * Only called from the mbm_over worker via update_mba_bw() where the returned + * domain is kept alive by cancel_delayed_work_sync() in + * resctrl_offline_ctrl_domain(). This drains this worker and then waits on + * rdtgroup_mutex held here before the architecture can free the ctrl domain. + * + * Context: Call from RCU read-side critical section. + */ +static struct rdt_ctrl_domain *get_sc_ctrl_domain_from_cpu(int cpu, + struct rdt_resource *r) { struct rdt_ctrl_domain *d; - lockdep_assert_cpus_held(); - - list_for_each_entry(d, &r->ctrl_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list) { /* Find the domain that contains this CPU */ if (cpumask_test_cpu(cpu, &d->hdr.cpu_mask)) return d; @@ -689,7 +713,8 @@ static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_l3_mon_domain *dom_m if (WARN_ON_ONCE(!pmbm_data)) return; - dom_mba = get_ctrl_domain_from_cpu(smp_processor_id(), r_mba); + guard(rcu)(); + dom_mba = get_sc_ctrl_domain_from_cpu(smp_processor_id(), r_mba); if (!dom_mba) { pr_warn_once("Failure to get domain for MBA update\n"); return; @@ -750,8 +775,8 @@ static void mbm_update_one_event(struct rdt_resource *r, struct rdt_l3_mon_domai } else { rr.arch_mon_ctx = resctrl_arch_mon_ctx_alloc(rr.r, evtid); if (IS_ERR(rr.arch_mon_ctx)) { - pr_warn_ratelimited("Failed to allocate monitor context: %ld", - PTR_ERR(rr.arch_mon_ctx)); + pr_warn_ratelimited("Failed to allocate monitor context: %pe", + rr.arch_mon_ctx); return; } } @@ -792,11 +817,25 @@ void cqm_handle_limbo(struct work_struct *work) unsigned long delay = msecs_to_jiffies(CQM_LIMBOCHECK_INTERVAL); struct rdt_l3_mon_domain *d; - cpus_read_lock(); + /* + * Safe to run without CPU hotplug lock. Work is guaranteed to be + * canceled before the domain structure is removed. + */ mutex_lock(&rdtgroup_mutex); + /* + * Ensure the worker is dedicated to a CPU as intended and not + * relocated by workqueue subsystem as part of CPU going offline. + */ + if (!is_percpu_thread()) + goto out_unlock; + d = container_of(work, struct rdt_l3_mon_domain, cqm_limbo.work); + /* Domain is going offline */ + if (cpumask_empty(&d->hdr.cpu_mask)) + goto out_unlock; + __check_limbo(d, false); if (has_busy_rmid(d)) { @@ -806,8 +845,8 @@ void cqm_handle_limbo(struct work_struct *work) delay); } +out_unlock: mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); } /** @@ -839,7 +878,10 @@ void mbm_handle_overflow(struct work_struct *work) struct list_head *head; struct rdt_resource *r; - cpus_read_lock(); + /* + * Safe to run without CPU hotplug lock. Work is guaranteed to be + * canceled before the domain structure is removed. + */ mutex_lock(&rdtgroup_mutex); /* @@ -849,9 +891,24 @@ void mbm_handle_overflow(struct work_struct *work) if (!resctrl_mounted || !resctrl_arch_mon_capable()) goto out_unlock; + /* + * Ensure the worker is dedicated to a CPU and not relocated by + * workqueue subsystem as part of CPU going offline since reading + * events depend on smp_processor_id(). After passing this check + * smp_processor_id() is valid for entire duration of this worker + * since it runs with rdtgroup_mutex held and the offline handler needs + * rdtgroup_mutex to offline the CPU being run on here. + */ + if (!is_percpu_thread()) + goto out_unlock; + r = resctrl_arch_get_resource(RDT_RESOURCE_L3); d = container_of(work, struct rdt_l3_mon_domain, mbm_over.work); + /* Domain is going offline */ + if (cpumask_empty(&d->hdr.cpu_mask)) + goto out_unlock; + list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) { mbm_update(r, d, prgrp); @@ -873,7 +930,6 @@ void mbm_handle_overflow(struct work_struct *work) out_unlock: mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); } /** @@ -1050,7 +1106,8 @@ int event_filter_show(struct kernfs_open_file *of, struct seq_file *seq, void *v bool sep = false; int ret = 0, i; - mutex_lock(&rdtgroup_mutex); + if (!info_kn_lock(of->kn)) + return -ENOENT; rdt_last_cmd_clear(); r = resctrl_arch_get_resource(mevt->rid); @@ -1071,7 +1128,7 @@ int event_filter_show(struct kernfs_open_file *of, struct seq_file *seq, void *v seq_putc(seq, '\n'); out_unlock: - mutex_unlock(&rdtgroup_mutex); + info_kn_unlock(of->kn); return ret; } @@ -1082,7 +1139,8 @@ int resctrl_mbm_assign_on_mkdir_show(struct kernfs_open_file *of, struct seq_fil struct rdt_resource *r = rdt_kn_parent_priv(of->kn); int ret = 0; - mutex_lock(&rdtgroup_mutex); + if (!info_kn_lock(of->kn)) + return -ENOENT; rdt_last_cmd_clear(); if (!resctrl_arch_mbm_cntr_assign_enabled(r)) { @@ -1094,7 +1152,7 @@ int resctrl_mbm_assign_on_mkdir_show(struct kernfs_open_file *of, struct seq_fil seq_printf(s, "%u\n", r->mon.mbm_assign_on_mkdir); out_unlock: - mutex_unlock(&rdtgroup_mutex); + info_kn_unlock(of->kn); return ret; } @@ -1106,13 +1164,16 @@ ssize_t resctrl_mbm_assign_on_mkdir_write(struct kernfs_open_file *of, char *buf bool value; int ret; - ret = kstrtobool(buf, &value); - if (ret) - return ret; - - mutex_lock(&rdtgroup_mutex); + if (!info_kn_lock(of->kn)) + return -ENOENT; rdt_last_cmd_clear(); + ret = kstrtobool(buf, &value); + if (ret) { + rdt_last_cmd_puts("mbm_assign_on_mkdir: Invalid input\n"); + goto out_unlock; + } + if (!resctrl_arch_mbm_cntr_assign_enabled(r)) { rdt_last_cmd_puts("mbm_event counter assignment mode is not enabled\n"); ret = -EINVAL; @@ -1122,7 +1183,7 @@ ssize_t resctrl_mbm_assign_on_mkdir_write(struct kernfs_open_file *of, char *buf r->mon.mbm_assign_on_mkdir = value; out_unlock: - mutex_unlock(&rdtgroup_mutex); + info_kn_unlock(of->kn); return ret ?: nbytes; } @@ -1209,9 +1270,10 @@ static int rdtgroup_alloc_assign_cntr(struct rdt_resource *r, struct rdt_l3_mon_ * NULL; otherwise, assign the counter to the specified domain @d. * * If all counters in a domain are already in use, rdtgroup_alloc_assign_cntr() - * will fail. The assignment process will abort at the first failure encountered - * during domain traversal, which may result in the event being only partially - * assigned. + * will fail. When attempting to assign counters to all domains, carry on trying + * to assign counters after a failure since only some domains may have counters + * and the goal is to assign counters where possible. If any counter assignment + * fails, return the error from the last failing assignment. * * Return: * 0 on success, < 0 on failure. @@ -1223,10 +1285,12 @@ static int rdtgroup_assign_cntr_event(struct rdt_l3_mon_domain *d, struct rdtgro int ret = 0; if (!d) { - list_for_each_entry(d, &r->mon_domains, hdr.list) { - ret = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt); - if (ret) - return ret; + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { + int err; + + err = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt); + if (err) + ret = err; } } else { ret = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt); @@ -1293,7 +1357,7 @@ static void rdtgroup_unassign_cntr_event(struct rdt_l3_mon_domain *d, struct rdt struct rdt_resource *r = resctrl_arch_get_resource(mevt->rid); if (!d) { - list_for_each_entry(d, &r->mon_domains, hdr.list) + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) rdtgroup_free_unassign_cntr(r, d, rdtgrp, mevt); } else { rdtgroup_free_unassign_cntr(r, d, rdtgrp, mevt); @@ -1365,7 +1429,7 @@ static void rdtgroup_update_cntr_event(struct rdt_resource *r, struct rdtgroup * struct rdt_l3_mon_domain *d; int cntr_id; - list_for_each_entry(d, &r->mon_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { cntr_id = mbm_cntr_get(r, d, rdtgrp, evtid); if (cntr_id >= 0) rdtgroup_assign_cntr(r, d, evtid, rdtgrp->mon.rmid, @@ -1403,16 +1467,19 @@ ssize_t event_filter_write(struct kernfs_open_file *of, char *buf, size_t nbytes u32 evt_cfg = 0; int ret = 0; - /* Valid input requires a trailing newline */ - if (nbytes == 0 || buf[nbytes - 1] != '\n') - return -EINVAL; + if (!info_kn_lock(of->kn)) + return -ENOENT; - buf[nbytes - 1] = '\0'; + rdt_last_cmd_clear(); - cpus_read_lock(); - mutex_lock(&rdtgroup_mutex); + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') { + rdt_last_cmd_puts("event_filter: Invalid input\n"); + ret = -EINVAL; + goto out_unlock; + } - rdt_last_cmd_clear(); + buf[nbytes - 1] = '\0'; r = resctrl_arch_get_resource(mevt->rid); if (!resctrl_arch_mbm_cntr_assign_enabled(r)) { @@ -1420,6 +1487,11 @@ ssize_t event_filter_write(struct kernfs_open_file *of, char *buf, size_t nbytes ret = -EINVAL; goto out_unlock; } + if (!r->mon.mbm_cntr_configurable) { + rdt_last_cmd_puts("event_filter is not configurable\n"); + ret = -EPERM; + goto out_unlock; + } ret = resctrl_parse_mem_transactions(buf, &evt_cfg); if (!ret && mevt->evt_cfg != evt_cfg) { @@ -1428,8 +1500,7 @@ ssize_t event_filter_write(struct kernfs_open_file *of, char *buf, size_t nbytes } out_unlock: - mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); + info_kn_unlock(of->kn); return ret ?: nbytes; } @@ -1440,7 +1511,8 @@ int resctrl_mbm_assign_mode_show(struct kernfs_open_file *of, struct rdt_resource *r = rdt_kn_parent_priv(of->kn); bool enabled; - mutex_lock(&rdtgroup_mutex); + if (!info_kn_lock(of->kn)) + return -ENOENT; enabled = resctrl_arch_mbm_cntr_assign_enabled(r); if (r->mon.mbm_cntr_assignable) { @@ -1449,7 +1521,7 @@ int resctrl_mbm_assign_mode_show(struct kernfs_open_file *of, else seq_puts(s, "[default]\n"); - if (!IS_ENABLED(CONFIG_RESCTRL_ASSIGN_FIXED)) { + if (!r->mon.mbm_cntr_assign_fixed) { if (enabled) seq_puts(s, "default\n"); else @@ -1459,7 +1531,7 @@ int resctrl_mbm_assign_mode_show(struct kernfs_open_file *of, seq_puts(s, "[default]\n"); } - mutex_unlock(&rdtgroup_mutex); + info_kn_unlock(of->kn); return 0; } @@ -1472,16 +1544,19 @@ ssize_t resctrl_mbm_assign_mode_write(struct kernfs_open_file *of, char *buf, int ret = 0; bool enable; - /* Valid input requires a trailing newline */ - if (nbytes == 0 || buf[nbytes - 1] != '\n') - return -EINVAL; + if (!info_kn_lock(of->kn)) + return -ENOENT; - buf[nbytes - 1] = '\0'; + rdt_last_cmd_clear(); - cpus_read_lock(); - mutex_lock(&rdtgroup_mutex); + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') { + rdt_last_cmd_puts("mbm_assign_mode: Invalid input\n"); + ret = -EINVAL; + goto out_unlock; + } - rdt_last_cmd_clear(); + buf[nbytes - 1] = '\0'; if (!strcmp(buf, "default")) { enable = 0; @@ -1500,6 +1575,12 @@ ssize_t resctrl_mbm_assign_mode_write(struct kernfs_open_file *of, char *buf, } if (enable != resctrl_arch_mbm_cntr_assign_enabled(r)) { + if (r->mon.mbm_cntr_assign_fixed) { + ret = -EINVAL; + rdt_last_cmd_puts("Counter assignment mode is not configurable\n"); + goto out_unlock; + } + ret = resctrl_arch_mbm_cntr_assign_set(r, enable); if (ret) goto out_unlock; @@ -1524,15 +1605,14 @@ ssize_t resctrl_mbm_assign_mode_write(struct kernfs_open_file *of, char *buf, /* * Reset all the non-achitectural RMID state and assignable counters. */ - list_for_each_entry(d, &r->mon_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { mbm_cntr_free_all(r, d); resctrl_reset_rmid_all(r, d); } } out_unlock: - mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); + info_kn_unlock(of->kn); return ret ?: nbytes; } @@ -1544,10 +1624,10 @@ int resctrl_num_mbm_cntrs_show(struct kernfs_open_file *of, struct rdt_l3_mon_domain *dom; bool sep = false; - cpus_read_lock(); - mutex_lock(&rdtgroup_mutex); + if (!info_kn_lock(of->kn)) + return -ENOENT; - list_for_each_entry(dom, &r->mon_domains, hdr.list) { + list_for_each_entry_rcu(dom, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { if (sep) seq_putc(s, ';'); @@ -1556,8 +1636,7 @@ int resctrl_num_mbm_cntrs_show(struct kernfs_open_file *of, } seq_putc(s, '\n'); - mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); + info_kn_unlock(of->kn); return 0; } @@ -1570,8 +1649,8 @@ int resctrl_available_mbm_cntrs_show(struct kernfs_open_file *of, u32 cntrs, i; int ret = 0; - cpus_read_lock(); - mutex_lock(&rdtgroup_mutex); + if (!info_kn_lock(of->kn)) + return -ENOENT; rdt_last_cmd_clear(); @@ -1581,7 +1660,7 @@ int resctrl_available_mbm_cntrs_show(struct kernfs_open_file *of, goto out_unlock; } - list_for_each_entry(dom, &r->mon_domains, hdr.list) { + list_for_each_entry_rcu(dom, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { if (sep) seq_putc(s, ';'); @@ -1597,8 +1676,7 @@ int resctrl_available_mbm_cntrs_show(struct kernfs_open_file *of, seq_putc(s, '\n'); out_unlock: - mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); + info_kn_unlock(of->kn); return ret; } @@ -1618,7 +1696,6 @@ int mbm_L3_assignments_show(struct kernfs_open_file *of, struct seq_file *s, voi goto out_unlock; } - rdt_last_cmd_clear(); if (!resctrl_arch_mbm_cntr_assign_enabled(r)) { rdt_last_cmd_puts("mbm_event counter assignment mode is not enabled\n"); ret = -EINVAL; @@ -1631,7 +1708,7 @@ int mbm_L3_assignments_show(struct kernfs_open_file *of, struct seq_file *s, voi sep = false; seq_printf(s, "%s:", mevt->name); - list_for_each_entry(d, &r->mon_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { if (sep) seq_putc(s, ';'); @@ -1729,7 +1806,7 @@ next: } /* Verify if the dom_id is valid */ - list_for_each_entry(d, &r->mon_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { if (d->hdr.id == dom_id) { ret = rdtgroup_modify_assign_state(dom_str, d, rdtgrp, mevt); if (ret) { @@ -1753,23 +1830,25 @@ ssize_t mbm_L3_assignments_write(struct kernfs_open_file *of, char *buf, char *token, *event; int ret = 0; - /* Valid input requires a trailing newline */ - if (nbytes == 0 || buf[nbytes - 1] != '\n') - return -EINVAL; - - buf[nbytes - 1] = '\0'; - rdtgrp = rdtgroup_kn_lock_live(of->kn); if (!rdtgrp) { rdtgroup_kn_unlock(of->kn); return -ENOENT; } - rdt_last_cmd_clear(); + + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') { + rdt_last_cmd_puts("mbm_L3_assignments: Invalid input\n"); + ret = -EINVAL; + goto out_unlock; + } + + buf[nbytes - 1] = '\0'; if (!resctrl_arch_mbm_cntr_assign_enabled(r)) { rdt_last_cmd_puts("mbm_event mode is not enabled\n"); - rdtgroup_kn_unlock(of->kn); - return -EINVAL; + ret = -EINVAL; + goto out_unlock; } while ((token = strsep(&buf, "\n")) != NULL) { @@ -1785,6 +1864,7 @@ ssize_t mbm_L3_assignments_write(struct kernfs_open_file *of, char *buf, break; } +out_unlock: rdtgroup_kn_unlock(of->kn); return ret ?: nbytes; @@ -1884,6 +1964,8 @@ int resctrl_l3_mon_resource_init(void) resctrl_file_fflags_init("available_mbm_cntrs", RFTYPE_MON_INFO | RFTYPE_RES_CACHE); resctrl_file_fflags_init("event_filter", RFTYPE_ASSIGN_CONFIG); + if (r->mon.mbm_cntr_configurable) + resctrl_file_mode_init("event_filter", 0644); resctrl_file_fflags_init("mbm_assign_on_mkdir", RFTYPE_MON_INFO | RFTYPE_RES_CACHE); resctrl_file_fflags_init("mbm_L3_assignments", RFTYPE_MON_BASE); diff --git a/fs/resctrl/pseudo_lock.c b/fs/resctrl/pseudo_lock.c index fa3687d69ebd..dea2b4bf966f 100644 --- a/fs/resctrl/pseudo_lock.c +++ b/fs/resctrl/pseudo_lock.c @@ -656,7 +656,7 @@ bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_ctrl_domain *d) * associated with them. */ for_each_alloc_capable_rdt_resource(r) { - list_for_each_entry(d_i, &r->ctrl_domains, hdr.list) { + list_for_each_entry_rcu(d_i, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { if (d_i->plr) cpumask_or(cpu_with_psl, cpu_with_psl, &d_i->hdr.cpu_mask); @@ -1044,7 +1044,7 @@ static int pseudo_lock_dev_mmap_prepare(struct vm_area_desc *desc) * Ensure changes are carried directly to the memory being mapped, * do not allow copy-on-write mapping. */ - if (!vma_desc_test_flags(desc, VMA_SHARED_BIT)) { + if (!vma_desc_test(desc, VMA_SHARED_BIT)) { mutex_unlock(&rdtgroup_mutex); return -EINVAL; } diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index 5da305bd36c9..5dcbb0a964e8 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -117,7 +117,7 @@ void rdt_staged_configs_clear(void) lockdep_assert_held(&rdtgroup_mutex); for_each_alloc_capable_rdt_resource(r) { - list_for_each_entry(dom, &r->ctrl_domains, hdr.list) + list_for_each_entry_rcu(dom, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) memset(dom->staged_config, 0, sizeof(dom->staged_config)); } } @@ -359,7 +359,6 @@ static int rdtgroup_cpus_show(struct kernfs_open_file *of, if (rdtgrp) { if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { if (!rdtgrp->plr->d) { - rdt_last_cmd_clear(); rdt_last_cmd_puts("Cache domain offline\n"); ret = -ENODEV; } else { @@ -511,38 +510,38 @@ static int cpus_ctrl_write(struct rdtgroup *rdtgrp, cpumask_var_t newmask, static ssize_t rdtgroup_cpus_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { - cpumask_var_t tmpmask, newmask, tmpmask1; + cpumask_var_t tmpmask = CPUMASK_VAR_NULL, newmask = CPUMASK_VAR_NULL; + cpumask_var_t tmpmask1 = CPUMASK_VAR_NULL; struct rdtgroup *rdtgrp; int ret; - if (!buf) - return -EINVAL; - - if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL)) - return -ENOMEM; - if (!zalloc_cpumask_var(&newmask, GFP_KERNEL)) { - free_cpumask_var(tmpmask); - return -ENOMEM; - } - if (!zalloc_cpumask_var(&tmpmask1, GFP_KERNEL)) { - free_cpumask_var(tmpmask); - free_cpumask_var(newmask); - return -ENOMEM; - } - rdtgrp = rdtgroup_kn_lock_live(of->kn); if (!rdtgrp) { ret = -ENOENT; - goto unlock; + goto out_unlock; } - rdt_last_cmd_clear(); + if (!buf) { + rdt_last_cmd_printf("%s: Invalid input\n", + is_cpu_list(of) ? "cpus_list" : "cpus"); + ret = -EINVAL; + goto out_unlock; + } + + if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL) || + !zalloc_cpumask_var(&newmask, GFP_KERNEL) || + !zalloc_cpumask_var(&tmpmask1, GFP_KERNEL)) { + rdt_last_cmd_printf("%s: Kernel allocation failure\n", + is_cpu_list(of) ? "cpus_list" : "cpus"); + ret = -ENOMEM; + goto out_free; + } if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED || rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { ret = -EINVAL; rdt_last_cmd_puts("Pseudo-locking in progress\n"); - goto unlock; + goto out_free; } if (is_cpu_list(of)) @@ -552,7 +551,7 @@ static ssize_t rdtgroup_cpus_write(struct kernfs_open_file *of, if (ret) { rdt_last_cmd_puts("Bad CPU list/mask\n"); - goto unlock; + goto out_free; } /* check that user didn't specify any offline cpus */ @@ -560,7 +559,7 @@ static ssize_t rdtgroup_cpus_write(struct kernfs_open_file *of, if (!cpumask_empty(tmpmask)) { ret = -EINVAL; rdt_last_cmd_puts("Can only assign online CPUs\n"); - goto unlock; + goto out_free; } if (rdtgrp->type == RDTCTRL_GROUP) @@ -570,11 +569,12 @@ static ssize_t rdtgroup_cpus_write(struct kernfs_open_file *of, else ret = -EINVAL; -unlock: - rdtgroup_kn_unlock(of->kn); +out_free: free_cpumask_var(tmpmask); free_cpumask_var(newmask); free_cpumask_var(tmpmask1); +out_unlock: + rdtgroup_kn_unlock(of->kn); return ret ?: nbytes; } @@ -585,14 +585,20 @@ unlock: * * On resource group creation via a mkdir, an extra kernfs_node reference is * taken to ensure that the rdtgroup structure remains accessible for the - * rdtgroup_kn_unlock() calls where it is removed. + * rdtgroup_kn_unlock() calls where it is removed. The default group is + * statically allocated: it does not have an extra reference but will have + * RDT_DELETED set on unmount to support safe access to its associated files + * via rdtgroup_kn_lock_live/rdtgroup_kn_unlock(). * - * Drop the extra reference here, then free the rdtgroup structure. + * For all but the default group: drop the extra reference, then free the + * rdtgroup structure. * * Return: void */ static void rdtgroup_remove(struct rdtgroup *rdtgrp) { + if (rdtgrp == &rdtgroup_default) + return; kernfs_put(rdtgrp->kn); kfree(rdtgrp); } @@ -779,7 +785,6 @@ static ssize_t rdtgroup_tasks_write(struct kernfs_open_file *of, rdtgroup_kn_unlock(of->kn); return -ENOENT; } - rdt_last_cmd_clear(); if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED || rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { @@ -971,13 +976,17 @@ static int rdt_last_cmd_status_show(struct kernfs_open_file *of, { int len; - mutex_lock(&rdtgroup_mutex); + if (!info_kn_lock(of->kn)) + return -ENOENT; len = seq_buf_used(&last_cmd_status); - if (len) + if (len) { seq_printf(seq, "%.*s", len, last_cmd_status_buf); - else + if (seq_buf_has_overflowed(&last_cmd_status)) + seq_puts(seq, "[truncated]\n"); + } else { seq_puts(seq, "ok\n"); - mutex_unlock(&rdtgroup_mutex); + } + info_kn_unlock(of->kn); return 0; } @@ -996,7 +1005,11 @@ static int rdt_num_closids_show(struct kernfs_open_file *of, { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); + if (!info_kn_lock(of->kn)) + return -ENOENT; seq_printf(seq, "%u\n", s->num_closid); + info_kn_unlock(of->kn); + return 0; } @@ -1004,9 +1017,14 @@ static int rdt_default_ctrl_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); - struct rdt_resource *r = s->res; + struct rdt_resource *r; + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; seq_printf(seq, "%x\n", resctrl_get_default_ctrl(r)); + info_kn_unlock(of->kn); + return 0; } @@ -1014,9 +1032,14 @@ static int rdt_min_cbm_bits_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); - struct rdt_resource *r = s->res; + struct rdt_resource *r; + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; seq_printf(seq, "%u\n", r->cache.min_cbm_bits); + info_kn_unlock(of->kn); + return 0; } @@ -1024,9 +1047,14 @@ static int rdt_shareable_bits_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); - struct rdt_resource *r = s->res; + struct rdt_resource *r; + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; seq_printf(seq, "%x\n", r->cache.shareable_bits); + info_kn_unlock(of->kn); + return 0; } @@ -1054,16 +1082,17 @@ static int rdt_bit_usage_show(struct kernfs_open_file *of, */ unsigned long sw_shareable = 0, hw_shareable = 0; unsigned long exclusive = 0, pseudo_locked = 0; - struct rdt_resource *r = s->res; struct rdt_ctrl_domain *dom; int i, hwb, swb, excl, psl; + struct rdt_resource *r; enum rdtgrp_mode mode; bool sep = false; u32 ctrl_val; - cpus_read_lock(); - mutex_lock(&rdtgroup_mutex); - list_for_each_entry(dom, &r->ctrl_domains, hdr.list) { + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; + list_for_each_entry_rcu(dom, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { if (sep) seq_putc(seq, ';'); hw_shareable = r->cache.shareable_bits; @@ -1138,8 +1167,7 @@ static int rdt_bit_usage_show(struct kernfs_open_file *of, sep = true; } seq_putc(seq, '\n'); - mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); + info_kn_unlock(of->kn); return 0; } @@ -1147,9 +1175,14 @@ static int rdt_min_bw_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); - struct rdt_resource *r = s->res; + struct rdt_resource *r; + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; seq_printf(seq, "%u\n", r->membw.min_bw); + info_kn_unlock(of->kn); + return 0; } @@ -1158,8 +1191,12 @@ static int rdt_num_rmids_show(struct kernfs_open_file *of, { struct rdt_resource *r = rdt_kn_parent_priv(of->kn); + if (!info_kn_lock(of->kn)) + return -ENOENT; seq_printf(seq, "%u\n", r->mon.num_rmid); + info_kn_unlock(of->kn); + return 0; } @@ -1169,6 +1206,8 @@ static int rdt_mon_features_show(struct kernfs_open_file *of, struct rdt_resource *r = rdt_kn_parent_priv(of->kn); struct mon_evt *mevt; + if (!info_kn_lock(of->kn)) + return -ENOENT; for_each_mon_event(mevt) { if (mevt->rid != r->rid || !mevt->enabled) continue; @@ -1178,6 +1217,8 @@ static int rdt_mon_features_show(struct kernfs_open_file *of, seq_printf(seq, "%s_config\n", mevt->name); } + info_kn_unlock(of->kn); + return 0; } @@ -1185,9 +1226,14 @@ static int rdt_bw_gran_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); - struct rdt_resource *r = s->res; + struct rdt_resource *r; + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; seq_printf(seq, "%u\n", r->membw.bw_gran); + info_kn_unlock(of->kn); + return 0; } @@ -1195,16 +1241,24 @@ static int rdt_delay_linear_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); - struct rdt_resource *r = s->res; + struct rdt_resource *r; + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; seq_printf(seq, "%u\n", r->membw.delay_linear); + info_kn_unlock(of->kn); + return 0; } static int max_threshold_occ_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { + if (!info_kn_lock(of->kn)) + return -ENOENT; seq_printf(seq, "%u\n", resctrl_rmid_realloc_threshold); + info_kn_unlock(of->kn); return 0; } @@ -1213,22 +1267,28 @@ static int rdt_thread_throttle_mode_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); - struct rdt_resource *r = s->res; + struct rdt_resource *r; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; switch (r->membw.throttle_mode) { case THREAD_THROTTLE_PER_THREAD: seq_puts(seq, "per-thread\n"); - return 0; + break; case THREAD_THROTTLE_MAX: seq_puts(seq, "max\n"); - return 0; + break; case THREAD_THROTTLE_UNDEFINED: seq_puts(seq, "undefined\n"); - return 0; + break; + default: + WARN_ON_ONCE(1); + break; } - WARN_ON_ONCE(1); - + info_kn_unlock(of->kn); return 0; } @@ -1238,16 +1298,30 @@ static ssize_t max_threshold_occ_write(struct kernfs_open_file *of, unsigned int bytes; int ret; + if (!info_kn_lock(of->kn)) + return -ENOENT; + + rdt_last_cmd_clear(); + ret = kstrtouint(buf, 0, &bytes); - if (ret) - return ret; + if (ret) { + rdt_last_cmd_puts("max_threshold_occupancy: Invalid input\n"); + goto out_unlock; + } - if (bytes > resctrl_rmid_realloc_limit) - return -EINVAL; + if (bytes > resctrl_rmid_realloc_limit) { + rdt_last_cmd_printf("max_threshold_occupancy: Exceeds limit (before adjustment) of %u bytes\n", + resctrl_rmid_realloc_limit); + ret = -EINVAL; + goto out_unlock; + } resctrl_rmid_realloc_threshold = resctrl_arch_round_mon_val(bytes); - return nbytes; +out_unlock: + info_kn_unlock(of->kn); + + return ret ?: nbytes; } /* @@ -1287,10 +1361,15 @@ static int rdt_has_sparse_bitmasks_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); - struct rdt_resource *r = s->res; + struct rdt_resource *r; + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; seq_printf(seq, "%u\n", r->cache.arch_has_sparse_bitmasks); + info_kn_unlock(of->kn); + return 0; } @@ -1415,7 +1494,7 @@ static bool rdtgroup_mode_test_exclusive(struct rdtgroup *rdtgrp) if (r->rid == RDT_RESOURCE_MBA || r->rid == RDT_RESOURCE_SMBA) continue; has_cache = true; - list_for_each_entry(d, &r->ctrl_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { ctrl = resctrl_arch_get_config(r, d, closid, s->conf_type); if (rdtgroup_cbm_overlaps(s, d, ctrl, closid, false)) { @@ -1443,18 +1522,20 @@ static ssize_t rdtgroup_mode_write(struct kernfs_open_file *of, enum rdtgrp_mode mode; int ret = 0; - /* Valid input requires a trailing newline */ - if (nbytes == 0 || buf[nbytes - 1] != '\n') - return -EINVAL; - buf[nbytes - 1] = '\0'; - rdtgrp = rdtgroup_kn_lock_live(of->kn); if (!rdtgrp) { rdtgroup_kn_unlock(of->kn); return -ENOENT; } - rdt_last_cmd_clear(); + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') { + rdt_last_cmd_puts("mode: Invalid input\n"); + ret = -EINVAL; + goto out; + } + + buf[nbytes - 1] = '\0'; mode = rdtgrp->mode; @@ -1519,6 +1600,8 @@ out: * * @cbm is unsigned long, even if only 32 bits are used to make the * bitmap functions work correctly. + * + * Return: Size (in bytes) of cache portion represented by CBM, 0 on failure. */ unsigned int rdtgroup_cbm_to_size(struct rdt_resource *r, struct rdt_ctrl_domain *d, unsigned long cbm) @@ -1581,7 +1664,6 @@ static int rdtgroup_size_show(struct kernfs_open_file *of, if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { if (!rdtgrp->plr->d) { - rdt_last_cmd_clear(); rdt_last_cmd_puts("Cache domain offline\n"); ret = -ENODEV; } else { @@ -1602,7 +1684,7 @@ static int rdtgroup_size_show(struct kernfs_open_file *of, type = schema->conf_type; sep = false; seq_printf(s, "%*s:", max_name_width, schema->name); - list_for_each_entry(d, &r->ctrl_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { if (sep) seq_putc(s, ';'); if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { @@ -1644,10 +1726,10 @@ static int mbm_config_show(struct seq_file *s, struct rdt_resource *r, u32 evtid struct rdt_l3_mon_domain *dom; bool sep = false; - cpus_read_lock(); - mutex_lock(&rdtgroup_mutex); + lockdep_assert_cpus_held(); + lockdep_assert_held(&rdtgroup_mutex); - list_for_each_entry(dom, &r->mon_domains, hdr.list) { + list_for_each_entry_rcu(dom, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { if (sep) seq_puts(s, ";"); @@ -1662,9 +1744,6 @@ static int mbm_config_show(struct seq_file *s, struct rdt_resource *r, u32 evtid } seq_puts(s, "\n"); - mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); - return 0; } @@ -1673,8 +1752,12 @@ static int mbm_total_bytes_config_show(struct kernfs_open_file *of, { struct rdt_resource *r = rdt_kn_parent_priv(of->kn); + if (!info_kn_lock(of->kn)) + return -ENOENT; + mbm_config_show(seq, r, QOS_L3_MBM_TOTAL_EVENT_ID); + info_kn_unlock(of->kn); return 0; } @@ -1683,8 +1766,12 @@ static int mbm_local_bytes_config_show(struct kernfs_open_file *of, { struct rdt_resource *r = rdt_kn_parent_priv(of->kn); + if (!info_kn_lock(of->kn)) + return -ENOENT; + mbm_config_show(seq, r, QOS_L3_MBM_LOCAL_EVENT_ID); + info_kn_unlock(of->kn); return 0; } @@ -1761,7 +1848,7 @@ next: return -EINVAL; } - list_for_each_entry(d, &r->mon_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { if (d->hdr.id == dom_id) { mbm_config_write_domain(r, d, evtid, val); goto next; @@ -1778,21 +1865,24 @@ static ssize_t mbm_total_bytes_config_write(struct kernfs_open_file *of, struct rdt_resource *r = rdt_kn_parent_priv(of->kn); int ret; - /* Valid input requires a trailing newline */ - if (nbytes == 0 || buf[nbytes - 1] != '\n') - return -EINVAL; - - cpus_read_lock(); - mutex_lock(&rdtgroup_mutex); + if (!info_kn_lock(of->kn)) + return -ENOENT; rdt_last_cmd_clear(); + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') { + rdt_last_cmd_puts("mbm_total_bytes_config: Invalid input\n"); + ret = -EINVAL; + goto out_unlock; + } + buf[nbytes - 1] = '\0'; ret = mon_config_write(r, buf, QOS_L3_MBM_TOTAL_EVENT_ID); - mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); +out_unlock: + info_kn_unlock(of->kn); return ret ?: nbytes; } @@ -1804,21 +1894,24 @@ static ssize_t mbm_local_bytes_config_write(struct kernfs_open_file *of, struct rdt_resource *r = rdt_kn_parent_priv(of->kn); int ret; - /* Valid input requires a trailing newline */ - if (nbytes == 0 || buf[nbytes - 1] != '\n') - return -EINVAL; - - cpus_read_lock(); - mutex_lock(&rdtgroup_mutex); + if (!info_kn_lock(of->kn)) + return -ENOENT; rdt_last_cmd_clear(); + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') { + rdt_last_cmd_puts("mbm_local_bytes_config: Invalid input\n"); + ret = -EINVAL; + goto out_unlock; + } + buf[nbytes - 1] = '\0'; ret = mon_config_write(r, buf, QOS_L3_MBM_LOCAL_EVENT_ID); - mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); +out_unlock: + info_kn_unlock(of->kn); return ret ?: nbytes; } @@ -2020,7 +2113,7 @@ static struct rftype res_common_files[] = { }, { .name = "event_filter", - .mode = 0644, + .mode = 0444, .kf_ops = &rdtgroup_kf_single_ops, .seq_show = event_filter_show, .write = event_filter_write, @@ -2213,6 +2306,15 @@ void resctrl_file_fflags_init(const char *config, unsigned long fflags) rft->fflags = fflags; } +void resctrl_file_mode_init(const char *config, umode_t mode) +{ + struct rftype *rft; + + rft = rdtgroup_get_rftype_by_name(config); + if (rft) + rft->mode = mode; +} + /** * rdtgroup_kn_mode_restrict - Restrict user access to named resctrl file * @r: The resource group with which the file is associated. @@ -2329,22 +2431,19 @@ static int resctrl_mkdir_event_configs(struct rdt_resource *r, struct kernfs_nod continue; kn_subdir2 = kernfs_create_dir(kn_subdir, mevt->name, kn_subdir->mode, mevt); - if (IS_ERR(kn_subdir2)) { - ret = PTR_ERR(kn_subdir2); - goto out; - } + if (IS_ERR(kn_subdir2)) + return PTR_ERR(kn_subdir2); ret = rdtgroup_kn_set_ugid(kn_subdir2); if (ret) - goto out; + return ret; ret = rdtgroup_add_files(kn_subdir2, RFTYPE_ASSIGN_CONFIG); if (ret) - break; + return ret; } -out: - return ret; + return 0; } static int rdtgroup_mkdir_info_resdir(void *priv, char *name, @@ -2508,10 +2607,13 @@ static void mba_sc_domain_destroy(struct rdt_resource *r, } /* - * MBA software controller is supported only if - * MBM is supported and MBA is in linear scale, - * and the MBM monitor scope is the same as MBA - * control scope. + * The MBA software controller is supported only if MBM is supported and MBA is + * in linear scale, and the MBM monitor scope is the same as MBA control scope. + * + * The software controller cannot be supported when the MBM counters are + * assignable. There is no guarantee that MBM counters are assigned to the + * event backing the software controller in all monitoring domains of all + * monitoring groups. */ static bool supports_mba_mbps(void) { @@ -2520,7 +2622,8 @@ static bool supports_mba_mbps(void) return (resctrl_is_mbm_enabled() && r->alloc_capable && is_mba_linear() && - r->ctrl_scope == rmbm->mon_scope); + r->ctrl_scope == rmbm->mon_scope && + !rmbm->mon.mbm_cntr_assignable); } /* @@ -2542,7 +2645,7 @@ static int set_mba_sc(bool mba_sc) rdtgroup_default.mba_mbps_event = mba_mbps_default_event; - list_for_each_entry(d, &r->ctrl_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { for (i = 0; i < num_closid; i++) d->mbps_val[i] = MBA_MAX_MBPS; } @@ -2588,15 +2691,24 @@ static void rdtgroup_kn_get(struct rdtgroup *rdtgrp, struct kernfs_node *kn) static void rdtgroup_kn_put(struct rdtgroup *rdtgrp, struct kernfs_node *kn) { - if (atomic_dec_and_test(&rdtgrp->waitcount) && - (rdtgrp->flags & RDT_DELETED)) { + bool needs_free; + + if (!atomic_dec_and_mutex_lock(&rdtgrp->waitcount, &rdtgroup_mutex)) { + kernfs_unbreak_active_protection(kn); + return; + } + + needs_free = rdtgrp->flags & RDT_DELETED; + + mutex_unlock(&rdtgroup_mutex); + + kernfs_unbreak_active_protection(kn); + + if (needs_free) { if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) rdtgroup_pseudo_lock_remove(rdtgrp); - kernfs_unbreak_active_protection(kn); rdtgroup_remove(rdtgrp); - } else { - kernfs_unbreak_active_protection(kn); } } @@ -2611,10 +2723,20 @@ struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn) cpus_read_lock(); mutex_lock(&rdtgroup_mutex); + rdt_last_cmd_clear(); /* Was this group deleted while we waited? */ - if (rdtgrp->flags & RDT_DELETED) + if (rdtgrp->flags & RDT_DELETED) { + /* + * It is safe to dereference kn to obtain the resource group's + * name because one extra reference to kn is obtained + * during resource group creation that will be released by + * rdtgroup_remove() called by rdtgroup_kn_put(). + */ + rdt_last_cmd_printf("Resource group %s deleted. No commands possible.\n", + rdt_kn_name(rdtgrp->kn)); return NULL; + } return rdtgrp; } @@ -2632,6 +2754,58 @@ void rdtgroup_kn_unlock(struct kernfs_node *kn) rdtgroup_kn_put(rdtgrp, kn); } +/* + * Accessing the kn after breaking active protection is safe since the open + * of resctrl file holds a kernfs base reference (different from active + * protection) on the kn ensuring that it remains accessible even if it was + * unlinked. Each kn in turn holds base reference to parent so the kn's + * genealogy remains in memory until all base references dropped. + */ +static bool is_active_resctrl_node(struct kernfs_node *kn) +{ + struct kernfs_node *p; + bool match = false; + + guard(rcu)(); + p = kn; + while (p) { + if (p == rdtgroup_default.kn) { + match = true; + break; + } + p = rcu_dereference(p->__parent); + } + + return match; +} + +bool info_kn_lock(struct kernfs_node *kn) +{ + kernfs_break_active_protection(kn); + cpus_read_lock(); + mutex_lock(&rdtgroup_mutex); + + /* + * Check both if resctrl is torn down (!rdtgroup_default.kn) and + * if the reader's kernfs_node originates from a dead mount. + */ + if (!rdtgroup_default.kn || !is_active_resctrl_node(kn)) { + mutex_unlock(&rdtgroup_mutex); + cpus_read_unlock(); + kernfs_unbreak_active_protection(kn); + return false; + } + + return true; +} + +void info_kn_unlock(struct kernfs_node *kn) +{ + mutex_unlock(&rdtgroup_mutex); + cpus_read_unlock(); + kernfs_unbreak_active_protection(kn); +} + static int mkdir_mondata_all(struct kernfs_node *parent_kn, struct rdtgroup *prgrp, struct kernfs_node **mon_data_kn); @@ -2780,205 +2954,6 @@ static void schemata_list_destroy(void) } } -static int rdt_get_tree(struct fs_context *fc) -{ - struct rdt_fs_context *ctx = rdt_fc2context(fc); - unsigned long flags = RFTYPE_CTRL_BASE; - struct rdt_l3_mon_domain *dom; - struct rdt_resource *r; - int ret; - - DO_ONCE_SLEEPABLE(resctrl_arch_pre_mount); - - cpus_read_lock(); - mutex_lock(&rdtgroup_mutex); - /* - * resctrl file system can only be mounted once. - */ - if (resctrl_mounted) { - ret = -EBUSY; - goto out; - } - - ret = setup_rmid_lru_list(); - if (ret) - goto out; - - ret = rdtgroup_setup_root(ctx); - if (ret) - goto out; - - ret = rdt_enable_ctx(ctx); - if (ret) - goto out_root; - - ret = schemata_list_create(); - if (ret) - goto out_schemata_free; - - ret = closid_init(); - if (ret) - goto out_schemata_free; - - if (resctrl_arch_mon_capable()) - flags |= RFTYPE_MON; - - ret = rdtgroup_add_files(rdtgroup_default.kn, flags); - if (ret) - goto out_closid_exit; - - kernfs_activate(rdtgroup_default.kn); - - ret = rdtgroup_create_info_dir(rdtgroup_default.kn); - if (ret < 0) - goto out_closid_exit; - - if (resctrl_arch_mon_capable()) { - ret = mongroup_create_dir(rdtgroup_default.kn, - &rdtgroup_default, "mon_groups", - &kn_mongrp); - if (ret < 0) - goto out_info; - - rdtgroup_assign_cntrs(&rdtgroup_default); - - ret = mkdir_mondata_all(rdtgroup_default.kn, - &rdtgroup_default, &kn_mondata); - if (ret < 0) - goto out_mongrp; - rdtgroup_default.mon.mon_data_kn = kn_mondata; - } - - ret = rdt_pseudo_lock_init(); - if (ret) - goto out_mondata; - - ret = kernfs_get_tree(fc); - if (ret < 0) - goto out_psl; - - if (resctrl_arch_alloc_capable()) - resctrl_arch_enable_alloc(); - if (resctrl_arch_mon_capable()) - resctrl_arch_enable_mon(); - - if (resctrl_arch_alloc_capable() || resctrl_arch_mon_capable()) - resctrl_mounted = true; - - if (resctrl_is_mbm_enabled()) { - r = resctrl_arch_get_resource(RDT_RESOURCE_L3); - list_for_each_entry(dom, &r->mon_domains, hdr.list) - mbm_setup_overflow_handler(dom, MBM_OVERFLOW_INTERVAL, - RESCTRL_PICK_ANY_CPU); - } - - goto out; - -out_psl: - rdt_pseudo_lock_release(); -out_mondata: - if (resctrl_arch_mon_capable()) - kernfs_remove(kn_mondata); -out_mongrp: - if (resctrl_arch_mon_capable()) { - rdtgroup_unassign_cntrs(&rdtgroup_default); - kernfs_remove(kn_mongrp); - } -out_info: - kernfs_remove(kn_info); -out_closid_exit: - closid_exit(); -out_schemata_free: - schemata_list_destroy(); - rdt_disable_ctx(); -out_root: - rdtgroup_destroy_root(); -out: - rdt_last_cmd_clear(); - mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); - return ret; -} - -enum rdt_param { - Opt_cdp, - Opt_cdpl2, - Opt_mba_mbps, - Opt_debug, - nr__rdt_params -}; - -static const struct fs_parameter_spec rdt_fs_parameters[] = { - fsparam_flag("cdp", Opt_cdp), - fsparam_flag("cdpl2", Opt_cdpl2), - fsparam_flag("mba_MBps", Opt_mba_mbps), - fsparam_flag("debug", Opt_debug), - {} -}; - -static int rdt_parse_param(struct fs_context *fc, struct fs_parameter *param) -{ - struct rdt_fs_context *ctx = rdt_fc2context(fc); - struct fs_parse_result result; - const char *msg; - int opt; - - opt = fs_parse(fc, rdt_fs_parameters, param, &result); - if (opt < 0) - return opt; - - switch (opt) { - case Opt_cdp: - ctx->enable_cdpl3 = true; - return 0; - case Opt_cdpl2: - ctx->enable_cdpl2 = true; - return 0; - case Opt_mba_mbps: - msg = "mba_MBps requires MBM and linear scale MBA at L3 scope"; - if (!supports_mba_mbps()) - return invalfc(fc, msg); - ctx->enable_mba_mbps = true; - return 0; - case Opt_debug: - ctx->enable_debug = true; - return 0; - } - - return -EINVAL; -} - -static void rdt_fs_context_free(struct fs_context *fc) -{ - struct rdt_fs_context *ctx = rdt_fc2context(fc); - - kernfs_free_fs_context(fc); - kfree(ctx); -} - -static const struct fs_context_operations rdt_fs_context_ops = { - .free = rdt_fs_context_free, - .parse_param = rdt_parse_param, - .get_tree = rdt_get_tree, -}; - -static int rdt_init_fs_context(struct fs_context *fc) -{ - struct rdt_fs_context *ctx; - - ctx = kzalloc_obj(*ctx); - if (!ctx) - return -ENOMEM; - - ctx->kfc.magic = RDTGROUP_SUPER_MAGIC; - fc->fs_private = &ctx->kfc; - fc->ops = &rdt_fs_context_ops; - put_user_ns(fc->user_ns); - fc->user_ns = get_user_ns(&init_user_ns); - fc->global = true; - return 0; -} - /* * Move tasks from one to the other group. If @from is NULL, then all tasks * in the systems are moved unconditionally (used for teardown). @@ -3057,10 +3032,6 @@ static void rmdir_all_sub(void) if (rdtgrp == &rdtgroup_default) continue; - if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || - rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) - rdtgroup_pseudo_lock_remove(rdtgrp); - /* * Give any CPUs back to the default group. We cannot copy * cpu_online_mask because a CPU might have executed the @@ -3071,7 +3042,13 @@ static void rmdir_all_sub(void) rdtgroup_unassign_cntrs(rdtgrp); - free_rmid(rdtgrp->closid, rdtgrp->mon.rmid); + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || + rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { + rdtgroup_pseudo_lock_remove(rdtgrp); + } else { + /* Pseudo-locked group's RMID is freed during setup. */ + free_rmid(rdtgrp->closid, rdtgrp->mon.rmid); + } kernfs_remove(rdtgrp->kn); list_del(&rdtgrp->rdtgroup_list); @@ -3102,6 +3079,8 @@ static void rmdir_all_sub(void) * @mevt: The type of event file being created. * @do_sum: Whether SNC summing monitors are being created. Only set * when @rid == RDT_RESOURCE_L3. + * + * Return: Pointer to mon_data private data of the event, NULL on failure. */ static struct mon_data *mon_get_kn_priv(enum resctrl_res_level rid, int domid, struct mon_evt *mevt, @@ -3160,12 +3139,13 @@ static void resctrl_fs_teardown(void) mon_put_kn_priv(); rdt_pseudo_lock_release(); rdtgroup_default.mode = RDT_MODE_SHAREABLE; + rdtgroup_default.flags = RDT_DELETED; closid_exit(); schemata_list_destroy(); rdtgroup_destroy_root(); } -static void rdt_kill_sb(struct super_block *sb) +static void resctrl_unmount(void) { struct rdt_resource *r; @@ -3184,11 +3164,246 @@ static void rdt_kill_sb(struct super_block *sb) if (resctrl_arch_mon_capable()) resctrl_arch_disable_mon(); resctrl_mounted = false; - kernfs_kill_sb(sb); mutex_unlock(&rdtgroup_mutex); cpus_read_unlock(); } +static int rdt_get_tree(struct fs_context *fc) +{ + struct rdt_fs_context *ctx = rdt_fc2context(fc); + unsigned long flags = RFTYPE_CTRL_BASE; + struct kernfs_node *rdt_root_kn; + struct rdt_l3_mon_domain *dom; + struct rdt_resource *r; + int ret; + + DO_ONCE_SLEEPABLE(resctrl_arch_pre_mount); + + cpus_read_lock(); + mutex_lock(&rdtgroup_mutex); + /* + * resctrl file system can only be mounted once. + */ + if (resctrl_mounted) { + ret = -EBUSY; + goto out; + } + + /* Avoid races from pending operations from a previous mount */ + if (atomic_read(&rdtgroup_default.waitcount) != 0) { + ret = -EBUSY; + goto out; + } + + ret = setup_rmid_lru_list(); + if (ret) + goto out; + + ret = rdtgroup_setup_root(ctx); + if (ret) + goto out; + + ret = rdt_enable_ctx(ctx); + if (ret) + goto out_root; + + ret = schemata_list_create(); + if (ret) + goto out_schemata_free; + + ret = closid_init(); + if (ret) + goto out_schemata_free; + + if (resctrl_arch_mon_capable()) + flags |= RFTYPE_MON; + + ret = rdtgroup_add_files(rdtgroup_default.kn, flags); + if (ret) + goto out_closid_exit; + + kernfs_activate(rdtgroup_default.kn); + + ret = rdtgroup_create_info_dir(rdtgroup_default.kn); + if (ret < 0) + goto out_closid_exit; + + if (resctrl_arch_mon_capable()) { + ret = mongroup_create_dir(rdtgroup_default.kn, + &rdtgroup_default, "mon_groups", + &kn_mongrp); + if (ret < 0) + goto out_info; + + rdtgroup_assign_cntrs(&rdtgroup_default); + + ret = mkdir_mondata_all(rdtgroup_default.kn, + &rdtgroup_default, &kn_mondata); + if (ret < 0) + goto out_mongrp; + rdtgroup_default.mon.mon_data_kn = kn_mondata; + } + + ret = rdt_pseudo_lock_init(); + if (ret) + goto out_mondata; + + if (resctrl_arch_alloc_capable()) + resctrl_arch_enable_alloc(); + if (resctrl_arch_mon_capable()) + resctrl_arch_enable_mon(); + + if (resctrl_arch_alloc_capable() || resctrl_arch_mon_capable()) + resctrl_mounted = true; + + if (resctrl_is_mbm_enabled()) { + r = resctrl_arch_get_resource(RDT_RESOURCE_L3); + list_for_each_entry_rcu(dom, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) + mbm_setup_overflow_handler(dom, MBM_OVERFLOW_INTERVAL, + RESCTRL_PICK_ANY_CPU); + } + + /* + * Ensure root remains accessible after mutex is unlocked so that + * kernfs_kill_sb() can run safely if called by kernfs_get_tree()'s + * failure path after creating a superblock but before taking reference + * on root kn (for example, if unable to get inode for root kn). + */ + kernfs_get(rdtgroup_default.kn); + + /* + * Make backup of the current root kn being created to be used in + * kernfs_put(). The additional reference taken above will prevent the + * kn from being freed before kernfs_kill_sb() can run but + * rdtgroup_default.kn may be set to NULL via rdtgroup_destroy_root() + * and its backing root (rdt_root) could be overwritten before + * kernfs_put() can run. + */ + rdt_root_kn = rdtgroup_default.kn; + + rdt_last_cmd_clear(); + mutex_unlock(&rdtgroup_mutex); + cpus_read_unlock(); + + ret = kernfs_get_tree(fc); + /* + * resctrl can only be mounted once, new superblock only expected + * to be created once. + */ + if (!ctx->kfc.new_sb_created) + resctrl_unmount(); + kernfs_put(rdt_root_kn); + return ret; + +out_mondata: + if (resctrl_arch_mon_capable()) + kernfs_remove(kn_mondata); +out_mongrp: + if (resctrl_arch_mon_capable()) { + mon_put_kn_priv(); + rdtgroup_unassign_cntrs(&rdtgroup_default); + kernfs_remove(kn_mongrp); + } +out_info: + kernfs_remove(kn_info); +out_closid_exit: + closid_exit(); +out_schemata_free: + schemata_list_destroy(); + rdt_disable_ctx(); +out_root: + rdtgroup_destroy_root(); +out: + mutex_unlock(&rdtgroup_mutex); + cpus_read_unlock(); + return ret; +} + +enum rdt_param { + Opt_cdp, + Opt_cdpl2, + Opt_mba_mbps, + Opt_debug, + nr__rdt_params +}; + +static const struct fs_parameter_spec rdt_fs_parameters[] = { + fsparam_flag("cdp", Opt_cdp), + fsparam_flag("cdpl2", Opt_cdpl2), + fsparam_flag("mba_MBps", Opt_mba_mbps), + fsparam_flag("debug", Opt_debug), + {} +}; + +static int rdt_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct rdt_fs_context *ctx = rdt_fc2context(fc); + struct fs_parse_result result; + const char *msg; + int opt; + + opt = fs_parse(fc, rdt_fs_parameters, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_cdp: + ctx->enable_cdpl3 = true; + return 0; + case Opt_cdpl2: + ctx->enable_cdpl2 = true; + return 0; + case Opt_mba_mbps: + msg = "mba_MBps requires MBM (mbm_event mode not supported) and linear scale MBA at L3 scope"; + if (!supports_mba_mbps()) + return invalfc(fc, msg); + ctx->enable_mba_mbps = true; + return 0; + case Opt_debug: + ctx->enable_debug = true; + return 0; + } + + return -EINVAL; +} + +static void rdt_fs_context_free(struct fs_context *fc) +{ + struct rdt_fs_context *ctx = rdt_fc2context(fc); + + kernfs_free_fs_context(fc); + kfree(ctx); +} + +static const struct fs_context_operations rdt_fs_context_ops = { + .free = rdt_fs_context_free, + .parse_param = rdt_parse_param, + .get_tree = rdt_get_tree, +}; + +static int rdt_init_fs_context(struct fs_context *fc) +{ + struct rdt_fs_context *ctx; + + ctx = kzalloc_obj(*ctx); + if (!ctx) + return -ENOMEM; + + ctx->kfc.magic = RDTGROUP_SUPER_MAGIC; + fc->fs_private = &ctx->kfc; + fc->ops = &rdt_fs_context_ops; + put_user_ns(fc->user_ns); + fc->user_ns = get_user_ns(&init_user_ns); + fc->global = true; + return 0; +} + +static void rdt_kill_sb(struct super_block *sb) +{ + resctrl_unmount(); + kernfs_kill_sb(sb); +} + static struct file_system_type rdt_fs_type = { .name = "resctrl", .init_fs_context = rdt_init_fs_context, @@ -3421,7 +3636,7 @@ static int mkdir_mondata_subdir_alldom(struct kernfs_node *parent_kn, /* Walking r->domains, ensure it can't race with cpuhp */ lockdep_assert_cpus_held(); - list_for_each_entry(hdr, &r->mon_domains, list) { + list_for_each_entry_rcu(hdr, &r->mon_domains, list, lockdep_is_cpus_held()) { ret = mkdir_mondata_subdir(parent_kn, hdr, r, prgrp); if (ret) return ret; @@ -3496,6 +3711,8 @@ out_destroy: * resource group is initialized. The user can follow this with a * modification to the CBM if the default does not satisfy the * requirements. + * + * Return: A CBM that is valid for resource @r. */ static u32 cbm_ensure_valid(u32 _val, struct rdt_resource *r) { @@ -3604,7 +3821,7 @@ int rdtgroup_init_cat(struct resctrl_schema *s, u32 closid) struct rdt_ctrl_domain *d; int ret; - list_for_each_entry(d, &s->res->ctrl_domains, hdr.list) { + list_for_each_entry_rcu(d, &s->res->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { ret = __init_one_rdt_domain(d, s, closid); if (ret < 0) return ret; @@ -3619,7 +3836,7 @@ static void rdtgroup_init_mba(struct rdt_resource *r, u32 closid) struct resctrl_staged_config *cfg; struct rdt_ctrl_domain *d; - list_for_each_entry(d, &r->ctrl_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { if (is_mba_sc(r)) { d->mbps_val[closid] = MBA_MAX_MBPS; continue; @@ -3733,8 +3950,6 @@ static int mkdir_rdt_prepare(struct kernfs_node *parent_kn, goto out_unlock; } - rdt_last_cmd_clear(); - /* * Check that the parent directory for a monitor group is a "mon_groups" * directory. @@ -4258,6 +4473,7 @@ static int rdtgroup_setup_root(struct rdt_fs_context *ctx) ctx->kfc.root = rdt_root; rdtgroup_default.kn = kernfs_root_to_node(rdt_root); + rdtgroup_default.flags = 0; return 0; } @@ -4298,6 +4514,29 @@ static void domain_destroy_l3_mon_state(struct rdt_l3_mon_domain *d) void resctrl_offline_ctrl_domain(struct rdt_resource *r, struct rdt_ctrl_domain *d) { + /* + * mbm_handle_overflow() may dereference this ctrl domain via + * update_mba_bw()->get_sc_ctrl_domain_from_cpu(). The architecture has + * unlinked the domain from the RCU list and waited a grace period, so + * no new worker iteration can find it; drain any worker that already + * holds a pointer to it before the architecture frees the domain. + * + * Software controller is enabled/disabled on mount/unmount with + * cpus_read_lock() held. Running here with cpus_write_lock() so + * there are no concurrent changes to software controller status. + */ + if (r->rid == RDT_RESOURCE_MBA && is_mba_sc(r)) { + struct rdt_resource *l3 = resctrl_arch_get_resource(RDT_RESOURCE_L3); + struct rdt_l3_mon_domain *mon_d; + + list_for_each_entry_rcu(mon_d, &l3->mon_domains, hdr.list, lockdep_is_cpus_held()) { + if (mon_d->hdr.id == d->hdr.id) { + cancel_delayed_work_sync(&mon_d->mbm_over); + break; + } + } + } + mutex_lock(&rdtgroup_mutex); if (supports_mba_mbps() && r->rid == RDT_RESOURCE_MBA) @@ -4310,6 +4549,24 @@ void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_domain_hdr *h { struct rdt_l3_mon_domain *d; + /* + * Called by architecture under CPU hotplug lock as it prepares to remove + * the domain which is guaranteed to be accessible here. + * The domain has been unlinked from the RCU list and a grace period + * has elapsed, so no new worker can be scheduled. Drain any worker that + * is in flight or pending before letting architecture proceed to free + * the domain that has the workers' struct delayed_work embedded. + * Do so before taking rdtgroup_mutex since the workers also acquire it. + */ + if (r->rid == RDT_RESOURCE_L3 && + domain_header_is_valid(hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3)) { + d = container_of(hdr, struct rdt_l3_mon_domain, hdr); + if (resctrl_is_mbm_enabled()) + cancel_delayed_work_sync(&d->mbm_over); + if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID)) + cancel_delayed_work_sync(&d->cqm_limbo); + } + mutex_lock(&rdtgroup_mutex); /* @@ -4326,8 +4583,6 @@ void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_domain_hdr *h goto out_unlock; d = container_of(hdr, struct rdt_l3_mon_domain, hdr); - if (resctrl_is_mbm_enabled()) - cancel_delayed_work(&d->mbm_over); if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID) && has_busy_rmid(d)) { /* * When a package is going down, forcefully @@ -4338,7 +4593,6 @@ void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_domain_hdr *h * package never comes back. */ __check_limbo(d, true); - cancel_delayed_work(&d->cqm_limbo); } domain_destroy_l3_mon_state(d); @@ -4490,7 +4744,7 @@ static struct rdt_l3_mon_domain *get_mon_domain_from_cpu(int cpu, lockdep_assert_cpus_held(); - list_for_each_entry(d, &r->mon_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { /* Find the domain that contains this CPU */ if (cpumask_test_cpu(cpu, &d->hdr.cpu_mask)) return d; @@ -4519,12 +4773,16 @@ void resctrl_offline_cpu(unsigned int cpu) d = get_mon_domain_from_cpu(cpu, l3); if (d) { if (resctrl_is_mbm_enabled() && cpu == d->mbm_work_cpu) { - cancel_delayed_work(&d->mbm_over); + mutex_unlock(&rdtgroup_mutex); + cancel_delayed_work_sync(&d->mbm_over); + mutex_lock(&rdtgroup_mutex); mbm_setup_overflow_handler(d, 0, cpu); } if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID) && cpu == d->cqm_work_cpu && has_busy_rmid(d)) { - cancel_delayed_work(&d->cqm_limbo); + mutex_unlock(&rdtgroup_mutex); + cancel_delayed_work_sync(&d->cqm_limbo); + mutex_lock(&rdtgroup_mutex); cqm_setup_limbo_handler(d, 0, cpu); } } |