diff options
Diffstat (limited to 'kernel')
224 files changed, 22184 insertions, 6961 deletions
diff --git a/kernel/Kconfig.kexec b/kernel/Kconfig.kexec index 15632358bcf7..a97ed9605602 100644 --- a/kernel/Kconfig.kexec +++ b/kernel/Kconfig.kexec @@ -167,7 +167,7 @@ config CRASH_MAX_MEMORY_RANGES memory regions that the elfcorehdr buffer/segment can accommodate. These regions are obtained via walk_system_ram_res(); eg. the 'System RAM' entries in /proc/iomem. - This value is combined with NR_CPUS_DEFAULT and multiplied by + This value is combined with NR_CPUS and multiplied by sizeof(Elf64_Phdr) to determine the final elfcorehdr memory buffer/ segment size. The value 8192, for example, covers a (sparsely populated) 1TiB system diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index f294dad43bd7..edc067a0c422 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -132,10 +132,9 @@ config PREEMPTION config PREEMPT_DYNAMIC bool "Preemption behaviour defined on boot" - depends on HAVE_PREEMPT_DYNAMIC - select JUMP_LABEL if HAVE_PREEMPT_DYNAMIC_KEY + depends on ARCH_HAS_PREEMPT_LAZY select PREEMPT_BUILD - default y if HAVE_PREEMPT_DYNAMIC_CALL + default y help This option allows to define the preemption model on the kernel command line parameter and thus override the default preemption @@ -145,9 +144,7 @@ config PREEMPT_DYNAMIC provide a pre-built kernel binary to reduce the number of kernel flavors they offer while still offering different usecases. - The runtime overhead is negligible with HAVE_STATIC_CALL_INLINE enabled - but if runtime patching is not available for the specific architecture - then the potential overhead should be considered. + The runtime overhead is negligible. Interesting if you want the same pre-built kernel should be used for both Server and Desktop workloads. @@ -197,3 +194,7 @@ config SCHED_CLASS_EXT For more information: Documentation/scheduler/sched-ext.rst https://github.com/sched-ext/scx + +config PREFERRED_CPU + bool + depends on SMP && PARAVIRT diff --git a/kernel/Makefile b/kernel/Makefile index 1e1a31673577..4ff92e24680c 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -141,6 +141,7 @@ obj-$(CONFIG_WATCH_QUEUE) += watch_queue.o obj-$(CONFIG_RESOURCE_KUNIT_TEST) += resource_kunit.o obj-$(CONFIG_SYSCTL_KUNIT_TEST) += sysctl-test.o +obj-$(CONFIG_USER_NS_MAP_KUNIT_TEST) += tests/user_ns_map_kunit.o CFLAGS_kstack_erase.o += $(DISABLE_KSTACK_ERASE) CFLAGS_kstack_erase.o += $(call cc-option,-mgeneral-regs-only) @@ -153,9 +154,13 @@ obj-$(CONFIG_SCF_TORTURE_TEST) += scftorture.o $(obj)/configs.o: $(obj)/config_data.gz +# .config is small, so it is compressed with gzip rather than the image compressor. +quiet_cmd_gzip_config = GZIP $@ + cmd_gzip_config = cat $(real-prereqs) | $(KGZIP) -n -f -9 > $@ + targets += config_data config_data.gz $(obj)/config_data.gz: $(obj)/config_data FORCE - $(call if_changed,gzip) + $(call if_changed,gzip_config) filechk_cat = cat $< diff --git a/kernel/audit.c b/kernel/audit.c index 9412af9144bc..631d6d4a23cf 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -871,7 +871,7 @@ static void kauditd_send_multicast_skb(struct sk_buff *skb) if (!copy) return; nlh = nlmsg_hdr(copy); - nlh->nlmsg_len = skb->len; + nlh->nlmsg_len = copy->len; nlmsg_multicast(sock, copy, 0, AUDIT_NLGRP_READLOG, GFP_KERNEL); } diff --git a/kernel/audit.h b/kernel/audit.h index 92d5e723d570..7640d2c0fba4 100644 --- a/kernel/audit.h +++ b/kernel/audit.h @@ -53,15 +53,18 @@ struct audit_entry { struct audit_krule rule; }; +struct audit_file_caps { + kernel_cap_t permitted; + kernel_cap_t inheritable; + unsigned int fE; /* effective bit of file cap */ + kuid_t rootid; +}; + struct audit_cap_data { kernel_cap_t permitted; kernel_cap_t inheritable; - union { - unsigned int fE; /* effective bit of file cap */ - kernel_cap_t effective; /* effective set of process */ - }; + kernel_cap_t effective; kernel_cap_t ambient; - kuid_t rootid; }; /* When fs/namei.c:getname() is called, we store the pointer in name and bump @@ -83,7 +86,7 @@ struct audit_names { kgid_t gid; dev_t rdev; struct lsm_prop oprop; - struct audit_cap_data fcap; + struct audit_file_caps fcap; unsigned int fcap_ver; unsigned char type; /* record type */ /* @@ -272,6 +275,9 @@ extern void audit_put_tty(struct tty_struct *tty); /* audit watch/mark/tree functions */ extern unsigned int audit_serial(void); #ifdef CONFIG_AUDITSYSCALL +void audit_rule_account(const struct audit_krule *rule); +void audit_rule_unaccount(const struct audit_krule *rule); + extern int auditsc_get_stamp(struct audit_context *ctx, struct audit_stamp *stamp); @@ -315,6 +321,8 @@ extern void audit_filter_inodes(struct task_struct *tsk, struct audit_context *ctx); extern struct list_head *audit_killed_trees(void); #else /* CONFIG_AUDITSYSCALL */ +#define audit_rule_account(...) do { } while (0) +#define audit_rule_unaccount(...) do { } while (0) #define auditsc_get_stamp(c, s) 0 #define audit_put_watch(w) do { } while (0) #define audit_get_watch(w) do { } while (0) diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c index 1ed19b775912..e83b9fef048f 100644 --- a/kernel/audit_tree.c +++ b/kernel/audit_tree.c @@ -545,22 +545,39 @@ static void kill_rules(struct audit_context *context, struct audit_tree *tree) { struct audit_krule *rule, *next; struct audit_entry *entry; + bool need_sync = false; list_for_each_entry_safe(rule, next, &tree->rules, rlist) { entry = container_of(rule, struct audit_entry, rule); - list_del_init(&rule->rlist); if (rule->tree) { /* not a half-baked one */ audit_tree_log_remove_rule(context, rule); - if (entry->rule.exe) - audit_remove_mark(entry->rule.exe); rule->tree = NULL; list_del_rcu(&entry->list); list_del(&entry->rule.list); - call_rcu(&entry->rcu, audit_free_rule_rcu); + if (entry->rule.exe) + need_sync = true; + audit_rule_unaccount(rule); + } else { + list_del_init(&rule->rlist); } } + + if (list_empty(&tree->rules)) + return; + + if (need_sync) + synchronize_rcu(); + + list_for_each_entry_safe(rule, next, &tree->rules, rlist) { + entry = container_of(rule, struct audit_entry, rule); + + list_del_init(&rule->rlist); + if (entry->rule.exe) + audit_remove_mark(entry->rule.exe); + call_rcu(&entry->rcu, audit_free_rule_rcu); + } } /* diff --git a/kernel/audit_watch.c b/kernel/audit_watch.c index 4ac8a91e9ba8..28fab822ca0c 100644 --- a/kernel/audit_watch.c +++ b/kernel/audit_watch.c @@ -284,6 +284,7 @@ static void audit_update_watch(struct audit_parent *parent, nentry = audit_dupe_rule(&oentry->rule, ctx); if (IS_ERR(nentry)) { list_del(&oentry->rule.list); + audit_rule_unaccount(r); audit_panic("error updating watch, removing"); } else { int h = audit_hash_ino(ino); @@ -336,6 +337,7 @@ static void audit_remove_parent_watches(struct audit_parent *parent) list_del(&r->rlist); list_del(&r->list); list_del_rcu(&e->list); + audit_rule_unaccount(r); call_rcu(&e->rcu, audit_free_rule_rcu); } audit_remove_watch(w); diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c index f52645625214..cf91ba6f3617 100644 --- a/kernel/auditfilter.c +++ b/kernel/auditfilter.c @@ -196,7 +196,7 @@ int audit_match_class(int class, unsigned int syscall) } #ifdef CONFIG_AUDITSYSCALL -static inline int audit_match_class_bits(int class, u32 *mask) +static inline int audit_match_class_bits(int class, const u32 *mask) { int i; @@ -208,30 +208,62 @@ static inline int audit_match_class_bits(int class, u32 *mask) return 1; } -static int audit_match_signal(struct audit_entry *entry) +static int audit_match_signal(const struct audit_krule *rule) { - struct audit_field *arch = entry->rule.arch_f; + struct audit_field *arch = rule->arch_f; if (!arch) { /* When arch is unspecified, we must check both masks on biarch * as syscall number alone is ambiguous. */ return (audit_match_class_bits(AUDIT_CLASS_SIGNAL, - entry->rule.mask) && + rule->mask) && audit_match_class_bits(AUDIT_CLASS_SIGNAL_32, - entry->rule.mask)); + rule->mask)); } switch (audit_classify_arch(arch->val)) { case 0: /* native */ return (audit_match_class_bits(AUDIT_CLASS_SIGNAL, - entry->rule.mask)); + rule->mask)); case 1: /* 32bit on biarch */ return (audit_match_class_bits(AUDIT_CLASS_SIGNAL_32, - entry->rule.mask)); + rule->mask)); default: return 1; } } + +static bool audit_rule_counts_syscalls(const struct audit_krule *rule) +{ + switch (rule->listnr) { + case AUDIT_FILTER_USER: + case AUDIT_FILTER_EXCLUDE: + case AUDIT_FILTER_FS: + return false; + default: + return true; + } +} + +void audit_rule_account(const struct audit_krule *rule) +{ + lockdep_assert_held(&audit_filter_mutex); + + if (audit_rule_counts_syscalls(rule)) + audit_n_rules++; + if (!audit_match_signal(rule)) + audit_signals++; +} + +void audit_rule_unaccount(const struct audit_krule *rule) +{ + lockdep_assert_held(&audit_filter_mutex); + + if (audit_rule_counts_syscalls(rule)) + audit_n_rules--; + if (!audit_match_signal(rule)) + audit_signals--; +} #endif /* Common user-space to kernel rule translation. */ @@ -943,17 +975,6 @@ static inline int audit_add_rule(struct audit_entry *entry) struct audit_tree *tree = entry->rule.tree; struct list_head *list; int err = 0; -#ifdef CONFIG_AUDITSYSCALL - int dont_count = 0; - - /* If any of these, don't count towards total */ - switch (entry->rule.listnr) { - case AUDIT_FILTER_USER: - case AUDIT_FILTER_EXCLUDE: - case AUDIT_FILTER_FS: - dont_count = 1; - } -#endif mutex_lock(&audit_filter_mutex); e = audit_find_rule(entry, &list); @@ -1007,13 +1028,7 @@ static inline int audit_add_rule(struct audit_entry *entry) &audit_rules_list[entry->rule.listnr]); list_add_tail_rcu(&entry->list, list); } -#ifdef CONFIG_AUDITSYSCALL - if (!dont_count) - audit_n_rules++; - - if (!audit_match_signal(entry)) - audit_signals++; -#endif + audit_rule_account(&entry->rule); mutex_unlock(&audit_filter_mutex); return err; @@ -1025,17 +1040,6 @@ int audit_del_rule(struct audit_entry *entry) struct audit_entry *e; struct list_head *list; int ret = 0; -#ifdef CONFIG_AUDITSYSCALL - int dont_count = 0; - - /* If any of these, don't count towards total */ - switch (entry->rule.listnr) { - case AUDIT_FILTER_USER: - case AUDIT_FILTER_EXCLUDE: - case AUDIT_FILTER_FS: - dont_count = 1; - } -#endif mutex_lock(&audit_filter_mutex); e = audit_find_rule(entry, &list); @@ -1057,14 +1061,7 @@ int audit_del_rule(struct audit_entry *entry) if (e->rule.exe) audit_remove_mark_rule(&e->rule); -#ifdef CONFIG_AUDITSYSCALL - if (!dont_count) - audit_n_rules--; - - if (!audit_match_signal(entry)) - audit_signals--; -#endif - + audit_rule_unaccount(&e->rule); call_rcu(&e->rcu, audit_free_rule_rcu); out: @@ -1427,6 +1424,7 @@ static int update_lsm_rule(struct audit_krule *r) list_del(&r->rlist); list_del_rcu(&entry->list); list_del(&r->list); + audit_rule_unaccount(r); } else { if (r->watch || r->tree) list_replace_init(&r->rlist, &nentry->rule.rlist); diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 2b9ce0b52511..552ba9f4fb0a 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -107,7 +107,7 @@ struct audit_aux_data_pids { struct audit_aux_data_bprm_fcaps { struct audit_aux_data d; - struct audit_cap_data fcap; + struct audit_file_caps fcap; unsigned int fcap_ver; struct audit_cap_data old_pcap; struct audit_cap_data new_pcap; @@ -459,7 +459,7 @@ static int audit_field_compare(struct task_struct *tsk, * * If task_creation is true, this is an explicit indication that we are * filtering a task rule at task creation time. This and tsk == current are - * the only situations where tsk->cred may be accessed without an rcu read lock. + * the only situations where tsk->cred may be accessed. */ static int audit_filter_rules(struct task_struct *tsk, struct audit_krule *rule, @@ -476,7 +476,8 @@ static int audit_filter_rules(struct task_struct *tsk, if (ctx && rule->prio <= ctx->prio) return 0; - cred = rcu_dereference_check(tsk->cred, tsk == current || task_creation); + WARN_ON(tsk != current && !task_creation); + cred = tsk->cred; for (i = 0; i < rule->field_count; i++) { struct audit_field *f = &rule->fields[i]; @@ -2601,6 +2602,8 @@ void __audit_bprm(struct linux_binprm *bprm) { struct audit_context *context = audit_context(); + /* clear proctitle in audit context to allow replacement */ + audit_proctitle_free(context); context->type = AUDIT_EXECVE; context->execve.argc = bprm->argc; } @@ -2883,6 +2886,19 @@ void __audit_log_nfcfg(const char *name, u8 af, unsigned int nentries, } EXPORT_SYMBOL_GPL(__audit_log_nfcfg); +void __audit_log_fsopen(const char *fs_name) +{ + struct audit_buffer *ab; + + ab = audit_log_start(audit_context(), GFP_KERNEL, AUDIT_FSOPEN); + if (!ab) + return; + + audit_log_format(ab, "fs_name="); + audit_log_untrustedstring(ab, fs_name); + audit_log_end(ab); +} + static void audit_log_task(struct audit_buffer *ab) { kuid_t auid, uid; diff --git a/kernel/bpf/Kconfig b/kernel/bpf/Kconfig index eb3de35734f0..98493e32db2a 100644 --- a/kernel/bpf/Kconfig +++ b/kernel/bpf/Kconfig @@ -17,6 +17,10 @@ config HAVE_CBPF_JIT config HAVE_EBPF_JIT bool +# KASAN support for JIT compiler +config HAVE_EBPF_JIT_KASAN + bool + # Used by archs to tell that they want the BPF JIT compiler enabled by # default for kernels that were compiled with BPF JIT support. config ARCH_WANT_DEFAULT_BPF_JIT @@ -87,6 +91,15 @@ config BPF_UNPRIV_DEFAULT_OFF If you are unsure how to answer this question, answer Y. +config BPF_CRYPTO + def_bool y + depends on BPF_SYSCALL + select CRYPTO_LIB_AES_CBC + select CRYPTO_LIB_AES_ECB + help + Provide the kfuncs needed for BPF programs to encrypt and decrypt + data. The supported algorithms are AES-CBC and AES-ECB. + source "kernel/bpf/preload/Kconfig" config BPF_LSM @@ -101,4 +114,17 @@ config BPF_LSM If you are unsure how to answer this question, answer N. +config BPF_JIT_KASAN + bool + depends on HAVE_EBPF_JIT_KASAN + depends on KASAN_GENERIC + depends on KASAN_VMALLOC + depends on BPF_JIT + default y + help + Makes JIT compiler insert generic outline KASAN checks in BPF + programs when they are inserted in the kernel. This feature is + automatically enabled if the needed set of KASAN and BPF + configuration options is enabled. + endmenu # "BPF subsystem" diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 90255d80e5be..269810b128c4 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -6,7 +6,7 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse endif CFLAGS_core.o += -Wno-override-init $(cflags-nogcse-yy) -obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o diagnostics.o +obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o diagnostics.o loops.o scev.o obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o @@ -27,6 +27,9 @@ obj-$(CONFIG_BPF_SYSCALL) += offload.o obj-$(CONFIG_BPF_SYSCALL) += net_namespace.o obj-$(CONFIG_BPF_SYSCALL) += tcx.o endif +ifeq ($(CONFIG_KEYS),y) +obj-$(CONFIG_BPF_SYSCALL) += keys.o +endif ifeq ($(CONFIG_PERF_EVENTS),y) obj-$(CONFIG_BPF_SYSCALL) += stackmap.o endif @@ -39,6 +42,7 @@ obj-$(CONFIG_BPF_SYSCALL) += reuseport_array.o endif ifeq ($(CONFIG_SYSFS),y) obj-$(CONFIG_DEBUG_INFO_BTF) += sysfs_btf.o +obj-$(CONFIG_DEBUG_INFO_BTF_INLINE) += btf_vmlinux_inline.o endif ifeq ($(CONFIG_BPF_JIT),y) obj-$(CONFIG_BPF_SYSCALL) += bpf_struct_ops.o @@ -55,9 +59,7 @@ obj-$(CONFIG_BPF_SYSCALL) += cpumask.o # semantics within pahole are revisited accordingly. obj-${CONFIG_BPF_LSM} += bpf_lsm_proto.o bpf_lsm.o endif -ifneq ($(CONFIG_CRYPTO),) -obj-$(CONFIG_BPF_SYSCALL) += crypto.o -endif +obj-$(CONFIG_BPF_CRYPTO) += crypto.o obj-$(CONFIG_BPF_PRELOAD) += preload/ obj-$(CONFIG_BPF_SYSCALL) += relo_core.o diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 7b6847200b43..868589f07330 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -146,7 +146,7 @@ static long compute_pgoff(struct bpf_arena *arena, long uaddr) struct apply_range_data { struct bpf_arena *arena; - struct page **pages; + struct llist_head *pages; int i; }; @@ -158,13 +158,17 @@ struct clear_range_data { static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) { struct apply_range_data *d = data; + struct llist_node *node; struct page *page; pte_t pteval; if (!data) return 0; - page = d->pages[d->i]; + node = READ_ONCE(d->pages->first); + if (WARN_ON_ONCE(!node)) + return -EINVAL; + page = llist_entry(node, struct page, pcp_llist); /* paranoia, similar to vmap_pages_pte_range() */ if (WARN_ON_ONCE(!pfn_valid(page_to_pfn(page)))) return -EINVAL; @@ -197,6 +201,7 @@ static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) return -EBUSY; set_pte_at(&init_mm, addr, pte, pteval); #endif + WARN_ON_ONCE(llist_del_first(d->pages) != node); d->i++; WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages + 1); return 0; @@ -312,8 +317,8 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr) INIT_WORK(&arena->free_work, arena_free_worker); bpf_map_init_from_attr(&arena->map, attr); - err = bpf_map_alloc_pages(&arena->map, NUMA_NO_NODE, 1, &arena->scratch_page); - if (err) + arena->scratch_page = bpf_alloc_page(NUMA_NO_NODE, true); + if (!arena->scratch_page) goto err_free_arena; range_tree_init(&arena->rt); @@ -481,7 +486,10 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) struct bpf_map *map = vmf->vma->vm_file->private_data; struct bpf_arena *arena = container_of(map, struct bpf_arena, map); struct mem_cgroup *new_memcg, *old_memcg; - struct page *page; + LLIST_HEAD(pages); + struct page *page, *new_page = NULL; + struct apply_range_data data; + vm_fault_t fault_ret; long kbase, kaddr; unsigned long flags; int ret; @@ -489,59 +497,123 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) kbase = bpf_arena_get_kern_vm_start(arena); kaddr = kbase + (u32)(vmf->address); - if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) + page = vmalloc_to_page((void *)kaddr); + if (!page && !(arena->map.map_flags & BPF_F_SEGV_ON_FAULT)) { + /* + * Preallocate outside the lock so the allocation can reclaim; + * __GFP_RETRY_MAYFAIL keeps the OOM killer out of it. + */ + bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); + new_page = alloc_pages_node(map->numa_node, + GFP_KERNEL | __GFP_ZERO | + __GFP_ACCOUNT | __GFP_NOWARN | + __GFP_RETRY_MAYFAIL, 0); + bpf_map_memcg_exit(old_memcg, new_memcg); + } + + if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) { /* * A failed lock means a possible deadlock was detected. Don't * return VM_FAULT_RETRY: this handler never took mmap_lock, but * the fault path would re-take it on retry and deadlock. Fail. */ + if (new_page) + free_pages_nolock(new_page, 0); return VM_FAULT_SIGBUS; + } page = vmalloc_to_page((void *)kaddr); if (page) { - if (page == arena->scratch_page) + if (page == arena->scratch_page) { /* BPF triggered scratch here; don't lazy-alloc over it */ - goto out_sigsegv; + fault_ret = (arena->map.map_flags & BPF_F_SEGV_ON_FAULT) + ? VM_FAULT_SIGSEGV : VM_FAULT_SIGBUS; + goto out_err_locked; + } /* already have a page vmap-ed */ goto out; } + if (arena->map.map_flags & BPF_F_SEGV_ON_FAULT) { + /* + * User space requested to segfault when page is not allocated + * by bpf prog + */ + fault_ret = VM_FAULT_SIGSEGV; + goto out_err_locked; + } + + if (is_range_tree_set(&arena->rt, vmf->pgoff, 1)) { + /* Allocated without a page: reserved or being freed */ + fault_ret = VM_FAULT_SIGBUS; + goto out_err_locked; + } + bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); - if (arena->map.map_flags & BPF_F_SEGV_ON_FAULT) - /* User space requested to segfault when page is not allocated by bpf prog */ - goto out_sigsegv_memcg; + if (!new_page) { + /* + * The probed page was freed meanwhile or preallocation failed; + * try the non-blocking allocator, we cannot sleep here. + */ + new_page = bpf_alloc_page(map->numa_node, false); + if (!new_page) { + fault_ret = VM_FAULT_SIGBUS; + goto out_err_locked_memcg; + } + } ret = range_tree_clear(&arena->rt, vmf->pgoff, 1); - if (ret) - goto out_sigsegv_memcg; - - struct apply_range_data data = { .arena = arena, .pages = &page, .i = 0 }; - /* Account into memcg of the process that created bpf_arena */ - ret = bpf_map_alloc_pages(map, NUMA_NO_NODE, 1, &page); if (ret) { - range_tree_set(&arena->rt, vmf->pgoff, 1); - goto out_sigsegv_memcg; + fault_ret = VM_FAULT_SIGBUS; + goto out_err_locked_memcg; } + llist_add(&new_page->pcp_llist, &pages); + data.arena = arena; + data.pages = &pages; + data.i = 0; ret = apply_to_page_range(&init_mm, kaddr, PAGE_SIZE, apply_range_set_cb, &data); if (ret) { + llist_del_first(&pages); range_tree_set(&arena->rt, vmf->pgoff, 1); - free_pages_nolock(page, 0); - goto out_sigsegv_memcg; + fault_ret = VM_FAULT_SIGBUS; + goto out_err_locked_memcg; } flush_vmap_cache(kaddr, PAGE_SIZE); bpf_map_memcg_exit(old_memcg, new_memcg); + /* new_page was consumed */ + page = new_page; + new_page = NULL; out: page_ref_add(page, 1); raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - vmf->page = page; - return 0; -out_sigsegv_memcg: + if (new_page) + free_pages_nolock(new_page, 0); + + /* + * A free clears the kernel pte and then takes the page lock before it + * zaps user vma-s. Either it waits for this fault and zaps its pte, or + * the page is gone from here. + */ + lock_page(page); + if (vmalloc_to_page((void *)kaddr) == page && !PageHWPoison(page)) { + vmf->page = page; + return VM_FAULT_LOCKED; + } + /* The page was freed meanwhile. Let the fault start over. */ + fault_ret = PageHWPoison(page) ? VM_FAULT_HWPOISON : VM_FAULT_NOPAGE; + unlock_page(page); + put_page(page); + return fault_ret; + +out_err_locked_memcg: bpf_map_memcg_exit(old_memcg, new_memcg); -out_sigsegv: +out_err_locked: raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - return VM_FAULT_SIGSEGV; + if (new_page) + free_pages_nolock(new_page, 0); + return fault_ret; } static const struct vm_operations_struct arena_vm_ops = { @@ -661,6 +733,27 @@ static u64 clear_lo32(u64 val) return val & ~(u64)~0U; } +static int arena_adjust_tree(struct bpf_arena *arena, long uaddr, long page_cnt, long *pgoff) +{ + int ret; + + /* Special case where user is requesting specific range. */ + if (uaddr) { + ret = is_range_tree_set(&arena->rt, *pgoff, page_cnt); + if (ret) + return ret; + return range_tree_clear(&arena->rt, *pgoff, page_cnt); + } + + ret = range_tree_find(&arena->rt, page_cnt); + if (ret < 0) + return ret; + + *pgoff = ret; + + return range_tree_clear(&arena->rt, *pgoff, page_cnt); +} + /* * Allocate pages and vmap them into kernel vmalloc area. * Later the pages will be mmaped into user space vma. @@ -673,13 +766,13 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt u64 kern_vm_start = bpf_arena_get_kern_vm_start(arena); struct mem_cgroup *new_memcg, *old_memcg; struct apply_range_data data; - struct page **pages = NULL; - long remaining, mapped = 0; - long alloc_pages; + LLIST_HEAD(pages); + long mapped = 0; unsigned long flags; long pgoff = 0; u32 uaddr32; - int ret, i; + long addr = 0; + int ret; if (node_id != NUMA_NO_NODE && ((unsigned int)node_id >= nr_node_ids || !node_online(node_id))) @@ -696,89 +789,63 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt /* requested address will be outside of user VMA */ return 0; } - bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); - /* Cap allocation size to KMALLOC_MAX_CACHE_SIZE so kmalloc_nolock() can succeed. */ - alloc_pages = min(page_cnt, KMALLOC_MAX_CACHE_SIZE / sizeof(struct page *)); - pages = kmalloc_nolock(alloc_pages * sizeof(struct page *), __GFP_ACCOUNT, NUMA_NO_NODE); - if (!pages) { - bpf_map_memcg_exit(old_memcg, new_memcg); - return 0; - } + + ret = bpf_alloc_pages(node_id, page_cnt, &pages, sleepable); + if (ret) + goto out_memcg; + data.arena = arena; - data.pages = pages; + data.pages = &pages; + data.i = 0; if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) goto out_free_pages; - if (uaddr) { - ret = is_range_tree_set(&arena->rt, pgoff, page_cnt); - if (ret) - goto out_unlock_free_pages; - ret = range_tree_clear(&arena->rt, pgoff, page_cnt); - } else { - ret = pgoff = range_tree_find(&arena->rt, page_cnt); - if (pgoff >= 0) - ret = range_tree_clear(&arena->rt, pgoff, page_cnt); + ret = arena_adjust_tree(arena, uaddr, page_cnt, &pgoff); + if (ret) { + raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + goto out_free_pages; } - if (ret) - goto out_unlock_free_pages; - remaining = page_cnt; uaddr32 = (u32)(arena->user_vm_start + pgoff * PAGE_SIZE); - while (remaining) { - long this_batch = min(remaining, alloc_pages); - - /* zeroing is needed, since alloc_pages_bulk() only fills in non-zero entries */ - memset(pages, 0, this_batch * sizeof(struct page *)); + /* + * Earlier checks made sure that uaddr32 + page_cnt * PAGE_SIZE - 1 + * will not overflow 32-bit. Lower 32-bit need to represent + * contiguous user address range. + * Map these pages at kern_vm_start base. + * kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE - 1 can overflow + * lower 32-bit and it's ok. + */ + ret = apply_to_page_range(&init_mm, kern_vm_start + uaddr32, + page_cnt << PAGE_SHIFT, apply_range_set_cb, &data); + mapped = data.i; + if (ret) + goto out_unmap; - ret = bpf_map_alloc_pages(&arena->map, node_id, this_batch, pages); - if (ret) - goto out; + flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT); + raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - /* - * Earlier checks made sure that uaddr32 + page_cnt * PAGE_SIZE - 1 - * will not overflow 32-bit. Lower 32-bit need to represent - * contiguous user address range. - * Map these pages at kern_vm_start base. - * kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE - 1 can overflow - * lower 32-bit and it's ok. - */ - data.i = 0; - ret = apply_to_page_range(&init_mm, - kern_vm_start + uaddr32 + (mapped << PAGE_SHIFT), - this_batch << PAGE_SHIFT, apply_range_set_cb, &data); - if (ret) { - /* data.i pages were mapped, account them and free the remaining */ - mapped += data.i; - for (i = data.i; i < this_batch; i++) - free_pages_nolock(pages[i], 0); - goto out; - } + addr = clear_lo32(arena->user_vm_start) + uaddr32; + goto out_memcg; - mapped += this_batch; - remaining -= this_batch; - } +out_unmap: + /* Error handling: Undo partial mappings. */ flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT); - raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - kfree_nolock(pages); - bpf_map_memcg_exit(old_memcg, new_memcg); - return clear_lo32(arena->user_vm_start) + uaddr32; -out: range_tree_set(&arena->rt, pgoff + mapped, page_cnt - mapped); raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - if (mapped) { - flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT); + if (mapped) arena_free_pages(arena, uaddr32, mapped, sleepable); - } - goto out_free_pages; -out_unlock_free_pages: - raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + out_free_pages: - kfree_nolock(pages); + /* Error handling: Free back any unmapped pages. */ + bpf_free_pages(&pages); + +out_memcg: bpf_map_memcg_exit(old_memcg, new_memcg); - return 0; + + return addr; } /* @@ -807,7 +874,7 @@ static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt) * otherwise call B's mark would make call A skip a vml that A has * not yet zapped for A's uaddr range. */ - mutex_lock(&arena->zap_mutex); + lockdep_assert_held(&arena->zap_mutex); mutex_lock(&arena->lock); my_gen = ++arena->zap_gen; for (;;) { @@ -841,49 +908,41 @@ static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt) mutex_lock(&arena->lock); } mutex_unlock(&arena->lock); - mutex_unlock(&arena->zap_mutex); } -static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable) +/* + * The range stays allocated in the range tree until the pages are unmapped + * from vmalloc area and from all user space vma-s, so that nobody can + * populate it while stale TLB entries or user ptes are still around. + */ +static int arena_free_range(struct bpf_arena *arena, long uaddr, long page_cnt) { - struct mem_cgroup *new_memcg, *old_memcg; - u64 full_uaddr, uaddr_end; - long kaddr, pgoff; - struct page *page; - struct llist_head free_pages; - struct llist_node *pos, *t; - struct arena_free_span *s; + u64 full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; + long kaddr = bpf_arena_get_kern_vm_start(arena) + uaddr; + long pgoff = compute_pgoff(arena, uaddr); struct clear_range_data cdata; + struct llist_node *pos, *t; + LLIST_HEAD(free_pages); unsigned long flags; - int ret = 0; - - /* only aligned lower 32-bit are relevant */ - uaddr = (u32)uaddr; - uaddr &= PAGE_MASK; - kaddr = bpf_arena_get_kern_vm_start(arena) + uaddr; - full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; - if (full_uaddr < arena->user_vm_start) - return; - uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT)); - if (full_uaddr >= uaddr_end) - return; - - page_cnt = (uaddr_end - full_uaddr) >> PAGE_SHIFT; - pgoff = compute_pgoff(arena, uaddr); - bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); + struct page *page; + int ret; - if (!sleepable) - goto defer; + /* An overlapping free must not release the range before this one is done */ + guard(mutex)(&arena->zap_mutex); ret = raw_res_spin_lock_irqsave(&arena->spinlock, flags); - - /* Can't proceed without holding the spinlock so defer the free */ if (ret) - goto defer; + return ret; - range_tree_set(&arena->rt, pgoff, page_cnt); + /* + * Take the free pages of the range too. It allocates, and can fail, + * only when the whole range is free. Then there is nothing to do. + */ + if (range_tree_clear(&arena->rt, pgoff, page_cnt)) { + raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + return 0; + } - init_llist_head(&free_pages); cdata.arena = arena; cdata.free_pages = &free_pages; /* clear ptes and collect struct pages */ @@ -893,6 +952,12 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, /* drop the lock to do the tlb flush and zap pages */ raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + /* wait for arena_vm_fault() that found the page before its pte was cleared */ + llist_for_each_entry(page, free_pages.first, pcp_llist) { + lock_page(page); + unlock_page(page); + } + /* ensure no stale TLB entries */ flush_tlb_kernel_range(kaddr, kaddr + (page_cnt * PAGE_SIZE)); @@ -911,8 +976,39 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, zap_pages(arena, full_uaddr, 1); __free_page(page); } - bpf_map_memcg_exit(old_memcg, new_memcg); + /* The callers hold no other rqspinlock, so this one can only time out */ + while (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) + cond_resched(); + range_tree_set(&arena->rt, pgoff, page_cnt); + raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + return 0; +} + +static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable) +{ + struct mem_cgroup *new_memcg, *old_memcg; + u64 full_uaddr, uaddr_end; + struct arena_free_span *s; + + /* only aligned lower 32-bit are relevant */ + uaddr = (u32)uaddr; + uaddr &= PAGE_MASK; + full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; + if (full_uaddr < arena->user_vm_start) + return; + uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT)); + if (full_uaddr >= uaddr_end) + return; + + page_cnt = (uaddr_end - full_uaddr) >> PAGE_SHIFT; + bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); + + /* Can't proceed without holding the spinlock so defer the free */ + if (!sleepable || arena_free_range(arena, uaddr, page_cnt)) + goto defer; + + bpf_map_memcg_exit(old_memcg, new_memcg); return; defer: @@ -974,66 +1070,18 @@ static void arena_free_worker(struct work_struct *work) { struct bpf_arena *arena = container_of(work, struct bpf_arena, free_work); struct mem_cgroup *new_memcg, *old_memcg; - struct llist_node *list, *pos, *t; + struct llist_node *pos, *t; struct arena_free_span *s; - u64 arena_vm_start, user_vm_start; - struct llist_head free_pages; - struct clear_range_data cdata; - struct page *page; - unsigned long full_uaddr; - long kaddr, page_cnt, pgoff; - unsigned long flags; - - if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) { - schedule_work(work); - return; - } bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); - init_llist_head(&free_pages); - cdata.arena = arena; - cdata.free_pages = &free_pages; - arena_vm_start = bpf_arena_get_kern_vm_start(arena); - user_vm_start = bpf_arena_get_user_vm_start(arena); - - list = llist_del_all(&arena->free_spans); - llist_for_each(pos, list) { + llist_for_each_safe(pos, t, llist_del_all(&arena->free_spans)) { s = llist_entry(pos, struct arena_free_span, node); - page_cnt = s->page_cnt; - kaddr = arena_vm_start + s->uaddr; - pgoff = compute_pgoff(arena, s->uaddr); - - /* clear ptes and collect pages in free_pages llist */ - apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT, - apply_range_clear_cb, &cdata); - - range_tree_set(&arena->rt, pgoff, page_cnt); - } - raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - - /* Iterate the list again without holding spinlock to do the tlb flush and zap_pages */ - llist_for_each_safe(pos, t, list) { - s = llist_entry(pos, struct arena_free_span, node); - page_cnt = s->page_cnt; - full_uaddr = clear_lo32(user_vm_start) + s->uaddr; - kaddr = arena_vm_start + s->uaddr; - - /* ensure no stale TLB entries */ - flush_tlb_kernel_range(kaddr, kaddr + (page_cnt * PAGE_SIZE)); - - /* remove pages from user vmas */ - zap_pages(arena, full_uaddr, page_cnt); - + while (arena_free_range(arena, s->uaddr, s->page_cnt)) + cond_resched(); kfree_nolock(s); } - /* free all pages collected by apply_to_existing_page_range() in the first loop */ - llist_for_each_safe(pos, t, __llist_del_all(&free_pages)) { - page = llist_entry(pos, struct page, pcp_llist); - __free_page(page); - } - bpf_map_memcg_exit(old_memcg, new_memcg); } diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 0ce26b538075..133f189bf6d2 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -438,15 +438,15 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, u32 size; int cpu, off = 0; - if (unlikely((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS)) - /* unknown flags */ + if (unlikely(((map_flags & BPF_EXIST_FLAGS) == BPF_EXIST_FLAGS) || + ((u32)map_flags & ~(BPF_EXIST_FLAGS | BPF_CPU_FLAGS)))) return -EINVAL; if (unlikely(index >= array->map.max_entries)) /* all elements were pre-allocated, cannot insert a new one */ return -E2BIG; - if (unlikely(map_flags == BPF_NOEXIST)) + if (unlikely(map_flags & BPF_NOEXIST)) /* all elements already exist */ return -EEXIST; @@ -1035,7 +1035,7 @@ static void *prog_fd_array_get_ptr(struct bpf_map *map, } mutex_lock(&prog->aux->ext_mutex); - is_extended = prog->aux->is_extended; + is_extended = prog->aux->freplace_link_cnt > 0; if (!is_extended) prog->aux->prog_array_member_cnt++; mutex_unlock(&prog->aux->ext_mutex); diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 47282ffeeaf9..43d5740034c0 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -9,7 +9,8 @@ /* for any branch, call, exit record the history of jmps in the given state */ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, - int insn_flags, int spi, int frame, u64 linked_regs) + int insn_flags, int spi, int frame, const u16 *linked_regs, + u8 linked_regs_cnt) { u32 cnt = cur->jmp_history_cnt; struct bpf_jmp_history_entry *p; @@ -27,10 +28,13 @@ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state env->cur_hist_ent->flags |= insn_flags; env->cur_hist_ent->spi = spi; env->cur_hist_ent->frame = frame; - verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env, - "insn history: insn_idx %d linked_regs: %#llx", - env->insn_idx, env->cur_hist_ent->linked_regs); - env->cur_hist_ent->linked_regs = linked_regs; + verifier_bug_if(env->cur_hist_ent->linked_regs_cnt != 0, env, + "insn history: insn_idx %d has %u linked regs", + env->insn_idx, env->cur_hist_ent->linked_regs_cnt); + if (linked_regs_cnt) + memcpy(env->cur_hist_ent->linked_regs, linked_regs, + linked_regs_cnt * sizeof(*linked_regs)); + env->cur_hist_ent->linked_regs_cnt = linked_regs_cnt; return 0; } @@ -47,7 +51,9 @@ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state p->flags = insn_flags; p->spi = spi; p->frame = frame; - p->linked_regs = linked_regs; + if (linked_regs_cnt) + memcpy(p->linked_regs, linked_regs, linked_regs_cnt * sizeof(*linked_regs)); + p->linked_regs_cnt = linked_regs_cnt; cur->jmp_history_cnt = cnt; env->cur_hist_ent = p; @@ -123,13 +129,26 @@ static inline void bt_reset(struct backtrack_state *bt) bt->env = env; } -static inline u32 bt_empty(struct backtrack_state *bt) +static inline bool bt_frame_stack_empty(struct backtrack_state *bt, u32 frame) { - u64 mask = 0; + return bitmap_empty(bt->stack_masks[frame], MAX_BPF_STACK_SLOTS); +} + +static inline bool bt_stack_empty(struct backtrack_state *bt) +{ + return bt_frame_stack_empty(bt, bt->frame); +} + +static inline bool bt_empty(struct backtrack_state *bt) +{ + u32 mask = 0; int i; - for (i = 0; i <= bt->frame; i++) - mask |= bt->reg_masks[i] | bt->stack_masks[i] | bt->stack_arg_masks[i]; + for (i = 0; i <= bt->frame; i++) { + mask |= bt->reg_masks[i] | bt->stack_arg_masks[i]; + if (!bt_frame_stack_empty(bt, i)) + return false; + } return mask == 0; } @@ -181,7 +200,7 @@ static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg) static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot) { - bt->stack_masks[frame] &= ~(1ull << slot); + __clear_bit(slot, bt->stack_masks[frame]); } static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame) @@ -194,14 +213,14 @@ static inline u32 bt_reg_mask(struct backtrack_state *bt) return bt->reg_masks[bt->frame]; } -static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame) +static inline unsigned long *bt_frame_stack_mask(struct backtrack_state *bt, u32 frame) { return bt->stack_masks[frame]; } -static inline u64 bt_stack_mask(struct backtrack_state *bt) +static inline unsigned long *bt_stack_mask(struct backtrack_state *bt) { - return bt->stack_masks[bt->frame]; + return bt_frame_stack_mask(bt, bt->frame); } static inline u8 bt_stack_arg_mask(struct backtrack_state *bt) @@ -233,17 +252,16 @@ static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask) break; } } -/* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */ -void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask) + +/* format stack slots bitmask, e.g., "-8,-24,-40" for slots 0, 2 and 4 */ +void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, const unsigned long *stack_mask) { - DECLARE_BITMAP(mask, 64); bool first = true; int i, n; buf[0] = '\0'; - bitmap_from_u64(mask, stack_mask); - for_each_set_bit(i, mask, 64) { + for_each_set_bit(i, stack_mask, MAX_BPF_STACK_SLOTS) { n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8); first = false; buf += n; @@ -406,15 +424,18 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, if (class == BPF_STX) bt_set_reg(bt, sreg); } else if (class == BPF_JMP || class == BPF_JMP32) { - if (bpf_pseudo_call(insn)) { - int subprog_insn_idx, subprog; + if (bpf_pseudo_call(insn) || bpf_is_callx(insn)) { + int subprog_insn_idx, subprog = -1; - subprog_insn_idx = idx + insn->imm + 1; - subprog = bpf_find_subprog(env, subprog_insn_idx); - if (subprog < 0) - return -EFAULT; + if (bpf_pseudo_call(insn)) { + subprog_insn_idx = idx + insn->imm + 1; + subprog = bpf_find_subprog(env, subprog_insn_idx); + if (subprog < 0) + return -EFAULT; + } - if (bpf_subprog_is_global(env, subprog)) { + /* callx calls static subprogs only */ + if (subprog >= 0 && bpf_subprog_is_global(env, subprog)) { /* check that jump history doesn't have any * extra instructions from subprog; the next * instruction after call to global subprog @@ -423,6 +444,10 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, */ verifier_bug_if(idx + 1 != subseq_idx, env, "extra insn from subprog"); + /* global subprog always sets R0 */ + bt_clear_reg(bt, BPF_REG_0); + /* and if it does not set R2, main pass would catch it */ + bt_clear_reg(bt, BPF_REG_2); /* r1-r5 are invalidated after subprog call, * so for global func call it shouldn't be set * anymore @@ -432,8 +457,6 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, bt_reg_mask(bt)); return -EFAULT; } - /* global subprog always sets R0 */ - bt_clear_reg(bt, BPF_REG_0); return 0; } else { /* static subprog call instruction, which @@ -450,10 +473,11 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, /* we are now tracking register spills correctly, * so any instance of leftover slots is a bug */ - if (bt_stack_mask(bt) != 0) { - verifier_bug(env, - "static subprog leftover stack slots %llx", - bt_stack_mask(bt)); + if (!bt_stack_empty(bt)) { + bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, + bt_stack_mask(bt)); + verifier_bug(env, "static subprog leftover stack slots %s", + env->tmp_str_buf); return -EFAULT; } /* propagate r1-r5 to the caller */ @@ -486,9 +510,11 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, bt_reg_mask(bt)); return -EFAULT; } - if (bt_stack_mask(bt) != 0) { - verifier_bug(env, "callback leftover stack slots %llx", - bt_stack_mask(bt)); + if (!bt_stack_empty(bt)) { + bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, + bt_stack_mask(bt)); + verifier_bug(env, "callback leftover stack slots %s", + env->tmp_str_buf); return -EFAULT; } /* clear r1-r5 in callback subprog's mask */ @@ -506,6 +532,8 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, return -ENOTSUPP; /* regular helper call sets R0 */ bt_clear_reg(bt, BPF_REG_0); + /* kfunc might also set R2 */ + bt_clear_reg(bt, BPF_REG_2); if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { /* if backtracking was looking for registers R1-R5 * they should have been found already. @@ -520,40 +548,51 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, return -EFAULT; } } else if (opcode == BPF_EXIT) { - bool from_subprog_call, r0_precise; + bool from_subprog_call, r0_precise, r2_precise; /* BPF_EXIT in subprog or callback always returns * right after the call instruction, so by checking * whether the instruction at subseq_idx-1 is subprog * call or not we can distinguish actual exit from * *subprog* from exit from *callback*. In the former - * case, we need to propagate r0 precision, if - * necessary. In the former we never do that. + * case, we need to propagate the precision of the + * return registers, if necessary. In the latter we + * never do that. */ from_subprog_call = subseq_idx - 1 >= 0 && - bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]); + (bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) || + bpf_is_callx(&env->prog->insnsi[subseq_idx - 1])); r0_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_0); + r2_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_2); /* Backtracking to a nested function call, 'idx' is a part of * the inner frame 'subseq_idx' is a part of the outer frame. - * In case of a regular function call, instructions giving - * precision to registers R1-R5 should have been found already. + * In case of a regular function call, the callee defines the + * return registers R0 and R2, so clear them before checking + * that instructions giving precision to registers R1-R5 have + * been found already. R2 is also an argument register, hence + * it has to be cleared before that check. * In case of a callback from bpf_loop(), R{1,4} in the calling - * frame would be set as precise and that is correct. + * frame would be set as precise and that is correct, and R2 + * might be a precise helper argument as well, so leave it set. */ + bt_clear_reg(bt, BPF_REG_0); + if (from_subprog_call) + bt_clear_reg(bt, BPF_REG_2); if (from_subprog_call && (bt_reg_mask(bt) & BPF_REGMASK_ARGS)) { verifier_bug(env, "backtracking exit unexpected regs %x", bt_reg_mask(bt)); return -EFAULT; } - bt_clear_reg(bt, BPF_REG_0); if (bt_subprog_enter(bt)) return -EFAULT; if (r0_precise) bt_set_reg(bt, BPF_REG_0); + if (r2_precise) + bt_set_reg(bt, BPF_REG_2); /* r6-r9 and stack slots will stay set in caller frame * bitmasks until we return back from callee(s) */ @@ -693,10 +732,12 @@ void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env, i, j); } } - for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { - if (!bpf_is_spilled_reg(&func->stack[j])) + for (j = 0; j < bpf_stack_nr_slots(func); j++) { + struct bpf_stack_state *ss = bpf_stack_slot(func, j); + + if (!bpf_is_spilled_reg(ss)) continue; - reg = &func->stack[j].spilled_ptr; + reg = &ss->spilled_ptr; if (reg->type != SCALAR_VALUE || reg->precise) continue; reg->precise = true; @@ -808,6 +849,7 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, int subseq_idx = -1; struct bpf_func_state *func; bool tmp, skip_first = true; + struct bpf_stack_state *ss; struct bpf_reg_state *reg; int i, fr, err; @@ -846,6 +888,10 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, } if (last_idx < 0) { + /* Consecutive checkpoints can have no instructions between them. */ + if (st->parent) + goto parent; + /* we are at the entry into subprog, which * is expected for global funcs, but only if * requested precise registers are R1-R5 @@ -854,7 +900,7 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, if (st->curframe == 0 && st->frame[0]->subprogno > 0 && st->frame[0]->callsite == BPF_MAIN_FUNC && - bt_stack_mask(bt) == 0 && + bt_stack_empty(bt) && (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) == 0) { bitmap_from_u64(mask, bt_reg_mask(bt)); for_each_set_bit(i, mask, 32) { @@ -868,8 +914,9 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, return 0; } - verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %llx", - st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt)); + bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt)); + verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %s", + st->frame[0]->subprogno, bt_reg_mask(bt), env->tmp_str_buf); return -EFAULT; } @@ -909,6 +956,7 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, return -EFAULT; } } +parent: st = st->parent; if (!st) break; @@ -930,18 +978,18 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, } } - bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr)); - for_each_set_bit(i, mask, 64) { - if (verifier_bug_if(i >= func->allocated_stack / BPF_REG_SIZE, + for_each_set_bit(i, bt_frame_stack_mask(bt, fr), MAX_BPF_STACK_SLOTS) { + if (verifier_bug_if(i >= bpf_stack_nr_slots(func), env, "stack slot %d, total slots %d", - i, func->allocated_stack / BPF_REG_SIZE)) + i, bpf_stack_nr_slots(func))) return -EFAULT; - if (!bpf_is_spilled_scalar_reg(&func->stack[i])) { + ss = bpf_stack_slot(func, i); + if (!bpf_is_spilled_scalar_reg(ss)) { bt_clear_frame_slot(bt, fr, i); continue; } - reg = &func->stack[i].spilled_ptr; + reg = &ss->spilled_ptr; if (reg->precise) { bt_clear_frame_slot(bt, fr, i); } else { diff --git a/kernel/bpf/bpf_insn_array.c b/kernel/bpf/bpf_insn_array.c index a2f84afe6f7c..92bbb71576ae 100644 --- a/kernel/bpf/bpf_insn_array.c +++ b/kernel/bpf/bpf_insn_array.c @@ -199,12 +199,16 @@ int bpf_insn_array_init(struct bpf_map *map, const struct bpf_prog *prog) return -EBUSY; /* - * Reset all the map indexes to the original values. This is needed, - * e.g., when a replay of verification with different log level should - * be performed. + * Reset the map to its pre-verification state. The xlated and jitted + * offsets and the jitted target pointers are recomputed by the verifier + * and the JIT for this program, so any values left by a previous owner + * must be cleared here. */ - for (i = 0; i < map->max_entries; i++) + for (i = 0; i < map->max_entries; i++) { values[i].xlated_off = values[i].orig_off; + values[i].jitted_off = 0; + insn_array->ips[i] = 0; + } return 0; } @@ -228,10 +232,11 @@ void bpf_insn_array_release(struct bpf_map *map) { struct bpf_insn_array *insn_array = cast_insn_array(map); - atomic_set(&insn_array->used, 0); + /* Paired with atomic_xchg() in bpf_insn_array_init(). */ + atomic_set_release(&insn_array->used, 0); } -void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len) +void bpf_insn_array_adjust(struct bpf_map *map, u32 first, u32 len) { struct bpf_insn_array *insn_array = cast_insn_array(map); int i; @@ -240,7 +245,7 @@ void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len) return; for (i = 0; i < map->max_entries; i++) { - if (insn_array->values[i].xlated_off <= off) + if (insn_array->values[i].xlated_off < first) continue; if (insn_array->values[i].xlated_off == INSN_DELETED) continue; diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index b40eb404adab..f0b29a170935 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -72,6 +72,15 @@ static inline bool bpf_iter_target_support_resched(const struct bpf_iter_target_ return tinfo->reg_info->feature & BPF_ITER_RESCHED; } +/* Only allow sleepable program for resched-able iterator that can run one */ +static bool bpf_iter_target_support_prog(const struct bpf_iter_target_info *tinfo, + const struct bpf_prog *prog) +{ + return !prog->sleepable || + (bpf_iter_target_support_resched(tinfo) && + !(tinfo->reg_info->feature & BPF_ITER_NOSLEEP)); +} + static bool bpf_iter_support_resched(struct seq_file *seq) { struct bpf_iter_priv_data *iter_priv; @@ -342,6 +351,13 @@ int bpf_iter_prog_supported(struct bpf_prog *prog) const char *prefix = BPF_ITER_FUNC_PREFIX; int prefix_len = strlen(prefix); + /* + * The attach function of a program that has a target program is a + * function of that program, whatever its name is. + */ + if (prog->aux->dst_prog) + return -EINVAL; + if (strncmp(attach_fname, prefix, prefix_len)) return -EINVAL; @@ -409,6 +425,8 @@ static int bpf_iter_link_replace(struct bpf_link *link, struct bpf_prog *new_prog, struct bpf_prog *old_prog) { + struct bpf_iter_link *iter_link = + container_of(link, struct bpf_iter_link, link); int ret = 0; mutex_lock(&link_mutex); @@ -419,11 +437,19 @@ static int bpf_iter_link_replace(struct bpf_link *link, if (link->prog->type != new_prog->type || link->prog->expected_attach_type != new_prog->expected_attach_type || - link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) { + link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id || + !bpf_iter_target_support_prog(iter_link->tinfo, new_prog)) { ret = -EINVAL; goto out_unlock; } + if (iter_link->tinfo->reg_info->check_prog) { + ret = iter_link->tinfo->reg_info->check_prog(new_prog, + &iter_link->aux); + if (ret) + goto out_unlock; + } + old_prog = xchg(&link->prog, new_prog); bpf_prog_put(old_prog); @@ -544,8 +570,7 @@ int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, if (!tinfo) return -ENOENT; - /* Only allow sleepable program for resched-able iterator */ - if (prog->sleepable && !bpf_iter_target_support_resched(tinfo)) + if (!bpf_iter_target_support_prog(tinfo, prog)) return -EINVAL; link = kzalloc_obj(*link, GFP_USER | __GFP_NOWARN); @@ -643,11 +668,11 @@ int bpf_iter_new_fd(struct bpf_link *link) flags = O_RDONLY | O_CLOEXEC; FD_PREPARE(fdf, flags, anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags)); - if (fdf.err) - return fdf.err; + if (fdf->fd < 0) + return fdf->fd; iter_link = container_of(link, struct bpf_iter_link, link); - err = prepare_seq_file(fd_prepare_file(fdf), iter_link); + err = prepare_seq_file(fdf->file, iter_link); if (err) return err; /* Automatic cleanup handles fput */ diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c index 6fc6a4b672b5..b12fcec47ffa 100644 --- a/kernel/bpf/bpf_local_storage.c +++ b/kernel/bpf/bpf_local_storage.c @@ -199,7 +199,7 @@ static void bpf_selem_unlink_storage_nolock_misc(struct bpf_local_storage_elem * } if (pin_owner) - refcount_dec(&local_storage->owner_refcnt); + bpf_local_storage_unpin_owner(local_storage); } /* local_storage->lock must be held and selem->local_storage == local_storage. @@ -240,24 +240,26 @@ void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage, hlist_add_head_rcu(&selem->snode, &local_storage->list); } +/* Must be called with the owning local_storage->lock held. */ static int bpf_selem_unlink_map(struct bpf_local_storage_elem *selem) { struct bpf_local_storage *local_storage; struct bpf_local_storage_map *smap; struct bpf_local_storage_map_bucket *b; - unsigned long flags; int err; + lockdep_assert_irqs_disabled(); + local_storage = rcu_dereference_check(selem->local_storage, bpf_rcu_lock_held()); smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); b = select_bucket(smap, local_storage); - err = raw_res_spin_lock_irqsave(&b->lock, flags); + err = raw_res_spin_lock(&b->lock); if (err) return err; hlist_del_init_rcu(&selem->map_node); - raw_res_spin_unlock_irqrestore(&b->lock, flags); + raw_res_spin_unlock(&b->lock); return 0; } @@ -350,7 +352,8 @@ out: static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem, struct bpf_local_storage_map_bucket *b) { - bool in_map_free = !!b, free_storage = false; + bool in_map_free = !!b, free_storage = false, sleepable_users = false; + bool keep_fields = false; struct bpf_local_storage *local_storage; struct bpf_local_storage_map *smap; unsigned long flags; @@ -360,17 +363,26 @@ static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem, smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); if (smap) { + /* A map that is freed, or NULL here, has no program left. */ + sleepable_users = !in_map_free && + (atomic64_read(&smap->map.sleepable_refcnt) || + READ_ONCE(smap->map.was_inner_map)); b = b ? : select_bucket(smap, local_storage); err = raw_res_spin_lock_irqsave(&b->lock, flags); if (!err) { /* * Call bpf_obj_free_fields() under b->lock to make sure it is done * exactly once for an selem. Safe to free special fields immediately - * as no BPF program should be referencing the selem. + * as no BPF program should be referencing the selem, unless it + * is sleepable: see below. */ if (likely(selem_linked_to_map(selem))) { hlist_del_init_rcu(&selem->map_node); - bpf_obj_free_fields(smap->map.record, SDATA(selem)->data); + if (sleepable_users) + keep_fields = true; + else + bpf_obj_free_fields(smap->map.record, + SDATA(selem)->data); unlink++; } raw_res_spin_unlock_irqrestore(&b->lock, flags); @@ -385,7 +397,15 @@ static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem, * with selem1 as raw_res_spin_lock_irqsave() returns -ETIMEDOUT. */ WARN_ON_ONCE(err && in_map_free); - if (!err || in_map_free) + /* + * A sleepable program can still use, and fill, the fields of a value + * that it got before it let go of the owner. As for bpf_selem_unlink(), + * bpf_selem_free_trace_rcu() frees them, and takes smap for that: + * map_free() waits for this RCU read side critical section and then + * for the callback. It must not take smap away from an selem that + * destroy() has unlinked from the map. + */ + if (in_map_free ? err || unlink : !err && !keep_fields) RCU_INIT_POINTER(SDATA(selem)->smap, NULL); } @@ -429,13 +449,18 @@ static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem, * then freed after an RCU grace period. However, if destroy() and map_free() are * racing or rqspinlock returns errors in unlikely situations (unlink != 2), free * the selem only after both map_free() and destroy() see the selem. + * + * The owner that map_free() takes an selem from is alive: sleepable code, which + * is in no RCU read side critical section, may be walking its list for another + * map. And it can still use a value that it got before it let go of an owner + * that destroy() is called for. */ if (unlink == 2 || atomic_cmpxchg(&selem->state, SELEM_UNLINKED, SELEM_TOFREE) == SELEM_UNLINKED) - bpf_selem_free(selem, true); + bpf_selem_free(selem, !in_map_free && !sleepable_users); if (free_storage) - bpf_local_storage_free(local_storage, true); + bpf_local_storage_free(local_storage, !in_map_free); } void __bpf_local_storage_insert_cache(struct bpf_local_storage *local_storage, @@ -507,8 +532,6 @@ int bpf_local_storage_alloc(void *owner, if (err) goto uncharge; - bpf_selem_link_map_nolock(b, first_selem); - owner_storage_ptr = (struct bpf_local_storage **)owner_storage(smap, owner); /* Publish storage to the owner. @@ -523,11 +546,17 @@ int bpf_local_storage_alloc(void *owner, */ prev_storage = cmpxchg(owner_storage_ptr, NULL, storage); if (unlikely(prev_storage)) { - bpf_selem_unlink_map_nolock(first_selem); raw_res_spin_unlock_irqrestore(&b->lock, flags); err = -EAGAIN; goto uncharge; } + /* + * Not before the storage is published: the owner does not wait for + * users of a storage that it knows nothing of, so that nobody may find + * such a storage through the map. b->lock holds off those who find + * first_selem through the owner and want to unlink it. + */ + bpf_selem_link_map_nolock(b, first_selem); raw_res_spin_unlock_irqrestore(&b->lock, flags); return 0; @@ -552,7 +581,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, struct bpf_local_storage *local_storage; struct bpf_local_storage_map_bucket *b; HLIST_HEAD(old_selem_free_list); - unsigned long flags, b_flags; + unsigned long flags; int err; /* BPF_EXIST and BPF_NOEXIST cannot be both set */ @@ -637,7 +666,8 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, b = select_bucket(smap, local_storage); - err = raw_res_spin_lock_irqsave(&b->lock, b_flags); + /* local_storage->lock is held, so IRQs are already disabled. */ + err = raw_res_spin_lock(&b->lock); if (err) goto unlock; @@ -655,7 +685,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, &old_selem_free_list); } - raw_res_spin_unlock_irqrestore(&b->lock, b_flags); + raw_res_spin_unlock(&b->lock); unlock: raw_res_spin_unlock_irqrestore(&local_storage->lock, flags); free_selem: @@ -750,8 +780,8 @@ u32 bpf_local_storage_destroy(struct bpf_local_storage *local_storage) while (refcount_read(&local_storage->owner_refcnt)) cpu_relax(); /* - * Paired with refcount_dec() in bpf_selem_unlink_nofail() - * to make sure destroy() sees the correct local_storage->mem_charge. + * Paired with bpf_local_storage_unpin_owner() to make sure + * destroy() sees the correct local_storage->mem_charge. */ smp_mb(); } diff --git a/kernel/bpf/bpf_lru_list.c b/kernel/bpf/bpf_lru_list.c index 5ed7cb4b98c0..3858886b0232 100644 --- a/kernel/bpf/bpf_lru_list.c +++ b/kernel/bpf/bpf_lru_list.c @@ -71,7 +71,7 @@ static void __bpf_lru_node_move_in(struct bpf_lru_list *l, return; bpf_lru_list_count_inc(l, tgt_type); - node->type = tgt_type; + WRITE_ONCE(node->type, tgt_type); bpf_lru_node_clear_ref(node); /* Reset pending_free only when moving to the free list */ if (tgt_type == BPF_LRU_LIST_T_FREE) @@ -94,7 +94,7 @@ static void __bpf_lru_node_move(struct bpf_lru_list *l, if (node->type != tgt_type) { bpf_lru_list_count_dec(l, node->type); bpf_lru_list_count_inc(l, tgt_type); - node->type = tgt_type; + WRITE_ONCE(node->type, tgt_type); } bpf_lru_node_clear_ref(node); @@ -298,9 +298,6 @@ static void bpf_lru_list_push_free(struct bpf_lru_list *l, { unsigned long flags; - if (WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(node->type))) - return; - if (raw_res_spin_lock_irqsave(&l->lock, flags)) { WRITE_ONCE(node->pending_free, 1); return; @@ -542,7 +539,7 @@ static void bpf_common_lru_push_free(struct bpf_lru *lru, return; } - if (unlikely(node->type != BPF_LRU_LOCAL_LIST_T_PENDING)) { + if (unlikely(READ_ONCE(node->type) != BPF_LRU_LOCAL_LIST_T_PENDING)) { raw_res_spin_unlock_irqrestore(&loc_l->lock, flags); goto check_lru_list; diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 82c5988417a0..f58dce888ff4 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -116,8 +116,11 @@ void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, } #endif +BTF_ID_LIST_SINGLE(bpf_lsm_inode_init_security_btf_id, func, + bpf_lsm_inode_init_security) + int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, - const struct bpf_prog *prog) + struct bpf_prog *prog) { u32 btf_id = prog->aux->attach_btf_id; const char *func_name = prog->aux->attach_func_name; @@ -140,6 +143,23 @@ int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, return -EINVAL; } + if (btf_id == bpf_lsm_inode_init_security_btf_id[0]) { + /* + * inode, dir, qstr, xattrs, xattr_count + * + * The trusted pointer this hands the program is only as + * trustworthy as the context it is loaded from, which + * lsm_verifier_ops keeps read-only. + */ + static const struct bpf_ctx_arg_aux xattr_count_arg_info = { + .offset = 4 * sizeof(u64), + .reg_type = PTR_TO_MEM | MEM_RDONLY | PTR_TRUSTED, + .mem_size = sizeof(int), + }; + + return bpf_prog_ctx_arg_info_init(prog, &xattr_count_arg_info, 1); + } + return 0; } @@ -414,8 +434,8 @@ const struct bpf_prog_ops lsm_prog_ops = { }; const struct bpf_verifier_ops lsm_verifier_ops = { - .get_func_proto = bpf_lsm_func_proto, - .is_valid_access = btf_ctx_access, + .get_func_proto = bpf_lsm_func_proto, + .is_valid_access = bpf_tracing_btf_ctx_access, }; /* hooks return 0 or 1 */ diff --git a/kernel/bpf/bpf_lsm_proto.c b/kernel/bpf/bpf_lsm_proto.c index 44a54fd8045e..b81179ae2e08 100644 --- a/kernel/bpf/bpf_lsm_proto.c +++ b/kernel/bpf/bpf_lsm_proto.c @@ -4,6 +4,7 @@ */ #include <linux/fs.h> +#include <linux/xattr.h> #include <linux/bpf_lsm.h> /* @@ -12,8 +13,22 @@ * explicitly enforces that BPF LSM programs check for NULL before attempting to * dereference it. */ -int bpf_lsm_mmap_file(struct file *file__nullable, unsigned long reqprot, - unsigned long prot, unsigned long flags) +noinline int bpf_lsm_mmap_file(struct file *file__nullable, unsigned long reqprot, + unsigned long prot, unsigned long flags) { return 0; } + +/* + * Strong definition of the inode_init_security() BPF LSM hook. Both the + * qstr and the xattr array are NULL for some callers, so the __nullable + * suffix marks it as PTR_MAYBE_NULL. BPF LSM programs have to check before + * dereferencing them or handing them to bpf_inode_init_xattr(). + */ +noinline int bpf_lsm_inode_init_security(struct inode *inode, struct inode *dir, + const struct qstr *qstr__nullable, + struct xattr *xattrs__nullable, + int *xattr_count) +{ + return -EOPNOTSUPP; +} diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index d7c3030bc63b..1178acd72296 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -13,6 +13,7 @@ #include <linux/btf_ids.h> #include <linux/rcupdate_wait.h> #include <linux/poll.h> +#include <linux/bpf-cgroup.h> struct bpf_struct_ops_value { struct bpf_struct_ops_common_value common; @@ -57,7 +58,7 @@ struct bpf_struct_ops_map { struct bpf_struct_ops_link { struct bpf_link link; - struct bpf_map __rcu *map; + struct bpf_map *map; wait_queue_head_t wait_hup; }; @@ -1024,9 +1025,18 @@ static void __bpf_struct_ops_map_free(struct bpf_map *map) bpf_map_area_free(st_map); } +static void bpf_struct_ops_map_free_pre_rcu(struct bpf_map *map) +{ + struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; + + bpf_struct_ops_map_del_ksyms(st_map); +} + static void bpf_struct_ops_map_free(struct bpf_map *map) { struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; + struct bpf_struct_ops *st_ops = st_map->st_ops_desc->st_ops; + bool tasks_rcu = st_ops->free_after_tasks_rcu_gp; /* st_ops->owner was acquired during map_alloc to implicitly holds * the btf's refcnt. The acquire was only done when btf_is_module() @@ -1037,24 +1047,8 @@ static void bpf_struct_ops_map_free(struct bpf_map *map) bpf_struct_ops_map_dissoc_progs(st_map); - bpf_struct_ops_map_del_ksyms(st_map); - - /* The struct_ops's function may switch to another struct_ops. - * - * For example, bpf_tcp_cc_x->init() may switch to - * another tcp_cc_y by calling - * setsockopt(TCP_CONGESTION, "tcp_cc_y"). - * During the switch, bpf_struct_ops_put(tcp_cc_x) is called - * and its refcount may reach 0 which then free its - * trampoline image while tcp_cc_x is still running. - * - * A vanilla rcu gp is to wait for all bpf-tcp-cc prog - * to finish. bpf-tcp-cc prog is non sleepable. - * A rcu_tasks gp is to wait for the last few insn - * in the tramopline image to finish before releasing - * the trampoline image. - */ - synchronize_rcu_mult(call_rcu, call_rcu_tasks); + if (tasks_rcu && IS_ENABLED(CONFIG_TASKS_RCU)) + synchronize_rcu_tasks(); __bpf_struct_ops_map_free(map); } @@ -1123,6 +1117,11 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) goto errout; } + if (st_ops_desc->st_ops->cgroup_atype && !(attr->map_flags & BPF_F_LINK)) { + ret = -EOPNOTSUPP; + goto errout; + } + vt = st_ops_desc->value_type; if (attr->value_size != vt->size) { ret = -EINVAL; @@ -1163,6 +1162,8 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) mutex_init(&st_map->lock); bpf_map_init_from_attr(map, attr); + map->free_after_mult_rcu_gp = st_ops_desc->st_ops->free_after_mult_rcu_gp; + map->free_after_rcu_gp = true; return map; @@ -1195,6 +1196,7 @@ const struct bpf_map_ops bpf_struct_ops_map_ops = { .map_alloc_check = bpf_struct_ops_map_alloc_check, .map_alloc = bpf_struct_ops_map_alloc, .map_free = bpf_struct_ops_map_free, + .map_free_pre_rcu = bpf_struct_ops_map_free_pre_rcu, .map_get_next_key = bpf_struct_ops_map_get_next_key, .map_lookup_elem = bpf_struct_ops_map_lookup_elem, .map_delete_elem = bpf_struct_ops_map_delete_elem, @@ -1281,7 +1283,31 @@ int bpf_struct_ops_for_each_prog(const void *kdata, } EXPORT_SYMBOL_GPL(bpf_struct_ops_for_each_prog); -static bool bpf_struct_ops_valid_to_reg(struct bpf_map *map) +void *bpf_struct_ops_map_kdata(struct bpf_map *map) +{ + struct bpf_struct_ops_map *st_map; + + st_map = container_of(map, struct bpf_struct_ops_map, map); + return st_map->kvalue.data; +} + +int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map) +{ + struct bpf_struct_ops_map *st_map; + + st_map = container_of(map, struct bpf_struct_ops_map, map); + return st_map->st_ops_desc->st_ops->cgroup_atype; +} + +void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map) +{ + struct bpf_struct_ops_map *st_map; + + st_map = container_of(map, struct bpf_struct_ops_map, map); + return st_map->st_ops_desc->st_ops->cfi_stubs; +} + +bool bpf_struct_ops_valid_to_reg(struct bpf_map *map) { struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; @@ -1297,8 +1323,7 @@ static void bpf_struct_ops_map_link_dealloc(struct bpf_link *link) struct bpf_struct_ops_map *st_map; st_link = container_of(link, struct bpf_struct_ops_link, link); - st_map = (struct bpf_struct_ops_map *) - rcu_dereference_protected(st_link->map, true); + st_map = (struct bpf_struct_ops_map *)st_link->map; if (st_map) { st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link); bpf_map_put(&st_map->map); @@ -1313,11 +1338,11 @@ static void bpf_struct_ops_map_link_show_fdinfo(const struct bpf_link *link, struct bpf_map *map; st_link = container_of(link, struct bpf_struct_ops_link, link); - rcu_read_lock(); - map = rcu_dereference(st_link->map); + mutex_lock(&update_mutex); + map = st_link->map; if (map) seq_printf(seq, "map_id:\t%d\n", map->id); - rcu_read_unlock(); + mutex_unlock(&update_mutex); } static int bpf_struct_ops_map_link_fill_link_info(const struct bpf_link *link, @@ -1327,11 +1352,31 @@ static int bpf_struct_ops_map_link_fill_link_info(const struct bpf_link *link, struct bpf_map *map; st_link = container_of(link, struct bpf_struct_ops_link, link); - rcu_read_lock(); - map = rcu_dereference(st_link->map); + mutex_lock(&update_mutex); + map = st_link->map; if (map) info->struct_ops.map_id = map->id; - rcu_read_unlock(); + mutex_unlock(&update_mutex); + return 0; +} + +int bpf_struct_ops_link_update_check(struct bpf_map *new_map, + struct bpf_map *old_map, + struct bpf_map *expected_old_map) +{ + struct bpf_struct_ops_map *st_map, *old_st_map; + + if (!old_map) + return -ENOLINK; + if (expected_old_map && old_map != expected_old_map) + return -EPERM; + + st_map = container_of(new_map, struct bpf_struct_ops_map, map); + old_st_map = container_of(old_map, struct bpf_struct_ops_map, map); + /* The new and old struct_ops must be the same type. */ + if (st_map->st_ops_desc != old_st_map->st_ops_desc) + return -EINVAL; + return 0; } @@ -1353,30 +1398,19 @@ static int bpf_struct_ops_map_link_update(struct bpf_link *link, struct bpf_map return -EOPNOTSUPP; mutex_lock(&update_mutex); - - old_map = rcu_dereference_protected(st_link->map, lockdep_is_held(&update_mutex)); - if (!old_map) { - err = -ENOLINK; - goto err_out; - } - if (expected_old_map && old_map != expected_old_map) { - err = -EPERM; + old_map = st_link->map; + err = bpf_struct_ops_link_update_check(new_map, old_map, expected_old_map); + if (err) goto err_out; - } old_st_map = container_of(old_map, struct bpf_struct_ops_map, map); - /* The new and old struct_ops must be the same type. */ - if (st_map->st_ops_desc != old_st_map->st_ops_desc) { - err = -EINVAL; - goto err_out; - } err = st_map->st_ops_desc->st_ops->update(st_map->kvalue.data, old_st_map->kvalue.data, link); if (err) goto err_out; bpf_map_inc(new_map); - rcu_assign_pointer(st_link->map, new_map); + WRITE_ONCE(st_link->map, new_map); bpf_map_put(old_map); err_out: @@ -1393,7 +1427,7 @@ static int bpf_struct_ops_map_link_detach(struct bpf_link *link) mutex_lock(&update_mutex); - map = rcu_dereference_protected(st_link->map, lockdep_is_held(&update_mutex)); + map = st_link->map; if (!map) { mutex_unlock(&update_mutex); return 0; @@ -1402,7 +1436,7 @@ static int bpf_struct_ops_map_link_detach(struct bpf_link *link) st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link); - RCU_INIT_POINTER(st_link->map, NULL); + WRITE_ONCE(st_link->map, NULL); /* Pair with bpf_map_get() in bpf_struct_ops_link_create() or * bpf_map_inc() in bpf_struct_ops_map_link_update(). */ @@ -1422,7 +1456,7 @@ static __poll_t bpf_struct_ops_map_link_poll(struct file *file, poll_wait(file, &st_link->wait_hup, pts); - return rcu_access_pointer(st_link->map) ? 0 : EPOLLHUP; + return READ_ONCE(st_link->map) ? 0 : EPOLLHUP; } static const struct bpf_link_ops bpf_struct_ops_map_lops = { @@ -1440,6 +1474,7 @@ int bpf_struct_ops_link_create(union bpf_attr *attr) struct bpf_link_primer link_primer; struct bpf_struct_ops_map *st_map; struct bpf_map *map; + int cgroup_atype; int err; map = bpf_map_get(attr->link_create.map_fd); @@ -1453,6 +1488,19 @@ int bpf_struct_ops_link_create(union bpf_attr *attr) goto err_out; } + cgroup_atype = st_map->st_ops_desc->st_ops->cgroup_atype; + if (cgroup_atype) { + err = cgroup_bpf_struct_ops_attach(map, attr); + bpf_map_put(map); + return err; + } + + if (memchr_inv(&attr->link_create.cgroup, 0, sizeof(attr->link_create.cgroup)) || + attr->link_create.target_fd) { + err = -EINVAL; + goto err_out; + } + link = kzalloc_obj(*link, GFP_USER); if (!link) { err = -ENOMEM; @@ -1478,7 +1526,7 @@ int bpf_struct_ops_link_create(union bpf_attr *attr) link = NULL; goto err_out; } - RCU_INIT_POINTER(link->map, map); + link->map = map; mutex_unlock(&update_mutex); return bpf_link_settle(&link_primer); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 9f33e95d5741..538129ed2418 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -20,15 +20,17 @@ #include <linux/btf.h> #include <linux/btf_ids.h> #include <linux/bpf.h> +#include <linux/bpf-cgroup.h> #include <linux/bpf_lsm.h> #include <linux/skmsg.h> #include <linux/perf_event.h> #include <linux/bsearch.h> -#include <linux/kobject.h> #include <linux/string.h> -#include <linux/sysfs.h> #include <linux/overflow.h> #include <linux/bitops.h> +#include <linux/unaligned.h> +#include <linux/vmalloc.h> +#include <crypto/sha2.h> #include <net/netfilter/nf_bpf_link.h> @@ -345,6 +347,9 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { [BTF_KIND_DECL_TAG] = "DECL_TAG", [BTF_KIND_TYPE_TAG] = "TYPE_TAG", [BTF_KIND_ENUM64] = "ENUM64", + [BTF_KIND_LOC_PARAM] = "LOC_PARAM", + [BTF_KIND_LOC_PROTO] = "LOC_PROTO", + [BTF_KIND_LOCSEC] = "LOCSEC", }; const char *btf_type_str(const struct btf_type *t) @@ -517,11 +522,27 @@ static bool btf_type_is_decl_tag(const struct btf_type *t) return BTF_INFO_KIND(t->info) == BTF_KIND_DECL_TAG; } +static bool btf_type_is_loc_param(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_LOC_PARAM; +} + +static bool btf_type_is_loc_proto(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_LOC_PROTO; +} + +static bool btf_type_is_locsec(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_LOCSEC; +} + static bool btf_type_nosize(const struct btf_type *t) { return btf_type_is_void(t) || btf_type_is_fwd(t) || btf_type_is_func(t) || btf_type_is_func_proto(t) || - btf_type_is_decl_tag(t); + btf_type_is_decl_tag(t) || btf_type_is_loc_param(t) || + btf_type_is_loc_proto(t) || btf_type_is_locsec(t); } static bool btf_type_nosize_or_null(const struct btf_type *t) @@ -768,7 +789,9 @@ static bool btf_type_is_resolve_source_only(const struct btf_type *t) { return btf_type_is_var(t) || btf_type_is_decl_tag(t) || - btf_type_is_datasec(t); + btf_type_is_datasec(t) || + btf_type_is_loc_proto(t) || + btf_type_is_locsec(t); } /* What types need to be resolved? @@ -796,7 +819,9 @@ static bool btf_type_needs_resolve(const struct btf_type *t) btf_type_is_var(t) || btf_type_is_func(t) || btf_type_is_decl_tag(t) || - btf_type_is_datasec(t); + btf_type_is_datasec(t) || + btf_type_is_loc_proto(t) || + btf_type_is_locsec(t); } /* t->size can be used */ @@ -877,16 +902,6 @@ static bool btf_name_offset_valid(const struct btf *btf, u32 offset) return offset < btf->hdr.str_len; } -static bool __btf_name_char_ok(char c, bool first) -{ - if ((first ? !isalpha(c) : - !isalnum(c)) && - c != '_' && - c != '.') - return false; - return true; -} - const char *btf_str_by_offset(const struct btf *btf, u32 offset) { while (offset < btf->start_str_off) @@ -899,20 +914,21 @@ const char *btf_str_by_offset(const struct btf *btf, u32 offset) return NULL; } +/* Names in BTF of Rust are not C identifiers. Allow any printable character */ static bool btf_name_valid_identifier(const struct btf *btf, u32 offset) { /* offset must be valid */ const char *src = btf_str_by_offset(btf, offset); const char *src_limit; - if (!__btf_name_char_ok(*src, true)) + if (!isprint(*src)) return false; /* set a limit on identifier length */ src_limit = src + KSYM_NAME_LEN; src++; while (*src && src < src_limit) { - if (!__btf_name_char_ok(*src, false)) + if (!isprint(*src)) return false; src++; } @@ -1065,6 +1081,8 @@ static const struct btf_type *btf_type_skip_qualifiers(const struct btf *btf, } #define BTF_SHOW_MAX_ITER 10 +/* nested structs and arrays shown; __btf_show_indent() has as many columns */ +#define BTF_SHOW_MAX_DEPTH 32 #define BTF_KIND_BIT(kind) (1ULL << kind) @@ -1458,8 +1476,18 @@ static void *btf_show_start_aggr_type(struct btf_show *show, const struct btf_type *t, u32 type_id, void *data) { - void *safe_data = btf_show_start_type(show, t, type_id, data); + void *safe_data; + + /* + * Structs and arrays are shown by recursion. BTF loading bounds the + * depth of one resolve walk, not how deep resolved types may nest. + */ + if (show->state.depth >= BTF_SHOW_MAX_DEPTH) { + show->state.status = -E2BIG; + return NULL; + } + safe_data = btf_show_start_type(show, t, type_id, data); if (!safe_data) return safe_data; @@ -1920,7 +1948,7 @@ void btf_set_base_btf(struct btf *btf, const struct btf *base_btf) { btf->base_btf = (struct btf *)base_btf; btf->start_id = btf_nr_types(base_btf); - btf->start_str_off = base_btf->hdr.str_len; + btf->start_str_off = base_btf->start_str_off + base_btf->hdr.str_len; } static int env_resolve_init(struct btf_verifier_env *env) @@ -2064,7 +2092,8 @@ static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env) * *elem_id: id of u32 * *total_nelems: (x * y). Hence, individual elem size is * (*type_size / *total_nelems) - * *type_id: id of type if it's changed within the function, 0 if not + * *type_id: ID of the returned type if leading modifiers were stripped; + * otherwise left unchanged. * * type: is not an array (e.g. const struct X) * return type: type "struct X" @@ -2072,7 +2101,8 @@ static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env) * *elem_type: same as return type ("struct X") * *elem_id: 0 * *total_nelems: 1 - * *type_id: id of type if it's changed within the function, 0 if not + * *type_id: ID of the returned type if leading modifiers were stripped; + * otherwise left unchanged. */ static const struct btf_type * __btf_resolve_size(const struct btf *btf, const struct btf_type *type, @@ -2105,7 +2135,8 @@ __btf_resolve_size(const struct btf *btf, const struct btf_type *type, case BTF_KIND_CONST: case BTF_KIND_RESTRICT: case BTF_KIND_TYPE_TAG: - id = type->type; + if (!array_type) + id = type->type; type = btf_type_by_id(btf, type->type); break; @@ -3538,6 +3569,28 @@ static int btf_type_tag_walk(const struct btf *btf, return 0; } +/* Whether the pointee of pointer type @t carries the type tag @name */ +static bool btf_ptr_has_type_tag(const struct btf *btf, const struct btf_type *t, const char *name) +{ + if (!btf_type_is_ptr(t)) + return false; + + for (t = btf_type_by_id(btf, t->type); btf_type_is_modifier(t); + t = btf_type_by_id(btf, t->type)) { + if (!btf_type_is_type_tag(t) || btf_type_kflag(t)) + continue; + if (!strcmp(__btf_name_by_offset(btf, t->name_off), name)) + return true; + } + + return false; +} + +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t) +{ + return btf_ptr_has_type_tag(btf, t, "arena"); +} + static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, u32 off, int sz, struct btf_field_info *info, u32 field_mask) { @@ -3679,6 +3732,7 @@ static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_ { BPF_TIMER, "bpf_timer", true }, { BPF_WORKQUEUE, "bpf_wq", true }, { BPF_TASK_WORK, "bpf_task_work", true }, + { BPF_RCU_HEAD, "bpf_rcu_head", true }, { BPF_LIST_HEAD, "bpf_list_head", false }, { BPF_LIST_NODE, "bpf_list_node", false }, { BPF_RB_ROOT, "bpf_rb_root", false }, @@ -3864,6 +3918,7 @@ static int btf_find_field_one(const struct btf *btf, case BPF_RB_NODE: case BPF_REFCOUNT: case BPF_TASK_WORK: + case BPF_RCU_HEAD: ret = btf_find_struct(btf, var_type, off, sz, field_type, info_cnt ? &info[0] : &tmp); if (ret < 0) @@ -4159,6 +4214,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type rec->wq_off = -EINVAL; rec->refcount_off = -EINVAL; rec->task_work_off = -EINVAL; + rec->rcu_head_off = -EINVAL; for (i = 0; i < cnt; i++) { field_type_size = btf_field_type_size(info_arr[i].type); if (info_arr[i].off + field_type_size > value_size) { @@ -4202,6 +4258,10 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type WARN_ON_ONCE(rec->task_work_off >= 0); rec->task_work_off = rec->fields[i].offset; break; + case BPF_RCU_HEAD: + WARN_ON_ONCE(rec->rcu_head_off >= 0); + rec->rcu_head_off = rec->fields[i].offset; + break; case BPF_REFCOUNT: WARN_ON_ONCE(rec->refcount_off >= 0); /* Cache offset for faster lookup at runtime */ @@ -4268,13 +4328,10 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec) { int i; - /* There are three types that signify ownership of some other type: - * kptr_ref, bpf_list_head, bpf_rb_root. - * kptr_ref only supports storing kernel types, which can't store - * references to program allocated local types. - * - * Hence we only need to ensure that bpf_{list_head,rb_root} ownership - * does not form cycles. + /* + * Check fields which require the complete BTF and initialize runtime + * metadata. Ownership relationships are validated after every record has + * been fixed up. */ if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & (BPF_GRAPH_ROOT | BPF_UPTR))) return 0; @@ -4305,51 +4362,88 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec) if (!meta) return -EFAULT; rec->fields[i].graph_root.value_rec = meta->record; + } + return 0; +} - /* We need to set value_rec for all root types, but no need - * to check ownership cycle for a type unless it's also a - * node type. - */ - if (!(rec->field_mask & BPF_GRAPH_NODE)) +static int btf_owned_type_idx(const struct btf *btf, struct btf_struct_metas *tab, + const struct btf_field *field) +{ + struct btf_struct_meta *meta; + u32 btf_id; + + if (field->type & BPF_GRAPH_ROOT) { + btf_id = field->graph_root.value_btf_id; + } else if (field->type == BPF_KPTR_REF || field->type == BPF_KPTR_PERCPU) { + if (btf_is_kernel(field->kptr.btf)) + return -ENOENT; + btf_id = field->kptr.btf_id; + } else { + return -ENOENT; + } + + meta = btf_find_struct_meta(btf, btf_id); + if (!meta) + return field->type & BPF_GRAPH_ROOT ? -EFAULT : -ENOENT; + return meta - tab->types; +} + +/* + * Each ownership edge adds kernel frames through bpf_obj_free_fields() and + * __bpf_obj_drop_impl(). Keep the bound deliberately small because object + * destruction can itself run below a BPF call chain. A final pointee without + * special fields is not present in the struct metadata table and adds only a + * non-recursing drop. + */ +#define BTF_MAX_OWNERSHIP_DEPTH 8 + +static int btf_ownership_depth(const struct btf *btf, + struct btf_struct_metas *tab, u8 *depth, + int idx, int depth_left) +{ + const struct btf_record *rec = tab->types[idx].record; + int i, ret, max_depth = 0; + + if (!depth_left) + return -ELOOP; + if (depth[idx]) + goto done; + + for (i = 0; i < rec->cnt; i++) { + ret = btf_owned_type_idx(btf, tab, &rec->fields[i]); + if (ret == -ENOENT) continue; + if (ret < 0) + return ret; + ret = btf_ownership_depth(btf, tab, depth, ret, depth_left - 1); + if (ret < 0) + return ret; + max_depth = max(max_depth, ret); + } + depth[idx] = max_depth + 1; +done: + return depth[idx] > depth_left ? -ELOOP : depth[idx]; +} - /* We need to ensure ownership acyclicity among all types. The - * proper way to do it would be to topologically sort all BTF - * IDs based on the ownership edges, since there can be multiple - * bpf_{list_head,rb_node} in a type. Instead, we use the - * following resaoning: - * - * - A type can only be owned by another type in user BTF if it - * has a bpf_{list,rb}_node. Let's call these node types. - * - A type can only _own_ another type in user BTF if it has a - * bpf_{list_head,rb_root}. Let's call these root types. - * - * We ensure that if a type is both a root and node, its - * element types cannot be root types. - * - * To ensure acyclicity: - * - * When A is an root type but not a node, its ownership - * chain can be: - * A -> B -> C - * Where: - * - A is an root, e.g. has bpf_rb_root. - * - B is both a root and node, e.g. has bpf_rb_node and - * bpf_list_head. - * - C is only an root, e.g. has bpf_list_node - * - * When A is both a root and node, some other type already - * owns it in the BTF domain, hence it can not own - * another root type through any of the ownership edges. - * A -> B - * Where: - * - A is both an root and node. - * - B is only an node. - */ - if (meta->record->field_mask & BPF_GRAPH_ROOT) - return -ELOOP; +static int btf_check_ownership_depth(const struct btf *btf, + struct btf_struct_metas *tab) +{ + u8 *depth; + int i, ret = 0; + + depth = kvcalloc(tab->cnt, sizeof(*depth), GFP_KERNEL | __GFP_NOWARN); + if (!depth) + return -ENOMEM; + + for (i = 0; i < tab->cnt; i++) { + ret = btf_ownership_depth(btf, tab, depth, i, + BTF_MAX_OWNERSHIP_DEPTH); + if (ret < 0) + break; + ret = 0; } - return 0; + kvfree(depth); + return ret; } static void __btf_struct_show(const struct btf *btf, const struct btf_type *t, @@ -4713,6 +4807,336 @@ static const struct btf_kind_operations enum64_ops = { .show = btf_enum64_show, }; +static s32 btf_loc_param_check_meta(struct btf_verifier_env *env, + const struct btf_type *t, + u32 meta_left) +{ + const struct btf_loc_param *p = btf_loc_param(t); + u32 size, meta_needed, vlen = btf_vlen(t); + + meta_needed = sizeof(*p) + sizeof(__u32) * vlen; + if (meta_left < meta_needed) { + btf_verifier_log_basic(env, t, + "meta_left:%u meta_needed:%u", + meta_left, meta_needed); + return -EINVAL; + } + + if (t->name_off) { + btf_verifier_log_type(env, t, "Invalid name"); + return -EINVAL; + } + size = t->size; + if (!size || size > 16) { + btf_verifier_log_type(env, t, "Unexpected size"); + return -EINVAL; + } + + if (btf_type_kflag(t)) { + btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); + return -EINVAL; + } + /* All LOC_PARAMs have vlen of at least 1 */ + if (vlen < 1) + goto invalid_vlen; + + switch (p->flags) { + case BTF_LOC_PARAM_CONST: + case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_SIGNED: + case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_ADDR: + case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_ADDR | BTF_LOC_PARAM_SIGNED: + if (vlen < 1 || vlen > 2) + goto invalid_vlen; + if (!size || size > sizeof(__u64)) + goto invalid_vlen; + /* Ensure size/vlen are consistent */ + if (vlen != (size > sizeof(__u32) ? 2 : 1)) + goto invalid_vlen; + break; + case BTF_LOC_PARAM_REG: + /* Reg or reg/reg pair */ + if (vlen > 2) + goto invalid_vlen; + break; + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF: + /* A reg deref without offset can only have vlen 1 */ + if (vlen != 1) + goto invalid_vlen; + break; + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_OFFSET: + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_OFFSET | BTF_LOC_PARAM_SIGNED: + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF | BTF_LOC_PARAM_OFFSET: + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF | BTF_LOC_PARAM_OFFSET | + BTF_LOC_PARAM_SIGNED: + if (vlen < 2 || vlen > 3) + goto invalid_vlen; + break; + default: + btf_verifier_log_type(env, t, "Invalid flags"); + return -EINVAL; + } + + btf_verifier_log_type(env, t, NULL); + + return meta_needed; + +invalid_vlen: + btf_verifier_log_type(env, t, "Invalid vlen"); + return -EINVAL; +} + +static void btf_loc_param_log(struct btf_verifier_env *env, + const struct btf_type *t) +{ + const struct btf_loc_param *p = btf_loc_param(t); + u32 i, vlen = btf_vlen(t); + + btf_verifier_log(env, "size=%u vlen=%u flags=0x%x", t->size, vlen, p->flags); + for (i = 0; i < vlen; i++) + btf_verifier_log(env, ", %u", p->values[i]); +} + +static const struct btf_kind_operations loc_param_ops = { + .check_meta = btf_loc_param_check_meta, + .resolve = btf_df_resolve, + .check_member = btf_df_check_member, + .check_kflag_member = btf_df_check_kflag_member, + .log_details = btf_loc_param_log, + .show = btf_df_show, +}; + +static s32 btf_loc_proto_check_meta(struct btf_verifier_env *env, + const struct btf_type *t, + u32 meta_left) +{ + u32 meta_needed; + + meta_needed = sizeof(__u32) * btf_type_vlen(t); + + if (meta_left < meta_needed) { + btf_verifier_log_basic(env, t, + "meta_left:%u meta_needed:%u", + meta_left, meta_needed); + return -EINVAL; + } + + if (t->name_off) { + btf_verifier_log_type(env, t, "Invalid name"); + return -EINVAL; + } + + if (btf_type_kflag(t)) { + btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); + return -EINVAL; + } + + btf_verifier_log_type(env, t, NULL); + + return meta_needed; +} + +static void btf_loc_proto_log(struct btf_verifier_env *env, + const struct btf_type *t) +{ + const __u32 *params = btf_loc_proto_params(t); + u32 i, nr_params = btf_type_vlen(t); + + btf_verifier_log(env, "vlen=%u", nr_params); + for (i = 0; i < nr_params; i++, params++) + btf_verifier_log(env, ", %u", *params); +} + +static int btf_loc_proto_resolve(struct btf_verifier_env *env, + const struct resolve_vertex *v) +{ + const struct btf_type *t = v->t; + const __u32 *params = btf_loc_proto_params(t); + u32 i, nr_params = btf_type_vlen(t); + struct btf *btf = env->btf; + + if (t->type) { + btf_verifier_log_type(env, t, "Invalid loc_proto type"); + return -EINVAL; + } + + for (i = 0; i < nr_params; i++) { + const struct btf_type *param_type; + u32 param_type_id = params[i]; + + if (!param_type_id) + continue; + + param_type = btf_type_by_id(btf, param_type_id); + if (!param_type || !btf_type_is_loc_param(param_type)) { + btf_verifier_log_type(env, t, + "Invalid loc_param#%u", i + 1); + return -EINVAL; + } + } + + env_stack_pop_resolved(env, 0, 0); + return 0; +} + +static const struct btf_kind_operations loc_proto_ops = { + .check_meta = btf_loc_proto_check_meta, + .resolve = btf_loc_proto_resolve, + .check_member = btf_df_check_member, + .check_kflag_member = btf_df_check_kflag_member, + .log_details = btf_loc_proto_log, + .show = btf_df_show, +}; + +__printf(4, 5) +static void btf_verifier_log_loc(struct btf_verifier_env *env, + const struct btf_type *locsec_type, + const struct btf_loc *loc, + const char *fmt, ...) +{ + struct bpf_verifier_log *log = &env->log; + va_list args; + + if (!bpf_verifier_log_needed(log)) + return; + if (log->level == BPF_LOG_KERNEL && !fmt) + return; + if (env->phase != CHECK_META) + btf_verifier_log_type(env, locsec_type, NULL); + + __btf_verifier_log(log, "\t func=%u loc_proto=%u offset=%u", + loc->func, loc->loc_proto, loc->offset); + if (fmt && *fmt) { + __btf_verifier_log(log, " "); + va_start(args, fmt); + bpf_verifier_vlog(log, fmt, args); + va_end(args); + } + + __btf_verifier_log(log, "\n"); +} + +static s32 btf_locsec_check_meta(struct btf_verifier_env *env, + const struct btf_type *t, + u32 meta_left) +{ + u32 i, meta_needed, vlen = btf_type_vlen(t); + const struct btf_loc *loc; + + meta_needed = sizeof(struct btf_loc) * vlen; + + if (meta_left < meta_needed) { + btf_verifier_log_basic(env, t, + "meta_left:%u meta_needed:%u", + meta_left, meta_needed); + return -EINVAL; + } + + if (btf_type_kflag(t)) { + btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); + return -EINVAL; + } + + if (!t->name_off || + !btf_name_valid_section(env->btf, t->name_off)) { + btf_verifier_log_type(env, t, "Invalid name"); + return -EINVAL; + } + + for_each_loc(i, t, loc) { + /* A loc func, loc proto cannot be in type void */ + if (!loc->func || !BTF_TYPE_ID_VALID(loc->func)) { + btf_verifier_log_loc(env, t, loc, "Invalid func"); + return -EINVAL; + } + if (!loc->loc_proto || !BTF_TYPE_ID_VALID(loc->loc_proto)) { + btf_verifier_log_loc(env, t, loc, "Invalid loc_proto"); + return -EINVAL; + } + btf_verifier_log_loc(env, t, loc, NULL); + } + + return meta_needed; +} + +static void btf_locsec_log(struct btf_verifier_env *env, + const struct btf_type *t) +{ + btf_verifier_log(env, "vlen=%u", btf_type_vlen(t)); +} + +static int btf_locsec_resolve(struct btf_verifier_env *env, + const struct resolve_vertex *v) +{ + const struct btf_type *t = v->t; + const struct btf_loc *loc; + struct btf *btf = env->btf; + u32 i; + + if (t->type) { + btf_verifier_log_type(env, t, "Invalid locsec type"); + return -EINVAL; + } + + env->resolve_mode = RESOLVE_TBD; + for (i = v->next_member, loc = btf_type_loc_secinfo(t) + i; + i < btf_type_vlen(t); i++, loc++) { + const struct btf_type *func_type, *func_proto_type; + const struct btf_type *loc_proto_type; + u32 func_type_id = loc->func; + u32 loc_proto_type_id = loc->loc_proto; + u32 proto_vlen; + + func_type = btf_type_by_id(btf, func_type_id); + if (!func_type || !btf_type_is_func(func_type)) { + btf_verifier_log_type(env, t, + "Invalid func#%u", i + 1); + return -EINVAL; + } + func_proto_type = btf_type_by_id(btf, func_type->type); + if (!func_proto_type || !btf_type_is_func_proto(func_proto_type)) { + btf_verifier_log_type(env, t, + "Invalid func#%u", i + 1); + return -EINVAL; + } + proto_vlen = btf_vlen(func_proto_type); + + if (!env_type_is_resolved(env, func_type_id)) { + env_stack_set_next_member(env, i); + return env_stack_push(env, func_type, func_type_id); + } + + loc_proto_type = btf_type_by_id(btf, loc_proto_type_id); + if (!loc_proto_type || !btf_type_is_loc_proto(loc_proto_type)) { + btf_verifier_log_type(env, t, + "Invalid loc_proto#%u", i + 1); + return -EINVAL; + } + if (proto_vlen != btf_vlen(loc_proto_type)) { + btf_verifier_log_type(env, t, + "Mismatched vlen for loc_proto#%u", i + 1); + return -EINVAL; + } + + if (!env_type_is_resolved(env, loc_proto_type_id)) { + env_stack_set_next_member(env, i + 1); + return env_stack_push(env, loc_proto_type, + loc_proto_type_id); + } + } + + env_stack_pop_resolved(env, 0, 0); + return 0; +} + +static const struct btf_kind_operations locsec_ops = { + .check_meta = btf_locsec_check_meta, + .resolve = btf_locsec_resolve, + .check_member = btf_df_check_member, + .check_kflag_member = btf_df_check_kflag_member, + .log_details = btf_locsec_log, + .show = btf_df_show, +}; + static s32 btf_func_proto_check_meta(struct btf_verifier_env *env, const struct btf_type *t, u32 meta_left) @@ -4995,7 +5419,7 @@ static int btf_datasec_resolve(struct btf_verifier_env *env, env->resolve_mode = RESOLVE_TBD; for_each_vsi_from(i, v->next_member, v->t, vsi) { - u32 var_type_id = vsi->type, type_id, type_size = 0; + u32 var_type_id = vsi->type, type_id; const struct btf_type *var_type = btf_type_by_id(env->btf, var_type_id); if (!var_type || !btf_type_is_var(var_type)) { @@ -5010,16 +5434,16 @@ static int btf_datasec_resolve(struct btf_verifier_env *env, return env_stack_push(env, var_type, var_type_id); } + /* + * The variable can be smaller than its type. It's a piece of + * a variable that the compiler split then, with the type of + * the whole variable. + */ type_id = var_type->type; - if (!btf_type_id_size(btf, &type_id, &type_size)) { + if (!btf_type_id_size(btf, &type_id, NULL)) { btf_verifier_log_vsi(env, v->t, vsi, "Invalid type"); return -EINVAL; } - - if (vsi->size < type_size) { - btf_verifier_log_vsi(env, v->t, vsi, "Invalid size"); - return -EINVAL; - } } env_stack_pop_resolved(env, 0, 0); @@ -5032,6 +5456,16 @@ static void btf_datasec_log(struct btf_verifier_env *env, btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); } +/* A piece of a variable is smaller than its type, which is the one of the whole variable */ +static bool btf_var_is_piece(const struct btf *btf, const struct btf_type *var, + const struct btf_var_secinfo *vsi) +{ + u32 size; + + return IS_ERR(btf_resolve_size(btf, btf_type_by_id(btf, var->type), &size)) || + vsi->size < size; +} + static void btf_datasec_show(const struct btf *btf, const struct btf_type *t, u32 type_id, void *data, u8 bits_offset, @@ -5039,6 +5473,7 @@ static void btf_datasec_show(const struct btf *btf, { const struct btf_var_secinfo *vsi; const struct btf_type *var; + bool comma = false; u32 i; if (!btf_show_start_type(show, t, type_id, data)) @@ -5048,8 +5483,12 @@ static void btf_datasec_show(const struct btf *btf, __btf_name_by_offset(btf, t->name_off)); for_each_vsi(i, t, vsi) { var = btf_type_by_id(btf, vsi->type); - if (i) + /* there is less data than the type takes */ + if (btf_var_is_piece(btf, var, vsi)) + continue; + if (comma) btf_show(show, ","); + comma = true; btf_type_ops(var)->show(btf, var, vsi->type, data + vsi->offset, bits_offset, show); } @@ -5350,6 +5789,10 @@ static int btf_func_check(struct btf_verifier_env *env, return -EINVAL; } + /* Rust leaves out names of some arguments of static functions */ + if (btf_func_linkage(t) == BTF_FUNC_STATIC) + return 0; + args = (const struct btf_param *)(proto_type + 1); nr_args = btf_type_vlen(proto_type); for (i = 0; i < nr_args; i++) { @@ -5382,6 +5825,9 @@ static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = { [BTF_KIND_DECL_TAG] = &decl_tag_ops, [BTF_KIND_TYPE_TAG] = &modifier_ops, [BTF_KIND_ENUM64] = &enum64_ops, + [BTF_KIND_LOC_PARAM] = &loc_param_ops, + [BTF_KIND_LOC_PROTO] = &loc_proto_ops, + [BTF_KIND_LOCSEC] = &locsec_ops, }; static s32 btf_check_meta(struct btf_verifier_env *env, @@ -5456,7 +5902,8 @@ static bool btf_resolve_valid(struct btf_verifier_env *env, if (!env_type_is_resolved(env, type_id)) return false; - if (btf_type_is_struct(t) || btf_type_is_datasec(t)) + if (btf_type_is_struct(t) || btf_type_is_datasec(t) || + btf_type_is_loc_proto(t) || btf_type_is_locsec(t)) return !btf_resolved_type_id(btf, type_id) && !btf_resolved_type_size(btf, type_id); @@ -5781,7 +6228,7 @@ static int btf_parse_hdr(struct btf_verifier_env *env) return btf_check_sec_info(env, btf_data_size); } -static const char *alloc_obj_fields[] = { +static const char * const alloc_obj_fields[] = { "bpf_spin_lock", "bpf_list_head", "bpf_list_node", @@ -6044,6 +6491,10 @@ static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, if (err < 0) goto errout_meta; } + + err = btf_check_ownership_depth(btf, struct_meta_tab); + if (err < 0) + goto errout_meta; } err = bpf_log_attr_finalize(attr_log, &env->log); @@ -6492,15 +6943,137 @@ __u32 btf_relocate_id(const struct btf *btf, __u32 id) #ifdef CONFIG_DEBUG_INFO_BTF_MODULES +static struct btf *btf_parse_split(struct btf_verifier_env *env, + const char *name, const void *data, + unsigned int data_size, struct btf *base_btf) +{ + struct btf *btf; + int err; + + btf = kzalloc_obj(*btf, GFP_KERNEL | __GFP_NOWARN); + if (!btf) + return ERR_PTR(-ENOMEM); + env->btf = btf; + + btf_set_base_btf(btf, base_btf); + btf->kernel_btf = true; + btf->named_start_id = 0; + strscpy(btf->name, name); + + btf->data = kvmemdup(data, data_size, GFP_KERNEL | __GFP_NOWARN); + if (!btf->data) { + err = -ENOMEM; + goto errout; + } + btf->data_size = data_size; + + err = btf_parse_hdr(env); + if (err) + goto errout; + + btf->nohdr_data = btf->data + btf->hdr.hdr_len; + + err = btf_parse_str_sec(env); + if (err) + goto errout; + + err = btf_check_all_metas(env); + if (err) + goto errout; + + err = btf_check_modifier_chain_length(env, btf, btf_nr_types(base_btf)); + if (err) + goto errout; + + return btf; + +errout: + btf_free(btf); + return ERR_PTR(err); +} + +static int btf_rebase_inline(struct btf *inline_btf, + const struct btf *module_btf, + const u32 *module_id_map, + const u32 *module_str_map, + u32 old_module_type_cnt) +{ + u32 old_start_id = inline_btf->start_id; + u32 old_start_str_off = inline_btf->start_str_off; + u32 old_module_start_str_off = old_start_str_off - module_btf->hdr.str_len; + u32 new_start_id = btf_nr_types(module_btf); + u32 new_start_str_off = module_btf->start_str_off + module_btf->hdr.str_len; + s64 id_delta = (s64)new_start_id - old_start_id; + s64 str_delta = (s64)new_start_str_off - old_start_str_off; + u32 i; + + /* + * The inline BTF was parsed relative to the original module BTF. Its + * base IDs must therefore use the map generated when that BTF was + * relocated, while IDs for inline-local types only move by the change + * in the module BTF's starting ID. + */ + for (i = 0; i < inline_btf->nr_types; i++) { + struct btf_field_iter it; + struct btf_type *t = inline_btf->types[i]; + u32 *id, *str_off; + int err; + + err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS); + if (err) + return err; + while ((id = btf_field_iter_next(&it))) { + if (!*id) + continue; + if (*id < old_module_type_cnt) { + if (module_id_map) + *id = module_id_map[*id]; + } else if (*id >= old_start_id) { + *id += id_delta; + } else { + return -EINVAL; + } + } + + err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS); + if (err) + return err; + while ((str_off = btf_field_iter_next(&it))) { + if (!*str_off) + continue; + if (*str_off < old_module_start_str_off) { + /* Vmlinux strings retain their offsets for in-tree modules. */ + if (!module_id_map) + continue; + if (!module_str_map || !module_str_map[*str_off]) + return -EINVAL; + *str_off = module_str_map[*str_off]; + continue; + } + *str_off += str_delta; + } + } + + btf_set_base_btf(inline_btf, module_btf); + btf_check_sorted(inline_btf); + return 0; +} + static struct btf *btf_parse_module(const char *module_name, const void *data, unsigned int data_size, void *base_data, - unsigned int base_data_size) + unsigned int base_data_size, + const void *inline_data, + unsigned int inline_data_size, + void **relocated_inline_data) { - struct btf *btf = NULL, *vmlinux_btf, *base_btf = NULL; + struct btf *btf = NULL, *inline_btf = NULL, *vmlinux_btf, *base_btf = NULL; struct btf_verifier_env *env = NULL; struct bpf_verifier_log *log; + u32 *module_str_map = NULL; + u32 old_module_type_cnt; int err = 0; + *relocated_inline_data = NULL; vmlinux_btf = bpf_get_btf_vmlinux(); if (IS_ERR(vmlinux_btf)) return vmlinux_btf; @@ -6524,67 +7097,68 @@ static struct btf *btf_parse_module(const char *module_name, const void *data, base_btf = vmlinux_btf; } - btf = kzalloc_obj(*btf, GFP_KERNEL | __GFP_NOWARN); - if (!btf) { - err = -ENOMEM; + btf = btf_parse_split(env, module_name, data, data_size, base_btf); + if (IS_ERR(btf)) { + err = PTR_ERR(btf); + btf = NULL; goto errout; } - env->btf = btf; + old_module_type_cnt = btf_nr_types(btf); - btf->base_btf = base_btf; - btf->start_id = base_btf->nr_types; - btf->start_str_off = base_btf->hdr.str_len; - btf->kernel_btf = true; - btf->named_start_id = 0; - strscpy(btf->name, module_name); - - btf->data = kvmemdup(data, data_size, GFP_KERNEL | __GFP_NOWARN); - if (!btf->data) { - err = -ENOMEM; - goto errout; + if (inline_data_size) { + inline_btf = btf_parse_split(env, module_name, inline_data, + inline_data_size, btf); + if (IS_ERR(inline_btf)) { + pr_warn("failed to validate module [%s] inline BTF: %ld\n", + module_name, PTR_ERR(inline_btf)); + inline_btf = NULL; + } } - btf->data_size = data_size; - - err = btf_parse_hdr(env); - if (err) - goto errout; - - btf->nohdr_data = btf->data + btf->hdr.hdr_len; - - err = btf_parse_str_sec(env); - if (err) - goto errout; - - err = btf_check_all_metas(env); - if (err) - goto errout; - - err = btf_check_modifier_chain_length(env, btf, btf_nr_types(base_btf)); - if (err) - goto errout; if (base_btf != vmlinux_btf) { - err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map); + err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map, + &module_str_map); if (err) goto errout; btf_free(base_btf); base_btf = vmlinux_btf; } - btf_verifier_env_free(env); + if (inline_btf) { + err = btf_rebase_inline(inline_btf, btf, btf->base_id_map, + module_str_map, old_module_type_cnt); + if (err) { + pr_warn("failed to relocate module [%s] inline BTF: %d\n", + module_name, err); + btf_free(inline_btf); + } else { + *relocated_inline_data = inline_btf->data; + inline_btf->data = NULL; + btf_free(inline_btf); + } + } + + /* + * With a distilled base, btf_relocate() replaces the base BTF and + * rewrites string offsets. Check ordering only after that final BTF + * view has been established, so named_start_id describes the BTF used + * by name lookups. + */ btf_check_sorted(btf); + btf_verifier_env_free(env); + kvfree(module_str_map); refcount_set(&btf->refcnt, 1); return btf; errout: + kvfree(module_str_map); btf_verifier_env_free(env); + if (inline_btf) + btf_free(inline_btf); if (!IS_ERR(base_btf) && base_btf != vmlinux_btf) btf_free(base_btf); - if (btf) { - kvfree(btf->data); - kvfree(btf->types); - kfree(btf); - } + if (btf) + btf_free(btf); return ERR_PTR(err); } @@ -6978,8 +7552,9 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, } /* - * Check for PTR_TO_RDONLY_BUF_OR_NULL, PTR_TO_RDWR_BUF_OR_NULL or - * PTR_TO_ARENA (both nullable and non-nullable cases). + * Check for PTR_TO_RDONLY_BUF_OR_NULL, PTR_TO_RDWR_BUF_OR_NULL, + * PTR_TO_ARENA (both nullable and non-nullable cases) or fixed-size + * PTR_TO_MEM. */ for (i = 0; i < prog->aux->ctx_arg_info_size; i++) { const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i]; @@ -6989,8 +7564,10 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, flag = type_flag(ctx_arg_info->reg_type); if (ctx_arg_info->offset == off && (type == PTR_TO_ARENA || + type == PTR_TO_MEM || (type == PTR_TO_BUF && (flag & PTR_MAYBE_NULL)))) { info->reg_type = ctx_arg_info->reg_type; + info->mem_size = ctx_arg_info->mem_size; return true; } } @@ -7117,18 +7694,60 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, } EXPORT_SYMBOL_GPL(btf_ctx_access); +#define MAX_ARRAYS_WALK 16 + +/* Description of an array crossed while walking a BTF access chain */ +struct array_access { + u32 id; /* BTF_KIND_ARRAY type id */ + u32 off; /* offset of the access within the array */ +}; + enum bpf_struct_walk_result { /* < 0 error */ WALK_SCALAR = 0, WALK_PTR, WALK_PTR_UNTRUSTED, + WALK_PTR_ARENA, WALK_STRUCT, }; +static const struct btf_member *find_flex_member(const struct btf *btf, const struct btf_type *t) +{ + const struct btf_array *array; + const struct btf_type *mtype; + const struct btf_member *member; + u32 vlen = btf_type_vlen(t); + + if (vlen == 0) + return NULL; + + member = btf_type_member(t) + vlen - 1; + mtype = btf_type_skip_modifiers(btf, member->type, NULL); + if (!btf_type_is_array(mtype)) + return NULL; + + array = (const struct btf_array *)(mtype + 1); + if (array->nelems != 0) + return NULL; + + return member; +} + +static int push_array(struct array_access *arrays, u32 *arrays_cnt, struct array_access item) +{ + if (arrays == NULL) + return 0; + if (*arrays_cnt == MAX_ARRAYS_WALK) + return -1; + arrays[(*arrays_cnt)++] = item; + return 0; +} + static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, const struct btf_type *t, int off, int size, u32 *next_btf_id, enum bpf_type_flag *flag, - const char **field_name, bool walk_flex_arrays) + const char **field_name, bool walk_flex_arrays, + struct array_access *arrays, u32 *arrays_cnt) { u32 i, moff, mtrue_end, msize = 0, total_nelems = 0; const struct btf_type *mtype, *elem_type = NULL; @@ -7163,22 +7782,19 @@ again: /* If the last element is a variable size array, we may * need to relax the rule. */ - if (vlen == 0) + member = find_flex_member(btf, t); + if (!member) goto error; - member = btf_type_member(t) + vlen - 1; - mtype = btf_type_skip_modifiers(btf, member->type, - NULL); - if (!btf_type_is_array(mtype)) + moff = __btf_member_bit_offset(t, member) / 8; + if (off < moff) goto error; + mtype = btf_type_skip_modifiers(btf, member->type, &mid); array_elem = (struct btf_array *)(mtype + 1); - if (array_elem->nelems != 0) - goto error; - moff = __btf_member_bit_offset(t, member) / 8; - if (off < moff) - goto error; + if (push_array(arrays, arrays_cnt, (struct array_access){ mid, off - moff })) + return -E2BIG; /* allow structure and integer */ t = btf_type_skip_modifiers(btf, array_elem->type, @@ -7187,7 +7803,7 @@ again: if (btf_type_is_int(t)) return WALK_SCALAR; - if (!btf_type_is_struct(t)) + if (!btf_type_is_struct(t) || !t->size) goto error; off = (off - moff) % t->size; @@ -7301,6 +7917,9 @@ error: * the array's element as long as it is * within the mtrue_end boundary. */ + if (push_array(arrays, arrays_cnt, + (struct array_access){ mid, off - moff })) + return -E2BIG; /* skip empty array */ if (moff == mtrue_end) @@ -7361,6 +7980,38 @@ error: id = ctx.id; stype = ctx.t; + /* + * __nullable combines with any of the above: it only says + * that the pointer may be NULL. The caller decides whether + * that matters for the register type it derives. + */ + if (btf_ptr_has_type_tag(btf, mtype, "nullable")) + tmp_flag |= PTR_MAYBE_NULL; + + /* + * An __arena member of a kernel struct holds the kernel + * address of arena memory, which the verifier turns into + * an arena pointer on load. Program BTF is not affected: + * there the tag is what the compiler emits for the arena + * address space. An offset into a union resolves to its + * first member, so a program reading another member there + * would get an arena pointer it did not ask for; the walk + * marked such a union untrusted. + */ + if (btf_is_kernel(btf) && btf_type_is_arena_ptr(btf, mtype)) { + if (*flag & PTR_UNTRUSTED) { + bpf_log(log, + "arena pointer member %s of %s is inside a union\n", + mname, tname); + return -EACCES; + } + *next_btf_id = id; + *flag |= tmp_flag & PTR_MAYBE_NULL; + if (field_name) + *field_name = mname; + return WALK_PTR_ARENA; + } + if (btf_type_is_struct(stype)) { *next_btf_id = id; *flag |= tmp_flag; @@ -7393,15 +8044,29 @@ error: int btf_struct_access(struct bpf_verifier_log *log, const struct bpf_reg_state *reg, - int off, int size, enum bpf_access_type atype __maybe_unused, + int _off, int size, enum bpf_access_type atype __maybe_unused, u32 *next_btf_id, enum bpf_type_flag *flag, const char **field_name) { const struct btf *btf = reg->btf; enum bpf_type_flag tmp_flag = 0; + struct array_access arrays[MAX_ARRAYS_WALK]; + const struct btf_member *member; const struct btf_type *t; + u32 i, arrays_cnt = 0; u32 id = reg->btf_id; - int err; + s64 min_off, max_off, flex_off; + int err, off, ret; + + if (check_add_overflow(reg_smin(reg), (s64)_off, &min_off) || + check_add_overflow(reg_smax(reg), (s64)_off, &max_off) || + min_off != (int)min_off) { + bpf_log(log, + "offset computation overflows: register offset range is [%lld, %lld], instruction offset is %d\n", + reg_smin(reg), reg_smax(reg), _off); + return -EINVAL; + } + off = min_off; while (type_is_alloc(reg->type)) { struct btf_struct_meta *meta; @@ -7428,26 +8093,35 @@ int btf_struct_access(struct bpf_verifier_log *log, t = btf_type_by_id(btf, id); do { err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, - field_name, !type_is_alloc(reg->type)); - + field_name, !type_is_alloc(reg->type), arrays, &arrays_cnt); switch (err) { case WALK_PTR: /* For local types, the destination register cannot * become a pointer again. */ - if (type_is_alloc(reg->type)) - return SCALAR_VALUE; + if (type_is_alloc(reg->type)) { + ret = SCALAR_VALUE; + goto check_variable_offset; + } /* If we found the pointer or scalar on t+off, * we're done. */ *next_btf_id = id; *flag = tmp_flag; - return PTR_TO_BTF_ID; + ret = PTR_TO_BTF_ID; + goto check_variable_offset; case WALK_PTR_UNTRUSTED: *flag = MEM_RDONLY | PTR_UNTRUSTED; - return PTR_TO_MEM; + ret = PTR_TO_MEM; + goto check_variable_offset; + case WALK_PTR_ARENA: + *next_btf_id = id; + *flag = tmp_flag; + ret = PTR_TO_ARENA; + goto check_variable_offset; case WALK_SCALAR: - return SCALAR_VALUE; + ret = SCALAR_VALUE; + goto check_variable_offset; case WALK_STRUCT: /* We found nested struct, so continue the search * by diving in it. At this point the offset is @@ -7467,6 +8141,55 @@ int btf_struct_access(struct bpf_verifier_log *log, } while (t); return -EINVAL; + +check_variable_offset: + if (min_off == max_off) + return ret; + + /* Find an offset at which access would go to a flexible array tail (if any). */ + t = btf_type_skip_modifiers(btf, reg->btf_id, NULL); + member = find_flex_member(btf, t); + flex_off = member ? __btf_member_bit_offset(t, member) / 8 : S64_MAX; + + /* + * If this is a varying offset access, the step recorded within a register + * should correspond to one of the arrays visited while walking. + */ + for (i = 0; i < arrays_cnt; i++) { + s64 array_start, array_end, access_end; + u32 asize, esize, elem_id; + + /* + * 'arrays' records top-level arrays only, use __btf_resolve_size() + * to get flattened representation. + */ + t = btf_type_by_id(btf, arrays[i].id); + if (IS_ERR(__btf_resolve_size(btf, t, &asize, NULL, &elem_id, NULL, NULL)) || + IS_ERR(btf_resolve_size(btf, btf_type_by_id(btf, elem_id), &esize))) + continue; + /* Make sure every accessed offset lands on the same field of some element. */ + if (esize == 0 || reg->step % esize != 0) + continue; + + array_start = min_off - arrays[i].off; + /* An array within the flexible tail has no upper bound to exceed */ + if (array_start >= flex_off && btf_type_is_array(t) && + btf_type_array(t)->nelems == 0) + return ret; + + /* the 'size' bytes read at max_off must stay within the array */ + if (check_add_overflow(array_start, (s64)asize, &array_end) || + check_add_overflow(max_off, (s64)size, &access_end) || + access_end > array_end) + continue; + return ret; + } + + bpf_log(log, + "invalid variable offset access into struct %s: offsets [%lld, %lld] size %d step %d do not match any array\n", + btf_name_by_offset(btf, btf_type_by_id(btf, reg->btf_id)->name_off), + min_off, max_off, size, reg->step); + return -EINVAL; } /* Check that two BTF types, each specified as an BTF object + id, are exactly @@ -7507,7 +8230,7 @@ again: if (!type) return false; err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL, - walk_flex_arrays); + walk_flex_arrays, NULL, NULL); if (err != WALK_STRUCT) return false; @@ -7563,6 +8286,9 @@ static u8 __get_arg_fmodel_flags(const struct btf *btf, { u8 flags = __get_type_fmodel_flags(t); + if (btf_func_arg_align(btf, t) > sizeof(u64)) + flags |= BTF_FMODEL_ALIGN16_ARG; + if (btf_param_match_suffix(btf, arg, "__arena__nullable")) flags |= BTF_FMODEL_ARENA_ARG | BTF_FMODEL_NULLABLE_ARG; else if (btf_param_match_suffix(btf, arg, "__arena")) @@ -7606,7 +8332,7 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, return -EINVAL; } ret = __get_type_size(btf, func->type, &t); - if (ret < 0 || btf_type_is_struct(t)) { + if (ret < 0 || ret > 16) { bpf_log(log, "The function %s return type %s is unsupported.\n", tname, btf_type_str(t)); @@ -7699,6 +8425,12 @@ static int btf_check_func_type_match(struct bpf_verifier_log *log, btf_type_str(t2), fn2); return -EINVAL; } + if (btf_type_has_size(t1) && (t1->size > 8 || t2->size > 8)) { + bpf_log(log, + "Return type of %s() has size %u and of %s() size %u, and a size above 8 bytes cannot be replaced\n", + fn1, t1->size, fn2, t2->size); + return -EINVAL; + } for (i = 0; i < nargs1; i++) { t1 = btf_type_skip_modifiers(btf1, args1[i].type, NULL); @@ -7943,51 +8675,19 @@ static int btf_scan_decl_tags(struct bpf_verifier_env *env, return 0; } -static int btf_scan_type_tags(struct bpf_verifier_env *env, - const struct btf *btf, u32 type_id, - u32 *tags) +static void btf_scan_type_tags(const struct btf *btf, u32 type_id, u32 *tags) { - static const struct btf_type_tag_match func_type_tags[] = { - { "arena", ARG_TAG_ARENA }, - }; - struct btf_type_tag_walk_ctx ctx; - const struct btf_type *t; - int err; - /* Find the first pointer type in the chain. */ - t = btf_type_skip_modifiers(btf, type_id, NULL); - - /* - * We currently reject type tags on non-pointer types, - * which neither LLVM nor GCC support anyway. - */ - if (!t || !btf_type_is_ptr(t)) - return 0; + const struct btf_type *t = btf_type_skip_modifiers(btf, type_id, NULL); - ctx.t = t; - err = btf_type_tag_walk(btf, &ctx, func_type_tags, - ARRAY_SIZE(func_type_tags)); - if (err) { - bpf_log(&env->log, - "function signature member has multiple type tags\n"); - return err; - } - *tags |= ctx.res; - - return 0; + if (btf_type_is_arena_ptr(btf, t)) + *tags |= ARG_TAG_ARENA; } /* Check whether the type is a valid return type. */ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf, - const struct btf_type *t, int subprog) + const struct btf_type *t, int subprog, bool is_global) { - u32 tags = 0; - int err; - - err = btf_scan_type_tags(env, btf, t->type, &tags); - if (err) - return err; - t = btf_type_skip_modifiers(btf, t->type, NULL); /* @@ -7995,16 +8695,62 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt * General arena variables are not allowed, since it makes no sense to return by value * a variable that's on the heap in the first place. */ - if (subprog && (tags & ARG_TAG_ARENA) && btf_type_is_ptr(t)) + if (subprog && btf_type_is_arena_ptr(btf, t)) return 0; /* We always accept void or scalars. */ if (btf_type_is_void(t) || btf_type_is_int(t) || btf_is_any_enum(t)) return 0; + if (btf_type_is_struct(t) && t->size <= 16) { + /* + * A global function's caller models the return as an opaque + * scalar pair, so a pointer member would be laundered into a + * scalar and escape provenance and reference tracking. Only + * scalars and arena pointers are allowed: an arena pointer has + * no provenance to lose, since a program may already derive one + * from any scalar with addr_space_cast(), which confines the + * result to the arena. The main program is the exception: it + * returns to the kernel, which has no arena to cast the address + * back into. A local function is verified inline, so its caller + * receives the real register state and any member is fine. + */ + bool local_func = subprog && !is_global; + u32 member_kinds = BTF_MEMBER_SCALAR; + + if (subprog) + member_kinds |= BTF_MEMBER_ARENA_PTR; + + if (local_func || btf_struct_is_composed_of(env, btf, t, member_kinds)) + return 0; + } + return -EOPNOTSUPP; } +static int btf_check_arg_slots(struct bpf_verifier_log *log, const char *tname, + bool is_global, u32 slot_cnt, + struct bpf_subprog_info *sub) +{ + if (slot_cnt <= MAX_BPF_FUNC_REG_ARGS) + return 0; + + if (is_global) { + bpf_log(log, + "global function %s() needs %d > %d argument slots, " + "stack args not supported\n", + tname, slot_cnt, MAX_BPF_FUNC_REG_ARGS); + return -EINVAL; + } + if (!bpf_jit_supports_stack_args()) { + bpf_log(log, "JIT does not support function %s() with %d argument slots\n", + tname, slot_cnt); + return -EFAULT; + } + sub->stack_arg_cnt = slot_cnt - MAX_BPF_FUNC_REG_ARGS; + return 0; +} + /* Process BTF of a function to produce high-level expectation of function * arguments (like ARG_PTR_TO_CTX, or ARG_PTR_TO_MEM, etc). This information * is cached in subprog info for reuse. @@ -8024,7 +8770,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) const struct btf_param *args; const struct btf_type *t, *ref_t, *fn_t; int err; - u32 i, nargs, btf_id; + u32 i, slots_used, nargs, btf_id; const char *tname; if (sub->args_cached) @@ -8068,34 +8814,29 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) } args = (const struct btf_param *)(t + 1); nargs = btf_type_vlen(t); - sub->arg_cnt = nargs; + sub->arg_slot_cnt = nargs; if (nargs > MAX_BPF_FUNC_ARGS) { bpf_log(log, "kernel supports at most %d parameters, function %s has %d\n", MAX_BPF_FUNC_ARGS, tname, nargs); return -EFAULT; } - if (nargs > MAX_BPF_FUNC_REG_ARGS) { - if (!bpf_jit_supports_stack_args()) { - bpf_log(log, "JIT does not support function %s() with %d args\n", - tname, nargs); - return -EFAULT; - } - sub->stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS; - } - - if (is_global && nargs > MAX_BPF_FUNC_REG_ARGS) { - bpf_log(log, "global function %s has %d > %d args, stack args not supported\n", - tname, nargs, MAX_BPF_FUNC_REG_ARGS); - return -EINVAL; - } + err = btf_check_arg_slots(log, tname, is_global, nargs, sub); + if (err) + return err; - err = btf_validate_return_type(env, btf, t, subprog); + err = btf_validate_return_type(env, btf, t, subprog, is_global); if (err) { if (is_global) { + /* Only a subprogram may return arena pointers. */ + const char *supported = subprog ? + "void, scalar, arena pointer, or a struct/union of " + "scalars and arena pointers" : + "void, scalar, or a scalar-only struct/union"; + bpf_log(log, - "Global function %s() return value not void or scalar. " - "Only those are supported.\n", - tname); + "Global function %s() has unsupported return type. " + "Only %s up to 16 bytes is supported.\n", + tname, supported); } return err; } @@ -8103,15 +8844,17 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) /* Convert BTF function arguments into verifier types. * Only PTR_TO_CTX and SCALAR are supported atm. */ - for (i = 0; i < nargs; i++) { + for (i = 0, slots_used = 0; i < nargs; i++) { u32 tags = 0; + + if (slots_used >= MAX_BPF_FUNC_ARGS) + goto too_many_slots; + err = btf_scan_decl_tags(env, btf, fn_t, i, is_global, &tags); if (err) return err; - err = btf_scan_type_tags(env, btf, args[i].type, &tags); - if (err) - return err; + btf_scan_type_tags(btf, args[i].type, &tags); t = btf_type_by_id(btf, args[i].type); while (btf_type_is_modifier(t)) @@ -8128,7 +8871,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) btf_validate_prog_ctx_type(log, btf, t, i, prog_type, prog->expected_attach_type)) return -EINVAL; - sub->args[i].arg_type = ARG_PTR_TO_CTX; + sub->args[slots_used++].arg_type = ARG_PTR_TO_CTX; continue; } if (btf_is_dynptr_ptr(btf, t)) { @@ -8136,7 +8879,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) bpf_log(log, "arg#%d has invalid combination of tags\n", i); return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_DYNPTR; + sub->args[slots_used++].arg_type = ARG_PTR_TO_DYNPTR; continue; } if (tags & ARG_TAG_TRUSTED) { @@ -8151,10 +8894,11 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) if (kern_type_id < 0) return kern_type_id; - sub->args[i].arg_type = ARG_PTR_TO_BTF_ID | PTR_TRUSTED; + sub->args[slots_used].arg_type = ARG_PTR_TO_BTF_ID | PTR_TRUSTED; if (tags & ARG_TAG_NULLABLE) - sub->args[i].arg_type |= PTR_MAYBE_NULL; - sub->args[i].btf_id = kern_type_id; + sub->args[slots_used].arg_type |= PTR_MAYBE_NULL; + sub->args[slots_used].btf_id = kern_type_id; + slots_used++; continue; } if (tags & ARG_TAG_UNTRUSTED) { @@ -8168,8 +8912,10 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) ref_t = btf_type_skip_modifiers(btf, t->type, NULL); if (btf_type_is_void(ref_t) || btf_type_is_primitive(ref_t)) { - sub->args[i].arg_type = ARG_PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED; - sub->args[i].mem_size = 0; + sub->args[slots_used].arg_type = ARG_PTR_TO_MEM | MEM_RDONLY | + PTR_UNTRUSTED; + sub->args[slots_used].mem_size = 0; + slots_used++; continue; } @@ -8185,8 +8931,9 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) i, btf_type_str(ref_t), tname); return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_BTF_ID | PTR_UNTRUSTED; - sub->args[i].btf_id = kern_type_id; + sub->args[slots_used].arg_type = ARG_PTR_TO_BTF_ID | PTR_UNTRUSTED; + sub->args[slots_used].btf_id = kern_type_id; + slots_used++; continue; } if (tags & ARG_TAG_ARENA) { @@ -8194,7 +8941,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) bpf_log(log, "arg#%d arena cannot be combined with any other tags\n", i); return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_ARENA; + sub->args[slots_used++].arg_type = ARG_PTR_TO_ARENA; continue; } if (is_global) { /* generic user data pointer */ @@ -8214,10 +8961,11 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL; + sub->args[slots_used].arg_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL; if (tags & ARG_TAG_NONNULL) - sub->args[i].arg_type &= ~PTR_MAYBE_NULL; - sub->args[i].mem_size = mem_size; + sub->args[slots_used].arg_type &= ~PTR_MAYBE_NULL; + sub->args[slots_used].mem_size = mem_size; + slots_used++; continue; } @@ -8226,8 +8974,32 @@ skip_pointer: bpf_log(log, "arg#%d has pointer tag, but is not a pointer type\n", i); return -EINVAL; } - if (btf_type_is_int(t) || btf_is_any_enum(t)) { - sub->args[i].arg_type = ARG_ANYTHING; + if (btf_type_is_int(t) || btf_is_any_enum(t) || btf_type_is_struct(t)) { + u32 nslots; + + if (!t->size || t->size > 2 * BPF_REG_SIZE) { + if (!is_global) + return -EINVAL; + bpf_log(log, + "Arg#%d type %s in %s() has size %u, only 1 to %d bytes " + "can be passed by value\n", + i, btf_type_str(t), tname, t->size, 2 * BPF_REG_SIZE); + return -EINVAL; + } + if (btf_type_is_struct(t) && + !btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR)) { + if (!is_global) + return -EINVAL; + bpf_log(log, "Arg#%d type %s in %s() is not composed of scalars\n", + i, btf_type_str(t), tname); + return -EINVAL; + } + + nslots = (t->size + BPF_REG_SIZE - 1) / BPF_REG_SIZE; + if (slots_used + nslots > MAX_BPF_FUNC_ARGS) + goto too_many_slots; + while (nslots--) + sub->args[slots_used++].arg_type = ARG_SCALAR; continue; } if (!is_global) @@ -8237,9 +9009,21 @@ skip_pointer: return -EINVAL; } + err = btf_check_arg_slots(log, tname, is_global, slots_used, sub); + if (err) + return err; + sub->arg_slot_cnt = slots_used; + sub->args_cached = true; return 0; + +too_many_slots: + if (!is_global) + return -EINVAL; + bpf_log(log, "Arguments of %s() need more than %d argument slots\n", + tname, MAX_BPF_FUNC_ARGS); + return -EINVAL; } static void btf_type_show(const struct btf *btf, u32 type_id, void *obj, @@ -8514,6 +9298,8 @@ struct btf_module { struct module *module; struct btf *btf; struct bin_attribute *sysfs_attr; + struct bin_attribute *sysfs_inline_attr; + void *btf_inline_data; int flags; }; @@ -8522,28 +9308,149 @@ static DEFINE_MUTEX(btf_module_mutex); static void purge_cand_cache(struct btf *btf); +#if IS_MODULE(CONFIG_DEBUG_INFO_BTF_INLINE) +extern char __start_BTF_inline_link[]; +extern char __stop_BTF_inline_link[]; + +static struct bin_attribute *vmlinux_inline_attr; +static const struct btf_link *vmlinux_inline_link; + +static const struct btf_link *btf_parse_link(void *start, unsigned int size) +{ + const struct btf_link *link = start; + unsigned int module_name_len; + + if (!link || size != sizeof(*link) || link->btf_size == 0) + return NULL; + + module_name_len = strnlen(link->module_name, sizeof(link->module_name)); + if (!module_name_len || module_name_len == sizeof(link->module_name) || + memchr_inv(link->module_name + module_name_len + 1, 0, + sizeof(link->module_name) - module_name_len - 1)) + return NULL; + return link; +} + +static bool btf_verify_link(const struct btf_link *link, const struct module *mod, + const void *data, size_t data_size) +{ + u8 actual_digest[BTF_LINK_SHA256_LEN]; + + if (!link || data_size != link->btf_size) { + pr_warn(" BTF link from module [%s] missing or has unexpected size\n", mod->name); + return false; + } + sha256(data, data_size, actual_digest); + if (memcmp(actual_digest, link->sha256, sizeof(actual_digest))) { + pr_warn("BTF link from module [%s] has unexpected digest\n", mod->name); + return false; + } + + return true; +} + +static void btf_vmlinux_inline_lazy_init(void) +{ + vmlinux_inline_link = btf_parse_link(__start_BTF_inline_link, + __stop_BTF_inline_link - + __start_BTF_inline_link); + if (!vmlinux_inline_link) { + pr_warn("invalid vmlinux inline BTF link\n"); + return; + } + vmlinux_inline_attr = sysfs_btf_add("vmlinux.inline", NULL, + vmlinux_inline_link->btf_size, true, + vmlinux_inline_link->module_name); + if (IS_ERR(vmlinux_inline_attr)) { + pr_warn("failed to register vmlinux inline BTF in sysfs: %ld\n", + PTR_ERR(vmlinux_inline_attr)); + vmlinux_inline_attr = NULL; + } +} + +static bool btf_module_is_vmlinux_inline(const struct module *mod) +{ + return vmlinux_inline_link && !strcmp(mod->name, vmlinux_inline_link->module_name); +} + +static int btf_module_handle_vmlinux_inline(const struct module *mod) +{ + void *data; + + if (!mod->btf_inline_data || + mod->btf_inline_data_size != vmlinux_inline_link->btf_size) + return -EINVAL; + + data = vmalloc_user(mod->btf_inline_data_size); + if (!data) + return -ENOMEM; + memcpy(data, mod->btf_inline_data, mod->btf_inline_data_size); + + if (!btf_verify_link(vmlinux_inline_link, mod, data, mod->btf_inline_data_size) || + !sysfs_btf_update(vmlinux_inline_attr, data)) { + kvfree(data); + return -EINVAL; + } + return 0; +} + +#else + +static bool btf_module_is_vmlinux_inline(const struct module *mod) +{ + return false; +} + +static int btf_module_handle_vmlinux_inline(const struct module *mod) +{ + return 0; +} + +static void btf_vmlinux_inline_lazy_init(void) +{ +} + +#endif /* IS_MODULE(CONFIG_DEBUG_INFO_BTF_INLINE) */ + static int btf_module_notify(struct notifier_block *nb, unsigned long op, void *module) { struct btf_module *btf_mod, *tmp; struct module *mod = module; + struct bin_attribute *attr; struct btf *btf; + void *inline_data = NULL, *relocated_inline_data = NULL; + unsigned int inline_data_size = 0; int err = 0; - if (mod->btf_data_size == 0 || - (op != MODULE_STATE_COMING && op != MODULE_STATE_LIVE && - op != MODULE_STATE_GOING)) + if (op != MODULE_STATE_COMING && op != MODULE_STATE_LIVE && + op != MODULE_STATE_GOING) goto out; switch (op) { case MODULE_STATE_COMING: + if (btf_module_is_vmlinux_inline(mod)) { + err = btf_module_handle_vmlinux_inline(mod); + break; + } + +#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE) + if (mod->btf_inline_data_size) { + inline_data = mod->btf_inline_data; + inline_data_size = mod->btf_inline_data_size; + } +#endif + if (!mod->btf_data_size) + break; btf_mod = kzalloc_obj(*btf_mod); if (!btf_mod) { err = -ENOMEM; goto out; } btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size, - mod->btf_base_data, mod->btf_base_data_size); + mod->btf_base_data, mod->btf_base_data_size, + inline_data, inline_data_size, + &relocated_inline_data); if (IS_ERR(btf)) { kfree(btf_mod); if (!IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) { @@ -8558,6 +9465,7 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, err = btf_alloc_id(btf); if (err) { btf_free(btf); + kvfree(relocated_inline_data); kfree(btf_mod); goto out; } @@ -8566,37 +9474,35 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, mutex_lock(&btf_module_mutex); btf_mod->module = module; btf_mod->btf = btf; + btf_mod->btf_inline_data = relocated_inline_data; list_add(&btf_mod->list, &btf_modules); mutex_unlock(&btf_module_mutex); - if (IS_ENABLED(CONFIG_SYSFS)) { - struct bin_attribute *attr; - - attr = kzalloc_obj(*attr); - if (!attr) - goto out; + attr = sysfs_btf_add(btf->name, btf->data, btf->data_size, false, NULL); + if (IS_ERR(attr)) { + err = 0; + goto out; + } + btf_mod->sysfs_attr = attr; - sysfs_bin_attr_init(attr); - attr->attr.name = btf->name; - attr->attr.mode = 0444; - attr->size = btf->data_size; - attr->private = btf->data; - attr->read = sysfs_bin_attr_simple_read; + if (relocated_inline_data) { + char name[MODULE_NAME_LEN + sizeof(".inline")]; - err = sysfs_create_bin_file(btf_kobj, attr); - if (err) { - pr_warn("failed to register module [%s] BTF in sysfs: %d\n", - mod->name, err); - kfree(attr); + snprintf(name, sizeof(name), "%s.inline", mod->name); + attr = sysfs_btf_add(name, relocated_inline_data, + inline_data_size, false, NULL); + if (IS_ERR(attr)) { + kvfree(relocated_inline_data); + btf_mod->btf_inline_data = NULL; err = 0; - goto out; + break; } - - btf_mod->sysfs_attr = attr; + btf_mod->sysfs_inline_attr = attr; } - break; case MODULE_STATE_LIVE: + if (!mod->btf_data_size) + break; mutex_lock(&btf_module_mutex); list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { if (btf_mod->module != module) @@ -8608,6 +9514,8 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, mutex_unlock(&btf_module_mutex); break; case MODULE_STATE_GOING: + if (!mod->btf_data_size) + break; mutex_lock(&btf_module_mutex); list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { if (btf_mod->module != module) @@ -8622,10 +9530,12 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, btf_free_id(btf_mod->btf); list_del(&btf_mod->list); if (btf_mod->sysfs_attr) - sysfs_remove_bin_file(btf_kobj, btf_mod->sysfs_attr); + sysfs_btf_remove(btf_mod->sysfs_attr); + if (btf_mod->sysfs_inline_attr) + sysfs_btf_remove(btf_mod->sysfs_inline_attr); + kvfree(btf_mod->btf_inline_data); purge_cand_cache(btf_mod->btf); btf_put(btf_mod->btf); - kfree(btf_mod->sysfs_attr); kfree(btf_mod); break; } @@ -8642,6 +9552,7 @@ static struct notifier_block btf_module_nb = { static int __init btf_module_init(void) { + btf_vmlinux_inline_lazy_init(); register_module_notifier(&btf_module_nb); return 0; } @@ -9915,6 +10826,7 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops, struct bpf_verifier_log *log) { struct btf_struct_ops_tab *tab, *new_tab; + int cgroup_atype; int i, err; tab = btf->struct_ops_tab; @@ -9926,8 +10838,10 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops, btf->struct_ops_tab = tab; } + cgroup_atype = st_ops->cgroup_atype; for (i = 0; i < tab->cnt; i++) - if (tab->ops[i].st_ops == st_ops) + if (tab->ops[i].st_ops == st_ops || + (cgroup_atype && cgroup_atype == tab->ops[i].st_ops->cgroup_atype)) return -EEXIST; if (tab->cnt == tab->capacity) { @@ -9947,6 +10861,31 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops, if (err) return err; + if (cgroup_atype) { + /* + * Cgroup struct_ops callers hold the RCU read lock around the + * entire trampoline call, including its trailing instructions. + * A regular RCU grace period therefore protects both the kdata + * and the trampoline image, so a tasks RCU grace period is not + * needed. + */ + if (!cgroup_bpf_is_struct_ops_atype(cgroup_atype) || + st_ops->reg || st_ops->unreg || st_ops->free_after_tasks_rcu_gp) { + bpf_struct_ops_desc_release(&tab->ops[btf->struct_ops_tab->cnt]); + return -EINVAL; + } + + /* + * There is no need to unregister from the cgroup when btf_free() + * runs. No struct_ops map or cgroup link can be created once its + * BTF is gone. + */ + cgroup_bpf_struct_ops_register(cgroup_atype, + tab->ops[btf->struct_ops_tab->cnt].type_id, + st_ops->cfi_stubs, + st_ops->free_after_mult_rcu_gp); + } + btf->struct_ops_tab->cnt++; return 0; diff --git a/kernel/bpf/btf_vmlinux_inline.c b/kernel/bpf/btf_vmlinux_inline.c new file mode 100644 index 000000000000..d5c599d9f729 --- /dev/null +++ b/kernel/bpf/btf_vmlinux_inline.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026, Oracle and/or its affiliates. */ +/* + * Provide kernel BTF inline function information for use by BPF tools. + */ +#include <linux/btf.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> + +#if IS_BUILTIN(CONFIG_DEBUG_INFO_BTF_INLINE) +extern char __start_BTF_inline[]; +extern char __stop_BTF_inline[]; +#endif + +static int __init btf_vmlinux_inline_init(void) +{ +#if IS_BUILTIN(CONFIG_DEBUG_INFO_BTF_INLINE) + size_t data_size = __stop_BTF_inline - __start_BTF_inline; + + if (data_size) + sysfs_btf_add("vmlinux.inline", __start_BTF_inline, data_size, true, NULL); +#endif + return 0; +} +subsys_initcall(btf_vmlinux_inline_init); + +MODULE_DESCRIPTION("BTF inline information for vmlinux"); +MODULE_LICENSE("GPL"); diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index 842c7d1eabcc..2ee950e9ac82 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -9,6 +9,8 @@ #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) +#define BPF_MAX_GOTOX_EDGES BPF_COMPLEXITY_LIMIT_INSNS + /* non-recursive DFS pseudo code * 1 procedure DFS-iterative(G,v): * 2 label v as discovered @@ -284,15 +286,17 @@ err_free: } /* - * Find and collect all maps which fit in the subprog. Return the result as one - * combined jump table in jt->items (allocated with kvcalloc) + * Collect the jump table of every subprogram that has one, as the combined + * table of all maps whose targets land inside that subprogram. All gotox + * instructions of a subprogram share the same table, so this is done in a + * single pass over the maps rather than once per gotox. */ -static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env, - int subprog_start, int subprog_end) +static int __compute_subprog_jts(struct bpf_verifier_env *env) { - struct bpf_iarray *jt = NULL; + struct bpf_subprog_info *subprog; + struct bpf_iarray *jt, *jt_cur; struct bpf_map *map; - struct bpf_iarray *jt_cur; + u32 old_cnt; int i; for (i = 0; i < env->insn_array_map_cnt; i++) { @@ -303,73 +307,100 @@ static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env, map = env->insn_array_maps[i]; jt_cur = jt_from_map(map); - if (IS_ERR(jt_cur)) { - kvfree(jt); - return jt_cur; + if (IS_ERR(jt_cur)) + return PTR_ERR(jt_cur); + + subprog = bpf_find_containing_subprog(env, jt_cur->items[0]); + if (verifier_bug_if(!subprog, env, "no subprog contains insn %u of an insn array map", + jt_cur->items[0])) { + kvfree(jt_cur); + return -EFAULT; + } + if (jt_cur->items[jt_cur->cnt - 1] >= (subprog + 1)->start) { + verbose(env, "jump table of subprog starting at %u spans multiple subprogs, map id=%d\n", + subprog->start, map->id); + bpf_diag_program_structure( + env, subprog->start, "jump table spans subprograms", + "Keep every entry of a jump table inside one subprogram.", + "The jump table in map id %d for the subprogram that starts at instruction %u reaches past its end at instruction %u.", + map->id, subprog->start, (subprog + 1)->start); + kvfree(jt_cur); + return -EINVAL; } - /* - * This is enough to check one element. The full table is - * checked to fit inside the subprog later in create_jt() - */ - if (jt_cur->items[0] >= subprog_start && jt_cur->items[0] < subprog_end) { - u32 old_cnt = jt ? jt->cnt : 0; - jt = bpf_iarray_realloc(jt, old_cnt + jt_cur->cnt); - if (!jt) { - kvfree(jt_cur); - return ERR_PTR(-ENOMEM); - } - memcpy(jt->items + old_cnt, jt_cur->items, jt_cur->cnt << 2); + old_cnt = subprog->jt ? subprog->jt->cnt : 0; + jt = bpf_iarray_realloc(subprog->jt, old_cnt + jt_cur->cnt); + if (!jt) { + subprog->jt = NULL; + kvfree(jt_cur); + return -ENOMEM; } + memcpy(jt->items + old_cnt, jt_cur->items, jt_cur->cnt << 2); + subprog->jt = jt; kvfree(jt_cur); } - if (!jt) { - verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start); - bpf_diag_program_structure( - env, subprog_start, "missing jump table", - "Make sure subprograms containing gotox instructions are accompanied by jump tables referencing these subprograms.", - "No jump table was found for the subprogram that starts at instruction %u.", - subprog_start); - return ERR_PTR(-EINVAL); + for (i = 0; i < env->subprog_cnt; i++) { + jt = env->subprog_info[i].jt; + if (jt) + jt->cnt = sort_insn_array_uniq(jt->items, jt->cnt); } - jt->cnt = sort_insn_array_uniq(jt->items, jt->cnt); - return jt; + env->cfg.subprog_jts_ready = true; + return 0; +} + +/* Leave nothing behind on failure. */ +static int compute_subprog_jts(struct bpf_verifier_env *env) +{ + int err; + + err = __compute_subprog_jts(env); + if (err) + bpf_free_subprog_jts(env); + return err; +} + +void bpf_free_subprog_jts(struct bpf_verifier_env *env) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(env->subprog_info); i++) { + kvfree(env->subprog_info[i].jt); + env->subprog_info[i].jt = NULL; + } + env->cfg.subprog_jts_ready = false; } static struct bpf_iarray * -create_jt(int t, struct bpf_verifier_env *env) +subprog_jt(int t, struct bpf_verifier_env *env) { struct bpf_subprog_info *subprog; - int subprog_start, subprog_end; - struct bpf_iarray *jt; - int i; + int subprog_start, err; + + if (!env->cfg.subprog_jts_ready) { + err = compute_subprog_jts(env); + if (err) + return ERR_PTR(err); + } subprog = bpf_find_containing_subprog(env, t); + if (verifier_bug_if(!subprog, env, "no subprog contains insn %d", t)) + return ERR_PTR(-EFAULT); subprog_start = subprog->start; - subprog_end = (subprog + 1)->start; - jt = jt_from_subprog(env, subprog_start, subprog_end); - if (IS_ERR(jt)) - return jt; - /* Check that the every element of the jump table fits within the given subprogram */ - for (i = 0; i < jt->cnt; i++) { - if (jt->items[i] < subprog_start || jt->items[i] >= subprog_end) { - verbose(env, "jump table for insn %d points outside of the subprog [%u,%u]\n", - t, subprog_start, subprog_end); - bpf_diag_program_structure( - env, t, "jump table target out of range", - "Keep every jump-table target inside the same subprogram.", - "The jump table for instruction %d points outside subprogram range [%u,%u).", - t, subprog_start, subprog_end); - kvfree(jt); - return ERR_PTR(-EINVAL); - } + if (!subprog->jt) { + verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start); + bpf_diag_program_structure( + env, t, "missing jump table", + "Make sure subprograms containing gotox instructions are accompanied by jump tables referencing these subprograms.", + "No jump table was found for the subprogram that starts at instruction %u.", + subprog_start); + return ERR_PTR(-EINVAL); } - return jt; + return subprog->jt; } /* "conditional jump with N edges" */ @@ -381,27 +412,39 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) struct bpf_iarray *jt; int i, w; - jt = env->insn_aux_data[t].jt; - if (!jt) { - jt = create_jt(t, env); - if (IS_ERR(jt)) - return PTR_ERR(jt); + /* + * The first visit pushes every target that still needs exploring, and + * those are all done by the time this insn is revisited, so there is + * nothing left to walk the jump table for. + */ + if (insn_state[t] & BRANCH) + return DONE_EXPLORING; + + jt = subprog_jt(t, env); + if (IS_ERR(jt)) + return PTR_ERR(jt); - env->insn_aux_data[t].jt = jt; + insn_state[t] |= BRANCH; + + if (check_add_overflow(env->cfg.gotox_edges, jt->cnt, + &env->cfg.gotox_edges) || + env->cfg.gotox_edges > BPF_MAX_GOTOX_EDGES) { + verbose(env, "number of indirect jump edges in the program exceeds %u\n", + BPF_MAX_GOTOX_EDGES); + bpf_diag_program_structure( + env, t, "too many indirect jump edges", + "Reduce the number of indirect jumps, or the number of distinct targets they can reach.", + "The program has more than %u indirect jump edges in total, counted over every gotox instruction.", + BPF_MAX_GOTOX_EDGES); + return -E2BIG; } mark_prune_point(env, t); for (i = 0; i < jt->cnt; i++) { w = jt->items[i]; - if (w < 0 || w >= env->prog->len) { - verbose(env, "indirect jump out of range from insn %d to %d\n", t, w); - bpf_diag_program_structure( - env, t, "indirect jump out of range", - "Keep indirect jump targets inside the program.", - "Instruction %d can jump indirectly to instruction %d, but the program only contains instructions 0 through %d.", - t, w, env->prog->len - 1); - return -EINVAL; - } + if (verifier_bug_if(w < 0 || w >= env->prog->len, env, + "indirect jump from insn %d to %d out of range", t, w)) + return -EFAULT; mark_jmp_point(env, w); mark_jump_target(env, w); @@ -422,27 +465,73 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) } /* - * Instructions that can abnormally return from a subprog (tail_call - * upon success, ld_{abs,ind} upon load failure) have a hidden exit - * that the verifier must account for. + * Return pointers to functions in the read-only map that ld_imm64 instruction + * 't' loads the address of, or of its value, if there are any. */ -static int visit_abnormal_return_insn(struct bpf_verifier_env *env, int t) +static struct bpf_func_ptr *insn_func_ptrs(struct bpf_verifier_env *env, int t, u32 *cnt) { - struct bpf_subprog_info *subprog; - struct bpf_iarray *jt; + struct bpf_insn *insn = &env->prog->insnsi[t]; - if (env->insn_aux_data[t].jt) - return 0; + *cnt = 0; + if (!env->func_ptr_cnt || !bpf_is_ldimm64(insn)) + return NULL; + if (insn->src_reg != BPF_PSEUDO_MAP_VALUE && + insn->src_reg != BPF_PSEUDO_MAP_IDX_VALUE && + insn->src_reg != BPF_PSEUDO_MAP_FD && + insn->src_reg != BPF_PSEUDO_MAP_IDX) + return NULL; - jt = bpf_iarray_realloc(NULL, 2); - if (!jt) - return -ENOMEM; + return bpf_map_func_ptrs(env, env->used_maps[env->insn_aux_data[t].map_index], cnt); +} - subprog = bpf_find_containing_subprog(env, t); - jt->items[0] = t + 1; - jt->items[1] = subprog->exit_idx; - env->insn_aux_data[t].jt = jt; - return 0; +/* + * ld_imm64 that loads the address of a map that has pointers to functions + * is similar to ld_imm64 with BPF_PSEUDO_FUNC that loads the address of one + * function: any of them may be read from the map and called via callx later. + * Treat it as a call of all of them. + */ +static int visit_func_ptrs_insn(int t, struct bpf_verifier_env *env, + struct bpf_func_ptr *ptrs, u32 cnt) +{ + int *insn_stack = env->cfg.insn_stack; + int *insn_state = env->cfg.insn_state; + int ret, w; + u32 i; + + ret = push_insn(t, t + 2, FALLTHROUGH, env); + if (ret) + return ret; + + /* + * Explore one function at a time like push_insn() does, so that + * DISCOVERED function is a caller of 't' and not the one that waits in + * insn_stack[], which merge_callee_effects() relies on. + */ + for (i = 0; i < cnt; i++) { + w = ptrs[i].xlated_off; + if (insn_state[w]) + continue; + + if (env->cfg.cur_stack >= env->prog->len) + return -E2BIG; + + insn_stack[env->cfg.cur_stack++] = w; + insn_state[w] = DISCOVERED; + return KEEP_EXPLORING; + } + + /* all functions are explored, their effects are known */ + mark_prune_point(env, t); + for (i = 0; i < cnt; i++) { + w = ptrs[i].xlated_off; + merge_callee_effects(env, t, w); + + /* the same marks as push_insn() leaves on a branch target */ + mark_prune_point(env, w); + mark_jmp_point(env, w); + mark_jump_target(env, w); + } + return DONE_EXPLORING; } /* Visits the instruction at index t and returns one of the following: @@ -453,21 +542,20 @@ static int visit_abnormal_return_insn(struct bpf_verifier_env *env, int t) static int visit_insn(int t, struct bpf_verifier_env *env) { struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t]; + struct bpf_func_ptr *ptrs; int ret, off, insn_sz; + u32 cnt; if (bpf_pseudo_func(insn)) return visit_func_call_insn(t, insns, env, true); + ptrs = insn_func_ptrs(env, t, &cnt); + if (ptrs) + return visit_func_ptrs_insn(t, env, ptrs, cnt); + /* All non-branch instructions have a single fall-through edge. */ if (BPF_CLASS(insn->code) != BPF_JMP && BPF_CLASS(insn->code) != BPF_JMP32) { - if (BPF_CLASS(insn->code) == BPF_LD && - (BPF_MODE(insn->code) == BPF_ABS || - BPF_MODE(insn->code) == BPF_IND)) { - ret = visit_abnormal_return_insn(env, t); - if (ret) - return ret; - } insn_sz = bpf_is_ldimm64(insn) ? 2 : 1; return push_insn(t, t + insn_sz, FALLTHROUGH, env); } @@ -510,13 +598,10 @@ static int visit_insn(int t, struct bpf_verifier_env *env) */ if (ret == 0 && fp->might_sleep) mark_subprog_might_sleep(env, t); - if (bpf_helper_changes_pkt_data(insn->imm)) + /* nobody knows yet whether the dynptr is that of an skb */ + if (bpf_helper_changes_pkt_data(insn->imm) || + insn->imm == BPF_FUNC_dynptr_write) mark_subprog_changes_pkt_data(env, t); - if (insn->imm == BPF_FUNC_tail_call) { - ret = visit_abnormal_return_insn(env, t); - if (ret) - return ret; - } } else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { struct bpf_call_arg_meta meta; @@ -684,6 +769,11 @@ err_free: return ret; } +struct dfs_state { + u32 traversed:1; + u32 next_succ:31; +}; + /* * For each subprogram 'i' fill array env->cfg.insn_subprogram sub-range * [env->subprog_info[i].postorder_start, env->subprog_info[i+1].postorder_start) @@ -691,43 +781,52 @@ err_free: */ int bpf_compute_postorder(struct bpf_verifier_env *env) { - u32 cur_postorder, i, top, stack_sz, s; - int *stack = NULL, *postorder = NULL, *state = NULL; - struct bpf_iarray *succ; + int *stack = NULL, *postorder = NULL, *postorder_nums = NULL; + int subprog_idx, stack_sz, cur, s, cur_postorder, start; + struct dfs_state *state = NULL; + struct bpf_iarray *succ = NULL; + postorder_nums = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT); postorder = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT); - state = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT); stack = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT); - if (!postorder || !state || !stack) { + state = kvzalloc_objs(struct dfs_state, env->prog->len, GFP_KERNEL_ACCOUNT); + if (!postorder_nums || !postorder || !stack || !state) { + kvfree(postorder_nums); kvfree(postorder); - kvfree(state); kvfree(stack); + kvfree(state); return -ENOMEM; } cur_postorder = 0; - for (i = 0; i < env->subprog_cnt; i++) { - env->subprog_info[i].postorder_start = cur_postorder; - stack[0] = env->subprog_info[i].start; + for (subprog_idx = 0; subprog_idx < env->subprog_cnt; subprog_idx++) { + start = env->subprog_info[subprog_idx].start; + env->subprog_info[subprog_idx].postorder_start = cur_postorder; + stack[0] = start; stack_sz = 1; + state[start].traversed = true; do { - top = stack[stack_sz - 1]; - state[top] |= DISCOVERED; - if (state[top] & EXPLORED) { - postorder[cur_postorder++] = top; + cur = stack[stack_sz - 1]; + succ = bpf_insn_successors(env, cur); + if (state[cur].next_succ == succ->cnt) { + postorder_nums[cur] = cur_postorder; + postorder[cur_postorder] = cur; + cur_postorder++; stack_sz--; continue; } - succ = bpf_insn_successors(env, top); - for (s = 0; s < succ->cnt; ++s) { - if (!state[succ->items[s]]) { - stack[stack_sz++] = succ->items[s]; - state[succ->items[s]] |= DISCOVERED; - } + s = succ->items[state[cur].next_succ]; + if (!state[s].traversed) { + state[s].traversed = true; + state[s].next_succ = 0; + stack[stack_sz] = s; + stack_sz++; + continue; } - state[top] |= EXPLORED; + state[cur].next_succ++; } while (stack_sz); } - env->subprog_info[i].postorder_start = cur_postorder; + env->subprog_info[subprog_idx].postorder_start = cur_postorder; + env->cfg.postorder_nums = postorder_nums; env->cfg.insn_postorder = postorder; env->cfg.cur_postorder = cur_postorder; kvfree(stack); @@ -749,7 +848,7 @@ int bpf_compute_scc(struct bpf_verifier_env *env) struct bpf_insn_aux_data *aux = env->insn_aux_data; const u32 insn_cnt = env->prog->len; int stack_sz, dfs_sz, err = 0; - u32 *stack, *pre, *low, *dfs; + u32 *stack, *pre, *low, *dfs, *dfs_pos; u32 i, j, t, w; u32 next_preorder_num; u32 next_scc_id; @@ -762,13 +861,16 @@ int bpf_compute_scc(struct bpf_verifier_env *env) * - 'stack' accumulates vertices in DFS order, see invariant comment below; * - 'pre[t] == p' => preorder number of vertex 't' is 'p'; * - 'low[t] == n' => smallest preorder number of the vertex reachable from 't' is 'n'; - * - 'dfs' DFS traversal stack, used to emulate explicit recursion. + * - 'dfs' DFS traversal stack, used to emulate explicit recursion; + * - 'dfs_pos[k] == j' => the frame 'dfs[k]' resumes visiting its + * successors at index 'j'. */ stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT); pre = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT); low = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT); dfs = kvcalloc(insn_cnt, sizeof(*dfs), GFP_KERNEL_ACCOUNT); - if (!stack || !pre || !low || !dfs) { + dfs_pos = kvcalloc(insn_cnt, sizeof(*dfs_pos), GFP_KERNEL_ACCOUNT); + if (!stack || !pre || !low || !dfs || !dfs_pos) { err = -ENOMEM; goto exit; } @@ -851,6 +953,7 @@ int bpf_compute_scc(struct bpf_verifier_env *env) stack_sz = 0; dfs_sz = 1; dfs[0] = i; + dfs_pos[0] = 0; dfs_continue: while (dfs_sz) { w = dfs[dfs_sz - 1]; @@ -860,13 +963,22 @@ dfs_continue: next_preorder_num++; stack[stack_sz++] = w; } - /* Visit 'w' successors */ + /* Visit remaining 'w' successors */ succ = bpf_insn_successors(env, w); - for (j = 0; j < succ->cnt; ++j) { + for (j = dfs_pos[dfs_sz - 1]; j < succ->cnt; ++j) { if (pre[succ->items[j]]) { low[w] = min(low[w], low[succ->items[j]]); } else { - dfs[dfs_sz++] = succ->items[j]; + /* + * Once DFS for succ->items[j] is complete, + * pre[succ->items[j]] is non-zero, hence + * resuming at 'j' allows to follow the + * low[w] = min(...) update branch above. + */ + dfs_pos[dfs_sz - 1] = j; + dfs_pos[dfs_sz] = 0; + dfs[dfs_sz] = succ->items[j]; + dfs_sz++; goto dfs_continue; } } @@ -916,5 +1028,6 @@ exit: kvfree(pre); kvfree(low); kvfree(dfs); + kvfree(dfs_pos); return err; } diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 149672c76c49..ce82963bcfc7 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -24,6 +24,34 @@ DEFINE_STATIC_KEY_ARRAY_FALSE(cgroup_bpf_enabled_key, MAX_CGROUP_BPF_ATTACH_TYPE); EXPORT_SYMBOL(cgroup_bpf_enabled_key); +struct cgroup_struct_ops { + u32 type_id; + void *cfi_stubs; + bool mult_rcu; +}; + +static struct cgroup_struct_ops cgroup_struct_ops[MAX_CGROUP_BPF_ATTACH_TYPE]; + +void cgroup_bpf_struct_ops_register(int atype, u32 type_id, void *cfi_stubs, bool mult_rcu) +{ + cgroup_struct_ops[atype].type_id = type_id; + cgroup_struct_ops[atype].cfi_stubs = cfi_stubs; + cgroup_struct_ops[atype].mult_rcu = mult_rcu; +} + +static enum cgroup_bpf_attach_type find_atype_by_struct_ops_id(u32 type_id) +{ + enum cgroup_bpf_attach_type atype; + + for (atype = 0; atype < MAX_CGROUP_BPF_ATTACH_TYPE; atype++) { + if (cgroup_bpf_is_struct_ops_atype(atype) && + cgroup_struct_ops[atype].type_id && + cgroup_struct_ops[atype].type_id == type_id) + return atype; + } + return CGROUP_BPF_ATTACH_TYPE_INVALID; +} + /* * cgroup bpf destruction makes heavy use of work items and there can be a lot * of concurrent destructions. Use a separate workqueue so that cgroup bpf @@ -306,6 +334,19 @@ static void bpf_cgroup_storages_link(struct bpf_cgroup_storage *storages[], bpf_cgroup_storage_link(storages[stype], cgrp, attach_type); } +static void cgroup_struct_ops_link_detach_wake(struct bpf_cgroup_link *link, bool wake_poll) +{ + cgroup_put(link->cgroup); + link->cgroup = NULL; + + bpf_map_put(link->map); + /* READ_ONCE in cgroup_struct_ops_link_poll */ + WRITE_ONCE(link->map, NULL); + + if (wake_poll) + wake_up_interruptible_poll(&link->wait_hup, EPOLLHUP); +} + /* Called when bpf_cgroup_link is auto-detached from dying cgroup. * It drops cgroup and bpf_prog refcounts, and marks bpf_link as defunct. It * doesn't free link memory, which will eventually be done by bpf_link's @@ -313,10 +354,96 @@ static void bpf_cgroup_storages_link(struct bpf_cgroup_storage *storages[], */ static void bpf_cgroup_link_auto_detach(struct bpf_cgroup_link *link) { + if (link->map) { + cgroup_struct_ops_link_detach_wake(link, true); + return; + } + + if (link->link.prog->expected_attach_type == BPF_LSM_CGROUP) + bpf_trampoline_unlink_cgroup_shim(link->link.prog); cgroup_put(link->cgroup); link->cgroup = NULL; } +static void bpf_cgroup_array_free_rcu(struct rcu_head *rcu) +{ + kfree(container_of(rcu, struct bpf_prog_array, rcu)); +} + +static void bpf_cgroup_array_free(struct bpf_prog_array *array, + enum cgroup_bpf_attach_type atype) +{ + if (!array || array == &bpf_empty_prog_array) + return; + if (cgroup_struct_ops[atype].mult_rcu) + /* RCU tasks trace grace period implies RCU grace period. */ + call_rcu_tasks_trace(&array->rcu, bpf_cgroup_array_free_rcu); + else + kfree_rcu(array, rcu); +} + +static void *bpf_cgroup_array_dummy(enum cgroup_bpf_attach_type atype) +{ + if (cgroup_bpf_is_struct_ops_atype(atype)) + return cgroup_struct_ops[atype].cfi_stubs; + return bpf_prog_dummy(); +} + +static int bpf_cgroup_array_length(struct bpf_prog_array *array, + enum cgroup_bpf_attach_type atype) +{ + struct bpf_prog_array_item *item; + int cnt = 0; + + for (item = array->items; item->prog; item++) { + if (item->prog != bpf_cgroup_array_dummy(atype)) + cnt++; + } + + return cnt; +} + +static int bpf_cgroup_array_copy_to_user(struct bpf_prog_array *array, + __u32 __user *prog_ids, int cnt, + enum cgroup_bpf_attach_type atype) +{ + struct bpf_prog_array_item *item; + int i = 0; + u32 id; + + for (item = array->items; item->prog && i < cnt; item++) { + if (item->prog == bpf_cgroup_array_dummy(atype)) + continue; + + if (cgroup_bpf_is_struct_ops_atype(atype)) + id = bpf_struct_ops_id(item->kdata); + else + id = item->prog->aux->id; + + if (copy_to_user(prog_ids + i, &id, sizeof(id))) + return -EFAULT; + i++; + } + return item->prog ? -ENOSPC : 0; +} + +static int bpf_cgroup_array_delete_safe_at(struct bpf_prog_array *array, + int index, enum cgroup_bpf_attach_type atype) +{ + struct bpf_prog_array_item *item; + + for (item = array->items; item->prog; item++) { + if (item->prog == bpf_cgroup_array_dummy(atype)) + continue; + if (!index) { + WRITE_ONCE(item->prog, bpf_cgroup_array_dummy(atype)); + return 0; + } + index--; + } + return -ENOENT; +} + /** * cgroup_bpf_release() - put references of all bpf programs and * release all cgroup bpf data @@ -346,18 +473,15 @@ static void cgroup_bpf_release(struct work_struct *work) bpf_trampoline_unlink_cgroup_shim(pl->prog); bpf_prog_put(pl->prog); } - if (pl->link) { - if (pl->link->link.prog->expected_attach_type == BPF_LSM_CGROUP) - bpf_trampoline_unlink_cgroup_shim(pl->link->link.prog); + if (pl->link) bpf_cgroup_link_auto_detach(pl->link); - } kfree(pl); static_branch_dec(&cgroup_bpf_enabled_key[atype]); } old_array = rcu_dereference_protected( cgrp->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); - bpf_prog_array_free(old_array); + bpf_cgroup_array_free(old_array, atype); } list_for_each_entry_safe(storage, stmp, storages, list_cg) { @@ -399,6 +523,37 @@ static struct bpf_prog *prog_list_prog(struct bpf_prog_list *pl) return NULL; } +static bool prog_list_is_struct_ops(const struct bpf_prog_list *pl) +{ + return pl->link && pl->link->map; +} + +static void prog_list_init_item(struct bpf_prog_list *pl, struct bpf_prog_array_item *item) +{ + if (prog_list_is_struct_ops(pl)) { + item->kdata = bpf_struct_ops_map_kdata(pl->link->map); + return; + } + + item->prog = prog_list_prog(pl); + bpf_cgroup_storages_assign(item->cgroup_storage, pl->storage); +} + +static void prog_list_replace_item(struct bpf_prog_list *pl, struct bpf_prog_array_item *item) +{ + if (prog_list_is_struct_ops(pl)) + WRITE_ONCE(item->kdata, bpf_struct_ops_map_kdata(pl->link->map)); + else + WRITE_ONCE(item->prog, pl->link->link.prog); +} + +static u32 prog_list_id(struct bpf_prog_list *pl) +{ + if (prog_list_is_struct_ops(pl)) + return pl->link->map->id; + return prog_list_prog(pl)->aux->id; +} + /* count number of elements in the list. * it's slow but the list cannot be long */ @@ -408,7 +563,7 @@ static u32 prog_list_length(struct hlist_head *head, int *preorder_cnt) u32 cnt = 0; hlist_for_each_entry(pl, head, node) { - if (!prog_list_prog(pl)) + if (!pl->prog && !pl->link) continue; if (preorder_cnt && (pl->flags & BPF_F_PREORDER)) (*preorder_cnt)++; @@ -482,7 +637,7 @@ static int compute_effective_progs(struct cgroup *cgrp, init_bstart = bstart; hlist_for_each_entry(pl, &p->bpf.progs[atype], node) { - if (!prog_list_prog(pl)) + if (!pl->prog && !pl->link) continue; if (pl->flags & BPF_F_PREORDER) { @@ -492,9 +647,7 @@ static int compute_effective_progs(struct cgroup *cgrp, item = &progs->items[fstart]; fstart++; } - item->prog = prog_list_prog(pl); - bpf_cgroup_storages_assign(item->cgroup_storage, - pl->storage); + prog_list_init_item(pl, item); cnt++; } @@ -517,7 +670,7 @@ static void activate_effective_progs(struct cgroup *cgrp, /* free prog array after grace period, since __cgroup_bpf_run_*() * might be still walking the array */ - bpf_prog_array_free(old_array); + bpf_cgroup_array_free(old_array, atype); } /** @@ -557,7 +710,7 @@ static int cgroup_bpf_inherit(struct cgroup *cgrp) return 0; cleanup: for (i = 0; i < NR; i++) - bpf_prog_array_free(arrays[i]); + bpf_cgroup_array_free(arrays[i], i); for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p)) cgroup_bpf_put(p); @@ -612,7 +765,7 @@ static int update_effective_progs(struct cgroup *cgrp, if (percpu_ref_is_zero(&desc->bpf.refcnt)) { if (unlikely(desc->bpf.inactive)) { - bpf_prog_array_free(desc->bpf.inactive); + bpf_cgroup_array_free(desc->bpf.inactive, atype); desc->bpf.inactive = NULL; } continue; @@ -631,7 +784,7 @@ cleanup: css_for_each_descendant_pre(css, &cgrp->self) { struct cgroup *desc = container_of(css, struct cgroup, self); - bpf_prog_array_free(desc->bpf.inactive); + bpf_cgroup_array_free(desc->bpf.inactive, atype); desc->bpf.inactive = NULL; } @@ -870,7 +1023,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, old_pl_flags = pl->flags; bpf_cgroup_storages_assign(old_storage, pl->storage); } else { - pl = kmalloc_obj(*pl); + pl = kzalloc_obj(*pl); if (!pl) { bpf_cgroup_storages_free(new_storage); return -ENOMEM; @@ -973,9 +1126,10 @@ static int effective_prog_pos(struct cgroup *cgrp, init_bstart = bstart; hlist_for_each_entry(pl, &p->bpf.progs[atype], node) { - if (!prog_list_prog(pl)) - continue; - + /* + * No detaching pl (NULL prog and link) is visible to the callers, + * so skip the check compute_effective_progs() needs. + */ if (pl->flags & BPF_F_PREORDER) { if (pl == target_pl) pos = bstart; @@ -1022,7 +1176,7 @@ static void replace_effective_prog(struct cgroup *cgrp, desc->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); item = &progs->items[pos]; - WRITE_ONCE(item->prog, pl->link->link.prog); + prog_list_replace_item(pl, item); } } @@ -1177,7 +1331,7 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl, lockdep_is_held(&cgroup_mutex)); /* Remove the program from the array */ - WARN_ONCE(bpf_prog_array_delete_safe_at(progs, pos), + WARN_ONCE(bpf_cgroup_array_delete_safe_at(progs, pos, atype), "Failed to purge a prog from array at index %d", pos); } } @@ -1287,7 +1441,15 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, if (effective_query && prog_attach_flags) return -EINVAL; - if (type == BPF_LSM_CGROUP) { + if (type == BPF_STRUCT_OPS) { + u32 type_id = attr->query.type_id; + + atype = find_atype_by_struct_ops_id(type_id); + if (atype == CGROUP_BPF_ATTACH_TYPE_INVALID) + return -ENOENT; + from_atype = to_atype = atype; + flags = 0; + } else if (type == BPF_LSM_CGROUP) { if (!effective_query && attr->query.prog_cnt && prog_ids && !prog_attach_flags) return -EINVAL; @@ -1307,7 +1469,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, if (effective_query) { effective = rcu_dereference_protected(cgrp->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); - total_cnt += bpf_prog_array_length(effective); + total_cnt += bpf_cgroup_array_length(effective, atype); } else { total_cnt += prog_list_length(&cgrp->bpf.progs[atype], NULL); } @@ -1337,35 +1499,33 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, if (effective_query) { effective = rcu_dereference_protected(cgrp->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); - cnt = min_t(int, bpf_prog_array_length(effective), total_cnt); - ret = bpf_prog_array_copy_to_user(effective, prog_ids, cnt); + cnt = min_t(int, bpf_cgroup_array_length(effective, atype), total_cnt); + ret = bpf_cgroup_array_copy_to_user(effective, prog_ids, cnt, atype); } else { struct hlist_head *progs; struct bpf_prog_list *pl; - struct bpf_prog *prog; u32 id; progs = &cgrp->bpf.progs[atype]; cnt = min_t(int, prog_list_length(progs, NULL), total_cnt); i = 0; hlist_for_each_entry(pl, progs, node) { - prog = prog_list_prog(pl); - id = prog->aux->id; + id = prog_list_id(pl); if (copy_to_user(prog_ids + i, &id, sizeof(id))) return -EFAULT; + if (prog_attach_flags) { + flags = cgrp->bpf.flags[atype] | + (pl->flags & BPF_F_PREORDER); + if (copy_to_user(prog_attach_flags + i, + &flags, sizeof(flags))) + return -EFAULT; + } if (++i == cnt) break; } - if (prog_attach_flags) { - flags = cgrp->bpf.flags[atype]; - - for (i = 0; i < cnt; i++) - if (copy_to_user(prog_attach_flags + i, - &flags, sizeof(flags))) - return -EFAULT; + if (prog_attach_flags) prog_attach_flags += cnt; - } } prog_ids += cnt; @@ -2329,7 +2489,7 @@ static const struct bpf_func_proto bpf_sysctl_get_name_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, + .arg2_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE, .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -2772,6 +2932,274 @@ const struct bpf_verifier_ops cg_sockopt_verifier_ops = { const struct bpf_prog_ops cg_sockopt_prog_ops = { }; +static int __cgroup_struct_ops_link_detach(struct bpf_link *link, bool wake_poll) +{ + struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); + enum cgroup_bpf_attach_type atype; + struct bpf_prog_list *pl; + struct bpf_map *map; + struct cgroup *cgrp; + + cgroup_lock(); + + cgrp = cg_link->cgroup; + if (!cgrp) { + cgroup_unlock(); + return 0; + } + + map = cg_link->map; + atype = bpf_struct_ops_map_cgroup_atype(map); + + hlist_for_each_entry(pl, &cgrp->bpf.progs[atype], node) { + if (pl->link == cg_link) + break; + } + if (WARN_ON_ONCE(!pl)) { + cgroup_unlock(); + return -ENOENT; + } + + /* mark deleted so compute_effective_progs() skips it */ + pl->link = NULL; + if (update_effective_progs(cgrp, atype)) { + pl->link = cg_link; + purge_effective_progs(cgrp, pl, atype); + } + + hlist_del(&pl->node); + cgroup_struct_ops_link_detach_wake(cg_link, wake_poll); + cgrp->bpf.revisions[atype]++; + + kfree(pl); + static_branch_dec(&cgroup_bpf_enabled_key[atype]); + + cgroup_unlock(); + + return 0; +} + +static int cgroup_struct_ops_link_detach(struct bpf_link *link) +{ + return __cgroup_struct_ops_link_detach(link, true); +} + +static void cgroup_struct_ops_link_dealloc(struct bpf_link *link) +{ + struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); + + __cgroup_struct_ops_link_detach(link, false); + kfree(cg_link); +} + +static void cgroup_struct_ops_link_show_fdinfo(const struct bpf_link *link, struct seq_file *seq) +{ + struct bpf_cgroup_link *cg_link = + container_of(link, struct bpf_cgroup_link, link); + + cgroup_lock(); + if (!cg_link->cgroup) { + cgroup_unlock(); + return; + } + + seq_printf(seq, "map_id:\t%u\n", cg_link->map->id); + seq_printf(seq, "cgroup_id:\t%llu\n", cgroup_id(cg_link->cgroup)); + cgroup_unlock(); +} + +static int cgroup_struct_ops_link_fill_link_info(const struct bpf_link *link, + struct bpf_link_info *info) +{ + struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); + + cgroup_lock(); + if (!cg_link->cgroup) { + cgroup_unlock(); + return 0; + } + + info->struct_ops.map_id = cg_link->map->id; + info->struct_ops.cgroup_id = cgroup_id(cg_link->cgroup); + cgroup_unlock(); + return 0; +} + +static int cgroup_struct_ops_link_update(struct bpf_link *link, struct bpf_map *new_map, + struct bpf_map *expected_old_map) +{ + struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); + enum cgroup_bpf_attach_type atype; + struct bpf_prog_list *pl; + struct bpf_map *old_map; + struct cgroup *cgrp; + bool found = false; + int err; + + if (!bpf_struct_ops_valid_to_reg(new_map)) + return -EINVAL; + + cgroup_lock(); + + cgrp = cg_link->cgroup; + if (!cgrp) { + err = -ENOLINK; + goto out; + } + + old_map = cg_link->map; + err = bpf_struct_ops_link_update_check(new_map, old_map, expected_old_map); + if (err) + goto out; + + atype = bpf_struct_ops_map_cgroup_atype(new_map); + + hlist_for_each_entry(pl, &cgrp->bpf.progs[atype], node) { + if (pl->link == cg_link) { + found = true; + break; + } + } + if (!found) { + err = -ENOENT; + goto out; + } + + bpf_map_inc(new_map); + WRITE_ONCE(cg_link->map, new_map); + replace_effective_prog(cgrp, atype, pl); + bpf_map_put(old_map); + cgrp->bpf.revisions[atype]++; + +out: + cgroup_unlock(); + return err; +} + +static __poll_t cgroup_struct_ops_link_poll(struct file *file, struct poll_table_struct *pts) +{ + struct bpf_cgroup_link *link = file->private_data; + + poll_wait(file, &link->wait_hup, pts); + + return READ_ONCE(link->map) ? 0 : EPOLLHUP; +} + +static const struct bpf_link_ops cgroup_struct_ops_link_ops = { + .dealloc = cgroup_struct_ops_link_dealloc, + .detach = cgroup_struct_ops_link_detach, + .show_fdinfo = cgroup_struct_ops_link_show_fdinfo, + .fill_link_info = cgroup_struct_ops_link_fill_link_info, + .update_map = cgroup_struct_ops_link_update, + .poll = cgroup_struct_ops_link_poll, +}; + +int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr) +{ + u32 flags = attr->link_create.flags; + u32 pl_flags = (flags & BPF_F_PREORDER) | BPF_F_ALLOW_MULTI; + enum cgroup_bpf_attach_type atype; + struct bpf_link_primer link_primer; + struct bpf_cgroup_link *link; + struct bpf_prog_list *pl = NULL; + struct hlist_head *progs; + struct cgroup *cgrp; + int err; + + if (flags & ~BPF_F_LINK_ATTACH_MASK) + return -EINVAL; + + /* + * Attaching struct_ops to cgroup is through link only. All relative + * position must be corresponding to a link id or fd. + */ + if (attr->link_create.cgroup.relative_fd && !(flags & BPF_F_LINK)) + return -EINVAL; + + link = kzalloc_obj(*link, GFP_USER); + if (!link) + return -ENOMEM; + + bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, + &cgroup_struct_ops_link_ops, NULL, + attr->link_create.attach_type); + + err = bpf_link_prime(&link->link, &link_primer); + if (err) { + kfree(link); + return err; + } + + cgrp = cgroup_get_from_fd(attr->link_create.target_fd); + if (IS_ERR(cgrp)) { + err = PTR_ERR(cgrp); + goto cleanup; + } + + bpf_map_inc(map); + link->map = map; + link->cgroup = cgrp; + init_waitqueue_head(&link->wait_hup); + + atype = bpf_struct_ops_map_cgroup_atype(map); + progs = &cgrp->bpf.progs[atype]; + + cgroup_lock(); + + if (attr->link_create.cgroup.expected_revision && + attr->link_create.cgroup.expected_revision != cgrp->bpf.revisions[atype]) { + err = -ESTALE; + goto unlock; + } + + if (prog_list_length(progs, NULL) >= BPF_CGROUP_MAX_PROGS) { + err = -E2BIG; + goto unlock; + } + + pl = kzalloc_obj(*pl); + if (!pl) { + err = -ENOMEM; + goto unlock; + } + + pl->link = link; + pl->flags = pl_flags; + cgrp->bpf.flags[atype] = BPF_F_ALLOW_MULTI; + + err = insert_pl_to_hlist(pl, progs, NULL, link, + flags | BPF_F_ALLOW_MULTI, attr->link_create.cgroup.relative_fd); + if (err) + goto unlock; + + err = update_effective_progs(cgrp, atype); + if (err) { + hlist_del(&pl->node); + goto unlock; + } + + cgrp->bpf.revisions[atype]++; + static_branch_inc(&cgroup_bpf_enabled_key[atype]); + + cgroup_unlock(); + + return bpf_link_settle(&link_primer); + +unlock: + cgroup_unlock(); + +cleanup: + kfree(pl); + if (link->cgroup) { + cgroup_put(link->cgroup); + link->cgroup = NULL; + bpf_map_put(link->map); + link->map = NULL; + } + bpf_link_cleanup(&link_primer); + return err; +} + /* Common helpers for cgroup hooks. */ const struct bpf_func_proto * cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) diff --git a/kernel/bpf/check_btf.c b/kernel/bpf/check_btf.c index 0e8b3ccc7a5b..4c1ed842f661 100644 --- a/kernel/bpf/check_btf.c +++ b/kernel/bpf/check_btf.c @@ -338,9 +338,9 @@ err_free: #define MIN_CORE_RELO_SIZE sizeof(struct bpf_core_relo) #define MAX_CORE_RELO_SIZE MAX_FUNCINFO_REC_SIZE -static int check_core_relo(struct bpf_verifier_env *env, - const union bpf_attr *attr, - bpfptr_t uattr) +int bpf_check_core_relo(struct bpf_verifier_env *env, + const union bpf_attr *attr, + bpfptr_t uattr) { u32 i, nr_core_relo, ncopy, expected_size, rec_size; struct bpf_core_relo core_relo = {}; @@ -414,7 +414,7 @@ int bpf_prepare_btf_info(struct bpf_verifier_env *env, struct btf *btf; int err; - if (!attr->func_info_cnt && !attr->line_info_cnt) { + if (!attr->func_info_cnt && !attr->line_info_cnt && !attr->core_relo_cnt) { if (check_abnormal_return(env)) return -EINVAL; return 0; @@ -455,9 +455,5 @@ int bpf_check_btf_info(struct bpf_verifier_env *env, if (err) return err; - err = check_core_relo(env, attr, uattr); - if (err) - return err; - return 0; } diff --git a/kernel/bpf/cnum.c b/kernel/bpf/cnum.c index 86142cb2aee5..8ec5187d7c75 100644 --- a/kernel/bpf/cnum.c +++ b/kernel/bpf/cnum.c @@ -2,6 +2,7 @@ /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ #include <linux/bits.h> +#include <linux/bpf_verifier.h> /* for imod() */ #define T 32 #include "cnum_defs.h" @@ -118,3 +119,38 @@ struct cnum64 cnum64_cnum32_intersect(struct cnum64 a, struct cnum32 b) } return t; } + +/* Intersect 'in' with the set of integers defined by equation 'base + step * k'. */ +struct cnum64 cnum64_intersect_linear(struct cnum64 in, u16 base, u16 step) +{ + s64 smin = cnum64_smin(in); + s64 smax = cnum64_smax(in); + s64 lo, hi; + u16 d; + + if (step <= 1 || cnum64_is_empty(in) || cnum64_srange_overflow(in)) + return in; + /* + * Round smin up to the next value congruent to 'base' modulo 'step', + * i.e. increase smin by d = (base - smin) mod step: + * + * |<---- d ---->| + * |-----------|=============|... + * base+step*k smin base+step*(k+1) + */ + d = imod(base - imod(smin, step), step); + if ((u64)smax - (u64)smin < d) + return CNUM64_EMPTY; + lo = smin + d; + /* + * Round smax down to the previous value congruent to 'base' modulo 'step', + * i.e. decrease smax by d = (smax - base) mod step: + * + * |<--- d --->| + * ...|===========|-------------| + * base+step*k smax base+step*(k+1) + */ + d = imod(imod(smax, step) - base, step); + hi = smax - d; + return cnum64_from_srange(lo, hi); +} diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h index a90e317e3578..05b17a8b4814 100644 --- a/kernel/bpf/cnum_defs.h +++ b/kernel/bpf/cnum_defs.h @@ -33,7 +33,12 @@ struct cnum_t FN(from_srange)(st min, st max) return (struct cnum_t){ .base = base, .size = size }; } -/* True if this cnum represents two unsigned ranges. */ +/* + * True if this cnum represents two unsigned ranges. + * + * The caller should ensure !is_empty(cnum) holds when calling + * this function. + */ static inline bool FN(urange_overflow)(struct cnum_t cnum) { /* Same as cnum.base + cnum.size > UT_MAX but avoids overflow */ @@ -44,6 +49,9 @@ static inline bool FN(urange_overflow)(struct cnum_t cnum) * cnum{T}_umin / cnum{T}_umax query an unsigned range represented by this cnum. * If cnum represents a range crossing the UT_MAX/0 boundary, the unbound range * [0..UT_MAX] is returned. + * + * The caller should ensure !is_empty(cnum) holds when calling + * cnum{T}_umin / cnum{T}_umax. */ ut FN(umin)(struct cnum_t cnum) { @@ -57,7 +65,12 @@ ut FN(umax)(struct cnum_t cnum) } EXPORT_SYMBOL_GPL(FN(umax)); -/* True if this cnum represents two signed ranges. */ +/* + * True if this cnum represents two signed ranges. + * + * The caller should ensure !is_empty(cnum) holds when calling + * this function. + */ static inline bool FN(srange_overflow)(struct cnum_t cnum) { return FN(contains)(cnum, (ut)ST_MAX) && FN(contains)(cnum, (ut)ST_MIN); @@ -67,19 +80,18 @@ static inline bool FN(srange_overflow)(struct cnum_t cnum) * cnum{T}_smin / cnum{T}_smax query a signed range represented by this cnum. * If cnum represents a range crossing the ST_MAX/ST_MIN boundary, the unbound range * [ST_MIN..ST_MAX] is returned. + * + * The caller should ensure !is_empty(cnum) holds when calling + * cnum{T}_smin / cnum{T}_smax. */ st FN(smin)(struct cnum_t cnum) { - return FN(srange_overflow)(cnum) - ? ST_MIN - : min((st)cnum.base, (st)(cnum.base + cnum.size)); + return FN(srange_overflow)(cnum) ? ST_MIN : (st)cnum.base; } st FN(smax)(struct cnum_t cnum) { - return FN(srange_overflow)(cnum) - ? ST_MAX - : max((st)cnum.base, (st)(cnum.base + cnum.size)); + return FN(srange_overflow)(cnum) ? ST_MAX : (st)(cnum.base + cnum.size); } /* @@ -181,11 +193,55 @@ void FN(intersect_with_srange)(struct cnum_t *dst, st min, st max) static inline struct cnum_t FN(normalize)(struct cnum_t cnum) { - if (cnum.size == UT_MAX && cnum.base != 0 && cnum.base != (ut)ST_MAX) + if (cnum.size == UT_MAX) cnum.base = 0; return cnum; } +/* + * Return a smallest arc containing both 'a' and 'b'. + * Break equal-size ties by choosing the smaller base. + */ +struct cnum_t FN(union)(struct cnum_t a, struct cnum_t b) +{ + struct cnum_t b1, ab, ba; + ut end; + + if (FN(is_empty)(a)) + return b; + if (FN(is_empty)(b)) + return a; + + /* + * Rotate so that a1.base == 0 and a1.end == a.size. + * Normalize b1 to preserve the full-circle representation. + */ + b1 = FN(normalize)((struct cnum_t){ b.base - a.base, b.size }); + end = max(a.size, (ut)(b1.base + b1.size)); + + if (FN(urange_overflow)(b1)) { + /* a1 reaches b1's main arc: together they cover the circle. */ + if (b1.base <= a.size) + return (struct cnum_t){ 0, UT_MAX }; + + /* Extend b1's tail through a1's end, then rotate back. */ + return FN(normalize)((struct cnum_t){ b.base, end - b1.base }); + } + + /* ab, rotated back, covers both nonwrapping arcs. */ + ab = (struct cnum_t){ a.base, end }; + if (b1.base <= a.size) + return FN(normalize)(ab); + + /* The arcs are disjoint; ba is the other possible covering arc. */ + ba = (struct cnum_t){ b.base, a.size - b1.base }; + if (ba.size < ab.size || + (ba.size == ab.size && ba.base < ab.base)) + ab = ba; + + return FN(normalize)(ab); +} + struct cnum_t FN(add)(struct cnum_t a, struct cnum_t b) { if (FN(is_empty)(a) || FN(is_empty)(b)) @@ -210,12 +266,7 @@ bool FN(is_empty)(struct cnum_t cnum) bool FN(contains)(struct cnum_t cnum, ut v) { - if (FN(is_empty)(cnum)) - return false; - if (FN(urange_overflow)(cnum)) - return v >= cnum.base || v <= (ut)cnum.base + cnum.size; - else - return v >= cnum.base && v <= (ut)cnum.base + cnum.size; + return !FN(is_empty)(cnum) && v - cnum.base <= cnum.size; } bool FN(is_const)(struct cnum_t cnum) diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c index 7f1b30059cc8..f9edfde965dc 100644 --- a/kernel/bpf/const_fold.c +++ b/kernel/bpf/const_fold.c @@ -180,9 +180,12 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info * bool is_ldsx = mode == BPF_MEMSX; int off = src->val + insn->off; u64 val = 0; + u32 cnt; if (!bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr || off < 0 || off + size > map->value_size || + /* so are the addresses of functions that the map points to */ + bpf_map_range_func_ptrs(env, map, off, size, &cnt) || bpf_map_direct_read(map, off, size, &val, is_ldsx)) { *dst = unknown; break; @@ -191,7 +194,8 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info * dst->val = val; break; case BPF_JMP: - if (opcode != BPF_CALL) + /* both 'call imm' and 'callx reg' clobber caller saved registers */ + if (BPF_OP(insn->code) != BPF_CALL) break; process_call: for (r = BPF_REG_0; r <= BPF_REG_5; r++) @@ -353,7 +357,7 @@ int bpf_prune_dead_branches(struct bpf_verifier_env *env) if (!bpf_insn_is_cond_jump(insn->code)) continue; - if (bpf_is_may_goto_insn(insn)) + if (BPF_OP(insn->code) == BPF_JCOND) continue; if (!(aux->const_reg_mask & BIT(insn->dst_reg))) @@ -394,5 +398,7 @@ int bpf_prune_dead_branches(struct bpf_verifier_env *env) /* recompute postorder, since CFG has changed */ kvfree(env->cfg.insn_postorder); env->cfg.insn_postorder = NULL; + kvfree(env->cfg.postorder_nums); + env->cfg.postorder_nums = NULL; return bpf_compute_postorder(env); } diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 8b294dfc1ad4..3030beb531d9 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -18,7 +18,9 @@ */ #include <uapi/linux/btf.h> +#include <linux/ctype.h> #include <linux/filter.h> +#include <linux/sched/signal.h> #include <linux/skbuff.h> #include <linux/static_call.h> #include <linux/vmalloc.h> @@ -141,10 +143,6 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag mutex_init(&fp->aux->dst_mutex); mutex_init(&fp->aux->st_ops_assoc_mutex); -#ifdef CONFIG_BPF_SYSCALL - bpf_prog_stream_init(fp); -#endif - return fp; } @@ -174,7 +172,6 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) } return prog; } -EXPORT_SYMBOL_GPL(bpf_prog_alloc); int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog) { @@ -272,6 +269,7 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE); fp->pages = pages; fp->aux->prog = fp; + bpf_prog_dev_bound_moved(fp); /* We keep fp->aux from fp_old around in the new * reallocated structure. @@ -288,6 +286,9 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, void __bpf_prog_free(struct bpf_prog *fp) { if (fp->aux) { +#ifdef CONFIG_BPF_SYSCALL + bpf_prog_stream_free(fp); +#endif mutex_destroy(&fp->aux->used_maps_mutex); mutex_destroy(&fp->aux->dst_mutex); mutex_destroy(&fp->aux->st_ops_assoc_mutex); @@ -589,6 +590,10 @@ bpf_prog_ksym_set_name(struct bpf_prog *prog) prog->aux->func_info[prog->aux->func_idx].type_id); func_name = btf_name_by_offset(prog->aux->btf, type->name_off); snprintf(sym, (size_t)(end - sym), "_%s", func_name); + /* the name of a function of Rust is not an identifier */ + for (; *sym; sym++) + if (!isalnum(*sym) && *sym != '_' && *sym != '.') + *sym = '_'; return; } @@ -1361,7 +1366,7 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from, { struct bpf_insn *to = to_buff; u32 imm_rnd = get_random_u32(); - s16 off; + int off; BUILD_BUG_ON(BPF_REG_PARAMS + 2 != MAX_BPF_JIT_REG); BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG); @@ -1437,6 +1442,8 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from, off = from->off; if (off < 0) off -= 2; + if (off < S16_MIN) + return -ERANGE; *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); *to++ = BPF_JMP_REG(from->code, from->dst_reg, BPF_REG_AX, off); @@ -1457,6 +1464,8 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from, off = from->off; if (off < 0) off -= 2; + if (off < S16_MIN) + return -ERANGE; *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); *to++ = BPF_JMP32_REG(from->code, from->dst_reg, BPF_REG_AX, @@ -1550,8 +1559,7 @@ void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other) * know whether fp here is the clone or the original. */ fp->aux->prog = fp; - if (fp->aux->offload) - fp->aux->offload->prog = fp; + bpf_prog_dev_bound_moved(fp); bpf_prog_clone_free(fp_other); } @@ -1605,7 +1613,9 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp if (!rewritten) continue; - if (env) + if (rewritten < 0) + tmp = ERR_PTR(rewritten); + else if (env) tmp = bpf_patch_insn_data(env, i, insn_buff, rewritten); else tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten); @@ -1619,6 +1629,8 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp * fix it up here on error. */ bpf_jit_prog_release_other(prog, clone); + if (env && fatal_signal_pending(current)) + return ERR_PTR(-EINTR); return IS_ERR(tmp) ? tmp : ERR_PTR(-ENOMEM); } @@ -1668,7 +1680,6 @@ noinline u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) { return 0; } -EXPORT_SYMBOL_GPL(__bpf_call_base); /* All UAPI available opcodes. */ #define BPF_INSN_MAP(INSN_2, INSN_3) \ @@ -1831,6 +1842,7 @@ bool bpf_opcode_in_insntable(u8 code) [BPF_LD | BPF_IND | BPF_H] = true, [BPF_LD | BPF_IND | BPF_W] = true, [BPF_JMP | BPF_JA | BPF_X] = true, + [BPF_JMP | BPF_CALL | BPF_X] = true, [BPF_JMP | BPF_JCOND] = true, }; #undef BPF_INSN_3_TBL @@ -2625,22 +2637,82 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp) return select_interpreter; } +/* + * Whether a function has an insn that works with the arena base, which a JIT + * sets up in the prologue when bpf_prog_uses_arena_insns() says so. Decided on + * the final insns of the function: an access through an arena pointer, or + * through a number in a program loaded with BPF_F_ARENA_SCALAR, has been + * rewritten to a BPF_PROBE_MEM32 form by then, a cast to the user address + * space keeps BPF_ADDR_SPACE_CAST, and a kfunc call with an __arena argument, + * which the JIT rebases, has a function model that says so. A load of an + * __arena struct member is rewritten with an immediate and needs none; + * dereferencing the result does. + */ +bool bpf_prog_has_arena_insns(const struct bpf_prog *prog) +{ + const struct bpf_insn *insn = prog->insnsi; + const struct btf_func_model *m; + int i, j; + + if (!prog->aux->arena) + return false; + + for (i = 0; i < prog->len; i++, insn++) { + switch (BPF_CLASS(insn->code)) { + case BPF_LDX: + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || + BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) + return true; + break; + case BPF_ST: + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) + return true; + break; + case BPF_STX: + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) + return true; + break; + case BPF_ALU64: + if (BPF_OP(insn->code) == BPF_MOV && BPF_SRC(insn->code) == BPF_X && + insn->off == BPF_ADDR_SPACE_CAST) + return true; + break; + case BPF_JMP: + if (!bpf_pseudo_kfunc_call(insn)) + break; + m = bpf_jit_find_kfunc_model(prog, insn); + if (!m) + break; + for (j = 0; j < m->nr_args; j++) + if (m->arg_flags[j] & BTF_FMODEL_ARENA_ARG) + return true; + break; + } + } + return false; +} + static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { #ifdef CONFIG_BPF_JIT struct bpf_prog *orig_prog; + prog->aux->uses_arena_insns = bpf_prog_has_arena_insns(prog); if (!bpf_prog_need_blind(prog)) return bpf_int_jit_compile(env, prog); orig_prog = prog; prog = bpf_jit_blind_constants(env, prog); /* - * If blinding was requested and we failed during blinding, we must fall - * back to the interpreter. + * Fall back to the interpreter after blinding failures, except when + * the loader was killed. */ - if (IS_ERR(prog)) + if (IS_ERR(prog)) { + if (PTR_ERR(prog) == -EINTR) + return prog; goto out_restore; + } prog = bpf_int_jit_compile(env, prog); if (prog->jited) { @@ -2659,6 +2731,8 @@ out_restore: struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp, int *err) { + struct bpf_prog *jit_prog; + /* In case of BPF to BPF calls, verifier did all the prep * work with regards to JITing, etc. */ @@ -2681,7 +2755,12 @@ struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct if (*err) return fp; - fp = bpf_prog_jit_compile(env, fp); + jit_prog = bpf_prog_jit_compile(env, fp); + if (IS_ERR(jit_prog)) { + *err = PTR_ERR(jit_prog); + return fp; + } + fp = jit_prog; bpf_prog_jit_attempt_done(fp); if (!fp->jited && jit_needed) { *err = -ENOTSUPP; @@ -2723,7 +2802,6 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) { return __bpf_prog_select_runtime(NULL, fp, err); } -EXPORT_SYMBOL_GPL(bpf_prog_select_runtime); static unsigned int __bpf_prog_ret1(const void *ctx, const struct bpf_insn *insn) @@ -2784,6 +2862,11 @@ void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs) call_rcu_tasks_trace(&progs->rcu, __bpf_prog_array_free_sleepable_cb); } +struct bpf_prog *bpf_prog_dummy(void) +{ + return &dummy_bpf_prog.prog; +} + int bpf_prog_array_length(struct bpf_prog_array *array) { struct bpf_prog_array_item *item; @@ -3016,13 +3099,18 @@ void __bpf_free_used_maps(struct bpf_prog_aux *aux, bool sleepable; u32 i; - sleepable = aux->prog->sleepable; + sleepable = bpf_prog_runs_sleepable(aux->prog); for (i = 0; i < len; i++) { map = used_maps[i]; if (map->ops->map_poke_untrack) map->ops->map_poke_untrack(map, aux); if (sleepable) atomic64_dec(&map->sleepable_refcnt); + /* + * The program that didn't load is not a user of the map. libbpf + * loads the program again to get the log of the verifier. + */ + cmpxchg(&map->user, (unsigned long)aux, 0); bpf_map_put(map); } } @@ -3054,15 +3142,42 @@ static void bpf_free_used_btfs(struct bpf_prog_aux *aux) kfree(aux->used_btfs); } -static void bpf_prog_free_deferred(struct work_struct *work) +static void bpf_prog_free_images(struct work_struct *work) { struct bpf_prog_aux *aux; int i; + aux = container_of(to_rcu_work(work), struct bpf_prog_aux, rwork); + for (i = 0; i < aux->real_func_cnt; i++) { + /* We can just unlink the subprog poke descriptor table as + * it was originally linked to the main program and is also + * released along with it. + */ + aux->func[i]->aux->poke_tab = NULL; + bpf_jit_free(aux->func[i]); + } + if (aux->real_func_cnt) { + kfree(aux->func); + bpf_prog_unlock_free(aux->prog); + } else { + bpf_jit_free(aux->prog); + } +} + +static void bpf_prog_free_deferred(struct work_struct *work) +{ + struct bpf_prog_aux *aux; + bool ksyms; + aux = container_of(work, struct bpf_prog_aux, work); + /* + * A prog that still runs when its last reference goes needs its symbols, + * by which its exception table is found. It has ended by now. + */ + ksyms = aux->real_func_cnt || !list_empty(&aux->ksym.lnode); + bpf_prog_kallsyms_del_all(aux->prog); #ifdef CONFIG_BPF_SYSCALL bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab); - bpf_prog_stream_free(aux->prog); #endif #ifdef CONFIG_CGROUP_BPF if (aux->cgroup_atype != CGROUP_BPF_ATTACH_TYPE_INVALID) @@ -3079,20 +3194,12 @@ static void bpf_prog_free_deferred(struct work_struct *work) #endif if (aux->dst_trampoline) bpf_trampoline_put(aux->dst_trampoline); - for (i = 0; i < aux->real_func_cnt; i++) { - /* We can just unlink the subprog poke descriptor table as - * it was originally linked to the main program and is also - * released along with it. - */ - aux->func[i]->aux->poke_tab = NULL; - bpf_jit_free(aux->func[i]); - } - if (aux->real_func_cnt) { - kfree(aux->func); - bpf_prog_unlock_free(aux->prog); - } else { - bpf_jit_free(aux->prog); - } + /* The symbols are looked up under rcu_read_lock() */ + INIT_RCU_WORK(&aux->rwork, bpf_prog_free_images); + if (ksyms) + queue_rcu_work(system_percpu_wq, &aux->rwork); + else + bpf_prog_free_images(&aux->rwork.work); } void bpf_prog_free(struct bpf_prog *fp) @@ -3105,7 +3212,6 @@ void bpf_prog_free(struct bpf_prog *fp) INIT_WORK(&aux->work, bpf_prog_free_deferred); schedule_work(&aux->work); } -EXPORT_SYMBOL_GPL(bpf_prog_free); /* RNG for unprivileged user space with separated state from prandom_u32(). */ static DEFINE_PER_CPU(struct rnd_state, bpf_user_rnd_state); @@ -3282,6 +3388,117 @@ bool __weak bpf_jit_supports_kfunc_call(void) return false; } +/* Return TRUE if the JIT backend supports callx (indirect call) instruction. */ +bool __weak bpf_jit_supports_callx(void) +{ + return false; +} + +bool __weak bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return false; +} + +/* unwind_goto and unwind_exit */ +bool __weak bpf_jit_supports_unwind(void) +{ + return false; +} + +/* + * How this arch places a by-value kfunc argument, or NULL for one that has + * not opted in and so only takes an argument of a single eightbyte, which + * every convention places in slot order. + */ +const struct bpf_jit_arg_abi * __weak bpf_jit_arg_abi(void) +{ + return NULL; +} + +u32 bpf_jit_place_args(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, u8 *pos_of_slot) +{ + u32 i, k, nslots, slot = 0, nregs_used = 0, stack_off = 0; + bool on_stack = false; + + for (i = 0; i < fm->nr_args; i++) { + bool align16 = fm->arg_flags[i] & BTF_FMODEL_ALIGN16_ARG; + u32 pos; + + nslots = btf_func_model_arg_slots(fm, i); + + if (align16 && abi->even_reg_align) + nregs_used = round_up(nregs_used, 2); + + if (!on_stack && nregs_used + nslots <= abi->nr_arg_regs) { + /* wholly in registers */ + pos = nregs_used; + nregs_used += nslots; + } else if (!on_stack && abi->split_at_boundary) { + /* the last registers hold what fits, the stack the rest */ + pos = nregs_used; + stack_off = (nregs_used + nslots - abi->nr_arg_regs) * BPF_REG_SIZE; + nregs_used = abi->nr_arg_regs; + on_stack = true; + } else { + /* wholly on the stack */ + if (align16 && abi->even_stack_align) + stack_off = round_up(stack_off, 2 * BPF_REG_SIZE); + pos = abi->nr_arg_regs + stack_off / BPF_REG_SIZE; + stack_off += nslots * BPF_REG_SIZE; + if (!abi->backfill_after_stack) + on_stack = true; + } + + for (k = 0; k < nslots; k++) + pos_of_slot[slot + k] = pos + k; + slot += nslots; + } + + return slot; +} + +u32 bpf_jit_plan_arg_moves(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, + struct bpf_jit_arg_move *moves) +{ + u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; + u32 nslots, n = 0, s, back; + + nslots = bpf_jit_place_args(abi, fm, pos_of_slot); + back = nslots; + + /* + * An argument is two eightbytes at most, so it frees one register at + * most and only one argument ever moves down. Its destination is + * still in use, so carry it in the scratch. Only a lower slot can + * take the one it leaves, so the walk reaches it first. + */ + for (s = nslots; s > 0; s--) { + u8 slot = s - 1, pos = pos_of_slot[slot]; + + if (pos == slot) + continue; + + if (pos < slot) { + moves[n].dst = BPF_JIT_ARG_TMP; + back = slot; + } else { + moves[n].dst = pos; + } + moves[n].src = slot; + n++; + } + + if (back < nslots) { + moves[n].dst = pos_of_slot[back]; + moves[n].src = BPF_JIT_ARG_TMP; + n++; + } + + return n; +} + bool __weak bpf_jit_supports_stack_args(void) { return false; @@ -3302,6 +3519,12 @@ bool __weak bpf_jit_supports_arena(void) return false; } +/* Whether JIT takes BPF_REG_AX as the address of arena access */ +bool __weak bpf_jit_supports_arena_scalar(void) +{ + return false; +} + bool __weak bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { return false; @@ -3367,6 +3590,19 @@ bool __weak bpf_jit_supports_private_stack(void) return false; } +/* + * Return TRUE if the JIT lays out frames of up to MAX_BPF_STACK_JIT bytes. + * Its prologue, epilogue and tail call sequences must encode such frame + * sizes and a private stack must be sized from the program's depth. The + * budget is only granted alongside bpf_jit_supports_subprog_tailcalls(), + * whose tail calls land before the target sets up its own frame; see + * bpf_prog_stack_limit(). + */ +bool __weak bpf_jit_supports_large_stack(void) +{ + return false; +} + void __weak arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie) { } diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index 5e59ab896f05..eb87131b1bef 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -45,6 +45,7 @@ */ #define CPU_MAP_BULK_SIZE 8 /* 8 == one cacheline on 64-bit archs */ +#define CPU_MAP_HELD_DEVS 2048 struct bpf_cpu_map_entry; struct bpf_cpu_map; @@ -74,6 +75,13 @@ struct bpf_cpu_map_entry { struct completion kthread_running; struct rcu_work free_work; + + /* + * Every frame and skb in the queue holds a reference to its device. + * The kthread keeps it until nothing is left in GRO. + */ + struct net_device *held[CPU_MAP_HELD_DEVS]; + u32 nr_held; }; struct bpf_cpu_map { @@ -129,9 +137,11 @@ static void __cpu_map_ring_cleanup(struct ptr_ring *ring) WARN_ON_ONCE(1); if (unlikely(__ptr_test_bit(0, &ptr))) { __ptr_clear_bit(0, &ptr); + dev_put(((struct sk_buff *)ptr)->dev); kfree_skb(ptr); continue; } + dev_put(((struct xdp_frame *)ptr)->dev_rx); xdp_return_frame(ptr); } } @@ -311,7 +321,7 @@ static int cpu_map_kthread_run(void *data) void *frames[CPUMAP_BATCH]; void *skbs[CPUMAP_BATCH]; u32 i, n, m; - bool empty; + bool empty, full; /* Release CPU reschedule checks */ if (__ptr_ring_empty(rcpu->queue)) { @@ -344,10 +354,12 @@ static int cpu_map_kthread_run(void *data) struct sk_buff *skb = f; __ptr_clear_bit(0, &skb); + rcpu->held[rcpu->nr_held++] = skb->dev; skbs[ret.skb_n++] = skb; continue; } + rcpu->held[rcpu->nr_held++] = ((struct xdp_frame *)f)->dev_rx; frames[ret.xdp_n++] = f; page = virt_to_page(f); @@ -398,10 +410,13 @@ stats: /* Flush either every 64 packets or in case of empty ring */ packets += n; empty = __ptr_ring_empty(rcpu->queue); - if (packets >= NAPI_POLL_WEIGHT || empty) { - cpu_map_gro_flush(rcpu, empty); + full = empty || rcpu->nr_held > CPU_MAP_HELD_DEVS - CPUMAP_BATCH; + if (packets >= NAPI_POLL_WEIGHT || full) { + cpu_map_gro_flush(rcpu, full); packets = 0; } + while (full && rcpu->nr_held) + dev_put(rcpu->held[--rcpu->nr_held]); local_bh_enable(); /* resched point, may call do_softirq() */ } @@ -738,12 +753,14 @@ static void bq_flush_to_queue(struct xdp_bulk_queue *bq) for (i = 0; i < bq->count; i++) { struct xdp_frame *xdpf = bq->q[i]; - int err; - err = __ptr_ring_produce(q, xdpf); - if (err) { + if (__ptr_ring_full(q)) { drops++; xdp_return_frame_rx_napi(xdpf); + } else { + /* The device cannot go before the flush, from now on it can */ + dev_hold(xdpf->dev_rx); + __ptr_ring_produce(q, xdpf); } processed++; } @@ -807,11 +824,15 @@ int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu, __skb_pull(skb, skb->mac_len); skb_set_redirected(skb, false); + dev_hold(skb->dev); __ptr_set_bit(0, &skb); ret = ptr_ring_produce(rcpu->queue, skb); - if (ret < 0) + if (ret < 0) { + __ptr_clear_bit(0, &skb); + dev_put(skb->dev); goto trace; + } wake_up_process(rcpu->kthread); trace: diff --git a/kernel/bpf/crypto.c b/kernel/bpf/crypto.c index 51f89cecefb4..69bc29d0a4d4 100644 --- a/kernel/bpf/crypto.c +++ b/kernel/bpf/crypto.c @@ -1,19 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright (c) 2024 Meta, Inc */ #include <linux/bpf.h> -#include <linux/bpf_crypto.h> #include <linux/bpf_mem_alloc.h> #include <linux/btf.h> #include <linux/btf_ids.h> #include <linux/filter.h> -#include <linux/scatterlist.h> #include <linux/skbuff.h> -#include <crypto/skcipher.h> - -struct bpf_crypto_type_list { - const struct bpf_crypto_type *type; - struct list_head list; -}; +#include <crypto/aes-cbc.h> +#include <crypto/aes-ecb.h> /* BPF crypto initialization parameters struct */ /** @@ -36,94 +30,53 @@ struct bpf_crypto_params { u32 authsize; }; -static LIST_HEAD(bpf_crypto_types); -static DECLARE_RWSEM(bpf_crypto_types_sem); +enum bpf_crypto_algo_id { + BPF_ALGO_AES_CBC, + BPF_ALGO_AES_ECB, +}; + +static const struct { + const char *type_name; + const char *algo_name; + enum bpf_crypto_algo_id algo; +} bpf_crypto_algos[] = { + { "skcipher", "cbc(aes)", BPF_ALGO_AES_CBC }, + { "skcipher", "ecb(aes)", BPF_ALGO_AES_ECB }, +}; + +static bool bpf_crypto_find_algo(const struct bpf_crypto_params *params, + enum bpf_crypto_algo_id *id_ret) +{ + for (size_t i = 0; i < ARRAY_SIZE(bpf_crypto_algos); i++) { + if (strncmp(bpf_crypto_algos[i].type_name, params->type, + sizeof(params->type)) == 0 && + strncmp(bpf_crypto_algos[i].algo_name, params->algo, + sizeof(params->algo)) == 0) { + *id_ret = bpf_crypto_algos[i].algo; + return true; + } + } + return false; +} /** * struct bpf_crypto_ctx - refcounted BPF crypto context structure - * @type: The pointer to bpf crypto type - * @tfm: The pointer to instance of crypto API struct. - * @siv_len: Size of IV and state storage for cipher + * @key: The crypto key + * @algo: The crypto algorithm ID * @rcu: The RCU head used to free the crypto context with RCU safety. * @usage: Object reference counter. When the refcount goes to 0, the * memory is released back to the BPF allocator, which provides * RCU safety. */ struct bpf_crypto_ctx { - const struct bpf_crypto_type *type; - void *tfm; - u32 siv_len; + union { + struct aes_key aes; + } key; + enum bpf_crypto_algo_id algo; struct rcu_head rcu; refcount_t usage; }; -int bpf_crypto_register_type(const struct bpf_crypto_type *type) -{ - struct bpf_crypto_type_list *node; - int err = -EBUSY; - - down_write(&bpf_crypto_types_sem); - list_for_each_entry(node, &bpf_crypto_types, list) { - if (!strcmp(node->type->name, type->name)) - goto unlock; - } - - node = kmalloc_obj(*node); - err = -ENOMEM; - if (!node) - goto unlock; - - node->type = type; - list_add(&node->list, &bpf_crypto_types); - err = 0; - -unlock: - up_write(&bpf_crypto_types_sem); - - return err; -} -EXPORT_SYMBOL_GPL(bpf_crypto_register_type); - -int bpf_crypto_unregister_type(const struct bpf_crypto_type *type) -{ - struct bpf_crypto_type_list *node; - int err = -ENOENT; - - down_write(&bpf_crypto_types_sem); - list_for_each_entry(node, &bpf_crypto_types, list) { - if (strcmp(node->type->name, type->name)) - continue; - - list_del(&node->list); - kfree(node); - err = 0; - break; - } - up_write(&bpf_crypto_types_sem); - - return err; -} -EXPORT_SYMBOL_GPL(bpf_crypto_unregister_type); - -static const struct bpf_crypto_type *bpf_crypto_get_type(const char *name) -{ - const struct bpf_crypto_type *type = ERR_PTR(-ENOENT); - struct bpf_crypto_type_list *node; - - down_read(&bpf_crypto_types_sem); - list_for_each_entry(node, &bpf_crypto_types, list) { - if (strcmp(node->type->name, name)) - continue; - - if (try_module_get(node->type->owner)) - type = node->type; - break; - } - up_read(&bpf_crypto_types_sem); - - return type; -} - __bpf_kfunc_start_defs(); /** @@ -132,92 +85,70 @@ __bpf_kfunc_start_defs(); * Allocates a crypto context that can be used, acquired, and released by * a BPF program. The crypto context returned by this function must either * be embedded in a map as a kptr, or freed with bpf_crypto_ctx_release(). - * As crypto API functions use GFP_KERNEL allocations, this function can - * only be used in sleepable BPF programs. + * As this uses a GFP_KERNEL allocation, this function can only be used in + * sleepable BPF programs. * * bpf_crypto_ctx_create() allocates memory for crypto context. * It may return NULL if no memory is available. * @params: pointer to struct bpf_crypto_params which contains all the * details needed to initialise crypto context. - * @params__sz: size of steuct bpf_crypto_params usef by bpf program - * @err: integer to store error code when NULL is returned. + * @params__sz: size of struct bpf_crypto_params used by bpf program + * @err_ret: integer to store error code when NULL is returned. */ __bpf_kfunc struct bpf_crypto_ctx * bpf_crypto_ctx_create(const struct bpf_crypto_params *params, u32 params__sz, - int *err) + int *err_ret) { - const struct bpf_crypto_type *type; struct bpf_crypto_ctx *ctx; + int err; - if (!params || params->reserved[0] || params->reserved[1] || - params__sz != sizeof(struct bpf_crypto_params)) { - *err = -EINVAL; - return NULL; - } - - type = bpf_crypto_get_type(params->type); - if (IS_ERR(type)) { - *err = PTR_ERR(type); + if (!params || + params__sz != sizeof(struct bpf_crypto_params) || + params->reserved[0] || params->reserved[1]) { + *err_ret = -EINVAL; return NULL; } - if (!type->has_algo(params->algo)) { - *err = -EOPNOTSUPP; - goto err_module_put; - } - - if (!!params->authsize ^ !!type->setauthsize) { - *err = -EOPNOTSUPP; - goto err_module_put; - } - if (!params->key_len || params->key_len > sizeof(params->key)) { - *err = -EINVAL; - goto err_module_put; + *err_ret = -EINVAL; + return NULL; } ctx = kzalloc_obj(*ctx); if (!ctx) { - *err = -ENOMEM; - goto err_module_put; + *err_ret = -ENOMEM; + return NULL; } - ctx->type = type; - ctx->tfm = type->alloc_tfm(params->algo); - if (IS_ERR(ctx->tfm)) { - *err = PTR_ERR(ctx->tfm); - goto err_free_ctx; + if (!bpf_crypto_find_algo(params, &ctx->algo)) { + err = -EOPNOTSUPP; + goto out; } - if (params->authsize) { - *err = type->setauthsize(ctx->tfm, params->authsize); - if (*err) - goto err_free_tfm; + switch (ctx->algo) { + case BPF_ALGO_AES_CBC: + case BPF_ALGO_AES_ECB: + if (params->authsize) + err = -EOPNOTSUPP; + else + err = aes_preparekey(&ctx->key.aes, params->key, + params->key_len); + break; + default: + WARN_ON_ONCE(1); + err = -EOPNOTSUPP; + break; } - *err = type->setkey(ctx->tfm, params->key, params->key_len); - if (*err) - goto err_free_tfm; - - if (type->get_flags(ctx->tfm) & CRYPTO_TFM_NEED_KEY) { - *err = -EINVAL; - goto err_free_tfm; +out: + if (err) { + kfree_sensitive(ctx); + *err_ret = err; + return NULL; } - - ctx->siv_len = type->ivsize(ctx->tfm) + type->statesize(ctx->tfm); - refcount_set(&ctx->usage, 1); - + *err_ret = 0; return ctx; - -err_free_tfm: - type->free_tfm(ctx->tfm); -err_free_ctx: - kfree(ctx); -err_module_put: - module_put(type->owner); - - return NULL; } static void crypto_free_cb(struct rcu_head *head) @@ -225,9 +156,7 @@ static void crypto_free_cb(struct rcu_head *head) struct bpf_crypto_ctx *ctx; ctx = container_of(head, struct bpf_crypto_ctx, rcu); - ctx->type->free_tfm(ctx->tfm); - module_put(ctx->type->owner); - kfree(ctx); + kfree_sensitive(ctx); } /** @@ -267,27 +196,53 @@ __bpf_kfunc void bpf_crypto_ctx_release_dtor(void *ctx) } CFI_NOSEAL(bpf_crypto_ctx_release_dtor); +static int bpf_aes_cbc_crypt(u8 *dst, u32 dst_len, const u8 *src, u32 src_len, + u8 *iv, u32 iv_len, + const struct bpf_crypto_ctx *ctx, bool decrypt) +{ + if (iv_len != AES_BLOCK_SIZE) + return -EINVAL; + if (src_len % AES_BLOCK_SIZE || dst_len < src_len) + return -EINVAL; + if (decrypt) + aes_cbc_decrypt(dst, src, src_len, iv, &ctx->key.aes); + else + aes_cbc_encrypt(dst, src, src_len, iv, &ctx->key.aes); + return 0; +} + +static int bpf_aes_ecb_crypt(u8 *dst, u32 dst_len, const u8 *src, u32 src_len, + u8 *iv, u32 iv_len, + const struct bpf_crypto_ctx *ctx, bool decrypt) +{ + if (iv_len != 0) + return -EINVAL; + if (src_len % AES_BLOCK_SIZE || dst_len < src_len) + return -EINVAL; + if (decrypt) + aes_ecb_decrypt(dst, src, src_len, &ctx->key.aes); + else + aes_ecb_encrypt(dst, src, src_len, &ctx->key.aes); + return 0; +} + static int bpf_crypto_crypt(const struct bpf_crypto_ctx *ctx, const struct bpf_dynptr_kern *src, const struct bpf_dynptr_kern *dst, - const struct bpf_dynptr_kern *siv, + const struct bpf_dynptr_kern *iv, bool decrypt) { - u32 src_len, dst_len, siv_len; + u32 src_len, dst_len, iv_len; const u8 *psrc; u8 *pdst, *piv; - int err; if (__bpf_dynptr_is_rdonly(dst)) return -EINVAL; - siv_len = siv ? __bpf_dynptr_size(siv) : 0; + iv_len = iv ? __bpf_dynptr_size(iv) : 0; src_len = __bpf_dynptr_size(src); dst_len = __bpf_dynptr_size(dst); - if (!src_len || !dst_len || src_len > dst_len) - return -EINVAL; - - if (siv_len != ctx->siv_len) + if (!src_len || !dst_len) return -EINVAL; psrc = __bpf_dynptr_data(src, src_len); @@ -297,14 +252,20 @@ static int bpf_crypto_crypt(const struct bpf_crypto_ctx *ctx, if (!pdst) return -EINVAL; - piv = siv_len ? __bpf_dynptr_data_rw(siv, siv_len) : NULL; - if (siv_len && !piv) + piv = iv_len ? __bpf_dynptr_data_rw(iv, iv_len) : NULL; + if (iv_len && !piv) return -EINVAL; - err = decrypt ? ctx->type->decrypt(ctx->tfm, psrc, pdst, src_len, piv) - : ctx->type->encrypt(ctx->tfm, psrc, pdst, src_len, piv); - - return err; + switch (ctx->algo) { + case BPF_ALGO_AES_CBC: + return bpf_aes_cbc_crypt(pdst, dst_len, psrc, src_len, piv, + iv_len, ctx, decrypt); + case BPF_ALGO_AES_ECB: + return bpf_aes_ecb_crypt(pdst, dst_len, psrc, src_len, piv, + iv_len, ctx, decrypt); + default: + return -EINVAL; + } } /** @@ -312,20 +273,20 @@ static int bpf_crypto_crypt(const struct bpf_crypto_ctx *ctx, * @ctx: The crypto context being used. The ctx must be a trusted pointer. * @src: bpf_dynptr to the encrypted data. Must be a trusted pointer. * @dst: bpf_dynptr to the buffer where to store the result. Must be a trusted pointer. - * @siv__nullable: bpf_dynptr to IV data and state data to be used by decryptor. May be NULL. + * @iv__nullable: bpf_dynptr to the initialization vector. May be NULL. * * Decrypts provided buffer using IV data and the crypto context. Crypto context must be configured. */ __bpf_kfunc int bpf_crypto_decrypt(struct bpf_crypto_ctx *ctx, const struct bpf_dynptr *src, const struct bpf_dynptr *dst, - const struct bpf_dynptr *siv__nullable) + const struct bpf_dynptr *iv__nullable) { const struct bpf_dynptr_kern *src_kern = (struct bpf_dynptr_kern *)src; const struct bpf_dynptr_kern *dst_kern = (struct bpf_dynptr_kern *)dst; - const struct bpf_dynptr_kern *siv_kern = (struct bpf_dynptr_kern *)siv__nullable; + const struct bpf_dynptr_kern *iv_kern = (struct bpf_dynptr_kern *)iv__nullable; - return bpf_crypto_crypt(ctx, src_kern, dst_kern, siv_kern, true); + return bpf_crypto_crypt(ctx, src_kern, dst_kern, iv_kern, true); } /** @@ -333,20 +294,20 @@ __bpf_kfunc int bpf_crypto_decrypt(struct bpf_crypto_ctx *ctx, * @ctx: The crypto context being used. The ctx must be a trusted pointer. * @src: bpf_dynptr to the plain data. Must be a trusted pointer. * @dst: bpf_dynptr to the buffer where to store the result. Must be a trusted pointer. - * @siv__nullable: bpf_dynptr to IV data and state data to be used by decryptor. May be NULL. + * @iv__nullable: bpf_dynptr to the initialization vector. May be NULL. * * Encrypts provided buffer using IV data and the crypto context. Crypto context must be configured. */ __bpf_kfunc int bpf_crypto_encrypt(struct bpf_crypto_ctx *ctx, const struct bpf_dynptr *src, const struct bpf_dynptr *dst, - const struct bpf_dynptr *siv__nullable) + const struct bpf_dynptr *iv__nullable) { const struct bpf_dynptr_kern *src_kern = (struct bpf_dynptr_kern *)src; const struct bpf_dynptr_kern *dst_kern = (struct bpf_dynptr_kern *)dst; - const struct bpf_dynptr_kern *siv_kern = (struct bpf_dynptr_kern *)siv__nullable; + const struct bpf_dynptr_kern *iv_kern = (struct bpf_dynptr_kern *)iv__nullable; - return bpf_crypto_crypt(ctx, src_kern, dst_kern, siv_kern, false); + return bpf_crypto_crypt(ctx, src_kern, dst_kern, iv_kern, false); } __bpf_kfunc_end_defs(); diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index dc7b859e8bbf..1b8f0d7f8455 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c @@ -350,6 +350,7 @@ static int dev_map_bpf_prog_run(struct bpf_prog *xdp_prog, xdp_convert_frame_to_buff(xdpf, &xdp); xdp.txq = &txq; + rxq.mem.type = xdpf->mem_type; xdp.rxq = &rxq; act = bpf_prog_run_xdp(xdp_prog, &xdp); diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index 0abbbe177e31..b5204798231d 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -351,7 +351,7 @@ static void diag_fmt_restore(struct bpf_verifier_env *env, struct diag_fmt_mark if (mark.chunk) { mark.chunk->seq.len = mark.len; - seq_buf_str(&mark.chunk->seq); + seq_buf_terminate(&mark.chunk->seq); } } @@ -631,11 +631,7 @@ static void format_disasm_line(struct bpf_verifier_env *env, int insn_idx, return; print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); - seq_buf_str(&ctx.seq); - ctx.seq.len = strnlen(line->text, sizeof(line->text)); - while (ctx.seq.len && line->text[ctx.seq.len - 1] == '\n') - seq_buf_pop(&ctx.seq); - seq_buf_str(&ctx.seq); + seq_buf_terminate(&ctx.seq); line->valid = true; } @@ -794,8 +790,9 @@ static void diag_print_insn_context(struct bpf_verifier_env *env, u32 insn_idx, } } -static void bpf_diag_source(struct bpf_verifier_env *env, u32 insn_idx, const char *label, - const char *fmt, ...) +static __printf(4, 5) void bpf_diag_source(struct bpf_verifier_env *env, + u32 insn_idx, const char *label, + const char *fmt, ...) { struct bpf_diag_scratch *scratch; struct bpf_linfo_source *source_lines; @@ -960,6 +957,37 @@ const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_t } } +const char *bpf_diag_arg_type_plain(enum bpf_arg_type type) +{ + switch (base_type(type)) { + case ARG_MEM_SIZE: + case ARG_CONST_MEM_SIZE: + return "an integer scalar length for this memory argument"; + case ARG_PTR_TO_CTX: + return "the original program context pointer or preserve it before modifying registers"; + case ARG_SCALAR: + case ARG_CONST_SCALAR: + case ARG_CONST_ALLOC_SIZE_OR_ZERO: + return "an integer scalar value for this argument, not a pointer or resource object"; + case ARG_PTR_TO_CONST_STR: + return "a constant string pointer that the verifier recognizes, such as a string stored in a read-only map value"; + case ARG_PTR_TO_DYNPTR: + return "the address of a stack dynptr object, or use a const dynptr pointer returned by the verifier-supported path"; + case ARG_PTR_TO_ALLOC_BTF_ID: + return "a pointer returned by the matching BPF object allocation path"; + case ARG_PTR_TO_REFCOUNTED_KPTR: + return "an owning or non-owning pointer to a BPF-managed object containing a bpf_refcount field"; + case ARG_PTR_TO_ITER: + return "the address of a stack iterator object for iterator new, next, and destroy calls"; + case ARG_PTR_TO_IRQ_FLAG: + return "the same stack slot used by bpf_local_irq_save() or bpf_res_spin_lock_irqsave()"; + case ARG_PTR_TO_CTX_OUT: + return "the attach hook's own output argument, loaded directly from the program context"; + default: + return "a value with one of the accepted pointer or scalar types for this call"; + } +} + static const char *diag_arg_ordinal(int argno) { switch (argno) { @@ -1087,16 +1115,17 @@ void bpf_diag_ctx_forbidden(struct bpf_verifier_env *env, u32 insn_idx, const char *constraint, *context; u32 depth; - if (env->cur_state->active_rcu_locks) - ctx_kind = BPF_DIAG_CONTEXT_RCU; - else if (env->cur_state->active_preempt_locks) - ctx_kind = BPF_DIAG_CONTEXT_PREEMPT; - else if (env->cur_state->active_irq_id) - ctx_kind = BPF_DIAG_CONTEXT_IRQ; - else if (env->cur_state->active_locks) - ctx_kind = BPF_DIAG_CONTEXT_LOCK; - else - ctx_kind = BPF_DIAG_CONTEXT_NONE; + ctx_kind = BPF_DIAG_CONTEXT_NONE; + if (env->cur_state->in_sleepable) { + if (env->cur_state->active_rcu_locks) + ctx_kind = BPF_DIAG_CONTEXT_RCU; + else if (env->cur_state->active_preempt_locks) + ctx_kind = BPF_DIAG_CONTEXT_PREEMPT; + else if (env->cur_state->active_irq_id) + ctx_kind = BPF_DIAG_CONTEXT_IRQ; + else if (env->cur_state->active_locks) + ctx_kind = BPF_DIAG_CONTEXT_LOCK; + } depth = diag_context_depth(env, ctx_kind); opts = (struct bpf_diag_history_opts) { @@ -1535,8 +1564,7 @@ static void diag_record_mod(struct bpf_verifier_env *env, u32 insn_idx, } else if (diag_mod_insn_origin(env, insn_idx, &target, &event.mod.origin)) { event.mod.origin_valid = true; } - if (old_reg && new_reg && - (reason == BPF_DIAG_MOD_WRITE || reason == BPF_DIAG_MOD_SPILL) && + if (old_reg && new_reg && reason == BPF_DIAG_MOD_WRITE && !memcmp(&event.mod.old, &event.mod.new, sizeof(event.mod.old)) && !event.mod.origin_valid && diag_mod_keeps_lineage(env, &event)) @@ -1568,9 +1596,9 @@ static struct bpf_reg_state *target_to_reg(struct bpf_verifier_env *env, return NULL; return &state->stack_arg_regs[target->stack_arg]; case BPF_DIAG_MOD_TARGET_STACK_SLOT: - if (target->spi >= state->allocated_stack / BPF_REG_SIZE) + if (target->spi >= bpf_stack_nr_slots(state)) return NULL; - return &state->stack[target->spi].spilled_ptr; + return &bpf_stack_slot(state, target->spi)->spilled_ptr; default: return NULL; } @@ -1586,7 +1614,7 @@ static bool reg_to_target(struct bpf_verifier_env *env, const struct bpf_reg_sta for (frame = 0; frame <= vstate->curframe; frame++) { struct bpf_func_state *state = vstate->frame[frame]; unsigned long start, end; - u32 nslots = state->allocated_stack / BPF_REG_SIZE; + u32 nslots = bpf_stack_nr_slots(state); int spi; start = (unsigned long)state->regs; @@ -1605,6 +1633,11 @@ static bool reg_to_target(struct bpf_verifier_env *env, const struct bpf_reg_sta return true; } + /* + * Map the pointer back to its slot by address, which relies on + * the slots forming one contiguous array as bpf_stack_slot() + * indexes it. + */ start = (unsigned long)state->stack; end = (unsigned long)(state->stack + nslots); if (nslots && addr >= start && addr < end) { diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h index d1b79945008a..a4102fb049ec 100644 --- a/kernel/bpf/diagnostics.h +++ b/kernel/bpf/diagnostics.h @@ -51,6 +51,7 @@ const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id); const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_type type); +const char *bpf_diag_arg_type_plain(enum bpf_arg_type type); u64 bpf_diag_event_log_save(struct bpf_verifier_env *env); void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos); u32 bpf_diag_irq_depth(const struct bpf_verifier_state *state); diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index 3ce8d74b0e40..889fc074496f 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -350,7 +350,10 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, if (opcode == BPF_CALL) { char tmp[64]; - if (insn->src_reg == BPF_PSEUDO_CALL) { + if (BPF_SRC(insn->code) == BPF_X) { + verbose(cbs->private_data, "(%02x) callx r%d", + insn->code, insn->dst_reg); + } else if (insn->src_reg == BPF_PSEUDO_CALL) { verbose(cbs->private_data, "(%02x) call pc%s", insn->code, __func_get_name(cbs, insn, @@ -372,11 +375,16 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->src_reg == BPF_MAY_GOTO) { verbose(cbs->private_data, "(%02x) may_goto pc%+d", insn->code, insn->off); + } else if (insn->code == (BPF_JMP | BPF_JCOND) && + insn->src_reg == BPF_UNWIND_GOTO) { + verbose(cbs->private_data, "(%02x) unwind_goto pc%+d", + insn->code, insn->off); } else if (insn->code == (BPF_JMP32 | BPF_JA)) { verbose(cbs->private_data, "(%02x) gotol pc%+d", insn->code, insn->imm); } else if (insn->code == (BPF_JMP | BPF_EXIT)) { - verbose(cbs->private_data, "(%02x) exit", insn->code); + verbose(cbs->private_data, "(%02x) %s", insn->code, + insn->src_reg == BPF_UNWIND_EXIT ? "unwind_exit" : "exit"); } else if (BPF_SRC(insn->code) == BPF_X) { verbose(cbs->private_data, "(%02x) if %c%d %s %c%d goto pc%+d", diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 52d3cec33672..f062ef7dd943 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -5,9 +5,8 @@ #include <linux/bpf_verifier.h> #include <linux/filter.h> #include <linux/vmalloc.h> -#include <linux/bsearch.h> -#include <linux/sort.h> #include <linux/perf_event.h> +#include <linux/sched/signal.h> #include <net/xdp.h> #include "disasm.h" @@ -45,6 +44,31 @@ static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *i return false; } +/* + * The insn accesses arena through a number. JITs add the base of arena + * to the register as it is, so the access goes through the low 32 bits of + * the number in BPF_REG_AX. Registers of the program are not changed: the + * number may be compared or stored later, and on another path the register + * may be PTR_TO_ARENA. + * + * Constant blinding leaves insns that use BPF_REG_AX alone, so the immediate + * of st through a number is not blinded. + */ +static int arena_scalar_access(const struct bpf_insn *insn, struct bpf_insn *buf) +{ + bool load = BPF_CLASS(insn->code) == BPF_LDX || bpf_atomic_is_load_acq(insn); + struct bpf_insn *patch = buf; + + *patch++ = BPF_MOV32_REG(BPF_REG_AX, load ? insn->src_reg : insn->dst_reg); + *patch = *insn; + if (load) + patch->src_reg = BPF_REG_AX; + else + patch->dst_reg = BPF_REG_AX; + patch++; + return patch - buf; +} + /* Return the regno defined by the insn, or -1. */ static int insn_def_regno(const struct bpf_insn *insn) { @@ -116,73 +140,26 @@ int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn) return dst_reg; } -static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b) -{ - const struct bpf_kfunc_desc *d0 = a; - const struct bpf_kfunc_desc *d1 = b; - - if (d0->imm != d1->imm) - return d0->imm < d1->imm ? -1 : 1; - if (d0->offset != d1->offset) - return d0->offset < d1->offset ? -1 : 1; - return 0; -} - const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn) { - const struct bpf_kfunc_desc desc = { - .imm = insn->imm, - .offset = insn->off, - }; const struct bpf_kfunc_desc *res; struct bpf_kfunc_desc_tab *tab; tab = prog->aux->kfunc_tab; - res = bsearch(&desc, tab->descs, tab->nr_descs, - sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm_off); - - return res ? &res->func_model : NULL; -} - -static int set_kfunc_desc_imm(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc) -{ - unsigned long call_imm; + if (insn->off < 0 || insn->off >= tab->nr_descs) + return NULL; + res = &tab->descs[insn->off]; if (bpf_jit_supports_far_kfunc_call()) { - call_imm = desc->func_id; - } else { - call_imm = BPF_CALL_IMM(desc->addr); - /* Check whether the relative offset overflows desc->imm */ - if ((unsigned long)(s32)call_imm != call_imm) { - verbose(env, "address of kernel func_id %u is out of range\n", - desc->func_id); - return -EINVAL; - } - } - desc->imm = call_imm; - return 0; -} - -static int sort_kfunc_descs_by_imm_off(struct bpf_verifier_env *env) -{ - struct bpf_kfunc_desc_tab *tab; - int i, err; - - tab = env->prog->aux->kfunc_tab; - if (!tab) - return 0; - - for (i = 0; i < tab->nr_descs; i++) { - err = set_kfunc_desc_imm(env, &tab->descs[i]); - if (err) - return err; + if (res->func_id != insn->imm) + return NULL; + } else if ((s32)BPF_CALL_IMM(res->addr) != insn->imm) { + return NULL; } - sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), - kfunc_desc_cmp_by_imm_off, NULL); - return 0; + return &res->func_model; } static int add_kfunc_in_insns(struct bpf_verifier_env *env, @@ -213,12 +190,19 @@ static int get_callee_stack_depth(struct bpf_verifier_env *env, } #endif +enum bpf_patch_mode { + BPF_PATCH_KEEP_TARGET, /* targets of 'off' keep naming 'off', the first new insn */ + BPF_PATCH_MOVE_TARGET, /* targets of 'off' follow the original insn to 'off + len - 1' */ +}; + /* single env->prog->insni[off] instruction was replaced with the range * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying * [0, off) and [off, end) to new locations, so the patched range stays zero */ static void adjust_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_prog *new_prog, u32 off, u32 cnt) + struct bpf_prog *new_prog, u32 off, u32 cnt, + struct bpf_insn *original_insn, + enum bpf_patch_mode mode) { struct bpf_insn_aux_data *data = env->insn_aux_data; struct bpf_insn *insn = new_prog->insnsi; @@ -232,8 +216,15 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, */ data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0; - if (cnt == 1) + if (cnt == 1) { + /* + * A non-memory accessing insn could have been replaced by a + * memory accessing insn, systematically mark it for non-stack + * access + */ + data[off].non_stack_access = bpf_is_mem_insn(insn + off); return; + } prog_len = new_prog->len; env->insn_aux_data_len = prog_len; @@ -244,16 +235,39 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, /* Expand insni[off]'s seen count to the patched range. */ data[i].seen = old_seen; data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; + /* + * In BPF_PATCH_MOVE_TARGET mode the original insn is the last + * slot by construction. + */ + if (mode == BPF_PATCH_KEEP_TARGET && + !memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) { + data[i].non_stack_access = + data[off + cnt - 1].non_stack_access; + data[off + cnt - 1].non_stack_access = false; + } else if (bpf_is_mem_insn(insn + i)) { + data[i].non_stack_access = true; + } } /* + * Last slot instruction could be a newly generated + * BPF_ST/BPF_LDX/BPF_STX, systematically mark it for non-stack access + * if it is not the original instruction, otherwise keep the + * original marking + */ + if (bpf_is_mem_insn(insn + off + cnt - 1) && + memcmp(insn + off + cnt - 1, original_insn, sizeof(struct bpf_insn))) + data[off + cnt - 1].non_stack_access = true; + + /* * The indirect_target flag of the original instruction was moved to the last of the * new instructions by the above memmove and memset, but the indirect jump target is * actually the first instruction, so move it back. This also matches with the behavior * of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new - * instruction. + * instruction. For BPF_PATCH_MOVE_TARGET the original instruction is the last one, + * so the flag already sits where needed. */ - if (data[off + cnt - 1].indirect_target) { + if (mode == BPF_PATCH_KEEP_TARGET && data[off + cnt - 1].indirect_target) { data[off].indirect_target = 1; data[off + cnt - 1].indirect_target = 0; } @@ -273,7 +287,7 @@ static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len } } -static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len) +static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 first, u32 len) { int i; @@ -281,7 +295,7 @@ static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len) return; for (i = 0; i < env->insn_array_map_cnt; i++) - bpf_insn_array_adjust(env->insn_array_maps[i], off, len); + bpf_insn_array_adjust(env->insn_array_maps[i], first, len); } static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 off, u32 len) @@ -292,7 +306,7 @@ static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 of bpf_insn_array_adjust_after_remove(env->insn_array_maps[i], off, len); } -static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) +static void adjust_poke_descs(struct bpf_prog *prog, u32 first, u32 len) { struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; int i, sz = prog->aux->size_poke_tab; @@ -300,17 +314,37 @@ static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) for (i = 0; i < sz; i++) { desc = &tab[i]; - if (desc->insn_idx <= off) + if (desc->insn_idx < first) continue; desc->insn_idx += len - 1; } } -struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, - const struct bpf_insn *patch, u32 len) +/* + * Some post-verification instruction rewriting passes require an + * O(prog->len) operation per instruction. Keep their shared primitives + * killable and preemptible. + */ +static bool bpf_rewrite_must_abort(void) +{ + if (fatal_signal_pending(current)) + return true; + cond_resched(); + return false; +} + +static struct bpf_prog *__bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, + const struct bpf_insn *patch, u32 len, + enum bpf_patch_mode mode) { struct bpf_prog *new_prog; struct bpf_insn_aux_data *new_data = NULL; + /* The first slot whose targets move. */ + u32 first = mode == BPF_PATCH_MOVE_TARGET ? off : off + 1; + struct bpf_insn original_insn; + + if (bpf_rewrite_must_abort()) + return NULL; if (len > 1) { new_data = vrealloc(env->insn_aux_data, @@ -323,6 +357,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, env->insn_aux_data = new_data; } + memcpy(&original_insn, env->prog->insnsi + off, sizeof(struct bpf_insn)); new_prog = bpf_patch_insn_single(env->prog, off, patch, len); if (IS_ERR(new_prog)) { if (PTR_ERR(new_prog) == -ERANGE) @@ -331,13 +366,58 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, env->insn_aux_data[off].orig_idx); return NULL; } - adjust_insn_aux_data(env, new_prog, off, len); + adjust_insn_aux_data(env, new_prog, off, len, &original_insn, mode); adjust_subprog_starts(env, off, len); - adjust_insn_arrays(env, off, len); - adjust_poke_descs(new_prog, off, len); + adjust_insn_arrays(env, first, len); + bpf_adjust_func_ptrs(env, off, len); + adjust_poke_descs(new_prog, first, len); return new_prog; } +struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, + const struct bpf_insn *patch, u32 len) +{ + return __bpf_patch_insn_data(env, off, patch, len, BPF_PATCH_KEEP_TARGET); +} + +/* + * insn was moved down by delta insns inside its own patch. Operands relative + * to the pc that point in front of the old position did not move with it. + */ +static int bpf_adj_moved_insn(struct bpf_insn *insn, u32 delta) +{ + u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code); + s64 off = insn->imm, off_min = S32_MIN; + bool is_imm = true; + + if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) { + /* subprog that started at the old position starts with the patch */ + if (off >= 0) + return 0; + } else if ((class == BPF_JMP || class == BPF_JMP32) && + op != BPF_CALL && op != BPF_EXIT) { + if (insn->code != (BPF_JMP32 | BPF_JA)) { + off = insn->off; + off_min = S16_MIN; + is_imm = false; + } + /* jump to itself stays */ + if (off >= -1) + return 0; + } else { + return 0; + } + + off -= delta; + if (off < off_min) + return -ERANGE; + if (is_imm) + insn->imm = off; + else + insn->off = off; + return 0; +} + /* * For all jmp insns in a given 'prog' that point to 'tgt_idx' insn adjust the * jump offset by 'delta'. @@ -423,6 +503,9 @@ static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, * in adjust_btf_func() - no need to adjust */ } + /* The exception callback is live, so it cannot sit inside the removed range. */ + if (env->exception_callback_subprog >= j) + env->exception_callback_subprog -= j - i; } else { /* convert i from "first prog to remove" to "first to adjust" */ if (env->subprog_info[i].start == off) @@ -500,9 +583,7 @@ static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, return 0; } -/* - * Clean up dynamically allocated fields of aux data for instructions [start, ...] - */ +/* Clean up dynamically allocated fields of aux data for instructions [start, ...] */ void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) { struct bpf_insn_aux_data *aux_data = env->insn_aux_data; @@ -510,9 +591,10 @@ void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) int i; for (i = start; i < end; i++) { - if (aux_data[i].jt) { - kvfree(aux_data[i].jt); - aux_data[i].jt = NULL; + if (aux_data[i].loop) { + kvfree(aux_data[i].loop->exits); + kvfree(aux_data[i].loop); + aux_data[i].loop = NULL; } } } @@ -523,6 +605,9 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) unsigned int orig_prog_len = env->prog->len; int err; + if (bpf_rewrite_must_abort()) + return -EINTR; + if (bpf_prog_is_offloaded(env->prog->aux)) bpf_prog_offload_remove_insns(env, off, cnt); @@ -532,6 +617,9 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) if (err) return err; + /* before subprogs are adjusted, since it looks at them */ + bpf_adjust_func_ptrs_after_remove(env, off, cnt); + err = adjust_subprog_starts_after_remove(env, off, cnt); if (err) return err; @@ -570,6 +658,7 @@ void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) { struct bpf_insn_aux_data *aux_data = env->insn_aux_data; struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); + struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); struct bpf_insn *insn = env->prog->insnsi; const int insn_cnt = env->prog->len; int i; @@ -578,6 +667,20 @@ void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) if (!bpf_insn_is_cond_jump(insn->code)) continue; + if (bpf_is_unwind_goto(insn) && aux_data[i + 1 + insn->off].seen) { + /* + * The callee never exits. Its unwind_exit needs + * unwind_goto all the same, and the cpu predicts that it + * returns to what follows. Don't let the removal of dead + * code put something else there. + */ + if (!aux_data[i + 1].seen) { + memcpy(insn + 1, &trap, sizeof(trap)); + aux_data[i + 1].seen = env->pass_cnt; + } + continue; + } + if (!aux_data[i + 1].seen) ja.off = insn->off; else if (!aux_data[i + 1 + insn->off].seen) @@ -592,12 +695,41 @@ void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) } } +/* + * The address of a function might be taken by the code that is not dead, + * while the function itself is dead code, since nothing calls it. Keep the first + * instruction of such function, so that the address remains valid and differs + * from the addresses of other functions. Make it a trap like + * sanitize_dead_code() does. + */ +static void keep_funcs_with_addr_taken(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux_data = env->insn_aux_data; + struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); + struct bpf_insn *insn = env->prog->insnsi; + int insn_cnt = env->prog->len; + int i, t; + + for (i = 0; i < insn_cnt; i++) { + if (!aux_data[i].seen || !bpf_pseudo_func(insn + i)) + continue; + t = i + insn[i].imm + 1; + if (aux_data[t].seen) + continue; + memcpy(insn + t, &trap, sizeof(trap)); + aux_data[t].zext_dst = false; + aux_data[t].seen = env->pass_cnt; + } +} + int bpf_opt_remove_dead_code(struct bpf_verifier_env *env) { struct bpf_insn_aux_data *aux_data = env->insn_aux_data; int insn_cnt = env->prog->len; int i, err; + keep_funcs_with_addr_taken(env); + for (i = 0; i < insn_cnt; i++) { int j; @@ -618,6 +750,7 @@ int bpf_opt_remove_dead_code(struct bpf_verifier_env *env) int bpf_opt_remove_nops(struct bpf_verifier_env *env) { + struct bpf_insn_aux_data *aux = env->insn_aux_data; struct bpf_insn *insn = env->prog->insnsi; int insn_cnt = env->prog->len; bool is_may_goto_0, is_ja; @@ -629,6 +762,8 @@ int bpf_opt_remove_nops(struct bpf_verifier_env *env) if (!is_may_goto_0 && !is_ja) continue; + if (aux[i].indirect_target) + continue; err = verifier_remove_insns(env, i, 1); if (err) @@ -738,11 +873,61 @@ apply_patch_buffer: return 0; } +/* + * The prologue takes the context from R1, and it can clobber R2-R5. An + * extension gets the arguments of the function that it replaces. + */ +static bool ext_prog_takes_prologue(const struct bpf_verifier_env *env) +{ + const struct bpf_subprog_info *sub = &env->subprog_info[0]; + int i, cnt = min_t(int, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); + + if (!cnt || sub->args[0].arg_type != ARG_PTR_TO_CTX) + return false; + + for (i = 1; i < cnt; i++) + if (sub->args[i].arg_type != ARG_SCALAR) + return false; + + return true; +} + /* convert load instructions that access fields of a context type into a * sequence of instructions that access fields of the underlying structure: * struct __sk_buff -> struct sk_buff * struct bpf_sock_ops -> struct sock */ +/* + * A load through an __arena pointer member of a kernel struct reads the kernel + * address of arena memory. Append the conversion to the arena pointer form, + * (u32)(kaddr - kern_vm_start), that the struct_ops trampoline applies to + * __arena arguments. A __nullable member keeps NULL as NULL, tested on the + * full kernel address, and so does a probed load, which reads as 0 when it + * faults. Returns the number of insns in @buf, 0 when the insn loads no such + * member, or an error when it does so on some paths only. + */ +static int arena_member_fixup(struct bpf_verifier_env *env, int idx, struct bpf_insn *insn, + struct bpf_insn *buf) +{ + struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx]; + int cnt = 0; + u32 base; + + if (!aux->arena_member) + return 0; + if (aux->non_arena_load) { + verbose(env, "an insn loads an arena pointer member on one path and something else on another\n"); + return -EINVAL; + } + + base = bpf_arena_get_kern_vm_start(env->prog->aux->arena); + buf[cnt++] = *insn; + if (aux->arena_member_nullable || BPF_MODE(insn->code) == BPF_PROBE_MEM) + buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, insn->dst_reg, 0, 1); + buf[cnt++] = BPF_ALU32_IMM(BPF_SUB, insn->dst_reg, (s32)base); + return cnt; +} + int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) { struct bpf_subprog_info *subprogs = env->subprog_info; @@ -771,7 +956,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1, -subprogs[0].stack_depth); insn_buf[cnt++] = env->prog->insnsi[0]; - new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); + new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt, + BPF_PATCH_MOVE_TARGET); if (!new_prog) return -ENOMEM; env->prog = new_prog; @@ -794,7 +980,20 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) verifier_bug(env, "prologue is too long"); return -EFAULT; } else if (cnt) { - new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); + if (env->prog->type == BPF_PROG_TYPE_EXT && + !ext_prog_takes_prologue(env)) { + verbose(env, + "Extension writing packet data must take ctx and scalars only\n"); + return -EINVAL; + } + if (env->prog->type == BPF_PROG_TYPE_EXT && + bpf_check_ext_prologue(&env->log, + env->prog->aux->dst_prog, + env->prog->aux->attach_btf_id)) + return -EINVAL; + env->prog->aux->has_prologue = true; + new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt, + BPF_PATCH_MOVE_TARGET); if (!new_prog) return -ENOMEM; @@ -825,7 +1024,13 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) struct bpf_insn *patch = insn_buf; *patch++ = BPF_ST_NOSPEC(); - *patch++ = *insn; + *patch = *insn; + ret = bpf_adj_moved_insn(patch++, 1); + if (ret) { + verbose(env, "insn %d cannot be patched due to 16-bit range\n", + env->insn_aux_data[i + delta].orig_idx); + return ret; + } cnt = patch - insn_buf; new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -867,6 +1072,21 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code); env->prog->aux->num_exentries++; continue; + } else if (insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW) && + insn->imm == BPF_LOAD_ACQ && + env->insn_aux_data[i + delta].arena_member) { + /* an acquire load of an __arena member is rewritten like a plain load */ + cnt = arena_member_fixup(env, i + delta, insn, insn_buf); + if (cnt < 0) + return cnt; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = new_prog; + insn = new_prog->insnsi + i + delta; + continue; } else if (insn->code == (BPF_JMP | BPF_EXIT) && epilogue_cnt && i + delta < subprogs[1].start) { @@ -958,11 +1178,23 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) insn->code = BPF_LDX | BPF_PROBE_MEMSX | BPF_SIZE(insn->code); env->prog->aux->num_exentries++; - continue; - } - if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, - "access to a fault prone pointer is not rewritten as a probed one")) + } else if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, + "access to a fault prone pointer is not rewritten as a probed one")) { return -EFAULT; + } + + cnt = arena_member_fixup(env, i + delta, insn, insn_buf); + if (cnt < 0) + return cnt; + if (!cnt) + continue; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = new_prog; + insn = new_prog->insnsi + i + delta; continue; } @@ -1068,6 +1300,59 @@ static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_s env->subprog_info[env->subprog_cnt].start = env->prog->len; } +/* + * Replace the offsets of functions with their addresses in the maps of + * the program, see resolve_func_ptrs() in verifier.c. The program must be + * the only user of such map. From now on no other program can use it, see + * bpf_map_claim(). + */ +static int resolve_func_ptrs(struct bpf_verifier_env *env, struct bpf_prog *prog, + struct bpf_prog **func) +{ + unsigned long me = (unsigned long)prog->aux; + struct bpf_func_ptr *ptr; + int i, err, subprog; + u64 addr, new, *slot; + + for (i = 0; i < env->func_ptr_cnt; i++) { + ptr = &env->func_ptrs[i]; + + /* pointers are sorted by map */ + if ((!i || ptr->map != ptr[-1].map) && + cmpxchg(&ptr->map->user, me, me | BPF_MAP_USER_PATCHED) != me) { + verbose(env, "map '%s' is used by another program\n", ptr->map->name); + return -EBUSY; + } + + /* it's the address of the value of the map whatever the offset is */ + err = ptr->map->ops->map_direct_value_addr(ptr->map, &addr, 0); + if (verifier_bug_if(err, env, "no value of map '%s'", ptr->map->name)) + return -EFAULT; + slot = (u64 *)(unsigned long)(addr + ptr->map_off); + + /* + * The function that is dead code is removed and the pointer is + * NULL. The program may read it, but callx of it is dead code + * too, see check_func_callx(). + */ + new = 0; + if (ptr->xlated_off != BPF_FUNC_PTR_DELETED) { + subprog = bpf_find_subprog(env, ptr->xlated_off); + if (verifier_bug_if(!func || subprog <= 0, env, "no function at insn %u", + ptr->xlated_off)) + return -EFAULT; + new = (unsigned long)func[subprog]->bpf_func; + } + + /* the map is frozen and nothing else uses it */ + if (verifier_bug_if(*slot != (u64)ptr->orig_off * sizeof(struct bpf_insn), env, + "map '%s' offset %u changed", ptr->map->name, ptr->map_off)) + return -EFAULT; + WRITE_ONCE(*slot, new); + } + return 0; +} + static int jit_subprogs(struct bpf_verifier_env *env) { struct bpf_prog *prog = env->prog, **func, *tmp; @@ -1178,6 +1463,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) func[i]->aux->jited_linfo = prog->aux->jited_linfo; func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; func[i]->aux->arena = prog->aux->arena; + func[i]->aux->uses_arena_insns = bpf_prog_has_arena_insns(func[i]); func[i]->aux->used_maps = env->used_maps; func[i]->aux->used_map_cnt = env->used_map_cnt; num_exentries = 0; @@ -1196,12 +1482,15 @@ static int jit_subprogs(struct bpf_verifier_env *env) if (BPF_CLASS(insn->code) == BPF_STX && BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) num_exentries++; + if (bpf_is_unwind_goto(insn)) + num_exentries++; } func[i]->aux->num_exentries = num_exentries; func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb; func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data; func[i]->aux->might_sleep = env->subprog_info[i].might_sleep; + func[i]->aux->can_unwind = env->subprog_info[i].can_unwind; func[i]->aux->token = prog->aux->token; if (!i) func[i]->aux->exception_boundary = env->seen_exception; @@ -1258,6 +1547,11 @@ static int jit_subprogs(struct bpf_verifier_env *env) cond_resched(); } + /* the addresses of all functions are final */ + err = resolve_func_ptrs(env, prog, func); + if (err) + goto out_free; + /* * Cleanup func[i]->aux fields which aren't required * or can become invalid in future @@ -1344,8 +1638,9 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) struct bpf_prog *prog, *orig_prog; u32 *orig_subprog_starts; + /* all functions that the maps point to are removed as dead code */ if (env->subprog_cnt <= 1) - return 0; + return resolve_func_ptrs(env, env->prog, NULL); prog = orig_prog = env->prog; if (bpf_prog_need_blind(prog)) { @@ -1356,7 +1651,7 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) } prog = bpf_jit_blind_constants(env, prog); if (IS_ERR(prog)) { - err = -ENOMEM; + err = PTR_ERR(prog); prog = orig_prog; goto out_restore; } @@ -1433,7 +1728,7 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) err = bpf_jit_subprogs(env); if (err == 0) return 0; - if (err == -EFAULT) + if (err == -EFAULT || err == -EINTR) return err; } #ifndef CONFIG_BPF_JIT_ALWAYS_ON @@ -1502,11 +1797,37 @@ static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *pat env->prog = prog; info[cnt + 1].start = info[cnt].start; info[cnt].start = prog->len - len + 1; + /* The patch ends with the exit. */ + info[cnt].exit_idx = prog->len - 1; env->subprog_cnt++; env->hidden_subprog_cnt++; return 0; } +/* + * Expand may_goto: load the count from the stack, jump to the target of + * may_goto when it is zero, then the tail that updates the count. + * Use gotol when the target is too far for 16-bit offset of a conditional jump. + */ +static int may_goto_expand(struct bpf_insn *insn_buf, int off, int stack_off, + const struct bpf_insn *tail, int tail_cnt) +{ + int cnt = 0; + + /* Forward jump has to step over the tail */ + off = off >= 0 ? off + tail_cnt : off - 1; + + insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off); + if (off == (s16)off) { + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, off); + } else { + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 1); + insn_buf[cnt++] = BPF_JMP32_A(off >= 0 ? off : off - 1); + } + memcpy(insn_buf + cnt, tail, tail_cnt * sizeof(*tail)); + return cnt + tail_cnt; +} + /* Do various post-verification rewrites in a single program pass. * These rewrites simplify JIT and interpreter implementations. */ @@ -1559,6 +1880,20 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */ insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code); + if (env->insn_aux_data[i + delta].arena_scalar) { + cnt = arena_scalar_access(insn, insn_buf); + + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + prog = new_prog; + env->prog = prog; + insn = prog->insnsi + i + delta; + goto next_insn; + } + /* Make sdiv/smod divide-by-minus-one exceptions impossible. */ if ((insn->code == (BPF_ALU64 | BPF_MOD | BPF_K) || insn->code == (BPF_ALU64 | BPF_DIV | BPF_K) || @@ -1784,6 +2119,18 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) if (bpf_is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) { int stack_off_cnt = -stack_depth - 16; + /* + * AX is used as an argument to pass in stack_off_cnt + * (to add to r10/fp), and also as the return value of + * the call to arch_bpf_timed_may_goto. + */ + struct bpf_insn tail[] = { + BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1), + BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2), + BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt), + BPF_EMIT_CALL(arch_bpf_timed_may_goto), + BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt), + }; /* * Two 8 byte slots, depth-16 stores the count, and @@ -1800,22 +2147,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) * after subtraction, rinse and repeat. */ stack_depth_extra = 16; - insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off_cnt); - if (insn->off >= 0) - insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 5); - else - insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); - insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); - insn_buf[3] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2); - /* - * AX is used as an argument to pass in stack_off_cnt - * (to add to r10/fp), and also as the return value of - * the call to arch_bpf_timed_may_goto. - */ - insn_buf[4] = BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt); - insn_buf[5] = BPF_EMIT_CALL(arch_bpf_timed_may_goto); - insn_buf[6] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt); - cnt = 7; + cnt = may_goto_expand(insn_buf, insn->off, stack_off_cnt, + tail, ARRAY_SIZE(tail)); new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -1827,16 +2160,14 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) goto next_insn; } else if (bpf_is_may_goto_insn(insn)) { int stack_off = -stack_depth - 8; + struct bpf_insn tail[] = { + BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1), + BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off), + }; stack_depth_extra = 8; - insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off); - if (insn->off >= 0) - insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 2); - else - insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); - insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); - insn_buf[3] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off); - cnt = 4; + cnt = may_goto_expand(insn_buf, insn->off, stack_off, + tail, ARRAY_SIZE(tail)); new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -1928,6 +2259,10 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) if (!bpf_allow_tail_call_in_subprogs(env)) prog->aux->stack_depth = MAX_BPF_STACK; prog->aux->max_pkt_offset = MAX_PACKET_OFF; + prog->aux->max_rdonly_access = U32_MAX; + prog->aux->max_rdwr_access = U32_MAX; + if (prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) + prog->aux->max_tp_access = U32_MAX; /* mark bpf_tail_call as different opcode to avoid * conditional branch in the interpreter for every normal @@ -1993,7 +2328,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) goto next_insn; } - if (insn->imm == BPF_FUNC_timer_set_callback) { + aux = &env->insn_aux_data[i + delta]; + if (aux->arg_prog) { /* The verifier will process callback_fn as many times as necessary * with different maps and the register states prepared by * set_timer_callback_state will be accurate. @@ -2008,7 +2344,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) * bpf_timer_set_callback-ed will return -EINVAL. */ struct bpf_insn ld_addrs[2] = { - BPF_LD_IMM64(BPF_REG_3, (long)prog->aux), + BPF_LD_IMM64(aux->arg_prog, (long)prog->aux), }; insn_buf[0] = ld_addrs[0]; @@ -2236,7 +2572,7 @@ patch_map_ops_generic: cnt = 2; } insn_buf[cnt++] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); - insn_buf[cnt++] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); + insn_buf[cnt++] = BPF_ALU32_IMM(BPF_LSH, BPF_REG_2, 3); insn_buf[cnt++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); @@ -2457,9 +2793,17 @@ next_insn: BPF_MAX_LOOPS); } /* Copy first actual insn to preserve it */ - insn_buf[cnt++] = env->prog->insnsi[subprog_start]; + insn_buf[cnt] = env->prog->insnsi[subprog_start]; + ret = bpf_adj_moved_insn(&insn_buf[cnt], cnt); + if (ret) { + verbose(env, "insn %d cannot be patched due to 16-bit range\n", + env->insn_aux_data[subprog_start].orig_idx); + return ret; + } + cnt++; - new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt); + new_prog = __bpf_patch_insn_data(env, subprog_start, insn_buf, cnt, + BPF_PATCH_MOVE_TARGET); if (!new_prog) return -ENOMEM; env->prog = prog = new_prog; @@ -2489,10 +2833,6 @@ next_insn: } } - ret = sort_kfunc_descs_by_imm_off(env); - if (ret) - return ret; - return 0; } @@ -2666,4 +3006,3 @@ int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env) return 0; } - diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 6f331c80130d..bf78e09f4d51 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -100,8 +100,10 @@ struct bpf_htab { struct percpu_counter pcount; atomic_t count; bool use_percpu_counter; + bool has_hash; u32 n_buckets; /* number of hash buckets */ u32 elem_size; /* size of each element in bytes */ + u32 elem_offset;/* offset of htab_elem in bytes */ u32 hashrnd; }; @@ -117,18 +119,46 @@ struct htab_elem { }; }; }; - union { - /* pointer to per-cpu pointer */ - void *ptr_to_pptr; - struct bpf_lru_node lru_node; - }; - u32 hash; char key[] __aligned(8); }; +struct htab_elem_hashed { + u32 hash __aligned(8); + struct htab_elem elem; +}; + +struct htab_elem_lru { + struct bpf_lru_node lru_node; + struct htab_elem_hashed; +}; + +/* + * Only for non-preallocated PCPU maps with small keys. Preallocated PCPU maps + * don't need ptr_to_pptr, and use htab_elem. + */ +struct htab_elem_pcpu { + /* pointer to per-cpu pointer */ + void *ptr_to_pptr; + struct htab_elem elem; +}; + +/* + * Only for non-preallocated PCPU maps with large keys. Preallocated PCPU maps + * don't need ptr_to_pptr, and use htab_elem_hashed. + */ +struct htab_elem_pcpu_hashed { + /* pointer to per-cpu pointer */ + void *ptr_to_pptr; + struct htab_elem_hashed; +}; + +static_assert(offsetof(struct htab_elem_pcpu_hashed, ptr_to_pptr) == + offsetof(struct htab_elem_pcpu, ptr_to_pptr)); + struct htab_btf_record { struct btf_record *record; - u32 key_size; + struct btf *btf; + u32 value_offset; }; static inline bool htab_is_prealloc(const struct bpf_htab *htab) @@ -183,6 +213,71 @@ static inline bool is_fd_htab(const struct bpf_htab *htab) return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS; } +/* + * u32 hash reads are always atomic. If the hash is elided, key comparisons + * must also be atomic to avoid false positive matches from torn key + * reads/writes on recycled elements. That is only possible when the + * key fits within a word. + */ +static __always_inline bool htab_key_size_needs_hash(u32 key_size) +{ + return key_size > sizeof(unsigned long); +} + +/* + * When hash is omitted, key reads and writes must be atomic to support lockless + * RCU readers. Zero-extend keys to the word size to support all sub-word key + * sizes, regardless of the size of the caller's key allocation. + */ +static __always_inline unsigned long htab_zero_extend_key(const void *key, + u32 key_size) +{ + unsigned long k = 0; + + memcpy(&k, key, min_t(u32, key_size, sizeof(k))); + return k; +} + +static bool htab_has_hash(const struct bpf_htab *htab) +{ + return htab->has_hash; +} + +static u32 htab_elem_hash(struct htab_elem *l) +{ + return container_of(l, struct htab_elem_hashed, elem)->hash; +} + +static void htab_elem_set_hash(struct htab_elem *l, u32 hash) +{ + container_of(l, struct htab_elem_hashed, elem)->hash = hash; +} + +static void *htab_elem_container(const struct bpf_htab *htab, struct htab_elem *l) +{ + return (void *)l - htab->elem_offset; +} + +static void *htab_elem_get_ptr_to_pptr(const struct bpf_htab *htab, struct htab_elem *l) +{ + struct htab_elem_pcpu *pcpu_elem = htab_elem_container(htab, l); + + return pcpu_elem->ptr_to_pptr; +} + +static void htab_elem_set_ptr_to_pptr(const struct bpf_htab *htab, struct htab_elem *l, + void *ptr) +{ + struct htab_elem_pcpu *pcpu_elem = htab_elem_container(htab, l); + + pcpu_elem->ptr_to_pptr = ptr; +} + +static struct bpf_lru_node *htab_elem_lru_node(struct htab_elem *l) +{ + return &container_of(l, struct htab_elem_lru, elem)->lru_node; +} + static inline void *htab_elem_value(struct htab_elem *l, u32 key_size) { return l->key + round_up(key_size, 8); @@ -206,7 +301,7 @@ static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l) static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i) { - return (struct htab_elem *) (htab->elems + i * (u64)htab->elem_size); + return htab->elems + i * (u64)htab->elem_size + htab->elem_offset; } /* Both percpu and fd htab support in-place update, so no need for @@ -300,16 +395,16 @@ free_elems: * bucket_lock followed by lru_lock is not allowed. In such cases, * bucket_lock needs to be released first before acquiring lru_lock. */ -static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key, - u32 hash) +static struct htab_elem_lru *prealloc_lru_pop(struct bpf_htab *htab, void *key, + u32 hash) { struct bpf_lru_node *node = bpf_lru_pop_free(&htab->lru, hash); - struct htab_elem *l; + struct htab_elem_lru *l; if (node) { bpf_map_inc_elem_count(&htab->map); - l = container_of(node, struct htab_elem, lru_node); - memcpy(l->key, key, htab->map.key_size); + l = container_of(node, struct htab_elem_lru, lru_node); + memcpy(l->elem.key, key, htab->map.key_size); return l; } @@ -349,8 +444,8 @@ skip_percpu_elems: if (htab_is_lru(htab)) err = bpf_lru_init(&htab->lru, htab->map.map_flags & BPF_F_NO_COMMON_LRU, - offsetof(struct htab_elem, hash) - - offsetof(struct htab_elem, lru_node), + offsetof(struct htab_elem_lru, hash) - + offsetof(struct htab_elem_lru, lru_node), htab_lru_map_delete_node, htab); else @@ -361,11 +456,12 @@ skip_percpu_elems: if (htab_is_lru(htab)) bpf_lru_populate(&htab->lru, htab->elems, - offsetof(struct htab_elem, lru_node), + offsetof(struct htab_elem_lru, lru_node), htab->elem_size, num_entries); else pcpu_freelist_populate(&htab->freelist, - htab->elems + offsetof(struct htab_elem, fnode), + htab->elems + htab->elem_offset + + offsetof(struct htab_elem, fnode), htab->elem_size, num_entries); return 0; @@ -453,7 +549,7 @@ static int htab_map_alloc_check(union bpf_attr *attr) return -EINVAL; if ((u64)attr->key_size + attr->value_size >= KMALLOC_MAX_SIZE - - sizeof(struct htab_elem)) + sizeof(struct htab_elem_lru)) /* if key_size + value_size is bigger, the user space won't be * able to access the elements via bpf syscall. This check * also makes sure that the elem_size doesn't overflow and it's @@ -470,14 +566,11 @@ static int htab_map_alloc_check(union bpf_attr *attr) static void htab_mem_dtor(void *obj, void *ctx) { struct htab_btf_record *hrec = ctx; - struct htab_elem *elem = obj; - void *map_value; if (IS_ERR_OR_NULL(hrec->record)) return; - map_value = htab_elem_value(elem, hrec->key_size); - bpf_obj_free_fields(hrec->record, map_value); + bpf_obj_free_fields(hrec->record, obj + hrec->value_offset); } static void htab_pcpu_mem_dtor(void *obj, void *ctx) @@ -497,12 +590,17 @@ static void htab_dtor_ctx_free(void *ctx) { struct htab_btf_record *hrec = ctx; + /* + * The duplicated record still points into the map BTF, so free it + * before dropping the reference that keeps that BTF alive. + */ btf_record_free(hrec->record); - kfree(ctx); + btf_put(hrec->btf); + kfree(hrec); } static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma, - void (*dtor)(void *, void *)) + void (*dtor)(void *, void *), u32 value_offset) { struct htab_btf_record *hrec; int err; @@ -514,13 +612,22 @@ static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma, hrec = kzalloc_obj(*hrec); if (!hrec) return -ENOMEM; - hrec->key_size = map->key_size; + hrec->value_offset = value_offset; hrec->record = btf_record_dup(map->record); if (IS_ERR(hrec->record)) { err = PTR_ERR(hrec->record); kfree(hrec); return err; } + /* + * btf_record_dup() only acquires kernel and module BTF. Fields whose + * types live in the map BTF keep pointing into it: kptrs to local + * types refer to map->btf, and graph roots carry a value record owned + * by its struct meta table. The context can outlive the map when the + * allocator defers its teardown, so hold a reference of our own. + */ + hrec->btf = map->btf; + btf_get(hrec->btf); bpf_mem_alloc_set_dtor(ma, dtor, htab_dtor_ctx_free, hrec); return 0; } @@ -540,9 +647,12 @@ static int htab_map_check_btf(struct bpf_map *map, const struct btf *btf, * populated in htab_map_alloc(), so it will always appear as NULL. */ if (htab_is_percpu(htab)) - return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor); + return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor, 0); else - return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor); + return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor, + htab->elem_offset + + offsetof(struct htab_elem, key) + + round_up(map->key_size, 8)); } static struct bpf_map *htab_map_alloc(union bpf_attr *attr) @@ -565,6 +675,10 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) bpf_map_init_from_attr(&htab->map, attr); + /* Avoid hash memory use and comparisons where unnecessary. */ + htab->has_hash = htab_is_lru(htab) || + htab_key_size_needs_hash(htab->map.key_size); + if (percpu_lru) { /* ensure each CPU's lru list has >=1 elements. * since we are at it, make each lru list has the same @@ -586,7 +700,17 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) htab->n_buckets = roundup_pow_of_two(htab->map.max_entries); - htab->elem_size = sizeof(struct htab_elem) + + if (htab_is_lru(htab)) + htab->elem_offset = offsetof(struct htab_elem_lru, elem); + else if (percpu && !prealloc) + htab->elem_offset = htab_has_hash(htab) ? + offsetof(struct htab_elem_pcpu_hashed, elem) : + offsetof(struct htab_elem_pcpu, elem); + else if (htab_has_hash(htab)) + htab->elem_offset = offsetof(struct htab_elem_hashed, elem); + + htab->elem_size = htab->elem_offset + + sizeof(struct htab_elem) + round_up(htab->map.key_size, 8); if (percpu) htab->elem_size += sizeof(void *); @@ -677,11 +801,18 @@ free_htab: return ERR_PTR(err); } -static inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd) +static __always_inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd) { - if (likely(key_len % 4 == 0)) + const u32 *k = key; + + if (key_len == sizeof(u32)) + return jhash_1word(k[0], hashrnd); + else if (key_len == sizeof(u64)) + return jhash_2words(k[0], k[1], hashrnd); + else if (likely(key_len % 4 == 0)) return jhash2(key, key_len / 4, hashrnd); - return jhash(key, key_len, hashrnd); + else + return jhash(key, key_len, hashrnd); } static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash) @@ -694,35 +825,57 @@ static inline struct hlist_nulls_head *select_bucket(struct bpf_htab *htab, u32 return &__select_bucket(htab, hash)->head; } -/* this lookup function can only be called with bucket lock taken */ -static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash, - void *key, u32 key_size) +static __always_inline struct hlist_nulls_node * +__lookup_elem_raw(struct hlist_nulls_head *head, u32 hash, void *key, + u32 key_size, bool elem_has_hash) { struct hlist_nulls_node *n; struct htab_elem *l; - hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) - if (l->hash == hash && !memcmp(&l->key, key, key_size)) - return l; + if (elem_has_hash) { + hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) + if (htab_elem_hash(l) == hash && + !memcmp(&l->key, key, key_size)) + break; + } else { + unsigned long k = htab_zero_extend_key(key, key_size); - return NULL; + hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) + if (READ_ONCE(*(unsigned long *)l->key) == k) + break; + } + + return n; +} + +/* this lookup function can only be called with bucket lock taken */ +static __always_inline struct htab_elem * +lookup_elem_raw(struct hlist_nulls_head *head, u32 hash, void *key, + u32 key_size, bool elem_has_hash) +{ + struct hlist_nulls_node *n; + + n = __lookup_elem_raw(head, hash, key, key_size, elem_has_hash); + if (is_a_nulls(n)) + return NULL; + + return hlist_nulls_entry(n, struct htab_elem, hash_node); } /* can be called without bucket lock. it will repeat the loop in * the unlikely event when elements moved from one bucket into another * while link list is being walked */ -static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head, - u32 hash, void *key, - u32 key_size, u32 n_buckets) +static __always_inline struct htab_elem * +lookup_nulls_elem_raw(struct hlist_nulls_head *head, u32 hash, void *key, + u32 key_size, bool elem_has_hash, u32 n_buckets) { struct hlist_nulls_node *n; - struct htab_elem *l; again: - hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) - if (l->hash == hash && !memcmp(&l->key, key, key_size)) - return l; + n = __lookup_elem_raw(head, hash, key, key_size, elem_has_hash); + if (!is_a_nulls(n)) + return hlist_nulls_entry(n, struct htab_elem, hash_node); if (unlikely(get_nulls_value(n) != (hash & (n_buckets - 1)))) goto again; @@ -730,31 +883,56 @@ again: return NULL; } -/* Called from syscall or from eBPF program directly, so - * arguments have to match bpf_map_lookup_elem() exactly. - * The return value is adjusted by BPF instructions - * in htab_map_gen_lookup(). - */ -static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) +static __always_inline void *__htab_lookup(struct bpf_map *map, void *key, + u32 key_size, bool elem_has_hash) { struct bpf_htab *htab = container_of(map, struct bpf_htab, map); struct hlist_nulls_head *head; struct htab_elem *l; - u32 hash, key_size; + u32 hash; WARN_ON_ONCE(!bpf_rcu_lock_held()); - key_size = map->key_size; - hash = htab_map_hash(key, key_size, htab->hashrnd); head = select_bucket(htab, hash); - l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); + l = lookup_nulls_elem_raw(head, hash, key, key_size, elem_has_hash, + htab->n_buckets); return l; } +/* + * Called from syscall or from eBPF program directly, so + * arguments have to match bpf_map_lookup_elem() exactly. + * The return value is adjusted by BPF instructions + * in htab_map_gen_lookup(). u32/u64 specializations + * allow utilizing compile-time key_size for optimization. + */ +static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) +{ + struct bpf_htab *htab = container_of(map, struct bpf_htab, map); + + return __htab_lookup(map, key, map->key_size, htab_has_hash(htab)); +} + +/* + * Specialized lookups for htab_map_gen_lookup(). These are only used for + * BPF_MAP_TYPE_HASH (never LRU), so elem_has_hash depends only on key size. + */ +static void *__htab_map_lookup_elem_u32(struct bpf_map *map, void *key) +{ + return __htab_lookup(map, key, sizeof(u32), + htab_key_size_needs_hash(sizeof(u32))); +} + +static void *__htab_map_lookup_elem_u64(struct bpf_map *map, void *key) +{ + return __htab_lookup(map, key, sizeof(u64), + htab_key_size_needs_hash(sizeof(u64))); +} + static void *htab_map_lookup_elem(struct bpf_map *map, void *key) { struct htab_elem *l = __htab_map_lookup_elem(map, key); @@ -783,7 +961,16 @@ static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, (void *(*)(struct bpf_map *map, void *key))NULL)); - *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); + BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem_u32, + (void *(*)(struct bpf_map *map, void *key))NULL)); + BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem_u64, + (void *(*)(struct bpf_map *map, void *key))NULL)); + if (map->key_size == sizeof(u32)) + *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem_u32); + else if (map->key_size == sizeof(u64)) + *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem_u64); + else + *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1); *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, offsetof(struct htab_elem, key) + @@ -798,7 +985,7 @@ static __always_inline void *__htab_lru_map_lookup_elem(struct bpf_map *map, if (l) { if (mark) - bpf_lru_node_set_ref(&l->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l)); return htab_elem_value(l, map->key_size); } @@ -821,19 +1008,17 @@ static int htab_lru_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn = insn_buf; const int ret = BPF_REG_0; const int ref_reg = BPF_REG_1; + const s16 ref_off = (int)offsetof(struct htab_elem_lru, lru_node) + + (int)offsetof(struct bpf_lru_node, ref) - + (int)offsetof(struct htab_elem_lru, elem); BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, (void *(*)(struct bpf_map *map, void *key))NULL)); *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 4); - *insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret, - offsetof(struct htab_elem, lru_node) + - offsetof(struct bpf_lru_node, ref)); + *insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret, ref_off); *insn++ = BPF_JMP_IMM(BPF_JNE, ref_reg, 0, 1); - *insn++ = BPF_ST_MEM(BPF_B, ret, - offsetof(struct htab_elem, lru_node) + - offsetof(struct bpf_lru_node, ref), - 1); + *insn++ = BPF_ST_MEM(BPF_B, ret, ref_off, 1); *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, offsetof(struct htab_elem, key) + round_up(map->key_size, 8)); @@ -865,14 +1050,15 @@ static void check_and_cancel_fields(struct bpf_htab *htab, static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node) { struct bpf_htab *htab = arg; - struct htab_elem *l = NULL, *tgt_l; + struct htab_elem_lru *tgt_l; + struct htab_elem *l = NULL; struct hlist_nulls_head *head; struct hlist_nulls_node *n; unsigned long flags; struct bucket *b; int ret; - tgt_l = container_of(node, struct htab_elem, lru_node); + tgt_l = container_of(node, struct htab_elem_lru, lru_node); b = __select_bucket(htab, tgt_l->hash); head = &b->head; @@ -881,7 +1067,7 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node) return false; hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) - if (l == tgt_l) { + if (l == &tgt_l->elem) { hlist_nulls_del_rcu(&l->hash_node); bpf_map_dec_elem_count(&htab->map); break; @@ -889,9 +1075,9 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node) htab_unlock_bucket(b, flags); - if (l == tgt_l) + if (l == &tgt_l->elem) check_and_cancel_fields(htab, l); - return l == tgt_l; + return l == &tgt_l->elem; } /* Called from syscall */ @@ -915,7 +1101,8 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key) head = select_bucket(htab, hash); /* lookup the key */ - l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); + l = lookup_nulls_elem_raw(head, hash, key, key_size, + htab_has_hash(htab), htab->n_buckets); if (!l) goto find_first_elem; @@ -958,8 +1145,8 @@ static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l) check_and_cancel_fields(htab, l); if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) - bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr); - bpf_mem_cache_free(&htab->ma, l); + bpf_mem_cache_free(&htab->pcpu_ma, htab_elem_get_ptr_to_pptr(htab, l)); + bpf_mem_cache_free(&htab->ma, htab_elem_container(htab, l)); } static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l) @@ -1054,14 +1241,17 @@ static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr, /* When not setting the initial value on all cpus, zero-fill element * values for other cpus. Otherwise, bpf program has no way to ensure * known initial values for cpus other than current one - * (onallcpus=false always when coming from bpf prog). + * (onallcpus=false always when coming from bpf prog, + * map_flags & BPF_F_CPU when coming from syscall but setting + * only one cpu). */ - if (!onallcpus) { - int current_cpu = raw_smp_processor_id(); + if (!onallcpus || (map_flags & BPF_F_CPU)) { + int init_cpu = (map_flags & BPF_F_CPU) ? map_flags >> 32 : + raw_smp_processor_id(); int cpu; for_each_possible_cpu(cpu) { - if (cpu == current_cpu) + if (cpu == init_cpu) copy_map_value(&htab->map, per_cpu_ptr(pptr, cpu), value); else /* Since elem is preallocated, we cannot touch special fields */ zero_map_value(&htab->map, per_cpu_ptr(pptr, cpu)); @@ -1089,11 +1279,17 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, if (prealloc) { if (old_elem) { /* if we're updating the existing element, - * use per-cpu extra elems to avoid freelist_pop/push + * use per-cpu extra elems to avoid freelist_pop/push. + * The caller puts old_elem into the slot once it is + * unlinked. Until then the slot is empty for an NMI + * or a kprobe that updates this map on this cpu. */ pl_new = this_cpu_ptr(htab->extra_elems); - l_new = *pl_new; - *pl_new = old_elem; + l_new = READ_ONCE(*pl_new); + do { + if (!l_new) + return ERR_PTR(-EBUSY); + } while (!try_cmpxchg_local(pl_new, &l_new, NULL)); } else { struct pcpu_freelist_node *l; @@ -1104,6 +1300,8 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, bpf_map_inc_elem_count(&htab->map); } } else { + void *container; + if (is_map_full(htab)) if (!old_elem) /* when map is full and update() is replacing @@ -1113,14 +1311,17 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, */ return ERR_PTR(-E2BIG); inc_elem_count(htab); - l_new = bpf_mem_cache_alloc(&htab->ma); - if (!l_new) { + container = bpf_mem_cache_alloc(&htab->ma); + if (!container) { l_new = ERR_PTR(-ENOMEM); goto dec_count; } + l_new = container + htab->elem_offset; } - memcpy(l_new->key, key, key_size); + if (htab_has_hash(htab)) + memcpy(l_new->key, key, key_size); + if (percpu) { if (prealloc) { pptr = htab_elem_get_ptr(l_new, key_size); @@ -1129,11 +1330,11 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, void *ptr = bpf_mem_cache_alloc(&htab->pcpu_ma); if (!ptr) { - bpf_mem_cache_free(&htab->ma, l_new); + bpf_mem_cache_free(&htab->ma, htab_elem_container(htab, l_new)); l_new = ERR_PTR(-ENOMEM); goto dec_count; } - l_new->ptr_to_pptr = ptr; + htab_elem_set_ptr_to_pptr(htab, l_new, ptr); pptr = *(void __percpu **)ptr; } @@ -1152,7 +1353,11 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, copy_map_value(&htab->map, htab_elem_value(l_new, key_size), value); } - l_new->hash = hash; + if (htab_has_hash(htab)) + htab_elem_set_hash(l_new, hash); + else + WRITE_ONCE(*(unsigned long *)l_new->key, + htab_zero_extend_key(key, key_size)); return l_new; dec_count: dec_elem_count(htab); @@ -1162,11 +1367,11 @@ dec_count: static int check_flags(struct bpf_htab *htab, struct htab_elem *l_old, u64 map_flags) { - if (l_old && (map_flags & ~BPF_F_LOCK) == BPF_NOEXIST) + if (l_old && (map_flags & BPF_NOEXIST)) /* elem already exists */ return -EEXIST; - if (!l_old && (map_flags & ~BPF_F_LOCK) == BPF_EXIST) + if (!l_old && (map_flags & BPF_EXIST)) /* elem doesn't exist, cannot update it */ return -ENOENT; @@ -1203,6 +1408,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, return -EINVAL; /* find an element without taking the bucket lock */ l_old = lookup_nulls_elem_raw(head, hash, key, key_size, + htab_has_hash(htab), htab->n_buckets); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1224,7 +1430,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, if (ret) return ret; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(head, hash, key, key_size, htab_has_hash(htab)); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1259,11 +1465,15 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, if (l_old) { hlist_nulls_del_rcu(&l_old->hash_node); - /* l_old has already been stashed in htab->extra_elems, cancel - * its reusable special fields before it is available for reuse. + /* + * l_old replaces the extra element that alloc_htab_elem() took. + * Cancel its reusable special fields first: once it is in the + * slot, the next update on this cpu can reuse it. */ - if (htab_is_prealloc(htab)) + if (htab_is_prealloc(htab)) { check_and_cancel_fields(htab, l_old); + WRITE_ONCE(*this_cpu_ptr(htab->extra_elems), l_old); + } } htab_unlock_bucket(b, flags); if (l_old && !htab_is_prealloc(htab)) @@ -1278,14 +1488,15 @@ static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem) { check_and_cancel_fields(htab, elem); bpf_map_dec_elem_count(&htab->map); - bpf_lru_push_free(&htab->lru, &elem->lru_node); + bpf_lru_push_free(&htab->lru, htab_elem_lru_node(elem)); } static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value, u64 map_flags) { struct bpf_htab *htab = container_of(map, struct bpf_htab, map); - struct htab_elem *l_new, *l_old = NULL; + struct htab_elem *l_old = NULL; + struct htab_elem_lru *l_new; struct hlist_nulls_head *head; unsigned long flags; struct bucket *b; @@ -1313,13 +1524,13 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value l_new = prealloc_lru_pop(htab, key, hash); if (!l_new) return -ENOMEM; - copy_map_value(&htab->map, htab_elem_value(l_new, map->key_size), value); + copy_map_value(&htab->map, htab_elem_value(&l_new->elem, map->key_size), value); ret = htab_lock_bucket(b, &flags); if (ret) goto err_lock_bucket; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(head, hash, key, key_size, true); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1328,7 +1539,7 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value /* add new element to the head of the list, so that * concurrent search will find it before old elem */ - hlist_nulls_add_head_rcu(&l_new->hash_node, head); + hlist_nulls_add_head_rcu(&l_new->elem.hash_node, head); if (l_old) { bpf_lru_node_set_ref(&l_new->lru_node); hlist_nulls_del_rcu(&l_old->hash_node); @@ -1340,7 +1551,7 @@ err: err_lock_bucket: if (ret) - htab_lru_push_free(htab, l_new); + htab_lru_push_free(htab, &l_new->elem); else if (l_old) htab_lru_push_free(htab, l_old); @@ -1349,9 +1560,11 @@ err_lock_bucket: static int htab_map_check_update_flags(bool onallcpus, u64 map_flags) { + if (unlikely((map_flags & BPF_EXIST_FLAGS) == BPF_EXIST_FLAGS)) + return -EINVAL; if (unlikely(!onallcpus && map_flags > BPF_EXIST)) return -EINVAL; - if (unlikely(onallcpus && ((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS))) + if (unlikely(onallcpus && ((u32)map_flags & ~(BPF_EXIST_FLAGS | BPF_CPU_FLAGS)))) return -EINVAL; return 0; } @@ -1386,7 +1599,7 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key, if (ret) return ret; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(head, hash, key, key_size, htab_has_hash(htab)); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1424,7 +1637,8 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, bool onallcpus) { struct bpf_htab *htab = container_of(map, struct bpf_htab, map); - struct htab_elem *l_new = NULL, *l_old; + struct htab_elem_lru *l_new = NULL; + struct htab_elem *l_old; struct hlist_nulls_head *head; unsigned long flags; struct bucket *b; @@ -1449,7 +1663,7 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, * to remove older elem from htab and this removal * operation will need a bucket lock. */ - if (map_flags != BPF_EXIST) { + if (!(map_flags & BPF_EXIST)) { l_new = prealloc_lru_pop(htab, key, hash); if (!l_new) return -ENOMEM; @@ -1459,22 +1673,22 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, if (ret) goto err_lock_bucket; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(head, hash, key, key_size, true); ret = check_flags(htab, l_old, map_flags); if (ret) goto err; if (l_old) { - bpf_lru_node_set_ref(&l_old->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l_old)); /* per-cpu hash map can update value in-place */ pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size), value, onallcpus, map_flags); } else { - pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size), + pcpu_init_value(htab, htab_elem_get_ptr(&l_new->elem, key_size), value, onallcpus, map_flags); - hlist_nulls_add_head_rcu(&l_new->hash_node, head); + hlist_nulls_add_head_rcu(&l_new->elem.hash_node, head); l_new = NULL; } ret = 0; @@ -1524,7 +1738,7 @@ static long htab_map_delete_elem(struct bpf_map *map, void *key) if (ret) return ret; - l = lookup_elem_raw(head, hash, key, key_size); + l = lookup_elem_raw(head, hash, key, key_size, htab_has_hash(htab)); if (l) hlist_nulls_del_rcu(&l->hash_node); else @@ -1559,7 +1773,7 @@ static long htab_lru_map_delete_elem(struct bpf_map *map, void *key) if (ret) return ret; - l = lookup_elem_raw(head, hash, key, key_size); + l = lookup_elem_raw(head, hash, key, key_size, true); if (l) hlist_nulls_del_rcu(&l->hash_node); @@ -1700,7 +1914,7 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, if (ret) return ret; - l = lookup_elem_raw(head, hash, key, key_size); + l = lookup_elem_raw(head, hash, key, key_size, htab_has_hash(htab)); if (!l) { ret = -ENOENT; goto out_unlock; @@ -1772,6 +1986,12 @@ static int htab_lru_percpu_map_lookup_and_delete_elem(struct bpf_map *map, flags); } +/* + * Max consecutive empty buckets to walk in one RCU + + * instrumentation-disabled section before rescheduling. + */ +#define HTAB_BATCH_EMPTY_RESCHED 64 + static int __htab_map_lookup_and_delete_batch(struct bpf_map *map, const union bpf_attr *attr, @@ -1793,6 +2013,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, unsigned long flags = 0; bool locked = false; struct htab_elem *l; + u32 empty_cnt = 0; struct bucket *b; int ret = 0; @@ -1971,30 +2192,41 @@ again_nocopy: } next_batch: - /* If we are not copying data, we can go to next bucket and avoid - * unlocking the rcu. + /* + * If we are not copying data, we can go to next bucket and avoid + * unlocking the rcu. Bound the walk though: after + * HTAB_BATCH_EMPTY_RESCHED consecutive empty buckets, fully exit + * the critical section (no locks are held here) and reschedule. */ if (!bucket_cnt && (batch + 1 < htab->n_buckets)) { batch++; - goto again_nocopy; + if (++empty_cnt < HTAB_BATCH_EMPTY_RESCHED) + goto again_nocopy; + empty_cnt = 0; + rcu_read_unlock(); + bpf_enable_instrumentation(); + cond_resched_tasks_rcu_qs(); + goto again; } rcu_read_unlock(); bpf_enable_instrumentation(); - if (bucket_cnt && (copy_to_user(ukeys + total * key_size, keys, - key_size * bucket_cnt) || - copy_to_user(uvalues + total * value_size, values, - value_size * bucket_cnt))) { + if (bucket_cnt && (copy_to_user(ukeys + (size_t)total * key_size, keys, + (size_t)key_size * bucket_cnt) || + copy_to_user(uvalues + (size_t)total * value_size, values, + (size_t)value_size * bucket_cnt))) { ret = -EFAULT; goto after_loop; } total += bucket_cnt; + empty_cnt = 0; batch++; if (batch >= htab->n_buckets) { ret = -ENOENT; goto after_loop; } + cond_resched_tasks_rcu_qs(); goto again; after_loop: @@ -2453,7 +2685,7 @@ static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key) struct htab_elem *l = __htab_map_lookup_elem(map, key); if (l) { - bpf_lru_node_set_ref(&l->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l)); return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); } @@ -2469,7 +2701,7 @@ static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *k l = __htab_map_lookup_elem(map, key); if (l) { - bpf_lru_node_set_ref(&l->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l)); return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu); } @@ -2868,18 +3100,6 @@ static int rhtab_map_alloc_check(union bpf_attr *attr) return htab_map_alloc_check(attr); } -static void rhtab_mem_dtor(void *obj, void *ctx) -{ - struct htab_btf_record *hrec = ctx; - struct rhtab_elem *elem = obj; - - if (IS_ERR_OR_NULL(hrec->record)) - return; - - bpf_obj_free_fields(hrec->record, - rhtab_elem_value(elem, hrec->key_size)); -} - static void rhtab_free_elem(void *ptr, void *arg) { struct bpf_rhtab *rhtab = arg; @@ -3105,7 +3325,9 @@ static int rhtab_map_check_btf(struct bpf_map *map, const struct btf *btf, if (btf_type_is_void(key_type)) return -EINVAL; - return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor); + return bpf_ma_set_dtor(map, &rhtab->ma, htab_mem_dtor, + offsetof(struct rhtab_elem, data) + + round_up(map->key_size, 8)); } static void rhtab_map_free_internal_structs(struct bpf_map *map) @@ -3338,8 +3560,10 @@ static int __rhtab_map_lookup_and_delete_batch(struct bpf_map *map, } if (do_delete) { + migrate_disable(); for (i = 0; i < total; i++) rhtab_delete_elem(rhtab, del_elems[i], NULL, 0); + migrate_enable(); } rcu_read_unlock(); diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index b3cc5c8fc875..648abe1ba63a 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -679,9 +679,44 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { .arg3_type = ARG_ANYTHING, }; +static int __bpf_copy_from_user_mm(void *dst, u32 size, + const void __user *user_ptr, + struct mm_struct *mm) +{ + int ret; + + ret = access_remote_vm(mm, (unsigned long)user_ptr, dst, size, 0); + if (ret == size) + return 0; + + memset(dst, 0, size); + /* Return -EFAULT for partial read */ + return ret < 0 ? ret : -EFAULT; +} + +static int __bpf_copy_from_user_mm_str(void *dst, u32 size, + const void __user *user_ptr, + struct mm_struct *mm, u64 flags) +{ + int ret; + + ret = copy_remote_mm_str(mm, (unsigned long)user_ptr, dst, size, 0); + if (ret < 0) { + if (flags & BPF_F_PAD_ZEROS) + memset(dst, 0, size); + return ret; + } + + if (flags & BPF_F_PAD_ZEROS) + memset(dst + ret, 0, size - ret); + + return ret + 1; +} + BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size, const void __user *, user_ptr, struct task_struct *, tsk, u64, flags) { + struct mm_struct *mm; int ret; /* flags is not used yet */ @@ -691,13 +726,16 @@ BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size, if (unlikely(!size)) return 0; - ret = access_process_vm(tsk, (unsigned long)user_ptr, dst, size, 0); - if (ret == size) - return 0; + mm = get_task_mm(tsk); + if (!mm) { + memset(dst, 0, size); + return -EFAULT; + } - memset(dst, 0, size); - /* Return -EFAULT for partial read */ - return ret < 0 ? ret : -EFAULT; + ret = __bpf_copy_from_user_mm(dst, size, user_ptr, mm); + mmput(mm); + + return ret; } const struct bpf_func_proto bpf_copy_from_user_task_proto = { @@ -1092,7 +1130,7 @@ const struct bpf_func_proto bpf_snprintf_proto = { .func = bpf_snprintf, .gpl_only = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, + .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_UNINIT | MEM_WRITE, .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_CONST_STR, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, @@ -1224,18 +1262,21 @@ static void bpf_wq_work(struct work_struct *work) BTF_TYPE_EMIT(struct bpf_wq); + /* keeps the program that callback_fn belongs to */ + rcu_read_lock_trace(); + callback_fn = READ_ONCE(cb->callback_fn); if (!callback_fn) - return; + goto out; key = map_key_from_value(map, value, &idx); - rcu_read_lock_trace(); - migrate_disable(); + migrate_disable(); callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0); migrate_enable(); +out: rcu_read_unlock_trace(); } @@ -1510,6 +1551,7 @@ static const struct bpf_func_proto bpf_timer_set_callback_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_TIMER, .arg2_type = ARG_PTR_TO_FUNC, + .arg3_type = ARG_PTR_TO_PROG_AUX, }; static bool defer_timer_wq_op(void) @@ -3659,6 +3701,92 @@ __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __user } /** + * bpf_copy_from_user_mm() - Copy data from an address space + * @dst: Destination address, in kernel space + * @dst__sz: Number of bytes to copy + * @unsafe_ptr__ign: Source address in the address space + * @mm: Address space to copy from + * @flags: Reserved for future use; must be zero + * + * Copies data from the user address space associated with @mm. The destination + * is zeroed if an attempted copy cannot be completed in full. Unsupported + * flags return -EINVAL without modifying @dst. + * + * Return: 0 on success, -EINVAL if @flags is non-zero, or -EFAULT if @mm is no + * longer active or the copy fails or is partial. + */ +__bpf_kfunc int bpf_copy_from_user_mm(void *dst, u32 dst__sz, + const void __user *unsafe_ptr__ign, + struct mm_struct *mm, u64 flags) +{ + int ret; + + if (unlikely(flags)) + return -EINVAL; + + if (unlikely(!dst__sz)) + return 0; + + if (unlikely(!mmget_not_zero(mm))) { + memset(dst, 0, dst__sz); + return -EFAULT; + } + + ret = __bpf_copy_from_user_mm(dst, dst__sz, unsafe_ptr__ign, mm); + mmput(mm); + + return ret; +} + +/** + * bpf_copy_from_user_mm_str() - Copy a string from an address space + * @dst: Destination address, in kernel space. This buffer must be + * at least @dst__sz bytes long + * @dst__sz: Maximum number of bytes to copy, including the trailing NUL + * @unsafe_ptr__ign: Source address in the address space + * @mm: Address space to copy from + * @flags: The only supported flag is BPF_F_PAD_ZEROS + * + * Copies a NUL-terminated string from the user address space associated with + * @mm. If the string is too long, @dst is still NUL-terminated unless @dst__sz + * is zero. + * + * If the flags are valid and BPF_F_PAD_ZEROS is set, the unused portion of + * @dst is cleared on success and all of @dst is cleared on a copy failure. + * Unsupported flags return -EINVAL without modifying @dst. + * + * Return: The number of copied bytes including the NUL terminator on success, + * -EINVAL if @flags is invalid, or -EFAULT if @mm is no longer active or the + * copy fails. + */ +__bpf_kfunc int bpf_copy_from_user_mm_str(void *dst, u32 dst__sz, + const void __user *unsafe_ptr__ign, + struct mm_struct *mm, u64 flags) +{ + int ret; + + if (unlikely(flags & ~BPF_F_PAD_ZEROS)) + return -EINVAL; + + if (unlikely(dst__sz == 0)) + return 0; + + if (unlikely(!mmget_not_zero(mm))) { + if (flags & BPF_F_PAD_ZEROS) + memset(dst, 0, dst__sz); + else + *(char *)dst = '\0'; + return -EFAULT; + } + + ret = __bpf_copy_from_user_mm_str(dst, dst__sz, unsafe_ptr__ign, + mm, flags); + mmput(mm); + + return ret; +} + +/** * bpf_copy_from_user_task_str() - Copy a string from an task's address space * @dst: Destination address, in kernel space. This buffer must be * at least @dst__sz bytes long. @@ -3681,6 +3809,7 @@ __bpf_kfunc int bpf_copy_from_user_task_str(void *dst, u32 dst__sz, const void __user *unsafe_ptr__ign, struct task_struct *tsk, u64 flags) { + struct mm_struct *mm; int ret; if (unlikely(flags & ~BPF_F_PAD_ZEROS)) @@ -3689,17 +3818,20 @@ __bpf_kfunc int bpf_copy_from_user_task_str(void *dst, u32 dst__sz, if (unlikely(dst__sz == 0)) return 0; - ret = copy_remote_vm_str(tsk, (unsigned long)unsafe_ptr__ign, dst, dst__sz, 0); - if (ret < 0) { + mm = get_task_mm(tsk); + if (!mm) { if (flags & BPF_F_PAD_ZEROS) memset(dst, 0, dst__sz); - return ret; + else + *(char *)dst = '\0'; + return -EFAULT; } - if (flags & BPF_F_PAD_ZEROS) - memset(dst + ret, 0, dst__sz - ret); + ret = __bpf_copy_from_user_mm_str(dst, dst__sz, unsafe_ptr__ign, + mm, flags); + mmput(mm); - return ret + 1; + return ret; } /* Keep unsigned long in prototype so that kfunc is usable when emitted to @@ -4437,15 +4569,21 @@ static void bpf_task_work_ctx_put(struct bpf_task_work_ctx *ctx) } } -static void bpf_task_work_cancel(struct bpf_task_work_ctx *ctx) +static void bpf_task_work_cancel(struct bpf_task_work_ctx *ctx, + struct task_struct *task) { /* * Scheduled task_work callback holds ctx ref, so if we successfully * cancelled, we put that ref on callback's behalf. If we couldn't * cancel, callback will inevitably run or has already completed * running, and it would have taken care of its ctx ref itself. + * + * The caller passes the task explicitly: the scheduling handler must + * not read ctx->task after the work is published (the callback may + * have reset it), while in the asynchronous cancellation path + * ctx->task is stable, see bpf_task_work_cancel_scheduled(). */ - if (task_work_cancel(ctx->task, &ctx->work)) + if (task_work_cancel(task, &ctx->work)) bpf_task_work_ctx_put(ctx); } @@ -4486,6 +4624,7 @@ static void bpf_task_work_callback(struct callback_head *cb) static void bpf_task_work_irq(struct irq_work *irq_work) { struct bpf_task_work_ctx *ctx = container_of(irq_work, struct bpf_task_work_ctx, irq_work); + struct task_struct *task; enum bpf_task_work_state state; int err; @@ -4496,7 +4635,16 @@ static void bpf_task_work_irq(struct irq_work *irq_work) return; } - err = task_work_add(ctx->task, &ctx->work, ctx->mode); + /* + * Capture the task pointer before the work is published: once the + * callback runs, it may reset ctx->task. The capture happens while + * this round's task reference is held, and every path that can + * release it runs after this handler entered its rcu read-side + * section. bpf_task_release() is put_task_struct_rcu_user(), so the + * task_struct itself cannot be freed before this section ends. + */ + task = ctx->task; + err = task_work_add(task, &ctx->work, ctx->mode); if (err) { bpf_task_work_ctx_reset(ctx); /* @@ -4510,14 +4658,14 @@ static void bpf_task_work_irq(struct irq_work *irq_work) /* * It's technically possible for just scheduled task_work callback to - * complete running by now, going SCHEDULING -> RUNNING and then - * dropping its ctx refcount. Instead of capturing an extra ref just - * to protect below ctx->state access, we rely on rcu_read_lock - * above to prevent kfree_rcu from freeing ctx before we return. + * complete running by now, going SCHEDULING -> RUNNING, resetting + * ctx->task and dropping its ctx refcount. The rcu read-side section + * above keeps both the ctx memory and the captured task pointer + * valid until this handler returns. */ state = cmpxchg(&ctx->state, BPF_TW_SCHEDULING, BPF_TW_SCHEDULED); if (state == BPF_TW_FREED) - bpf_task_work_cancel(ctx); /* clean up if we switched into FREED state */ + bpf_task_work_cancel(ctx, task); /* clean up if we switched into FREED state */ } static struct bpf_task_work_ctx *bpf_task_work_fetch_ctx(struct bpf_task_work *tw, @@ -4537,6 +4685,7 @@ static struct bpf_task_work_ctx *bpf_task_work_fetch_ctx(struct bpf_task_work *t memset(ctx, 0, sizeof(*ctx)); refcount_set(&ctx->refcnt, 1); /* map's own ref */ ctx->state = BPF_TW_STANDBY; + init_irq_work(&ctx->irq_work, bpf_task_work_irq); old_ctx = cmpxchg(&twk->ctx, NULL, ctx); if (old_ctx) { @@ -4579,6 +4728,18 @@ static struct bpf_task_work_ctx *bpf_task_work_acquire_ctx(struct bpf_task_work bpf_task_work_ctx_put(ctx); return ERR_PTR(-EBUSY); } + /* + * STANDBY can be published before the previous scheduling irq_work + * returns. Claim PENDING before checking BUSY to order the check after + * that publication. No irq_work has been queued for this attempt yet, + * so reject reuse and roll back to STANDBY while BUSY is set. Preserve + * FREED if deletion wins the rollback race. + */ + if (unlikely(irq_work_is_busy(&ctx->irq_work))) { + (void)cmpxchg(&ctx->state, BPF_TW_PENDING, BPF_TW_STANDBY); + bpf_task_work_ctx_put(ctx); + return ERR_PTR(-EBUSY); + } /* * If no process or bpffs is holding a reference to the map, no new callbacks should be @@ -4628,7 +4789,6 @@ static int bpf_task_work_schedule(struct task_struct *task, struct bpf_task_work ctx->map = map; ctx->map_val = (void *)tw - map->record->task_work_off; init_task_work(&ctx->work, bpf_task_work_callback); - init_irq_work(&ctx->irq_work, bpf_task_work_irq); irq_work_queue(&ctx->irq_work); return 0; @@ -4676,6 +4836,106 @@ __bpf_kfunc int bpf_task_work_schedule_resume(struct task_struct *task, struct b return bpf_task_work_schedule(task, tw, map__const_map, callback, aux, TWA_RESUME); } +typedef int (*bpf_rcu_callback_t)(struct bpf_map *map, void *key, void *value); + +/* Actual type for struct bpf_rcu_head */ +struct bpf_rcu_head_kern { + struct rcu_head rcu; + bpf_callback_t callback_fn; + struct bpf_map *map; + struct bpf_prog *prog; + u32 armed; +} __aligned(8); + +static void bpf_rcu_run_callback(struct rcu_head *rcu) +{ + struct bpf_rcu_head_kern *rh = container_of(rcu, struct bpf_rcu_head_kern, rcu); + bpf_callback_t callback_fn = rh->callback_fn; + struct bpf_prog *prog = rh->prog; + struct bpf_map *map = rh->map; + void *value, *key; + u32 idx; + + value = (void *)rh - map->record->rcu_head_off; + key = map_key_from_value(map, value, &idx); + + /* Pairs with the arming cmpxchg(): rh may be re-armed as soon as this store lands. */ + smp_store_release(&rh->armed, 0); + + rcu_read_lock_dont_migrate(); + callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0); + rcu_read_unlock_migrate(); + + bpf_prog_put(prog); +} + +static int __bpf_call_rcu(struct bpf_rcu_head *rh, struct bpf_map *map, void *callback, + struct bpf_prog_aux *aux, bool trace) +{ + struct bpf_rcu_head_kern *rhk = (void *)rh; + struct bpf_prog *prog; + + BUILD_BUG_ON(sizeof(struct bpf_rcu_head_kern) > sizeof(struct bpf_rcu_head)); + BUILD_BUG_ON(__alignof__(struct bpf_rcu_head_kern) != __alignof__(struct bpf_rcu_head)); + BTF_TYPE_EMIT(struct bpf_rcu_head); + + /* A queued callback cannot be cancelled, so a self-rearming one would pin prog and map. */ + if (!atomic64_read(&map->usercnt)) + return -EPERM; + + if (cmpxchg(&rhk->armed, 0, 1)) + return -EBUSY; + + prog = bpf_prog_inc_not_zero(aux->prog); + if (IS_ERR(prog)) { + WRITE_ONCE(rhk->armed, 0); + return -EBADF; + } + + rhk->callback_fn = (bpf_callback_t)callback; + rhk->map = map; + rhk->prog = prog; + if (trace) + call_rcu_tasks_trace(&rhk->rcu, bpf_rcu_run_callback); + else + call_rcu(&rhk->rcu, bpf_rcu_run_callback); + return 0; +} + +/** + * bpf_call_rcu - Invoke a BPF callback after an RCU grace period + * @rh: struct bpf_rcu_head in a BPF map value + * @map__const_map: bpf_map that embeds struct bpf_rcu_head in the values + * @callback: BPF subprogram, invoked as callback(map, key, value) for the value holding @rh + * @aux: bpf_prog_aux of the caller, implicitly set by the verifier + * + * Return: 0, -EBUSY if @rh is already queued, -EPERM if @map is held by neither a process + * nor bpffs, or -EBADF if the calling program is going away. + */ +__bpf_kfunc int bpf_call_rcu(struct bpf_rcu_head *rh, void *map__const_map, + bpf_rcu_callback_t callback, struct bpf_prog_aux *aux) +{ + return __bpf_call_rcu(rh, map__const_map, callback, aux, false); +} + +/** + * bpf_call_rcu_tasks_trace - Invoke a BPF callback after an RCU tasks trace grace period + * @rh: struct bpf_rcu_head in a BPF map value + * @map__const_map: bpf_map that embeds struct bpf_rcu_head in the values + * @callback: BPF subprogram, invoked as callback(map, key, value) for the value holding @rh + * @aux: bpf_prog_aux of the caller, implicitly set by the verifier + * + * Waits for sleepable BPF programs too. The callback itself is not sleepable either way. + * + * Return: 0, -EBUSY if @rh is already queued, -EPERM if @map is held by neither a process + * nor bpffs, or -EBADF if the calling program is going away. + */ +__bpf_kfunc int bpf_call_rcu_tasks_trace(struct bpf_rcu_head *rh, void *map__const_map, + bpf_rcu_callback_t callback, struct bpf_prog_aux *aux) +{ + return __bpf_call_rcu(rh, map__const_map, callback, aux, true); +} + static int make_file_dynptr(struct file *file, u32 flags, bool may_sleep, struct bpf_dynptr_kern *ptr) { @@ -4777,7 +5037,13 @@ static void bpf_task_work_cancel_scheduled(struct irq_work *irq_work) { struct bpf_task_work_ctx *ctx = container_of(irq_work, struct bpf_task_work_ctx, irq_work); - bpf_task_work_cancel(ctx); /* this might put task_work callback's ref */ + /* + * Deletion observed SCHEDULED and won against the callback's + * transition to RUNNING. The callback takes its FREED exit without + * resetting ctx->task, and the transferred map reference prevents + * destruction until cancellation completes. + */ + bpf_task_work_cancel(ctx, ctx->task); /* this might put task_work callback's ref */ bpf_task_work_ctx_put(ctx); /* and here we put map's own ref that was transferred to us */ } @@ -4883,7 +5149,7 @@ BTF_ID(func, bpf_cgroup_release_dtor) BTF_KFUNCS_START(common_btf_ids) BTF_ID_FLAGS(func, bpf_cast_to_kern_ctx, KF_FASTCALL) -BTF_ID_FLAGS(func, bpf_rdonly_cast, KF_FASTCALL) +BTF_ID_FLAGS(func, bpf_rdonly_cast, KF_FASTCALL | KF_PERFMON) BTF_ID_FLAGS(func, bpf_rcu_read_lock) BTF_ID_FLAGS(func, bpf_rcu_read_unlock) BTF_ID_FLAGS(func, bpf_dynptr_slice, KF_RET_NULL) @@ -4920,26 +5186,28 @@ BTF_ID_FLAGS(func, bpf_wq_set_callback, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_wq_start) BTF_ID_FLAGS(func, bpf_preempt_disable) BTF_ID_FLAGS(func, bpf_preempt_enable) -BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW) +BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW | KF_PERFMON) BTF_ID_FLAGS(func, bpf_iter_bits_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_bits_destroy, KF_ITER_DESTROY) -BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE) -BTF_ID_FLAGS(func, bpf_copy_from_user_task_str, KF_SLEEPABLE) -BTF_ID_FLAGS(func, bpf_get_kmem_cache) +BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_mm, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_mm_str, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_task_str, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_get_kmem_cache, KF_PERFMON) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_local_irq_save) BTF_ID_FLAGS(func, bpf_local_irq_restore) #ifdef CONFIG_BPF_EVENTS -BTF_ID_FLAGS(func, bpf_probe_read_user_dynptr) -BTF_ID_FLAGS(func, bpf_probe_read_kernel_dynptr) -BTF_ID_FLAGS(func, bpf_probe_read_user_str_dynptr) -BTF_ID_FLAGS(func, bpf_probe_read_kernel_str_dynptr) -BTF_ID_FLAGS(func, bpf_copy_from_user_dynptr, KF_SLEEPABLE) -BTF_ID_FLAGS(func, bpf_copy_from_user_str_dynptr, KF_SLEEPABLE) -BTF_ID_FLAGS(func, bpf_copy_from_user_task_dynptr, KF_SLEEPABLE) -BTF_ID_FLAGS(func, bpf_copy_from_user_task_str_dynptr, KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_probe_read_user_dynptr, KF_PERFMON) +BTF_ID_FLAGS(func, bpf_probe_read_kernel_dynptr, KF_PERFMON) +BTF_ID_FLAGS(func, bpf_probe_read_user_str_dynptr, KF_PERFMON) +BTF_ID_FLAGS(func, bpf_probe_read_kernel_str_dynptr, KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_dynptr, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_str_dynptr, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_task_dynptr, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_task_str_dynptr, KF_SLEEPABLE | KF_PERFMON) #endif #ifdef CONFIG_DMA_SHARED_BUFFER BTF_ID_FLAGS(func, bpf_iter_dmabuf_new, KF_ITER_NEW | KF_SLEEPABLE) @@ -4947,28 +5215,30 @@ BTF_ID_FLAGS(func, bpf_iter_dmabuf_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPAB BTF_ID_FLAGS(func, bpf_iter_dmabuf_destroy, KF_ITER_DESTROY | KF_SLEEPABLE) #endif BTF_ID_FLAGS(func, __bpf_trap) -BTF_ID_FLAGS(func, bpf_strcmp); -BTF_ID_FLAGS(func, bpf_strcasecmp); -BTF_ID_FLAGS(func, bpf_strncasecmp); -BTF_ID_FLAGS(func, bpf_strchr); -BTF_ID_FLAGS(func, bpf_strchrnul); -BTF_ID_FLAGS(func, bpf_strnchr); -BTF_ID_FLAGS(func, bpf_strrchr); -BTF_ID_FLAGS(func, bpf_strlen); -BTF_ID_FLAGS(func, bpf_strnlen); -BTF_ID_FLAGS(func, bpf_strspn); -BTF_ID_FLAGS(func, bpf_strcspn); -BTF_ID_FLAGS(func, bpf_strstr); -BTF_ID_FLAGS(func, bpf_strcasestr); -BTF_ID_FLAGS(func, bpf_strnstr); -BTF_ID_FLAGS(func, bpf_strncasestr); +BTF_ID_FLAGS(func, bpf_strcmp, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strcasecmp, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strncasecmp, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strchr, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strchrnul, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strnchr, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strrchr, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strlen, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strnlen, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strspn, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strcspn, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strstr, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strcasestr, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strnstr, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strncasestr, KF_PERFMON); #if defined(CONFIG_BPF_LSM) && defined(CONFIG_CGROUPS) BTF_ID_FLAGS(func, bpf_cgroup_read_xattr, KF_RCU) #endif -BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) -BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE | KF_PERFMON) BTF_ID_FLAGS(func, bpf_task_work_schedule_signal, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_task_work_schedule_resume, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, bpf_call_rcu, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, bpf_call_rcu_tasks_trace, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_dynptr_from_file) BTF_ID_FLAGS(func, bpf_dynptr_file_discard, KF_RELEASE) BTF_ID_FLAGS(func, bpf_timer_cancel_async) diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 7837968c0842..c6f328e4752e 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -176,7 +176,7 @@ static void bpf_dentry_finalize(struct dentry *dentry, struct inode *inode, inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); } -static struct dentry *bpf_mkdir(struct mnt_idmap *idmap, struct inode *dir, +static struct dentry *bpf_mkdir(const struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, umode_t mode) { struct inode *inode; @@ -424,7 +424,7 @@ bpf_lookup(struct inode *dir, struct dentry *dentry, unsigned flags) return simple_lookup(dir, dentry, flags); } -static int bpf_symlink(struct mnt_idmap *idmap, struct inode *dir, +static int bpf_symlink(const struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, const char *target) { struct inode *inode; @@ -874,7 +874,7 @@ enum { }; static int bpf_fs_xattr_set(const struct xattr_handler *handler, - struct mnt_idmap *idmap, struct dentry *unused, + const struct mnt_idmap *idmap, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) { diff --git a/kernel/bpf/keys.c b/kernel/bpf/keys.c new file mode 100644 index 000000000000..bffc356839ac --- /dev/null +++ b/kernel/bpf/keys.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Isovalent */ + +#include <linux/bpf.h> +#include <linux/cred.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/key.h> +#include <linux/moduleparam.h> +#include <linux/slab.h> + +#undef MODULE_PARAM_PREFIX +#define MODULE_PARAM_PREFIX "bpf." + +static struct key *bpf_keyring; + +static bool bpf_keyring_unsealed __ro_after_init; +module_param_named(keyring_unsealed, bpf_keyring_unsealed, bool, 0444); +MODULE_PARM_DESC(keyring_unsealed, "Leave the bpf keyring unsealed"); + +bool bpf_keyring_enforced(void) +{ + return bpf_keyring_unsealed; +} + +struct bpf_key *bpf_lookup_keyring(void) +{ + struct bpf_key *bkey; + + if (!bpf_keyring) + return NULL; + if (!READ_ONCE(bpf_keyring->keys.nr_leaves_on_tree) || + !READ_ONCE(bpf_keyring->restrict_link)) + return NULL; + + bkey = kmalloc_obj(*bkey); + if (!bkey) + return NULL; + + bkey->key = bpf_keyring; + bkey->has_ref = false; + return bkey; +} + +static int __init bpf_keyring_init(void) +{ + struct key *keyring; + + keyring = keyring_alloc(".bpf", + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, + current_cred(), KEY_POS_SEARCH | + KEY_USR_VIEW | KEY_USR_READ | + KEY_USR_WRITE | KEY_USR_SEARCH | + KEY_USR_SETATTR, KEY_ALLOC_NOT_IN_QUOTA, + NULL, NULL); + if (IS_ERR(keyring)) { + pr_err("bpf: cannot allocate bpf keyring: %ld\n", + PTR_ERR(keyring)); + return 0; + } + if (!bpf_keyring_unsealed && + keyring_restrict(make_key_ref(keyring, true), NULL, NULL)) { + pr_err("bpf: cannot seal bpf keyring\n"); + key_revoke(keyring); + key_put(keyring); + return 0; + } + + bpf_keyring = keyring; + key_register_bpf_keyring(keyring); + return 0; +} +late_initcall(bpf_keyring_init); diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 301fc60bddc4..e657e6d95408 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -10,10 +10,40 @@ #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) -struct per_frame_masks { - spis_t may_read; /* stack slots that may be read by this instruction */ - spis_t must_write; /* stack slots written by this instruction */ - spis_t live_before; /* stack slots that may be read by this insn and its successors */ +/* + * Stack liveness is tracked with a 4-byte (half register) granularity. + * Half-slot 0 covers [fp-4, fp), half-slot 1 covers [fp-8, fp-4), and so on, + * hence FRAME_HALF_SPIS - 1 is the deepest half-slot a frame can have. + */ +#define FRAME_HALF_SPIS (MAX_BPF_STACK_JIT / BPF_HALF_REG_SIZE) +#define FRAME_MAX_WORDS BITS_TO_LONGS(FRAME_HALF_SPIS) + +/* Masks tracked for each instruction of a frame */ +enum { + FM_MAY_READ, /* stack slots that may be read by this instruction */ + FM_MUST_WRITE, /* stack slots written by this instruction */ + FM_LIVE_BEFORE, /* stack slots that may be read by this insn and its successors */ + FM_MAY_WRITE, /* stack slots that may be written by this instruction */ + FM_MASK_CNT, +}; + +/* + * Per instruction stack masks for one frame of a function instance. + * + * Most frames use only a fraction of the stack budget, so instead of masks + * wide enough for every half-slot of the largest possible frame, all masks + * of one array share the width @words, and the marking functions widen the + * array as deeper half-slots are recorded. Mask @kind of the instruction at + * relative index @i is at &bits[(i * FM_MASK_CNT + kind) * words]. A + * half-slot at or past @words * BITS_PER_LONG is never read by this frame, + * hence never live. An instruction that may read the whole frame, such as a + * call passing a frame pointer to another subprog, widens the array to the + * program's stack budget, the deepest an accepted program can reach, so + * that the read cannot lose half-slots to a later widening. + */ +struct frame_masks { + u32 words; + unsigned long bits[]; }; /* @@ -29,7 +59,7 @@ struct func_instance { u32 subprog_start; /* cached env->subprog_info[subprog].start */ u32 insn_cnt; /* cached number of insns in the function */ /* Per frame, per instruction masks, frames allocated lazily. */ - struct per_frame_masks *frames[MAX_CALL_FRAMES]; + struct frame_masks *frames[MAX_CALL_FRAMES]; bool must_write_initialized; }; @@ -151,53 +181,145 @@ static int relative_idx(struct func_instance *instance, u32 insn_idx) return insn_idx - instance->subprog_start; } -static struct per_frame_masks *get_frame_masks(struct func_instance *instance, - u32 frame, u32 insn_idx) +static u32 frame_mask_bits(struct frame_masks *fm) { - if (!instance->frames[frame]) - return NULL; + return fm->words * BITS_PER_LONG; +} + +static size_t frame_mask_words(struct func_instance *instance, u32 words) +{ + return (size_t)instance->insn_cnt * FM_MASK_CNT * words; +} - return &instance->frames[frame][relative_idx(instance, insn_idx)]; +/* Mask @kind of the instruction at relative index @rel */ +static unsigned long *rel_mask(struct frame_masks *fm, u32 rel, u32 kind) +{ + return fm->bits + ((size_t)rel * FM_MASK_CNT + kind) * fm->words; } -static struct per_frame_masks *alloc_frame_masks(struct func_instance *instance, - u32 frame, u32 insn_idx) +/* + * Make sure @frame has a mask array at least @words wide, allocating it or + * copying the existing masks into the wider stride as needed. + * @words must be in range [1, FRAME_MAX_WORDS]. + */ +static struct frame_masks *widen_frame_masks(struct func_instance *instance, + u32 frame, u32 words) { - struct per_frame_masks *arr; + struct frame_masks *old = instance->frames[frame], *new; + u32 i, kind; - if (!instance->frames[frame]) { - arr = kvzalloc_objs(*arr, instance->insn_cnt, - GFP_KERNEL_ACCOUNT); - instance->frames[frame] = arr; - if (!arr) - return ERR_PTR(-ENOMEM); + if (old && old->words >= words) + return old; + new = kvzalloc_flex(*new, bits, frame_mask_words(instance, words), GFP_KERNEL_ACCOUNT); + if (!new) + return NULL; + new->words = words; + if (old) { + for (i = 0; i < instance->insn_cnt; i++) + for (kind = 0; kind < FM_MASK_CNT; kind++) + memcpy(rel_mask(new, i, kind), rel_mask(old, i, kind), + old->words * sizeof(*old->bits)); + kvfree(old); } - return get_frame_masks(instance, frame, insn_idx); + instance->frames[frame] = new; + return new; } -/* Accumulate may_read masks for @frame at @insn_idx */ -static int mark_stack_read(struct func_instance *instance, u32 frame, u32 insn_idx, spis_t mask) +/* + * Set the inclusive half-slot range [lo, hi] in mask @kind of @frame at @insn_idx. + * An empty range, including one with a negative @hi as computed for a write + * that does not fully cover any half-slot, marks nothing. + */ +static int mark_stack_range(struct func_instance *instance, u32 frame, u32 insn_idx, + u32 kind, s32 lo, s32 hi) { - struct per_frame_masks *masks; + struct frame_masks *fm; - masks = alloc_frame_masks(instance, frame, insn_idx); - if (IS_ERR(masks)) - return PTR_ERR(masks); - masks->may_read = spis_or(masks->may_read, mask); + /* + * An access past the frame bottom is rejected by the main verifier + * pass later, liveness only has to avoid running off the masks. + */ + hi = min_t(s32, hi, FRAME_HALF_SPIS - 1); + if (lo > hi) + return 0; + fm = widen_frame_masks(instance, frame, BITS_TO_LONGS(hi + 1)); + if (!fm) + return -ENOMEM; + bitmap_set(rel_mask(fm, relative_idx(instance, insn_idx), kind), lo, hi - lo + 1); return 0; } -static int mark_stack_write(struct func_instance *instance, u32 frame, u32 insn_idx, spis_t mask) +/* Accumulate may_read for half-slots [lo, hi] of @frame at @insn_idx */ +static int mark_stack_read(struct func_instance *instance, u32 frame, u32 insn_idx, + s32 lo, s32 hi) { - struct per_frame_masks *masks; + return mark_stack_range(instance, frame, insn_idx, FM_MAY_READ, lo, hi); +} - masks = alloc_frame_masks(instance, frame, insn_idx); - if (IS_ERR(masks)) - return PTR_ERR(masks); - masks->must_write = spis_or(masks->must_write, mask); +/* Accumulate must_write for half-slots [lo, hi] of @frame at @insn_idx */ +static int mark_stack_write(struct func_instance *instance, u32 frame, u32 insn_idx, + s32 lo, s32 hi) +{ + return mark_stack_range(instance, frame, insn_idx, FM_MUST_WRITE, lo, hi); +} + +static int mark_stack_may_write(struct func_instance *instance, u32 frame, u32 insn_idx, + s32 lo, s32 hi) +{ + return mark_stack_range(instance, frame, insn_idx, FM_MAY_WRITE, lo, hi); +} + +/* + * Mark every half-slot of @frame as possibly read by @insn_idx. This widens + * the masks to the program's stack budget: a full read recorded at a narrower + * width would leave the bits added by a later widening clear and lose part of + * it. No verifier state holds a slot past the budget, so no liveness query + * reaches beyond it; an access past the budget in unreachable code may still + * widen a frame's masks further, which is harmless. + */ +static int mark_stack_read_all(struct bpf_verifier_env *env, struct func_instance *instance, + u32 frame, u32 insn_idx) +{ + return mark_stack_read(instance, frame, insn_idx, 0, + env->stack_limit / BPF_HALF_REG_SIZE - 1); +} + +static int mark_stack_may_write_all(struct bpf_verifier_env *env, struct func_instance *instance, + u32 frame, u32 insn_idx) +{ + return mark_stack_may_write(instance, frame, insn_idx, 0, + env->stack_limit / BPF_HALF_REG_SIZE - 1); +} + +/* Accumulate @src, a mask @src_words wide, into @kind mask of @frame at @insn_idx */ +static int mark_stack_mask(struct func_instance *instance, u32 frame, u32 insn_idx, u32 kind, + const unsigned long *src, u32 src_words) +{ + u32 nbits = src_words * BITS_PER_LONG; + struct frame_masks *fm; + unsigned long *dst; + u32 last, w; + + last = find_last_bit(src, nbits); + if (last == nbits) + return 0; + fm = widen_frame_masks(instance, frame, BITS_TO_LONGS(last + 1)); + if (!fm) + return -ENOMEM; + dst = rel_mask(fm, relative_idx(instance, insn_idx), kind); + /* @src has no bits set past @last, hence none past @fm->words either */ + src_words = min(src_words, fm->words); + for (w = 0; w < src_words; w++) + dst[w] |= src[w]; return 0; } +static int mark_stack_read_mask(struct func_instance *instance, u32 frame, u32 insn_idx, + const unsigned long *src, u32 src_words) +{ + return mark_stack_mask(instance, frame, insn_idx, FM_MAY_READ, src, src_words); +} + int bpf_jmp_offset(struct bpf_insn *insn) { u8 code = insn->code; @@ -248,14 +370,18 @@ bpf_insn_successors(struct bpf_verifier_env *env, u32 idx) struct bpf_iarray *succ, *jt; int insn_sz; - jt = env->insn_aux_data[idx].jt; - if (unlikely(jt)) - return jt; - /* pre-allocated array of size up to 2; reset cnt, as it may have been used already */ succ = env->succ; succ->cnt = 0; + /* All gotox of a subprogram share its jump table, see compute_subprog_jts() */ + if (unlikely(insn_is_gotox(insn))) { + jt = bpf_find_containing_subprog(env, idx)->jt; + if (verifier_bug_if(!jt, env, "no jump table for gotox insn %u", idx)) + return succ; + return jt; + } + opcode_info = &opcode_info_tbl[BPF_CLASS(insn->code) | BPF_OP(insn->code)]; insn_sz = bpf_is_ldimm64(insn) ? 2 : 1; if (opcode_info->can_fallthrough) @@ -264,6 +390,20 @@ bpf_insn_successors(struct bpf_verifier_env *env, u32 idx) if (opcode_info->can_jump) succ->items[succ->cnt++] = idx + bpf_jmp_offset(insn) + 1; + /* + * tail_call upon success and ld_{abs,ind} upon load failure return + * from the subprogram, which is a hidden edge to its exit. A subprogram + * without an exit insn has no exit to model. + */ + if (unlikely((bpf_helper_call(insn) && insn->imm == BPF_FUNC_tail_call) || + (BPF_CLASS(insn->code) == BPF_LD && + (BPF_MODE(insn->code) == BPF_ABS || BPF_MODE(insn->code) == BPF_IND)))) { + u32 exit_idx = bpf_find_containing_subprog(env, idx)->exit_idx; + + if (exit_idx != U32_MAX) + succ->items[succ->cnt++] = exit_idx; + } + return succ; } @@ -272,33 +412,42 @@ __diag_pop(); static inline bool update_insn(struct bpf_verifier_env *env, struct func_instance *instance, u32 frame, u32 insn_idx) { - spis_t new_before, new_after; - struct per_frame_masks *insn, *succ_insn; + unsigned long new_after[FRAME_MAX_WORDS] = {}; + unsigned long *may_read, *must_write, *live_before; + struct frame_masks *fm = instance->frames[frame]; + u32 rel = relative_idx(instance, insn_idx); struct bpf_iarray *succ; - u32 s; - bool changed; + bool changed = false; + u32 s, w; succ = bpf_insn_successors(env, insn_idx); if (succ->cnt == 0) return false; - changed = false; - insn = get_frame_masks(instance, frame, insn_idx); - new_before = SPIS_ZERO; - new_after = SPIS_ZERO; + /* All instructions of one frame array share the same mask width */ for (s = 0; s < succ->cnt; ++s) { - succ_insn = get_frame_masks(instance, frame, succ->items[s]); - new_after = spis_or(new_after, succ_insn->live_before); + unsigned long *succ_live; + + succ_live = rel_mask(fm, relative_idx(instance, succ->items[s]), FM_LIVE_BEFORE); + for (w = 0; w < fm->words; w++) + new_after[w] |= succ_live[w]; } + may_read = rel_mask(fm, rel, FM_MAY_READ); + must_write = rel_mask(fm, rel, FM_MUST_WRITE); + live_before = rel_mask(fm, rel, FM_LIVE_BEFORE); /* * New "live_before" is a union of all "live_before" of successors * minus slots written by instruction plus slots read by instruction. * new_before = (new_after & ~insn->must_write) | insn->may_read */ - new_before = spis_or(spis_and(new_after, spis_not(insn->must_write)), - insn->may_read); - changed |= !spis_equal(new_before, insn->live_before); - insn->live_before = new_before; + for (w = 0; w < fm->words; w++) { + unsigned long new_before = (new_after[w] & ~must_write[w]) | may_read[w]; + + if (new_before != live_before[w]) { + live_before[w] = new_before; + changed = true; + } + } return changed; } @@ -329,10 +478,12 @@ static void update_instance(struct bpf_verifier_env *env, struct func_instance * static bool is_live_before(struct func_instance *instance, u32 insn_idx, u32 frameno, u32 half_spi) { - struct per_frame_masks *masks; + struct frame_masks *fm = instance->frames[frameno]; - masks = get_frame_masks(instance, frameno, insn_idx); - return masks && spis_test_bit(masks->live_before, half_spi); + /* No recorded access reaches past the masks, so nothing there is live */ + if (!fm || half_spi >= frame_mask_bits(fm)) + return false; + return test_bit(half_spi, rel_mask(fm, relative_idx(instance, insn_idx), FM_LIVE_BEFORE)); } int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st) @@ -356,12 +507,29 @@ int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_ return 0; } +/* + * Stack slots of outer frames that the callee may read are accounted as read + * by the @callsite insn itself, i.e. they are live before @callsite and may be + * dead after it, when: + * - @callsite calls a callback, which may be called several times; + * - @callsite is callx. Its callee is not known when stack liveness is + * computed, so there is no func instance for it at @callsite. While + * the verifier is in the callee lookup_instance() finds its standalone + * instance that knows nothing about outer frames. What the callee may read + * there is recorded by record_call_access(). + */ +static bool callee_stack_access_at_callsite(struct bpf_verifier_env *env, u32 callsite) +{ + return bpf_calls_callback(env, callsite) || + bpf_is_callx(&env->prog->insnsi[callsite]); +} + bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 half_spi) { /* * Slot is alive if it is read before q->insn_idx in current func instance, * or if for some outer func instance: - * - alive before callsite if callsite calls callback, otherwise + * - alive before callsite if callsite calls callback or is callx, otherwise * - alive after callsite */ struct live_stack_query *q = &env->liveness->live_stack_query; @@ -394,7 +562,7 @@ bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 half_sp /* Get callsite from verifier state, not from instance callchain */ callsite = q->callsites[i]; - alive = bpf_calls_callback(env, callsite) + alive = callee_stack_access_at_callsite(env, callsite) ? is_live_before(instance, callsite, rel, half_spi) : is_live_before(instance, callsite + 1, rel, half_spi); if (alive) @@ -430,17 +598,19 @@ static int spi_off(int spi) * When only one half is set, print as "-4h","-8h",... * Runs of 3+ consecutive fully-set SPIs are collapsed: "fp0-8..-24" */ -static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first, spis_t spis) +static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first, + const unsigned long *spis, u32 words) { int buf_sz = sizeof(env->tmp_str_buf); + int spi_cnt = words * BITS_PER_LONG / 2; char *buf = env->tmp_str_buf; int spi, n, run_start; buf[0] = '\0'; - for (spi = 0; spi < STACK_SLOTS / 2 && buf_sz > 0; spi++) { - bool lo = spis_test_bit(spis, spi * 2); - bool hi = spis_test_bit(spis, spi * 2 + 1); + for (spi = 0; spi < spi_cnt && buf_sz > 0; spi++) { + bool lo = test_bit(spi * 2, spis); + bool hi = test_bit(spi * 2 + 1, spis); const char *space = first ? "" : " "; if (!lo && !hi) @@ -450,16 +620,16 @@ static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first, /* half-spi */ n = scnprintf(buf, buf_sz, "%sfp%d%d%s", space, frame, spi_off(spi) + (lo ? STACK_SLOT_SZ : 0), "h"); - } else if (spi + 2 < STACK_SLOTS / 2 && - spis_test_bit(spis, spi * 2 + 2) && - spis_test_bit(spis, spi * 2 + 3) && - spis_test_bit(spis, spi * 2 + 4) && - spis_test_bit(spis, spi * 2 + 5)) { + } else if (spi + 2 < spi_cnt && + test_bit(spi * 2 + 2, spis) && + test_bit(spi * 2 + 3, spis) && + test_bit(spi * 2 + 4, spis) && + test_bit(spi * 2 + 5, spis)) { /* 3+ consecutive full spis */ run_start = spi; - while (spi + 1 < STACK_SLOTS / 2 && - spis_test_bit(spis, (spi + 1) * 2) && - spis_test_bit(spis, (spi + 1) * 2 + 1)) + while (spi + 1 < spi_cnt && + test_bit((spi + 1) * 2, spis) && + test_bit((spi + 1) * 2 + 1, spis)) spi++; n = scnprintf(buf, buf_sz, "%sfp%d%d..%d", space, frame, spi_off(run_start), spi_off(spi)); @@ -474,15 +644,41 @@ static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first, return env->tmp_str_buf; } +/* Print mask @kind of the instruction at relative index @i for every frame, if any bit is set. */ +static bool print_mask(struct bpf_verifier_env *env, struct func_instance *instance, int i, + const char *name, u32 kind) +{ + struct frame_masks *fm; + bool printed = false; + unsigned long *mask; + int frame; + u64 pos; + + pos = env->log.end_pos; + verbose(env, "%s", name); + for (frame = instance->depth; frame >= 0; --frame) { + fm = instance->frames[frame]; + if (!fm) + continue; + mask = rel_mask(fm, i, kind); + if (bitmap_empty(mask, frame_mask_bits(fm))) + continue; + verbose(env, "%s", fmt_spis_mask(env, frame, !printed, mask, fm->words)); + printed = true; + } + if (!printed) + bpf_vlog_reset(&env->log, pos); + return printed; +} + static void print_instance(struct bpf_verifier_env *env, struct func_instance *instance) { int start = env->subprog_info[instance->subprog].start; struct bpf_insn *insns = env->prog->insnsi; - struct per_frame_masks *masks; int len = instance->insn_cnt; - int insn_idx, frame, i; - bool has_use, has_def; u64 pos, insn_pos; + int insn_idx, i; + bool printed; if (!(env->log.level & BPF_LOG_LEVEL2)) return; @@ -491,35 +687,17 @@ static void print_instance(struct bpf_verifier_env *env, struct func_instance *i verbose(env, "%s:\n", fmt_instance(env, instance)); for (i = 0; i < len; i++) { insn_idx = start + i; - has_use = false; - has_def = false; pos = env->log.end_pos; verbose(env, "%3d: ", insn_idx); bpf_verbose_insn(env, &insns[insn_idx]); insn_pos = env->log.end_pos; verbose(env, "%*c;", bpf_vlog_alignment(insn_pos - pos), ' '); - pos = env->log.end_pos; - verbose(env, " use: "); - for (frame = instance->depth; frame >= 0; --frame) { - masks = get_frame_masks(instance, frame, insn_idx); - if (!masks || spis_is_zero(masks->may_read)) - continue; - verbose(env, "%s", fmt_spis_mask(env, frame, !has_use, masks->may_read)); - has_use = true; - } - if (!has_use) - bpf_vlog_reset(&env->log, pos); - pos = env->log.end_pos; - verbose(env, " def: "); - for (frame = instance->depth; frame >= 0; --frame) { - masks = get_frame_masks(instance, frame, insn_idx); - if (!masks || spis_is_zero(masks->must_write)) - continue; - verbose(env, "%s", fmt_spis_mask(env, frame, !has_def, masks->must_write)); - has_def = true; - } - if (!has_def) - bpf_vlog_reset(&env->log, has_use ? pos : insn_pos); + printed = false; + printed |= print_mask(env, instance, i, " use: ", FM_MAY_READ); + printed |= print_mask(env, instance, i, " def: ", FM_MUST_WRITE); + printed |= print_mask(env, instance, i, " may_def: ", FM_MAY_WRITE); + if (!printed) + bpf_vlog_reset(&env->log, insn_pos); verbose(env, "\n"); if (bpf_is_ldimm64(&insns[insn_idx])) i++; @@ -540,6 +718,49 @@ static int cmp_instances(const void *pa, const void *pb) return 0; } +/* OR the 8-byte slots touched by a half-slot (4-byte) mask into @slots. */ +static void half_spis_to_slots(unsigned long *slots, const unsigned long *mask, u32 nbits) +{ + u32 slot; + + for (slot = 0; slot < MAX_BPF_STACK_SLOTS && slot * 2 + 1 < nbits; slot++) + if (test_bit(slot * 2, mask) || test_bit(slot * 2 + 1, mask)) + __set_bit(slot, slots); +} + +/* + * Precompute, for each instruction, the OR of may_write and live_before masks + * over its top frame across all func_instances reaching it, stash them in the + * insn_aux_data. + */ +static void compute_may_write_masks(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_liveness *liveness = env->liveness; + struct func_instance *instance; + struct frame_masks *fm; + u32 nbits; + int bkt, i; + + hash_for_each(liveness->func_instances, bkt, instance, hl_node) { + fm = instance->frames[instance->depth]; + if (!fm) + continue; + nbits = frame_mask_bits(fm); + for (i = 0; i < instance->insn_cnt; i++) { + half_spis_to_slots(aux[instance->subprog_start + i].may_write_mask, + rel_mask(fm, i, FM_MAY_WRITE), nbits); + half_spis_to_slots(aux[instance->subprog_start + i].live_stack_before, + rel_mask(fm, i, FM_LIVE_BEFORE), nbits); + } + } +} + +const unsigned long *bpf_may_write_mask(struct bpf_verifier_env *env, u32 insn_idx) +{ + return env->insn_aux_data[insn_idx].may_write_mask; +} + /* print use/def slots for all instances ordered by callsite first, then by depth */ static int print_instances(struct bpf_verifier_env *env) { @@ -584,9 +805,9 @@ static int print_instances(struct bpf_verifier_env *env) * - same frame + different offset -> offset-imprecise * - different frames -> fully-imprecise (bitmask OR) * - * At memory access sites (LDX/STX/ST), offset-imprecise marks only - * the known frame's access mask as SPIS_ALL, while fully-imprecise - * iterates bits in the bitmask and routes each frame to its target. + * At memory access sites (LDX/STX/ST), offset-imprecise marks the known + * frame as fully read, while fully-imprecise iterates bits in the bitmask + * and routes each frame to its target. */ #define MAX_ARG_OFFSETS 4 @@ -605,8 +826,15 @@ enum arg_track_state { ARG_IMPRECISE = -3, /* lost identity; .mask is arg bitmask */ }; -/* Track callee stack slots fp-8 through fp-512 (64 slots of 8 bytes each) */ -#define MAX_ARG_SPILL_SLOTS 64 +/* + * The tracker follows as many spill slots as a subprog addresses directly + * through R10, see subprog_spill_slots(). A frame's slots at a call site are + * kept in a snapshot for the callees that receive pointers into that frame. + */ +struct spill_snapshot { + u32 slots; + struct arg_track at[]; +}; /* * Combined register + stack arg tracking: R0-R10 at indices 0-10, @@ -881,13 +1109,60 @@ static void arg_padd(struct arg_track *at, s64 delta) } /* - * Convert a byte offset from FP to a callee stack slot index. + * Slots the spill tracker may follow for a subprog of @len instructions + * without its per-instruction tables costing more than they could for the + * largest program while every frame stayed within MAX_BPF_STACK: as many + * entries as 64 slots need for BPF_COMPLEXITY_LIMIT_INSNS instructions. + */ +static u32 spill_slots_affordable(int len) +{ + u32 base = MAX_BPF_STACK / BPF_REG_SIZE; + + return max_t(u32, base, base * BPF_COMPLEXITY_LIMIT_INSNS / len); +} + +/* + * Number of 8-byte spill slots to track for the instructions in [@start, @end): + * the 64 slots of a MAX_BPF_STACK frame, which the tracker has always + * followed, or the deepest 8-byte stack access made directly through R10 when + * that reaches further, up to the program's stack budget. Spills and fills are + * compiled as direct R10 accesses, so the count covers them whatever the frame + * size; a fill through a derived pointer that reaches past it is treated as + * imprecise, as one past the frame always was. + */ +static u32 subprog_spill_slots(struct bpf_verifier_env *env, int start, int end) +{ + struct bpf_insn *insns = env->prog->insnsi; + int deepest = 0; + + for (int i = start; i < end; i++) { + struct bpf_insn *insn = &insns[i]; + u8 class = BPF_CLASS(insn->code); + u8 base; + + if (BPF_SIZE(insn->code) != BPF_DW) + continue; + if (class == BPF_LDX) + base = insn->src_reg; + else if (class == BPF_STX || class == BPF_ST) + base = insn->dst_reg; + else + continue; + if (base == BPF_REG_FP && insn->off < 0) + deepest = max(deepest, -insn->off); + } + return clamp_t(u32, DIV_ROUND_UP(deepest, 8), MAX_BPF_STACK / BPF_REG_SIZE, + env->stack_limit / BPF_REG_SIZE); +} + +/* + * Convert a byte offset from FP to one of the @nslots tracked stack slots. * Returns -1 if out of range or not 8-byte aligned. * Slot 0 = fp-8, slot 1 = fp-16, ..., slot 7 = fp-64, .... */ -static int fp_off_to_slot(s16 off) +static int fp_off_to_slot(s16 off, u32 nslots) { - if (off >= 0 || off < -(int)(MAX_ARG_SPILL_SLOTS * 8)) + if (off >= 0 || off < -(int)(nslots * 8)) return -1; if (off % 8) return -1; @@ -896,7 +1171,7 @@ static int fp_off_to_slot(s16 off) static struct arg_track fill_from_stack(struct bpf_insn *insn, struct arg_track *at_out, int reg, - struct arg_track *at_stack_out, + struct arg_track *at_stack_out, u32 nslots, int depth) { struct arg_track imp = { @@ -907,7 +1182,7 @@ static struct arg_track fill_from_stack(struct bpf_insn *insn, int cnt, i; if (reg == BPF_REG_FP) { - int slot = fp_off_to_slot(insn->off); + int slot = fp_off_to_slot(insn->off, nslots); return slot >= 0 ? at_stack_out[slot] : imp; } @@ -920,7 +1195,7 @@ static struct arg_track fill_from_stack(struct bpf_insn *insn, if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) return imp; - slot = fp_off_to_slot(fp_off); + slot = fp_off_to_slot(fp_off, nslots); if (slot < 0) return imp; result = __arg_track_join(result, at_stack_out[slot]); @@ -935,7 +1210,7 @@ static struct arg_track fill_from_stack(struct bpf_insn *insn, * When exact offset is unknown conservatively add reg values to all slots in at_stack_out. */ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, - int reg, struct arg_track *at_stack_out, + int reg, struct arg_track *at_stack_out, u32 nslots, struct arg_track *val, u32 sz) { struct arg_track none = { .frame = ARG_NONE }; @@ -943,7 +1218,7 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, int cnt, i; if (reg == BPF_REG_FP) { - int slot = fp_off_to_slot(insn->off); + int slot = fp_off_to_slot(insn->off, nslots); if (slot >= 0) at_stack_out[slot] = new_val; @@ -951,7 +1226,7 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, } cnt = at_out[reg].off_cnt; if (cnt == 0) { - for (int slot = 0; slot < MAX_ARG_SPILL_SLOTS; slot++) + for (int slot = 0; slot < nslots; slot++) at_stack_out[slot] = __arg_track_join(at_stack_out[slot], new_val); return; } @@ -961,7 +1236,7 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) continue; - slot = fp_off_to_slot(fp_off); + slot = fp_off_to_slot(fp_off, nslots); if (slot < 0) continue; if (cnt == 1) @@ -975,16 +1250,17 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, * Clear all tracked callee stack slots overlapping the byte range * [off, off+sz-1] where off is a negative FP-relative offset. */ -static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u32 sz, int cnt) +static void clear_overlapping_stack_slots(struct arg_track *at_stack, u32 nslots, s16 off, + u32 sz, int cnt) { struct arg_track none = { .frame = ARG_NONE }; if (cnt == 0) { - for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) + for (int i = 0; i < nslots; i++) at_stack[i] = __arg_track_join(at_stack[i], none); return; } - for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) { + for (int i = 0; i < nslots; i++) { int slot_start = -((i + 1) * 8); int slot_end = slot_start + 8; @@ -1002,33 +1278,33 @@ static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u */ static void clear_stack_for_all_offs(struct bpf_insn *insn, struct arg_track *at_out, int reg, - struct arg_track *at_stack_out, u32 sz) + struct arg_track *at_stack_out, u32 nslots, u32 sz) { int cnt, i; if (reg == BPF_REG_FP) { - clear_overlapping_stack_slots(at_stack_out, insn->off, sz, 1); + clear_overlapping_stack_slots(at_stack_out, nslots, insn->off, sz, 1); return; } cnt = at_out[reg].off_cnt; if (cnt == 0) { - clear_overlapping_stack_slots(at_stack_out, 0, sz, cnt); + clear_overlapping_stack_slots(at_stack_out, nslots, 0, sz, cnt); return; } for (i = 0; i < cnt; i++) { s16 fp_off; if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) { - clear_overlapping_stack_slots(at_stack_out, 0, sz, 0); + clear_overlapping_stack_slots(at_stack_out, nslots, 0, sz, 0); break; } - clear_overlapping_stack_slots(at_stack_out, fp_off, sz, cnt); + clear_overlapping_stack_slots(at_stack_out, nslots, fp_off, sz, cnt); } } static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, int idx, struct arg_track *at_in, struct arg_track *at_stack_in, - struct arg_track *at_out, struct arg_track *at_stack_out) + struct arg_track *at_out, struct arg_track *at_stack_out, u32 nslots) { bool printed = false; int i; @@ -1060,7 +1336,7 @@ static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, i verbose(env, "\tsa%d: ", i); verbose_arg_track(env, &at_in[ai]); verbose(env, " -> "); verbose_arg_track(env, &at_out[ai]); } - for (i = 0; i < MAX_ARG_SPILL_SLOTS; i++) { + for (i = 0; i < nslots; i++) { if (arg_track_eq(&at_stack_out[i], &at_stack_in[i])) continue; if (!printed) { @@ -1082,13 +1358,36 @@ static bool can_be_local_fp(int depth, int regno, struct arg_track *at) } /* + * Model 8-byte reads through @reg, including load-acquire + * and the old value fetched by atomic operations. + */ +static struct arg_track arg_track_deref(struct bpf_verifier_env *env, struct bpf_insn *insn, + struct arg_track *at_out, int reg, + struct arg_track *at_stack_out, u32 nslots, + int depth, u32 *callsites) +{ + struct arg_track *ptr = &at_out[reg]; + + if (can_be_local_fp(depth, reg, ptr)) + return fill_from_stack(insn, at_out, reg, at_stack_out, nslots, depth); + if (ptr->frame >= 0 && ptr->frame < depth) { + struct spill_snapshot *parent = env->callsite_at_stack[callsites[ptr->frame]]; + + return fill_from_stack(insn, at_out, reg, parent->at, parent->slots, ptr->frame); + } + if (ptr->frame == ARG_IMPRECISE && !(ptr->mask & BIT(depth)) && ptr->mask) + return *ptr; + return (struct arg_track){ .frame = ARG_NONE }; +} + +/* * Pure dataflow transfer function for arg_track state. * Updates at_out[] based on how the instruction modifies registers. * Tracks spill/fill, but not other memory accesses. */ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, int insn_idx, - struct arg_track *at_out, struct arg_track *at_stack_out, + struct arg_track *at_out, struct arg_track *at_stack_out, u32 nslots, const struct arg_track *at_stack_arg_entry, struct func_instance *instance, u32 *callsites) @@ -1162,55 +1461,42 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, for (r = BPF_REG_0; r <= BPF_REG_5; r++) at_out[r] = none; } else if (class == BPF_LDX) { - u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); - bool src_is_local_fp = can_be_local_fp(depth, insn->src_reg, src); - - /* - * Reload from callee stack: if src is current-frame FP-derived - * and the load is an 8-byte BPF_MEM, try to restore the spill - * identity. For imprecise sources fill_from_stack() returns - * ARG_IMPRECISE (off_cnt == 0). - */ - if (src_is_local_fp && BPF_MODE(insn->code) == BPF_MEM && sz == 8) { - *dst = fill_from_stack(insn, at_out, insn->src_reg, at_stack_out, depth); - } else if (src->frame >= 0 && src->frame < depth && - BPF_MODE(insn->code) == BPF_MEM && sz == 8) { - struct arg_track *parent_stack = - env->callsite_at_stack[callsites[src->frame]]; - - *dst = fill_from_stack(insn, at_out, insn->src_reg, - parent_stack, src->frame); - } else if (src->frame == ARG_IMPRECISE && - !(src->mask & BIT(depth)) && src->mask && - BPF_MODE(insn->code) == BPF_MEM && sz == 8) { - /* - * Imprecise src with only parent-frame bits: - * conservative fallback. - */ - *dst = *src; - } else { + /* Only an 8-byte BPF_MEM load can be a fill, see arg_track_deref(). */ + if (BPF_MODE(insn->code) == BPF_MEM && BPF_SIZE(insn->code) == BPF_DW) + *dst = arg_track_deref(env, insn, at_out, insn->src_reg, + at_stack_out, nslots, depth, callsites); + else *dst = none; - } } else if (class == BPF_LD && BPF_MODE(insn->code) == BPF_IMM) { *dst = none; } else if (class == BPF_STX) { u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); - bool dst_is_local_fp; - - /* Track spills to current-frame FP-derived callee stack */ - dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst); - if (dst_is_local_fp && BPF_MODE(insn->code) == BPF_MEM) - spill_to_stack(insn, at_out, insn->dst_reg, - at_stack_out, src, sz); - - if (BPF_MODE(insn->code) == BPF_ATOMIC) { - if (dst_is_local_fp && insn->imm != BPF_LOAD_ACQ) - clear_stack_for_all_offs(insn, at_out, insn->dst_reg, - at_stack_out, sz); + bool dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst); + u8 mode = BPF_MODE(insn->code); + + if (mode == BPF_MEM || (mode == BPF_ATOMIC && insn->imm == BPF_STORE_REL)) { + if (dst_is_local_fp) + spill_to_stack(insn, at_out, insn->dst_reg, + at_stack_out, nslots, src, sz); + } else if (mode == BPF_ATOMIC && insn->imm == BPF_LOAD_ACQ) { + if (sz == 8) + *dst = arg_track_deref(env, insn, at_out, insn->src_reg, + at_stack_out, nslots, depth, callsites); + else + *dst = none; + } else if (mode == BPF_ATOMIC) { + /* BPF_XCHG, BPF_CMPXCHG or BPF_{ADD,AND,OR,XOR} | [BPF_FETCH] */ + struct arg_track old = none; r = bpf_atomic_load_reg(insn); + if (r >= 0 && sz == 8) + old = arg_track_deref(env, insn, at_out, insn->dst_reg, + at_stack_out, nslots, depth, callsites); + if (dst_is_local_fp) + clear_stack_for_all_offs(insn, at_out, insn->dst_reg, + at_stack_out, nslots, sz); if (r >= 0) - at_out[r] = none; + at_out[r] = old; } } else if (class == BPF_ST && BPF_MODE(insn->code) == BPF_MEM) { u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); @@ -1219,23 +1505,21 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, /* BPF_ST to FP-derived dst: clear overlapping stack slots */ if (dst_is_local_fp) clear_stack_for_all_offs(insn, at_out, insn->dst_reg, - at_stack_out, sz); + at_stack_out, nslots, sz); } } /* - * Record access_bytes from helper/kfunc or load/store insn. - * access_bytes > 0: stack read - * access_bytes < 0: stack write - * access_bytes == S64_MIN: unknown — conservative, mark [0..slot] as read - * access_bytes == 0: no access - * + * Record possible reads and writes for every touched half-slot. A definite + * write requires full coverage, a known size, and a single possible offset. */ -static int record_stack_access_off(struct func_instance *instance, s64 fp_off, - s64 access_bytes, u32 frame, u32 insn_idx) +static int record_stack_access_off(struct func_instance *instance, const struct arg_track *arg, + s64 off_idx, struct arg_access_info info, + u32 frame, u32 insn_idx) { + s64 fp_off = arg->off[off_idx]; s32 slot_hi, slot_lo; - spis_t mask; + int err; if (fp_off >= 0) /* @@ -1244,68 +1528,68 @@ static int record_stack_access_off(struct func_instance *instance, s64 fp_off, * by the main verifier pass later. */ return 0; - if (access_bytes == S64_MIN) { - /* helper/kfunc read unknown amount of bytes from fp_off until fp+0 */ - slot_hi = (-fp_off - 1) / STACK_SLOT_SZ; - mask = SPIS_ZERO; - spis_or_range(&mask, 0, slot_hi); - return mark_stack_read(instance, frame, insn_idx, mask); - } - if (access_bytes > 0) { - /* Mark any touched slot as use */ - slot_hi = (-fp_off - 1) / STACK_SLOT_SZ; - slot_lo = max_t(s32, (-fp_off - access_bytes) / STACK_SLOT_SZ, 0); - mask = SPIS_ZERO; - spis_or_range(&mask, slot_lo, slot_hi); - return mark_stack_read(instance, frame, insn_idx, mask); - } else if (access_bytes < 0) { + + /* + * Read and may_write marks include partially covered slots. + * An unknown access size may reach from fp_off to the frame top. + */ + slot_hi = (-fp_off - 1) / STACK_SLOT_SZ; + slot_lo = info.size == U32_MAX + ? 0 + : max_t(s32, (-fp_off - info.size) / STACK_SLOT_SZ, 0); + if (info.may_read) { + err = mark_stack_read(instance, frame, insn_idx, slot_lo, slot_hi); + if (err) + return err; + } + if (info.may_write) { + err = mark_stack_may_write(instance, frame, insn_idx, slot_lo, slot_hi); + if (err) + return err; + } + if (info.must_write && info.size != U32_MAX && arg->off_cnt == 1) { /* Mark only fully covered slots as def */ - access_bytes = -access_bytes; slot_hi = (-fp_off) / STACK_SLOT_SZ - 1; - slot_lo = max_t(s32, (-fp_off - access_bytes + STACK_SLOT_SZ - 1) / STACK_SLOT_SZ, 0); - if (slot_lo <= slot_hi) { - mask = SPIS_ZERO; - spis_or_range(&mask, slot_lo, slot_hi); - return mark_stack_write(instance, frame, insn_idx, mask); - } + slot_lo = max_t(s32, (-fp_off - info.size + STACK_SLOT_SZ - 1) / STACK_SLOT_SZ, 0); + return mark_stack_write(instance, frame, insn_idx, slot_lo, slot_hi); } return 0; } -/* - * 'arg' is FP-derived argument to helper/kfunc or load/store that - * reads (positive) or writes (negative) 'access_bytes' into 'use' or 'def'. - */ -static int record_stack_access(struct func_instance *instance, +/* Record access through a pointer with a known frame and possibly known offsets. */ +static int record_stack_access(struct bpf_verifier_env *env, + struct func_instance *instance, const struct arg_track *arg, - s64 access_bytes, u32 frame, u32 insn_idx) + struct arg_access_info info, u32 frame, u32 insn_idx) { int i, err; - if (access_bytes == 0) + if (!info.size) return 0; if (arg->off_cnt == 0) { - if (access_bytes > 0 || access_bytes == S64_MIN) - return mark_stack_read(instance, frame, insn_idx, SPIS_ALL); + if (info.may_read) { + err = mark_stack_read_all(env, instance, frame, insn_idx); + if (err) + return err; + } + if (info.may_write) { + err = mark_stack_may_write_all(env, instance, frame, insn_idx); + if (err) + return err; + } return 0; } - if (access_bytes != S64_MIN && access_bytes < 0 && arg->off_cnt != 1) - /* multi-offset write cannot set stack_def */ - return 0; - for (i = 0; i < arg->off_cnt; i++) { - err = record_stack_access_off(instance, arg->off[i], access_bytes, frame, insn_idx); + err = record_stack_access_off(instance, arg, i, info, frame, insn_idx); if (err) return err; } return 0; } -/* - * When a pointer is ARG_IMPRECISE, conservatively mark every frame in - * the bitmask as fully used. - */ -static int record_imprecise(struct func_instance *instance, u32 mask, u32 insn_idx) +/* Record possible effects on every candidate frame of an imprecise pointer. */ +static int record_imprecise(struct bpf_verifier_env *env, struct func_instance *instance, + struct arg_access_info info, u32 mask, u32 insn_idx) { int depth = instance->depth; int f, err; @@ -1314,9 +1598,16 @@ static int record_imprecise(struct func_instance *instance, u32 mask, u32 insn_i if (!(mask & 1)) continue; if (f <= depth) { - err = mark_stack_read(instance, f, insn_idx, SPIS_ALL); - if (err) - return err; + if (info.may_read) { + err = mark_stack_read_all(env, instance, f, insn_idx); + if (err) + return err; + } + if (info.may_write) { + err = mark_stack_may_write_all(env, instance, f, insn_idx); + if (err) + return err; + } } } return 0; @@ -1328,9 +1619,10 @@ static int record_load_store_access(struct bpf_verifier_env *env, struct arg_track *at, int insn_idx) { struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; + struct arg_access_info info = {}; int depth = instance->depth; - s32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); u8 class = BPF_CLASS(insn->code); + bool read = false, write = false; struct arg_track resolved, *ptr; int oi; @@ -1347,23 +1639,34 @@ static int record_load_store_access(struct bpf_verifier_env *env, switch (class) { case BPF_LDX: ptr = &at[insn->src_reg]; + read = true; break; case BPF_STX: if (BPF_MODE(insn->code) == BPF_ATOMIC) { - if (insn->imm == BPF_STORE_REL) - sz = -sz; - if (insn->imm == BPF_LOAD_ACQ) + switch (insn->imm) { + case BPF_LOAD_ACQ: ptr = &at[insn->src_reg]; - else + read = true; + break; + case BPF_STORE_REL: + ptr = &at[insn->dst_reg]; + write = true; + break; + default: + /* ADD/AND/OR/XOR(+FETCH), XCHG, CMPXCHG */ ptr = &at[insn->dst_reg]; + read = true; + write = true; + break; + } } else { ptr = &at[insn->dst_reg]; - sz = -sz; + write = true; } break; case BPF_ST: ptr = &at[insn->dst_reg]; - sz = -sz; + write = true; break; default: return 0; @@ -1382,10 +1685,14 @@ static int record_load_store_access(struct bpf_verifier_env *env, ptr = &resolved; } + info.may_read = read; + info.may_write = write; + info.must_write = write; + info.size = bpf_size_to_bytes(BPF_SIZE(insn->code)); if (ptr->frame >= 0 && ptr->frame <= depth) - return record_stack_access(instance, ptr, sz, ptr->frame, insn_idx); + return record_stack_access(env, instance, ptr, info, ptr->frame, insn_idx); if (ptr->frame == ARG_IMPRECISE) - return record_imprecise(instance, ptr->mask, insn_idx); + return record_imprecise(env, instance, info, ptr->mask, insn_idx); /* ARG_NONE: not derived from any frame pointer, skip */ return 0; } @@ -1396,37 +1703,42 @@ static int record_arg_access(struct bpf_verifier_env *env, struct arg_track *at, int arg_idx, int insn_idx) { + struct arg_access_info info; int depth = instance->depth; int frame = at->frame; int err = 0; - s64 bytes; if (!arg_is_fp(at)) return 0; if (bpf_helper_call(insn)) { - bytes = bpf_helper_stack_access_bytes(env, insn, arg_idx, insn_idx); + info = bpf_helper_stack_access_bytes(env, insn, arg_idx, insn_idx); } else if (bpf_pseudo_kfunc_call(insn)) { - bytes = bpf_kfunc_stack_access_bytes(env, insn, arg_idx, insn_idx); + info = bpf_kfunc_stack_access_bytes(env, insn, arg_idx, insn_idx); + } else if (bpf_pseudo_call(insn)) { + info = bpf_global_subprog_stack_access_bytes(env, insn, arg_idx, insn_idx); } else { for (int f = 0; f <= depth; f++) { - err = mark_stack_read(instance, f, insn_idx, SPIS_ALL); + err = mark_stack_read_all(env, instance, f, insn_idx); + if (err) + return err; + err = mark_stack_may_write_all(env, instance, f, insn_idx); if (err) return err; } return 0; } - if (bytes == 0) + if (!info.size) return 0; if (frame >= 0 && frame <= depth) - err = record_stack_access(instance, at, bytes, frame, insn_idx); + err = record_stack_access(env, instance, at, info, frame, insn_idx); else if (frame == ARG_IMPRECISE) - err = record_imprecise(instance, at->mask, insn_idx); + err = record_imprecise(env, instance, info, at->mask, insn_idx); return err; } -/* Record stack access for a given 'at' state of helper/kfunc 'insn' */ +/* Record stack access for a given 'at' state of helper/kfunc/global subprog 'insn' */ static int record_call_access(struct bpf_verifier_env *env, struct func_instance *instance, struct arg_track *at, @@ -1434,21 +1746,36 @@ static int record_call_access(struct bpf_verifier_env *env, { struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; struct bpf_call_summary cs; - int r, err, num_params = 5; - - if (bpf_pseudo_call(insn)) - return 0; + int r, err, callee, arg_slot_cnt = 5; - if (bpf_get_call_summary(env, insn, &cs)) - num_params = cs.num_params; + if (bpf_pseudo_call(insn)) { + /* + * analyze_subprog() handles static callees. For global callees, + * derive effects from argument types so may_write marks remain + * conservative under freplace. + */ + callee = bpf_find_subprog(env, insn_idx + insn->imm + 1); + if (callee < 0 || !bpf_subprog_is_global(env, callee)) + return 0; + arg_slot_cnt = MAX_BPF_FUNC_ARGS; + } else if (bpf_is_callx(insn)) + /* + * The callee is not known statically. Assume that all arg + * slots are passed and let record_arg_access() conservatively + * mark the stack of all frames as read if any of them is + * derived from a frame pointer. + */ + arg_slot_cnt = MAX_BPF_FUNC_REG_ARGS + MAX_STACK_ARG_SLOTS; + else if (bpf_get_call_summary(env, insn, &cs)) + arg_slot_cnt = cs.arg_slot_cnt; - for (r = BPF_REG_1; r < BPF_REG_1 + min(num_params, MAX_BPF_FUNC_REG_ARGS); r++) { + for (r = BPF_REG_1; r < BPF_REG_1 + min(arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); r++) { err = record_arg_access(env, instance, insn, &at[r], r - 1, insn_idx); if (err) return err; } - for (r = 0; r < MAX_STACK_ARG_SLOTS && r < num_params - MAX_BPF_FUNC_REG_ARGS; r++) { + for (r = 0; r < MAX_STACK_ARG_SLOTS && r < arg_slot_cnt - MAX_BPF_FUNC_REG_ARGS; r++) { err = record_arg_access(env, instance, insn, &at[MAX_BPF_REG + r], r + MAX_BPF_FUNC_REG_ARGS, insn_idx); if (err) @@ -1517,7 +1844,7 @@ struct subprog_at_info { static void print_subprog_arg_access(struct bpf_verifier_env *env, int subprog, struct subprog_at_info *info, - struct arg_track (*at_stack_in)[MAX_ARG_SPILL_SLOTS]) + struct arg_track *at_stack_in, u32 nslots) { struct bpf_insn *insns = env->prog->insnsi; int start = env->subprog_info[subprog].start; @@ -1533,7 +1860,7 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, bool has_extra = false; u8 cls = BPF_CLASS(insns[idx].code); bool is_ldx_stx_call = cls == BPF_LDX || cls == BPF_STX || - insns[idx].code == (BPF_JMP | BPF_CALL); + (cls == BPF_JMP && BPF_OP(insns[idx].code) == BPF_CALL); verbose(env, "%3d: ", idx); bpf_verbose_insn(env, &insns[idx]); @@ -1550,8 +1877,8 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, has_extra = true; } if (is_ldx_stx_call) { - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) - if (arg_is_fp(&at_stack_in[i][r])) + for (r = 0; r < nslots; r++) + if (arg_is_fp(&at_stack_in[(size_t)i * nslots + r])) has_extra = true; } @@ -1581,11 +1908,11 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, } if (is_ldx_stx_call) { - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) { - if (!arg_is_fp(&at_stack_in[i][r])) + for (r = 0; r < nslots; r++) { + if (!arg_is_fp(&at_stack_in[(size_t)i * nslots + r])) continue; verbose(env, " fp%+d=", -(r + 1) * 8); - verbose_arg_track(env, &at_stack_in[i][r]); + verbose_arg_track(env, &at_stack_in[(size_t)i * nslots + r]); } } @@ -1595,6 +1922,17 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, } } +static void record_stack_ptrs(struct bpf_verifier_env *env, int idx, struct arg_track *at_in) +{ + u16 r, mask = 0; + + for (r = 0; r < MAX_BPF_REG; r++) + if (arg_is_fp(&at_in[r])) + mask |= BIT(r); + + env->insn_aux_data[idx].stack_ptrs |= mask; +} + /* * Compute arg tracking dataflow for a single subprog. * Runs forward fixed-point with arg_track_xfer(), then records @@ -1618,9 +1956,10 @@ static int compute_subprog_args(struct bpf_verifier_env *env, int end = env->subprog_info[subprog + 1].start; int po_end = env->subprog_info[subprog + 1].postorder_start; int len = end - start; + u32 nslots = min(subprog_spill_slots(env, start, end), spill_slots_affordable(len)); struct arg_track (*at_in)[MAX_AT_TRACK_REGS] = NULL; struct arg_track at_out[MAX_AT_TRACK_REGS]; - struct arg_track (*at_stack_in)[MAX_ARG_SPILL_SLOTS] = NULL; + struct arg_track *at_stack_in = NULL; struct arg_track *at_stack_out = NULL; struct arg_track at_stack_arg_entry[MAX_STACK_ARG_SLOTS]; struct arg_track unvisited = { .frame = ARG_UNVISITED }; @@ -1632,19 +1971,19 @@ static int compute_subprog_args(struct bpf_verifier_env *env, if (!at_in) goto err_free; - at_stack_in = kvmalloc_objs(*at_stack_in, len, GFP_KERNEL_ACCOUNT); + at_stack_in = kvmalloc_objs(*at_stack_in, (size_t)len * nslots, GFP_KERNEL_ACCOUNT); if (!at_stack_in) goto err_free; - at_stack_out = kvmalloc_objs(*at_stack_out, MAX_ARG_SPILL_SLOTS, GFP_KERNEL_ACCOUNT); + at_stack_out = kvmalloc_objs(*at_stack_out, nslots, GFP_KERNEL_ACCOUNT); if (!at_stack_out) goto err_free; for (i = 0; i < len; i++) { for (r = 0; r < MAX_AT_TRACK_REGS; r++) at_in[i][r] = unvisited; - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) - at_stack_in[i][r] = unvisited; + for (r = 0; r < nslots; r++) + at_stack_in[(size_t)i * nslots + r] = unvisited; } for (r = 0; r < MAX_AT_TRACK_REGS; r++) @@ -1660,8 +1999,8 @@ static int compute_subprog_args(struct bpf_verifier_env *env, } /* Entry: all stack slots are ARG_NONE */ - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) - at_stack_in[0][r] = none; + for (r = 0; r < nslots; r++) + at_stack_in[r] = none; /* Entry: incoming stack args from caller, or ARG_NONE for main */ for (r = 0; r < MAX_STACK_ARG_SLOTS; r++) @@ -1683,11 +2022,12 @@ redo: continue; memcpy(at_out, at_in[i], sizeof(at_out)); - memcpy(at_stack_out, at_stack_in[i], MAX_ARG_SPILL_SLOTS * sizeof(*at_stack_out)); + memcpy(at_stack_out, &at_stack_in[(size_t)i * nslots], nslots * sizeof(*at_stack_out)); - arg_track_xfer(env, insn, idx, at_out, at_stack_out, + arg_track_xfer(env, insn, idx, at_out, at_stack_out, nslots, at_stack_arg_entry, instance, callsites); - arg_track_log(env, insn, idx, at_in[i], at_stack_in[i], at_out, at_stack_out); + arg_track_log(env, insn, idx, at_in[i], &at_stack_in[(size_t)i * nslots], at_out, + at_stack_out, nslots); /* Propagate to successors within this subprogram */ succ = bpf_insn_successors(env, idx); @@ -1704,9 +2044,10 @@ redo: changed |= arg_track_join(env, idx, target, r, &at_in[ti][r], at_out[r]); - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) + for (r = 0; r < nslots; r++) changed |= arg_track_join(env, idx, target, -r - 1, - &at_stack_in[ti][r], at_stack_out[r]); + &at_stack_in[(size_t)ti * nslots + r], + at_stack_out[r]); } } if (changed) @@ -1722,30 +2063,33 @@ redo: if (err) goto err_free; - if (insn->code == (BPF_JMP | BPF_CALL)) { + if (BPF_CLASS(insn->code) == BPF_JMP && BPF_OP(insn->code) == BPF_CALL) { err = record_call_access(env, instance, at_in[i], idx); if (err) goto err_free; } if (bpf_pseudo_call(insn) || bpf_calls_callback(env, idx)) { + struct spill_snapshot *snap; + kvfree(env->callsite_at_stack[idx]); - env->callsite_at_stack[idx] = - kvmalloc_objs(*env->callsite_at_stack[idx], - MAX_ARG_SPILL_SLOTS, GFP_KERNEL_ACCOUNT); - if (!env->callsite_at_stack[idx]) { + env->callsite_at_stack[idx] = kvmalloc_flex(*snap, at, nslots, GFP_KERNEL_ACCOUNT); + snap = env->callsite_at_stack[idx]; + if (!snap) { err = -ENOMEM; goto err_free; } - memcpy(env->callsite_at_stack[idx], - at_stack_in[i], sizeof(struct arg_track) * MAX_ARG_SPILL_SLOTS); + snap->slots = nslots; + memcpy(snap->at, &at_stack_in[(size_t)i * nslots], nslots * sizeof(*snap->at)); } + + record_stack_ptrs(env, idx, at_in[i]); } info->at_in = at_in; at_in = NULL; info->len = len; - print_subprog_arg_access(env, subprog, info, at_stack_in); + print_subprog_arg_access(env, subprog, info, at_stack_in, nslots); err = 0; err_free: @@ -1770,38 +2114,87 @@ static bool has_fp_args(struct arg_track *args) /* * Merge a freshly analyzed instance into the original. * may_read: union (any pass might read the slot). + * may_write: union (slots written on ANY pass). * must_write: intersection (only slots written on ALL passes are guaranteed). * live_before is recomputed by a subsequent update_instance() on @dst. + * + * The two instances may have settled on different mask widths for the same + * frame, so @dst is widened to cover @src first. A word only @dst has counts + * as zero on the @src side: it unions into may_read as a no-op and intersects + * must_write to empty. */ -static void merge_instances(struct func_instance *dst, struct func_instance *src) +static int merge_instances(struct func_instance *dst, struct func_instance *src) { - int f, i; + struct frame_masks *d, *s; + u32 f, i, w; for (f = 0; f <= dst->depth; f++) { - if (!src->frames[f]) { + s = src->frames[f]; + d = dst->frames[f]; + if (!s) { /* This pass didn't touch frame f — must_write intersects with empty. */ - if (dst->frames[f]) + if (d) for (i = 0; i < dst->insn_cnt; i++) - dst->frames[f][i].must_write = SPIS_ZERO; + bitmap_zero(rel_mask(d, i, FM_MUST_WRITE), + frame_mask_bits(d)); continue; } - if (!dst->frames[f]) { + if (!d) { /* Previous pass didn't touch frame f — take src, zero must_write. */ - dst->frames[f] = src->frames[f]; + dst->frames[f] = s; src->frames[f] = NULL; for (i = 0; i < dst->insn_cnt; i++) - dst->frames[f][i].must_write = SPIS_ZERO; + bitmap_zero(rel_mask(s, i, FM_MUST_WRITE), frame_mask_bits(s)); continue; } + d = widen_frame_masks(dst, f, s->words); + if (!d) + return -ENOMEM; for (i = 0; i < dst->insn_cnt; i++) { - dst->frames[f][i].may_read = - spis_or(dst->frames[f][i].may_read, - src->frames[f][i].may_read); - dst->frames[f][i].must_write = - spis_and(dst->frames[f][i].must_write, - src->frames[f][i].must_write); + unsigned long *dst_read = rel_mask(d, i, FM_MAY_READ); + unsigned long *dst_write = rel_mask(d, i, FM_MUST_WRITE); + unsigned long *dst_may_write = rel_mask(d, i, FM_MAY_WRITE); + unsigned long *src_read = rel_mask(s, i, FM_MAY_READ); + unsigned long *src_write = rel_mask(s, i, FM_MUST_WRITE); + unsigned long *src_may_write = rel_mask(s, i, FM_MAY_WRITE); + + for (w = 0; w < d->words; w++) { + dst_read[w] |= w < s->words ? src_read[w] : 0; + dst_write[w] &= w < s->words ? src_write[w] : 0; + dst_may_write[w] |= w < s->words ? src_may_write[w] : 0; + } } } + return 0; +} + +/* + * Fold a fully analyzed callee instance writes to upper frames as + * may_write marks at callsite in caller's frames. + */ +static int merge_may_write(struct func_instance *caller, struct func_instance *callee) +{ + DECLARE_BITMAP(acc, FRAME_HALF_SPIS); + u32 call_idx = callee->callsite; + struct frame_masks *fm; + u32 f, i, nbits; + int err; + + for (f = 0; f < callee->depth; f++) { + fm = callee->frames[f]; + if (!fm) + continue; + nbits = frame_mask_bits(fm); + bitmap_zero(acc, nbits); + for (i = 0; i < callee->insn_cnt; i++) + bitmap_or(acc, acc, rel_mask(fm, i, FM_MAY_WRITE), nbits); + if (bitmap_empty(acc, nbits)) + continue; + err = mark_stack_mask(caller, f, call_idx, FM_MAY_WRITE, acc, fm->words); + if (err) + return err; + } + return 0; } static struct func_instance *fresh_instance(struct func_instance *src) @@ -1898,7 +2291,12 @@ static int analyze_subprog(struct bpf_verifier_env *env, if (bpf_pseudo_call(insn)) { target = idx + insn->imm + 1; callee = bpf_find_subprog(env, target); - if (callee < 0) + /* + * Global subprograms can be freplaced, so conservatively derive + * their stack effects from their BTF signatures instead of + * analyzing them in the context of their callers. + */ + if (callee < 0 || bpf_subprog_is_global(env, callee)) continue; /* Build entry args: R1-R5 and stack args from at_in at call site */ @@ -1916,7 +2314,10 @@ static int analyze_subprog(struct bpf_verifier_env *env, if (info[subprog].at_in[j][caller_reg].frame == ARG_NONE) continue; for (int f = 0; f <= depth; f++) { - err = mark_stack_read(instance, f, idx, SPIS_ALL); + err = mark_stack_read_all(env, instance, f, idx); + if (err) + goto out_free; + err = mark_stack_may_write_all(env, instance, f, idx); if (err) goto out_free; } @@ -1955,23 +2356,35 @@ static int analyze_subprog(struct bpf_verifier_env *env, /* Pull callee's entry liveness back to caller's callsite */ { u32 callee_start = callee_instance->subprog_start; - struct per_frame_masks *entry; + struct frame_masks *callee_fm; for (int f = 0; f < callee_instance->depth; f++) { - entry = get_frame_masks(callee_instance, f, callee_start); - if (!entry) + callee_fm = callee_instance->frames[f]; + if (!callee_fm) continue; - err = mark_stack_read(instance, f, idx, entry->live_before); + err = mark_stack_read_mask(instance, f, idx, + rel_mask(callee_fm, + relative_idx(callee_instance, + callee_start), + FM_LIVE_BEFORE), + callee_fm->words); if (err) goto out_free; } } + + /* Summarize callee's writes to ancestor frames onto the callsite */ + err = merge_may_write(instance, callee_instance); + if (err) + goto out_free; } if (prev_instance) { - merge_instances(prev_instance, instance); + err = merge_instances(prev_instance, instance); free_instance(instance); instance = prev_instance; + if (err) + return err; } update_instance(env, instance); return 0; @@ -2026,6 +2439,8 @@ int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env) goto out; } + compute_may_write_masks(env); + if (env->log.level & BPF_LOG_LEVEL2) err = print_instances(env); @@ -2057,10 +2472,17 @@ static inline u32 mask_widen(u32 m) { return m | (m << 16); } static inline u16 mask_lo(u32 m) { return (u16)m; } static inline u16 mask_hi(u32 m) { return (u16)(m >> 16); } +/* Index of the subprogram called by pseudo call @insn */ +static int callee_subprog(struct bpf_verifier_env *env, const struct bpf_insn *insn) +{ + return bpf_find_subprog(env, insn - env->prog->insnsi + insn->imm + 1); +} + /* Compute info->{use,def} fields for the instruction */ static void compute_insn_live_regs(struct bpf_verifier_env *env, struct bpf_insn *insn, - struct insn_live_regs *info) + struct insn_live_regs *info, + bool ret_reg_pair) { struct bpf_call_summary cs; const u8 class = BPF_CLASS(insn->code); @@ -2072,6 +2494,8 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, const u32 src32 = mask_lo(src); const u32 dst32 = mask_lo(dst); const u32 r0 = reg64_mask(0); + const u32 r2 = reg64_mask(BPF_REG_2); + int subprog, reg_args; u32 def = 0; u32 use = U32_MAX; @@ -2191,15 +2615,28 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, break; case BPF_EXIT: def = 0; - use = r0; + use = ret_reg_pair ? (r0 | r2) : r0; break; case BPF_CALL: def = ALL_CALLER_SAVED_REGS; use = def & ~BIT(BPF_REG_0); - if (bpf_get_call_summary(env, insn, &cs)) - use = GENMASK(min_t(u8, cs.num_params, MAX_BPF_FUNC_REG_ARGS), 1); + if (bpf_get_call_summary(env, insn, &cs)) { + use = GENMASK(min_t(u8, cs.arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS), 1); + } else if (bpf_pseudo_call(insn)) { + /* a static callee's reads are added in bpf_compute_live_registers() */ + subprog = callee_subprog(env, insn); + if (bpf_subprog_is_global(env, subprog)) { + reg_args = min_t(u8, env->subprog_info[subprog].arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); + use = GENMASK(reg_args, 1); + } else { + use = 0; + } + } def = mask_widen(def); use = mask_widen(use); + /* callx reads the address of the callee from dst_reg */ + if (bpf_is_callx(insn)) + use |= dst; break; default: def = 0; @@ -2227,9 +2664,8 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) struct bpf_insn *insns = env->prog->insnsi; struct insn_live_regs *state; int insn_cnt = env->prog->len; - u64 pos, insn_pos; - int err = 0, i, j; - bool changed; + int err = 0, i, subprog, start, end; + bool changed, ret_reg_pair; /* Use the following algorithm: * - define the following: @@ -2256,8 +2692,14 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) goto out; } - for (i = 0; i < insn_cnt; ++i) - compute_insn_live_regs(env, &insns[i], &state[i]); + for (subprog = 0; subprog < env->subprog_cnt; subprog++) { + start = env->subprog_info[subprog].start; + end = env->subprog_info[subprog + 1].start; + ret_reg_pair = bpf_ret_reg_pair(env, subprog); + + for (i = start; i < end; ++i) + compute_insn_live_regs(env, &insns[i], &state[i], ret_reg_pair); + } /* Forward pass: resolve stack access through FP-derived pointers */ err = bpf_compute_subprog_arg_access(env); @@ -2273,11 +2715,21 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) struct bpf_iarray *succ; u32 new_out = 0; u32 new_in = 0; + u32 use; succ = bpf_insn_successors(env, insn_idx); for (int s = 0; s < succ->cnt; ++s) new_out |= state[succ->items[s]].in; - new_in = (new_out & ~live->def) | live->use; + use = live->use; + /* + * For a call to a static subprogram include the callee's + * entry 'in' as computed so far. + */ + if (bpf_pseudo_call(&insns[insn_idx]) && + !bpf_subprog_is_global(env, callee_subprog(env, &insns[insn_idx]))) + use |= state[insn_idx + insns[insn_idx].imm + 1].in & + mask_widen(GENMASK(BPF_REG_5, BPF_REG_1)); + new_in = (new_out & ~live->def) | use; if (new_out != live->out || new_in != live->in) { live->in = new_in; live->out = new_out; @@ -2301,31 +2753,6 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) insn_aux[i].zext_dst = def32 >= 0 && (mask_hi(out) & BIT(def32)); } - if (env->log.level & BPF_LOG_LEVEL2) { - verbose(env, "Live regs before insn:\n"); - for (i = 0; i < insn_cnt; ++i) { - if (env->insn_aux_data[i].scc) - verbose(env, "%3d ", env->insn_aux_data[i].scc); - else - verbose(env, " "); - verbose(env, "%3d: ", i); - for (j = BPF_REG_0; j < BPF_REG_10; ++j) - if (insn_aux[i].live_regs_before & BIT(j)) - verbose(env, "%d", j); - else - verbose(env, "."); - verbose(env, " "); - pos = env->log.end_pos; - bpf_verbose_insn(env, &insns[i]); - insn_pos = env->log.end_pos; - if (insn_aux[i].zext_dst) - verbose(env, "%*c; zext", bpf_vlog_alignment(insn_pos - pos), ' '); - verbose(env, "\n"); - if (bpf_is_ldimm64(&insns[i])) - i++; - } - } - out: kvfree(state); return err; diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index fb032dfdc0de..5432c0fc442b 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -527,7 +527,7 @@ static void verbose_unum(struct bpf_verifier_env *env, u64 num) verbose(env, "%#llx", num); } -static void verbose_snum(struct bpf_verifier_env *env, s64 num) +void bpf_verbose_snum(struct bpf_verifier_env *env, s64 num) { if (is_snum_decimal(num)) verbose(env, "%lld", num); @@ -609,7 +609,7 @@ static void print_scalar_ranges(struct bpf_verifier_env *env, } if (m1->name[0] == 's') - verbose_snum(env, m1->val); + bpf_verbose_snum(env, m1->val); else verbose_unum(env, m1->val); } @@ -632,7 +632,7 @@ static void print_reg_state(struct bpf_verifier_env *env, if (t == SCALAR_VALUE && reg->precise) verbose(env, "P"); if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) { - verbose_snum(env, reg->var_off.value); + bpf_verbose_snum(env, reg->var_off.value); return; } @@ -643,7 +643,7 @@ static void print_reg_state(struct bpf_verifier_env *env, if (state->frameno != reg->frameno) verbose(env, "[%d]", reg->frameno); if (tnum_is_const(reg->var_off)) { - verbose_snum(env, reg->var_off.value + reg->delta); + bpf_verbose_snum(env, reg->var_off.value + reg->delta); return; } } @@ -667,7 +667,7 @@ static void print_reg_state(struct bpf_verifier_env *env, } if (t != SCALAR_VALUE && reg->delta) { verbose_a("off="); - verbose_snum(env, reg->delta); + bpf_verbose_snum(env, reg->delta); } if (type_is_pkt_pointer(t)) { verbose_a("r="); @@ -683,7 +683,7 @@ static void print_reg_state(struct bpf_verifier_env *env, /* a pointer register with fixed offset */ if (reg->var_off.value) { verbose_a("imm="); - verbose_snum(env, reg->var_off.value); + bpf_verbose_snum(env, reg->var_off.value); } } else { print_scalar_ranges(env, reg, &sep); @@ -693,6 +693,8 @@ static void print_reg_state(struct bpf_verifier_env *env, tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose_a("var_off=%s", tn_buf); } + if (reg->base != 0 || reg->step != 1) + verbose_a("step=%d+%d", reg->base, reg->step); } verbose(env, ")"); } @@ -716,7 +718,8 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie verbose(env, "="); print_reg_state(env, state, reg); } - for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { + for (i = 0; i < bpf_stack_nr_slots(state); i++) { + struct bpf_stack_state *slot = bpf_stack_slot(state, i); char types_buf[BPF_REG_SIZE + 1]; const char *sep = ""; bool valid = false; @@ -727,7 +730,7 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie continue; for (j = 0; j < BPF_REG_SIZE; j++) { - slot_type = state->stack[i].slot_type[j]; + slot_type = slot->slot_type[j]; if (slot_type != STACK_INVALID && slot_type != STACK_POISON) valid = true; types_buf[j] = slot_type_char[slot_type]; @@ -736,12 +739,12 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie if (!valid) continue; - reg = &state->stack[i].spilled_ptr; - switch (state->stack[i].slot_type[BPF_REG_SIZE - 1]) { + reg = &slot->spilled_ptr; + switch (slot->slot_type[BPF_REG_SIZE - 1]) { case STACK_SPILL: /* print MISC/ZERO/INVALID slots above subreg spill */ for (j = 0; j < BPF_REG_SIZE; j++) - if (state->stack[i].slot_type[j] == STACK_SPILL) + if (slot->slot_type[j] == STACK_SPILL) break; types_buf[j] = '\0'; @@ -751,7 +754,7 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie case STACK_DYNPTR: /* skip to main dynptr slot */ i += BPF_DYNPTR_NR_SLOTS - 1; - reg = &state->stack[i].spilled_ptr; + reg = &bpf_stack_slot(state, i)->spilled_ptr; verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); verbose(env, "=dynptr_%s(", dynptr_type_str(reg->dynptr.type)); @@ -789,6 +792,11 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie verbose(env, " cb"); if (state->in_async_callback_fn) verbose(env, " async_cb"); + if (state->loop_stack_cnt) { + verbose(env, " loop_stack="); + for (i = 0; i < state->loop_stack_cnt; i++) + verbose(env, "%s%d", i ? "," : "", state->loop_stack[i].loop_id); + } verbose(env, "\n"); if (!print_all) mark_verifier_state_clean(env); diff --git a/kernel/bpf/loops.c b/kernel/bpf/loops.c new file mode 100644 index 000000000000..2568079c2945 --- /dev/null +++ b/kernel/bpf/loops.c @@ -0,0 +1,616 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <linux/slab.h> +#include <linux/sched/signal.h> +#include <linux/bpf_verifier.h> + +static bool is_cfg_jump(struct bpf_insn *insn) +{ + u8 class = BPF_CLASS(insn->code); + u8 opcode = BPF_OP(insn->code); + + switch (class) { + case BPF_JMP: + case BPF_JMP32: + return opcode != BPF_CALL; + default: + return false; + } +} + +static struct bpf_iarray **compute_predecessors(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_iarray *succ, *preds, **result; + struct bpf_prog *prog = env->prog; + u32 *num_preds, i, s, len = prog->len; + struct bpf_insn *insn; + size_t sz; + void *tmp; + + num_preds = kvcalloc(prog->len, sizeof(u32), GFP_KERNEL_ACCOUNT); + if (!num_preds) + return NULL; + + /* + * 'result' layout: + * - array of pointers (struct bpf_iarray *)[len] + * - struct bpf_iarray one after another + */ + sz = size_mul(len, sizeof(*result) + sizeof(**result)); + for (i = 0; i < len; i++) { + insn = env->prog->insnsi + i; + succ = bpf_insn_successors(env, i); + sz = size_add(sz, array_size(succ->cnt, sizeof(*succ->items))); + iarray_for_each(s, succ) { + num_preds[s]++; + } + if (bpf_is_ldimm64(insn)) + i++; + } + + result = kvzalloc(sz, GFP_KERNEL_ACCOUNT); + if (!result) { + kvfree(num_preds); + return NULL; + } + + tmp = (void *)&result[len]; + for (i = 0; i < len; i++) { + result[i] = tmp; + tmp += struct_size(result[i], items, num_preds[i]); + } + + for (i = 0; i < len; i++) { + insn = env->prog->insnsi + i; + succ = bpf_insn_successors(env, i); + iarray_for_each(s, succ) { + preds = result[s]; + preds->items[preds->cnt++] = i; + } + if (succ->cnt > 1 || + (succ->cnt == 1 && num_preds[succ->items[0]] > 1) || + is_cfg_jump(insn)) + aux[i].bb_end = true; + if (bpf_is_ldimm64(insn)) + i++; + } + + kvfree(num_preds); + return result; +} + +static int idoms_intersect(struct bpf_verifier_env *env, int a, int b) +{ + int *postorder_nums = env->cfg.postorder_nums; + int *idoms = env->idoms; + + while (a != b) { + while (postorder_nums[a] < postorder_nums[b]) { + a = idoms[a]; + } + while (postorder_nums[b] < postorder_nums[a]) { + b = idoms[b]; + } + } + return a; +} + +/* See "A Simple, Fast Dominance Algorithm" by Cooper et al. for details. */ +static int compute_subprog_idoms(struct bpf_verifier_env *env, struct bpf_iarray **preds, + int subprog_idx) +{ + struct bpf_subprog_info *subprog = &env->subprog_info[subprog_idx]; + int start = subprog->start; + int po_first = subprog->postorder_start; + int po_last = (subprog + 1)->postorder_start - 1; + int *idoms = env->idoms; + int po_num, pred; + u32 work = 0; + bool changed; + + idoms[start] = start; + changed = true; + do { + changed = false; + /* + * Iterate in reverse postorder, skipping the subprog entry: + * insn_postorder[po_last] is the subprogram entry. + */ + for (po_num = po_last - 1; po_num >= po_first; po_num--) { + int idx = env->cfg.insn_postorder[po_num]; + int new_idom = -1; + + iarray_for_each(pred, preds[idx]) { + if (++work % 1024 == 0) { + if (signal_pending(current)) + return -EAGAIN; + cond_resched(); + } + + if (idoms[pred] == -1) + continue; + if (new_idom == -1) + new_idom = pred; + else + new_idom = idoms_intersect(env, pred, new_idom); + } + if (new_idom != -1 && idoms[idx] != new_idom) { + idoms[idx] = new_idom; + changed = true; + } + } + } while (changed); + idoms[start] = -1; + return 0; +} + +int bpf_compute_idoms(struct bpf_verifier_env *env) +{ + struct bpf_iarray **preds; + u32 len = env->prog->len; + int *idoms, i, err = 0; + + preds = compute_predecessors(env); + if (!preds) + return -ENOMEM; + + idoms = kvcalloc(len, sizeof(*idoms), GFP_KERNEL_ACCOUNT); + if (!idoms) { + kvfree(preds); + return -ENOMEM; + } + + env->idoms = idoms; + for (i = 0; i < len; i++) + idoms[i] = -1; + + for (i = 0; i < env->subprog_cnt; i++) { + err = compute_subprog_idoms(env, preds, i); + if (err) + break; + } + + kvfree(preds); + return err; +} + +struct dfs_state { + u32 traversed:1; + u32 next_succ:31; +}; + +struct loops_dfs { + struct dfs_state *state; + int *dfs_pos; + int *stack; +}; + +static void mark_irreducible(struct bpf_verifier_env *env, int h) +{ + env->insn_aux_data[h].loop->irreducible = true; +} + +static void add_backedge(struct bpf_verifier_env *env, int from, int h) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *loop = aux[h].loop; + int cnt = loop->backedges_cnt; + + if (cnt == MAX_BACKEDGES) { + loop->backedges_overflow = true; + return; + } + loop->backedges[cnt].from = from; + loop->backedges[cnt].latches_cnt = 0; + loop->backedges_cnt++; +} + +static int add_exit(struct bpf_loop *loop, int from, int to) +{ + if (loop->exits_overflow) + return 0; + if (loop->exits_cnt == MAX_LOOP_EXITS) { + loop->exits_overflow = true; + return 0; + } + if (!loop->exits) { + loop->exits = kvcalloc(MAX_LOOP_EXITS, sizeof(*loop->exits), GFP_KERNEL_ACCOUNT); + if (!loop->exits) + return -ENOMEM; + } + loop->exits[loop->exits_cnt] = (struct bpf_loop_exit) { + .from = from, + .to = to, + }; + loop->exits_cnt++; + return 0; +} + +static int mark_as_header(struct bpf_verifier_env *env, int h) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + + if (!aux[h].loop) { + aux[h].loop = kvzalloc_obj(struct bpf_loop, GFP_KERNEL_ACCOUNT); + if (!aux[h].loop) + return -ENOMEM; + } + return 0; +} + +static int assign_header(struct bpf_verifier_env *env, struct loops_dfs *dfs, int n, int h) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + int *dfs_pos = dfs->dfs_pos; + int err, nh; + + err = mark_as_header(env, h); + if (err) + return err; + + /* Don't encode self-loops, otherwise can't reflect loops nesting structure. */ + if (n == h) + return 0; + + /* Make sure that loop headers up the chain are sorted by dfs_pos. */ + while (aux[n].loop_header != -1) { + nh = aux[n].loop_header; + if (nh == h) + return 0; + if (dfs_pos[nh] < dfs_pos[h]) { + aux[n].loop_header = h; + n = h; + h = nh; + } else { + n = nh; + } + } + aux[n].loop_header = h; + return 0; +} + +static bool is_cond_jmp_insn(struct bpf_insn *insn) +{ + u8 class = BPF_CLASS(insn->code); + u8 opcode = BPF_OP(insn->code); + + if (class != BPF_JMP && class != BPF_JMP32) + return false; + + switch (opcode) { + case BPF_JEQ: + case BPF_JGE: + case BPF_JGT: + case BPF_JLE: + case BPF_JLT: + case BPF_JNE: + case BPF_JSET: + case BPF_JSGE: + case BPF_JSGT: + case BPF_JSLE: + case BPF_JSLT: + return true; + default: + return false; + } +} + +int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + + return aux[idx].loop ? idx : aux[idx].loop_header; +} + +/* + * Collect conditional jumps within n's loop that dominate 'n' and have + * a successor outside the loop, nearest first. + */ +static void find_dominating_conditions(struct bpf_verifier_env *env, int n, + struct bpf_backedge *backedge) +{ + struct bpf_insn *insns = env->prog->insnsi; + bool t_exits_loop, f_exits_loop; + int n_loop, t_loop, f_loop; + int *idoms = env->idoms; + + n_loop = bpf_loop_at_index(env, n); + if (idoms_intersect(env, n, n_loop) != n_loop) + return; + while (n >= 0) { + if (is_cond_jmp_insn(&insns[n]) && bpf_loop_at_index(env, n) == n_loop) { + t_loop = bpf_loop_at_index(env, n + insns[n].off + 1); + f_loop = bpf_loop_at_index(env, n + 1); + t_exits_loop = t_loop != n_loop && !bpf_is_nested_loop(env, t_loop, n_loop); + f_exits_loop = f_loop != n_loop && !bpf_is_nested_loop(env, f_loop, n_loop); + if (t_exits_loop || f_exits_loop) { + if (backedge->latches_cnt == MAX_LATCHES) + return; + backedge->latches[backedge->latches_cnt++] = n; + } + } + if (n == n_loop) + break; + n = idoms[n]; + } +} + +/* + * As described in "A New Algorithm for Identifying Loops in Decompilation" by Wei et al, + * adapted to be non-recursive. + */ +static int compute_loops_in_subprog(struct bpf_verifier_env *env, struct loops_dfs *dfs, + int subprog_idx) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct dfs_state *state = dfs->state; + int start = env->subprog_info[subprog_idx].start; + int *dfs_pos = dfs->dfs_pos; + int *stack = dfs->stack; + int s, h, err, cur, stack_sz; + struct bpf_iarray *succ; + u32 i; + + stack[0] = start; + state[start].traversed = true; + state[start].next_succ = 0; + dfs_pos[start] = 1; + stack_sz = 1; + i = 0; + do { + /* + * The algorithm should be very fast in practice, + * guard against pathological inputs, just in case. + */ + if ((++i % 1024) == 0) { + if (signal_pending(current)) + return -EAGAIN; + cond_resched(); + } + + cur = stack[stack_sz - 1]; + succ = bpf_insn_successors(env, cur); + if (state[cur].next_succ == succ->cnt) { + dfs_pos[cur] = 0; + stack_sz--; + continue; + } + s = succ->items[state[cur].next_succ]; + if (!state[s].traversed) { + /* Case A: start -> ... -> cur -> s [unexplored] */ + state[s].traversed = true; + state[s].next_succ = 0; + stack[stack_sz] = s; + dfs_pos[s] = stack_sz + 1; + stack_sz++; + continue; + } + /* 's' is fully explored at this point */ + if (dfs_pos[s]) { + /* + * start -> ... -> s -> cur --. + * ^ | + * '----------' + * Case B: 's' is in the current DFS path. + */ + err = assign_header(env, dfs, cur, s); + if (err) + return err; + add_backedge(env, cur, s); + } else if (aux[s].loop_header == -1) { + /* + * start -> ... -> ... -> s -> ... -> end + * | ^ + * '---> cur ---' + * Case C: 's' is explored, not in the current DFS path, + * and not a part of any loop. + */ + } else if (dfs_pos[aux[s].loop_header]) { + /* + * .----------------------. + * v | + * start -> ... -> h -> ... -> ... -> s --' + * | ^ + * '---> cur ---' + * Case D: 's' is explored, not in current DFS path, + * but its innermost loop header is. + */ + err = assign_header(env, dfs, cur, aux[s].loop_header); + if (err) + return err; + } else { + /* + * case E: 's' is explored, not in current DFS path, + * its innermost loop header is not in current DFS path, + * hence 's' is another entry into the same loop. + */ + h = aux[s].loop_header; + mark_irreducible(env, h); + while (aux[h].loop_header != -1) { + h = aux[h].loop_header; + if (dfs_pos[h]) { + err = assign_header(env, dfs, cur, h); + if (err) + return err; + break; + } + mark_irreducible(env, h); + } + } + state[cur].next_succ++; + } while (stack_sz); + + return 0; +} + +bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + int idx; + + for (idx = inner_header; idx >= 0; idx = aux[idx].loop_header) + if (aux[idx].loop_header == outer_header) + return true; + + return false; +} + +int bpf_compute_loops(struct bpf_verifier_env *env) +{ + int i, j, k, s, t, iloop, sloop, err = 0, len = env->prog->len; + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_verifier_log *log = &env->log; + struct bpf_backedge *backedge; + struct loops_dfs dfs = {}; + struct bpf_iarray *succ; + struct bpf_loop *loop; + + dfs.dfs_pos = kvcalloc(len, sizeof(int), GFP_KERNEL_ACCOUNT); + dfs.state = kvcalloc(len, sizeof(struct dfs_state), GFP_KERNEL_ACCOUNT); + dfs.stack = kvcalloc(len, sizeof(int), GFP_KERNEL_ACCOUNT); + if (!dfs.dfs_pos || !dfs.state || !dfs.stack) { + err = -ENOMEM; + goto out; + } + for (i = 0; i < len; i++) + aux[i].loop_header = -1; + for (i = 0; i < env->subprog_cnt; i++) { + err = compute_loops_in_subprog(env, &dfs, i); + if (err) + goto out; + } + /* find latches */ + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop) + continue; + for (j = 0; j < loop->backedges_cnt; j++) { + backedge = &loop->backedges[j]; + /* + * In theory, the backedge->from and it's latch can reside in + * an inner loop of `i`, e.g.: + * + * 1: r7 += 1; + * 2: r6 += 1; + * if r7 == 2 goto 1b; + * if r6 < 2 goto 2b; + * + * For now, let's assume that latches are unknown for such cases. + * (GCC/LLVM handle this by inserting artificial cfg nodes). + */ + if (bpf_loop_at_index(env, backedge->from) != i) + continue; + find_dominating_conditions(env, backedge->from, backedge); + } + } + /* find exits */ + for (i = 0; i < len; i++) { + iloop = aux[i].loop ? i : aux[i].loop_header; + if (iloop < 0) + continue; + succ = bpf_insn_successors(env, i); + iarray_for_each(s, succ) { + /* + * Nothing left to record once the innermost loop of 'i' + * overflowed: the walk below would stop at it right away. + */ + if (aux[iloop].loop->exits_overflow) + break; + sloop = aux[s].loop ? s : aux[s].loop_header; + if (iloop == sloop) + continue; + if (bpf_is_nested_loop(env, sloop, iloop)) + continue; + /* + * At this point 'sloop' is either -1, an outer loop, + * or a loop in another branch of the loop hierarchy. + */ + for (t = iloop; t >= 0 && t != sloop; t = aux[t].loop_header) { + /* + * Account for the following configuration: + * + * tloop { + * iloop { + * ... i: goto s; + * } + * sloop { + * s: + * ... + * } + * } + */ + if (bpf_is_nested_loop(env, sloop, t)) + break; + /* + * Record edges i -> s as exits from tloop when: + * + * sloop { + * tloop { + * iloop { + * ... i: goto s; + * } + * } + * s: ... + * } + */ + /* Enclosing loops inherit the overflow, see below */ + if (aux[t].loop->exits_overflow) + break; + err = add_exit(aux[t].loop, i, s); + if (err) + goto out; + } + } + if (bpf_is_ldimm64(env->prog->insnsi + i)) + i++; + } + /* A nested loop with a truncated exit list can't be abstracted by SCEV */ + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop || !loop->exits_overflow) + continue; + for (t = aux[i].loop_header; t >= 0; t = aux[t].loop_header) + aux[t].loop->exits_overflow = true; + } + + if (env->log.level & BPF_LOG_LEVEL2) { + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop) + continue; + bpf_log(log, "loop at %d", i); + if (aux[i].loop_header >= 0) + bpf_log(log, ", nested in %d", aux[i].loop_header); + if (loop->irreducible) + bpf_log(log, ", irreducible"); + if (loop->exits_overflow) + bpf_log(log, ", too many exits"); + bpf_log(log, "\n"); + for (j = 0; j < loop->backedges_cnt; j++) { + backedge = &loop->backedges[j]; + bpf_log(log, " backedge from %d", backedge->from); + for (k = 0; k < backedge->latches_cnt; k++) { + if (k == 0) + bpf_log(log, ", latch at "); + if (k == 1) + bpf_log(log, ", dominating exits at "); + if (k > 1) + bpf_log(log, ", "); + bpf_log(log, "%d", backedge->latches[k]); + } + bpf_log(log, "\n"); + } + for (j = 0; j < loop->exits_cnt; j++) + bpf_log(log, " exit from %d to %d\n", + loop->exits[j].from, loop->exits[j].to); + } + } + +out: + kvfree(dfs.dfs_pos); + kvfree(dfs.stack); + kvfree(dfs.state); + return err; +} diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c index d2cbab4bdf64..0fa3f2ac0f99 100644 --- a/kernel/bpf/map_in_map.c +++ b/kernel/bpf/map_in_map.c @@ -25,6 +25,10 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd) if (!inner_map->ops->map_meta_equal) return ERR_PTR(-ENOTSUPP); + /* An inner map has no used_maps reference to hold it under a queued callback. */ + if (btf_record_has_field(inner_map->record, BPF_RCU_HEAD)) + return ERR_PTR(-EOPNOTSUPP); + inner_map_meta_size = sizeof(*inner_map_meta); /* In some cases verifier needs to access beyond just base map. */ if (inner_map->ops == &array_map_ops || inner_map->ops == &percpu_array_map_ops) @@ -106,10 +110,13 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map, return ERR_PTR(-ENOTSUPP); inner_map_meta = map->inner_map_meta; - if (inner_map_meta->ops->map_meta_equal(inner_map_meta, inner_map)) + if (inner_map_meta->ops->map_meta_equal(inner_map_meta, inner_map)) { bpf_map_inc(inner_map); - else + /* Its users are those of the outer maps from now on */ + WRITE_ONCE(inner_map->was_inner_map, true); + } else { inner_map = ERR_PTR(-EINVAL); + } return inner_map; } diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c index c19b360bad9e..a58c148692e6 100644 --- a/kernel/bpf/map_iter.c +++ b/kernel/bpf/map_iter.c @@ -97,13 +97,57 @@ static struct bpf_iter_reg bpf_map_reg_info = { .seq_info = &bpf_map_seq_info, }; +/* + * For an iterator that hands the live map value to the program as a writable + * buffer: the verifier knows nothing of the map, so the special fields of the + * value have to lie outside of what the program may write. + */ +int bpf_iter_map_check_value_write(const struct bpf_prog *prog, + const struct bpf_map *map) +{ + const struct btf_record *rec = map->record; + const struct btf_field *field; + u32 i; + + if (IS_ERR_OR_NULL(rec) || !prog->aux->max_rdwr_write) + return 0; + + for (i = 0; i < rec->cnt; i++) { + field = &rec->fields[i]; + if (field->offset < prog->aux->max_rdwr_write && + field->offset + field->size > prog->aux->min_rdwr_write) + return -EACCES; + } + + return 0; +} + +static int bpf_iter_map_check_prog(const struct bpf_prog *prog, + const struct bpf_iter_aux_info *aux) +{ + const struct bpf_map *map = aux->map; + u32 value_size = map->value_size; + bool is_percpu; + + is_percpu = map->map_type == BPF_MAP_TYPE_PERCPU_HASH || + map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || + map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; + if (is_percpu) + value_size = round_up(value_size, 8) * num_possible_cpus(); + + if (prog->aux->max_rdonly_access > map->key_size || + prog->aux->max_rdwr_access > value_size) + return -EACCES; + + /* the program gets a copy of a per-cpu value */ + return is_percpu ? 0 : bpf_iter_map_check_value_write(prog, map); +} + static int bpf_iter_attach_map(struct bpf_prog *prog, union bpf_iter_link_info *linfo, struct bpf_iter_aux_info *aux) { - u32 key_acc_size, value_acc_size, key_size, value_size; struct bpf_map *map; - bool is_percpu = false; int err = -EINVAL; if (!linfo->map.map_fd) @@ -117,30 +161,26 @@ static int bpf_iter_attach_map(struct bpf_prog *prog, goto put_map; } - if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || - map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || - map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) - is_percpu = true; - else if (map->map_type != BPF_MAP_TYPE_HASH && - map->map_type != BPF_MAP_TYPE_LRU_HASH && - map->map_type != BPF_MAP_TYPE_ARRAY && - map->map_type != BPF_MAP_TYPE_RHASH) + /* The value ctx arg is writable and aliases the live element. */ + if (btf_record_has_field(map->record, BPF_RCU_HEAD)) { + err = -EOPNOTSUPP; goto put_map; + } - key_acc_size = prog->aux->max_rdonly_access; - value_acc_size = prog->aux->max_rdwr_access; - key_size = map->key_size; - if (!is_percpu) - value_size = map->value_size; - else - value_size = round_up(map->value_size, 8) * num_possible_cpus(); - - if (key_acc_size > key_size || value_acc_size > value_size) { - err = -EACCES; + if (map->map_type != BPF_MAP_TYPE_PERCPU_HASH && + map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH && + map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY && + map->map_type != BPF_MAP_TYPE_HASH && + map->map_type != BPF_MAP_TYPE_LRU_HASH && + map->map_type != BPF_MAP_TYPE_ARRAY && + map->map_type != BPF_MAP_TYPE_RHASH) goto put_map; - } aux->map = map; + err = bpf_iter_map_check_prog(prog, aux); + if (err) + goto put_map; + return 0; put_map: @@ -173,6 +213,7 @@ static const struct bpf_iter_reg bpf_map_elem_reg_info = { .target = "bpf_map_elem", .attach_target = bpf_iter_attach_map, .detach_target = bpf_iter_detach_map, + .check_prog = bpf_iter_map_check_prog, .show_fdinfo = bpf_iter_map_show_fdinfo, .fill_link_info = bpf_iter_map_fill_link_info, .ctx_arg_info_size = 2, diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c index e9662db7198f..15684d0fc883 100644 --- a/kernel/bpf/memalloc.c +++ b/kernel/bpf/memalloc.c @@ -118,7 +118,13 @@ struct bpf_mem_cache { struct llist_head free_by_rcu_ttrace; struct llist_head waiting_for_gp_ttrace; struct rcu_head rcu_ttrace; + /* + * 0 - idle + * 1 - __free_rcu() is queued + * 2 - __free_rcu() is queued and free_by_rcu_ttrace got more objects since + */ atomic_t call_rcu_ttrace_in_progress; + raw_spinlock_t lock; }; struct bpf_mem_caches { @@ -214,25 +220,24 @@ static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node, bool atomic) gfp = __GFP_NOWARN | __GFP_ACCOUNT; gfp |= atomic ? GFP_NOWAIT : GFP_KERNEL; - for (i = 0; i < cnt; i++) { - /* - * For every 'c' llist_del_first(&c->free_by_rcu_ttrace); is - * done only by one CPU == current CPU. Other CPUs might - * llist_add() and llist_del_all() in parallel. - */ - obj = llist_del_first(&c->free_by_rcu_ttrace); - if (!obj) - break; - add_obj_to_free_list(c, obj); - } - if (i >= cnt) - return; + /* + * c->lock serializes concurrent llist_del_first() against + * llist_del_all() in __free_rcu() and do_call_rcu_ttrace(). + */ + scoped_guard(raw_spinlock_irqsave, &c->lock) { + for (i = 0; i < cnt; i++) { + obj = llist_del_first(&c->free_by_rcu_ttrace); + if (!obj) + break; + add_obj_to_free_list(c, obj); + } - for (; i < cnt; i++) { - obj = llist_del_first(&c->waiting_for_gp_ttrace); - if (!obj) - break; - add_obj_to_free_list(c, obj); + for (; i < cnt; i++) { + obj = llist_del_first(&c->waiting_for_gp_ttrace); + if (!obj) + break; + add_obj_to_free_list(c, obj); + } } if (i >= cnt) return; @@ -276,12 +281,30 @@ static int free_all(struct bpf_mem_cache *c, struct llist_node *llnode, bool per return cnt; } +static void __do_call_rcu_ttrace(struct bpf_mem_cache *c); + static void __free_rcu(struct rcu_head *head) { struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu_ttrace); + struct llist_node *llnode; + + scoped_guard(raw_spinlock_irqsave, &c->lock) + llnode = llist_del_all(&c->waiting_for_gp_ttrace); + + free_all(c, llnode, !!c->percpu_size); - free_all(c, llist_del_all(&c->waiting_for_gp_ttrace), !!c->percpu_size); - atomic_set(&c->call_rcu_ttrace_in_progress, 0); + /* + * do_call_rcu_ttrace() that ran while GP was in flight left its objects + * in free_by_rcu_ttrace. This cache may never free or alloc in bulk + * again, so start the next GP from here. + * 'c' can be freed as soon as call_rcu_ttrace_in_progress is zero. + */ + if (atomic_cmpxchg(&c->call_rcu_ttrace_in_progress, 1, 0) == 1) + return; + + /* Pairs with synchronize_rcu() in free_mem_alloc() */ + guard(rcu)(); + __do_call_rcu_ttrace(c); } static void enque_to_free(struct bpf_mem_cache *c, void *obj) @@ -294,17 +317,15 @@ static void enque_to_free(struct bpf_mem_cache *c, void *obj) llist_add(llnode, &c->free_by_rcu_ttrace); } -static void do_call_rcu_ttrace(struct bpf_mem_cache *c) +static void __do_call_rcu_ttrace(struct bpf_mem_cache *c) { struct llist_node *llnode, *t; - if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 1)) { - if (unlikely(READ_ONCE(c->draining))) { - llnode = llist_del_all(&c->free_by_rcu_ttrace); - free_all(c, llnode, !!c->percpu_size); - } - return; - } + /* + * Must be done before llist_del_all(). Objects that it misses were + * added by do_call_rcu_ttrace() that will set 2 after this store. + */ + atomic_set(&c->call_rcu_ttrace_in_progress, 1); WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp_ttrace)); llist_for_each_safe(llnode, t, llist_del_all(&c->free_by_rcu_ttrace)) @@ -323,6 +344,22 @@ static void do_call_rcu_ttrace(struct bpf_mem_cache *c) call_rcu_tasks_trace(&c->rcu_ttrace, __free_rcu); } +static void do_call_rcu_ttrace(struct bpf_mem_cache *c) +{ + struct llist_node *llnode; + + if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 2)) { + if (unlikely(READ_ONCE(c->draining))) { + scoped_guard(raw_spinlock_irqsave, &c->lock) + llnode = llist_del_all(&c->free_by_rcu_ttrace); + free_all(c, llnode, !!c->percpu_size); + } + return; + } + + __do_call_rcu_ttrace(c); +} + static void free_bulk(struct bpf_mem_cache *c) { struct bpf_mem_cache *tgt = c->tgt; @@ -535,6 +572,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) c->objcg = objcg; c->percpu_size = percpu_size; c->tgt = c; + raw_spin_lock_init(&c->lock); init_refill_work(c); prefill_mem_cache(c, cpu); } @@ -557,7 +595,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) c->objcg = objcg; c->percpu_size = percpu_size; c->tgt = c; - + raw_spin_lock_init(&c->lock); init_refill_work(c); prefill_mem_cache(c, cpu); } @@ -609,7 +647,7 @@ int bpf_mem_alloc_percpu_unit_init(struct bpf_mem_alloc *ma, int size) c->objcg = objcg; c->percpu_size = percpu_size; c->tgt = c; - + raw_spin_lock_init(&c->lock); init_refill_work(c); prefill_mem_cache(c, cpu); } @@ -694,7 +732,12 @@ static void free_mem_alloc(struct bpf_mem_alloc *ma) * to wait for the pending __free_by_rcu(), and __free_rcu(). RCU Tasks * Trace grace period implies RCU grace period, so all __free_rcu don't * need extra call_rcu() (and thus extra rcu_barrier() here). + * + * __free_rcu() queues itself again unless it sees 'draining'. After + * synchronize_rcu() it either did that already or will not do it, so + * rcu_barrier_tasks_trace() cannot miss it. */ + synchronize_rcu(); rcu_barrier(); /* wait for __free_by_rcu */ rcu_barrier_tasks_trace(); /* wait for __free_rcu */ free_mem_alloc_no_barrier(ma); diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c index 0d6f5569588c..f7bfff1760cc 100644 --- a/kernel/bpf/offload.c +++ b/kernel/bpf/offload.c @@ -96,6 +96,22 @@ err_free: return err; } +/* + * @prog takes the place of a copy that the caller is about to free. Nothing + * keeps the netdev while a program is loaded: once the lock is dropped, no + * unregistration is left that has got the old copy from offload->prog. + */ +void bpf_prog_dev_bound_moved(struct bpf_prog *prog) +{ + if (!bpf_prog_is_dev_bound(prog->aux)) + return; + + down_read(&bpf_devs_lock); + if (prog->aux->offload) + prog->aux->offload->prog = prog; + up_read(&bpf_devs_lock); +} + static void __bpf_prog_offload_destroy(struct bpf_prog *prog) { struct bpf_prog_offload *offload = prog->aux->offload; @@ -250,7 +266,14 @@ int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr) prog->aux->offload_requested = !(attr->prog_flags & BPF_F_XDP_DEV_BOUND_ONLY); down_write(&bpf_devs_lock); - err = __bpf_prog_dev_bound_init(prog, netdev); + /* + * Behind bpf_dev_bound_netdev_unregister() nobody would take the + * program off the device, which goes once the reference is dropped. + */ + if (READ_ONCE(netdev->reg_state) == NETREG_REGISTERED) + err = __bpf_prog_dev_bound_init(prog, netdev); + else + err = -EINVAL; up_write(&bpf_devs_lock); out: @@ -698,6 +721,8 @@ static bool __bpf_offload_dev_match(struct bpf_prog *prog, return false; if (offload->netdev == netdev) return true; + if (!bpf_prog_is_offloaded(prog->aux)) + return false; ondev1 = bpf_offload_find_netdev(offload->netdev); ondev2 = bpf_offload_find_netdev(netdev); diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c index 6e8b18c32a10..1cee2ba6e332 100644 --- a/kernel/bpf/queue_stack_maps.c +++ b/kernel/bpf/queue_stack_maps.c @@ -59,6 +59,9 @@ static int queue_stack_map_alloc_check(union bpf_attr *attr) */ return -E2BIG; + if (attr->max_entries >= U32_MAX / attr->value_size) + return -E2BIG; + return 0; } diff --git a/kernel/bpf/range_tree.c b/kernel/bpf/range_tree.c index 2f28886f3ff7..2460e78d6a55 100644 --- a/kernel/bpf/range_tree.c +++ b/kernel/bpf/range_tree.c @@ -143,16 +143,22 @@ int range_tree_clear(struct range_tree *rt, u32 start, u32 len) if (rn->rn_start < start && rn->rn_last > last) { u32 old_last = rn->rn_last; + /* + * Pre-allocate the right-half node before modifying + * the tree. If allocation fails we return -ENOMEM + * without altering the range tree. + */ + new_rn = kmalloc_nolock(sizeof(struct range_node), + __GFP_ACCOUNT, NUMA_NO_NODE); + if (!new_rn) + return -ENOMEM; + /* Overlaps with the entire clearing range */ range_it_remove(rn, rt); rn->rn_last = start - 1; range_it_insert(rn, rt); - /* Add a range */ - new_rn = kmalloc_nolock(sizeof(struct range_node), __GFP_ACCOUNT, - NUMA_NO_NODE); - if (!new_rn) - return -ENOMEM; + /* Add right-half range */ new_rn->rn_start = last + 1; new_rn->rn_last = old_last; range_it_insert(new_rn, rt); @@ -193,6 +199,7 @@ int is_range_tree_set(struct range_tree *rt, u32 start, u32 len) int range_tree_set(struct range_tree *rt, u32 start, u32 len) { u32 last = start + len - 1; + struct range_node *new_rn = NULL; struct range_node *right; struct range_node *left; int err; @@ -202,20 +209,19 @@ int range_tree_set(struct range_tree *rt, u32 start, u32 len) if (left && left->rn_start <= start && left->rn_last >= last) return 0; + left = range_it_iter_first(rt, start - 1, start - 1); + right = range_it_iter_first(rt, last + 1, last + 1); + if (!left && !right) { + new_rn = kmalloc_nolock(sizeof(struct range_node), + __GFP_ACCOUNT, NUMA_NO_NODE); + if (!new_rn) + return -ENOMEM; + } + /* Clear out everything in the range we want to set. */ err = range_tree_clear(rt, start, len); if (err) - return err; - - /* Do we have a left-adjacent range ? */ - left = range_it_iter_first(rt, start - 1, start - 1); - if (left && left->rn_last + 1 != start) - return -EFAULT; - - /* Do we have a right-adjacent range ? */ - right = range_it_iter_first(rt, last + 1, last + 1); - if (right && right->rn_start != last + 1) - return -EFAULT; + goto out_free_new; if (left && right) { /* Combine left and right adjacent ranges */ @@ -235,14 +241,16 @@ int range_tree_set(struct range_tree *rt, u32 start, u32 len) right->rn_start = start; range_it_insert(right, rt); } else { - left = kmalloc_nolock(sizeof(struct range_node), __GFP_ACCOUNT, NUMA_NO_NODE); - if (!left) - return -ENOMEM; - left->rn_start = start; - left->rn_last = last; - range_it_insert(left, rt); + /* No adjacent ranges; use the pre-allocated node */ + new_rn->rn_start = start; + new_rn->rn_last = last; + range_it_insert(new_rn, rt); } return 0; + +out_free_new: + kfree_nolock(new_rn); + return err; } void range_tree_destroy(struct range_tree *rt) diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index 3f1013d80544..447863ece26a 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -722,7 +722,7 @@ BPF_CALL_2(bpf_ringbuf_submit_dynptr, struct bpf_dynptr_kern *, ptr, u64, flags) const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto = { .func = bpf_ringbuf_submit_dynptr, .ret_type = RET_VOID, - .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE, + .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE | MEM_WRITE, .arg2_type = ARG_ANYTHING, }; @@ -741,7 +741,7 @@ BPF_CALL_2(bpf_ringbuf_discard_dynptr, struct bpf_dynptr_kern *, ptr, u64, flags const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto = { .func = bpf_ringbuf_discard_dynptr, .ret_type = RET_VOID, - .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE, + .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE | MEM_WRITE, .arg2_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c new file mode 100644 index 000000000000..fea77772ca96 --- /dev/null +++ b/kernel/bpf/scev.c @@ -0,0 +1,2916 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ + +#include "linux/cnum.h" +#include <linux/bpf_verifier.h> +#include <linux/jhash.h> +#include <linux/log2.h> +#include <linux/math64.h> +#include <linux/min_heap.h> +#include <linux/bug.h> +#include <linux/tnum.h> +#include <linux/overflow.h> + +#define REGS_NUM BPF_SCEV_REGS_NUM +#define UNKNOWN_EXPR_ID 0 +#define OPAQUE_EXPR_ID 1 + +/* + * BPF instructions use 'code', 'src_reg', 'off' and 'imm' fields for instruction encoding. + * For scalar evolution purpose we want to reuse most of the opcode definitions, + * but also add a few custom operations (REG and IMM). + * Use the enum below to uniformly represent all operations relevant for SCEV. + */ +enum expr_op { + /* leave range 0..255 for standard bpf opcodes */ + UNKNOWN = 256, /* start custom opcodes from the second byte */ + REG, + IMM, + SDIV, SMOD, + SEXT8, SEXT16, SEXT32, + ZEXT8, ZEXT16, ZEXT32, + BSWAP16, BSWAP32, BSWAP64, + /* during the loop body execution the value can be either of param[0] or param[1] */ + ANY, + /* some value that verifier is not going to track precisely */ + OPAQUE, + /* '*(u8/16/32 *)(r10 + X) = Y' writes define slot contents only partially */ + SPILL8, SPILL16, SPILL32, + /* + * SCEV expression corresponding to linear equation 'param[0] + param[1] * k', + * where k is a loop iteration number. Loop here refers to innermost loop + * containing instruction associated with this expression, as returned by + * bpf_loop_at_index(). + */ + LINEAR_SCEV, +}; + +struct expr { + u32 op; + union { + u32 params[2]; + s64 imm; + }; +}; + +struct expr_bucket { + u32 cnt; + u32 cap; + u32 ids[]; +}; + +struct env { + bool empty; + u32 reg2expr[REGS_NUM]; + u32 reg2scev[REGS_NUM]; +}; + +struct insn_envs { + u32 cnt; + struct { + int loop_header; + struct env *env; + } entries[]; +}; + +#define EXPR_STACK_DEPTH 8 + +struct expr_stack_elt { + u32 id:28; + u32 pre:1; + u32 next_param:2; +}; + +DEFINE_MIN_HEAP(int, scev_worklist); + +struct scev { + /* Read-only representatives for stack slots without a saved register. */ + struct bpf_reg_state zero_reg; + struct bpf_reg_state unknown_reg; + /* + * Expressions are identified by id, exprs_ht ensures that + * each expression exists as a unique instance. + * This allows for fast equivalence check: id1 === id2. + */ + struct expr_bucket **exprs_ht; + struct scev_worklist worklist; + struct insn_envs **envs; + struct expr *exprs; + /* + * Loops are analyzed one by one, this array keeps track if a particular + * instruction was visited on a current pass. + */ + u32 *discovered; + u32 exprs_ht_cnt; + int exprs_cnt; + int exprs_cap; + int envs_cnt; + int stack_sz; + struct expr_stack_elt expr_stack[EXPR_STACK_DEPTH]; + u32 ids_buf[EXPR_STACK_DEPTH]; +}; + +static u32 expr_hash(struct expr *e) +{ + return jhash_3words(e->op, e->params[0], e->params[1], 0); +} + +static int expr_eq(struct expr *a, struct expr *b) +{ + return a->op == b->op && a->imm == b->imm; +} + +static int add_expr(struct scev *scev, struct expr e) +{ + struct expr_bucket *bucket; + u32 i, id, hash, new_cap; + void *tmp; + + hash = expr_hash(&e) & (scev->exprs_ht_cnt - 1); + bucket = scev->exprs_ht[hash]; + + if (bucket) { + for (i = 0; i < bucket->cnt; i++) { + id = bucket->ids[i]; + if (expr_eq(&e, &scev->exprs[id])) + return id; + } + } + + if (!bucket || bucket->cap == bucket->cnt) { + new_cap = bucket ? bucket->cap * 2 : 4; + bucket = kvrealloc(bucket, sizeof(*bucket) + sizeof(u32) * new_cap, GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!bucket) + return -ENOMEM; + scev->exprs_ht[hash] = bucket; + bucket->cap = new_cap; + } + + if (scev->exprs_cnt == scev->exprs_cap) { + new_cap = scev->exprs_cap ? scev->exprs_cap * 2 : 256; + tmp = kvrealloc(scev->exprs, sizeof(struct expr) * new_cap, GFP_KERNEL_ACCOUNT); + if (!tmp) + return -ENOMEM; + scev->exprs = tmp; + scev->exprs_cap = new_cap; + } + + id = scev->exprs_cnt++; + scev->exprs[id] = e; + bucket->ids[bucket->cnt++] = id; + return id; +} + +static int expr2(struct scev *scev, u32 op, int a, int b) +{ + if (a < 0) + return a; + if (b < 0) + return b; + return add_expr(scev, (struct expr){ .op = op, .params = {a, b} }); +} + +static int expr1(struct scev *scev, u32 op, int a) +{ + if (a < 0) + return a; + return expr2(scev, op, a, 0); +} + +static int expr0(struct scev *scev, u32 op) +{ + return expr2(scev, op, 0, 0); +} + +static int imm_expr(struct scev *scev, s64 value) +{ + return add_expr(scev, (struct expr){ .op = IMM, .imm = value }); +} + +static bool same_exprs(struct scev *scev, int id_a, int id_b) +{ + return id_a == id_b; +} + +static bool is_imm(struct scev *scev, int id, s64 *imm) +{ + if (scev->exprs[id].op != IMM) + return false; + *imm = scev->exprs[id].imm; + return true; +} + +static bool is_op(struct scev *scev, int id, u32 op) +{ + if (scev->exprs[id].op != op) + return false; + return true; +} + +static bool is_unop(struct scev *scev, enum expr_op op, int id, u32 *p0) +{ + if (scev->exprs[id].op != op) + return false; + *p0 = scev->exprs[id].params[0]; + return true; +} + +static bool is_binop(struct scev *scev, enum expr_op op, int id, u32 *p0, u32 *p1) +{ + if (scev->exprs[id].op != op) + return false; + *p0 = scev->exprs[id].params[0]; + *p1 = scev->exprs[id].params[1]; + return true; +} + +static bool is_reg(struct scev *scev, int id, u32 *reg) +{ + return is_unop(scev, REG, id, reg); +} + +static bool is_add(struct scev *scev, int id, u32 *left, u32 *right) +{ + return is_binop(scev, BPF_ADD, id, left, right); +} + +static bool is_zext32(struct scev *scev, int id, u32 *left) +{ + return is_unop(scev, ZEXT32, id, left); +} + +static bool is_any(struct scev *scev, int id, u32 *left, u32 *right) +{ + return is_binop(scev, ANY, id, left, right); +} + +static bool is_linear(struct scev *scev, int id, u32 *base, u32 *slope) +{ + return is_binop(scev, LINEAR_SCEV, id, base, slope); +} + +static bool is_linear_or_linear32(struct scev *scev, int id, u32 *base, u32 *slope, bool *zext32) +{ + u32 t; + + *zext32 = is_zext32(scev, id, &t); + return is_linear(scev, *zext32 ? t : id, base, slope); +} + +static bool is_opaque(struct scev *scev, int id) +{ + return is_op(scev, id, OPAQUE); +} + +static void log_reg(struct bpf_verifier_env *env, u32 reg) +{ + if (reg < MAX_BPF_REG) + bpf_log(&env->log, "r%d", reg); + else + bpf_log(&env->log, "*fp%d", (MAX_BPF_REG - reg - 1) * 8); +} + +static const char *op_str(u32 op) +{ + switch (op) { + case BPF_ADD: return "+"; + case BPF_SUB: return "-"; + case BPF_MUL: return "*"; + case BPF_DIV: return "/"; + case SDIV: return "s/"; + case BPF_OR: return "|"; + case BPF_AND: return "&"; + case BPF_LSH: return "<<"; + case BPF_RSH: return ">>"; + case BPF_NEG: return "-"; + case BPF_MOD: return "%"; + case SMOD: return "s%"; + case BPF_XOR: return "^"; + case BPF_ARSH: return "s>>"; + case SEXT8: return "sext8"; + case SEXT16: return "sext16"; + case SEXT32: return "sext32"; + case ZEXT8: return "zext8"; + case ZEXT16: return "zext16"; + case ZEXT32: return "zext32"; + case BSWAP16: return "bswap16"; + case BSWAP32: return "bswap32"; + case BSWAP64: return "bswap64"; + case SPILL8: return "spill8"; + case SPILL16: return "spill16"; + case SPILL32: return "spill32"; + case ANY: return "any"; + case LINEAR_SCEV: return "linear"; + } + return NULL; +} + +static u32 op_params_num(u32 op) +{ + switch (op) { + case BPF_ADD: + case BPF_SUB: + case BPF_MUL: + case BPF_DIV: + case BPF_MOD: + case BPF_OR: + case BPF_XOR: + case BPF_AND: + case BPF_LSH: + case BPF_RSH: + case BPF_ARSH: + case SDIV: + case SMOD: + case ANY: + case LINEAR_SCEV: + return 2; + case BPF_NEG: + case SEXT8: + case SEXT16: + case SEXT32: + case ZEXT8: + case ZEXT16: + case ZEXT32: + case BSWAP16: + case BSWAP32: + case BSWAP64: + case SPILL8: + case SPILL16: + case SPILL32: + return 1; + case REG: + case IMM: + case OPAQUE: + return 0; + default: + return 0; + } +} + +static bool expr_stack_push(struct scev *scev, u32 id) +{ + if (scev->stack_sz >= EXPR_STACK_DEPTH) + return false; + scev->expr_stack[scev->stack_sz].id = id; + scev->expr_stack[scev->stack_sz].pre = true; + scev->expr_stack[scev->stack_sz].next_param = 0; + scev->stack_sz++; + return true; +} + +enum { + PRE = BIT(1), POST = BIT(2), DEPTH_LIMIT = BIT(3) +}; + +static bool expr_next(struct scev *scev, u32 *id, u32 *order) +{ + struct expr_stack_elt *elt; + struct expr *expr; + u32 num_params; + + if (scev->stack_sz == 0) + return false; + + elt = &scev->expr_stack[scev->stack_sz - 1]; + *id = elt->id; + *order = 0; + expr = &scev->exprs[elt->id]; + num_params = op_params_num(expr->op); + if (elt->pre) { + elt->pre = false; + *order = PRE; + return true; + } + if (elt->next_param == num_params) { + *order = POST; + scev->stack_sz--; + return true; + } + if (scev->stack_sz == EXPR_STACK_DEPTH) { + *order = POST | DEPTH_LIMIT; + scev->stack_sz--; + return true; + } + expr_stack_push(scev, expr->params[elt->next_param]); + elt->next_param++; + return expr_next(scev, id, order); +} + +static void log_expr(struct bpf_verifier_env *env, u32 id) +{ + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + struct expr *expr; + const char *str; + u32 order; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if ((order & PRE) && scev->stack_sz > 1) + bpf_log(log, " "); + expr = &scev->exprs[id]; + switch (expr->op) { + case UNKNOWN: + if (order & PRE) + bpf_log(log, "?"); + break; + case OPAQUE: + if (order & PRE) + bpf_log(log, "_"); + break; + case REG: + if (order & PRE) + log_reg(env, expr->params[0]); + break; + case IMM: + if (order & PRE) + bpf_log(log, "%lld", expr->imm); + break; + default: + if (order & PRE) { + str = op_str(expr->op); + bpf_log(log, "("); + if (str) + bpf_log(log, "%s", str); + else + bpf_log(log, "bad-expr-op %x", expr->op); + } + if (order & DEPTH_LIMIT) + bpf_log(log, "..."); + if (order & POST) + bpf_log(log, ")"); + } + } +} + +static bool reg_alive_at(struct bpf_verifier_env *env, u32 reg, u32 insn_idx) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + u16 live_regs = aux[insn_idx].live_regs_before; + + return reg < MAX_BPF_REG + ? (BIT(reg) & live_regs) + : test_bit(reg - MAX_BPF_REG, aux[insn_idx].live_stack_before); +} + +static void print_env(struct bpf_verifier_env *env, struct env *e, u32 insn_idx) +{ + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + bool printed_some = false; + bool is_self_scev; + bool is_self_reg; + int i, r; + + if (e->empty) { + bpf_log(log, " <empty>\n"); + return; + } + + for (i = 0; i < REGS_NUM; i++) { + if (!reg_alive_at(env, i, insn_idx)) + continue; + is_self_reg = is_reg(scev, e->reg2expr[i], &r) && i == r; + is_self_scev = is_reg(scev, e->reg2scev[i], &r) && i == r; + if (is_self_reg && is_self_scev) + continue; + printed_some = true; + bpf_log(log, " "); + log_reg(env, i); + bpf_log(log, "="); + log_expr(env, e->reg2expr[i]); + if (e->reg2expr[i] != UNKNOWN_EXPR_ID) { + bpf_log(log, " / "); + log_expr(env, e->reg2scev[i]); + } + bpf_log(log, "\n"); + } + if (!printed_some) + bpf_log(log, " <all regs unchanged>\n"); +} +static struct env *find_loop_env(struct scev *scev, int loop_header, int insn_idx) +{ + struct insn_envs *envs = scev->envs[insn_idx]; + u32 i; + + if (!envs) + return NULL; + for (i = 0; i < envs->cnt; i++) + if (envs->entries[i].loop_header == loop_header) + return envs->entries[i].env; + return NULL; +} + +static struct env *find_header_env(struct scev *scev, int header) +{ + return find_loop_env(scev, header, header); +} + +static struct env *get_loop_env(struct scev *scev, int loop_header, int insn_idx) +{ + struct insn_envs *envs = scev->envs[insn_idx]; + u32 cnt = envs ? envs->cnt : 0; + struct insn_envs *tmp; + struct env *e; + + e = find_loop_env(scev, loop_header, insn_idx); + if (e) + return e; + + e = kzalloc(sizeof(*e), GFP_KERNEL_ACCOUNT); + if (!e) + return NULL; + + tmp = krealloc(envs, struct_size(envs, entries, cnt + 1), GFP_KERNEL_ACCOUNT); + if (!tmp) { + kfree(e); + return NULL; + } + + e->empty = true; + tmp->entries[cnt].loop_header = loop_header; + tmp->entries[cnt].env = e; + tmp->cnt = cnt + 1; + scev->envs[insn_idx] = tmp; + return e; +} + +static int replace_reg(struct scev *scev, struct env *e, u32 reg, int id) +{ + if (id < 0) + return id; + e->reg2expr[reg] = id; + return 0; +} + +static int setup_initial_loop_env(struct bpf_verifier_env *env, struct env *e, int insn_idx) +{ + struct scev *scev = env->scev; + int i, err; + + /* + * Initial assignments are of form rX -> rX are needed as seeds for subsequent + * work queue pop / transfer() calls. Because e->empty is true, the first backedge + * join() will overwrite the them. compute_header_scevs() accounts for register + * states at loop entry. + */ + for (i = 0; i < REGS_NUM; i++) { + err = replace_reg(scev, e, i, expr1(scev, REG, i)); + if (err) + return err; + } + return 0; +} + +static void forget_call_regs(struct env *e) +{ + int i; + + for (i = BPF_REG_0; i <= BPF_REG_5; i++) + e->reg2expr[i] = UNKNOWN_EXPR_ID; +} + +static u32 spill_spi(struct bpf_insn *insn) +{ + return -insn->off / BPF_REG_SIZE - 1; +} + +static u32 off_to_reg(struct bpf_insn *insn) +{ + return spill_spi(insn) + MAX_BPF_REG; +} + +static bool is_spill_off(int off) +{ + return off % BPF_REG_SIZE == 0 && + off <= -BPF_REG_SIZE && + off >= -MAX_BPF_STACK_JIT; +} + +static void mark_opaque(struct env *e, int off, int size) +{ + int b, spi; + + for (b = off; b < off + size; b++) { + if (b >= 0 || b < -MAX_BPF_STACK_JIT) + continue; + spi = (-b - 1) / BPF_REG_SIZE; + e->reg2expr[MAX_BPF_REG + spi] = OPAQUE_EXPR_ID; + } +} + +static int mk_spill(struct scev *scev, u8 size, int id) +{ + switch (size) { + case BPF_B: return expr1(scev, SPILL8, id); + case BPF_H: return expr1(scev, SPILL16, id); + case BPF_W: return expr1(scev, SPILL32, id); + case BPF_DW: return id; + } + return UNKNOWN_EXPR_ID; +} + +static int mk_fill(struct scev *scev, int id, u8 code) +{ + bool sx = BPF_MODE(code) == BPF_MEMSX; + +#ifdef __BIG_ENDIAN + /* A narrow fill from a DW spill reads high bits on big-endian. */ + if (BPF_SIZE(code) != BPF_DW) + return OPAQUE_EXPR_ID; +#endif + + switch (BPF_SIZE(code)) { + case BPF_B: return expr1(scev, sx ? SEXT8 : ZEXT8, id); + case BPF_H: return expr1(scev, sx ? SEXT16 : ZEXT16, id); + case BPF_W: return expr1(scev, sx ? SEXT32 : ZEXT32, id); + case BPF_DW: return sx ? UNKNOWN_EXPR_ID : id; /* DW sign-extended load is invalid */ + } + return UNKNOWN_EXPR_ID; +} + +static int maybe_store_fp(struct scev *scev, struct env *e, struct bpf_insn *insn, int id) +{ + u8 size = BPF_SIZE(insn->code); + + if (insn->dst_reg != BPF_REG_FP) + return 0; + if (is_spill_off(insn->off)) + return replace_reg(scev, e, off_to_reg(insn), mk_spill(scev, size, id)); + mark_opaque(e, insn->off, bpf_size_to_bytes(size)); + return 0; +} + +static int maybe_load_fp(struct scev *scev, struct env *e, struct bpf_insn *insn) +{ + int id; + + if (insn->src_reg == BPF_REG_FP && is_spill_off(insn->off)) + id = mk_fill(scev, e->reg2expr[off_to_reg(insn)], insn->code); + else + id = OPAQUE_EXPR_ID; + return replace_reg(scev, e, insn->dst_reg, id); +} + +static int transfer(struct bpf_verifier_env *env, struct env *e, int idx) +{ + const bool little_endian = htons(0x3412) == 0x1234; + struct bpf_insn *insn = &env->prog->insnsi[idx]; + struct scev *scev = env->scev; + u32 *reg2expr = e->reg2expr; + u8 class = BPF_CLASS(insn->code); + u8 x_or_k = BPF_SRC(insn->code); + u8 opcode = BPF_OP(insn->code); + u8 mode = BPF_MODE(insn->code); + u32 dst = insn->dst_reg; + u32 src = insn->src_reg; + u32 op, sext; + bool need_bswap; + int i, id; + s64 imm; + + switch (class) { + case BPF_ALU: + case BPF_ALU64: + switch (opcode) { + case BPF_MOV: + switch (insn->off) { + case 0: sext = 0; break; + case 8: sext = SEXT8; break; + case 16: sext = SEXT16; break; + case 32: sext = SEXT32; break; + default: + goto mark_dst_unknown; + } + + if (x_or_k == BPF_X && insn->imm == 0) + id = reg2expr[src]; + else if (x_or_k == BPF_K && src == 0 && insn->off == 0) + id = imm_expr(scev, insn->imm); + else + goto mark_dst_unknown; + + if (sext) + id = expr1(scev, sext, id); + if (class == BPF_ALU) + id = expr1(scev, ZEXT32, id); + + return replace_reg(scev, e, dst, id); + + case BPF_ADD: + case BPF_SUB: + case BPF_MUL: + case BPF_DIV: + case BPF_MOD: + case BPF_OR: + case BPF_XOR: + case BPF_AND: + case BPF_LSH: + case BPF_RSH: + case BPF_ARSH: + if (opcode == BPF_DIV && insn->off == 1) + op = SDIV; + else if (opcode == BPF_MOD && insn->off == 1) + op = SMOD; + else if (insn->off == 0) + op = opcode; + else + goto mark_dst_unknown; + + if (x_or_k == BPF_X && insn->imm == 0) + id = reg2expr[src]; + else if (x_or_k == BPF_K && src == 0) + id = imm_expr(scev, insn->imm); + else + goto mark_dst_unknown; + + id = expr2(scev, op, reg2expr[dst], id); + + if (class == BPF_ALU) + id = expr1(scev, ZEXT32, id); + + return replace_reg(scev, e, dst, id); + + case BPF_NEG: + if (src == 0 && insn->off == 0 && insn->imm == 0) + id = expr1(scev, BPF_NEG, reg2expr[dst]); + else + goto mark_dst_unknown; + + if (class == BPF_ALU) + id = expr1(scev, ZEXT32, id); + + return replace_reg(scev, e, dst, id); + + case BPF_END: + if (src != 0 || insn->off != 0 || (class == BPF_ALU64 && x_or_k != 0)) + goto mark_dst_unknown; + + need_bswap = class == BPF_ALU64 || ((x_or_k == BPF_TO_LE) != little_endian); + switch (insn->imm) { + case 16: op = need_bswap ? BSWAP16 : ZEXT16; break; + case 32: op = need_bswap ? BSWAP32 : ZEXT32; break; + case 64: op = need_bswap ? BSWAP64 : 0; break; + default: + goto mark_dst_unknown; + } + + id = reg2expr[dst]; + if (op) + id = expr1(scev, op, reg2expr[dst]); + + return replace_reg(scev, e, dst, id); + default: + goto mark_dst_unknown; + } + break; + case BPF_LDX: + switch (mode) { + case BPF_MEM: + case BPF_MEMSX: + return maybe_load_fp(scev, e, insn); + default: + goto mark_dst_unknown; + } + case BPF_STX: + switch (mode) { + case BPF_MEM: + return maybe_store_fp(scev, e, insn, reg2expr[src]); + case BPF_ATOMIC: + if (insn->imm == BPF_LOAD_ACQ) + return maybe_load_fp(scev, e, insn); + if (insn->imm == BPF_STORE_REL) { + return maybe_store_fp(scev, e, insn, reg2expr[src]); + } + /* + * verifier does not track other atomic ops precisely, + * hence mark the results as opaque. + */ + if (insn->dst_reg == BPF_REG_FP) + mark_opaque(e, insn->off, bpf_size_to_bytes(BPF_SIZE(insn->code))); + if (insn->imm == BPF_CMPXCHG) + return replace_reg(scev, e, BPF_REG_0, OPAQUE_EXPR_ID); + if (insn->imm & BPF_FETCH) + return replace_reg(scev, e, src, OPAQUE_EXPR_ID); + break; + } + break; + case BPF_ST: + if (insn->dst_reg == BPF_REG_FP) { + id = imm_expr(scev, insn->imm); + if (id < 0) + return id; + return maybe_store_fp(scev, e, insn, id); + } + break; + case BPF_JMP: + case BPF_JMP32: + if (opcode == BPF_CALL) + forget_call_regs(e); + /* for non-CALL there are no changes in register states */ + break; + case BPF_LD: + switch (mode) { + case BPF_IMM: + /* rX = imm ll */ + if (BPF_SIZE(insn->code) == BPF_DW && insn->src_reg == 0) { + imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; + return replace_reg(scev, e, dst, imm_expr(scev, imm)); + } + /* map, map value, BTF id, function */ + if (BPF_SIZE(insn->code) == BPF_DW) + return replace_reg(scev, e, dst, OPAQUE_EXPR_ID); + goto mark_dst_unknown; + case BPF_ABS: + case BPF_IND: + forget_call_regs(e); + break; + default: + goto mark_dst_unknown; + } + break; + default: + /* unknown instruction, nuke state */ + for (i = 0; i < REGS_NUM; i++) + reg2expr[i] = UNKNOWN_EXPR_ID; + break; + } + return 0; + +mark_dst_unknown: + reg2expr[dst] = UNKNOWN_EXPR_ID; + return 0; +} + +/* + * Construct minimal 'ANY' expression by traversing 'a' and 'b' + * and accumulating non-duplicated non-ANY entries. + */ +static int mk_any(struct scev *scev, int a, int b) +{ + struct expr_stack_elt *elt; + struct expr *expr; + u32 i, j, ids_buf_sz; + u32 roots[2] = {a, b}; + int id; + + if (a < 0) + return a; + if (b < 0) + return b; + ids_buf_sz = 0; + for (i = 0; i < ARRAY_SIZE(roots); i++) { + scev->stack_sz = 0; + expr_stack_push(scev, roots[i]); + while (scev->stack_sz) { + elt = &scev->expr_stack[--scev->stack_sz]; + id = elt->id; + expr = &scev->exprs[elt->id]; + if (expr->op == ANY) { + if (!expr_stack_push(scev, expr->params[1]) || + !expr_stack_push(scev, expr->params[0])) + return UNKNOWN_EXPR_ID; + } else { + for (j = 0; j < ids_buf_sz; j++) { + if (same_exprs(scev, scev->ids_buf[j], id)) + goto next; + } + if (ids_buf_sz == ARRAY_SIZE(scev->ids_buf)) + return UNKNOWN_EXPR_ID; + scev->ids_buf[ids_buf_sz++] = id; + } +next:; + } + } + if (WARN_ON(ids_buf_sz == 0)) + return -EFAULT; + id = scev->ids_buf[0]; + for (i = 1; i < ids_buf_sz; i++) { + id = expr2(scev, ANY, id, scev->ids_buf[i]); + if (id < 0) + return id; + } + return id; +} + +static int join(struct scev *scev, struct env *acc, struct env *cur) +{ + int i, id; + + if (acc->empty) { + memcpy(acc, cur, sizeof(*acc)); + acc->empty = false; + return 0; + } + + for (i = 0; i < REGS_NUM; i++) { + if (!same_exprs(scev, acc->reg2expr[i], cur->reg2expr[i])) { + id = mk_any(scev, acc->reg2expr[i], cur->reg2expr[i]); + if (id < 0) + return id; + acc->reg2expr[i] = id; + } + } + return 0; +} + +/* Mark any register modified in 'e' as unknown in 'acc'. */ +static void forget_modified_regs(struct scev *scev, struct env *acc, struct env *e) +{ + u32 i, reg; + + for (i = 0; i < REGS_NUM; i++) { + if (e && is_reg(scev, e->reg2expr[i], ®) && reg == i) + continue; + acc->reg2expr[i] = UNKNOWN_EXPR_ID; + } +} + +/* For a loop H at `header`, mark any register modified as a result of H execution as unknown. */ +static void forget_non_invariants(struct bpf_verifier_env *env, struct env *acc, int header) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *loop = aux[header].loop; + struct scev *scev = env->scev; + int i, idx, h; + + acc->empty = false; + forget_modified_regs(scev, acc, find_header_env(scev, header)); + for (i = 0; i < loop->exits_cnt; i++) { + idx = loop->exits[i].from; + h = bpf_loop_at_index(env, idx); + /* + * In order to account for situations like this: + * + * header: + * do { + * saved = rX; + * rX = -1000; + * do { // H + * if (random()) + * goto out; // E + * } while (inner_again); + * rX = saved; + * } while (outer_again); + * out: + * + * For exits originating in nested loops check that rX is invariant + * both at the header (H) and at exit (E). + */ + while (h != header) { + forget_modified_regs(scev, acc, find_header_env(scev, h)); + forget_modified_regs(scev, acc, find_loop_env(scev, h, idx)); + idx = h; + h = aux[h].loop_header; + } + /* + * For exits originating at the same loop the header is already accounted for, + * check only the exit environment. + */ + forget_modified_regs(scev, acc, find_loop_env(scev, header, idx)); + } +} + +/* Order worklist in reverse post-order. */ +static bool reverse_postorder_less(const void *a, const void *b, void *arg) +{ + const int *rank = arg; + + return rank[*(const int *)a] > rank[*(const int *)b]; +} + +static const struct min_heap_callbacks worklist_cb = { + .less = reverse_postorder_less, +}; + +static int worklist_push(struct bpf_verifier_env *env, int loop_header, int idx) +{ + struct scev *scev = env->scev; + struct scev_worklist *worklist = &scev->worklist; + u32 tag = (u32)loop_header + 1; + + if (scev->discovered[idx] == tag) + return 0; + + if (min_heap_full_inline(worklist)) { + size_t size = worklist->size ? worklist->size * 2 : 16; + int *data; + + data = kvrealloc(worklist->data, array_size(size, sizeof(*data)), + GFP_KERNEL_ACCOUNT); + if (!data) + return -ENOMEM; + worklist->data = data; + worklist->size = size; + } + if (!min_heap_push_inline(worklist, &idx, &worklist_cb, env->cfg.postorder_nums)) + return -EFAULT; + + scev->discovered[idx] = tag; + return 0; +} + +static bool reg_alive_at_succ(struct bpf_verifier_env *env, u32 r, u32 insn_idx) +{ + struct bpf_iarray *succ = bpf_insn_successors(env, insn_idx); + u32 i; + + for (i = 0; i < succ->cnt; i++) + if (reg_alive_at(env, r, succ->items[i])) + return true; + return false; +} + +enum changes_at { + LOG_AT_TRANSFER, + LOG_AT_JOIN, +}; + +static void log_env_changes(struct bpf_verifier_env *env, enum changes_at at, + struct env *old, struct env *new, int insn_idx) +{ + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + u64 null_pos = log->end_pos; + u32 old_id, new_id, regno; + bool any_changes = false; + bool is_self_scev; + u64 len; + int r; + + bpf_log(log, "%s %4d: ", at == LOG_AT_TRANSFER ? "t" : "j", insn_idx); + bpf_verbose_insn(env, &env->prog->insnsi[insn_idx]); + len = log->end_pos - null_pos; + bpf_log(log, "%*s", max(37 - (int)len, 1), " "); + bpf_log(log, " ; "); + for (r = 0; r < REGS_NUM; r++) { + old_id = old->reg2expr[r]; + new_id = new->reg2expr[r]; + if (at == LOG_AT_TRANSFER && !reg_alive_at_succ(env, r, insn_idx)) + continue; + if (at == LOG_AT_JOIN && !reg_alive_at(env, r, insn_idx)) + continue; + if (same_exprs(scev, old_id, new_id)) + continue; + is_self_scev = is_reg(scev, new_id, ®no) && r == regno; + if (old->empty && is_self_scev) + continue; + if (any_changes) + bpf_log(log, ", "); + log_reg(env, r); + bpf_log(log, " "); + log_expr(env, old_id); + bpf_log(log, " -> "); + log_expr(env, new_id); + any_changes = true; + } + if (old->empty && !any_changes) + bpf_log(log, "<empty join>"); + bpf_log(log, "\n"); + if (!old->empty && !any_changes) + bpf_vlog_reset(log, null_pos); +} + +static bool is_probe_read_helper(u32 func_id) +{ + return func_id == BPF_FUNC_probe_read || + func_id == BPF_FUNC_probe_read_kernel || + func_id == BPF_FUNC_probe_read_user || + func_id == BPF_FUNC_probe_read_str || + func_id == BPF_FUNC_probe_read_kernel_str || + func_id == BPF_FUNC_probe_read_user_str; +} + +/* + * If instruction is an indirect write to stack, invalidate SCEVs for spi's + * that this instruction can touch. + */ +static void reset_scevs_at_indirect_writes(struct bpf_verifier_env *env, struct env *cur_env, int idx) +{ + struct bpf_insn *insn = &env->prog->insnsi[idx]; + struct scev *scev = env->scev; + u8 class = BPF_CLASS(insn->code); + const unsigned long *mask; + u32 spi, reg; + bool opaque; + + /* Direct fp stores are fine. */ + if ((class == BPF_ST || class == BPF_STX) && insn->dst_reg == BPF_REG_FP) + return; + + mask = bpf_may_write_mask(env, idx); + opaque = bpf_helper_call(insn) && is_probe_read_helper(insn->imm); + for_each_set_bit(spi, mask, MAX_BPF_STACK_SLOTS) { + reg = MAX_BPF_REG + spi; + if (cur_env->reg2expr[reg] == UNKNOWN_EXPR_ID) + continue; + replace_reg(scev, cur_env, reg, opaque ? OPAQUE_EXPR_ID : UNKNOWN_EXPR_ID); + } +} + +/* + * Collect loop-entry registers referenced by expr 'id', best effort. + * DEPTH_LIMIT may leave dependencies unrecorded. This mask only avoids + * widening that would lose stack-access precision; missing dependencies + * may cause false rejections, but this is not a soundness issue. + */ +static void or_expr_regs(struct bpf_verifier_env *env, u32 id, unsigned long *mask) +{ + struct scev *scev = env->scev; + u32 order; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if ((order & PRE) && scev->exprs[id].op == REG) + __set_bit(scev->exprs[id].params[0], mask); + } +} + +/* Mask of argument registers (R1..R5) a call at 'idx' passes by register. */ +static u16 call_params_mask(struct bpf_verifier_env *env, int idx) +{ + struct bpf_insn *insn = &env->prog->insnsi[idx]; + struct bpf_call_summary cs; + int n = bpf_get_call_summary(env, insn, &cs) ? cs.arg_slot_cnt : MAX_BPF_FUNC_REG_ARGS; + + return n ? GENMASK(BPF_REG_1 + n - 1, BPF_REG_1) : 0; +} + +/* + * For instructions like: + * - *(u64 *)(rBase + off) = rX + * - rX = *(u64 *)(rBase + off) + * - calls that construct objects on stack (e.g. dynptr_from_mem(rBase, ...)) + * When 'rBase' can be a stack pointer and is derived from some registers Rs + * defined at loop entry, record Rs into 'mask'. + */ +static void collect_store_base_regs(struct bpf_verifier_env *env, + struct env *cur_env, int idx, unsigned long *mask) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_insn *insn = &env->prog->insnsi[idx]; + u8 class = BPF_CLASS(insn->code); + u8 size = BPF_SIZE(insn->code); + u16 base_regs = 0; + u32 r; + + if (size == BPF_W || size == BPF_DW) { + if ((class == BPF_STX || class == BPF_ST) && insn->dst_reg != BPF_REG_FP) + base_regs |= BIT(insn->dst_reg); + else if (class == BPF_LDX && insn->src_reg != BPF_REG_FP) + base_regs |= BIT(insn->src_reg); + } + + if (class == BPF_JMP && BPF_OP(insn->code) == BPF_CALL && + bpf_needs_fixed_stack_off(env, idx)) + base_regs |= call_params_mask(env, idx); + + base_regs &= aux[idx].stack_ptrs; + for (r = 0; r < MAX_BPF_REG; r++) + if (base_regs & BIT(r)) + or_expr_regs(env, cur_env->reg2expr[r], mask); +} + +/* Find the topmost loop header containing idx inside cur_header, or -1 if none. */ +static int topmost_nested_loop(struct bpf_verifier_env *env, int idx, int cur_header) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + int header; + + for (header = bpf_loop_at_index(env, idx); header >= 0; + header = aux[header].loop_header) + if (aux[header].loop_header == cur_header) + return header; + + return -1; +} + +/* + * Join cur_env into the successor's environment and schedule their traversal. + * Map successors in nested loops to their topmost nested header. + * Ignore successors outside the current loop and its nested loops. + */ +static int join_successor(struct bpf_verifier_env *env, int cur_header, int succ_idx, + struct env *old_env, struct env *cur_env) +{ + bool log_level2 = env->log.level & BPF_LOG_LEVEL2; + struct scev *scev = env->scev; + int err, nested_header; + struct env *succ_env; + + /* + * There are several possibilities for a successor: + * - succ_idx can be a part of a loop outside of the cur_header's loop, + * such edges are ignored. + * - succ_idx can be a part of the same loop as cur_header, + * for such edges succ_idx environment is updated: + * e[succ_idx] = join(e[succ_idx], cur_env) + * - succ_idx can be a part of some loop inner to cur_header, + * in such a case there exists some loop header H, + * such that H.loop_header == cur_header + * and H is the same as succ_idx's loop or contains it. + */ + if (bpf_loop_at_index(env, succ_idx) != cur_header) { + /* + * topmost_nested_loop() either finds H or returns -1, + * in case if succ_idx is a part of a loop outer to cur_header. + */ + nested_header = topmost_nested_loop(env, succ_idx, cur_header); + if (nested_header < 0) + return 0; + succ_idx = nested_header; + } + succ_env = get_loop_env(scev, cur_header, succ_idx); + if (!succ_env) + return -ENOMEM; + if (log_level2) + memcpy(old_env, succ_env, sizeof(*old_env)); + err = join(scev, succ_env, cur_env); + if (err) + return err; + err = worklist_push(env, cur_header, succ_idx); + if (err) + return err; + if (log_level2) + log_env_changes(env, LOG_AT_JOIN, old_env, succ_env, succ_idx); + return 0; +} + +/* Unsupported loops get only an unknown header environment. */ +static bool can_compute_loop_scev(const struct bpf_loop *loop) +{ + return !loop->irreducible && !loop->exits_overflow; +} + +static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header) +{ + struct scev_worklist *worklist = &env->scev->worklist; + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *cur_loop = aux[cur_header].loop; + struct env *header_env; + struct scev *scev = env->scev; + struct bpf_loop *nested_loop; + struct env *cur_env = NULL; + struct env *old_env = NULL; + struct bpf_iarray *succ; + bool log_level2 = env->log.level & BPF_LOG_LEVEL2; + int s, i, err, idx, succ_idx; + u32 r; + + if (log_level2) + bpf_log(&env->log, "Computing SCEV for loop at %d:\n", cur_header); + + /* + * For irreducible loops, and for loops nesting a loop with a truncated + * exit list, just assume that everything is clobbered for now. + */ + if (!can_compute_loop_scev(cur_loop)) { + header_env = get_loop_env(scev, cur_header, cur_header); + if (!header_env) + return -ENOMEM; + /* The freshly allocated environment has all expressions unknown. */ + bitmap_fill(cur_loop->store_base_regs, REGS_NUM); + header_env->empty = false; + return 0; + } + + cur_env = kzalloc(sizeof(*cur_env), GFP_KERNEL_ACCOUNT); + if (!cur_env) + goto nomem; + if (log_level2) { + old_env = kzalloc(sizeof(*old_env), GFP_KERNEL_ACCOUNT); + if (!old_env) + goto nomem; + } + header_env = get_loop_env(scev, cur_header, cur_header); + if (!header_env) + goto nomem; + err = setup_initial_loop_env(env, header_env, cur_header); + if (err) + goto out; + err = worklist_push(env, cur_header, cur_header); + if (err) + goto out; + + while (worklist->nr) { + idx = *min_heap_peek_inline(worklist); + min_heap_pop_inline(worklist, &worklist_cb, env->cfg.postorder_nums); + + /* join_successor() maps nested-loop entries to their representative header. */ + nested_loop = idx != cur_header ? aux[idx].loop : NULL; + memcpy(cur_env, find_loop_env(scev, cur_header, idx), sizeof(*cur_env)); + if (nested_loop) { + /* + * Process nested loop as a single instruction by + * forgetting anything non-invariant in the nested loop + */ + if (log_level2) + memcpy(old_env, cur_env, sizeof(*old_env)); + /* Pull the nested loop's stack-store base dependencies up. */ + for_each_set_bit(r, nested_loop->store_base_regs, BPF_SCEV_REGS_NUM) + or_expr_regs(env, cur_env->reg2expr[r], cur_loop->store_base_regs); + forget_non_invariants(env, cur_env, idx); + if (log_level2) + log_env_changes(env, LOG_AT_TRANSFER, old_env, cur_env, idx); + /* + * Treat nested loop exits as successors, + * join cur_env into successor's envs. + */ + for (i = 0; i < nested_loop->exits_cnt; i++) { + s = nested_loop->exits[i].to; + err = join_successor(env, cur_header, s, old_env, cur_env); + if (err) + goto out; + } + } else { + /* + * Iterate instructions within a single basic block + * starting at 'idx' mutating 'cur_env'. + */ + for (;;) { + if (log_level2) + memcpy(old_env, cur_env, sizeof(*old_env)); + collect_store_base_regs(env, cur_env, idx, cur_loop->store_base_regs); + err = transfer(env, cur_env, idx); + if (err) + goto out; + reset_scevs_at_indirect_writes(env, cur_env, idx); + if (log_level2) + log_env_changes(env, LOG_AT_TRANSFER, old_env, cur_env, idx); + succ = bpf_insn_successors(env, idx); + if (succ->cnt != 1) + break; + succ_idx = succ->items[0]; + if (aux[idx].bb_end || aux[succ_idx].need_scev || + bpf_loop_at_index(env, succ_idx) != cur_header) + break; + idx = succ_idx; + } + /* Join cur_env into basic block successor's envs. */ + iarray_for_each(s, succ) { + err = join_successor(env, cur_header, s, old_env, cur_env); + if (err) + goto out; + } + } + } + + err = 0; +out: + kfree(cur_env); + kfree(old_env); + return err; +nomem: + err = -ENOMEM; + goto out; +} + +static void mark_latches(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *loop; + int len = env->prog->len; + int i, j, k, from; + + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop) + continue; + aux[i].need_scev = true; + if (!can_compute_loop_scev(loop)) + continue; + /* + * Make sure to only set .need_scev for exits and latches + * belonging to the loop at `i`. If can_compute_loop_scev() + * fails for a loop, it should hide it's instructions from + * compute_insn_scevs(). + */ + for (j = 0; j < loop->backedges_cnt; j++) + for (k = 0; k < loop->backedges[j].latches_cnt; k++) + aux[loop->backedges[j].latches[k]].need_scev = true; + for (j = 0; j < loop->exits_cnt; j++) { + from = loop->exits[j].from; + if (bpf_loop_at_index(env, from) == i) + aux[from].need_scev = true; + } + } +} + +static bool is_any_imm_reg_opaque(struct scev *scev, u32 id) +{ + u32 l, r, reg, order; + s64 imm; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if (order & DEPTH_LIMIT) + return false; + if (!(order & PRE) || + is_imm(scev, id, &imm) || + is_reg(scev, id, ®) || + is_opaque(scev, id) || + is_any(scev, id, &l, &r)) + continue; + return false; + } + return true; +} + +/* + * Can implement explicit stack version, but it is harder to read. + * Stick with recursive version for now. + */ +static int transform_expr_once(struct scev *scev, u32 lvl, u32 root, void *priv, + int (*fn)(struct scev *scev, u32 id, void *priv)) +{ + struct expr expr; + int p0, p1, id; + + if (lvl >= EXPR_STACK_DEPTH) + return UNKNOWN_EXPR_ID; + + expr = scev->exprs[root]; /* snapshot the expr before potential realloc */ + switch (op_params_num(expr.op)) { + case 0: + id = root; + break; + case 1: + p0 = transform_expr_once(scev, lvl + 1, expr.params[0], priv, fn); + id = expr1(scev, expr.op, p0); + break; + case 2: + p0 = transform_expr_once(scev, lvl + 1, expr.params[0], priv, fn); + p1 = transform_expr_once(scev, lvl + 1, expr.params[1], priv, fn); + id = expr2(scev, expr.op, p0, p1); + break; + } + return id < 0 ? id : fn(scev, id, priv); +} + +static int transform_expr(struct scev *scev, u32 root, void *priv, + int (*fn)(struct scev *scev, u32 id, void *priv)) +{ + int i = 0, id_old, id_new = root; + + do { + /* + * The depth limit is somewhat arbitrary and is essentially a failsafe. + * oscillation should not really happen. Still, it seems harsh to error out. + */ + if (i++ == 4 * EXPR_STACK_DEPTH) + return UNKNOWN_EXPR_ID; + id_old = id_new; + id_new = transform_expr_once(scev, 0, id_old, priv, fn); + if (id_new < 0) + return id_new; + } while (id_old != id_new); + return id_new; +} + +static int simplify(struct scev *scev, u32 id, void *priv) +{ + int a, b, l, r, t, base, slope; + s64 imm1, imm2; + + /* (+ (linear base slope) imm) -> (linear (+ base imm) slope) */ + if (is_add(scev, id, &l, &r) && + is_linear(scev, l, &base, &slope) && + is_imm(scev, r, &imm1)) + return expr2(scev, LINEAR_SCEV, expr2(scev, BPF_ADD, base, r), slope); + + /* (+ imm imm) -> imm */ + if (is_add(scev, id, &l, &r) && + is_imm(scev, l, &imm1) && + is_imm(scev, r, &imm2)) + return imm_expr(scev, imm1 + imm2); + + /* (zext32 (imm X)) -> (imm (u32)X) */ + if (is_zext32(scev, id, &l) && is_imm(scev, l, &imm1)) + return imm_expr(scev, (u64)(u32)imm1); + + /* + * (zext32 (+ (zext32 $a) $b)) -> (zext32 (+ $a $b)) + * (zext32 (+ $a (zext32 $b))) -> (zext32 (+ $a $b)) + */ + if ((is_zext32(scev, id, &t) && is_add(scev, t, &l, &b) && is_zext32(scev, l, &a)) || + (is_zext32(scev, id, &t) && is_add(scev, t, &a, &r) && is_zext32(scev, r, &b))) { + t = expr2(scev, BPF_ADD, a, b); + t = expr1(scev, ZEXT32, t); + return t; + } + + return id; +} + +static int compute_header_scevs(struct bpf_verifier_env *env, struct env *header_env) +{ + struct scev *scev = env->scev; + int l, r, t, id, ra, rb, ra_expr; + s64 imm; + + /* + * Note: + * reg2expr[ra] describes a change in rA's value over a single loop iteration. + * It does not account for rA's entry value. However, the SCEV constructed for rA + * must include an additional rA entry edge. For example: + * + * r0 = ..., r1 = 7 + * l: if r0 == 10 goto e // reg2expr[r1] == (imm 42) => reg2scev[r1] == (any r1 42) + * r0 += 1 + * r1 = 42 + * goto l + * e: + * ... at exit r1 is in {7,42} ... + */ + for (ra = 0; ra < REGS_NUM; ra++) { + ra_expr = transform_expr(scev, header_env->reg2expr[ra], NULL, simplify); + if (ra_expr < 0) + return ra_expr; + header_env->reg2expr[ra] = ra_expr; + /* rA = (+ rA IMM) -> (linear rA IMM) */ + if (is_add(scev, ra_expr, &l, &r) && + is_reg(scev, l, &rb) && + is_imm(scev, r, &imm) && + ra == rb) { + id = expr2(scev, LINEAR_SCEV, l, r); + if (id < 0) + return id; + header_env->reg2scev[ra] = id; + continue; + } + /* rA = (is_zext32 (+ rA IMM)) -> (is_zext32 (linear rA IMM)) */ + if (is_zext32(scev, ra_expr, &t) && + is_add(scev, t, &l, &r) && + is_reg(scev, l, &rb) && + is_imm(scev, r, &imm) && + ra == rb) { + id = expr2(scev, LINEAR_SCEV, l, r); + id = expr1(scev, ZEXT32, id); + if (id < 0) + return id; + header_env->reg2scev[ra] = id; + continue; + } + /* rA = rA -> rA*/ + if (is_reg(scev, ra_expr, &rb) && ra == rb) { + header_env->reg2scev[ra] = ra_expr; + continue; + } + /* rA = (any 1 2 3 4 ...) -> (any rA 1 2 3 4 ...) */ + if (is_any_imm_reg_opaque(scev, ra_expr)) { + id = mk_any(scev, expr1(scev, REG, ra), ra_expr); + if (id < 0) + return id; + header_env->reg2scev[ra] = id; + continue; + } + + } + return 0; +} +static int instantiate_header_scevs(struct scev *scev, u32 id, void *priv) +{ + u32 reg, *reg2scev = priv; + + /* (reg r) -> ... equation at header ... */ + if (is_reg(scev, id, ®)) + return reg2scev[reg]; + /* keep expression shapes */ + return id; +} + +static int compute_insn_scevs(struct bpf_verifier_env *env, struct env *eheader, struct env *einsn) +{ + struct scev *scev = env->scev; + int id, reg; + + for (reg = 0; reg < REGS_NUM; reg++) { + id = einsn->reg2expr[reg]; + id = transform_expr_once(scev, 0, id, eheader->reg2scev, instantiate_header_scevs); + if (id < 0) + return id; + id = transform_expr(scev, id, NULL, simplify); + if (id < 0) + return id; + einsn->reg2scev[reg] = id; + } + + return 0; +} + +static int log_scevs(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_verifier_log *log = &env->log; + struct env *header_env, *latch_env; + struct scev *scev = env->scev; + struct bpf_loop *loop; + int i, j, k, len, latch; + + len = env->prog->len; + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop) + continue; + header_env = find_header_env(scev, i); + bpf_log(log, "scev at header %d:\n", i); + print_env(env, header_env, i); + if (!can_compute_loop_scev(loop)) + continue; + for (j = 0; j < loop->backedges_cnt; j++) { + for (k = 0; k < loop->backedges[j].latches_cnt; k++) { + latch = loop->backedges[j].latches[k]; + bpf_log(log, " scev at %s %d:\n", k ? "exit" : "latch", latch); + latch_env = find_loop_env(scev, i, latch); + if (verifier_bug_if(!latch_env, env, "i=%d, latch=%d", i, latch)) + return -EFAULT; + print_env(env, latch_env, latch); + } + } + } + return 0; +} + +int bpf_compute_scev(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct scev *scev = env->scev; + int *postorder = env->cfg.insn_postorder; + int cnt = env->cfg.cur_postorder; + int i, idx, err, header; + + mark_latches(env); + /* + * Visit loop headers in postorder, to guarantee that scevs + * for innermost loops are computed first. + */ + for (i = 0; i < cnt; i++) { + idx = postorder[i]; + if (!aux[idx].loop) + continue; + err = compute_scev_for_loop(env, idx); + if (err) + return err; + } + + /* + * Compute scevs from exprs collected on a previous step. Iterate instructions in + * reverse post-order so that each loop header is processed before instructions + * reachable from it. + */ + for (i = cnt - 1; i >= 0; i--) { + idx = postorder[i]; + if (!aux[idx].need_scev) + continue; + header = bpf_loop_at_index(env, idx); + err = aux[idx].loop + ? compute_header_scevs(env, find_header_env(scev, header)) + : compute_insn_scevs(env, + find_header_env(scev, header), + find_loop_env(scev, header, idx)); + if (err) + return err; + } + + for (i = 0; i < env->prog->len; i++) + if (aux[i].loop) + aux[i].prune_point = true; + + if (env->log.level & BPF_LOG_LEVEL2) { + err = log_scevs(env); + if (err) + return err; + } + + return 0; +} + +void bpf_free_scev(struct bpf_verifier_env *env) +{ + struct scev *scev = env->scev; + struct insn_envs *envs; + int i; + u32 j; + + if (!scev) + return; + for (i = 0; i < scev->envs_cnt; i++) { + envs = scev->envs[i]; + if (!envs) + continue; + for (j = 0; j < envs->cnt; j++) + kfree(envs->entries[j].env); + kfree(envs); + } + if (scev->exprs_ht) + for (i = 0; i < scev->exprs_ht_cnt; i++) + kvfree(scev->exprs_ht[i]); + kvfree(scev->exprs_ht); + kvfree(scev->worklist.data); + kvfree(scev->envs); + kvfree(scev->exprs); + kvfree(scev->discovered); + kfree(scev); + env->scev = NULL; +} + +int bpf_init_scev(struct bpf_verifier_env *env) +{ + struct scev *scev; + + scev = kzalloc(sizeof(struct scev), GFP_KERNEL_ACCOUNT); + if (!scev) + return -ENOMEM; + env->scev = scev; + bpf_mark_reg_known_scalar(&scev->zero_reg, 0); + bpf_mark_reg_unknown_imprecise(&scev->unknown_reg); + /* A power of two for hash & (exprs_ht_cnt - 1)` indexing. */ + scev->exprs_ht_cnt = roundup_pow_of_two(max(256U, DIV_ROUND_UP(env->prog->len, 4))); + scev->exprs_ht = kvcalloc(scev->exprs_ht_cnt, sizeof(*scev->exprs_ht), GFP_KERNEL_ACCOUNT); + if (!scev->exprs_ht) + goto nomem; + if (expr0(scev, UNKNOWN) < 0 || expr0(scev, OPAQUE) < 0) + goto nomem; + scev->envs = kvcalloc(env->prog->len, sizeof(*scev->envs), GFP_KERNEL_ACCOUNT); + scev->discovered = kvcalloc(env->prog->len, sizeof(*scev->discovered), GFP_KERNEL_ACCOUNT); + if (!scev->envs || !scev->discovered) + goto nomem; + /* + * Remember original program length, in case bpf_free_scev() + * is called after bpf program rewrites that increase program + * length. + */ + scev->envs_cnt = env->prog->len; + return 0; +nomem: + bpf_free_scev(env); + return -ENOMEM; +} + +static struct bpf_stack_state *scev_regno_to_stack(const struct bpf_func_state *st, u32 r) +{ + int spi, slots_available; + + if (r < MAX_BPF_REG) + return NULL; + + slots_available = st->allocated_stack / BPF_REG_SIZE; + spi = r - MAX_BPF_REG; + if (spi < slots_available) + return &st->stack[spi]; + + return NULL; +} + +static struct bpf_reg_state *scev_regno_to_reg(struct bpf_func_state *st, u32 r) +{ + return r < MAX_BPF_REG ? &st->regs[r] : NULL; +} + +/* Linear widening requires a physical register or a full spill. */ +static struct bpf_reg_state *scev_reg_for_linear(struct bpf_func_state *st, u32 r) +{ + struct bpf_stack_state *stack; + + if (r < MAX_BPF_REG) + return scev_regno_to_reg(st, r); + stack = scev_regno_to_stack(st, r); + if (!stack || bpf_spill_size(stack) != BPF_REG_SIZE) + return NULL; + return &stack->spilled_ptr; +} + +/* + * Return a borrowed value; never use it as a widening destination. + * NULL denotes no readable value. + */ +static const struct bpf_reg_state * +scev_reg_for_read(struct bpf_verifier_env *env, const struct bpf_func_state *st, u32 r) +{ + const struct bpf_stack_state *stack; + bool zero = true; + int i; + + if (r < MAX_BPF_REG) + return st->regs[r].type == NOT_INIT ? NULL : &st->regs[r]; + if (r >= REGS_NUM || r - MAX_BPF_REG >= env->stack_limit / BPF_REG_SIZE) + return NULL; + + stack = scev_regno_to_stack(st, r); + if (!stack) + return &env->scev->unknown_reg; + if (bpf_spill_size(stack) == BPF_REG_SIZE) + return &stack->spilled_ptr; + + /* Partial spills do not describe the value of the whole slot. */ + for (i = 0; i < BPF_REG_SIZE; i++) { + switch (stack->slot_type[i]) { + case STACK_ZERO: + break; + case STACK_INVALID: + case STACK_POISON: + case STACK_MISC: + zero = false; + break; + case STACK_SPILL: + if (!bpf_is_spilled_scalar_reg(stack)) + return NULL; + zero = false; + break; + default: + return NULL; + } + } + return zero ? &env->scev->zero_reg : &env->scev->unknown_reg; +} + +/* Materialize a writable value in the current state, growing its stack if needed. */ +static int scev_reg_for_write(struct bpf_verifier_env *env, struct bpf_func_state *st, + u32 r, struct bpf_reg_state **out) +{ + const struct bpf_reg_state *value; + struct bpf_stack_state *stack; + u32 spi; + int err; + + *out = scev_regno_to_reg(st, r); + if (*out) + return 0; + if (r >= REGS_NUM || r - MAX_BPF_REG >= env->stack_limit / BPF_REG_SIZE) + return -EINVAL; + + spi = r - MAX_BPF_REG; + err = bpf_grow_stack_state(env, st, (spi + 1) * BPF_REG_SIZE); + if (err) + return err; + + /* Resolve pointers after growth, which may relocate the stack. */ + value = scev_reg_for_read(env, st, r); + if (!value) + return -EINVAL; + stack = scev_regno_to_stack(st, r); + stack->spilled_ptr = *value; + memset(stack->slot_type, STACK_SPILL, sizeof(stack->slot_type)); + *out = &stack->spilled_ptr; + return 0; +} + +static bool scev_reg_alive(struct bpf_verifier_env *env, struct bpf_verifier_state *st, u32 r) +{ + int insn_idx = bpf_frame_insn_idx(st, st->curframe); + u16 live_regs = env->insn_aux_data[insn_idx].live_regs_before; + int spi; + + if (r < MAX_BPF_REG) { + return BIT(r) & live_regs; + } else { + spi = r - MAX_BPF_REG; + return bpf_stack_slot_alive(env, st->curframe, spi * 2) || + bpf_stack_slot_alive(env, st->curframe, spi * 2 + 1); + } +} + +/* + * Latch is a condition deciding if execution remains inside a loop. + * Linear latch represents a condition 'if <reg> <op> <loop invariant> goto <loop-header>', + * where equation '<base> + i * <step> <op> <bound>' describes values taken by register <reg>, + * 'i' is the loop iteration number, starting from 0. + * 'zext32' indicates the latch is for (zext32 (linear ...)) SCEV. + * 'jmp32' indicates that the latch is a JMP32 instruction. + */ +struct linear_latch { + u32 insn_idx; + u32 base_expr; + u32 step_expr; + u32 bound_expr; + u8 op; + bool zext32; + bool jmp32; +}; + +static bool loop_invariant(struct scev *scev, u32 id) +{ + s64 imm; + u32 reg; + + return is_reg(scev, id, ®) || is_imm(scev, id, &imm); +} + +static int match_linear_latch(struct bpf_verifier_env *env, + u32 header, + u32 latch_idx, + struct linear_latch *latch) +{ + struct bpf_insn *insn = &env->prog->insnsi[latch_idx]; + struct scev *scev = env->scev; + struct env *latch_env; + u32 true_branch_loop; + u32 true_branch_tgt; + u32 src_reg_scev; + u32 dst_reg_scev; + u32 t, l, r, op; + int id; + + op = BPF_OP(insn->code); + latch->jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; + /* Flip the condition if true branch jumps out of the loop. */ + true_branch_tgt = latch_idx + bpf_jmp_offset(insn) + 1; + true_branch_loop = bpf_loop_at_index(env, true_branch_tgt); + if (true_branch_loop != header && + !bpf_is_nested_loop(env, true_branch_loop, header)) + op = bpf_rev_opcode(op); + switch (op) { + case BPF_JSLT: + case BPF_JSLE: + case BPF_JSGT: + case BPF_JSGE: + case BPF_JLT: + case BPF_JLE: + case BPF_JGT: + case BPF_JGE: + case BPF_JNE: + break; + default: + return false; + } + + latch->insn_idx = latch_idx; + + latch_env = find_loop_env(scev, header, latch_idx); + if (verifier_bug_if(!latch_env, env, "header_idx=%d, latch_idx=%d\n", header, latch_idx)) + return -EFAULT; + dst_reg_scev = latch_env->reg2scev[insn->dst_reg]; + src_reg_scev = latch_env->reg2scev[insn->src_reg]; + /* + * Make sure (linear ...) is in dst_reg_scev: + * 'if <invariant> <op> <linear>' ⇔ + * 'if <linear> <flip(op)> <invariant>'. + */ + if (BPF_SRC(insn->code) == BPF_X && + is_linear_or_linear32(scev, src_reg_scev, &l, &r, &latch->zext32)) { + swap(dst_reg_scev, src_reg_scev); + op = bpf_flip_opcode(op); + } + latch->op = op; + /* + * simplify() produces two shapes for linear latches: + * - (linear register step) + * ^----------------------------+-- base_expr + * base_expr | + * - (linear (+ register imm) step) | + * ^----------------------------' + */ + if (!is_linear_or_linear32(scev, dst_reg_scev, + &latch->base_expr, &latch->step_expr, &latch->zext32)) + return false; + + if (BPF_SRC(insn->code) == BPF_K) { + id = imm_expr(scev, insn->imm); + if (id < 0) + return id; + latch->bound_expr = id; + } else { + latch->bound_expr = src_reg_scev; + } + + if (!loop_invariant(scev, latch->step_expr) || + !loop_invariant(scev, latch->bound_expr)) + return false; + + /* (linear register step) */ + if (is_reg(scev, latch->base_expr, &t)) + return true; + + /* (linear (+ register imm) step) */ + if (is_add(scev, latch->base_expr, &l, &r) && + is_reg(scev, l, &t) && + loop_invariant(scev, r)) + return true; + + return false; +} + +struct scev_value { + u64 value; /* scalar value or a pointer offset */ + int ptr_reg; /* SCEV register identifying the pointer origin, or -1. */ +}; + +static void log_scev_value(struct bpf_verifier_env *env, struct bpf_func_state *st, + const struct scev_value *v) +{ + const struct bpf_reg_state *reg; + + if (v->ptr_reg >= 0) { + reg = scev_reg_for_read(env, st, v->ptr_reg); + log_reg(env, v->ptr_reg); + bpf_log(&env->log, " (%s)", reg ? reg_type_str(env, reg->type) : "unavailable"); + } else { + bpf_log(&env->log, "scalar %llu", v->value); + } +} + +static void log_incompatible_scev_values(struct bpf_verifier_env *env, const char *reason, + struct bpf_func_state *st, + const struct scev_value *a, const struct scev_value *b) +{ + bpf_log(&env->log, "scev: %s ", reason); + log_scev_value(env, st, a); + bpf_log(&env->log, ", "); + log_scev_value(env, st, b); + bpf_log(&env->log, "\n"); +} + +static bool scev_value_compatible(struct bpf_verifier_env *env, struct bpf_func_state *st, + const struct scev_value *a, const struct scev_value *b) +{ + const struct bpf_reg_state *ra, *rb; + + if (a->ptr_reg < 0 || b->ptr_reg < 0) { + if (a->ptr_reg < 0 && b->ptr_reg < 0) + return true; + goto incompatible; + } + ra = scev_reg_for_read(env, st, a->ptr_reg); + rb = scev_reg_for_read(env, st, b->ptr_reg); + if (ra && rb && bpf_same_memory_origin(ra, rb)) + return true; + +incompatible: + if (env->log.level & BPF_LOG_LEVEL2) + log_incompatible_scev_values(env, "incompatible latch operands", st, a, b); + return false; +} + +static bool scev_value_add(struct bpf_verifier_env *env, struct bpf_func_state *st, + const struct scev_value *a, const struct scev_value *b, + struct scev_value *result) +{ + if (a->ptr_reg >= 0 && b->ptr_reg >= 0) { + if (env->log.level & BPF_LOG_LEVEL2) + log_incompatible_scev_values(env, "can't add two pointers", st, a, b); + return false; + } + result->value = a->value + b->value; + result->ptr_reg = a->ptr_reg >= 0 ? a->ptr_reg : b->ptr_reg; + return true; +} + +static bool eval_expr(struct bpf_verifier_env *env, struct scev *scev, + struct bpf_func_state *st, u32 id, struct scev_value *result) +{ + struct scev_value lval, rval; + const struct bpf_reg_state *reg; + u32 l, r, regno; + s64 imm; + + if (is_reg(scev, id, ®no)) { + reg = scev_reg_for_read(env, st, regno); + if (!reg || reg->type == NOT_INIT || !tnum_is_const(reg->var_off)) + return false; + result->value = reg->var_off.value; + result->ptr_reg = reg->type == SCALAR_VALUE ? -1 : (int)regno; + return true; + } else if (is_imm(scev, id, &imm)) { + result->value = imm; + result->ptr_reg = -1; + return true; + } else if (is_add(scev, id, &l, &r) && + eval_expr(env, scev, st, l, &lval) && + eval_expr(env, scev, st, r, &rval)) { + return scev_value_add(env, st, &lval, &rval, result); + } + return false; +} + +static bool is_signed_op(u8 op) +{ + switch (op) { + case BPF_JSLT: + case BPF_JSLE: + case BPF_JSGT: + case BPF_JSGE: + return true; + default: + return false; + } +} + +static bool compute_backedge_taken_count(u8 op, u64 initial, u64 step, u64 bound, u64 *result) +{ + u64 rem, diff, backedge_taken_count; + bool up, down; + + if (step == 0) + return false; + down = (s64)step < 0; + up = (s64)step > 0; + diff = bound - initial; + switch (op) { + case BPF_JLT: + /* + * E.g.: 3 + 2*i < 7 ⇔ 2*i < 4 ⇔ i ∈ [0,1]. + * Here the inequality describes a set of values `i` might take + * while the latch condition remains true (the backedge is taken). + * The backedge_taken_count is the size of this set. + */ + if (/* + * If the condition is initially true, decreasing values won't + * change that without underflow. If the condition is initially false, + * the main pass would do a single iteration anyway. + */ + down || + /* Is the latch condition false on the first iteration? */ + initial >= bound || + /* Count iterations satisfying the strict inequality, including i=0. */ + check_add_overflow(div64_u64(diff - 1, step), 1, &backedge_taken_count) || + /* + * Check if the last iteration overflows the counter: + * initial + backedge_taken_count * step > U64_MAX + */ + backedge_taken_count > div64_u64(U64_MAX - initial, step)) + return false; + break; + case BPF_JLE: + /* E.g.: 3 + 2*i ≤ 7 ⇔ 2*i ≤ 4 ⇔ i ∈ [0,1,2]. */ + if (down || initial > bound || + /* Count iterations satisfying the non-strict inequality. */ + check_add_overflow(div64_u64(diff, step), 1, &backedge_taken_count) || + backedge_taken_count > div64_u64(U64_MAX - initial, step)) + return false; + break; + case BPF_JGT: + /* E.g.: 7 - 2*i > 3 ⇔ 2*i < 4 ⇔ i ∈ [0,1]. */ + if (up || initial <= bound || + check_add_overflow(div64_u64(-diff - 1, -step), 1, &backedge_taken_count) || + /* + * The last iteration underflows the counter if: + * initial + backedge_taken_count * step < 0 ⇔ + * backedge_taken_count * step < -initial ⇔ + * backedge_taken_count > initial / -step + * (direction flips because step < 0) + */ + backedge_taken_count > div64_u64(initial, -step)) + return false; + break; + case BPF_JGE: + /* E.g.: 7 - 2*i ≥ 3 ⇔ 2*i ≤ 4 ⇔ i ∈ [0,1,2]. */ + if (up || initial < bound || + check_add_overflow(div64_u64(-diff, -step), 1, &backedge_taken_count) || + backedge_taken_count > div64_u64(initial, -step)) + return false; + break; + case BPF_JSLT: + if (down || (s64)initial >= (s64)bound || + check_add_overflow(div64_u64(diff - 1, step), 1, &backedge_taken_count) || + /* + * Check if the last iteration overflows the signed counter: + * (s64)initial + backedge_taken_count * step > S64_MAX + */ + backedge_taken_count > div64_u64((u64)S64_MAX - initial, step)) + return false; + break; + case BPF_JSLE: + if (down || (s64)initial > (s64)bound || + check_add_overflow(div64_u64(diff, step), 1, &backedge_taken_count) || + backedge_taken_count > div64_u64((u64)S64_MAX - initial, step)) + return false; + break; + case BPF_JSGT: + if (up || (s64)initial <= (s64)bound || + check_add_overflow(div64_u64(-diff - 1, -step), 1, &backedge_taken_count) || + /* + * The last iteration underflows the signed counter if: + * (s64)initial + backedge_taken_count * step < S64_MIN ⇔ + * backedge_taken_count * step < S64_MIN - (s64)initial ⇔ + * backedge_taken_count > initial - (u64)S64_MIN / -step + * (direction flips because step < 0) + */ + backedge_taken_count > div64_u64(initial - (u64)S64_MIN, -step)) + return false; + break; + case BPF_JSGE: + if (up || (s64)initial < (s64)bound || + check_add_overflow(div64_u64(-diff, -step), 1, &backedge_taken_count) || + backedge_taken_count > div64_u64(initial - (u64)S64_MIN, -step)) + return false; + break; + case BPF_JNE: + /* + * For a decreasing counter flip the diff and step: + * 7 - 2*i ≠3 ⇔ 2*i ≠7 - 3. + */ + diff = down ? -diff : diff; + step = down ? -step : step; + backedge_taken_count = div64_u64_rem(diff, step, &rem); + if (diff == 0 || rem != 0) + return false; + break; + default: + return false; + } + + *result = backedge_taken_count; + return true; +} + +/* + * Loop with post-condition: + * + * r0 = 0 + * l: ... r0 ∈ [0,1,2] header executed 3 times + * r0 += 1 r0 ∈ [0,1,2] + * ... r0 ∈ [1,2,3] + * if r0 != 3 goto l r0 ∈ [1,2,3] backedge taken 2 times + * ... r0 ∈ [3] + * + * SCEV at header: r0 = k + * SCEV at latch: r0 = 1 + k + * + * Loop with pre-condition: + * + * r0 = 0 + * l: ... r0 ∈ [0,1,2,3] header executed 4 times + * if r0 == 3 goto e r0 ∈ [0,1,2,3] backedge taken 3 times + * r0 += 1 r0 ∈ [0,1,2] + * ... r0 ∈ [1,2,3] + * goto l r0 ∈ [1,2,3] + * e: ... r0 ∈ [3] + * + * SCEV at header: r0 = k + * SCEV at latch: r0 = k + */ +static bool compute_max_iters(struct bpf_verifier_env *env, + struct bpf_func_state *st, + struct linear_latch *latch, + struct bpf_loop_iters *iters) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *loop = aux[bpf_loop_at_index(env, latch->insn_idx)].loop; + struct scev *scev = env->scev; + struct scev_value initial_val, step_val, bound_val; + u64 step, bound, initial, total_change, backedge_taken_count; + u8 op = latch->op; + + if (!eval_expr(env, scev, st, latch->base_expr, &initial_val) || + !eval_expr(env, scev, st, latch->step_expr, &step_val) || + !eval_expr(env, scev, st, latch->bound_expr, &bound_val)) + return false; + + if (step_val.ptr_reg >= 0) { + if (env->log.level & BPF_LOG_LEVEL2) { + bpf_log(&env->log, "scev: latch step is a pointer "); + log_scev_value(env, st, &step_val); + bpf_log(&env->log, "\n"); + } + return false; + } + if (!scev_value_compatible(env, st, &initial_val, &bound_val)) + return false; + + initial = initial_val.value; + step = step_val.value; + bound = bound_val.value; + + /* + * The code below only handles mixed 64-bit/32-bit comparisons/recurrences + * when initial bounds and step fit in 32-bit domain. + */ + if (latch->zext32 || latch->jmp32) { + /* + * With the above constraint doing `l: w0 += 1; if r0 s< 256 goto l;` + * is equivalent to `l: w0 += 1; if r0 < 256 goto l;` + */ + if (latch->jmp32 && is_signed_op(op)) { + initial = (s64)(s32)(u32)initial; + bound = (s64)(s32)(u32)bound; + step = (s64)(s32)(u32)step; + } else { + initial = (u32)initial; + bound = (u32)bound; + step = (s64)(s32)(u32)step; + } + } + if (latch->zext32 ^ latch->jmp32 && (initial != initial_val.value || + bound != bound_val.value || + step != step_val.value)) { + if (env->log.level & BPF_LOG_LEVEL2) + bpf_log(&env->log, + "scev: mixed-width latch has operands outside 32-bit ranges\n"); + return false; + } + + + if (!compute_backedge_taken_count(op, initial, step, bound, &backedge_taken_count)) + return false; + + if (latch->zext32 || latch->jmp32) { + /* + * compute_backedge_taken_count() guarantees that 'total_change' + * does not overflow in 64-bit domain. + */ + total_change = initial + step * backedge_taken_count; + if (latch->jmp32 && is_signed_op(op)) { + if ((s64)total_change < S32_MIN || (s64)total_change > S32_MAX) + return false; + } else { + if (total_change > U32_MAX) + return false; + } + } + + /* + * max_header_count is u32 and U32_MAX means "infinite", + * check to avoid overflow below. + */ + if (backedge_taken_count >= U32_MAX - 1) + return false; + + /* The header executes once on entry and once for each taken backedge, hence +1. */ + iters->max_header_count = backedge_taken_count + 1; + /* + * The latch is a conditional jump with one jump target exiting the loop. + * Linear latch is matched only if the loop has a single backedge. + * The loop still, however can have multiple exits. + * In such case, conservatively assume that non-latch exit can happen + * at any iteration, thus setting minimal number of iterations as 0. + */ + iters->min_header_count = loop->exits_cnt == 1 ? iters->max_header_count : 0; + return true; +} + +static void mark_scev_reg_scratched(struct bpf_verifier_env *env, u32 r) +{ + if (r < MAX_BPF_REG) + mark_reg_scratched(env, r); + else + mark_stack_slot_scratched(env, r - __MAX_BPF_REG); +} + +/* + * An unknown scalar is its own widening. + * Stack bytes without a value are represented by scev->unknown_reg. + */ +static bool is_unbound_scev_reg(struct bpf_verifier_env *env, const struct bpf_func_state *st, + u32 r) +{ + const struct bpf_reg_state *reg = scev_reg_for_read(env, st, r); + + return reg && reg->type == SCALAR_VALUE && !reg->id && reg->step <= 1 && + tnum_is_unknown(reg->var_off) && + reg->r64.size == U64_MAX && reg->r32.size == U32_MAX; +} + +/* Main logic in verifier.c forbids varying offsets for certain register types. */ +static bool is_widenable_reg_type(const struct bpf_reg_state *reg) +{ + if (type_may_be_null(reg->type)) + return false; + + switch (base_type(reg->type)) { + case SCALAR_VALUE: + case PTR_TO_MAP_VALUE: + case PTR_TO_MAP_KEY: + case PTR_TO_STACK: + case PTR_TO_PACKET: + case PTR_TO_PACKET_META: + case PTR_TO_MEM: + case PTR_TO_BUF: + case PTR_TO_BTF_ID: + return true; + default: + return false; + } +} + +static void scratch_widened_reg_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) +{ + switch (base_type(reg->type)) { + case SCALAR_VALUE: + reg->id = 0; + break; + case PTR_TO_PACKET: + case PTR_TO_PACKET_META: + reg->id = ++env->id_gen; + break; + default: + break; + } +} + +/* Check that ANY leaves can be unioned with r's loop-entry value. */ +static bool is_widenable_any(struct bpf_verifier_env *env, struct bpf_func_state *loop_entry, + struct env *header_env, u32 r, u32 id) +{ + const struct bpf_reg_state *reg, *leaf_reg; + struct scev *scev = env->scev; + u32 l, ra, rr, leaf, order; + s64 imm; + + if (!is_any(scev, id, &l, &rr)) + return false; + reg = scev_reg_for_read(env, loop_entry, r); + if (!reg || !is_widenable_reg_type(reg)) + return false; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if (order & DEPTH_LIMIT) + return false; + if (!(order & PRE) || is_any(scev, id, &l, &rr)) + continue; + if (is_imm(scev, id, &imm)) { + if (reg->type != SCALAR_VALUE) + return false; + } else if (is_reg(scev, id, &leaf)) { + /* Other leaves must denote loop-invariant registers. */ + if (leaf != r && + !(is_reg(scev, header_env->reg2scev[leaf], &ra) && ra == leaf)) + return false; + leaf_reg = scev_reg_for_read(env, loop_entry, leaf); + if (!leaf_reg || !bpf_reg_union_compatible(reg, leaf_reg)) + return false; + } else { + return false; + } + } + return true; +} + +struct bounds { + struct cnum64 r64; + struct cnum32 r32; + u16 base; + u16 step; +}; + +static void log_bounds(struct bpf_verifier_env *env, const struct bounds *bounds) +{ + struct bpf_verifier_log *log = &env->log; + + bpf_verbose_snum(env, cnum64_smin(bounds->r64)); + bpf_log(log, ".."); + bpf_verbose_snum(env, cnum64_smax(bounds->r64)); + if (bounds->r32.size < U32_MAX) { + bpf_log(log, " (s32 "); + bpf_verbose_snum(env, cnum32_smin(bounds->r32)); + bpf_log(log, ".."); + bpf_verbose_snum(env, cnum32_smax(bounds->r32)); + bpf_log(log, ")"); + } + bpf_log(log, " step %u", bounds->step); +} + +struct linear_coefs { + u32 base_reg; + s64 slope; + bool zext32; +}; + +static bool is_simple_linear(struct scev *scev, u32 id, struct linear_coefs *out) +{ + u32 base, slope; + + return is_linear_or_linear32(scev, id, &base, &slope, &out->zext32) && + is_reg(scev, base, &out->base_reg) && + is_imm(scev, slope, &out->slope) && + out->slope <= S16_MAX && + out->slope >= S16_MIN && + out->slope != 0; +} + +static struct cnum64 cnum64_zext32(struct cnum32 r32) +{ + return cnum64_from_urange(cnum32_umin(r32), cnum32_umax(r32)); +} + +/* Compute the range an induction variable in `reg` spans over the loop. */ +static void linear_bounds(struct bpf_reg_state *reg, u32 max_header_count, + struct linear_coefs *coefs, struct bounds *out) +{ + s64 slope = coefs->slope; + s64 slope_abs = slope < 0 ? -slope : slope; + s64 min_val = reg_smin(reg); + s64 max_val = reg_smax(reg); + struct cnum64 r64; + struct cnum32 r32; + s64 total_change; + bool overflow; + + overflow = check_mul_overflow(slope, (s64)max_header_count - 1, &total_change) || + (slope > 0 && check_add_overflow(max_val, total_change, &max_val)) || + (slope < 0 && check_add_overflow(min_val, total_change, &min_val)); + r64 = overflow ? CNUM64_UNBOUNDED : cnum64_from_srange(min_val, max_val); + r32 = CNUM32_UNBOUNDED; + if (coefs->zext32) { + r32 = cnum32_from_cnum64(r64); + r64 = cnum64_zext32(r32); + /* + * 32-bit iteration overflow is equivalent to subtracting N * 2**32: + * (base + slope * k) % slope ≠(base + slope * k - N * 2**32) % slope. + */ + overflow |= cnum32_contains(r32, 0) && cnum32_contains(r32, U32_MAX); + /* + * The entry and zero-extended values must agree on how they define `base`, + * otherwise `base + slope * k` would define different sets of points between + * the two. + */ + overflow |= imod(reg_smin(reg), slope_abs) != + imod(cnum64_smin(r64), slope_abs); + /* + * Full range is a union of a reg's value at loop entry, + * plus a range produced by 32-bit iteration. + */ + r64 = cnum64_union(reg->r64, r64); + r32 = cnum32_union(reg->r32, r32); + } + /* + * If the entry value is a single point the value set is 'v + slope * k', + * so the step is |slope|. Otherwise, only the power-of-two alignment + * shared by the entry value and the slope. + */ + if (cnum64_is_const(reg->r64) && !overflow) { + out->step = slope_abs; + out->base = imod(reg_smin(reg), out->step); + } else { + out->step = 1u << min_t(u32, tnum_alignment(reg->var_off), __ffs(slope_abs)); + out->base = 0; + } + out->r64 = r64; + out->r32 = r32; +} + +static int mark_expr_precise(struct bpf_verifier_env *env, struct bpf_func_state *st, u32 id, + unsigned long *precise_regs) +{ + struct scev *scev = env->scev; + struct bpf_reg_state *reg; + u32 r, order; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if (order & DEPTH_LIMIT) { + verifier_bug(env, "scev precision marking exceeds expression depth limit"); + return -EFAULT; + } + if (!(order & PRE) || !is_reg(scev, id, &r)) + continue; + /* Synthetic values have no spilled register to mark precise. */ + reg = scev_reg_for_linear(st, r); + if (reg && reg->type == SCALAR_VALUE) + __set_bit(r, precise_regs); + } + return 0; +} + +int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_state *st, + struct bpf_loop_iters *iters) +{ + struct bpf_func_state *cur_func = st->frame[st->curframe]; + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + struct env *header_env; + struct linear_latch latch, best; + struct linear_coefs coefs; + struct bpf_backedge *backedge; + struct bpf_loop_iters tmp; + struct bpf_loop *loop; + int insn_idx = st->insn_idx; + bool found = false; + int linear_latch; + int latch_idx; + int err, j; + u32 r; + + if (!env->allow_uninit_stack) + return 0; + + /* + * If insn_idx is a loop header for a reducible loop with a single backedge. + * loop is NULL for secondary entries to irreducible loops. + */ + loop = aux[insn_idx].loop; + if (!loop || loop->irreducible || loop->backedges_cnt != 1 || loop->backedges_overflow || + loop->exits_overflow) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, unsupported loop:%s%s%s\n", insn_idx, + !loop || loop->irreducible ? " irreducible" : "", + loop && loop->backedges_cnt > 1 ? " multiple backedges" : "", + loop && loop->exits_overflow ? " too many exits" : ""); + return 0; + } + + /* If this backedge has a latch */ + backedge = &loop->backedges[0]; + if (!backedge->latches_cnt) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, latch not identified\n", insn_idx); + return 0; + } + + /* + * Every condition dominating the backedge bounds the number of + * iterations from above, keep the smallest bound. + */ + for (j = 0; j < backedge->latches_cnt; j++) { + latch_idx = backedge->latches[j]; + linear_latch = match_linear_latch(env, insn_idx, latch_idx, &latch); + if (linear_latch < 0) + return linear_latch; + if (!linear_latch) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, non-linear %s at %d\n", + insn_idx, j ? "exit" : "latch", latch_idx); + continue; + } + if (!compute_max_iters(env, cur_func, &latch, &tmp)) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, + "loop header at %d, can't compute iterations count at %d\n", + insn_idx, latch_idx); + continue; + } + if (!found || tmp.max_header_count < iters->max_header_count) { + iters->max_header_count = tmp.max_header_count; + best = latch; + } + iters->min_header_count = found ? min(iters->min_header_count, tmp.min_header_count) + : tmp.min_header_count; + found = true; + } + if (!found) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, can't compute iterations count\n", insn_idx); + return 0; + } + latch = best; + + if (iters->max_header_count == 0) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, 0 iterations count\n", insn_idx); + return 0; + } + + if (iters->max_header_count == U32_MAX) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, inf iterations count\n", insn_idx); + return 0; + } + + if (log->level & BPF_LOG_LEVEL2) { + bpf_log(log, "loop header at %d, header_count is ", insn_idx); + if (iters->min_header_count == iters->max_header_count) + bpf_log(log, "%u ", iters->max_header_count); + else + bpf_log(log, "[%u..%u] ", iters->min_header_count, iters->max_header_count); + bpf_log(log, "\n"); + } + + err = bpf_live_stack_query_init(env, st); + if (err) + return err; + + header_env = find_header_env(scev, insn_idx); + for (r = 0; r < REGS_NUM; r++) { + struct bpf_reg_state *reg; + u32 ra, r_scev, r_expr; + bool spill_base = false; + + if (!scev_reg_alive(env, st, r)) + continue; + /* + * A heuristic: + * Ignore SCEVs computed for unbound values and treat those as widened. + * This is poor man's modelling of e.g. bpf_probe_read_kernel() effect + * on stack, when called from a subprogram. Currently this would produce + * an unknown ('?') SCEV as a summary of a subprogram call effects. + * Ideally, this should be computed as an OPAQUE value. + * However, that would require tracking STACK_MISC writes in the + * liveness analysis. Not implemented yet. + */ + if (is_unbound_scev_reg(env, cur_func, r)) + continue; + + r_scev = header_env->reg2scev[r]; + reg = scev_reg_for_linear(cur_func, r); + /* If SCEV for r is (linear <reg> <slope>) */ + if (reg && is_simple_linear(scev, r_scev, &coefs) && is_widenable_reg_type(reg)) { + /* Spills at varying offsets lose precision */ + if (test_bit(r, loop->store_base_regs)) { + spill_base = true; + goto cant_widen; + } + continue; + } + + /* rA = rA, loop does not change this reg */ + if (is_reg(scev, r_scev, &ra) && r == ra) + continue; + + /* (any 1 (any 2 (any 3 4))) */ + if (is_widenable_any(env, cur_func, header_env, r, r_scev)) { + if (test_bit(r, loop->store_base_regs)) { + spill_base = true; + goto cant_widen; + } + continue; + } +cant_widen: + if (log->level & BPF_LOG_LEVEL2) { + r_expr = header_env->reg2expr[r]; + bpf_log(log, "loop header at %d, can't widen ", insn_idx); + log_reg(env, r); + bpf_log(log, ", expr is "); + log_expr(env, r_expr); + if (spill_base) + bpf_log(log, ", requires exact stack-offset tracking"); + bpf_log(log, "\n"); + } + return 0; + } + + bitmap_zero(iters->precise_regs, REGS_NUM); + err = mark_expr_precise(env, cur_func, latch.base_expr, iters->precise_regs); + if (err) + return err; + err = mark_expr_precise(env, cur_func, latch.step_expr, iters->precise_regs); + if (err) + return err; + err = mark_expr_precise(env, cur_func, latch.bound_expr, iters->precise_regs); + if (err) + return err; + + return 1; +} + +/* bpf_compute_loop_iters() checked that every leaf can be unioned into acc. */ +static int union_any_reg(struct bpf_verifier_env *env, struct bpf_func_state *loop_entry, + struct bpf_reg_state *acc, u32 id) +{ + struct bpf_reg_state *tmp = &env->fake_reg[0]; + const struct bpf_reg_state *leaf_reg; + struct scev *scev = env->scev; + u32 l, r, leaf, order; + s64 imm; + int err; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if (order & DEPTH_LIMIT) { + verifier_bug(env, "scev ANY union exceeds expression depth limit"); + return -EFAULT; + } + if (!(order & PRE) || is_any(scev, id, &l, &r)) + continue; + if (is_imm(scev, id, &imm)) { + bpf_mark_reg_known_scalar(tmp, imm); + leaf_reg = tmp; + } else if (is_reg(scev, id, &leaf)) { + leaf_reg = scev_reg_for_read(env, loop_entry, leaf); + if (verifier_bug_if(!leaf_reg, env, "unreadable SCEV ANY leaf %u", leaf)) + return -EFAULT; + } else { + verifier_bug(env, "scev ANY union has an unsupported leaf"); + return -EFAULT; + } + err = bpf_reg_union(env, acc, leaf_reg); + if (err) + return err; + } + return 0; +} + +int bpf_widen_scev_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *st, + struct bpf_verifier_state *loop_entry, struct bpf_loop_iters *iters) +{ + struct bpf_func_state *cur_func = st->frame[st->curframe]; + struct bpf_func_state *entry_func = loop_entry->frame[loop_entry->curframe]; + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + struct bpf_reg_state *reg; + struct env *header_env; + struct linear_coefs coefs; + struct bounds bounds; + u32 r, r_expr, a, b; + int insn_idx = st->insn_idx; + int err; + + for_each_set_bit(r, iters->precise_regs, REGS_NUM) { + if (r < MAX_BPF_REG) + bpf_bt_set_frame_reg(&env->bt, st->curframe, r); + else + bpf_bt_set_frame_slot(&env->bt, st->curframe, r - MAX_BPF_REG); + } + err = bpf_mark_chain_precision(env, st, -1, NULL); + if (err) + return err; + + header_env = find_header_env(scev, insn_idx); + for (r = 0; r < REGS_NUM; r++) { + if (!scev_reg_alive(env, st, r) || is_unbound_scev_reg(env, cur_func, r)) + continue; + + r_expr = header_env->reg2scev[r]; + /* If SCEV for r is (linear <reg> <slope>)*/ + if (is_simple_linear(scev, r_expr, &coefs) && coefs.base_reg == r) { + err = scev_reg_for_write(env, cur_func, r, ®); + if (err) + return err; + linear_bounds(reg, iters->max_header_count, &coefs, &bounds); + if (log->level & BPF_LOG_LEVEL2) { + bpf_log(log, "loop header at %d, widening ", insn_idx); + log_reg(env, r); + bpf_log(log, " to "); + log_bounds(env, &bounds); + bpf_log(log, "\n"); + } + scratch_widened_reg_id(env, reg); + err = bpf_set_reg_range(env, reg, bounds.r64, bounds.r32, + bounds.base, bounds.step); + if (err) + return err; + mark_scev_reg_scratched(env, r); + } else if (is_any(scev, r_expr, &a, &b)) { + err = scev_reg_for_write(env, cur_func, r, ®); + if (err) + return err; + err = union_any_reg(env, entry_func, reg, r_expr); + if (err) + return err; + if (log->level & BPF_LOG_LEVEL2) { + bpf_log(log, "loop header at %d, widening ", insn_idx); + log_reg(env, r); + bpf_log(log, " to "); + bpf_verbose_snum(env, reg_smin(reg)); + bpf_log(log, ".."); + bpf_verbose_snum(env, reg_smax(reg)); + bpf_log(log, " step %u\n", reg->step); + } + scratch_widened_reg_id(env, reg); + mark_scev_reg_scratched(env, r); + } + } + return 1; +} + +int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *cur_func_state, u32 insn_idx, + struct bpf_verifier_state *entry_state, u32 max_header_count) +{ + struct bpf_func_state *entry_st = entry_state->frame[entry_state->curframe]; + struct bpf_verifier_log *log = &env->log; + struct bpf_reg_state *reg, *entry_reg; + struct scev *scev = env->scev; + struct linear_coefs coefs; + struct env *header_env; + struct bounds bounds; + u32 r; + int err; + + if (entry_state->curframe != cur_func_state->frameno) { + verifier_bug(env, "clamping registers for a wrong frame: %d vs %d\n", + entry_state->curframe, cur_func_state->frameno); + return -EFAULT; + } + + header_env = find_header_env(scev, insn_idx); + for (r = 0; r < REGS_NUM; r++) { + /* If SCEV for r is (linear <reg> <slope>)*/ + if (!is_simple_linear(scev, header_env->reg2scev[r], &coefs) || + coefs.base_reg != r) + continue; + + entry_reg = scev_reg_for_linear(entry_st, r); + reg = scev_reg_for_linear(cur_func_state, r); + if (!entry_reg || !reg || entry_reg->type == NOT_INIT || reg->type == NOT_INIT || + is_unbound_scev_reg(env, entry_st, r)) + continue; + + linear_bounds(entry_reg, max_header_count, &coefs, &bounds); + bounds.r64 = cnum64_intersect(reg->r64, bounds.r64); + bounds.r32 = cnum32_intersect(reg->r32, bounds.r32); + if (cnum64_is_empty(bounds.r64)) { + verifier_bug(env, "scev clamp produced empty range for r%d", r); + return -EFAULT; + } + if (log->level & BPF_LOG_LEVEL2) { + bpf_log(log, "loop header at %d, clamping ", insn_idx); + log_reg(env, r); + bpf_log(log, " to "); + log_bounds(env, &bounds); + bpf_log(log, "\n"); + } + scratch_widened_reg_id(env, reg); + err = bpf_set_reg_range(env, reg, bounds.r64, bounds.r32, + bounds.base, bounds.step); + if (err) + return err; + mark_scev_reg_scratched(env, r); + } + return 0; +} diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index d09d4c3fe547..f7e8d766d128 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -272,13 +272,10 @@ struct stack_map_vma_lock { /* * Acquire a stable read-side reference on the VMA covering @ip. * - * With CONFIG_PER_VMA_LOCK=y this returns a VMA with its per-VMA read - * lock held and mmap_lock dropped, so the caller may sleep. - * - * With CONFIG_PER_VMA_LOCK=n it returns a VMA with mmap_lock still - * held; the caller must snapshot any fields it needs and pin vm_file - * with get_file() before stack_map_unlock_vma() drops mmap_lock, as - * the VMA may be split, merged, or freed after that. + * On NOMMU configurations, returns with the mmap_lock held. If the MMU + * is enabled, the per-VMA lock will be held instead. The lock + * should be released with stack_map_unlock_vma() which will release the + * appropriate lock. Once the lock is released, the VMA may be freed. * * Returns NULL on failure, in which case no lock is held. */ @@ -288,7 +285,6 @@ stack_map_lock_vma(struct stack_map_vma_lock *lock, unsigned long ip) struct mm_struct *mm = lock->mm; struct vm_area_struct *vma; - /* noop under !CONFIG_PER_VMA_LOCK */ vma = lock_vma_under_rcu(mm, ip); if (vma) { lock->vma = vma; @@ -308,21 +304,20 @@ stack_map_lock_vma(struct stack_map_vma_lock *lock, unsigned long ip) return NULL; } -#ifdef CONFIG_PER_VMA_LOCK +#ifdef CONFIG_MMU if (!vma_start_read_locked(vma)) { mmap_read_unlock(mm); return NULL; } mmap_read_unlock(mm); #endif - lock->vma = vma; return vma; } static void stack_map_unlock_vma(struct stack_map_vma_lock *lock) { -#ifdef CONFIG_PER_VMA_LOCK +#ifdef CONFIG_MMU vma_end_read(lock->vma); #else mmap_read_unlock(lock->mm); diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 66fb11b6c6a7..d7df6632a5c4 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -302,8 +302,30 @@ int bpf_update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_s static bool range_within(const struct bpf_reg_state *old, const struct bpf_reg_state *cur) { - return cnum64_is_subset(old->r64, cur->r64) && - cnum32_is_subset(old->r32, cur->r32); + if (!cnum64_is_subset(old->r64, cur->r64) || + !cnum32_is_subset(old->r32, cur->r32)) + return false; + + if (old->step <= 1) + return true; + + if (cnum64_is_const(cur->r64)) + return imod((s64)cnum64_smin(cur->r64), old->step) == old->base; + + /* + * Both `old` and `cur` define some sets of points. + * Return true, if points defined by `cur` are a subset of points defined by `old`: + * - bounds for `cur` should be within bounds for `old`; + * - cur->step should be dividable by old->step; + * - cur->base should start at integer number of old->step + * steps from old->base. + * + * E.g. the following ranges are compatible: + * - old [0, 10] step 2 + * - new [2, 10] step 4 + */ + return cur->step % old->step == 0 && + (cur->base - old->base) % old->step == 0; } /* If in the old state two registers had the same id, then they need to have @@ -415,14 +437,14 @@ static void __clean_func_state(struct bpf_verifier_env *env, * half_spi 2*i → lower half: slot_type[0..3] (closer to FP) * half_spi 2*i+1 → upper half: slot_type[4..7] (farther from FP) */ - for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { + for (i = 0; i < bpf_stack_nr_slots(st); i++) { bool lo_live = bpf_stack_slot_alive(env, frame, i * 2); bool hi_live = bpf_stack_slot_alive(env, frame, i * 2 + 1); if (!hi_live || !lo_live) { int start = !lo_live ? 0 : BPF_REG_SIZE / 2; int end = !hi_live ? BPF_REG_SIZE : BPF_REG_SIZE / 2; - u8 stype = st->stack[i].slot_type[7]; + u8 stype = bpf_stack_slot(st, i)->slot_type[7]; /* * Don't clear special slots. @@ -442,7 +464,7 @@ static void __clean_func_state(struct bpf_verifier_env *env, * rejecting as non-scalar register fills. */ if (!hi_live) { - struct bpf_reg_state *spill = &st->stack[i].spilled_ptr; + struct bpf_reg_state *spill = &bpf_stack_slot(st, i)->spilled_ptr; if (lo_live && stype == STACK_SPILL) { if (spill->type != SCALAR_VALUE) @@ -454,7 +476,7 @@ static void __clean_func_state(struct bpf_verifier_env *env, if (bpf_register_is_null(spill)) continue; for (j = 0; j < 4; j++) { - u8 *t = &st->stack[i].slot_type[j]; + u8 *t = &bpf_stack_slot(st, i)->slot_type[j]; if (*t == STACK_SPILL) *t = STACK_MISC; @@ -463,7 +485,7 @@ static void __clean_func_state(struct bpf_verifier_env *env, bpf_mark_reg_not_init(env, spill); } for (j = start; j < end; j++) - st->stack[i].slot_type[j] = STACK_POISON; + bpf_stack_slot(st, i)->slot_type[j] = STACK_POISON; } } } @@ -491,7 +513,8 @@ static bool regs_exact(const struct bpf_reg_state *rold, { return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && check_ids(rold->id, rcur->id, idmap) && - check_ids(rold->parent_id, rcur->parent_id, idmap); + check_ids(rold->parent_id, rcur->parent_id, idmap) && + check_ids(rold->map_uid, rcur->map_uid, idmap); } enum exact_level { @@ -609,6 +632,8 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, case PTR_TO_MEM: case PTR_TO_BUF: case PTR_TO_TP_BUFFER: + case PTR_TO_STACK: + case PTR_TO_BTF_ID: /* If the new min/max/var_off satisfy the old ones and * everything else matches, we are OK. */ @@ -616,7 +641,8 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off) && check_ids(rold->id, rcur->id, idmap) && - check_ids(rold->parent_id, rcur->parent_id, idmap); + check_ids(rold->parent_id, rcur->parent_id, idmap) && + check_ids(rold->map_uid, rcur->map_uid, idmap); case PTR_TO_PACKET_META: case PTR_TO_PACKET: /* We must have at least as much range as the old ptr @@ -635,14 +661,12 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, /* id relations must be preserved */ if (!check_ids(rold->id, rcur->id, idmap)) return false; + /* Preserve displacements between pointers sharing an ID. */ + if (rold->id && rold->r64.base != rcur->r64.base) + return false; /* new val must satisfy old val knowledge */ return range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off); - case PTR_TO_STACK: - /* two stack pointers are equal only if they're pointing to - * the same stack frame, since fp-8 in foo != fp-8 in bar - */ - return regs_exact(rold, rcur, idmap) && rold->frameno == rcur->frameno; case PTR_TO_ARENA: return true; case PTR_TO_INSN: @@ -707,37 +731,38 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * didn't use them */ for (i = 0; i < old->allocated_stack; i++) { + struct bpf_stack_state *old_slot, *cur_slot; struct bpf_reg_state *old_reg, *cur_reg; int im = i % BPF_REG_SIZE; + u8 old_type; spi = i / BPF_REG_SIZE; + old_slot = bpf_stack_slot(old, spi); + old_type = old_slot->slot_type[im]; + cur_slot = i < cur->allocated_stack ? bpf_stack_slot(cur, spi) : NULL; if (exact == EXACT) { - u8 old_type = old->stack[spi].slot_type[i % BPF_REG_SIZE]; - u8 cur_type = i < cur->allocated_stack ? - cur->stack[spi].slot_type[i % BPF_REG_SIZE] : STACK_INVALID; + u8 cur_type = cur_slot ? cur_slot->slot_type[im] : STACK_INVALID; /* STACK_INVALID and STACK_POISON are equivalent for pruning */ if (old_type == STACK_POISON) old_type = STACK_INVALID; if (cur_type == STACK_POISON) cur_type = STACK_INVALID; - if (i >= cur->allocated_stack || old_type != cur_type) + if (!cur_slot || old_type != cur_type) return false; } - if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID || - old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_POISON) + if (old_type == STACK_INVALID || old_type == STACK_POISON) continue; - if (env->allow_uninit_stack && - old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC) + if (env->allow_uninit_stack && old_type == STACK_MISC) continue; /* explored stack has more populated slots than current stack * and these slots were used */ - if (i >= cur->allocated_stack) + if (!cur_slot) return false; /* @@ -747,8 +772,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * regsafe() to ensure scalar ids are compared. */ if (im == 0 || im == 4) { - old_reg = scalar_reg_for_stack(env, &old->stack[spi], im); - cur_reg = scalar_reg_for_stack(env, &cur->stack[spi], im); + old_reg = scalar_reg_for_stack(env, old_slot, im); + cur_reg = scalar_reg_for_stack(env, cur_slot, im); if (old_reg && cur_reg) { if (!regsafe(env, old_reg, cur_reg, idmap, exact)) return false; @@ -761,21 +786,19 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * it will be safe with zero-initialized stack. * The opposite is not true */ - if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && - cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) + if (old_type == STACK_MISC && cur_slot->slot_type[im] == STACK_ZERO) continue; - if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != - cur->stack[spi].slot_type[i % BPF_REG_SIZE]) + if (old_type != cur_slot->slot_type[im]) /* Ex: old explored (safe) state has STACK_SPILL in * this stack slot, but current has STACK_MISC -> * this verifier states are not equivalent, * return false to continue verification of this path */ return false; - if (i % BPF_REG_SIZE != BPF_REG_SIZE - 1) + if (im != BPF_REG_SIZE - 1) continue; /* Both old and cur are having same slot_type */ - switch (old->stack[spi].slot_type[BPF_REG_SIZE - 1]) { + switch (old_type) { case STACK_SPILL: /* when explored and current stack slot are both storing * spilled registers, check that stored pointers types @@ -787,13 +810,13 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * such verifier states are not equivalent. * return false to continue verification of this path */ - if (!regsafe(env, &old->stack[spi].spilled_ptr, - &cur->stack[spi].spilled_ptr, idmap, exact)) + if (!regsafe(env, &old_slot->spilled_ptr, &cur_slot->spilled_ptr, + idmap, exact)) return false; break; case STACK_DYNPTR: - old_reg = &old->stack[spi].spilled_ptr; - cur_reg = &cur->stack[spi].spilled_ptr; + old_reg = &old_slot->spilled_ptr; + cur_reg = &cur_slot->spilled_ptr; if (old_reg->dynptr.type != cur_reg->dynptr.type || old_reg->dynptr.first_slot != cur_reg->dynptr.first_slot || !check_ids(old_reg->id, cur_reg->id, idmap) || @@ -801,8 +824,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, return false; break; case STACK_ITER: - old_reg = &old->stack[spi].spilled_ptr; - cur_reg = &cur->stack[spi].spilled_ptr; + old_reg = &old_slot->spilled_ptr; + cur_reg = &cur_slot->spilled_ptr; /* iter.depth is not compared between states as it * doesn't matter for correctness and would otherwise * prevent convergence; we maintain it only to prevent @@ -818,8 +841,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, return false; break; case STACK_IRQ_FLAG: - old_reg = &old->stack[spi].spilled_ptr; - cur_reg = &cur->stack[spi].spilled_ptr; + old_reg = &old_slot->spilled_ptr; + cur_reg = &cur_slot->spilled_ptr; if (!check_ids(old_reg->id, cur_reg->id, idmap) || old_reg->irq.kfunc_class != cur_reg->irq.kfunc_class) return false; @@ -913,6 +936,27 @@ static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *c return true; } +static bool loop_stack_safe(struct bpf_verifier_env *env, struct bpf_func_state *old, + struct bpf_func_state *cur) +{ + struct loop_stack_entry *old_stack = old->loop_stack; + struct loop_stack_entry *cur_stack = cur->loop_stack; + u32 i; + + if (old->loop_stack_cnt != cur->loop_stack_cnt) + return false; + + for (i = 0; i < old->loop_stack_cnt; i++) { + if (old_stack[i].loop_id == cur_stack[i].loop_id && + old_stack[i].max_header_count == cur_stack[i].max_header_count && + old_stack[i].terminates == cur_stack[i].terminates) + continue; + return false; + } + + return true; +} + /* compare two verifier states * * all states stored in state_list are known to be valid, since @@ -951,6 +995,9 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat if (!old->no_stack_arg_load && cur->no_stack_arg_load) return false; + if (old->in_rcu_callback_fn != cur->in_rcu_callback_fn) + return false; + for (i = 0; i < MAX_BPF_REG; i++) if (((1 << i) & live_regs) && !regsafe(env, &old->regs[i], &cur->regs[i], @@ -963,6 +1010,9 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat if (!stack_arg_safe(env, old, cur, &env->idmap_scratch, exact)) return false; + if (!loop_stack_safe(env, old, cur)) + return false; + return true; } @@ -996,6 +1046,9 @@ static bool states_equal(struct bpf_verifier_env *env, if (old->in_sleepable != cur->in_sleepable) return false; + if (old->unwound != cur->unwound) + return false; + if (!refsafe(old, cur, &env->idmap_scratch)) return false; @@ -1009,6 +1062,7 @@ static bool states_equal(struct bpf_verifier_env *env, if (!func_states_equal(env, old->frame[i], cur->frame[i], insn_idx, exact)) return false; } + return true; } @@ -1043,10 +1097,12 @@ static int propagate_precision(struct bpf_verifier_env *env, first = false; } - for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { - if (!bpf_is_spilled_reg(&state->stack[i])) + for (i = 0; i < bpf_stack_nr_slots(state); i++) { + struct bpf_stack_state *ss = bpf_stack_slot(state, i); + + if (!bpf_is_spilled_reg(ss)) continue; - state_reg = &state->stack[i].spilled_ptr; + state_reg = &ss->spilled_ptr; if (state_reg->type != SCALAR_VALUE || !state_reg->precise) continue; @@ -1121,7 +1177,7 @@ static bool states_maybe_looping(struct bpf_verifier_state *old, fcur = cur->frame[fr]; for (i = 0; i < MAX_BPF_REG; i++) if (memcmp(&fold->regs[i], &fcur->regs[i], - offsetof(struct bpf_reg_state, frameno))) + offsetofend(struct bpf_reg_state, step))) return false; return true; } @@ -1192,15 +1248,17 @@ static bool iter_active_depths_differ(struct bpf_verifier_state *old, struct bpf for (fr = old->curframe; fr >= 0; fr--) { state = old->frame[fr]; - for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { - if (state->stack[i].slot_type[0] != STACK_ITER) + for (i = 0; i < bpf_stack_nr_slots(state); i++) { + struct bpf_stack_state *ss = bpf_stack_slot(state, i); + + if (ss->slot_type[0] != STACK_ITER) continue; - slot = &state->stack[i].spilled_ptr; + slot = &ss->spilled_ptr; if (slot->iter.state != BPF_ITER_STATE_ACTIVE) continue; - cur_slot = &cur->frame[fr]->stack[i].spilled_ptr; + cur_slot = &bpf_stack_slot(cur->frame[fr], i)->spilled_ptr; if (cur_slot->iter.depth != slot->iter.depth) return true; } @@ -1222,10 +1280,12 @@ static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_ continue; reg->precise = false; } - for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { - if (!bpf_is_spilled_reg(&func->stack[j])) + for (j = 0; j < bpf_stack_nr_slots(func); j++) { + struct bpf_stack_state *ss = bpf_stack_slot(func, j); + + if (!bpf_is_spilled_reg(ss)) continue; - reg = &func->stack[j].spilled_ptr; + reg = &ss->spilled_ptr; if (reg->type != SCALAR_VALUE) continue; reg->precise = false; @@ -1233,16 +1293,92 @@ static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_ } } -int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) +int bpf_split_cur_state(struct bpf_verifier_env *env) { + struct bpf_verifier_state *cur = env->cur_state, *new; struct bpf_verifier_state_list *new_sl; + struct list_head *head; + int insn_idx = cur->insn_idx; + int err; + + head = bpf_explored_state(env, insn_idx); + new_sl = kzalloc_obj(struct bpf_verifier_state_list, GFP_KERNEL_ACCOUNT); + if (!new_sl) + return -ENOMEM; + env->total_states++; + env->explored_states_size++; + update_peak_states(env); + env->prev_jmps_processed = env->jmps_processed; + env->prev_insn_processed = env->insn_processed; + + /* forget precise markings we inherited, see __mark_chain_precision */ + if (env->bpf_capable) + mark_all_scalars_imprecise(env, cur); + + bpf_clear_singular_ids(env, cur); + + /* add new state to the head of linked list */ + new = &new_sl->state; + err = bpf_copy_verifier_state(new, cur); + if (err) { + bpf_free_verifier_state(new, false); + kfree(new_sl); + return err; + } + new->insn_idx = insn_idx; + verifier_bug_if(new->branches != 1, env, + "%s:branches_to_explore=%d insn %d", + __func__, new->branches, insn_idx); + err = maybe_enter_scc(env, new); + if (err) { + bpf_free_verifier_state(new, false); + kfree(new_sl); + return err; + } + + cur->parent = new; + cur->last_insn_idx = -1; + cur->first_insn_idx = insn_idx; + cur->dfs_depth = new->dfs_depth + 1; + bpf_clear_jmp_history(cur); + list_add(&new_sl->node, head); + return 0; +} + +/* Force a checkpoint upon reaching a loop header for a terminating loop. */ +static bool need_loop_checkpoint(struct bpf_verifier_env *env, int insn_idx) +{ + struct bpf_verifier_state *cur = env->cur_state; + struct bpf_func_state *frame = cur->frame[cur->curframe]; + struct loop_stack_entry *top; + + if (bpf_loop_at_index(env, insn_idx) != insn_idx || !frame->loop_stack_cnt) + return false; + top = &frame->loop_stack[frame->loop_stack_cnt - 1]; + return top->loop_id == insn_idx && top->terminates; +} + +static struct loop_stack_entry *current_loop(struct bpf_verifier_state *st) +{ + struct bpf_func_state *frame = st->frame[st->curframe]; + + if (frame->loop_stack_cnt == 0) + return NULL; + return &frame->loop_stack[frame->loop_stack_cnt - 1]; +} + +int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) +{ + struct loop_stack_entry *old_loop, *cur_loop; struct bpf_verifier_state_list *sl; - struct bpf_verifier_state *cur = env->cur_state, *new; - bool force_new_state, add_new_state, loop; + struct bpf_verifier_state *cur = env->cur_state; + bool force_new_state, add_new_state, loop, loop_checkpoint; int n, err, states_cnt = 0; struct list_head *pos, *tmp, *head; + loop_checkpoint = need_loop_checkpoint(env, insn_idx); force_new_state = env->test_state_freq || bpf_is_force_checkpoint(env, insn_idx) || + loop_checkpoint || /* Avoid accumulating infinitely long jmp history */ cur->jmp_history_cnt > 40; @@ -1273,10 +1409,10 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) continue; if (sl->state.branches) { - struct bpf_func_state *frame = sl->state.frame[sl->state.curframe]; + struct bpf_func_state *old_top_frame = sl->state.frame[0]; - if (frame->in_async_callback_fn && - frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) { + if (old_top_frame->in_async_callback_fn && + old_top_frame->async_entry_cnt != cur->frame[0]->async_entry_cnt) { /* Different async_entry_cnt means that the verifier is * processing another entry into async callback. * Seeing the same state is not an indication of infinite @@ -1328,8 +1464,9 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) */ if (is_iter_next_insn(env, insn_idx)) { if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) { - struct bpf_func_state *cur_frame; + struct bpf_func_state *cur_frame, *iter_frame, *old_frame; struct bpf_reg_state *iter_state, *iter_reg; + struct bpf_reg_state *old_iter; int spi; cur_frame = cur->frame[cur->curframe]; @@ -1342,8 +1479,18 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) * no need for extra (re-)validations */ spi = bpf_get_spi(iter_reg->var_off.value); - iter_state = &bpf_func(env, iter_reg)->stack[spi].spilled_ptr; - if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE) { + iter_frame = bpf_func(env, iter_reg); + iter_state = &bpf_stack_slot(iter_frame, spi)->spilled_ptr; + old_frame = sl->state.frame[iter_reg->frameno]; + old_iter = &bpf_stack_slot(old_frame, spi)->spilled_ptr; + /* + * states_equal() matches ids through idmap. + * The loop converged only if it's the same + * iterator. The one that was destroyed and + * created again starts from the beginning. + */ + if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE && + iter_state->id == old_iter->id) { loop = true; goto hit; } @@ -1364,6 +1511,33 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) } goto skip_inf_loop_check; } + /* + * If old state belongs to a control flow loop that we know terminates, + * it should be safe to prune current state. However, account for the + * following situation: + * + * for (;;) { + * for (i = 0; i < 10; i++) + * ... + * } + * + * Here a backedge to a non-terminating outer loop is dominated by an exit + * from the inner loop. Pruning at a re-entry to the inner loop therefore + * would hide the fact that outer loop is non-terminating. + * Hence, only allow pruning states within the same entry state. + */ + old_loop = current_loop(&sl->state); + cur_loop = current_loop(cur); + if (old_loop && cur_loop && + old_loop->terminates && cur_loop->terminates && + old_loop->entry_state == cur_loop->entry_state) { + if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) { + loop = true; + goto hit; + } + goto skip_inf_loop_check; + } + /* attempt to detect infinite loop to avoid unnecessary doomed work */ if (states_maybe_looping(&sl->state, cur) && states_equal(env, &sl->state, cur, EXACT) && @@ -1410,7 +1584,7 @@ hit: */ err = 0; if (bpf_is_jmp_point(env, env->insn_idx)) - err = bpf_push_jmp_history(env, cur, 0, 0, 0, 0); + err = bpf_push_jmp_history(env, cur, 0, 0, 0, NULL, 0); err = err ? : propagate_precision(env, &sl->state, cur, NULL); if (err) return err; @@ -1522,7 +1696,8 @@ miss: * Use bigger 'n' for checkpoints because evicting checkpoint states * too early would hinder iterator convergence. */ - n = bpf_is_force_checkpoint(env, insn_idx) && sl->state.branches > 0 ? 64 : 3; + n = (bpf_is_force_checkpoint(env, insn_idx) || loop_checkpoint) && + sl->state.branches > 0 ? 64 : 3; if (sl->miss_cnt > sl->hit_cnt * n + n) { /* the state is unlikely to be useful. Remove it to * speed up verification @@ -1554,44 +1729,5 @@ miss: * When looping the sl->state.branches will be > 0 and this state * will not be considered for equivalence until branches == 0. */ - new_sl = kzalloc_obj(struct bpf_verifier_state_list, GFP_KERNEL_ACCOUNT); - if (!new_sl) - return -ENOMEM; - env->total_states++; - env->explored_states_size++; - update_peak_states(env); - env->prev_jmps_processed = env->jmps_processed; - env->prev_insn_processed = env->insn_processed; - - /* forget precise markings we inherited, see __mark_chain_precision */ - if (env->bpf_capable) - mark_all_scalars_imprecise(env, cur); - - bpf_clear_singular_ids(env, cur); - - /* add new state to the head of linked list */ - new = &new_sl->state; - err = bpf_copy_verifier_state(new, cur); - if (err) { - bpf_free_verifier_state(new, false); - kfree(new_sl); - return err; - } - new->insn_idx = insn_idx; - verifier_bug_if(new->branches != 1, env, - "%s:branches_to_explore=%d insn %d", - __func__, new->branches, insn_idx); - err = maybe_enter_scc(env, new); - if (err) { - bpf_free_verifier_state(new, false); - kfree(new_sl); - return err; - } - - cur->parent = new; - cur->first_insn_idx = insn_idx; - cur->dfs_depth = new->dfs_depth + 1; - bpf_clear_jmp_history(cur); - list_add(&new_sl->node, head); - return 0; + return bpf_split_cur_state(env); } diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index be9ce98e9469..83e803ea54ef 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -2,11 +2,15 @@ /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ #include <linux/bpf.h> +#include <linux/anon_inodes.h> #include <linux/filter.h> #include <linux/bpf_mem_alloc.h> #include <linux/gfp.h> +#include <linux/irq_work.h> #include <linux/memory.h> #include <linux/mutex.h> +#include <linux/poll.h> +#include <linux/refcount.h> static void bpf_stream_elem_init(struct bpf_stream_elem *elem, int len) { @@ -22,11 +26,11 @@ static struct bpf_stream_elem *bpf_stream_elem_alloc(int len) size_t alloc_size; /* - * Length denotes the amount of data to be written as part of stream element, - * thus includes '\0' byte. We're capped by how much bpf_bprintf_buffers can - * accomodate, therefore deny allocations that won't fit into them. + * Length is the payload pushed into the stream, excluding the + * trailing NUL of the bprintf buffer. Reject anything that cannot + * fit without copying that NUL into the stream element. */ - if (len < 0 || len > max_len) + if (len < 0 || len >= max_len) return NULL; alloc_size = offsetof(struct bpf_stream_elem, str[len]); @@ -68,25 +72,82 @@ static int bpf_stream_consume_capacity(struct bpf_stream *stream, int len) return 0; } -static void bpf_stream_release_capacity(struct bpf_stream *stream, struct bpf_stream_elem *elem) +static void bpf_stream_release_capacity(struct bpf_stream *stream, int len) { - int len = elem->total_len; - atomic_sub(len, &stream->capacity); } +static void bpf_stream_notify(struct irq_work *work) +{ + struct bpf_stream *stream = container_of(work, struct bpf_stream, notify_work); + + /* + * Writers run in arbitrary program contexts, including NMI and regions + * that already hold wait queue or epoll locks. Wake waiters from + * irq_work instead, where taking those locks is safe. + */ + wake_up_interruptible_poll(&stream->waitq, EPOLLIN | EPOLLRDNORM); +} + +static void bpf_stream_queue_notify(struct bpf_stream *stream) +{ + /* + * Record that the work has been used so that teardown only pays for + * irq_work_sync(), which may wait for an RCU grace period, when a + * callback could actually be in flight. + */ + if (!READ_ONCE(stream->notify_used)) + WRITE_ONCE(stream->notify_used, true); + irq_work_queue(&stream->notify_work); +} + +static int bpf_stream_readable_bytes(struct bpf_stream *stream) +{ + return atomic_read_acquire(&stream->readable); +} + +static void bpf_stream_publish(struct bpf_stream *stream, int len) +{ + /* + * Pairs with atomic_read_acquire() in bpf_stream_readable_bytes(). + * + * Notify only when the stream turns from empty to readable. Readers + * block and pollers wait only after finding it empty, and a read never + * takes the count below zero, so the publication that makes it positive + * is the one they wait for. Publishing into a stream that already holds + * data would only raise an interrupt for waiters that are being woken + * already, or for nobody at all. + */ + if (atomic_add_return_release(len, &stream->readable) == len) + bpf_stream_queue_notify(stream); +} + static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int len) { - int ret = bpf_stream_consume_capacity(stream, len); + int ret; + + /* Nothing to publish; do not allocate an element for it. */ + if (!len) + return 0; + + ret = bpf_stream_consume_capacity(stream, len); + if (ret) + return ret; + + ret = __bpf_stream_push_str(&stream->log, str, len); + if (ret) + bpf_stream_release_capacity(stream, len); + else + bpf_stream_publish(stream, len); - return ret ?: __bpf_stream_push_str(&stream->log, str, len); + return ret; } static struct bpf_stream *bpf_stream_get(enum bpf_stream_id stream_id, struct bpf_prog_aux *aux) { if (stream_id != BPF_STDOUT && stream_id != BPF_STDERR) return NULL; - return &aux->stream[stream_id - 1]; + return aux->stream[stream_id - 1]; } static void bpf_stream_free_elem(struct bpf_stream_elem *elem) @@ -154,14 +215,17 @@ static bool bpf_stream_consume_elem(struct bpf_stream_elem *elem, int *len) static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) { - int rem_len = len, cons_len, ret = 0; + int read_len, rem_len, cons_len, ret = 0; struct bpf_stream_elem *elem = NULL; struct llist_node *node; mutex_lock(&stream->lock); + read_len = min(len, bpf_stream_readable_bytes(stream)); + rem_len = read_len; while (rem_len) { - int pos = len - rem_len; + int pos = read_len - rem_len; + int chunk, n; bool cont; node = bpf_stream_backlog_peek(stream); @@ -175,37 +239,146 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) cons_len = elem->consumed_len; cont = bpf_stream_consume_elem(elem, &rem_len) == false; - - ret = copy_to_user(buf + pos, elem->str + cons_len, - elem->consumed_len - cons_len); - /* Restore in case of error. */ - if (ret) { - ret = -EFAULT; - elem->consumed_len = cons_len; + chunk = elem->consumed_len - cons_len; + + n = copy_to_user(buf + pos, elem->str + cons_len, chunk); + if (n) { + /* Keep any successfully copied bytes; -EFAULT only if none. */ + elem->consumed_len -= n; + rem_len += n; + ret = (read_len == rem_len) ? -EFAULT : 0; break; } if (cont) continue; bpf_stream_backlog_pop(stream); - bpf_stream_release_capacity(stream, elem); + bpf_stream_release_capacity(stream, elem->total_len); bpf_stream_free_elem(elem); } + atomic_sub(read_len - rem_len, &stream->readable); mutex_unlock(&stream->lock); - return ret ? ret : len - rem_len; + return ret ? ret : read_len - rem_len; } -int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len) +int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len) { struct bpf_stream *stream; stream = bpf_stream_get(stream_id, prog->aux); if (!stream) return -ENOENT; + if (len > INT_MAX) + return -EINVAL; return bpf_stream_read(stream, buf, len); } +static bool bpf_stream_has_data(struct bpf_stream *stream) +{ + return bpf_stream_readable_bytes(stream) > 0; +} + +static void bpf_stream_put(struct bpf_stream *stream) +{ + if (refcount_dec_and_test(&stream->refcnt)) { + struct llist_node *list; + + /* Only a stream that ever queued its work can have a callback in flight. */ + if (READ_ONCE(stream->notify_used)) + irq_work_sync(&stream->notify_work); + list = llist_del_all(&stream->log); + bpf_stream_free_list(list); + bpf_stream_free_list(stream->backlog_head); + mutex_destroy(&stream->lock); + kfree(stream); + } +} + +static int bpf_stream_release(struct inode *inode, struct file *file) +{ + bpf_stream_put(file->private_data); + return 0; +} + +static ssize_t bpf_stream_file_read(struct file *file, char __user *buf, size_t len, + loff_t *ppos) +{ + struct bpf_stream *stream = file->private_data; + bool dead; + int ret; + + if (!len) + return 0; + + for (;;) { + /* + * Sample teardown state before looking for data. Nothing is + * published once the program is gone, so finding the stream + * empty after observing dead means EOF. The opposite order could + * report EOF while data published just before teardown is still + * buffered. + */ + dead = smp_load_acquire(&stream->dead); + ret = bpf_stream_read(stream, buf, len); + if (ret) + return ret; + if (dead) + return 0; + if (file->f_flags & O_NONBLOCK) + return -EAGAIN; + + ret = wait_event_interruptible(stream->waitq, + bpf_stream_has_data(stream) || + READ_ONCE(stream->dead)); + if (ret) + return ret; + } +} + +static __poll_t bpf_stream_poll(struct file *file, struct poll_table_struct *pts) +{ + struct bpf_stream *stream = file->private_data; + __poll_t events = 0; + + /* + * poll_wait() only registers the wait queue callback. Register before + * checking persistent state so a concurrent publication or teardown is + * observed either by the callback or by the checks below. + */ + poll_wait(file, &stream->waitq, pts); + if (bpf_stream_has_data(stream)) + events |= EPOLLIN | EPOLLRDNORM; + if (READ_ONCE(stream->dead)) + events |= EPOLLHUP; + return events; +} + +static const struct file_operations bpf_stream_fops = { + .release = bpf_stream_release, + .read = bpf_stream_file_read, + .poll = bpf_stream_poll, +}; + +int bpf_prog_stream_new_fd(struct bpf_prog *prog, enum bpf_stream_id stream_id, u32 flags) +{ + struct bpf_stream *stream; + int fd_flags = O_RDONLY | O_CLOEXEC; + int fd; + + stream = bpf_stream_get(stream_id, prog->aux); + if (!stream) + return -ENOENT; + if (flags & BPF_F_STREAM_NONBLOCK) + fd_flags |= O_NONBLOCK; + + refcount_inc(&stream->refcnt); + fd = anon_inode_getfd("bpf-stream", &bpf_stream_fops, stream, fd_flags); + if (fd < 0) + bpf_stream_put(stream); + return fd; +} + __bpf_kfunc_start_defs(); /* @@ -238,6 +411,11 @@ __bpf_kfunc int bpf_stream_vprintk(int stream_id, const char *fmt__str, const vo return ret; ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt__str, data.bin_args); + /* Truncation: reject before capacity charge (not -ENOMEM). */ + if (ret >= MAX_BPRINTF_BUF) { + bpf_bprintf_cleanup(&data); + return -E2BIG; + } /* Exclude NULL byte during push. */ ret = bpf_stream_push_str(stream, data.buf, ret); bpf_bprintf_cleanup(&data); @@ -268,28 +446,47 @@ __bpf_kfunc_end_defs(); /* Added kfunc to common_btf_ids */ -void bpf_prog_stream_init(struct bpf_prog *prog) +int bpf_prog_stream_init(struct bpf_prog *prog, gfp_t gfp_extra_flags) { int i; for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) { - atomic_set(&prog->aux->stream[i].capacity, 0); - init_llist_head(&prog->aux->stream[i].log); - mutex_init(&prog->aux->stream[i].lock); - prog->aux->stream[i].backlog_head = NULL; - prog->aux->stream[i].backlog_tail = NULL; + struct bpf_stream *stream; + + /* On failure, bpf_prog_stream_free() releases the streams allocated so far. */ + stream = kzalloc_obj(*stream, + bpf_memcg_flags(GFP_KERNEL | gfp_extra_flags)); + if (!stream) + return -ENOMEM; + + refcount_set(&stream->refcnt, 1); + init_llist_head(&stream->log); + mutex_init(&stream->lock); + init_waitqueue_head(&stream->waitq); + init_irq_work(&stream->notify_work, bpf_stream_notify); + prog->aux->stream[i] = stream; } + return 0; } void bpf_prog_stream_free(struct bpf_prog *prog) { - struct llist_node *list; int i; for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) { - list = llist_del_all(&prog->aux->stream[i].log); - bpf_stream_free_list(list); - bpf_stream_free_list(prog->aux->stream[i].backlog_head); + struct bpf_stream *stream = prog->aux->stream[i]; + + if (!stream) + continue; + /* + * Pairs with smp_load_acquire() in bpf_stream_file_read(): every + * publication precedes the dead flag, so a reader that observes + * it also observes all buffered data. + */ + smp_store_release(&stream->dead, true); + wake_up_interruptible_poll(&stream->waitq, EPOLLHUP); + bpf_stream_put(stream); + prog->aux->stream[i] = NULL; } } @@ -311,17 +508,18 @@ int bpf_stream_stage_printk(struct bpf_stream_stage *ss, const char *fmt, ...) { struct bpf_bprintf_buffers *buf; va_list args; - int ret; + int len, ret; if (bpf_try_get_buffers(&buf)) return -EBUSY; va_start(args, fmt); - ret = vsnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args); + len = vscnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args); va_end(args); - ss->len += ret; - /* Exclude NULL byte during push. */ - ret = __bpf_stream_push_str(&ss->log, buf->buf, ret); + /* Exclude NULL byte during push; skip empty output entirely. */ + ret = len ? __bpf_stream_push_str(&ss->log, buf->buf, len) : 0; + if (!ret) + ss->len += len; bpf_put_buffers(); return ret; } @@ -351,6 +549,7 @@ int bpf_stream_stage_commit(struct bpf_stream_stage *ss, struct bpf_prog *prog, list = tail; } llist_add_batch(head, tail, &stream->log); + bpf_stream_publish(stream, ss->len); return 0; } diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index c7bc9ba9b331..90bc7230afa8 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -602,38 +602,45 @@ static bool can_alloc_pages(void) !IS_ENABLED(CONFIG_PREEMPT_RT); } -static struct page *__bpf_alloc_page(int nid) +struct page *bpf_alloc_page(int nid, bool sleepable) { - if (!can_alloc_pages()) + if (!sleepable || !can_alloc_pages()) return alloc_pages_nolock(__GFP_ACCOUNT, nid, 0); return alloc_pages_node(nid, GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT - | __GFP_NOWARN, + | __GFP_NOWARN | __GFP_RETRY_MAYFAIL, 0); } -int bpf_map_alloc_pages(const struct bpf_map *map, int nid, - unsigned long nr_pages, struct page **pages) +void bpf_free_pages(struct llist_head *pages) { - unsigned long i, j; + struct llist_node *node; + struct page *page, *tmp; + + node = llist_del_all(pages); + llist_for_each_entry_safe(page, tmp, node, pcp_llist) + free_pages_nolock(page, 0); +} + +int bpf_alloc_pages(int nid, unsigned long nr_pages, + struct llist_head *pages, bool sleepable) +{ + unsigned long i; struct page *pg; - int ret = 0; for (i = 0; i < nr_pages; i++) { - pg = __bpf_alloc_page(nid); - - if (pg) { - pages[i] = pg; - continue; - } - for (j = 0; j < i; j++) - free_pages_nolock(pages[j], 0); - ret = -ENOMEM; - break; + pg = bpf_alloc_page(nid, sleepable); + if (!pg) + goto free_pages; + llist_add(&pg->pcp_llist, pages); } - return ret; + return 0; + +free_pages: + bpf_free_pages(pages); + return -ENOMEM; } static int btf_field_cmp(const void *a, const void *b) @@ -687,6 +694,7 @@ void btf_record_free(struct btf_record *rec) case BPF_REFCOUNT: case BPF_WORKQUEUE: case BPF_TASK_WORK: + case BPF_RCU_HEAD: /* Nothing to release */ break; default: @@ -741,6 +749,7 @@ struct btf_record *btf_record_dup(const struct btf_record *rec) case BPF_REFCOUNT: case BPF_WORKQUEUE: case BPF_TASK_WORK: + case BPF_RCU_HEAD: /* Nothing to acquire */ break; default: @@ -874,6 +883,7 @@ void bpf_obj_free_fields(const struct btf_record *rec, void *obj) case BPF_LIST_NODE: case BPF_RB_NODE: case BPF_REFCOUNT: + case BPF_RCU_HEAD: break; default: WARN_ON_ONCE(1); @@ -954,6 +964,9 @@ void bpf_map_put(struct bpf_map *map) /* bpf_map_free_id() must be called first */ bpf_map_free_id(map); + if (map->ops->map_free_pre_rcu) + map->ops->map_free_pre_rcu(map); + WARN_ON_ONCE(atomic64_read(&map->sleepable_refcnt)); /* RCU tasks trace grace period implies RCU grace period. */ if (READ_ONCE(map->free_after_mult_rcu_gp)) @@ -1277,7 +1290,7 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, map->record = btf_parse_fields(btf, value_type, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK | BPF_TIMER | BPF_KPTR | BPF_LIST_HEAD | BPF_RB_ROOT | BPF_REFCOUNT | BPF_WORKQUEUE | BPF_UPTR | - BPF_TASK_WORK, + BPF_TASK_WORK | BPF_RCU_HEAD, map->value_size); if (!IS_ERR_OR_NULL(map->record)) { int i; @@ -1319,6 +1332,12 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, goto free_map_tab; } break; + case BPF_RCU_HEAD: + if (map->map_type != BPF_MAP_TYPE_ARRAY) { + ret = -EOPNOTSUPP; + goto free_map_tab; + } + break; case BPF_KPTR_UNREF: case BPF_KPTR_REF: case BPF_KPTR_PERCPU: @@ -2036,7 +2055,7 @@ int generic_map_delete_batch(struct bpf_map *map, for (cp = 0; cp < max_count; cp++) { err = -EFAULT; - if (copy_from_user(key, keys + cp * map->key_size, + if (copy_from_user(key, keys + (size_t)cp * map->key_size, map->key_size)) break; @@ -2098,9 +2117,9 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file, for (cp = 0; cp < max_count; cp++) { err = -EFAULT; - if (copy_from_user(key, keys + cp * map->key_size, + if (copy_from_user(key, keys + (size_t)cp * map->key_size, map->key_size) || - copy_from_user(value, values + cp * value_size, value_size)) + copy_from_user(value, values + (size_t)cp * value_size, value_size)) break; err = bpf_map_update_value(map, map_file, key, value, @@ -2179,12 +2198,12 @@ int generic_map_lookup_batch(struct bpf_map *map, if (err) goto free_buf; - if (copy_to_user(keys + cp * map->key_size, key, + if (copy_to_user(keys + (size_t)cp * map->key_size, key, map->key_size)) { err = -EFAULT; goto free_buf; } - if (copy_to_user(values + cp * value_size, value, value_size)) { + if (copy_to_user(values + (size_t)cp * value_size, value, value_size)) { err = -EFAULT; goto free_buf; } @@ -2441,6 +2460,9 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) { struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); + btf_put(aux->btf); + kvfree(aux->jited_linfo); + kvfree(aux->linfo); kvfree(aux->func_info); kfree(aux->func_info_aux); free_uid(aux->user); @@ -2448,20 +2470,33 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) bpf_prog_free(aux->prog); } +/* + * Progs called from a trampoline can also be reached by a task that was + * preempted in the trampoline before the prog's enter helper took its RCU + * read lock, wait for those first. + */ +static void __bpf_prog_put_rcu_tasks(struct rcu_head *rcu) +{ + struct bpf_prog *prog = container_of(rcu, struct bpf_prog_aux, rcu)->prog; + + if (bpf_prog_runs_sleepable(prog)) + call_rcu_tasks_trace(rcu, __bpf_prog_put_rcu); + else + call_rcu(rcu, __bpf_prog_put_rcu); +} + static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) { - bpf_prog_kallsyms_del_all(prog); - btf_put(prog->aux->btf); module_put(prog->aux->mod); - kvfree(prog->aux->jited_linfo); - kvfree(prog->aux->linfo); kfree(prog->aux->kfunc_tab); kfree(prog->aux->ctx_arg_info); if (prog->aux->attach_btf) btf_put(prog->aux->attach_btf); if (deferred) { - if (prog->sleepable) + if (IS_ENABLED(CONFIG_TASKS_RCU) && prog->aux->tramp_linked) + call_rcu_tasks(&prog->aux->rcu, __bpf_prog_put_rcu_tasks); + else if (bpf_prog_runs_sleepable(prog)) call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu); else call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); @@ -2936,9 +2971,13 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at BPF_F_XDP_HAS_FRAGS | BPF_F_XDP_DEV_BOUND_ONLY | BPF_F_TEST_REG_INVARIANTS | + BPF_F_ARENA_SCALAR | BPF_F_TOKEN_FD)) return -EINVAL; + if ((attr->prog_flags & BPF_F_ARENA_SCALAR) && !bpf_jit_supports_arena_scalar()) + return -EOPNOTSUPP; + bpf_prog_load_fixup_attach_type(attr); if (attr->prog_flags & BPF_F_TOKEN_FD) { @@ -3068,6 +3107,10 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at prog->aux->user = get_current_user(); prog->len = attr->insn_cnt; + err = bpf_prog_stream_init(prog, GFP_USER); + if (err) + goto free_prog; + err = -EFAULT; if (copy_from_bpfptr(prog->insns, make_bpfptr(attr->insns, uattr.is_kernel), @@ -3807,6 +3850,19 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, err = -EINVAL; goto out_unlock; } + /* + * The size of these contexts depends on what the program is attached + * to, or run with. That is compared with max_ctx_offset of the program, + * and nobody looks at the one of its extensions. + */ + if (prog->type == BPF_PROG_TYPE_EXT && + (tgt_prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT || + tgt_prog->type == BPF_PROG_TYPE_TRACEPOINT || + tgt_prog->type == BPF_PROG_TYPE_SYSCALL) && + prog->aux->max_ctx_offset > tgt_prog->aux->max_ctx_offset) { + err = -EINVAL; + goto out_unlock; + } err = bpf_link_prime(&link->link.link, &link_primer); if (err) @@ -3976,19 +4032,19 @@ static int bpf_perf_link_fill_common(const struct perf_event *event, *ulenp = 1; } if (!uname) - return 0; + goto out; if (buf) { err = bpf_copy_to_user(uname, buf, ulen, len); - if (err) - return err; } else { char zero = '\0'; if (put_user(zero, uname)) - return -EFAULT; + err = -EFAULT; } - return 0; +out: + kfree_const(buf); + return err; } #ifdef CONFIG_KPROBE_EVENTS @@ -4045,6 +4101,7 @@ static void bpf_perf_link_fdinfo_kprobe(const struct perf_event *event, name, offset, missed, addr, type == BPF_FD_TYPE_KRETPROBE ? "kretprobe" : "kprobe", event->bpf_cookie); + kfree_const(name); } #endif @@ -4098,6 +4155,7 @@ static void bpf_perf_link_fdinfo_uprobe(const struct perf_event *event, name, offset, ref_ctr_offset, type == BPF_FD_TYPE_URETPROBE ? "uretprobe" : "uprobe", event->bpf_cookie); + kfree_const(name); } #endif @@ -4196,6 +4254,7 @@ static void bpf_tracepoint_link_show_fdinfo(const struct perf_event *event, "event_type:\t%s\n" "cookie:\t%llu\n", name, "tracepoint", event->bpf_cookie); + kfree_const(name); } static void bpf_probe_link_show_fdinfo(const struct perf_event *event, @@ -4755,6 +4814,7 @@ static int bpf_prog_query(const union bpf_attr *attr, case BPF_CGROUP_GETSOCKOPT: case BPF_CGROUP_SETSOCKOPT: case BPF_LSM_CGROUP: + case BPF_STRUCT_OPS: return cgroup_bpf_prog_query(attr, uattr, uattr_size); case BPF_LIRC_MODE2: return lirc_prog_query(attr, uattr); @@ -5027,6 +5087,12 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, insns[i].imm = BPF_FUNC_tail_call; /* fall-through */ } + /* + * BPF_JCOND is the same opcode as BPF_CALL_ARGS. The latter is for + * the interpreter, and a prog with unwind_goto is always JITed. + */ + if (prog->jited && bpf_is_unwind_goto(&insns[i])) + continue; if (code == (BPF_JMP | BPF_CALL) || code == (BPF_JMP | BPF_CALL_ARGS)) { /* Restore the legacy xlated dump layout. */ @@ -5403,7 +5469,11 @@ static int bpf_map_get_info_by_fd(struct file *file, info_len = min_t(u32, sizeof(info), info_len); memset(&info, 0, sizeof(info)); - if (copy_from_user(&info, uinfo, info_len)) + if (info_len >= offsetofend(struct bpf_map_info, hash) && + get_user(info.hash, &uinfo->hash)) + return -EFAULT; + if (info_len >= offsetofend(struct bpf_map_info, hash_size) && + get_user(info.hash_size, &uinfo->hash_size)) return -EFAULT; info.type = map->map_type; @@ -5718,11 +5788,13 @@ static int bpf_task_fd_query(const union bpf_attr *attr, err = bpf_get_perf_event_info(event, &prog_id, &fd_type, &buf, &probe_offset, &probe_addr, NULL); - if (!err) + if (!err) { err = bpf_task_fd_query_copy(attr, uattr, prog_id, fd_type, buf, probe_offset, probe_addr); + kfree_const(buf); + } goto put_file; } @@ -6042,7 +6114,10 @@ struct bpf_link *bpf_link_get_curr_or_next(u32 *id) again: link = idr_get_next(&link_idr, id); if (link) { - link = bpf_link_inc_not_zero(link); + if (link->id) + link = bpf_link_inc_not_zero(link); + else + link = ERR_PTR(-EAGAIN); if (IS_ERR(link)) { (*id)++; goto again; @@ -6200,7 +6275,7 @@ static int bpf_prog_bind_map(union bpf_attr *attr) /* The bpf program will not access the bpf map, but for the sake of * simplicity, increase sleepable_refcnt for sleepable program as well. */ - if (prog->sleepable) + if (bpf_prog_runs_sleepable(prog)) atomic64_inc(&map->sleepable_refcnt); memcpy(used_maps_new, used_maps_old, sizeof(used_maps_old[0]) * prog->aux->used_map_cnt); @@ -6300,6 +6375,28 @@ put_prog: return ret; } +#define BPF_PROG_STREAM_OPEN_LAST_FIELD prog_stream_open.flags + +static int prog_stream_open(union bpf_attr *attr) +{ + struct bpf_prog *prog; + u32 flags = attr->prog_stream_open.flags; + int ret; + + if (CHECK_ATTR(BPF_PROG_STREAM_OPEN)) + return -EINVAL; + if (flags & ~BPF_F_STREAM_NONBLOCK) + return -EINVAL; + + prog = bpf_prog_get(attr->prog_stream_open.prog_fd); + if (IS_ERR(prog)) + return PTR_ERR(prog); + + ret = bpf_prog_stream_new_fd(prog, attr->prog_stream_open.stream_id, flags); + bpf_prog_put(prog); + return ret; +} + static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size, bpfptr_t uattr_common, unsigned int size_common) { @@ -6472,6 +6569,9 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size, case BPF_PROG_ASSOC_STRUCT_OPS: err = prog_assoc_struct_ops(&attr); break; + case BPF_PROG_STREAM_OPEN: + err = prog_stream_open(&attr); + break; default: err = -EINVAL; break; diff --git a/kernel/bpf/sysfs_btf.c b/kernel/bpf/sysfs_btf.c index 9cbe15ce3540..556fe92c3649 100644 --- a/kernel/bpf/sysfs_btf.c +++ b/kernel/bpf/sysfs_btf.c @@ -6,8 +6,10 @@ #include <linux/module.h> #include <linux/kobject.h> #include <linux/init.h> +#include <linux/slab.h> #include <linux/sysfs.h> #include <linux/mm.h> +#include <linux/vmalloc.h> #include <linux/io.h> #include <linux/btf.h> @@ -15,16 +17,12 @@ extern char __start_BTF[]; extern char __stop_BTF[]; -static int btf_sysfs_vmlinux_mmap(struct file *filp, struct kobject *kobj, - const struct bin_attribute *attr, - struct vm_area_struct *vma) +static int btf_sysfs_mmap_check(void *data, size_t size, struct vm_area_struct *vma) { - unsigned long pages = PAGE_ALIGN(attr->size) >> PAGE_SHIFT; + unsigned long pages = PAGE_ALIGN(size) >> PAGE_SHIFT; size_t vm_size = vma->vm_end - vma->vm_start; - phys_addr_t addr = __pa_symbol(__start_BTF); - unsigned long pfn = addr >> PAGE_SHIFT; - if (attr->private != __start_BTF || !PAGE_ALIGNED(addr)) + if (!data || !PAGE_ALIGNED((unsigned long)data)) return -EINVAL; if (vma->vm_pgoff) @@ -33,24 +31,153 @@ static int btf_sysfs_vmlinux_mmap(struct file *filp, struct kobject *kobj, if (vma->vm_flags & (VM_WRITE | VM_EXEC | VM_MAYSHARE)) return -EACCES; - if (pfn + pages < pfn) - return -EINVAL; - if ((vm_size >> PAGE_SHIFT) > pages) return -EINVAL; vm_flags_mod(vma, VM_DONTDUMP, VM_MAYEXEC | VM_MAYWRITE); + return 0; +} + +static int btf_sysfs_mmap_image(struct file *filp, struct kobject *kobj, + const struct bin_attribute *attr, + struct vm_area_struct *vma) +{ + void *data = READ_ONCE(attr->private); + phys_addr_t addr; + unsigned long pfn; + size_t vm_size = vma->vm_end - vma->vm_start; + int err; + + err = btf_sysfs_mmap_check(data, attr->size, vma); + if (err) + return err; + + /* Kernel image addresses can lie inside the vmalloc range on arm64. */ + addr = __pa_symbol(data); + pfn = addr >> PAGE_SHIFT; + if (pfn + (PAGE_ALIGN(attr->size) >> PAGE_SHIFT) < pfn) + return -EINVAL; + return remap_pfn_range(vma, vma->vm_start, pfn, vm_size, vma->vm_page_prot); } static struct bin_attribute bin_attr_btf_vmlinux __ro_after_init = { .attr = { .name = "vmlinux", .mode = 0444, }, .read = sysfs_bin_attr_simple_read, - .mmap = btf_sysfs_vmlinux_mmap, + .mmap = btf_sysfs_mmap_image, }; struct kobject *btf_kobj; +struct btf_sysfs_entry { + struct bin_attribute attr; + char *module_name; +}; + +static void *btf_sysfs_lazy_data(const struct bin_attribute *attr) +{ + struct btf_sysfs_entry *entry = container_of(attr, struct btf_sysfs_entry, attr); + void *data; + + data = smp_load_acquire(&attr->private); + if (!data) { + request_module("%s", entry->module_name); + data = smp_load_acquire(&attr->private); + } + return data; +} + +static ssize_t btf_sysfs_read_lazy(struct file *filp, struct kobject *kobj, + const struct bin_attribute *attr, char *buf, + loff_t off, size_t count) +{ + void *data = btf_sysfs_lazy_data(attr); + + if (!data) + return -ENODEV; + + return memory_read_from_buffer(buf, count, &off, data, attr->size); +} + +static int btf_sysfs_mmap_lazy(struct file *filp, struct kobject *kobj, + const struct bin_attribute *attr, + struct vm_area_struct *vma) +{ + void *data = btf_sysfs_lazy_data(attr); + int err; + + if (!data) + return -ENODEV; + + err = btf_sysfs_mmap_check(data, attr->size, vma); + if (err) + return err; + + return remap_vmalloc_range(vma, data, 0); +} + +struct bin_attribute *sysfs_btf_add(const char *name, void *data, size_t data_size, + bool mmap, const char *lazy_module_name) +{ + struct btf_sysfs_entry *entry; + struct bin_attribute *attr; + int err; + + entry = kzalloc_obj(*entry); + if (!entry) + return ERR_PTR(-ENOMEM); + + attr = &entry->attr; + sysfs_bin_attr_init(attr); + attr->attr.mode = 0444; + attr->size = data_size; + attr->private = data; + attr->read = lazy_module_name ? btf_sysfs_read_lazy : sysfs_bin_attr_simple_read; + if (mmap) + attr->mmap = lazy_module_name ? btf_sysfs_mmap_lazy : btf_sysfs_mmap_image; + attr->attr.name = kstrdup(name, GFP_KERNEL); + if (!attr->attr.name) { + err = -ENOMEM; + goto err_free; + } + if (lazy_module_name) { + entry->module_name = kstrdup(lazy_module_name, GFP_KERNEL); + if (!entry->module_name) { + err = -ENOMEM; + goto err_free; + } + } + + err = sysfs_create_bin_file(btf_kobj, attr); + if (err) { + pr_warn("failed to register [%s] BTF in sysfs: %d\n", name, err); + goto err_free; + } + + return attr; + +err_free: + kfree(entry->module_name); + kfree(attr->attr.name); + kfree(entry); + return ERR_PTR(err); +} + +bool sysfs_btf_update(struct bin_attribute *attr, void *data) +{ + return attr && !cmpxchg(&attr->private, NULL, data); +} + +void sysfs_btf_remove(struct bin_attribute *attr) +{ + struct btf_sysfs_entry *entry = container_of(attr, struct btf_sysfs_entry, attr); + + sysfs_remove_bin_file(btf_kobj, attr); + kfree(entry->module_name); + kfree(attr->attr.name); + kfree(entry); +} + static int __init btf_vmlinux_init(void) { bin_attr_btf_vmlinux.private = __start_BTF; diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index 13e1aabe6f88..2ee7b3b3c10b 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -737,7 +737,11 @@ static const struct bpf_iter_seq_info task_vma_seq_info = { static struct bpf_iter_reg task_vma_reg_info = { .target = "task_vma", .attach_target = bpf_iter_attach_task, - .feature = BPF_ITER_RESCHED, + /* + * ->show() runs with the mmap_lock of the iterated mm held, which a + * sleepable program takes again when it faults on user memory. + */ + .feature = BPF_ITER_RESCHED | BPF_ITER_NOSLEEP, .ctx_arg_info_size = 2, .ctx_arg_info = { { offsetof(struct bpf_iter__task_vma, task), @@ -806,8 +810,14 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, vma = find_vma(mm, start); if (vma && vma->vm_start <= start && vma->vm_end > start) { + /* + * The callback must not sleep while mmap_lock is held, and the + * verifier checks it as an RCU read-side critical section. + */ + rcu_read_lock(); callback_fn((u64)(long)task, (u64)(long)vma, (u64)(long)callback_ctx, 0, 0); + rcu_read_unlock(); ret = 0; } bpf_mmap_unlock_mm(work, mm); @@ -869,7 +879,7 @@ __bpf_kfunc int bpf_iter_task_vma_new(struct bpf_iter_task_vma *it, BUILD_BUG_ON(sizeof(struct bpf_iter_task_vma_kern) != sizeof(struct bpf_iter_task_vma)); BUILD_BUG_ON(__alignof__(struct bpf_iter_task_vma_kern) != __alignof__(struct bpf_iter_task_vma)); - if (!IS_ENABLED(CONFIG_PER_VMA_LOCK)) { + if (!IS_ENABLED(CONFIG_MMU)) { kit->data = NULL; return -EOPNOTSUPP; } diff --git a/kernel/bpf/tnum.c b/kernel/bpf/tnum.c index ec9c310cf5d7..1f620facd1b7 100644 --- a/kernel/bpf/tnum.c +++ b/kernel/bpf/tnum.c @@ -317,3 +317,14 @@ u64 tnum_step(struct tnum t, u64 z) inc = (filled + 1) & t.mask; return t.value | inc; } + +/* + * Return the number of trailing bits known to be zero in a tnum. + * Return 64 for a tnum representing zero. + */ +u32 tnum_alignment(struct tnum a) +{ + u64 v = a.value | a.mask; + + return v ? __ffs64(v) : 64; +} diff --git a/kernel/bpf/token.c b/kernel/bpf/token.c index e85a179523f0..aa42243af16b 100644 --- a/kernel/bpf/token.c +++ b/kernel/bpf/token.c @@ -169,8 +169,11 @@ int bpf_token_create(union bpf_attr *attr) FD_PREPARE(fdf, O_CLOEXEC, alloc_file_pseudo(inode, path.mnt, BPF_TOKEN_INODE_NAME, O_RDWR, &bpf_token_fops)); - if (fdf.err) - return fdf.err; + if (fdf->fd < 0) { + /* no file took over the inode reference */ + iput(inode); + return fdf->fd; + } token = kzalloc_obj(*token, GFP_USER); if (!token) @@ -190,7 +193,7 @@ int bpf_token_create(union bpf_attr *attr) return err; get_user_ns(token->userns); - fd_prepare_file(fdf)->private_data = no_free_ptr(token); + fdf->file->private_data = no_free_ptr(token); return fd_publish(fdf); } diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 90b70ea0d370..21d460866dfe 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -401,6 +401,7 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip) head = &trampoline_ip_table[hash_64(tr->ip, TRAMPOLINE_HASH_BITS)]; hlist_add_head(&tr->hlist_ip, head); refcount_set(&tr->refcnt, 1); + INIT_LIST_HEAD(&tr->images); for (i = 0; i < BPF_TRAMP_MAX; i++) INIT_HLIST_HEAD(&tr->progs_hlist[i]); out: @@ -474,18 +475,11 @@ static int register_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im void *data __maybe_unused) { void *new_addr = im->image; - void *ip = tr->func.addr; - unsigned long faddr; int ret; - faddr = ftrace_location((unsigned long)ip); - if (faddr) { + if (tr->func.ftrace_managed) { if (!tr->fops) return -ENOTSUPP; - tr->func.ftrace_managed = true; - } - - if (tr->func.ftrace_managed) { ret = direct_ops_add(tr, new_addr); } else { ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr); @@ -565,15 +559,22 @@ static void bpf_tramp_image_free(struct bpf_tramp_image *im) arch_free_bpf_trampoline(im->image, im->size); bpf_jit_uncharge_modmem(im->size); percpu_ref_exit(&im->pcref); + kfree(im->skips); kfree_rcu(im, rcu); } static void __bpf_tramp_image_put_deferred(struct work_struct *work) { struct bpf_tramp_image *im; + struct bpf_trampoline *tr; im = container_of(work, struct bpf_tramp_image, work); + tr = im->tr; + trampoline_lock(tr); + list_del(&im->list); + trampoline_unlock(tr); bpf_tramp_image_free(im); + bpf_trampoline_put(tr); } /* callback, fexit step 3 or fentry step 2 */ @@ -601,7 +602,7 @@ static void __bpf_tramp_image_put_rcu_tasks(struct rcu_head *rcu) struct bpf_tramp_image *im; im = container_of(rcu, struct bpf_tramp_image, rcu); - if (im->ip_after_call) + if (im->call_orig) /* the case of fmod_ret/fexit trampoline and CONFIG_PREEMPTION=y */ percpu_ref_kill(&im->pcref); else @@ -621,9 +622,9 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im) * * The trampoline is unreachable before bpf_tramp_image_put(). * - * First, patch the trampoline to avoid calling into fexit progs. - * The progs will be freed even if the original function is still - * executing or sleeping. + * Progs are patched out of the image when they are detached, see + * bpf_trampoline_skip_prog(), so they can be freed even if a task is + * still in the image. * In case of CONFIG_PREEMPT=y use call_rcu_tasks() to wait on * first few asm instructions to execute and call into * __bpf_tramp_enter->percpu_ref_get. @@ -637,11 +638,7 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im) * percpu_ref_kill will be waiting for. Hence the first * call_rcu_tasks() is not necessary. */ - if (im->ip_after_call) { - int err = bpf_arch_text_poke(im->ip_after_call, BPF_MOD_NOP, - BPF_MOD_JUMP, NULL, - im->ip_epilogue); - WARN_ON(err); + if (im->call_orig) { if (IS_ENABLED(CONFIG_TASKS_RCU)) call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu_tasks); else @@ -658,7 +655,7 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im) call_rcu_tasks_trace(&im->rcu, __bpf_tramp_image_put_rcu_tasks); } -static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size) +static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size, int nr_progs) { struct bpf_tramp_image *im; struct bpf_ksym *ksym; @@ -669,6 +666,10 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size) if (!im) goto out; + im->skips = kzalloc_objs(*im->skips, nr_progs); + if (!im->skips) + goto out_free_im; + err = bpf_jit_charge_modmem(size); if (err) goto out_free_im; @@ -695,6 +696,7 @@ out_free_image: out_uncharge: bpf_jit_uncharge_modmem(size); out_free_im: + kfree(im->skips); kfree(im); out: return ERR_PTR(err); @@ -771,11 +773,12 @@ again: goto out; } - im = bpf_tramp_image_alloc(tr->key, size); + im = bpf_tramp_image_alloc(tr->key, size, total); if (IS_ERR(im)) { err = PTR_ERR(im); goto out; } + im->call_orig = tr->flags & BPF_TRAMP_F_CALL_ORIG; err = arch_prepare_bpf_trampoline(im, im->image, im->image + size, &tr->func.model, tr->flags, tnodes, @@ -806,8 +809,14 @@ again: #endif out_free: - if (err) + if (err) { bpf_tramp_image_free(im); + } else { + /* track the image until it is freed, for bpf_trampoline_skip_prog() */ + refcount_inc(&tr->refcnt); + im->tr = tr; + list_add(&im->list, &tr->images); + } out: /* If any error happens, restore previous flags */ if (err) @@ -843,7 +852,7 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) } } -static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog) +static int bpf_freplace_link_tgt_prog(struct bpf_prog *tgt_prog) { struct bpf_prog_aux *aux = tgt_prog->aux; @@ -857,7 +866,7 @@ static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog) */ return -EBUSY; - aux->is_extended = true; + aux->freplace_link_cnt++; return 0; } @@ -907,6 +916,7 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, } hlist_add_head(&node->tramp_hlist, prog_list); + node->link->prog->aux->tramp_linked = true; if (kind == BPF_TRAMP_FSESSION) { tr->progs_cnt[BPF_TRAMP_FENTRY]++; fexit = fsession_exit(node); @@ -920,6 +930,41 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, return 0; } +/* + * Patch the nop in front of a prog call to a jump over it. A task can be + * preempted anywhere in the image, so archs that need several instructions for + * a jump of any range patch a single near branch here instead. + */ +int __weak arch_bpf_trampoline_skip(void *nop, void *target) +{ + return bpf_arch_text_poke(nop, BPF_MOD_NOP, BPF_MOD_JUMP, NULL, target); +} + +/* + * prog was detached and can be freed, but tasks may still be running in images + * that call it, sleeping in an earlier prog for example. They can be in any + * image that is not freed yet, not only in cur_image, so patch all of them to + * jump over prog. + */ +static void bpf_trampoline_skip_prog(struct bpf_trampoline *tr, struct bpf_prog *prog) +{ + struct bpf_tramp_image *im; + int i, err; + + list_for_each_entry(im, &tr->images, list) { + for (i = 0; i < im->nr_skips; i++) { + struct bpf_tramp_skip *skip = &im->skips[i]; + + if (skip->prog != prog) + continue; + err = arch_bpf_trampoline_skip(skip->nop, skip->target); + WARN_ON_ONCE(err); + /* not a nop anymore, and prog's address can be reused */ + skip->prog = NULL; + } + } +} + static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr, struct bpf_tramp_node *node) { @@ -937,6 +982,7 @@ static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr, } hlist_del_init(&node->tramp_hlist); tr->progs_cnt[kind]--; + bpf_trampoline_skip_prog(tr, node->link->prog); } static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node, @@ -970,13 +1016,20 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node, /* Cannot attach extension if fentry/fexit are in use. */ if (cnt) return -EBUSY; - err = bpf_freplace_check_tgt_prog(tgt_prog); + err = bpf_freplace_link_tgt_prog(tgt_prog); if (err) return err; + err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_NOP, + BPF_MOD_JUMP, NULL, + node->link->prog->bpf_func); + if (err) { + /* Undo the claim from bpf_freplace_link_tgt_prog(). */ + guard(mutex)(&tgt_prog->aux->ext_mutex); + tgt_prog->aux->freplace_link_cnt--; + return err; + } tr->extension_prog = node->link->prog; - return bpf_arch_text_poke(tr->func.addr, BPF_MOD_NOP, - BPF_MOD_JUMP, NULL, - node->link->prog->bpf_func); + return 0; } err = bpf_trampoline_add_prog(tr, node, cnt); if (err) @@ -1016,7 +1069,7 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_node *node, tr->extension_prog->bpf_func, NULL); tr->extension_prog = NULL; guard(mutex)(&tgt_prog->aux->ext_mutex); - tgt_prog->aux->is_extended = false; + tgt_prog->aux->freplace_link_cnt--; return err; } bpf_trampoline_remove_prog(tr, node); @@ -1226,6 +1279,7 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key, memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel)); tr->func.addr = (void *)tgt_info->tgt_addr; + tr->func.ftrace_managed = tr->ip != 0; out: trampoline_unlock(tr); return tr; @@ -1245,11 +1299,9 @@ void bpf_trampoline_put(struct bpf_trampoline *tr) if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[i]))) goto out; - /* This code will be executed even when the last bpf_tramp_image - * is alive. All progs are detached from the trampoline and the - * trampoline image is patched with jmp into epilogue to skip - * fexit progs. The fentry-only trampoline will be freed via - * multiple rcu callbacks. + /* + * All progs are detached and the last image has been freed, images + * hold a reference on the trampoline until then. */ hlist_del(&tr->hlist_key); hlist_del(&tr->hlist_ip); @@ -1577,13 +1629,20 @@ static void ftrace_hash_add(struct ftrace_hash *hash, struct ftrace_func_entry * static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *ptr) { unsigned long addr = (unsigned long) im->image; - unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; + unsigned long ip = tr->ip; + + /* + * Different BTF ids get different trampolines, but they can still + * resolve to the same address (aliases, functions of the same name). + * There's only one call site to attach to. + */ + if (ftrace_lookup_ip(data->reg, ip)) + return -EINVAL; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); - tr->func.ftrace_managed = true; ftrace_hash_add(data->reg, data->entry, ip, addr); tr->cur_image = im; return 0; @@ -1592,13 +1651,12 @@ static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_ima static int unregister_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, void *ptr) { unsigned long addr = (unsigned long) tr->cur_image->image; - unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); - ftrace_hash_add(data->unreg, data->entry, ip, addr); + ftrace_hash_add(data->unreg, data->entry, tr->ip, addr); tr->cur_image = NULL; return 0; } @@ -1607,13 +1665,12 @@ static int modify_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, struct bool lock_direct_mutex, void *ptr) { unsigned long addr = (unsigned long) im->image; - unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); - ftrace_hash_add(data->modify, data->entry, ip, addr); + ftrace_hash_add(data->modify, data->entry, tr->ip, addr); tr->cur_image = im; return 0; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 72a3f5998dd2..5601d2056218 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5,6 +5,7 @@ */ #include <uapi/linux/btf.h> #include <linux/bpf-cgroup.h> +#include <linux/count_zeros.h> #include <linux/kernel.h> #include <linux/types.h> #include <linux/slab.h> @@ -24,11 +25,13 @@ #include <linux/bpf_lsm.h> #include <linux/security.h> #include <linux/verification.h> +#include <linux/keyctl.h> #include <linux/btf_ids.h> #include <linux/poison.h> #include <linux/module.h> #include <linux/cpumask.h> #include <linux/cnum.h> +#include <linux/gcd.h> #include <linux/bpf_mem_alloc.h> #include <net/xdp.h> #include <linux/trace_events.h> @@ -216,6 +219,7 @@ static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg); static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg); static inline bool in_sleepable_context(struct bpf_verifier_env *env); +static bool in_rcu_callback(struct bpf_verifier_env *env); static const char *non_sleepable_context_description(struct bpf_verifier_env *env); static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg); static void scalar_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg); @@ -301,6 +305,12 @@ static int arg_idx_from_argno(argno_t a) return arg_from_argno(a) - 1; } +/* Normalize helper register numbers and kfunc argument numbers to ABI slots. */ +static u32 arg_slot_from_argno(argno_t a) +{ + return abs(a.argno) - 1; +} + static const char *btf_type_name(const struct btf *btf, u32 id) { return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); @@ -394,27 +404,69 @@ bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog) return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL; } -static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) +static const struct btf_type *subprog_ret_type(struct bpf_verifier_env *env, int subprog) { - const struct btf_type *type, *func, *func_proto; + const struct btf_type *func, *func_proto; const struct btf *btf = env->prog->aux->btf; u32 btf_id; + if (!btf || !env->prog->aux->func_info) + return NULL; + btf_id = env->prog->aux->func_info[subprog].type_id; + /* Both already validated by prepare_btf_func() at prog load. */ func = btf_type_by_id(btf, btf_id); - if (verifier_bug_if(!func, env, "btf_id %u not found", btf_id)) - return false; - func_proto = btf_type_by_id(btf, func->type); - if (!func_proto) - return false; - type = btf_type_skip_modifiers(btf, func_proto->type, NULL); - if (!type) - return false; + return btf_type_skip_modifiers(btf, func_proto->type, NULL); +} + +static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) +{ + const struct btf_type *type = subprog_ret_type(env, subprog); + + return type && btf_type_is_void(type); +} + +static u32 ret_regs_cnt(u32 size) +{ + return size > 8 && size <= 16 ? 2 : 1; +} + +/* Registers holding a function return value, in order. See ret_regs_cnt(). */ +static const int ret_regs[] = { BPF_REG_0, BPF_REG_2 }; + +static int bpf_compute_subprog_ret_regs(struct bpf_verifier_env *env) +{ + const struct btf *btf = env->prog->aux->btf; + const struct btf_type *type; + int subprog; + u32 size; - return btf_type_is_void(type); + if (!env->prog->jit_requested || bpf_prog_is_offloaded(env->prog->aux)) + return 0; + + /* + * Skip the main program: its return value is the program's exit code, + * read out of R0, so it never uses the register pair. An extension does + * have a real prototype for subprog 0, but btf_check_func_type_match() + * refuses to replace a function returning more than 8 bytes. + */ + for (subprog = 1; subprog < env->subprog_cnt; subprog++) { + type = subprog_ret_type(env, subprog); + if (!type || btf_type_is_void(type)) + continue; + if (verifier_bug_if(IS_ERR(btf_resolve_size(btf, type, &size)), env, + "cannot size return type of subprog %d", subprog)) + return -EFAULT; + if (ret_regs_cnt(size) > 1) { + subprog_info(env, subprog)->ret_reg_pair = true; + env->prog->jit_required = 1; + } + } + + return 0; } const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog) @@ -486,6 +538,7 @@ static bool is_ptr_cast_function(enum bpf_func_id func_id) static bool is_sync_callback_calling_kfunc(u32 btf_id); static bool is_async_callback_calling_kfunc(u32 btf_id); +static bool is_call_rcu_kfunc(u32 btf_id); static bool is_callback_calling_kfunc(u32 btf_id); static bool is_bpf_wq_set_callback_kfunc(u32 btf_id); @@ -528,6 +581,10 @@ static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_timer_set_callback) return false; + /* bpf_call_rcu and bpf_call_rcu_tasks_trace callbacks are never sleepable. */ + if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 && is_call_rcu_kfunc(insn->imm)) + return false; + /* bpf_wq and bpf_task_work callbacks are always sleepable. */ if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 && (is_bpf_wq_set_callback_kfunc(insn->imm) || is_task_work_add_kfunc(insn->imm))) @@ -537,6 +594,19 @@ static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn return false; } +/* From here on the maps of the program count it as a sleepable user. */ +static void mark_sleepable_async_cb(struct bpf_verifier_env *env) +{ + int i; + + if (bpf_prog_runs_sleepable(env->prog)) + return; + + env->prog->aux->sleepable_async_cb = true; + for (i = 0; i < env->used_map_cnt; i++) + atomic64_inc(&env->used_maps[i]->sleepable_refcnt); +} + bool bpf_is_may_goto_insn(struct bpf_insn *insn) { return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO; @@ -544,16 +614,17 @@ bool bpf_is_may_goto_insn(struct bpf_insn *insn) static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots) { - int allocated_slots = state->allocated_stack / BPF_REG_SIZE; + int allocated_slots = bpf_stack_nr_slots(state); - /* We need to check that slots between [spi - nr_slots + 1, spi] are - * within [0, allocated_stack). - * - * Please note that the spi grows downwards. For example, a dynptr - * takes the size of two stack slots; the first slot will be at - * spi and the second slot will be at spi - 1. - */ - return spi - nr_slots + 1 >= 0 && spi < allocated_slots; + /* + * We need to check that slots between [spi - nr_slots + 1, spi] are + * within [0, allocated_stack). + * + * Please note that the spi grows downwards. For example, a dynptr + * takes the size of two stack slots; the first slot will be at + * spi and the second slot will be at spi - 1. + */ + return spi - nr_slots + 1 >= 0 && spi < allocated_slots; } static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, @@ -567,7 +638,7 @@ static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_s } off = reg->var_off.value; - if (off % BPF_REG_SIZE) { + if (off >= 0 || off % BPF_REG_SIZE) { verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off); return -EINVAL; } @@ -668,6 +739,21 @@ static void mark_dynptr_cb_reg(struct bpf_verifier_env *env, static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi); +static int destroy_dynptrs_in_stack_slots(struct bpf_verifier_env *env, + struct bpf_func_state *state, + int first_spi, int last_spi) +{ + int spi, err; + + for (spi = first_spi; spi <= last_spi; spi++) { + err = destroy_if_dynptr_stack_slot(env, state, spi); + if (err) + return err; + } + + return 0; +} + static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg, enum bpf_arg_type arg_type, int insn_idx, struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr) @@ -681,7 +767,7 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ return spi; /* We cannot assume both spi and spi - 1 belong to the same dynptr, - * hence we need to call destroy_if_dynptr_stack_slot twice for both, + * hence we need to destroy dynptrs in both slots, * to ensure that for the following example: * [d1][d1][d2][d2] * spi 3 2 1 0 @@ -689,16 +775,13 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ * case they do belong to same dynptr, second call won't see slot_type * as STACK_DYNPTR and will simply skip destruction. */ - err = destroy_if_dynptr_stack_slot(env, state, spi); - if (err) - return err; - err = destroy_if_dynptr_stack_slot(env, state, spi - 1); + err = destroy_dynptrs_in_stack_slots(env, state, spi - 1, spi); if (err) return err; for (i = 0; i < BPF_REG_SIZE; i++) { - state->stack[spi].slot_type[i] = STACK_DYNPTR; - state->stack[spi - 1].slot_type[i] = STACK_DYNPTR; + bpf_stack_slot(state, spi)->slot_type[i] = STACK_DYNPTR; + bpf_stack_slot(state, spi - 1)->slot_type[i] = STACK_DYNPTR; } type = arg_to_dynptr_type(arg_type); @@ -731,8 +814,8 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ parent_id = dynptr->parent_id; } - mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr, - &state->stack[spi - 1].spilled_ptr, type, parent_id); + mark_dynptr_stack_regs(env, &bpf_stack_slot(state, spi)->spilled_ptr, + &bpf_stack_slot(state, spi - 1)->spilled_ptr, type, parent_id); return 0; } @@ -764,7 +847,7 @@ static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_re * all clones and derived slices. For non-referenced dynptr, only * the dynptr and slices derived from it will be invalidated. */ - reg = &state->stack[spi].spilled_ptr; + reg = &bpf_stack_slot(state, spi)->spilled_ptr; return release_reference(env, dynptr_type_referenced(reg->dynptr.type) ? reg->parent_id : reg->id); @@ -803,6 +886,7 @@ static int dynptr_ref_cnt(struct bpf_verifier_env *env, int v_parent_id) static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi) { + struct bpf_stack_state *slot = bpf_stack_slot(state, spi); int err = 0; /* We always ensure that STACK_DYNPTR is never set partially, @@ -810,20 +894,22 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, * different for STACK_SPILL, where it may be only set for * 1 byte, so code has to use is_spilled_reg. */ - if (state->stack[spi].slot_type[0] != STACK_DYNPTR) + if (slot->slot_type[0] != STACK_DYNPTR) return 0; /* Reposition spi to first slot */ - if (!state->stack[spi].spilled_ptr.dynptr.first_slot) + if (!slot->spilled_ptr.dynptr.first_slot) { spi = spi + 1; + slot = bpf_stack_slot(state, spi); + } /* * A referenced dynptr can be overwritten only if there is at * least one other dynptr sharing the same virtual ref parent, * ensuring the reference can still be properly released. */ - if (dynptr_type_referenced(state->stack[spi].spilled_ptr.dynptr.type) && - dynptr_ref_cnt(env, state->stack[spi].spilled_ptr.parent_id) <= 1) { + if (dynptr_type_referenced(slot->spilled_ptr.dynptr.type) && + dynptr_ref_cnt(env, slot->spilled_ptr.parent_id) <= 1) { verbose(env, "cannot overwrite referenced dynptr\n"); bpf_diag_res( env, env->insn_idx, "referenced dynptr overwrite", @@ -833,7 +919,7 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, } /* Invalidate the dynptr and any derived slices */ - err = release_reference(env, state->stack[spi].spilled_ptr.id); + err = release_reference(env, slot->spilled_ptr.id); if (!err) { mark_stack_slot_scratched(env, spi); mark_stack_slot_scratched(env, spi - 1); @@ -873,6 +959,7 @@ static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_ static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg) { struct bpf_func_state *state = bpf_func(env, reg); + struct bpf_stack_state *slot; int i, spi; /* This already represents first slot of initialized bpf_dynptr. @@ -887,12 +974,13 @@ static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_re spi = dynptr_get_spi(env, reg); if (spi < 0) return false; - if (!state->stack[spi].spilled_ptr.dynptr.first_slot) + slot = bpf_stack_slot(state, spi); + if (!slot->spilled_ptr.dynptr.first_slot) return false; for (i = 0; i < BPF_REG_SIZE; i++) { - if (state->stack[spi].slot_type[i] != STACK_DYNPTR || - state->stack[spi - 1].slot_type[i] != STACK_DYNPTR) + if (slot->slot_type[i] != STACK_DYNPTR || + bpf_stack_slot(state, spi - 1)->slot_type[i] != STACK_DYNPTR) return false; } @@ -911,14 +999,23 @@ static enum bpf_dynptr_type dynptr_reg_type(struct bpf_verifier_env *env, struct if (spi < 0) return BPF_DYNPTR_TYPE_INVALID; state = bpf_func(env, reg); - return state->stack[spi].spilled_ptr.dynptr.type; + return bpf_stack_slot(state, spi)->spilled_ptr.dynptr.type; +} + +static bool dynptr_write_changes_pkt_data(enum bpf_dynptr_type type) +{ + return type == BPF_DYNPTR_TYPE_SKB || type == BPF_DYNPTR_TYPE_SKB_META; } static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, enum bpf_arg_type arg_type) { - /* ARG_PTR_TO_DYNPTR takes any type of dynptr */ - if (arg_type == ARG_PTR_TO_DYNPTR) + /* + * ARG_PTR_TO_DYNPTR without a type flag takes any type of dynptr. + * Test the flags rather than the whole arg_type, which may carry + * unrelated ones such as PTR_MAYBE_NULL. + */ + if (!(arg_type & DYNPTR_TYPE_FLAG_MASK)) return true; return dynptr_reg_type(env, reg) == arg_to_dynptr_type(arg_type); @@ -947,7 +1044,7 @@ static int mark_stack_slots_iter(struct bpf_verifier_env *env, return id; for (i = 0; i < nr_slots; i++) { - struct bpf_stack_state *slot = &state->stack[spi - i]; + struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); struct bpf_reg_state *st = &slot->spilled_ptr; __mark_reg_known_zero(st); @@ -984,7 +1081,7 @@ static int unmark_stack_slots_iter(struct bpf_verifier_env *env, return spi; for (i = 0; i < nr_slots; i++) { - struct bpf_stack_state *slot = &state->stack[spi - i]; + struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); struct bpf_reg_state *st = &slot->spilled_ptr; if (i == 0) @@ -1018,7 +1115,7 @@ static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env, return false; for (i = 0; i < nr_slots; i++) { - struct bpf_stack_state *slot = &state->stack[spi - i]; + struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); for (j = 0; j < BPF_REG_SIZE; j++) if (slot->slot_type[j] == STACK_ITER) @@ -1039,7 +1136,7 @@ static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_s return -EINVAL; for (i = 0; i < nr_slots; i++) { - struct bpf_stack_state *slot = &state->stack[spi - i]; + struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); struct bpf_reg_state *st = &slot->spilled_ptr; if (st->type & PTR_UNTRUSTED) @@ -1081,7 +1178,7 @@ static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env, if (id < 0) return id; - slot = &state->stack[spi]; + slot = bpf_stack_slot(state, spi); st = &slot->spilled_ptr; __mark_reg_known_zero(st); @@ -1108,7 +1205,7 @@ static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_r if (spi < 0) return spi; - slot = &state->stack[spi]; + slot = bpf_stack_slot(state, spi); st = &slot->spilled_ptr; if (st->irq.kfunc_class != kfunc_class) { @@ -1177,7 +1274,7 @@ static bool is_irq_flag_reg_valid_uninit(struct bpf_verifier_env *env, struct bp if (spi < 0) return false; - slot = &state->stack[spi]; + slot = bpf_stack_slot(state, spi); for (i = 0; i < BPF_REG_SIZE; i++) if (slot->slot_type[i] == STACK_IRQ_FLAG) @@ -1196,7 +1293,7 @@ static int is_irq_flag_reg_valid_init(struct bpf_verifier_env *env, struct bpf_r if (spi < 0) return -EINVAL; - slot = &state->stack[spi]; + slot = bpf_stack_slot(state, spi); st = &slot->spilled_ptr; if (!st->id) @@ -1343,7 +1440,7 @@ static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src) { - size_t n = src->allocated_stack / BPF_REG_SIZE; + size_t n = bpf_stack_nr_slots(src); dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state), GFP_KERNEL_ACCOUNT); @@ -1380,9 +1477,9 @@ static int resize_reference_state(struct bpf_verifier_state *state, size_t n) /* Possibly update state->allocated_stack to be at least size bytes. Also * possibly update the function's high-water mark in its bpf_subprog_info. */ -static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size) +int bpf_grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size) { - size_t old_n = state->allocated_stack / BPF_REG_SIZE, n; + size_t old_n = bpf_stack_nr_slots(state), n; /* The stack size is always a multiple of BPF_REG_SIZE. */ size = round_up(size, BPF_REG_SIZE); @@ -1603,6 +1700,7 @@ static void free_func_state(struct bpf_func_state *state) { if (!state) return; + kfree(state->loop_stack); kfree(state->stack_arg_regs); kfree(state->stack); kfree(state); @@ -1639,6 +1737,10 @@ static int copy_func_state(struct bpf_func_state *dst, memcpy(dst, src, offsetof(struct bpf_func_state, stack)); /* Instruction accounting is path-local, not part of verifier state. */ dst->insns_subtotal = 0; + dst->loop_stack = copy_array(dst->loop_stack, src->loop_stack, src->loop_stack_cnt, + sizeof(*src->loop_stack), GFP_KERNEL_ACCOUNT); + if (!dst->loop_stack) + return -ENOMEM; return copy_stack_state(dst, src); } @@ -1667,6 +1769,7 @@ int bpf_copy_verifier_state(struct bpf_verifier_state *dst_state, return err; dst_state->speculative = src->speculative; dst_state->in_sleepable = src->in_sleepable; + dst_state->unwound = src->unwound; dst_state->curframe = src->curframe; dst_state->branches = src->branches; dst_state->parent = src->parent; @@ -1846,12 +1949,124 @@ static void bpf_diag_record_caller_saved(struct bpf_verifier_env *env, } } +static void reg_step_reset(struct bpf_reg_state *reg) +{ + reg->base = 0; + reg->step = 1; +} + +static void scalar_step_add(struct bpf_reg_state *dst_reg, + const struct bpf_reg_state *a, + const struct bpf_reg_state *b) +{ + const struct bpf_reg_state *reg; + s64 amount, tmp; + + if (tnum_is_const(b->var_off)) { + reg = a; + amount = (s64)b->var_off.value; + } else if (tnum_is_const(a->var_off)) { + reg = b; + amount = (s64)a->var_off.value; + } else { + reg_step_reset(dst_reg); + return; + } + + /* + * Let x be a possible signed value of the register, + * d = amount, s = reg->step, and b = reg->base. + * Then x mod s = b, with s > 0 and 0 <= b < s. + * If the addition causes signed underflow, the resulting value + * is x + d + 2**64. + * + * Taking the result modulo s and substituting x mod s = b gives: + * (x + d + 2**64) mod s = (b + d + (2**64 mod s)) mod s (1) + * Keeping reg->step = s and updating reg->base to (b + d) mod s + * requires (1) to equal (b + d) mod s, which holds if and only + * if 2**64 mod s = 0. + * Signed overflow subtracts 2**64 and has the same requirement. + * + * Conservatively reset the stride if either wrap is possible. + */ + if (check_add_overflow(reg_smin(reg), amount, &tmp) || + check_add_overflow(reg_smax(reg), amount, &tmp)) { + reg_step_reset(dst_reg); + return; + } + + dst_reg->base = (reg->base + imod(amount, reg->step)) % reg->step; + dst_reg->step = reg->step; +} + +static void scalar_step_scale(struct bpf_reg_state *dst_reg, u64 amount) +{ + u16 step; + s64 tmp; + + /* + * Using the same notation as scalar_step_add(), with d > 0, + * the scaled value satisfies (x * d) mod (s * d) = b * d. + * If the multiplication wraps, its signed result can be represented + * as x * d - n * 2**64 for some nonzero integer n. + * + * Taking the result modulo s * d gives: + * (x * d - n * 2**64) mod (s * d) = + * (b * d - (n * 2**64 mod (s * d))) mod (s * d) (1) + * Updating dst_reg->base to b * d and dst_reg->step to s * d + * requires (1) to equal b * d, which holds if and only if + * n * 2**64 mod (s * d) = 0. + * + * Conservatively reset the stride if a wrap is possible. + */ + if (amount == 0 || check_mul_overflow(dst_reg->step, amount, &step) || + check_mul_overflow(reg_smin(dst_reg), (s64)amount, &tmp) || + check_mul_overflow(reg_smax(dst_reg), (s64)amount, &tmp)) { + reg_step_reset(dst_reg); + return; + } + /* + * The above established that `step = dst_reg->step * amount` + * does not overflow, therefore the base < step invariant holds + * after assignments below. + */ + dst_reg->base = dst_reg->base * amount; + dst_reg->step = step; +} + +/* + * Let x be a possible signed value of the register, + * s = reg->step, and b = reg->base. + * Then x mod s = b, with s > 0 and 0 <= b < s. + * Truncating to N bits and then zero- or sign-extending x + * is equivalent to computing x + k * 2**N for some integer k. + * For example, sign-extending the low 8 bits of x = 255 can + * be expressed as 255 - 2**8 = -1. + * + * Taking the result modulo s and substituting x mod s = b gives: + * (x + k * 2**N) mod s = (b + (k * 2**N mod s)) mod s + * Keeping reg->base = b and reg->step = s requires this to equal b, + * which holds if and only if k * 2**N mod s = 0. + * Continuing the example with s = 3 and b = 0, we have 255 mod 3 = 0, + * but (-1) mod 3 = 2, so the original base and step no longer hold. + * + * Conservatively reset the stride unless all possible values fit in + * the range where the conversion leaves them unchanged. + * Call before updating the register's bounds. Requires 0 < bits < 64. + */ +static void reg_step_check_unsigned(struct bpf_reg_state *reg, u32 bits) +{ + if (reg_umax(reg) >= (1ULL << bits)) + reg_step_reset(reg); +} + /* This helper doesn't clear reg->id */ static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) { reg->var_off = tnum_const(imm); reg->r64 = cnum64_from_urange(imm, imm); reg->r32 = cnum32_from_urange((u32)imm, (u32)imm); + reg_step_reset(reg); } /* Mark the unknown part of a register (variable offset or scalar value) as @@ -1864,9 +2079,16 @@ static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); reg->id = 0; reg->parent_id = 0; + reg->map_uid = 0; ___mark_reg_known(reg, imm); } +void bpf_mark_reg_known_scalar(struct bpf_reg_state *reg, u64 imm) +{ + __mark_reg_known(reg, imm); + reg->type = SCALAR_VALUE; +} + static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm) { reg->var_off = tnum_const_subreg(reg->var_off, imm); @@ -1925,17 +2147,18 @@ static void refine_map_lookup_value(struct bpf_reg_state *reg) if (map->inner_map_meta) { reg->type = CONST_PTR_TO_MAP | maybe_null; reg->map_ptr = map->inner_map_meta; - /* transfer reg's id which is unique for every map_lookup_elem + /* + * transfer reg's id which is unique for every map_lookup_elem * as UID of the inner map. */ - if (btf_record_has_field(map->inner_map_meta->record, - BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) - reg->map_uid = reg->id; + reg->map_uid = reg->id; } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { reg->type = PTR_TO_XDP_SOCK | maybe_null; + reg->map_uid = 0; } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || map->map_type == BPF_MAP_TYPE_SOCKHASH) { reg->type = PTR_TO_SOCKET | maybe_null; + reg->map_uid = 0; } } @@ -2107,8 +2330,24 @@ static void __reg_bound_offset(struct bpf_reg_state *reg) struct tnum var32_off = tnum_intersect(tnum_subreg(var64_off), tnum_range(reg_u32_min(reg), reg_u32_max(reg))); + u32 trailing_zero_bits; + u16 base = reg->base; + u16 step = reg->step; reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off); + + if (base == 0) + trailing_zero_bits = count_trailing_zeros(step); + else + trailing_zero_bits = min(count_trailing_zeros(base), + count_trailing_zeros(step)); + reg->var_off = tnum_and(reg->var_off, tnum_const(~0ULL << trailing_zero_bits)); +} + +static void __update_bounds_from_step(struct bpf_reg_state *reg) +{ + reg->r64 = cnum64_intersect_linear(reg->r64, reg->base, reg->step); + deduce_bounds_32_from_64(reg); } static bool range_bounds_violation(struct bpf_reg_state *reg); @@ -2130,6 +2369,15 @@ static void reg_bounds_sync(struct bpf_reg_state *reg) * then new var_off is (0; 0x7f...fc) which improves our umax. */ __update_reg_bounds(reg); + /* + * Both tnum and base+step define a sequence of points, + * potentially w/o intersection, or intersecting after N + * tnum_step()'s, where N can be large. On the other hand, + * step+base *must* be in sync with r64 in order for + * array traversal logic starting at reg_smin() to be sound. + * Hence, place this as a final step in the reg_bounds_sync(). + */ + __update_bounds_from_step(reg); } static bool const_tnum_range_mismatch(struct bpf_reg_state *reg) @@ -2173,17 +2421,24 @@ static int reg_bounds_sanity_check(struct bpf_verifier_env *env, goto out; } + if (reg->step == 0 || reg->base >= reg->step) { + msg = "invalid base/step"; + goto out; + } + return 0; out: verifier_bug(env, "REG INVARIANTS VIOLATION (%s): %s r64={.base=%#llx, .size=%#llx} " - "r32={.base=%#x, .size=%#x} var_off=(%#llx, %#llx)", + "r32={.base=%#x, .size=%#x} var_off=(%#llx, %#llx) step=%u+%u", ctx, msg, reg->r64.base, reg->r64.size, reg->r32.base, reg->r32.size, - reg->var_off.value, reg->var_off.mask); + reg->var_off.value, reg->var_off.mask, + reg->base, reg->step); if (env->test_reg_invariants) return -EFAULT; __mark_reg_unbounded(reg); + reg_step_reset(reg); return 0; } @@ -2194,6 +2449,7 @@ void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg) reg->type = SCALAR_VALUE; reg->var_off = tnum_unknown; __mark_reg_unbounded(reg); + reg_step_reset(reg); } /* Mark a register as having a completely unknown (scalar) value, @@ -2404,6 +2660,7 @@ static int add_subprog(struct bpf_verifier_env *env, int off) return -E2BIG; } /* determine subprog starts. The end is one before the next starts */ + env->subprog_info[env->subprog_cnt].exit_idx = U32_MAX; env->subprog_info[env->subprog_cnt++].start = off; sort(env->subprog_info, env->subprog_cnt, sizeof(env->subprog_info[0]), cmp_subprogs, NULL); @@ -2512,17 +2769,21 @@ find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset) struct bpf_kfunc_desc_tab *tab; tab = prog->aux->kfunc_tab; - return bsearch(&desc, tab->descs, tab->nr_descs, + return bsearch(&desc, tab->descs, tab->nr_base_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off); } int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, - u16 btf_fd_idx, u8 **func_addr) + u16 desc_idx, u8 **func_addr) { + struct bpf_kfunc_desc_tab *tab; const struct bpf_kfunc_desc *desc; - desc = find_kfunc_desc(prog, func_id, btf_fd_idx); - if (!desc) + tab = prog->aux->kfunc_tab; + if (desc_idx >= tab->nr_descs) + return -EFAULT; + desc = &tab->descs[desc_idx]; + if (desc->func_id != func_id) return -EFAULT; *func_addr = (u8 *)desc->addr; @@ -2797,7 +3058,7 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env, } static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - struct bpf_func_proto *proto); + const struct btf_func_model *fm, struct bpf_func_proto *proto); int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) { @@ -2862,10 +3123,15 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) if (find_kfunc_desc(env->prog, func_id, offset)) return 0; - if (tab->nr_descs == MAX_KFUNC_DESCS) { + if (tab->nr_base_descs == MAX_KFUNC_DESCS) { verbose(env, "too many different kernel function calls\n"); return -E2BIG; } + if (tab->nr_descs != tab->nr_base_descs) { + verifier_bug(env, "unexpected specialized func desc found (%d descs, %d base descs)", + tab->nr_descs, tab->nr_base_descs); + return -EFAULT; + } err = fetch_kfunc_meta(env, func_id, offset, &kfunc); if (err) @@ -2886,6 +3152,18 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) err = btf_distill_func_proto(&env->log, kfunc.btf, kfunc.proto, kfunc.name, &func_model); if (err) return err; + if (func_model.ret_size > 8) { + if (kfunc.flags && (*kfunc.flags & KF_FASTCALL)) { + verbose(env, "kfunc %s with >8-byte return is not supported with KF_FASTCALL\n", + kfunc.name); + return -EOPNOTSUPP; + } + if (!bpf_jit_supports_kfunc_ret_reg_pair()) { + verbose(env, "kfunc %s with >8-byte return is not supported by JIT\n", + kfunc.name); + return -EOPNOTSUPP; + } + } memset(&meta, 0, sizeof(meta)); meta.btf = kfunc.btf; @@ -2902,7 +3180,7 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) desc = &tab->descs[tab->nr_descs]; memset(desc, 0, sizeof(*desc)); - err = gen_kfunc_arg_proto(env, &meta, &desc->proto); + err = gen_kfunc_arg_proto(env, &meta, &func_model, &desc->proto); if (err) return err; @@ -2911,7 +3189,8 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) desc->addr = addr; desc->func_model = func_model; tab->nr_descs++; - sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), + tab->nr_base_descs++; + sort(tab->descs, tab->nr_base_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off, NULL); return 0; } @@ -2953,6 +3232,22 @@ static int add_subprogs(struct bpf_verifier_env *env) return ret; } + /* + * func_info describes all functions of the program. Those that are + * referenced only from data, e.g. from a table of functions to be + * called via callx, are not seen by the loop above. They are possible + * callees that have to be known upfront as well. + */ + if (env->bpf_capable) { + struct bpf_prog_aux *aux = env->prog->aux; + + for (i = 1; i < aux->func_info_cnt; i++) { + ret = add_subprog(env, aux->func_info[i].insn_off); + if (ret < 0) + return ret; + } + } + ret = bpf_find_exception_callback_insn_off(env); if (ret < 0) return ret; @@ -3012,6 +3307,34 @@ static int add_kfuncs(struct bpf_verifier_env *env) return 0; } +static void find_subprog_properties(struct bpf_verifier_env *env) +{ + struct bpf_subprog_info *subprog = env->subprog_info; + struct bpf_insn *insn = env->prog->insnsi; + int cur_subprog; + + for (cur_subprog = 0; cur_subprog < env->subprog_cnt; cur_subprog++) { + int i; + + for (i = subprog[cur_subprog].start; + i < subprog[cur_subprog + 1].start; i++) { + u8 code = insn[i].code; + + if (code == (BPF_JMP | BPF_CALL) && + insn[i].src_reg == 0 && + insn[i].imm == BPF_FUNC_tail_call) { + subprog[cur_subprog].has_tail_call = true; + subprog[cur_subprog].tail_call_reachable = true; + } + if (BPF_CLASS(code) == BPF_LD && + (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) + subprog[cur_subprog].has_ld_abs = true; + if (bpf_is_callx(&insn[i])) + env->has_callx = true; + } + } +} + static int check_subprogs(struct bpf_verifier_env *env) { int i, subprog_start, subprog_end, off, cur_subprog = 0; @@ -3025,15 +3348,6 @@ static int check_subprogs(struct bpf_verifier_env *env) for (i = 0; i < insn_cnt; i++) { u8 code = insn[i].code; - if (code == (BPF_JMP | BPF_CALL) && - insn[i].src_reg == 0 && - insn[i].imm == BPF_FUNC_tail_call) { - subprog[cur_subprog].has_tail_call = true; - subprog[cur_subprog].tail_call_reachable = true; - } - if (BPF_CLASS(code) == BPF_LD && - (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) - subprog[cur_subprog].has_ld_abs = true; if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) goto next; if (BPF_OP(code) == BPF_CALL) @@ -3042,6 +3356,8 @@ static int check_subprogs(struct bpf_verifier_env *env) subprog[cur_subprog].exit_idx = i; goto next; } + if (insn_is_gotox(&insn[i])) + goto next; off = i + bpf_jmp_offset(&insn[i]) + 1; if (off < subprog_start || off >= subprog_end) { verbose(env, "jump out of range from insn %d to %d\n", i, off); @@ -3055,13 +3371,15 @@ static int check_subprogs(struct bpf_verifier_env *env) } next: if (i == subprog_end - 1) { - /* to avoid fall-through from one subprog into another + /* + * To avoid fall-through from one subprog into another, * the last insn of the subprog should be either exit - * or unconditional jump back or bpf_throw call + * or unconditional jump back or bpf_throw call. */ if (code != (BPF_JMP | BPF_EXIT) && code != (BPF_JMP32 | BPF_JA) && - code != (BPF_JMP | BPF_JA)) { + code != (BPF_JMP | BPF_JA) && + !insn_is_gotox(&insn[i])) { verbose(env, "last insn is not an exit or jmp\n"); bpf_diag_program_structure( env, i, "subprogram can fall through", @@ -3080,11 +3398,89 @@ next: } /* + * The callee of callx is known to the main verification pass only, which + * records the 'caller' -> 'callee' edge of the call graph for the checks + * that follow it: absence of recursion and the maximum stack depth. + */ +static int record_callx_edge(struct bpf_verifier_env *env, int caller, int callee) +{ + u32 cnt = env->subprog_cnt; + + if (!env->callx_edges) { + env->callx_edges = kvcalloc(BITS_TO_LONGS(cnt * cnt), sizeof(long), + GFP_KERNEL_ACCOUNT); + if (!env->callx_edges) + return -ENOMEM; + } + __set_bit(caller * cnt + callee, env->callx_edges); + return 0; +} + +/* + * Return the first subprog with the number >= 'from' that 'caller' calls + * via callx, or -1 when there is none. + */ +static int next_callx_callee(struct bpf_verifier_env *env, int caller, int from) +{ + u32 cnt = env->subprog_cnt; + unsigned long bit, end = (caller + 1) * cnt; + + if (!env->callx_edges || from >= cnt) + return -1; + bit = find_next_bit(env->callx_edges, end, caller * cnt + from); + return bit < end ? bit - caller * cnt : -1; +} + +/* + * 'call foo; unwind_goto pad' goes to the pad when foo left with unwind_exit. + * JITs don't emit a branch for it. unwind_exit finds the pad by its return + * address. So unwind_goto has to follow the call and nothing may jump to it. + */ +static int check_unwind_insns(struct bpf_verifier_env *env) +{ + struct bpf_insn *insn = env->prog->insnsi; + bool seen = false; + int i; + + for (i = 0; i < env->prog->len; i++, insn++) { + if (bpf_is_unwind_exit(insn)) { + bpf_find_containing_subprog(env, i)->can_unwind = true; + seen = true; + } + if (!bpf_is_unwind_goto(insn)) + continue; + seen = true; + if (!i || !(bpf_pseudo_call(insn - 1) || bpf_is_callx(insn - 1))) { + verbose(env, "unwind_goto at insn %d doesn't follow a call of a subprog\n", + i); + return -EINVAL; + } + if (bpf_is_jump_target(env, i)) { + verbose(env, "jump to unwind_goto at insn %d\n", i); + return -EINVAL; + } + } + if (!seen) + return 0; + if (!bpf_jit_supports_unwind() || bpf_prog_is_offloaded(env->prog->aux)) { + verbose(env, "JIT does not support unwinding\n"); + return -ENOTSUPP; + } + env->prog->jit_required = 1; + env->prog->aux->can_unwind = env->subprog_info[0].can_unwind; + return 0; +} + +/* * Sort subprogs in topological order so that leaf subprogs come first and * their callers come later. This is a DFS post-order traversal of the call * graph. Scan only reachable instructions (those in the computed postorder) of * the current subprog to discover callees (direct subprogs and sync * callbacks). + * + * The callees of callx are not known before the main verification pass. + * When callx is used the sort is repeated after it with the recorded callx + * edges added to the call graph to reject recursion through indirect calls. */ static int sort_subprogs_topo(struct bpf_verifier_env *env) { @@ -3125,12 +3521,22 @@ static int sort_subprogs_topo(struct bpf_verifier_env *env) int idx = insn_postorder[j]; int callee; - if (!bpf_pseudo_call(&insn[idx]) && !bpf_pseudo_func(&insn[idx])) + if (bpf_is_callx(&insn[idx])) { + /* find a callee that is not explored yet */ + callee = -1; + do { + callee = next_callx_callee(env, cur, callee + 1); + } while (callee >= 0 && color[callee] == 2); + if (callee < 0) + continue; + } else if (bpf_pseudo_call(&insn[idx]) || bpf_pseudo_func(&insn[idx])) { + callee = bpf_find_subprog(env, idx + insn[idx].imm + 1); + if (callee < 0) { + ret = -EFAULT; + goto out; + } + } else { continue; - callee = bpf_find_subprog(env, idx + insn[idx].imm + 1); - if (callee < 0) { - ret = -EFAULT; - goto out; } if (color[callee] == 2) continue; @@ -3227,26 +3633,30 @@ static void mark_indirect_target(struct bpf_verifier_env *env, int idx) env->insn_aux_data[idx].indirect_target = true; } +static void mark_non_stack_access(struct bpf_verifier_env *env, int idx) +{ + env->insn_aux_data[idx].non_stack_access = true; +} + +/* Layout of one packed linked register in the jump history, see linked_regs_pack() */ #define LR_FRAMENO_BITS 4 -#define LR_SPI_BITS 6 -#define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1) -#define LR_SIZE_BITS 4 -#define LR_FRAMENO_MASK ((1ull << LR_FRAMENO_BITS) - 1) -#define LR_SPI_MASK ((1ull << LR_SPI_BITS) - 1) -#define LR_SIZE_MASK ((1ull << LR_SIZE_BITS) - 1) -#define LR_SPI_OFF LR_FRAMENO_BITS -#define LR_IS_REG_OFF (LR_SPI_BITS + LR_FRAMENO_BITS) -#define LINKED_REGS_MAX 5 +#define LR_INDEX_BITS 11 +#define LR_FRAMENO_MASK ((1u << LR_FRAMENO_BITS) - 1) +#define LR_IS_REG BIT(LR_FRAMENO_BITS) +#define LR_INDEX_OFF (LR_FRAMENO_BITS + 1) +#define LR_INDEX_MASK ((1u << LR_INDEX_BITS) - 1) +#define LINKED_REGS_MAX BPF_LINKED_REGS_MAX static_assert(MAX_CALL_FRAMES <= (1 << LR_FRAMENO_BITS)); -static_assert(LINKED_REGS_MAX < (1 << LR_SIZE_BITS)); -static_assert(LINKED_REGS_MAX * LR_ENTRY_BITS + LR_SIZE_BITS <= 64); +static_assert(MAX_BPF_REG <= (1 << LR_INDEX_BITS)); +static_assert(MAX_BPF_STACK_SLOTS <= (1 << LR_INDEX_BITS)); +static_assert(LR_INDEX_OFF + LR_INDEX_BITS <= 16); struct linked_reg { u8 frameno; union { - u8 spi; - u8 regno; + u16 spi; + u16 regno; }; bool is_reg; }; @@ -3265,48 +3675,34 @@ static struct linked_reg *linked_regs_push(struct linked_regs *s) } /* - * Use u64 as a vector of 5 11-bit values, use first 4-bits to track - * number of elements currently in stack. - * Pack one history entry for linked registers as 11 bits in the following format: - * - 4-bits frameno - * - 6-bits spi_or_reg - * - 1-bit is_reg + * Pack linked registers for a jump history entry, one u16 each: + * - 4 bits frameno + * - 1 bit is_reg + * - 11 bits register or stack slot index */ -static u64 linked_regs_pack(struct linked_regs *s) +static void linked_regs_pack(const struct linked_regs *s, u16 *packed) { - u64 val = 0; int i; for (i = 0; i < s->cnt; ++i) { - struct linked_reg *e = &s->entries[i]; - u64 tmp = 0; - - tmp |= e->frameno; - tmp |= e->spi << LR_SPI_OFF; - tmp |= (e->is_reg ? 1 : 0) << LR_IS_REG_OFF; + const struct linked_reg *e = &s->entries[i]; - val <<= LR_ENTRY_BITS; - val |= tmp; + packed[i] = e->frameno | (e->is_reg ? LR_IS_REG : 0) | (e->spi << LR_INDEX_OFF); } - val <<= LR_SIZE_BITS; - val |= s->cnt; - return val; } -static void linked_regs_unpack(u64 val, struct linked_regs *s) +static void linked_regs_unpack(const struct bpf_jmp_history_entry *hist, struct linked_regs *s) { int i; - s->cnt = val & LR_SIZE_MASK; - val >>= LR_SIZE_BITS; - + s->cnt = hist->linked_regs_cnt; for (i = 0; i < s->cnt; ++i) { struct linked_reg *e = &s->entries[i]; + u16 packed = hist->linked_regs[i]; - e->frameno = val & LR_FRAMENO_MASK; - e->spi = (val >> LR_SPI_OFF) & LR_SPI_MASK; - e->is_reg = (val >> LR_IS_REG_OFF) & 0x1; - val >>= LR_ENTRY_BITS; + e->frameno = packed & LR_FRAMENO_MASK; + e->is_reg = packed & LR_IS_REG; + e->spi = (packed >> LR_INDEX_OFF) & LR_INDEX_MASK; } } @@ -3348,10 +3744,10 @@ void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_ bool some_precise = false; int i; - if (!hist || hist->linked_regs == 0) + if (!hist || !hist->linked_regs_cnt) return; - linked_regs_unpack(hist->linked_regs, &linked_regs); + linked_regs_unpack(hist, &linked_regs); for (i = 0; i < linked_regs.cnt; ++i) { struct linked_reg *e = &linked_regs.entries[i]; @@ -3448,17 +3844,18 @@ static void save_register_state(struct bpf_verifier_env *env, int spi, struct bpf_reg_state *reg, int size) { + struct bpf_stack_state *slot = bpf_stack_slot(state, spi); int i; - bpf_diag_mod_begin(env, &state->stack[spi].spilled_ptr, reg, BPF_DIAG_MOD_SPILL); - state->stack[spi].spilled_ptr = *reg; + bpf_diag_mod_begin(env, &slot->spilled_ptr, reg, BPF_DIAG_MOD_SPILL); + slot->spilled_ptr = *reg; for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--) - state->stack[spi].slot_type[i - 1] = STACK_SPILL; + slot->slot_type[i - 1] = STACK_SPILL; /* size < 8 bytes spill */ for (; i; i--) - mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]); + mark_stack_slot_misc(env, &slot->slot_type[i - 1]); bpf_diag_mod_end(env); } @@ -3500,14 +3897,15 @@ static void check_fastcall_stack_contract(struct bpf_verifier_env *env, static void scrub_special_slot(struct bpf_func_state *state, int spi) { + struct bpf_stack_state *slot = bpf_stack_slot(state, spi); int i; /* regular write of data into stack destroys any spilled ptr */ - state->stack[spi].spilled_ptr.type = NOT_INIT; + slot->spilled_ptr.type = NOT_INIT; /* Mark slots as STACK_MISC if they belonged to spilled ptr/dynptr/iter. */ - if (is_stack_slot_special(&state->stack[spi])) + if (is_stack_slot_special(slot)) for (i = 0; i < BPF_REG_SIZE; i++) - scrub_spilled_slot(&state->stack[spi].slot_type[i]); + scrub_spilled_slot(&slot->slot_type[i]); } /* check_stack_{read,write}_fixed_off functions track spill/fill of registers, @@ -3525,13 +3923,15 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, struct bpf_reg_state *reg = NULL; int insn_flags = INSN_F_STACK_ACCESS; int hist_spi = spi, hist_frame = state->frameno; + struct bpf_stack_state *ss = bpf_stack_slot(state, spi); - /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, + /* + * caller checked that off % size == 0 and -env->stack_limit <= off < 0, * so it's aligned access and [off, off + size) are within stack limits */ if (!env->allow_ptr_leaks && - bpf_is_spilled_reg(&state->stack[spi]) && - !bpf_is_spilled_scalar_reg(&state->stack[spi]) && + bpf_is_spilled_reg(ss) && + !bpf_is_spilled_scalar_reg(ss) && size != BPF_REG_SIZE) { const char *reason; @@ -3553,7 +3953,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, bool sanitize = reg && is_pointer_regtype(reg->type); for (i = 0; i < size; i++) { - u8 type = state->stack[spi].slot_type[(slot - i) % + u8 type = ss->slot_type[(slot - i) % BPF_REG_SIZE]; if (type != STACK_MISC && type != STACK_ZERO) { @@ -3570,7 +3970,6 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, if (err) return err; - check_fastcall_stack_contract(env, state, insn_idx, off); mark_stack_slot_scratched(env, spi); if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) { bool reg_value_fits; @@ -3582,7 +3981,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, save_register_state(env, state, spi, reg, size); /* Break the relation on a narrowing spill. */ if (!reg_value_fits) - state->stack[spi].spilled_ptr.id = 0; + clear_scalar_id(&ss->spilled_ptr); } else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) && env->bpf_capable) { struct bpf_reg_state *tmp_reg = &env->fake_reg[0]; @@ -3606,8 +4005,8 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, } else { u8 type = STACK_MISC; - if (bpf_is_spilled_reg(&state->stack[spi])) - bpf_diag_record_scrub(env, &state->stack[spi].spilled_ptr, + if (bpf_is_spilled_reg(ss)) + bpf_diag_record_scrub(env, &ss->spilled_ptr, BPF_DIAG_MOD_WRITE); scrub_special_slot(state, spi); @@ -3628,13 +4027,13 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, /* Mark slots affected by this stack write. */ for (i = 0; i < size; i++) - state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = type; + ss->slot_type[(slot - i) % BPF_REG_SIZE] = type; insn_flags = 0; /* not a register spill */ } if (insn_flags) return bpf_push_jmp_history(env, env->cur_state, insn_flags, - hist_spi, hist_frame, 0); + hist_spi, hist_frame, NULL, 0); return 0; } @@ -3682,24 +4081,21 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, (!value_reg && is_bpf_st_mem(insn) && insn->imm == 0)) writing_zero = true; - for (i = min_off; i < max_off; i++) { - int spi; - - spi = bpf_get_spi(i); - err = destroy_if_dynptr_stack_slot(env, state, spi); - if (err) - return err; - } + err = destroy_dynptrs_in_stack_slots(env, state, bpf_get_spi(max_off - 1), + bpf_get_spi(min_off)); + if (err) + return err; - check_fastcall_stack_contract(env, state, insn_idx, min_off); /* Variable offset writes destroy any spilled pointers in range. */ for (i = min_off; i < max_off; i++) { + struct bpf_stack_state *ss; u8 new_type, *stype; int slot, spi; slot = -i - 1; spi = slot / BPF_REG_SIZE; - stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; + ss = bpf_stack_slot(state, spi); + stype = &ss->slot_type[slot % BPF_REG_SIZE]; mark_stack_slot_scratched(env, spi); if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) { @@ -3722,9 +4118,8 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, /* If writing_zero and the spi slot contains a spill of value 0, * maintain the spill type. */ - if (writing_zero && *stype == STACK_SPILL && - bpf_is_spilled_scalar_reg(&state->stack[spi])) { - struct bpf_reg_state *spill_reg = &state->stack[spi].spilled_ptr; + if (writing_zero && *stype == STACK_SPILL && bpf_is_spilled_scalar_reg(ss)) { + struct bpf_reg_state *spill_reg = &ss->spilled_ptr; if (tnum_is_const(spill_reg->var_off) && spill_reg->var_off.value == 0) { zero_used = true; @@ -3795,23 +4190,22 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env, { struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; - u64 zero_spill_mask = 0; int i, slot, spi; u8 *stype; - int zeros = 0; + int zeros = 0, zero_spills = 0; for (i = min_off; i < max_off; i++) { slot = -i - 1; spi = slot / BPF_REG_SIZE; mark_stack_slot_scratched(env, spi); - stype = ptr_state->stack[spi].slot_type; + stype = bpf_stack_slot(ptr_state, spi)->slot_type; if (stype[slot % BPF_REG_SIZE] == STACK_ZERO) { zeros++; continue; } if (stype[slot % BPF_REG_SIZE] == STACK_SPILL && - bpf_register_is_null(&ptr_state->stack[spi].spilled_ptr)) { - zero_spill_mask |= 1ull << spi; + bpf_register_is_null(&bpf_stack_slot(ptr_state, spi)->spilled_ptr)) { + zero_spills++; zeros++; continue; } @@ -3822,8 +4216,14 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env, * so the whole register == const_zero. */ __mark_reg_const_zero(env, &state->regs[dst_regno]); - if (zero_spill_mask) { - bpf_bt_set_frame_slot_mask(&env->bt, ptr_state->frameno, zero_spill_mask); + if (zero_spills) { + for (i = min_off; i < max_off; i++) { + slot = -i - 1; + spi = slot / BPF_REG_SIZE; + stype = bpf_stack_slot(ptr_state, spi)->slot_type; + if (stype[slot % BPF_REG_SIZE] == STACK_SPILL) + bpf_bt_set_frame_slot(&env->bt, ptr_state->frameno, spi); + } return mark_chain_precision_batch(env, env->cur_state); } } else { @@ -3834,19 +4234,46 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env, return 0; } -static void bpf_diag_stack_read_uninit(struct bpf_verifier_env *env, int off, int i, - int size) +static void bpf_diag_stack_read_invalid(struct bpf_verifier_env *env, int off, int i, int size, + enum bpf_stack_slot_type type) { - const char *reason; + const char *problem, *reason, *suggestion, *kind; + + if (type == STACK_INVALID) { + reason = bpf_diag_fmt( + env, "This rejected read uses %d bytes at stack offset %d, but byte %d in that range is uninitialized on this path. " + "Programs loaded with CAP_PERFMON can be allowed to read uninitialized stack bytes, but this program is being rejected without that allowance.", + size, off, i); + bpf_diag_memory( + env, env->insn_idx, "uninitialized stack read", reason, + "Initialize every byte in the stack range before reading it, adjust the offset and size so the read covers only initialized bytes, " + "or load with CAP_PERFMON if uninitialized stack reads are intended."); + return; + } + + switch (type) { + case STACK_DYNPTR: + kind = "dynptr"; + suggestion = "Use dynptr helpers or kfuncs to access the object represented by the dynptr instead of reading the dynptr state directly."; + break; + case STACK_ITER: + kind = "iterator"; + suggestion = "Use iterator kfuncs to advance or destroy the iterator instead of reading its state directly."; + break; + case STACK_IRQ_FLAG: + kind = "IRQ flag"; + suggestion = "Pass the saved IRQ flag to the matching restore kfunc instead of reading its state directly."; + break; + default: + return; + } - reason = bpf_diag_fmt(env, - "This rejected read uses %d bytes at stack offset %d, but byte %d in that range is uninitialized on this path. " - "Programs loaded with CAP_PERFMON can be allowed to read uninitialized stack bytes, but this program is being rejected without that allowance.", - size, off, i); - bpf_diag_memory( - env, env->insn_idx, "uninitialized stack read", reason, - "Initialize every byte in the stack range before reading it, adjust the offset and size so the read covers only initialized bytes, " - "or load with CAP_PERFMON if uninitialized stack reads are intended."); + problem = bpf_diag_fmt(env, "direct read of %s stack state", kind); + reason = bpf_diag_fmt( + env, "This rejected read uses %d bytes at stack offset %d, but byte %d in that range belongs to verifier-managed %s state. " + "This state has an opaque representation that BPF programs cannot read directly.", + size, off, i, kind); + bpf_diag_memory(env, env->insn_idx, problem, reason, suggestion); } /* Read the stack at 'off' and put the results into the register indicated by @@ -3871,12 +4298,12 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, int err; int insn_flags = INSN_F_STACK_ACCESS; int hist_spi = spi, hist_frame = reg_state->frameno; + struct bpf_stack_state *ss = bpf_stack_slot(reg_state, spi); - stype = reg_state->stack[spi].slot_type; - reg = ®_state->stack[spi].spilled_ptr; + stype = ss->slot_type; + reg = &ss->spilled_ptr; mark_stack_slot_scratched(env, spi); - check_fastcall_stack_contract(env, state, env->insn_idx, off); /* * Refine the in-progress load record's origin to the source stack slot. @@ -3884,11 +4311,8 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, if (dst_regno >= 0) bpf_diag_mod_begin(env, &state->regs[dst_regno], reg, BPF_DIAG_MOD_WRITE); - if (bpf_is_spilled_reg(®_state->stack[spi])) { - u8 spill_size = 1; - - for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--) - spill_size++; + if (bpf_is_spilled_reg(ss)) { + u8 spill_size = bpf_spill_size(ss); if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) { if (reg->type != SCALAR_VALUE) { @@ -3897,11 +4321,11 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, return -EACCES; } - if (dst_regno < 0) - return 0; - if (size <= spill_size && bpf_stack_narrow_access_ok(off, size, spill_size)) { + if (dst_regno < 0) + return 0; + if (env->bpf_capable && size == 4 && spill_size == 4 && get_reg_width(reg) <= 32) /* Ensure stack slot has an ID to build a relation @@ -3938,11 +4362,14 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } else { verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); - bpf_diag_stack_read_uninit(env, off, i, size); + bpf_diag_stack_read_invalid(env, off, i, size, type); } return -EACCES; } + if (dst_regno < 0) + return 0; + if (spill_cnt == size && tnum_is_const(reg->var_off) && reg->var_off.value == 0) { __mark_reg_const_zero(env, &state->regs[dst_regno]); @@ -3997,7 +4424,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } else { verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); - bpf_diag_stack_read_uninit(env, off, i, size); + bpf_diag_stack_read_invalid(env, off, i, size, type); } return -EACCES; } @@ -4010,7 +4437,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } if (insn_flags) return bpf_push_jmp_history(env, env->cur_state, insn_flags, - hist_spi, hist_frame, 0); + hist_spi, hist_frame, NULL, 0); return 0; } @@ -4063,7 +4490,6 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg dst_regno); if (err) return err; - check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off); return 0; } @@ -4096,13 +4522,13 @@ static int check_stack_read(struct bpf_verifier_env *env, tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", tn_buf, off, size); - reason = bpf_diag_fmt(env, - "The helper would access the stack through variable offset %s plus fixed offset %d and size %d. " - "Helper stack memory arguments require a constant stack offset and a precise initialized range.", + reason = bpf_diag_fmt( + env, "The instruction would access the stack through variable offset %s plus fixed offset %d and size %d. " + "This stack access requires a constant stack offset and a precise initialized range.", tn_buf, off, size); bpf_diag_memory( - env, env->insn_idx, "variable stack access", reason, - "Use a fixed stack offset for helper memory arguments, or copy the needed bytes into a fixed stack slot first."); + env, env->insn_idx, "variable-offset stack access", reason, + "Use a fixed stack offset for the instruction, selecting the target stack slot on separate control-flow paths if necessary."); return -EACCES; } /* Variable offset is prohibited for unprivileged mode for simplicity @@ -4200,7 +4626,7 @@ static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_s bpf_diag_mod_end(env); state->no_stack_arg_load = true; return bpf_push_jmp_history(env, env->cur_state, - INSN_F_STACK_ARG_ACCESS, spi, 0, 0); + INSN_F_STACK_ARG_ACCESS, spi, 0, NULL, 0); } /* @@ -4234,7 +4660,7 @@ static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_st cur->regs[dst_regno] = *arg; bpf_diag_mod_end(env); return bpf_push_jmp_history(env, env->cur_state, - INSN_F_STACK_ARG_ACCESS, spi, 0, 0); + INSN_F_STACK_ARG_ACCESS, spi, 0, NULL, 0); } static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx) @@ -4594,7 +5020,7 @@ BTF_ID(struct, cgroup) BTF_ID(struct, bpf_cpumask) #endif BTF_ID(struct, task_struct) -#ifdef CONFIG_CRYPTO +#ifdef CONFIG_BPF_CRYPTO BTF_ID(struct, bpf_crypto_ctx) #endif #ifdef CONFIG_INET @@ -4817,7 +5243,7 @@ static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state * } static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, - const struct bpf_func_proto *fn, + const struct bpf_call_arg_meta *meta, enum bpf_access_type t) { enum bpf_prog_type prog_type = resolve_prog_type(env->prog); @@ -4841,10 +5267,11 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, case BPF_PROG_TYPE_LWT_XMIT: case BPF_PROG_TYPE_SK_SKB: case BPF_PROG_TYPE_SK_MSG: - if (fn) - return fn->pkt_access; + if (meta && !meta->btf && meta->func_id) + return meta->fn->pkt_access; - env->seen_direct_write = true; + if (t == BPF_WRITE) + env->seen_direct_write = true; return true; case BPF_PROG_TYPE_CGROUP_SOCKOPT: @@ -5056,6 +5483,19 @@ static bool is_arena_reg(struct bpf_verifier_env *env, int regno) return reg->type == PTR_TO_ARENA; } +/* + * There are no address spaces in Rust, addresses of arena are plain numbers. + * When the program is loaded with BPF_F_ARENA_SCALAR and has an arena a load or + * a store through a number is an access to arena at the low 32 bits of it, + * like the access through PTR_TO_ARENA is. The register stays a number. + */ +static bool is_arena_scalar(struct bpf_verifier_env *env, int regno) +{ + const struct bpf_reg_state *reg = reg_state(env, regno); + + return reg->type == SCALAR_VALUE && env->arena_scalar && env->prog->aux->arena; +} + static bool is_load_acq_unsafe(struct bpf_verifier_env *env, int regno, struct bpf_insn *insn) { @@ -5086,7 +5526,7 @@ static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno, return false; if (is_sk_reg(env, regno)) return false; - if (is_arena_reg(env, regno)) + if (is_arena_reg(env, regno) || is_arena_scalar(env, regno)) return bpf_jit_supports_insn(insn, true); if (is_load_acq_unsafe(env, regno, insn)) return false; @@ -5102,27 +5542,17 @@ static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { [CONST_PTR_TO_MAP] = btf_bpf_map_id, }; -static enum bpf_reg_type lookup_reg2btf_ids(u32 ref_id) -{ - enum bpf_reg_type type; - - for (type = 0; type < __BPF_REG_TYPE_MAX; type++) { - if (reg2btf_ids[type] && *reg2btf_ids[type] == ref_id) - return type; - } - - return NOT_INIT; -} - -static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg) +static bool is_trusted_reg_type(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, + enum bpf_reg_type type) { /* A referenced register is always trusted. */ if (reg_is_referenced(env, reg)) return true; /* Types listed in the reg2btf_ids are always trusted */ - if (reg2btf_ids[base_type(reg->type)] && - !bpf_type_has_unsafe_modifiers(reg->type)) + if (reg2btf_ids[base_type(type)] && + !bpf_type_has_unsafe_modifiers(type)) return true; /* If a register is not referenced, it is trusted if it has the @@ -5134,8 +5564,14 @@ static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_st * Eventually, we should make PTR_TRUSTED the single source of truth * for whether a register is trusted. */ - return type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS && - !bpf_type_has_unsafe_modifiers(reg->type); + return type_flag(type) & BPF_REG_TRUSTED_MODIFIERS && + !bpf_type_has_unsafe_modifiers(type); +} + +static bool is_trusted_reg(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg) +{ + return is_trusted_reg_type(env, reg, reg->type); } static bool is_rcu_reg(const struct bpf_reg_state *reg) @@ -5308,6 +5744,9 @@ struct bpf_subprog_call_depth_info { int ret_insn; /* caller instruction where we return to. */ int caller; /* caller subprogram idx */ int frame; /* # of consecutive static call stack frames on top of stack */ + int callx_insn; /* callx instruction whose callees are being walked */ + int callx_next; /* next callee of callx_insn to walk */ + bool via_callx; /* the subprogram is entered via callx */ }; /* starting from main bpf function walk all instructions of the function @@ -5327,11 +5766,13 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, /* no caller idx */ dinfo[idx].caller = -1; + dinfo[idx].via_callx = false; i = subprog[idx].start; if (!priv_stack_supported) subprog[idx].priv_stack_mode = NO_PRIV_STACK; process_func: + dinfo[idx].callx_insn = -1; if (subprog[idx].has_ld_abs) { for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { if (subprog[tmp].is_cb) { @@ -5384,7 +5825,7 @@ process_func: if (subprog[idx].priv_stack_mode == PRIV_STACK_ADAPTIVE) { if (subprog_depth > env->max_stack_depth) env->max_stack_depth = subprog_depth; - if (subprog_depth > MAX_BPF_STACK) { + if (subprog_depth > env->stack_limit) { verbose(env, "stack size of subprog %d is %d. Too large\n", idx, subprog_depth); return -EACCES; @@ -5393,7 +5834,7 @@ process_func: depth += subprog_depth; if (depth > env->max_stack_depth) env->max_stack_depth = depth; - if (depth > MAX_BPF_STACK) { + if (depth > env->stack_limit) { total = 0; for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) total++; @@ -5408,11 +5849,10 @@ continue_func: for (; i < subprog_end; i++) { int next_insn, sidx; - if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) { + if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off && + bpf_is_throw_kfunc(insn + i)) { bool err = false; - if (!bpf_is_throw_kfunc(insn + i)) - continue; for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) { if (subprog[tmp].is_cb) { err = true; @@ -5427,31 +5867,70 @@ continue_func: return -EINVAL; } - if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i)) + if (bpf_is_callx(insn + i) || bpf_calls_callback(env, i)) { + /* + * Walk the callees and the callbacks recorded by + * the main verification pass one by one, returning to + * this insn after each. + */ + if (dinfo[idx].callx_insn != i) { + dinfo[idx].callx_insn = i; + dinfo[idx].callx_next = 0; + } + sidx = next_callx_callee(env, idx, dinfo[idx].callx_next); + if (sidx < 0) + continue; + dinfo[idx].callx_next = sidx + 1; + dinfo[idx].ret_insn = i; + next_insn = subprog[sidx].start; + } else if (bpf_pseudo_call(insn + i) || bpf_pseudo_func(insn + i)) { + /* find the callee */ + next_insn = i + insn[i].imm + 1; + sidx = bpf_find_subprog(env, next_insn); + if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn)) + return -EFAULT; + if (subprog[sidx].is_async_cb) { + /* async callbacks don't increase bpf prog stack size unless called directly */ + if (!bpf_pseudo_call(insn + i)) + continue; + if (subprog[sidx].is_exception_cb) { + verbose(env, "insn %d cannot call exception cb directly", i); + return -EINVAL; + } + } + /* remember insn to return to */ + dinfo[idx].ret_insn = i + 1; + } else { continue; - /* remember insn and function to return to */ + } - /* find the callee */ - next_insn = i + insn[i].imm + 1; - sidx = bpf_find_subprog(env, next_insn); - if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn)) - return -EFAULT; - if (subprog[sidx].is_async_cb) { - /* async callbacks don't increase bpf prog stack size unless called directly */ - if (!bpf_pseudo_call(insn + i)) + /* + * sort_subprogs_topo() tolerates cycles in the call graph that + * go through the address of a function being taken, since it + * doesn't know what it is taken for. Such cycle is a recursion + * unless it's an async callback, which are skipped above. + * The main verification pass limits the depth of the recursion, + * but it doesn't follow calls of global functions. + */ + for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { + if (tmp != sidx) continue; - if (subprog[sidx].is_exception_cb) { - verbose(env, "insn %d cannot call exception cb directly", i); - return -EINVAL; - } + verbose(env, "recursive call from %s() to %s()\n", + bpf_subprog_name(env, idx), bpf_subprog_name(env, sidx)); + bpf_diag_program_structure( + env, i, "recursive subprogram call", + "Rewrite the recursion as an explicit bounded loop, or split the logic so subprogram calls do not form a cycle.", + "This call or callback would make the subprogram call graph recursive. " + "The verifier requires a finite, acyclic call graph so it can bound stack depth and analysis."); + return -EINVAL; } /* store caller info for after we return from callee */ dinfo[idx].frame = frame; - dinfo[idx].ret_insn = i + 1; /* push caller idx into callee's dinfo */ dinfo[sidx].caller = idx; + dinfo[sidx].via_callx = bpf_is_callx(insn + i); i = next_insn; @@ -5485,6 +5964,14 @@ continue_func: verbose(env, "tail_calls are not allowed in programs with stack args\n"); return -EINVAL; } + /* + * JITs pass tail call counter in a register that is + * not available when the callee is called via callx. + */ + if (dinfo[tmp].via_callx) { + verbose(env, "tail_calls are not allowed in functions called via callx\n"); + return -EINVAL; + } subprog[tmp].tail_call_reachable = true; } } else if (!idx && subprog[0].has_tail_call && subprog[0].stack_arg_cnt) { @@ -5620,9 +6107,10 @@ static int check_buffer_access(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, argno_t argno, int off, int size, bool zero_size_allowed, - u32 *max_access) + u32 *max_access, bool write) { const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr"; + struct bpf_prog_aux *aux = env->prog->aux; u32 access_end; int err; @@ -5631,6 +6119,11 @@ static int check_buffer_access(struct bpf_verifier_env *env, return err; *max_access = max(access_end, *max_access); + if (write && size) { + if (!aux->max_rdwr_write || access_end - size < aux->min_rdwr_write) + aux->min_rdwr_write = access_end - size; + aux->max_rdwr_write = max(access_end, aux->max_rdwr_write); + } return 0; } @@ -5638,6 +6131,7 @@ static int check_buffer_access(struct bpf_verifier_env *env, /* BPF architecture zero extends alu32 ops into 64-bit registesr */ static void zext_32_to_64(struct bpf_reg_state *reg) { + reg_step_check_unsigned(reg, 32); reg->var_off = tnum_subreg(reg->var_off); reg_set_urange64(reg, reg_u32_min(reg), reg_u32_max(reg)); } @@ -5647,13 +6141,14 @@ static void zext_32_to_64(struct bpf_reg_state *reg) */ static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) { - u64 mask; + u64 mask = (1ULL << (size * 8)) - 1; + + reg_step_check_unsigned(reg, size * 8); /* clear high bits in bit representation */ reg->var_off = tnum_cast(reg->var_off, size); /* fix arithmetic bounds */ - mask = ((u64)1 << (size * 8)) - 1; if ((reg_umin(reg) & ~mask) == (reg_umax(reg) & ~mask)) reg_set_urange64(reg, reg_umin(reg) & mask, reg_umax(reg) & mask); else @@ -5690,6 +6185,14 @@ static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size) s64 init_s64_max, init_s64_min, s64_max, s64_min, u64_cval; u64 top_smax_value, top_smin_value; u64 num_bits = size * 8; + s64 limit = 1LL << (num_bits - 1); + + /* If every value fits the signed N-bit range sign extension is a noop. */ + if (reg_smin(reg) >= -limit && reg_smax(reg) < limit) + return; + + /* See commen at reg_step_check_unsigned(). */ + reg_step_reset(reg); if (tnum_is_const(reg->var_off)) { u64_cval = reg->var_off.value; @@ -5756,6 +6259,16 @@ static void coerce_subreg_to_size_sx(struct bpf_reg_state *reg, int size) u32 top_smax_value, top_smin_value; u32 num_bits = size * 8; + /* + * If value fits the non-negative half of the signed N-bit range, + * sign extension is a noop. + */ + if (reg_umax(reg) < (1ULL << (num_bits - 1))) + return; + + /* See comment at reg_step_check_unsigned(). */ + reg_step_reset(reg); + if (tnum_is_const(reg->var_off)) { u32_val = reg->var_off.value; if (size == 1) @@ -5849,6 +6362,110 @@ int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, return 0; } +static int cmp_func_ptrs(const void *_a, const void *_b) +{ + const struct bpf_func_ptr *a = _a, *b = _b; + + if (a->map != b->map) + return a->map < b->map ? -1 : 1; + if (a->map_off != b->map_off) + return a->map_off < b->map_off ? -1 : 1; + return 0; +} + +/* Find the first pointer to a function at or after 'off' in the value of 'map' */ +static u32 func_ptr_lower_bound(struct bpf_verifier_env *env, const struct bpf_map *map, u64 off) +{ + u32 l = 0, r = env->func_ptr_cnt, m; + struct bpf_func_ptr *p; + + while (l < r) { + m = l + (r - l) / 2; + p = &env->func_ptrs[m]; + if (p->map < map || (p->map == map && p->map_off < off)) + l = m + 1; + else + r = m; + } + return l; +} + +/* + * Return pointers to functions that overlap with 'size' bytes at offset 'off' + * of the value of 'map' and their number in 'cnt'. + */ +struct bpf_func_ptr *bpf_map_range_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, + u64 off, u64 size, u32 *cnt) +{ + u32 first, last; + + *cnt = 0; + if (!env->func_ptr_cnt || !size) + return NULL; + + /* a pointer that starts up to 7 bytes before 'off' overlaps too */ + first = func_ptr_lower_bound(env, map, off >= sizeof(u64) ? off - sizeof(u64) + 1 : 0); + last = func_ptr_lower_bound(env, map, off + size); + if (first >= last) + return NULL; + + *cnt = last - first; + return &env->func_ptrs[first]; +} + +/* Return all pointers to functions in the value of 'map' */ +struct bpf_func_ptr *bpf_map_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, u32 *cnt) +{ + return bpf_map_range_func_ptrs(env, map, 0, (u64)map->value_size, cnt); +} + +/* instructions [off, off + len) replaced the instruction at 'off' */ +void bpf_adjust_func_ptrs(struct bpf_verifier_env *env, u32 off, u32 len) +{ + struct bpf_func_ptr *p; + u32 i; + + if (len <= 1) + return; + + for (i = 0; i < env->func_ptr_cnt; i++) { + p = &env->func_ptrs[i]; + if (p->xlated_off <= off || p->xlated_off == BPF_FUNC_PTR_DELETED) + continue; + p->xlated_off += len - 1; + } +} + +/* + * Instructions [off, off + len) are about to be removed. It's called before + * the starts of subprogs are adjusted. A subprog is gone when all of its + * instructions are. Otherwise, e.g. when its first instruction is a nop, + * it starts where the removed instructions did. + */ +void bpf_adjust_func_ptrs_after_remove(struct bpf_verifier_env *env, u32 off, u32 len) +{ + struct bpf_func_ptr *p; + int subprog; + u32 i; + + for (i = 0; i < env->func_ptr_cnt; i++) { + p = &env->func_ptrs[i]; + if (p->xlated_off < off || p->xlated_off == BPF_FUNC_PTR_DELETED) + continue; + if (p->xlated_off >= off + len) { + p->xlated_off -= len; + continue; + } + subprog = bpf_find_subprog(env, p->xlated_off); + if (subprog > 0 && env->subprog_info[subprog + 1].start > off + len) + p->xlated_off = off; + else + p->xlated_off = BPF_FUNC_PTR_DELETED; + } +} + #define BTF_TYPE_SAFE_RCU(__type) __PASTE(__type, __safe_rcu) #define BTF_TYPE_SAFE_RCU_OR_NULL(__type) __PASTE(__type, __safe_rcu_or_null) #define BTF_TYPE_SAFE_TRUSTED(__type) __PASTE(__type, __safe_trusted) @@ -5918,10 +6535,36 @@ BTF_TYPE_SAFE_TRUSTED(struct file) { struct inode *f_inode; }; +/* + * The pointer fields in the sched_ext ops argument containers are pinned by the + * callers for the duration of the ops calls and are never NULL. + */ +BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args) { +#ifdef CONFIG_EXT_GROUP_SCHED + struct cgroup *cgroup; +#endif +}; + +BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args) { + struct task_struct *task; +}; + +BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args) { + struct sched_ext_ops *ops; +}; + +BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args) { + struct sched_ext_ops *ops; +}; + BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry) { struct inode *d_inode; }; +BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct linux_binprm) { + struct mm_struct *mm; +}; + BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket) { struct sock *sk; }; @@ -5962,6 +6605,10 @@ static bool type_is_trusted(struct bpf_verifier_env *env, BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct linux_binprm)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct file)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args)); return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted"); } @@ -5972,12 +6619,52 @@ static bool type_is_trusted_or_null(struct bpf_verifier_env *env, { BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct linux_binprm)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct vm_area_struct)); return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted_or_null"); } +/* + * The member holds the kernel address of arena memory, which the rewritten + * load turns into an arena pointer, see arena_member_fixup(). NULL needs no + * tracking, PTR_TO_ARENA covers it. + */ +static int check_arena_member_load(struct bpf_verifier_env *env, struct bpf_reg_state *regs, + int value_regno, enum bpf_type_flag flag, const char *tname, + const char *field_name) +{ + struct bpf_insn_aux_data *aux = &env->insn_aux_data[env->insn_idx]; + struct bpf_insn *insn; + + /* a helper reading the member as bytes needs no rewrite */ + if (value_regno < 0) + return 0; + if (!env->prog->aux->arena) { + verbose(env, + "arena pointer member %s of struct %s requires a program with an associated arena\n", + field_name, tname); + return -EINVAL; + } + /* only a plain load or an acquire load is rewritten */ + insn = &env->prog->insnsi[env->insn_idx]; + if (BPF_CLASS(insn->code) != BPF_LDX && + !(insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW) && insn->imm == BPF_LOAD_ACQ)) { + verbose(env, + "arena pointer member %s of struct %s can only be read with a plain or an acquire load\n", + field_name, tname); + return -EACCES; + } + + aux->arena_member = true; + if (flag & PTR_MAYBE_NULL) + aux->arena_member_nullable = true; + mark_reg_unknown(env, regs, value_regno); + regs[value_regno].type = PTR_TO_ARENA; + return 0; +} + static int check_ptr_to_btf_access(struct bpf_verifier_env *env, struct bpf_reg_state *regs, struct bpf_reg_state *reg, argno_t argno, int off, int size, @@ -5989,6 +6676,7 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, const char *field_name = NULL; enum bpf_type_flag flag = 0; u32 btf_id = 0; + s64 min_off; int ret; if (!env->allow_ptr_leaks) { @@ -6004,36 +6692,31 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, return -EINVAL; } - if (!tnum_is_const(reg->var_off)) { - char tn_buf[48]; - - tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); + if (check_add_overflow(reg_smin(reg), off, &min_off)) { verbose(env, - "%s is ptr_%s invalid variable offset: off=%d, var_off=%s\n", - reg_arg_name(env, argno), tname, off, tn_buf); + "%s is ptr_%s access, offset computation overflows: register's minimal offset is %lld, instruction offset is %d\n", + reg_arg_name(env, argno), tname, reg_smin(reg), off); return -EACCES; } - off += reg->var_off.value; - - if (off < 0) { + if (min_off < 0) { verbose(env, - "%s is ptr_%s invalid negative access: off=%d\n", - reg_arg_name(env, argno), tname, off); + "%s is ptr_%s invalid negative access: off=%lld\n", + reg_arg_name(env, argno), tname, min_off); return -EACCES; } if (reg->type & MEM_USER) { verbose(env, - "%s is ptr_%s access user memory: off=%d\n", - reg_arg_name(env, argno), tname, off); + "%s is ptr_%s access user memory\n", + reg_arg_name(env, argno), tname); return -EACCES; } if (reg->type & MEM_PERCPU) { verbose(env, - "%s is ptr_%s access percpu memory: off=%d\n", - reg_arg_name(env, argno), tname, off); + "%s is ptr_%s access percpu memory\n", + reg_arg_name(env, argno), tname); return -EACCES; } @@ -6043,6 +6726,18 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, } if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) { + if (!tnum_is_const(reg->var_off)) { + char tn_buf[48]; + + tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); + verbose(env, + "%s is ptr_%s invalid variable offset: off=%d, var_off=%s\n", + reg_arg_name(env, argno), tname, off, tn_buf); + return -EACCES; + } + + off += reg->var_off.value; + if (!btf_is_kernel(reg->btf)) { verifier_bug(env, "reg->btf must be kernel btf"); return -EFAULT; @@ -6079,6 +6774,9 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, if (ret < 0) return ret; + if (ret == PTR_TO_ARENA) + return check_arena_member_load(env, regs, value_regno, flag, tname, field_name); + if (ret != PTR_TO_BTF_ID) { /* just mark; */ @@ -6139,6 +6837,16 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, clear_trusted_flags(&flag); } + /* + * A __nullable member only matters where NULL would otherwise be + * assumed away, which is anywhere but on a plain or an untrusted + * pointer: those are probed on a dereference, which tolerates NULL, and + * the BTF_TYPE_SAFE_*_OR_NULL lists never applied to them either. A + * per-CPU pointer counts, per_cpu_ptr() does not check for NULL. + */ + if (bpf_may_fault_on_deref(PTR_TO_BTF_ID | (flag & ~PTR_MAYBE_NULL))) + flag &= ~PTR_MAYBE_NULL; + if (atype == BPF_READ && value_regno >= 0) { ret = mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); if (ret < 0) @@ -6213,10 +6921,11 @@ static int check_ptr_to_map_access(struct bpf_verifier_env *env, return 0; } -/* Check that the stack access at the given offset is within bounds. The +/* + * Check that the stack access at the given offset is within bounds. The * maximum valid offset is -1. * - * The minimum valid offset is -MAX_BPF_STACK for writes, and + * The minimum valid offset is -env->stack_limit for writes, and * -state->allocated_stack for reads. */ static int check_stack_slot_within_bounds(struct bpf_verifier_env *env, @@ -6227,7 +6936,7 @@ static int check_stack_slot_within_bounds(struct bpf_verifier_env *env, int min_valid_off; if (t == BPF_WRITE || env->allow_uninit_stack) - min_valid_off = -MAX_BPF_STACK; + min_valid_off = -(int)env->stack_limit; else min_valid_off = -state->allocated_stack; @@ -6293,10 +7002,19 @@ static int check_stack_access_within_bounds( return err; } + /* + * Every stack access passes through here, including buffers passed to + * helpers and kfuncs and accesses into a caller's frame, so check the + * fastcall contract of the frame that owns the slots once for all of + * them. Zero-sized accesses touch no stack and keep the rewrite enabled. + */ + if (access_size) + check_fastcall_stack_contract(env, state, env->insn_idx, min_off); + /* Note that there is no stack access with offset zero, so the needed stack * size is -min_off, not -min_off+1. */ - return grow_stack_state(env, state, -min_off /* size */); + return bpf_grow_stack_state(env, state, -min_off /* size */); } static bool get_func_retval_range(struct bpf_prog *prog, @@ -6320,6 +7038,7 @@ static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val) fake_reg.type = SCALAR_VALUE; __mark_reg_known(&fake_reg, val); + scalar_step_add(dst_reg, dst_reg, &fake_reg); scalar32_min_max_add(dst_reg, &fake_reg); scalar_min_max_add(dst_reg, &fake_reg); dst_reg->var_off = tnum_add(dst_reg->var_off, fake_reg.var_off); @@ -6327,12 +7046,147 @@ static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val) reg_bounds_sync(dst_reg); } +/* there is no support for callx in the interpreter */ +static int require_callx_jit(struct bpf_verifier_env *env) +{ + if (!env->prog->jit_requested) { + verbose(env, "JIT is required to use callx\n"); + return -EOPNOTSUPP; + } + if (!bpf_jit_supports_callx()) { + verbose(env, "JIT doesn't support callx\n"); + return -EOPNOTSUPP; + } + env->prog->jit_required = true; + return 0; +} + +static void mark_reg_func_ptr(struct bpf_verifier_env *env, struct bpf_reg_state *regs, + int regno, int subprog) +{ + mark_reg_known_zero(env, regs, regno); + regs[regno].type = PTR_TO_FUNC; + regs[regno].subprogno = subprog; +} + +/* a read from a table of functions branches into that many states at most */ +#define BPF_MAX_FUNC_PTR_TARGETS 64 +/* and that many pointers at most are next to what is read at variable offset */ +#define BPF_MAX_FUNC_PTR_RANGE 4096 + +/* Count up to 'limit' offsets in [min_off, max_off] that the read is possible at */ +static u32 count_read_offsets(struct tnum offs, u64 min_off, u64 max_off, u32 limit) +{ + u64 o = min_off, next; + u32 cnt = 0; + + if (!tnum_in(offs, tnum_const(o))) { + next = tnum_step(offs, o); + if (next <= o) + return 0; + o = next; + } + while (o <= max_off && cnt < limit) { + cnt++; + next = tnum_step(offs, o); + if (next <= o) + break; + o = next; + } + return cnt; +} + +/* + * A read from a frozen read-only map that has pointers to functions, see + * resolve_func_ptrs(). The bounds and var_off of the offset tell where the read + * is possible at, e.g. var_off is the stride of the elements of an array. + * A read of exactly one pointer yields PTR_TO_FUNC. When the offset is variable + * and only pointers can be read, which is how an element of a table of + * functions is loaded, the verification continues with each of them. Other + * reads that overlap with a pointer are rejected, because their result is not + * known until the program is jitted. + * + * Return -ENOENT if there are no pointers to functions in the bytes that are read. + */ +static int check_func_ptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, + int size, int value_regno) +{ + u64 min_off = reg_umin(reg) + off, max_off = reg_umax(reg) + off; + struct tnum offs = tnum_add(reg->var_off, tnum_const(off)); + struct bpf_reg_state *regs = cur_regs(env); + struct bpf_map *map = reg->map_ptr; + struct bpf_verifier_state *branch; + int subprog, targets[BPF_MAX_FUNC_PTR_TARGETS]; + struct bpf_func_ptr *ptrs; + u32 i, cnt, n = 0; + u64 o, lo, hi; + int err; + + ptrs = bpf_map_range_func_ptrs(env, map, min_off, max_off - min_off + size, &cnt); + if (!ptrs) + return -ENOENT; + if (cnt > BPF_MAX_FUNC_PTR_RANGE) + goto overlap; + + for (i = 0; i < cnt; i++) { + /* the offsets at which the read overlaps with the pointer */ + lo = ptrs[i].map_off >= size ? ptrs[i].map_off - size + 1 : 0; + lo = max(lo, min_off); + hi = min(ptrs[i].map_off + sizeof(u64) - 1, max_off); + for (o = lo; o <= hi; o++) { + if (!tnum_in(offs, tnum_const(o))) + continue; + if (o != ptrs[i].map_off || size != sizeof(u64) || value_regno < 0) + goto overlap; + if (n == BPF_MAX_FUNC_PTR_TARGETS) { + verbose(env, "read from map '%s' may yield more than %d pointers to functions\n", + map->name, BPF_MAX_FUNC_PTR_TARGETS); + return -E2BIG; + } + subprog = bpf_find_subprog(env, ptrs[i].xlated_off); + if (verifier_bug_if(subprog <= 0, env, + "no function at insn %u for map '%s' offset %u", + ptrs[i].xlated_off, map->name, ptrs[i].map_off)) + return -EFAULT; + targets[n++] = subprog; + } + } + /* pointers are next to what is read, e.g. the data of an array of vtables */ + if (!n) + return -ENOENT; + /* either every offset that the read is possible at is a pointer or none */ + if (count_read_offsets(offs, min_off, max_off, n + 1) != n) + goto overlap; + + /* the map has the offset of the function until the program is jitted */ + err = require_callx_jit(env); + if (err) + return err; + + for (i = 0; i < n - 1; i++) { + branch = push_stack(env, env->insn_idx + 1, env->insn_idx, + env->cur_state->speculative); + if (IS_ERR(branch)) + return PTR_ERR(branch); + mark_reg_func_ptr(env, branch->frame[branch->curframe]->regs, value_regno, + targets[i]); + } + mark_reg_func_ptr(env, regs, value_regno, targets[n - 1]); + return 0; + +overlap: + verbose(env, "read of %d bytes at offset [%llu,%llu] of map '%s' overlaps with a pointer to a function\n", + size, min_off, max_off, map->name); + return -EACCES; +} + static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, int bpf_size, int value_regno, bool is_ldsx) { struct bpf_reg_state *regs = cur_regs(env); int size = bpf_size_to_bytes(bpf_size); struct bpf_map *map = reg->map_ptr; + int err; switch (map->map_type) { case BPF_MAP_TYPE_INSN_ARRAY: @@ -6350,13 +7204,18 @@ static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state break; } + if (env->func_ptr_cnt) { + err = check_func_ptr_read(env, reg, off, size, value_regno); + if (err != -ENOENT) + return err; + } + /* If map is read-only, track its contents as scalars. */ if (tnum_is_const(reg->var_off) && bpf_map_is_rdonly(map) && map->ops->map_direct_value_addr) { int map_off = off + reg->var_off.value; u64 val = 0; - int err; err = bpf_map_direct_read(map, map_off, size, &val, is_ldsx); if (err) @@ -6383,6 +7242,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b int value_regno, bool strict_alignment_once, bool is_ldsx) { struct bpf_reg_state *regs = cur_regs(env); + enum bpf_reg_type ptr_type = reg->type; int size, err = 0; size = bpf_size_to_bytes(bpf_size); @@ -6453,6 +7313,15 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b return -EACCES; } + if (rdonly_untrusted && !env->allow_ptr_leaks) { + verbose(env, "%s access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", + reg_type_str(env, reg->type)); + bpf_diag_policy(env, insn_idx, "read from untrusted read-only memory", + "the access requires CAP_PERFMON", + "Load the program with CAP_PERFMON, or avoid dereferencing untrusted pointers."); + return -EPERM; + } + /* * Accesses to untrusted PTR_TO_MEM are done through probe * instructions, hence no need to check bounds in that case. @@ -6499,6 +7368,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b regs[value_regno].btf = info.btf; regs[value_regno].btf_id = info.btf_id; regs[value_regno].id = info.ref_id; + } else if (base_type(info.reg_type) == PTR_TO_MEM) { + regs[value_regno].mem_size = info.mem_size; } if (type_may_be_null(info.reg_type) && !regs[value_regno].id) regs[value_regno].id = ++env->id_gen; @@ -6580,7 +7451,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b } err = check_buffer_access(env, reg, argno, off, size, false, - max_access); + max_access, t == BPF_WRITE); if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ)) mark_reg_unknown(env, regs, value_regno); @@ -6621,12 +7492,32 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b clear_scalar_id(®s[value_regno]); } } + + if (!err && bpf_is_mem_insn(&env->prog->insnsi[insn_idx]) && ptr_type != PTR_TO_STACK) + mark_non_stack_access(env, insn_idx); + return err; } static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, bool allow_trust_mismatch); +/* + * Returns the register to check the access of the current insn with. + * For a number in a program with an arena that is 'arena'. + */ +static struct bpf_reg_state *mem_access_reg(struct bpf_verifier_env *env, int regno, + struct bpf_reg_state *arena) +{ + if (!is_arena_scalar(env, regno)) + return cur_regs(env) + regno; + + memset(arena, 0, sizeof(*arena)); + arena->type = PTR_TO_ARENA; + env->insn_aux_data[env->insn_idx].arena_scalar = true; + return arena; +} + static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, bool strict_alignment_once, bool is_ldsx, bool allow_trust_mismatch, const char *ctx) @@ -6634,6 +7525,7 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); + struct bpf_reg_state arena, *src_reg; enum bpf_reg_type src_reg_type; int err; @@ -6655,16 +7547,28 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; - src_reg_type = regs[insn->src_reg].type; + src_reg = mem_access_reg(env, insn->src_reg, &arena); + src_reg_type = src_reg->type; /* * check_stack_read_fixed_off() may refine the modification's origin to * the source stack slot. */ bpf_diag_mod_begin(env, ®s[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE); - err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, argno_from_reg(insn->src_reg), insn->off, - BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, + err = check_mem_access(env, env->insn_idx, src_reg, argno_from_reg(insn->src_reg), + insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, strict_alignment_once, is_ldsx); + /* + * Paths through a kernel struct pointer and through other pointers + * may share the insn, see save_aux_ptr_type(). Record a load that did + * not produce an arena pointer from an __arena member, the only load + * through a kernel struct pointer that does, next to the one + * check_arena_member_load() records, so that arena_member_fixup() can + * tell when the paths disagree. + */ + if (!err && !(base_type(src_reg_type) == PTR_TO_BTF_ID && + regs[insn->dst_reg].type == PTR_TO_ARENA)) + env->insn_aux_data[env->insn_idx].non_arena_load = true; err = err ?: save_aux_ptr_type(env, src_reg_type, allow_trust_mismatch); err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], ctx); @@ -6680,6 +7584,7 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); + struct bpf_reg_state arena, *dst_reg; enum bpf_reg_type dst_reg_type; int err; @@ -6701,11 +7606,12 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; - dst_reg_type = regs[insn->dst_reg].type; + dst_reg = mem_access_reg(env, insn->dst_reg, &arena); + dst_reg_type = dst_reg->type; /* Check if (dst_reg + off) is writeable. */ - err = check_mem_access(env, env->insn_idx, regs + insn->dst_reg, argno_from_reg(insn->dst_reg), insn->off, - BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, + err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), + insn->off, BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, strict_alignment_once, false); err = err ?: save_aux_ptr_type(env, dst_reg_type, false); @@ -6715,7 +7621,7 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, static int check_atomic_rmw(struct bpf_verifier_env *env, struct bpf_insn *insn) { - struct bpf_reg_state *dst_reg; + struct bpf_reg_state arena, *dst_reg; int load_reg; int err; @@ -6760,6 +7666,9 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return -EACCES; } + /* load_reg may be dst_reg. Look at dst_reg before it's marked as unknown. */ + dst_reg = mem_access_reg(env, insn->dst_reg, &arena); + load_reg = bpf_atomic_load_reg(insn); if (load_reg >= 0) { /* check and record load of old value */ @@ -6768,8 +7677,6 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return err; } - dst_reg = cur_regs(env) + insn->dst_reg; - /* Check whether we can read the memory, with second call for fetch * case to simulate the register fill. */ @@ -6896,8 +7803,11 @@ static int check_stack_range_initialized( * but BTF based global subprog validation isn't accurate enough. */ bool allow_poison = access_size < 0 || clobber; - /* The call will initialize the memory; uninitialized stack allowed */ - bool raw_mode = meta && meta->arg_raw_mem.regno == reg_from_argno(argno); + u32 arg_slot = arg_slot_from_argno(argno); + bool uninit = clobber && meta && arg_slot < MAX_BPF_FUNC_ARGS && + (meta->arg_raw_mem.mask & BIT(arg_slot)); + bool raw_mode = uninit && env->allow_uninit_stack && + !(meta->arg_raw_mem.var_size_mask & BIT(arg_slot)); access_size = abs(access_size); @@ -6926,11 +7836,9 @@ static int check_stack_range_initialized( reg_arg_name(env, argno), tn_buf); return -EACCES; } - /* Only initialized buffer on stack is allowed to be accessed - * with variable offset. With uninitialized buffer it's hard to - * guarantee that whole memory is marked as initialized on - * helper return since specific bounds are unknown what may - * cause uninitialized stack leaking. + /* + * The call may touch any byte in the possible range, but does not + * definitely initialize all of it. Fall back to ordinary stack checks. */ raw_mode = false; @@ -6938,12 +7846,14 @@ static int check_stack_range_initialized( max_off = reg_smax(reg) + off; } + /* Unprivileged outputs retain each byte's initialization state. */ if (raw_mode) { - meta->arg_raw_mem.size = access_size; + meta->arg_raw_mem.size[arg_slot] = access_size; return 0; } for (i = min_off; i < max_off + access_size; i++) { + struct bpf_stack_state *ss; u8 *stype; slot = -i - 1; @@ -6953,32 +7863,41 @@ static int check_stack_range_initialized( return -EFAULT; } - stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; + ss = bpf_stack_slot(state, spi); + stype = &ss->slot_type[slot % BPF_REG_SIZE]; if (*stype == STACK_MISC) goto mark; if ((*stype == STACK_ZERO) || - (*stype == STACK_INVALID && env->allow_uninit_stack)) { - if (clobber) { + (*stype == STACK_INVALID && (uninit || env->allow_uninit_stack))) { + if (clobber && (*stype != STACK_INVALID || env->allow_uninit_stack)) { /* helper can write anything into the stack */ *stype = STACK_MISC; } goto mark; } - if (bpf_is_spilled_reg(&state->stack[spi]) && - (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || + /* + * Only the bytes marked STACK_SPILL hold the spilled register. + * The rest of a narrowly spilled slot keeps its previous type + * and must be initialized on its own. + */ + if (*stype == STACK_SPILL && + (ss->spilled_ptr.type == SCALAR_VALUE || env->allow_ptr_leaks)) { if (clobber) { - __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); + __mark_reg_unknown(env, &ss->spilled_ptr); for (j = 0; j < BPF_REG_SIZE; j++) - scrub_spilled_slot(&state->stack[spi].slot_type[j]); + scrub_spilled_slot(&ss->slot_type[j]); } goto mark; } if (*stype == STACK_POISON) { - if (allow_poison) + if (allow_poison) { + if (uninit && env->allow_uninit_stack) + *stype = STACK_MISC; goto mark; + } verbose(env, "reading from stack %s off %d+%d size %d, slot poisoned by dead code elimination\n", reg_arg_name(env, argno), min_off, i - min_off, access_size); } else if (tnum_is_const(reg->var_off)) { @@ -7012,6 +7931,10 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ switch (base_type(reg->type)) { case PTR_TO_PACKET: case PTR_TO_PACKET_META: + if (!may_access_direct_pkt_data(env, meta, access_type)) { + verbose(env, "function access to the packet is not allowed\n"); + return -EACCES; + } return check_packet_access(env, reg, argno, 0, access_size, zero_size_allowed); case PTR_TO_MAP_KEY: @@ -7052,7 +7975,7 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ } return check_buffer_access(env, reg, argno, 0, access_size, zero_size_allowed, - max_access); + max_access, access_type == BPF_WRITE); case PTR_TO_STACK: return check_stack_range_initialized( env, reg, @@ -7122,13 +8045,13 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, */ meta->msize_max_value = reg_umax(size_reg); - /* The register is SCALAR_VALUE; the access check happens using - * its boundaries. For unprivileged variable accesses, disable - * raw mode so that the program is required to initialize all - * the memory that the helper could just partially fill up. + /* + * Check variable ranges byte by byte instead of using raw mode. Keep the + * MEM_UNINIT annotation so invalid bytes are accepted without marking them + * initialized when the caller cannot read uninitialized stack memory. */ if (!tnum_is_const(size_reg->var_off)) - meta = NULL; + meta->arg_raw_mem.var_size_mask |= BIT(arg_slot_from_argno(mem_argno)); if (reg_smin(size_reg) < 0) { verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n", @@ -7171,6 +8094,11 @@ size_error: return err; } +static bool is_subprog(const struct bpf_call_arg_meta *meta) +{ + return meta->btf && !meta->func_id; +} + static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, u32 mem_size, enum bpf_access_type access_type, struct bpf_call_arg_meta *meta, bool *known_memory) @@ -7189,10 +8117,11 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg } /* - * Only a global subprog (meta == NULL) may read poisoned stack slots: + * Only a global subprog may read poisoned stack slots: * its static stack liveness proved the callee body skips them. */ - size = (!meta && base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size; + size = (is_subprog(meta) && + base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size; if (access_type & BPF_READ) err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta, @@ -7484,6 +8413,9 @@ static int check_map_field_pointer(struct bpf_verifier_env *env, struct bpf_reg_ case BPF_TASK_WORK: field_off = map->record->task_work_off; break; + case BPF_RCU_HEAD: + field_off = map->record->rcu_head_off; + break; case BPF_WORKQUEUE: field_off = map->record->wq_off; break; @@ -7572,11 +8504,11 @@ __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u /* * Validate dynptr arguments for helper, kfunc and subprog. * - * @dynptr is both input and output. It is populated when the argument is - * tagged with MEM_UNINIT (i.e., the dynptr argument that will be constructed) - * and consumed when the argument is expecting to be an initialized dynptr. - * @parent_id is used to track the referenced parent object (e.g., file or skb in - * qdisc program) when constructing a dynptr. + * @meta carries the dynptr and referenced-object state. The dynptr is populated + * when the argument is tagged with MEM_UNINIT (i.e., the dynptr argument that + * will be constructed) and consumed when the argument is expected to be an + * initialized dynptr. The reference tracks the parent object (e.g., file or skb + * in qdisc program) when constructing a dynptr. * * There are two register types representing a bpf_dynptr, one is PTR_TO_STACK * which points to a stack slot, and the other is CONST_PTR_TO_DYNPTR. @@ -7593,24 +8525,11 @@ __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u * and checked dynamically during runtime. */ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - argno_t argno, int insn_idx, const char *call_name, - enum bpf_arg_type arg_type, - struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr) + argno_t argno, int insn_idx, enum bpf_arg_type arg_type, + struct bpf_call_arg_meta *meta) { int spi, err = 0; - if (reg->type != PTR_TO_STACK && reg->type != CONST_PTR_TO_DYNPTR) { - verbose(env, - "%s expected pointer to stack or const struct bpf_dynptr\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt( - env, insn_idx, argno, call_name, - "Pass the address of a stack dynptr object, or use a const dynptr pointer returned by the verifier-supported path.", - "a dynptr argument must be a pointer to a dynptr stack slot or a verifier-provided const struct bpf_dynptr, but %s is %s", - reg_arg_name(env, argno), bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - /* MEM_UNINIT - Points to memory that is an appropriate candidate for * constructing a mutable bpf_dynptr object. * @@ -7632,7 +8551,7 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat verbose(env, "Dynptr has to be an uninitialized dynptr\n"); bpf_diag_res( env, insn_idx, "dynptr is already initialized", - "This kfunc constructs a dynptr and requires an uninitialized dynptr stack slot, but the selected slot already holds dynptr state.", + "This function constructs a dynptr and requires an uninitialized dynptr stack slot, but the selected slot already holds dynptr state.", "Use a fresh stack dynptr slot, or release/destroy the existing dynptr before reusing the slot."); return -EINVAL; } @@ -7645,7 +8564,8 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat return err; } - err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, ref_obj, dynptr); + err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, + &meta->ref_obj, &meta->dynptr); } else /* OBJ_RELEASE and None case from above */ { /* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */ if (reg->type == CONST_PTR_TO_DYNPTR && (arg_type & OBJ_RELEASE)) { @@ -7675,7 +8595,7 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat verbose(env, "Expected a dynptr of type %s as %s\n", dynptr_type_str(expected_type), reg_arg_name(env, argno)); bpf_diag_call_arg_fmt( - env, insn_idx, argno, call_name, + env, insn_idx, argno, meta->func_name, "Use a dynptr constructor that matches this operation, or call an operation that accepts the dynptr's current type.", "the dynptr is initialized with backing object type %s, but this operation expects dynptr type %s", dynptr_type_str(actual_type), dynptr_type_str(expected_type)); @@ -7691,14 +8611,12 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat mark_stack_slots_scratched(env, spi, BPF_DYNPTR_NR_SLOTS); - reg = &state->stack[spi].spilled_ptr; + reg = &bpf_stack_slot(state, spi)->spilled_ptr; } - if (dynptr) { - dynptr->type = reg->dynptr.type; - dynptr->id = reg->id; - dynptr->parent_id = reg->parent_id; - } + meta->dynptr.type = reg->dynptr.type; + meta->dynptr.id = reg->id; + meta->dynptr.parent_id = reg->parent_id; } return err; } @@ -7731,12 +8649,12 @@ static bool is_kfunc_arg_iter(struct bpf_call_arg_meta *meta, int arg_idx, return btf_param_match_suffix(meta->btf, arg, "__iter"); } -static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx, +static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + u32 arg, argno_t argno, int insn_idx, struct bpf_call_arg_meta *meta) { struct bpf_func_state *state = bpf_func(env, reg); const struct btf_type *t; - u32 arg_idx = arg_idx_from_argno(argno); int spi, err, i, nr_slots, btf_id; if (reg->type != PTR_TO_STACK) { @@ -7756,14 +8674,14 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * * to any kfunc, if arg has "__iter" suffix, we need to be a bit more * conservative here. */ - btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, arg_idx); + btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, arg); if (btf_id < 0) { verbose(env, "expected valid iter pointer as %s\n", reg_arg_name(env, argno)); bpf_diag_call_arg( env, insn_idx, argno, meta->func_name, - "the kfunc expects a recognized iterator state pointer, but this argument does not match a valid iterator type", - "Pass the exact iterator state type expected by this kfunc."); + "the function expects a recognized iterator state pointer, but this argument does not match a valid iterator type", + "Pass the exact iterator state type expected by this function."); return -EINVAL; } t = btf_type_by_id(meta->btf, btf_id); @@ -7826,7 +8744,7 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * /* remember meta->iter info for process_iter_next_call() */ meta->iter.spi = spi; meta->iter.frameno = reg->frameno; - update_ref_obj(&meta->ref_obj, &state->stack[spi].spilled_ptr); + update_ref_obj(&meta->ref_obj, &bpf_stack_slot(state, spi)->spilled_ptr); if (is_iter_destroy_kfunc(meta)) { err = unmark_stack_slots_iter(env, reg, nr_slots); @@ -7903,16 +8821,15 @@ static int widen_imprecise_scalars(struct bpf_verifier_env *env, &fold->regs[i], &fcur->regs[i]); - num_slots = min(fold->allocated_stack / BPF_REG_SIZE, - fcur->allocated_stack / BPF_REG_SIZE); + num_slots = min(bpf_stack_nr_slots(fold), bpf_stack_nr_slots(fcur)); for (i = 0; i < num_slots; i++) { - if (!bpf_is_spilled_reg(&fold->stack[i]) || - !bpf_is_spilled_reg(&fcur->stack[i])) + struct bpf_stack_state *old_ss = bpf_stack_slot(fold, i); + struct bpf_stack_state *cur_ss = bpf_stack_slot(fcur, i); + + if (!bpf_is_spilled_reg(old_ss) || !bpf_is_spilled_reg(cur_ss)) continue; - maybe_widen_reg(env, - &fold->stack[i].spilled_ptr, - &fcur->stack[i].spilled_ptr); + maybe_widen_reg(env, &old_ss->spilled_ptr, &cur_ss->spilled_ptr); } } return 0; @@ -7924,7 +8841,7 @@ static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_ int iter_frameno = meta->iter.frameno; int iter_spi = meta->iter.spi; - return &cur_st->frame[iter_frameno]->stack[iter_spi].spilled_ptr; + return &bpf_stack_slot(cur_st->frame[iter_frameno], iter_spi)->spilled_ptr; } /* process_iter_next_call() is called when verifier gets to iterator's next @@ -8087,9 +9004,52 @@ static bool arg_type_is_dynptr(enum bpf_arg_type type) return base_type(type) == ARG_PTR_TO_DYNPTR; } +/* + * An argument that only ever takes a scalar, so a zero register passed to it + * is a value rather than a NULL pointer. + */ +static bool arg_type_is_scalar(enum bpf_arg_type type) +{ + switch (base_type(type)) { + case ARG_SCALAR: + case ARG_CONST_SCALAR: + case ARG_MEM_SIZE: + case ARG_MEM_SIZE_OR_ZERO: + case ARG_CONST_MEM_SIZE: + case ARG_CONST_ALLOC_SIZE_OR_ZERO: + return true; + default: + return false; + } +} + +/* + * A helper has no BTF and a nonzero function ID. A kfunc has both, while a + * BPF subprogram has BTF and a zero function ID. + */ +static bool is_helper(const struct bpf_call_arg_meta *meta) +{ + return !meta->btf && meta->func_id; +} + +static bool is_helper_call(const struct bpf_call_arg_meta *meta, enum bpf_func_id func_id) +{ + return is_helper(meta) && meta->func_id == func_id; +} + +static bool is_kfunc(const struct bpf_call_arg_meta *meta) +{ + return meta->btf && meta->func_id; +} + +static bool is_kfunc_call(const struct bpf_call_arg_meta *meta, u32 btf_id) +{ + return is_kfunc(meta) && meta->func_id == btf_id; +} + static int resolve_map_arg_type(struct bpf_verifier_env *env, - const struct bpf_call_arg_meta *meta, - enum bpf_arg_type *arg_type) + const struct bpf_call_arg_meta *meta, + enum bpf_arg_type *arg_type) { if (!meta->map.ptr) { /* kernel subsystem misconfigured verifier */ @@ -8108,7 +9068,7 @@ static int resolve_map_arg_type(struct bpf_verifier_env *env, } break; case BPF_MAP_TYPE_BLOOM_FILTER: - if (meta->func_id == BPF_FUNC_map_peek_elem) + if (is_helper_call(meta, BPF_FUNC_map_peek_elem)) *arg_type = ARG_PTR_TO_MAP_VALUE; break; default: @@ -8117,6 +9077,49 @@ static int resolve_map_arg_type(struct bpf_verifier_env *env, return 0; } +static int resolve_func_arg_type(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, u32 arg, argno_t argno, + struct bpf_call_arg_meta *meta, + enum bpf_arg_type *arg_type, u32 *arg_size); +static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, enum bpf_arg_type arg_type, + const struct btf *arg_btf, u32 arg_btf_id, + struct bpf_call_arg_meta *meta, int insn_idx); +static bool is_kfunc_arg_nonown_allowed(const struct btf *btf, + const struct btf_param *arg); +static bool is_kfunc_arg_scalar_with_name(const struct btf *btf, + const struct btf_param *arg, + const char *name); +static bool is_bpf_cast_to_kern_ctx_kfunc(const struct bpf_call_arg_meta *meta); +static bool is_bpf_dynptr_clone_kfunc(const struct bpf_call_arg_meta *meta); +static bool is_bpf_iter_css_task_new_kfunc(const struct bpf_call_arg_meta *meta); +static bool is_kfunc_dynptr_write(struct bpf_call_arg_meta *meta); +static bool is_bpf_obj_drop_kfunc(u32 func_id); +static bool is_bpf_percpu_obj_drop_kfunc(u32 func_id); +static bool is_bpf_rbtree_add_kfunc(u32 func_id); +static int get_bpf_res_spin_lock_kfunc_flags(const struct bpf_call_arg_meta *meta); +static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env); +static int process_irq_flag(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, argno_t argno, + struct bpf_call_arg_meta *meta); +static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, + argno_t argno, + struct bpf_call_arg_meta *meta); +static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, + argno_t argno, + struct bpf_call_arg_meta *meta); +static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, + argno_t argno, + struct bpf_call_arg_meta *meta); +static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, + argno_t argno, + struct bpf_call_arg_meta *meta); +static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env); + struct bpf_reg_types { const enum bpf_reg_type types[10]; u32 *btf_id; @@ -8160,7 +9163,7 @@ static const struct bpf_reg_types mem_types = { }, }; -static const struct bpf_reg_types spin_lock_types = { +static const struct bpf_reg_types map_value_or_alloc_obj_types = { .types = { PTR_TO_MAP_VALUE, PTR_TO_BTF_ID | MEM_ALLOC, @@ -8189,7 +9192,33 @@ static const struct bpf_reg_types percpu_btf_ptr_types = { static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } }; static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } }; static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } }; -static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } }; +static const struct bpf_reg_types map_value_types = { .types = { PTR_TO_MAP_VALUE } }; +static const struct bpf_reg_types arena_types = { + .types = { + PTR_TO_ARENA, + SCALAR_VALUE, + } +}; +static const struct bpf_reg_types ctx_out_types = { + .types = { PTR_TO_MEM | MEM_RDONLY | PTR_TRUSTED }, +}; + +static const struct bpf_reg_types alloc_obj_drop_types = { + .types = { + PTR_TO_BTF_ID | MEM_ALLOC, + PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU, + } +}; + +static const struct bpf_reg_types alloc_obj_types = { + .types = { + PTR_TO_BTF_ID | MEM_ALLOC, + PTR_TO_BTF_ID | MEM_ALLOC | MEM_RCU, + PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF, + PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU, + } +}; + static const struct bpf_reg_types kptr_xchg_dest_types = { .types = { PTR_TO_MAP_VALUE, @@ -8220,16 +9249,32 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { #endif [ARG_PTR_TO_SOCKET] = &fullsock_types, [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, - [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types, + [ARG_PTR_TO_SPIN_LOCK] = &map_value_or_alloc_obj_types, [ARG_PTR_TO_MEM] = &mem_types, [ARG_PTR_TO_RINGBUF_MEM] = &ringbuf_mem_types, [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, [ARG_PTR_TO_FUNC] = &func_ptr_types, [ARG_PTR_TO_STACK] = &stack_ptr_types, [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types, - [ARG_PTR_TO_TIMER] = &timer_types, + [ARG_PTR_TO_TIMER] = &map_value_types, [ARG_KPTR_XCHG_DEST] = &kptr_xchg_dest_types, [ARG_PTR_TO_DYNPTR] = &dynptr_types, + [ARG_CONST_SCALAR] = &scalar_types, + [ARG_CONST_MEM_SIZE] = &scalar_types, + [ARG_PTR_TO_ALLOC_BTF_ID] = &alloc_obj_drop_types, + [ARG_PTR_TO_REFCOUNTED_KPTR] = &alloc_obj_types, + [ARG_PTR_TO_ITER] = &stack_ptr_types, + [ARG_PTR_TO_LIST_HEAD] = &map_value_or_alloc_obj_types, + [ARG_PTR_TO_LIST_NODE] = &alloc_obj_types, + [ARG_PTR_TO_RB_ROOT] = &map_value_or_alloc_obj_types, + [ARG_PTR_TO_RB_NODE] = &alloc_obj_types, + [ARG_PTR_TO_RES_SPIN_LOCK] = &map_value_or_alloc_obj_types, + [ARG_PTR_TO_WORKQUEUE] = &map_value_types, + [ARG_PTR_TO_TASK_WORK] = &map_value_types, + [ARG_PTR_TO_RCU_HEAD] = &map_value_types, + [ARG_PTR_TO_IRQ_FLAG] = &stack_ptr_types, + [ARG_PTR_TO_ARENA] = &arena_types, + [ARG_PTR_TO_CTX_OUT] = &ctx_out_types, }; static void bpf_diag_call_arg(struct bpf_verifier_env *env, u32 insn_idx, argno_t argno, @@ -8269,6 +9314,71 @@ __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u bpf_diag_call_arg(env, insn_idx, argno, call_name, reason, suggestion); } +static int check_func_arg_nullability(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, u32 arg, argno_t argno, + enum bpf_arg_type arg_type, + struct bpf_call_arg_meta *meta, int insn_idx) +{ + const char *expected_type = "pointer"; + + if (arg_type_is_scalar(arg_type) || type_may_be_null(arg_type) || + (!bpf_register_is_null(reg) && !type_may_be_null(reg->type))) + return 0; + + if (meta->btf) { + u32 arg_btf_id; + + arg_btf_id = btf_params(meta->func_proto)[arg].type; + expected_type = bpf_diag_fmt(env, "value of type %s", + bpf_diag_fmt_btf_type(env, meta->btf, arg_btf_id)); + } + + verbose(env, "Possibly NULL pointer passed to trusted %s\n", + reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Add a NULL check and make the call only on the non-NULL path.", + "the pointer may be NULL, but this call requires a non-NULL %s", + expected_type); + return -EACCES; +} + +static int check_func_arg_release(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + u32 arg, argno_t argno, enum bpf_arg_type arg_type, + struct bpf_call_arg_meta *meta, int insn_idx) +{ + const char *expected_type = "pointer"; + + if (!arg_type_is_release(arg_type)) + return 0; + + if (arg_type_is_dynptr(arg_type) || reg_is_referenced(env, reg) || + bpf_register_is_null(reg)) + return 0; + + verbose(env, "release function %s expects referenced PTR_TO_BTF_ID passed to %s\n", + meta->func_name, reg_arg_name(env, argno)); + + if (is_kfunc(meta)) { + const struct btf_param *btf_arg; + const struct btf_type *t; + u32 ref_id; + + btf_arg = &btf_params(meta->func_proto)[arg]; + ref_id = btf_arg->type; + t = btf_type_skip_modifiers(meta->btf, btf_arg->type, NULL); + if (btf_type_is_ptr(t)) + btf_type_skip_modifiers(meta->btf, t->type, &ref_id); + expected_type = bpf_diag_fmt(env, "value of type %s", + bpf_diag_fmt_btf_type(env, meta->btf, ref_id)); + } + + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + bpf_diag_fmt(env, "Pass the resource-owning %s returned by the matching acquire call, or avoid the release function after ownership has already been transferred or released.", + expected_type), + "release functions require a value that owns a live resource returned by a matching acquire function"); + return -EINVAL; +} + static const char *bpf_diag_expected_reg_types(struct bpf_verifier_env *env, const enum bpf_reg_type *types, int count) { @@ -8290,19 +9400,21 @@ static const char *bpf_diag_expected_reg_types(struct bpf_verifier_env *env, } static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type, const u32 *arg_btf_id, - struct bpf_call_arg_meta *meta, const char *call_name) + enum bpf_arg_type arg_type, struct bpf_call_arg_meta *meta) { enum bpf_reg_type expected, type = reg->type; const struct bpf_reg_types *compatible; const char *actual, *accepted; - int i, j, err; + int i, j; compatible = compatible_reg_types[base_type(arg_type)]; if (!compatible) { verifier_bug(env, "unsupported arg type %d", arg_type); return -EFAULT; } + if (is_kfunc(meta) && base_type(arg_type) == ARG_PTR_TO_BTF_ID && + (base_type(type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(type)])) + goto found; /* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY, * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY @@ -8322,9 +9434,14 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re type &= ~PTR_MAYBE_NULL; if (base_type(arg_type) == ARG_PTR_TO_MEM) type &= ~DYNPTR_TYPE_FLAG_MASK; + /* Allow allocated memory for BTF-defined ARG_PTR_TO_MEM but not helpers. */ + if (!is_helper(meta) && base_type(arg_type) == ARG_PTR_TO_MEM && + type_is_ptr_alloc_obj(type)) + type = PTR_TO_MEM; /* Local kptr types are allowed as the source argument of bpf_kptr_xchg */ - if (meta->func_id == BPF_FUNC_kptr_xchg && type_is_alloc(type) && reg_from_argno(argno) == BPF_REG_2) { + if (is_helper_call(meta, BPF_FUNC_kptr_xchg) && type_is_alloc(type) && + reg_from_argno(argno) == BPF_REG_2) { type &= ~MEM_ALLOC; type &= ~MEM_PERCPU; } @@ -8344,115 +9461,13 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); actual = bpf_diag_fmt(env, "%s", reg_type_str(env, reg->type)); accepted = bpf_diag_expected_reg_types(env, compatible->types, i); - bpf_diag_call_arg_fmt(env, env->insn_idx, argno, call_name, - "Pass a value with one of the accepted pointer or scalar types for this call.", + bpf_diag_call_arg_fmt(env, env->insn_idx, argno, meta->func_name, + bpf_diag_fmt(env, "Pass %s.", bpf_diag_arg_type_plain(arg_type)), "it has type %s, but this argument accepts %s", actual, accepted); return -EACCES; found: - if (base_type(reg->type) != PTR_TO_BTF_ID) - return 0; - - if (compatible == &mem_types) { - if (!(arg_type & MEM_RDONLY)) { - verbose(env, - "%s() may write into memory pointed by %s type=%s\n", - func_id_name(meta->func_id), - reg_arg_name(env, argno), reg_type_str(env, reg->type)); - return -EACCES; - } - return 0; - } - - switch ((int)reg->type) { - case PTR_TO_BTF_ID: - case PTR_TO_BTF_ID | PTR_TRUSTED: - case PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL: - case PTR_TO_BTF_ID | MEM_RCU: - case PTR_TO_BTF_ID | PTR_MAYBE_NULL: - case PTR_TO_BTF_ID | PTR_MAYBE_NULL | MEM_RCU: - { - /* For bpf_sk_release, it needs to match against first member - * 'struct sock_common', hence make an exception for it. This - * allows bpf_sk_release to work for multiple socket types. - */ - bool strict_type_match = arg_type_is_release(arg_type) && - meta->func_id != BPF_FUNC_sk_release; - - if (type_may_be_null(reg->type) && - (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) { - verbose(env, "Possibly NULL pointer passed to helper %s\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg( - env, env->insn_idx, argno, call_name, - "the pointer may be NULL, but this call requires a non-NULL pointer", - "Add a NULL check and make the call only on the non-NULL path."); - return -EACCES; - } - - if (!arg_btf_id) { - if (!compatible->btf_id) { - verifier_bug(env, "missing arg compatible BTF ID"); - return -EFAULT; - } - arg_btf_id = compatible->btf_id; - } - - if (meta->func_id == BPF_FUNC_kptr_xchg) { - if (map_kptr_match_type(env, meta->kptr_field, reg, reg_from_argno(argno))) - return -EACCES; - } else { - if (arg_btf_id == BPF_PTR_POISON) { - verbose(env, "verifier internal error:"); - verbose(env, "%s has non-overwritten BPF_PTR_POISON type\n", - reg_arg_name(env, argno)); - return -EACCES; - } - - err = __check_ptr_off_reg(env, reg, argno, true); - if (err) - return err; - - if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, - reg->var_off.value, btf_vmlinux, *arg_btf_id, - strict_type_match, !type_is_alloc(reg->type))) { - verbose(env, "%s is of type %s but %s is expected\n", - reg_arg_name(env, argno), - btf_type_name(reg->btf, reg->btf_id), - btf_type_name(btf_vmlinux, *arg_btf_id)); - return -EACCES; - } - } - break; - } - case PTR_TO_BTF_ID | MEM_ALLOC: - case PTR_TO_BTF_ID | MEM_PERCPU | MEM_ALLOC: - case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF: - case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: - if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock && - meta->func_id != BPF_FUNC_kptr_xchg) { - verifier_bug(env, "unimplemented handling of MEM_ALLOC"); - return -EFAULT; - } - /* Check if local kptr in src arg matches kptr in dst arg */ - if (meta->func_id == BPF_FUNC_kptr_xchg) { - int regno = reg_from_argno(argno); - - if (regno == BPF_REG_2 && - map_kptr_match_type(env, meta->kptr_field, reg, regno)) - return -EACCES; - } - break; - case PTR_TO_BTF_ID | MEM_PERCPU: - case PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU: - case PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED: - /* Handled by helper specific checks */ - break; - default: - verifier_bug(env, "invalid PTR_TO_BTF_ID register for type match"); - return -EFAULT; - } return 0; } @@ -8473,10 +9488,9 @@ reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields) return field; } -static int __check_func_arg_reg_off(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type, - bool btf_id_fixed_off_ok) +static int check_func_arg_reg_off(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, argno_t argno, + enum bpf_arg_type arg_type) { u32 type = reg->type; @@ -8532,12 +9546,15 @@ static int __check_func_arg_reg_off(struct bpf_verifier_env *env, case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF: case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: /* When referenced PTR_TO_BTF_ID is passed to release function, - * its fixed offset must be 0. In the other cases, fixed offset - * can be non-zero unless the caller requires otherwise. - * var_off always must be 0 for PTR_TO_BTF_ID, hence we still - * need to do checks instead of returning. + * its fixed offset must be 0. bpf_refcount_acquire() returns the + * pointer it was given while incrementing the refcount at the + * refcount field offset, so it needs a zero offset too. In the + * other cases, fixed offset can be non-zero. var_off always must + * be 0 for PTR_TO_BTF_ID, hence we still need to do checks + * instead of returning. */ - return __check_ptr_off_reg(env, reg, argno, btf_id_fixed_off_ok); + return __check_ptr_off_reg(env, reg, argno, + base_type(arg_type) != ARG_PTR_TO_REFCOUNTED_KPTR); case PTR_TO_CTX: /* * Allow fixed and variable offsets for syscall context, but @@ -8545,7 +9562,7 @@ static int __check_func_arg_reg_off(struct bpf_verifier_env *env, * otherwise we may get modified ctx in tail called programs and * global subprogs (that may act as extension prog hooks). */ - if (arg_type != ARG_PTR_TO_CTX && is_var_ctx_off_allowed(env->prog)) + if (base_type(arg_type) != ARG_PTR_TO_CTX && is_var_ctx_off_allowed(env->prog)) return 0; fallthrough; default: @@ -8553,13 +9570,6 @@ static int __check_func_arg_reg_off(struct bpf_verifier_env *env, } } -static int check_func_arg_reg_off(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type) -{ - return __check_func_arg_reg_off(env, reg, argno, arg_type, true); -} - static int check_arg_const_str(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno) { @@ -8568,6 +9578,7 @@ static int check_arg_const_str(struct bpf_verifier_env *env, int map_off; u64 map_addr; char *str_ptr; + u32 cnt; if (reg->type != PTR_TO_MAP_VALUE) return -EINVAL; @@ -8617,6 +9628,11 @@ static int check_arg_const_str(struct bpf_verifier_env *env, verbose(env, "string is not zero-terminated\n"); return -EINVAL; } + /* the bytes of a pointer to a function are not known until the program is jitted */ + if (bpf_map_range_func_ptrs(env, map, map_off, strlen(str_ptr + map_off) + 1, &cnt)) { + verbose(env, "string overlaps with a pointer to a function\n"); + return -EACCES; + } return 0; } @@ -8627,6 +9643,7 @@ static int get_constant_map_key(struct bpf_verifier_env *env, s64 *value) { struct bpf_func_state *state = bpf_func(env, key); + struct bpf_stack_state *ss; struct bpf_reg_state *reg; int slot, spi, off; int spill_size = 0; @@ -8646,7 +9663,8 @@ static int get_constant_map_key(struct bpf_verifier_env *env, slot = -stack_off - 1; spi = slot / BPF_REG_SIZE; off = slot % BPF_REG_SIZE; - stype = state->stack[spi].slot_type; + ss = bpf_stack_slot(state, spi); + stype = ss->slot_type; /* First handle precisely tracked STACK_ZERO */ for (i = off; i >= 0 && stype[i] == STACK_ZERO; i--) @@ -8657,14 +9675,14 @@ static int get_constant_map_key(struct bpf_verifier_env *env, } /* Check that stack contains a scalar spill of expected size */ - if (!bpf_is_spilled_scalar_reg(&state->stack[spi])) + if (!bpf_is_spilled_scalar_reg(ss)) return -EOPNOTSUPP; for (i = off; i >= 0 && stype[i] == STACK_SPILL; i--) spill_size++; if (spill_size != key_size) return -EOPNOTSUPP; - reg = &state->stack[spi].spilled_ptr; + reg = &ss->spilled_ptr; if (!tnum_is_const(reg->var_off)) /* Stack value not statically known */ return -EOPNOTSUPP; @@ -8708,6 +9726,8 @@ static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_stat obj_name = "timer"; else if (rec->task_work_off >= 0) obj_name = "bpf_task_work"; + else if (rec->rcu_head_off >= 0) + obj_name = "bpf_rcu_head"; verbose(env, "%s pointer in %s map_uid=%d ", obj_name, reg_arg_name(env, obj_argno), meta->map.uid); @@ -8721,65 +9741,89 @@ static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_stat return 0; } -static int check_func_arg(struct bpf_verifier_env *env, u32 arg, +/* + * MEM_WRITE alone denotes an input/output buffer. MEM_UNINIT marks an output + * whose incoming contents are not read. + */ +static enum bpf_access_type func_arg_access_type(enum bpf_arg_type arg_type) +{ + if (!(arg_type & MEM_WRITE)) + return BPF_READ; + if (arg_type & MEM_UNINIT) + return BPF_WRITE; + return BPF_READ | BPF_WRITE; +} + +static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 prev_slot, struct bpf_call_arg_meta *meta, int insn_idx) { + const struct btf_param *btf_arg = is_kfunc(meta) ? + &btf_params(meta->func_proto)[arg] : NULL; const struct bpf_func_proto *fn = meta->fn; - u32 regno = BPF_REG_1 + arg; - struct bpf_reg_state *reg = reg_state(env, regno); + struct bpf_func_state *caller = cur_func(env); + struct bpf_reg_state *regs = cur_regs(env); + argno_t argno = argno_from_arg(slot + 1); + struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, slot); enum bpf_arg_type arg_type = fn->arg_type[arg]; - argno_t argno = argno_from_reg(regno); - enum bpf_reg_type type = reg->type; - u32 *arg_btf_id = NULL; + int regno = reg_from_argno(argno); + u32 arg_size = arg_type & MEM_FIXED_SIZE ? fn->arg_size[arg] : 0; u32 key_size; int err = 0; - if (arg_type == ARG_DONTCARE) + if (arg_type == ARG_PTR_TO_PROG_AUX) { + cur_aux(env)->arg_prog = regno; return 0; + } - err = check_reg_arg(env, regno, SRC_OP); - if (err) - return err; + if (arg_type == ARG_IGNORE) + return 0; + if (regno >= 0) { + err = check_reg_arg(env, regno, SRC_OP); + if (err) + return err; + } + + /* Preserve the legacy helper behavior for privileged pointer leaks. */ if (arg_type == ARG_ANYTHING) { - if (is_pointer_value(env, regno)) { - verbose(env, "R%d leaks addr into helper function\n", - regno); + if (__is_pointer_value(env->allow_ptr_leaks, reg)) { + verbose(env, "%s leaks addr into helper function\n", + reg_arg_name(env, argno)); return -EACCES; } return 0; } - if (type_is_pkt_pointer(type) && - !may_access_direct_pkt_data(env, fn, BPF_READ)) { - verbose(env, "helper access to the packet is not allowed\n"); + err = resolve_func_arg_type(env, reg, arg, argno, meta, &arg_type, &arg_size); + if (err) + return err; + + /* + * A slice of a packet is packet data under another name. A helper which + * may reallocate the packet must not go on reading from the old one. + */ + if (is_helper(meta) && reg_is_dynptr_slice_pkt(reg) && + bpf_helper_changes_pkt_data(meta->func_id)) { + verbose(env, "function access to the packet is not allowed\n"); return -EACCES; } - if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE) { - err = resolve_map_arg_type(env, meta, &arg_type); - if (err) - return err; - } + if (arg_type_is_raw_mem(arg_type)) + meta->arg_raw_mem.mask |= BIT(slot); if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) { - /* A NULL register has a SCALAR_VALUE type, so skip - * type checking. - */ - err = mark_chain_precision(env, regno); + err = mark_arg_precision(env, argno); if (err) return err; - goto skip_type_check; + return 0; } - /* arg_btf_id and arg_size are in a union. */ - if (base_type(arg_type) == ARG_PTR_TO_BTF_ID || - base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) - arg_btf_id = fn->arg_btf_id[arg]; + err = check_func_arg_nullability(env, reg, arg, argno, arg_type, meta, insn_idx); + if (err) + return err; - err = check_reg_type(env, reg, argno, arg_type, arg_btf_id, meta, - func_id_name(meta->func_id)); + err = check_reg_type(env, reg, argno, arg_type, meta); if (err) return err; @@ -8787,22 +9831,27 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, if (err) return err; -skip_type_check: - if (arg_type_is_release(arg_type) && !arg_type_is_dynptr(arg_type) && - !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { - verbose(env, "release helper %s expects referenced PTR_TO_BTF_ID passed to %s\n", - func_id_name(meta->func_id), reg_arg_name(env, argno)); - bpf_diag_call_arg( - env, insn_idx, argno, func_id_name(meta->func_id), - "release helpers require a value that owns a live resource returned by a matching acquire helper", - "Pass the resource-owning pointer returned by the matching acquire helper, and avoid calling the release helper after ownership has already been transferred or released."); - return -EINVAL; - } + err = check_func_arg_release(env, reg, arg, argno, arg_type, meta, insn_idx); + if (err) + return err; if (reg_is_referenced(env, reg)) update_ref_obj(&meta->ref_obj, reg); switch (base_type(arg_type)) { + case ARG_CONST_SCALAR: + err = process_const_arg(env, reg, argno, meta); + if (err < 0) { + if (err == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass a compile-time constant or a value the verifier can prove is constant at this call.", + "the function requires this scalar argument to be a verifier-known constant, but %s is variable on this path", + reg_arg_name(env, argno)); + return err; + } + break; + case ARG_SCALAR: + break; case ARG_CONST_MAP_PTR: /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ err = process_map_ptr_arg(env, reg, argno, meta); @@ -8824,7 +9873,7 @@ skip_type_check: return -EFAULT; } key_size = meta->map.ptr->key_size; - err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, NULL, + err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, meta, NULL); if (err) return err; @@ -8852,21 +9901,86 @@ skip_type_check: return -EFAULT; } - /* - * Disable raw mode for bpf_map_peek_elem() on a bloom filter. The helper reads - * the value buffer as an input rather than filling it. - */ - if (meta->func_id == BPF_FUNC_map_peek_elem && - meta->map.ptr->map_type == BPF_MAP_TYPE_BLOOM_FILTER) - meta->arg_raw_mem.regno = 0; - err = check_helper_mem_access(env, reg, argno, meta->map.ptr->value_size, arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta, NULL); break; + case ARG_PTR_TO_BTF_ID: + case ARG_PTR_TO_BTF_ID_SOCK_COMMON: + { + const u32 *arg_btf_id = fn->arg_btf_id[arg]; + const struct btf *arg_btf = is_kfunc(meta) ? meta->btf : btf_vmlinux; + enum bpf_reg_type reg_type; + + if (!meta->btf) { + const struct bpf_reg_types *compatible; + + if (base_type(reg->type) != PTR_TO_BTF_ID) + break; + + if (is_helper_call(meta, BPF_FUNC_kptr_xchg)) + return map_kptr_match_type(env, meta->kptr_field, reg, regno) ? + -EACCES : 0; + + if (!arg_btf_id) { + compatible = compatible_reg_types[base_type(arg_type)]; + if (!compatible->btf_id) { + verifier_bug(env, "missing arg compatible BTF ID"); + return -EFAULT; + } + arg_btf_id = compatible->btf_id; + } + if (arg_btf_id == BPF_PTR_POISON) { + verbose(env, "verifier internal error:"); + verbose(env, "%s has non-overwritten BPF_PTR_POISON type\n", + reg_arg_name(env, argno)); + return -EACCES; + } + } + + reg_type = reg->type & ~(arg_type & PTR_MAYBE_NULL); + if ((arg_type & PTR_TRUSTED) && + (!is_trusted_reg_type(env, reg, reg_type) || + bpf_type_has_unsafe_modifiers(reg_type))) { + if (!(arg_type & MEM_RCU)) { + const char *actual_type, *arg_name, *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, arg_btf, *arg_btf_id); + verbose(env, "%s must be referenced or trusted\n", + reg_arg_name(env, argno)); + arg_name = reg_arg_name(env, argno); + actual_type = bpf_diag_reg_type_plain(env, reg->type); + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass a pointer acquired from a verifier-tracked source, or call this function only inside the required protection if it accepts RCU pointers.", + "the function requires a trusted or resource-owning pointer to %s, but %s is %s", + expected_type, arg_name, actual_type); + return -EINVAL; + } + if (!is_rcu_reg(reg)) { + const char *actual_type, *arg_name, *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, arg_btf, *arg_btf_id); + verbose(env, "%s must be a rcu pointer\n", + reg_arg_name(env, argno)); + arg_name = reg_arg_name(env, argno); + actual_type = bpf_diag_reg_type_plain(env, reg->type); + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Use this function with a pointer that is valid in an RCU read lock region.", + "the function requires an RCU-protected pointer to %s, but %s is %s", + expected_type, arg_name, actual_type); + return -EINVAL; + } + } + + err = process_arg_ptr_to_btf_id(env, reg, argno, arg_type, arg_btf, + *arg_btf_id, meta, insn_idx); + if (err < 0) + return err; + break; + } case ARG_PTR_TO_PERCPU_BTF_ID: if (!reg->btf_id) { - verbose(env, "Helper has invalid btf_id in R%d\n", regno); + verbose(env, "Helper has invalid btf_id in %s\n", reg_arg_name(env, argno)); return -EACCES; } meta->ret_btf = reg->btf; @@ -8877,11 +9991,11 @@ skip_type_check: verbose(env, "can't spin_{lock,unlock} in rbtree cb\n"); return -EACCES; } - if (meta->func_id == BPF_FUNC_spin_lock) { + if (is_helper_call(meta, BPF_FUNC_spin_lock)) { err = process_spin_lock(env, reg, argno, PROCESS_SPIN_LOCK); if (err) return err; - } else if (meta->func_id == BPF_FUNC_spin_unlock) { + } else if (is_helper_call(meta, BPF_FUNC_spin_unlock)) { err = process_spin_lock(env, reg, argno, 0); if (err) return err; @@ -8895,45 +10009,283 @@ skip_type_check: if (err) return err; break; + case ARG_PTR_TO_CTX: + if (is_bpf_cast_to_kern_ctx_kfunc(meta)) { + err = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); + if (err < 0) + return -EINVAL; + meta->ret_btf_id = err; + } + break; + case ARG_PTR_TO_ARENA: + break; + case ARG_PTR_TO_ALLOC_BTF_ID: + if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { + if (!is_bpf_obj_drop_kfunc(meta->func_id)) { + verbose(env, "%s expected for bpf_obj_drop()\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + } else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) { + if (!is_bpf_percpu_obj_drop_kfunc(meta->func_id)) { + verbose(env, "%s expected for bpf_percpu_obj_drop()\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + } + if (!reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass the owned object pointer before it is released or transferred.", + "the allocated object pointer in %s must still carry verifier-tracked ownership, but this pointer no longer owns a live resource", + reg_arg_name(env, argno)); + return -EINVAL; + } + if (meta->btf == btf_vmlinux) { + meta->arg_btf = reg->btf; + meta->arg_btf_id = reg->btf_id; + } + break; case ARG_PTR_TO_FUNC: meta->subprogno = reg->subprogno; break; case ARG_PTR_TO_MEM: + { + enum bpf_access_type access_type; + bool known_memory; + /* The access to this pointer is only checked when we hit the * next is_mem_size argument below. */ - if (arg_type & MEM_FIXED_SIZE) { - err = check_mem_reg(env, reg, argno_from_reg(regno), fn->arg_size[arg], - arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, meta, NULL); - if (err) - return err; - if (arg_type & MEM_ALIGNED) - err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true); + if (!(arg_type & MEM_FIXED_SIZE)) + break; + + access_type = func_arg_access_type(arg_type); + + err = check_mem_reg(env, reg, argno, arg_size, access_type, meta, &known_memory); + if (err < 0) { + if (known_memory) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass memory with at least the required number of accessible bytes and suitable read or write access.", + "the function expects %u bytes of memory, but the verifier cannot prove that %s provides a range of that size with the required read or write access", + arg_size, + bpf_diag_reg_type_plain(env, reg->type)); + else + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass stack, map, context, or other verifier-known memory of the expected type and size, not an integer cast to a pointer.", + "the function expects %u bytes of memory, but it is %s and not verifier-known memory", + arg_size, + bpf_diag_reg_type_plain(env, reg->type)); + return err; + } + /* + * PTR_TO_PACKET gets passed as PTR_TO_MEM, preventing us from adjusting + * bounds tracking information. + */ + if (is_subprog(meta) && meta->subprog_may_change_pkt && + (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg))) { + verbose(env, + "cannot pass packet pointer %s to %s(): function may change packet data\n", + reg_arg_name(env, argno), meta->func_name); + return -EINVAL; } + if (arg_type & MEM_ALIGNED) + err = check_ptr_alignment(env, reg, 0, arg_size, true); break; + } + case ARG_CONST_MEM_SIZE: + err = process_const_arg(env, reg, argno, meta); + if (err < 0) { + if (err == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass a compile-time constant or a value the verifier can prove is constant at this call.", + "the function requires this memory size to be a verifier-known constant, but %s is variable on this path", + reg_arg_name(env, argno)); + return err; + } + fallthrough; case ARG_MEM_SIZE: - err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, - argno_from_reg(regno - 1), argno, - fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, - false, meta, NULL); - break; case ARG_MEM_SIZE_OR_ZERO: - err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, - argno_from_reg(regno - 1), argno, - fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, - true, meta, NULL); + { + struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, prev_slot); + argno_t buff_argno = argno_from_arg(prev_slot + 1); + enum bpf_mem_size_failure failure; + const char *buff_arg, *size_arg; + bool zero_size_allowed; + u32 access_type; + + if (meta->btf && bpf_register_is_null(buff_reg)) + break; + + access_type = func_arg_access_type(fn->arg_type[arg - 1]); + + zero_size_allowed = meta->btf || base_type(arg_type) == ARG_MEM_SIZE_OR_ZERO; + + err = check_mem_size_reg(env, buff_reg, reg, buff_argno, argno, + access_type, zero_size_allowed, meta, &failure); + if (!err) + break; + + buff_arg = bpf_diag_arg_name(env, buff_argno); + size_arg = bpf_diag_arg_name(env, argno); + verbose(env, "%s and ", reg_arg_name(env, buff_argno)); + verbose(env, "%s memory, len pair leads to invalid memory access\n", + reg_arg_name(env, argno)); + if (failure == BPF_MEM_SIZE_FAIL_MEMORY) { + bpf_diag_call_arg_fmt(env, insn_idx, buff_argno, meta->func_name, + "Pass a stack, map, context, or other verifier-known memory pointer, and keep the paired length within that object.", + "it is the memory pointer in a memory/length pair with %s, but %s does not provide a verifier-accessible range of the requested length", + size_arg, buff_arg); + } else if (failure == BPF_MEM_SIZE_FAIL_SIZE) { + if (reg_smin(reg) < 0) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", + "the memory size in %s may be negative because its signed minimum is %lld", + size_arg, reg_smin(reg)); + else if (!zero_size_allowed && reg_umin(reg) == 0) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Ensure the memory size is non-zero before this call.", + "the memory size in %s may be zero, but the function requires a non-zero size", + size_arg); + else + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", + "the memory size in %s may reach %llu bytes, but variable memory accesses must stay below %u bytes", + size_arg, reg_umax(reg), BPF_MAX_VAR_SIZ); + } break; - case ARG_PTR_TO_DYNPTR: - err = process_dynptr_func(env, reg, argno, insn_idx, func_id_name(meta->func_id), - arg_type, &meta->ref_obj, &meta->dynptr); + } + case ARG_PTR_TO_DYNPTR: { + if (is_bpf_dynptr_clone_kfunc(meta) && + (arg_type & MEM_UNINIT)) { + enum bpf_dynptr_type parent_type = meta->dynptr.type; + + if (parent_type == BPF_DYNPTR_TYPE_INVALID) { + verifier_bug(env, "no dynptr type for parent of clone"); + return -EFAULT; + } + + arg_type |= (unsigned int)get_dynptr_type_flag(parent_type); + } + + err = process_dynptr_func(env, reg, argno, insn_idx, arg_type, meta); if (err) return err; + if (!arg && is_kfunc_dynptr_write(meta)) + meta->dynptr_write_type = meta->dynptr.type; + break; + } + case ARG_PTR_TO_ITER: + if (is_bpf_iter_css_task_new_kfunc(meta) && + !check_css_task_iter_allowlist(env)) { + verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n"); + return -EINVAL; + } + err = process_iter_arg(env, reg, arg, argno, insn_idx, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_LIST_HEAD: + if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && + !reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + return -EINVAL; + } + err = process_kf_arg_ptr_to_list_head(env, reg, argno, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_RB_ROOT: + if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && + !reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + return -EINVAL; + } + err = process_kf_arg_ptr_to_rbtree_root(env, reg, argno, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_LIST_NODE: + if (!(is_kfunc_arg_nonown_allowed(meta->btf, btf_arg) && + type_is_non_owning_ref(reg->type) && !reg_is_referenced(env, reg))) { + if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { + verbose(env, "%s expected pointer to allocated object\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + if (!reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + return -EINVAL; + } + } + err = process_kf_arg_ptr_to_list_node(env, reg, argno, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_RB_NODE: + if (is_bpf_rbtree_add_kfunc(meta->func_id)) { + if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { + verbose(env, "%s expected pointer to allocated object\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + if (!reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + return -EINVAL; + } + } else { + if (!type_is_non_owning_ref(reg->type) && + !reg_is_referenced(env, reg)) { + verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n", + meta->func_name); + return -EINVAL; + } + if (in_rbtree_lock_required_cb(env)) { + verbose(env, "%s not allowed in rbtree cb\n", meta->func_name); + return -EINVAL; + } + } + err = process_kf_arg_ptr_to_rbtree_node(env, reg, argno, meta); + if (err < 0) + return err; break; case ARG_CONST_ALLOC_SIZE_OR_ZERO: + if (meta->btf && is_kfunc_arg_scalar_with_name(meta->btf, btf_arg, + "rdonly_buf_size")) + meta->r0_rdonly = true; err = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); - if (err) + if (err < 0) { + if (meta->btf && err == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass a verifier-known constant size for this function's buffer argument.", + "the function uses this argument as a return-buffer size, but %s is invalid or variable on this path", + reg_arg_name(env, argno)); return err; + } break; + case ARG_PTR_TO_REFCOUNTED_KPTR: + { + struct btf_record *rec; + + if (!type_is_non_owning_ref(reg->type) && reg_is_referenced(env, reg)) + meta->arg_owning_ref = true; + + rec = reg_btf_record(reg); + if (!rec) { + verifier_bug(env, "Couldn't find btf_record"); + return -EFAULT; + } + + if (rec->refcount_off < 0) { + verbose(env, "%s doesn't point to a type with bpf_refcount field\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + + meta->arg_btf = reg->btf; + meta->arg_btf_id = reg->btf_id; + break; + } case ARG_PTR_TO_CONST_STR: { err = check_arg_const_str(env, reg, argno); @@ -8941,6 +10293,50 @@ skip_type_check: return err; break; } + case ARG_PTR_TO_WORKQUEUE: + err = check_map_field_pointer(env, reg, argno, BPF_WORKQUEUE, &meta->map); + if (err < 0) + return err; + break; + case ARG_PTR_TO_TASK_WORK: + err = check_map_field_pointer(env, reg, argno, BPF_TASK_WORK, &meta->map); + if (err < 0) + return err; + break; + case ARG_PTR_TO_RCU_HEAD: + err = check_map_field_pointer(env, reg, argno, BPF_RCU_HEAD, &meta->map); + if (err < 0) + return err; + break; + case ARG_PTR_TO_IRQ_FLAG: + err = process_irq_flag(env, reg, argno, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_CTX_OUT: + if (reg->mem_size != arg_size) { + verbose(env, "%s expected %u bytes of ctx-provided memory, got %u\n", + reg_arg_name(env, argno), arg_size, reg->mem_size); + return -EINVAL; + } + break; + case ARG_PTR_TO_RES_SPIN_LOCK: + { + int flags; + + if (in_rbtree_lock_required_cb(env)) { + verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n"); + return -EACCES; + } + + flags = get_bpf_res_spin_lock_kfunc_flags(meta); + if (!flags) + return -EFAULT; + err = process_spin_lock(env, reg, argno, flags); + if (err < 0) + return err; + break; + } case ARG_KPTR_XCHG_DEST: err = process_kptr_func(env, regno, meta); if (err) @@ -8951,6 +10347,139 @@ skip_type_check: return err; } +/* + * The slots a parameter takes, from its BTF type. This has to agree with + * btf_func_model_arg_slots(), which answers the same from the size the + * func model recorded, or the verifier would check an argument at a slot + * the JIT does not place it at. + */ +static u32 btf_arg_slots(const struct btf_type *t) +{ + if (btf_type_is_int(t) || btf_type_is_struct(t)) + return (t->size + BPF_REG_SIZE - 1) / BPF_REG_SIZE; + return 1; +} + +static u32 btf_proto_slots(const struct btf *btf, const struct btf_type *func_proto) +{ + const struct btf_param *args = btf_params(func_proto); + u32 i, nargs = btf_type_vlen(func_proto), slots_used = 0; + + for (i = 0; i < nargs; i++) + slots_used += btf_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL)); + + return slots_used; +} + +/* The argument slot @slot holds an eightbyte of a by-value argument. */ +static int check_arg_extra_slot(struct bpf_verifier_env *env, struct bpf_func_state *caller, + u32 slot, struct bpf_call_arg_meta *meta) +{ + struct bpf_reg_state *reg = get_func_arg_reg(caller, cur_regs(env), slot); + argno_t argno = argno_from_arg(slot + 1); + int regno = reg_from_argno(argno); + int err; + + if (regno >= 0) { + err = check_reg_arg(env, regno, SRC_OP); + if (err) + return err; + } + + return check_reg_type(env, reg, argno, ARG_SCALAR, meta); +} + +static int check_func_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + int insn_idx) +{ + u32 slot = 0, prev_slot = 0, proto_slots, nslots; + struct bpf_func_state *caller = cur_func(env); + const struct btf_param *args = NULL; + u32 arg, nargs = MAX_BPF_FUNC_REG_ARGS; + int err; + + if (meta->btf) { + args = btf_params(meta->func_proto); + nargs = btf_type_vlen(meta->func_proto); + } + + /* + * A by-value argument of more than one eightbyte takes a slot per + * eightbyte, so the slots a call occupies are no longer its parameter + * count. Only a proto whose parameters take a slot each can name the + * argument a stack slot belongs to. + */ + proto_slots = meta->btf ? btf_proto_slots(meta->btf, meta->func_proto) : nargs; + + if (proto_slots > MAX_BPF_FUNC_REG_ARGS) { + err = check_outgoing_stack_args(env, caller, proto_slots, meta->func_name, + meta->btf, proto_slots == nargs ? args : NULL); + if (err) + return err; + } + + for (arg = 0; arg < nargs; arg++, prev_slot = slot, slot += nslots) { + const struct btf_type *t; + u32 k; + + nslots = 1; + if (args) { + t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL); + nslots = btf_arg_slots(t); + } + + if (meta->fn->arg_type[arg] == ARG_UNUSED) + break; + err = check_func_arg(env, arg, slot, prev_slot, meta, insn_idx); + if (err) + return err; + + /* + * check_func_arg() took the first slot. A parameter of more + * than one eightbyte is always a scalar, so every slot it + * takes beyond the first holds one too. + */ + for (k = 1; k < nslots; k++) { + err = check_arg_extra_slot(env, caller, slot + k, meta); + if (err) + return err; + } + } + + return 0; +} + +static int mark_raw_stack(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + int insn_idx) +{ + struct bpf_func_state *caller = cur_func(env); + u32 slot; + int i, err; + + /* + * Validate every argument before initializing outputs: an input argument + * may alias an output buffer. Use the normal stack-write checks to discard + * stale spills and preserve the rules for special stack objects. + */ + for (slot = 0; slot < MAX_BPF_FUNC_ARGS; slot++) { + struct bpf_reg_state *reg; + argno_t argno = argno_from_arg(slot + 1); + + if (!meta->arg_raw_mem.size[slot]) + continue; + reg = get_func_arg_reg(caller, cur_regs(env), slot); + + for (i = 0; i < meta->arg_raw_mem.size[slot]; i++) { + err = check_mem_access(env, insn_idx, reg, argno, i, BPF_B, + BPF_WRITE, -1, false, false); + if (err) + return err; + } + } + + return 0; +} + static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) { enum bpf_attach_type eatype = env->prog->expected_attach_type; @@ -8990,12 +10519,71 @@ bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env) bpf_jit_supports_subprog_tailcalls(); } +/* + * Is rcu_read_lock_held() true at run time? Unlike for in_rcu_cs(), disabled + * preemption or interrupts will not do. + */ +static bool has_rcu_read_lock(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *state = env->cur_state; + + if (state->active_rcu_locks || in_rcu_callback(env)) + return true; + if (in_sleepable(env)) + return false; + return !env->no_rcu_read_lock || state->frame[0]->in_async_callback_fn; +} + +/* May the map be used outside of an RCU read side critical section? */ +static bool map_needs_no_rcu(const struct bpf_map *map) +{ + switch (map->map_type) { + case BPF_MAP_TYPE_HASH: + case BPF_MAP_TYPE_RHASH: + case BPF_MAP_TYPE_LRU_HASH: + case BPF_MAP_TYPE_ARRAY: + case BPF_MAP_TYPE_PERCPU_HASH: + case BPF_MAP_TYPE_PERCPU_ARRAY: + case BPF_MAP_TYPE_LRU_PERCPU_HASH: + case BPF_MAP_TYPE_LPM_TRIE: + case BPF_MAP_TYPE_ARRAY_OF_MAPS: + case BPF_MAP_TYPE_HASH_OF_MAPS: + case BPF_MAP_TYPE_RINGBUF: + case BPF_MAP_TYPE_USER_RINGBUF: + case BPF_MAP_TYPE_INODE_STORAGE: + case BPF_MAP_TYPE_SK_STORAGE: + case BPF_MAP_TYPE_TASK_STORAGE: + case BPF_MAP_TYPE_CGRP_STORAGE: + case BPF_MAP_TYPE_QUEUE: + case BPF_MAP_TYPE_STACK: + case BPF_MAP_TYPE_ARENA: + case BPF_MAP_TYPE_INSN_ARRAY: + case BPF_MAP_TYPE_PROG_ARRAY: + case BPF_MAP_TYPE_BLOOM_FILTER: + return true; + default: + return false; + } +} + static int check_map_func_compatibility(struct bpf_verifier_env *env, struct bpf_map *map, int func_id) { if (!map) return 0; + /* + * check_map_prog_compatibility() asks the program. That misses the + * sleepable callback of another program, and an inner map. They may + * use any map under the RCU read lock. Disabled preemption or + * interrupts will not do: the maps ask for rcu_read_lock_held(). + */ + if (!has_rcu_read_lock(env) && !map_needs_no_rcu(map)) { + verbose(env, "map '%s' needs bpf_rcu_read_lock() in sleepable context\n", + map->name); + return -EINVAL; + } + /* We need a two way check, first is from map perspective ... */ switch (map->map_type) { case BPF_MAP_TYPE_PROG_ARRAY: @@ -9243,23 +10831,6 @@ error: return -EINVAL; } -static bool check_raw_mode_ok(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { - if (fn->arg_type[i] == ARG_DONTCARE) - break; - if (!arg_type_is_raw_mem(fn->arg_type[i])) - continue; - if (meta->arg_raw_mem.regno) - return false; - meta->arg_raw_mem.regno = i + 1; - } - - return true; -} - static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg) { bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE; @@ -9298,7 +10869,7 @@ static bool check_btf_id_ok(const struct bpf_func_proto *fn) int i; for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { - if (fn->arg_type[i] == ARG_DONTCARE) + if (fn->arg_type[i] == ARG_UNUSED) break; if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID) return !!fn->arg_btf_id[i]; @@ -9321,7 +10892,7 @@ static bool check_mem_arg_rw_flag_ok(const struct bpf_func_proto *fn) for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i]; - if (arg_type == ARG_DONTCARE) + if (arg_type == ARG_UNUSED) break; if (base_type(arg_type) != ARG_PTR_TO_MEM) continue; @@ -9339,7 +10910,7 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_ for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i]; - if (arg_type == ARG_DONTCARE) + if (arg_type == ARG_UNUSED) break; if (arg_type_is_release(arg_type)) { if (meta->release_regno) @@ -9351,9 +10922,41 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_ return true; } -static int check_func_proto(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta) +static bool check_arg_prog_aux(struct bpf_verifier_env *env, + const struct bpf_func_proto *proto) +{ + bool seen = false; + argno_t argno; + u32 i; + + for (i = 0; i < ARRAY_SIZE(proto->arg_type); i++) { + if (proto->arg_type[i] == ARG_UNUSED) + break; + if (proto->arg_type[i] != ARG_PTR_TO_PROG_AUX) + continue; + + if (seen) { + verifier_bug(env, "Only 1 prog->aux argument supported"); + return false; + } + + argno = argno_from_arg(i + 1); + if (reg_from_argno(argno) < 0) { + verbose(env, "%s prog->aux cannot be a stack argument\n", + reg_arg_name(env, argno)); + return false; + } + + seen = true; + } + + return true; +} + +static int check_func_proto(struct bpf_verifier_env *env, const struct bpf_func_proto *fn, + struct bpf_call_arg_meta *meta) { - return check_raw_mode_ok(fn, meta) && + return check_arg_prog_aux(env, fn) && check_arg_pair_ok(fn) && check_mem_arg_rw_flag_ok(fn) && check_proto_release_reg(fn, meta) && @@ -9379,6 +10982,20 @@ static void clear_all_pkt_pointers(struct bpf_verifier_env *env) })); } +static bool holds_pkt_pointers(struct bpf_verifier_env *env) +{ + struct bpf_func_state *state; + struct bpf_reg_state *reg; + bool ret = false; + + bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ + if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) + ret = true; + })); + + return ret; +} + enum { AT_PKT_END = -1, BEYOND_PKT_END = -2, @@ -9573,7 +11190,8 @@ static int ref_convert_alloc_rcu_protected(struct bpf_verifier_env *env, u32 id) continue; if ((reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) { bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE); - reg->id = 0; + if (!type_may_be_null(reg->type)) + reg->id = 0; reg->type &= ~MEM_ALLOC; reg->type |= MEM_RCU; bpf_diag_mod_end(env); @@ -9665,23 +11283,57 @@ err_out: return err; } -static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, - const struct btf *btf, - struct bpf_reg_state *regs) +static void gen_subprog_arg_proto(const struct bpf_subprog_info *sub, const struct btf *btf, + const struct btf_type *func_proto, struct bpf_func_proto *proto) +{ + const struct btf_param *args = btf_params(func_proto); + u32 arg, slot = 0; + + memset(proto, 0, sizeof(*proto)); + for (arg = 0; arg < btf_type_vlen(func_proto); arg++) { + enum bpf_arg_type arg_type = sub->args[slot].arg_type; + const struct btf_type *t; + + if (arg_type & PTR_UNTRUSTED) { + /* + * An __arg_untrusted argument accepts any caller value. The + * callee treats it as read-only and uses probe-read instructions + * to protect against invalid memory access. + */ + arg_type = ARG_IGNORE; + } else if (base_type(arg_type) == ARG_PTR_TO_ARENA) { + arg_type |= PTR_MAYBE_NULL; + } else if (base_type(arg_type) == ARG_PTR_TO_MEM) { + proto->arg_size[arg] = sub->args[slot].mem_size; + arg_type |= MEM_FIXED_SIZE | MEM_WRITE; + } else if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) { + proto->arg_btf_id[arg] = &sub->args[slot].btf_id; + } + proto->arg_type[arg] = arg_type; + t = btf_type_skip_modifiers(btf, args[arg].type, NULL); + slot += btf_arg_slots(t); + } +} + +static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct btf *btf) { struct bpf_subprog_info *sub = subprog_info(env, subprog); struct bpf_func_state *caller = cur_func(env); - struct bpf_verifier_log *log = &env->log; - struct ref_obj_desc ref_obj = {}; - const struct btf_param *args; const struct btf_type *func, *func_proto; - u32 i; + struct bpf_call_arg_meta meta; + struct bpf_func_proto *fn; int ret, err; + u32 i; + + memset(&meta, 0, sizeof(meta)); + meta.btf = btf; + meta.subprog_may_change_pkt = sub->changes_pkt_data; + meta.func_name = bpf_subprog_name(env, subprog); ret = btf_prepare_func_args(env, subprog); if (ret) { if (bpf_in_stack_arg_cnt(sub) > 0) { - err = check_outgoing_stack_args(env, caller, sub->arg_cnt, + err = check_outgoing_stack_args(env, caller, sub->arg_slot_cnt, bpf_subprog_name(env, subprog), NULL, NULL); if (err) @@ -9692,115 +11344,33 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, func = btf_type_by_id(btf, env->prog->aux->func_info[subprog].type_id); func_proto = btf_type_by_id(btf, func->type); - args = btf_params(func_proto); - ret = check_outgoing_stack_args(env, caller, sub->arg_cnt, - bpf_subprog_name(env, subprog), btf, args); - if (ret) - return ret; - /* check that BTF function arguments match actual types that the - * verifier sees. - */ - for (i = 0; i < sub->arg_cnt; i++) { - argno_t argno = argno_from_arg(i + 1); - struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); - struct bpf_subprog_arg_info *arg = &sub->args[i]; + fn = &env->bpf_subprog_scratch; + gen_subprog_arg_proto(sub, btf, func_proto, fn); + meta.fn = fn; + meta.func_proto = func_proto; - if (arg->arg_type == ARG_ANYTHING) { - if (reg->type != SCALAR_VALUE) { - bpf_log(log, "%s is not a scalar\n", reg_arg_name(env, argno)); - return -EINVAL; - } - } else if (arg->arg_type & PTR_UNTRUSTED) { - /* - * Anything is allowed for untrusted arguments, as these are - * read-only and probe read instructions would protect against - * invalid memory access. - */ - } else if (arg->arg_type == ARG_PTR_TO_CTX) { - ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_CTX); - if (ret < 0) - return ret; - /* If function expects ctx type in BTF check that caller - * is passing PTR_TO_CTX. - */ - if (reg->type != PTR_TO_CTX) { - bpf_log(log, "%s expects pointer to ctx\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) { - ret = check_func_arg_reg_off(env, reg, argno, ARG_DONTCARE); - if (ret < 0) - return ret; - if (check_mem_reg(env, reg, argno, arg->mem_size, BPF_READ | BPF_WRITE, NULL, - NULL)) - return -EINVAL; - if (!(arg->arg_type & PTR_MAYBE_NULL) && - (type_may_be_null(reg->type) || bpf_register_is_null(reg))) { - bpf_log(log, "%s is expected to be non-NULL\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) { - /* - * Can pass any value and the kernel won't crash, but - * only PTR_TO_ARENA or SCALAR make sense. Everything - * else is a bug in the bpf program. Point it out to - * the user at the verification time instead of - * run-time debug nightmare. - */ - if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) { - bpf_log(log, "%s is not a pointer to arena or scalar.\n", - reg_arg_name(env, argno)); + /* + * Check if all register arguments declared in BTF are initialized. + * If they aren't the BTF is unreliable. + */ + if (!bpf_subprog_is_global(env, subprog)) { + for (i = 0; i < min_t(u32, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); i++) + if (caller->regs[BPF_REG_1 + i].type == NOT_INIT) return -EINVAL; - } - } else if (arg->arg_type == ARG_PTR_TO_DYNPTR) { - ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_DYNPTR); - if (ret) - return ret; - - ret = process_dynptr_func(env, reg, argno, env->insn_idx, - bpf_subprog_name(env, subprog), arg->arg_type, - &ref_obj, NULL); - if (ret) - return ret; - } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { - struct bpf_call_arg_meta meta; - int err; - - if (bpf_register_is_null(reg) && type_may_be_null(arg->arg_type)) { - err = mark_arg_precision(env, argno); - if (err) - return err; - continue; - } - - memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */ - err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta, - bpf_subprog_name(env, subprog)); - err = err ?: check_func_arg_reg_off(env, reg, argno, arg->arg_type); - if (err) - return err; - } else { - verifier_bug(env, "unrecognized %s type %d", - reg_arg_name(env, argno), arg->arg_type); - return -EFAULT; - } } - - return 0; + return check_func_args(env, &meta, env->insn_idx); } -/* Compare BTF of a function call with given bpf_reg_state. +/* + * Check that call-site argument states match a subprog's BTF signature. + * * Returns: * EFAULT - there is a verifier bug. Abort verification. - * EINVAL - there is a type mismatch or BTF is not available. + * Other errors - there is a type mismatch or BTF is not available. * 0 - BTF matches with what bpf_reg_state expects. - * Only PTR_TO_CTX and SCALAR_VALUE states are recognized. */ -static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, - struct bpf_reg_state *regs) +static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog) { struct bpf_prog *prog = env->prog; struct btf *btf = prog->aux->btf; @@ -9817,7 +11387,7 @@ static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, if (prog->aux->func_info_aux[subprog].unreliable) return -EINVAL; - err = btf_check_func_arg_match(env, subprog, btf, regs); + err = btf_check_func_arg_match(env, subprog, btf); /* Compiler optimizations can remove arguments from static functions * or mismatched type can be passed into a global function. * In such cases mark the function as unreliable from BTF point of view. @@ -9836,7 +11406,7 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins int err; caller = state->frame[state->curframe]; - err = btf_check_subprog_call(env, subprog, caller->regs); + err = btf_check_subprog_call(env, subprog); if (err == -EFAULT) return err; @@ -9867,8 +11437,10 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins is_async_cb_sleepable(env, insn)); if (IS_ERR(async_cb)) return PTR_ERR(async_cb); + if (is_async_cb_sleepable(env, insn)) + mark_sleepable_async_cb(env); callee = async_cb->frame[0]; - callee->async_entry_cnt = caller->async_entry_cnt + 1; + callee->async_entry_cnt = state->frame[0]->async_entry_cnt + 1; /* Convert bpf_timer_set_callback() args into timer callback args */ err = set_callee_state_cb(env, caller, callee, insn_idx); @@ -9878,6 +11450,15 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins return 0; } + /* + * The address of the callback might be taken in another function or + * read from a map, see check_func_ptr_read(). Like with callx it's + * known here what is called. + */ + err = record_callx_edge(env, caller->subprogno, subprog); + if (err) + return err; + /* for callback functions enqueue entry to callback and * proceed with next instruction within current frame. */ @@ -9899,14 +11480,63 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins static int process_bpf_exit_full(struct bpf_verifier_env *env, bool *do_print_state, bool exception_exit); -static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, - int *insn_idx) +/* + * Call of a static subprog. The callee is verified in the context of + * the caller, hence set up a new frame and continue from the first + * instruction of the callee. + */ +static int check_static_func_call(struct bpf_verifier_env *env, int subprog, + int *insn_idx) { struct bpf_verifier_state *state = env->cur_state; struct bpf_subprog_info *caller_info; u16 callee_incoming, stack_arg_cnt; struct bpf_func_state *caller; + int err; + + caller = state->frame[state->curframe]; + + /* + * Track caller's total stack arg count (incoming + max outgoing). + * This is needed so the JIT knows how much stack arg space to allocate. + */ + caller_info = &env->subprog_info[caller->subprogno]; + callee_incoming = bpf_in_stack_arg_cnt(&env->subprog_info[subprog]); + stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + callee_incoming; + if (stack_arg_cnt > caller_info->stack_arg_cnt) + caller_info->stack_arg_cnt = stack_arg_cnt; + + /* + * For regular function entry setup new frame and continue + * from that frame. + */ + err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state); + if (err) + return err; + + bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); + clear_caller_saved_regs(env, caller->regs); + + /* and go analyze first insn of the callee */ + *insn_idx = env->subprog_info[subprog].start - 1; + + if (env->log.level & BPF_LOG_LEVEL) { + verbose(env, "caller:\n"); + print_verifier_state(env, state, caller->frameno, true); + verbose(env, "callee:\n"); + print_verifier_state(env, state, state->curframe, true); + } + + return 0; +} + +static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, + int *insn_idx) +{ + struct bpf_verifier_state *state = env->cur_state; + struct bpf_func_state *caller; int err, subprog, target_insn; + u32 i, nregs; target_insn = *insn_idx + insn->imm + 1; subprog = bpf_find_subprog(env, target_insn); @@ -9915,10 +11545,11 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return -EFAULT; caller = state->frame[state->curframe]; - err = btf_check_subprog_call(env, subprog, caller->regs); + err = btf_check_subprog_call(env, subprog); if (err == -EFAULT) return err; if (bpf_subprog_is_global(env, subprog)) { + struct bpf_func_info_aux *sub_aux = subprog_aux(env, subprog); const char *sub_name = bpf_subprog_name(env, subprog); const char *operation; bool returns_void; @@ -9933,11 +11564,16 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } if (env->subprog_info[subprog].might_sleep && !in_sleepable_context(env)) { + const char *suggestion; + verbose(env, "sleepable global function %s() called in %s\n", sub_name, non_sleepable_context_description(env)); + if (in_sleepable(env)) + suggestion = "Move the call outside the critical section, or use a non-sleepable function."; + else + suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable function."; operation = bpf_diag_fmt(env, "sleepable global function %s()", sub_name); - bpf_diag_ctx_forbidden(env, *insn_idx, operation, - "Move the call outside the critical section, or use a non-sleepable function."); + bpf_diag_ctx_forbidden(env, *insn_idx, operation, suggestion); return -EINVAL; } @@ -9950,22 +11586,56 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (env->log.level & BPF_LOG_LEVEL) verbose(env, "Func#%d ('%s') is global and assumed valid.\n", subprog, sub_name); + if (in_sleepable_context(env)) + sub_aux->called[BPF_FUNC_CTX_SLEEPABLE] = true; + else if (has_rcu_read_lock(env)) + sub_aux->called[BPF_FUNC_CTX_NON_SLEEPABLE] = true; + else + sub_aux->called[BPF_FUNC_CTX_NO_RCU_LOCK] = true; returns_void = subprog_returns_void(env, subprog); if (env->subprog_info[subprog].changes_pkt_data) clear_all_pkt_pointers(env); - /* mark global subprog for verifying after main prog */ - subprog_aux(env, subprog)->called = true; if (returns_void) bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); else bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); clear_caller_saved_regs(env, caller->regs); invalidate_outgoing_stack_args(env, cur_func(env)); + /* + * What the callers of a global function hold is not known + * while it is verified. + */ + if (!env->subprog_info[subprog].changes_pkt_data && + (state->frame[0]->subprogno || holds_pkt_pointers(env))) + sub_aux->called_with_pkt_ptrs = true; - /* All non-void global functions return a 64-bit SCALAR_VALUE. */ + /* + * A non-void global function returns a 64-bit SCALAR_VALUE in + * R0, or a >8 byte SCALAR_VALUE in the R0:R2 register pair. + */ if (!returns_void) { - mark_reg_unknown(env, caller->regs, BPF_REG_0); + nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; + mark_reg_unknown(env, caller->regs, ret_regs[0]); bpf_diag_mod_end(env); + for (i = 1; i < nregs; i++) { + bpf_diag_mod_begin(env, &caller->regs[ret_regs[i]], NULL, + BPF_DIAG_MOD_WRITE); + mark_reg_unknown(env, caller->regs, ret_regs[i]); + bpf_diag_mod_end(env); + } + } + + if (env->subprog_info[subprog].can_unwind) { + struct bpf_verifier_state *branch; + + if (!bpf_is_unwind_goto(insn + 1)) { + verbose(env, "no unwind_goto after the call of %s()\n", sub_name); + return -EINVAL; + } + branch = push_stack(env, *insn_idx + 1, *insn_idx, false); + if (IS_ERR(branch)) + return PTR_ERR(branch); + branch->unwound = true; } if (env->subprog_info[subprog].might_throw) { @@ -9983,37 +11653,69 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; } - /* - * Track caller's total stack arg count (incoming + max outgoing). - * This is needed so the JIT knows how much stack arg space to allocate. - */ - caller_info = &env->subprog_info[caller->subprogno]; - callee_incoming = bpf_in_stack_arg_cnt(&env->subprog_info[subprog]); - stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + callee_incoming; - if (stack_arg_cnt > caller_info->stack_arg_cnt) - caller_info->stack_arg_cnt = stack_arg_cnt; + return check_static_func_call(env, subprog, insn_idx); +} - /* for regular function entry setup new frame and continue - * from that frame. +/* + * callx dst_reg: call a bpf subprog whose address is in dst_reg. + * + * The address of a subprog is either loaded into a register by ld_imm64 with + * src_reg == BPF_PSEUDO_FUNC, or it is read from a frozen read-only map, see + * resolve_func_ptrs(). Both are possible for static subprogs only. Hence all + * possible callees of callx are discovered by add_subprogs() and are reachable + * in the control flow graph before the main verification pass begins. + * PTR_TO_FUNC register identifies the callee, so from here on callx is verified + * as a direct call of that static subprog. + */ +static int check_func_callx(struct bpf_verifier_env *env, struct bpf_insn *insn, + int *insn_idx) +{ + struct bpf_func_state *caller = cur_func(env); + struct bpf_reg_state *reg; + const char *reason; + int err, subprog; + + err = check_reg_arg(env, insn->dst_reg, SRC_OP); + if (err) + return err; + + reg = reg_state(env, insn->dst_reg); + if (reg->type != PTR_TO_FUNC) { + verbose(env, "R%d has type %s, expected func\n", insn->dst_reg, + reg_type_str(env, reg->type)); + reason = bpf_diag_fmt( + env, "R%d holds %s, but callx can only call through the address of a static BPF function.", + insn->dst_reg, bpf_diag_reg_type_plain(env, reg->type)); + bpf_diag_register_type( + env, *insn_idx, insn->dst_reg, "indirect call through a non-function pointer", reason, + "Load the address of a static BPF function into the register before callx."); + return -EACCES; + } + + /* + * Arithmetic on PTR_TO_FUNC is allowed, but only unmodified address + * of a subprog can be called. */ - err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state); + err = check_ptr_off_reg(env, reg, insn->dst_reg); if (err) return err; - bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); - clear_caller_saved_regs(env, caller->regs); + /* the callx is valid, but only a JIT can execute it */ + err = require_callx_jit(env); + if (err) + return err; - /* and go analyze first insn of the callee */ - *insn_idx = env->subprog_info[subprog].start - 1; + /* PTR_TO_FUNC is a pointer to a static subprog */ + subprog = reg->subprogno; + err = btf_check_subprog_call(env, subprog); + if (err == -EFAULT) + return err; - if (env->log.level & BPF_LOG_LEVEL) { - verbose(env, "caller:\n"); - print_verifier_state(env, state, caller->frameno, true); - verbose(env, "callee:\n"); - print_verifier_state(env, state, state->curframe, true); - } + err = record_callx_edge(env, caller->subprogno, subprog); + if (err) + return err; - return 0; + return check_static_func_call(env, subprog, insn_idx); } int map_set_for_each_callback_args(struct bpf_verifier_env *env, @@ -10036,6 +11738,7 @@ int map_set_for_each_callback_args(struct bpf_verifier_env *env, __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr; callee->regs[BPF_REG_3].map_uid = caller->regs[BPF_REG_1].map_uid; + callee->regs[BPF_REG_3].id = ++env->id_gen; /* pointer to stack or null */ callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3]; @@ -10080,6 +11783,19 @@ static int set_map_elem_callback_state(struct bpf_verifier_env *env, if (err) return err; + switch (map->map_type) { + case BPF_MAP_TYPE_HASH: + case BPF_MAP_TYPE_PERCPU_HASH: + case BPF_MAP_TYPE_LRU_HASH: + case BPF_MAP_TYPE_LRU_PERCPU_HASH: + case BPF_MAP_TYPE_RHASH: + /* Hash tables run the callback under rcu_read_lock(). */ + callee->in_rcu_callback_fn = true; + break; + default: + break; + } + callee->in_callback_fn = true; callee->callback_ret_range = retval_range(0, 1); return 0; @@ -10132,6 +11848,7 @@ static int set_timer_callback_state(struct bpf_verifier_env *env, __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = map_ptr; callee->regs[BPF_REG_3].map_uid = map_uid; + callee->regs[BPF_REG_3].id = ++env->id_gen; /* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); @@ -10165,6 +11882,8 @@ static int set_find_vma_callback_state(struct bpf_verifier_env *env, bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_callback_fn = true; + /* bpf_find_vma() runs the callback under mmap_lock and RCU. */ + callee->in_rcu_callback_fn = true; callee->callback_ret_range = retval_range(0, 1); return 0; } @@ -10250,6 +11969,41 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = map_ptr; callee->regs[BPF_REG_3].map_uid = map_uid; + callee->regs[BPF_REG_3].id = ++env->id_gen; + + /* unused */ + bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); + bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); + callee->in_async_callback_fn = true; + callee->callback_ret_range = retval_range(S32_MIN, S32_MAX); + return 0; +} + +static int set_rcu_callback_state(struct bpf_verifier_env *env, + struct bpf_func_state *caller, + struct bpf_func_state *callee, + int insn_idx) +{ + struct bpf_map *map_ptr = caller->regs[BPF_REG_2].map_ptr; + u32 map_uid = caller->regs[BPF_REG_2].map_uid; + + /* + * callback_fn(struct bpf_map *map, void *key, void *value); + */ + callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; + __mark_reg_known_zero(&callee->regs[BPF_REG_1]); + callee->regs[BPF_REG_1].map_ptr = map_ptr; + callee->regs[BPF_REG_1].map_uid = map_uid; + + callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; + __mark_reg_known_zero(&callee->regs[BPF_REG_2]); + callee->regs[BPF_REG_2].map_ptr = map_ptr; + callee->regs[BPF_REG_2].map_uid = map_uid; + + callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; + __mark_reg_known_zero(&callee->regs[BPF_REG_3]); + callee->regs[BPF_REG_3].map_ptr = map_ptr; + callee->regs[BPF_REG_3].map_uid = map_uid; /* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); @@ -10337,11 +12091,15 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) struct bpf_func_state *caller, *callee; struct bpf_reg_state *r0; bool in_callback_fn; + u32 i, nregs; int err; callee = state->frame[state->curframe]; r0 = &callee->regs[BPF_REG_0]; - if (r0->type == PTR_TO_STACK) { + nregs = bpf_ret_reg_pair(env, callee->subprogno) ? 2 : 1; + for (i = 0; i < nregs; i++) { + if (callee->regs[ret_regs[i]].type != PTR_TO_STACK) + continue; /* technically it's ok to return caller's stack pointer * (or caller's caller's pointer) back to the caller, * since these pointers are valid. Only current stack @@ -10353,6 +12111,19 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) } caller = state->frame[state->curframe - 1]; + if (bpf_is_unwind_exit(&env->prog->insnsi[*insn_idx])) { + /* the helper or the kfunc that called it doesn't expect that */ + if (callee->in_callback_fn) { + verbose(env, "callback cannot unwind\n"); + return -EINVAL; + } + if (!bpf_is_unwind_goto(&env->prog->insnsi[callee->callsite + 1])) { + verbose(env, "unwind_exit, but no unwind_goto after the call at insn %d\n", + callee->callsite); + return -EINVAL; + } + state->unwound = true; + } if (callee->in_callback_fn) { if (r0->type != SCALAR_VALUE) { verbose(env, "R0 not a scalar value\n"); @@ -10376,12 +12147,26 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) return -EFAULT; } } else { - /* return to the caller whatever r0 had in the callee */ - bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE); - caller->regs[BPF_REG_0] = *r0; - bpf_diag_mod_end(env); + /* + * return to the caller whatever the callee had in the + * return register(s) + */ + for (i = 0; i < nregs; i++) { + u32 regno = ret_regs[i]; + + bpf_diag_mod_begin(env, &caller->regs[regno], &callee->regs[regno], + BPF_DIAG_MOD_WRITE); + caller->regs[regno] = callee->regs[regno]; + bpf_diag_mod_end(env); + } } + /* Invalidate callee-local dynptrs and their slices before the frame goes away. */ + err = destroy_dynptrs_in_stack_slots(env, callee, 0, + callee->allocated_stack / BPF_REG_SIZE - 1); + if (err) + return err; + /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite, * there function call logic would reschedule callback visit. If iteration * converges is_state_visited() would prune that visit eventually. @@ -10769,26 +12554,37 @@ int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, return *ptr && (*ptr)->func ? 0 : -EINVAL; } +/* Is any frame on the current path a callback that runs under RCU? */ +static bool in_rcu_callback(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *state = env->cur_state; + int i; + + for (i = 0; i <= state->curframe; i++) + if (state->frame[i]->in_rcu_callback_fn) + return true; + return false; +} + /* Check if we're in a sleepable context. */ static inline bool in_sleepable_context(struct bpf_verifier_env *env) { - return !env->cur_state->active_rcu_locks && - !env->cur_state->active_preempt_locks && - !env->cur_state->active_locks && - !env->cur_state->active_irq_id && - in_sleepable(env); + return !in_rcu_cs(env) && !in_rcu_callback(env); } static const char *non_sleepable_context_description(struct bpf_verifier_env *env) { - if (env->cur_state->active_rcu_locks) - return "rcu_read_lock region"; - if (env->cur_state->active_preempt_locks) - return "non-preemptible region"; - if (env->cur_state->active_irq_id) - return "IRQ-disabled region"; - if (env->cur_state->active_locks) - return "lock region"; + if (in_sleepable(env)) { + if (env->cur_state->active_rcu_locks) + return "rcu_read_lock region"; + if (env->cur_state->active_preempt_locks) + return "non-preemptible region"; + if (env->cur_state->active_irq_id) + return "IRQ-disabled region"; + if (env->cur_state->active_locks) + return "lock region"; + return "callback running under rcu_read_lock"; + } return "non-sleepable prog"; } @@ -10873,19 +12669,24 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn memset(&meta, 0, sizeof(meta)); - err = check_func_proto(fn, &meta); + err = check_func_proto(env, fn, &meta); if (err) { verifier_bug(env, "incorrect func proto %s#%d", func_id_name(func_id), func_id); return err; } if (fn->might_sleep && !in_sleepable_context(env)) { + const char *suggestion; + verbose(env, "sleepable helper %s#%d in %s\n", func_id_name(func_id), func_id, non_sleepable_context_description(env)); + if (in_sleepable(env)) + suggestion = "Move the helper call outside the critical section, or use a non-sleepable helper."; + else + suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable helper."; operation = bpf_diag_fmt(env, "sleepable helper %s#%d", func_id_name(func_id), func_id); - bpf_diag_ctx_forbidden(env, insn_idx, operation, - "Move the helper call outside the critical section, or use a non-sleepable helper."); + bpf_diag_ctx_forbidden(env, insn_idx, operation, suggestion); return -EINVAL; } @@ -10894,13 +12695,11 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn env->insn_aux_data[insn_idx].non_sleepable = true; meta.func_id = func_id; + meta.func_name = func_id_name(func_id); meta.fn = fn; - /* check args */ - for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { - err = check_func_arg(env, i, &meta, insn_idx); - if (err) - return err; - } + err = check_func_args(env, &meta, insn_idx); + if (err) + return err; err = record_func_map(env, &meta, func_id, insn_idx); if (err) @@ -10912,16 +12711,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn regs = cur_regs(env); - /* Mark slots with STACK_MISC in case of raw mode, stack offset - * is inferred from register state. - */ - for (i = 0; i < meta.arg_raw_mem.size; i++) { - err = check_mem_access(env, insn_idx, regs + meta.arg_raw_mem.regno, - argno_from_reg(meta.arg_raw_mem.regno), i, BPF_B, - BPF_WRITE, -1, false, false); - if (err) - return err; - } + err = mark_raw_stack(env, &meta, insn_idx); + if (err) + return err; if (meta.release_regno) { struct bpf_reg_state *reg = ®s[meta.release_regno]; @@ -11042,8 +12834,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (dynptr_type == BPF_DYNPTR_TYPE_INVALID) return -EFAULT; - if (dynptr_type == BPF_DYNPTR_TYPE_SKB || - dynptr_type == BPF_DYNPTR_TYPE_SKB_META) + if (dynptr_write_changes_pkt_data(dynptr_type)) /* this will trigger clear_all_pkt_pointers(), which will * invalidate all dynptr slices associated with the skb */ @@ -11341,6 +13132,37 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return 0; } +/* + * Mark the register(s) holding a @size byte kfunc return value as unknown + * scalars and return how many registers that took. Both halves of a register + * pair are treated the same way. + * + * R0 is written in the diagnostic modification scope the caller opened before + * the caller-saved scrub, since R0 gets no scrub record of its own and that + * scope is what carries its pre-call value into the history. A scope tracks a + * single register and does not nest, so close it before recording the rest of + * the return value. A value that fits in R0 leaves the caller's scope open for + * the caller to close, so a later write to R0 still lands in it. + */ +static u32 mark_kfunc_ret_regs(struct bpf_verifier_env *env, + struct bpf_reg_state *regs, u32 size) +{ + u32 i, nregs = ret_regs_cnt(size); + + mark_reg_unknown(env, regs, ret_regs[0]); + if (nregs == 1) + return nregs; + + bpf_diag_mod_end(env); + for (i = 1; i < nregs; i++) { + bpf_diag_mod_begin(env, ®s[ret_regs[i]], NULL, BPF_DIAG_MOD_WRITE); + mark_reg_unknown(env, regs, ret_regs[i]); + bpf_diag_mod_end(env); + } + + return nregs; +} + static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ACQUIRE; @@ -11356,6 +13178,11 @@ static bool is_kfunc_destructive(struct bpf_call_arg_meta *meta) return meta->kfunc_flags & KF_DESTRUCTIVE; } +static bool is_kfunc_perfmon(struct bpf_call_arg_meta *meta) +{ + return meta->kfunc_flags & KF_PERFMON; +} + static bool is_kfunc_rcu(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_RCU; @@ -11446,6 +13273,11 @@ static bool is_kfunc_arg_irq_flag(const struct btf *btf, const struct btf_param return btf_param_match_suffix(btf, arg, "__irq_flag"); } +static bool is_kfunc_arg_ctx_out(const struct btf *btf, const struct btf_param *arg) +{ + return btf_param_match_suffix(btf, arg, "__ctx_out"); +} + static bool is_kfunc_arg_arena(const struct btf *btf, const struct btf_param *arg) { return btf_param_match_suffix(btf, arg, "__arena__nullable") || @@ -11481,7 +13313,8 @@ enum { KF_ARG_RES_SPIN_LOCK_ID, KF_ARG_TASK_WORK_ID, KF_ARG_PROG_AUX_ID, - KF_ARG_TIMER_ID + KF_ARG_TIMER_ID, + KF_ARG_RCU_HEAD_ID }; BTF_ID_LIST(kf_arg_btf_ids) @@ -11495,6 +13328,7 @@ BTF_ID(struct, bpf_res_spin_lock) BTF_ID(struct, bpf_task_work) BTF_ID(struct, bpf_prog_aux) BTF_ID(struct, bpf_timer) +BTF_ID(struct, bpf_rcu_head) static bool __is_kfunc_ptr_arg_type(const struct btf *btf, const struct btf_param *arg, int type) @@ -11553,6 +13387,11 @@ static bool is_kfunc_arg_task_work(const struct btf *btf, const struct btf_param return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_TASK_WORK_ID); } +static bool is_kfunc_arg_rcu_head(const struct btf *btf, const struct btf_param *arg) +{ + return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RCU_HEAD_ID); +} + static bool is_kfunc_arg_res_spin_lock(const struct btf *btf, const struct btf_param *arg) { return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RES_SPIN_LOCK_ID); @@ -11607,13 +13446,39 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ return argn <= arg_idx; } -/* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ -static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, - const struct btf *btf, - const struct btf_type *t, int rec) +#define BTF_MEMBER_MAX_DEPTH 4 +#define BTF_MEMBER_PATH_LEN 64 + +struct btf_member_path { + const struct btf_member *member[BTF_MEMBER_MAX_DEPTH]; + bool in_array[BTF_MEMBER_MAX_DEPTH]; + const struct btf_type *bad_type; + int depth; + bool too_deep; +}; + +static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type *t, + u32 member_kinds) +{ + if ((member_kinds & BTF_MEMBER_SCALAR) && btf_type_is_scalar(t)) + return true; + if ((member_kinds & BTF_MEMBER_ARENA_PTR) && btf_type_is_arena_ptr(btf, t)) + return true; + return false; +} + +/* + * Returns true if every member of struct @t is of a kind listed in + * @member_kinds, 4 levels of nesting allowed. An array member counts as its + * element type. + */ +static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, u32 member_kinds, int rec, + struct btf_member_path *path) { const struct btf_type *member_type; const struct btf_member *member; + bool in_array; u32 i; if (!btf_type_is_struct(t)) @@ -11622,58 +13487,128 @@ static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, for_each_member(i, t, member) { const struct btf_array *array; + in_array = false; member_type = btf_type_skip_modifiers(btf, member->type, NULL); + /* + * Every element of an array is laid out in the value being + * returned, so an array counts as its element type however many + * dimensions deep that is. + */ + while (btf_type_is_array(member_type)) { + array = btf_array(member_type); + if (!array->nelems) { + member_type = NULL; + goto bad_member; + } + member_type = btf_type_skip_modifiers(btf, array->type, NULL); + in_array = true; + } if (btf_type_is_struct(member_type)) { - if (rec >= 3) { + if (rec >= BTF_MEMBER_MAX_DEPTH - 1) { verbose(env, "max struct nesting depth exceeded\n"); + if (path) + path->too_deep = true; return false; } - if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) - return false; - continue; - } - if (btf_type_is_array(member_type)) { - array = btf_array(member_type); - if (!array->nelems) - return false; - member_type = btf_type_skip_modifiers(btf, array->type, NULL); - if (!btf_type_is_scalar(member_type)) - return false; + if (!btf_struct_member_walk(env, btf, member_type, member_kinds, + rec + 1, path)) + goto bad_path; continue; } - if (!btf_type_is_scalar(member_type)) - return false; + if (!btf_member_kind_allowed(btf, member_type, member_kinds)) + goto bad_member; } return true; + +bad_member: + if (path) { + path->depth = rec + 1; + path->bad_type = member_type; + } +bad_path: + if (path && path->depth) { + path->member[rec] = member; + path->in_array[rec] = in_array; + } + return false; } -enum kfunc_ptr_arg_type { - KF_ARG_CONST_MEM_SIZE, - KF_ARG_MEM_SIZE, - KF_ARG_CONST, - KF_ARG_CONST_ALLOC_SIZE_OR_ZERO, - KF_ARG_ANYTHING, - KF_ARG_PTR_TO_CTX, - KF_ARG_PTR_TO_ALLOC_BTF_ID, /* Allocated object */ - KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */ - KF_ARG_PTR_TO_DYNPTR, - KF_ARG_PTR_TO_ITER, - KF_ARG_PTR_TO_LIST_HEAD, - KF_ARG_PTR_TO_LIST_NODE, - KF_ARG_PTR_TO_BTF_ID, /* Also covers reg2btf_ids conversions */ - KF_ARG_PTR_TO_MEM, - KF_ARG_PTR_TO_CALLBACK, - KF_ARG_PTR_TO_RB_ROOT, - KF_ARG_PTR_TO_RB_NODE, - KF_ARG_PTR_TO_CONST_STR, - KF_ARG_CONST_MAP_PTR, - KF_ARG_PTR_TO_TIMER, - KF_ARG_PTR_TO_WORKQUEUE, - KF_ARG_PTR_TO_IRQ_FLAG, - KF_ARG_PTR_TO_RES_SPIN_LOCK, - KF_ARG_PTR_TO_TASK_WORK, - KF_ARG_PTR_TO_ARENA, -}; +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t, u32 member_kinds) +{ + return btf_struct_member_walk(env, btf, t, member_kinds, 0, NULL); +} + +static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t) +{ + return btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR); +} + +static int resolve_func_arg_type(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, u32 arg, argno_t argno, + struct bpf_call_arg_meta *meta, + enum bpf_arg_type *arg_type, u32 *arg_size) +{ + const struct btf_param *args; + const struct btf_type *ref_t, *resolve_ret; + const struct btf *btf; + const char *ref_tname; + u32 ref_id; + + if (base_type(*arg_type) == ARG_PTR_TO_MAP_VALUE) + return resolve_map_arg_type(env, meta, arg_type); + + if (base_type(*arg_type) != ARG_PTR_TO_BTF_ID) + return 0; + + if (!is_kfunc(meta) || arg_type_is_release(*arg_type) || + base_type(reg->type) == PTR_TO_BTF_ID || + reg2btf_ids[base_type(reg->type)]) + return 0; + + args = btf_params(meta->func_proto); + ref_id = *meta->fn->arg_btf_id[arg]; + btf = is_kfunc_arg_map(meta->btf, &args[arg]) ? btf_vmlinux : meta->btf; + ref_t = btf_type_skip_modifiers(btf, ref_id, &ref_id); + ref_tname = btf_name_by_offset(btf, ref_t->name_off); + + if (!btf_type_is_scalar_struct(env, btf, ref_t)) + return 0; + + resolve_ret = btf_resolve_size(btf, ref_t, arg_size); + if (IS_ERR(resolve_ret)) { + verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname, + PTR_ERR(resolve_ret)); + return -EINVAL; + } + *arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE | MEM_WRITE | + (*arg_type & (PTR_MAYBE_NULL | MEM_UNINIT)); + + return 0; +} + +static void btf_member_path_str(const struct btf *btf, const struct btf_member_path *path, + char *buf, size_t buf_sz) +{ + size_t len = 0; + int i; + + buf[0] = '\0'; + for (i = 0; i < path->depth; i++) { + const char *name = btf_name_by_offset(btf, path->member[i]->name_off); + + /* An anonymous struct or union has no name to spell. */ + if (!name || !name[0]) + continue; + len += scnprintf(buf + len, buf_sz - len, "%s%s", len ? "." : "", name); + if (path->in_array[i]) + len += scnprintf(buf + len, buf_sz - len, "[]"); + } +} enum special_kfunc_type { KF_bpf_obj_new_impl, @@ -11740,6 +13675,8 @@ enum special_kfunc_type { KF___bpf_trap, KF_bpf_task_work_schedule_signal, KF_bpf_task_work_schedule_resume, + KF_bpf_call_rcu, + KF_bpf_call_rcu_tasks_trace, KF_bpf_arena_alloc_pages, KF_bpf_arena_free_pages, KF_bpf_session_is_return, @@ -11830,14 +13767,36 @@ BTF_ID(func, bpf_dynptr_file_discard) BTF_ID(func, __bpf_trap) BTF_ID(func, bpf_task_work_schedule_signal) BTF_ID(func, bpf_task_work_schedule_resume) +BTF_ID(func, bpf_call_rcu) +BTF_ID(func, bpf_call_rcu_tasks_trace) +#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) BTF_ID(func, bpf_arena_alloc_pages) BTF_ID(func, bpf_arena_free_pages) +#else +BTF_ID_UNUSED +BTF_ID_UNUSED +#endif #ifdef CONFIG_BPF_EVENTS BTF_ID(func, bpf_session_is_return) #else BTF_ID_UNUSED #endif +static bool is_bpf_cast_to_kern_ctx_kfunc(const struct bpf_call_arg_meta *meta) +{ + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx]); +} + +static bool is_bpf_dynptr_clone_kfunc(const struct bpf_call_arg_meta *meta) +{ + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_clone]); +} + +static bool is_bpf_iter_css_task_new_kfunc(const struct bpf_call_arg_meta *meta) +{ + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_iter_css_task_new]); +} + static bool is_bpf_obj_new_kfunc(u32 func_id) { return func_id == special_kfunc_list[KF_bpf_obj_new] || @@ -11883,6 +13842,12 @@ static bool is_bpf_rbtree_add_kfunc(u32 func_id) func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]; } +static bool is_call_rcu_kfunc(u32 func_id) +{ + return func_id == special_kfunc_list[KF_bpf_call_rcu] || + func_id == special_kfunc_list[KF_bpf_call_rcu_tasks_trace]; +} + static bool is_task_work_add_kfunc(u32 func_id) { return func_id == special_kfunc_list[KF_bpf_task_work_schedule_signal] || @@ -11899,52 +13864,159 @@ static bool is_kfunc_ret_null(struct bpf_call_arg_meta *meta) static bool is_kfunc_bpf_rcu_read_lock(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_rcu_read_lock]); } static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_rcu_read_unlock]); } static bool is_kfunc_bpf_preempt_disable(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_preempt_disable]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_preempt_disable]); } static bool is_kfunc_bpf_preempt_enable(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_preempt_enable]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_preempt_enable]); +} + +/* These write to what their first argument, a dynptr, points to */ +BTF_SET_START(dynptr_write_kfuncs) +BTF_ID(func, bpf_dynptr_copy) +BTF_ID(func, bpf_dynptr_memset) +#ifdef CONFIG_BPF_EVENTS +BTF_ID(func, bpf_probe_read_user_dynptr) +BTF_ID(func, bpf_probe_read_kernel_dynptr) +BTF_ID(func, bpf_probe_read_user_str_dynptr) +BTF_ID(func, bpf_probe_read_kernel_str_dynptr) +BTF_ID(func, bpf_copy_from_user_dynptr) +BTF_ID(func, bpf_copy_from_user_str_dynptr) +BTF_ID(func, bpf_copy_from_user_task_dynptr) +BTF_ID(func, bpf_copy_from_user_task_str_dynptr) +#endif +BTF_SET_END(dynptr_write_kfuncs) + +static bool is_kfunc_dynptr_write(struct bpf_call_arg_meta *meta) +{ + return meta->btf == btf_vmlinux && + btf_id_set_contains(&dynptr_write_kfuncs, meta->func_id); } +/* + * A write through the dynptr of an skb may reallocate its head. Before the + * arguments are checked nobody knows what kind of dynptr it is. + */ bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_xdp_pull_data]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_xdp_pull_data]) || + is_kfunc_dynptr_write(meta); +} + +static u32 kfunc_abi_slots(const struct btf_func_model *fm) +{ + const struct bpf_jit_arg_abi *abi = bpf_jit_arg_abi(); + u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; + u32 i, nslots, slots = 0; + + for (i = 0; i < fm->nr_args; i++) + slots += btf_func_model_arg_slots(fm, i); + + if (!abi) + return slots; + + nslots = bpf_jit_place_args(abi, fm, pos_of_slot); + for (i = 0; i < nslots; i++) + if (pos_of_slot[i] + 1 > slots) + slots = pos_of_slot[i] + 1; + + return slots; +} + +static u32 __btf_func_arg_align(const struct btf *btf, const struct btf_type *t, int rec) +{ + const struct btf_member *member; + const struct btf_type *mt; + u32 align, i; + + while (btf_type_is_array(t)) + t = btf_type_skip_modifiers(btf, btf_array(t)->type, NULL); + + if (btf_type_is_int(t)) + return t->size > BPF_REG_SIZE ? t->size : BPF_REG_SIZE; + if (!btf_type_is_struct(t)) + return BPF_REG_SIZE; + if (rec >= BTF_MEMBER_MAX_DEPTH) + return 0; + + for_each_member(i, t, member) { + mt = btf_type_skip_modifiers(btf, member->type, NULL); + align = __btf_func_arg_align(btf, mt, rec + 1); + if (!align) + return 0; + if (align > BPF_REG_SIZE) + return 2 * BPF_REG_SIZE; + } + return BPF_REG_SIZE; +} + +u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t) +{ + return __btf_func_arg_align(btf, t, 0); } static int get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - const struct btf_param *args, int arg, int nargs) + const struct btf_param *args, int arg, int nargs, u32 slot, + struct bpf_func_proto *proto) { - const struct btf_type *t, *ref_t = NULL; - argno_t argno = argno_from_arg(arg + 1); + const struct btf_type *t, *ref_t = NULL, *resolve_ret; + const u32 *ref_id_ptr = NULL; + argno_t argno = argno_from_arg(slot + 1); const char *ref_tname = NULL; + u32 ref_id, type_size; int arg_type; + proto->arg_btf_id[arg] = NULL; + + if (is_kfunc_arg_prog_aux(meta->btf, &args[arg])) + return ARG_PTR_TO_PROG_AUX; + + if (is_kfunc_arg_ignore(meta->btf, &args[arg]) || is_kfunc_arg_implicit(meta, arg)) + return ARG_IGNORE; + t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL); /* Scalar arguments are classified from their BTF suffix/name alone. */ if (btf_type_is_scalar(t)) { if (is_kfunc_arg_constant(meta->btf, &args[arg])) - return KF_ARG_CONST; + return ARG_CONST_SCALAR; if (is_kfunc_arg_const_mem_size(meta->btf, &args[arg])) - return KF_ARG_CONST_MEM_SIZE; + return ARG_CONST_MEM_SIZE; if (is_kfunc_arg_mem_size(meta->btf, &args[arg])) - return KF_ARG_MEM_SIZE; + return ARG_MEM_SIZE; if (is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdonly_buf_size") || is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdwr_buf_size")) - return KF_ARG_CONST_ALLOC_SIZE_OR_ZERO; - return KF_ARG_ANYTHING; + return ARG_CONST_ALLOC_SIZE_OR_ZERO; + return ARG_SCALAR; + } + + if (btf_type_is_struct(t)) { + if (!t->size || t->size > 2 * BPF_REG_SIZE) { + verbose(env, + "%s type %s has size %u, only 1 to %d bytes " + "can be passed by value\n", + reg_arg_name(env, argno), btf_type_str(t), t->size, + 2 * BPF_REG_SIZE); + return -EINVAL; + } + if (!btf_type_is_scalar_struct(env, meta->btf, t)) { + verbose(env, "%s type %s is not composed of scalars\n", + reg_arg_name(env, argno), btf_type_str(t)); + return -EINVAL; + } + return ARG_SCALAR; } if (!btf_type_is_ptr(t)) { @@ -11952,54 +14024,87 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, reg_arg_name(env, argno), btf_type_str(t)); return -EINVAL; } - ref_t = btf_type_skip_modifiers(meta->btf, t->type, NULL); + /* Keep a pointer to the BTF field containing the resolved referent ID. */ + ref_id_ptr = &t->type; + ref_t = btf_type_skip_modifiers(meta->btf, *ref_id_ptr, &ref_id); + while (*ref_id_ptr != ref_id) + ref_id_ptr = &btf_type_by_id(meta->btf, *ref_id_ptr)->type; ref_tname = btf_name_by_offset(meta->btf, ref_t->name_off); /* In this function, we verify the kfunc's BTF as per the argument type, * leaving the rest of the verification with respect to the register * type to our caller. When a set of conditions hold in the BTF type of - * arguments, we resolve it to a known kfunc_ptr_arg_type. + * arguments, we resolve it to a known bpf_arg_type. */ - if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || - meta->func_id == special_kfunc_list[KF_bpf_session_is_return] || - meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) - arg_type = KF_ARG_PTR_TO_CTX; + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_session_is_return]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_session_cookie])) + arg_type = ARG_PTR_TO_CTX; else if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) - arg_type = KF_ARG_PTR_TO_CTX; + arg_type = ARG_PTR_TO_CTX; else if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_ALLOC_BTF_ID; + arg_type = ARG_PTR_TO_ALLOC_BTF_ID; else if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_REFCOUNTED_KPTR; - else if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_DYNPTR; - else if (is_kfunc_arg_iter(meta, arg, &args[arg])) - arg_type = KF_ARG_PTR_TO_ITER; + arg_type = ARG_PTR_TO_REFCOUNTED_KPTR; + else if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) { + arg_type = ARG_PTR_TO_DYNPTR; + + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_skb])) + arg_type |= DYNPTR_TYPE_SKB; + else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_xdp])) + arg_type |= DYNPTR_TYPE_XDP; + else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_skb_meta])) + arg_type |= DYNPTR_TYPE_SKB_META; + else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_file]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_file_discard])) + /* OBJ_RELEASE for the latter comes from KF_RELEASE below */ + arg_type |= DYNPTR_TYPE_FILE; + } else if (is_kfunc_arg_iter(meta, arg, &args[arg])) + arg_type = ARG_PTR_TO_ITER; else if (is_kfunc_arg_list_head(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_LIST_HEAD; + arg_type = ARG_PTR_TO_LIST_HEAD; else if (is_kfunc_arg_list_node(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_LIST_NODE; + arg_type = ARG_PTR_TO_LIST_NODE; else if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_RB_ROOT; + arg_type = ARG_PTR_TO_RB_ROOT; else if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_RB_NODE; + arg_type = ARG_PTR_TO_RB_NODE; else if (is_kfunc_arg_const_str(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_CONST_STR; + arg_type = ARG_PTR_TO_CONST_STR; else if (is_kfunc_arg_const_map(meta->btf, &args[arg])) - arg_type = KF_ARG_CONST_MAP_PTR; + arg_type = ARG_CONST_MAP_PTR; else if (is_kfunc_arg_map(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_BTF_ID; + arg_type = ARG_PTR_TO_BTF_ID; else if (is_kfunc_arg_wq(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_WORKQUEUE; + arg_type = ARG_PTR_TO_WORKQUEUE; else if (is_kfunc_arg_timer(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_TIMER; + arg_type = ARG_PTR_TO_TIMER; else if (is_kfunc_arg_task_work(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_TASK_WORK; + arg_type = ARG_PTR_TO_TASK_WORK; + else if (is_kfunc_arg_rcu_head(meta->btf, &args[arg])) + arg_type = ARG_PTR_TO_RCU_HEAD; else if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_IRQ_FLAG; + arg_type = ARG_PTR_TO_IRQ_FLAG; else if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_RES_SPIN_LOCK; - else if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_CALLBACK; + arg_type = ARG_PTR_TO_RES_SPIN_LOCK; + else if (is_kfunc_arg_ctx_out(meta->btf, &args[arg])) { + if (!btf_type_is_scalar(ref_t)) { + verbose(env, "%s __ctx_out argument must point to a scalar\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + resolve_ret = btf_resolve_size(meta->btf, ref_t, &type_size); + if (IS_ERR(resolve_ret)) { + verbose(env, + "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), + ref_tname, PTR_ERR(resolve_ret)); + return -EINVAL; + } + proto->arg_size[arg] = type_size; + arg_type = ARG_PTR_TO_CTX_OUT | MEM_FIXED_SIZE; + } else if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) + arg_type = ARG_PTR_TO_FUNC; else if (is_kfunc_arg_arena(meta->btf, &args[arg])) { if (!bpf_jit_supports_arena_args()) { verbose(env, "JIT does not support kfunc %s() with arena pointer arguments\n", @@ -12022,20 +14127,20 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * whether the JIT rebases it to the arena base or preserves NULL. * The common nullable path below records that verifier property. */ - arg_type = KF_ARG_PTR_TO_ARENA; + arg_type = ARG_PTR_TO_ARENA; } else if (arg + 1 < nargs && (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && - !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; } - arg_type = KF_ARG_PTR_TO_MEM; + arg_type = ARG_PTR_TO_MEM | MEM_WRITE; } else if (btf_type_is_struct(ref_t)) - /* A pointer to a struct without a size argument is classified as KF_ARG_PTR_TO_BTF_ID */ - arg_type = KF_ARG_PTR_TO_BTF_ID; + /* A pointer to a struct without a size argument is classified as ARG_PTR_TO_BTF_ID */ + arg_type = ARG_PTR_TO_BTF_ID; else { /* * Otherwise this is a fixed-size memory buffer supported by @@ -12043,26 +14148,64 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * scalars. The access size is derived from the pointed-to BTF type. */ if (!btf_type_is_scalar(ref_t) && - !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; } - arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; + resolve_ret = btf_resolve_size(meta->btf, ref_t, &type_size); + if (IS_ERR(resolve_ret)) { + verbose(env, + "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), + ref_tname, PTR_ERR(resolve_ret)); + return -EINVAL; + } + proto->arg_size[arg] = type_size; + arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE | MEM_WRITE; } + if (is_kfunc_arg_uninit(meta->btf, &args[arg])) + arg_type |= MEM_UNINIT; + if (is_kfunc_arg_nullable(meta->btf, &args[arg])) arg_type |= PTR_MAYBE_NULL; + /* + * Only the first argument of a KF_RELEASE kfunc releases anything, and + * bpf_fetch_kfunc_arg_meta() only ever records BPF_REG_1 for it. + */ + if (is_kfunc_release(meta) && arg == 0) + arg_type |= OBJ_RELEASE; + + if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) { + if (is_kfunc_arg_map(meta->btf, &args[arg])) + proto->arg_btf_id[arg] = reg2btf_ids[CONST_PTR_TO_MAP]; + else + proto->arg_btf_id[arg] = ref_id_ptr; + + arg_type |= PTR_TRUSTED; + + /* + * A KF_RCU kfunc accepts an RCU-protected pointer where it would + * otherwise demand a referenced or trusted one. Other argument kinds + * have their own provenance requirements and must not inherit MEM_RCU. + */ + if (is_kfunc_rcu(meta)) + arg_type |= MEM_RCU; + } + return arg_type; } static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - struct bpf_func_proto *proto) + const struct btf_func_model *fm, struct bpf_func_proto *proto) { + const struct bpf_jit_arg_abi *abi; const struct btf *btf = meta->btf; const struct btf_param *args; - u32 i, nargs; + const struct btf_type *t; + u32 i, nargs, slots_used; int arg_type; args = (const struct btf_param *)(meta->func_proto + 1); @@ -12072,54 +14215,84 @@ static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg nargs, MAX_BPF_FUNC_ARGS); return -EINVAL; } - if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { - verbose(env, "JIT does not support kfunc %s() with %d args\n", - meta->func_name, nargs); - return -ENOTSUPP; + + for (i = 0, slots_used = 0; i < nargs; i++) { + u32 nslots = btf_func_model_arg_slots(fm, i); + + if (nslots > 1) { + t = btf_type_skip_modifiers(btf, args[i].type, NULL); + if (!btf_func_arg_align(btf, t)) { + verbose(env, + "Function %s arg#%d type %s nests structs more than " + "%d levels deep\n", + meta->func_name, i, btf_type_str(t), + BTF_MEMBER_MAX_DEPTH); + return -EINVAL; + } + } + slots_used += nslots; } - for (i = 0; i < nargs; i++) { - if (is_kfunc_arg_prog_aux(btf, &args[i]) || - is_kfunc_arg_ignore(btf, &args[i]) || - is_kfunc_arg_implicit(meta, i)) - continue; + if (slots_used > MAX_BPF_FUNC_ARGS) { + verbose(env, "Function %s needs %d > %d argument slots\n", meta->func_name, + slots_used, MAX_BPF_FUNC_ARGS); + return -EINVAL; + } + + abi = bpf_jit_arg_abi(); + + for (i = 0, slots_used = 0; i < nargs; i++) { + u32 nslots = btf_func_model_arg_slots(fm, i); + + if (!abi && nslots > 1) { + t = btf_type_skip_modifiers(btf, args[i].type, NULL); + verbose(env, + "Function %s arg#%d type %s cannot be passed at " + "argument slot %d on this architecture\n", + meta->func_name, i, btf_type_str(t), slots_used); + return -EINVAL; + } + slots_used += nslots; - arg_type = get_kfunc_arg_type(env, meta, args, i, nargs); + arg_type = get_kfunc_arg_type(env, meta, args, i, nargs, + slots_used - nslots, proto); if (arg_type < 0) return arg_type; proto->arg_type[i] = arg_type; } - return 0; + if (slots_used > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { + verbose(env, "JIT does not support kfunc %s() with %d argument slots\n", + meta->func_name, slots_used); + return -ENOTSUPP; + } + + return check_arg_prog_aux(env, proto) ? 0 : -EINVAL; } -static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, - struct bpf_reg_state *reg, - const struct btf_type *ref_t, - const char *ref_tname, u32 ref_id, - struct bpf_call_arg_meta *meta, - int arg, argno_t argno) +static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, enum bpf_arg_type arg_type, + const struct btf *arg_btf, u32 arg_btf_id, + struct bpf_call_arg_meta *meta, int insn_idx) { - const struct btf_type *reg_ref_t; - bool strict_type_match = false; + bool taking_projection, struct_same, strict_type_match = false; + const struct btf_type *arg_t, *reg_t; + const char *arg_tname, *reg_tname; const struct btf *reg_btf; - const char *reg_ref_tname; - bool taking_projection; - bool struct_same; - u32 reg_ref_id; + u32 reg_btf_id; if (base_type(reg->type) == PTR_TO_BTF_ID) { reg_btf = reg->btf; - reg_ref_id = reg->btf_id; + reg_btf_id = reg->btf_id; } else { reg_btf = btf_vmlinux; - reg_ref_id = *reg2btf_ids[base_type(reg->type)]; + reg_btf_id = *reg2btf_ids[base_type(reg->type)]; } - /* Enforce strict type matching for calls to kfuncs that are acquiring - * or releasing a reference, or are no-cast aliases. We do _not_ - * enforce strict matching for kfuncs by default, + /* + * Enforce strict type matching for arguments that release a reference, + * or are no-cast aliases. We do _not_ enforce strict matching by default, * as we want to enable BPF programs to pass types that are bitwise * equivalent without forcing them to explicitly cast with something * like bpf_cast_to_kern_ctx(). @@ -12141,27 +14314,30 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, * btf_struct_ids_match() to walk the struct at the 0th offset, and * resolve types. */ - if ((is_kfunc_release(meta) && reg_is_referenced(env, reg)) || - btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id)) + if ((arg_type_is_release(arg_type) && !is_helper_call(meta, BPF_FUNC_sk_release)) || + (is_kfunc(meta) && btf_type_ids_nocast_alias(&env->log, reg_btf, reg_btf_id, + arg_btf, arg_btf_id))) strict_type_match = true; - WARN_ON_ONCE(is_kfunc_release(meta) && !tnum_is_const(reg->var_off)); + arg_t = btf_type_skip_modifiers(arg_btf, arg_btf_id, &arg_btf_id); + arg_tname = btf_name_by_offset(arg_btf, arg_t->name_off); + reg_t = btf_type_skip_modifiers(reg_btf, reg_btf_id, ®_btf_id); + reg_tname = btf_name_by_offset(reg_btf, reg_t->name_off); + + struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_btf_id, + reg->var_off.value, arg_btf, arg_btf_id, + strict_type_match, !type_is_alloc(reg->type)); - reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, ®_ref_id); - reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off); - struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->var_off.value, - meta->btf, ref_id, strict_type_match, - !type_is_alloc(reg->type)); /* If kfunc is accepting a projection type (ie. __sk_buff), it cannot * actually use it -- it must cast to the underlying type. So we allow * caller to pass in the underlying type. */ - taking_projection = btf_is_projection_of(ref_tname, reg_ref_tname); + taking_projection = is_kfunc(meta) && btf_is_projection_of(arg_tname, reg_tname); if (!taking_projection && !struct_same) { - verbose(env, "kernel function %s %s expected pointer to %s %s but %s has a pointer to %s %s\n", + verbose(env, "%s %s expected pointer to %s %s but %s has a pointer to %s %s\n", meta->func_name, reg_arg_name(env, argno), - btf_type_str(ref_t), ref_tname, reg_arg_name(env, argno), - btf_type_str(reg_ref_t), reg_ref_tname); + btf_type_str(arg_t), arg_tname, + reg_arg_name(env, argno), btf_type_str(reg_t), reg_tname); return -EINVAL; } return 0; @@ -12173,15 +14349,15 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * int err, spi, kfunc_class = IRQ_NATIVE_KFUNC; bool irq_save; - if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_save] || - meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) { + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_local_irq_save]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) { irq_save = true; - if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) kfunc_class = IRQ_LOCK_KFUNC; - } else if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_restore] || - meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_local_irq_restore]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) { irq_save = false; - if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) kfunc_class = IRQ_LOCK_KFUNC; } else { verifier_bug(env, "unknown irq flags kfunc"); @@ -12374,12 +14550,20 @@ static bool is_bpf_rbtree_api_kfunc(u32 btf_id) btf_id == special_kfunc_list[KF_bpf_rbtree_right]; } -static bool is_bpf_res_spin_lock_kfunc(u32 btf_id) +static int get_bpf_res_spin_lock_kfunc_flags(const struct bpf_call_arg_meta *meta) { - return btf_id == special_kfunc_list[KF_bpf_res_spin_lock] || - btf_id == special_kfunc_list[KF_bpf_res_spin_unlock] || - btf_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] || - btf_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]; + int flags = PROCESS_RES_LOCK; + + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) + flags |= PROCESS_SPIN_LOCK; + else if (!is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock]) && + !is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) + return 0; + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) + flags |= PROCESS_LOCK_IRQ; + return flags; } static bool kfunc_spin_allowed(struct bpf_verifier_env *env, s32 func_id, s16 offset) @@ -12394,6 +14578,55 @@ static bool kfunc_spin_allowed(struct bpf_verifier_env *env, s32 func_id, s16 of return *kfunc.flags & KF_SPINLOCK_SAFE; } +/* + * True if insn calls a helper/kfunc that requires one of its arguments to + * be a stack pointer with a constant offset. + */ +bool bpf_needs_fixed_stack_off(struct bpf_verifier_env *env, int insn_idx) +{ + const struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; + const struct bpf_func_proto *fn; + struct bpf_kfunc_desc *desc; + u32 *flags, btf_id; + int i; + + if (bpf_helper_call(insn)) { + if (bpf_get_helper_proto(env, insn->imm, &fn) < 0) + return false; + } else if (bpf_pseudo_kfunc_call(insn)) { + desc = find_kfunc_desc(env->prog, insn->imm, insn->off); + if (!desc) + return false; + fn = &desc->proto; + } else { + return false; + } + + /* Both initialized and uninitialized stack dynptrs need a fixed offset. */ + for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) + if (arg_type_is_dynptr(fn->arg_type[i])) + return true; + + /* vmlinux kfuncs only */ + if (!bpf_pseudo_kfunc_call(insn) || insn->off != 0) + return false; + btf_id = insn->imm; + + flags = btf_kfunc_flags(btf_vmlinux, btf_id, env->prog); + if (flags && (*flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY))) + return true; + + if (btf_id == special_kfunc_list[KF_bpf_res_spin_lock] || + btf_id == special_kfunc_list[KF_bpf_res_spin_unlock] || + btf_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] || + btf_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore] || + btf_id == special_kfunc_list[KF_bpf_local_irq_save] || + btf_id == special_kfunc_list[KF_bpf_local_irq_restore]) + return true; + + return false; +} + static bool is_sync_callback_calling_kfunc(u32 btf_id) { return is_bpf_rbtree_add_kfunc(btf_id); @@ -12402,7 +14635,8 @@ static bool is_sync_callback_calling_kfunc(u32 btf_id) static bool is_async_callback_calling_kfunc(u32 btf_id) { return is_bpf_wq_set_callback_kfunc(btf_id) || - is_task_work_add_kfunc(btf_id); + is_task_work_add_kfunc(btf_id) || + is_call_rcu_kfunc(btf_id); } bool bpf_is_throw_kfunc(struct bpf_insn *insn) @@ -12656,707 +14890,6 @@ static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env) } } -static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - int insn_idx) -{ - const char *func_name = meta->func_name, *ref_tname; - struct bpf_func_state *caller = cur_func(env); - struct bpf_reg_state *regs = cur_regs(env); - const struct btf *btf = meta->btf; - const struct btf_param *args; - struct btf_record *rec; - u32 i, nargs; - int ret; - - args = (const struct btf_param *)(meta->func_proto + 1); - nargs = btf_type_vlen(meta->func_proto); - - ret = check_outgoing_stack_args(env, caller, nargs, func_name, btf, args); - if (ret) - return ret; - - /* Check that BTF function arguments match actual types that the - * verifier sees. - */ - for (i = 0; i < nargs; i++) { - struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); - const struct btf_type *t, *ref_t, *resolve_ret; - enum bpf_arg_type arg_type = ARG_DONTCARE; - argno_t argno = argno_from_arg(i + 1); - int regno = reg_from_argno(argno); - bool btf_id_fixed_off_ok = true; - u32 ref_id = args[i].type, type_size; - int kf_arg_type = meta->fn->arg_type[i]; - - if (is_kfunc_arg_prog_aux(btf, &args[i])) { - /* Reject repeated use bpf_prog_aux */ - if (meta->arg_prog) { - verifier_bug(env, "Only 1 prog->aux argument supported per-kfunc"); - return -EFAULT; - } - if (regno < 0) { - verbose(env, "%s prog->aux cannot be a stack argument\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - meta->arg_prog = true; - cur_aux(env)->arg_prog = regno; - continue; - } - - if (is_kfunc_arg_ignore(btf, &args[i]) || is_kfunc_arg_implicit(meta, i)) - continue; - - t = btf_type_skip_modifiers(btf, args[i].type, NULL); - - if (btf_type_is_ptr(t)) { - ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); - } - - if (btf_type_is_ptr(t) && - (bpf_register_is_null(reg) || type_may_be_null(reg->type)) && - !type_may_be_null(kf_arg_type)) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, args[i].type); - verbose(env, "Possibly NULL pointer passed to trusted %s\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Add a NULL check and call the kfunc only on the non-NULL path.", - "the pointer may be NULL, but this kfunc requires a non-NULL value of type %s", - expected_type); - return -EACCES; - } - - if (regno == meta->release_regno && !is_kfunc_arg_dynptr(meta->btf, &args[i]) && - !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - verbose(env, "release kfunc %s expects referenced PTR_TO_BTF_ID passed to %s\n", - func_name, reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass the resource-owning pointer returned by the matching acquire kfunc, and avoid calling the release kfunc after ownership has already been transferred or released.", - "release kfuncs require a resource-owning value of type %s returned by a matching acquire kfunc", - expected_type); - return -EINVAL; - } - - if (reg_is_referenced(env, reg)) - update_ref_obj(&meta->ref_obj, reg); - - if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) { - ret = mark_arg_precision(env, argno); - if (ret) - return ret; - continue; - } - - if (is_kfunc_arg_map(btf, &args[i])) { - ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; - ref_t = btf_type_by_id(btf_vmlinux, ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); - } - - switch (base_type(kf_arg_type)) { - case KF_ARG_CONST: - case KF_ARG_CONST_MEM_SIZE: - case KF_ARG_MEM_SIZE: - case KF_ARG_ANYTHING: - case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: - case KF_ARG_PTR_TO_ALLOC_BTF_ID: - case KF_ARG_PTR_TO_BTF_ID: - case KF_ARG_CONST_MAP_PTR: - case KF_ARG_PTR_TO_ITER: - case KF_ARG_PTR_TO_LIST_HEAD: - case KF_ARG_PTR_TO_LIST_NODE: - case KF_ARG_PTR_TO_RB_ROOT: - case KF_ARG_PTR_TO_RB_NODE: - case KF_ARG_PTR_TO_MEM: - case KF_ARG_PTR_TO_CALLBACK: - case KF_ARG_PTR_TO_CONST_STR: - case KF_ARG_PTR_TO_WORKQUEUE: - case KF_ARG_PTR_TO_TIMER: - case KF_ARG_PTR_TO_TASK_WORK: - case KF_ARG_PTR_TO_IRQ_FLAG: - case KF_ARG_PTR_TO_RES_SPIN_LOCK: - case KF_ARG_PTR_TO_ARENA: - break; - case KF_ARG_PTR_TO_DYNPTR: - arg_type = ARG_PTR_TO_DYNPTR; - break; - case KF_ARG_PTR_TO_CTX: - arg_type = ARG_PTR_TO_CTX; - break; - case KF_ARG_PTR_TO_REFCOUNTED_KPTR: - arg_type = ARG_PTR_TO_BTF_ID; - btf_id_fixed_off_ok = false; - break; - default: - verifier_bug(env, "unknown kfunc arg type %d", kf_arg_type); - return -EFAULT; - } - - if (regno == meta->release_regno) - arg_type |= OBJ_RELEASE; - ret = __check_func_arg_reg_off(env, reg, argno, arg_type, - btf_id_fixed_off_ok); - if (ret < 0) - return ret; - - switch (base_type(kf_arg_type)) { - case KF_ARG_CONST: - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an integer scalar value for this argument, not a pointer or resource object.", - "the kfunc expects an integer scalar, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - - ret = process_const_arg(env, reg, argno, meta); - if (ret < 0) { - if (ret == -EINVAL) - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a compile-time constant or a value the verifier can prove is constant at this call.", - "the kfunc requires this scalar argument to be a verifier-known constant, but %s is variable on this path", - reg_arg_name(env, argno)); - return ret; - } - break; - case KF_ARG_ANYTHING: - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an integer scalar value for this argument, not a pointer or resource object.", - "the kfunc expects an integer scalar, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - break; - case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an integer scalar value for this argument, not a pointer or resource object.", - "the kfunc expects an integer scalar, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - - if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) - meta->r0_rdonly = true; - ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); - if (ret < 0) { - if (ret == -EINVAL) - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a verifier-known constant size for this kfunc buffer argument.", - "the kfunc uses this argument as a return-buffer size, but %s is invalid or variable on this path", - reg_arg_name(env, argno)); - return ret; - } - break; - case KF_ARG_PTR_TO_CTX: - if (reg->type != PTR_TO_CTX) { - verbose(env, "%s expected pointer to ctx, but got %s\n", - reg_arg_name(env, argno), reg_type_str(env, reg->type)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass the original program context pointer or preserve it before modifying registers.", - "the kfunc expects a context pointer, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - - if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { - ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); - if (ret < 0) - return -EINVAL; - meta->ret_btf_id = ret; - } - break; - case KF_ARG_PTR_TO_ARENA: - if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a pointer to arena or scalar\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - break; - case KF_ARG_PTR_TO_ALLOC_BTF_ID: - if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { - if (!is_bpf_obj_drop_kfunc(meta->func_id)) { - verbose(env, "%s expected for bpf_obj_drop()\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) { - if (!is_bpf_percpu_obj_drop_kfunc(meta->func_id)) { - verbose(env, "%s expected for bpf_percpu_obj_drop()\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } else { - verbose(env, "%s expected pointer to allocated object\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a pointer returned by the matching BPF object allocation path.", - "the kfunc expects an allocated object pointer, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - if (!reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass the owned object pointer before it is released or transferred.", - "the allocated object pointer in %s must still carry verifier-tracked ownership, but this pointer no longer owns a live resource", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (meta->btf == btf_vmlinux) { - meta->arg_btf = reg->btf; - meta->arg_btf_id = reg->btf_id; - } - break; - case KF_ARG_PTR_TO_DYNPTR: - { - enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR; - - if (is_kfunc_arg_uninit(btf, &args[i])) - dynptr_arg_type |= MEM_UNINIT; - - if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) { - dynptr_arg_type |= DYNPTR_TYPE_SKB; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_xdp]) { - dynptr_arg_type |= DYNPTR_TYPE_XDP; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb_meta]) { - dynptr_arg_type |= DYNPTR_TYPE_SKB_META; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_file]) { - dynptr_arg_type |= DYNPTR_TYPE_FILE; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_file_discard]) { - dynptr_arg_type |= DYNPTR_TYPE_FILE | OBJ_RELEASE; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] && - (dynptr_arg_type & MEM_UNINIT)) { - enum bpf_dynptr_type parent_type = meta->dynptr.type; - - if (parent_type == BPF_DYNPTR_TYPE_INVALID) { - verifier_bug(env, "no dynptr type for parent of clone"); - return -EFAULT; - } - - dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type); - } - - ret = process_dynptr_func(env, reg, argno, insn_idx, func_name, - dynptr_arg_type, &meta->ref_obj, &meta->dynptr); - if (ret < 0) - return ret; - break; - } - case KF_ARG_PTR_TO_ITER: - if (meta->func_id == special_kfunc_list[KF_bpf_iter_css_task_new]) { - if (!check_css_task_iter_allowlist(env)) { - verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n"); - return -EINVAL; - } - } - ret = process_iter_arg(env, reg, argno, insn_idx, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_LIST_HEAD: - if (reg->type != PTR_TO_MAP_VALUE && - reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s expected pointer to map value or allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && - !reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - return -EINVAL; - } - ret = process_kf_arg_ptr_to_list_head(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_RB_ROOT: - if (reg->type != PTR_TO_MAP_VALUE && - reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s expected pointer to map value or allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && - !reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - return -EINVAL; - } - ret = process_kf_arg_ptr_to_rbtree_root(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_LIST_NODE: - if (is_kfunc_arg_nonown_allowed(btf, &args[i]) && - type_is_non_owning_ref(reg->type) && !reg_is_referenced(env, reg)) { - /* Allow bpf_list_front/back return value for - * __nonown_allowed list-node arguments. - */ - goto check_ok; - } - if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s expected pointer to allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (!reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - return -EINVAL; - } -check_ok: - ret = process_kf_arg_ptr_to_list_node(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_RB_NODE: - if (is_bpf_rbtree_add_kfunc(meta->func_id)) { - if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s expected pointer to allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (!reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - return -EINVAL; - } - } else { - if (!type_is_non_owning_ref(reg->type) && - !reg_is_referenced(env, reg)) { - verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n", func_name); - return -EINVAL; - } - if (in_rbtree_lock_required_cb(env)) { - verbose(env, "%s not allowed in rbtree cb\n", func_name); - return -EINVAL; - } - } - - ret = process_kf_arg_ptr_to_rbtree_node(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_CONST_MAP_PTR: - if (base_type(reg->type) != CONST_PTR_TO_MAP || - type_may_be_null(reg->type)) { - verbose(env, "pointer in %s isn't map pointer\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - ret = process_map_ptr_arg(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_BTF_ID: - /* Only base_type is checked, further checks are done here */ - if (base_type(reg->type) == PTR_TO_BTF_ID || - reg2btf_ids[base_type(reg->type)]) { - if (!is_trusted_reg(env, reg) || - bpf_type_has_unsafe_modifiers(reg->type)) { - if (!is_kfunc_rcu(meta)) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - verbose(env, "%s must be referenced or trusted\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a pointer acquired from a verifier-tracked source, or call this kfunc only inside the required protection if it accepts RCU pointers.", - "the kfunc requires a trusted or resource-owning pointer to %s, but %s is %s", - expected_type, - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - if (!is_rcu_reg(reg)) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - verbose(env, "%s must be a rcu pointer\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Use this kfunc with a pointer that is valid in an RCU read lock region.", - "the kfunc requires an RCU-protected pointer to %s, but %s is %s", - expected_type, - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - } - - ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno); - if (ret < 0) - return ret; - break; - } - - if (!__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { - enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); - const char *expected_type; - - verbose(env, "%s is %s expected %s %s", - reg_arg_name(env, argno), reg_type_str(env, reg->type), - btf_type_str(ref_t), ref_tname); - if (reg2btf_type != NOT_INIT) - verbose(env, " or %s", reg_type_str(env, reg2btf_type)); - verbose(env, "\n"); - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a verifier-tracked pointer to the expected kernel object type, not a pointer to stack storage or another memory buffer.", - "the kfunc expects a pointer to %s, but this argument is %s and cannot be used as that kernel object pointer", - expected_type, - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - - /* - * If the register does not contain btf id but the argument type is a pointer to - * scalar-only struct, allow verifying it as a fixed size memory. - */ - kf_arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; - fallthrough; - case KF_ARG_PTR_TO_MEM: - if (kf_arg_type & MEM_FIXED_SIZE) { - bool known_memory; - - resolve_ret = btf_resolve_size(btf, ref_t, &type_size); - if (IS_ERR(resolve_ret)) { - verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", - reg_arg_name(env, argno), btf_type_str(ref_t), - ref_tname, PTR_ERR(resolve_ret)); - return -EINVAL; - } - ret = check_mem_reg(env, reg, argno, type_size, BPF_READ | BPF_WRITE, - meta, &known_memory); - if (ret < 0) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - if (known_memory) - bpf_diag_call_arg_fmt( - env, insn_idx, argno, func_name, - "Pass memory with at least the required number of accessible bytes and suitable read and write access.", - "the kfunc expects %u bytes of memory for %s, but the verifier cannot prove that %s provides a readable and writable range of that size", - type_size, expected_type, - bpf_diag_reg_type_plain(env, reg->type)); - else - bpf_diag_call_arg_fmt( - env, insn_idx, argno, func_name, - "Pass stack, map, context, or other verifier-known memory of the expected type and size, not an integer cast to a pointer.", - "the kfunc expects %u bytes of memory for %s, but it is %s and not verifier-known memory", - type_size, expected_type, - bpf_diag_reg_type_plain(env, reg->type)); - return ret; - } - } - break; - case KF_ARG_CONST_MEM_SIZE: - ret = process_const_arg(env, reg, argno, meta); - if (ret < 0) { - if (ret == -EINVAL) - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a compile-time constant or a value the verifier can prove is constant at this call.", - "the kfunc requires this memory size to be a verifier-known constant, but %s is variable on this path", - reg_arg_name(env, argno)); - return ret; - } - fallthrough; - case KF_ARG_MEM_SIZE: - { - struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, i - 1); - struct bpf_reg_state *size_reg = reg; - argno_t buff_argno = argno_from_arg(i); - enum bpf_mem_size_failure failure; - - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an integer scalar length for this memory argument.", - "the kfunc expects a scalar memory size, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - - if (bpf_register_is_null(buff_reg)) - break; - - ret = check_mem_size_reg(env, buff_reg, size_reg, buff_argno, argno, - BPF_READ | BPF_WRITE, true, meta, &failure); - if (ret < 0) { - const char *buff_arg, *size_arg; - - buff_arg = bpf_diag_arg_name(env, buff_argno); - size_arg = bpf_diag_arg_name(env, argno); - verbose(env, "%s and ", reg_arg_name(env, buff_argno)); - verbose(env, "%s memory, len pair leads to invalid memory access\n", - reg_arg_name(env, argno)); - if (failure == BPF_MEM_SIZE_FAIL_MEMORY) { - bpf_diag_call_arg_fmt(env, insn_idx, buff_argno, func_name, - "Pass a stack, map, context, or other verifier-known memory pointer, and keep the paired length within that object.", - "it is the memory pointer in a memory/length pair with %s, but %s does not describe verifier-readable memory for the requested length", - size_arg, buff_arg); - } else if (failure == BPF_MEM_SIZE_FAIL_SIZE) { - if (reg_smin(size_reg) < 0) - bpf_diag_call_arg_fmt( - env, insn_idx, argno, func_name, - "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", - "the memory size in %s may be negative because its signed minimum is %lld", - size_arg, reg_smin(size_reg)); - else - bpf_diag_call_arg_fmt( - env, insn_idx, argno, func_name, - "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", - "the memory size in %s may reach %llu bytes, but variable memory accesses must stay below %u bytes", - size_arg, reg_umax(size_reg), BPF_MAX_VAR_SIZ); - } - return ret; - } - break; - } - case KF_ARG_PTR_TO_CALLBACK: - if (reg->type != PTR_TO_FUNC) { - verbose(env, "%s expected pointer to func\n", reg_arg_name(env, argno)); - return -EINVAL; - } - meta->subprogno = reg->subprogno; - break; - case KF_ARG_PTR_TO_REFCOUNTED_KPTR: - if (!type_is_ptr_alloc_obj(reg->type)) { - verbose(env, "%s is neither owning or non-owning ref\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an owning or non-owning pointer to a BPF-managed object containing a bpf_refcount field.", - "the kfunc expects a pointer to a BPF-managed refcounted object, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - if (!type_is_non_owning_ref(reg->type) && reg_is_referenced(env, reg)) - meta->arg_owning_ref = true; - - rec = reg_btf_record(reg); - if (!rec) { - verifier_bug(env, "Couldn't find btf_record"); - return -EFAULT; - } - - if (rec->refcount_off < 0) { - verbose(env, "%s doesn't point to a type with bpf_refcount field\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - - meta->arg_btf = reg->btf; - meta->arg_btf_id = reg->btf_id; - break; - case KF_ARG_PTR_TO_CONST_STR: - if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "%s doesn't point to a const string\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a constant string pointer that the verifier recognizes, such as a string stored in a read-only map value.", - "the kfunc expects a pointer to a constant string stored in verifier-known memory, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - ret = check_arg_const_str(env, reg, argno); - if (ret) - return ret; - break; - case KF_ARG_PTR_TO_WORKQUEUE: - if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "%s doesn't point to a map value\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - ret = check_map_field_pointer(env, reg, argno, BPF_WORKQUEUE, &meta->map); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_TIMER: - if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "%s doesn't point to a map value\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - ret = process_timer_func(env, reg, argno, &meta->map); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_TASK_WORK: - if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "%s doesn't point to a map value\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - ret = check_map_field_pointer(env, reg, argno, BPF_TASK_WORK, &meta->map); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_IRQ_FLAG: - if (reg->type != PTR_TO_STACK) { - verbose(env, "%s doesn't point to an irq flag on stack\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass the same stack slot used by bpf_local_irq_save() or bpf_res_spin_lock_irqsave().", - "the kfunc expects a stack pointer to an IRQ flag slot, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - ret = process_irq_flag(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_RES_SPIN_LOCK: - { - int flags = PROCESS_RES_LOCK; - - if (in_rbtree_lock_required_cb(env)) { - verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n"); - return -EACCES; - } - - if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s doesn't point to map value or allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - - if (!is_bpf_res_spin_lock_kfunc(meta->func_id)) - return -EFAULT; - if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock] || - meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) - flags |= PROCESS_SPIN_LOCK; - if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] || - meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) - flags |= PROCESS_LOCK_IRQ; - ret = process_spin_lock(env, reg, argno, flags); - if (ret < 0) - return ret; - break; - } - } - } - - return 0; -} - int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, s32 func_id, s16 offset, @@ -13389,27 +14922,32 @@ int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, } /* - * Determine how many bytes a helper accesses through a stack pointer at - * argument position @arg (0-based, corresponding to R1-R5). - * - * Returns: - * > 0 known read access size in bytes - * 0 doesn't read anything directly - * S64_MIN unknown - * < 0 known write access of (-return) bytes + * Describe a helper's stack access through argument @arg (0-based, R1-R5). + * must_write means the verifier destroys prior state throughout size bytes. + * In other words, must_write follows the verifier's model of the + * function's behaviour, which is safe to use for is_state_visited() pruning. */ -s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, - int arg, int insn_idx) +struct arg_access_info +bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, + int arg, int insn_idx) { struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; + struct arg_access_info info = { + .size = U32_MAX, + .may_read = true, + .may_write = true, + }; const struct bpf_func_proto *fn; + enum bpf_access_type access_type; enum bpf_arg_type at; - s64 size; + bool exact_size = true; + u64 size = U32_MAX; if (bpf_get_helper_proto(env, insn->imm, &fn) < 0) - return S64_MIN; + return info; at = fn->arg_type[arg]; + access_type = func_arg_access_type(at); switch (base_type(at)) { case ARG_PTR_TO_MAP_KEY: @@ -13432,6 +14970,10 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn i = aux->const_reg_vals[map_reg]; if (i < env->used_map_cnt) { + /* Bloom-filter peek reads the value buffer. */ + if (!is_key && insn->imm == BPF_FUNC_map_peek_elem && + env->used_maps[i]->map_type == BPF_MAP_TYPE_BLOOM_FILTER) + access_type = BPF_READ; size = is_key ? env->used_maps[i]->key_size : env->used_maps[i]->value_size; goto out; @@ -13441,7 +14983,12 @@ scan_all_maps: * Map pointer is not known at this call site (e.g. different * maps on merged paths). Conservatively return the largest * key_size or value_size across all maps used by the program. + * This is only an upper bound, so it cannot establish a definite + * write. Map-dependent argument types can also turn an output + * into an input, so conservatively retain a read dependency. */ + exact_size = false; + access_type |= BPF_READ; val = 0; for (i = 0; i < env->used_map_cnt; i++) { struct bpf_map *map = env->used_maps[i]; @@ -13457,7 +15004,7 @@ scan_all_maps: } } if (!val) - return S64_MIN; + return info; size = val; goto out; } @@ -13471,18 +15018,12 @@ scan_all_maps: int size_reg = BPF_REG_1 + arg + 1; if (aux->const_reg_mask & BIT(size_reg)) { - size = (s64)aux->const_reg_vals[size_reg]; + size = aux->const_reg_vals[size_reg]; goto out; } - /* - * Size arg is const on each path but differs across merged - * paths. MAX_BPF_STACK is a safe upper bound for reads. - */ - if (at & MEM_UNINIT) - return 0; - return MAX_BPF_STACK; } - return S64_MIN; + /* Preserve access directions even when the extent is unknown. */ + goto out; case ARG_PTR_TO_DYNPTR: size = BPF_DYNPTR_SIZE; break; @@ -13492,71 +15033,86 @@ scan_all_maps: * doesn't access stack. The callback subprog does and it's * analyzed separately. */ - return 0; + return (struct arg_access_info) {}; default: - return S64_MIN; + return info; } out: - /* - * MEM_UNINIT args are write-only: the helper initializes the - * buffer without reading it. - */ - if (at & MEM_UNINIT) - return -size; - return size; + info.size = min_t(u64, size, U32_MAX); + info.may_read = !!(access_type & BPF_READ); + info.may_write = !!(access_type & BPF_WRITE); + info.must_write = info.may_write && exact_size && info.size != U32_MAX; + /* Generic unprivileged outputs retain prior initialization state. */ + if (!env->allow_uninit_stack && arg_type_is_raw_mem(at)) { + info.may_read = true; + info.must_write = false; + } + return info; } /* - * Determine how many bytes a kfunc accesses through a stack pointer at - * argument position @arg (0-based, corresponding to R1-R5). - * - * Returns: - * > 0 known read access size in bytes - * 0 doesn't access memory through that argument (ex: not a pointer) - * S64_MIN unknown - * < 0 known write access of (-return) bytes + * Describe a kfunc's stack access through argument slot @arg (0-based). + * As for helpers, must_write describes destruction of prior verifier state, + * rather than a guarantee that the callee writes every byte at runtime. */ -s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, - int arg, int insn_idx) +struct arg_access_info +bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, + int arg, int insn_idx) { struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; + struct arg_access_info info = { + .size = U32_MAX, + .may_read = true, + .may_write = true, + }; struct bpf_call_arg_meta meta; const struct btf_param *args; const struct btf_type *t, *ref_t; const struct btf *btf; - u32 nargs, type_size; - s64 size; + u32 i, slot, nargs, type_size; + u64 size; if (bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta) < 0) - return S64_MIN; + return info; btf = meta.btf; args = btf_params(meta.func_proto); nargs = btf_type_vlen(meta.func_proto); - if (arg >= nargs) - return 0; - t = btf_type_skip_modifiers(btf, args[arg].type, NULL); + /* + * @arg is an argument slot and a 16-byte parameter takes two of them, + * so walk the parameters to find the one that starts at this slot. A + * slot holding the upper eightbyte of such a parameter belongs to no + * pointer, and neither does a slot past the last parameter. + */ + for (i = 0, slot = 0; i < nargs && slot < arg; i++) + slot += btf_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL)); + if (i >= nargs || slot != arg) + return (struct arg_access_info) {}; + + t = btf_type_skip_modifiers(btf, args[i].type, NULL); if (!btf_type_is_ptr(t)) - return 0; + return (struct arg_access_info) {}; /* dynptr: fixed 16-byte on-stack representation */ - if (is_kfunc_arg_dynptr(btf, &args[arg])) { + if (is_kfunc_arg_dynptr(btf, &args[i])) { size = BPF_DYNPTR_SIZE; goto out; } - /* ptr + __sz/__szk pair: size is in the next register */ - if (arg + 1 < nargs && - (btf_param_match_suffix(btf, &args[arg + 1], "__sz") || - btf_param_match_suffix(btf, &args[arg + 1], "__szk"))) { + /* ptr + __sz/__szk pair: the size follows the pointer */ + if (i + 1 < nargs && + (btf_param_match_suffix(btf, &args[i + 1], "__sz") || + btf_param_match_suffix(btf, &args[i + 1], "__szk"))) { int size_reg = BPF_REG_1 + arg + 1; - if (aux->const_reg_mask & BIT(size_reg)) { - size = (s64)aux->const_reg_vals[size_reg]; + if (size_reg <= MAX_BPF_FUNC_REG_ARGS && + (aux->const_reg_mask & BIT(size_reg))) { + size = aux->const_reg_vals[size_reg]; goto out; } - return MAX_BPF_STACK; + /* Unknown size: the read may extend anywhere up to the frame top. */ + return info; } /* fixed-size pointed-to type: resolve via BTF */ @@ -13566,14 +15122,99 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * goto out; } - return S64_MIN; + return info; out: + info.size = min_t(u64, size, U32_MAX); /* KF_ITER_NEW kfuncs initialize the iterator state at arg 0 */ if (arg == 0 && meta.kfunc_flags & KF_ITER_NEW) - return -size; - if (is_kfunc_arg_uninit(btf, &args[arg])) - return -size; - return size; + info.may_read = false; + if (is_kfunc_arg_uninit(btf, &args[i]) && + (is_kfunc_arg_dynptr(btf, &args[i]) || env->allow_uninit_stack)) + info.may_read = false; + info.must_write = !info.may_read && info.size != U32_MAX; + return info; +} + +/* + * Called before dead code elimination, some subprograms may be genuinely unreachable. + * Suppress diagnostics so callers can use a conservative result on error. + * The main pass would report errors for reachable subprograms. + */ +static bool try_btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) +{ + struct bpf_prog *prog = env->prog; + u32 log_level = env->log.level; + int err; + + if (!prog->aux->func_info || !prog->aux->func_info[subprog].type_id) + return false; + + env->log.level = 0; + err = btf_prepare_func_args(env, subprog); + env->log.level = log_level; + return !err; +} + +/* Prepare argument metadata for liveness; the main program is handled separately. */ +static void prepare_subprog_args(struct bpf_verifier_env *env) +{ + int i; + + for (i = 1; i < env->subprog_cnt; i++) { + if (!try_btf_prepare_func_args(env, i) && bpf_subprog_is_global(env, i)) + /* Keep all argument registers live if the prototype is unavailable. */ + env->subprog_info[i].arg_slot_cnt = MAX_BPF_FUNC_REG_ARGS; + } +} + +/* + * Derive stack effects for a global subprog's argument slot @arg (0-based) + * from its declared type, matching the main pass's argument checks. + */ +struct arg_access_info +bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, + int arg, int insn_idx) +{ + struct arg_access_info info = { + .size = U32_MAX, + .may_read = true, + .may_write = true, + }; + struct bpf_subprog_info *sub; + int callee; + + callee = bpf_find_subprog(env, insn_idx + insn->imm + 1); + if (callee < 0) + return info; + sub = subprog_info(env, callee); + if (!try_btf_prepare_func_args(env, callee)) + return info; + + switch (base_type(sub->args[arg].arg_type)) { + case ARG_UNUSED: /* Not an argument slot. */ + case ARG_SCALAR: /* A scalar, the callee can't dereference it. */ + return (struct arg_access_info) {}; + case ARG_PTR_TO_BTF_ID: + /* + * Trusted arguments cannot be stack pointers. + * Untrusted arguments use ARG_IGNORE: + * probe reads need no caller stack state, writes are prohibited. + */ + return (struct arg_access_info) {}; + case ARG_PTR_TO_MEM: + /* mem_size is zero for untrusted pointers of unknown size. */ + if (sub->args[arg].mem_size) { + info.size = sub->args[arg].mem_size; + info.must_write = true; + } + return info; + case ARG_PTR_TO_DYNPTR: + /* The callee may adjust the view, but cannot reinitialize the dynptr. */ + info.size = BPF_DYNPTR_SIZE; + return info; + default: + return info; + } } /* check special kfuncs and return: @@ -13650,7 +15291,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { - if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { + if (!btf_type_is_scalar_struct(env, ret_btf, ret_t)) { verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); return -EINVAL; } @@ -13687,12 +15328,12 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg struct btf_field *field = meta->arg_rbtree_root.field; mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root); - } else if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx])) { mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED; regs[BPF_REG_0].btf = desc_btf; regs[BPF_REG_0].btf_id = meta->ret_btf_id; - } else if (meta->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_rdonly_cast])) { ret_t = btf_type_by_id(desc_btf, meta->arg_constant.value); if (!ret_t) { verbose(env, "Unknown type ID %lld passed to kfunc bpf_rdonly_cast\n", @@ -13712,8 +15353,8 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg "kfunc bpf_rdonly_cast type ID argument must be of a struct or void\n"); return -EINVAL; } - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice] || - meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice_rdwr]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice_rdwr])) { enum bpf_type_flag type_flag = get_dynptr_type_flag(meta->dynptr.type); mark_reg_known_zero(env, regs, BPF_REG_0); @@ -13728,7 +15369,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg /* PTR_MAYBE_NULL will be added when is_kfunc_ret_null is checked */ regs[BPF_REG_0].type = PTR_TO_MEM | type_flag; - if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice]) { + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice])) { regs[BPF_REG_0].type |= MEM_RDONLY; } else { /* this will set env->seen_direct_write to true */ @@ -13764,7 +15405,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn_aux_data *insn_aux; const char *operation; int err, insn_idx = *insn_idx_p; - u32 i, nargs, ptr_type_id; + u32 i, proto_slots, ptr_type_id, ret_nregs = 1; struct bpf_kfunc_desc *desc; struct btf *desc_btf; int id; @@ -13834,12 +15475,30 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return -EACCES; } + if (is_kfunc_perfmon(&meta) && !env->allow_ptr_leaks) { + verbose(env, "%s is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", + func_name); + operation = bpf_diag_fmt(env, "kfunc %s", func_name); + bpf_diag_policy(env, insn_idx, operation, "the kfunc requires CAP_PERFMON", + "Load the program with CAP_PERFMON, or avoid the kfunc."); + return -EPERM; + } + sleepable = bpf_is_kfunc_sleepable(&meta); - if (sleepable && !in_sleepable(env)) { - verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); + if (sleepable && !in_sleepable_context(env)) { + const char *suggestion; + + if (in_sleepable(env)) { + verbose(env, "kernel func %s is sleepable within %s\n", + func_name, non_sleepable_context_description(env)); + suggestion = "Move the kfunc call outside the critical section, or use a non-sleepable kfunc."; + } else { + verbose(env, "program must be sleepable to call sleepable kfunc %s\n", + func_name); + suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable kfunc."; + } operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name); - bpf_diag_ctx_forbidden(env, insn_idx, operation, - "Mark the program sleepable if the program type allows it, or use a non-sleepable kfunc."); + bpf_diag_ctx_forbidden(env, insn_idx, operation, suggestion); return -EACCES; } @@ -13848,10 +15507,14 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn_aux->non_sleepable = true; /* Check the arguments */ - err = check_kfunc_args(env, &meta, insn_idx); + err = check_func_args(env, &meta, insn_idx); if (err < 0) return err; + err = mark_raw_stack(env, &meta, insn_idx); + if (err) + return err; + if ((is_bpf_obj_drop_kfunc(meta.func_id) || is_bpf_percpu_obj_drop_kfunc(meta.func_id)) && (is_tracing_prog_type(prog_type) || /* is_tracing_prog_type() for now doesn't cover non-iterator tracing progs. */ @@ -13897,6 +15560,16 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } } + if (is_call_rcu_kfunc(meta.func_id)) { + err = push_callback_call(env, insn, insn_idx, meta.subprogno, + set_rcu_callback_state); + if (err) { + verbose(env, "kfunc %s#%d failed callback verification\n", + func_name, meta.func_id); + return err; + } + } + rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta); rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta); @@ -13939,15 +15612,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, invalidate_rcu_protected_refs(env); } - if (sleepable && !in_sleepable_context(env)) { - verbose(env, "kernel func %s is sleepable within %s\n", - func_name, non_sleepable_context_description(env)); - operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name); - bpf_diag_ctx_forbidden(env, insn_idx, operation, - "Move the kfunc call outside the critical section, or use a non-sleepable kfunc."); - return -EACCES; - } - if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) { verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n"); return -EACCES; @@ -14018,10 +15682,63 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } if (btf_type_is_scalar(t)) { - mark_reg_unknown(env, regs, BPF_REG_0); + ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] || meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); + } else if (btf_type_is_struct(t)) { + struct btf_member_path path = {}; + const char *member_note = ""; + + /* + * The returned struct comes back as raw register bits modeled + * as an unknown scalar, so a pointer member would be laundered + * into a scalar and escape provenance and reference tracking. + * Only scalars and arena pointers are allowed; see + * btf_validate_return_type() in btf.c for why an arena pointer + * is safe here. + */ + if (!btf_struct_member_walk(env, desc_btf, t, + BTF_MEMBER_SCALAR | BTF_MEMBER_ARENA_PTR, 0, &path)) { + verbose(env, + "kernel function %s returns %s %s that is not composed of scalars or arena pointers\n", + func_name, btf_type_str(t), + btf_name_by_offset(desc_btf, t->name_off)); + if (path.too_deep) { + member_note = bpf_diag_fmt( + env, " It nests structs more than %d levels deep.", + BTF_MEMBER_MAX_DEPTH); + } else if (path.depth) { + char bad_name[BTF_MEMBER_PATH_LEN]; + + btf_member_path_str(desc_btf, &path, bad_name, sizeof(bad_name)); + if (!path.bad_type) { + verbose(env, "member '%s' is a zero-length array\n", + bad_name); + member_note = bpf_diag_fmt( + env, " Its member '%s' is a zero-length array.", + bad_name); + } else { + verbose(env, "member '%s' has type %s\n", bad_name, + btf_type_str(path.bad_type)); + member_note = bpf_diag_fmt( + env, + " Its member '%s' is %s, not a scalar or an arena pointer.", + bad_name, btf_type_str(path.bad_type)); + } + } + bpf_diag_program_structure( + env, insn_idx, "unsupported kernel function return type", + "Call a kernel function that returns only scalars or arena pointers by value.", + "%s() returns %s %s by value.%s " + "Only kfuncs returning scalar values or arena pointers, or " + "structures composed of scalar values or arena pointers are " + "supported.", + func_name, btf_type_str(t), + btf_name_by_offset(desc_btf, t->name_off), member_note); + return -EINVAL; + } + ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); } else if (btf_type_is_ptr(t)) { ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf); @@ -14134,14 +15851,24 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } } - if (bpf_is_kfunc_pkt_changing(&meta)) + if (is_kfunc_dynptr_write(&meta) ? + dynptr_write_changes_pkt_data(meta.dynptr_write_type) : + bpf_is_kfunc_pkt_changing(&meta)) clear_all_pkt_pointers(env); - nargs = btf_type_vlen(meta.func_proto); - if (nargs > MAX_BPF_FUNC_REG_ARGS) { + /* + * These kfuncs write to the head of the skb in place if what they write + * to is linear. Like a direct write that takes bpf_unclone_prologue(). + */ + if (is_kfunc_dynptr_write(&meta) && + dynptr_write_changes_pkt_data(meta.dynptr_write_type)) + may_access_direct_pkt_data(env, NULL, BPF_WRITE); + + proto_slots = kfunc_abi_slots(&desc->func_model); + if (proto_slots > MAX_BPF_FUNC_REG_ARGS) { struct bpf_func_state *caller = cur_func(env); struct bpf_subprog_info *caller_info = &env->subprog_info[caller->subprogno]; - u16 out_stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS; + u16 out_stack_arg_cnt = proto_slots - MAX_BPF_FUNC_REG_ARGS; u16 stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + out_stack_arg_cnt; if (stack_arg_cnt > caller_info->stack_arg_cnt) @@ -14152,7 +15879,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, * Record R0 before process_iter_next_call() snapshots the alternate * iterator path's diagnostic position. */ - bpf_diag_mod_end(env); + if (ret_nregs == 1) + bpf_diag_mod_end(env); if (bpf_is_iter_next_kfunc(&meta)) { err = process_iter_next_call(env, insn_idx, &meta); @@ -14229,7 +15957,8 @@ enum { REASON_STACK = -5, }; -static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, +static int retrieve_ptr_limit(const struct bpf_verifier_env *env, + const struct bpf_reg_state *ptr_reg, u32 *alu_limit, bool mask_to_left) { u32 max = 0, ptr_limit = 0; @@ -14241,7 +15970,7 @@ static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, * offset where we would need to deal with min/max bounds is * currently prohibited for unprivileged. */ - max = MAX_BPF_STACK + mask_to_left; + max = env->stack_limit + mask_to_left; ptr_limit = -ptr_reg->var_off.value; break; case PTR_TO_MAP_VALUE: @@ -14361,7 +16090,7 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env, (opcode == BPF_SUB && !off_is_neg); } - err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left); + err = retrieve_ptr_limit(env, ptr_reg, &alu_limit, info->mask_to_left); if (err < 0) return err; @@ -14491,7 +16220,7 @@ static int check_stack_access_for_ptr_arithmetic( return -EACCES; } - if (off >= 0 || off < -MAX_BPF_STACK) { + if (off >= 0 || off < -(int)env->stack_limit) { verbose(env, "R%d stack pointer arithmetic goes out of range, " "prohibited for !root; off=%d\n", regno, off); return -EACCES; @@ -14532,6 +16261,24 @@ static int sanitize_check_bounds(struct bpf_verifier_env *env, return 0; } +static void scalar_step_mul(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) +{ + if (tnum_is_const(src_reg->var_off)) + scalar_step_scale(dst_reg, src_reg->var_off.value); + else + reg_step_reset(dst_reg); +} + +static void scalar_step_lsh(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) +{ + u64 amount = src_reg->var_off.value; + + if (tnum_is_const(src_reg->var_off) && amount < 64) + scalar_step_scale(dst_reg, 1ULL << amount); + else + reg_step_reset(dst_reg); +} + /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. * Caller should also handle BPF_MOV case separately. * If we return -EACCES, caller may want to try again treating pointer as a @@ -14552,8 +16299,14 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn u32 dst = insn->dst_reg; const char *reason; int ret, bounds_ret; + bool to_scalar; dst_reg = ®s[dst]; + /* + * Only += and -= keep a pointer. Any other op makes a number of it, + * which is fine when the program may see values of pointers anyway. + */ + to_scalar = opcode != BPF_ADD && opcode != BPF_SUB && env->allow_ptr_leaks; if ((known && (smin_val != smax_val || umin_val != umax_val)) || smin_val > smax_val || umin_val > umax_val) { @@ -14564,7 +16317,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn return 0; } - if (BPF_CLASS(insn->code) != BPF_ALU64) { + if (BPF_CLASS(insn->code) != BPF_ALU64 && !to_scalar) { /* 32-bit ALU ops on pointers produce (meaningless) scalars */ if (opcode == BPF_SUB && env->allow_ptr_leaks) { __mark_reg_unknown(env, dst_reg); @@ -14630,6 +16383,11 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn return -EACCES; } + if (to_scalar) { + __mark_reg_unknown(env, dst_reg); + return 0; + } + /* For 'scalar += pointer', dst_reg inherits the complete pointer * register state. Individual fields may be adjusted later by pointer * arithmetic. Callers guarantee that below does not overwrite off_reg. @@ -14689,6 +16447,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn * added into the variable offset, and we copy the fixed offset * from ptr_reg. */ + scalar_step_add(dst_reg, ptr_reg, off_reg); dst_reg->r64 = cnum64_add(ptr_reg->r64, off_reg->r64); dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); dst_reg->raw = ptr_reg->raw; @@ -14750,6 +16509,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn if ((!known && smin_val < 0) || dst_reg->range < 0) memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); } + reg_step_reset(dst_reg); break; case BPF_AND: case BPF_OR: @@ -14789,7 +16549,15 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn "Tighten the scalar bounds before the arithmetic so the resulting pointer remains within the allowed range."); return -EINVAL; } - reg_bounds_sync(dst_reg); + /* + * A packet pointer that keeps its id or range is checked against a + * range set from the checked pointer's umax, so var_off must not tighten + * its umax. r32 must still match var_off for reg_bounds_sanity_check(). + */ + if (reg_is_pkt_pointer(dst_reg) && (known || dst_reg->range > 0)) + __update_reg32_bounds(dst_reg); + else + reg_bounds_sync(dst_reg); bounds_ret = sanitize_check_bounds(env, insn, dst_reg); if (bounds_ret == -EACCES) return bounds_ret; @@ -15449,6 +17217,7 @@ static void scalar_byte_swap(struct bpf_reg_state *dst_reg, struct bpf_insn *ins * Bounds will be re-derived from the new tnum later. */ __mark_reg_unbounded(dst_reg); + reg_step_reset(dst_reg); } /* For bswap16/32, truncate dst register to match the swapped size */ if (insn->imm == 16 || insn->imm == 32) @@ -15575,6 +17344,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, */ switch (opcode) { case BPF_ADD: + scalar_step_add(dst_reg, dst_reg, &src_reg); scalar32_min_max_add(dst_reg, &src_reg); scalar_min_max_add(dst_reg, &src_reg); dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); @@ -15583,6 +17353,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar32_min_max_sub(dst_reg, &src_reg); scalar_min_max_sub(dst_reg, &src_reg); dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); + reg_step_reset(dst_reg); break; case BPF_NEG: env->fake_reg[0] = *dst_reg; @@ -15590,8 +17361,10 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar32_min_max_sub(dst_reg, &env->fake_reg[0]); scalar_min_max_sub(dst_reg, &env->fake_reg[0]); dst_reg->var_off = tnum_neg(env->fake_reg[0].var_off); + reg_step_reset(dst_reg); break; case BPF_MUL: + scalar_step_mul(dst_reg, &src_reg); dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); scalar32_min_max_mul(dst_reg, &src_reg); scalar_min_max_mul(dst_reg, &src_reg); @@ -15612,6 +17385,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar_min_max_sdiv(dst_reg, &src_reg); else scalar_min_max_udiv(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_MOD: /* BPF mod specification: x % 0 = x */ @@ -15627,6 +17401,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar_min_max_smod(dst_reg, &src_reg); else scalar_min_max_umod(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_AND: if (tnum_is_const(src_reg.var_off)) { @@ -15637,6 +17412,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); scalar32_min_max_and(dst_reg, &src_reg); scalar_min_max_and(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_OR: if (tnum_is_const(src_reg.var_off)) { @@ -15647,13 +17423,16 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); scalar32_min_max_or(dst_reg, &src_reg); scalar_min_max_or(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_XOR: dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); scalar32_min_max_xor(dst_reg, &src_reg); scalar_min_max_xor(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_LSH: + scalar_step_lsh(dst_reg, &src_reg); if (alu32) scalar32_min_max_lsh(dst_reg, &src_reg); else @@ -15664,15 +17443,18 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar32_min_max_rsh(dst_reg, &src_reg); else scalar_min_max_rsh(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_ARSH: if (alu32) scalar32_min_max_arsh(dst_reg, &src_reg); else scalar_min_max_arsh(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_END: scalar_byte_swap(dst_reg, insn); + reg_step_reset(dst_reg); break; default: break; @@ -15704,6 +17486,7 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); + struct bpf_insn_aux_data *aux = cur_aux(env); u8 opcode = BPF_OP(insn->code); int err; @@ -15715,13 +17498,24 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, /* Case where at least one operand is an arena. */ if (dst_reg->type == PTR_TO_ARENA || (src_reg && src_reg->type == PTR_TO_ARENA)) { - struct bpf_insn_aux_data *aux = cur_aux(env); if (dst_reg->type != PTR_TO_ARENA) *dst_reg = *src_reg; if (BPF_CLASS(insn->code) == BPF_ALU64) { /* + * Only arena pointers set needs_zext, but doing so + * modifies the instruction at fixup time to an ALU32 + * and makes it unsuitable for 64-bit scalar args. We + * prevent zext from being set if the instruction has + * been previously called with non-arena registers. + */ + if (aux->prevent_zext) { + verbose(env, "same insn cannot be used with and without arena pointer\n"); + return -EINVAL; + } + + /* * 32-bit operations zero upper bits automatically. * 64-bit operations need to be converted to 32. */ @@ -15733,6 +17527,16 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, return 0; } + /* Prevent the instruction from being used with arena pointers (see above). */ + if (env->prog->aux->arena && BPF_CLASS(insn->code) == BPF_ALU64) { + if (aux->needs_zext) { + verbose(env, "same insn cannot be used with and without arena pointer\n"); + return -EINVAL; + } + + aux->prevent_zext = true; + } + if (dst_reg->type != SCALAR_VALUE) ptr_reg = dst_reg; @@ -15862,6 +17666,112 @@ clear_id: return 0; } +/* + * Many checks done by this function are quite conservative. + * This is because main verification pass does not maintain + * enough information to track object identities for some + * of the interesting types, e.g. PTR_TO_MEM. + */ +bool bpf_same_memory_origin(const struct bpf_reg_state *reg_a, + const struct bpf_reg_state *reg_b) +{ + if (reg_a == reg_b) + return true; + /* Require matching flags and base types. */ + if (reg_a->type != reg_b->type) + return false; + /* NULL can't be compared to some base+offset pointer. */ + if (type_may_be_null(reg_a->type)) + return false; + + switch (base_type(reg_a->type)) { + case PTR_TO_STACK: + return reg_a->frameno == reg_b->frameno; + case PTR_TO_MAP_VALUE: + if (reg_a->map_ptr != reg_b->map_ptr || reg_a->map_uid != reg_b->map_uid) + return false; + if (reg_a->id && reg_a->id == reg_b->id) + return true; + /* + * A plain single-element array has one stable value address, + * but not when BPF_F_INNER_MAP is set, see arraymap.c:array_map_meta_equal(). + */ + if (reg_a->map_ptr->map_type == BPF_MAP_TYPE_ARRAY && + reg_a->map_ptr->max_entries == 1 && + !(reg_a->map_ptr->map_flags & BPF_F_INNER_MAP)) + return true; + /* + * The rules above can be simplified / relaxed if: + * - fresh IDs would always be assigned for map-value lookups; + * - direct map value loads would always have and ID of zero. + */ + return false; + case PTR_TO_MEM: + case PTR_TO_BUF: + return reg_a->id && reg_a->id == reg_b->id; + + default: + return false; + } +} + +int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + struct cnum64 r64, struct cnum32 r32, u16 base, u16 step) +{ + reg->r64 = r64; + reg->r32 = r32; + reg->step = step; + reg->base = base; + reg->var_off = tnum_unknown; + reg_bounds_sync(reg); /* this should infer the tnum alignment */ + return reg_bounds_sanity_check(env, reg, "bpf_set_reg_range"); +} + +/* + * Checks if pointer and packet metadata matches between two registers. + * For use with bpf_reg_union(). + */ +bool bpf_reg_union_compatible(const struct bpf_reg_state *a, const struct bpf_reg_state *b) +{ + if (a->type != b->type) + return false; + if (a->type == SCALAR_VALUE) + return true; + return !memcmp(a, b, offsetof(struct bpf_reg_state, var_off)) && + a->id == b->id && a->parent_id == b->parent_id && a->map_uid == b->map_uid; +} + +/* + * Extends acc's scalar range to include src's scalar range. + * Caller must ensure that other kinds of metadata is compatible. + * Caller must clear acc's scalar ID. + */ +int bpf_reg_union(struct bpf_verifier_env *env, struct bpf_reg_state *acc, + const struct bpf_reg_state *src) +{ + u16 base, step; + + if (!bpf_reg_union_compatible(acc, src)) { + verifier_bug(env, "union of incompatible registers"); + return -EFAULT; + } + acc->r64 = cnum64_union(acc->r64, src->r64); + acc->r32 = cnum32_union(acc->r32, src->r32); + acc->var_off = tnum_union(acc->var_off, src->var_off); + + /* Retain a common congruence if the bases agree modulo the gcd. */ + step = gcd(acc->step, src->step); + base = acc->base % step; + if (base != src->base % step) { + reg_step_reset(acc); + } else { + acc->base = base; + acc->step = step; + } + reg_bounds_sync(acc); + return reg_bounds_sanity_check(env, acc, "bpf_reg_union"); +} + /* check validity of 32-bit and 64-bit arithmetic operations */ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) { @@ -16131,8 +18041,6 @@ static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2, u8 opcode, bool is_jmp32); -static u8 rev_opcode(u8 opcode); - /* * Learn more information about live branches by simulating refinement on both branches. * regs_refine_cond_op() is sound, so producing ill-formed register bounds for the branch means @@ -16141,7 +18049,7 @@ static u8 rev_opcode(u8 opcode); static int simulate_both_branches_taken(struct bpf_verifier_env *env, u8 opcode, bool is_jmp32) { /* Fallthrough (FALSE) branch */ - regs_refine_cond_op(&env->false_reg1, &env->false_reg2, rev_opcode(opcode), is_jmp32); + regs_refine_cond_op(&env->false_reg1, &env->false_reg2, bpf_rev_opcode(opcode), is_jmp32); reg_bounds_sync(&env->false_reg1); reg_bounds_sync(&env->false_reg2); /* @@ -16327,7 +18235,7 @@ static int is_scalar_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_s return simulate_both_branches_taken(env, opcode, is_jmp32); } -static int flip_opcode(u32 opcode) +int bpf_flip_opcode(u32 opcode) { /* How can we transform "a <op> b" into "b <op> a"? */ static const u8 opcode_flip[16] = { @@ -16358,7 +18266,7 @@ static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, pkt = dst_reg; } else if (dst_reg->type == PTR_TO_PACKET_END) { pkt = src_reg; - opcode = flip_opcode(opcode); + opcode = bpf_flip_opcode(opcode); } else { return -1; } @@ -16413,7 +18321,7 @@ static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *r /* arrange that reg2 is a scalar, and reg1 is a pointer */ if (!is_reg_const(reg2, is_jmp32)) { - opcode = flip_opcode(opcode); + opcode = bpf_flip_opcode(opcode); swap(reg1, reg2); } /* and ensure that reg2 is a constant */ @@ -16447,7 +18355,7 @@ static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *r /* Opcode that corresponds to a *false* branch condition. * E.g., if r1 < r2, then reverse (false) condition is r1 >= r2 */ -static u8 rev_opcode(u8 opcode) +u8 bpf_rev_opcode(u8 opcode) { switch (opcode) { case BPF_JEQ: return BPF_JNE; @@ -16482,7 +18390,7 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state case BPF_JGT: case BPF_JSGE: case BPF_JSGT: - opcode = flip_opcode(opcode); + opcode = bpf_flip_opcode(opcode); swap(reg1, reg2); break; default: @@ -16540,7 +18448,7 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state * this restriction. I.e., !(r1 & 0x3) means neither bit 0 nor * bit 1 is set, which we can readily use in adjustments. */ - if (!is_power_of_2(val)) + if (hweight64(val) != 1) break; if (is_jmp32) { t = tnum_or(tnum_subreg(reg1->var_off), tnum_const(val)); @@ -16549,7 +18457,7 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state reg1->var_off = tnum_or(reg1->var_off, tnum_const(val)); } break; - case BPF_JSET | BPF_X: /* reverse of BPF_JSET, see rev_opcode() */ + case BPF_JSET | BPF_X: /* reverse of BPF_JSET, see bpf_rev_opcode() */ if (!is_reg_const(reg2, is_jmp32)) swap(reg1, reg2); if (!is_reg_const(reg2, is_jmp32)) @@ -16559,6 +18467,7 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state * violations if we're on a dead branch. */ __mark_reg_unbounded(reg1); + reg_step_reset(reg1); if (is_jmp32) { t = tnum_and(tnum_subreg(reg1->var_off), tnum_const(~val)); reg1->var_off = tnum_with_subreg(reg1->var_off, t); @@ -16826,10 +18735,12 @@ static void collect_linked_regs(struct bpf_verifier_env *env, reg = &func->regs[j]; __collect_linked_regs(linked_regs, reg, id, i, j, true); } - for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { - if (!bpf_is_spilled_reg(&func->stack[j])) + for (j = 0; j < bpf_stack_nr_slots(func); j++) { + struct bpf_stack_state *ss = bpf_stack_slot(func, j); + + if (!bpf_is_spilled_reg(ss)) continue; - reg = &func->stack[j].spilled_ptr; + reg = &ss->spilled_ptr; __collect_linked_regs(linked_regs, reg, id, i, j, false); } } @@ -16849,7 +18760,7 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s for (i = 0; i < linked_regs->cnt; ++i) { e = &linked_regs->entries[i]; reg = e->is_reg ? &vstate->frame[e->frameno]->regs[e->regno] - : &vstate->frame[e->frameno]->stack[e->spi].spilled_ptr; + : &bpf_stack_slot(vstate->frame[e->frameno], e->spi)->spilled_ptr; if (reg->type != SCALAR_VALUE || reg == known_reg) continue; if ((reg->id & ~BPF_ADD_CONST) != (known_reg->id & ~BPF_ADD_CONST)) @@ -16879,6 +18790,7 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s reg->delta = saved_off; reg->id = saved_id; + scalar_step_add(reg, reg, &fake_reg); scalar32_min_max_add(reg, &fake_reg); scalar_min_max_add(reg, &fake_reg); reg->var_off = tnum_add(reg->var_off, fake_reg.var_off); @@ -16914,6 +18826,23 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, return -EINVAL; } + if (bpf_is_unwind_goto(insn)) { + if (!env->cur_state->unwound) + return 0; + env->cur_state->unwound = false; + /* + * unwind_exit changed its return address, so the return is + * mispredicted and what follows unwind_goto runs speculatively. + */ + if (!env->bypass_spec_v1) { + err = sanitize_speculative_path(env, NULL, *insn_idx + 1, *insn_idx); + if (err) + return err; + } + *insn_idx += insn->off; + return 0; + } + if (opcode == BPF_JCOND) { struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st; int idx = *insn_idx; @@ -16967,7 +18896,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, } if (insn_flags) { - err = bpf_push_jmp_history(env, this_branch, insn_flags, 0, 0, 0); + err = bpf_push_jmp_history(env, this_branch, insn_flags, 0, 0, NULL, 0); if (err) return err; } @@ -17037,7 +18966,10 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, * if parent state is created. */ if (linked_regs.cnt > 1) { - err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, linked_regs_pack(&linked_regs)); + u16 packed[LINKED_REGS_MAX]; + + linked_regs_pack(&linked_regs, packed); + err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, packed, linked_regs.cnt); if (err) return err; } @@ -17602,37 +19534,54 @@ enforce_retval: return 0; } -static int check_global_subprog_return_code(struct bpf_verifier_env *env) +static int check_global_ret_scalar_reg(struct bpf_verifier_env *env, u32 regno) { - struct bpf_reg_state *reg = reg_state(env, BPF_REG_0); - struct bpf_func_state *cur_frame = cur_func(env); + struct bpf_reg_state *reg; int err; - if (subprog_returns_void(env, cur_frame->subprogno)) - return 0; - - err = check_reg_arg(env, BPF_REG_0, SRC_OP); + err = check_reg_arg(env, regno, SRC_OP); if (err) return err; /* Pointers to arena are safe to pass between subprograms. */ - if (is_arena_reg(env, BPF_REG_0)) + if (is_arena_reg(env, regno)) return 0; - if (is_pointer_value(env, BPF_REG_0)) { - verbose(env, "R%d leaks addr as return value\n", BPF_REG_0); + if (is_pointer_value(env, regno)) { + verbose(env, "R%d leaks addr as return value\n", regno); return -EACCES; } + reg = reg_state(env, regno); if (reg->type != SCALAR_VALUE) { - verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", - reg_type_str(env, reg->type)); + verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n", + regno, reg_type_str(env, reg->type)); return -EINVAL; } return 0; } +static int check_global_subprog_return_code(struct bpf_verifier_env *env) +{ + struct bpf_func_state *cur_frame = cur_func(env); + u32 subprog = cur_frame->subprogno; + u32 i, nregs; + int err; + + if (subprog_returns_void(env, subprog)) + return 0; + + nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; + for (i = 0; i < nregs; i++) { + err = check_global_ret_scalar_reg(env, ret_regs[i]); + if (err) + return err; + } + + return 0; +} + /* Bitmask with 1s for all caller saved registers */ #define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1) @@ -17674,11 +19623,11 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, (bpf_verifier_inlines_helper_call(env, call->imm) || bpf_jit_inlines_helper_call(call->imm)); cs->is_void = fn->ret_type == RET_VOID; - cs->num_params = 0; + cs->arg_slot_cnt = 0; for (i = 0; i < ARRAY_SIZE(fn->arg_type); ++i) { - if (fn->arg_type[i] == ARG_DONTCARE) + if (fn->arg_type[i] == ARG_UNUSED) break; - cs->num_params++; + cs->arg_slot_cnt++; } return true; } @@ -17690,7 +19639,7 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, if (err < 0) /* error would be reported later */ return false; - cs->num_params = btf_type_vlen(meta.func_proto); + cs->arg_slot_cnt = btf_proto_slots(meta.btf, meta.func_proto); cs->fastcall = meta.kfunc_flags & KF_FASTCALL; cs->is_void = btf_type_is_void(btf_type_by_id(meta.btf, meta.func_proto->type)); return true; @@ -17799,7 +19748,7 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, * - includes R1-R5 if corresponding parameter has is described * in the function prototype. */ - clobbered_regs_mask = GENMASK(cs.num_params, cs.is_void ? 1 : 0); + clobbered_regs_mask = GENMASK(cs.arg_slot_cnt, cs.is_void ? 1 : 0); /* e.g. if helper call clobbers r{0,1}, expect r{2,3,4,5} in the pattern */ expected_regs_mask = ~clobbered_regs_mask & ALL_CALLER_SAVED_REGS; @@ -18165,10 +20114,25 @@ static int indirect_jump_min_max_index(struct bpf_verifier_env *env, return 0; } +static int reject_gotox_out_of_subprog(struct bpf_verifier_env *env, u32 target, + u32 subprog_start, u32 subprog_end) +{ + verbose(env, "indirect jump from insn %d to %u leaves the subprog [%u,%u)\n", + env->insn_idx, target, subprog_start, subprog_end); + bpf_diag_program_structure( + env, env->insn_idx, "indirect jump leaves subprogram", + "Keep every reachable jump-table target inside the subprogram of the indirect jump.", + "Instruction %d can jump indirectly to instruction %u, which is outside its own subprogram [%u,%u).", + env->insn_idx, target, subprog_start, subprog_end); + return -EINVAL; +} + /* gotox *dst_reg */ static int check_indirect_jump(struct bpf_verifier_env *env, struct bpf_insn *insn) { struct bpf_verifier_state *other_branch; + struct bpf_subprog_info *subprog; + u32 subprog_start, subprog_end; struct bpf_reg_state *dst_reg; struct bpf_map *map; u32 min_index, max_index; @@ -18212,6 +20176,24 @@ static int check_indirect_jump(struct bpf_verifier_env *env, struct bpf_insn *in return -EINVAL; } + subprog = bpf_find_containing_subprog(env, env->insn_idx); + if (verifier_bug_if(!subprog, env, "no subprog contains insn %d", env->insn_idx)) + return -EFAULT; + subprog_start = subprog->start; + subprog_end = (subprog + 1)->start; + + /* + * Maps are confined to a subprog, see compute_subprog_jts(), and the + * targets are sorted and free of duplicates, see sort_insn_array_uniq(), + * so either all of them belong to this subprog or none of them does. + */ + if (env->gotox_tmp_buf->items[0] < subprog_start) + return reject_gotox_out_of_subprog(env, env->gotox_tmp_buf->items[0], + subprog_start, subprog_end); + if (env->gotox_tmp_buf->items[n - 1] >= subprog_end) + return reject_gotox_out_of_subprog(env, env->gotox_tmp_buf->items[n - 1], + subprog_start, subprog_end); + for (i = 0; i < n - 1; i++) { mark_indirect_target(env, env->gotox_tmp_buf->items[i]); other_branch = push_stack(env, env->gotox_tmp_buf->items[i], @@ -18254,15 +20236,17 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) return check_stack_arg_write(env, state, insn->off, NULL); } + struct bpf_reg_state arena, *dst_reg; enum bpf_reg_type dst_reg_type; err = check_reg_arg(env, insn->dst_reg, SRC_OP); if (err) return err; - dst_reg_type = cur_regs(env)[insn->dst_reg].type; + dst_reg = mem_access_reg(env, insn->dst_reg, &arena); + dst_reg_type = dst_reg->type; - err = check_mem_access(env, env->insn_idx, cur_regs(env) + insn->dst_reg, argno_from_reg(insn->dst_reg), + err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, false, false); if (err) @@ -18277,11 +20261,13 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) env->jmps_processed++; if (opcode == BPF_CALL) { if (env->cur_state->active_locks) { - if ((insn->src_reg == BPF_REG_0 && - insn->imm != BPF_FUNC_spin_unlock && - insn->imm != BPF_FUNC_kptr_xchg) || - (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && - !kfunc_spin_allowed(env, insn->imm, insn->off))) { + /* similar to static subprog calls callx is allowed under a lock */ + if (!bpf_is_callx(insn) && + ((insn->src_reg == BPF_REG_0 && + insn->imm != BPF_FUNC_spin_unlock && + insn->imm != BPF_FUNC_kptr_xchg) || + (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && + !kfunc_spin_allowed(env, insn->imm, insn->off)))) { verbose(env, "function calls are not allowed while holding a lock\n"); bpf_diag_ctx_active( @@ -18294,6 +20280,8 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) mark_reg_scratched(env, BPF_REG_0); if (bpf_in_stack_arg_cnt(&env->subprog_info[cur_func(env)->subprogno])) cur_func(env)->no_stack_arg_load = true; + if (bpf_is_callx(insn)) + return check_func_callx(env, insn, &env->insn_idx); if (insn->src_reg == BPF_PSEUDO_CALL) return check_func_call(env, insn, &env->insn_idx); if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) @@ -18334,6 +20322,178 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) return -EFAULT; } +/* + * Push the loop entered at insn_idx onto the stack. Usually this adds a single + * entry on top of its enclosing loop. However, an edge may enter an inner loop + * directly, bypassing the header(s) of its enclosing loop(s), e.g.: + * + * 1: for (...): // enclosing loop, header at 1 + * 2: for (...): // inner loop, header at 2 + * ... + * 3: if ...: + * goto 2b; // enters loop 2 without going through header 1 + * + * Such a bypass makes the enclosing loop irreducible, so its header is missing + * from the stack and both headers (1) and (2) need to be pushed onto stack. + * Only the innermost loop (the one actually entered at insn_idx) carries SCEV bounds; + * the bypassed ancestors are irreducible and pushed as non-terminating. + * + * Assumes loop_stack_pop() has already truncated the stack to the common ancestor + * of the bpf_loop_at_index(env->insn_idx) and whatever was at the top of the loop stack. + */ +static int loop_stack_push(struct bpf_verifier_env *env, bool *pushed) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_func_state *frame = cur_func(env); + struct loop_stack_entry *loop_stack = frame->loop_stack; + u32 missing_headers[LOOP_STACK_SIZE]; + u32 cnt = frame->loop_stack_cnt; + u32 num_missing = 0; + int h; + + *pushed = false; + + for (h = bpf_loop_at_index(env, env->insn_idx); h >= 0; h = aux[h].loop_header, num_missing++) { + if (cnt && loop_stack[cnt - 1].loop_id == h) + break; + if (num_missing == LOOP_STACK_SIZE) + goto e2big; + missing_headers[num_missing] = h; + } + + if (num_missing == 0) + return 0; + if (cnt + num_missing > LOOP_STACK_SIZE) + goto e2big; + frame->loop_stack = realloc_array(frame->loop_stack, cnt, cnt + num_missing, + sizeof(*frame->loop_stack)); + if (!frame->loop_stack) + return -ENOMEM; + loop_stack = frame->loop_stack; + + for (; num_missing; num_missing--, cnt++) { + h = missing_headers[num_missing - 1]; + loop_stack[cnt] = (struct loop_stack_entry){ .loop_id = h }; + if (env->log.level & BPF_LOG_LEVEL2) + verbose(env, "entering loop %d\n", h); + *pushed = true; + } + frame->loop_stack_cnt = cnt; + return 0; + +e2big: + verbose(env, "Too many nested loops (%d/%d) at %d\n", cnt, num_missing, env->insn_idx); + return -E2BIG; +} + +/* + * An exit from a loop can cross several nested loops, e.g.: + * + * 1: for (...): + * 2: for (...): + * 3: for (...): + * if ...: // before goto the loop stack is [1, 2, 3 <top>] + * goto 1b; // after goto it should become [1] + */ +static int loop_stack_pop(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_func_state *frame = cur_func(env); + int new_cnt = 0; + int h, i; + + /* + * Walk the loop nest of insn_idx outwards (innermost first) and stop at + * the first loop that is present on the stack - that loop is the common + * ancestor and becomes the new top. If no enclosing loop is on the + * stack, drop all loops. + */ + for (h = bpf_loop_at_index(env, env->insn_idx); h >= 0; h = aux[h].loop_header) { + for (i = frame->loop_stack_cnt - 1; i >= 0; i--) { + if (frame->loop_stack[i].loop_id == h) { + new_cnt = i + 1; + goto pop; + } + } + } + +pop: + if (env->log.level & BPF_LOG_LEVEL2) + for (i = frame->loop_stack_cnt - 1; i >= new_cnt; i--) + verbose(env, "exiting loop %d\n", frame->loop_stack[i].loop_id); + frame->loop_stack_cnt = new_cnt; + return 0; +} + +static int maybe_clamp_scev_regs(struct bpf_verifier_env *env) +{ + struct bpf_func_state *frame = cur_func(env); + struct loop_stack_entry *entry; + + if (frame->loop_stack_cnt == 0) { + verifier_bug(env, "%s: loop stack empty at %d", __FUNCTION__, env->insn_idx); + return -EFAULT; + } + + entry = &frame->loop_stack[frame->loop_stack_cnt - 1]; + if (!entry->terminates) + return 0; + + return bpf_clamp_scev_regs(env, frame, env->insn_idx, entry->entry_state, + entry->max_header_count); +} + +static int handle_loop_entry_exit(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *cur = env->cur_state; + struct bpf_func_state *frame = cur_func(env); + struct loop_stack_entry *entry; + struct bpf_loop_iters iters; + bool pushed, terminates; + int err; + + err = loop_stack_pop(env); + if (err) + return err; + + err = loop_stack_push(env, &pushed); + if (err) + return err; + + if (frame->loop_stack_cnt == 0) + return 0; + + entry = &frame->loop_stack[frame->loop_stack_cnt - 1]; + if (env->insn_idx != entry->loop_id) + return 0; + + /* If nothing was pushed and env->insn_idx is a loop header, we've taken a backedge. */ + if (!pushed) + return maybe_clamp_scev_regs(env); + + err = bpf_compute_loop_iters(env, env->cur_state, &iters); + if (err < 0) + return err; + + terminates = err == 1; + if (!terminates) + return 0; + + /* Create a loop_entry checkpoint before widening */ + err = bpf_split_cur_state(env); + if (err) + return err; + + entry->min_header_count = iters.min_header_count; + entry->max_header_count = iters.max_header_count; + entry->entry_state = cur->parent; + entry->terminates = true; + err = bpf_widen_scev_regs(env, cur, cur->parent, &iters); + if (err < 0) + return err; + return 0; +} + static int do_check(struct bpf_verifier_env *env) { bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); @@ -18348,6 +20508,12 @@ static int do_check(struct bpf_verifier_env *env) struct bpf_insn_aux_data *insn_aux; int err; + if (signal_pending(current)) + return -EAGAIN; + + if (need_resched()) + cond_resched(); + /* reset current history entry on each new instruction */ env->cur_hist_ent = NULL; @@ -18394,6 +20560,15 @@ static int do_check(struct bpf_verifier_env *env) } } + /* + * Possibly widen the registers before creating a checkpoint + * in bpf_is_state_visited(). The next loop iteration will + * have a chance to hit this checkpoint and converge. + */ + err = handle_loop_entry_exit(env); + if (err) + return err; + if (bpf_is_prune_point(env, env->insn_idx)) { err = bpf_is_state_visited(env, env->insn_idx); if (err < 0) @@ -18414,17 +20589,11 @@ static int do_check(struct bpf_verifier_env *env) } if (bpf_is_jmp_point(env, env->insn_idx)) { - err = bpf_push_jmp_history(env, state, 0, 0, 0, 0); + err = bpf_push_jmp_history(env, state, 0, 0, 0, NULL, 0); if (err) return err; } - if (signal_pending(current)) - return -EAGAIN; - - if (need_resched()) - cond_resched(); - if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) { verbose(env, "\nfrom %d to %d%s:", env->prev_insn_idx, env->insn_idx, @@ -18434,6 +20603,13 @@ static int do_check(struct bpf_verifier_env *env) do_print_state = false; } + if (bpf_loop_at_index(env, env->insn_idx) >= 0 && + cur_func(env)->loop_stack_cnt == 0) { + verifier_bug(env, "loop stack empty at %d, while inside the loop %d\n", + env->insn_idx, bpf_loop_at_index(env, env->insn_idx)); + return -EFAULT; + } + if (env->log.level & BPF_LOG_LEVEL) { if (verifier_state_scratched(env)) print_insn_state(env, state, state->curframe); @@ -18766,12 +20942,6 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, { enum bpf_prog_type prog_type = resolve_prog_type(prog); - if (map->excl_prog_sha && - memcmp(map->excl_prog_sha, prog->digest, SHA256_DIGEST_SIZE)) { - verbose(env, "program's hash doesn't match map's excl_prog_hash\n"); - return -EACCES; - } - if (btf_record_has_field(map->record, BPF_LIST_HEAD) || btf_record_has_field(map->record, BPF_RB_ROOT)) { if (is_tracing_prog_type(prog_type)) { @@ -18805,35 +20975,11 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, return -EINVAL; } - if (prog->sleepable) - switch (map->map_type) { - case BPF_MAP_TYPE_HASH: - case BPF_MAP_TYPE_RHASH: - case BPF_MAP_TYPE_LRU_HASH: - case BPF_MAP_TYPE_ARRAY: - case BPF_MAP_TYPE_PERCPU_HASH: - case BPF_MAP_TYPE_PERCPU_ARRAY: - case BPF_MAP_TYPE_LRU_PERCPU_HASH: - case BPF_MAP_TYPE_LPM_TRIE: - case BPF_MAP_TYPE_ARRAY_OF_MAPS: - case BPF_MAP_TYPE_HASH_OF_MAPS: - case BPF_MAP_TYPE_RINGBUF: - case BPF_MAP_TYPE_USER_RINGBUF: - case BPF_MAP_TYPE_INODE_STORAGE: - case BPF_MAP_TYPE_SK_STORAGE: - case BPF_MAP_TYPE_TASK_STORAGE: - case BPF_MAP_TYPE_CGRP_STORAGE: - case BPF_MAP_TYPE_QUEUE: - case BPF_MAP_TYPE_STACK: - case BPF_MAP_TYPE_ARENA: - case BPF_MAP_TYPE_INSN_ARRAY: - case BPF_MAP_TYPE_PROG_ARRAY: - break; - default: - verbose(env, - "Sleepable programs can only use array, hash, ringbuf and local storage maps\n"); - return -EINVAL; - } + if (prog->sleepable && !map_needs_no_rcu(map)) { + verbose(env, + "Sleepable programs can only use array, hash, ringbuf and local storage maps\n"); + return -EINVAL; + } if (bpf_map_is_cgroup_storage(map) && bpf_cgroup_storage_assign(env->prog->aux, map)) { @@ -18869,6 +21015,85 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, return 0; } +/* + * Keep track of whether the map is used by one program only. Such program may + * store the addresses of its functions into the map when it's frozen, see + * resolve_func_ptrs(), since nothing else relies on what the map has. After + * that the map is not available to other programs. + */ +static int bpf_map_claim(struct bpf_verifier_env *env, struct bpf_map *map) +{ + unsigned long me = (unsigned long)env->prog->aux, old; + + for (;;) { + old = READ_ONCE(map->user); + if (old == me || old == BPF_MAP_USER_MANY) + return 0; + if (old & BPF_MAP_USER_PATCHED) { + verbose(env, "map '%s' has addresses of functions of another program\n", + map->name); + return -EBUSY; + } + if (cmpxchg(&map->user, old, old ? BPF_MAP_USER_MANY : me) == old) + return 0; + } +} + +/* + * An exclusive map is bound to the digest of its program. The digest covers + * the insns as they are verified, after the CO-RE relocations the kernel + * applies, so it exists only once check_and_resolve_insns() has computed it, + * and the maps are checked against it there, whether an insn referred to + * them or a continuous fd_array bound them before. + */ +static int check_excl_maps(struct bpf_verifier_env *env) +{ + int i; + + for (i = 0; i < env->used_map_cnt; i++) { + struct bpf_map *map = env->used_maps[i]; + + if (map->excl_prog_sha && + memcmp(map->excl_prog_sha, env->prog->digest, SHA256_DIGEST_SIZE)) { + verbose(env, "program's hash doesn't match map's excl_prog_hash\n"); + return -EACCES; + } + } + return 0; +} + +/* + * The program may use every map it referred to, see check_excl_maps(). Only now + * does it leave a trace on them: it claims each as its user, see + * bpf_map_claim(), and takes over its insn arrays. A program rejected before + * this point, one that is not the owner of an exclusive map in particular, + * leaves the maps as it found them. Nothing consults a claim before + * resolve_func_ptrs() and the fixup passes, which run later. + */ +static int claim_used_maps(struct bpf_verifier_env *env) +{ + int i, err; + + for (i = 0; i < env->used_map_cnt; i++) { + struct bpf_map *map = env->used_maps[i]; + + err = bpf_map_claim(env, map); + if (err) + return err; + + if (map->map_type != BPF_MAP_TYPE_INSN_ARRAY) + continue; + err = bpf_insn_array_init(map, env->prog); + if (err) { + verbose(env, "Failed to properly initialize insn array\n"); + return err; + } + env->insn_array_maps[env->insn_array_map_cnt++] = map; + env->prog->jit_required = true; + } + return 0; +} + static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) { int i, err; @@ -18894,7 +21119,7 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) if (err) return err; - if (env->prog->sleepable) + if (bpf_prog_runs_sleepable(env->prog)) atomic64_inc(&map->sleepable_refcnt); /* hold the map. If the program is rejected by verifier, @@ -18906,16 +21131,6 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) env->used_maps[env->used_map_cnt++] = map; - if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) { - err = bpf_insn_array_init(map, env->prog); - if (err) { - verbose(env, "Failed to properly initialize insn array\n"); - return err; - } - env->insn_array_maps[env->insn_array_map_cnt++] = map; - env->prog->jit_required = true; - } - return env->used_map_cnt - 1; } @@ -19060,6 +21275,14 @@ static int check_jmp_fields(struct bpf_verifier_env *env, struct bpf_insn *insn) switch (opcode) { case BPF_CALL: + if (bpf_is_callx(insn)) { + /* callx dst_reg */ + if (insn->src_reg != BPF_REG_0 || insn->imm != 0 || insn->off != 0) { + verbose(env, "BPF_CALL|BPF_X uses reserved fields\n"); + return -EINVAL; + } + return 0; + } if (BPF_SRC(insn->code) != BPF_K || (insn->src_reg != BPF_PSEUDO_KFUNC_CALL && insn->off != 0) || (insn->src_reg != BPF_REG_0 && insn->src_reg != BPF_PSEUDO_CALL && @@ -19084,14 +21307,14 @@ static int check_jmp_fields(struct bpf_verifier_env *env, struct bpf_insn *insn) return 0; case BPF_EXIT: if (BPF_SRC(insn->code) != BPF_K || insn->imm != 0 || - insn->src_reg != BPF_REG_0 || insn->dst_reg != BPF_REG_0 || + insn->src_reg > BPF_UNWIND_EXIT || insn->dst_reg != BPF_REG_0 || class == BPF_JMP32) { verbose(env, "BPF_EXIT uses reserved fields\n"); return -EINVAL; } return 0; case BPF_JCOND: - if (insn->code != (BPF_JMP | BPF_JCOND) || insn->src_reg != BPF_MAY_GOTO || + if (insn->code != (BPF_JMP | BPF_JCOND) || insn->src_reg > BPF_UNWIND_GOTO || insn->dst_reg || insn->imm) { verbose(env, "invalid may_goto imm %d\n", insn->imm); return -EINVAL; @@ -19311,6 +21534,107 @@ next_insn: * 'struct bpf_map *' into a register instead of user map_fd. * These pointers will be used later by verifier to validate map access. */ + err = check_excl_maps(env); + if (err) + return err; + return claim_used_maps(env); +} + +/* that many pointers to functions are recognized in the maps of a program at most */ +#define BPF_MAX_FUNC_PTRS (1 << 20) + +static int add_func_ptr(struct bpf_verifier_env *env, struct bpf_map *map, u32 map_off, + u32 xlated_off) +{ + struct bpf_func_ptr *ptrs; + + if (env->func_ptr_cnt == BPF_MAX_FUNC_PTRS) { + verbose(env, "too many pointers to functions in maps, the limit is %d\n", + BPF_MAX_FUNC_PTRS); + return -E2BIG; + } + + /* grow by doubling, the array is sorted and searched later */ + if (!(env->func_ptr_cnt & (env->func_ptr_cnt - 1))) { + ptrs = kvrealloc(env->func_ptrs, + array_size(max(2 * env->func_ptr_cnt, 16U), sizeof(*ptrs)), + GFP_KERNEL_ACCOUNT); + if (!ptrs) + return -ENOMEM; + env->func_ptrs = ptrs; + } + env->func_ptrs[env->func_ptr_cnt++] = (struct bpf_func_ptr){ + .map = map, + .map_off = map_off, + .orig_off = xlated_off, + .xlated_off = xlated_off, + }; + return 0; +} + +/* + * Compilers put pointers to functions into read-only data: tables of functions, + * structures of operations, vtables, where they are mixed with other data. + * The loader stores such data in a frozen read-only array map and resolves + * a pointer to a static function to the offset in bytes of its first + * instruction in the program: the address of the function in the program. + * + * Find 64-bit values that look like that in the maps of a program that uses + * callx. It's a guess. When the value is not a pointer, the program either + * fails to load, because it does with a pointer what can be done with + * a number only, or it sees the address of a function instead of the number. + * It's known before the control flow graph of the program is built and the main + * verification pass begins which functions may be called via callx. + * + * When the program is jitted the offsets are replaced with the addresses of + * the functions in the map itself, see jit_subprogs(). Hence the program has to + * be the only user of the map, see bpf_map_claim(): nothing else may rely on + * what the map had. + * + * The program reads the addresses of its functions from there like any other + * data, so it has to be allowed to leak pointers. + */ +static int resolve_func_ptrs(struct bpf_verifier_env *env) +{ + int insn_cnt = env->prog->len; + int i, err, subprog; + struct bpf_map *map; + u64 addr, val; + u32 off; + + if (!env->has_callx || !env->allow_ptr_leaks) + return 0; + + for (i = 0; i < env->used_map_cnt; i++) { + map = env->used_maps[i]; + /* coincidences in maps that are shared with other programs don't matter */ + if (READ_ONCE(map->user) != (unsigned long)env->prog->aux) + continue; + if (map->map_type != BPF_MAP_TYPE_ARRAY || map->max_entries != 1 || + !bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr || + !IS_ERR_OR_NULL(map->record)) + continue; + if (map->ops->map_direct_value_addr(map, &addr, 0)) + continue; + + for (off = 0; off + sizeof(u64) <= map->value_size; off += sizeof(u64)) { + if (!(off % SZ_64K)) + cond_resched(); + val = *(u64 *)(unsigned long)(addr + off); + if (!val || val % sizeof(struct bpf_insn) || + val / sizeof(struct bpf_insn) >= insn_cnt) + continue; + subprog = bpf_find_subprog(env, val / sizeof(struct bpf_insn)); + if (subprog <= 0 || bpf_subprog_is_global(env, subprog)) + continue; + err = add_func_ptr(env, map, off, val / sizeof(struct bpf_insn)); + if (err) + return err; + } + } + if (env->func_ptr_cnt) + sort(env->func_ptrs, env->func_ptr_cnt, sizeof(*env->func_ptrs), + cmp_func_ptrs, NULL); return 0; } @@ -19421,13 +21745,15 @@ static void free_states(struct bpf_verifier_env *env) } } -static int do_check_common(struct bpf_verifier_env *env, int subprog) +static int do_check_common(struct bpf_verifier_env *env, int subprog, + enum bpf_func_context context) { bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); struct bpf_subprog_info *sub = subprog_info(env, subprog); struct bpf_prog_aux *aux = env->prog->aux; struct bpf_verifier_state *state; struct bpf_reg_state *regs; + u32 old_insns_total = sub->insns_total; u32 insn_processed = env->insn_processed; int ret, i; @@ -19440,7 +21766,8 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) state->curframe = 0; state->speculative = false; state->branches = 1; - state->in_sleepable = env->prog->sleepable; + state->in_sleepable = context == BPF_FUNC_CTX_SLEEPABLE; + env->no_rcu_read_lock = context == BPF_FUNC_CTX_NO_RCU_LOCK; state->frame[0] = kzalloc_obj(struct bpf_func_state, GFP_KERNEL_ACCOUNT); if (!state->frame[0]) { kfree(state); @@ -19480,25 +21807,30 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) } /* Also ensure the callback only has a single scalar argument. */ - if (sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_ANYTHING) { + if (sub->arg_slot_cnt != 1 || sub->args[0].arg_type != ARG_SCALAR) { verbose(env, "exception cb only supports single integer argument\n"); ret = -EINVAL; goto out; } } - for (i = BPF_REG_1; i <= min_t(u32, sub->arg_cnt, MAX_BPF_FUNC_REG_ARGS); i++) { + for (i = BPF_REG_1; + i <= min_t(u32, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); i++) { arg = &sub->args[i - BPF_REG_1]; reg = ®s[i]; if (arg->arg_type == ARG_PTR_TO_CTX) { reg->type = PTR_TO_CTX; mark_reg_known_zero(env, regs, i); - } else if (arg->arg_type == ARG_ANYTHING) { + } else if (arg->arg_type == ARG_SCALAR) { reg->type = SCALAR_VALUE; mark_reg_unknown(env, regs, i); } else if (arg->arg_type == ARG_PTR_TO_DYNPTR) { - /* assume unspecial LOCAL dynptr type */ - __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen, 0); + /* + * The callers may hand in a dynptr of any type. + * Take it for that of an skb, so that a write + * through it and its slices are treated with care. + */ + __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_SKB, true, ++env->id_gen, 0); } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) { reg->type = PTR_TO_MEM; reg->type |= arg->arg_type & @@ -19529,7 +21861,8 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) goto out; } } - if (env->prog->type == BPF_PROG_TYPE_EXT && sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS) { + if (env->prog->type == BPF_PROG_TYPE_EXT && + sub->arg_slot_cnt > MAX_BPF_FUNC_REG_ARGS) { verbose(env, "freplace programs with >%d args not supported yet\n", MAX_BPF_FUNC_REG_ARGS); ret = -EINVAL; @@ -19542,9 +21875,10 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) */ if (env->prog->aux->func_info_aux) { ret = btf_prepare_func_args(env, 0); - if (ret || sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_PTR_TO_CTX) { + if (ret || sub->arg_slot_cnt != 1 || + sub->args[0].arg_type != ARG_PTR_TO_CTX) { env->prog->aux->func_info_aux[0].unreliable = true; - sub->arg_cnt = 1; + sub->arg_slot_cnt = 1; sub->stack_arg_cnt = 0; } } @@ -19581,8 +21915,10 @@ out: * not accounted as callees by account_current_path(). * Accumulate their total counts as total counts of the main or * global subprog hosting the async call. + * Start from the saved total of earlier contexts: adding to the current + * total would count this pass's synchronous paths twice. */ - env->subprog_info[subprog].insns_total = env->insn_processed - insn_processed; + sub->insns_total = old_insns_total + (env->insn_processed - insn_processed); return ret; } @@ -19610,14 +21946,19 @@ static int do_check_subprogs(struct bpf_verifier_env *env) { struct bpf_prog_aux *aux = env->prog->aux; struct bpf_func_info_aux *sub_aux; - int i, ret, new_cnt; + int context, i, ret, new_cnt; if (!aux->func_info) return 0; - /* exception callback is presumed to be always called */ - if (env->exception_callback_subprog) - subprog_aux(env, env->exception_callback_subprog)->called = true; + /* + * Callbacks cannot throw, so the exception callback always runs in the + * main program's context. It is presumed to be always called. + */ + if (env->exception_callback_subprog) { + sub_aux = subprog_aux(env, env->exception_callback_subprog); + sub_aux->called[env->prog->sleepable] = true; + } again: new_cnt = 0; @@ -19626,29 +21967,28 @@ again: continue; sub_aux = subprog_aux(env, i); - if (!sub_aux->called || sub_aux->verified) - continue; + for (context = 0; context < ARRAY_SIZE(sub_aux->called); context++) { + if (!sub_aux->called[context] || sub_aux->verified[context]) + continue; - env->insn_idx = env->subprog_info[i].start; - WARN_ON_ONCE(env->insn_idx == 0); - ret = do_check_common(env, i); - if (ret) { - return ret; - } else if (env->log.level & BPF_LOG_LEVEL) { - verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n", - i, bpf_subprog_name(env, i)); - } + env->insn_idx = env->subprog_info[i].start; + WARN_ON_ONCE(env->insn_idx == 0); + ret = do_check_common(env, i, context); + if (ret) + return ret; + if (env->log.level & BPF_LOG_LEVEL) + verbose(env, "Func#%d ('%s') is safe for any args " + "that match its prototype\n", + i, bpf_subprog_name(env, i)); - /* We verified new global subprog, it might have called some - * more global subprogs that we haven't verified yet, so we - * need to do another pass over subprogs to verify those. - */ - sub_aux->verified = true; - new_cnt++; + sub_aux->verified[context] = true; + new_cnt++; + } } - /* We can't loop forever as we verify at least one global subprog on - * each pass. + /* + * We can't loop forever as each pass verifies at least one new context, + * and there are only three contexts per global subprog. */ if (new_cnt) goto again; @@ -19661,7 +22001,7 @@ static int do_check_main(struct bpf_verifier_env *env) int ret; env->insn_idx = 0; - ret = do_check_common(env, 0); + ret = do_check_common(env, 0, env->prog->sleepable); if (!ret) env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; return ret; @@ -20034,6 +22374,109 @@ static bool attach_uses_trampoline_retval(enum bpf_attach_type type) } } +/* + * BPF code passes argument slot N where the kernel convention passes its Nth + * u64 argument, so describe a BPF function to its trampoline as one u64 per + * slot, and every arch places its arguments right. + */ +static int flatten_bpf_func_model(struct bpf_verifier_log *log, const char *tname, + struct btf_func_model *m) +{ + u32 i, nslots = 0; + + for (i = 0; i < m->nr_args; i++) + nslots += btf_func_model_arg_slots(m, i); + if (nslots > MAX_BPF_FUNC_ARGS) { + bpf_log(log, "The function %s has %u argument slots. Too many.\n", + tname, nslots); + return -EINVAL; + } + + for (i = 0; i < nslots; i++) { + m->arg_size[i] = sizeof(u64); + m->arg_flags[i] = 0; + } + m->nr_args = nslots; + return 0; +} + +/* + * The verifier lets a static function use stack arguments counted from its + * BTF, even if that BTF turns out unreliable. A trampoline on the function is + * then given only MAX_BPF_FUNC_REG_ARGS u64 arguments and would lose the + * stack arguments, so such a function cannot be attached to. + */ +static int check_unreliable_func_args(struct bpf_verifier_log *log, struct btf *btf, + const struct btf_type *func_proto, const char *tname) +{ + struct btf_func_model m; + u32 i, nslots = 0; + + if (btf_distill_func_proto(NULL, btf, func_proto, tname, &m)) { + /* at least a slot for each argument */ + nslots = btf_type_vlen(func_proto); + } else { + for (i = 0; i < m.nr_args; i++) + nslots += btf_func_model_arg_slots(&m, i); + } + if (nslots <= MAX_BPF_FUNC_REG_ARGS) + return 0; + + bpf_log(log, "Cannot attach to static function %s: unreliable BTF and stack arguments\n", + tname); + return -EINVAL; +} + +/* Functions may share a type id. The first of them is attached to. */ +static int find_tgt_subprog(const struct bpf_prog *tgt_prog, u32 btf_id) +{ + const struct bpf_prog_aux *aux = tgt_prog->aux; + int i; + + for (i = 0; i < aux->func_info_cnt; i++) + if (aux->func_info[i].type_id == btf_id) + return i; + + return -1; +} + +/* + * The prologue of an extension gives a cloned skb a new head, unless the + * prologue of the target program has done that. The callers of the function + * that it replaces must not hold on to the old head. + */ +int bpf_check_ext_prologue(struct bpf_verifier_log *log, + const struct bpf_prog *tgt_prog, u32 btf_id) +{ + const struct bpf_prog_aux *aux = tgt_prog->aux; + int subprog = find_tgt_subprog(tgt_prog, btf_id); + + if (aux->has_prologue) + return 0; + + if (subprog >= 0 && !aux->func_info_aux[subprog].called_with_pkt_ptrs) + return 0; + + bpf_log(log, + "Extension writing packet data cannot replace a function whose callers keep packet pointers\n"); + return -EINVAL; +} + +/* + * An extension is verified as a program that runs under the RCU read lock. Does + * the function that it is to replace? + */ +static bool func_has_rcu_read_lock(const struct bpf_prog *prog, int subprog) +{ + const struct bpf_func_info_aux *sub_aux = &prog->aux->func_info_aux[subprog]; + + if (!subprog) + return !prog->sleepable; + + return !sub_aux->called[BPF_FUNC_CTX_SLEEPABLE] && + !sub_aux->called[BPF_FUNC_CTX_NO_RCU_LOCK]; +} + int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog, @@ -20045,7 +22488,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, char trace_symbol[KSYM_SYMBOL_LEN]; const char prefix[] = "btf_trace_"; struct bpf_raw_event_map *btp; - int ret = 0, subprog = -1, i; + int ret = 0, subprog = -1; const struct btf_type *t; bool conservative = true; const char *tname, *fname; @@ -20077,6 +22520,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, struct bpf_prog_aux *aux = tgt_prog->aux; bool tgt_changes_pkt_data; bool tgt_might_sleep; + bool tgt_can_unwind; if (bpf_prog_is_dev_bound(prog->aux) && !bpf_prog_dev_bound_match(prog, tgt_prog)) { @@ -20084,11 +22528,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, return -EINVAL; } - for (i = 0; i < aux->func_info_cnt; i++) - if (aux->func_info[i].type_id == btf_id) { - subprog = i; - break; - } + subprog = find_tgt_subprog(tgt_prog, btf_id); if (subprog == -1) { bpf_log(log, "Subprog %s doesn't exist\n", tname); return -EINVAL; @@ -20109,9 +22549,20 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, prog_extension ? "Extension" : "Tracing"); return -EINVAL; } + /* unwind_exit of the main prog is exit, unless it's an extension */ + tgt_can_unwind = (subprog || tgt_prog->type == BPF_PROG_TYPE_EXT) && + (aux->func ? aux->func[subprog]->aux->can_unwind + : aux->can_unwind); + /* unwind_exit would return to the trampoline that called the target */ + if (tgt_can_unwind && !prog_extension && + prog->expected_attach_type != BPF_TRACE_FENTRY) { + bpf_log(log, "Only fentry can attach to %s, which can unwind\n", tname); + return -EINVAL; + } conservative = aux->func_info_aux[subprog].unreliable; if (prog_extension) { - if (conservative) { + if (aux->func_info_aux[subprog].linkage != BTF_FUNC_GLOBAL || + conservative) { bpf_log(log, "Cannot replace static functions\n"); return -EINVAL; @@ -20121,6 +22572,20 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, "Extension programs should be JITed\n"); return -EINVAL; } + /* + * How much of the buffer of the tracepoint is used + * is checked for the program that is attached to it. + */ + if (tgt_prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) { + bpf_log(log, + "Cannot extend a writable raw tracepoint program\n"); + return -EINVAL; + } + if (!func_has_rcu_read_lock(tgt_prog, subprog)) { + bpf_log(log, + "Cannot extend a function that runs without RCU read lock\n"); + return -EINVAL; + } tgt_changes_pkt_data = aux->func ? aux->func[subprog]->aux->changes_pkt_data : aux->changes_pkt_data; @@ -20129,6 +22594,9 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, "Extension program changes packet data, while original does not\n"); return -EINVAL; } + if (prog->aux->has_prologue && + bpf_check_ext_prologue(log, tgt_prog, btf_id)) + return -EINVAL; tgt_might_sleep = aux->func ? aux->func[subprog]->aux->might_sleep @@ -20138,6 +22606,12 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, "Extension program may sleep, while original does not\n"); return -EINVAL; } + /* There is no unwind_goto after the calls of the original */ + if (prog->aux->can_unwind && !tgt_can_unwind) { + bpf_log(log, + "Extension program can unwind, while original does not\n"); + return -EINVAL; + } } if (!tgt_prog->jited) { bpf_log(log, "Can attach to only JITed progs\n"); @@ -20301,12 +22775,23 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) return -EINVAL; - if (tgt_prog && conservative) + if (tgt_prog && conservative) { + if (subprog > 0) { + ret = check_unreliable_func_args(log, btf, t, tname); + if (ret) + return ret; + } t = NULL; + } ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); if (ret < 0) return ret; + if (tgt_prog) { + ret = flatten_bpf_func_model(log, tname, &tgt_info->fmodel); + if (ret < 0) + return ret; + } if (tgt_info->fmodel.ret_size > 8 && attach_uses_trampoline_retval(prog->expected_attach_type)) { @@ -20783,6 +23268,66 @@ static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc return 0; } +static int add_kfunc_desc_target(struct bpf_verifier_env *env, + const struct bpf_kfunc_desc *target_desc, + u16 base_desc_idx, u16 *desc_idx) +{ + struct bpf_kfunc_desc desc = *target_desc; + struct bpf_kfunc_desc_tab *new_tab; + struct bpf_kfunc_desc_tab *tab; + struct bpf_prog_aux *prog_aux; + u32 i; + + prog_aux = env->prog->aux; + tab = prog_aux->kfunc_tab; + + if (base_desc_idx >= tab->nr_base_descs) { + verifier_bug(env, "kfunc desc_idx %d out of bounds (%d descriptors)", + base_desc_idx, tab->nr_base_descs); + return -EFAULT; + } + if (tab->descs[base_desc_idx].func_id != desc.func_id) { + verifier_bug(env, "kfunc desc %d has invalid id (expected %d, got %d)", + base_desc_idx, tab->descs[base_desc_idx].func_id, desc.func_id); + return -EFAULT; + } + if (tab->descs[base_desc_idx].offset != desc.offset) { + verifier_bug(env, "kfunc desc %d has invalid offset (expected %d, got %d)", + base_desc_idx, tab->descs[base_desc_idx].offset, desc.offset); + return -EFAULT; + } + + if (tab->descs[base_desc_idx].addr == desc.addr) { + *desc_idx = base_desc_idx; + return 0; + } + + for (i = tab->nr_base_descs; i < tab->nr_descs; i++) { + if (tab->descs[i].func_id == desc.func_id && + tab->descs[i].offset == desc.offset && + tab->descs[i].addr == desc.addr) { + *desc_idx = i; + return 0; + } + } + + if (tab->nr_descs == MAX_KFUNC_CALL_DESCS) { + verbose(env, "too many different kernel function call targets\n"); + return -E2BIG; + } + + new_tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1), + GFP_KERNEL_ACCOUNT); + if (!new_tab) + return -ENOMEM; + tab = new_tab; + prog_aux->kfunc_tab = tab; + + *desc_idx = tab->nr_descs; + tab->descs[tab->nr_descs++] = desc; + return 0; +} + static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux, u16 struct_meta_reg, u16 node_offset_reg, @@ -20803,7 +23348,11 @@ static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux, int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn *insn_buf, int insn_idx, int *cnt) { + struct bpf_kfunc_desc desc_copy; struct bpf_kfunc_desc *desc; + unsigned long call_imm; + u16 base_desc_idx, desc_idx; + bool near_call; int err; if (!insn->imm) { @@ -20823,13 +23372,30 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn->imm); return -EFAULT; } + base_desc_idx = desc - env->prog->aux->kfunc_tab->descs; + + near_call = !bpf_jit_supports_far_kfunc_call(); + desc_copy = *desc; + desc = &desc_copy; err = specialize_kfunc(env, desc, insn_idx); if (err) return err; - if (!bpf_jit_supports_far_kfunc_call()) - insn->imm = BPF_CALL_IMM(desc->addr); + if (near_call) { + call_imm = BPF_CALL_IMM(desc->addr); + if ((unsigned long)(s32)call_imm != call_imm) { + verbose(env, "address of kernel func_id %u is out of range\n", + desc->func_id); + return -EINVAL; + } + insn->imm = call_imm; + } + + err = add_kfunc_desc_target(env, desc, base_desc_idx, &desc_idx); + if (err) + return err; + insn->off = desc_idx; if (is_bpf_obj_new_kfunc(desc->func_id) || is_bpf_percpu_obj_new_kfunc(desc->func_id)) { struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; @@ -20989,6 +23555,14 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; } +/* + * Upper bound on the PKCS#7 signature blob passed with a program. Comfortably + * above the largest signature the kernel can verify, and far below anything + * that would make rejecting a load expensive. Deliberately a fixed number so + * that what the syscall accepts does not depend on PAGE_SIZE. + */ +#define BPF_PROG_MAX_SIGNATURE_SIZE (64 * 1024) + static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) { switch (keyring_id) { @@ -20998,6 +23572,8 @@ static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) return BPF_SIG_KEYRING_SECONDARY; case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING: return BPF_SIG_KEYRING_PLATFORM; + case KEY_SPEC_BPF_KEYRING: + return BPF_SIG_KEYRING_BPF; default: return BPF_SIG_KEYRING_USER; } @@ -21027,17 +23603,33 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env, int err = 0; /* - * Don't attempt to use kmalloc_large or vmalloc for signatures. - * Practical signature for BPF program should be below this limit. + * The signature covers the insns as submitted. CO-RE relocations would + * have the kernel rewrite them afterwards, with values the loader picks + * through the BTF it supplies, see bpf_check_core_relo(). */ + if (attr->core_relo_cnt) { + verbose(env, "signed program cannot carry CO-RE relocations\n"); + return -EACCES; + } if (!attr->signature_size || - attr->signature_size > KMALLOC_MAX_CACHE_SIZE) + attr->signature_size > BPF_PROG_MAX_SIGNATURE_SIZE) return -EINVAL; - if (system_keyring_id_check(attr->keyring_id) == 0) + + if (system_keyring_id_check(attr->keyring_id) == 0) { key = bpf_lookup_system_key(attr->keyring_id); - else + } else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { + key = bpf_lookup_keyring(); + } else if (bpf_keyring_enforced()) { + verbose(env, "caller-supplied keyring refused, use bpf keyring\n"); + return -EPERM; + } else { key = bpf_lookup_user_key(attr->keyring_id, 0); + } if (!key) { + if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { + verbose(env, "the bpf keyring is empty or has not been restricted\n"); + return -ENOKEY; + } verbose(env, "cannot resolve signing keyring with keyring_id %d\n", attr->keyring_id); return -EINVAL; @@ -21129,6 +23721,65 @@ out: return err; } +/* + * bpf_clone_redirect() transmits from inside the program, so a tc egress or + * lwt_xmit program that redirects to its own device runs again on top of its + * own frame, and again from there, until the datapath's recursion limit drops + * the packet: ten frames deep. Ten MAX_BPF_STACK frames fit the kernel stack + * as they always did; ten MAX_BPF_STACK_JIT frames would not, so a program + * calling it keeps the smaller budget. The redirect helpers that transmit + * after the program has returned leave no frame behind. + */ +static bool bpf_prog_reenters_datapath(const struct bpf_prog *prog) +{ + const struct bpf_insn *insn = prog->insnsi; + int i; + + for (i = 0; i < prog->len; i++, insn++) + if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_clone_redirect) + return true; + return false; +} + +/* Various log level 2 information about the program */ +static void log_program(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *insn_aux = env->insn_aux_data; + struct bpf_insn *insns = env->prog->insnsi; + u32 insn_cnt = env->prog->len; + u64 pos, insn_pos; + u32 i, j; + + verbose(env, "Program dump (scc? loop_header? idom insn#: live_regs_before):\n"); + for (i = 0; i < insn_cnt; ++i) { + verbose_linfo(env, i, " ; "); + if (env->insn_aux_data[i].scc) + verbose(env, "%3d ", env->insn_aux_data[i].scc); + else + verbose(env, " "); + if (env->insn_aux_data[i].loop_header >= 0) + verbose(env, "%3d ", env->insn_aux_data[i].loop_header); + else + verbose(env, " "); + verbose(env, "%3d ", env->idoms[i]); + verbose(env, "%3d: ", i); + for (j = BPF_REG_0; j < BPF_REG_10; ++j) + if (insn_aux[i].live_regs_before & BIT(j)) + verbose(env, "%d", j); + else + verbose(env, "."); + verbose(env, " "); + pos = env->log.end_pos; + bpf_verbose_insn(env, &insns[i]); + insn_pos = env->log.end_pos; + if (insn_aux[i].zext_dst) + verbose(env, "%*c; zext", bpf_vlog_alignment(insn_pos - pos), ' '); + verbose(env, "\n"); + if (bpf_is_ldimm64(&insns[i])) + i++; + } +} + int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log) { @@ -21153,6 +23804,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, env->bt.env = env; env->prog = *prog; env->ops = bpf_verifier_ops[env->prog->type]; + env->stack_limit = bpf_prog_stack_limit(env->prog); + if (bpf_prog_reenters_datapath(env->prog)) + env->stack_limit = MAX_BPF_STACK; env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token); env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token); @@ -21170,10 +23824,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, ret = bpf_diag_init(env); if (ret) goto err_prep; - if (env->signature) { - ret = bpf_prog_calc_tag(env->prog); - if (ret < 0) - goto err_prep; + if (env->prog->insnsi[env->prog->len - 1].code == (BPF_LD | BPF_IMM | BPF_DW)) { + verbose(env, "invalid bpf_ld_imm64 insn\n"); + ret = -EINVAL; + goto err_prep; } ret = process_fd_array(env, attr, uattr); @@ -21228,6 +23882,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (is_priv) env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS; + env->arena_scalar = attr->prog_flags & BPF_F_ARENA_SCALAR; env->explored_states = kvzalloc_objs(struct list_head, state_htab_size(env), @@ -21245,25 +23900,37 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; - /* Discover all subprograms before validating their layout and BTF. */ - ret = add_subprogs(env); + /* Apply CO-RE before validating the program's instruction layout. */ + ret = bpf_check_core_relo(env, attr, uattr); if (ret < 0) goto skip_full_check; - ret = check_subprogs(env); + /* Discover all subprograms and collect the properties needed by BTF validation. */ + ret = add_subprogs(env); if (ret < 0) goto skip_full_check; - /* Validate BTF against the complete subprogram layout and apply CO-RE. */ + find_subprog_properties(env); + + /* Validate BTF before reporting subprogram layout errors. */ ret = bpf_check_btf_info(env, attr, uattr); if (ret < 0) goto skip_full_check; + ret = check_subprogs(env); + if (ret < 0) + goto skip_full_check; + /* Validate instructions and resolve the program's referenced resources. */ ret = check_and_resolve_insns(env); if (ret < 0) goto skip_full_check; + /* Find pointers to functions in the read-only maps of the program. */ + ret = resolve_func_ptrs(env); + if (ret < 0) + goto skip_full_check; + /* Build kfunc prototypes after resolving program resources. */ ret = add_kfuncs(env); if (ret < 0) @@ -21279,6 +23946,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + ret = check_unwind_insns(env); + if (ret < 0) + goto skip_full_check; + ret = bpf_compute_postorder(env); if (ret < 0) goto skip_full_check; @@ -21307,10 +23978,38 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + /* must precede the first bpf_ret_reg_pair() user below */ + ret = bpf_compute_subprog_ret_regs(env); + if (ret < 0) + goto skip_full_check; + + prepare_subprog_args(env); + + ret = bpf_compute_idoms(env); + if (ret < 0) + goto skip_full_check; + + ret = bpf_compute_loops(env); + if (ret < 0) + goto skip_full_check; + ret = bpf_compute_live_registers(env); if (ret < 0) goto skip_full_check; + if (env->log.level & BPF_LOG_LEVEL2) + log_program(env); + + ret = bpf_init_scev(env); + if (ret < 0) + goto skip_full_check; + + ret = bpf_compute_scev(env); + if (ret < 0) + goto skip_full_check; + + bpf_free_subprog_jts(env); + ret = mark_fastcall_patterns(env); if (ret < 0) goto skip_full_check; @@ -21318,6 +24017,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, ret = do_check_main(env); ret = ret ?: do_check_subprogs(env); + /* reject recursion through the callx edges found by the main pass */ + if (ret == 0 && env->callx_edges) + ret = sort_subprogs_topo(env); + if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux)) ret = bpf_prog_offload_finalize(env); @@ -21451,16 +24154,22 @@ err_prep: release_maps(env); release_btfs(env); err_free_env: + bpf_free_subprog_jts(env); if (env->insn_aux_data) bpf_clear_insn_aux_data(env, 0, env->insn_aux_data_len); vfree(env->insn_aux_data); kvfree(env->fd_array); + bpf_free_scev(env); bpf_stack_liveness_free(env); + kvfree(env->cfg.postorder_nums); kvfree(env->cfg.insn_postorder); kvfree(env->scc_info); kvfree(env->succ); kvfree(env->gotox_tmp_buf); + kvfree(env->callx_edges); + kvfree(env->func_ptrs); bpf_diag_free(env); + kvfree(env->idoms); kvfree(env); return ret; } diff --git a/kernel/capability.c b/kernel/capability.c index 90e6ab62f6db..a689dae590ff 100644 --- a/kernel/capability.c +++ b/kernel/capability.c @@ -470,7 +470,7 @@ EXPORT_SYMBOL(file_ns_capable); * Return true if the inode uid and gid are within the namespace. */ bool privileged_wrt_inode_uidgid(struct user_namespace *ns, - struct mnt_idmap *idmap, + const struct mnt_idmap *idmap, const struct inode *inode) { return vfsuid_has_mapping(ns, i_uid_into_vfsuid(idmap, inode)) && @@ -487,7 +487,7 @@ bool privileged_wrt_inode_uidgid(struct user_namespace *ns, * its own user namespace and that the given inode's uid and gid are * mapped into the current user namespace. */ -bool capable_wrt_inode_uidgid(struct mnt_idmap *idmap, +bool capable_wrt_inode_uidgid(const struct mnt_idmap *idmap, const struct inode *inode, int cap) { struct user_namespace *ns = current_user_ns(); diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h index 58797123b752..233fb8b45d2c 100644 --- a/kernel/cgroup/cgroup-internal.h +++ b/kernel/cgroup/cgroup-internal.h @@ -254,7 +254,7 @@ void cgroup_procs_write_finish(struct task_struct *task, enum cgroup_attach_lock_mode lock_mode) __releases(&cgroup_threadgroup_rwsem); -void cgroup_lock_and_drain_offline(struct cgroup *cgrp); +int cgroup_lock_and_drain_offline(struct cgroup *cgrp); int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode); int cgroup_rmdir(struct kernfs_node *kn); @@ -285,7 +285,6 @@ extern struct kernfs_syscall_ops cgroup1_kf_syscall_ops; extern const struct fs_parameter_spec cgroup1_fs_parameters[]; int proc_cgroupstats_show(struct seq_file *m, void *v); -bool cgroup1_ssid_disabled(int ssid); void cgroup1_pidlist_destroy_all(struct cgroup *cgrp); void cgroup1_release_agent(struct work_struct *work); void cgroup1_check_for_release(struct cgroup *cgrp); diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c index a4337c9b5287..7f55a7bc8402 100644 --- a/kernel/cgroup/cgroup-v1.c +++ b/kernel/cgroup/cgroup-v1.c @@ -2,7 +2,7 @@ #include "cgroup-internal.h" #include <linux/ctype.h> -#include <linux/kmod.h> +#include <linux/umh.h> #include <linux/sort.h> #include <linux/delay.h> #include <linux/mm.h> @@ -506,8 +506,8 @@ static ssize_t __cgroup1_procs_write(struct kernfs_open_file *of, enum cgroup_attach_lock_mode lock_mode; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENODEV; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); task = cgroup_procs_write_start(buf, threadgroup, &lock_mode); ret = PTR_ERR_OR_ZERO(task); @@ -569,8 +569,8 @@ static ssize_t cgroup_release_agent_write(struct kernfs_open_file *of, return -EPERM; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENODEV; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); spin_lock(&release_agent_path_lock); strscpy(cgrp->root->release_agent_path, strstrip(buf), sizeof(cgrp->root->release_agent_path)); @@ -605,10 +605,7 @@ static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css, static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css, struct cftype *cft, u64 val) { - if (val) - set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags); - else - clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags); + assign_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags, val); return 0; } @@ -621,10 +618,7 @@ static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css, static int cgroup_clone_children_write(struct cgroup_subsys_state *css, struct cftype *cft, u64 val) { - if (val) - set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); - else - clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); + assign_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags, val); return 0; } @@ -1097,7 +1091,9 @@ int cgroup1_reconfigure(struct fs_context *fc) int ret = 0; u32 added_mask, removed_mask; - cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); + ret = cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); + if (unlikely(ret)) + return ret; /* See what subsystems are wanted */ ret = check_cgroupfs_options(fc); @@ -1262,7 +1258,9 @@ int cgroup1_get_tree(struct fs_context *fc) if (!ns_capable(ctx->ns->user_ns, CAP_SYS_ADMIN)) return -EPERM; - cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); + ret = cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); + if (unlikely(ret)) + return ret; ret = cgroup1_root_to_use(fc); if (!ret && !percpu_ref_tryget_live(&ctx->root->cgrp.self.refcnt)) diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 227d09704ca5..14634246a666 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -548,20 +548,11 @@ static struct cgroup_subsys_state *cgroup_e_css_by_mask(struct cgroup *cgrp, struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp, struct cgroup_subsys *ss) { - struct cgroup_subsys_state *css; - if (!CGROUP_HAS_SUBSYS_CONFIG) return NULL; - do { - css = cgroup_css(cgrp, ss); - - if (css) - return css; - cgrp = cgroup_parent(cgrp); - } while (cgrp); - - return init_css_set.subsys[ss->id]; + return rcu_dereference_check(cgrp->e_css[ss->id], + lockdep_is_held(&cgroup_mutex)); } /** @@ -585,17 +576,10 @@ struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgrp, rcu_read_lock(); - do { - css = cgroup_css(cgrp, ss); - - if (css && css_tryget_online(css)) - goto out_unlock; - cgrp = cgroup_parent(cgrp); - } while (cgrp); + css = cgroup_e_css(cgrp, ss); + while (!css_tryget_online(css)) + css = cgroup_e_css(cgroup_parent(css->cgroup), ss); - css = init_css_set.subsys[ss->id]; - css_get(css); -out_unlock: rcu_read_unlock(); return css; } @@ -1379,7 +1363,10 @@ static void cgroup_destroy_root(struct cgroup_root *root) trace_cgroup_destroy_root(root); - cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); + /* runs off a workqueue, no signal can interrupt the drain */ + ret = cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); + if (WARN_ON_ONCE(ret)) + cgroup_lock(); BUG_ON(atomic_read(&root->nr_cgrps)); BUG_ON(!list_empty(&cgrp->self.children)); @@ -1685,9 +1672,10 @@ void cgroup_kn_unlock(struct kernfs_node *kn) * This helper is to be used by a cgroup kernfs method currently servicing * @kn. It breaks the active protection, performs cgroup locking and * verifies that the associated cgroup is alive. Returns the cgroup if - * alive; otherwise, %NULL. A successful return should be undone by a - * matching cgroup_kn_unlock() invocation. If @drain_offline is %true, the - * cgroup is drained of offlining csses before return. + * alive; otherwise, an ERR_PTR value. A successful return should be undone by + * a matching cgroup_kn_unlock() invocation. If @drain_offline is %true, the + * cgroup is drained of offlining csses before return, and an interrupted drain + * fails with -ERESTARTSYS. * * Any cgroup kernfs method implementation which requires locking the * associated cgroup should use this helper. It avoids nesting cgroup @@ -1697,6 +1685,7 @@ void cgroup_kn_unlock(struct kernfs_node *kn) struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn, bool drain_offline) { struct cgroup *cgrp; + int ret; if (kernfs_type(kn) == KERNFS_DIR) cgrp = kn->priv; @@ -1710,19 +1699,25 @@ struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn, bool drain_offline) * break the active_ref protection. */ if (!cgroup_tryget(cgrp)) - return NULL; + return ERR_PTR(-ENODEV); kernfs_break_active_protection(kn); - if (drain_offline) - cgroup_lock_and_drain_offline(cgrp); - else + if (drain_offline) { + ret = cgroup_lock_and_drain_offline(cgrp); + if (unlikely(ret)) { + kernfs_unbreak_active_protection(kn); + cgroup_put(cgrp); + return ERR_PTR(ret); + } + } else { cgroup_lock(); + } if (!cgroup_is_dead(cgrp)) return cgrp; cgroup_kn_unlock(kn); - return NULL; + return ERR_PTR(-ENODEV); } static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft) @@ -2131,12 +2126,24 @@ void init_cgroup_root(struct cgroup_fs_context *ctx) { struct cgroup_root *root = ctx->root; struct cgroup *cgrp = &root->cgrp; + struct cgroup_subsys *ss; + int ssid; INIT_LIST_HEAD_RCU(&root->root_list); atomic_set(&root->nr_cgrps, 1); cgrp->root = root; init_cgroup_housekeeping(cgrp); + /* + * A root cgroup's effective css is always the root css in + * init_css_set.subsys[], whichever hierarchy the subsystem is bound + * to, so rebind_subsystems() doesn't need to update e_css[]. For + * cgrp_dfl_root this runs before the root csses exist, and + * online_css() sets the entries when they come online. + */ + for_each_subsys(ss, ssid) + RCU_INIT_POINTER(cgrp->e_css[ssid], init_css_set.subsys[ssid]); + /* DYNMODS must be modified through cgroup_favor_dynmods() */ root->flags = ctx->flags & ~CGRP_ROOT_FAVOR_DYNMODS; if (ctx->release_agent) @@ -3320,16 +3327,20 @@ out_finish: * @cgrp: root of the target subtree * * Because css offlining is asynchronous, userland may try to re-enable a - * controller while the previous css is still around. This function grabs - * cgroup_mutex and drains the previous css instances of @cgrp's subtree. + * controller while the previous css is still around. This function grabs + * cgroup_mutex and waits until no css in @cgrp's subtree is dying. A dying css + * offlines only after every task that still pins it has finished exiting, which + * can take arbitrarily long, so the wait is interruptible. + * + * Returns 0 with cgroup_mutex held once the subtree is drained, or -ERESTARTSYS + * without it if interrupted by a signal. */ -void cgroup_lock_and_drain_offline(struct cgroup *cgrp) - __acquires(&cgroup_mutex) +int cgroup_lock_and_drain_offline(struct cgroup *cgrp) { struct cgroup *dsct; struct cgroup_subsys_state *d_css; struct cgroup_subsys *ss; - int ssid; + int ssid, ret; restart: cgroup_lock(); @@ -3343,17 +3354,20 @@ restart: continue; cgroup_get_live(dsct); - prepare_to_wait(&dsct->offline_waitq, &wait, - TASK_UNINTERRUPTIBLE); - + ret = prepare_to_wait_event(&dsct->offline_waitq, &wait, + TASK_INTERRUPTIBLE); cgroup_unlock(); - schedule(); + if (!ret) + schedule(); finish_wait(&dsct->offline_waitq, &wait); - cgroup_put(dsct); + if (unlikely(ret)) + return ret; goto restart; } } + + return 0; } /** @@ -3650,8 +3664,8 @@ static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of, } cgrp = cgroup_kn_lock_live(of->kn, true); - if (!cgrp) - return -ENODEV; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); for_each_subsys(ss, ssid) { if (enable & (1 << ssid)) { @@ -3790,8 +3804,8 @@ static ssize_t cgroup_type_write(struct kernfs_open_file *of, char *buf, /* drain dying csses before we re-apply (threaded) subtree control */ cgrp = cgroup_kn_lock_live(of->kn, true); - if (!cgrp) - return -ENOENT; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); /* threaded can only be enabled */ ret = cgroup_enable_threaded(cgrp); @@ -3833,8 +3847,8 @@ static ssize_t cgroup_max_descendants_write(struct kernfs_open_file *of, return -ERANGE; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENOENT; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); WRITE_ONCE(cgrp->max_descendants, descendants); @@ -3876,8 +3890,8 @@ static ssize_t cgroup_max_depth_write(struct kernfs_open_file *of, return -ERANGE; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENOENT; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); WRITE_ONCE(cgrp->max_depth, depth); @@ -4075,8 +4089,8 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf, ssize_t ret = 0; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENODEV; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); ctx = of->priv; if (!ctx) { @@ -4192,8 +4206,8 @@ static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, return -ERANGE; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENOENT; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); psi = cgroup_psi(cgrp); if (psi->enabled != enable) { @@ -4268,8 +4282,8 @@ static ssize_t cgroup_freeze_write(struct kernfs_open_file *of, return -ERANGE; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENOENT; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); cgroup_freeze(cgrp, freeze); @@ -4330,8 +4344,8 @@ static ssize_t cgroup_kill_write(struct kernfs_open_file *of, char *buf, return -ERANGE; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENOENT; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); /* * Killing is a process directed operation, i.e. the whole thread-group @@ -5488,8 +5502,8 @@ static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf, enum cgroup_attach_lock_mode lock_mode; dst_cgrp = cgroup_kn_lock_live(of->kn, false); - if (!dst_cgrp) - return -ENODEV; + if (IS_ERR(dst_cgrp)) + return PTR_ERR(dst_cgrp); task = cgroup_procs_write_start(buf, threadgroup, &lock_mode); ret = PTR_ERR_OR_ZERO(task); @@ -5859,6 +5873,22 @@ static void init_and_link_css(struct cgroup_subsys_state *css, BUG_ON(cgroup_css(cgrp, ss)); } +static void cgroup_update_e_css(struct cgroup *cgrp, struct cgroup_subsys *ss) +{ + struct cgroup_subsys_state *d_css; + + lockdep_assert_held(&cgroup_mutex); + + css_for_each_descendant_pre(d_css, &cgrp->self) { + struct cgroup *dsct = d_css->cgroup; + struct cgroup_subsys_state *css = cgroup_css(dsct, ss); + + if (!css) + css = cgroup_e_css(cgroup_parent(dsct), ss); + rcu_assign_pointer(dsct->e_css[ss->id], css); + } +} + /* invoke ->css_online() on a new CSS and mark it online if successful */ static int online_css(struct cgroup_subsys_state *css) { @@ -5872,6 +5902,7 @@ static int online_css(struct cgroup_subsys_state *css) if (!ret) { css->flags |= CSS_ONLINE; rcu_assign_pointer(css->cgroup->subsys[ss->id], css); + cgroup_update_e_css(css->cgroup, ss); atomic_inc(&css->online_cnt); if (css->parent) { @@ -5898,6 +5929,7 @@ static void offline_css(struct cgroup_subsys_state *css) css->flags &= ~CSS_ONLINE; RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL); + cgroup_update_e_css(css->cgroup, ss); wake_up_all(&css->cgroup->offline_waitq); } @@ -5969,6 +6001,7 @@ static struct cgroup *cgroup_create(struct cgroup *parent, const char *name, { struct cgroup_root *root = parent->root; struct cgroup *cgrp, *tcgrp; + struct cgroup_subsys *ss; struct kernfs_node *kn; int i, level = parent->level + 1; int ret; @@ -6013,6 +6046,9 @@ static struct cgroup *cgroup_create(struct cgroup *parent, const char *name, for (tcgrp = cgrp; tcgrp; tcgrp = cgroup_parent(tcgrp)) cgrp->ancestors[tcgrp->level] = tcgrp; + for_each_subsys(ss, i) + RCU_INIT_POINTER(cgrp->e_css[i], cgroup_e_css(parent, ss)); + /* * New cgroup inherits effective freeze counter, and * if the parent has to be frozen, the child has too. @@ -6123,8 +6159,8 @@ int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode) return -EINVAL; parent = cgroup_kn_lock_live(parent_kn, false); - if (!parent) - return -ENODEV; + if (IS_ERR(parent)) + return PTR_ERR(parent); if (!cgroup_check_hierarchy_limits(parent)) { ret = -EAGAIN; @@ -6400,7 +6436,7 @@ int cgroup_rmdir(struct kernfs_node *kn) int ret = 0; cgrp = cgroup_kn_lock_live(kn, false); - if (!cgrp) + if (IS_ERR(cgrp)) return 0; ret = cgroup_destroy_locked(cgrp); @@ -7197,15 +7233,19 @@ static void do_cgroup_task_dead(struct task_struct *tsk) * the cgroup and task_struct can be pinned indefinitely. Bounce through lazy * irq_work to allow batching while ensuring timely completion. */ -static DEFINE_PER_CPU(struct llist_head, cgrp_dead_tasks); -static DEFINE_PER_CPU(struct irq_work, cgrp_dead_tasks_iwork); +struct cgroup_dead { + struct irq_work iwork; + struct llist_head tasks; +}; +static DEFINE_PER_CPU(struct cgroup_dead, cgroup_dead); static void cgrp_dead_tasks_iwork_fn(struct irq_work *iwork) { + struct cgroup_dead *cgrp_dead = container_of(iwork, struct cgroup_dead, iwork); struct llist_node *lnode; struct task_struct *task, *next; - lnode = llist_del_all(this_cpu_ptr(&cgrp_dead_tasks)); + lnode = llist_del_all(&cgrp_dead->tasks); llist_for_each_entry_safe(task, next, lnode, cg_dead_lnode) { do_cgroup_task_dead(task); put_task_struct(task); @@ -7217,17 +7257,20 @@ static void __init cgroup_rt_init(void) int cpu; for_each_possible_cpu(cpu) { - init_llist_head(per_cpu_ptr(&cgrp_dead_tasks, cpu)); - per_cpu(cgrp_dead_tasks_iwork, cpu) = - IRQ_WORK_INIT_LAZY(cgrp_dead_tasks_iwork_fn); + struct cgroup_dead *cgrp_dead = per_cpu_ptr(&cgroup_dead, cpu); + + init_llist_head(&cgrp_dead->tasks); + cgrp_dead->iwork = IRQ_WORK_INIT_LAZY(cgrp_dead_tasks_iwork_fn); } } void cgroup_task_dead(struct task_struct *task) { + struct cgroup_dead *cgrp_dead = this_cpu_ptr(&cgroup_dead); + get_task_struct(task); - llist_add(&task->cg_dead_lnode, this_cpu_ptr(&cgrp_dead_tasks)); - irq_work_queue(this_cpu_ptr(&cgrp_dead_tasks_iwork)); + llist_add(&task->cg_dead_lnode, &cgrp_dead->tasks); + irq_work_queue(&cgrp_dead->iwork); } #else /* CONFIG_PREEMPT_RT */ static void __init cgroup_rt_init(void) {} diff --git a/kernel/cgroup/cpuset-internal.h b/kernel/cgroup/cpuset-internal.h index e7d010661fd3..5da9c91295b0 100644 --- a/kernel/cgroup/cpuset-internal.h +++ b/kernel/cgroup/cpuset-internal.h @@ -304,7 +304,9 @@ void cpuset_callback_lock_irq(void); void cpuset_callback_unlock_irq(void); void cpuset_update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus); void cpuset_update_tasks_nodemask(struct cpuset *cs); -int cpuset_update_flag(cpuset_flagbits_t bit, struct cpuset *cs, int turning_on); +struct cpuset *dup_or_alloc_cpuset(struct cpuset *cs); +void free_cpuset(struct cpuset *cs); +int validate_change(struct cpuset *cur, struct cpuset *trial); ssize_t cpuset_write_resmask(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off); int cpuset_common_seq_show(struct seq_file *sf, void *v); @@ -318,7 +320,6 @@ void cpuset_full_unlock(void); extern struct cftype cpuset1_files[]; void cpuset1_update_task_spread_flags(struct cpuset *cs, struct task_struct *tsk); -void cpuset1_update_tasks_flags(struct cpuset *cs); void cpuset1_hotplug_update_tasks(struct cpuset *cs, struct cpumask *new_cpus, nodemask_t *new_mems, bool cpus_updated, bool mems_updated); @@ -328,11 +329,11 @@ void cpuset1_init(struct cpuset *cs); void cpuset1_online_css(struct cgroup_subsys_state *css); int cpuset1_generate_sched_domains(cpumask_var_t **domains, struct sched_domain_attr **attributes); +void cpuset1_offline_css(struct cpuset *cs); #else static inline void cpuset1_update_task_spread_flags(struct cpuset *cs, struct task_struct *tsk) {} -static inline void cpuset1_update_tasks_flags(struct cpuset *cs) {} static inline void cpuset1_hotplug_update_tasks(struct cpuset *cs, struct cpumask *new_cpus, nodemask_t *new_mems, bool cpus_updated, bool mems_updated) {} @@ -344,6 +345,7 @@ static inline void cpuset1_init(struct cpuset *cs) {} static inline void cpuset1_online_css(struct cgroup_subsys_state *css) {} static inline int cpuset1_generate_sched_domains(cpumask_var_t **domains, struct sched_domain_attr **attributes) { return 0; }; +static inline void cpuset1_offline_css(struct cpuset *cs) {} #endif /* CONFIG_CPUSETS_V1 */ diff --git a/kernel/cgroup/cpuset-v1.c b/kernel/cgroup/cpuset-v1.c index 562ad35f00d0..7c7f2887dfaf 100644 --- a/kernel/cgroup/cpuset-v1.c +++ b/kernel/cgroup/cpuset-v1.c @@ -229,7 +229,7 @@ void cpuset1_update_task_spread_flags(struct cpuset *cs, * function is called with cpuset_mutex held, cpuset membership stays * stable. */ -void cpuset1_update_tasks_flags(struct cpuset *cs) +static void cpuset1_update_tasks_flags(struct cpuset *cs) { struct css_task_iter it; struct task_struct *task; @@ -473,6 +473,51 @@ static u64 cpuset_read_u64(struct cgroup_subsys_state *css, struct cftype *cft) return 0; } +/* + * cpuset_update_flag - read a 0 or a 1 in a file and update associated flag + * bit: the bit to update (see cpuset_flagbits_t) + * cs: the cpuset to update + * turning_on: whether the flag is being set or cleared + * + * Call with cpuset_mutex held. + */ +static int cpuset_update_flag(cpuset_flagbits_t bit, struct cpuset *cs, + int turning_on) +{ + struct cpuset *trialcs; + int balance_flag_changed; + int spread_page_changed; + int err; + + trialcs = dup_or_alloc_cpuset(cs); + if (!trialcs) + return -ENOMEM; + + assign_bit(bit, &trialcs->flags, turning_on); + + err = validate_change(cs, trialcs); + if (err < 0) + goto out; + + balance_flag_changed = (is_sched_load_balance(cs) != + is_sched_load_balance(trialcs)); + + spread_page_changed = is_spread_page(cs) != is_spread_page(trialcs); + + cpuset_callback_lock_irq(); + cs->flags = trialcs->flags; + cpuset_callback_unlock_irq(); + + if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed) + rebuild_sched_domains_locked(); + + if (spread_page_changed) + cpuset1_update_tasks_flags(cs); +out: + free_cpuset(trialcs); + return err; +} + static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft, u64 val) { @@ -581,6 +626,18 @@ void cpuset1_online_css(struct cgroup_subsys_state *css) cpuset_callback_unlock_irq(); } +/* + * Legacy-hierarchy handling when a cpuset is taken offline: if it had + * sched_load_balance enabled, turn it off so the scheduler domains are + * rebuilt. A no-op on the default hierarchy. + */ +void cpuset1_offline_css(struct cpuset *cs) +{ + if (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) && + is_sched_load_balance(cs)) + cpuset_update_flag(CS_SCHED_LOAD_BALANCE, cs, 0); +} + static void update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c) { diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 2538faac9aba..0ebba2d646c3 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -153,6 +153,12 @@ static cpumask_var_t isolated_cpus; /* CSCB */ static bool update_housekeeping; /* RWCS */ /* + * Set if "cpuset_v2_mode" mount option is used + * Cached at bind time and not lock protected; accessed via {READ,WRITE}_ONCE + */ +static bool cpuset_v2_mode; + +/* * Copy of isolated_cpus to be passed to housekeeping_update() */ static cpumask_var_t isolated_hk_cpus; /* T */ @@ -162,7 +168,6 @@ static cpumask_var_t isolated_hk_cpus; /* T */ * It can be set in * - update_partition_sd_lb() * - update_cpumasks_hier() - * - cpuset_update_flag() * - cpuset_hotplug_update_tasks() * - cpuset_handle_hotplug() * @@ -439,8 +444,7 @@ static inline bool cpuset_v2(void) */ static inline bool is_in_v2_mode(void) { - return cpuset_v2() || - (cpuset_cgrp_subsys.root->flags & CGRP_ROOT_CPUSET_V2_MODE); + return cpuset_v2() || READ_ONCE(cpuset_v2_mode); } /** @@ -513,10 +517,26 @@ static void guarantee_active_cpus(struct task_struct *tsk, rcu_read_lock(); cs = task_cs(tsk); - while (!cpumask_intersects(cs->effective_cpus, pmask)) + while (!cpumask_intersects(cs->effective_cpus, pmask)) { cs = parent_cs(cs); - + if (unlikely(!cs)) { + /* + * The top cpuset doesn't have any active cpu as a + * consequence of a race between its caller and the cpu + * hotplug operation where cpu_active_mask is updated + * asynchronously before cpuset_handle_hotplug() is + * being called to adjust the effective_cpus of the + * affected cpusets. But we know the top cpuset's + * effective_cpus is on its way to be identical to + * cpu_active_mask minus the exclusive CPUs dedicated + * to other valid cpuset partitions. Just pass back + * the filtered cpu_active_mask in this case. + */ + goto out_unlock; + } + } cpumask_and(pmask, pmask, cs->effective_cpus); +out_unlock: rcu_read_unlock(); } @@ -608,7 +628,7 @@ static inline void free_tmpmasks(struct tmpmasks *tmp) * * Return: Pointer to newly allocated cpuset on success, NULL on failure */ -static struct cpuset *dup_or_alloc_cpuset(struct cpuset *cs) +struct cpuset *dup_or_alloc_cpuset(struct cpuset *cs) { struct cpuset *trial; @@ -649,7 +669,7 @@ static struct cpuset *dup_or_alloc_cpuset(struct cpuset *cs) * free_cpuset - free the cpuset * @cs: the cpuset to be freed */ -static inline void free_cpuset(struct cpuset *cs) +void free_cpuset(struct cpuset *cs) { free_cpumask_var(cs->cpus_allowed); free_cpumask_var(cs->effective_cpus); @@ -743,7 +763,7 @@ static inline bool mems_excl_conflict(struct cpuset *cs1, struct cpuset *cs2) * Return 0 if valid, -errno if not. */ -static int validate_change(struct cpuset *cur, struct cpuset *trial) +int validate_change(struct cpuset *cur, struct cpuset *trial) { struct cgroup_subsys_state *css; struct cpuset *c, *par; @@ -771,8 +791,8 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial) * For v1, effective_cpus == cpus_allowed & user_xcpus() returns * cpus_allowed. * - * For v2, is_cpu_exclusive() & is_sched_load_balance() are true only - * for non-isolated partition root. At this point, the target + * For v2, is_partition_valid(cur) & is_sched_load_balance() are true + * only for non-isolated partition root. At this point, the target * effective_cpus isn't computed yet. user_xcpus() is the best * approximation. * @@ -781,7 +801,8 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial) * becomes an issue. */ ret = -EBUSY; - if (is_cpu_exclusive(cur) && is_sched_load_balance(cur) && + if ((is_partition_valid(cur) || (!cpuset_v2() && is_cpu_exclusive(cur))) && + is_sched_load_balance(cur) && !cpuset_cpumask_can_shrink(cur->effective_cpus, user_xcpus(trial))) goto out; @@ -1174,25 +1195,6 @@ static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs, struct tmpmasks *tmp); /* - * Update partition exclusive flag - * - * Return: 0 if successful, an error code otherwise - */ -static int update_partition_exclusive_flag(struct cpuset *cs, int new_prs) -{ - bool exclusive = (new_prs > PRS_MEMBER); - - if (exclusive && !is_cpu_exclusive(cs)) { - if (cpuset_update_flag(CS_CPU_EXCLUSIVE, cs, 1)) - return PERR_NOTEXCL; - } else if (!exclusive && is_cpu_exclusive(cs)) { - /* Turning off CS_CPU_EXCLUSIVE will not return error */ - cpuset_update_flag(CS_CPU_EXCLUSIVE, cs, 0); - } - return 0; -} - -/* * Update partition load balance flag and/or rebuild sched domain * * Changing load balance flag will automatically call @@ -1216,10 +1218,7 @@ static void update_partition_sd_lb(struct cpuset *cs, int old_prs) } if (new_lb != !!is_sched_load_balance(cs)) { rebuild_domains = true; - if (new_lb) - set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); - else - clear_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); + assign_bit(CS_SCHED_LOAD_BALANCE, &cs->flags, new_lb); } if (rebuild_domains) @@ -1250,11 +1249,9 @@ static void reset_partition_data(struct cpuset *cs) lockdep_assert_held(&callback_lock); - if (cpumask_empty(cs->exclusive_cpus)) { + if (cpumask_empty(cs->exclusive_cpus)) cpumask_clear(cs->effective_xcpus); - if (is_cpu_exclusive(cs)) - clear_bit(CS_CPU_EXCLUSIVE, &cs->flags); - } + if (!cpumask_and(cs->effective_cpus, parent->effective_cpus, cs->cpus_allowed)) cpumask_copy(cs->effective_cpus, parent->effective_cpus); } @@ -1591,10 +1588,11 @@ static int remote_partition_enable(struct cpuset *cs, int new_prs, * above it or remote partition root underneath it is not allowed. */ compute_excpus(cs, tmp->new_cpus); - WARN_ON_ONCE(cpumask_intersects(tmp->new_cpus, subpartitions_cpus)); if (!cpumask_intersects(tmp->new_cpus, cpu_active_mask) || cpumask_subset(top_cpuset.effective_cpus, tmp->new_cpus)) return PERR_INVCPUS; + if (cpumask_intersects(tmp->new_cpus, subpartitions_cpus)) + return PERR_NOCPUS; if (((new_prs == PRS_ISOLATED) && !isolated_cpus_can_update(tmp->new_cpus, NULL)) || prstate_housekeeping_conflict(new_prs, tmp->new_cpus)) @@ -2031,19 +2029,6 @@ write_error: return 0; /* - * Transitioning between invalid to valid or vice versa may require - * changing CS_CPU_EXCLUSIVE. In the case of partcmd_update, - * validate_change() has already been successfully called and - * CPU lists in cs haven't been updated yet. So defer it to later. - */ - if ((old_prs != new_prs) && (cmd != partcmd_update)) { - int err = update_partition_exclusive_flag(cs, new_prs); - - if (err) - return err; - } - - /* * Change the parent's effective_cpus & effective_xcpus (top cpuset * only). * @@ -2065,9 +2050,6 @@ write_error: spin_unlock_irq(&callback_lock); - if ((old_prs != new_prs) && (cmd == partcmd_update)) - update_partition_exclusive_flag(cs, new_prs); - if (adding || deleting) { cpuset_update_tasks_cpumask(parent, tmp->addmask); update_sibling_cpumasks(parent, cs, tmp); @@ -2308,10 +2290,8 @@ get_css: */ if (cpuset_v2() && !is_partition_valid(cp) && (is_sched_load_balance(parent) != is_sched_load_balance(cp))) { - if (is_sched_load_balance(parent)) - set_bit(CS_SCHED_LOAD_BALANCE, &cp->flags); - else - clear_bit(CS_SCHED_LOAD_BALANCE, &cp->flags); + assign_bit(CS_SCHED_LOAD_BALANCE, &cp->flags, + is_sched_load_balance(parent)); } /* @@ -2869,59 +2849,6 @@ bool current_cpuset_is_being_rebound(void) return ret; } -/* - * cpuset_update_flag - read a 0 or a 1 in a file and update associated flag - * bit: the bit to update (see cpuset_flagbits_t) - * cs: the cpuset to update - * turning_on: whether the flag is being set or cleared - * - * Call with cpuset_mutex held. - */ - -int cpuset_update_flag(cpuset_flagbits_t bit, struct cpuset *cs, - int turning_on) -{ - struct cpuset *trialcs; - int balance_flag_changed; - int spread_page_changed; - int err; - - trialcs = dup_or_alloc_cpuset(cs); - if (!trialcs) - return -ENOMEM; - - if (turning_on) - set_bit(bit, &trialcs->flags); - else - clear_bit(bit, &trialcs->flags); - - err = validate_change(cs, trialcs); - if (err < 0) - goto out; - - balance_flag_changed = (is_sched_load_balance(cs) != - is_sched_load_balance(trialcs)); - - spread_page_changed = is_spread_page(cs) != is_spread_page(trialcs); - - spin_lock_irq(&callback_lock); - cs->flags = trialcs->flags; - spin_unlock_irq(&callback_lock); - - if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed) { - if (cpuset_v2()) - cpuset_force_rebuild(); - else - rebuild_sched_domains_locked(); - } - - if (spread_page_changed) - cpuset1_update_tasks_flags(cs); -out: - free_cpuset(trialcs); - return err; -} - /** * update_prstate - update partition_root_state * @cs: the cpuset to update @@ -2949,10 +2876,6 @@ static int update_prstate(struct cpuset *cs, int new_prs) if (alloc_tmpmasks(&tmpmask)) return -ENOMEM; - err = update_partition_exclusive_flag(cs, new_prs); - if (err) - goto out; - if (!old_prs) { /* * cpus_allowed and exclusive_cpus cannot be both empty. @@ -3016,13 +2939,10 @@ static int update_prstate(struct cpuset *cs, int new_prs) } out: /* - * Make partition invalid & disable CS_CPU_EXCLUSIVE if an error - * happens. + * Make partition invalid if an error happens. */ - if (err) { + if (err) new_prs = -new_prs; - update_partition_exclusive_flag(cs, new_prs); - } spin_lock_irq(&callback_lock); cs->partition_root_state = new_prs; @@ -3044,8 +2964,6 @@ out: update_partition_sd_lb(cs, old_prs); notify_partition_change(cs, old_prs); - if (force_sd_rebuild) - rebuild_sched_domains_locked(); free_tmpmasks(&tmpmask); return 0; } @@ -3684,21 +3602,12 @@ static int cpuset_css_online(struct cgroup_subsys_state *css) return 0; } -/* - * If the cpuset being removed has its flag 'sched_load_balance' - * enabled, then simulate turning sched_load_balance off, which - * will call rebuild_sched_domains_locked(). That is not needed - * in the default hierarchy where only changes in partition - * will cause repartitioning. - */ static void cpuset_css_offline(struct cgroup_subsys_state *css) { struct cpuset *cs = css_cs(css); cpuset_full_lock(); - if (!cpuset_v2() && is_sched_load_balance(cs)) - cpuset_update_flag(CS_SCHED_LOAD_BALANCE, cs, 0); - + cpuset1_offline_css(cs); cpuset_dec(); cpuset_full_unlock(); } @@ -3731,6 +3640,8 @@ static void cpuset_bind(struct cgroup_subsys_state *root_css) mutex_lock(&cpuset_mutex); spin_lock_irq(&callback_lock); + WRITE_ONCE(cpuset_v2_mode, + !!(cpuset_cgrp_subsys.root->flags & CGRP_ROOT_CPUSET_V2_MODE)); if (is_in_v2_mode()) { cpumask_copy(top_cpuset.cpus_allowed, cpu_possible_mask); cpumask_copy(top_cpuset.effective_xcpus, cpu_possible_mask); @@ -4043,15 +3954,16 @@ static void cpuset_handle_hotplug(void) static cpumask_t new_cpus; static nodemask_t new_mems; bool cpus_updated, mems_updated; - bool on_dfl = is_in_v2_mode(); + bool on_dfl; struct tmpmasks tmp, *ptmp = NULL; - if (on_dfl && !alloc_tmpmasks(&tmp)) - ptmp = &tmp; - lockdep_assert_cpus_held(); mutex_lock(&cpuset_mutex); + on_dfl = is_in_v2_mode(); + if (on_dfl && !alloc_tmpmasks(&tmp)) + ptmp = &tmp; + /* fetch the available cpus/mems and find out which changed how */ cpumask_copy(&new_cpus, cpu_active_mask); new_mems = node_states[N_MEMORY]; @@ -4311,7 +4223,7 @@ int cpuset_num_cpus(struct cgroup *cgrp) int nr = num_online_cpus(); struct cpuset *cs; - if (is_in_v2_mode()) { + if (cpuset_v2()) { guard(rcu)(); cs = css_cs(cgroup_e_css(cgrp, &cpuset_cgrp_subsys)); if (cs) diff --git a/kernel/cgroup/debug.c b/kernel/cgroup/debug.c index 883347b87842..aba243b5fd5f 100644 --- a/kernel/cgroup/debug.c +++ b/kernel/cgroup/debug.c @@ -45,13 +45,13 @@ static int current_css_set_read(struct seq_file *seq, void *v) struct cgroup_subsys_state *css; int i, refcnt; - if (!cgroup_kn_lock_live(of->kn, false)) + if (IS_ERR(cgroup_kn_lock_live(of->kn, false))) return -ENODEV; spin_lock_irq(&css_set_lock); cset = task_css_set(current); refcnt = refcount_read(&cset->refcount); - seq_printf(seq, "css_set %pK %d", cset, refcnt); + seq_printf(seq, "css_set %p %d", cset, refcnt); if (refcnt > cset->nr_tasks) seq_printf(seq, " +%d", refcnt - cset->nr_tasks); seq_puts(seq, "\n"); @@ -125,10 +125,10 @@ static int cgroup_css_links_read(struct seq_file *seq, void *v) * Print out the proc_cset and threaded_cset relationship * and highlight difference between refcount and task_count. */ - seq_printf(seq, "css_set %pK", cset); + seq_printf(seq, "css_set %p", cset); if (rcu_dereference_protected(cset->dom_cset, 1) != cset) { threaded_csets++; - seq_printf(seq, "=>%pK", cset->dom_cset); + seq_printf(seq, "=>%p", cset->dom_cset); } if (!list_empty(&cset->threaded_csets)) { struct css_set *tcset; @@ -137,7 +137,7 @@ static int cgroup_css_links_read(struct seq_file *seq, void *v) list_for_each_entry(tcset, &cset->threaded_csets, threaded_csets_node) { seq_puts(seq, idx ? "," : "<="); - seq_printf(seq, "%pK", tcset); + seq_printf(seq, "%p", tcset); idx++; } } else { @@ -206,8 +206,8 @@ static int cgroup_subsys_states_read(struct seq_file *seq, void *v) int i; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENODEV; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); for_each_subsys(ss, i) { css = rcu_dereference_check(cgrp->subsys[ss->id], true); @@ -254,8 +254,8 @@ static int cgroup_masks_read(struct seq_file *seq, void *v) struct cgroup *cgrp; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENODEV; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); cgroup_masks_read_one(seq, "subtree_control", cgrp->subtree_control); cgroup_masks_read_one(seq, "subtree_ss_mask", cgrp->subtree_ss_mask); diff --git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c index 4683f3d68022..d49ed5d4a923 100644 --- a/kernel/cgroup/dmem.c +++ b/kernel/cgroup/dmem.c @@ -28,7 +28,7 @@ DEFINE_STATIC_SRCU(dmemcg_srcu); struct dmem_cgroup_region { /** * @ref: References keeping the region alive. - * Keeps the region reference alive after a succesful RCU lookup. + * Keeps the region reference alive after a successful RCU lookup. */ struct kref ref; @@ -696,7 +696,7 @@ EXPORT_SYMBOL_GPL(dmem_cgroup_uncharge); * dmem_cgroup_try_charge() - Try charging a new allocation to a region. * @region: dmem region to charge * @size: Size (in bytes) to charge. - * @ret_pool: On succesfull allocation, the pool that is charged. + * @ret_pool: On successful allocation, the pool that is charged. * @ret_limit_pool: On a failed allocation, the limiting pool. * * This function charges the @region region for a size of @size bytes. @@ -706,8 +706,8 @@ EXPORT_SYMBOL_GPL(dmem_cgroup_uncharge); * * When this function fails with -EAGAIN and @ret_limit_pool is non-null, it * will be set to the pool for which the limit is hit. This can be used for - * eviction as argument to dmem_cgroup_evict_valuable(). This reference must be freed - * with @dmem_cgroup_pool_state_put(). + * eviction as argument to dmem_cgroup_state_evict_valuable(). This reference + * must be freed with @dmem_cgroup_pool_state_put(). * * Return: 0 on success, -EAGAIN on hitting a limit, or a negative errno on failure. */ diff --git a/kernel/cgroup/freezer.c b/kernel/cgroup/freezer.c index 6c18854bff34..35112d62318a 100644 --- a/kernel/cgroup/freezer.c +++ b/kernel/cgroup/freezer.c @@ -20,10 +20,7 @@ static bool cgroup_update_frozen_flag(struct cgroup *cgrp, bool frozen) if (test_bit(CGRP_FROZEN, &cgrp->flags) == frozen) return false; - if (frozen) - set_bit(CGRP_FROZEN, &cgrp->flags); - else - clear_bit(CGRP_FROZEN, &cgrp->flags); + assign_bit(CGRP_FROZEN, &cgrp->flags, frozen); cgroup_file_notify(&cgrp->events_file); TRACE_CGROUP_PATH(notify_frozen, cgrp, frozen); diff --git a/kernel/cgroup/namespace.c b/kernel/cgroup/namespace.c index ea4ee13936be..76660332ec64 100644 --- a/kernel/cgroup/namespace.c +++ b/kernel/cgroup/namespace.c @@ -53,8 +53,6 @@ struct cgroup_namespace *copy_cgroup_ns(u64 flags, struct ucounts *ucounts; struct css_set *cset; - BUG_ON(!old_ns); - if (!(flags & CLONE_NEWCGROUP)) { get_cgroup_ns(old_ns); return old_ns; diff --git a/kernel/cgroup/pids.c b/kernel/cgroup/pids.c index ecbb839d2acb..78cdc0558d0c 100644 --- a/kernel/cgroup/pids.c +++ b/kernel/cgroup/pids.c @@ -253,6 +253,11 @@ static void pids_event(struct pids_cgroup *pids_forking, } if (!cgroup_subsys_on_dfl(pids_cgrp_subsys) || cgrp_dfl_root.flags & CGRP_ROOT_PIDS_LOCAL_EVENTS) { + /* + * pids.events reports the local counter on legacy hierarchies + * and when pids_localevents is enabled. + */ + cgroup_file_notify(&p->events_file); cgroup_file_notify(&p->events_local_file); return; } diff --git a/kernel/compat.c b/kernel/compat.c index fb50f29d9b36..c754793c7d65 100644 --- a/kernel/compat.c +++ b/kernel/compat.c @@ -180,14 +180,18 @@ int get_compat_sigevent(struct sigevent *event, const struct compat_sigevent __user *u_event) { memset(event, 0, sizeof(*event)); - return (!access_ok(u_event, sizeof(*u_event)) || - __get_user(event->sigev_value.sival_int, - &u_event->sigev_value.sival_int) || - __get_user(event->sigev_signo, &u_event->sigev_signo) || - __get_user(event->sigev_notify, &u_event->sigev_notify) || - __get_user(event->sigev_notify_thread_id, - &u_event->sigev_notify_thread_id)) - ? -EFAULT : 0; + scoped_user_read_access(u_event, Efault) { + unsafe_get_user(event->sigev_value.sival_int, + &u_event->sigev_value.sival_int, Efault); + unsafe_get_user(event->sigev_signo, &u_event->sigev_signo, Efault); + unsafe_get_user(event->sigev_notify, &u_event->sigev_notify, Efault); + unsafe_get_user(event->sigev_notify_thread_id, + &u_event->sigev_notify_thread_id, Efault); + } + return 0; + +Efault: + return -EFAULT; } long compat_get_bitmap(unsigned long *mask, const compat_ulong_t __user *umask, diff --git a/kernel/cpu.c b/kernel/cpu.c index b3c8553d7bd6..376d297a6292 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -3103,6 +3103,11 @@ EXPORT_SYMBOL(__cpu_dying_mask); atomic_t __num_online_cpus __read_mostly; EXPORT_SYMBOL(__num_online_cpus); +#ifdef CONFIG_PREFERRED_CPU +struct cpumask __cpu_preferred_mask __read_mostly; +EXPORT_SYMBOL_GPL(__cpu_preferred_mask); +#endif + void init_cpu_present(const struct cpumask *src) { cpumask_copy(&__cpu_present_mask, src); @@ -3160,6 +3165,7 @@ void __init boot_cpu_init(void) /* Mark the boot cpu "present", "online" etc for SMP and UP case */ set_cpu_online(cpu, true); set_cpu_active(cpu, true); + set_cpu_preferred(cpu, true); set_cpu_present(cpu, true); set_cpu_possible(cpu, true); diff --git a/kernel/cpu_pm.c b/kernel/cpu_pm.c index 7481fbb947d3..a2a598ad6e6d 100644 --- a/kernel/cpu_pm.c +++ b/kernel/cpu_pm.c @@ -10,6 +10,7 @@ #include <linux/cpu_pm.h> #include <linux/module.h> #include <linux/notifier.h> +#include <linux/rcupdate.h> #include <linux/spinlock.h> #include <linux/syscore_ops.h> @@ -76,7 +77,8 @@ EXPORT_SYMBOL_GPL(cpu_pm_register_notifier); * * Remove a driver from the CPU PM notifier list. * - * This function has the same return conditions as raw_notifier_chain_unregister. + * This function may sleep, and has the same return conditions as + * raw_notifier_chain_unregister. */ int cpu_pm_unregister_notifier(struct notifier_block *nb) { @@ -86,6 +88,11 @@ int cpu_pm_unregister_notifier(struct notifier_block *nb) raw_spin_lock_irqsave(&cpu_pm_notifier.lock, flags); ret = raw_notifier_chain_unregister(&cpu_pm_notifier.chain, nb); raw_spin_unlock_irqrestore(&cpu_pm_notifier.lock, flags); + + /* Wait for the rcu_read_lock() walkers in cpu_pm_notify(). */ + if (!ret) + synchronize_rcu(); + return ret; } EXPORT_SYMBOL_GPL(cpu_pm_unregister_notifier); diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 2b36aa9fade0..d0bd2d0cf899 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -648,7 +648,7 @@ int crash_check_hotplug_support(void) * new list of CPUs and memory. To make changes to the elfcorehdr, it * should be large enough to permit a growing number of CPU and Memory * resources. One can estimate the elfcorehdr memory size based on - * NR_CPUS_DEFAULT and CRASH_MAX_MEMORY_RANGES. The elfcorehdr is + * NR_CPUS and CRASH_MAX_MEMORY_RANGES. The elfcorehdr is * excluded from SHA verification by default if the architecture * supports crash hotplug. */ diff --git a/kernel/cred.c b/kernel/cred.c index 3df4e15bd67f..d01df40a9a38 100644 --- a/kernel/cred.c +++ b/kernel/cred.c @@ -414,7 +414,7 @@ int commit_creds(struct cred *new) inc_rlimit_ucounts(new->ucounts, UCOUNT_RLIMIT_NPROC, 1); rcu_assign_pointer(task->real_cred, new); - rcu_assign_pointer(task->cred, new); + task->cred = new; if (new->user != old->user || new->user_ns != old->user_ns) dec_rlimit_ucounts(old->ucounts, UCOUNT_RLIMIT_NPROC, 1); if (new->user_ns != old->user_ns) @@ -537,7 +537,8 @@ void __init cred_init(void) { /* allocate a slab in which we can store credentials */ cred_jar = KMEM_CACHE(cred, - SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT); + SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT | + SLAB_NO_MERGE); } /** diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c index 66093460584e..90937dc7c627 100644 --- a/kernel/dma/contiguous.c +++ b/kernel/dma/contiguous.c @@ -451,8 +451,8 @@ struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) * * This function releases memory allocated by dma_alloc_contiguous(). As the * cma_release returns false when provided pages do not belong to contiguous - * area and true otherwise, this function then does a fallback __free_pages() - * upon a false-return. + * area and true otherwise, this function then does a fallback + * free_pages_exact() upon a false-return. */ void dma_free_contiguous(struct device *dev, struct page *page, size_t size) { @@ -476,7 +476,7 @@ void dma_free_contiguous(struct device *dev, struct page *page, size_t size) } /* not in any cma, free from buddy */ - __free_pages(page, get_order(size)); + free_pages_exact(page_address(page), size); } /* diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index 026d50de340e..36c1f7cbb2ae 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -489,8 +489,10 @@ static int active_cacheline_insert(struct dma_debug_entry *entry, /* If the device is not writing memory then we don't have any * concerns about the cpu consuming stale data. This mitigates * legitimate usages of overlapping mappings. + * MMIO mappings are never CPU-cacheable and do not need tracking either. */ - if (entry->direction == DMA_TO_DEVICE) + if (entry->direction == DMA_TO_DEVICE || + entry->attrs & DMA_ATTR_MMIO) return 0; spin_lock_irqsave(&radix_lock, flags); @@ -519,7 +521,8 @@ static void active_cacheline_remove(struct dma_debug_entry *entry) unsigned long flags; /* ...mirror the insert case */ - if (entry->direction == DMA_TO_DEVICE) + if (entry->direction == DMA_TO_DEVICE || + entry->attrs & DMA_ATTR_MMIO) return; spin_lock_irqsave(&radix_lock, flags); diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index da665ca22d5c..25e9a86835fd 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -120,6 +120,7 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, int node = dev_to_node(dev); struct page *page; u64 phys_limit; + void *va; WARN_ON_ONCE(!PAGE_ALIGNED(size)); @@ -133,9 +134,9 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, dma_free_contiguous(dev, page, size); } - while ((page = alloc_pages_node(node, gfp, get_order(size))) - && !dma_coherent_ok(dev, page_to_phys(page), size)) { - __free_pages(page, get_order(size)); + while ((va = alloc_pages_exact_nid(node, size, gfp)) && + !dma_coherent_ok(dev, virt_to_phys(va), size)) { + free_pages_exact(va, size); if (IS_ENABLED(CONFIG_ZONE_DMA32) && phys_limit < DMA_BIT_MASK(64) && @@ -146,6 +147,7 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, else return NULL; } + page = va ? virt_to_page(va) : NULL; return page; } @@ -179,6 +181,22 @@ static struct page *dma_direct_alloc_from_pool(struct device *dev, size_t size, return page; } +/* + * Check if the allocation can skip the kernel mapping and return the page + * pointer as the cookie. + */ +static bool dma_direct_use_no_mapping(struct device *dev, unsigned long attrs) +{ + if ((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) != + DMA_ATTR_NO_KERNEL_MAPPING) + return false; + if (is_swiotlb_for_alloc(dev)) + return false; + if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_ALLOC) && !dev_is_dma_coherent(dev)) + return false; + return true; +} + static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp) { @@ -226,8 +244,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, if (attrs & DMA_ATTR_NO_WARN) gfp |= __GFP_NOWARN; - if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) == - DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) + if (dma_direct_use_no_mapping(dev, attrs)) return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp); if (!dev_is_dma_coherent(dev)) { @@ -374,8 +391,7 @@ void dma_direct_free(struct device *dev, size_t size, if (attrs & __DMA_ATTR_ALLOC_CC_SHARED) mark_mem_encrypted = true; - if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) == - DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) { + if (dma_direct_use_no_mapping(dev, attrs)) { /* cpu_addr is a struct page cookie, not a kernel address */ dma_free_contiguous(dev, cpu_addr, size); return; diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index bf2651a70b7c..82e701648d60 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -1012,19 +1012,19 @@ size_t dma_max_mapping_size(struct device *dev) } EXPORT_SYMBOL_GPL(dma_max_mapping_size); -size_t dma_opt_mapping_size(struct device *dev) +size_t dma_max_opt_mapping_size(struct device *dev) { const struct dma_map_ops *ops = get_dma_ops(dev); size_t size = SIZE_MAX; if (use_dma_iommu(dev)) - size = iommu_dma_opt_mapping_size(); - else if (ops && ops->opt_mapping_size) - size = ops->opt_mapping_size(); + size = iommu_dma_max_opt_mapping_size(); + else if (ops && ops->max_opt_mapping_size) + size = ops->max_opt_mapping_size(); return min(dma_max_mapping_size(dev), size); } -EXPORT_SYMBOL_GPL(dma_opt_mapping_size); +EXPORT_SYMBOL_GPL(dma_max_opt_mapping_size); unsigned long dma_get_merge_boundary(struct device *dev) { diff --git a/kernel/dma/ops_helpers.c b/kernel/dma/ops_helpers.c index 6b5f9208d31c..5cdf3f60cf47 100644 --- a/kernel/dma/ops_helpers.c +++ b/kernel/dma/ops_helpers.c @@ -67,10 +67,13 @@ struct page *dma_common_alloc_pages(struct device *dev, size_t size, phys_addr_t phys; page = dma_alloc_contiguous(dev, size, gfp); - if (!page) - page = alloc_pages_node(dev_to_node(dev), gfp, get_order(size)); - if (!page) - return NULL; + if (!page) { + void *va = alloc_pages_exact_nid(dev_to_node(dev), size, gfp); + + if (!va) + return NULL; + page = virt_to_page(va); + } phys = page_to_phys(page); if (use_dma_iommu(dev)) diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index aa2f1c4588b9..2c57b49ac7c0 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -100,6 +100,7 @@ static struct io_tlb_mem io_tlb_default_mem; static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT; static unsigned long default_nareas; +static bool swiotlb_cmdline_size_set __initdata; /** * struct io_tlb_area - IO TLB memory area descriptor @@ -117,27 +118,20 @@ struct io_tlb_area { spinlock_t lock; }; -/* - * Round up number of slabs to the next power of 2. The last area is going - * be smaller than the rest if default_nslabs is not power of two. - * The number of slot in an area should be a multiple of IO_TLB_SEGSIZE, - * otherwise a segment may span two or more areas. It conflicts with free - * contiguous slots tracking: free slots are treated contiguous no matter - * whether they cross an area boundary. - * - * Return true if default_nslabs is rounded up. - */ -static bool round_up_default_nslabs(void) +/* Return a power-of-two number of slabs that can be split between areas. */ +static unsigned long swiotlb_calc_nslabs(unsigned long size, + unsigned long nareas) { - if (!default_nareas) - return false; + unsigned long nslabs; - if (default_nslabs < IO_TLB_SEGSIZE * default_nareas) - default_nslabs = IO_TLB_SEGSIZE * default_nareas; - else if (is_power_of_2(default_nslabs)) - return false; - default_nslabs = roundup_pow_of_two(default_nslabs); - return true; + nslabs = ALIGN(DIV_ROUND_UP(size, IO_TLB_SIZE), IO_TLB_SEGSIZE); + if (nareas && nslabs < IO_TLB_SEGSIZE * nareas) + nslabs = IO_TLB_SEGSIZE * nareas; + + if (!is_power_of_2(nslabs)) + nslabs = roundup_pow_of_two(nslabs); + + return nslabs; } /** @@ -150,6 +144,8 @@ static bool round_up_default_nslabs(void) */ static void swiotlb_adjust_nareas(unsigned int nareas) { + unsigned long nslabs; + if (!nareas) nareas = 1; else if (!is_power_of_2(nareas)) @@ -158,9 +154,12 @@ static void swiotlb_adjust_nareas(unsigned int nareas) default_nareas = nareas; pr_info("area num %d.\n", nareas); - if (round_up_default_nslabs()) + nslabs = swiotlb_calc_nslabs(default_nslabs << IO_TLB_SHIFT, nareas); + if (nslabs != default_nslabs) { + default_nslabs = nslabs; pr_info("SWIOTLB bounce buffer size roundup to %luMB", (default_nslabs << IO_TLB_SHIFT) >> 20); + } } /** @@ -255,6 +254,7 @@ setup_io_tlb_npages(char *str) /* avoid tail segment of size < IO_TLB_SEGSIZE */ default_nslabs = ALIGN(simple_strtoul(str, &str, 0), IO_TLB_SEGSIZE); + swiotlb_cmdline_size_set = true; } if (*str == ',') ++str; @@ -285,7 +285,7 @@ setup_io_tlb_npages(char *str) } early_param("swiotlb", setup_io_tlb_npages); -unsigned long swiotlb_size_or_default(void) +unsigned long swiotlb_default_pool_size(void) { return default_nslabs << IO_TLB_SHIFT; } @@ -297,13 +297,11 @@ void __init swiotlb_adjust_size(unsigned long size) * architectures such as those supporting memory encryption to * adjust/expand SWIOTLB size for their use. */ - if (default_nslabs != IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT) + if (swiotlb_cmdline_size_set) return; - size = ALIGN(size, IO_TLB_SIZE); - default_nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE); - if (round_up_default_nslabs()) - size = default_nslabs << IO_TLB_SHIFT; + default_nslabs = swiotlb_calc_nslabs(size, default_nareas); + size = default_nslabs << IO_TLB_SHIFT; pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20); } @@ -1844,10 +1842,10 @@ phys_addr_t default_swiotlb_base(void) phys_addr_t default_swiotlb_limit(void) { #ifdef CONFIG_SWIOTLB_DYNAMIC - return io_tlb_default_mem.phys_limit; -#else - return io_tlb_default_mem.defpool.end - 1; + if (io_tlb_default_mem.can_grow) + return io_tlb_default_mem.phys_limit; #endif + return io_tlb_default_mem.defpool.end - 1; } #ifdef CONFIG_DEBUG_FS @@ -2083,8 +2081,8 @@ static int __init rmem_swiotlb_setup(unsigned long node, of_get_flat_dt_prop(node, "no-map", NULL)) return -EINVAL; - pr_info("Reserved memory: created restricted DMA pool at %pa, size %ld MiB\n", - &rmem->base, (unsigned long)rmem->size / SZ_1M); + pr_info("Reserved memory: created restricted DMA pool at %pa, size %llu KiB\n", + &rmem->base, (unsigned long long)(rmem->size / SZ_1K)); return 0; } diff --git a/kernel/entry/common.c b/kernel/entry/common.c index e3d381fd3d25..e234b04373fe 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -123,7 +123,7 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) /** * arch_irqentry_exit_need_resched - Architecture specific need resched function * - * Invoked from raw_irqentry_exit_cond_resched() to check if resched is needed. + * Invoked from irqentry_exit_cond_resched() to check if resched is needed. * Defaults return true. * * The main purpose is to permit arch to avoid preemption of a task from an IRQ. @@ -134,7 +134,7 @@ static inline bool arch_irqentry_exit_need_resched(void); static inline bool arch_irqentry_exit_need_resched(void) { return true; } #endif -void raw_irqentry_exit_cond_resched(void) +void irqentry_exit_cond_resched(void) { if (!preempt_count()) { /* Sanity check RCU and thread stack */ @@ -145,19 +145,6 @@ void raw_irqentry_exit_cond_resched(void) preempt_schedule_irq(); } } -#ifdef CONFIG_PREEMPT_DYNAMIC -#if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) -DEFINE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); -#elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); -void dynamic_irqentry_exit_cond_resched(void) -{ - if (!static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched)) - return; - raw_irqentry_exit_cond_resched(); -} -#endif -#endif noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) { diff --git a/kernel/entry/syscall_user_dispatch.c b/kernel/entry/syscall_user_dispatch.c index 59c861866941..7d1dd7f1bf38 100644 --- a/kernel/entry/syscall_user_dispatch.c +++ b/kernel/entry/syscall_user_dispatch.c @@ -45,11 +45,7 @@ bool syscall_user_dispatch(struct pt_regs *regs) return false; if (likely(sd->selector)) { - /* - * access_ok() is performed once, at prctl time, when - * the selector is loaded by userspace. - */ - if (unlikely(__get_user(state, sd->selector))) { + if (unlikely(get_user(state, untagged_addr(sd->selector)))) { force_exit_sig(SIGSEGV); return true; } diff --git a/kernel/events/core.c b/kernel/events/core.c index db7b76d6b68a..a9ee91151b2d 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -3764,6 +3764,9 @@ static void perf_ctx_sched_task_cb(struct perf_event_context *ctx, list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) { cpc = this_cpc(pmu_ctx->pmu); + if (cpc->task_epc != pmu_ctx) + continue; + if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task) pmu_ctx->pmu->sched_task(pmu_ctx, task, sched_in); } @@ -3914,7 +3917,7 @@ static void __perf_pmu_sched_task(struct perf_cpu_pmu_context *cpc, perf_ctx_lock(cpuctx, cpuctx->task_ctx); perf_pmu_disable(pmu); - pmu->sched_task(cpc->task_epc, task, sched_in); + pmu->sched_task(&cpc->epc, task, sched_in); perf_pmu_enable(pmu); perf_ctx_unlock(cpuctx, cpuctx->task_ctx); @@ -3924,15 +3927,17 @@ static void perf_pmu_sched_task(struct task_struct *prev, struct task_struct *next, bool sched_in) { - struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); struct perf_cpu_pmu_context *cpc, *cpc2; - /* cpuctx->task_ctx will be handled in perf_event_context_sched_in/out */ - if (prev == next || cpuctx->task_ctx) + if (prev == next) return; - list_for_each_entry_safe(cpc, cpc2, this_cpu_ptr(&sched_cb_list), sched_cb_entry) + list_for_each_entry_safe(cpc, cpc2, this_cpu_ptr(&sched_cb_list), sched_cb_entry) { + if (cpc->task_epc) + continue; + __perf_pmu_sched_task(cpc, sched_in ? next : prev, sched_in); + } } static void perf_event_switch(struct task_struct *task, @@ -4634,6 +4639,7 @@ static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc) if (cpu_event) { update_context_time(&cpuctx->ctx); + update_cgrp_time_from_cpuctx(cpuctx, false); __pmu_ctx_sched_out(cpu_epc, EVENT_FLEXIBLE); rotate_ctx(&cpuctx->ctx, cpu_event); __pmu_ctx_sched_in(cpu_epc, EVENT_FLEXIBLE); @@ -5454,6 +5460,8 @@ attach_task_ctx_data(struct task_struct *task, struct kmem_cache *ctx_cache, } if (refcount_inc_not_zero(&old->refcount)) { + if (global) + old->global = true; free_perf_ctx_data(cd); /* unused */ return 0; } @@ -7052,7 +7060,7 @@ static void perf_mmap_close(struct vm_area_struct *vma) perf_pmu_output_stop(event); /* now it's safe to free the pages */ - atomic_long_sub(rb->aux_nr_pages - rb->aux_mmap_locked, &mmap_user->locked_vm); + atomic_long_sub(rb->aux_nr_pages - rb->aux_mmap_locked, &mmap_user->perf_mlock); atomic64_sub(rb->aux_mmap_locked, &vma->vm_mm->pinned_vm); /* this has to be the last one */ @@ -7235,11 +7243,11 @@ static bool perf_mmap_calc_limits(struct vm_area_struct *vma, long *user_extra, /* Increase the limit linearly with more CPUs */ user_lock_limit *= num_online_cpus(); - user_locked = atomic_long_read(&user->locked_vm); + user_locked = atomic_long_read(&user->perf_mlock); /* * sysctl_perf_event_mlock may have changed, so that - * user->locked_vm > user_lock_limit + * user->perf_mlock > user_lock_limit */ if (user_locked > user_lock_limit) user_locked = user_lock_limit; @@ -7247,7 +7255,7 @@ static bool perf_mmap_calc_limits(struct vm_area_struct *vma, long *user_extra, if (user_locked > user_lock_limit) { /* - * charge locked_vm until it hits user_lock_limit; + * charge perf_mlock until it hits user_lock_limit; * charge the rest from pinned_vm */ *extra = user_locked - user_lock_limit; @@ -7265,7 +7273,7 @@ static void perf_mmap_account(struct vm_area_struct *vma, long user_extra, long { struct user_struct *user = current_user(); - atomic_long_add(user_extra, &user->locked_vm); + atomic_long_add(user_extra, &user->perf_mlock); atomic64_add(extra, &vma->vm_mm->pinned_vm); } @@ -7274,7 +7282,7 @@ static void perf_mmap_unaccount(struct vm_area_struct *vma, struct perf_buffer * struct user_struct *user = rb->mmap_user; atomic_long_sub((perf_data_size(rb) >> PAGE_SHIFT) + 1 - rb->mmap_locked, - &user->locked_vm); + &user->perf_mlock); atomic64_sub(rb->mmap_locked, &vma->vm_mm->pinned_vm); } @@ -7619,9 +7627,11 @@ static void perf_sigtrap(struct perf_event *event) { /* * Both perf_pending_task() and perf_pending_irq() can race with the - * task exiting. + * task exiting or exec-ing. We can determine if such a race has + * occurred by checking if perf_event_exit_task(), which will set + * ctx->task to TASK_TOMBSTONE, has already been called. */ - if (current->flags & PF_EXITING) + if (event->ctx->task == TASK_TOMBSTONE) return; /* @@ -7821,22 +7831,82 @@ unsigned long perf_instruction_pointer(struct perf_event *event, 0 : perf_arch_instruction_pointer(regs); } +u64 __weak perf_reg_value(struct pt_regs *regs, int idx) +{ + return 0; +} + +int __weak perf_reg_validate(u64 mask, bool simd_enabled) +{ + return mask ? -ENOSYS : 0; +} + +u64 __weak perf_reg_abi(struct task_struct *task) +{ + return PERF_SAMPLE_REGS_ABI_NONE; +} + +void __weak perf_get_regs_user(struct perf_regs *regs_user, + struct pt_regs *regs) +{ + regs_user->regs = task_pt_regs(current); + regs_user->abi = perf_reg_abi(current); +} + +#define word_for_each_set_bit(bit, val) \ + for (unsigned long long __v = (val); \ + __v && ((bit = __builtin_ctzll(__v)), 1); \ + __v &= __v - 1) + static void perf_output_sample_regs(struct perf_output_handle *handle, struct pt_regs *regs, u64 mask) { int bit; - DECLARE_BITMAP(_mask, 64); - bitmap_from_u64(_mask, mask); - for_each_set_bit(bit, _mask, sizeof(mask) * BITS_PER_BYTE) { - u64 val; - - val = perf_reg_value(regs, bit); + word_for_each_set_bit(bit, mask) { + u64 val = perf_reg_value(regs, bit); perf_output_put(handle, val); } } +static void +perf_output_sample_simd_regs(struct perf_output_handle *handle, + struct perf_event *event, + struct pt_regs *regs, + u64 mask, u32 pred_mask) +{ + u64 pred_qwords = event->attr.sample_simd_pred_reg_qwords; + u64 vec_qwords = event->attr.sample_simd_vec_reg_qwords; + u64 nr_vectors = hweight64(mask); + u64 nr_pred = hweight32(pred_mask); + int bit; + + perf_output_put(handle, nr_vectors); + perf_output_put(handle, vec_qwords); + perf_output_put(handle, nr_pred); + perf_output_put(handle, pred_qwords); + + if (nr_vectors) { + word_for_each_set_bit(bit, mask) { + for (int i = 0; i < vec_qwords; i++) { + u64 val = perf_simd_reg_value(regs, bit, + i, false); + perf_output_put(handle, val); + } + } + } + if (nr_pred) { + word_for_each_set_bit(bit, pred_mask) { + for (int i = 0; i < pred_qwords; i++) { + u64 val = perf_simd_reg_value(regs, bit, + i, true); + perf_output_put(handle, val); + } + } + } +} + static void perf_sample_regs_user(struct perf_regs *regs_user, struct pt_regs *regs) { @@ -7870,6 +7940,17 @@ static void perf_sample_regs_intr(struct perf_regs *regs_intr, } } +int __weak perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask, + u16 pred_qwords, u32 pred_mask) +{ + return -EINVAL; +} + +u64 __weak perf_simd_reg_value(struct pt_regs *regs, int idx, + u16 qwords_idx, bool pred) +{ + return 0; +} /* * Get remaining task size from user stack pointer. @@ -8122,10 +8203,15 @@ static void __perf_event_header__init_id(struct perf_sample_data *data, } } -void perf_event_header__init_id(struct perf_event_header *header, - struct perf_sample_data *data, - struct perf_event *event) +void perf_event_header__init(struct perf_event_header *header, + struct perf_sample_data *data, + u32 type, u16 misc, u16 size, + struct perf_event *event) { + header->type = type; + header->misc = misc; + header->size = size; + if (event->attr.sample_id_all) { header->size += event->id_header_size; __perf_event_header__init_id(data, event, event->attr.sample_type); @@ -8400,10 +8486,17 @@ void perf_output_sample(struct perf_output_handle *handle, perf_output_put(handle, abi); if (abi) { - u64 mask = event->attr.sample_regs_user; + struct perf_event_attr *attr = &event->attr; + u64 mask = attr->sample_regs_user; perf_output_sample_regs(handle, data->regs_user.regs, mask); + if (abi & PERF_SAMPLE_REGS_ABI_SIMD) { + perf_output_sample_simd_regs(handle, event, + data->regs_user.regs, + attr->sample_simd_vec_reg_user, + attr->sample_simd_pred_reg_user); + } } } @@ -8431,11 +8524,18 @@ void perf_output_sample(struct perf_output_handle *handle, perf_output_put(handle, abi); if (abi) { - u64 mask = event->attr.sample_regs_intr; + struct perf_event_attr *attr = &event->attr; + u64 mask = attr->sample_regs_intr; perf_output_sample_regs(handle, data->regs_intr.regs, mask); + if (abi & PERF_SAMPLE_REGS_ABI_SIMD) { + perf_output_sample_simd_regs(handle, event, + data->regs_intr.regs, + attr->sample_simd_vec_reg_intr, + attr->sample_simd_pred_reg_intr); + } } } @@ -8638,6 +8738,29 @@ static __always_inline u64 __cond_set(u64 flags, u64 s, u64 d) return d * !!(flags & s); } +u64 perf_update_xregs_size(struct perf_event *event, bool intr) +{ + u16 pred_qwords = event->attr.sample_simd_pred_reg_qwords; + u16 vec_qwords = event->attr.sample_simd_vec_reg_qwords; + u64 pred_mask; + u64 mask; + int size; + + if (intr) { + mask = event->attr.sample_simd_vec_reg_intr; + pred_mask = event->attr.sample_simd_pred_reg_intr; + } else { + mask = event->attr.sample_simd_vec_reg_user; + pred_mask = event->attr.sample_simd_pred_reg_user; + } + + size = sizeof(u64) * 4; + size += (hweight64(mask) * vec_qwords + + hweight64(pred_mask) * pred_qwords) * sizeof(u64); + + return size; +} + void perf_prepare_sample(struct perf_sample_data *data, struct perf_event *event, struct pt_regs *regs) @@ -8700,7 +8823,12 @@ void perf_prepare_sample(struct perf_sample_data *data, if (data->regs_user.regs) { u64 mask = event->attr.sample_regs_user; + size += hweight64(mask) * sizeof(u64); + if (event_has_simd_regs(event)) { + size += perf_update_xregs_size(event, false); + data->regs_user.abi |= PERF_SAMPLE_REGS_ABI_SIMD; + } } data->dyn_size += size; @@ -8765,6 +8893,10 @@ void perf_prepare_sample(struct perf_sample_data *data, u64 mask = event->attr.sample_regs_intr; size += hweight64(mask) * sizeof(u64); + if (event_has_simd_regs(event)) { + size += perf_update_xregs_size(event, true); + data->regs_intr.abi |= PERF_SAMPLE_REGS_ABI_SIMD; + } } data->dyn_size += size; @@ -8962,17 +9094,16 @@ perf_event_read_event(struct perf_event *event, struct perf_output_handle handle; struct perf_sample_data sample; struct perf_read_event read_event = { - .header = { - .type = PERF_RECORD_READ, - .misc = 0, - .size = sizeof(read_event) + event->read_size, - }, .pid = perf_event_pid(event, task), .tid = perf_event_tid(event, task), }; int ret; - perf_event_header__init_id(&read_event.header, &sample, event); + perf_event_header__init(&read_event.header, &sample, + PERF_RECORD_READ, + /* misc= */ 0, + sizeof(read_event) + event->read_size, + event); ret = perf_output_begin(&handle, &sample, event, read_event.header.size); if (ret) return; @@ -9213,6 +9344,7 @@ struct perf_task_event { u32 ptid; u64 time; } event_id; + int new; }; static int perf_event_task_match(struct perf_event *event) @@ -9229,17 +9361,21 @@ static void perf_event_task_output(struct perf_event *event, struct perf_output_handle handle; struct perf_sample_data sample; struct task_struct *task = task_event->task; - int ret, size = task_event->event_id.header.size; + int ret; if (!perf_event_task_match(event)) return; - perf_event_header__init_id(&task_event->event_id.header, &sample, event); + perf_event_header__init(&task_event->event_id.header, &sample, + task_event->new ? PERF_RECORD_FORK : PERF_RECORD_EXIT, + /* misc= */ 0, + sizeof(task_event->event_id), + event); ret = perf_output_begin(&handle, &sample, event, task_event->event_id.header.size); if (ret) - goto out; + return; task_event->event_id.pid = perf_event_pid(event, task); task_event->event_id.tid = perf_event_tid(event, task); @@ -9261,8 +9397,6 @@ static void perf_event_task_output(struct perf_event *event, perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - task_event->event_id.header.size = size; } static void perf_event_task(struct task_struct *task, @@ -9279,18 +9413,7 @@ static void perf_event_task(struct task_struct *task, task_event = (struct perf_task_event){ .task = task, .task_ctx = task_ctx, - .event_id = { - .header = { - .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT, - .misc = 0, - .size = sizeof(task_event.event_id), - }, - /* .pid */ - /* .ppid */ - /* .tid */ - /* .ptid */ - /* .time */ - }, + .new = new, }; perf_iterate_sb(perf_event_task_output, @@ -9367,6 +9490,7 @@ struct perf_comm_event { u32 pid; u32 tid; } event_id; + bool exec; }; static int perf_event_comm_match(struct perf_event *event) @@ -9380,18 +9504,21 @@ static void perf_event_comm_output(struct perf_event *event, struct perf_comm_event *comm_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - int size = comm_event->event_id.header.size; int ret; if (!perf_event_comm_match(event)) return; - perf_event_header__init_id(&comm_event->event_id.header, &sample, event); + perf_event_header__init(&comm_event->event_id.header, &sample, + PERF_RECORD_COMM, + comm_event->exec ? PERF_RECORD_MISC_COMM_EXEC : 0, + sizeof(comm_event->event_id) + comm_event->comm_size, + event); ret = perf_output_begin(&handle, &sample, event, comm_event->event_id.header.size); if (ret) - goto out; + return; comm_event->event_id.pid = perf_event_pid(event, comm_event->task); comm_event->event_id.tid = perf_event_tid(event, comm_event->task); @@ -9403,8 +9530,6 @@ static void perf_event_comm_output(struct perf_event *event, perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - comm_event->event_id.header.size = size; } static void perf_event_comm_event(struct perf_comm_event *comm_event) @@ -9419,8 +9544,6 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event) comm_event->comm = comm; comm_event->comm_size = size; - comm_event->event_id.header.size = sizeof(comm_event->event_id) + size; - perf_iterate_sb(perf_event_comm_output, comm_event, NULL); @@ -9437,15 +9560,8 @@ void perf_event_comm(struct task_struct *task, bool exec) .task = task, /* .comm */ /* .comm_size */ - .event_id = { - .header = { - .type = PERF_RECORD_COMM, - .misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0, - /* .size */ - }, - /* .pid */ - /* .tid */ - }, + /* .event_id */ + .exec = exec, }; perf_event_comm_event(&comm_event); @@ -9479,18 +9595,20 @@ static void perf_event_namespaces_output(struct perf_event *event, struct perf_namespaces_event *namespaces_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - u16 header_size = namespaces_event->event_id.header.size; int ret; if (!perf_event_namespaces_match(event)) return; - perf_event_header__init_id(&namespaces_event->event_id.header, - &sample, event); + perf_event_header__init(&namespaces_event->event_id.header, &sample, + PERF_RECORD_NAMESPACES, + /* misc= */ 0, + sizeof(namespaces_event->event_id), + event); ret = perf_output_begin(&handle, &sample, event, namespaces_event->event_id.header.size); if (ret) - goto out; + return; namespaces_event->event_id.pid = perf_event_pid(event, namespaces_event->task); @@ -9502,8 +9620,6 @@ static void perf_event_namespaces_output(struct perf_event *event, perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - namespaces_event->event_id.header.size = header_size; } static void perf_fill_ns_link_info(struct perf_ns_link_info *ns_link_info, @@ -9534,11 +9650,7 @@ void perf_event_namespaces(struct task_struct *task) namespaces_event = (struct perf_namespaces_event){ .task = task, .event_id = { - .header = { - .type = PERF_RECORD_NAMESPACES, - .misc = 0, - .size = sizeof(namespaces_event.event_id), - }, + /* .header */ /* .pid */ /* .tid */ .nr_namespaces = NR_NAMESPACES, @@ -9606,18 +9718,19 @@ static void perf_event_cgroup_output(struct perf_event *event, void *data) struct perf_cgroup_event *cgroup_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - u16 header_size = cgroup_event->event_id.header.size; int ret; + u16 size = sizeof(cgroup_event->event_id) + cgroup_event->path_size; if (!perf_event_cgroup_match(event)) return; - perf_event_header__init_id(&cgroup_event->event_id.header, - &sample, event); + perf_event_header__init(&cgroup_event->event_id.header, &sample, + PERF_RECORD_CGROUP, /* misc= */ 0, size, + event); ret = perf_output_begin(&handle, &sample, event, cgroup_event->event_id.header.size); if (ret) - goto out; + return; perf_output_put(&handle, cgroup_event->event_id); __output_copy(&handle, cgroup_event->path, cgroup_event->path_size); @@ -9625,8 +9738,6 @@ static void perf_event_cgroup_output(struct perf_event *event, void *data) perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - cgroup_event->event_id.header.size = header_size; } static void perf_event_cgroup(struct cgroup *cgrp) @@ -9641,11 +9752,6 @@ static void perf_event_cgroup(struct cgroup *cgrp) cgroup_event = (struct perf_cgroup_event){ .event_id = { - .header = { - .type = PERF_RECORD_CGROUP, - .misc = 0, - .size = sizeof(cgroup_event.event_id), - }, .id = cgroup_id(cgrp), }, }; @@ -9668,7 +9774,6 @@ static void perf_event_cgroup(struct cgroup *cgrp) while (!IS_ALIGNED(size, sizeof(u64))) cgroup_event.path[size++] = '\0'; - cgroup_event.event_id.header.size += size; cgroup_event.path_size = size; perf_iterate_sb(perf_event_cgroup_output, @@ -9724,38 +9829,40 @@ static void perf_event_mmap_output(struct perf_event *event, struct perf_mmap_event *mmap_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - int size = mmap_event->event_id.header.size; - u32 type = mmap_event->event_id.header.type; - bool use_build_id; + int size = sizeof(mmap_event->event_id) + mmap_event->file_size; + u32 type = PERF_RECORD_MMAP; + u16 misc = PERF_RECORD_MISC_USER; + bool use_build_id = false; int ret; if (!perf_event_mmap_match(event, data)) return; if (event->attr.mmap2) { - mmap_event->event_id.header.type = PERF_RECORD_MMAP2; - mmap_event->event_id.header.size += sizeof(mmap_event->maj); - mmap_event->event_id.header.size += sizeof(mmap_event->min); - mmap_event->event_id.header.size += sizeof(mmap_event->ino); - mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation); - mmap_event->event_id.header.size += sizeof(mmap_event->prot); - mmap_event->event_id.header.size += sizeof(mmap_event->flags); - } - - perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); + type = PERF_RECORD_MMAP2; + size += sizeof(mmap_event->maj); + size += sizeof(mmap_event->min); + size += sizeof(mmap_event->ino); + size += sizeof(mmap_event->ino_generation); + size += sizeof(mmap_event->prot); + size += sizeof(mmap_event->flags); + use_build_id = event->attr.build_id && mmap_event->build_id_size; + if (use_build_id) + misc |= PERF_RECORD_MISC_MMAP_BUILD_ID; + } + if (!(mmap_event->vma->vm_flags & VM_EXEC)) + misc |= PERF_RECORD_MISC_MMAP_DATA; + + perf_event_header__init(&mmap_event->event_id.header, &sample, + type, misc, size, event); ret = perf_output_begin(&handle, &sample, event, mmap_event->event_id.header.size); if (ret) - goto out; + return; mmap_event->event_id.pid = perf_event_pid(event, current); mmap_event->event_id.tid = perf_event_tid(event, current); - use_build_id = event->attr.build_id && mmap_event->build_id_size; - - if (event->attr.mmap2 && use_build_id) - mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_BUILD_ID; - perf_output_put(&handle, mmap_event->event_id); if (event->attr.mmap2) { @@ -9780,9 +9887,6 @@ static void perf_event_mmap_output(struct perf_event *event, perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - mmap_event->event_id.header.size = size; - mmap_event->event_id.header.type = type; } static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) @@ -9878,11 +9982,6 @@ got_name: mmap_event->prot = prot; mmap_event->flags = flags; - if (!(vma->vm_flags & VM_EXEC)) - mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA; - - mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size; - if (atomic_read(&nr_build_id_events)) build_id_parse_nofault(vma, mmap_event->build_id, &mmap_event->build_id_size); @@ -10002,11 +10101,7 @@ void perf_event_mmap(struct vm_area_struct *vma) /* .file_name */ /* .file_size */ .event_id = { - .header = { - .type = PERF_RECORD_MMAP, - .misc = PERF_RECORD_MISC_USER, - /* .size */ - }, + /* .header */ /* .pid */ /* .tid */ .start = vma->vm_start, @@ -10036,18 +10131,15 @@ void perf_event_aux_event(struct perf_event *event, unsigned long head, u64 size; u64 flags; } rec = { - .header = { - .type = PERF_RECORD_AUX, - .misc = 0, - .size = sizeof(rec), - }, .offset = head, .size = size, .flags = flags, }; int ret; - perf_event_header__init_id(&rec.header, &sample, event); + perf_event_header__init(&rec.header, &sample, + PERF_RECORD_AUX, /* misc= */ 0, sizeof(rec), + event); ret = perf_output_begin(&handle, &sample, event, rec.header.size); if (ret) @@ -10072,15 +10164,14 @@ void perf_log_lost_samples(struct perf_event *event, u64 lost) struct perf_event_header header; u64 lost; } lost_samples_event = { - .header = { - .type = PERF_RECORD_LOST_SAMPLES, - .misc = 0, - .size = sizeof(lost_samples_event), - }, .lost = lost, }; - perf_event_header__init_id(&lost_samples_event.header, &sample, event); + perf_event_header__init(&lost_samples_event.header, &sample, + PERF_RECORD_LOST_SAMPLES, + /* misc= */ 0, + sizeof(lost_samples_event), + event); ret = perf_output_begin(&handle, &sample, event, lost_samples_event.header.size); @@ -10105,6 +10196,8 @@ struct perf_switch_event { u32 next_prev_pid; u32 next_prev_tid; } event_id; + bool sched_in; + bool preempt; }; static int perf_event_switch_match(struct perf_event *event) @@ -10117,6 +10210,9 @@ static void perf_event_switch_output(struct perf_event *event, void *data) struct perf_switch_event *se = data; struct perf_output_handle handle; struct perf_sample_data sample; + u32 type; + u16 misc; + u16 size; int ret; if (!perf_event_switch_match(event)) @@ -10124,18 +10220,22 @@ static void perf_event_switch_output(struct perf_event *event, void *data) /* Only CPU-wide events are allowed to see next/prev pid/tid */ if (event->ctx->task) { - se->event_id.header.type = PERF_RECORD_SWITCH; - se->event_id.header.size = sizeof(se->event_id.header); + type = PERF_RECORD_SWITCH; + size = sizeof(se->event_id.header); } else { - se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE; - se->event_id.header.size = sizeof(se->event_id); + type = PERF_RECORD_SWITCH_CPU_WIDE; + size = sizeof(se->event_id); se->event_id.next_prev_pid = perf_event_pid(event, se->next_prev); se->event_id.next_prev_tid = perf_event_tid(event, se->next_prev); } + misc = se->sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT; + if (se->preempt) + misc |= PERF_RECORD_MISC_SWITCH_OUT_PREEMPT; - perf_event_header__init_id(&se->event_id.header, &sample, event); + perf_event_header__init(&se->event_id.header, &sample, + type, misc, size, event); ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size); if (ret) @@ -10161,22 +10261,11 @@ static void perf_event_switch(struct task_struct *task, switch_event = (struct perf_switch_event){ .task = task, .next_prev = next_prev, - .event_id = { - .header = { - /* .type */ - .misc = sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT, - /* .size */ - }, - /* .next_prev_pid */ - /* .next_prev_tid */ - }, + /* .event_id */ + .sched_in = sched_in, + .preempt = !sched_in && task_is_runnable(task), }; - if (!sched_in && task_is_runnable(task)) { - switch_event.event_id.header.misc |= - PERF_RECORD_MISC_SWITCH_OUT_PREEMPT; - } - perf_iterate_sb(perf_event_switch_output, &switch_event, NULL); } @@ -10196,20 +10285,17 @@ static void perf_log_throttle(struct perf_event *event, int enable) u64 id; u64 stream_id; } throttle_event = { - .header = { - .type = PERF_RECORD_THROTTLE, - .misc = 0, - .size = sizeof(throttle_event), - }, .time = perf_event_clock(event), .id = primary_event_id(event), .stream_id = event->id, }; - if (enable) - throttle_event.header.type = PERF_RECORD_UNTHROTTLE; - - perf_event_header__init_id(&throttle_event.header, &sample, event); + perf_event_header__init(&throttle_event.header, &sample, + enable ? PERF_RECORD_UNTHROTTLE + : PERF_RECORD_THROTTLE, + /* misc= */ 0, + sizeof(throttle_event), + event); ret = perf_output_begin(&handle, &sample, event, throttle_event.header.size); @@ -10248,12 +10334,14 @@ static void perf_event_ksymbol_output(struct perf_event *event, void *data) struct perf_output_handle handle; struct perf_sample_data sample; int ret; + u16 size = sizeof(ksymbol_event->event_id) + ksymbol_event->name_len; if (!perf_event_ksymbol_match(event)) return; - perf_event_header__init_id(&ksymbol_event->event_id.header, - &sample, event); + perf_event_header__init(&ksymbol_event->event_id.header, &sample, + PERF_RECORD_KSYMBOL, /* misc= */ 0, size, + event); ret = perf_output_begin(&handle, &sample, event, ksymbol_event->event_id.header.size); if (ret) @@ -10294,11 +10382,6 @@ void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len, bool unregister, .name = name, .name_len = name_len, .event_id = { - .header = { - .type = PERF_RECORD_KSYMBOL, - .size = sizeof(ksymbol_event.event_id) + - name_len, - }, .addr = addr, .len = len, .ksym_type = ksym_type, @@ -10342,8 +10425,11 @@ static void perf_event_bpf_output(struct perf_event *event, void *data) if (!perf_event_bpf_match(event)) return; - perf_event_header__init_id(&bpf_event->event_id.header, - &sample, event); + perf_event_header__init(&bpf_event->event_id.header, &sample, + PERF_RECORD_BPF_EVENT, + /* misc= */ 0, + sizeof(bpf_event->event_id), + event); ret = perf_output_begin(&handle, &sample, event, bpf_event->event_id.header.size); if (ret) @@ -10399,10 +10485,6 @@ void perf_event_bpf_event(struct bpf_prog *prog, bpf_event = (struct perf_bpf_event){ .prog = prog, .event_id = { - .header = { - .type = PERF_RECORD_BPF_EVENT, - .size = sizeof(bpf_event.event_id), - }, .type = type, .flags = flags, .id = prog->aux->id, @@ -10430,18 +10512,23 @@ static void perf_callchain_deferred_output(struct perf_event *event, void *data) struct perf_callchain_deferred_event *deferred_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - int ret, size = deferred_event->event.header.size; + int ret; + u16 size = sizeof(deferred_event->event) + (deferred_event->trace->nr * sizeof(u64)); if (!event->attr.defer_output) return; /* XXX do we really need sample_id_all for this ??? */ - perf_event_header__init_id(&deferred_event->event.header, &sample, event); + perf_event_header__init(&deferred_event->event.header, &sample, + PERF_RECORD_CALLCHAIN_DEFERRED, + PERF_RECORD_MISC_USER, + size, + event); ret = perf_output_begin(&handle, &sample, event, deferred_event->event.header.size); if (ret) - goto out; + return; perf_output_put(&handle, deferred_event->event); for (int i = 0; i < deferred_event->trace->nr; i++) { @@ -10451,8 +10538,6 @@ static void perf_callchain_deferred_output(struct perf_event *event, void *data) perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - deferred_event->event.header.size = size; } static void perf_unwind_deferred_callback(struct unwind_work *work, @@ -10461,12 +10546,6 @@ static void perf_unwind_deferred_callback(struct unwind_work *work, struct perf_callchain_deferred_event deferred_event = { .trace = trace, .event = { - .header = { - .type = PERF_RECORD_CALLCHAIN_DEFERRED, - .misc = PERF_RECORD_MISC_USER, - .size = sizeof(deferred_event.event) + - (trace->nr * sizeof(u64)), - }, .cookie = cookie, .nr = trace->nr, }, @@ -10478,7 +10557,8 @@ static void perf_unwind_deferred_callback(struct unwind_work *work, struct perf_text_poke_event { const void *old_bytes; const void *new_bytes; - size_t pad; + u16 tot; + u16 pad; u16 old_len; u16 new_len; @@ -10499,13 +10579,18 @@ static void perf_event_text_poke_output(struct perf_event *event, void *data) struct perf_text_poke_event *text_poke_event = data; struct perf_output_handle handle; struct perf_sample_data sample; + u16 size = sizeof(text_poke_event->event_id) + text_poke_event->tot + text_poke_event->pad; u64 padding = 0; int ret; if (!perf_event_text_poke_match(event)) return; - perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event); + perf_event_header__init(&text_poke_event->event_id.header, &sample, + PERF_RECORD_TEXT_POKE, + PERF_RECORD_MISC_KERNEL, + size, + event); ret = perf_output_begin(&handle, &sample, event, text_poke_event->event_id.header.size); @@ -10543,15 +10628,11 @@ void perf_event_text_poke(const void *addr, const void *old_bytes, text_poke_event = (struct perf_text_poke_event){ .old_bytes = old_bytes, .new_bytes = new_bytes, + .tot = tot, .pad = pad, .old_len = old_len, .new_len = new_len, .event_id = { - .header = { - .type = PERF_RECORD_TEXT_POKE, - .misc = PERF_RECORD_MISC_KERNEL, - .size = sizeof(text_poke_event.event_id) + tot + pad, - }, .addr = (unsigned long)addr, }, }; @@ -10563,6 +10644,7 @@ void perf_event_itrace_started(struct perf_event *event) { WRITE_ONCE(event->attach_state, event->attach_state | PERF_ATTACH_ITRACE); } +EXPORT_SYMBOL_GPL(perf_event_itrace_started); static void perf_log_itrace_start(struct perf_event *event) { @@ -10582,13 +10664,12 @@ static void perf_log_itrace_start(struct perf_event *event) event->attach_state & PERF_ATTACH_ITRACE) return; - rec.header.type = PERF_RECORD_ITRACE_START; - rec.header.misc = 0; - rec.header.size = sizeof(rec); rec.pid = perf_event_pid(event, current); rec.tid = perf_event_tid(event, current); - perf_event_header__init_id(&rec.header, &sample, event); + perf_event_header__init(&rec.header, &sample, + PERF_RECORD_ITRACE_START, /* misc= */ 0, sizeof(rec), + event); ret = perf_output_begin(&handle, &sample, event, rec.header.size); if (ret) @@ -10613,12 +10694,10 @@ void perf_report_aux_output_id(struct perf_event *event, u64 hw_id) if (event->parent) event = event->parent; - rec.header.type = PERF_RECORD_AUX_OUTPUT_HW_ID; - rec.header.misc = 0; - rec.header.size = sizeof(rec); - rec.hw_id = hw_id; - - perf_event_header__init_id(&rec.header, &sample, event); + rec.hw_id = hw_id; + perf_event_header__init(&rec.header, &sample, + PERF_RECORD_AUX_OUTPUT_HW_ID, /* misc= */ 0, + sizeof(rec), event); ret = perf_output_begin(&handle, &sample, event, rec.header.size); if (ret) @@ -13109,12 +13188,6 @@ int perf_pmu_unregister(struct pmu *pmu) } EXPORT_SYMBOL_GPL(perf_pmu_unregister); -static inline bool has_extended_regs(struct perf_event *event) -{ - return (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) || - (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK); -} - static int perf_try_init_event(struct pmu *pmu, struct perf_event *event) { struct perf_event_context *ctx = NULL; @@ -13148,8 +13221,14 @@ static int perf_try_init_event(struct pmu *pmu, struct perf_event *event) if (ret) goto err_pmu; + if (!(pmu->capabilities & PERF_PMU_CAP_SIMD_REGS) && + event_has_simd_regs(event)) { + ret = -EOPNOTSUPP; + goto err_destroy; + } + if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) && - has_extended_regs(event)) { + event_has_extended_regs(event)) { ret = -EOPNOTSUPP; goto err_destroy; } @@ -13643,7 +13722,8 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr, attr->size = size; - if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3) + if (attr->__reserved_1 || attr->__reserved_2 || + attr->__reserved_3 || attr->__reserved_4) return -EINVAL; if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) @@ -13689,9 +13769,18 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr, } if (attr->sample_type & PERF_SAMPLE_REGS_USER) { - ret = perf_reg_validate(attr->sample_regs_user); + ret = perf_reg_validate(attr->sample_regs_user, + attr->sample_simd_regs_enabled); if (ret) return ret; + if (attr->sample_simd_regs_enabled) { + ret = perf_simd_reg_validate(attr->sample_simd_vec_reg_qwords, + attr->sample_simd_vec_reg_user, + attr->sample_simd_pred_reg_qwords, + attr->sample_simd_pred_reg_user); + if (ret) + return ret; + } } if (attr->sample_type & PERF_SAMPLE_STACK_USER) { @@ -13712,8 +13801,20 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr, if (!attr->sample_max_stack) attr->sample_max_stack = sysctl_perf_event_max_stack; - if (attr->sample_type & PERF_SAMPLE_REGS_INTR) - ret = perf_reg_validate(attr->sample_regs_intr); + if (attr->sample_type & PERF_SAMPLE_REGS_INTR) { + ret = perf_reg_validate(attr->sample_regs_intr, + attr->sample_simd_regs_enabled); + if (ret) + return ret; + if (attr->sample_simd_regs_enabled) { + ret = perf_simd_reg_validate(attr->sample_simd_vec_reg_qwords, + attr->sample_simd_vec_reg_intr, + attr->sample_simd_pred_reg_qwords, + attr->sample_simd_pred_reg_intr); + if (ret) + return ret; + } + } #ifndef CONFIG_CGROUP_PERF if (attr->sample_type & PERF_SAMPLE_CGROUP) @@ -13946,7 +14047,7 @@ SYSCALL_DEFINE5(perf_event_open, if (err) return err; - if (!attr.exclude_kernel || + if (!attr.exclude_kernel || attr.text_poke || ((attr.sample_type & PERF_SAMPLE_CALLCHAIN) && !attr.exclude_callchain_kernel)) { err = perf_allow_kernel(); @@ -14434,23 +14535,18 @@ err: } EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter); -static void __perf_pmu_remove(struct perf_event_context *ctx, - int cpu, struct pmu *pmu, - struct perf_event_groups *groups, +static void __perf_pmu_remove(struct perf_event_context *ctx, struct pmu *pmu, struct list_head *events) { - struct perf_event *event, *sibling; + struct perf_event *event, *tmp; + + list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry) { + if (event->pmu_ctx->pmu != pmu) + continue; - perf_event_groups_for_cpu_pmu(event, groups, cpu, pmu) { perf_remove_from_context(event, 0); put_pmu_ctx(event->pmu_ctx); list_add(&event->migrate_entry, events); - - for_each_sibling_event(sibling, event) { - perf_remove_from_context(sibling, 0); - put_pmu_ctx(sibling->pmu_ctx); - list_add(&sibling->migrate_entry, events); - } } } @@ -14526,8 +14622,7 @@ void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu) */ mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex); - __perf_pmu_remove(src_ctx, src_cpu, pmu, &src_ctx->pinned_groups, &events); - __perf_pmu_remove(src_ctx, src_cpu, pmu, &src_ctx->flexible_groups, &events); + __perf_pmu_remove(src_ctx, pmu, &events); if (!list_empty(&events)) { /* diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index 1b1ffe0533e5..64d4dbdc6c07 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -246,14 +246,13 @@ __perf_output_begin(struct perf_output_handle *handle, handle->size = (1UL << page_shift) - offset; if (unlikely(have_lost)) { - lost_event.header.size = sizeof(lost_event); - lost_event.header.type = PERF_RECORD_LOST; - lost_event.header.misc = 0; lost_event.id = event->id; lost_event.lost = local_xchg(&rb->lost, 0); /* XXX mostly redundant; @data is already fully initializes */ - perf_event_header__init_id(&lost_event.header, data, event); + perf_event_header__init(&lost_event.header, data, + PERF_RECORD_LOST, /* misc= */ 0, + sizeof(lost_event), event); perf_output_put(handle, lost_event); perf_event__output_id_sample(event, handle, data); } @@ -635,11 +634,10 @@ static struct page *rb_alloc_aux_page(int node, int order) /* * Communicate the allocation size to the driver: * if we managed to secure a high-order allocation, - * set its first page's private to this order; - * !PagePrivate(page) means it's just a normal page. + * set its first page's private to this order, otherwise page's + * private remains zero. */ split_page(page, order); - SetPagePrivate(page); set_page_private(page, order); } @@ -650,7 +648,7 @@ static void rb_free_aux_page(struct perf_buffer *rb, int idx) { struct page *page = virt_to_page(rb->aux_pages[idx]); - ClearPagePrivate(page); + set_page_private(page, 0); __free_page(page); } diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 7709ea882477..3192806eebfe 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -2404,7 +2404,7 @@ static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr) return -EINVAL; pagefault_disable(); - result = __get_user(opcode, (uprobe_opcode_t __user *)vaddr); + result = get_user(opcode, (uprobe_opcode_t __user *)vaddr); pagefault_enable(); if (likely(result == 0)) diff --git a/kernel/exit.c b/kernel/exit.c index 424c44a42a4d..d5e7bc2af804 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -17,6 +17,7 @@ #include <linux/module.h> #include <linux/capability.h> #include <linux/completion.h> +#include <linux/wait_bit.h> #include <linux/personality.h> #include <linux/tty.h> #include <linux/iocontext.h> @@ -167,12 +168,6 @@ static void __exit_signal(struct release_task_post *post, struct task_struct *ts lockdep_tasklist_lock_is_held()); spin_lock(&sighand->siglock); -#ifdef CONFIG_POSIX_TIMERS - posix_cpu_timers_exit(tsk); - if (group_dead) - posix_cpu_timers_exit_group(tsk); -#endif - if (group_dead) { tty = sig->tty; sig->tty = NULL; @@ -436,15 +431,14 @@ static void coredump_task_exit(struct task_struct *tsk, self.task = tsk; if (self.task->flags & PF_SIGNALED) - self.next = xchg(&core_state->dumper.next, &self); + self.next = xchg(&core_state->tasks, &self); else self.task = NULL; /* * Implies mb(), the result of xchg() must be visible - * to core_state->dumper. + * to the dumper. */ - if (atomic_dec_and_test(&core_state->nr_threads)) - complete(&core_state->startup); + atomic_dec_and_wake_up(&core_state->threads_remaining); for (;;) { set_current_state(TASK_IDLE|TASK_FREEZABLE); @@ -551,32 +545,6 @@ void mm_update_next_owner(struct mm_struct *mm) } #endif /* CONFIG_MEMCG */ -#if defined(CONFIG_SCHED_CACHE) && defined(CONFIG_NUMA_BALANCING) -/* - * Subtract the memory footprint of the current task from - * mm. - */ -static void exit_mm_sched_cache(struct mm_struct *mm) -{ - unsigned long fp, sub; - - if (!current->total_numa_faults) - return; - /* - * No lock protection due to performance considerations. - * Make sure mm->sc_stat.footprint does not become - * negative. - */ - fp = READ_ONCE(mm->sc_stat.footprint); - sub = min(fp, current->total_numa_faults); - WRITE_ONCE(mm->sc_stat.footprint, fp - sub); -} -#else -static inline void exit_mm_sched_cache(struct mm_struct *mm) -{ -} -#endif /* CONFIG_SCHED_CACHE CONFIG_NUMA_BALANCING */ - /* * Turn us into a lazy TLB process if we * aren't already.. @@ -589,7 +557,7 @@ static void exit_mm(void) if (!mm) return; - exit_mm_sched_cache(mm); + sched_cache_exit_mm(current); mmap_read_lock(mm); mmgrab_lazy_tlb(mm); @@ -918,7 +886,7 @@ static void synchronize_group_exit(struct task_struct *tsk, long code) * Serialize with any possible pending coredump. * We must hold siglock around checking core_state * and setting PF_POSTCOREDUMP. The core-inducing thread - * will increment ->nr_threads for each thread in the + * will increment ->threads_remaining for each thread in the * group without PF_POSTCOREDUMP set. */ tsk->flags |= PF_POSTCOREDUMP; @@ -966,13 +934,12 @@ void __noreturn do_exit(long code) panic("Attempted to kill init! exitcode=0x%08x\n", tsk->signal->group_exit_code ?: (int)code); -#ifdef CONFIG_POSIX_TIMERS - hrtimer_cancel(&tsk->signal->real_timer); - exit_itimers(tsk); -#endif if (tsk->mm) setmax_mm_hiwater_rss(&tsk->signal->maxrss, tsk->mm); } + + posixtimer_exit(group_dead); + acct_collect(code, group_dead); if (group_dead) tty_audit_exit(); @@ -1004,10 +971,11 @@ void __noreturn do_exit(long code) exit_sem(tsk); exit_shm(tsk); - exit_files(tsk); - exit_fs(tsk); + /* Hang the tty up before the last close of it can clear the session. */ if (group_dead) disassociate_ctty(1); + exit_files(tsk); + exit_fs(tsk); exit_nsproxy_namespaces(tsk); exit_task_work(tsk); exit_thread(tsk); @@ -1755,7 +1723,7 @@ static long do_wait(struct wait_opts *wo) add_wait_queue(¤t->signal->wait_chldexit, &wo->child_wait); do { - set_current_state(TASK_INTERRUPTIBLE); + set_current_state(TASK_INTERRUPTIBLE | TASK_FREEZABLE); retval = __do_wait(wo); if (retval != -ERESTARTSYS) break; diff --git a/kernel/fork.c b/kernel/fork.c index 5ef413368912..2142c6f38119 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1083,9 +1083,7 @@ static void mmap_init_lock(struct mm_struct *mm) { init_rwsem(&mm->mmap_lock); mm_lock_seqcount_init(mm); -#ifdef CONFIG_PER_VMA_LOCK rcuwait_init(&mm->vma_writer_wait); -#endif } static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p) @@ -1599,6 +1597,7 @@ static int copy_mm(u64 clone_flags, struct task_struct *tsk) tsk->mm = mm; tsk->active_mm = mm; + sched_cache_fork(tsk); return 0; } @@ -1618,7 +1617,13 @@ static int copy_exec_state(u64 clone_flags, struct task_struct *tsk) return task_exec_state_copy(tsk); } -static int copy_fs(u64 clone_flags, struct task_struct *tsk, bool umh) +enum copy_fs_flags_t { + COPY_FS_UMH = (1 << 0), + COPY_FS_IDLE = (1 << 1), +}; + +static int copy_fs(u64 clone_flags, struct task_struct *tsk, + enum copy_fs_flags_t flags) { struct fs_struct *fs; @@ -1628,19 +1633,20 @@ static int copy_fs(u64 clone_flags, struct task_struct *tsk, bool umh) * filesystem state, or be started from a non-initial mount * namespace. */ - if (umh) { + if (flags & COPY_FS_UMH) { if (clone_flags & (CLONE_NEWNS | CLONE_FS)) return -EINVAL; if (current->nsproxy->mnt_ns != &init_mnt_ns) return -EINVAL; fs = userspace_init_fs; + } else if (flags & COPY_FS_IDLE) { + fs = &init_fs; } else { fs = current->fs; VFS_WARN_ON_ONCE(current->fs != current->real_fs); } if (clone_flags & CLONE_FS) { - /* tsk->fs is already what we want */ read_seqlock_excl(&fs->seq); /* "users" and "in_exec" locked for check_unsafe_exec() */ if (fs->in_exec) { @@ -1649,6 +1655,7 @@ static int copy_fs(u64 clone_flags, struct task_struct *tsk, bool umh) } fs->users++; read_sequnlock_excl(&fs->seq); + tsk->real_fs = tsk->fs = fs; return 0; } tsk->real_fs = tsk->fs = copy_fs_struct(fs); @@ -1676,6 +1683,7 @@ static int copy_files(u64 clone_flags, struct task_struct *tsk, if (clone_flags & CLONE_FILES) { atomic_inc(&oldf->count); + tsk->files = oldf; return 0; } @@ -2145,12 +2153,9 @@ __latent_entropy struct task_struct *copy_process( if (args->kthread) p->flags |= PF_KTHREAD; if (args->user_worker) { - /* - * Mark us a user worker, and block any signal that isn't - * fatal or STOP - */ + /* A user worker takes only the signals nobody can block. */ p->flags |= PF_USER_WORKER; - siginitsetinv(&p->blocked, sigmask(SIGKILL)|sigmask(SIGSTOP)); + siginitsetinv(&p->blocked, SIG_KERNEL_ONLY_MASK); } if (args->io_thread) p->flags |= PF_IO_WORKER; @@ -2199,6 +2204,8 @@ __latent_entropy struct task_struct *copy_process( INIT_LIST_HEAD(&p->sibling); rcu_copy_process(p); p->vfork_done = NULL; + /* Set by copy_files(), exit_files() on the error path skips NULL. */ + p->files = NULL; spin_lock_init(&p->alloc_lock); init_sigpending(&p->pending); @@ -2298,9 +2305,11 @@ __latent_entropy struct task_struct *copy_process( retval = copy_files(clone_flags, p, args->no_files); if (retval) goto bad_fork_cleanup_semundo; - retval = copy_fs(clone_flags, p, args->umh); + retval = copy_fs(clone_flags, p, + (args->umh ? COPY_FS_UMH : 0) | + (args->idle ? COPY_FS_IDLE : 0)); if (retval) - goto bad_fork_cleanup_files; + goto bad_fork_cleanup_semundo; retval = copy_sighand(clone_flags, p); if (retval) goto bad_fork_cleanup_fs; @@ -2602,6 +2611,7 @@ bad_fork_cleanup_io: bad_fork_cleanup_namespaces: exit_nsproxy_namespaces(p); bad_fork_cleanup_mm: + sched_cache_fork_cleanup(p); if (p->mm) { mm_clear_owner(p->mm, p); mmput(p->mm); @@ -2613,8 +2623,6 @@ bad_fork_cleanup_sighand: __cleanup_sighand(p->sighand); bad_fork_cleanup_fs: exit_fs(p); /* blocking */ -bad_fork_cleanup_files: - exit_files(p); /* blocking */ bad_fork_cleanup_semundo: exit_sem(p); bad_fork_cleanup_security: @@ -2625,6 +2633,8 @@ bad_fork_cleanup_perf: perf_event_free_task(p); bad_fork_sched_cancel_fork: sched_cancel_fork(p); + /* ->release() of a file may need scx_fork_rwsem for write. */ + exit_files(p); /* blocking */ bad_fork_cleanup_policy: lockdep_free_task(p); #ifdef CONFIG_NUMA @@ -2669,7 +2679,7 @@ struct task_struct * __init fork_idle(int cpu) { struct task_struct *task; struct kernel_clone_args args = { - .flags = CLONE_VM, + .flags = CLONE_VM | CLONE_FS, .fn = &idle_dummy, .fn_arg = NULL, .kthread = 1, @@ -2703,6 +2713,10 @@ struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node) .user_worker = 1, }; + /* A creator past its fatal signal or its coredump point gets no thread. */ + if (current->flags & (PF_SIGNALED | PF_POSTCOREDUMP)) + return ERR_PTR(-EINTR); + return copy_process(NULL, 0, node, &args); } @@ -3211,24 +3225,6 @@ static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp) } /* - * Unshare file descriptor table if it is being shared - */ -static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp) -{ - struct files_struct *fd = current->files; - - if ((unshare_flags & CLONE_FILES) && - (fd && atomic_read(&fd->count) > 1)) { - fd = dup_fd(fd, NULL); - if (IS_ERR(fd)) - return PTR_ERR(fd); - *new_fdp = fd; - } - - return 0; -} - -/* * unshare allows a process to 'unshare' part of the process * context which was originally shared using clone. copy_* * functions used by kernel_clone() cannot be used here directly @@ -3315,18 +3311,19 @@ int ksys_unshare(unsigned long unshare_flags) shm_init_task(current); } + if (new_fs) { + new_fs = switch_fs_struct(new_fs); + if (new_fs) + free_fs_struct(no_free_ptr(new_fs)); + } + if (new_nsproxy) { switch_task_namespaces(current, new_nsproxy); new_nsproxy = NULL; } - if (new_fs) - new_fs = switch_fs_struct(new_fs); - - if (new_fd) { - guard(task_lock)(current); - swap(current->files, new_fd); - } + if (new_fd) + switch_files_struct(current, no_free_ptr(new_fd)); if (new_cred) { /* Install the new user namespace */ @@ -3338,8 +3335,9 @@ int ksys_unshare(unsigned long unshare_flags) perf_event_namespaces(current); bad_unshare_cleanup_nsproxy: + /* never installed, so no active references to drop */ if (new_nsproxy) - put_nsproxy(new_nsproxy); + nsproxy_free(new_nsproxy); bad_unshare_cleanup_cred: if (new_cred) put_cred(new_cred); @@ -3359,30 +3357,6 @@ SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags) return ksys_unshare(unshare_flags); } -/* - * Helper to unshare the files of the current task. - * We don't want to expose copy_files internals to - * the exec layer of the kernel. - */ - -int unshare_files(void) -{ - struct task_struct *task = current; - struct files_struct *old, *copy = NULL; - int error; - - error = unshare_fd(CLONE_FILES, ©); - if (error || !copy) - return error; - - old = task->files; - task_lock(task); - task->files = copy; - task_unlock(task); - put_files_struct(old); - return 0; -} - static int sysctl_max_threads(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { diff --git a/kernel/futex/core.c b/kernel/futex/core.c index a061f54b606d..a240638f69e5 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -213,10 +213,12 @@ static bool __futex_pivot_hash(struct mm_struct *mm, struct futex_private_hash * futex_rehash_private(fph, new); } new->state = FR_PERCPU; - scoped_guard(rcu) { - mmph->batches = get_state_synchronize_rcu(); - rcu_assign_pointer(mmph->hash, new); - } + rcu_assign_pointer(mmph->hash, new); + /* + * mmph->batches must reference a grace period which started after + * mmph->hash was assigned. See futex_ref_drop(). + */ + mmph->batches = get_state_synchronize_rcu(); kvfree_rcu(fph, rcu); return true; } @@ -437,45 +439,51 @@ struct hrtimer_sleeper *futex_setup_timer(ktime_t *time, struct hrtimer_sleeper } /* - * Generate a machine wide unique identifier for this inode. + * Generate a machine wide unique identifier for this object (inode or mm). * * This relies on u64 not wrapping in the life-time of the machine; which with * 1ns resolution means almost 585 years. - * - * This further relies on the fact that a well formed program will not unmap - * the file while it has a (shared) futex waiting on it. This mapping will have - * a file reference which pins the mount and inode. - * - * If for some reason an inode gets evicted and read back in again, it will get - * a new sequence number and will _NOT_ match, even though it is the exact same - * file. - * - * It is important that futex_match() will never have a false-positive, esp. - * for PI futexes that can mess up the state. The above argues that false-negatives - * are only possible for malformed programs. */ -static u64 get_inode_sequence_number(struct inode *inode) +static u64 get_object_id(atomic64_t *object_seq) { - static atomic64_t i_seq; + static atomic64_t last_assigned_seq; u64 old; - /* Does the inode already have a sequence number? */ - old = atomic64_read(&inode->i_sequence); + /* Does the object already have a sequence number? */ + old = atomic64_read(object_seq); if (likely(old)) return old; for (;;) { - u64 new = atomic64_inc_return(&i_seq); + u64 new = atomic64_inc_return(&last_assigned_seq); if (WARN_ON_ONCE(!new)) continue; old = 0; - if (!atomic64_try_cmpxchg_relaxed(&inode->i_sequence, &old, new)) + if (!atomic64_try_cmpxchg_relaxed(object_seq, &old, new)) return old; return new; } } +/* + * This relies on the fact that a well formed program will not unmap + * the file while it has a (shared) futex waiting on it. This mapping will have + * a file reference which pins the mount and inode. + * + * If for some reason an inode gets evicted and read back in again, it will get + * a new sequence number and will _NOT_ match, even though it is the exact same + * file. + * + * It is important that futex_match() will never have a false-positive, esp. + * for PI futexes that can mess up the state. The above argues that false-negatives + * are only possible for malformed programs. + */ +static u64 get_inode_sequence_number(struct inode *inode) +{ + return get_object_id(&inode->i_sequence); +} + /** * get_futex_key() - Get parameters which are the keys for a futex * @uaddr: virtual address of the futex @@ -681,8 +689,8 @@ again: goto out; } - key->both.offset |= FUT_OFF_MMSHARED; /* ref taken on mm */ - key->private.mm = mm; + key->both.offset |= FUT_OFF_MMSHARED; + key->private.mm_seq = get_object_id(&mm->futex.unique_id); key->private.address = address; } else { @@ -2053,13 +2061,12 @@ static void futex_robust_unlock_init_mm(struct futex_mm_data *fd) static inline void futex_robust_unlock_init_mm(struct futex_mm_data *fd) { } #endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ -#if defined(CONFIG_FUTEX_PRIVATE_HASH) || defined(CONFIG_FUTEX_ROBUST_UNLOCK) void futex_mm_init(struct mm_struct *mm) { + atomic64_set(&mm->futex.unique_id, 0); futex_hash_init_mm(&mm->futex); futex_robust_unlock_init_mm(&mm->futex); } -#endif int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4) { diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c index b3f4a4bccb12..842d852302dd 100644 --- a/kernel/futex/requeue.c +++ b/kernel/futex/requeue.c @@ -744,7 +744,7 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb, /* Handle spurious wakeups gracefully */ ret = -EWOULDBLOCK; - if (timeout && !timeout->task) + if (timeout && !hrtimer_sleeper_task_get(timeout)) ret = -ETIMEDOUT; else if (signal_pending(current)) ret = -ERESTARTNOINTR; diff --git a/kernel/futex/waitwake.c b/kernel/futex/waitwake.c index d4483d15d30a..cf18309e5770 100644 --- a/kernel/futex/waitwake.c +++ b/kernel/futex/waitwake.c @@ -383,7 +383,7 @@ void futex_do_wait(struct futex_q *q, struct hrtimer_sleeper *timeout) * flagged for rescheduling. Only call schedule if there * is no timeout, or if it has yet to expire. */ - if (!timeout || timeout->task) + if (!timeout || hrtimer_sleeper_task_get(timeout)) schedule(); } __set_current_state(TASK_RUNNING); @@ -539,7 +539,7 @@ retry: static void futex_sleep_multiple(struct futex_vector *vs, unsigned int count, struct hrtimer_sleeper *to) { - if (to && !to->task) + if (to && !hrtimer_sleeper_task_get(to)) return; for (; count; count--, vs++) { @@ -590,7 +590,7 @@ int futex_wait_multiple(struct futex_vector *vs, unsigned int count, if (ret >= 0) return ret; - if (to && !to->task) + if (to && !hrtimer_sleeper_task_get(to)) return -ETIMEDOUT; else if (signal_pending(current)) return -ERESTARTSYS; @@ -725,7 +725,7 @@ retry: if (!futex_unqueue(&q)) return 0; - if (to && !to->task) + if (to && !hrtimer_sleeper_task_get(to)) return -ETIMEDOUT; /* diff --git a/kernel/irq/Kconfig b/kernel/irq/Kconfig index 05cba4e16dad..6923f37eaab4 100644 --- a/kernel/irq/Kconfig +++ b/kernel/irq/Kconfig @@ -150,6 +150,18 @@ config IRQ_KUNIT_TEST If unsure, say N. +config REFCOUNT_INTERRUPT_KUNIT_TEST + tristate "Test refcounted interrupt enable/disable" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + This builds the kunit tests for the refcounted interrupt + infrastructure. It verifies the correctness of single, nested, + and multiple interrupt enable/disable state changes and ensures + that the underlying reference counting mechanisms work as expected. + + If unsure, say N. + endmenu config GENERIC_IRQ_MULTI_HANDLER diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile index 44c4d6fc502a..0e5df962a149 100644 --- a/kernel/irq/Makefile +++ b/kernel/irq/Makefile @@ -16,4 +16,4 @@ obj-$(CONFIG_SMP) += affinity.o obj-$(CONFIG_GENERIC_IRQ_DEBUGFS) += debugfs.o obj-$(CONFIG_GENERIC_IRQ_MATRIX_ALLOCATOR) += matrix.o obj-$(CONFIG_IRQ_KUNIT_TEST) += irq_test.o -obj-$(CONFIG_KUNIT) += refcount_interrupt_test.o +obj-$(CONFIG_REFCOUNT_INTERRUPT_KUNIT_TEST) += refcount_interrupt_test.o diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index 0ce21dd45404..2c30e7537987 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -36,6 +36,7 @@ extern struct irqaction chained_action; * IRQTF_AFFINITY - irq thread is requested to adjust affinity * IRQTF_FORCED_THREAD - irq action is force threaded * IRQTF_READY - signals that irq thread is ready + * IRQTF_EXIT_WORK - irq thread exit task_work is pending */ enum { IRQTF_RUNTHREAD, @@ -43,6 +44,7 @@ enum { IRQTF_AFFINITY, IRQTF_FORCED_THREAD, IRQTF_READY, + IRQTF_EXIT_WORK, }; /* diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c index 3a818f07a101..7cbcc3fb9e6d 100644 --- a/kernel/irq/irqdesc.c +++ b/kernel/irq/irqdesc.c @@ -28,7 +28,15 @@ static struct lock_class_key irq_desc_lock_class; static int __init irq_affinity_setup(char *str) { alloc_bootmem_cpumask_var(&irq_default_affinity); - cpulist_parse(str, irq_default_affinity); + if (cpulist_parse(str, irq_default_affinity) < 0) { + /* + * Clear the mask so that init_irq_default_affinity() + * defaults it later. + */ + cpumask_clear(irq_default_affinity); + pr_warn("irqaffinity: incorrect CPU range, using default\n"); + return 1; + } /* * Set at least the boot cpu. We don't want to end up with * bugreports caused by random commandline masks diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 57c819da30c2..4fdcb6df5306 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -344,6 +344,7 @@ static struct irq_domain *__irq_domain_instantiate(const struct irq_domain_info err = irq_domain_alloc_generic_chips(domain, info->dgc_info); if (err) goto err_domain_free; + domain->flags |= IRQ_DOMAIN_FLAG_DESTROY_GC; } if (info->init) { diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 57eff26fa646..3b647d9b458a 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -21,6 +21,7 @@ #include <linux/sched/isolation.h> #include <uapi/linux/sched/types.h> #include <linux/task_work.h> +#include <linux/wait_bit.h> #include "internals.h" @@ -1176,17 +1177,14 @@ void wake_threads_waitq(struct irq_desc *desc) wake_up(&desc->wait_for_threads); } -static void irq_thread_dtor(struct callback_head *unused) +static void __irq_thread_dtor(struct irqaction *action) { struct task_struct *tsk = current; struct irq_desc *desc; - struct irqaction *action; if (WARN_ON_ONCE(!(current->flags & PF_EXITING))) return; - action = kthread_data(tsk); - pr_err("exiting task \"%s\" (%d) is an active IRQ thread (irq %d)\n", tsk->comm, tsk->pid, action->irq); @@ -1203,6 +1201,14 @@ static void irq_thread_dtor(struct callback_head *unused) irq_finalize_oneshot(desc, action); } +static void irq_thread_dtor(struct callback_head *work) +{ + struct irqaction *action = container_of(work, struct irqaction, thread_dtor); + + __irq_thread_dtor(action); + clear_and_wake_up_bit(IRQTF_EXIT_WORK, &action->thread_flags); +} + static void irq_wake_secondary(struct irq_desc *desc, struct irqaction *action) { struct irqaction *secondary = action->secondary; @@ -1244,7 +1250,6 @@ static void wake_up_and_wait_for_irq_thread_ready(struct irq_desc *desc, */ static int irq_thread(void *data) { - struct callback_head on_exit_work; struct irqaction *action = data; struct irq_desc *desc = irq_to_desc(action->irq); irqreturn_t (*handler_fn)(struct irq_desc *desc, @@ -1263,8 +1268,9 @@ static int irq_thread(void *data) else handler_fn = irq_thread_fn; - init_task_work(&on_exit_work, irq_thread_dtor); - task_work_add(current, &on_exit_work, TWA_NONE); + set_bit(IRQTF_EXIT_WORK, &action->thread_flags); + init_task_work(&action->thread_dtor, irq_thread_dtor); + task_work_add(current, &action->thread_dtor, TWA_NONE); while (!irq_wait_for_interrupt(desc, action)) { irqreturn_t action_ret; @@ -1283,6 +1289,7 @@ static int irq_thread(void *data) * oneshot mask bit can be set. */ task_work_cancel_func(current, irq_thread_dtor); + clear_and_wake_up_bit(IRQTF_EXIT_WORK, &action->thread_flags); return 0; } @@ -1853,6 +1860,12 @@ out_mput: return ret; } +static void irq_stop_thread(struct irqaction *action) +{ + kthread_stop_put(action->thread); + wait_on_bit(&action->thread_flags, IRQTF_EXIT_WORK, TASK_UNINTERRUPTIBLE); +} + /* * Internal function to unregister an irqaction - used to free * regular and special interrupts that are part of the architecture. @@ -1958,9 +1971,9 @@ static struct irqaction *__free_irq(struct irq_desc *desc, void *dev_id) * the same bit to a newly requested action. */ if (action->thread) { - kthread_stop_put(action->thread); + irq_stop_thread(action); if (action->secondary && action->secondary->thread) - kthread_stop_put(action->secondary->thread); + irq_stop_thread(action->secondary); } /* Last action releases resources */ diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index fb5f372215bf..b22d74496af1 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -142,7 +142,7 @@ fail: * insert it at @init_desc->msi_index * * @dev: Pointer to the device for which the descriptor is allocated - * @domid: The id of the interrupt domain to which the desriptor is added + * @domid: The id of the interrupt domain to which the descriptor is added * @init_desc: Pointer to an MSI descriptor to initialize the new descriptor * * Return: 0 on success or an appropriate failure code. @@ -416,7 +416,7 @@ EXPORT_SYMBOL_GPL(msi_domain_first_desc); * @domid: The id of the interrupt domain which should be walked. * @filter: Descriptor state filter * - * The first invocation of msi_next_desc() has to be preceeded by a + * The first invocation of msi_next_desc() has to be preceded by a * successful invocation of __msi_first_desc(). Consecutive invocations are * only valid if the previous one was successful. All these operations have * to be done within the same MSI mutex held region. diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index 1b835725f7b1..0c0df16de581 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -462,7 +462,7 @@ int __weak arch_show_interrupts(struct seq_file *p, int prec) static DEFINE_RAW_SPINLOCK(irq_proc_constraints_lock); static struct irq_proc_constraints { - bool print_header; + loff_t first_pos; unsigned int num_prec; unsigned int chip_width; } irq_proc_constraints __read_mostly = { @@ -543,14 +543,13 @@ static int irq_seq_show(struct seq_file *p, void *v) struct irqaction *action; /* Print header for the first interrupt? */ - if (constr->print_header) { + if (p->index == constr->first_pos) { unsigned int cpu; seq_printf(p, "%*s", constr->num_prec + 8, ""); for_each_online_cpu(cpu) seq_printf(p, "CPU%-8d", cpu); seq_putc(p, '\n'); - constr->print_header = false; } if (desc == ARCH_PROC_IRQDESC) @@ -635,14 +634,25 @@ static void *irq_seq_next_desc(loff_t *pos) static void *irq_seq_start(struct seq_file *f, loff_t *pos) { - if (!*pos) { - struct irq_proc_constraints *constr = f->private; + struct irq_proc_constraints *constr = f->private; + bool first = !*pos; + void *ret; + if (first) { constr->num_prec = READ_ONCE(irq_proc_constraints.num_prec); constr->chip_width = READ_ONCE(irq_proc_constraints.chip_width); - constr->print_header = true; } - return irq_seq_next_desc(pos); + + ret = irq_seq_next_desc(pos); + + /* + * Record position of the first output record so that irq_seq_show() + * knows where to write the header. + */ + if (first) + constr->first_pos = *pos; + + return ret; } static void *irq_seq_next(struct seq_file *f, void *v, loff_t *pos) diff --git a/kernel/irq/refcount_interrupt_test.c b/kernel/irq/refcount_interrupt_test.c index ca904dba24b9..38dfccbaa4d4 100644 --- a/kernel/irq/refcount_interrupt_test.c +++ b/kernel/irq/refcount_interrupt_test.c @@ -52,6 +52,27 @@ static void test_multiple_irq_change(struct kunit *test) TEST_IRQ_ON(); } +static void test_max_nesting_irq_change(struct kunit *test) +{ + for (int i = 0; i < __IRQ_MASK(HARDIRQ_DISABLE_BITS); i++) { + local_interrupt_disable(); + TEST_IRQ_OFF(); + } + + + for (int i = 0; i < __IRQ_MASK(HARDIRQ_DISABLE_BITS); i++) { + TEST_IRQ_OFF(); + local_interrupt_enable(); + } + + TEST_IRQ_ON(); + + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); +} + static void test_irq_save(struct kunit *test) { unsigned long flags; @@ -79,6 +100,7 @@ static struct kunit_case test_cases[] = { KUNIT_CASE(test_single_irq_change), KUNIT_CASE(test_nested_irq_change), KUNIT_CASE(test_multiple_irq_change), + KUNIT_CASE(test_max_nesting_irq_change), KUNIT_CASE(test_irq_save), {}, }; diff --git a/kernel/irq_work.c b/kernel/irq_work.c index f7e2dc2c30c6..73eabcbdcd50 100644 --- a/kernel/irq_work.c +++ b/kernel/irq_work.c @@ -252,10 +252,7 @@ static void irq_work_run_list(struct llist_head *list) irq_work_single(work); } -/* - * hotplug calls this through: - * hotplug_cfd() -> flush_smp_call_function_queue() - */ +/* CPU hotplug calls this through smpcfd_dying_cpu() */ void irq_work_run(void) { irq_work_run_list(this_cpu_ptr(&raised_list)); @@ -266,6 +263,17 @@ void irq_work_run(void) } EXPORT_SYMBOL_GPL(irq_work_run); +void irq_work_run_cpu(unsigned int cpu) +{ + if (WARN_ON_ONCE(!cpumask_test_cpu(cpu, cpu_dying_mask))) + return; + + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + return; + + irq_work_run_list(per_cpu_ptr(&lazy_list, cpu)); +} + void irq_work_tick(void) { struct llist_head *raised = this_cpu_ptr(&raised_list); diff --git a/kernel/kcsan/core.c b/kernel/kcsan/core.c index 2db82661cd60..a3aa5e5b039d 100644 --- a/kernel/kcsan/core.c +++ b/kernel/kcsan/core.c @@ -290,8 +290,9 @@ should_watch(struct kcsan_ctx *ctx, const volatile void *ptr, size_t size, int t } /* - * Returns a pseudo-random number in interval [0, ep_ro). Simple linear - * congruential generator, using constants from "Numerical Recipes". + * Returns a pseudo-random number in interval [0, ep_ro), or 0 if ep_ro is 0, + * which the runtime-tunable callers may pass. Simple linear congruential + * generator, using constants from "Numerical Recipes". */ static u32 kcsan_prandom_u32_max(u32 ep_ro) { @@ -300,7 +301,7 @@ static u32 kcsan_prandom_u32_max(u32 ep_ro) state = 1664525 * state + 1013904223; this_cpu_write(kcsan_rand_state, state); - return state % ep_ro; + return reciprocal_scale(state, ep_ro); } static inline void reset_kcsan_skip(void) diff --git a/kernel/kcsan/kcsan_test.c b/kernel/kcsan/kcsan_test.c index ae758150ccb9..760686e130ad 100644 --- a/kernel/kcsan/kcsan_test.c +++ b/kernel/kcsan/kcsan_test.c @@ -48,6 +48,9 @@ static void (*access_kernels[2])(void); static struct task_struct **threads; /* Lists of threads. */ static unsigned long end_time; /* End time of test. */ +static unsigned long end_time_max; /* Upper bound for end time of test. */ +static unsigned long report_budget; /* Time left when report printing began. */ +static bool report_paused; /* Paused while a report is printing. */ /* Report as observed from console. */ static struct { @@ -69,17 +72,46 @@ begin_test_checks(void (*func1)(void), void (*func2)(void)) * least one race is reported. */ end_time = jiffies + msecs_to_jiffies(CONFIG_KCSAN_REPORT_ONCE_IN_MS + 500); + end_time_max = end_time + msecs_to_jiffies(10000); + report_paused = false; /* Signal start; release potential initialization of shared data. */ smp_store_release(&access_kernels[0], func1); smp_store_release(&access_kernels[1], func2); } +/* + * Printing a report may take a while, e.g. if a printk emergency section (such + * as with CONFIG_KCSAN_VERBOSE) flushes a console backlog synchronously. Do not + * count the time a test-related report is being printed against the test's + * time budget, so that a slow report is neither missed, nor prevents observing + * other reports. Returns true if checking must continue. + */ +static __no_kcsan bool pause_for_report(void) +{ + const int nlines = READ_ONCE(observed.nlines); + const bool in_progress = nlines == 1 || nlines == 2; + + if (in_progress && !report_paused) { + report_paused = true; + report_budget = time_before(jiffies, end_time) ? end_time - jiffies : 0; + } else if (!in_progress && report_paused) { + report_paused = false; + end_time = jiffies + report_budget; + if (time_after(end_time, end_time_max)) + end_time = end_time_max; + /* Check the now complete report at least once. */ + return true; + } + + return report_paused && time_before(jiffies, end_time_max); +} + /* End test checking loop. */ static __no_kcsan inline bool end_test_checks(bool stop) { - if (!stop && time_before(jiffies, end_time)) { + if (!stop && (pause_for_report() || time_before(jiffies, end_time))) { /* Continue checking */ might_sleep(); return false; diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 6337da5cab9e..2441b9e39343 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -42,6 +42,8 @@ #include <linux/execmem.h> #include <linux/cleanup.h> #include <linux/wait.h> +#include <linux/wait_bit.h> +#include <linux/rcupdate.h> #include <asm/sections.h> #include <asm/cacheflush.h> @@ -66,7 +68,7 @@ static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; /* NOTE: change this value only with 'kprobe_mutex' held */ static bool kprobes_all_disarmed; -/* This protects 'kprobe_table' and 'optimizing_list' */ +/* This protects 'kprobe_table' and 'optprobe_gens' */ static DEFINE_MUTEX(kprobe_mutex); static DEFINE_PER_CPU(struct kprobe *, kprobe_instance); @@ -475,18 +477,6 @@ bool kprobe_disarmed(struct kprobe *p) return kprobe_disabled(p) && list_empty(&op->list); } -/* Return true if the probe is queued on (un)optimizing lists */ -static bool kprobe_queued(struct kprobe *p) -{ - struct optimized_kprobe *op; - - if (kprobe_aggrprobe(p)) { - op = container_of(p, struct optimized_kprobe, kp); - if (!list_empty(&op->list)) - return true; - } - return false; -} /* * Return an optimized kprobe whose optimizing code replaces @@ -495,14 +485,16 @@ static bool kprobe_queued(struct kprobe *p) static struct kprobe *get_optimized_kprobe(kprobe_opcode_t *addr) { int i; - struct kprobe *p = NULL; + struct kprobe *p; struct optimized_kprobe *op; /* Don't check i == 0, since that is a breakpoint case. */ - for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++) + for (i = 1; i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++) { p = get_kprobe(addr - i); + /* A disabled probe can have prepared, but inactive, optinsns. */ + if (!p || !kprobe_optready(p) || kprobe_disarmed(p)) + continue; - if (p && kprobe_optready(p)) { op = container_of(p, struct optimized_kprobe, kp); if (arch_within_optimized_kprobe(op, addr)) return p; @@ -511,10 +503,39 @@ static struct kprobe *get_optimized_kprobe(kprobe_opcode_t *addr) return NULL; } -/* Optimization staging list, protected by 'kprobe_mutex' */ -static LIST_HEAD(optimizing_list); -static LIST_HEAD(unoptimizing_list); -static LIST_HEAD(freeing_list); +#define OPTPROBE_GEN_MAX 2 + +struct optprobe_generation { + /* Probes waiting for grace period to drain tasks from original text */ + struct list_head optimizing_list; + /* Probes queued for batch unoptimization upon dispatch */ + struct list_head unoptimizing_list; + /* In-use probes waiting for grace period to drain optinsn slot */ + struct list_head cooling_list; + /* Unused probes unhashed and waiting for batch release */ + struct list_head freeing_list; + /* Tasks RCU callback head for asynchronous waiting */ + struct rcu_head rcu; + /* True if dispatched and awaiting Tasks RCU callback */ + bool in_flight; + /* True when Tasks RCU callback has fired and ready to finalize */ + bool ready; +}; + +/* + * Generational ring of optprobes. + * + * Incoming probe requests are queued into the waiting room generation + * (optprobe_gens[optprobe_cur_gen]). When dispatched, the generation + * unoptimizes its probes, invokes call_rcu_tasks(), and optprobe_cur_gen + * advances to the next slot. + * + * To ensure an idle generation is always available to collect incoming + * requests without dynamic allocation, the last available generation slot + * is never dispatched until another generation has finished. + */ +static struct optprobe_generation optprobe_gens[OPTPROBE_GEN_MAX]; +static int optprobe_cur_gen; static void optimize_kprobe(struct kprobe *p); static struct task_struct *kprobe_optimizer_task; @@ -526,54 +547,90 @@ enum { OPTIMIZER_ST_FLUSHING = 2, }; -static DECLARE_COMPLETION(optimizer_completion); +/* Bumped at the end of each kprobe_optimizer() pass, under 'kprobe_mutex' */ +static unsigned long optimizer_passes; #define OPTIMIZE_DELAY 5 /* - * Optimize (replace a breakpoint with a jump) kprobes listed on - * 'optimizing_list'. + * Note: gen->cooling_list is not checked here because it is strictly + * an in-flight holding list for dispatched generations, so it is always + * empty in optprobe_cur_gen. */ -static void do_optimize_kprobes(void) +static bool optprobe_has_queued_probes(void) { - lockdep_assert_held(&text_mutex); - /* - * The optimization/unoptimization refers 'online_cpus' via - * stop_machine() and cpu-hotplug modifies the 'online_cpus'. - * And same time, 'text_mutex' will be held in cpu-hotplug and here. - * This combination can cause a deadlock (cpu-hotplug tries to lock - * 'text_mutex' but stop_machine() can not be done because - * the 'online_cpus' has been changed) - * To avoid this deadlock, caller must have locked cpu-hotplug - * for preventing cpu-hotplug outside of 'text_mutex' locking. - */ - lockdep_assert_cpus_held(); + struct optprobe_generation *gen = &optprobe_gens[optprobe_cur_gen]; - /* Optimization never be done when disarmed */ - if (kprobes_all_disarmed || !kprobes_allow_optimization || - list_empty(&optimizing_list)) - return; + return !list_empty(&gen->optimizing_list) || + !list_empty(&gen->unoptimizing_list) || + !list_empty(&gen->freeing_list); +} + +static int optprobe_active_gens_count(void) +{ + int count = 0; + int i; + + for (i = 0; i < OPTPROBE_GEN_MAX; i++) { + if (optprobe_gens[i].in_flight || optprobe_gens[i].ready) + count++; + } + return count; +} + +/* + * The last generation must not be fired until another generation is done. + * (Thus the last generation acts as the waiting room.) + */ +static bool optprobe_can_fire(void) +{ + return optprobe_active_gens_count() < OPTPROBE_GEN_MAX - 1; +} - arch_optimize_kprobes(&optimizing_list); +static bool optprobe_has_ready_gens(void) +{ + int i; + + for (i = 0; i < OPTPROBE_GEN_MAX; i++) { + if (READ_ONCE(optprobe_gens[i].ready)) + return true; + } + return false; +} + +static bool optprobe_optimizer_busy(void) +{ + return optprobe_has_queued_probes() || (optprobe_active_gens_count() > 0); } /* * Unoptimize (replace a jump with a breakpoint and remove the breakpoint - * if need) kprobes listed on 'unoptimizing_list'. + * if need) kprobes listed on 'unopt_list'. */ -static void do_unoptimize_kprobes(void) +static void do_unoptimize_kprobes(struct list_head *unopt_list, + struct list_head *free_list, + struct list_head *cooling_list) { struct optimized_kprobe *op, *tmp; lockdep_assert_held(&text_mutex); - /* See comment in do_optimize_kprobes() */ + /* + * The optimization/unoptimization refers 'online_cpus' via + * stop_machine() and cpu-hotplug modifies the 'online_cpus'. + * And same time, 'text_mutex' will be held in cpu-hotplug and here. + * This combination can cause a deadlock (cpu-hotplug tries to lock + * 'text_mutex' but stop_machine() can not be done because + * the 'online_cpus' has been changed) + * To avoid this deadlock, caller must have locked cpu-hotplug + * for preventing cpu-hotplug outside of 'text_mutex' locking. + */ lockdep_assert_cpus_held(); - if (!list_empty(&unoptimizing_list)) - arch_unoptimize_kprobes(&unoptimizing_list, &freeing_list); + if (!list_empty(unopt_list)) + arch_unoptimize_kprobes(unopt_list, free_list); - /* Loop on 'freeing_list' for disarming and removing from kprobe hash list */ - list_for_each_entry_safe(op, tmp, &freeing_list, list) { + /* Loop on 'free_list' for disarming and removing from kprobe hash list */ + list_for_each_entry_safe(op, tmp, free_list, list) { /* Switching from detour code to origin */ op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; /* Disarm probes if marked disabled and not gone */ @@ -585,18 +642,26 @@ static void do_unoptimize_kprobes(void) * for synchronization, these probes are reclaimed. * (reclaiming is done by do_free_cleaned_kprobes().) */ - hlist_del_rcu(&op->kp.hlist); - } else - list_del_init(&op->list); + hlist_del_init_rcu(&op->kp.hlist); + } else { + /* + * Keep on cooling_list until the quiescence period + * completes so that kprobe_disarmed() remains false and + * unregister_kprobes() does not prematurely free it. + */ + list_move(&op->list, cooling_list); + } } } -/* Reclaim all kprobes on the 'freeing_list' */ -static void do_free_cleaned_kprobes(void) +/* Reclaim all kprobes on the 'free_list' */ +static void do_free_cleaned_kprobes(struct list_head *free_list) { struct optimized_kprobe *op, *tmp; - list_for_each_entry_safe(op, tmp, &freeing_list, list) { + list_for_each_entry_safe(op, tmp, free_list, list) { + struct kprobe *_p; + list_del_init(&op->list); if (WARN_ON_ONCE(!kprobe_unused(&op->kp))) { /* @@ -608,11 +673,10 @@ static void do_free_cleaned_kprobes(void) /* * The aggregator was holding back another probe while it sat on the - * unoptimizing/freeing lists. Now that the aggregator has been fully + * unoptimizing/freeing lists. Now that the aggregator has been fully * reverted we can safely retry the optimization of that sibling. */ - - struct kprobe *_p = get_optimized_kprobe(op->kp.addr); + _p = get_optimized_kprobe(op->kp.addr); if (unlikely(_p)) optimize_kprobe(_p); @@ -622,67 +686,146 @@ static void do_free_cleaned_kprobes(void) static void kick_kprobe_optimizer(void); -/* Kprobe jump optimizer */ -static void kprobe_optimizer(void) +static void optprobe_generation_rcu_cb(struct rcu_head *rcu) { - guard(mutex)(&kprobe_mutex); + struct optprobe_generation *gen; + + gen = container_of(rcu, struct optprobe_generation, rcu); + WRITE_ONCE(gen->ready, true); + wake_up(&kprobe_optimizer_wait); +} + +static void optprobe_finalize_generation(struct optprobe_generation *gen) +{ + struct optimized_kprobe *op, *tmp; + + lockdep_assert_held(&kprobe_mutex); scoped_guard(cpus_read_lock) { guard(mutex)(&text_mutex); - /* - * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed) - * kprobes before waiting for quiesence period. - */ - do_unoptimize_kprobes(); + /* Optimization never be done when disarmed */ + if (!kprobes_all_disarmed && kprobes_allow_optimization && + !list_empty(&gen->optimizing_list)) + arch_optimize_kprobes(&gen->optimizing_list); + } + + /* Free cleaned kprobes after quiescence period */ + do_free_cleaned_kprobes(&gen->freeing_list); + + /* Finalize unoptimized kprobes whose quiescence period completed */ + list_for_each_entry_safe(op, tmp, &gen->cooling_list, list) { + if (kprobe_unused(&op->kp)) { + /* + * Unregistered while quiescence period was in flight. + * Unhash it now and move to cur_gen's freeing list so + * it will be reclaimed after the next quiescence period. + */ + hlist_del_init_rcu(&op->kp.hlist); + list_move(&op->list, &optprobe_gens[optprobe_cur_gen].freeing_list); + kick_kprobe_optimizer(); + } else { + /* Still in use; now safely disarmed */ + list_del_init(&op->list); + if (!kprobe_disabled(&op->kp)) + optimize_kprobe(&op->kp); + } + } + + gen->in_flight = false; + WRITE_ONCE(gen->ready, false); +} + +static bool optprobe_dispatch_generation(void) +{ + struct optprobe_generation *gen; + + lockdep_assert_held(&kprobe_mutex); + + if (!optprobe_can_fire() || !optprobe_has_queued_probes()) + return false; + + gen = &optprobe_gens[optprobe_cur_gen]; + + scoped_guard(cpus_read_lock) { + guard(mutex)(&text_mutex); /* - * Step 2: Wait for quiesence period to ensure all potentially - * preempted tasks to have normally scheduled. Because optprobe - * may modify multiple instructions, there is a chance that Nth - * instruction is preempted. In that case, such tasks can return - * to 2nd-Nth byte of jump instruction. This wait is for avoiding it. - * Note that on non-preemptive kernel, this is transparently converted - * to synchronoze_sched() to wait for all interrupts to have completed. + * Unoptimize kprobes and collect cleaned (unused and disarmed) + * kprobes before waiting for quiescence period. */ - synchronize_rcu_tasks(); + do_unoptimize_kprobes(&gen->unoptimizing_list, &gen->freeing_list, + &gen->cooling_list); + } - /* Step 3: Optimize kprobes after quiesence period */ - do_optimize_kprobes(); + /* Advance cur_gen to the next generation slot */ + optprobe_cur_gen = (optprobe_cur_gen + 1) % OPTPROBE_GEN_MAX; - /* Step 4: Free cleaned kprobes after quiesence period */ - do_free_cleaned_kprobes(); + gen->in_flight = true; + WRITE_ONCE(gen->ready, false); + + call_rcu_tasks(&gen->rcu, optprobe_generation_rcu_cb); + return true; +} + +/* Kprobe jump optimizer */ +static void kprobe_optimizer(void) +{ + bool progress = false; + int i; + + guard(mutex)(&kprobe_mutex); + + /* Step 1: Finalize any generation whose Tasks RCU grace period completed */ + for (i = 0; i < OPTPROBE_GEN_MAX; i++) { + if (READ_ONCE(optprobe_gens[i].ready)) { + optprobe_finalize_generation(&optprobe_gens[i]); + progress = true; + } } - /* Step 5: Kick optimizer again if needed. But if there is a flush requested, */ - if (completion_done(&optimizer_completion)) - complete(&optimizer_completion); + /* Step 2: Dispatch waiting room generation if allowed */ + if (optprobe_dispatch_generation()) + progress = true; + + /* Step 3: Wake up flushers if progress was made */ + if (progress) { + optimizer_passes++; + wake_up_var_locked(&optimizer_passes, &kprobe_mutex); + } - if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) - kick_kprobe_optimizer(); /*normal kick*/ + if (optprobe_has_queued_probes() && optprobe_can_fire()) { + /* Probes remain and can be fired immediately (e.g. retried siblings) */ + kick_kprobe_optimizer(); + } } static int kprobe_optimizer_thread(void *data) { while (!kthread_should_stop()) { - /* To avoid hung_task, wait in interruptible state. */ + /* Wait until there is work to do or a generation is ready */ wait_event_interruptible(kprobe_optimizer_wait, - atomic_read(&optimizer_state) != OPTIMIZER_ST_IDLE || - kthread_should_stop()); + atomic_read(&optimizer_state) != OPTIMIZER_ST_IDLE || + optprobe_has_ready_gens() || + kthread_should_stop()); if (kthread_should_stop()) break; /* - * If it was a normal kick, wait for OPTIMIZE_DELAY. - * This wait can be interrupted by a flush request. + * If it was a normal kick and no generation is ready to finalize, + * wait for OPTIMIZE_DELAY to batch incoming requests. + * This wait can be interrupted by a flush request or a ready generation. */ - if (atomic_read(&optimizer_state) == 1) + if (atomic_read(&optimizer_state) == OPTIMIZER_ST_KICKED && + !optprobe_has_ready_gens()) { wait_event_interruptible_timeout( kprobe_optimizer_wait, atomic_read(&optimizer_state) == OPTIMIZER_ST_FLUSHING || + optprobe_has_ready_gens() || kthread_should_stop(), OPTIMIZE_DELAY); + } if (kthread_should_stop()) break; @@ -707,19 +850,23 @@ static void wait_for_kprobe_optimizer_locked(void) { lockdep_assert_held(&kprobe_mutex); - while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) { - init_completion(&optimizer_completion); + while (optprobe_optimizer_busy()) { + unsigned long passes = optimizer_passes; + + /* Wake up optimizer thread if it can make progress */ + if ((optprobe_can_fire() && optprobe_has_queued_probes()) || + optprobe_has_ready_gens()) { + if (atomic_xchg_acquire(&optimizer_state, + OPTIMIZER_ST_FLUSHING) != OPTIMIZER_ST_FLUSHING) + wake_up(&kprobe_optimizer_wait); + } + /* - * Set state to OPTIMIZER_ST_FLUSHING and wake up the thread if it's - * idle. If it's already kicked, it will see the state change. + * kprobe_optimizer() holds 'kprobe_mutex' for a whole pass, which + * this drops while sleeping, so a new count means progress was made. */ - if (atomic_xchg_acquire(&optimizer_state, - OPTIMIZER_ST_FLUSHING) != OPTIMIZER_ST_FLUSHING) - wake_up(&kprobe_optimizer_wait); - - mutex_unlock(&kprobe_mutex); - wait_for_completion(&optimizer_completion); - mutex_lock(&kprobe_mutex); + wait_var_event_mutex(&optimizer_passes, + optimizer_passes != passes, &kprobe_mutex); } } @@ -734,10 +881,43 @@ void wait_for_kprobe_optimizer(void) bool optprobe_queued_unopt(struct optimized_kprobe *op) { struct optimized_kprobe *_op; + int i; - list_for_each_entry(_op, &unoptimizing_list, list) { - if (op == _op) - return true; + for (i = 0; i < OPTPROBE_GEN_MAX; i++) { + list_for_each_entry(_op, &optprobe_gens[i].unoptimizing_list, list) { + if (op == _op) + return true; + } + } + + return false; +} + +static bool optprobe_queued_freeing(struct optimized_kprobe *op) +{ + struct optimized_kprobe *_op; + int i; + + for (i = 0; i < OPTPROBE_GEN_MAX; i++) { + list_for_each_entry(_op, &optprobe_gens[i].freeing_list, list) { + if (op == _op) + return true; + } + } + + return false; +} + +static bool optprobe_queued_cooling(struct optimized_kprobe *op) +{ + struct optimized_kprobe *_op; + int i; + + for (i = 0; i < OPTPROBE_GEN_MAX; i++) { + list_for_each_entry(_op, &optprobe_gens[i].cooling_list, list) { + if (op == _op) + return true; + } } return false; @@ -771,6 +951,15 @@ static void optimize_kprobe(struct kprobe *p) } return; } + + if (optprobe_queued_cooling(op)) { + /* Under in-flight unoptimization. It will be re-optimized upon finalize */ + return; + } + + if (optprobe_queued_freeing(op)) + list_del_init(&op->list); + op->kp.flags |= KPROBE_FLAG_OPTIMIZED; /* @@ -780,7 +969,7 @@ static void optimize_kprobe(struct kprobe *p) if (WARN_ON_ONCE(!list_empty(&op->list))) return; - list_add(&op->list, &optimizing_list); + list_add(&op->list, &optprobe_gens[optprobe_cur_gen].optimizing_list); kick_kprobe_optimizer(); } @@ -795,14 +984,14 @@ static void force_unoptimize_kprobe(struct optimized_kprobe *op) /* Unoptimize a kprobe if p is optimized */ static void unoptimize_kprobe(struct kprobe *p, bool force) { + struct optprobe_generation *cur; struct optimized_kprobe *op; if (!kprobe_aggrprobe(p) || kprobe_disarmed(p)) return; /* This is not an optprobe nor optimized */ op = container_of(p, struct optimized_kprobe, kp); - if (!kprobe_optimized(p)) - return; + cur = &optprobe_gens[optprobe_cur_gen]; if (!list_empty(&op->list)) { if (optprobe_queued_unopt(op)) { @@ -810,10 +999,30 @@ static void unoptimize_kprobe(struct kprobe *p, bool force) if (force) { /* * Forcibly unoptimize the kprobe here, and queue it - * in the freeing list for release afterwards. + * for cooling down (or freeing if unused). */ force_unoptimize_kprobe(op); - list_move(&op->list, &freeing_list); + if (kprobe_unused(&op->kp)) { + if (!kprobe_gone(&op->kp)) + arch_disarm_kprobe(&op->kp); + hlist_del_init_rcu(&op->kp.hlist); + list_move(&op->list, &cur->freeing_list); + } else { + list_move(&op->list, &cur->cooling_list); + } + kick_kprobe_optimizer(); + } + } else if (optprobe_queued_cooling(op)) { + if (force && kprobe_unused(&op->kp)) { + /* + * Already unoptimized, move to freeing list for + * release afterwards. + */ + if (!kprobe_gone(&op->kp)) + arch_disarm_kprobe(&op->kp); + hlist_del_init_rcu(&op->kp.hlist); + list_move(&op->list, &cur->freeing_list); + kick_kprobe_optimizer(); } } else { /* Dequeue from the optimizing queue */ @@ -823,12 +1032,15 @@ static void unoptimize_kprobe(struct kprobe *p, bool force) return; } + if (!kprobe_optimized(p)) + return; + /* Optimized kprobe case */ if (force) { /* Forcibly update the code: this is a special case */ force_unoptimize_kprobe(op); } else { - list_add(&op->list, &unoptimizing_list); + list_add(&op->list, &cur->unoptimizing_list); kick_kprobe_optimizer(); } } @@ -860,23 +1072,29 @@ static void kill_optimized_kprobe(struct kprobe *p) struct optimized_kprobe *op; op = container_of(p, struct optimized_kprobe, kp); - if (!list_empty(&op->list)) - /* Dequeue from the (un)optimization queue */ - list_del_init(&op->list); - op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; - - if (kprobe_unused(p)) { - /* - * Unused kprobe is on unoptimizing or freeing list. We move it - * to freeing_list and let the kprobe_optimizer() remove it from - * the kprobe hash list and free it. - */ - if (optprobe_queued_unopt(op)) - list_move(&op->list, &freeing_list); + if (!list_empty(&op->list)) { + if (kprobe_unused(p)) { + if (optprobe_queued_unopt(op) || optprobe_queued_cooling(op)) { + hlist_del_init_rcu(&op->kp.hlist); + list_move(&op->list, + &optprobe_gens[optprobe_cur_gen].freeing_list); + kick_kprobe_optimizer(); + } else if (!optprobe_queued_freeing(op)) { + list_del_init(&op->list); + } + } else { + list_del_init(&op->list); + } } + op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; - /* Don't touch the code, because it is already freed. */ - arch_remove_optimized_kprobe(op); + /* + * Don't remove the slot if it is queued for freeing or unoptimization; + * the optimizer will reclaim it after the quiescence period. + */ + if (!optprobe_queued_freeing(op) && !optprobe_queued_unopt(op) && + !optprobe_queued_cooling(op)) + arch_remove_optimized_kprobe(op); } static inline @@ -1052,6 +1270,19 @@ static void __arm_kprobe(struct kprobe *p) optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */ } +/* Return true if the probe is queued on unoptimizing lists */ +static bool kprobe_queued_unopt(struct kprobe *p) +{ + struct optimized_kprobe *op; + + if (kprobe_aggrprobe(p)) { + op = container_of(p, struct optimized_kprobe, kp); + if (optprobe_queued_unopt(op)) + return true; + } + return false; +} + /* Remove the breakpoint of a probe. */ static void __disarm_kprobe(struct kprobe *p, bool reopt) { @@ -1062,7 +1293,7 @@ static void __disarm_kprobe(struct kprobe *p, bool reopt) /* Try to unoptimize */ unoptimize_kprobe(p, kprobes_all_disarmed); - if (!kprobe_queued(p)) { + if (!kprobe_queued_unopt(p)) { arch_disarm_kprobe(p); /* If another kprobe was blocked, re-optimize it. */ _p = get_optimized_kprobe(p->addr); @@ -1073,11 +1304,23 @@ static void __disarm_kprobe(struct kprobe *p, bool reopt) static void __init init_optprobe(void) { + int i; + #ifdef __ARCH_WANT_KPROBES_INSN_SLOT /* Init 'kprobe_optinsn_slots' for allocation */ kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE; #endif + for (i = 0; i < OPTPROBE_GEN_MAX; i++) { + INIT_LIST_HEAD(&optprobe_gens[i].optimizing_list); + INIT_LIST_HEAD(&optprobe_gens[i].unoptimizing_list); + INIT_LIST_HEAD(&optprobe_gens[i].cooling_list); + INIT_LIST_HEAD(&optprobe_gens[i].freeing_list); + optprobe_gens[i].in_flight = false; + optprobe_gens[i].ready = false; + } + optprobe_cur_gen = 0; + init_waitqueue_head(&kprobe_optimizer_wait); atomic_set(&optimizer_state, OPTIMIZER_ST_IDLE); kprobe_optimizer_task = kthread_run(kprobe_optimizer_thread, NULL, @@ -3019,47 +3262,22 @@ static int disarm_all_kprobes(void) return ret; } -/* - * XXX: The debugfs bool file interface doesn't allow for callbacks - * when the bool state is switched. We can reuse that facility when - * available - */ -static ssize_t read_enabled_file_bool(struct file *file, - char __user *user_buf, size_t count, loff_t *ppos) +static int kprobes_enabled_set(void *data, u64 val) { - char buf[3]; + if (val) + return arm_all_kprobes(); - if (!kprobes_all_disarmed) - buf[0] = '1'; - else - buf[0] = '0'; - buf[1] = '\n'; - buf[2] = 0x00; - return simple_read_from_buffer(user_buf, count, ppos, buf, 2); + return disarm_all_kprobes(); } -static ssize_t write_enabled_file_bool(struct file *file, - const char __user *user_buf, size_t count, loff_t *ppos) +static int kprobes_enabled_get(void *data, u64 *val) { - bool enable; - int ret; - - ret = kstrtobool_from_user(user_buf, count, &enable); - if (ret) - return ret; - - ret = enable ? arm_all_kprobes() : disarm_all_kprobes(); - if (ret) - return ret; - - return count; + *val = !kprobes_all_disarmed; + return 0; } -static const struct file_operations fops_kp = { - .read = read_enabled_file_bool, - .write = write_enabled_file_bool, - .llseek = default_llseek, -}; +DEFINE_DEBUGFS_ATTRIBUTE(fops_kp, kprobes_enabled_get, + kprobes_enabled_set, "%llu\n"); static int __init debugfs_kprobe_init(void) { diff --git a/kernel/kthread.c b/kernel/kthread.c index a3f95c90456b..cc8cb5d3eab7 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -81,7 +81,7 @@ enum KTHREAD_BITS { static inline struct kthread *to_kthread(struct task_struct *k) { - WARN_ON(!(k->flags & PF_KTHREAD)); + WARN_ON(!(READ_ONCE(k->flags) & PF_KTHREAD)); return k->worker_private; } diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index a240d1144e89..517fe427ff92 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -646,12 +646,15 @@ static const struct kobj_type klp_ktype_patch = { static void klp_kobj_release_object(struct kobject *kobj) { + struct klp_patch *patch; struct klp_object *obj; obj = container_of(kobj, struct klp_object, kobj); + patch = obj->patch; if (obj->dynamic) klp_free_object_dynamic(obj); + kobject_put(&patch->kobj); } static const struct kobj_type klp_ktype_object = { @@ -662,12 +665,15 @@ static const struct kobj_type klp_ktype_object = { static void klp_kobj_release_func(struct kobject *kobj) { + struct klp_object *obj; struct klp_func *func; func = container_of(kobj, struct klp_func, kobj); + obj = func->obj; if (func->nop) klp_free_func_nop(func); + kobject_put(&obj->kobj); } static const struct kobj_type klp_ktype_func = { @@ -943,7 +949,9 @@ static void klp_init_func_early(struct klp_object *obj, struct klp_func *func) { kobject_init(&func->kobj, &klp_ktype_func); + kobject_get(&obj->kobj); list_add_tail(&func->node, &obj->func_list); + func->obj = obj; } static void klp_init_object_early(struct klp_patch *patch, @@ -951,7 +959,9 @@ static void klp_init_object_early(struct klp_patch *patch, { INIT_LIST_HEAD(&obj->func_list); kobject_init(&obj->kobj, &klp_ktype_object); + kobject_get(&patch->kobj); list_add_tail(&obj->node, &patch->obj_list); + obj->patch = patch; } static void klp_init_patch_early(struct klp_patch *patch) diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c index 4728631ae719..5a9534c715b8 100644 --- a/kernel/locking/rtmutex.c +++ b/kernel/locking/rtmutex.c @@ -1644,7 +1644,7 @@ static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock, break; } - if (timeout && !timeout->task) { + if (timeout && !hrtimer_sleeper_task_get(timeout)) { ret = -ETIMEDOUT; break; } diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c index f23126d804b2..1ff8ce857d92 100644 --- a/kernel/module/kallsyms.c +++ b/kernel/module/kallsyms.c @@ -75,23 +75,29 @@ static char elf_type(const Elf_Sym *sym, const struct load_info *info) return '?'; } -static bool is_core_symbol(const Elf_Sym *src, const Elf_Shdr *sechdrs, - unsigned int shnum, unsigned int pcpundx) +static bool is_core_symbol(struct module *mod, const struct load_info *info, + unsigned int symnum, const Elf_Sym *src) { const Elf_Shdr *sec; enum mod_mem_type type; + if (symnum == 0 || is_livepatch_module(mod)) + return true; + if (src->st_shndx == SHN_UNDEF || - src->st_shndx >= shnum || + src->st_shndx >= info->hdr->e_shnum || !src->st_name) return false; + if (is_ignored_kernel_symbol(&info->strtab[src->st_name])) + return false; + #ifdef CONFIG_KALLSYMS_ALL - if (src->st_shndx == pcpundx) + if (src->st_shndx == info->index.pcpu) return true; #endif - sec = sechdrs + src->st_shndx; + sec = info->sechdrs + src->st_shndx; type = sec->sh_entsize >> SH_ENTSIZE_TYPE_SHIFT; if (!(sec->sh_flags & SHF_ALLOC) #ifndef CONFIG_KALLSYMS_ALL @@ -130,12 +136,11 @@ void layout_symtab(struct module *mod, struct load_info *info) /* Compute total space required for the core symbols' strtab. */ for (ndst = i = 0; i < nsrc; i++) { - if (i == 0 || is_livepatch_module(mod) || - is_core_symbol(src + i, info->sechdrs, info->hdr->e_shnum, - info->index.pcpu)) { - strtab_size += strlen(&info->strtab[src[i].st_name]) + 1; - ndst++; - } + if (!is_core_symbol(mod, info, i, src + i)) + continue; + + strtab_size += strlen(&info->strtab[src[i].st_name]) + 1; + ndst++; } /* Append room for core symbols at end of core part. */ @@ -197,23 +202,21 @@ void add_kallsyms(struct module *mod, const struct load_info *info) strtab_size = info->core_typeoffs - info->stroffs; src = kallsyms->symtab; for (ndst = i = 0; i < kallsyms->num_symtab; i++) { + ssize_t ret; + kallsyms->typetab[i] = elf_type(src + i, info); - if (i == 0 || is_livepatch_module(mod) || - is_core_symbol(src + i, info->sechdrs, info->hdr->e_shnum, - info->index.pcpu)) { - ssize_t ret; - - mod->core_kallsyms.typetab[ndst] = - kallsyms->typetab[i]; - dst[ndst] = src[i]; - dst[ndst++].st_name = s - mod->core_kallsyms.strtab; - ret = strscpy(s, &kallsyms->strtab[src[i].st_name], - strtab_size); - if (ret < 0) - break; - s += ret + 1; - strtab_size -= ret + 1; - } + + if (!is_core_symbol(mod, info, i, src + i)) + continue; + + mod->core_kallsyms.typetab[ndst] = kallsyms->typetab[i]; + dst[ndst] = src[i]; + dst[ndst++].st_name = s - mod->core_kallsyms.strtab; + ret = strscpy(s, &kallsyms->strtab[src[i].st_name], strtab_size); + if (ret < 0) + break; + s += ret + 1; + strtab_size -= ret + 1; } /* Set up to point into init section. */ @@ -294,7 +297,7 @@ static const char *find_kallsyms_symbol(struct module *mod, * and inserted at a whim. */ if (*kallsyms_symbol_name(kallsyms, i) == '\0' || - is_mapping_symbol(kallsyms_symbol_name(kallsyms, i))) + is_ignored_kernel_symbol(kallsyms_symbol_name(kallsyms, i))) continue; if (thisval <= addr && thisval > bestval) { diff --git a/kernel/module/kmod.c b/kernel/module/kmod.c index a25dccdf7aa7..24091210741e 100644 --- a/kernel/module/kmod.c +++ b/kernel/module/kmod.c @@ -28,6 +28,7 @@ #include <linux/ptrace.h> #include <linux/async.h> #include <linux/uaccess.h> +#include <linux/umh.h> #include <trace/events/module.h> #include "internal.h" @@ -81,7 +82,7 @@ static int call_modprobe(char *orig_module_name, int wait) char *module_name; int ret; - char **argv = kmalloc(sizeof(char *[5]), GFP_KERNEL); + char **argv = kmalloc_array(5, sizeof(*argv), GFP_KERNEL); if (!argv) goto out; diff --git a/kernel/module/main.c b/kernel/module/main.c index d0e1e0bd2ad0..34e0204ea37c 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -60,6 +60,7 @@ #include <linux/codetag.h> #include <linux/debugfs.h> #include <linux/execmem.h> +#include <linux/error-injection.h> #include <uapi/linux/module.h> #include "internal.h" @@ -191,11 +192,22 @@ static inline int strong_try_module_get(struct module *mod) return -ENOENT; } -static inline void add_taint_module(struct module *mod, unsigned flag, - enum lockdep_ok lockdep_ok) +/** + * add_taint_module: add a taint flag if not already set for a specific module + * @mod: pointer to the module that caused the problem + * @flag: one of the TAINT_* constants. + * @lockdep_ok: whether lock debugging is still OK. + * + * If something bad has gone wrong, you'll want + * @lockdep_ok = LOCKDEP_NOW_UNRELIABLE, but for some + * noteworthy-but-not-corrupting cases, it can be set to LOCKDEP_STILL_OK. + */ +void add_taint_module(struct module *mod, unsigned flag, + enum lockdep_ok lockdep_ok) { add_taint(flag, lockdep_ok); - set_bit(flag, &mod->taints); + if (mod) + set_bit(flag, &mod->taints); } /* @@ -2723,6 +2735,10 @@ static int find_module_sections(struct module *mod, struct load_info *info) mod->btf_base_data = any_section_objs(info, ".BTF.base", 1, &mod->btf_base_data_size); #endif +#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE) + mod->btf_inline_data = any_section_objs(info, ".BTF.inline", 1, + &mod->btf_inline_data_size); +#endif #ifdef CONFIG_JUMP_LABEL mod->jump_entries = section_objs(info, "__jump_table", sizeof(*mod->jump_entries), @@ -3177,6 +3193,9 @@ static noinline int do_init_module(struct module *mod) mod->btf_data = NULL; mod->btf_base_data = NULL; #endif +#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE) + mod->btf_inline_data = NULL; +#endif /* * We want to free module_init, but be aware that kallsyms may be * walking this within an RCU read section. In all the failure paths, we @@ -3581,8 +3600,9 @@ static int load_module(struct load_info *info, const char __user *uargs, goto sysfs_cleanup; } - if (codetag_load_module(mod)) - goto sysfs_cleanup; + err = codetag_load_module(mod); + if (err) + goto livepatch_cleanup; /* Get rid of temporary copy. */ free_copy(info, flags); @@ -3592,6 +3612,9 @@ static int load_module(struct load_info *info, const char __user *uargs, return do_init_module(mod); + livepatch_cleanup: + if (is_livepatch_module(mod)) + free_module_elf(mod); sysfs_cleanup: mod_sysfs_teardown(mod); coming_cleanup: diff --git a/kernel/module/tracking.c b/kernel/module/tracking.c index 9033ff54c4e2..9e8902600c32 100644 --- a/kernel/module/tracking.c +++ b/kernel/module/tracking.c @@ -38,8 +38,8 @@ int try_add_tainted_module(struct module *mod) return -ENOMEM; strscpy(mod_taint->name, mod->name, MODULE_NAME_LEN); mod_taint->taints = mod->taints; - list_add_rcu(&mod_taint->list, &unloaded_tainted_modules); mod_taint->count = 1; + list_add_rcu(&mod_taint->list, &unloaded_tainted_modules); out: return 0; } diff --git a/kernel/notifier.c b/kernel/notifier.c index 2f9fe7c30287..55fbd2be7407 100644 --- a/kernel/notifier.c +++ b/kernel/notifier.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only +#include <linux/device/devres.h> #include <linux/kdebug.h> #include <linux/kprobes.h> #include <linux/export.h> @@ -197,6 +198,58 @@ int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh, } EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister); +struct atomic_notifier_chain_devres { + struct atomic_notifier_head *nh; + struct notifier_block *nb; +}; + +static void devm_atomic_notifier_chain_unregister(struct device *dev, void *res) +{ + struct atomic_notifier_chain_devres *dr = res; + + atomic_notifier_chain_unregister(dr->nh, dr->nb); +} + +/** + * devm_atomic_notifier_chain_register - Device-managed atomic notifier registration + * @dev: Device to tie the notifier lifetime to + * @nh: Pointer to head of the atomic notifier chain + * @nb: New entry in notifier chain + * + * Adds a notifier to an atomic notifier chain and registers a cleanup + * action to automatically unregister it when @dev is unbound. + * + * Must be called in process context. + * + * Return: + * 0 on success, negative errno on error. + */ +int devm_atomic_notifier_chain_register(struct device *dev, + struct atomic_notifier_head *nh, + struct notifier_block *nb) +{ + struct atomic_notifier_chain_devres *dr; + int ret; + + dr = devres_alloc(devm_atomic_notifier_chain_unregister, + sizeof(*dr), GFP_KERNEL); + if (!dr) + return -ENOMEM; + + ret = atomic_notifier_chain_register(nh, nb); + if (ret) { + devres_free(dr); + return ret; + } + + dr->nh = nh; + dr->nb = nb; + devres_add(dev, dr); + + return 0; +} +EXPORT_SYMBOL_GPL(devm_atomic_notifier_chain_register); + /** * atomic_notifier_call_chain - Call functions in an atomic notifier chain * @nh: Pointer to head of the atomic notifier chain @@ -349,6 +402,58 @@ int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh, } EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust); +struct blocking_notifier_chain_devres { + struct blocking_notifier_head *nh; + struct notifier_block *nb; +}; + +static void devm_blocking_notifier_chain_unregister(struct device *dev, + void *res) +{ + struct blocking_notifier_chain_devres *dr = res; + + blocking_notifier_chain_unregister(dr->nh, dr->nb); +} + +/** + * devm_blocking_notifier_chain_register - Device-managed blocking notifier registration + * @dev: Device to tie the notifier lifetime to + * @nh: Pointer to head of the blocking notifier chain + * @nb: New entry in notifier chain + * + * Adds a notifier to a blocking notifier chain and registers a cleanup + * action to automatically unregister it when @dev is unbound. + * Must be called in process context. + * + * Return: + * 0 on success, negative errno on error. + */ +int devm_blocking_notifier_chain_register(struct device *dev, + struct blocking_notifier_head *nh, + struct notifier_block *nb) +{ + struct blocking_notifier_chain_devres *dr; + int ret; + + dr = devres_alloc(devm_blocking_notifier_chain_unregister, + sizeof(*dr), GFP_KERNEL); + if (!dr) + return -ENOMEM; + + ret = blocking_notifier_chain_register(nh, nb); + if (ret) { + devres_free(dr); + return ret; + } + + dr->nh = nh; + dr->nb = nb; + devres_add(dev, dr); + + return 0; +} +EXPORT_SYMBOL_GPL(devm_blocking_notifier_chain_register); + /** * blocking_notifier_call_chain - Call functions in a blocking notifier chain * @nh: Pointer to head of the blocking notifier chain diff --git a/kernel/nscommon.c b/kernel/nscommon.c index 3166c1fd844a..e72426bba29a 100644 --- a/kernel/nscommon.c +++ b/kernel/nscommon.c @@ -4,6 +4,7 @@ #include <linux/ns_common.h> #include <linux/nstree.h> #include <linux/proc_ns.h> +#include <linux/security.h> #include <linux/user_namespace.h> #include <linux/vfsdebug.h> @@ -59,6 +60,9 @@ int __ns_common_init(struct ns_common *ns, u32 ns_type, const struct proc_ns_ope refcount_set(&ns->__ns_ref, 1); ns->stashed = NULL; +#ifdef CONFIG_SECURITY + ns->ns_security = NULL; +#endif ns->ops = ops; ns->ns_id = 0; ns->ns_type = ns_type; @@ -77,6 +81,14 @@ int __ns_common_init(struct ns_common *ns, u32 ns_type, const struct proc_ns_ope ret = proc_alloc_inum(&ns->inum); if (ret) return ret; + + ret = security_namespace_init(ns); + if (ret) { + if (!inum) + proc_free_inum(ns->inum); + return ret; + } + /* * Tree ref starts at 0. It's incremented when namespace enters * active use (installed in nsproxy) and decremented when all @@ -91,7 +103,10 @@ int __ns_common_init(struct ns_common *ns, u32 ns_type, const struct proc_ns_ope void __ns_common_free(struct ns_common *ns) { - proc_free_inum(ns->inum); + security_namespace_free(ns); + + if (ns->inum > MNT_NS_INO_SPECIAL_MAX) + proc_free_inum(ns->inum); } struct ns_common *__must_check ns_owner(struct ns_common *ns) diff --git a/kernel/nsproxy.c b/kernel/nsproxy.c index d9d3d5973bf5..454427b09ed9 100644 --- a/kernel/nsproxy.c +++ b/kernel/nsproxy.c @@ -61,7 +61,7 @@ static inline struct nsproxy *create_nsproxy(void) return nsproxy; } -static inline void nsproxy_free(struct nsproxy *ns) +void nsproxy_free(struct nsproxy *ns) { put_mnt_ns(ns->mnt_ns); put_uts_ns(ns->uts_ns); @@ -385,6 +385,12 @@ out: static inline int validate_ns(struct nsset *nsset, struct ns_common *ns) { + int ret; + + ret = security_namespace_install(nsset, ns); + if (ret) + return ret; + return ns->ops->install(nsset, ns); } diff --git a/kernel/padata.c b/kernel/padata.c index 0d3ea1b68b1f..5a113c667d4c 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -534,7 +534,7 @@ static void padata_init_reorder_list(struct parallel_data *pd) } } -/* Allocate and initialize the internal cpumask dependend resources. */ +/* Allocate and initialize the internal cpumask dependent resources. */ static struct parallel_data *padata_alloc_pd(struct padata_shell *ps, int offlining_cpu) { diff --git a/kernel/panic.c b/kernel/panic.c index 213725b612aa..a4dbb6b8cd2b 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -826,6 +826,7 @@ const struct taint_flag taint_flags[TAINT_FLAGS_COUNT] = { TAINT_FLAG(RANDSTRUCT, 'T', ' '), TAINT_FLAG(TEST, 'N', ' '), TAINT_FLAG(FWCTL, 'J', ' '), + TAINT_FLAG(FORCED_BIND, 'Y', ' '), }; #undef TAINT_FLAG @@ -862,9 +863,9 @@ static void print_tainted_seq(struct seq_buf *s, bool verbose) * exact size is allocated dynamically; the initial buffer remains * as a fallback if allocation fails. * - * The verbose taint string currently requires up to 327 characters. + * The verbose taint string currently requires up to 344 characters. */ -#define INIT_TAINT_BUF_MAX 350 +#define INIT_TAINT_BUF_MAX 370 static char init_taint_buf[INIT_TAINT_BUF_MAX] __initdata; static char *taint_buf __refdata = init_taint_buf; @@ -948,8 +949,9 @@ unsigned long get_taint(void) * @flag: one of the TAINT_* constants. * @lockdep_ok: whether lock debugging is still OK. * - * If something bad has gone wrong, you'll want @lockdebug_ok = false, but for - * some notewortht-but-not-corrupting cases, it can be set to true. + * If something bad has gone wrong, you'll want + * @lockdep_ok = LOCKDEP_NOW_UNRELIABLE, but for some + * noteworthy-but-not-corrupting cases, it can be set to LOCKDEP_STILL_OK. */ void add_taint(unsigned flag, enum lockdep_ok lockdep_ok) { diff --git a/kernel/pid.c b/kernel/pid.c index 95b8ccfa8269..adf721606768 100644 --- a/kernel/pid.c +++ b/kernel/pid.c @@ -411,7 +411,7 @@ static void __change_pid(struct pid **pids, struct task_struct *task, pid = *pid_ptr; hlist_del_rcu(&task->pid_links[type]); - *pid_ptr = new; + WRITE_ONCE(*pid_ptr, new); for (tmp = PIDTYPE_MAX; --tmp >= 0; ) if (pid_has_task(pid, tmp)) diff --git a/kernel/pid_namespace.c b/kernel/pid_namespace.c index d36afc58ee1d..127a5fd23e70 100644 --- a/kernel/pid_namespace.c +++ b/kernel/pid_namespace.c @@ -238,9 +238,10 @@ void zap_pid_ns_processes(struct pid_namespace *pid_ns) * kernel_wait4() will also block until our children traced from the * parent namespace are detached and become EXIT_DEAD. */ + /* Task work must not busy-loop the reaper, see signal_pending(). */ + guard(no_notify_signal)(); do { clear_thread_flag(TIF_SIGPENDING); - clear_thread_flag(TIF_NOTIFY_SIGNAL); rc = kernel_wait4(-1, NULL, __WALL, NULL); } while (rc != -ECHILD); @@ -268,7 +269,7 @@ void zap_pid_ns_processes(struct pid_namespace *pid_ns) * free_pid() will awaken this task. */ for (;;) { - set_current_state(TASK_INTERRUPTIBLE); + set_current_state(TASK_IDLE | TASK_FREEZABLE); if (pid_ns->pid_allocated == init_pids) break; schedule(); diff --git a/kernel/power/em_netlink.c b/kernel/power/em_netlink.c index 4d4fd29bd2be..1f68d456a713 100644 --- a/kernel/power/em_netlink.c +++ b/kernel/power/em_netlink.c @@ -95,41 +95,62 @@ static int __em_nl_get_pd_for_dump(struct em_perf_domain *pd, void *data) return ret; } +struct em_nl_doit_ctx { + struct genl_info *info; + int cmd; + struct sk_buff *msg; +}; + +static int __em_nl_get_pd_doit_fill(struct em_perf_domain *pd, void *data) +{ + struct em_nl_doit_ctx *ctx = data; + void *hdr; + + hdr = genlmsg_put_reply(ctx->msg, ctx->info, &dev_energymodel_nl_family, 0, + ctx->cmd); + if (!hdr) + return -EMSGSIZE; + + if (__em_nl_get_pd(pd, ctx->msg)) { + genlmsg_cancel(ctx->msg, hdr); + return -EMSGSIZE; + } + + genlmsg_end(ctx->msg, hdr); + return 0; +} + int dev_energymodel_nl_get_perf_domains_doit(struct sk_buff *skb, - struct genl_info *info) + struct genl_info *info) { - int id, ret = -EMSGSIZE, msg_sz = 0; - int cmd = info->genlhdr->cmd; - struct em_perf_domain *pd; + struct em_nl_doit_ctx ctx = { + .info = info, + .cmd = info->genlhdr->cmd, + }; struct sk_buff *msg; - void *hdr; + int id, ret, msg_sz = 0; if (!info->attrs[DEV_ENERGYMODEL_A_PERF_DOMAIN_PERF_DOMAIN_ID]) return -EINVAL; id = nla_get_u32(info->attrs[DEV_ENERGYMODEL_A_PERF_DOMAIN_PERF_DOMAIN_ID]); - pd = em_perf_domain_get_by_id(id); - if (!pd) - return -EINVAL; - __em_nl_get_pd_size(pd, &msg_sz); + /* Encode under em_pd_list_mutex, like the dumpit path. */ + ret = em_perf_domain_for_id(id, __em_nl_get_pd_size, &msg_sz); + if (ret) + return ret; + msg = genlmsg_new(msg_sz, GFP_KERNEL); if (!msg) return -ENOMEM; - hdr = genlmsg_put_reply(msg, info, &dev_energymodel_nl_family, 0, cmd); - if (!hdr) - goto out_free_msg; - - ret = __em_nl_get_pd(pd, msg); + ctx.msg = msg; + ret = em_perf_domain_for_id(id, __em_nl_get_pd_doit_fill, &ctx); if (ret) - goto out_cancel_msg; - genlmsg_end(msg, hdr); + goto out_free_msg; return genlmsg_reply(msg, info); -out_cancel_msg: - genlmsg_cancel(msg, hdr); out_free_msg: nlmsg_free(msg); return ret; @@ -148,19 +169,6 @@ int dev_energymodel_nl_get_perf_domains_dumpit(struct sk_buff *skb, return for_each_em_perf_domain(__em_nl_get_pd_for_dump, &ctx); } -static struct em_perf_domain *__em_nl_get_pd_table_id(struct nlattr **attrs) -{ - struct em_perf_domain *pd; - int id; - - if (!attrs[DEV_ENERGYMODEL_A_PERF_TABLE_PERF_DOMAIN_ID]) - return NULL; - - id = nla_get_u32(attrs[DEV_ENERGYMODEL_A_PERF_TABLE_PERF_DOMAIN_ID]); - pd = em_perf_domain_get_by_id(id); - return pd; -} - static int __em_nl_get_pd_table_size(const struct em_perf_domain *pd) { int id_sz, ps_sz; @@ -245,34 +253,58 @@ out_err: return -EMSGSIZE; } +static int __em_nl_get_pd_table_size_cb(struct em_perf_domain *pd, void *data) +{ + *(int *)data = __em_nl_get_pd_table_size(pd); + return 0; +} + +static int __em_nl_get_pd_table_doit_fill(struct em_perf_domain *pd, + void *data) +{ + struct em_nl_doit_ctx *ctx = data; + void *hdr; + + hdr = genlmsg_put_reply(ctx->msg, ctx->info, &dev_energymodel_nl_family, 0, + ctx->cmd); + if (!hdr) + return -EMSGSIZE; + + if (__em_nl_get_pd_table(ctx->msg, pd)) + return -EMSGSIZE; + + genlmsg_end(ctx->msg, hdr); + return 0; +} + int dev_energymodel_nl_get_perf_table_doit(struct sk_buff *skb, - struct genl_info *info) + struct genl_info *info) { - int cmd = info->genlhdr->cmd; - int msg_sz, ret = -EMSGSIZE; - struct em_perf_domain *pd; + struct em_nl_doit_ctx ctx = { + .info = info, + .cmd = info->genlhdr->cmd, + }; struct sk_buff *msg; - void *hdr; + int id, ret, msg_sz; - pd = __em_nl_get_pd_table_id(info->attrs); - if (!pd) + if (!info->attrs[DEV_ENERGYMODEL_A_PERF_TABLE_PERF_DOMAIN_ID]) return -EINVAL; - msg_sz = __em_nl_get_pd_table_size(pd); + id = nla_get_u32(info->attrs[DEV_ENERGYMODEL_A_PERF_TABLE_PERF_DOMAIN_ID]); + + ret = em_perf_domain_for_id(id, __em_nl_get_pd_table_size_cb, &msg_sz); + if (ret) + return ret; msg = genlmsg_new(msg_sz, GFP_KERNEL); if (!msg) return -ENOMEM; - hdr = genlmsg_put_reply(msg, info, &dev_energymodel_nl_family, 0, cmd); - if (!hdr) - goto out_free_msg; - - ret = __em_nl_get_pd_table(msg, pd); + ctx.msg = msg; + ret = em_perf_domain_for_id(id, __em_nl_get_pd_table_doit_fill, &ctx); if (ret) goto out_free_msg; - genlmsg_end(msg, hdr); return genlmsg_reply(msg, info); out_free_msg: diff --git a/kernel/power/em_netlink.h b/kernel/power/em_netlink.h index 583d7f1c3939..bc98a3c278b9 100644 --- a/kernel/power/em_netlink.h +++ b/kernel/power/em_netlink.h @@ -12,7 +12,8 @@ #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_NET) int for_each_em_perf_domain(int (*cb)(struct em_perf_domain*, void *), void *data); -struct em_perf_domain *em_perf_domain_get_by_id(int id); +int em_perf_domain_for_id(int id, int (*cb)(struct em_perf_domain *, void *), + void *data); void em_notify_pd_created(const struct em_perf_domain *pd); void em_notify_pd_deleted(const struct em_perf_domain *pd); void em_notify_pd_updated(const struct em_perf_domain *pd); @@ -24,9 +25,10 @@ int for_each_em_perf_domain(int (*cb)(struct em_perf_domain*, void *), return -EINVAL; } static inline -struct em_perf_domain *em_perf_domain_get_by_id(int id) +int em_perf_domain_for_id(int id, int (*cb)(struct em_perf_domain *, void *), + void *data) { - return NULL; + return -EINVAL; } static inline void em_notify_pd_created(const struct em_perf_domain *pd) {} diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index e610cf8e9a06..a76089e2ce8c 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -1031,7 +1031,9 @@ int for_each_em_perf_domain(int (*cb)(struct em_perf_domain*, void *), return 0; } -struct em_perf_domain *em_perf_domain_get_by_id(int id) +/* Run @cb on the matching domain with em_pd_list_mutex held. */ +int em_perf_domain_for_id(int id, int (*cb)(struct em_perf_domain *, void *), + void *data) { struct em_perf_domain *pd; @@ -1040,9 +1042,9 @@ struct em_perf_domain *em_perf_domain_get_by_id(int id) list_for_each_entry(pd, &em_pd_list, node) { if (pd->id == id) - return pd; + return cb(pd, data); } - return NULL; + return -EINVAL; } #endif diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index d2479c69d71a..e365e440cd52 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -106,11 +106,17 @@ bool hibernation_in_progress(void) return !atomic_read(&hibernate_atomic); } +__weak bool arch_hibernation_available(void) +{ + return true; +} + bool hibernation_available(void) { return nohibernate == 0 && !security_locked_down(LOCKDOWN_HIBERNATION) && - !secretmem_active() && !cxl_mem_active(); + !secretmem_active() && !cxl_mem_active() && + arch_hibernation_available(); } /** @@ -408,9 +414,18 @@ int hibernation_snapshot(int platform_mode) if (error) goto Close; + error = dpm_prepare(PMSG_FREEZE); + if (error) + goto Complete; + + /* Preallocate image memory before freezing kernel threads and shutting down devices. */ + error = hibernate_preallocate_memory(); + if (error) + goto Complete; + error = freeze_kernel_threads(); if (error) - goto Close; + goto Cleanup; if (hibernation_test(TEST_FREEZER)) { @@ -422,15 +437,6 @@ int hibernation_snapshot(int platform_mode) goto Thaw; } - error = dpm_prepare(PMSG_FREEZE); - if (error) - goto Complete; - - /* Preallocate image memory before shutting down devices. */ - error = hibernate_preallocate_memory(); - if (error) - goto Complete; - console_suspend_all(); pm_restrict_gfp_mask(); @@ -464,10 +470,12 @@ int hibernation_snapshot(int platform_mode) platform_end(platform_mode); return error; - Complete: - dpm_complete(PMSG_RECOVER); Thaw: thaw_kernel_threads(); + Cleanup: + swsusp_free(); + Complete: + dpm_complete(PMSG_RECOVER); goto Close; } diff --git a/kernel/power/process.c b/kernel/power/process.c index dc0dfc349f22..295904ec9a82 100644 --- a/kernel/power/process.c +++ b/kernel/power/process.c @@ -16,7 +16,7 @@ #include <linux/freezer.h> #include <linux/delay.h> #include <linux/workqueue.h> -#include <linux/kmod.h> +#include <linux/umh.h> #include <trace/events/power.h> #include <linux/cpuset.h> diff --git a/kernel/power/qos.c b/kernel/power/qos.c index 1944dbeb0d4c..c241da5bccf6 100644 --- a/kernel/power/qos.c +++ b/kernel/power/qos.c @@ -159,7 +159,7 @@ static void pm_qos_flags_remove_req(struct pm_qos_flags *pqf, list_for_each_entry(req, &pqf->list, node) val |= req->flags; - pqf->effective_flags = val; + WRITE_ONCE(pqf->effective_flags, val); } /** @@ -193,7 +193,7 @@ bool pm_qos_update_flags(struct pm_qos_flags *pqf, req->flags = val; INIT_LIST_HEAD(&req->node); list_add_tail(&req->node, &pqf->list); - pqf->effective_flags |= val; + WRITE_ONCE(pqf->effective_flags, pqf->effective_flags | val); break; default: /* no action */ diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index b209712cb2c3..d5dba0e50b2e 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -88,7 +88,7 @@ static inline int hibernate_restore_unprotect_page(void *page_address) {return 0 static inline void hibernate_map_page(struct page *page) { if (IS_ENABLED(CONFIG_ARCH_HAS_SET_DIRECT_MAP)) { - int ret = set_direct_map_default_noflush(page); + int ret = set_direct_map_default_noflush(page, 1); if (ret) pr_warn_once("Failed to remap page\n"); @@ -101,7 +101,7 @@ static inline void hibernate_unmap_page(struct page *page) { if (IS_ENABLED(CONFIG_ARCH_HAS_SET_DIRECT_MAP)) { unsigned long addr = (unsigned long)page_address(page); - int ret = set_direct_map_invalid_noflush(page); + int ret = set_direct_map_invalid_noflush(page, 1); if (ret) pr_warn_once("Failed to remap page\n"); diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index d17704fe93ae..5f8afaca8866 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -1887,6 +1887,7 @@ bool nbcon_device_try_acquire(struct console *con) memset(ctxt, 0, sizeof(*ctxt)); ctxt->console = con; + /* Keep in sync with nbcon_braille_try_acquire(). */ ctxt->prio = NBCON_PRIO_NORMAL; if (!nbcon_context_try_acquire(ctxt, false)) @@ -2002,3 +2003,82 @@ void nbcon_kdb_release(struct nbcon_write_context *wctxt) */ __nbcon_atomic_flush_pending_con(ctxt->console, prb_next_reserve_seq(prb)); } + +/** + * nbcon_write_context_is_braille - Checks whether the nbcon write context + * is using Braille console + * + * @wctxt: checked nbcon write context + * + * Return: True when the write context is associated with a Braille console. + * Otherwise, return false. + * + * Context: Can be called in any context but only when Braille console is + * registered and the struct console could not disappear. + */ +bool nbcon_write_context_is_braille(struct nbcon_write_context *wctxt) +{ + struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); + struct console *con = ctxt->console; + + return con && con->flags & CON_BRL; +} +EXPORT_SYMBOL_GPL(nbcon_write_context_is_braille); + +/** + * nbcon_braille_try_acquire - Try to acquire nbcon console for braille_write() + * + * @con: The nbcon console to acquire + * @wctxt: The nbcon write context to be used on success + * + * Context: braille_write() for emitting a single buffer on Braille console. + * + * Return: True if the console was acquired. False otherwise. + * + * Braille console is not registered as a proper printk consoles. Instead, + * it is integrated with the graphical virtual terminal. + * + * This function is going to synchronize the Braille write against other + * operations on the used serial port. The port can be used also for a user + * input but printk() won't emit the messages there directly. It means + * the other operations will get synchronized using nbcon_device_try_acquire(). + */ +bool nbcon_braille_try_acquire(struct console *con, + struct nbcon_write_context *wctxt) +{ + struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); + bool success; + + memset(ctxt, 0, sizeof(*ctxt)); + ctxt->console = con; + /* Keep in sync with nbcon_device_try_acquire(). */ + if (panic_on_this_cpu()) + ctxt->prio = NBCON_PRIO_PANIC; + else + ctxt->prio = NBCON_PRIO_NORMAL; + + success = nbcon_context_try_acquire(ctxt, false); + /* + * Try hard in panic, otherwise the Braille console would + * miss a text on the related VT. + */ + if (panic_on_this_cpu() && !success) { + ctxt->allow_unsafe_takeover = true; + success = nbcon_context_try_acquire(ctxt, false); + } + + return success; +} + +/** + * nbcon_braille_release - Release the nbcon console + * + * @wctxt: The nbcon write context initialized by a successful + * nbcon_braille_try_acquire() + */ +void nbcon_braille_release(struct nbcon_write_context *wctxt) +{ + struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); + + nbcon_context_release(ctxt); +} diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 6d3d18a50da7..9a205f94bfac 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -1085,9 +1085,9 @@ static void __init log_buf_add_cpu(void) if (num_possible_cpus() == 1) return; - cpu_extra = (num_possible_cpus() - 1) * __LOG_CPU_MAX_BUF_LEN; + cpu_extra = num_possible_cpus() * __LOG_CPU_MAX_BUF_LEN; - /* by default this will only continue through for large > 64 CPUs */ + /* by default this will only continue through for more than 16 CPUs */ if (cpu_extra <= __LOG_BUF_LEN / 2) return; @@ -2646,24 +2646,25 @@ static int __init console_setup(char *str) if (_braille_console_setup(&str, &brl_options)) return 1; + /* + * Decode str into name, index and options. Start with options, since + * it might also contain a ':' used for DEVNAME. + */ + options = strchr(str, ','); + if (options) + *(options++) = 0; + /* For a DEVNAME:0.0 style console the character device is unknown early */ if (strchr(str, ':')) devname = buf; else ttyname = buf; - /* - * Decode str into name, index, options. - */ if (ttyname && isdigit(str[0])) scnprintf(buf, sizeof(buf), "ttyS%s", str); else strscpy(buf, str); - options = strchr(str, ','); - if (options) - *(options++) = 0; - #ifdef __sparc__ if (!strcmp(str, "ttya")) strscpy(buf, "ttyS0"); @@ -2671,17 +2672,18 @@ static int __init console_setup(char *str) strscpy(buf, "ttyS1"); #endif - for (s = buf; *s; s++) - if ((ttyname && isdigit(*s)) || *s == ',') - break; - - /* @idx will get defined when devname matches. */ - if (devname) - idx = -1; - else + if (ttyname) { + /* Detect @idx in ttyname and remove it. */ + for (s = ttyname; *s; s++) { + if (isdigit(*s)) + break; + } idx = simple_strtoul(s, NULL, 10); - - *s = 0; + *s = 0; + } else { + /* @idx will get defined when devname matches. */ + idx = -1; + } __add_preferred_console(ttyname, idx, devname, options, brl_options, true); return 1; @@ -3862,7 +3864,7 @@ static int __init printk_set_kthreads_ready(void) return 0; } -early_initcall(printk_set_kthreads_ready); +subsys_initcall(printk_set_kthreads_ready); #endif /* CONFIG_PRINTK */ static int __read_mostly keep_bootcon; @@ -4424,36 +4426,51 @@ void __init console_init(void) * be a brief interval in which no messages are logged to the console, which * makes it difficult to diagnose problems that occur during this time. * - * To mitigate this problem somewhat, only unregister consoles whose memory - * intersects with the init section. Note that all other boot consoles will - * get unregistered when the real preferred console is registered. + * If a real console has already registered, remove all remaining boot consoles. + * Otherwise, mitigate the no-console interval by removing only boot consoles + * whose memory intersects with the init section. */ static int __init printk_late_init(void) { + bool have_real_console = false; struct hlist_node *tmp; struct console *con; int ret; console_list_lock(); + for_each_console(con) { + if (!(con->flags & CON_BOOT)) { + have_real_console = true; + break; + } + } + hlist_for_each_entry_safe(con, tmp, &console_list, node) { if (!(con->flags & CON_BOOT)) continue; - /* Check addresses that might be used for enabled consoles. */ - if (init_section_intersects(con, sizeof(*con)) || - init_section_contains(con->write, 0) || - init_section_contains(con->read, 0) || - init_section_contains(con->device, 0) || - init_section_contains(con->unblank, 0) || - init_section_contains(con->data, 0)) { + /* + * Keep the boot console when requested or as a fallback + * unless it is using an init section. + */ + if (keep_bootcon || !have_real_console) { + if (!init_section_intersects(con, sizeof(*con)) && + !init_section_contains(con->write, 0) && + !init_section_contains(con->read, 0) && + !init_section_contains(con->device, 0) && + !init_section_contains(con->unblank, 0) && + !init_section_contains(con->data, 0)) + continue; + /* * Please, consider moving the reported consoles out * of the init section. */ pr_warn("bootconsole [%s%d] uses init memory and must be disabled even before the real one is ready\n", con->name, con->index); - unregister_console_locked(con); } + + unregister_console_locked(con); } console_list_unlock(); @@ -4593,31 +4610,26 @@ bool pr_flush(int timeout_ms, bool reset_on_progress) /* * Delayed printk version, for scheduler-internal messages: */ -#define PRINTK_PENDING_WAKEUP 0x01 -#define PRINTK_PENDING_OUTPUT 0x02 - -static DEFINE_PER_CPU(int, printk_pending); +#define PRINTK_PENDING_OUTPUT 0x01 -static void wake_up_klogd_work_func(struct irq_work *irq_work) +static void pending_wake_fn(struct irq_work *irq_work) { - int pending = this_cpu_xchg(printk_pending, 0); + wake_up_interruptible(&log_wait); +} - if (pending & PRINTK_PENDING_OUTPUT) { - if (force_legacy_kthread()) { - if (printk_legacy_kthread) - wake_up_interruptible(&legacy_wait); - } else { - if (console_trylock()) - console_unlock(); - } +static void pending_output_fn(struct irq_work *irq_work) +{ + if (force_legacy_kthread()) { + if (printk_legacy_kthread) + wake_up_interruptible(&legacy_wait); + } else { + if (console_trylock()) + console_unlock(); } - - if (pending & PRINTK_PENDING_WAKEUP) - wake_up_interruptible(&log_wait); } -static DEFINE_PER_CPU(struct irq_work, wake_up_klogd_work) = - IRQ_WORK_INIT_LAZY(wake_up_klogd_work_func); +static struct irq_work pending_wakeup_work = IRQ_WORK_INIT_LAZY(pending_wake_fn); +static struct irq_work pending_output_work = IRQ_WORK_INIT_LAZY(pending_output_fn); static void __wake_up_klogd(int val) { @@ -4631,7 +4643,6 @@ static void __wake_up_klogd(int val) if (WARN_ON_ONCE(console_irqwork_blocked)) return; - preempt_disable(); /* * Guarantee any new records can be seen by tasks preparing to wait * before this context checks if the wait queue is empty. @@ -4643,12 +4654,11 @@ static void __wake_up_klogd(int val) * * This pairs with devkmsg_read:A and syslog_print:A. */ - if (wq_has_sleeper(&log_wait) || /* LMM(__wake_up_klogd:A) */ - (val & PRINTK_PENDING_OUTPUT)) { - this_cpu_or(printk_pending, val); - irq_work_queue(this_cpu_ptr(&wake_up_klogd_work)); - } - preempt_enable(); + if (wq_has_sleeper(&log_wait)) /* LMM(__wake_up_klogd:A) */ + irq_work_queue(&pending_wakeup_work); + + if (val & PRINTK_PENDING_OUTPUT) + irq_work_queue(&pending_output_work); } /** @@ -4663,7 +4673,7 @@ static void __wake_up_klogd(int val) */ void wake_up_klogd(void) { - __wake_up_klogd(PRINTK_PENDING_WAKEUP); + __wake_up_klogd(0); } /** @@ -4684,7 +4694,7 @@ void defer_console_output(void) * New messages may have been added directly to the ringbuffer * using vprintk_store(), so wake any waiters as well. */ - __wake_up_klogd(PRINTK_PENDING_WAKEUP | PRINTK_PENDING_OUTPUT); + __wake_up_klogd(PRINTK_PENDING_OUTPUT); } /** diff --git a/kernel/printk/printk_ringbuffer.c b/kernel/printk/printk_ringbuffer.c index 85c0c854b3ce..d38e7c2f6887 100644 --- a/kernel/printk/printk_ringbuffer.c +++ b/kernel/printk/printk_ringbuffer.c @@ -439,13 +439,12 @@ static enum desc_state get_desc_state(unsigned long id, } /* - * Get a copy of a specified descriptor and return its queried state. If the - * descriptor is in an inconsistent state (miss or reserved), the caller can - * only expect the descriptor's @state_var field to be valid. + * Get the state of a specified descriptor. * - * The sequence number and caller_id can be optionally retrieved. Like all - * non-state_var data, they are only valid if the descriptor is in a - * consistent state. + * Optionally, a copy of the descriptor, sequence number and/or caller ID can + * be retrieved. However, (with the exception of @desc_out->state_var) all + * retrieved values are invalid if the descriptor is in an inconsistent state + * (miss or reserved). */ static enum desc_state desc_read(struct prb_desc_ring *desc_ring, unsigned long id, struct prb_desc *desc_out, @@ -1335,7 +1334,6 @@ static struct prb_desc *desc_reopen_last(struct prb_desc_ring *desc_ring, { unsigned long prev_state_val; enum desc_state d_state; - struct prb_desc desc; struct prb_desc *d; unsigned long id; u32 cid; @@ -1346,7 +1344,7 @@ static struct prb_desc *desc_reopen_last(struct prb_desc_ring *desc_ring, * To reduce unnecessarily reopening, first check if the descriptor * state and caller ID are correct. */ - d_state = desc_read(desc_ring, id, &desc, NULL, &cid); + d_state = desc_read(desc_ring, id, NULL, NULL, &cid); if (d_state != desc_committed || cid != caller_id) return NULL; @@ -2007,14 +2005,13 @@ u64 prb_first_seq(struct printk_ringbuffer *rb) { struct prb_desc_ring *desc_ring = &rb->desc_ring; enum desc_state d_state; - struct prb_desc desc; unsigned long id; u64 seq; for (;;) { id = atomic_long_read(&rb->desc_ring.tail_id); /* LMM(prb_first_seq:A) */ - d_state = desc_read(desc_ring, id, &desc, &seq, NULL); /* LMM(prb_first_seq:B) */ + d_state = desc_read(desc_ring, id, NULL, &seq, NULL); /* LMM(prb_first_seq:B) */ /* * This loop will not be infinite because the tail is diff --git a/kernel/ptrace.c b/kernel/ptrace.c index d041645d9d17..3f2fe68ccb14 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -1227,6 +1227,12 @@ int ptrace_request(struct task_struct *child, long request, case PTRACE_SETSIGMASK: { sigset_t new_set; + /* A user worker only ever takes SIGKILL and SIGSTOP. */ + if (child->flags & PF_USER_WORKER) { + ret = -EPERM; + break; + } + if (addr != sizeof(sigset_t)) { ret = -EINVAL; break; @@ -1359,16 +1365,13 @@ int ptrace_request(struct task_struct *child, long request, struct iovec kiov; struct iovec __user *uiov = datavp; - if (!access_ok(uiov, sizeof(*uiov))) - return -EFAULT; - - if (__get_user(kiov.iov_base, &uiov->iov_base) || - __get_user(kiov.iov_len, &uiov->iov_len)) + if (get_user(kiov.iov_base, &uiov->iov_base) || + get_user(kiov.iov_len, &uiov->iov_len)) return -EFAULT; ret = ptrace_regset(child, request, addr, &kiov); if (!ret) - ret = __put_user(kiov.iov_len, &uiov->iov_len); + ret = put_user(kiov.iov_len, &uiov->iov_len); break; } @@ -1525,11 +1528,8 @@ int compat_ptrace_request(struct task_struct *child, compat_long_t request, compat_uptr_t ptr; compat_size_t len; - if (!access_ok(uiov, sizeof(*uiov))) - return -EFAULT; - - if (__get_user(ptr, &uiov->iov_base) || - __get_user(len, &uiov->iov_len)) + if (get_user(ptr, &uiov->iov_base) || + get_user(len, &uiov->iov_len)) return -EFAULT; kiov.iov_base = compat_ptr(ptr); @@ -1537,7 +1537,7 @@ int compat_ptrace_request(struct task_struct *child, compat_long_t request, ret = ptrace_regset(child, request, addr, &kiov); if (!ret) - ret = __put_user(kiov.iov_len, &uiov->iov_len); + ret = put_user(kiov.iov_len, &uiov->iov_len); break; } #endif diff --git a/kernel/rcu/Kconfig b/kernel/rcu/Kconfig index 332df7a7a634..bc9483c0b5d8 100644 --- a/kernel/rcu/Kconfig +++ b/kernel/rcu/Kconfig @@ -175,6 +175,12 @@ config RCU_STALL_COMMON config RCU_NEED_SEGCBLIST def_bool ( TREE_RCU || TREE_SRCU || TASKS_RCU_GENERIC ) +# The deferral (and the IRQ_WORK it uses) is only needed where call_rcu() / +# call_srcu() can be invoked while a callback-list operation is in flight. +config RCU_DEFER + def_bool HAVE_NMI || KPROBES || FUNCTION_TRACER || TRACEPOINTS + select IRQ_WORK + config RCU_FANOUT int "Tree-based hierarchical RCU fanout value" range 2 64 if 64BIT diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index 39a9f6fa9a7b..d60444bf3a02 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -572,6 +572,20 @@ static inline void tasks_cblist_init_generic(void) { } #define RCU_SCHEDULER_INIT 1 #define RCU_SCHEDULER_RUNNING 2 +/* + * Defer whenever interrupts are disabled, since a callback-list operation may + * be in flight on this CPU. Not before the scheduler is up: irq_work is not + * usable that early, and rcu_init() itself calls call_rcu(). + */ +static inline bool should_rcu_defer(void) +{ + return IS_ENABLED(CONFIG_RCU_DEFER) && irqs_disabled() && + rcu_scheduler_active != RCU_SCHEDULER_INACTIVE; +} + +/* Drain an outgoing CPU's deferred SRCU callbacks; see rcutree_migrate_callbacks(). */ +void srcu_offline_drain(int cpu); + enum rcutorture_type { RCU_FLAVOR, RCU_TASKS_FLAVOR, diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 794937e13e7c..968343be4e0d 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -48,6 +48,7 @@ #include <linux/tick.h> #include <linux/rcupdate_trace.h> #include <linux/nmi.h> +#include <linux/perf_event.h> #include "rcu.h" @@ -115,6 +116,7 @@ torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers"); torture_param(int, n_barrier_cbs, 0, "# of callbacks/kthreads for barrier testing"); torture_param(int, n_up_down, 32, "# of concurrent up/down hrtimer-based RCU readers"); torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); +torture_param(bool, nmi_calls, true, "Exercise ->call() from NMI on nmi_capable flavors"); torture_param(int, nreaders, -1, "Number of RCU reader threads"); torture_param(bool, nwriters, 1, "Number of RCU writer threads (0 or 1)"); torture_param(int, object_debug, 0, "Enable debug-object double call_rcu() testing"); @@ -216,6 +218,8 @@ static long n_rcu_torture_boost_failure; static long n_rcu_torture_boosts; static atomic_long_t n_rcu_torture_timers; static atomic_long_t n_rcu_torture_irqs; +static atomic_long_t n_rcu_torture_nmi_call; +static atomic_long_t n_rcu_torture_nmi_cb; static long n_barrier_attempts; static long n_barrier_successes; /* did rcu_barrier test succeed? */ static unsigned long n_read_exits; @@ -433,6 +437,7 @@ struct rcu_torture_ops { bool (*is_task_rcu_boosted)(void); long cbflood_max; int irq_capable; + int nmi_capable; int can_boost; int extendables; int slow_gps; @@ -440,6 +445,7 @@ struct rcu_torture_ops { int debug_objects; int start_poll_irqsoff; int have_up_down; + int rdrs_handle_load; const char *name; }; @@ -648,6 +654,8 @@ static struct rcu_torture_ops rcu_ops = { .extendables = RCUTORTURE_MAX_EXTEND, .debug_objects = 1, .start_poll_irqsoff = 1, + .rdrs_handle_load = !IS_ENABLED(CONFIG_PREEMPT_RCU) || IS_ENABLED(CONFIG_RCU_BOOST), + .nmi_capable = 1, .name = "rcu" }; @@ -702,10 +710,21 @@ static struct rcu_torture_ops rcu_busted_ops = { DEFINE_STATIC_SRCU(srcu_ctl); DEFINE_STATIC_SRCU_FAST(srcu_ctlf); DEFINE_STATIC_SRCU_FAST_UPDOWN(srcu_ctlfud); +DEFINE_STATIC_SRCU_ATOMIC(srcu_ctla); static struct srcu_struct srcu_ctld; static struct srcu_struct *srcu_ctlp = &srcu_ctl; static struct rcu_torture_ops srcud_ops; +// Restrict APIs permitted for atomic SRCU. +static void srcu_torture_init_forbidden_apis(void) +{ + cur_ops->call = NULL; + cur_ops->cb_barrier = NULL; + cur_ops->deferred_free = NULL; + cur_ops->exp_current = NULL; + cur_ops->start_gp_poll = NULL; +} + static void srcu_torture_init(void) { rcu_sync_torture_init(); @@ -721,6 +740,11 @@ static void srcu_torture_init(void) srcu_ctlp = &srcu_ctlfud; VERBOSE_TOROUT_STRING("srcu_torture_init fast-up/down SRCU"); } + if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) { + srcu_ctlp = &srcu_ctla; + VERBOSE_TOROUT_STRING("srcu_torture_init atomic SRCU"); + srcu_torture_init_forbidden_apis(); + } } static void srcu_get_gp_data(int *flags, unsigned long *gp_seq) @@ -758,6 +782,11 @@ static int srcu_torture_read_lock(void) WARN_ON_ONCE(idx & ~0x1); ret += idx << 3; } + if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) { + idx = srcu_read_lock_atomic(srcu_ctlp); + WARN_ON_ONCE(idx & ~0x1); + ret += idx << 4; + } return ret; } @@ -777,7 +806,8 @@ srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) delay = torture_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick); - if (!delay && !in_atomic() && !rcu_preempt_depth() && !irqs_disabled()) { + if (!delay && !in_atomic() && !rcu_preempt_depth() && !irqs_disabled() && + !(reader_flavor & SRCU_READ_FLAVOR_ATOMIC)) { schedule_timeout_interruptible(longdelay); rtrsp->rt_delay_jiffies = longdelay; } else { @@ -788,6 +818,8 @@ srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) static void srcu_torture_read_unlock(int idx) { WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1))); + if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) + srcu_read_unlock_atomic(srcu_ctlp, (idx & 0x10) >> 4); if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) srcu_read_unlock_fast_updown(srcu_ctlp, __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x8) >> 3)); @@ -867,7 +899,10 @@ static void srcu_torture_deferred_free(struct rcu_torture *rp) static void srcu_torture_synchronize(void) { - synchronize_srcu(srcu_ctlp); + if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) + synchronize_srcu_atomic(srcu_ctlp); + else + synchronize_srcu(srcu_ctlp); } static unsigned long srcu_torture_get_gp_state(void) @@ -903,7 +938,10 @@ static void srcu_torture_stats(void) static void srcu_torture_synchronize_expedited(void) { - synchronize_srcu_expedited(srcu_ctlp); + if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) + synchronize_srcu_atomic(srcu_ctlp); + else + synchronize_srcu_expedited(srcu_ctlp); } static void srcu_torture_expedite_current(void) @@ -942,6 +980,7 @@ static struct rcu_torture_ops srcu_ops = { .debug_objects = 1, .have_up_down = IS_ENABLED(CONFIG_TINY_SRCU) ? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST_UPDOWN, + .nmi_capable = 1, .name = "srcu" }; @@ -960,6 +999,10 @@ static void srcud_torture_init(void) } else if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) { WARN_ON(init_srcu_struct_fast_updown(&srcu_ctld)); VERBOSE_TOROUT_STRING("srcud_torture_init fast-up/down SRCU"); + } else if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) { + WARN_ON(init_srcu_struct_atomic(&srcu_ctld)); + VERBOSE_TOROUT_STRING("srcud_torture_init atomic SRCU"); + srcu_torture_init_forbidden_apis(); } else { WARN_ON(init_srcu_struct(&srcu_ctld)); } @@ -1005,6 +1048,7 @@ static struct rcu_torture_ops srcud_ops = { .debug_objects = 1, .have_up_down = IS_ENABLED(CONFIG_TINY_SRCU) ? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST_UPDOWN, + .nmi_capable = 1, .name = "srcud" }; @@ -1269,6 +1313,7 @@ static struct rcu_torture_ops tasks_tracing_ops = { .cbflood_max = 50000, .irq_capable = 1, .slow_gps = 1, + .nmi_capable = 1, .name = "tasks-tracing" }; @@ -1738,7 +1783,7 @@ rcu_torture_writer(void *arg) pr_alert("%s" TORTURE_FLAG " Waited %lu jiffies for boot to complete.\n", torture_type, jiffies - j); - if (IS_ENABLED(CONFIG_RCU_LAZY)) + if (IS_ENABLED(CONFIG_RCU_LAZY) && cur_ops->call) INIT_WORK_ONSTACK(&lazy_work, rcu_torture_writer_work); do { @@ -1933,7 +1978,7 @@ rcu_torture_writer(void *arg) !rcu_gp_is_normal(); } rcu_torture_writer_state = RTWS_STUTTER; - if (IS_ENABLED(CONFIG_RCU_LAZY)) + if (IS_ENABLED(CONFIG_RCU_LAZY) && cur_ops->call) queue_work(system_percpu_wq, &lazy_work); stutter_waited = stutter_wait("rcu_torture_writer"); if (stutter_waited && @@ -1966,7 +2011,7 @@ rcu_torture_writer(void *arg) " Dynamic grace-period expediting was disabled.\n", torture_type); - if (IS_ENABLED(CONFIG_RCU_LAZY)) { + if (IS_ENABLED(CONFIG_RCU_LAZY) && cur_ops->call) { cancel_work_sync(&lazy_work); destroy_work_on_stack(&lazy_work); } @@ -2534,8 +2579,10 @@ static bool rcu_torture_one_read_start(struct rcu_torture_one_read_state *rtorsp rtorsp->p = rcu_dereference_check(rcu_torture_current, !cur_ops->readlock_held || cur_ops->readlock_held() || (rtorsp->readstate & RCUTORTURE_RDR_UPDOWN)); - if (rtorsp->p == NULL) { - /* Wait for rcu_torture_writer to get underway */ + if ((!cur_ops->rdrs_handle_load && atomic_read(&rcu_fwd_cb_nodelay)) || rtorsp->p == NULL) { + // Wait for rcu_torture_writer to get underway and + // (if readers cannot handle heavy loads) for any + // forward-progress testing to complete. rcutorture_one_extend(&rtorsp->readstate, 0, trsp, rtorsp->rtrsp); return false; } @@ -2660,6 +2707,124 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_timer_rand); /* + * Exercise ->call() from NMI context for flavors that set ->nmi_capable. A + * per-CPU hardware perf counter overflows into an NMI, and its handler submits + * a preallocated callback via ->call(). One callback per CPU is in flight at a + * time (guarded by an atomic) to avoid allocating in NMI. + */ +#ifdef CONFIG_PERF_EVENTS +static struct perf_event_attr rcu_torture_nmi_attr = { + .type = PERF_TYPE_HARDWARE, + .config = PERF_COUNT_HW_CPU_CYCLES, + .size = sizeof(struct perf_event_attr), + .pinned = 1, + .disabled = 1, + /* + * A fixed period rather than .freq: a frequency-based event bumps + * nr_freq_events, which sets TICK_DEP_BIT_PERF_EVENTS and would pin the + * tick for the whole run on NO_HZ_FULL kernels. + */ + .sample_period = 20 * 1000 * 1000, +}; + +/* One in-flight callback per CPU; ->inuse is released by the callback. */ +struct rcu_torture_nmi_cb { + struct rcu_head rh; + atomic_t inuse; +}; + +static struct perf_event **rcu_torture_nmi_events; +static int rcu_torture_nmi_hp_state; +static DEFINE_PER_CPU(struct rcu_torture_nmi_cb, rcu_torture_nmi_cb); + +static void rcu_torture_nmi_invoked(struct rcu_head *rhp) +{ + struct rcu_torture_nmi_cb *rtncp = container_of(rhp, struct rcu_torture_nmi_cb, rh); + + atomic_long_inc(&n_rcu_torture_nmi_cb); + atomic_set(&rtncp->inuse, 0); +} + +static void rcu_torture_nmi_overflow(struct perf_event *event, + struct perf_sample_data *data, + struct pt_regs *regs) +{ + struct rcu_torture_nmi_cb *rtncp = this_cpu_ptr(&rcu_torture_nmi_cb); + + if (!in_nmi()) + return; + if (cur_ops->call && !atomic_xchg(&rtncp->inuse, 1)) { + atomic_long_inc(&n_rcu_torture_nmi_call); + cur_ops->call(&rtncp->rh, rcu_torture_nmi_invoked); + } +} + +static int rcu_torture_nmi_online(unsigned int cpu) +{ + struct perf_event *event; + + event = perf_event_create_kernel_counter(&rcu_torture_nmi_attr, cpu, NULL, + rcu_torture_nmi_overflow, NULL); + if (IS_ERR(event)) + return 0; + rcu_torture_nmi_events[cpu] = event; + perf_event_enable(event); + return 0; +} + +static int rcu_torture_nmi_offline(unsigned int cpu) +{ + struct perf_event *event = rcu_torture_nmi_events[cpu]; + + if (event) { + rcu_torture_nmi_events[cpu] = NULL; + perf_event_disable(event); + perf_event_release_kernel(event); + } + return 0; +} + +/* Drive the counters from hotplug callbacks so coverage survives onoff. */ +static void rcu_torture_nmi_init(void) +{ + int ret; + + if (!nmi_calls || !cur_ops->nmi_capable || !cur_ops->call) + return; + rcu_torture_nmi_events = kcalloc(nr_cpu_ids, sizeof(*rcu_torture_nmi_events), + GFP_KERNEL); + if (!rcu_torture_nmi_events) + return; + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "rcutorture/nmi:online", + rcu_torture_nmi_online, rcu_torture_nmi_offline); + if (ret < 0) { + kfree(rcu_torture_nmi_events); + rcu_torture_nmi_events = NULL; + return; + } + rcu_torture_nmi_hp_state = ret; +} + +static void rcu_torture_nmi_cleanup(void) +{ + if (!rcu_torture_nmi_events) + return; + if (rcu_torture_nmi_hp_state > 0) { + cpuhp_remove_state(rcu_torture_nmi_hp_state); + rcu_torture_nmi_hp_state = 0; + } + kfree(rcu_torture_nmi_events); + rcu_torture_nmi_events = NULL; + if (!atomic_long_read(&n_rcu_torture_nmi_call)) + pr_alert("%s: nmi_calls set but no ->call() ever issued from NMI, so NMI ->call() went untested (no PMU, or NMIs unavailable here).\n", + __func__); +} +#else /* #ifdef CONFIG_PERF_EVENTS */ +static void rcu_torture_nmi_init(void) { } +static void rcu_torture_nmi_cleanup(void) { } +#endif /* #else #ifdef CONFIG_PERF_EVENTS */ + +/* * RCU torture reader from timer handler. Dereferences rcu_torture_current, * incrementing the corresponding element of the pipeline array. The * counter in the element should never be greater than 1, otherwise, the @@ -3047,6 +3212,9 @@ rcu_torture_stats_print(void) data_race(n_barrier_attempts), data_race(n_rcu_torture_barrier_error)); pr_cont("read-exits: %ld ", data_race(n_read_exits)); // Statistic. + pr_cont("nmi-calls: %ld nmi-cbs: %ld ", + atomic_long_read(&n_rcu_torture_nmi_call), + atomic_long_read(&n_rcu_torture_nmi_cb)); pr_cont("nocb-toggles: %ld:%ld ", atomic_long_read(&n_nocb_offload), atomic_long_read(&n_nocb_deoffload)); pr_cont("gpwraps: %ld\n", n_gpwraps); @@ -3195,7 +3363,7 @@ rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) "read_exit_delay=%d read_exit_burst=%d " "reader_flavor=%x " "nocbs_nthreads=%d nocbs_toggle=%d " - "test_nmis=%d " + "test_nmis=%d nmi_calls=%d " "preempt_duration=%d preempt_interval=%d n_up_down=%d\n", torture_type, tag, nrealreaders, nwriters, nrealfakewriters, stat_interval, verbose, test_no_idle_hz, shuffle_interval, @@ -3209,7 +3377,7 @@ rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) read_exit_delay, read_exit_burst, reader_flavor, nocbs_nthreads, nocbs_toggle, - test_nmis, + test_nmis, nmi_calls, preempt_duration, preempt_interval, n_up_down); } @@ -3856,7 +4024,7 @@ static int __init rcu_torture_fwd_prog_init(void) } if (fwd_progress_holdoff <= 0) fwd_progress_holdoff = 1; - if (fwd_progress_div <= 0) + if (fwd_progress_div < 2) fwd_progress_div = 4; rfp = kzalloc_objs(*rfp, fwd_progress); fwd_prog_tasks = kzalloc_objs(*fwd_prog_tasks, fwd_progress); @@ -4283,6 +4451,7 @@ rcu_torture_cleanup(void) int i; if (torture_cleanup_begin()) { + rcu_torture_nmi_cleanup(); if (cur_ops->cb_barrier != NULL) { pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier); cur_ops->cb_barrier(); @@ -4322,9 +4491,12 @@ rcu_torture_cleanup(void) for (i = 0; i < nrealreaders; i++) torture_stop_kthread(rcu_torture_reader, reader_tasks[i]); + if (irqreader && cur_ops->irq_capable) + kick_all_cpus_sync(); kfree(reader_tasks); reader_tasks = NULL; } + rcu_torture_nmi_cleanup(); kfree(rcu_torture_reader_mbchk); rcu_torture_reader_mbchk = NULL; @@ -4354,6 +4526,19 @@ rcu_torture_cleanup(void) pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier); cur_ops->cb_barrier(); } + + /* + * cb_barrier() above drained every deferred callback, so the count + * issued from NMI must equal the count invoked. + */ + if (atomic_long_read(&n_rcu_torture_nmi_call) != + atomic_long_read(&n_rcu_torture_nmi_cb)) { + pr_alert("%s: NMI ->call() lost a callback: issued %ld invoked %ld\n", + __func__, atomic_long_read(&n_rcu_torture_nmi_call), + atomic_long_read(&n_rcu_torture_nmi_cb)); + atomic_inc(&n_rcu_torture_error); + } + if (cur_ops->cleanup != NULL) cur_ops->cleanup(); @@ -4477,6 +4662,17 @@ static DECLARE_RWSEM(rwsem7); static DECLARE_RWSEM(rwsem8); static DECLARE_RWSEM(rwsem9); +static DEFINE_RAW_SPINLOCK(spin0); +static DEFINE_RAW_SPINLOCK(spin1); +static DEFINE_RAW_SPINLOCK(spin2); +static DEFINE_RAW_SPINLOCK(spin3); +static DEFINE_RAW_SPINLOCK(spin4); +static DEFINE_RAW_SPINLOCK(spin5); +static DEFINE_RAW_SPINLOCK(spin6); +static DEFINE_RAW_SPINLOCK(spin7); +static DEFINE_RAW_SPINLOCK(spin8); +static DEFINE_RAW_SPINLOCK(spin9); + DEFINE_STATIC_SRCU(srcu0); DEFINE_STATIC_SRCU(srcu1); DEFINE_STATIC_SRCU(srcu2); @@ -4488,6 +4684,19 @@ DEFINE_STATIC_SRCU(srcu7); DEFINE_STATIC_SRCU(srcu8); DEFINE_STATIC_SRCU(srcu9); +DEFINE_STATIC_SRCU_ATOMIC(srcu0_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu1_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu2_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu3_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu4_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu5_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu6_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu7_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu8_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu9_atomic); + +DEFINE_STATIC_SRCU_ATOMIC(srcu_irq); + static int srcu_lockdep_next(const char *f, const char *fl, const char *fs, const char *fu, int i, int cyclelen, int deadlock) { @@ -4502,6 +4711,11 @@ static int srcu_lockdep_next(const char *f, const char *fl, const char *fs, cons return j; } +static void srcu_sync_irq(void *unused) +{ + synchronize_srcu_atomic(&srcu_irq); +} + // Test lockdep on SRCU-based deadlock scenarios. static void rcu_torture_init_srcu_lockdep(void) { @@ -4517,6 +4731,12 @@ static void rcu_torture_init_srcu_lockdep(void) &rwsem5, &rwsem6, &rwsem7, &rwsem8, &rwsem9 }; struct srcu_struct *srcus[] = { &srcu0, &srcu1, &srcu2, &srcu3, &srcu4, &srcu5, &srcu6, &srcu7, &srcu8, &srcu9 }; + raw_spinlock_t *spins[] = { &spin0, &spin1, &spin2, &spin3, &spin4, + &spin5, &spin6, &spin7, &spin8, &spin9 }; + struct srcu_struct *srcus_atomic[] = { &srcu0_atomic, &srcu1_atomic, &srcu2_atomic, + &srcu3_atomic, &srcu4_atomic, &srcu5_atomic, + &srcu6_atomic, &srcu7_atomic, &srcu8_atomic, + &srcu9_atomic }; int testtype; if (!test_srcu_lockdep) @@ -4627,11 +4847,83 @@ static void rcu_torture_init_srcu_lockdep(void) } #endif // #ifdef CONFIG_TASKS_TRACE_RCU + if (testtype == 4) { + pr_info("%s: test_srcu_lockdep = %05d: SRCU_ATOMIC %d-way %sdeadlock.\n", + __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); + if (deadlock && cyclelen == 1) + pr_info("%s: Expect hang.\n", __func__); + for (i = 0; i < cyclelen; i++) { + j = srcu_lockdep_next(__func__, "srcu_read_lock_atomic", + "synchronize_srcu_atomic", + "srcu_read_unlock_atomic", i, + cyclelen, deadlock); + idx = srcu_read_lock_atomic(srcus_atomic[i]); + if (j >= 0) + synchronize_srcu_atomic(srcus_atomic[j]); + srcu_read_unlock_atomic(srcus_atomic[i], idx); + } + return; + } + + if (testtype == 5) { + pr_info("%s: test_srcu_lockdep = %05d: SRCU_ATOMIC/raw_spinlock %d-way %sdeadlock.\n", + __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); + for (i = 0; i < cyclelen; i++) { + pr_info("%s: srcu_read_lock_atomic(%d), raw_spin_lock(%d), raw_spin_unlock(%d), srcu_read_unlock_atomic(%d)\n", + __func__, i, i, i, i); + idx = srcu_read_lock_atomic(srcus_atomic[i]); + raw_spin_lock(spins[i]); + raw_spin_unlock(spins[i]); + srcu_read_unlock_atomic(srcus_atomic[i], idx); + + j = srcu_lockdep_next(__func__, "raw_spin_lock", + "synchronize_srcu_atomic", + "raw_spin_unlock", i, cyclelen, + deadlock); + raw_spin_lock(spins[i]); + if (j >= 0) + synchronize_srcu_atomic(srcus_atomic[j]); + raw_spin_unlock(spins[i]); + } + return; + } + + if (testtype == 6) { + pr_info("%s: test_srcu_lockdep = %05d: synchronize_srcu_atomic() inside rcu_read_lock() %d-way.\n", + __func__, test_srcu_lockdep, cyclelen); + for (i = 0; i < cyclelen; i++) { + rcu_read_lock(); + synchronize_srcu_atomic(srcus_atomic[i]); + rcu_read_unlock(); + } + return; + } + + if (testtype == 7) { + int cpu; + + for (i = 0; i < cyclelen; i++) { + idx = srcu_read_lock_atomic(&srcu_irq); + cpu = cpumask_any_but(cpu_online_mask, + smp_processor_id()); + if (cpu < nr_cpu_ids) { + pr_info("%s: CPU%d sending IPI to CPU%d\n", + __func__, smp_processor_id(), cpu); + smp_call_function_single(cpu, srcu_sync_irq, + NULL, 1); + } + srcu_read_unlock_atomic(&srcu_irq, idx); + } + return; + } + err_out: pr_info("%s: test_srcu_lockdep = %05d does nothing.\n", __func__, test_srcu_lockdep); pr_info("%s: test_srcu_lockdep = DNNL.\n", __func__); pr_info("%s: D: Deadlock if nonzero.\n", __func__); - pr_info("%s: NN: Test number, 0=SRCU, 1=SRCU/mutex, 2=SRCU/rwsem, 3=SRCU/Tasks Trace RCU.\n", __func__); + pr_info("%s: NN: Test number, 0=SRCU, 1=SRCU/mutex, 2=SRCU/rwsem, 3=SRCU/Tasks Trace RCU, 4=SRCU_ATOMIC, ", + __func__); + pr_cont("5=SRCU_ATOMIC/raw_spinlock, 6=synchronize_srcu_atomic inside rcu_read_lock, 7=atomic SRCU cross-CPU IRQ context mismatch.\n"); pr_info("%s: L: Cycle length.\n", __func__); if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU)) pr_info("%s: NN=3 disallowed because kernel is built with CONFIG_TASKS_TRACE_RCU=n\n", __func__); @@ -4786,6 +5078,7 @@ rcu_torture_init(void) firsterr = -ENOMEM; goto unwind; } + rcu_torture_nmi_init(); for (i = 0; i < nrealreaders; i++) { rcu_torture_reader_mbchk[i].rtc_chkrdr = -1; firsterr = torture_create_kthread(rcu_torture_reader, (void *)i, diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index 558ba8d316db..63a3f0a8bc53 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -10,6 +10,7 @@ #include <linux/export.h> #include <linux/irq_work.h> +#include <linux/llist.h> #include <linux/mutex.h> #include <linux/preempt.h> #include <linux/rcupdate_wait.h> @@ -29,6 +30,8 @@ extern int rcu_scheduler_active; static LIST_HEAD(srcu_boot_list); static bool srcu_init_done; +static void __srcu_defer_drain(struct srcu_struct *ssp); + static int init_srcu_struct_fields(struct srcu_struct *ssp) { ssp->srcu_lock_nesting[0] = 0; @@ -38,11 +41,14 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp) ssp->srcu_cb_tail = &ssp->srcu_cb_head; ssp->srcu_gp_running = false; ssp->srcu_gp_waiting = false; + ssp->srcu_atomic_gp_flag = 0; ssp->srcu_idx = 0; ssp->srcu_idx_max = 0; INIT_WORK(&ssp->srcu_work, srcu_drive_gp); INIT_LIST_HEAD(&ssp->srcu_work.entry); init_irq_work(&ssp->srcu_irq_work, srcu_tiny_irq_work); + init_llist_head(&ssp->defer_cbs); + ssp->defer_iw = IRQ_WORK_INIT_HARD(srcu_defer_drain); return 0; } @@ -86,6 +92,16 @@ EXPORT_SYMBOL_GPL(init_srcu_struct_generic); void cleanup_srcu_struct(struct srcu_struct *ssp) { WARN_ON(srcu_readers_active(ssp)); + /* + * Re-issue any deferred callbacks, then wait out ->defer_iw before it is + * freed. Skipped entirely with CONFIG_RCU_DEFER=n: irq_work_sync() ends + * in an unconditional synchronize_rcu() wherever + * arch_irq_work_has_interrupt() is false, which is every !SMP target. + */ + if (IS_ENABLED(CONFIG_RCU_DEFER)) { + __srcu_defer_drain(ssp); + irq_work_sync(&ssp->defer_iw); + } irq_work_sync(&ssp->srcu_irq_work); flush_work(&ssp->srcu_work); WARN_ON(ssp->srcu_gp_running); @@ -144,6 +160,8 @@ void srcu_drive_gp(struct work_struct *wp) WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1); WRITE_ONCE(ssp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */ preempt_enable(); + if (IS_ENABLED(CONFIG_PREEMPTION)) + synchronize_rcu(); // Needed for RCU Tasks Trace to imply RCU grace period do { // Deadlock issues prevent __srcu_read_unlock() from // doing an unconditional wakeup, so polling is required. @@ -213,11 +231,11 @@ static void srcu_gp_start_if_needed(struct srcu_struct *ssp) } /* - * Enqueue an SRCU callback on the specified srcu_struct structure, - * initiating grace-period processing if it is not already running. + * Also called by __srcu_defer_drain() to re-issue a deferred callback, so it + * must not re-check the deferral condition. */ -void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, - rcu_callback_t func) +static void srcu_do_enqueue(struct srcu_struct *ssp, struct rcu_head *rhp, + rcu_callback_t func) { unsigned long flags; @@ -231,6 +249,68 @@ void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, srcu_gp_start_if_needed(ssp); preempt_enable(); } + +/* + * Set only by the irq_work drain, the one drain its own re-issue can re-feed; + * a callback staged during a direct drain is taken by ->defer_iw afterwards. + * Global rather than per-srcu_struct: a re-entrant call_srcu(B) inside a drain + * of A raises B's own ->defer_iw, whose drain can stage back onto A. + */ +static bool srcu_defer_draining; + +static void __srcu_defer_drain(struct srcu_struct *ssp) +{ + struct llist_node *node, *next; + unsigned long flags; + + if (!IS_ENABLED(CONFIG_RCU_DEFER)) + return; + + /* Re-issued newest-first; nothing depends on call_srcu() ordering. */ + local_irq_save(flags); + llist_for_each_safe(node, next, llist_del_all(&ssp->defer_cbs)) { + struct rcu_head *rhp = (struct rcu_head *)node; + + srcu_do_enqueue(ssp, rhp, rhp->func); + } + local_irq_restore(flags); +} + +/* Only the irq_work drain can be re-fed by its own re-issue; see Tree SRCU. */ +void srcu_defer_drain(struct irq_work *iw) +{ + struct srcu_struct *ssp = container_of(iw, struct srcu_struct, defer_iw); + + WRITE_ONCE(srcu_defer_draining, true); + __srcu_defer_drain(ssp); + WRITE_ONCE(srcu_defer_draining, false); +} +EXPORT_SYMBOL_GPL(srcu_defer_drain); + +void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, + rcu_callback_t func) +{ + if (should_rcu_defer()) { + /* A re-entrant call_srcu() during the drain would livelock it. */ + if (READ_ONCE(srcu_defer_draining) && !in_nmi()) { + WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU), + "call_srcu() re-entered during callback drain; leaking callback\n"); + return; + } + rhp->func = func; + if (llist_add((struct llist_node *)rhp, &ssp->defer_cbs)) + irq_work_queue(&ssp->defer_iw); + return; + } + + /* + * Only reachable from an NMI when deferral is off: before the scheduler + * is up, or with CONFIG_RCU_DEFER=n. The enqueue can then race. + */ + WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi()); + + srcu_do_enqueue(ssp, rhp, func); +} EXPORT_SYMBOL_GPL(call_srcu); /* @@ -261,6 +341,102 @@ void synchronize_srcu(struct srcu_struct *ssp) EXPORT_SYMBOL_GPL(synchronize_srcu); /* + * synchronize_srcu_atomic - spinning grace period for atomic-reader domains + * @ssp: srcu_struct with which to synchronize. + * + * On !SMP this cannot spin: a reader observed mid-section is preempted + * or interrupted-out, and can only finish if we yield the CPU. But it + * is also never needed: an atomic-flavor reader (preemption disabled) + * cannot be observed mid-section from process context on the sole CPU. + * So a reader observed here has broken the atomic-domain promise, and + * the only correct wait for it is a real grace period. + * + * (Actual kernel-doc header is in Tree SRCU.) + */ +void synchronize_srcu_atomic(struct srcu_struct *ssp) +{ + int idx; + bool ret; + unsigned long srcu_state = get_state_synchronize_srcu(ssp); + + srcu_lock_sync(&ssp->dep_map); + + RCU_LOCKDEP_WARN(lockdep_is_held(ssp), + "Illegal synchronize_srcu_atomic() in same-type SRCU read-side critical section"); + + if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE) + return; + + if (IS_ENABLED(CONFIG_PREEMPTION)) + synchronize_rcu(); // Needed for RCU Tasks Trace to imply RCU grace period. + // And in Tiny RCU, it is near zero cost and doesn't block. + + // Usually, there will be no readers. + preempt_disable(); // Guard against lazy preemption and some other grace period. + ret = !READ_ONCE(ssp->srcu_lock_nesting[0]) && !READ_ONCE(ssp->srcu_lock_nesting[1]); + if (ret) { + WRITE_ONCE(ssp->srcu_idx_max, ssp->srcu_idx + 2); + WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 2); + preempt_enable(); + return; + } + + // Because readers disable preemption, we should never get here. + // However, a splat and some spinning is usually preferable to + // memory corruption due to a too-short grace period. There is + // the possibility that this will hang if the preempted reader is + // not looked upon favorably by the scheduler, but this is still + // preferable to memory corruption. + WARN_ON_ONCE(1); + + // Wait to drive a grace period or for someone else to do it + // for us while we are lazily preempted. + while (ssp->srcu_atomic_gp_flag) { + if (poll_state_synchronize_srcu(ssp, srcu_state)) { + preempt_enable(); + return; + } + preempt_enable(); + cpu_relax(); + cond_resched_tasks_rcu_qs(); + preempt_disable(); + } + ssp->srcu_atomic_gp_flag = 1; // Acquire SRCU grace-period "lock". + preempt_enable(); + + // We get here if a reader has been lazily preempted. + // First, wait for old readers, which are quite unlikely. + WRITE_ONCE(ssp->srcu_idx_max, get_state_synchronize_srcu(ssp)); + idx = !(((READ_ONCE(ssp->srcu_idx) + 1) & 0x2) >> 1); + while (READ_ONCE(ssp->srcu_lock_nesting[idx])) { + cond_resched_tasks_rcu_qs(); + cpu_relax(); + } + + // Next, flip the index and wait for the other group of readers. + WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1); + idx = !idx; + while (READ_ONCE(ssp->srcu_lock_nesting[idx])) { + cond_resched_tasks_rcu_qs(); + cpu_relax(); + } + + // Finally, flip the index again for poll_state_synchronize_srcu(). + WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1); + WARN_ON_ONCE(!poll_state_synchronize_srcu(ssp, srcu_state)); + ssp->srcu_atomic_gp_flag = 0; // Release SRCU grace-period "lock". +} +EXPORT_SYMBOL_GPL(synchronize_srcu_atomic); + +/* Register any deferred callbacks, then wait for all in-flight ones. */ +void srcu_barrier(struct srcu_struct *ssp) +{ + __srcu_defer_drain(ssp); + synchronize_srcu(ssp); +} +EXPORT_SYMBOL_GPL(srcu_barrier); + +/* * get_state_synchronize_srcu - Provide an end-of-grace-period cookie */ unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp) @@ -280,6 +456,11 @@ EXPORT_SYMBOL_GPL(get_state_synchronize_srcu); * The difference between this and get_state_synchronize_srcu() is that * this function ensures that the poll_state_synchronize_srcu() will * eventually return the value true. + * + * This function cannot be used with atomic SRCU, which only has + * atomic grace periods. Doing so will silently corrupt internal + * SRCU state. Tree SRCU has appropriate checking with splats, + * so please test with CONFIG_SMP=y as well as CONFIG_SMP=n. */ unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp) { diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index ed204b3f4b84..2b0e2b2d2c18 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -20,6 +20,7 @@ #include <linux/percpu.h> #include <linux/preempt.h> #include <linux/irq_work.h> +#include <linux/llist.h> #include <linux/rcupdate_wait.h> #include <linux/sched.h> #include <linux/smp.h> @@ -79,6 +80,38 @@ static void process_srcu(struct work_struct *work); static void srcu_irq_work(struct irq_work *work); static void srcu_delay_timer(struct timer_list *t); +struct srcu_defer; +static void srcu_defer_drain(struct irq_work *iw); +static void __srcu_defer_drain(struct srcu_defer *sndp); + +/* + * Per-CPU call_srcu() deferral state, shared by every srcu_struct. A deferred + * callback is staged on its srcu_data's ->defer_cbs; that srcu_data is chained + * via ->defer_link onto ->list, which the irq_work walks. + */ +struct srcu_defer { + struct llist_head list; + struct irq_work iw; + raw_spinlock_t lock; + bool draining; +}; + +static DEFINE_PER_CPU(struct srcu_defer, srcu_defer) = { + .lock = __RAW_SPIN_LOCK_UNLOCKED(srcu_defer.lock), + .iw = IRQ_WORK_INIT_HARD(srcu_defer_drain), +}; + +/* + * Flush pending deferred callbacks so a following srcu_barrier() waits for them. + */ +static void srcu_defer_flush(void) +{ + int cpu; + + for_each_possible_cpu(cpu) + __srcu_defer_drain(&per_cpu(srcu_defer, cpu)); +} + /* * Initialize SRCU per-CPU data. Note that statically allocated * srcu_struct structures might already have srcu_read_lock() and @@ -107,6 +140,11 @@ static void init_srcu_struct_data(struct srcu_struct *ssp) sdp->cpu = cpu; INIT_WORK(&sdp->work, srcu_invoke_callbacks); timer_setup(&sdp->delay_work, srcu_delay_timer, 0); + /* + * ->defer_cbs and ->defer_link are valid when zeroed and are not + * reinitialized here: that would clobber callbacks a reentrant + * call_srcu() already staged. See __call_srcu(). + */ sdp->ssp = ssp; } } @@ -200,8 +238,10 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags) * Initialize non-compile-time initialized fields, including the * associated srcu_node and srcu_data structures. The is_static parameter * tells us that ->sda has already been wired up to srcu_data. + * The is_atomic parameter tells us that there is no reason to + * ever transition to big. */ -static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) +static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static, bool is_atomic) { if (!is_static) ssp->srcu_sup = kzalloc_obj(*ssp->srcu_sup); @@ -213,12 +253,18 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) ssp->srcu_sup->node = NULL; mutex_init(&ssp->srcu_sup->srcu_cb_mutex); mutex_init(&ssp->srcu_sup->srcu_gp_mutex); + atomic_set(&ssp->srcu_sup->srcu_atomic_gp_flag, 0); ssp->srcu_sup->srcu_gp_seq = SRCU_GP_SEQ_INITIAL_VAL; ssp->srcu_sup->srcu_barrier_seq = 0; mutex_init(&ssp->srcu_sup->srcu_barrier_mutex); atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0); INIT_DELAYED_WORK(&ssp->srcu_sup->work, process_srcu); - init_irq_work(&ssp->srcu_sup->irq_work, srcu_irq_work); + /* + * trace events started on the command line require SRCU before + * the irq_work kthread starts. Since all it does is a simple + * wakeup, having it as a hard irq, even on PREEMPT_RT is fine. + */ + ssp->srcu_sup->irq_work = IRQ_WORK_INIT_HARD(srcu_irq_work); ssp->srcu_sup->sda_is_static = is_static; if (!is_static) { ssp->sda = alloc_percpu(struct srcu_data); @@ -229,7 +275,8 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) init_srcu_struct_data(ssp); ssp->srcu_sup->srcu_gp_seq_needed_exp = SRCU_GP_SEQ_INITIAL_VAL; ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns(); - if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) { + if (!is_atomic && + READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) { if (!preemptible()) WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_ALLOC); else if (init_srcu_struct_nodes(ssp, GFP_KERNEL)) @@ -258,26 +305,27 @@ err_free_sup: #ifdef CONFIG_DEBUG_LOCK_ALLOC static int -__init_srcu_struct_common(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) +__init_srcu_struct_common(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key, bool is_atomic) { /* Don't re-initialize a lock while it is held. */ debug_check_no_locks_freed((void *)ssp, sizeof(*ssp)); lockdep_init_map(&ssp->dep_map, name, key, 0); - return init_srcu_struct_fields(ssp, false); + return init_srcu_struct_fields(ssp, false, is_atomic); } int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) { ssp->srcu_reader_flavor = 0; - return __init_srcu_struct_common(ssp, name, key); + return __init_srcu_struct_common(ssp, name, key, false); } EXPORT_SYMBOL_GPL(init_srcu_struct_lockdep); int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) { ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST; - return __init_srcu_struct_common(ssp, name, key); + return __init_srcu_struct_common(ssp, name, key, false); } EXPORT_SYMBOL_GPL(__init_srcu_struct_fast); @@ -285,10 +333,17 @@ int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) { ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST_UPDOWN; - return __init_srcu_struct_common(ssp, name, key); + return __init_srcu_struct_common(ssp, name, key, false); } EXPORT_SYMBOL_GPL(__init_srcu_struct_fast_updown); +int __init_srcu_struct_atomic(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) +{ + ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_ATOMIC; + return __init_srcu_struct_common(ssp, name, key, true); +} +EXPORT_SYMBOL_GPL(__init_srcu_struct_atomic); + #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ /** @@ -305,7 +360,7 @@ EXPORT_SYMBOL_GPL(__init_srcu_struct_fast_updown); int init_srcu_struct_generic(struct srcu_struct *ssp) { ssp->srcu_reader_flavor = 0; - return init_srcu_struct_fields(ssp, false); + return init_srcu_struct_fields(ssp, false, false); } EXPORT_SYMBOL_GPL(init_srcu_struct_generic); @@ -322,7 +377,7 @@ EXPORT_SYMBOL_GPL(init_srcu_struct_generic); int init_srcu_struct_fast(struct srcu_struct *ssp) { ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST; - return init_srcu_struct_fields(ssp, false); + return init_srcu_struct_fields(ssp, false, false); } EXPORT_SYMBOL_GPL(init_srcu_struct_fast); @@ -340,10 +395,30 @@ EXPORT_SYMBOL_GPL(init_srcu_struct_fast); int init_srcu_struct_fast_updown(struct srcu_struct *ssp) { ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST_UPDOWN; - return init_srcu_struct_fields(ssp, false); + return init_srcu_struct_fields(ssp, false, false); } EXPORT_SYMBOL_GPL(init_srcu_struct_fast_updown); +/** + * init_srcu_struct_atomic - initialize an atomic sleep-RCU structure + * @ssp: structure to initialize. + * + * Use this in place of DEFINE_SRCU_ATOMIC() and DEFINE_STATIC_SRCU_ATOMIC() + * for non-static srcu_struct structures that are to be passed to + * srcu_read_lock_atomic() and friends. It is necessary to invoke this on a + * given srcu_struct before passing that srcu_struct to any other function. + * Each srcu_struct represents a separate domain of SRCU protection. + * + * And yes, we really are defining a sleepable RCU implementation that + * cannot sleep. Strange universe we live in, isn't it? + */ +int init_srcu_struct_atomic(struct srcu_struct *ssp) +{ + ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_ATOMIC; + return init_srcu_struct_fields(ssp, false, true); +} +EXPORT_SYMBOL_GPL(init_srcu_struct_atomic); + #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ /* @@ -432,9 +507,11 @@ static void raw_spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsign * done with compile-time initialization, so this check is added * to each update-side SRCU primitive. Use ssp->lock, which -is- * compile-time initialized, to resolve races involving multiple - * CPUs trying to garner first-use privileges. + * CPUs trying to garner first-use privileges. The is_atomic + * parameter tells us that there will never be a reason to + * transition to big. */ -static void check_init_srcu_struct(struct srcu_struct *ssp) +static void check_init_srcu_struct(struct srcu_struct *ssp, bool is_atomic) { unsigned long flags; @@ -446,7 +523,7 @@ static void check_init_srcu_struct(struct srcu_struct *ssp) raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); return; } - init_srcu_struct_fields(ssp, true); + init_srcu_struct_fields(ssp, true, is_atomic); raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); } @@ -624,7 +701,7 @@ module_param(srcu_retry_check_delay, ulong, 0444); #define SRCU_UL_CLAMP_LO(val, low) ((val) > (low) ? (val) : (low)) #define SRCU_UL_CLAMP_HI(val, high) ((val) < (high) ? (val) : (high)) #define SRCU_UL_CLAMP(val, low, high) SRCU_UL_CLAMP_HI(SRCU_UL_CLAMP_LO((val), (low)), (high)) -// per-GP-phase no-delay instances adjusted to allow non-sleeping poll upto +// per-GP-phase no-delay instances adjusted to allow non-sleeping poll up to // one jiffies time duration. Mult by 2 is done to factor in the srcu_get_delay() // called from process_srcu(). #define SRCU_DEFAULT_MAX_NODELAY_PHASE_ADJUSTED \ @@ -688,6 +765,14 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) unsigned long delay; struct srcu_usage *sup = ssp->srcu_sup; + /* + * Drain before the early returns below: they leak the srcu_struct, but + * srcu_module_going() frees ->sda regardless, and a staged srcu_data + * left chained on a per-CPU list would then dangle. Draining first also + * has to precede the ->irq_work sync, since re-issuing a callback can + * start a grace period and re-queue ->irq_work, which schedules ->work. + */ + srcu_defer_flush(); raw_spin_lock_irq_rcu_node(ssp->srcu_sup); delay = srcu_get_delay(ssp); raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); @@ -695,7 +780,6 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) return; /* Just leak it! */ if (WARN_ON(srcu_readers_active(ssp))) return; /* Just leak it! */ - /* Wait for irq_work to finish first as it may queue a new work. */ irq_work_sync(&sup->irq_work); flush_delayed_work(&sup->work); for_each_possible_cpu(cpu) { @@ -704,7 +788,7 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) // Call srcu_barrier() before this cleanup_srcu_struct() // to avoid triggering this WARN_ON(). if (WARN_ON(timer_delete_sync(&sdp->delay_work) && - rcu_segcblist_n_cbs(&sdp->srcu_cblist)) && + !rcu_segcblist_empty(&sdp->srcu_cblist)) && rcu_cpu_beenfullyonline(sdp->cpu)) queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work); flush_work(&sdp->work); @@ -752,7 +836,10 @@ void __srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor) WARN_ON_ONCE(ssp->srcu_reader_flavor && read_flavor != ssp->srcu_reader_flavor); WARN_ON_ONCE(old_read_flavor && ssp->srcu_reader_flavor && old_read_flavor != ssp->srcu_reader_flavor); - WARN_ON_ONCE(read_flavor == SRCU_READ_FLAVOR_FAST && !ssp->srcu_reader_flavor); + WARN_ON_ONCE(!!(read_flavor & SRCU_READ_FLAVOR_PREDEF) && + read_flavor != ssp->srcu_reader_flavor); + WARN_ON_ONCE(!!(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_PREDEF) && + read_flavor != ssp->srcu_reader_flavor); if (!old_read_flavor) { old_read_flavor = cmpxchg(&sdp->srcu_reader_flavor, 0, read_flavor); if (!old_read_flavor) @@ -892,12 +979,13 @@ static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp * are initiating callback invocation. This allows the ->srcu_have_cbs[] * array to have a finite number of elements. */ -static void srcu_gp_end(struct srcu_struct *ssp) +static void srcu_gp_end(struct srcu_struct *ssp, bool is_atomic) { unsigned long cbdelay = 1; bool cbs; bool last_lvl; int cpu; + unsigned long flags; unsigned long gpseq; int idx; unsigned long mask; @@ -908,10 +996,11 @@ static void srcu_gp_end(struct srcu_struct *ssp) struct srcu_usage *sup = ssp->srcu_sup; /* Prevent more than one additional grace period. */ - mutex_lock(&sup->srcu_cb_mutex); + if (!is_atomic) + mutex_lock(&sup->srcu_cb_mutex); /* End the current grace period. */ - raw_spin_lock_irq_rcu_node(sup); + raw_spin_lock_irqsave_rcu_node(sup, flags); idx = rcu_seq_state(sup->srcu_gp_seq); WARN_ON_ONCE(idx != SRCU_STATE_SCAN2); if (srcu_gp_is_expedited(ssp)) @@ -922,16 +1011,17 @@ static void srcu_gp_end(struct srcu_struct *ssp) gpseq = rcu_seq_current(&sup->srcu_gp_seq); if (ULONG_CMP_LT(sup->srcu_gp_seq_needed_exp, gpseq)) WRITE_ONCE(sup->srcu_gp_seq_needed_exp, gpseq); - raw_spin_unlock_irq_rcu_node(sup); - mutex_unlock(&sup->srcu_gp_mutex); + raw_spin_unlock_irqrestore_rcu_node(sup, flags); + if (!is_atomic) + mutex_unlock(&sup->srcu_gp_mutex); /* A new grace period can start at this point. But only one. */ /* Initiate callback invocation as needed. */ ss_state = smp_load_acquire(&sup->srcu_size_state); - if (ss_state < SRCU_SIZE_WAIT_BARRIER) { + if (!is_atomic && ss_state < SRCU_SIZE_WAIT_BARRIER) { srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, get_boot_cpu_id()), cbdelay); - } else { + } else if (!is_atomic) { idx = rcu_seq_ctr(gpseq) % ARRAY_SIZE(snp->srcu_have_cbs); srcu_for_each_node_breadth_first(ssp, snp) { raw_spin_lock_irq_rcu_node(snp); @@ -959,31 +1049,35 @@ static void srcu_gp_end(struct srcu_struct *ssp) if (!(gpseq & counter_wrap_check)) for_each_possible_cpu(cpu) { sdp = per_cpu_ptr(ssp->sda, cpu); - raw_spin_lock_irq_rcu_node(sdp); + raw_spin_lock_irqsave_rcu_node(sdp, flags); if (ULONG_CMP_GE(gpseq, sdp->srcu_gp_seq_needed + 100)) sdp->srcu_gp_seq_needed = gpseq; if (ULONG_CMP_GE(gpseq, sdp->srcu_gp_seq_needed_exp + 100)) sdp->srcu_gp_seq_needed_exp = gpseq; - raw_spin_unlock_irq_rcu_node(sdp); + raw_spin_unlock_irqrestore_rcu_node(sdp, flags); } /* Callback initiation done, allow grace periods after next. */ - mutex_unlock(&sup->srcu_cb_mutex); + if (!is_atomic) + mutex_unlock(&sup->srcu_cb_mutex); /* Start a new grace period if needed. */ - raw_spin_lock_irq_rcu_node(sup); + raw_spin_lock_irqsave_rcu_node(sup, flags); gpseq = rcu_seq_current(&sup->srcu_gp_seq); if (!rcu_seq_state(gpseq) && ULONG_CMP_LT(gpseq, sup->srcu_gp_seq_needed)) { + WARN_ON_ONCE(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_ATOMIC); srcu_gp_start(ssp); - raw_spin_unlock_irq_rcu_node(sup); + raw_spin_unlock_irqrestore_rcu_node(sup, flags); srcu_reschedule(ssp, 0); } else { - raw_spin_unlock_irq_rcu_node(sup); + raw_spin_unlock_irqrestore_rcu_node(sup, flags); } - /* Transition to big if needed. */ - if (ss_state != SRCU_SIZE_SMALL && ss_state != SRCU_SIZE_BIG) { + /* Transition to big if needed, but never for atomic SRCU. */ + if (ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC && ss_state != SRCU_SIZE_SMALL) { + WARN_ON_ONCE(1); + } else if (ss_state != SRCU_SIZE_SMALL && ss_state != SRCU_SIZE_BIG) { if (ss_state == SRCU_SIZE_ALLOC) init_srcu_struct_nodes(ssp, GFP_KERNEL); else @@ -1098,6 +1192,7 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, /* If grace period not already in progress, start it. */ if (!WARN_ON_ONCE(rcu_seq_done(&sup->srcu_gp_seq, s)) && rcu_seq_state(sup->srcu_gp_seq) == SRCU_STATE_IDLE) { + WARN_ON_ONCE(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_ATOMIC); srcu_gp_start(ssp); // And how can that list_add() in the "else" clause @@ -1126,10 +1221,11 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount) { unsigned long curdelay; + unsigned long flags; - raw_spin_lock_irq_rcu_node(ssp->srcu_sup); + raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags); curdelay = !srcu_get_delay(ssp); - raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); + raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); for (;;) { if (srcu_readers_active_idx_check(ssp, idx)) @@ -1234,7 +1330,7 @@ static bool srcu_should_expedite(struct srcu_struct *ssp) unsigned long t; unsigned long tlast; - check_init_srcu_struct(ssp); + check_init_srcu_struct(ssp, false); /* If _lite() readers, don't do unsolicited expediting. */ if (this_cpu_read(ssp->sda->srcu_reader_flavor) & SRCU_READ_FLAVOR_SLOWGP) return false; @@ -1293,7 +1389,7 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, struct srcu_node *sdp_mynode; int ss_state; - check_init_srcu_struct(ssp); + check_init_srcu_struct(ssp, false); /* * While starting a new grace period, make sure we are in an * SRCU read-side critical section so that the grace-period @@ -1411,8 +1507,8 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, * srcu_read_lock(), and srcu_read_unlock() that are all passed the same * srcu_struct structure. */ -static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, - rcu_callback_t func, bool do_norm) +static void srcu_do_enqueue(struct srcu_struct *ssp, struct rcu_head *rhp, + rcu_callback_t func, bool do_norm) { if (debug_rcu_head_queue(rhp)) { /* Probable double call_srcu(), so leak the callback. */ @@ -1424,6 +1520,110 @@ static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, (void)srcu_gp_start_if_needed(ssp, rhp, do_norm); } +/* + * The srcu_cblist and srcu_node tree are only accessed with interrupts + * disabled, so defer when interrupts are already off rather than enqueue into + * an operation that may be in flight on this CPU. + */ +static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, + rcu_callback_t func, bool do_norm) +{ + if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) + return; // Leak the callback rather than corrupt SRCU state. + if (should_rcu_defer()) { + struct srcu_defer *sndp = this_cpu_ptr(&srcu_defer); + struct srcu_data *sdp; + + /* + * Instrumentation on the enqueue path can re-enter here from + * inside the drain. Re-queuing would livelock it, so drop the + * callback; an NMI cannot loop, so let it through. + */ + if (READ_ONCE(sndp->draining) && !in_nmi()) { + WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU), + "call_srcu() re-entered during callback drain; leaking callback\n"); + return; + } + sdp = this_cpu_ptr(ssp->sda); + rhp->func = func; + if (llist_add((struct llist_node *)rhp, &sdp->defer_cbs)) { + /* + * Chain this srcu_data for the drain. ->ssp must be + * published here: deferral skips + * check_init_srcu_struct(), so on a never-initialized + * static srcu_struct the srcu_data are still zeroed and + * the drain would read a NULL ->ssp. + */ + sdp->ssp = ssp; + if (llist_add(&sdp->defer_link, &sndp->list)) + irq_work_queue(&sndp->iw); + } + return; + } + + /* + * Only reachable from an NMI when deferral is off: before the scheduler + * is up, or with CONFIG_RCU_DEFER=n. The enqueue can then race. + */ + WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi()); + + srcu_do_enqueue(ssp, rhp, func, do_norm); +} + +/* + * Re-issue deferred callbacks straight to srcu_do_enqueue() so they cannot defer + * again. ->lock serializes the drainers: the irq_work, srcu_defer_flush() and + * srcu_offline_drain(). + */ +static void __srcu_defer_drain(struct srcu_defer *sndp) +{ + struct llist_node *snode, *snext; + unsigned long flags; + + if (!IS_ENABLED(CONFIG_RCU_DEFER)) + return; + + raw_spin_lock_irqsave(&sndp->lock, flags); + llist_for_each_safe(snode, snext, llist_del_all(&sndp->list)) { + struct srcu_data *sdp = container_of(snode, struct srcu_data, defer_link); + struct srcu_struct *ssp = sdp->ssp; + struct llist_node *cnode, *cnext; + + cnode = llist_del_all(&sdp->defer_cbs); + llist_for_each_safe(cnode, cnext, cnode) { + struct rcu_head *rhp = (struct rcu_head *)cnode; + + /* Bounds a node self-linked by a double call_srcu(). */ + rhp->next = NULL; + srcu_do_enqueue(ssp, rhp, rhp->func, true); + } + } + raw_spin_unlock_irqrestore(&sndp->lock, flags); +} + +/* + * Only the irq_work drain can be re-fed by its own re-issue, so only it sets + * ->draining. A direct drain re-issues onto this CPU, and anything staged + * during it is picked up by that CPU's own irq_work. + */ +static void srcu_defer_drain(struct irq_work *iw) +{ + struct srcu_defer *sndp = container_of(iw, struct srcu_defer, iw); + + WRITE_ONCE(sndp->draining, true); + __srcu_defer_drain(sndp); + WRITE_ONCE(sndp->draining, false); +} + +/* + * Drain @cpu's deferred call_srcu() callbacks once @cpu is dead. One pass + * covers every srcu_struct; the re-issue lands on the current CPU. + */ +void srcu_offline_drain(int cpu) +{ + __srcu_defer_drain(&per_cpu(srcu_defer, cpu)); +} + /** * call_srcu() - Queue a callback for invocation after an SRCU grace period * @ssp: srcu_struct in queue the callback @@ -1469,8 +1669,13 @@ static void __synchronize_srcu(struct srcu_struct *ssp, bool do_norm) if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE) return; + if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) { + // This works, and exposes a possible bug. + synchronize_srcu_atomic(ssp); + return; + } might_sleep(); - check_init_srcu_struct(ssp); + check_init_srcu_struct(ssp, false); init_completion(&rcu.completion); init_rcu_head_on_stack(&rcu.head); __call_srcu(ssp, &rcu.head, wakeme_after_rcu, do_norm); @@ -1589,10 +1794,21 @@ EXPORT_SYMBOL_GPL(get_state_synchronize_srcu); * period has elapsed in the meantime. Unlike get_state_synchronize_srcu(), * this function also ensures that any needed SRCU grace period will be * started. This convenience does come at a cost in terms of CPU overhead. + * + * This function cannot be used with atomic SRCU, which only has + * atomic grace periods. Give a warning if someone tries, and return + * the same cookie that would have been returned, but refrain from + * messing up state by starting a grace period. If someone somewhere + * somehow invokes synchronize_srcu_atomic(), passing this cookie to + * poll_state_synchronize_srcu() will return true. If no one ever invokes + * synchronize_srcu_atomic(), too bad. */ unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp) { - return srcu_gp_start_if_needed(ssp, NULL, true); + if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) + return get_state_synchronize_srcu(ssp); + else + return srcu_gp_start_if_needed(ssp, NULL, true); } EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu); @@ -1678,9 +1894,21 @@ void srcu_barrier(struct srcu_struct *ssp) { int cpu; int idx; - unsigned long s = rcu_seq_snap(&ssp->srcu_sup->srcu_barrier_seq); + unsigned long s; + + check_init_srcu_struct(ssp, false); + // Atomic SRCU has no callbacks, so there is nothing to wait on. + if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) + return; - check_init_srcu_struct(ssp); + /* + * Register any deferred callbacks before snapshotting the sequence. The + * staging list is per-CPU, not per-srcu_struct, so this also drains + * other srcu_structs'. + */ + srcu_defer_flush(); + + s = rcu_seq_snap(&ssp->srcu_sup->srcu_barrier_seq); mutex_lock(&ssp->srcu_sup->srcu_barrier_mutex); if (rcu_seq_done(&ssp->srcu_sup->srcu_barrier_seq, s)) { smp_mb(); /* Force ordering following return. */ @@ -1743,6 +1971,9 @@ static void srcu_expedite_current_cb(struct rcu_head *rhp) * no current grace period, one might be created. If the current grace * period is currently sleeping, that sleep will complete before expediting * will take effect. + * + * This function must not be invoked on srcu_struct structures that are + * used with srcu_read_lock_atomic() and synchronize_srcu_atomic(). */ void srcu_expedite_current(struct srcu_struct *ssp) { @@ -1750,6 +1981,9 @@ void srcu_expedite_current(struct srcu_struct *ssp) bool needcb = false; struct srcu_data *sdp; + // Atomic SRCU has no callbacks, so there is nothing to expedite. + if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) + return; migrate_disable(); sdp = this_cpu_ptr(ssp->sda); raw_spin_lock_irqsave_sdp_contention(sdp, &flags); @@ -1785,13 +2019,17 @@ EXPORT_SYMBOL_GPL(srcu_batches_completed); /* * Core SRCU state machine. Push state bits of ->srcu_gp_seq * to SRCU_STATE_SCAN2, and invoke srcu_gp_end() when scan has - * completed in that state. + * completed in that state. Set is_atomic to indicate that + * the caller is excluding other calls so that ->srcu_gp_mutex + * is not needed, and to indicate that blocking is forbidden. */ -static void srcu_advance_state(struct srcu_struct *ssp) +static void srcu_advance_state(struct srcu_struct *ssp, bool is_atomic) { + unsigned long flags; int idx; - mutex_lock(&ssp->srcu_sup->srcu_gp_mutex); + if (!is_atomic) + mutex_lock(&ssp->srcu_sup->srcu_gp_mutex); /* * Because readers might be delayed for an extended period after @@ -1805,19 +2043,23 @@ static void srcu_advance_state(struct srcu_struct *ssp) */ idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq)); /* ^^^ */ if (idx == SRCU_STATE_IDLE) { - raw_spin_lock_irq_rcu_node(ssp->srcu_sup); + raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags); if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) { WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq)); - raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); - mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); + raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); + if (!is_atomic) + mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); return; } idx = rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)); - if (idx == SRCU_STATE_IDLE) + if (idx == SRCU_STATE_IDLE) { + WARN_ON_ONCE(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_ATOMIC); srcu_gp_start(ssp); - raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); + } + raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); if (idx != SRCU_STATE_IDLE) { - mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); + if (!is_atomic) + mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); return; /* Someone else started the grace period. */ } } @@ -1825,14 +2067,15 @@ static void srcu_advance_state(struct srcu_struct *ssp) if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN1) { idx = !(ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0]); if (!try_check_zero(ssp, idx, 1)) { - mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); + if (!is_atomic) + mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); return; /* readers present, retry later. */ } srcu_flip(ssp); - raw_spin_lock_irq_rcu_node(ssp->srcu_sup); + raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags); rcu_seq_set_state(&ssp->srcu_sup->srcu_gp_seq, SRCU_STATE_SCAN2); ssp->srcu_sup->srcu_n_exp_nodelay = 0; - raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); + raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); } if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN2) { @@ -1843,13 +2086,145 @@ static void srcu_advance_state(struct srcu_struct *ssp) */ idx = !(ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0]); if (!try_check_zero(ssp, idx, 2)) { - mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); + if (!is_atomic) + mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); return; /* readers present, retry later. */ } ssp->srcu_sup->srcu_n_exp_nodelay = 0; - srcu_gp_end(ssp); /* Releases ->srcu_gp_mutex. */ + srcu_gp_end(ssp, is_atomic); /* Releases ->srcu_gp_mutex. */ + } +} + +/** + * synchronize_srcu_atomic - spin for prior SRCU read-side critical-section completion + * @ssp: srcu_struct with which to synchronize. + * + * Similar to synchronize_srcu(), but spins rather than blocking. + * Use only with srcu_read_lock_atomic() and srcu_read_unlock_atomic(), + * which are forbidden from voluntarily context switching. If + * synchronize_srcu_atomic() is invoked from a more restrictive context + * (for example, interrupts disabled) for a given srcu_struct structure, + * then for that structure, all calls to both srcu_read_lock_atomic() + * and srcu_read_unlock_atomic() must be invoked from that same context, + * or one that is even more strict. + * + * If synchronize_srcu_atomic() is invoked on a given srcu_struct + * structure, then none of call_srcu(), synchronize_srcu(), + * synchronize_srcu_expedited(), start_poll_synchronize_srcu(), + * srcu_barrier(), or srcu_expedite_current() may be invoked on that + * same structure. + * + * Because synchronize_srcu_atomic() is even more expedited than is + * synchronize_srcu_expedited(), there is no expedited counterpart to + * this function. + */ +void synchronize_srcu_atomic(struct srcu_struct *ssp) +{ + unsigned long srcu_state; + struct srcu_usage *sup = ssp->srcu_sup; + unsigned long flags; + unsigned long rdm0, rdm1; + unsigned long unlocks0, unlocks1; + + srcu_lock_sync(&ssp->dep_map); + + RCU_LOCKDEP_WARN(lockdep_is_held(ssp), + "Illegal synchronize_srcu_atomic() in same-type SRCU read-side critical section"); + + if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE) + return; + + // Initialize. Either init_srcu_struct() was invoked or + // DEFINE_SRCU() or similar was used. Therefore, no allocation + // will be done here. + check_init_srcu_struct(ssp, true); + srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_ATOMIC); + + // Perhaps others will do our work for us. + srcu_state = get_state_synchronize_srcu(ssp); + preempt_disable(); + while (atomic_read(&sup->srcu_atomic_gp_flag) || + atomic_xchg(&sup->srcu_atomic_gp_flag, 1)) { + preempt_enable(); + if (poll_state_synchronize_srcu(ssp, srcu_state)) + return; + cpu_relax(); + preempt_disable(); } + + // One last check for others doing our work for us under the lock. + raw_spin_lock_irqsave_rcu_node(sup, flags); + if (poll_state_synchronize_srcu(ssp, srcu_state)) { + raw_spin_unlock_irqrestore_rcu_node(sup, flags); + atomic_set(&sup->srcu_atomic_gp_flag, 0); + preempt_enable(); + return; + } + + // OK, we really have to do it ourselves. Start the grace period. + non_block_start(); // We must not voluntarily block! + smp_store_release(&sup->srcu_gp_seq_needed, srcu_state); // See srcu_funnel_gp_start(). + ASSERT_EXCLUSIVE_WRITER(ssp->srcu_sup->srcu_gp_seq); + srcu_gp_start(ssp); + raw_spin_unlock_irqrestore_rcu_node(sup, flags); + + // + // Fastpath: If there are no readers at all, neither grace-period + // scan need wait, so both can be satisfied at once without doing + // the index flip. The counter-sum proof is the same as that of + // srcu_readers_active_idx_check(), but spanning both indices. + // Atomic SRCU guarantees that all readers are of + // SRCU_READ_FLAVOR_ATOMIC, so the SLOWGP check never triggers and + // the ->srcu_reader_flavor masks returned by + // srcu_readers_unlock_idx() are unused. + // + // This proof must follow the grace-period anchor written by the + // srcu_gp_start() above, never precede it. With the anchor first, + // a reader whose lock increment is missed by the sums below cannot + // have incremented its lock counter before the anchor, and therefore + // cannot be a pre-existing reader of this grace period. Placing the + // proof before the anchor would let this grace period miss a + // pre-existing reader and return without waiting for it. + // + // The smp_mb() pairs with the smp_mb() in __srcu_read_lock() + // (store-buffering pattern), which guarantees that a lock is always + // counted if the corresponding unlock is counted, the same + // memory-ordering guarantee as is provided by + // srcu_readers_active_idx_check(). + // + unlocks0 = srcu_readers_unlock_idx(ssp, 0, &rdm0); + unlocks1 = srcu_readers_unlock_idx(ssp, 1, &rdm1); + smp_mb(); /* A */ + if (srcu_readers_lock_idx(ssp, 0, false, unlocks0) && + srcu_readers_lock_idx(ssp, 1, false, unlocks1)) { + // No readers, so end this grace period manually, skipping + // the index flip. Advancing the sequence number via + // rcu_seq_start() in srcu_gp_start() above and rcu_seq_end() + // below keeps get_state_synchronize_srcu() and + // poll_state_synchronize_srcu() working, all under ->lock + // and ->srcu_atomic_gp_flag, which excludes concurrent + // sequence-number updates. + raw_spin_lock_irqsave_rcu_node(sup, flags); + rcu_seq_end(&sup->srcu_gp_seq); + raw_spin_unlock_irqrestore_rcu_node(sup, flags); + WARN_ON_ONCE(!poll_state_synchronize_srcu(ssp, srcu_state)); + atomic_set_release(&sup->srcu_atomic_gp_flag, 0); + preempt_enable(); + non_block_end(); + return; + } + + // Wait for it to complete, helping it along. + while (!poll_state_synchronize_srcu(ssp, srcu_state)) { + cpu_relax(); + srcu_advance_state(ssp, true); + } + ASSERT_EXCLUSIVE_WRITER(sup->srcu_atomic_gp_flag); + atomic_set_release(&sup->srcu_atomic_gp_flag, 0); + preempt_enable(); + non_block_end(); } +EXPORT_SYMBOL_GPL(synchronize_srcu_atomic); /* * Invoke a limited number of SRCU callbacks that have passed through @@ -1930,6 +2305,7 @@ static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay) } } else if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq)) { /* Outstanding request and no GP. Start one. */ + WARN_ON_ONCE(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_ATOMIC); srcu_gp_start(ssp); } raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); @@ -1951,7 +2327,7 @@ static void process_srcu(struct work_struct *work) sup = container_of(work, struct srcu_usage, work.work); ssp = sup->srcu_ssp; - srcu_advance_state(ssp); + srcu_advance_state(ssp, false); raw_spin_lock_irq_rcu_node(ssp->srcu_sup); curdelay = srcu_get_delay(ssp); raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); @@ -2058,8 +2434,10 @@ void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf) } pr_cont(" T(%ld,%ld)\n", s0, s1); } - if (SRCU_SIZING_IS_TORTURE()) - srcu_transition_to_big(ssp); + if (SRCU_SIZING_IS_TORTURE()) { + if (!WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) + srcu_transition_to_big(ssp); + } } EXPORT_SYMBOL_GPL(srcu_torture_stats_print); @@ -2135,6 +2513,12 @@ static void srcu_module_going(struct module *mod) struct srcu_struct *ssp; struct srcu_struct **sspp = mod->srcu_struct_ptrs; + /* + * Deferral skips check_init_srcu_struct(), so cleanup_srcu_struct() + * below can be skipped for an srcu_struct that has staged callbacks. + * Drain them before any ->sda is freed. + */ + srcu_defer_flush(); for (i = 0; i < mod->num_srcu_structs; i++) { ssp = *(sspp++); if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed)) && diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 627295396cd9..fcac7361ec51 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -178,6 +178,8 @@ static int rcu_task_contend_lim __read_mostly = 100; module_param(rcu_task_contend_lim, int, 0444); static int rcu_task_collapse_lim __read_mostly = 10; module_param(rcu_task_collapse_lim, int, 0444); +static bool rcu_task_collapse_debug __read_mostly; +module_param(rcu_task_collapse_debug, bool, 0644); static int rcu_task_lazy_lim __read_mostly = 32; module_param(rcu_task_lazy_lim, int, 0444); @@ -390,7 +392,8 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, WRITE_ONCE(rtp->percpu_enqueue_shift, 0); WRITE_ONCE(rtp->percpu_dequeue_lim, rcu_task_cpu_ids); smp_store_release(&rtp->percpu_enqueue_lim, rcu_task_cpu_ids); - pr_info("Switching %s to per-CPU callback queuing.\n", rtp->name); + if (data_race(rcu_task_collapse_debug)) + pr_info("Switching %s to per-CPU callback queuing.\n", rtp->name); } raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); } @@ -511,7 +514,9 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp) smp_store_release(&rtp->percpu_enqueue_lim, 1); rtp->percpu_dequeue_gpseq = get_state_synchronize_rcu(); gpdone = false; - pr_info("Starting switch %s to CPU-0 callback queuing.\n", rtp->name); + if (data_race(rcu_task_collapse_debug)) + pr_info("Starting switch %s to CPU-0 callback queuing.\n", + rtp->name); } raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); } @@ -519,7 +524,9 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp) raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags); if (rtp->percpu_enqueue_lim < rtp->percpu_dequeue_lim) { WRITE_ONCE(rtp->percpu_dequeue_lim, 1); - pr_info("Completing switch %s to CPU-0 callback queuing.\n", rtp->name); + if (data_race(rcu_task_collapse_debug)) + pr_info("Completing switch %s to CPU-0 callback queuing.\n", + rtp->name); } if (rtp->percpu_dequeue_lim == 1) { for (cpu = rtp->percpu_dequeue_lim; cpu < rcu_task_cpu_ids; cpu++) { @@ -704,6 +711,8 @@ static void __init rcu_tasks_bootup_oddness(void) pr_info("\tTasks-RCU CPU stall info multiplier clamped to %d (rcu_task_stall_info_mult).\n", rtsimc); rcu_task_stall_info_mult = rtsimc; } + if (rcu_task_collapse_debug) + pr_info("\tTasks-RCU callback contend/collapse debug enabled.\n"); #endif /* #ifdef CONFIG_TASKS_RCU */ #ifdef CONFIG_TASKS_RCU pr_info("\tTrampoline variant of Tasks RCU enabled.\n"); diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c index dccccd6be941..656b6a682e31 100644 --- a/kernel/rcu/tiny.c +++ b/kernel/rcu/tiny.c @@ -11,6 +11,8 @@ */ #include <linux/completion.h> #include <linux/interrupt.h> +#include <linux/irq_work.h> +#include <linux/llist.h> #include <linux/notifier.h> #include <linux/rcupdate_wait.h> #include <linux/kernel.h> @@ -42,8 +44,100 @@ static struct rcu_ctrlblk rcu_ctrlblk = { .gp_seq = 0 - 300UL, }; +/* + * The callback list is only accessed with interrupts disabled, so a call_rcu() + * that arrives with interrupts off stages the callback on a lockless list that + * an irq_work re-issues later. One global list and irq_work suffice, as Tiny + * RCU is uniprocessor. + */ +static void rcu_defer_drain(struct irq_work *iw); +static LLIST_HEAD(rcu_defer_list); +static struct irq_work rcu_defer_iw = IRQ_WORK_INIT_HARD(rcu_defer_drain); +static bool rcu_defer_draining; + +/* + * Also called by __rcu_defer_drain() to re-issue a deferred callback, so it + * must not re-check the deferral condition. + */ +static void rcu_do_enqueue(struct rcu_head *head, rcu_callback_t func) +{ + static atomic_t doublefrees; + unsigned long flags; + + if (debug_rcu_head_queue(head)) { + if (atomic_inc_return(&doublefrees) < 4) { + pr_err("%s(): Double-freed CB %p->%pS()!!! ", __func__, head, head->func); + mem_dump_obj(head); + } + return; + } + + head->func = func; + head->next = NULL; + + local_irq_save(flags); + *rcu_ctrlblk.curtail = head; + rcu_ctrlblk.curtail = &head->next; + local_irq_restore(flags); +} + +/* Force scheduling for rcu_qs() when enqueuing from the idle task. */ +static void rcu_resched_if_idle(void) +{ + if (unlikely(is_idle_task(current))) + resched_cpu(0); +} + +static void __rcu_defer_drain(void) +{ + struct llist_node *node, *next; + bool drained = false; + unsigned long flags; + + if (!IS_ENABLED(CONFIG_RCU_DEFER)) + return; + + /* Re-issued newest-first; nothing depends on call_rcu() ordering. */ + local_irq_save(flags); + llist_for_each_safe(node, next, llist_del_all(&rcu_defer_list)) { + struct rcu_head *head = (struct rcu_head *)node; + + /* Bounds a node self-linked by a double call_rcu(). */ + head->next = NULL; + rcu_do_enqueue(head, head->func); + drained = true; + } + local_irq_restore(flags); + + if (drained) + rcu_resched_if_idle(); +} + +/* Only the irq_work drain can be re-fed by its own re-issue; see Tree RCU. */ +static void rcu_defer_drain(struct irq_work *iw) +{ + WRITE_ONCE(rcu_defer_draining, true); + __rcu_defer_drain(); + WRITE_ONCE(rcu_defer_draining, false); +} + +static void call_rcu_defer(struct rcu_head *head, rcu_callback_t func) +{ + /* A re-entrant call_rcu() during the drain would livelock it; drop it. */ + if (READ_ONCE(rcu_defer_draining) && !in_nmi()) { + WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU), + "call_rcu() re-entered during callback drain; leaking callback\n"); + return; + } + head->func = func; + if (llist_add((struct llist_node *)head, &rcu_defer_list)) + irq_work_queue(&rcu_defer_iw); +} + void rcu_barrier(void) { + /* Register any deferred callbacks so the wait below covers them. */ + __rcu_defer_drain(); wait_rcu_gp(call_rcu_hurry); } EXPORT_SYMBOL(rcu_barrier); @@ -157,29 +251,19 @@ EXPORT_SYMBOL_GPL(synchronize_rcu); */ void call_rcu(struct rcu_head *head, rcu_callback_t func) { - static atomic_t doublefrees; - unsigned long flags; - - if (debug_rcu_head_queue(head)) { - if (atomic_inc_return(&doublefrees) < 4) { - pr_err("%s(): Double-freed CB %p->%pS()!!! ", __func__, head, head->func); - mem_dump_obj(head); - } + if (should_rcu_defer()) { + call_rcu_defer(head, func); return; } - head->func = func; - head->next = NULL; + /* + * Only reachable from an NMI when deferral is off: before the scheduler + * is up, or with CONFIG_RCU_DEFER=n. The enqueue can then race. + */ + WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi()); - local_irq_save(flags); - *rcu_ctrlblk.curtail = head; - rcu_ctrlblk.curtail = &head->next; - local_irq_restore(flags); - - if (unlikely(is_idle_task(current))) { - /* force scheduling for rcu_qs() */ - resched_cpu(0); - } + rcu_do_enqueue(head, func); + rcu_resched_if_idle(); } EXPORT_SYMBOL_GPL(call_rcu); @@ -211,10 +295,7 @@ unsigned long start_poll_synchronize_rcu(void) { unsigned long gp_seq = get_state_synchronize_rcu(); - if (unlikely(is_idle_task(current))) { - /* force scheduling for rcu_qs() */ - resched_cpu(0); - } + rcu_resched_if_idle(); return gp_seq; } EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 96848fc1f02b..aea3aef7397e 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -24,6 +24,7 @@ #include <linux/smp.h> #include <linux/rcupdate_wait.h> #include <linux/interrupt.h> +#include <linux/llist.h> #include <linux/sched.h> #include <linux/sched/debug.h> #include <linux/nmi.h> @@ -64,7 +65,6 @@ #include <linux/mm.h> #include <linux/kasan.h> #include <linux/context_tracking.h> -#include "../time/tick-internal.h" #include "tree.h" #include "rcu.h" @@ -3148,21 +3148,19 @@ static void check_cb_ovld(struct rcu_data *rdp) raw_spin_unlock_rcu_node(rnp); } -static void -__call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in) +/* + * Also called by __rcu_defer_drain() to re-issue a deferred callback, so it + * must not re-check the deferral condition. Either caller may have interrupts + * already disabled, and a drain of a remote CPU re-issues onto the draining + * CPU. + */ +static void rcu_do_enqueue(struct rcu_head *head, rcu_callback_t func, bool lazy_in) { static atomic_t doublefrees; unsigned long flags; bool lazy; struct rcu_data *rdp; - /* Misaligned rcu_head! */ - WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1)); - - /* Avoid NULL dereference if callback is NULL. */ - if (WARN_ON_ONCE(!func)) - return; - if (debug_rcu_head_queue(head)) { /* * Probable double call_rcu(), so leak the callback. @@ -3178,7 +3176,6 @@ __call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in) } head->func = func; head->next = NULL; - kasan_record_aux_stack(head); local_irq_save(flags); rdp = this_cpu_ptr(&rcu_data); @@ -3206,6 +3203,103 @@ __call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in) local_irq_restore(flags); } +/* + * Re-issue deferred callbacks straight to the enqueue so they cannot defer + * again. ->defer_lock serializes the drainers: this CPU's irq_work, + * rcu_defer_flush() and rcutree_migrate_callbacks(). + */ +static void __rcu_defer_drain(struct rcu_data *rdp) +{ + struct llist_node *node, *next; + unsigned long flags; + + if (!IS_ENABLED(CONFIG_RCU_DEFER)) + return; + + raw_spin_lock_irqsave(&rdp->defer_lock, flags); + llist_for_each_safe(node, next, llist_del_all(&rdp->defer_head)) { + struct rcu_head *head = (struct rcu_head *)node; + + /* Bounds a node self-linked by a double call_rcu(). */ + head->next = NULL; + rcu_do_enqueue(head, head->func, false); + } + raw_spin_unlock_irqrestore(&rdp->defer_lock, flags); +} + +/* + * Only the irq_work drain can be re-fed by its own re-issue, so only it sets + * ->defer_draining. Anything staged during a direct drain is picked up by the + * staging CPU's own irq_work. Every caller of irq_work_run_list() has + * interrupts disabled, so the flag is never visible with them enabled. + */ +static void rcu_defer_drain(struct irq_work *iw) +{ + struct rcu_data *rdp = container_of(iw, struct rcu_data, defer_work); + + WRITE_ONCE(rdp->defer_draining, true); + __rcu_defer_drain(rdp); + WRITE_ONCE(rdp->defer_draining, false); +} + +/* + * Stage @head for this CPU's irq_work to re-issue once interrupts are on. Only + * the drain side takes a lock, so this stays safe from NMI. + */ +static void call_rcu_defer(struct rcu_head *head, rcu_callback_t func) +{ + struct rcu_data *rdp = this_cpu_ptr(&rcu_data); + + /* + * Instrumentation on the enqueue path can re-enter here from inside the + * drain. Re-queuing would livelock it, so drop the callback; an NMI + * cannot loop, so let it through. + */ + if (READ_ONCE(rdp->defer_draining) && !in_nmi()) { + WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU), + "call_rcu() re-entered during callback drain; leaking callback\n"); + return; + } + head->func = func; + if (llist_add((struct llist_node *)head, &rdp->defer_head)) + irq_work_queue(&rdp->defer_work); +} + +static void rcu_defer_flush(void) +{ + int cpu; + + for_each_possible_cpu(cpu) + __rcu_defer_drain(per_cpu_ptr(&rcu_data, cpu)); +} + +static void +__call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in) +{ + /* Misaligned rcu_head! */ + WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1)); + + /* Avoid NULL dereference if callback is NULL. */ + if (WARN_ON_ONCE(!func)) + return; + + /* Record the caller: the irq_work's stack says nothing about it. */ + kasan_record_aux_stack(head); + + if (should_rcu_defer()) { + call_rcu_defer(head, func); + return; + } + + /* + * Only reachable from an NMI when deferral is off: before the scheduler + * is up, or with CONFIG_RCU_DEFER=n. The enqueue can then race. + */ + WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi()); + + rcu_do_enqueue(head, func, lazy_in); +} + #ifdef CONFIG_RCU_LAZY static bool enable_rcu_lazy __read_mostly = !IS_ENABLED(CONFIG_RCU_LAZY_DEFAULT_OFF); module_param(enable_rcu_lazy, bool, 0444); @@ -3896,8 +3990,12 @@ void rcu_barrier(void) unsigned long flags; unsigned long gseq; struct rcu_data *rdp; - unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence); + unsigned long s; + /* Register any deferred callbacks before snapshotting the sequence. */ + rcu_defer_flush(); + + s = rcu_seq_snap(&rcu_state.barrier_sequence); rcu_barrier_trace(TPS("Begin"), -1, s); /* Take mutex to serialize concurrent rcu_barrier() requests. */ @@ -3989,12 +4087,12 @@ EXPORT_SYMBOL_GPL(rcu_barrier); static unsigned long rcu_barrier_last_throttle; /** - * rcu_barrier_throttled - Do rcu_barrier(), but limit to one per second + * rcu_barrier_throttled - Drain deferred RCU frees, but rate-limit starts * - * This can be thought of as guard rails around rcu_barrier() that - * permits unrestricted userspace use, at least assuming the hardware's - * try_cmpxchg() is robust. There will be at most one call per second to - * rcu_barrier() system-wide from use of this function, which means that + * This can be thought of as guard rails around the deferred-free barriers + * that permit unrestricted userspace use, at least assuming the hardware's + * try_cmpxchg() is robust. There will be at most one drain operation started + * per sixteenth of a second from use of this function, which means that * callers might needlessly wait a second or three. * * This is intended for use by test suites to avoid OOM by flushing RCU @@ -4016,14 +4114,24 @@ static void rcu_barrier_throttled(void) while (time_in_range(j, old, old + HZ / 16) || !try_cmpxchg(&rcu_barrier_last_throttle, &old, j)) { schedule_timeout_idle(HZ / 16); - if (rcu_seq_done(&rcu_state.barrier_sequence, s)) { - smp_mb(); /* caller's subsequent code after above check. */ - return; - } j = jiffies; old = READ_ONCE(rcu_barrier_last_throttle); } - rcu_barrier(); + /* + * kfree_rcu() can retain objects outside the ordinary callback lists in + * per-CPU SLUB sheaves and kvfree_rcu batches. Always drain those queues: + * an ordinary barrier does not establish that this work was drained. + */ + kvfree_rcu_barrier(); + /* + * A completed barrier can still cover ordinary callbacks queued before + * our entry snapshot. Otherwise, retain an explicit ordinary barrier + * without depending on the implementation of kvfree_rcu_barrier(). + */ + if (rcu_seq_done(&rcu_state.barrier_sequence, s)) + smp_mb(); /* caller's subsequent code after above check. */ + else + rcu_barrier(); } /* @@ -4231,6 +4339,9 @@ rcu_boot_init_percpu_data(int cpu) rdp->rcu_onl_gp_state = RCU_GP_CLEANED; rdp->last_sched_clock = jiffies; rdp->cpu = cpu; + init_llist_head(&rdp->defer_head); + raw_spin_lock_init(&rdp->defer_lock); + rdp->defer_work = IRQ_WORK_INIT_HARD(rcu_defer_drain); rcu_boot_init_nocb_percpu_data(rdp); } @@ -4528,6 +4639,15 @@ void rcutree_migrate_callbacks(int cpu) struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); bool needwake; + /* + * Callbacks deferred past the point the outgoing CPU's irq_work can run + * sit on ->defer_head, which the ->cblist migration below does not + * cover. Drain them here, before the early returns. + */ + __rcu_defer_drain(rdp); + /* Likewise for the outgoing CPU's deferred call_srcu() callbacks. */ + srcu_offline_drain(cpu); + if (rcu_rdp_is_offloaded(rdp)) return; diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index eedfa43059e8..b7cac7a13b4f 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -229,6 +229,12 @@ struct rcu_data { struct rcu_head barrier_head; int exp_watching_snap; /* Double-check need for IPI. */ + /* Deferral of an NMI/reentrant call_rcu(); see __call_rcu_common(). */ + struct llist_head defer_head; + struct irq_work defer_work; + raw_spinlock_t defer_lock; + bool defer_draining; + /* 5) Callback offloading. */ #ifdef CONFIG_RCU_NOCB_CPU struct swait_queue_head nocb_cb_wq; /* For nocb kthreads to sleep on. */ diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 81d42b6e2e31..01bdbd7e28cc 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -678,7 +678,7 @@ static void synchronize_rcu_expedited_wait(void) nbcon_cpu_emergency_exit(); - panic_on_rcu_stall(&rcu_exp_stall_cpumask); + panic_on_rcu_stall(&rcu_exp_stall_cpumask, NULL, 0); } } diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 19bb42672baf..b6cfee4e5039 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -1332,9 +1332,8 @@ lazy_rcu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) rcu_nocb_try_flush_bypass(rdp, jiffies); rcu_nocb_unlock_irqrestore(rdp, flags); wake_nocb_gp(rdp); - sc->nr_to_scan -= _count; count += _count; - if (sc->nr_to_scan <= 0) + if (count >= sc->nr_to_scan) break; } diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 743c16247fc0..ae5d21bae7bc 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -11,8 +11,6 @@ * Paul E. McKenney <paulmck@linux.ibm.com> */ -#include "../locking/rtmutex_common.h" - static bool rcu_rdp_is_offloaded(struct rcu_data *rdp) { /* diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index 091e7850ab6e..803a56af3259 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -11,6 +11,8 @@ #include <linux/kvm_para.h> #include <linux/rcu_notifier.h> #include <linux/smp.h> +#include <linux/rtmutex.h> +#include "../locking/rtmutex_common.h" ////////////////////////////////////////////////////////////////////////////// // @@ -159,7 +161,8 @@ static int __init check_cpu_stall_init(void) early_initcall(check_cpu_stall_init); /* If so specified via sysctl, panic, yielding cleaner stall-warning output. */ -static void panic_on_rcu_stall(const struct cpumask *stalled_mask) +static void panic_on_rcu_stall(const struct cpumask *stalled_mask, + unsigned long *gsp, unsigned long gp_seq) { static int cpu_stall; @@ -170,6 +173,11 @@ static void panic_on_rcu_stall(const struct cpumask *stalled_mask) if (scx_rcu_cpu_stall(stalled_mask)) return; + if (gsp && rcu_seq_current(gsp) != gp_seq) { + pr_err("INFO: Stall ended before panic check.\n"); + return; + } + if (++cpu_stall < sysctl_max_rcu_stall_to_panic) return; @@ -299,6 +307,8 @@ struct rcu_stall_chk_rdr { int nesting; union rcu_special rs; bool on_blkd_list; + bool rcu_rdr_running; + int rcu_rdr_boosted; }; /* @@ -307,6 +317,7 @@ struct rcu_stall_chk_rdr { */ static int check_slow_task(struct task_struct *t, void *arg) { + struct rcu_node *rnp; struct rcu_stall_chk_rdr *rscrp = arg; if (task_curr(t)) @@ -314,6 +325,19 @@ static int check_slow_task(struct task_struct *t, void *arg) rscrp->nesting = t->rcu_read_lock_nesting; rscrp->rs = t->rcu_read_unlock_special; rscrp->on_blkd_list = !list_empty(&t->rcu_node_entry); + rscrp->rcu_rdr_running = task_curr(t); + rscrp->rcu_rdr_boosted = 0; + if (rscrp->on_blkd_list) { + rnp = READ_ONCE(t->rcu_blocked_node); + raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ + if (rnp == READ_ONCE(t->rcu_blocked_node)) { + if (rt_mutex_owner(&rnp->boost_mtx.rtmutex) == t) + rscrp->rcu_rdr_boosted = 1; + } else { + rscrp->rcu_rdr_boosted = 2; + } + raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ + } return 0; } @@ -351,12 +375,14 @@ static int rcu_print_task_stall(struct rcu_node *rnp, unsigned long flags) if (task_call_func(t, check_slow_task, &rscr)) pr_cont(" P%d", t->pid); else - pr_cont(" P%d/%d:%c%c%c%c", + pr_cont(" P%d/%d:%c%c%c%c%c%c", t->pid, rscr.nesting, ".b"[rscr.rs.b.blocked], ".q"[rscr.rs.b.need_qs], ".e"[rscr.rs.b.exp_hint], - ".l"[rscr.on_blkd_list]); + ".l"[rscr.on_blkd_list], + ".R"[rscr.rcu_rdr_running], + ".B?"[rscr.rcu_rdr_boosted]); lockdep_assert_irqs_disabled(); put_task_struct(t); ndetected++; @@ -703,7 +729,7 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps) nbcon_cpu_emergency_exit(); - panic_on_rcu_stall(&rcu_stall_cpumask); + panic_on_rcu_stall(&rcu_stall_cpumask, &rcu_state.gp_seq, gp_seq); rcu_force_quiescent_state(); /* Kick them all. */ } @@ -758,7 +784,7 @@ static void print_cpu_stall(unsigned long gp_seq, unsigned long gps) cpumask_clear(&rcu_stall_cpumask); cpumask_set_cpu(smp_processor_id(), &rcu_stall_cpumask); - panic_on_rcu_stall(&rcu_stall_cpumask); + panic_on_rcu_stall(&rcu_stall_cpumask, &rcu_state.gp_seq, gp_seq); /* * Attempt to revive the RCU machinery by forcing a context switch. diff --git a/kernel/reboot.c b/kernel/reboot.c index d177d89fcc33..a93c23b9e194 100644 --- a/kernel/reboot.c +++ b/kernel/reboot.c @@ -8,10 +8,10 @@ #define pr_fmt(fmt) "reboot: " fmt #include <linux/atomic.h> +#include <linux/console.h> #include <linux/ctype.h> #include <linux/export.h> #include <linux/kexec.h> -#include <linux/kmod.h> #include <linux/kmsg_dump.h> #include <linux/rcupdate.h> #include <linux/reboot.h> @@ -20,6 +20,7 @@ #include <linux/syscalls.h> #include <linux/syscore_ops.h> #include <linux/uaccess.h> +#include <linux/umh.h> /* * this indicates whether you can reboot with ctrl-alt-del: the default is yes @@ -94,6 +95,14 @@ void emergency_restart(void) { kmsg_dump(KMSG_DUMP_EMERG); system_state = SYSTEM_RESTART; + + /* + * NBCON kthreads won't be flushed and stopped properly in + * the emergency restart. Move them to the atomic mode. + */ + nbcon_cpu_emergency_enter(); + printk_trigger_flush(); + machine_emergency_restart(); } EXPORT_SYMBOL_GPL(emergency_restart); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 0b846a13c628..0ab15f03cb94 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -805,7 +805,7 @@ struct rq *_task_rq_lock(struct task_struct *p, struct rq_flags *rf) /* Use CONFIG_PARAVIRT as this will avoid more #ifdef in arch code. */ #ifdef CONFIG_PARAVIRT -struct static_key paravirt_steal_rq_enabled; +DEFINE_STATIC_KEY_FALSE(paravirt_steal_rq_enabled); #endif static void update_rq_clock_task(struct rq *rq, s64 delta) @@ -844,7 +844,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta) } #endif #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING - if (static_key_false((¶virt_steal_rq_enabled))) { + if (static_branch_unlikely(¶virt_steal_rq_enabled)) { u64 prev_steal; steal = prev_steal = paravirt_steal_clock(cpu_of(rq)); @@ -1196,6 +1196,7 @@ static void __resched_curr(struct rq *rq, int tif) { struct task_struct *curr = rq->curr; struct thread_info *cti = task_thread_info(curr); + bool need_ipi; int cpu; lockdep_assert_rq_held(rq); @@ -1212,15 +1213,17 @@ static void __resched_curr(struct rq *rq, int tif) cpu = cpu_of(rq); - trace_sched_set_need_resched_tp(curr, cpu, tif); if (cpu == smp_processor_id()) { set_ti_thread_flag(cti, tif); if (tif == TIF_NEED_RESCHED) set_preempt_need_resched(); + trace_sched_set_need_resched_tp(curr, cpu, tif); return; } - if (set_nr_and_not_polling(cti, tif)) { + need_ipi = set_nr_and_not_polling(cti, tif); + trace_sched_set_need_resched_tp(curr, cpu, tif); + if (need_ipi) { if (tif == TIF_NEED_RESCHED) smp_send_reschedule(cpu); } else { @@ -2252,7 +2255,8 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) dequeue_task(rq, p, flags); } -static void block_task(struct rq *rq, struct task_struct *p, unsigned long task_state) +static bool dequeue_block_task(struct rq *rq, struct task_struct *p, + unsigned long task_state) { int flags = DEQUEUE_NOCLOCK; @@ -2273,9 +2277,15 @@ static void block_task(struct rq *rq, struct task_struct *p, unsigned long task_ * * Where __schedule() and ttwu() have matching control dependencies. * - * After this, schedule() must not care about p->state any more. + * Once the caller invokes __block_task(), schedule() must not care about + * p->state any more. */ - if (dequeue_task(rq, p, DEQUEUE_SLEEP | flags)) + return dequeue_task(rq, p, DEQUEUE_SLEEP | flags); +} + +static void block_task(struct rq *rq, struct task_struct *p, unsigned long task_state) +{ + if (dequeue_block_task(rq, p, task_state)) __block_task(rq, p); } @@ -2504,6 +2514,24 @@ static inline bool rq_has_pinned_tasks(struct rq *rq) return rq->nr_pinned; } +static inline bool task_can_migrate_to_preferred(struct task_struct *p, int cpu) +{ + /* No need to migrate from a preferred CPU */ + if (cpu_preferred(cpu)) + return false; + + /* Only FAIR tasks honor preferred CPU state */ + if (unlikely(p->sched_class != &fair_sched_class)) + return false; + + /* Ignore preferred state if task affinity is changing */ + if (unlikely(!cpumask_test_cpu(task_cpu(p), p->cpus_ptr))) + return false; + + return cpumask_intersects_and(p->cpus_ptr, cpu_preferred_mask, + task_cpu_possible_mask(p)); +} + /* * Per-CPU kthreads are allowed to run on !active && online CPUs, see * __set_cpus_allowed_ptr() and select_fallback_rq(). @@ -2519,8 +2547,12 @@ static inline bool is_cpu_allowed(struct task_struct *p, int cpu) return cpu_online(cpu); /* Non kernel threads are not allowed during either online or offline. */ - if (!(p->flags & PF_KTHREAD)) + if (!(p->flags & PF_KTHREAD)) { + /* Try to use preferred CPU if task's affinity allows */ + if (task_can_migrate_to_preferred(p, cpu)) + return false; return cpu_active(cpu); + } /* KTHREAD_IS_PER_CPU is always allowed. */ if (kthread_is_per_cpu(p)) @@ -2530,7 +2562,11 @@ static inline bool is_cpu_allowed(struct task_struct *p, int cpu) if (cpu_dying(cpu)) return false; - /* But are allowed during online. */ + /* Try to keep unbound kthreads on a preferred CPU if possible. */ + if (task_can_migrate_to_preferred(p, cpu)) + return false; + + /* Otherwise, they are allowed to run on online CPU. */ return cpu_online(cpu); } @@ -3773,6 +3809,7 @@ static inline void proxy_reset_donor(struct rq *rq) WARN_ON_ONCE(rq->donor == rq->curr); put_prev_set_next_task(rq, rq->donor, rq->curr); + rq->next_class = rq->curr->sched_class; rq_set_donor(rq, rq->curr); zap_balance_callbacks(rq); resched_curr(rq); @@ -3787,6 +3824,8 @@ static inline void proxy_reset_donor(struct rq *rq) */ static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p) { + bool dequeued; + /* * Typically per __set_task_cpu(), task_cpu(p) == p->wake_cpu. * @@ -3809,12 +3848,23 @@ static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p) /* If already current, don't need to return migrate */ if (task_current(rq, p)) return false; - - /* If we're return migrating the rq->donor, switch it out for idle */ - if (task_current_donor(rq, p)) - proxy_reset_donor(rq); } - block_task(rq, p, TASK_WAKING); + + dequeued = dequeue_block_task(rq, p, TASK_WAKING); + + /* + * Dequeue @p from its scheduling class before resetting rq->donor. + * In particular, sched_ext needs to end the donor's running session + * and clear SCX_TASK_QUEUED before put_prev_task_scx() is called by + * proxy_reset_donor(); otherwise it would reenqueue the blocked donor. + * + * Keep on_rq set until all donor references have been replaced. + */ + if (task_current_donor(rq, p)) + proxy_reset_donor(rq); + + if (dequeued) + __block_task(rq, p); return true; } #else /* !CONFIG_SCHED_PROXY_EXEC */ @@ -3905,7 +3955,7 @@ static int ttwu_runnable(struct task_struct *p, int wake_flags) * When on_rq && !on_cpu the task is preempted, see if * it should preempt the task that is current now. */ - wakeup_preempt(rq, p, wake_flags); + wakeup_preempt(rq, p, wake_flags | WF_TTWU_RQ); } ttwu_do_wakeup(p); return 1; @@ -5149,7 +5199,7 @@ static void do_balance_callbacks(struct rq *rq, struct balance_callback *head) lockdep_assert_rq_held(rq); while (head) { - func = (void (*)(struct rq *))head->func; + func = head->func; next = head->next; head->next = NULL; head = next; @@ -5588,15 +5638,12 @@ unsigned long long nr_context_switches(void) } /* - * Consumers of these two interfaces, like for example the cpuidle menu - * governor, are using nonsensical data. Preferring shallow idle state selection - * for a CPU that has IO-wait which might not even end up running the task when - * it does become runnable. + * Return whether any tasks are accounted as I/O waiters on this CPU. + * They may wake up on a different CPU. */ - -unsigned int nr_iowait_cpu(int cpu) +bool cpu_has_iowait(int cpu) { - return atomic_read(&cpu_rq(cpu)->nr_iowait); + return atomic_read(&cpu_rq(cpu)->nr_iowait) != 0; } /* @@ -5634,7 +5681,7 @@ unsigned int nr_iowait(void) unsigned int i, sum = 0; for_each_possible_cpu(i) - sum += nr_iowait_cpu(i); + sum += atomic_read(&cpu_rq(i)->nr_iowait); return sum; } @@ -5789,6 +5836,9 @@ void sched_tick(void) unsigned long hw_pressure; u64 resched_latency; + if (!cpu_preferred(cpu)) + sched_push_current_non_preferred_cpu(rq); + if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE)) arch_scale_freq_tick(); @@ -5798,7 +5848,7 @@ void sched_tick(void) curr = rq->curr; donor = rq->donor; - psi_account_irqtime(rq, donor, NULL); + psi_account_irqtime(rq, curr, NULL); update_rq_clock(rq); hw_pressure = arch_scale_hw_pressure(cpu_of(rq)); @@ -6283,10 +6333,7 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) * selection. In this case, do a core-wide selection. */ if (rq->core->core_pick_seq == rq->core->core_task_seq && - rq->core->core_pick_seq != rq->core_sched_seq && rq->core_pick) { - WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq); - next = rq->core_pick; rq->dl_server = rq->core_dl_server; rq->core_pick = NULL; @@ -6318,11 +6365,13 @@ restart: } /* - * core->core_task_seq, core->core_pick_seq, rq->core_sched_seq + * core->core_task_seq, core->core_pick_seq * * @task_seq guards the task state ({en,de}queues) * @pick_seq is the @task_seq we did a selection on - * @sched_seq is the @pick_seq we scheduled + * + * Once a core-wide selection is committed, a non-NULL core_pick denotes + * a pick which still needs to be consumed on this CPU. * * However, preemptions can cause multiple picks on the same task set. * 'Fix' this by also increasing @task_seq for every pick. @@ -6429,7 +6478,6 @@ restart: rq->core->core_pick_seq = rq->core->core_task_seq; next = rq->core_pick; - rq->core_sched_seq = rq->core->core_pick_seq; /* Something should have been selected for current CPU */ WARN_ON_ONCE(!next); @@ -6517,7 +6565,10 @@ static bool try_steal_cookie(int this, int that) return false; do { - if (p == src->core_pick || p == src->curr) + if (p == src->core_pick || p == src->curr || p == src->donor) + goto next; + + if (task_is_blocked(p)) goto next; if (!is_cpu_allowed(p, this)) @@ -6820,6 +6871,34 @@ static void proxy_deactivate(struct rq *rq, struct task_struct *donor) block_task(rq, donor, state); } +/* + * Remove a retained proxy donor before changing its scheduler ownership. + * The caller holds p->pi_lock, so p cannot wake and migrate if block_task() + * drops it from the runqueue. If DELAY_DEQUEUE keeps a blocked fair task + * queued, switching_from_fair() completes the dequeue in the immediately + * following sched_change_begin(). + */ +void sched_proxy_block_task(struct rq *rq, struct task_struct *p) +{ + unsigned long state = READ_ONCE(p->__state); + + lockdep_assert_held(&p->pi_lock); + lockdep_assert_rq_held(rq); + + if (!p->is_blocked || !task_on_rq_queued(p)) + return; + if (WARN_ON_ONCE(state == TASK_RUNNING)) + return; + + if (task_current_donor(rq, p)) + proxy_reset_donor(rq); + + if (!p->se.sched_delayed) + block_task(rq, p, state); + + WARN_ON_ONCE(task_on_rq_queued(p) && !p->se.sched_delayed); +} + static inline void proxy_release_rq_lock(struct rq *rq, struct rq_flags *rf) __releases(__rq_lockp(rq)) { @@ -6865,9 +6944,9 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf, __must_hold(__rq_lockp(rq)) { struct rq *target_rq = cpu_rq(target_cpu); + LIST_HEAD(migrate_list); lockdep_assert_rq_held(rq); - WARN_ON(p == rq->curr); /* * Since we are migrating a blocked donor, it could be rq->donor, * and we want to make sure there aren't any references from this @@ -6880,13 +6959,20 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf, * before we release the lock. */ proxy_resched_idle(rq); - - deactivate_task(rq, p, DEQUEUE_NOCLOCK); - proxy_set_task_cpu(p, target_cpu); - + for (; p; p = p->blocked_donor) { + WARN_ON(p == rq->curr); + deactivate_task(rq, p, DEQUEUE_NOCLOCK); + proxy_set_task_cpu(p, target_cpu); + /* + * We can re-use se.group_node to migrate the thing, + * because @p is deactivated (won't be balanced) and + * we hold the rq_lock. + */ + list_add(&p->se.group_node, &migrate_list); + } proxy_release_rq_lock(rq, rf); - attach_one_task(target_rq, p); + __attach_tasks(target_rq, &migrate_list); proxy_reacquire_rq_lock(rq, rf); } @@ -6979,7 +7065,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) if (!READ_ONCE(owner->on_rq) || owner->se.sched_delayed) { /* XXX Don't handle blocked owners/delayed dequeue yet */ if (curr_in_chain) - return proxy_resched_idle(rq); + goto resched_idle; __clear_task_blocked_on(p, NULL); goto deactivate; } @@ -6991,7 +7077,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) * and leave that CPU to sort things out. */ if (curr_in_chain) - return proxy_resched_idle(rq); + goto resched_idle; goto migrate_task; } @@ -7004,7 +7090,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) * case we should end up back in find_proxy_task(), this time * hopefully with all relevant tasks already enqueued. */ - return proxy_resched_idle(rq); + goto resched_idle; } /* @@ -7041,7 +7127,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) * So schedule rq->idle so that ttwu_runnable() can get the rq * lock and mark owner as running. */ - return proxy_resched_idle(rq); + goto resched_idle; } /* * OK, now we're absolutely sure @owner is on this @@ -7051,8 +7137,18 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) owner->blocked_donor = p; } WARN_ON_ONCE(owner && !owner->on_rq); + + if (owner && !sched_cpu_cookie_match(rq, owner)) { + if (curr_in_chain) + return proxy_resched_idle(rq); + p = donor; /* Deactivate the donor, not the runnable owner */ + clear_task_blocked_on(p, NULL); + goto deactivate; + } return owner; +resched_idle: + return proxy_resched_idle(rq); deactivate: proxy_deactivate(rq, p); return NULL; @@ -7184,13 +7280,12 @@ static void __sched notrace __schedule(int sched_mode) } } else if (!preempt && prev_state) { /* - * We pass task_is_blocked() as the should_block arg - * in order to keep mutex-blocked tasks on the runqueue - * for slection with proxy-exec (without proxy-exec - * task_is_blocked() will always be false). + * Keep mutex-blocked tasks on the runqueue for proxy execution + * only when their scheduling class allows it. Without proxy + * execution, task_is_blocked() always returns false. */ try_to_block_task(rq, prev, &prev_state, - !task_is_blocked(prev)); + !task_is_blocked(prev) || !scx_allow_proxy_exec(prev)); switch_count = &prev->nvcsw; } @@ -7211,6 +7306,7 @@ pick_again: } if (next == rq->idle) { zap_balance_callbacks(rq); + scx_proxy_reenqueue_retry(rq, next); goto keep_resched; } } @@ -7229,8 +7325,10 @@ pick_again: * on_cpu. */ donor->sched_class->put_prev_task(rq, donor, donor); - donor->sched_class->set_next_task(rq, donor, true); + donor->sched_class->set_next_task(rq, donor, SNT_PICK); } + scx_proxy_donor_start(rq); + scx_proxy_reenqueue_retry(rq, next); } else { rq_set_donor(rq, next); } @@ -7489,27 +7587,6 @@ asmlinkage __visible void __sched notrace preempt_schedule(void) NOKPROBE_SYMBOL(preempt_schedule); EXPORT_SYMBOL(preempt_schedule); -#ifdef CONFIG_PREEMPT_DYNAMIC -# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL -# ifndef preempt_schedule_dynamic_enabled -# define preempt_schedule_dynamic_enabled preempt_schedule -# define preempt_schedule_dynamic_disabled NULL -# endif -DEFINE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled); -EXPORT_STATIC_CALL_TRAMP(preempt_schedule); -# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule); -void __sched notrace dynamic_preempt_schedule(void) -{ - if (!static_branch_unlikely(&sk_dynamic_preempt_schedule)) - return; - preempt_schedule(); -} -NOKPROBE_SYMBOL(dynamic_preempt_schedule); -EXPORT_SYMBOL(dynamic_preempt_schedule); -# endif -#endif /* CONFIG_PREEMPT_DYNAMIC */ - /** * preempt_schedule_notrace - preempt_schedule called by tracing * @@ -7562,27 +7639,6 @@ asmlinkage __visible void __sched notrace preempt_schedule_notrace(void) } EXPORT_SYMBOL_GPL(preempt_schedule_notrace); -#ifdef CONFIG_PREEMPT_DYNAMIC -# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) -# ifndef preempt_schedule_notrace_dynamic_enabled -# define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace -# define preempt_schedule_notrace_dynamic_disabled NULL -# endif -DEFINE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled); -EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace); -# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule_notrace); -void __sched notrace dynamic_preempt_schedule_notrace(void) -{ - if (!static_branch_unlikely(&sk_dynamic_preempt_schedule_notrace)) - return; - preempt_schedule_notrace(); -} -NOKPROBE_SYMBOL(dynamic_preempt_schedule_notrace); -EXPORT_SYMBOL(dynamic_preempt_schedule_notrace); -# endif -#endif - #endif /* CONFIG_PREEMPTION */ /* @@ -7799,7 +7855,7 @@ out_unlock: } #endif /* CONFIG_RT_MUTEXES */ -#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) +#if !defined(CONFIG_PREEMPTION) int __sched __cond_resched(void) { if (should_resched(0) && !irqs_disabled()) { @@ -7827,38 +7883,6 @@ int __sched __cond_resched(void) EXPORT_SYMBOL(__cond_resched); #endif -#ifdef CONFIG_PREEMPT_DYNAMIC -# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL -# define cond_resched_dynamic_enabled __cond_resched -# define cond_resched_dynamic_disabled ((void *)&__static_call_return0) -DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched); -EXPORT_STATIC_CALL_TRAMP(cond_resched); - -# define might_resched_dynamic_enabled __cond_resched -# define might_resched_dynamic_disabled ((void *)&__static_call_return0) -DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched); -EXPORT_STATIC_CALL_TRAMP(might_resched); -# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched); -int __sched dynamic_cond_resched(void) -{ - if (!static_branch_unlikely(&sk_dynamic_cond_resched)) - return 0; - return __cond_resched(); -} -EXPORT_SYMBOL(dynamic_cond_resched); - -static DEFINE_STATIC_KEY_FALSE(sk_dynamic_might_resched); -int __sched dynamic_might_resched(void) -{ - if (!static_branch_unlikely(&sk_dynamic_might_resched)) - return 0; - return __cond_resched(); -} -EXPORT_SYMBOL(dynamic_might_resched); -# endif -#endif /* CONFIG_PREEMPT_DYNAMIC */ - /* * __cond_resched_lock() - if a reschedule is pending, drop the given lock, * call schedule, and on return reacquire the lock. @@ -7928,50 +7952,21 @@ EXPORT_SYMBOL(__cond_resched_rwlock_write); # endif /* - * SC:cond_resched - * SC:might_resched - * SC:preempt_schedule - * SC:preempt_schedule_notrace - * SC:irqentry_exit_cond_resched - * - * * NONE: - * cond_resched <- __cond_resched - * might_resched <- RET0 - * preempt_schedule <- NOP - * preempt_schedule_notrace <- NOP - * irqentry_exit_cond_resched <- NOP - * dynamic_preempt_lazy <- false + * (unselectable) * * VOLUNTARY: - * cond_resched <- __cond_resched - * might_resched <- __cond_resched - * preempt_schedule <- NOP - * preempt_schedule_notrace <- NOP - * irqentry_exit_cond_resched <- NOP - * dynamic_preempt_lazy <- false + * (unselectable) * * FULL: - * cond_resched <- RET0 - * might_resched <- RET0 - * preempt_schedule <- preempt_schedule - * preempt_schedule_notrace <- preempt_schedule_notrace - * irqentry_exit_cond_resched <- irqentry_exit_cond_resched * dynamic_preempt_lazy <- false * * LAZY: - * cond_resched <- RET0 - * might_resched <- RET0 - * preempt_schedule <- preempt_schedule - * preempt_schedule_notrace <- preempt_schedule_notrace - * irqentry_exit_cond_resched <- irqentry_exit_cond_resched * dynamic_preempt_lazy <- true */ enum { preempt_dynamic_undefined = -1, - preempt_dynamic_none, - preempt_dynamic_voluntary, preempt_dynamic_full, preempt_dynamic_lazy, }; @@ -7980,21 +7975,11 @@ int preempt_dynamic_mode = preempt_dynamic_undefined; int sched_dynamic_mode(const char *str) { -# if !(defined(CONFIG_PREEMPT_RT) || defined(CONFIG_ARCH_HAS_PREEMPT_LAZY)) - if (!strcmp(str, "none")) - return preempt_dynamic_none; - - if (!strcmp(str, "voluntary")) - return preempt_dynamic_voluntary; -# endif - if (!strcmp(str, "full")) return preempt_dynamic_full; -# ifdef CONFIG_ARCH_HAS_PREEMPT_LAZY if (!strcmp(str, "lazy")) return preempt_dynamic_lazy; -# endif return -EINVAL; } @@ -8002,71 +7987,18 @@ int sched_dynamic_mode(const char *str) # define preempt_dynamic_key_enable(f) static_key_enable(&sk_dynamic_##f.key) # define preempt_dynamic_key_disable(f) static_key_disable(&sk_dynamic_##f.key) -# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) -# define preempt_dynamic_enable(f) static_call_update(f, f##_dynamic_enabled) -# define preempt_dynamic_disable(f) static_call_update(f, f##_dynamic_disabled) -# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -# define preempt_dynamic_enable(f) preempt_dynamic_key_enable(f) -# define preempt_dynamic_disable(f) preempt_dynamic_key_disable(f) -# else -# error "Unsupported PREEMPT_DYNAMIC mechanism" -# endif - static DEFINE_MUTEX(sched_dynamic_mutex); static void __sched_dynamic_update(int mode) { - /* - * Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in - * the ZERO state, which is invalid. - */ - preempt_dynamic_enable(cond_resched); - preempt_dynamic_enable(might_resched); - preempt_dynamic_enable(preempt_schedule); - preempt_dynamic_enable(preempt_schedule_notrace); - preempt_dynamic_enable(irqentry_exit_cond_resched); - preempt_dynamic_key_disable(preempt_lazy); - switch (mode) { - case preempt_dynamic_none: - preempt_dynamic_enable(cond_resched); - preempt_dynamic_disable(might_resched); - preempt_dynamic_disable(preempt_schedule); - preempt_dynamic_disable(preempt_schedule_notrace); - preempt_dynamic_disable(irqentry_exit_cond_resched); - preempt_dynamic_key_disable(preempt_lazy); - if (mode != preempt_dynamic_mode) - pr_info("Dynamic Preempt: none\n"); - break; - - case preempt_dynamic_voluntary: - preempt_dynamic_enable(cond_resched); - preempt_dynamic_enable(might_resched); - preempt_dynamic_disable(preempt_schedule); - preempt_dynamic_disable(preempt_schedule_notrace); - preempt_dynamic_disable(irqentry_exit_cond_resched); - preempt_dynamic_key_disable(preempt_lazy); - if (mode != preempt_dynamic_mode) - pr_info("Dynamic Preempt: voluntary\n"); - break; - case preempt_dynamic_full: - preempt_dynamic_disable(cond_resched); - preempt_dynamic_disable(might_resched); - preempt_dynamic_enable(preempt_schedule); - preempt_dynamic_enable(preempt_schedule_notrace); - preempt_dynamic_enable(irqentry_exit_cond_resched); preempt_dynamic_key_disable(preempt_lazy); if (mode != preempt_dynamic_mode) pr_info("Dynamic Preempt: full\n"); break; case preempt_dynamic_lazy: - preempt_dynamic_disable(cond_resched); - preempt_dynamic_disable(might_resched); - preempt_dynamic_enable(preempt_schedule); - preempt_dynamic_enable(preempt_schedule_notrace); - preempt_dynamic_enable(irqentry_exit_cond_resched); preempt_dynamic_key_enable(preempt_lazy); if (mode != preempt_dynamic_mode) pr_info("Dynamic Preempt: lazy\n"); @@ -8099,11 +8031,7 @@ __setup("preempt=", setup_preempt_mode); static void __init preempt_dynamic_init(void) { if (preempt_dynamic_mode == preempt_dynamic_undefined) { - if (IS_ENABLED(CONFIG_PREEMPT_NONE)) { - sched_dynamic_update(preempt_dynamic_none); - } else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) { - sched_dynamic_update(preempt_dynamic_voluntary); - } else if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) { + if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) { sched_dynamic_update(preempt_dynamic_lazy); } else { /* Default static call setting, nothing to do */ @@ -8123,8 +8051,6 @@ static void __init preempt_dynamic_init(void) } \ EXPORT_SYMBOL_GPL(preempt_model_##mode) -PREEMPT_MODEL_ACCESSOR(none); -PREEMPT_MODEL_ACCESSOR(voluntary); PREEMPT_MODEL_ACCESSOR(full); PREEMPT_MODEL_ACCESSOR(lazy); @@ -8137,7 +8063,7 @@ static inline void preempt_dynamic_init(void) { } #endif /* CONFIG_PREEMPT_DYNAMIC */ const char *preempt_modes[] = { - "none", "voluntary", "full", "lazy", NULL, + "full", "lazy", NULL, }; const char *preempt_model_str(void) @@ -8759,6 +8685,9 @@ int sched_cpu_activate(unsigned int cpu) */ sched_set_rq_online(rq, cpu); + /* preferred is subset of active and follows its state */ + set_cpu_preferred(cpu, true); + return 0; } @@ -8772,6 +8701,8 @@ int sched_cpu_deactivate(unsigned int cpu) if (ret) return ret; + set_cpu_preferred(cpu, false); + /* * Remove CPU from nohz.idle_cpus_mask to prevent participating in * load balancing when not active @@ -11327,18 +11258,17 @@ void sched_change_end(struct sched_change_ctx *ctx) if (p->sched_class->switched_to) p->sched_class->switched_to(rq, p); + /* + * Check if the task can preempt the current in case the class + * was upgraded. Note that none of the switch*_from() methods + * know the new class and none of the switch*_to() methods know + * the old class so we must resched here. + */ + if (ctx->queued) + wakeup_preempt(rq, p, 0); + if (ctx->running) { /* - * If this was a class promotion; let the old class - * know it got preempted. Note that none of the - * switch*_from() methods know the new class and none - * of the switch*_to() methods know the old class. - */ - if (sched_class_above(p->sched_class, ctx->class)) { - rq->next_class->wakeup_preempt(rq, p, 0); - rq->next_class = p->sched_class; - } - /* * If this was a degradation in class; make sure to * reschedule. */ @@ -11349,3 +11279,88 @@ void sched_change_end(struct sched_change_ctx *ctx) p->sched_class->prio_changed(rq, p, ctx->prio); } } + +#ifdef CONFIG_PREFERRED_CPU +static DEFINE_PER_CPU(struct cpu_stop_work, npc_push_task_work); + +static int sched_non_preferred_cpu_push_stop(void *arg) +{ + struct task_struct *p = arg; + struct rq *rq = this_rq(); + struct rq_flags rf; + int cpu; + + if (cpu_preferred(rq->cpu)) { + scoped_guard(rq_lock_irqsave, rq) + rq->npc_push_work_pending = false; + put_task_struct(p); + return 0; + } + + scoped_guard (raw_spinlock_irq, &p->pi_lock) { + /* + * select_fallback_rq() may acquire the rq lock in case of + * fallback. So call it before grabbing rq lock. If the task + * migrates to another CPU before the rq lock is acquired, + * subsequent validation of task's current rq will help to + * safely bail out. + */ + cpu = select_fallback_rq(rq->cpu, p); + context_unsafe_alias(rq); + rq_lock(rq, &rf); + rq->npc_push_work_pending = false; + update_rq_clock(rq); + + if (task_rq(p) == rq && task_on_rq_queued(p)) { + struct rq *dest_rq = __migrate_task(rq, &rf, p, cpu); + + if (rq != dest_rq) + schedstat_inc(p->stats.nr_migrations_cpu_non_preferred); + rq = dest_rq; + } + rq_unlock(rq, &rf); + } + + put_task_struct(p); + return 0; +} + +/* + * Push the current task running on non-preferred CPU(npc). + * Using this non preferred CPU will lead to more contention + * in the host. So it is better not to use this CPU. + * + * Since task is running, call a stopper to push the task out. This is + * similar to how task moves during hotplug. In select_fallback_rq() a + * preferred CPU will be chosen and henceforth task shouldn't come back to + * this CPU again. + * + * Works for FAIR class only. + * + * If task is affined only on non-preferred CPUs, no point in moving it out. + */ +void sched_push_current_non_preferred_cpu(struct rq *rq) +{ + struct task_struct *push_task = rq->curr; + + scoped_guard(rq_lock, rq) { + /* Push the task if its explicit affinity allows */ + if (!task_can_migrate_to_preferred(push_task, rq->cpu)) + return; + + /* There is already a stopper thread. Don't race with it. */ + if (rq->npc_push_work_pending) + return; + + if (is_migration_disabled(push_task)) + return; + + rq->npc_push_work_pending = true; + } + + /* sched_tick runs with interrupts disabled. */ + get_task_struct(push_task); + stop_one_cpu_nowait(rq->cpu, sched_non_preferred_cpu_push_stop, + push_task, this_cpu_ptr(&npc_push_task_work)); +} +#endif diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index 49ccd6f1c185..14405a614991 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -73,6 +73,18 @@ static void sugov_update_rate_limit_us(struct sugov_policy *sg_policy) (s64)sg_policy->tunables->rate_limit_us * NSEC_PER_USEC; } +static void sugov_update_rate_limit_us_sync(struct sugov_policy *sg_policy) +{ + /* + * Serialize with rate_limit_us_store() to prevent this path from using + * a stale rate limit tunable value to set the limit for the given + * policy. + */ + guard(mutex)(&sg_policy->tunables->attr_set.update_lock); + + sugov_update_rate_limit_us(sg_policy); +} + static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time) { s64 delta_ns; @@ -861,7 +873,7 @@ static int sugov_start(struct cpufreq_policy *policy) void (*uu)(struct update_util_data *data, u64 time, unsigned int flags); unsigned int cpu; - sugov_update_rate_limit_us(sg_policy); + sugov_update_rate_limit_us_sync(sg_policy); sg_policy->last_freq_update_time = 0; sg_policy->next_freq = 0; sg_policy->work_in_progress = false; diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 06bddaa738e5..9e0fb8633d82 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -226,9 +226,8 @@ void account_steal_time(u64 cputime) void account_idle_time(u64 cputime) { u64 *cpustat = kcpustat_this_cpu->cpustat; - struct rq *rq = this_rq(); - if (atomic_read(&rq->nr_iowait) > 0) + if (cpu_has_iowait(smp_processor_id())) cpustat[CPUTIME_IOWAIT] += cputime; else cpustat[CPUTIME_IDLE] += cputime; @@ -255,7 +254,7 @@ void __account_forceidle_time(struct task_struct *p, u64 delta) * occasion account more time than the calling functions think elapsed. */ #ifdef CONFIG_PARAVIRT -struct static_key paravirt_steal_enabled; +DEFINE_STATIC_KEY_FALSE(paravirt_steal_enabled); #ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN static u64 native_steal_clock(int cpu) @@ -270,7 +269,7 @@ DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); static __always_inline u64 steal_account_process_time(u64 maxtime) { #ifdef CONFIG_PARAVIRT - if (static_key_false(¶virt_steal_enabled)) { + if (static_branch_unlikely(¶virt_steal_enabled)) { u64 steal; steal = paravirt_steal_clock(smp_processor_id()); @@ -381,9 +380,9 @@ void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times) * softirq as those do not count in task exec_runtime any more. */ static void irqtime_account_process_tick(struct task_struct *p, int user_tick, - int ticks) + u64 cputime) { - u64 other, cputime = TICK_NSEC * ticks; + u64 other; /* * When returning from idle, many ticks can get accounted at @@ -418,7 +417,7 @@ static void irqtime_account_process_tick(struct task_struct *p, int user_tick, #else /* !CONFIG_IRQ_TIME_ACCOUNTING: */ static inline void irqtime_account_process_tick(struct task_struct *p, int user_tick, - int nr_ticks) { } + u64 cputime) { } #endif /* !CONFIG_IRQ_TIME_ACCOUNTING */ #if defined(CONFIG_NO_HZ_COMMON) && !defined(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) @@ -431,7 +430,7 @@ static void kcpustat_idle_stop(struct kernel_cpustat *kc, u64 now) if (!kc->idle_elapse) return; - iowait = nr_iowait_cpu(smp_processor_id()) > 0; + iowait = cpu_has_iowait(smp_processor_id()); delta = now - kc->idle_entrytime; steal = steal_account_process_time(delta); @@ -468,18 +467,38 @@ static void kcpustat_idle_start(struct kernel_cpustat *kc, u64 now) write_seqcount_end(&kc->idle_sleeptime_seq); } -void kcpustat_dyntick_stop(u64 now) +void kcpustat_dyntick_stop(u64 now, u64 tick_start) { struct kernel_cpustat *kc = kcpustat_this_cpu; if (!vtime_generic_enabled_this_cpu()) { WARN_ON_ONCE(!kc->idle_dyntick); + if (kc->idle_tick_period != tick_start) { + kc->idle_tick_period = tick_start; + kc->idle_tick_overlap = 0; + } + tick_start = max(tick_start, kc->idle_dyntick_entry); + if (!WARN_ON_ONCE(now < tick_start)) + kc->idle_tick_overlap += now - tick_start; kcpustat_idle_stop(kc, now); kc->idle_dyntick = false; vtime_dyntick_stop(); } } +/* + * The first tick after dyntick-idle covers a period that the dyntick-idle + * accounting may already have accounted in part. + */ +static inline u64 kcpustat_tick_overlap(void) +{ + struct kernel_cpustat *kc = kcpustat_this_cpu; + u64 overlap = kc->idle_tick_overlap; + + kc->idle_tick_overlap = 0; + return overlap; +} + void kcpustat_dyntick_start(u64 now) { struct kernel_cpustat *kc = kcpustat_this_cpu; @@ -487,6 +506,7 @@ void kcpustat_dyntick_start(u64 now) if (!vtime_generic_enabled_this_cpu()) { vtime_dyntick_start(); kc->idle_dyntick = true; + kc->idle_dyntick_entry = now; kcpustat_idle_start(kc, now); } } @@ -544,17 +564,22 @@ static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx, u64 kcpustat_field_idle(int cpu) { return kcpustat_field_dyntick(cpu, CPUTIME_IDLE, - !nr_iowait_cpu(cpu), ktime_get()); + !cpu_has_iowait(cpu), ktime_get()); } EXPORT_SYMBOL_GPL(kcpustat_field_idle); u64 kcpustat_field_iowait(int cpu) { return kcpustat_field_dyntick(cpu, CPUTIME_IOWAIT, - nr_iowait_cpu(cpu), ktime_get()); + cpu_has_iowait(cpu), ktime_get()); } EXPORT_SYMBOL_GPL(kcpustat_field_iowait); #else +static inline u64 kcpustat_tick_overlap(void) +{ + return 0; +} + static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx, bool compute_delta, ktime_t now) { @@ -601,7 +626,7 @@ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) { return get_cpu_sleep_time_us(cpu, CPUTIME_IDLE, - !nr_iowait_cpu(cpu), last_update_time); + !cpu_has_iowait(cpu), last_update_time); } EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); @@ -625,7 +650,7 @@ EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time) { return get_cpu_sleep_time_us(cpu, CPUTIME_IOWAIT, - nr_iowait_cpu(cpu), last_update_time); + cpu_has_iowait(cpu), last_update_time); } EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); @@ -695,12 +720,13 @@ void account_process_tick(struct task_struct *p, int user_tick) if (kcpustat_idle_dyntick()) return; + cputime = TICK_NSEC - kcpustat_tick_overlap(); + if (irqtime_enabled()) { - irqtime_account_process_tick(p, user_tick, 1); + irqtime_account_process_tick(p, user_tick, cputime); return; } - cputime = TICK_NSEC; steal = steal_account_process_time(ULONG_MAX); if (steal >= cputime) @@ -1109,7 +1135,6 @@ static int kcpustat_field_vtime(u64 *cpustat, int cpu, u64 *val) { struct vtime *vtime = &tsk->vtime; - struct rq *rq = cpu_rq(cpu); unsigned int seq; do { @@ -1152,11 +1177,11 @@ static int kcpustat_field_vtime(u64 *cpustat, *val += vtime->gtime + vtime_delta(vtime); break; case CPUTIME_IDLE: - if (state == VTIME_IDLE && !atomic_read(&rq->nr_iowait)) + if (state == VTIME_IDLE && !cpu_has_iowait(cpu)) *val += vtime_delta(vtime); break; case CPUTIME_IOWAIT: - if (state == VTIME_IDLE && atomic_read(&rq->nr_iowait) > 0) + if (state == VTIME_IDLE && cpu_has_iowait(cpu)) *val += vtime_delta(vtime); break; default: @@ -1251,7 +1276,7 @@ static int kcpustat_cpu_fetch_vtime(struct kernel_cpustat *dst, } break; case VTIME_IDLE: - if (atomic_read(&cpu_rq(cpu)->nr_iowait) > 0) + if (cpu_has_iowait(cpu)) cpustat[CPUTIME_IOWAIT] += delta; else cpustat[CPUTIME_IDLE] += delta; diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 0663c00c41c0..c0ebdcde5fe5 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -1097,7 +1097,7 @@ static int start_dl_timer(struct sched_dl_entity *dl_se) * chosen as the deadline is too small, don't even try to * start the timer in the past! */ - if (ktime_us_delta(act, now) < 0) + if (ktime_before(act, now)) return 0; /* @@ -2773,11 +2773,14 @@ static void start_hrtick_dl(struct rq *rq, struct sched_dl_entity *dl_se) * DL keeps current in tree, because ->deadline is not typically changed while * a task is runnable. */ -static void set_next_task_dl(struct rq *rq, struct task_struct *p, bool first) +static void set_next_task_dl(struct rq *rq, struct task_struct *p, enum snt_e type) { struct sched_dl_entity *dl_se = &p->dl; struct dl_rq *dl_rq = &rq->dl; + if (type == SNT_REPICK) + return; + p->se.exec_start = rq_clock_task(rq); if (on_dl_rq(&p->dl)) update_stats_wait_end_dl(dl_rq, dl_se); @@ -2788,7 +2791,7 @@ static void set_next_task_dl(struct rq *rq, struct task_struct *p, bool first) WARN_ON_ONCE(dl_rq->curr); dl_rq->curr = dl_se; - if (!first) + if (type != SNT_PICK) return; if (rq->donor->sched_class != &dl_sched_class) diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 72236db67983..e6a3b516c703 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -73,13 +73,13 @@ static int sched_feat_show(struct seq_file *m, void *v) #ifdef CONFIG_JUMP_LABEL -#define jump_label_key__true STATIC_KEY_INIT_TRUE -#define jump_label_key__false STATIC_KEY_INIT_FALSE +#define jump_label_key__true { .key_true = STATIC_KEY_TRUE_INIT } +#define jump_label_key__false { .key_false = STATIC_KEY_FALSE_INIT } #define SCHED_FEAT(name, enabled) \ jump_label_key__##enabled , -struct static_key sched_feat_keys[__SCHED_FEAT_NR] = { +union sched_feat_key sched_feat_keys[__SCHED_FEAT_NR] = { #include "features.h" }; @@ -87,12 +87,12 @@ struct static_key sched_feat_keys[__SCHED_FEAT_NR] = { static void sched_feat_disable(int i) { - static_key_disable_cpuslocked(&sched_feat_keys[i]); + static_branch_disable_cpuslocked(&sched_feat_keys[i].key_true); } static void sched_feat_enable(int i) { - static_key_enable_cpuslocked(&sched_feat_keys[i]); + static_branch_enable_cpuslocked(&sched_feat_keys[i].key_false); } #else /* !CONFIG_JUMP_LABEL: */ static void sched_feat_disable(int i) { }; @@ -280,16 +280,10 @@ static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf, static int sched_dynamic_show(struct seq_file *m, void *v) { - int i = (IS_ENABLED(CONFIG_PREEMPT_RT) || IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY)) * 2; int mode = READ_ONCE(preempt_dynamic_mode); - int j; - /* Count entries in NULL terminated preempt_modes */ - for (j = 0; preempt_modes[j]; j++) - ; - j -= !IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY); - - for (; i < j; i++) { + /* Stop at NULL terminator */ + for (int i = 0; preempt_modes[i]; i++) { if (mode == i) seq_puts(m, "("); seq_puts(m, preempt_modes[i]); @@ -1446,6 +1440,7 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, P_SCHEDSTAT(nr_failed_migrations_running); P_SCHEDSTAT(nr_failed_migrations_hot); P_SCHEDSTAT(nr_forced_migrations); + P_SCHEDSTAT(nr_migrations_cpu_non_preferred); P_SCHEDSTAT(nr_wakeups); P_SCHEDSTAT(nr_wakeups_sync); P_SCHEDSTAT(nr_wakeups_migrate); diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index bc4eee5bb4cb..a5330b760ecd 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -31,6 +31,7 @@ static struct scx_cid_tables *scx_cid_tables; /* used only during alloc/free */ #define SCX_CID_TOPO_NEG (struct scx_cid_topo) { \ .core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1, \ .node_cid = -1, .node_idx = -1, .shard_cid = -1, .shard_idx = -1, \ + .cluster_cid = -1, .cluster_idx = -1, \ } /* @@ -182,12 +183,14 @@ s32 scx_cid_init(struct scx_sched *sch) cpumask_var_t to_walk __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t node_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t llc_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t cluster_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t core_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL; struct scx_cid_tables *tbls; u32 next_cid = 0; s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0; + s32 next_cluster_idx = 0; s32 next_shard_idx = 0; u32 shard_size, max_cids; u32 notopo_in_shard; @@ -215,6 +218,7 @@ s32 scx_cid_init(struct scx_sched *sch) if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) || !zalloc_cpumask_var(&node_scratch, GFP_KERNEL) || !zalloc_cpumask_var(&llc_scratch, GFP_KERNEL) || + !zalloc_cpumask_var(&cluster_scratch, GFP_KERNEL) || !zalloc_cpumask_var(&core_scratch, GFP_KERNEL) || !zalloc_cpumask_var(&llc_fallback, GFP_KERNEL) || !zalloc_cpumask_var(&online_no_topo, GFP_KERNEL)) @@ -256,6 +260,7 @@ s32 scx_cid_init(struct scx_sched *sch) u32 cores_per_shard, nr_large; u32 shard_local = 0, cores_in_shard = 0, cids_in_shard = 0; s32 shard_cid, shard_idx; + s32 cluster_cid = -1, cluster_idx = -1; /* llc_scratch = node_scratch & this llc */ cpumask_and(llc_scratch, node_scratch, llc_mask); @@ -268,15 +273,32 @@ s32 scx_cid_init(struct scx_sched *sch) tbls->shard_node[shard_idx] = nid; while (!cpumask_empty(llc_scratch)) { - s32 lcpu = cpumask_first(llc_scratch); - const struct cpumask *sib = topology_sibling_cpumask(lcpu); - s32 core_cid = next_cid; - s32 core_idx = next_core_idx++; - s32 ccpu; + const struct cpumask *sib; + s32 core_cid, core_idx, lcpu, ccpu; u32 max_cores, cids_in_core; - /* core_scratch = llc_scratch & this core */ - cpumask_and(core_scratch, llc_scratch, sib); + /* + * Take the cores of one cluster before moving + * on, so that a cluster is a contiguous cid + * range like the core, LLC and node levels. + */ + if (cpumask_empty(cluster_scratch)) { + s32 xcpu = cpumask_first(llc_scratch); + + cpumask_or(cluster_scratch, topology_cluster_cpumask(xcpu), + topology_sibling_cpumask(xcpu)); + cpumask_and(cluster_scratch, cluster_scratch, llc_scratch); + cluster_cid = next_cid; + cluster_idx = next_cluster_idx++; + } + + lcpu = cpumask_first(cluster_scratch); + sib = topology_sibling_cpumask(lcpu); + core_cid = next_cid; + core_idx = next_core_idx++; + + /* core_scratch = cluster_scratch & this core */ + cpumask_and(core_scratch, cluster_scratch, sib); if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch))) return -EINVAL; @@ -316,8 +338,11 @@ s32 scx_cid_init(struct scx_sched *sch) .node_idx = node_idx, .shard_cid = shard_cid, .shard_idx = shard_idx, + .cluster_cid = cluster_cid, + .cluster_idx = cluster_idx, }; + cpumask_clear_cpu(ccpu, cluster_scratch); cpumask_clear_cpu(ccpu, llc_scratch); cpumask_clear_cpu(ccpu, node_scratch); cpumask_clear_cpu(ccpu, to_walk); @@ -461,8 +486,8 @@ __bpf_kfunc_start_defs(); * starts must be strictly increasing with the first entry 0 and all values < * num_possible_cpus(). The last shard extends to num_possible_cpus() and no * shard may span more than SCX_CID_SHARD_MAX_CPUS cids. Topo info - * (core/LLC/node) is cleared and the shard layout is set from the input. On - * invalid input, abort the scheduler. + * (core/cluster/LLC/node) is cleared and the shard layout is set from the + * input. On invalid input, abort the scheduler. */ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid__arena, u32 cpu_to_cid_cnt, const s32 *shard_start__arena, u32 shard_start_cnt, @@ -641,6 +666,29 @@ __bpf_kfunc s32 scx_bpf_cid_to_cpu(s32 cid, const struct bpf_prog_aux *aux) } /** + * scx_bpf_cid_node - Return the NUMA node the given @cid belongs to + * @cid: cid to look up + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return NUMA_NO_NODE if @cid is invalid or there is no scheduler. Unlike + * scx_bpf_cpu_node(), this kfunc is available only with CID support, so BPF + * schedulers can use its presence to detect NUMA-aware CID lookups. + */ +__bpf_kfunc s32 scx_bpf_cid_node(s32 cid, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return NUMA_NO_NODE; + cpu = scx_cid_to_cpu(sch, cid); + return cpu < 0 ? NUMA_NO_NODE : cpu_to_node(cpu); +} + +/** * scx_bpf_cpu_to_cid - Return the cid for @cpu * @cpu: cpu to look up * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs @@ -912,30 +960,36 @@ bool scx_cmask_empty(const struct scx_cmask *m) /** * scx_bpf_cid_topo - Copy out per-cid topology info * @cid: cid to look up - * @out__uninit: where to copy the topology info; fully written by this call + * @out: where to copy the topology info + * @out__sz: size of @out, the program's sizeof(struct scx_cid_topo) * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * - * Fill @out__uninit with the topology info for @cid. Trigger scx_error() if - * @cid is out of range. If @cid is valid but in the no-topo section, all fields - * are set to -1. All fields are also set to -1 when no cid tables have been - * published yet, which a program may observe while racing the root enable. + * Fill @out with the topology info for @cid. Trigger scx_error() if @cid is out + * of range. If @cid is valid but in the no-topo section, all fields are set to + * -1. All fields are also set to -1 when no cid tables have been published yet, + * which a program may observe while racing the root enable. + * + * The program's struct may be older or newer than the kernel's. The smaller of + * @out__sz and the kernel's size is copied and the rest of @out is set to -1. */ -__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out__uninit, +__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out, size_t out__sz, const struct bpf_prog_aux *aux) { + size_t len = min(out__sz, sizeof(*out)); struct scx_cid_topo *topo; struct scx_sched *sch; + /* the error cases and fields the kernel lacks read as -1 */ + memset(out, 0xff, out__sz); + guard(rcu)(); sch = scx_prog_sched(aux); topo = rcu_dereference(scx_cid_topo); - if (unlikely(!sch) || !cid_valid(sch, cid) || unlikely(!topo)) { - *out__uninit = SCX_CID_TOPO_NEG; + if (unlikely(!sch) || !cid_valid(sch, cid) || unlikely(!topo)) return; - } - *out__uninit = topo[cid]; + memcpy(out, &topo[cid], len); } __bpf_kfunc_end_defs(); @@ -952,6 +1006,7 @@ static const struct btf_kfunc_id_set scx_kfunc_set_init_cids = { BTF_KFUNCS_START(scx_kfunc_ids_cid) BTF_ID_FLAGS(func, scx_bpf_cid_to_cpu, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cid_node, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cpu_to_cid, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cid_topo, KF_IMPLICIT_ARGS) BTF_KFUNCS_END(scx_kfunc_ids_cid) diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h index 2fe2311a0f99..709afbfb97c6 100644 --- a/kernel/sched/ext/cid.h +++ b/kernel/sched/ext/cid.h @@ -13,11 +13,12 @@ * kernel type sized for the maximum NR_CPUS (4k), with verbose helper sequences * for every op. * - * cids give every cpu a dense, topology-ordered id. CPUs sharing a core, LLC or - * NUMA node get contiguous cid ranges, so a topology unit becomes a (start, - * length) slice of cid space. Communication can pass a slice instead of a - * cpumask, and BPF code can process, for example, a u64 word's worth of cids at - * a time. + * cids give every cpu a dense, topology-ordered id. CPUs in each core, + * cluster, LLC or NUMA node get contiguous cid ranges, so a topology unit + * becomes a (start, length) slice of cid space. A core without a wider cluster + * level forms a cluster of its own. Communication can pass a slice + * instead of a cpumask, and BPF code can process, for example, a u64 word's + * worth of cids at a time. * * The mapping is built once at root scheduler enable time by walking the * topology of online cpus only. Going by online cpus is out of necessity: diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 3219f0da0fe4..9bae826d69f9 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -25,6 +25,43 @@ DEFINE_RAW_SPINLOCK(scx_sched_lock); /* + * Block a retained proxy donor, telling put_prev_task_scx() that the donor is + * about to be dequeued and must not be reenqueued. + */ +static void scx_block_proxy_donor(struct rq *rq, struct task_struct *p) +{ + rq->scx.flags |= SCX_RQ_PROXY_BLOCKING; + sched_proxy_block_task(rq, p); + rq->scx.flags &= ~SCX_RQ_PROXY_BLOCKING; +} + +bool __scx_allow_proxy_exec(const struct task_struct *p) +{ + struct scx_sched *sch; + + if (p->sched_class != &ext_sched_class) + return true; + + sch = scx_task_sched(p); + return !sch || (sch->ops.flags & SCX_OPS_ENQ_BLOCKED); +} + +/* + * End retained proxy execution before changing @p's BPF scheduler ownership. + * Called with @p's pi and rq locks held immediately before + * sched_change_begin(). The caller must pass DEQUEUE_NOCLOCK so the rq clock + * is updated only once. + */ +void scx_prepare_task_sched_change(struct task_struct *p) +{ + lockdep_assert_held(&p->pi_lock); + lockdep_assert_rq_held(task_rq(p)); + + update_rq_clock(task_rq(p)); + scx_block_proxy_donor(task_rq(p), p); +} + +/* * NOTE: sched_ext is in the process of growing multiple scheduler support and * scx_root usage is in a transitional state. Naked dereferences are safe if the * caller is one of the tasks attached to SCX and explicit RCU dereference is @@ -41,10 +78,21 @@ struct scx_sched __rcu *scx_root; LIST_HEAD(scx_sched_all); #ifdef CONFIG_EXT_SUB_SCHED +static __always_inline int scx_sched_cmpfn(struct rhashtable_compare_arg *arg, + const void *ptr) +{ + const struct scx_sched *sch = ptr; + + BUILD_BUG_ON(sizeof_field(struct scx_sched, ops.sub_cgroup_id) != sizeof(u64)); + + return sch->ops.sub_cgroup_id != *(const u64 *)arg->key; +} + const struct rhashtable_params scx_sched_hash_params = { .key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id), .key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id), .head_offset = offsetof(struct scx_sched, hash_node), + .obj_cmpfn = scx_sched_cmpfn, .insecure_elasticity = true, /* inserted under scx_sched_lock */ }; @@ -52,10 +100,20 @@ struct rhashtable scx_sched_hash; #endif /* see SCX_OPS_TID_TO_TASK */ +static __always_inline int scx_tid_cmpfn(struct rhashtable_compare_arg *arg, const void *ptr) +{ + const struct sched_ext_entity *scx = ptr; + + BUILD_BUG_ON(sizeof_field(struct sched_ext_entity, tid) != sizeof(u64)); + + return scx->tid != *(const u64 *)arg->key; +} + static const struct rhashtable_params scx_tid_hash_params = { .key_len = sizeof_field(struct sched_ext_entity, tid), .key_offset = offsetof(struct sched_ext_entity, tid), .head_offset = offsetof(struct sched_ext_entity, tid_hash_node), + .obj_cmpfn = scx_tid_cmpfn, .insecure_elasticity = true, /* inserted/removed under scx_tasks_lock */ }; static struct rhashtable scx_tid_hash; @@ -189,10 +247,20 @@ static DEFINE_PER_CPU(struct scx_tid_alloc, scx_tid_alloc); */ static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); +static __always_inline int dsq_cmpfn(struct rhashtable_compare_arg *arg, const void *ptr) +{ + const struct scx_dispatch_q *dsq = ptr; + + BUILD_BUG_ON(sizeof_field(struct scx_dispatch_q, id) != sizeof(u64)); + + return dsq->id != *(const u64 *)arg->key; +} + static const struct rhashtable_params dsq_hash_params = { .key_len = sizeof_field(struct scx_dispatch_q, id), .key_offset = offsetof(struct scx_dispatch_q, id), .head_offset = offsetof(struct scx_dispatch_q, hash_node), + .obj_cmpfn = dsq_cmpfn, }; static LLIST_HEAD(dsqs_to_free); @@ -322,7 +390,6 @@ static const struct sched_class *scx_setscheduler_class(struct task_struct *p) static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu) { -#ifdef CONFIG_EXT_SUB_SCHED /* * If @sch is a sub-sched which is bypassing, its tasks should go into * the bypass DSQs of the nearest ancestor which is not bypassing. The @@ -334,11 +401,7 @@ static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 c * are re-enqueued after scx_bypassing() is turned on, guaranteeing that * all tasks are transferred to the right DSQs. */ - while (scx_parent(sch) && scx_bypassing(sch, cpu)) - sch = scx_parent(sch); -#endif /* CONFIG_EXT_SUB_SCHED */ - - return scx_bypass_dsq(sch, cpu); + return scx_bypass_dsq(scx_exec_sched(sch, cpu), cpu); } /** @@ -374,28 +437,28 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) */ /* - * If we're in the dispatch path holding rq lock, $curr may or may not + * If we're in the dispatch path holding rq lock, $donor may or may not * be ready depending on whether the on-going dispatch decides to extend - * $curr's slice. We say yes here and resolve it at the end of dispatch. + * $donor's slice. We say yes here and resolve it at the end of dispatch. * See dispatch_one(). */ if (rq->scx.flags & SCX_RQ_IN_DISPATCH) return true; /* - * %SCX_ENQ_PREEMPT clears $curr's slice if on SCX and kicks dispatch, + * The preemption flags clear $donor's slice if on SCX and kick dispatch, * so allow it to avoid spuriously triggering reenq on a combined * PREEMPT|IMMED insertion. */ - if (enq_flags & SCX_ENQ_PREEMPT) { - struct task_struct *curr = rq->curr; + if (enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) { + struct task_struct *donor = rq->donor; /* * A protected slice refuses the preemption and the cpu stays * occupied. See rq_owned_post_enq(). */ - return curr->sched_class != &ext_sched_class || - likely(!(curr->scx.flags & SCX_TASK_PROTECTED)); + return donor->sched_class != &ext_sched_class || + likely(!(donor->scx.flags & SCX_TASK_PROTECTED)); } /* @@ -447,48 +510,58 @@ static void switch_rq_lock(struct rq *from, struct rq *to) DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type); /** - * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task + * scx_fill_cmask_scratch - Build this cpu's arena cmask from @cpumask + * @sch: scx_sched whose scratch to fill + * @cpumask: cpus to translate into cids + * + * The scratch lives in BPF-writable arena memory and its header can't be + * trusted, so it is rewritten from kernel geometry rather than read. Caller + * must hold an rq lock so this cpu is the sole kernel writer for as long as the + * returned address is in use. + */ +static struct scx_cmask *scx_fill_cmask_scratch(struct scx_sched *sch, + const struct cpumask *cpumask) +{ + struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); + struct scx_cmask_ref ref; + + scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref); + scx_cmask_ref_from_cpumask(&ref, cpumask); + return kern_va; +} + +/** + * scx_call_op_set_cpumask - Invoke the set_cpumask or set_cmask op for @task * @sch: scx_sched being invoked * @rq: rq to update as the currently-locked rq, or NULL * @task: task whose affinity is changing * @cpumask: new cpumask * - * For cid-form schedulers, translate @cpumask to a cmask via the per-cpu - * scratch in cid.c and dispatch through the ops_cid union view. Caller - * must hold @rq's rq lock so this_cpu_ptr is stable across the call. + * For cid-form schedulers, translate @cpumask to a cmask in the per-cpu scratch + * and dispatch through the ops_cid union view. Caller must hold @rq's rq lock. */ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, struct task_struct *task, const struct cpumask *cpumask) { - if (scx_is_cid_type()) { - struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); - struct scx_cmask_ref ref; - - /* - * Build the per-cpu arena cmask from kernel geometry via @ref, - * never reading its BPF-writable header. set_cmask()'s __arena - * argument takes the kernel address and the struct_ops - * trampoline rebases it into BPF's arena pointer form. The rq - * lock makes this cpu the sole kernel writer. - */ - scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref); - scx_cmask_ref_from_cpumask(&ref, cpumask); - SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, kern_va); - } else { + if (scx_is_cid_type()) + SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, + scx_fill_cmask_scratch(sch, cpumask)); + else SCX_CALL_OP_TASK(sch, set_cpumask, rq, task, cpumask); - } } enum scx_dsq_iter_flags { /* iterate in the reverse dispatch order */ SCX_DSQ_ITER_REV = 1U << 16, + __SCX_DSQ_ITER_HAS_REENQ = 1U << 29, __SCX_DSQ_ITER_HAS_SLICE = 1U << 30, __SCX_DSQ_ITER_HAS_VTIME = 1U << 31, __SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV, __SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS | + __SCX_DSQ_ITER_HAS_REENQ | __SCX_DSQ_ITER_HAS_SLICE | __SCX_DSQ_ITER_HAS_VTIME, }; @@ -1079,6 +1152,65 @@ static void schedule_deferred_locked(struct rq *rq) schedule_deferred(rq); } +#ifdef CONFIG_NO_HZ_FULL +static void scx_proxy_tick_bal_cb(struct rq *rq) +{ + sched_update_tick_dependency(rq); +} + +static void scx_proxy_update_tick(struct rq *rq, struct task_struct *next) +{ + bool proxy = next != rq->donor; + bool was_proxy = rq->scx.flags & SCX_RQ_PROXY_TICK; + + if (likely(proxy == was_proxy)) + return; + + if (proxy) { + /* Keep proxy execution tick-driven for now. */ + rq->scx.flags |= SCX_RQ_PROXY_TICK; + tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED); + } else { + rq->scx.flags &= ~SCX_RQ_PROXY_TICK; + /* + * The selected donor is already visible, but rq->curr still + * identifies the outgoing execution context. Reevaluate after + * context_switch() updates rq->curr so sched_can_stop_tick() sees + * the complete selection. + */ + queue_balance_callback(rq, &rq->scx.proxy_tick_bal_cb, + scx_proxy_tick_bal_cb); + } +} +#endif + +/* + * Complete sched_ext bookkeeping after proxy resolution and retry tasks which + * couldn't be reenqueued by an earlier reject DSQ drain. + */ +void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next) +{ + lockdep_assert_rq_held(rq); + +#ifdef CONFIG_NO_HZ_FULL + if (scx_enabled() && tick_nohz_full_cpu(cpu_of(rq))) + scx_proxy_update_tick(rq, next); +#endif + + /* + * A higher priority class may wake up while @rq is transiently idle + * for proxy resolution. wakeup_preempt_scx() isn't invoked in that + * case, so reenqueue IMMED donors parked on the local DSQ here. + */ + if (rq->scx.nr_immed && sched_class_above(rq->next_class, &ext_sched_class)) + scx_schedule_reenq_local(rq, 0); + + if (rq->scx.flags & SCX_RQ_PROXY_RETRY) { + rq->scx.flags &= ~SCX_RQ_PROXY_RETRY; + schedule_deferred_locked(rq); + } +} + void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, u64 reenq_flags, struct rq *locked_rq) { @@ -1095,6 +1227,18 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, rq = container_of(dsq, struct rq, scx.local_dsq); /* + * An inactive cpu's local DSQ holds only tasks that are bound + * to it or are being moved off, so a reenq of it is pointless. + * Once balance push is installed, queue_balance_callback() also + * drops the callback that would run a reenq requested from + * dispatch, and irq_work_queue_on() can't reach an offline cpu. + * The reenq's node would then stay on this rq's list after + * @sch's pcpu is freed. + */ + if (unlikely(!cpu_active(cpu_of(rq)))) + return; + + /* * A sub-sched lacking baseline access on the target cid has no * business triggering IPIs. The lockless test is fine: slipping * through right after a revoke is harmless and a wrong denial @@ -1127,7 +1271,7 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, } else if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) { rq = this_rq(); - struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu, cpu_of(rq)); + struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu_user, cpu_of(rq)); struct scx_deferred_reenq_user *dru = &dsq_pcpu->deferred_reenq_user; /* @@ -1141,12 +1285,18 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); + /* @dsq is being destroyed, see free_dsq_irq_workfn() */ + if (unlikely(READ_ONCE(dsq->id) == SCX_DSQ_INVALID)) + return; + if (list_empty(&dru->node)) list_move_tail(&dru->node, &rq->scx.deferred_reenq_users); WRITE_ONCE(dru->flags, dru->flags | reenq_flags); } } else { - scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id); + /* destroyed since the caller resolved it, nothing to reenq */ + if (READ_ONCE(dsq->id) != SCX_DSQ_INVALID) + scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id); return; } @@ -1390,20 +1540,27 @@ static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 e static void update_curr_scx(struct rq *rq) { - struct task_struct *curr = rq->curr; + struct task_struct *donor; s64 delta_exec; + /* + * update_curr_scx() is selected through rq->donor->sched_class, not + * rq->curr->sched_class, so @donor is always an EXT task here. If an EXT + * owner executes for a FAIR donor, FAIR's update_curr() runs instead. + */ + donor = rq->donor; + /* apply even on 0 delta_exec, callers may still act on the slice */ - apply_task_slice_oob(rq, curr); + apply_task_slice_oob(rq, donor); delta_exec = update_curr_common(rq); if (unlikely(delta_exec <= 0)) return; - if (curr->scx.slice != SCX_SLICE_INF) - curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); + if (donor->scx.slice != SCX_SLICE_INF) + donor->scx.slice -= min_t(u64, donor->scx.slice, delta_exec); - if (unlikely(curr == scx_rescuee(rq))) + if (unlikely(donor == scx_rescuee(rq))) scx_rescue_charge(rq, delta_exec); dl_server_update(&rq->ext_server, delta_exec); @@ -1499,31 +1656,48 @@ static inline bool task_scx_migrating(struct task_struct *p) return p->scx.sticky_cpu >= 0; } -/* - * Call ops.dequeue() if the task is in BPF custody and not migrating. - * Clears %SCX_TASK_IN_CUSTODY when the callback is invoked. - */ -static void call_task_dequeue(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 deq_flags) +/* Must be called under the lock serializing @p's custody transfers. */ +static bool task_leave_custody(struct task_struct *p) { if (!(p->scx.flags & SCX_TASK_IN_CUSTODY) || task_scx_migrating(p)) - return; - - if (SCX_HAS_OP(sch, dequeue)) - SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags); + return false; p->scx.flags &= ~SCX_TASK_IN_CUSTODY; + return true; +} + +/* + * A task with an infinite slice may be running with its tick stopped. Lazy + * rescheduling doesn't send an IPI, so restore the tick dependency to guarantee + * that the lazy request is promoted by a real scheduler tick. Set the lazy + * request first so that the dependency update's IPI can also serve it on return + * to user space. + */ +static void scx_resched_curr_lazy(struct rq *rq) +{ + resched_curr_lazy(rq); + + if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) { + rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK; + if (rq->clock_update_flags < RQCF_UPDATED) + update_rq_clock(rq); + update_other_load_avgs(rq); + sched_update_tick_dependency(rq); + } } static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) { - call_task_dequeue(sch, rq, p, 0); + if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue)) + SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0); /* - * Only local inserts get the wakeup treatment below. Rejects kick the - * deferred reenq and rescue parks are paced by the rescue timer. + * Only local inserts get the wakeup treatment below. Rejects kick a + * deferred reenq and rescue parks are paced by the rescue timer. Proxy + * rejects which aren't ready when drained request a later retry from + * scx_proxy_reenqueue_retry(). */ if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { if (dsq->id == SCX_DSQ_REJECT) @@ -1577,12 +1751,16 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, if (rq->scx.flags & SCX_RQ_IN_DISPATCH) return; - if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && - rq->curr->sched_class == &ext_sched_class) { - if (likely(scx_set_task_slice(rq->curr, 0))) - resched_curr(rq); - else + if ((enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) && + p != rq->donor && rq->donor->sched_class == &ext_sched_class) { + if (likely(scx_set_task_slice(rq->donor, 0))) { + if (enq_flags & SCX_ENQ_PREEMPT) + resched_curr(rq); + else + scx_resched_curr_lazy(rq); + } else { __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); + } } } @@ -1590,12 +1768,10 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, u64 slice, u64 vtime, u64 enq_flags) { - bool is_rq_owned = false; + bool is_rq_owned = dsq_is_rq_owned(dsq); - if (dsq->id == SCX_DSQ_LOCAL) { + if (dsq->id == SCX_DSQ_LOCAL) dsq = scx_resolve_local_dsq(sch, rq, p, &enq_flags); - is_rq_owned = true; - } WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || @@ -1672,7 +1848,7 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, scx_error(sch, "DSQ ID 0x%016llx already had PRIQ-enqueued tasks", dsq->id); - if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) { + if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) { /* new task inserted at head - use fastpath */ if (dsq_insert_head(dsq, p) && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) rcu_assign_pointer(dsq->first_task, p); @@ -1705,20 +1881,28 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, if (is_rq_owned) { rq_owned_post_enq(sch, rq, dsq, p, enq_flags); } else { + bool call_dequeue = false; + /* * Global and bypass DSQs are terminal - the task leaves the - * scheduler's custody, so ops.dequeue() fires here. It can run + * scheduler's custody, so ops.dequeue() fires. It can run * without @p's rq lock (finish_dispatch() passes the dispatch * rq); that's safe because dequeue_task_scx() waits on * SCX_OPSS_DISPATCHING (see the ops_state note above) and so * can't race it. A non-terminal DSQ keeps the task in custody. + * The custody transfer happens under @dsq->lock so that later + * consumers see the flag clear; the callback runs after + * @dsq->lock is dropped because it may lock a DSQ itself. */ if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS) - call_task_dequeue(sch, rq, p, 0); + call_dequeue = task_leave_custody(p); else p->scx.flags |= SCX_TASK_IN_CUSTODY; raw_spin_unlock(&dsq->lock); + + if (call_dequeue && SCX_HAS_OP(sch, dequeue)) + SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0); } /* @@ -1738,6 +1922,7 @@ void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq) RB_CLEAR_NODE(&p->scx.dsq_priq); p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; } + p->scx.dsq_flags &= ~SCX_TASK_DSQ_REENQ; list_del_init(&p->scx.dsq_list.node); dsq_dec_nr(dsq, p); @@ -1766,10 +1951,10 @@ void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) list_del_init(&p->scx.dsq_list.node); /* - * When dispatching directly from the BPF scheduler to a local - * DSQ, the task isn't associated with any DSQ but - * @p->scx.holding_cpu may be set under the protection of - * %SCX_OPSS_DISPATCHING. + * When dispatch_to_local_dsq() or remote consumption moves a + * task to a local DSQ, the task isn't associated with any DSQ + * but @p->scx.holding_cpu may be set. Clearing holding_cpu + * tells dispatch_to_local_dsq() that it lost to a dequeue. */ if (p->scx.holding_cpu >= 0) p->scx.holding_cpu = -1; @@ -1789,10 +1974,10 @@ void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) scx_task_unlink_from_dsq(p, dsq); } else { /* - * We're racing against dispatch_to_local_dsq() which already - * removed @p from @dsq and set @p->scx.holding_cpu. Clear the - * holding_cpu which tells dispatch_to_local_dsq() that it lost - * the race. + * We're racing against unlink_dsq_and_switch_rq_lock(), + * which already removed @p from @dsq and set + * @p->scx.holding_cpu. Clear holding_cpu to tell + * unlink_dsq_and_switch_rq_lock() that it lost the race. */ WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node)); p->scx.holding_cpu = -1; @@ -1975,6 +2160,12 @@ bool scx_rq_online(struct rq *rq) void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, int sticky_cpu) { + enum { + ENQ_ACTION_NONE, + ENQ_ACTION_DIRECT, + ENQ_ACTION_LOCAL_NOREFILL, + ENQ_ACTION_ENQUEUE, + } action = ENQ_ACTION_NONE; struct scx_sched *sch = scx_task_sched(p); struct task_struct **ddsp_taskp; struct scx_dispatch_q *dsq; @@ -2008,7 +2199,7 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, __scx_exit(sch, SCX_EXIT_ERROR_REENQ, 0, cpu_of(rq), "%s[%d] reenqueued %u times without running", p->comm, p->pid, p->scx.reenq_cnt); - return; + goto out; } } @@ -2028,26 +2219,41 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) goto direct; - /* see %SCX_OPS_ENQ_EXITING */ - if (!(sch->ops.flags & SCX_OPS_ENQ_EXITING) && - unlikely(p->flags & PF_EXITING)) { - __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); - enq_flags |= SCX_ENQ_RESCUE; /* avoid looping on cap rejection */ - goto local; - } + /* + * A full wakeup enqueue happens before is_blocked is cleared. Don't + * report it as another blocked-donor admission. + */ + if ((sch->ops.flags & SCX_OPS_ENQ_BLOCKED) && p->is_blocked && + !(enq_flags & SCX_ENQ_WAKEUP)) { + enq_flags |= SCX_ENQ_BLOCKED; + } else { + /* see %SCX_OPS_ENQ_EXITING */ + if (unlikely(p->flags & PF_EXITING) && + !(sch->ops.flags & SCX_OPS_ENQ_EXITING)) { + __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); + enq_flags |= SCX_ENQ_RESCUE; /* avoid looping on cap rejection */ + goto local; + } - /* see %SCX_OPS_ENQ_MIGRATION_DISABLED */ - if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) && - is_migration_disabled(p)) { - __scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1); - goto local; + /* see %SCX_OPS_ENQ_MIGRATION_DISABLED */ + if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) && + is_migration_disabled(p)) { + __scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1); + goto local; + } } if (unlikely(!SCX_HAS_OP(sch, enqueue))) goto global; - /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ - qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; + /* + * DSQ bypass didn't trigger, enqueue on the BPF scheduler. Wrap the + * per-task qseq counter where the QSEQ field wraps and skip 0, which is + * what scx_bpf_dsq_insert() records for a task in NONE or DISPATCHING. + */ + p->scx.ops_qseq = ((p->scx.ops_qseq + 1) & + (SCX_OPSS_QSEQ_MASK >> SCX_OPSS_QSEQ_SHIFT)) ?: 1; + qseq = (unsigned long)p->scx.ops_qseq << SCX_OPSS_QSEQ_SHIFT; WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq); @@ -2073,14 +2279,14 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, * dequeue may be waiting. The store_release matches their load_acquire. */ atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq); - return; + goto out; direct: - direct_dispatch(sch, p, enq_flags); - return; + action = ENQ_ACTION_DIRECT; + goto out; local_norefill: - scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, enq_flags); - return; + action = ENQ_ACTION_LOCAL_NOREFILL; + goto out; local: dsq = &rq->scx.local_dsq; goto enqueue; @@ -2092,9 +2298,27 @@ bypass: goto enqueue; enqueue: - refill_task_slice_dfl(sch, p); - clear_direct_dispatch(p); - scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); + action = ENQ_ACTION_ENQUEUE; +out: + /* The reason is input to ops.enqueue(), not to the resulting placement. */ + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + + switch (action) { + case ENQ_ACTION_DIRECT: + direct_dispatch(sch, p, enq_flags); + break; + case ENQ_ACTION_LOCAL_NOREFILL: + scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, + enq_flags); + break; + case ENQ_ACTION_ENQUEUE: + refill_task_slice_dfl(sch, p); + clear_direct_dispatch(p); + scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); + break; + case ENQ_ACTION_NONE: + break; + } } static bool task_runnable(const struct task_struct *p) @@ -2112,8 +2336,8 @@ static void set_task_runnable(struct rq *rq, struct task_struct *p) } /* - * list_add_tail() must be used. scx_bypass() depends on tasks being - * appended to the runnable_list. + * list_add_tail() must be used. scx_bypass() and rq_offline_scx() + * depend on tasks being appended to the runnable_list. */ list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list); @@ -2141,17 +2365,30 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ int sticky_cpu = p->scx.sticky_cpu; u64 enq_flags = core_enq_flags | rq->scx.remote_activate_enq_flags; - if (enq_flags & ENQUEUE_WAKEUP) + /* + * An SCX-internal migration ends on arrival. Clear sticky_cpu so @p can + * leave custody when inserted into the destination DSQ. + */ + if (sticky_cpu >= 0) + p->scx.sticky_cpu = -1; + + /* + * SCX_RQ_IN_WAKEUP promises a task_woken_scx() call once this enqueue + * returns. Only the core's wakeup path delivers one. The flags stashed + * for a remote activation may carry the wakeup bit without it. + */ + if (core_enq_flags & ENQUEUE_WAKEUP) rq->scx.flags |= SCX_RQ_IN_WAKEUP; /* - * Restoring a running task will be immediately followed by - * set_next_task_scx() which expects the task to not be on the BPF + * Restoring the current scheduling context will be immediately followed + * by set_next_task_scx() which expects the task to not be on the BPF * scheduler as tasks can only start running through local DSQs. Force * direct-dispatch into the local DSQ by setting the sticky_cpu. Mark * IGNORE_CAPS to force entry into the local DSQ. */ - if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) { + if (unlikely(enq_flags & ENQUEUE_RESTORE) && + task_current_donor(rq, p)) { sticky_cpu = cpu_of(rq); enq_flags |= SCX_ENQ_IGNORE_CAPS; } @@ -2174,9 +2411,6 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ dl_server_start(&rq->ext_server); scx_do_enqueue_task(rq, p, enq_flags, sticky_cpu); - - if (sticky_cpu >= 0) - p->scx.sticky_cpu = -1; out: rq->scx.flags &= ~SCX_RQ_IN_WAKEUP; @@ -2210,7 +2444,7 @@ retry: /* * A queued task must always be in BPF scheduler's custody. If * SCX_TASK_IN_CUSTODY is clear, finish_dispatch() on another - * CPU has already passed call_task_dequeue() (which clears the + * CPU has already passed task_leave_custody() (which clears the * flag), but has not yet written SCX_OPSS_NONE. That final * store does not require this rq's lock, so retrying with * cpu_relax() is bounded: we will observe NONE (or DISPATCHING, @@ -2258,7 +2492,8 @@ retry: * NONE but the task may still have %SCX_TASK_IN_CUSTODY set until * it is enqueued on the destination. */ - call_task_dequeue(sch, rq, p, deq_flags); + if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue)) + SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags); } static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_flags) @@ -2281,10 +2516,10 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ ops_dequeue(rq, p, deq_flags); /* - * A currently running task which is going off @rq first gets dequeued - * and then stops running. As we want running <-> stopping transitions - * to be contained within runnable <-> quiescent transitions, trigger - * ->stopping() early here instead of in put_prev_task_scx(). + * A current scheduling context which is going off @rq first gets + * dequeued and then stops running. As we want running <-> stopping + * transitions to be contained within runnable <-> quiescent transitions, + * trigger ->stopping() early here instead of in put_prev_task_scx(). * * @p may go through multiple stopping <-> running transitions between * here and put_prev_task_scx() if task attribute changes occur while @@ -2292,11 +2527,13 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ * information meaningful to the BPF scheduler and can be suppressed by * skipping the callbacks if the task is !QUEUED. */ - if (task_current(rq, p) && - (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq)))) { - update_curr_scx(rq); - if (SCX_HAS_OP(sch, stopping)) - SCX_CALL_OP_TASK(sch, stopping, rq, p, false); + if (task_current_donor(rq, p) && (p->scx.flags & SCX_TASK_RUN_TRACKED)) { + if (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq))) { + update_curr_scx(rq); + if (SCX_HAS_OP(sch, stopping)) + SCX_CALL_OP_TASK(sch, stopping, rq, p, false); + } + p->scx.flags &= ~SCX_TASK_RUN_TRACKED; } if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p)) @@ -2312,9 +2549,10 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ sub_nr_running(rq, 1); scx_dispatch_dequeue(rq, p); + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; /* see scx_task_slice_ended() for the save/restore exception */ - if (!((deq_flags & DEQUEUE_SAVE) && task_current(rq, p))) + if (!((deq_flags & DEQUEUE_SAVE) && task_current_donor(rq, p))) scx_task_slice_ended(rq, p); clear_direct_dispatch(p); @@ -2354,10 +2592,23 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl /* * Preemption between SCX tasks is implemented by resetting the victim * task's slice to 0 and triggering reschedule on the target CPU. - * Nothing to do. - */ - if (p->sched_class == &ext_sched_class) + * + * A retained proxy donor can wake through ttwu_runnable() without another + * ops.enqueue(). WF_TTWU_RQ identifies this path. Request rescheduling so + * that ops.dispatch() can reconsider the task after ttwu_runnable() clears + * is_blocked. A full wakeup activation has already enqueued the task and + * doesn't need the additional reschedule. + */ + if (p->sched_class == &ext_sched_class) { + if (sched_proxy_exec() && (wake_flags & WF_TTWU_RQ) && + p->is_blocked) { + struct scx_sched *sch = scx_task_sched(p); + + if (sch && (sch->ops.flags & SCX_OPS_ENQ_BLOCKED)) + resched_curr(rq); + } return; + } /* * Getting preempted by a higher-priority class. Reenqueue IMMED tasks. @@ -2374,19 +2625,18 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl } void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p, - u64 enq_flags, struct scx_dispatch_q *src_dsq, - struct rq *dst_rq) + u64 enq_flags, struct rq *dst_rq) { - struct scx_dispatch_q *dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags); + struct scx_dispatch_q *dst_dsq; - /* @p is on @dst_rq, an rq-owned @src_dsq is covered by the rq lock */ - if (!dsq_is_rq_owned(src_dsq)) - lockdep_assert_held(&src_dsq->lock); lockdep_assert_rq_held(dst_rq); WARN_ON_ONCE(p->scx.holding_cpu >= 0); + scx_wait_dispatching(p); + + dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags); - if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) + if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) dsq_insert_head(dst_dsq, p); else list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list); @@ -2456,8 +2706,10 @@ static void move_remote_task_to_local_dsq(struct scx_sched *sch, * - The BPF scheduler is bypassed while the rq is offline and we can always say * no to the BPF scheduler initiated migrations while offline. * - * The caller must ensure that @p and @rq are on different CPUs. - * If enforce == true, caller must hold @p's rq lock. + * The caller must ensure that @p and @rq are on different CPUs. If @enforce is + * true, report violations attributable to BPF-directed migrations. The caller + * must hold @p's rq lock to avoid reporting a transient race as a scheduler + * error. */ static bool task_can_run_on_remote_rq(struct scx_sched *sch, struct task_struct *p, struct rq *rq, @@ -2465,17 +2717,27 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, { s32 cpu = cpu_of(rq); - /* - * To prevent races with @p still running on its old CPU while switching - * out, make sure we're holding @p's rq lock so as not to risk - * erroneously killing the BPF scheduler. - */ if (enforce) lockdep_assert_rq_held(task_rq(p)); WARN_ON_ONCE(task_cpu(p) == cpu); /* + * proxy_set_task_cpu() preserves wake_cpu when moving a donor's scheduling + * context to its lock owner's rq. For example, if BPF places donor D on + * CPU0 while its lock owner runs on CPU1, proxy execution moves D to CPU1 + * but leaves D->wake_cpu pointing to CPU0. If D is later put on a shared + * DSQ, CPU0 could otherwise consume it and move it back, only for proxy + * execution to return it to CPU1 again. + * + * When task_cpu() == wake_cpu, the donor has not been proxy-migrated and + * BPF may still choose its placement. An actively donating task is + * rejected separately by task_proxy_running_or_donating(). + */ + if (sched_proxy_exec() && p->is_blocked && task_cpu(p) != p->wake_cpu) + return false; + + /* * If @p has migration disabled, @p->cpus_ptr is updated to contain only * the pinned CPU in migrate_disable_switch() while @p is being switched * out. However, put_prev_task_scx() is called before @p->cpus_ptr is @@ -2516,6 +2778,64 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, return true; } +/* + * Proxy execution can change @p's execution and migration-disabled state + * without touching its DSQ entry or clearing holding_cpu. Check those states + * with @p's rq locked. Without proxy execution, the holding_cpu handshake is + * sufficient and this must not affect the existing migration path. + * + * A BPF-directed transfer to a remote local DSQ performs a normal task + * migration and thus cannot move a migration-disabled task. In contrast, + * proxy_migrate_task() moves only a blocked donor's scheduling context towards + * the mutex owner and preserves its execution home in wake_cpu. The latter is + * therefore allowed even when the donor is migration-disabled. + */ +static bool task_proxy_running_or_donating(struct task_struct *p) +{ + struct rq *src_rq = task_rq(p); + + lockdep_assert_rq_held(src_rq); + + if (!sched_proxy_exec()) + return false; + + /* @p may be rq->curr under another task's scheduling context. */ + if (task_on_cpu(src_rq, p)) + return true; + + /* Don't move an active scheduling context off its source rq. */ + if (task_current_donor(src_rq, p)) + return true; + + return false; +} + +static bool task_proxy_unsafe_to_move(struct task_struct *p) +{ + if (!sched_proxy_exec()) + return false; + + return task_proxy_running_or_donating(p) || is_migration_disabled(p); +} + +/* + * Park a task whose remote transfer raced with proxy execution. Reenqueueing + * from the source rq makes the task's owning scheduler choose its placement + * again and preserves sub-scheduler containment. + */ +static void scx_proxy_reject_task(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 enq_flags) +{ + lockdep_assert_rq_held(rq); + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); + + p->scx.holding_cpu = -1; + p->scx.flags |= SCX_TASK_REENQ_PROXY; + scx_divert_strip_flags(p, &enq_flags); + + scx_dispatch_enqueue(sch, rq, &rq->scx.reject_dsq, p, 0, 0, enq_flags); +} + /** * unlink_dsq_and_switch_rq_lock() - Unlink task and switch to its rq lock * @p: target task @@ -2544,7 +2864,7 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, * points to this CPU. See scx_dispatch_dequeue() for the counterpart. * * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is - * still valid. %false if lost to dequeue. + * still valid and detached from @dsq. %false if lost to dequeue. */ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, struct scx_dispatch_q *dsq, @@ -2564,8 +2884,19 @@ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, switch_rq_lock(locked_rq, src_rq); /* task_rq couldn't have changed if we're still the holding cpu */ - return likely(p->scx.holding_cpu == cpu) && - !WARN_ON_ONCE(src_rq != task_rq(p)); + if (likely(p->scx.holding_cpu == cpu) && !WARN_ON_ONCE(src_rq != task_rq(p))) { + /* + * Keep ->dsq set until we own @src_rq so that a racing dequeue + * can find @dsq and clear holding_cpu. Once ownership is + * confirmed, clear it under @src_rq so that deactivate_task() + * takes the !dsq path instead of retaking @dsq->lock. + */ + p->scx.dsq = NULL; + scx_wait_dispatching(p); + return true; + } + + return false; } static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq, @@ -2573,6 +2904,19 @@ static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq, struct scx_dispatch_q *dsq, struct rq *src_rq) { if (unlink_dsq_and_switch_rq_lock(p, dsq, this_rq, src_rq)) { + /* + * Proxy execution may have changed @p's running or + * migration-disabled state while switching rq locks without + * clearing holding_cpu. Park it on the source rq and let its + * owning scheduler choose its placement again. + */ + if (unlikely(task_proxy_unsafe_to_move(p))) { + p->scx.dsq = NULL; + scx_proxy_reject_task(sch, src_rq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); + switch_rq_lock(src_rq, this_rq); + return false; + } + move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, this_rq); return true; } else { @@ -2608,8 +2952,23 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, lockdep_assert_held(&src_dsq->lock); lockdep_assert_rq_held(src_rq); + scx_wait_dispatching(p); + if (dst_dsq->id == SCX_DSQ_LOCAL) { dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); + /* + * Unlike the rq-lock handoff paths, @src_rq has been locked + * throughout this operation. Only active proxy state can race the + * move here; let the enforcing check below diagnose an ordinary + * migration-disabled task. + */ + if (src_rq != dst_rq && + unlikely(task_proxy_running_or_donating(p))) { + dispatch_dequeue_locked(p, src_dsq); + raw_spin_unlock(&src_dsq->lock); + scx_proxy_reject_task(sch, src_rq, p, enq_flags); + return src_rq; + } if (src_rq != dst_rq && unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { dst_dsq = find_global_dsq(sch, task_cpu(p)); @@ -2629,9 +2988,10 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, /* @p is going from a non-local DSQ to a local DSQ */ if (src_rq == dst_rq) { scx_task_unlink_from_dsq(p, src_dsq); - scx_move_local_task_to_local_dsq(sch, p, enq_flags, src_dsq, dst_rq); raw_spin_unlock(&src_dsq->lock); + scx_move_local_task_to_local_dsq(sch, p, enq_flags, dst_rq); } else { + dispatch_dequeue_locked(p, src_dsq); raw_spin_unlock(&src_dsq->lock); move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); } @@ -2680,8 +3040,8 @@ retry: if (rq == task_rq) { scx_task_unlink_from_dsq(p, dsq); - scx_move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq); raw_spin_unlock(&dsq->lock); + scx_move_local_task_to_local_dsq(sch, p, enq_flags, rq); return true; } @@ -2766,6 +3126,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, if (likely(p->scx.holding_cpu == raw_smp_processor_id()) && !WARN_ON_ONCE(src_rq != task_rq(p))) { bool fallback = false; + /* * If @p is staying on the same rq, there's no need to go * through the full deactivate/activate cycle. Optimize by @@ -2775,6 +3136,9 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, p->scx.holding_cpu = -1; scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, slice, vtime, enq_flags | SCX_ENQ_APPLY_SLICE); + } else if (unlikely(task_proxy_unsafe_to_move(p))) { + fallback = true; + scx_proxy_reject_task(sch, src_rq, p, enq_flags); } else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { p->scx.holding_cpu = -1; fallback = true; @@ -2790,7 +3154,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, } /* if the destination CPU is idle, wake it up */ - if (!fallback && sched_class_above(p->sched_class, dst_rq->curr->sched_class)) + if (!fallback && sched_class_above(p->sched_class, + dst_rq->donor->sched_class)) resched_curr(dst_rq); } @@ -3008,10 +3373,39 @@ has_tasks: return verdict; } -static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) +static void scx_start_task_running(struct rq *rq, struct task_struct *p) { struct scx_sched *sch = scx_task_sched(p); + if (p->scx.flags & SCX_TASK_RUN_TRACKED) + return; + + scx_cid_sched_update(rq, sch); + + if (SCX_HAS_OP(sch, running)) + SCX_CALL_OP_TASK(sch, running, rq, p); + + p->scx.flags |= SCX_TASK_RUN_TRACKED; +} + +static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e type) +{ + bool first = type == SNT_PICK; + bool can_stop_tick; + + /* + * A blocked pick is provisional until proxy resolution completes, see + * put_prev_task_scx(). A retry can repick the same donor, so + * SNT_REPICK must set it again. + */ + rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; + if (sched_proxy_exec() && p->is_blocked && + (type == SNT_PICK || type == SNT_REPICK)) + rq->scx.flags |= SCX_RQ_PROXY_PENDING; + + if (type == SNT_REPICK) + return; + if (p->scx.flags & SCX_TASK_QUEUED) { /* * Core-sched might decide to execute @p before it is @@ -3019,18 +3413,31 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) */ ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC); scx_dispatch_dequeue(rq, p); + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } p->se.exec_start = rq_clock_task(rq); - /* see dequeue_task_scx() on why we skip when !QUEUED */ - if (SCX_HAS_OP(sch, running) && (p->scx.flags & SCX_TASK_QUEUED)) - SCX_CALL_OP_TASK(sch, running, rq, p); + /* + * See dequeue_task_scx() for why we skip when !QUEUED. + * + * During a normal switch (@first), a blocked task is only a provisional + * donor. Proxy resolution may fail or migrate the donor to another CPU, + * so defer ops.running() until scx_proxy_donor_start() confirms that + * resolution succeeded. + * + * !@first denotes restoration after a SAVE/RESTORE cycle. The matching + * dequeue already issued ops.stopping(), so restart the session here + * regardless of the donor state. + */ + if ((p->scx.flags & SCX_TASK_QUEUED) && !(p->is_blocked && first)) + scx_start_task_running(rq, p); clr_task_runnable(p, true); /* apply any pending out-of-band slice request before the tick decision */ apply_task_slice_oob(rq, p); + can_stop_tick = p->scx.slice == SCX_SLICE_INF && !p->is_blocked; /* * @p is getting newly scheduled or got kicked after someone updated its @@ -3041,7 +3448,7 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) * nohz. In the future, we might want to add a mechanism to update * load_avgs periodically on tick-stopped CPUs. */ - if (p->scx.slice == SCX_SLICE_INF) { + if (can_stop_tick) { if (!(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) { /* * Bypass mode always assigns finite slices, so @p @@ -3062,13 +3469,25 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) /* * @rq still references the outgoing scheduling context. A finite - * slice is sufficient by itself to require the tick. + * slice or a blocked proxy donor is sufficient by itself to require + * the tick. */ if (tick_nohz_full_cpu(cpu_of(rq))) tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED); } } +void scx_proxy_donor_start(struct rq *rq) +{ + struct task_struct *donor = rq->donor; + + lockdep_assert_rq_held(rq); + rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; + + if (donor->sched_class == &ext_sched_class && (donor->scx.flags & SCX_TASK_QUEUED)) + scx_start_task_running(rq, donor); +} + static enum scx_cpu_preempt_reason preempt_reason_from_class(const struct sched_class *class) { @@ -3124,8 +3543,13 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, struct task_struct *next) { struct scx_sched *sch = scx_task_sched(p); + bool proxy_put = p->is_blocked && next == rq->idle && + (rq->scx.flags & SCX_RQ_PROXY_PENDING); + bool proxy_block = rq->scx.flags & SCX_RQ_PROXY_BLOCKING; bool rescue_keep = false; + rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; + /* see kick_sync_wait_bal_cb() */ smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); @@ -3142,29 +3566,51 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, scx_task_slice_ended(rq, p); } - /* see dequeue_task_scx() on why we skip when !QUEUED */ - if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED)) - SCX_CALL_OP_TASK(sch, stopping, rq, p, true); + /* + * Preserve the running session when proxy execution refreshes the same + * donor around an execution-context switch on this rq. + */ + if (next != p && (p->scx.flags & SCX_TASK_QUEUED) && + (p->scx.flags & SCX_TASK_RUN_TRACKED)) { + if (SCX_HAS_OP(sch, stopping)) + SCX_CALL_OP_TASK(sch, stopping, rq, p, !proxy_block); + + p->scx.flags &= ~SCX_TASK_RUN_TRACKED; + } if (p->scx.flags & SCX_TASK_QUEUED) { + if (proxy_block) + goto switch_class; + set_task_runnable(rq, p); /* + * If the donor is the same and only the mutex owner changes, + * avoid triggering another ops.enqueue(): the BPF scheduler has + * already admitted the donor, so it can continue running. + */ + if (p->is_blocked && next == p) + goto switch_class; + + /* * If @p has slice left and is being put, @p is getting * preempted by a higher priority scheduler class or core-sched * forcing a different task. Leave it at the head of the local * DSQ unless it was an IMMED task. IMMED tasks should not * linger on a busy CPU, reenqueue them to the BPF scheduler. * + * The exception is an IMMED donor put by proxy_resched_idle(): + * the CPU isn't busy, proxy resolution is only retrying through + * idle, so keep the donor where it can be picked again. + * * An open rescue must keep @p on the local DSQ even if the * scheduler zeroed the slice in ops.stopping() above. */ if ((p->scx.slice || unlikely(p == scx_rescuee(rq))) && !scx_bypassing(sch, cpu_of(rq))) { - if (p->scx.flags & SCX_TASK_IMMED) { + if ((p->scx.flags & SCX_TASK_IMMED) && !proxy_put) { p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } else { u64 enq_flags = 0; @@ -3179,6 +3625,9 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, enq_flags |= SCX_ENQ_HEAD; } else { enq_flags |= SCX_ENQ_HEAD; + /* only require SCX_CAP_ENQ_IMMED, see scx_caps_for_enq() */ + if (proxy_put && (p->scx.flags & SCX_TASK_IMMED)) + enq_flags |= SCX_ENQ_IMMED; } scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, @@ -3190,27 +3639,40 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, /* * If @p is runnable but we're about to enter a lower * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell - * ops.enqueue() that @p is the only one available for this cpu, - * which should trigger an explicit follow-up scheduling event. - * This doesn't apply if the baseline access on the CPU is lost. + * ops.enqueue() that @p is the only one available for this cpu. + * This doesn't apply if the baseline access on the CPU is lost or + * proxy resolution temporarily switches to idle. * * Under core scheduling, a pick dispatches only when nothing is * locally runnable and can legitimately go idle with @p still * runnable (see do_pick_task_scx()). */ - if (next && sched_class_above(&ext_sched_class, next->sched_class) && + if (!proxy_put && next && + sched_class_above(&ext_sched_class, next->sched_class) && scx_task_can_stay_on_cpu(rq, p)) { WARN_ON_ONCE(!sched_core_enabled(rq) && !(sch->ops.flags & SCX_OPS_ENQ_LAST)); scx_do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); + + /* + * A task put on the local DSQ runs, as anywhere else. + * Here the insert can't reschedule on its own: the pick + * has settled on @next and @p is still curr, so + * resched_curr() would flag @p and __schedule() clears + * that right after the pick. Flag @next instead. + */ + if (p->scx.dsq == &rq->scx.local_dsq) + set_tsk_need_resched(next); } else { scx_do_enqueue_task(rq, p, 0, -1); } } switch_class: - if (next && next->sched_class != &ext_sched_class) + if (next && next->sched_class != &ext_sched_class) { + scx_cid_sched_update(rq, NULL); switch_class(rq, next); + } } static void kick_sync_wait_bal_cb(struct rq *rq) @@ -3278,7 +3740,8 @@ static struct task_struct *first_local_task(struct rq *rq) } /* - * Run dispatch and queue the follow-up work for a pick. + * Run dispatch for a pick. The follow-up work is queued by do_pick_task_scx() + * once the pick is final. */ static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf, struct task_struct *prev) @@ -3288,18 +3751,6 @@ static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf, rq_unpin_lock(rq, rf); verdict = dispatch_one(rq, prev); rq_repin_lock(rq, rf); - maybe_queue_balance_callback(rq); - - /* - * Defer to a balance callback which can drop rq lock and enable IRQs. - * Waiting directly in the pick path would deadlock against CPUs sending - * us IPIs (e.g. TLB flushes) while we wait for them. - */ - if (unlikely(rq->scx.kick_sync_pending)) { - rq->scx.kick_sync_pending = false; - queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, - kick_sync_wait_bal_cb); - } return verdict; } @@ -3326,23 +3777,20 @@ static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *r verdict = dispatch_one(rq, prev); - if (cpu_of(rq) == smp_processor_id()) { - maybe_queue_balance_callback(rq); - - /* see dispatch_pick() */ - if (unlikely(rq->scx.kick_sync_pending)) { - rq->scx.kick_sync_pending = false; - queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, - kick_sync_wait_bal_cb); - } - } else if (unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { - /* - * Balance callbacks must run in the context that queued them, - * so they can't be queued on another CPU's rq. Run the deferred - * work directly instead. - */ + /* + * A pick that returns a task isn't final under core scheduling: the + * selection goes on to pick the siblings, and their dispatch can + * release the rq lock, which can't happen with balance callbacks + * queued. Don't queue the deferred work as a balance callback, for this + * CPU or a sibling, and run it from irq_work instead. + * + * do_pick_task_scx() leaves %SCX_RQ_BAL_CB_PENDING alone under core + * scheduling, so that the early return above leaves it to the dispatch + * in flight, which handles it here. + */ + if (unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; - run_deferred(rq); + schedule_deferred(rq); } rq_repin_lock(rq, rf); @@ -3364,22 +3812,33 @@ static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *r static struct task_struct * do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) { - struct task_struct *prev = rq->curr; + struct task_struct *prev = rq->donor; + bool core_sched = sched_core_enabled(rq); enum scx_dsp_verdict verdict; struct task_struct *p; + /* A retry can abandon a provisional blocked-donor pick. */ + rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; + /* see kick_sync_wait_bal_cb() */ smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); rq_modified_begin(rq, &ext_sched_class); - if (sched_core_enabled(rq)) + /* + * Core scheduling can get enabled while dispatch_pick() has the rq lock + * released. Stick to @core_sched, so that the follow-up work below + * matches the dispatch that ran. + */ + if (core_sched) verdict = dispatch_core_pick(rq, rf, prev); else verdict = dispatch_pick(rq, rf, prev); - if (verdict == SCX_DSP_RETRY) - return RETRY_TASK; + if (verdict == SCX_DSP_RETRY) { + p = RETRY_TASK; + goto out; + } /* * If any higher-priority sched class enqueued a runnable task on this @@ -3389,8 +3848,10 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) * If @force_scx is true, always try to pick a SCHED_EXT task, * regardless of any higher-priority sched classes activity. */ - if (!force_scx && rq_modified_above(rq, &ext_sched_class)) - return RETRY_TASK; + if (!force_scx && rq_modified_above(rq, &ext_sched_class)) { + p = RETRY_TASK; + goto out; + } /* * If we're keeping @prev, replenish slice if necessary and keep running @@ -3406,7 +3867,7 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) } else { p = first_local_task(rq); if (!p) - return NULL; + goto out; if (unlikely(!p->scx.slice) && scx_task_can_stay_on_cpu(rq, p)) { struct scx_sched *sch = scx_task_sched(p); @@ -3421,6 +3882,40 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) } } +out: + if (cpu_of(rq) != smp_processor_id()) + return p; + + /* + * Balance callbacks must be consumed before the rq lock is released, so + * queue them only when the pick is final. On RETRY_TASK, or after a + * NULL pick from the dl_server, the core continues picking and can + * release the rq lock (e.g., prev_balance() or another dispatch). In + * that case, run the deferred work from irq_work, which doesn't depend + * on the current rq lock section, and leave the kick sync wait pending + * for the next pick. + */ + if (p != RETRY_TASK && (p || !force_scx)) { + /* under core scheduling, see dispatch_core_pick() */ + if (!core_sched) + maybe_queue_balance_callback(rq); + + /* + * Defer to a balance callback which can drop rq lock and enable + * IRQs. Waiting directly in the pick path would deadlock against + * CPUs sending us IPIs (e.g., TLB flushes) while we wait for + * them. + */ + if (unlikely(rq->scx.kick_sync_pending)) { + rq->scx.kick_sync_pending = false; + queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, + kick_sync_wait_bal_cb); + } + } else if (!core_sched && (rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { + rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; + schedule_deferred(rq); + } + return p; } @@ -3629,8 +4124,12 @@ static void set_cpus_allowed_scx(struct task_struct *p, * * Fine-grained memory write control is enforced by BPF making the const * designation pointless. Cast it away when calling the operation. + * + * The cid form receives the initial mask when the task is enabled and + * hears about changes only afterwards, see struct scx_enable_args. */ - if (SCX_HAS_OP(sch, set_cpumask)) + if (SCX_HAS_OP(sch, set_cpumask) && + (!scx_is_cid_type() || scx_get_task_state(p) == SCX_TASK_ENABLED)) scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr); } @@ -3714,8 +4213,26 @@ static void rq_online_scx(struct rq *rq) static void rq_offline_scx(struct rq *rq) { + struct task_struct *p, *n; + rq->scx.flags &= ~SCX_RQ_ONLINE; + + /* sched domain rebuilds call rq_offline with the CPU staying alive */ + if (cpu_active(cpu_of(rq))) + return; + scx_rescue_flush(rq); + + /* + * An offline CPU no longer calls ops.dispatch(). Re-enqueue its tasks + * onto the local DSQ so that they run here and balance_push() moves + * them off. + */ + list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, scx.runnable_node) { + if (p->scx.dsq == &rq->scx.local_dsq) + continue; + guard(sched_change)(p, DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK); + } } static bool check_rq_for_timeouts(struct rq *rq) @@ -3806,9 +4323,9 @@ void scx_tick(struct rq *rq) update_other_load_avgs(rq); } -static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) +static void task_tick_scx(struct rq *rq, struct task_struct *donor, int queued) { - struct scx_sched *sch = scx_task_sched(curr); + struct scx_sched *sch = scx_task_sched(donor); update_curr_scx(rq); @@ -3817,12 +4334,18 @@ static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) * management. */ if (scx_bypassing(sch, cpu_of(rq))) - scx_set_task_slice(curr, 0); + scx_set_task_slice(donor, 0); else if (SCX_HAS_OP(sch, tick)) - SCX_CALL_OP_TASK(sch, tick, rq, curr); + SCX_CALL_OP_TASK(sch, tick, rq, donor); - if (!curr->scx.slice) - resched_curr(rq); + if (!donor->scx.slice) { + /* the slice can't be trusted while bypassing */ + if (READ_ONCE(donor->scx.lazy_resched) && + !scx_bypassing(sch, cpu_of(rq))) + scx_resched_curr_lazy(rq); + else + resched_curr(rq); + } } #ifdef CONFIG_EXT_GROUP_SCHED @@ -3938,9 +4461,29 @@ static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p) weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; p->scx.weight = sched_weight_to_cgroup(weight); + p->scx.lazy_resched = sch->ops.flags & SCX_OPS_LAZY_RESCHED; + + if (SCX_HAS_OP(sch, enable)) { + if (scx_is_cid_type()) { + struct scx_cmask *cmask = scx_fill_cmask_scratch(sch, p->cpus_ptr); + struct scx_enable_args args = { + .cmask_arena_addr = scx_kaddr_to_arena(sch, cmask), + }; + + SCX_CALL_CID_OP_TASK(sch, enable, rq, p, &args); + } else { + SCX_CALL_OP_TASK(sch, enable, rq, p); + } + } - if (SCX_HAS_OP(sch, enable)) - SCX_CALL_OP_TASK(sch, enable, rq, p); + /* + * The initial mask also goes out through set_cmask() so a scheduler can + * track affinity there alone, and before set_weight() so that the mask + * is in place when weight-dependent state is derived, see struct + * scx_enable_args. + */ + if (scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask)) + scx_call_op_set_cpumask(sch, rq, p, p->cpus_ptr); if (SCX_HAS_OP(sch, set_weight)) SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); @@ -4283,14 +4826,22 @@ static void switching_to_scx(struct rq *rq, struct task_struct *p) /* * set_cpus_allowed_scx() is not called while @p is associated with a - * different scheduler class. Keep the BPF scheduler up-to-date. + * different scheduler class. Keep the BPF scheduler up-to-date. The cid + * form gets its mask from scx_enable_task(). */ - if (SCX_HAS_OP(sch, set_cpumask)) + if (!scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask)) scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr); } static void switched_from_scx(struct rq *rq, struct task_struct *p) { + /* + * A class change of the running task: sched_change_begin() put @p with + * no successor, which leaves rq->scx.sched set. + */ + if (task_current_donor(rq, p)) + scx_cid_sched_update(rq, NULL); + if (task_dead_and_done(p)) return; @@ -4313,7 +4864,7 @@ int scx_check_setscheduler(struct task_struct *p, int policy) { lockdep_assert_rq_held(task_rq(p)); - /* if disallow, reject transitioning into SCX */ + /* If disallow, reject transitioning into SCX. */ if (scx_enabled() && READ_ONCE(p->scx.disallow) && p->policy != policy && policy == SCHED_EXT) return -EACCES; @@ -4321,6 +4872,20 @@ int scx_check_setscheduler(struct task_struct *p, int policy) return 0; } +/* + * Don't carry a donor retained by another class into sched_ext. The caller + * has updated the rq clock and invokes this immediately before + * sched_change_begin() records the task's queued state. + */ +void scx_prepare_setscheduler(struct task_struct *p, int policy) +{ + lockdep_assert_held(&p->pi_lock); + lockdep_assert_rq_held(task_rq(p)); + + if (scx_enabled() && p->policy != policy && policy == SCHED_EXT) + scx_block_proxy_donor(task_rq(p), p); +} + static void process_ddsp_deferred_locals(struct rq *rq) { struct task_struct *p; @@ -4404,6 +4969,18 @@ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, return *reenq_flags & SCX_REENQ_ANY; } +/* + * The dispatcher runs ops.dequeue() and stores the final ops_state after + * dropping the DSQ lock, so @p can be found on a DSQ while still + * %SCX_OPSS_DISPATCHING. Wait for the dispatch to finish before operating on + * @p. + */ +void scx_wait_dispatching(struct task_struct *p) +{ + if (unlikely(atomic_long_read_acquire(&p->scx.ops_state) == SCX_OPSS_DISPATCHING)) + wait_ops_state(p, SCX_OPSS_DISPATCHING); +} + static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) { LIST_HEAD(tasks); @@ -4447,10 +5024,10 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) if (!local_task_should_reenq(rq, p, &reenq_flags, &reason)) continue; + scx_wait_dispatching(p); scx_dispatch_dequeue(rq, p); - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); p->scx.flags |= reason; list_add_tail(&p->scx.dsq_list.node, &tasks); @@ -4461,21 +5038,20 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; nr_enqueued++; } /* - * The revoke that scheduled this scan may have raced the pick: curr + * The revoke that scheduled this scan may have raced the pick: donor * may be a now-capless task, either one that kept running or one * promoted off the local DSQ between the ecaps sync and this scan. * Zero the slice to evict it. The enqueue gate blocks new capless * inserts, so no later pick can slip through after the scan. */ if ((reenq_flags & SCX_REENQ_CAP_REVOKE) && - rq->curr->sched_class == &ext_sched_class && - scx_task_reenq_on_cap_revoke(rq, rq->curr)) { - scx_set_task_slice(rq->curr, 0); + rq->donor->sched_class == &ext_sched_class && + scx_task_reenq_on_cap_revoke(rq, rq->donor)) { + scx_set_task_slice(rq->donor, 0); resched_curr(rq); } @@ -4524,7 +5100,9 @@ static void process_deferred_reenq_locals(struct rq *rq) static bool user_task_should_reenq(struct task_struct *p, u64 reenq_flags, u32 *reason) { *reason = SCX_TASK_REENQ_KFUNC; - return reenq_flags & SCX_REENQ_ANY; + if (reenq_flags & SCX_REENQ_ANY) + return true; + return (reenq_flags & SCX_REENQ_MARKED) && (p->scx.dsq_flags & SCX_TASK_DSQ_REENQ); } static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flags) @@ -4533,7 +5111,7 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag struct scx_sched *sch = dsq->sched; struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, dsq, 0); struct task_struct *p; - s32 nr_enqueued = 0; + s32 nr_visited = 0; lockdep_assert_rq_held(rq); @@ -4543,6 +5121,21 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag struct rq *task_rq; u32 reason; + /* + * Count skipped tasks too so that a long walk doesn't hold the + * locks throughout. The cursor keeps the position. + */ + if (!(++nr_visited % SCX_TASK_ITER_BATCH)) { + raw_spin_unlock(&dsq->lock); + if (locked_rq) { + scx_rq_lock_drop(locked_rq); + raw_spin_rq_unlock(locked_rq); + locked_rq = NULL; + } + cpu_relax(); + raw_spin_lock(&dsq->lock); + } + p = nldsq_cursor_next_task(&cursor, dsq); if (!p) break; @@ -4570,24 +5163,15 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag } /* @p is on @dsq, its rq and @dsq are locked */ + scx_wait_dispatching(p); dispatch_dequeue_locked(p, dsq); raw_spin_unlock(&dsq->lock); - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); p->scx.flags |= reason; scx_do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1); - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - - if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) { - scx_rq_lock_drop(locked_rq); - raw_spin_rq_unlock(locked_rq); - locked_rq = NULL; - cpu_relax(); - } - raw_spin_lock(&dsq->lock); } @@ -4642,6 +5226,60 @@ static void process_deferred_reenq_users(struct rq *rq) } } +/* + * Drain ready tasks from @rq->scx.reject_dsq and reenqueue them so that their + * owning BPF schedulers choose placement again. Proxy-active tasks remain + * parked and rearm the retry notification for a later proxy resolution. + * + * A task can be re-rejected repeatedly. Reenqueues are bounded per task by + * SCX_REENQ_MAX_REPEAT in scx_do_enqueue_task(), which ejects the owning + * scheduler. + */ +static void scx_reenq_reject(struct rq *rq) +{ + LIST_HEAD(tasks); + struct task_struct *p, *n; + + lockdep_assert_rq_held(rq); + + if (list_empty(&rq->scx.reject_dsq.list)) + return; + + /* + * Move ready tasks to a private list so a task re-rejected by + * scx_do_enqueue_task() below isn't revisited this round. + */ + list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) { + u32 reason = p->scx.flags & SCX_TASK_REENQ_REASON_MASK; + + WARN_ON_ONCE(!reason); + + if (sched_proxy_exec() && reason == SCX_TASK_REENQ_PROXY) { + /* Affinity machinery will dequeue and reactivate @p. */ + if (p->migration_pending) + continue; + + if (task_on_cpu(rq, p) || task_current_donor(rq, p)) { + rq->scx.flags |= SCX_RQ_PROXY_RETRY; + continue; + } + } else { + WARN_ON_ONCE(p->migration_pending); + } + + scx_wait_dispatching(p); + scx_dispatch_dequeue(rq, p); + + list_add_tail(&p->scx.dsq_list.node, &tasks); + } + + list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { + list_del_init(&p->scx.dsq_list.node); + + scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); + } +} + static void run_deferred(struct rq *rq) { process_ddsp_deferred_locals(rq); @@ -4658,14 +5296,18 @@ static void run_deferred(struct rq *rq) #ifdef CONFIG_NO_HZ_FULL bool scx_can_stop_tick(struct rq *rq) { - struct task_struct *p = rq->curr; + struct task_struct *p = rq->donor; struct scx_sched *sch = scx_task_sched(p); + /* Proxy execution is conservatively tick-driven for now. */ + if (rq->scx.flags & SCX_RQ_PROXY_TICK) + return false; + if (p->sched_class != &ext_sched_class) return true; /* - * @rq->curr may still reference an outgoing EXT task after it has been + * @rq->donor may still reference an outgoing EXT task after it has been * dequeued. If no EXT tasks are accounted on @rq, ignore its stale * slice state. If another task is dispatched from a DSQ, * set_next_task_scx() will update the dependency for the incoming task. @@ -4686,7 +5328,8 @@ bool scx_can_stop_tick(struct rq *rq) /* * @rq can dispatch from different DSQs, so we can't tell whether it * needs the tick or not by looking at nr_running. Allow stopping ticks - * iff the BPF scheduler indicated so. See set_next_task_scx(). + * iff set_next_task_scx() determined that the selected scheduling context + * can run tickless. */ return rq->scx.flags & SCX_RQ_CAN_STOP_TICK; } @@ -4701,7 +5344,7 @@ void scx_tg_init(struct task_group *tg) tg->scx.weight = CGROUP_WEIGHT_DFL; tg->scx.bw_period_us = default_bw_period_us(); tg->scx.bw_quota_us = RUNTIME_INF; - tg->scx.idle = false; + tg->scx.sched_idle = false; } /** @@ -4784,7 +5427,7 @@ int scx_tg_online(struct task_group *tg) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, - .sched_idle = tg->scx.idle }; + .sched_idle = tg->scx.sched_idle }; ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, tg->css.cgroup, &args); @@ -4951,11 +5594,11 @@ void scx_group_set_idle(struct task_group *tg, bool idle) sch = scx_tg_knob_sched(tg); if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle) && - tg->scx.idle != idle) + tg->scx.sched_idle != idle) SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); /* Update the task group's idle state */ - tg->scx.idle = idle; + tg->scx.sched_idle = idle; percpu_up_read(&scx_cgroup_ops_rwsem); } @@ -5069,12 +5712,16 @@ s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch) dsq->id = dsq_id; dsq->sched = sch; - dsq->pcpu = alloc_percpu(struct scx_dsq_pcpu); - if (!dsq->pcpu) + /* per-DSQ deferred reenq state is only needed for user DSQs */ + if (dsq_id & SCX_DSQ_FLAG_BUILTIN) + return 0; + + dsq->pcpu_user = alloc_percpu(struct scx_dsq_pcpu); + if (!dsq->pcpu_user) return -ENOMEM; for_each_possible_cpu(cpu) { - struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); + struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu); pcpu->dsq = dsq; INIT_LIST_HEAD(&pcpu->deferred_reenq_user.node); @@ -5087,14 +5734,17 @@ static void exit_dsq(struct scx_dispatch_q *dsq) { s32 cpu; + if (!dsq->pcpu_user) + return; + for_each_possible_cpu(cpu) { - struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); + struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu); struct scx_deferred_reenq_user *dru = &pcpu->deferred_reenq_user; struct rq *rq = cpu_rq(cpu); /* - * There must have been a RCU grace period since the last - * insertion and @dsq should be off the deferred list by now. + * free_dsq_irq_workfn() cancelled the reenqs before the grace + * period and schedule_dsq_reenq() queues nothing after that. */ if (WARN_ON_ONCE(!list_empty(&dru->node))) { guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); @@ -5102,7 +5752,7 @@ static void exit_dsq(struct scx_dispatch_q *dsq) } } - free_percpu(dsq->pcpu); + free_percpu(dsq->pcpu_user); } static void free_dsq_rcufn(struct rcu_head *rcu) @@ -5118,8 +5768,26 @@ static void free_dsq_irq_workfn(struct irq_work *irq_work) struct llist_node *to_free = llist_del_all(&dsqs_to_free); struct scx_dispatch_q *dsq, *tmp_dsq; - llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) + llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) { + s32 cpu; + + /* + * Cancel pending reenqs. Only a run_deferred() that started + * before the grace period can hold one, and it keeps IRQs off, + * so the grace period waits for it. After this sweep, + * schedule_dsq_reenq() sees the invalid id under the same lock + * and queues nothing. + */ + for_each_possible_cpu(cpu) { + struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu); + struct rq *rq = cpu_rq(cpu); + + guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); + list_del_init(&pcpu->deferred_reenq_user.node); + } + call_rcu(&dsq->rcu, free_dsq_rcufn); + } } static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); @@ -5206,7 +5874,7 @@ static int scx_cgroup_init(struct scx_sched *sch) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, - .sched_idle = tg->scx.idle, + .sched_idle = tg->scx.sched_idle, }; ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args); @@ -5400,6 +6068,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) free_cpumask_var(pcpu->cpus_to_kick); free_cpumask_var(pcpu->cpus_to_kick_if_idle); free_cpumask_var(pcpu->cpus_to_preempt); + free_cpumask_var(pcpu->cpus_to_preempt_lazy); free_cpumask_var(pcpu->cpus_to_wait); exit_dsq(scx_bypass_dsq(sch, cpu)); @@ -5878,7 +6547,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) /* * We don't want CPUs to have more than $nr_donor_target tasks and - * balancing to fill donee CPUs upto $nr_target. Once targets are + * balancing to fill donee CPUs up to $nr_target. Once targets are * calculated, find the donee CPUs. */ nr_target = DIV_ROUND_UP(nr_tasks, nr_cpus); @@ -5990,15 +6659,12 @@ static void enable_bypass_dsp(struct scx_sched *sch) /* * When a sub-sched bypasses, its tasks are queued on the bypass DSQs of - * the nearest non-bypassing ancestor or root. As enable_bypass_dsp() is - * called iff @sch is not already bypassed due to an ancestor bypassing, - * we can assume that the parent is not bypassing and thus will be the - * host of the bypass DSQs. - * - * While the situation may change in the future, the following - * guarantees that the nearest non-bypassing ancestor or root has bypass - * dispatch enabled while a descendant is bypassing, which is all that's - * required. + * the nearest non-bypassing ancestor or root. The parent hosts them + * unless it is bypassing too. Then they sit further up and the parent + * consumes only its own bypass DSQs, empty until it leaves bypass and + * becomes the host. Either way the nearest non-bypassing ancestor or + * root has bypass dispatch enabled while a descendant is bypassing, + * which is all that's required. * * scx_bypass_dsp_enabled() test is used to determine whether to enter * the bypass dispatch handling path from both bypassing and hosting @@ -6107,35 +6773,46 @@ void scx_bypass(struct scx_sched *sch, bool bypass) struct scx_sched *pos; unsigned long flags; int cpu; + bool changed; raw_spin_lock_irqsave(&scx_bypass_lock, flags); if (bypass) { - if (!inc_bypass_depth(sch)) - goto unlock; - - enable_bypass_dsp(sch); + changed = inc_bypass_depth(sch); + if (changed) + enable_bypass_dsp(sch); } else { - if (!dec_bypass_depth(sch)) - goto unlock; + changed = dec_bypass_depth(sch); } /* * Bypass state is propagated to all descendants - an scx_sched bypasses - * if itself or any of its ancestors are in bypass mode. + * if itself or any of its ancestors are in bypass mode. A descendant's + * depth is the sum of its own bypasses and its ancestors', so every + * call propagates whether or not @sch changed state. The descendants' + * claims are released below with @sch's, after their tasks have moved. */ raw_spin_lock(&scx_sched_lock); scx_for_each_descendant_pre(pos, sch) { if (pos == sch) continue; - if (bypass) - inc_bypass_depth(pos); - else + if (bypass) { + if (inc_bypass_depth(pos)) + enable_bypass_dsp(pos); + } else { dec_bypass_depth(pos); + } } raw_spin_unlock(&scx_sched_lock); /* + * A descendant's depth is at least @sch's, so no bypass state changed + * unless @sch's did. + */ + if (!changed) + goto unlock; + + /* * No task property is changing. We just need to make sure all currently * queued tasks are re-queued according to the new scx_bypassing() * state. As an optimization, walk each rq's runnable_list instead of @@ -6180,6 +6857,24 @@ void scx_bypass(struct scx_sched *sch, bool bypass) } /* + * Bypass trumps protection. Cycling the queued tasks below + * clears theirs. The current donor isn't on the runnable list, + * so strip its protection here. + * + * When bypass lifts, the current donor keeps running on its + * slice without a cap check. Evict it if it can't stay on the + * cpu. + */ + p = rq->donor; + if (p->sched_class == &ext_sched_class && + scx_is_descendant(scx_task_sched(p), sch)) { + if (bypass) + scx_task_slice_ended(rq, p); + else if (!scx_task_can_stay_on_cpu(rq, p)) + scx_set_task_slice(p, 0); + } + + /* * The use of list_for_each_entry_safe_reverse() is required * because each task is going to be removed from and added back * to the runnable_list during iteration. Because they're added @@ -6191,13 +6886,6 @@ void scx_bypass(struct scx_sched *sch, bool bypass) if (!scx_is_descendant(scx_task_sched(p), sch)) continue; - /* - * Bypass trumps protection. Cycling clears for queued - * tasks but current task needs explicit stripping. - */ - if (bypass && task_current(rq, p)) - scx_task_slice_ended(rq, p); - /* cycling deq/enq is enough, see the function comment */ scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { /* nothing */ ; @@ -6213,8 +6901,12 @@ void scx_bypass(struct scx_sched *sch, bool bypass) } /* disarming must come after moving all tasks out of the bypass DSQs */ - if (!bypass) - scx_disable_bypass_dsp(sch); + if (!bypass) { + guard(raw_spinlock)(&scx_sched_lock); + scx_for_each_descendant_pre(pos, sch) + if (!pos->bypass_depth) + scx_disable_bypass_dsp(pos); + } unlock: raw_spin_unlock_irqrestore(&scx_bypass_lock, flags); } @@ -6525,6 +7217,8 @@ static void scx_root_disable(struct scx_sched *sch) } } + scx_ops_cid_sched_updated_disable(sch); + /* no task is on scx, turn off all the switches and flush in-progress calls */ static_branch_disable(&__scx_enabled); static_branch_disable(&__scx_is_cid_type); @@ -6897,12 +7591,14 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, seq_buf_init(&ns, buf, avail); dump_newline(&ns); - scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", + scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ksync=%lu", cpu, rq->scx.nr_running, rq->scx.flags, rq->scx.cpu_released, - rq->scx.ops_qseq, rq->scx.kick_sync); + rq->scx.kick_sync); scx_rescue_dump(&ns, rq); scx_dump_line(&ns, " curr=%s[%d] class=%ps", rq->curr->comm, rq->curr->pid, rq->curr->sched_class); + scx_dump_line(&ns, " donor=%s[%d] class=%ps", + rq->donor->comm, rq->donor->pid, rq->donor->sched_class); if (!cpumask_empty(pcpu->cpus_to_kick)) scx_dump_line(&ns, " cpus_to_kick : %*pb", cpumask_pr_args(pcpu->cpus_to_kick)); @@ -6912,6 +7608,9 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, if (!cpumask_empty(pcpu->cpus_to_preempt)) scx_dump_line(&ns, " cpus_to_preempt: %*pb", cpumask_pr_args(pcpu->cpus_to_preempt)); + if (!cpumask_empty(pcpu->cpus_to_preempt_lazy)) + scx_dump_line(&ns, " preempt_lazy : %*pb", + cpumask_pr_args(pcpu->cpus_to_preempt_lazy)); if (!cpumask_empty(pcpu->cpus_to_wait)) scx_dump_line(&ns, " cpus_to_wait : %*pb", cpumask_pr_args(pcpu->cpus_to_wait)); @@ -6946,6 +7645,10 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, if (rq->curr->sched_class == &ext_sched_class && (dump_all_tasks || scx_task_on_sched(sch, rq->curr))) scx_dump_task(sch, s, dctx, rq, rq->curr, '*'); + if (rq->donor != rq->curr && + rq->donor->sched_class == &ext_sched_class && + (dump_all_tasks || scx_task_on_sched(sch, rq->donor))) + scx_dump_task(sch, s, dctx, rq, rq->donor, ' '); list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) if (dump_all_tasks || scx_task_on_sched(sch, p)) @@ -7229,6 +7932,7 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, if (!zalloc_cpumask_var_node(&pcpu->cpus_to_kick, GFP_KERNEL, node) || !zalloc_cpumask_var_node(&pcpu->cpus_to_kick_if_idle, GFP_KERNEL, node) || !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt, GFP_KERNEL, node) || + !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt_lazy, GFP_KERNEL, node) || !zalloc_cpumask_var_node(&pcpu->cpus_to_wait, GFP_KERNEL, node)) { ret = -ENOMEM; goto err_free_pcpu; @@ -7364,6 +8068,7 @@ err_free_pcpu: free_cpumask_var(pcpu->cpus_to_kick); free_cpumask_var(pcpu->cpus_to_kick_if_idle); free_cpumask_var(pcpu->cpus_to_preempt); + free_cpumask_var(pcpu->cpus_to_preempt_lazy); free_cpumask_var(pcpu->cpus_to_wait); } for_each_possible_cpu(cpu) { @@ -7454,6 +8159,11 @@ int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) return -EINVAL; } + if ((ops->flags & SCX_OPS_ENQ_BLOCKED) && !ops->enqueue) { + scx_error(sch, "SCX_OPS_ENQ_BLOCKED requires ops.enqueue() to be implemented"); + return -EINVAL; + } + /* * SCX_OPS_TID_TO_TASK is enabled by the root scheduler. A sub-sched * may set it to declare a dependency; reject if the root hasn't @@ -7717,6 +8427,8 @@ static void scx_root_enable_workfn(struct kthread_work *work) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op); + scx_ops_cid_sched_updated_enable(sch); + if (sch->ops.cpu_acquire || sch->ops.cpu_release) sch->ops.flags |= SCX_OPS_HAS_CPU_PREEMPT; @@ -7843,6 +8555,10 @@ static void scx_root_enable_workfn(struct kthread_work *work) if (old_class != new_class) queue_flags |= DEQUEUE_CLASS; + if (new_class == &ext_sched_class) { + scx_prepare_task_sched_change(p); + queue_flags |= DEQUEUE_NOCLOCK; + } scoped_guard (sched_change, p, queue_flags) { scx_set_task_slice(p, READ_ONCE(sch->slice_dfl)); @@ -8356,12 +9072,15 @@ static struct bpf_struct_ops bpf_sched_ext_ops = { /* * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types * identical, only param names differ across structs) are reused. Some need - * fresh stubs, set_cmask due to an argument type difference and the sub-sched - * notifiers because no cpu-form stub exists to reuse. + * fresh stubs, set_cmask and enable due to argument differences and the + * sub-sched notifiers because no cpu-form stub exists to reuse. */ static void sched_ext_ops_cid__set_cmask(struct task_struct *p, const struct scx_cmask *cmask__arena) {} +static void sched_ext_ops_cid__enable(struct task_struct *p, struct scx_enable_args *args) {} static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask__arena, u64 caps) {} static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {} +static void sched_ext_ops__sub_cid_sched_updated(s32 cid, u64 sched) {} +static void sched_ext_ops__sub_child_ecaps_updated(u64 cgroup_id, s32 cid, u64 before, u64 after) {} static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .select_cid = sched_ext_ops__select_cpu, @@ -8380,7 +9099,7 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .update_idle = sched_ext_ops__update_idle, .init_task = sched_ext_ops__init_task, .exit_task = sched_ext_ops__exit_task, - .enable = sched_ext_ops__enable, + .enable = sched_ext_ops_cid__enable, .disable = sched_ext_ops__disable, #ifdef CONFIG_EXT_GROUP_SCHED .cpuctl_init = sched_ext_ops__cgroup_init, @@ -8396,6 +9115,8 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .sub_detach = sched_ext_ops__sub_detach, .sub_caps_updated = sched_ext_ops__sub_caps_updated, .sub_ecaps_updated = sched_ext_ops__sub_ecaps_updated, + .sub_cid_sched_updated = sched_ext_ops__sub_cid_sched_updated, + .sub_child_ecaps_updated = sched_ext_ops__sub_child_ecaps_updated, .cid_online = sched_ext_ops__cpu_online, .cid_offline = sched_ext_ops__cpu_offline, .init_cids = sched_ext_ops__init_cids, @@ -8490,13 +9211,23 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r { struct rq *rq = cpu_rq(cpu); struct scx_rq *this_scx = &this_rq->scx; + struct rq_flags rf; const struct sched_class *cur_class; bool should_wait = false; bool kickable; - unsigned long flags; + bool preempt, preempt_lazy, wait, immediate; - raw_spin_rq_lock_irqsave(rq, flags); - cur_class = rq->curr->sched_class; + rq_lock_irqsave(rq, &rf); + cur_class = rq->donor->sched_class; + preempt = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt); + preempt_lazy = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt_lazy); + wait = cpumask_test_cpu(cpu, pcpu->cpus_to_wait); + /* + * Immediate preemption, waiting and a plain kick take precedence over + * lazy preemption. The lazy request still clears the slice, so all + * accumulated requests are served. + */ + immediate = preempt || wait || cpumask_test_cpu(cpu, pcpu->cpus_to_kick); /* * During CPU hotplug, a CPU may depend on kicking itself to make @@ -8510,19 +9241,23 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r !sched_class_above(cur_class, &ext_sched_class); if (kickable && !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) { - if (cpumask_test_cpu(cpu, pcpu->cpus_to_preempt)) { + if (preempt || preempt_lazy) { if (cur_class == &ext_sched_class) { u64 caps = scx_caps_for_preempt(pcpu->sch, rq, 0); - if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps))) + if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps))) { __scx_add_event(pcpu->sch, SCX_EV_SUB_PREEMPT_DENIED, 1); - else if (unlikely(!scx_set_task_slice(rq->curr, 0))) + /* degrade to a plain, immediate kick */ + immediate = true; + } else if (unlikely(!scx_set_task_slice(rq->donor, 0))) { __scx_add_event(pcpu->sch, SCX_EV_SLICE_DENIED, 1); + } } cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); + cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt_lazy); } - if (cpumask_test_cpu(cpu, pcpu->cpus_to_wait)) { + if (wait) { if (cur_class == &ext_sched_class) { cpumask_set_cpu(cpu, this_scx->cpus_to_sync); ksyncs[cpu] = rq->scx.kick_sync; @@ -8531,17 +9266,21 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r cpumask_clear_cpu(cpu, pcpu->cpus_to_wait); } - resched_curr(rq); + if (immediate) + resched_curr(rq); + else + scx_resched_curr_lazy(rq); } else { /* a kickable cpu was skipped solely for the missing caps */ if (kickable) __scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1); cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); + cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt_lazy); cpumask_clear_cpu(cpu, pcpu->cpus_to_wait); } scx_rq_lock_drop(rq); - raw_spin_rq_unlock_irqrestore(rq, flags); + rq_unlock_irqrestore(rq, &rf); return should_wait; } @@ -8591,7 +9330,7 @@ static void kick_cpus_irq_workfn(struct irq_work *irq_work) list_for_each_entry_safe(pcpu, tmp, &this_scx->sched_pcpus_to_kick, to_kick_node) { list_del_init(&pcpu->to_kick_node); - for_each_cpu(cpu, pcpu->cpus_to_kick) { + for_each_cpu_or(cpu, pcpu->cpus_to_kick, pcpu->cpus_to_preempt_lazy) { should_wait |= kick_one_cpu(cpu, pcpu, this_rq, ksyncs); cpumask_clear_cpu(cpu, pcpu->cpus_to_kick); cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle); @@ -8722,8 +9461,8 @@ void __init init_sched_ext_class(void) /* local_dsq's sch will be set during scx_root_enable() */ BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); -#ifdef CONFIG_EXT_SUB_SCHED BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL)); +#ifdef CONFIG_EXT_SUB_SCHED scx_rescue_init(rq); #endif @@ -8856,7 +9595,7 @@ __bpf_kfunc_start_defs(); * task is inserted. * * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id - * and this function can be called upto ops.dispatch_max_batch times to insert + * and this function can be called up to ops.dispatch_max_batch times to insert * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the * remaining slots. scx_bpf_dsq_move_to_local() flushes the batch and resets the * counter. @@ -8941,7 +9680,9 @@ struct scx_bpf_dsq_insert_vtime_args { * * @args->vtime ordering is according to time_before64() which considers * wrapping. A numerically larger vtime may indicate an earlier position in the - * ordering and vice-versa. + * ordering and vice-versa. vtime is a rolling cursor and values used for + * ordering within a given DSQ should stay less than 2^63 apart for + * time_before64() ordering to remain well-defined. * * A DSQ can only be used as a FIFO or priority queue at any given time and this * function must not be called on a DSQ which already has one or more FIFO tasks @@ -9019,6 +9760,40 @@ static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { .filter = scx_kfunc_context_filter, }; +/* + * Lock @rq from a kfunc called either from ops.dispatch(), which holds the + * dispatched rq's lock, or from a context holding no rq lock. Under core + * scheduling, dispatch can run for a sibling rq, so the locked rq is not + * necessarily this CPU's. Return the rq locked on entry, NULL if none. + */ +static struct rq *kf_switch_rq_lock(struct rq *rq, unsigned long *flags) +{ + struct rq *entry_rq; + + local_irq_save(*flags); + + entry_rq = scx_locked_rq(); + if (entry_rq) { + if (entry_rq != rq) + switch_rq_lock(entry_rq, rq); + } else { + raw_spin_rq_lock(rq); + } + return entry_rq; +} + +/* undo kf_switch_rq_lock(), @rq is the rq locked now */ +static void kf_restore_rq_lock(struct rq *rq, struct rq *entry_rq, unsigned long flags) +{ + if (entry_rq) { + if (entry_rq != rq) + switch_rq_lock(rq, entry_rq); + } else { + scx_rq_lock_drop(rq); + raw_spin_rq_unlock_irqrestore(rq, flags); + } +} + static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, struct task_struct *p, u64 dsq_id, u64 enq_flags, bool priq) @@ -9053,27 +9828,8 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, if (unlikely(READ_ONCE(sch->aborting))) return false; - /* - * Can be called from either ops.dispatch() holding the dispatched rq's - * lock or any context where no rq lock is held. If latter, lock @p's - * task_rq which we'll likely need anyway. - */ src_rq = task_rq(p); - - local_irq_save(flags); - - /* - * Under core scheduling, dispatch can run for a sibling rq, so the - * locked rq is not necessarily this CPU's. - */ - locked_rq = scx_locked_rq(); - - if (locked_rq) { - if (locked_rq != src_rq) - switch_rq_lock(locked_rq, src_rq); - } else { - raw_spin_rq_lock(src_rq); - } + locked_rq = kf_switch_rq_lock(src_rq, &flags); p_rq = src_rq; raw_spin_lock(&src_dsq->lock); @@ -9112,13 +9868,7 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, p_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); dispatched = true; out: - if (locked_rq) { - if (locked_rq != p_rq) - switch_rq_lock(p_rq, locked_rq); - } else { - scx_rq_lock_drop(p_rq); - raw_spin_rq_unlock_irqrestore(p_rq, flags); - } + kf_restore_rq_lock(p_rq, locked_rq, flags); kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE | __SCX_DSQ_ITER_HAS_VTIME); @@ -9352,6 +10102,8 @@ BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) +/* also in scx_kfunc_ids_unlocked, same contexts as scx_bpf_dsq_move() */ +BTF_ID_FLAGS(func, scx_bpf_dsq_reenq_task, KF_RCU) #ifdef CONFIG_EXT_SUB_SCHED BTF_ID_FLAGS(func, scx_bpf_sub_dispatch, KF_IMPLICIT_ARGS) #endif @@ -9467,6 +10219,7 @@ BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_reenq_task, KF_RCU) /* also in scx_kfunc_ids_select_cpu: also callable from ops.select_cpu()/ops.enqueue() */ BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) @@ -9511,8 +10264,10 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, return false; /* - * Directly write only when we hold the lock of the rq @p is queued or - * running on. See the write rules above. + * Directly write only when we hold the lock of the rq @p is queued on or + * provides the current scheduling context for. Under proxy execution, + * rq->donor owns and consumes the slice while rq->curr executes on its + * behalf. See the slice write rules above. * * While @p is queued on a user DSQ or in the BPF scheduler, * synchronization is the scheduler's responsibility. This write can @@ -9526,7 +10281,7 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, locked_rq = scx_locked_rq(); if (!locked_rq || (READ_ONCE(p->scx.runnable_cpu) != cpu_of(locked_rq) && - !task_current(locked_rq, p))) { + !task_current_donor(locked_rq, p))) { set_task_slice_oob(sch, p, slice); return true; } @@ -9567,18 +10322,56 @@ __bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime, return true; } +/** + * scx_bpf_task_set_lazy_resched - Set task's tick reschedule mode + * @p: task of interest + * @lazy: whether slice expiry should request lazy rescheduling + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Choose whether depletion of @p's slice at the scheduler tick requests lazy or + * immediate rescheduling. This is a persistent per-task policy and does not + * itself clear @p's slice or request rescheduling, unlike the + * %SCX_ENQ_PREEMPT_LAZY and %SCX_KICK_PREEMPT_LAZY one-shot operations. + * + * With lazy rescheduling enabled, a task interrupted in user space still + * reschedules before returning to user space. A task interrupted in the kernel + * may continue until its next return to user space or the next scheduler tick, + * which promotes the request. If lazy preemption is disabled at runtime, the + * request behaves like an immediate reschedule. + * + * @p must be on the calling scheduler. + * + * Return %true on success, %false if @p is not on the calling scheduler. + */ +__bpf_kfunc bool scx_bpf_task_set_lazy_resched(struct task_struct *p, bool lazy, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + /* + * clang's register locations for @lazy make pahole 1.32 drop this kfunc + * from BTF. Keep @lazy in memory so its location names no register. + */ + barrier_data(&lazy); + guard(rcu)(); + sch = scx_prog_sched(aux); + if (unlikely(!sch || !scx_task_on_sched(sch, p))) + return false; + + WRITE_ONCE(p->scx.lazy_resched, lazy); + return true; +} + void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) { struct scx_sched_pcpu *pcpu; struct rq *this_rq; unsigned long irq_flags; - /* - * The per-cpu kick list is guarded only by local_irq_save(), which does - * not mask NMIs, so kicking from NMI could corrupt it and is unsupported. - */ - if (unlikely(in_nmi())) { - scx_error(sch, "scx_bpf_kick_cpu() called from NMI"); + if (!scx_kf_allowed_ctx(sch)) + return; + if (unlikely(flags & ~SCX_KICK_ALL_FLAGS)) { + scx_error(sch, "invalid kick flags 0x%llx", flags); return; } @@ -9606,7 +10399,8 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) if (flags & SCX_KICK_IDLE) { struct rq *target_rq = cpu_rq(cpu); - if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_WAIT))) + if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_PREEMPT_LAZY | + SCX_KICK_WAIT))) scx_error(sch, "PREEMPT/WAIT cannot be used with SCX_KICK_IDLE"); if (raw_spin_rq_trylock(target_rq)) { @@ -9620,12 +10414,17 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) } cpumask_set_cpu(cpu, pcpu->cpus_to_kick_if_idle); } else { - cpumask_set_cpu(cpu, pcpu->cpus_to_kick); - + /* + * Accumulate requests and resolve their precedence at delivery. + */ if (flags & SCX_KICK_PREEMPT) cpumask_set_cpu(cpu, pcpu->cpus_to_preempt); + if (flags & SCX_KICK_PREEMPT_LAZY) + cpumask_set_cpu(cpu, pcpu->cpus_to_preempt_lazy); if (flags & SCX_KICK_WAIT) cpumask_set_cpu(cpu, pcpu->cpus_to_wait); + if (!(flags & SCX_KICK_PREEMPT_LAZY)) + cpumask_set_cpu(cpu, pcpu->cpus_to_kick); } if (list_empty(&pcpu->to_kick_node)) @@ -9665,8 +10464,9 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux * cid-addressed equivalent of scx_bpf_kick_cpu(). An invalid @cid aborts the * scheduler via scx_cid_to_cpu(). Caps are enforced on the delivery path: a * kick is dropped if the caller lacks baseline access on @cid, and a - * %SCX_KICK_PREEMPT degrades to a plain reschedule if the caller lacks - * %SCX_CAP_PREEMPT for a task outside its subtree. + * %SCX_KICK_PREEMPT or %SCX_KICK_PREEMPT_LAZY request degrades to a plain + * reschedule if the caller lacks %SCX_CAP_PREEMPT for a task outside its + * subtree. */ __bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) { @@ -9748,8 +10548,13 @@ __bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux) guard(rcu)(); sch = scx_prog_sched(aux); - if (sch) - destroy_dsq(sch, dsq_id); + if (unlikely(!sch)) + return; + + if (!scx_kf_allowed_ctx(sch)) + return; + + destroy_dsq(sch, dsq_id); } /** @@ -9786,6 +10591,9 @@ __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, if (unlikely(!sch)) return -ENODEV; + if (!scx_kf_allowed_ctx(sch)) + return -EDEADLK; + if (flags & ~__SCX_DSQ_ITER_USER_FLAGS) return -EINVAL; @@ -9820,7 +10628,8 @@ __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *i * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator * @it: iterator to destroy * - * Undo bpf_iter_scx_dsq_new(). + * Undo bpf_iter_scx_dsq_new(). If the iteration marked tasks with + * scx_bpf_dsq_reenq_task(), schedule their re-enqueue. */ __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) { @@ -9836,6 +10645,13 @@ __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) list_del_init(&kit->cursor.node); raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); } + + if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_REENQ) { + guard(preempt)(); + schedule_dsq_reenq(kit->dsq->sched, kit->dsq, SCX_REENQ_MARKED, + scx_locked_rq()); + } + kit->dsq = NULL; } @@ -9888,7 +10704,8 @@ __bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id, * - Local DSQs (%SCX_DSQ_LOCAL or %SCX_DSQ_LOCAL_ON | $cpu) * - User DSQs * - * Re-enqueues are performed asynchronously. Can be called from anywhere. + * Re-enqueues are performed asynchronously. Can be called from anywhere. A + * re-enqueue of an inactive cpu's local DSQ is ignored. * * %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler * operation is locked to - e.g. the rq being dispatched for in ops.dispatch() - @@ -9912,6 +10729,9 @@ __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, return; } + if (!scx_kf_allowed_ctx(sch)) + return; + /* not specifying any filter bits is the same as %SCX_REENQ_ANY */ if (!(reenq_flags & __SCX_REENQ_FILTER_MASK)) reenq_flags |= SCX_REENQ_ANY; @@ -9921,6 +10741,64 @@ __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, } /** + * scx_bpf_dsq_reenq_task - Re-enqueue a task from a DSQ iteration + * @it__iter: DSQ iterator in progress + * @p: task to re-enqueue + * + * Mark @p, which is on the DSQ iterated by @it__iter and was queued before the + * iteration started, for re-enqueue. When the iteration ends, the DSQ's marked + * tasks go through ops.enqueue() with %SCX_ENQ_REENQ in DSQ order and the other + * tasks on the DSQ keep their positions. A marked task that leaves the DSQ in + * the meantime is not re-enqueued. The mark belongs to @p, so a re-enqueue + * scheduled by another iteration of the DSQ may re-enqueue @p before this + * iteration ends. @p must be on the DSQ's scheduler. + * + * Re-enqueues are performed asynchronously. Can be called from ops.dispatch() + * or any BPF context which doesn't hold a rq lock (e.g. BPF timers or SYSCALL + * programs). + * + * Return %true if @p was marked, %false if @p left the DSQ or was queued after + * the iteration started, @p is on another scheduler, or the iterator holds no + * DSQ. + */ +__bpf_kfunc bool scx_bpf_dsq_reenq_task(struct bpf_iter_scx_dsq *it__iter, + struct task_struct *p) +{ + struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; + struct scx_dispatch_q *dsq = kit->dsq; + struct rq *src_rq, *entry_rq; + unsigned long flags; + bool marked = false; + + /* NULL after a failed bpf_iter_scx_dsq_new(), see scx_dsq_move() */ + if (unlikely(!dsq)) + return false; + + /* the cursor check needs @p's rq locked */ + src_rq = task_rq(p); + entry_rq = kf_switch_rq_lock(src_rq, &flags); + + scoped_guard (raw_spinlock, &dsq->lock) { + if (nldsq_cursor_lost_task(&kit->cursor, src_rq, dsq, p)) + break; + + if (unlikely(!scx_task_on_sched(dsq->sched, p))) { + scx_error(dsq->sched, "scx_bpf_dsq_reenq_task() on %s[%d] but the task belongs to a different scheduler", + p->comm, p->pid); + break; + } + + p->scx.dsq_flags |= SCX_TASK_DSQ_REENQ; + /* the walk is scheduled once, by bpf_iter_scx_dsq_destroy() */ + kit->cursor.flags |= __SCX_DSQ_ITER_HAS_REENQ; + marked = true; + } + + kf_restore_rq_lock(src_rq, entry_rq, flags); + return marked; +} + +/** * scx_bpf_reenqueue_local___v2 - Re-enqueue tasks on a local DSQ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * @@ -10299,6 +11177,9 @@ __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_au if (unlikely(!sch)) return; + if (!scx_kf_allowed_ctx(sch)) + return; + scx_cpuperf_set(sch, cpu, perf); } @@ -10324,6 +11205,10 @@ __bpf_kfunc s32 scx_bpf_cidperf_set(s32 cid, u32 perf, sch = scx_prog_sched(aux); if (unlikely(!sch)) return -ENODEV; + + if (!scx_kf_allowed_ctx(sch)) + return -EDEADLK; + cpu = scx_cid_to_cpu(sch, cid); if (cpu < 0) return cpu; @@ -10403,8 +11288,7 @@ __bpf_kfunc const void *scx_bpf_online_cmask(const struct bpf_prog_aux *aux) if (unlikely(!online)) return NULL; - /* BPF rebases by the low 32 bits, like __arena callback args */ - return (void *)((unsigned long)online - sch->arena_kern_base); + return (void *)scx_kaddr_to_arena(sch, online); } /** @@ -10457,12 +11341,17 @@ __bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask) } /** - * scx_bpf_task_running - Is task currently running? + * scx_bpf_task_running - Is task the current scheduling context? * @p: task of interest + * + * Under proxy execution, this reports the donor rather than the task whose + * code is physically executing. The physical execution context is intentionally + * not exposed to the BPF scheduler, which continues to observe the donor as the + * running scheduling context. */ __bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p) { - return task_rq(p)->curr == p; + return rcu_access_pointer(task_rq(p)->donor) == p; } /** @@ -10523,10 +11412,15 @@ __bpf_kfunc struct rq *scx_bpf_locked_rq(const struct bpf_prog_aux *aux) } /** - * scx_bpf_cpu_curr - Return remote CPU's curr task + * scx_bpf_cpu_curr - Return remote CPU's current scheduling context * @cpu: CPU of interest * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * + * Under proxy execution, this returns the donor, which supplies the scheduling + * policy and runtime budget, rather than the task whose code is physically + * executing. The physical execution context is intentionally not exposed to + * the BPF scheduler. + * * Callers must hold RCU read lock (KF_RCU). */ __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_aux *aux) @@ -10542,7 +11436,7 @@ __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_ if (!scx_cpu_valid(sch, cpu, NULL)) return NULL; - return rcu_dereference(cpu_rq(cpu)->curr); + return rcu_dereference(cpu_rq(cpu)->donor); } /** @@ -10566,7 +11460,7 @@ __bpf_kfunc struct task_struct *scx_bpf_cid_curr(s32 cid, const struct bpf_prog_ cpu = scx_cid_to_cpu(sch, cid); if (cpu < 0) return NULL; - return rcu_dereference(cpu_rq(cpu)->curr); + return rcu_dereference(cpu_rq(cpu)->donor); } /** @@ -10740,6 +11634,23 @@ out: cgroup_get(cgrp); return cgrp; } + +/** + * scx_bpf_cgroup_nr_cpus - Return the number of CPUs in a cgroup's cpuset + * @cgrp: cgroup of interest + * + * Return the number of CPUs in @cgrp's effective cpuset, which is inherited + * from the nearest ancestor with the cpuset controller enabled. This matches + * the count used by fair's group share calculation and lets hierarchical BPF + * schedulers bound a group's weight by the CPUs it can actually run on. + * + * The value is a snapshot and may change at any time through cpuset updates or + * CPU hotplug. Schedulers should re-read it when recomputing group state. + */ +__bpf_kfunc u32 scx_bpf_cgroup_nr_cpus(struct cgroup *cgrp) +{ + return cpuset_num_cpus(cgrp); +} #endif /* CONFIG_CGROUP_SCHED */ __bpf_kfunc_end_defs(); @@ -10747,6 +11658,7 @@ __bpf_kfunc_end_defs(); BTF_KFUNCS_START(scx_kfunc_ids_any) BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS | KF_RCU); BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS | KF_RCU); +BTF_ID_FLAGS(func, scx_bpf_task_set_lazy_resched, KF_IMPLICIT_ARGS | KF_RCU); BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_kick_cid, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS) @@ -10786,6 +11698,7 @@ BTF_ID_FLAGS(func, scx_bpf_now) BTF_ID_FLAGS(func, scx_bpf_events, KF_IMPLICIT_ARGS) #ifdef CONFIG_CGROUP_SCHED BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS | KF_RCU | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_cgroup_nr_cpus, KF_RCU) #endif BTF_ID_FLAGS(func, scx_bpf_sub_grant, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_sub_revoke, KF_IMPLICIT_ARGS) @@ -10879,6 +11792,7 @@ static const u32 scx_kf_allow_flags[] = { [SCX_OP_IDX(sub_attach)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(sub_detach)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(sub_ecaps_updated)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH, + [SCX_OP_IDX(sub_child_ecaps_updated)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH, [SCX_OP_IDX(cpu_online)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(cpu_offline)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(init_cids)] = SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT_CIDS, @@ -11022,6 +11936,8 @@ static int __init scx_init(void) CID_OFFSET_MATCH(sub_detach, sub_detach); CID_OFFSET_MATCH(sub_caps_updated, sub_caps_updated); CID_OFFSET_MATCH(sub_ecaps_updated, sub_ecaps_updated); + CID_OFFSET_MATCH(sub_cid_sched_updated, sub_cid_sched_updated); + CID_OFFSET_MATCH(sub_child_ecaps_updated, sub_child_ecaps_updated); CID_OFFSET_MATCH(init_cids, init_cids); CID_OFFSET_MATCH(init, init); CID_OFFSET_MATCH(exit, exit); diff --git a/kernel/sched/ext/ext.h b/kernel/sched/ext/ext.h index 0b7fc46aee08..d32bc01071b0 100644 --- a/kernel/sched/ext/ext.h +++ b/kernel/sched/ext/ext.h @@ -18,9 +18,28 @@ bool scx_can_stop_tick(struct rq *rq); void scx_rq_activate(struct rq *rq); void scx_rq_deactivate(struct rq *rq); int scx_check_setscheduler(struct task_struct *p, int policy); +void scx_prepare_setscheduler(struct task_struct *p, int policy); bool task_should_scx(int policy); bool scx_allow_ttwu_queue(const struct task_struct *p); +bool __scx_allow_proxy_exec(const struct task_struct *p); +void scx_proxy_donor_start(struct rq *rq); +void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next); void init_sched_ext_class(void); +void __scx_update_idle(struct rq *rq, bool idle, bool do_notify); + +static inline bool scx_allow_proxy_exec(const struct task_struct *p) +{ + if (scx_enabled()) + return __scx_allow_proxy_exec(p); + + return true; +} + +static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) +{ + if (scx_enabled()) + __scx_update_idle(rq, idle, do_notify); +} static inline u32 scx_cpuperf_target(s32 cpu) { @@ -52,24 +71,18 @@ static inline bool scx_can_stop_tick(struct rq *rq) { return true; } static inline void scx_rq_activate(struct rq *rq) {} static inline void scx_rq_deactivate(struct rq *rq) {} static inline int scx_check_setscheduler(struct task_struct *p, int policy) { return 0; } +static inline void scx_prepare_setscheduler(struct task_struct *p, int policy) {} static inline bool task_on_scx(const struct task_struct *p) { return false; } static inline bool scx_allow_ttwu_queue(const struct task_struct *p) { return true; } +static inline bool scx_allow_proxy_exec(const struct task_struct *p) { return true; } +static inline void scx_proxy_donor_start(struct rq *rq) {} +static inline void scx_proxy_reenqueue_retry(struct rq *rq, + struct task_struct *next) {} static inline void init_sched_ext_class(void) {} +static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) {} #endif /* CONFIG_SCHED_CLASS_EXT */ -#ifdef CONFIG_SCHED_CLASS_EXT -void __scx_update_idle(struct rq *rq, bool idle, bool do_notify); - -static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) -{ - if (scx_enabled()) - __scx_update_idle(rq, idle, do_notify); -} -#else -static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) {} -#endif - #ifdef CONFIG_CGROUP_SCHED #ifdef CONFIG_EXT_GROUP_SCHED void scx_tg_init(struct task_group *tg); diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index aa9fb6de0ad6..922fbc3065cc 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -153,7 +153,18 @@ static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, i nodemask_t *unvisited; s32 cpu = -EBUSY; - preempt_disable(); + /* + * @per_cpu_unvisited is per-CPU scratch and the idle kfuncs can be + * called from IRQ-enabled contexts, so mask IRQs to keep a nested + * invocation from clobbering the mask an outer invocation is still + * iterating. + * + * NMI nesting is not handled: there is no legitimate reason to call + * pick_idle from NMI and doing so poses no crash risk, so such a + * caller is on its own. + */ + guard(irqsave)(); + unvisited = this_cpu_ptr(&per_cpu_unvisited); /* @@ -183,7 +194,6 @@ static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, i if (cpu >= 0) break; } - preempt_enable(); return cpu; } @@ -952,12 +962,43 @@ static bool is_bpf_migration_disabled(const struct task_struct *p) return p->migration_disabled; } +static s32 select_cpu_restricted(struct task_struct *p, s32 prev_cpu, + const struct cpumask *allowed, u64 flags, + bool migration_disabled) +{ + s32 cpu; + + if (migration_disabled) + cpu = task_cpu(p); + else + cpu = cpumask_first(p->cpus_ptr); + + if (!cpumask_test_cpu(cpu, p->cpus_ptr)) + return -EBUSY; + + if (allowed && !cpumask_test_cpu(cpu, allowed)) + return -EBUSY; + + if (flags & SCX_PICK_IDLE_IN_NODE) { + int node = scx_cpu_node_if_enabled(prev_cpu); + + if (scx_cpu_node_if_enabled(cpu) != node) + return -EBUSY; + } + + if (!scx_idle_test_and_clear_cpu(cpu)) + return -EBUSY; + + return cpu; +} + static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p, s32 prev_cpu, u64 wake_flags, const struct cpumask *allowed, u64 flags) { unsigned long irq_flags; bool we_locked = false; + bool migration_disabled; s32 cpu; if (!scx_cpu_valid(sch, prev_cpu, NULL)) @@ -994,20 +1035,17 @@ static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p, /* * This may also be called from ops.enqueue(), so we need to handle - * per-CPU tasks as well. For these tasks, we can skip all idle CPU - * selection optimizations and simply check whether the previously - * used CPU is idle and within the allowed cpumask. + * per-CPU tasks as well. For these tasks, skip general idle CPU + * selection and validate the task's only usable CPU directly. */ - if (p->nr_cpus_allowed == 1 || is_bpf_migration_disabled(p)) { - if (cpumask_test_cpu(prev_cpu, allowed ?: p->cpus_ptr) && - scx_idle_test_and_clear_cpu(prev_cpu)) - cpu = prev_cpu; - else - cpu = -EBUSY; - } else { + migration_disabled = is_bpf_migration_disabled(p); + + if (p->nr_cpus_allowed == 1 || migration_disabled) + cpu = select_cpu_restricted(p, prev_cpu, allowed, flags, + migration_disabled); + else cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, allowed ?: p->cpus_ptr, flags); - } if (we_locked) raw_spin_unlock_irqrestore(&p->pi_lock, irq_flags); diff --git a/kernel/sched/ext/inlines.h b/kernel/sched/ext/inlines.h index ed423bcc26b8..8de9d8fb4b25 100644 --- a/kernel/sched/ext/inlines.h +++ b/kernel/sched/ext/inlines.h @@ -70,7 +70,13 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, #endif /* CONFIG_EXT_SUB_SCHED */ } - if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) + /* + * scx_rq_online() can't be used. Its cpu_active() test goes false + * before CPU hotplug waits for an RCU grace period, and + * rq_offline_scx() moves this CPU's tasks to the local DSQ only after + * the wait. The grace period can depend on those tasks running. + */ + if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !(rq->scx.flags & SCX_RQ_ONLINE)) return SCX_DSP_NONE; dspc->rq = rq; @@ -129,8 +135,10 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, * scheduler's ops.dispatch() doesn't yield any tasks. */ if (scx_bypass_dsp_enabled(sch) && - scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) + scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { + __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); return SCX_DSP_LOCAL; + } return SCX_DSP_NONE; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 3464e0f113c1..29e69a83d876 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -215,6 +215,34 @@ enum scx_ops_flags { */ SCX_OPS_TID_TO_TASK = 1LLU << 8, + /* + * If set, tasks default to requesting lazy rescheduling when their + * slice runs out at the tick, the way fair.c expires a slice from + * update_curr(). The default is copied to p->scx.lazy_resched + * immediately before ops.enable(), after which + * scx_bpf_task_set_lazy_resched() may override it per task. + * + * A task in user space still reschedules on the way back from the tick; + * a task in the kernel runs on to its next return to user space or to + * the next tick, which promotes the request. When lazy preemption is + * disabled at runtime, the request behaves like an immediate + * reschedule. Rescheduling while bypassing stays immediate. + */ + SCX_OPS_LAZY_RESCHED = 1LLU << 9, + + /* + * If set, mutex-blocked tasks remain runnable as proxy donors and are + * passed to ops.enqueue() with %SCX_ENQ_BLOCKED. The BPF scheduler controls + * when donors are dispatched and whether they should preempt other work. + * + * If clear, mutex-blocked tasks are removed from the runqueue normally + * and cannot donate their scheduling context through proxy execution. + * + * For blocked donors, this flag takes precedence over + * %SCX_OPS_ENQ_EXITING and %SCX_OPS_ENQ_MIGRATION_DISABLED. + */ + SCX_OPS_ENQ_BLOCKED = 1LLU << 10, + SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | SCX_OPS_ENQ_LAST | SCX_OPS_ENQ_EXITING | @@ -223,7 +251,9 @@ enum scx_ops_flags { SCX_OPS_SWITCH_PARTIAL | SCX_OPS_BUILTIN_IDLE_PER_NODE | SCX_OPS_ALWAYS_ENQ_IMMED | - SCX_OPS_TID_TO_TASK, + SCX_OPS_TID_TO_TASK | + SCX_OPS_LAZY_RESCHED | + SCX_OPS_ENQ_BLOCKED, /* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */ __SCX_OPS_INTERNAL_MASK = 0xffLLU << 56, @@ -250,6 +280,31 @@ struct scx_exit_task_args { bool cancelled; }; +/** + * struct scx_enable_args - Argument container for cid-form ops.enable() + * @cmask_arena_addr: BPF arena address of the cmask of cids the task may run on + * + * @cmask_arena_addr is the task's affinity as it enters the scheduler. + * set_cmask() delivers the same mask right after enable(), before set_weight() + * and the first enqueue, then every affinity change afterwards, and is never + * called before enable(). A scheduler may therefore track affinity in + * set_cmask() alone. + * + * The kernel builds the mask in the scheduler arena from its own geometry, so + * the header is valid regardless of what the scheduler last wrote there. The + * memory is per-cpu scratch reused once the callback returns: copy the bits + * out, don't keep the address. The set_cmask() argument follows the same rules. + * + * The address is a plain value rather than a typed pointer because BTF can't + * mark a struct member as an arena pointer yet and a pointer member would reach + * the program typed as a kernel pointer. Cast it to struct scx_cmask __arena * + * before use. Once arena members can be typed, a typed alias will join this + * field in an anonymous union at the same offset. + */ +struct scx_enable_args { + u64 cmask_arena_addr; +}; + /* argument container for ops.cgroup_init() */ struct scx_cgroup_init_args { /* the weight of the cgroup [1..10000] */ @@ -311,6 +366,18 @@ struct scx_sub_detach_args { char *cgroup_path; }; +/* + * What a sched sees running on a cid, as reported by + * ops.sub_cid_sched_updated(): NONE when the cid is idle or runs a task outside + * the sched's subtree, SELF for the sched itself, or a direct child by its + * cgroup id. The sentinels lie outside the cgroup id space, whose low 32 bits + * are an idr value of at most INT_MAX. + */ +enum scx_cid_sched_consts { + SCX_CID_SCHED_NONE = (u64)-2, + SCX_CID_SCHED_SELF = (u64)-1, +}; + /** * struct sched_ext_ops - Operation table for BPF scheduler implementation * @@ -452,6 +519,12 @@ struct sched_ext_ops { * Therefore, always use scx_bpf_task_cpu(@p) to determine the * target CPU the task is going to use. * + * Under proxy execution, the BPF scheduler continues to observe the + * donor as the current scheduling context. A blocked donor enters a + * ->running()/->stopping() session while its scheduling context drives + * the lock owner. The lock owner executing on its behalf is intentionally + * not reported through these callbacks. + * * See ->runnable() for explanation on the task state notifiers. */ void (*running)(struct task_struct *p); @@ -803,6 +876,11 @@ struct sched_ext_ops { /** * @sub_detach: Detach a sub-scheduler * @args: argument container, see the struct definition + * + * The sub-scheduler holds no caps by this point and can no longer + * affect any cid. Its caps went away without a + * sub_child_ecaps_updated() report, so whatever was delegated to it, + * e.g. cpuperf targets, is the parent's to restore here. */ void (*sub_detach)(struct scx_sub_detach_args *args); @@ -839,6 +917,41 @@ struct sched_ext_ops { */ void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); + /** + * @sub_cid_sched_updated: The sched running on a cid changed + * @cid: cid whose running sched changed + * @sched: what this sched now sees running there + * + * Invoked when the sched owning the task running on @cid changes. + * @sched is %SCX_CID_SCHED_NONE when no task of this sched's subtree + * runs there, %SCX_CID_SCHED_SELF for its own task, or the cgroup id of + * the direct child whose subtree the task belongs to. Changes inside a + * child's subtree, whether across tasks or nested sub-scheds, are not + * reported. Every sched for which @sched changes is notified. + * + * Runs with the rq lock held on the context switch path. + */ + void (*sub_cid_sched_updated)(s32 cid, u64 sched); + + /** + * @sub_child_ecaps_updated: A child's effective caps on a cid changed + * @cgroup_id: cgroup id of the direct child + * @cid: the cid whose effective caps changed + * @before: the child's effective caps as of the last delivery + * @after: the child's effective caps now + * + * Invoked right after the child's sub_ecaps_updated(), once a grant or + * revoke is in effect on the cpu. From here on the parent can act on a + * cap the child lost, e.g. reset the cpuperf target or schedule on the + * cid. A detaching child's caps go away without a report, see + * sub_detach(). A cpu going offline drops caps without a report. + * + * Runs in dispatch context with rq lock held, and can perform all + * operations allowed in ops.dispatch() including inserting/moving + * tasks. + */ + void (*sub_child_ecaps_updated)(u64 cgroup_id, s32 cid, u64 before, u64 after); + /* * All online ops must come before ops.cpu_online(). */ @@ -886,6 +999,9 @@ struct sched_ext_ops { * ops.exit() is also called on ops.init() failure, which is a bit * unusual. This is to allow rich reporting through @info on how * ops.init() failed. + * + * A sub-scheduler holds no caps by the time ops.exit() runs. Its + * cap-gated kfuncs are denied. */ void (*exit)(struct scx_exit_info *info); @@ -1037,6 +1153,7 @@ struct sched_ext_ops { * - dispatch -> dispatch (cpu arg is now cid) * - update_idle -> update_idle (cpu arg is now cid) * - set_cpumask -> set_cmask (cmask instead of cpumask) + * - enable -> enable (takes struct scx_enable_args) * - cpu_online -> cid_online * - cpu_offline -> cid_offline * - dump_cpu -> dump_cid @@ -1070,7 +1187,7 @@ struct sched_ext_ops_cid { struct scx_init_task_args *args); void (*exit_task)(struct task_struct *p, struct scx_exit_task_args *args); - void (*enable)(struct task_struct *p); + void (*enable)(struct task_struct *p, struct scx_enable_args *args); void (*disable)(struct task_struct *p); void (*dump)(struct scx_dump_ctx *ctx); void (*dump_cid)(struct scx_dump_ctx *ctx, s32 cid, bool idle); @@ -1092,6 +1209,8 @@ struct sched_ext_ops_cid { void (*sub_detach)(struct scx_sub_detach_args *args); void (*sub_caps_updated)(const struct scx_cmask *cmask__arena, u64 caps); void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); + void (*sub_cid_sched_updated)(s32 cid, u64 sched); + void (*sub_child_ecaps_updated)(u64 cgroup_id, s32 cid, u64 before, u64 after); void (*cid_online)(s32 cid); void (*cid_offline)(s32 cid); s32 (*init_cids)(void); @@ -1333,6 +1452,7 @@ struct scx_sched_pcpu { cpumask_var_t cpus_to_kick; cpumask_var_t cpus_to_kick_if_idle; cpumask_var_t cpus_to_preempt; + cpumask_var_t cpus_to_preempt_lazy; cpumask_var_t cpus_to_wait; struct list_head to_kick_node; @@ -1354,7 +1474,10 @@ struct scx_sched_pcpu { struct llist_node ecaps_to_sync_node; /* owed a forced update_idle() re-notify on this cpu */ bool idle_renotify; - /* effective caps as of the last sub_ecaps_updated() delivery */ + /* + * ecaps as of the last sub_ecaps_updated() and + * sub_child_ecaps_updated() delivery. See __scx_process_sync_ecaps(). + */ u64 reported_ecaps; /* @@ -1413,15 +1536,16 @@ struct scx_sched_pnode { * the allocation pattern. * * ENQ_IMMED insert an IMMED task onto the cid's local DSQ - * - kick the cid's cpu (except SCX_KICK_PREEMPT) + * - kick the cid's cpu (except SCX_KICK_PREEMPT and + * SCX_KICK_PREEMPT_LAZY) * * ENQ insert any task onto the cid's local DSQ (implies ENQ_IMMED) * * PREEMPT preempt any task running on the cid regardless of the owning * sched (implies ENQ). Preempting a task in the sched's own subtree * doesn't require any cap. - * - SCX_ENQ_PREEMPT inserts - * - SCX_KICK_PREEMPT kicks + * - SCX_ENQ_PREEMPT and SCX_ENQ_PREEMPT_LAZY insert + * - SCX_KICK_PREEMPT and SCX_KICK_PREEMPT_LAZY kick * * PERF control the cid's cpu power/perf management state, currently the * cpufreq target set through scx_bpf_cidperf_set(). Hardware @@ -1533,7 +1657,8 @@ struct scx_sched { * by BUILD_BUG_ON in scx_init()). The anonymous union lets the kernel * access either view of the same storage without function-pointer * casts: use .ops for cpu-form and shared fields, .ops_cid for the - * cid-renamed callbacks (set_cmask, select_cid, cid_online, ...). + * callbacks whose cid-form signature differs (set_cmask, enable, + * select_cid, cid_online, ...). */ union { struct sched_ext_ops ops; @@ -1556,9 +1681,9 @@ struct scx_sched { uintptr_t arena_kern_base; /* - * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask - * to ops_cid.set_cmask(). The kernel writes through the stored kern_va - * and passes it to the callback's __arena argument. + * Per-CPU arena cmask the kernel fills from a task's cpumask and hands + * to ops_cid.enable() and ops_cid.set_cmask(). The stored pointers are + * the kernel addresses. */ struct scx_cmask * __percpu *set_cmask_scratch; struct scx_cmask *online_cmask; @@ -1669,6 +1794,19 @@ static inline void *scx_arena_to_kaddr(struct scx_sched *sch, const void *bpf_pt return (void *)(sch->arena_kern_base + (u32)(uintptr_t)bpf_ptr); } +/** + * scx_kaddr_to_arena - Translate a kernel arena address to the BPF form + * @sch: scheduler whose arena hosts @kaddr + * @kaddr: kernel address inside @sch's arena + * + * __arena callback arguments need no translation. Addresses handed to BPF any + * other way, such as struct fields and kfunc return values, go through this. + */ +static inline uintptr_t scx_kaddr_to_arena(struct scx_sched *sch, const void *kaddr) +{ + return (uintptr_t)kaddr - sch->arena_kern_base; +} + enum scx_wake_flags { /* expose select WF_* flags as enums */ SCX_WAKE_FORK = WF_FORK, @@ -1693,10 +1831,19 @@ enum scx_enq_flags { SCX_ENQ_PREEMPT = 1LLU << 32, /* + * Like %SCX_ENQ_PREEMPT, but request lazy rescheduling. The current + * task's slice is still cleared immediately so that the next scheduling + * boundary observes the new ordering. %SCX_ENQ_PREEMPT takes precedence + * if both are specified. Implies %SCX_ENQ_HEAD, which is all it means + * on a non-local DSQ, as with %SCX_ENQ_PREEMPT. + */ + SCX_ENQ_PREEMPT_LAZY = 1LLU << 35, + + /* * Only allowed on local DSQs. Guarantees that the task either gets * on the CPU immediately and stays on it, or gets reenqueued back - * to the BPF scheduler. It will never linger on a local DSQ or be - * silently put back after preemption. + * to the BPF scheduler. A blocked proxy donor may be put back on the + * local DSQ temporarily while proxy execution resolves its mutex owner. * * The protection persists until the next fresh enqueue - it * survives SAVE/RESTORE cycles, slice extensions and preemption. @@ -1734,11 +1881,24 @@ enum scx_enq_flags { * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with the * %SCX_ENQ_LAST flag set. * - * The BPF scheduler is responsible for triggering a follow-up - * scheduling event. Otherwise, Execution may stall. + * If the task is queued on the local DSQ of the CPU it was running on, + * it continues to run. Otherwise, the CPU goes idle. A scheduler that + * wants a full dispatch cycle on the CPU should kick it. */ SCX_ENQ_LAST = 1LLU << 41, + /* + * The task is blocked on a mutex and is being kept runnable as a proxy + * donor. Only passed to ops.enqueue() when %SCX_OPS_ENQ_BLOCKED is set. + * + * Blocking on the mutex does not enqueue the task by itself. A donor put + * with slice left stays at the head of the local DSQ, except when an + * IMMED donor is preempted or cannot stay local. It is passed to + * ops.enqueue() when its slice runs out, the IMMED placement cannot be + * kept, or proxy execution moves it to the owner's CPU. + */ + SCX_ENQ_BLOCKED = 1LLU << 42, + /* high 8 bits are internal */ __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, @@ -1751,6 +1911,15 @@ enum scx_enq_flags { SCX_ENQ_SLICE_DFL = 1LLU << 62, /* carried slice is a default refill */ }; +/* Strip priority and carried slice state when diverting from a local DSQ. */ +static inline void scx_divert_strip_flags(struct task_struct *p, u64 *enq_flags) +{ + *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | + SCX_ENQ_PREEMPT_LAZY | SCX_ENQ_HEAD | + SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL); + p->scx.flags &= ~SCX_TASK_IMMED; +} + enum scx_deq_flags { /* expose select DEQUEUE_* flags as enums */ SCX_DEQ_SLEEP = DEQUEUE_SLEEP, @@ -1777,6 +1946,8 @@ enum scx_reenq_flags { /* internal: kernel-issued on cap revoke, not accepted from BPF */ SCX_REENQ_CAP_REVOKE = 1LLU << 1, + /* internal: tasks marked by scx_bpf_dsq_reenq_task() */ + SCX_REENQ_MARKED = 1LLU << 2, __SCX_REENQ_FILTER_MASK = 0xffffLLU, @@ -1818,6 +1989,15 @@ enum scx_kick_flags { * is not on SCX. */ SCX_KICK_WAIT = 1LLU << 2, + + /* + * Like %SCX_KICK_PREEMPT, but request lazy rescheduling. If combined + * with %SCX_KICK_PREEMPT or %SCX_KICK_WAIT, rescheduling is immediate. + */ + SCX_KICK_PREEMPT_LAZY = 1LLU << 3, + + SCX_KICK_ALL_FLAGS = SCX_KICK_IDLE | SCX_KICK_PREEMPT | + SCX_KICK_WAIT | SCX_KICK_PREEMPT_LAZY, }; enum scx_tg_flags { @@ -1958,6 +2138,11 @@ enum scx_ops_state { * dequeue/requeue, the dispatcher can tell whether it still has a claim * on the task being dispatched. * + * QSEQ is generated from the per-task p->scx.ops_qseq counter so that + * it doesn't repeat across QUEUED instances of the same task even if + * the task moves between rqs. 0 is never used as a valid QSEQ since + * NONE and DISPATCHING map to this value. + * * As some 32bit archs can't do 64bit store_release/load_acquire, * p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on * 32bit machines. The dispatch race window QSEQ protects is very narrow @@ -2061,12 +2246,12 @@ struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter); bool scx_set_task_slice(struct task_struct *p, u64 slice); void scx_task_slice_ended(struct rq *rq, struct task_struct *p); void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq); +void scx_prepare_task_sched_change(struct task_struct *p); void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p); void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, int sticky_cpu); void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p, - u64 enq_flags, struct scx_dispatch_q *src_dsq, - struct rq *dst_rq); + u64 enq_flags, struct rq *dst_rq); bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, u64 enq_flags); bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq); @@ -2078,6 +2263,7 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); u64 __scx_bpf_now(struct rq *rq); void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, u64 reenq_flags, struct rq *locked_rq); +void scx_wait_dispatching(struct task_struct *p); int __scx_init_task(struct scx_sched *sch, struct task_struct *p, struct cgroup *cgrp, bool fork); void scx_enable_task(struct scx_sched *sch, struct task_struct *p); @@ -2122,6 +2308,22 @@ extern struct scx_sched *scx_enabling_sub_sched; #define scx_error(sch, fmt, args...) \ scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) +/* + * Tracing progs can call kfuncs from NMI. Kfuncs that take scheduler locks or + * touch the kick lists, which are only protected by irq masking, can't run + * there, so abort the scheduler instead. scx_error() is NMI-safe. + */ +static __always_inline bool __scx_kf_allowed_ctx(struct scx_sched *sch, const char *who) +{ + if (unlikely(in_nmi())) { + scx_error(sch, "%s called from NMI", who); + return false; + } + return true; +} + +#define scx_kf_allowed_ctx(sch) __scx_kf_allowed_ctx((sch), __func__) + /** * scx_root_protected_live - Root sched for paths that only run while live * @@ -2212,6 +2414,15 @@ static inline void scx_schedule_reenq_local(struct rq *rq, u64 reenq_flags) */ static inline struct rq *scx_locked_rq(void) { + /* + * Tracing progs can call kfuncs from NMI. scx_locked_rq_state tracks + * the rq locked by the interrupted context, so a non-NULL read from + * NMI would falsely claim its lock. Return NULL from NMI so that + * callers take their unlocked paths. + */ + if (unlikely(in_nmi())) + return NULL; + return __this_cpu_read(scx_locked_rq_state); } @@ -2302,9 +2513,9 @@ do { \ } while (0) /* - * Dispatch a task op through the cid-form ops_cid table. Only set_cmask() needs - * this: it takes an arena cmask address instead of a cpumask, so it cannot be - * invoked via its cpu-form set_cpumask() slot. + * Dispatch a task op through the cid-form ops_cid table, for the ops whose + * cid-form signature differs from the cpu-form slot: set_cmask() takes an arena + * cmask instead of a cpumask and enable() takes scx_enable_args. */ #define SCX_CALL_CID_OP_TASK(sch, op, locked_rq, task, args...) \ __SCX_CALL_OP_TASK(sch, ops_cid, op, locked_rq, task, ##args) @@ -2493,4 +2704,21 @@ static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; #endif /* CONFIG_EXT_SUB_SCHED */ +/** + * scx_exec_sched - Find the sched running @sch's tasks on @cpu + * @sch: sched owning the tasks + * @cpu: cpu of interest + * + * A bypassing sched's tasks are run by its nearest ancestor that isn't + * bypassing. Return @sch if it isn't bypassing on @cpu, otherwise that + * ancestor, or the root if every ancestor is bypassing. Hold @cpu's rq lock for + * a stable answer. + */ +static inline struct scx_sched *scx_exec_sched(struct scx_sched *sch, s32 cpu) +{ + while (scx_parent(sch) && scx_bypassing(sch, cpu)) + sch = scx_parent(sch); + return sch; +} + #endif /* _KERNEL_SCHED_EXT_INTERNAL_H */ diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 34e642a1a403..eb1270f7723c 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -28,6 +28,13 @@ */ DEFINE_STATIC_KEY_FALSE(__scx_has_subs); +/* + * On while a loaded sched implements ops.sub_cid_sched_updated(), see + * scx_ops_cid_sched_updated_enable(). + */ +static DEFINE_STATIC_KEY_FALSE(__scx_ops_cid_sched_updated_enabled); +static s32 scx_nr_ops_cid_sched_updated; /* such scheds, under scx_enable_mutex */ + /* latched at root enable before any rescue runs */ static s32 scx_rescue_bw_1024; static s64 scx_rescue_quantum_ns; @@ -92,6 +99,161 @@ struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sche return scx_skip_subtree_pre(pos, root); } +/** + * scx_cid_sched_id_from - Resolve what @from sees when @sch runs on a cid + * @sch: sched whose task runs there, %NULL for none + * @from: sched asking + * + * Return %SCX_CID_SCHED_NONE when @sch is %NULL or outside @from's subtree, + * %SCX_CID_SCHED_SELF for @from itself, or else the cgroup id of the direct + * child of @from on the way down to @sch. + */ +static u64 scx_cid_sched_id_from(struct scx_sched *sch, struct scx_sched *from) +{ + if (!sch || !scx_is_descendant(sch, from)) + return SCX_CID_SCHED_NONE; + if (sch == from) + return SCX_CID_SCHED_SELF; + return sch->ancestors[from->level + 1]->ops.sub_cgroup_id; +} + +/* notify @from of what it now sees running on @rq's cid */ +static void scx_cid_notify_sched_updated(struct rq *rq, struct scx_sched *sch, + struct scx_sched *from) +{ + if (SCX_HAS_OP(from, sub_cid_sched_updated)) + SCX_CALL_OP(from, sub_cid_sched_updated, rq, __scx_cpu_to_cid(cpu_of(rq)), + scx_cid_sched_id_from(sch, from)); +} + +/** + * scx_cid_sched_update - Update rq->scx.sched and notify the scheds affected + * @rq: rq whose running session opens or closes + * @sch: sched starting a session, %NULL to close the open one + * + * Every sched whose scx_cid_sched_id_from() value changes is notified of the + * new one. Above the deepest common ancestor of the two scheds, both map to the + * same child, so the walk stops there. With root R, children A and B, and A's + * child A1, a switch from an A1 task to a B task reports: + * + * R R: A -> B + * / \ + * A B A: A1 -> NONE B: NONE -> SELF + * | + * A1 A1: SELF -> NONE + * + * When the task on one side is not an ext task, there is no common ancestor and + * only the other side is walked. + */ +void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch) +{ + struct scx_sched *prev = rq->scx.sched; + s32 level, common = -1; + + lockdep_assert_rq_held(rq); + + if (!static_branch_unlikely(&__scx_ops_cid_sched_updated_enabled) || prev == sch) + return; + + rq->scx.sched = sch; + + if (prev && sch) { + for (level = min(prev->level, sch->level); level >= 0; level--) + if (prev->ancestors[level] == sch->ancestors[level]) + break; + common = level; + scx_cid_notify_sched_updated(rq, sch, prev->ancestors[common]); + } + if (prev) + for (level = common + 1; level <= prev->level; level++) + scx_cid_notify_sched_updated(rq, sch, prev->ancestors[level]); + if (sch) + for (level = common + 1; level <= sch->level; level++) + scx_cid_notify_sched_updated(rq, sch, sch->ancestors[level]); +} + +/** + * scx_ops_cid_sched_updated_enable - Maintain rq->scx.sched if @sch has the op + * @sch: sched being enabled, its has_op filled in + * + * rq->scx.sched is maintained only while a loaded sched implements + * ops.sub_cid_sched_updated(), so that everyone else pays one static branch per + * session open or close. The first such sched turns the key on and then + * initializes every rq from the task running there, in that order so that no + * session change in between goes unrecorded. While the key is off every + * rq->scx.sched is NULL, see scx_ops_cid_sched_updated_disable(). + */ +void scx_ops_cid_sched_updated_enable(struct scx_sched *sch) +{ + s32 cpu; + + lockdep_assert_held(&scx_enable_mutex); + + if (!SCX_HAS_OP(sch, sub_cid_sched_updated) || scx_nr_ops_cid_sched_updated++) + return; + + static_branch_enable(&__scx_ops_cid_sched_updated_enabled); + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + struct task_struct *p; + + guard(rq_lock_irqsave)(rq); + p = rq->donor; + if (p->sched_class == &ext_sched_class && + (p->scx.flags & SCX_TASK_RUN_TRACKED)) + rq->scx.sched = scx_task_sched(p); + } +} + +/** + * scx_ops_cid_sched_updated_disable - Stop maintaining rq->scx.sched for @sch + * @sch: sched being disabled, its tasks all gone + * + * Close the session on every rq that still points at @sch, before ops.exit(), + * so that no sched is notified of @sch once it is gone. The last sched with the + * op turns the key off first and clears every rq, so while the key is off every + * rq->scx.sched is NULL. A sched whose enable failed before its has_op fill was + * never counted, so has_op gates the decrement. + */ +void scx_ops_cid_sched_updated_disable(struct scx_sched *sch) +{ + bool last; + s32 cpu; + + lockdep_assert_held(&scx_enable_mutex); + + last = SCX_HAS_OP(sch, sub_cid_sched_updated) && !--scx_nr_ops_cid_sched_updated; + if (last) + static_branch_disable(&__scx_ops_cid_sched_updated_enabled); + + /* + * Re-homing restarts a running task's session under its new sched. A + * task that blocked while its cpu's dispatch had the rq lock dropped is + * put and set next again without a restart, so rq->scx.sched keeps @sch + * until the pick completes: + * + * cpu 0, __schedule() cpu 1, scx_sub_disable(@sch) + * T of @sch blocks, stops running + * dispatch drops the rq lock + * re-home T, nothing to restart + * this sweep + * pick N, N's session opens + * + * Closing the session here reports NONE to @sch and its ancestors now + * and lets the pick report N's sched from NONE instead of from @sch. + */ + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + guard(rq_lock_irqsave)(rq); + if (rq->scx.sched == sch) + scx_cid_sched_update(rq, NULL); + if (last) + rq->scx.sched = NULL; + } +} + static struct scx_sched *scx_find_sub_sched(u64 cgroup_id) { return rhashtable_lookup(&scx_sched_hash, &cgroup_id, @@ -285,7 +447,8 @@ void scx_rescue_charge(struct rq *rq, s64 delta_exec) rq->scx.rescue.budget -= delta_exec; /* per-cpu usage average feeds the overload victim pick */ - pcpu = per_cpu_ptr(scx_task_sched(rq->curr)->pcpu, cpu_of(rq)); + pcpu = per_cpu_ptr(scx_task_sched(rq->scx.rescue.curr)->pcpu, + cpu_of(rq)); pcpu->rescue_avg = scx_rescue_decay_avg(pcpu) + delta_exec; if (!scx_rescue_slice_remaining(rq)) @@ -555,9 +718,9 @@ static void scx_rescue_timerfn(struct timer_list *timer) scx.dsq_list.node); scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq); scx_rescue_admit(rq, p, slice); - scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, - &rq->scx.rescue.dsq, rq); - if (sched_class_above(&ext_sched_class, rq->curr->sched_class)) + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, + SCX_ENQ_IGNORE_CAPS, rq); + if (sched_class_above(&ext_sched_class, rq->donor->sched_class)) resched_curr(rq); } else if (p->scx.dsq && rq->scx.rescue.budget > 2 * scx_rescue_quantum_ns) { /* @@ -572,7 +735,7 @@ static void scx_rescue_timerfn(struct timer_list *timer) scx_task_unlink_from_dsq(p, &rq->scx.local_dsq); scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_IGNORE_CAPS, - &rq->scx.local_dsq, rq); + rq); } out_arm: scx_rescue_timer_arm(rq); @@ -585,10 +748,6 @@ void scx_rescue_flush(struct rq *rq) lockdep_assert_rq_held(rq); - /* sched domain rebuilds call rq_offline with the CPU staying alive */ - if (cpu_active(cpu_of(rq))) - return; - /* end the current rescue */ if (rq->scx.rescue.curr) scx_task_slice_ended(rq, rq->scx.rescue.curr); @@ -596,8 +755,8 @@ void scx_rescue_flush(struct rq *rq) /* and flush out all pending ones */ list_for_each_entry_safe(p, n, &rq->scx.rescue.dsq.list, scx.dsq_list.node) { scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq); - scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, - &rq->scx.rescue.dsq, rq); + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, + SCX_ENQ_IGNORE_CAPS, rq); } timer_delete(&rq->scx.rescue.timer); @@ -675,7 +834,7 @@ void scx_rescue_init(struct rq *rq) } /** - * scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert + * __scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert * @sch: enqueuing sub-sched * @rq: rq whose local DSQ @p targets * @p: task being inserted @@ -686,20 +845,18 @@ void scx_rescue_init(struct rq *rq) * rescue is enabled, or @rq's reject DSQ after recording the reenq reason on * @p. * - * %SCX_ENQ_IMMED, %SCX_ENQ_PREEMPT and %SCX_ENQ_HEAD are cleared when diverting - * to rescue or reject. %SCX_ENQ_PREEMPT is also cleared on a fallback - * migration-disabled admission. + * %SCX_ENQ_IMMED, %SCX_ENQ_PREEMPT, %SCX_ENQ_PREEMPT_LAZY and %SCX_ENQ_HEAD are + * cleared when diverting to rescue or reject. %SCX_ENQ_PREEMPT and + * %SCX_ENQ_PREEMPT_LAZY are also cleared on a fallback migration-disabled + * admission. * * Bypass doesn't need special-casing as a bypassing sched's tasks are enqueued * to and run by its nearest non-bypassing ancestor. If root is bypassing, it * always holds all caps. */ -struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 *enq_flags) +struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags) { - if (!scx_has_subs()) - return &rq->scx.local_dsq; - s32 cid = __scx_cpu_to_cid(cpu_of(rq)); struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch; u64 needed = scx_caps_for_enq(*enq_flags); @@ -709,7 +866,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r * On a remote activation the scheduling sched (@asch) differs from * @p's owner (@sch). Check caps against the scheduling sched. */ - if (*enq_flags & SCX_ENQ_PREEMPT) + if (*enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) needed |= scx_caps_for_preempt(asch, rq, *enq_flags); missing = scx_missing_caps(asch, cpu_of(rq), needed); @@ -726,7 +883,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r if (unlikely(!scx_rq_online(rq) || is_migration_disabled(p) || p->migration_pending)) { __scx_add_event(sch, SCX_EV_SUB_FORCED_ADMIT, 1); - *enq_flags &= ~SCX_ENQ_PREEMPT; + *enq_flags &= ~(SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY); return &rq->scx.local_dsq; } @@ -735,9 +892,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r * or HEAD - a diversion has no priority and IMMED is not allowed on * non-local DSQs. Strip the enq and task flags along with the slice. */ - *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | SCX_ENQ_HEAD | - SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL); - p->scx.flags &= ~SCX_TASK_IMMED; + scx_divert_strip_flags(p, enq_flags); /* the enqueuer opted for rescue instead of rejection and reenqueue */ if ((*enq_flags & SCX_ENQ_RESCUE) && likely(scx_rescue_bw_1024)) { @@ -752,6 +907,8 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r p->scx.reenq_reason_caps = missing; p->scx.reenq_reason_cid = cid; + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); + p->scx.flags |= SCX_TASK_REENQ_CAP; return &rq->scx.reject_dsq; } @@ -759,66 +916,24 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r /* @p lost the caps needed to stay on @rq's local DSQ? Record reason if so. */ bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { + s32 cpu = cpu_of(rq); u64 missing; /* migration-disabled tasks and the rescuee are admitted capless */ if (is_migration_disabled(p) || p == scx_rescuee(rq)) return false; - missing = scx_missing_caps(scx_task_sched(p), cpu_of(rq), scx_caps_for_task(p)); + /* check caps against the sched running a bypassed task */ + missing = scx_missing_caps(scx_exec_sched(scx_task_sched(p), cpu), cpu, + scx_caps_for_task(p)); if (likely(!missing)) return false; p->scx.reenq_reason_caps = missing; - p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu_of(rq)); + p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu); return true; } -/* - * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides - * from p->scx.reenq_reason_*. - * - * A task can be re-rejected repeatedly. The reenqueue is bounded per task in - * scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT. - * Rejection can't happen for root. - */ -void scx_reenq_reject(struct rq *rq) -{ - LIST_HEAD(tasks); - struct task_struct *p, *n; - - lockdep_assert_rq_held(rq); - - if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list)) - return; - - /* - * Move to a private list so a task re-rejected by the - * scx_do_enqueue_task() below isn't revisited this round. - */ - list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) { - /* migration_pending tasks should have bypassed to local DSQ */ - if (WARN_ON_ONCE(p->migration_pending)) - continue; - - scx_dispatch_dequeue(rq, p); - - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - p->scx.flags |= SCX_TASK_REENQ_CAP; - - list_add_tail(&p->scx.dsq_list.node, &tasks); - } - - list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { - list_del_init(&p->scx.dsq_list.node); - - scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - } -} - /* record a caps change, see struct scx_caps_updated */ static void caps_updated_record(struct scx_pshard *ps, const struct scx_cmask *cids, u64 caps, struct list_head *to_deliver) @@ -924,7 +1039,7 @@ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid) struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); /* - * Pairs with smp_mb() in scx_process_sync_ecaps(). Either the check + * Pairs with smp_mb() in __scx_process_sync_ecaps(). Either the check * below sees the node off the list and queues it, or the in-flight sync * sees the caps[] update made before this call. */ @@ -949,7 +1064,7 @@ static void discard_queued_syncs(struct rq *rq) } /** - * scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] + * __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] * @rq: the cid's cpu rq * @prev: @rq's previous task from the in-progress dispatch * @@ -961,16 +1076,14 @@ static void discard_queued_syncs(struct rq *rq) * learns the cid's idle state. Such a gain arms the per-rq * %SCX_RQ_SUB_IDLE_RENOTIFY gate so the next idle pick delivers it. */ -void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) +void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) { s32 cpu = cpu_of(rq); s32 cid, shard; struct llist_node *batch, *pos, *tmp; u64 lost_all = 0; - lockdep_assert_rq_held(rq); - - if (!scx_has_subs() || likely(llist_empty(&rq->scx.ecaps_to_sync))) + if (likely(llist_empty(&rq->scx.ecaps_to_sync))) return; /* @@ -1011,27 +1124,43 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) lost_all |= lost; /* - * Tell the sched its effective caps on this cid changed. The - * invocation is equivalent to the dispatch path and may drop - * and re-acquire the rq lock temporarily while the rest of - * @batch is held privately, see scx_discard_ecaps_to_sync(). - * The dispatch kfuncs resolve their context on the executing - * cpu, which under core scheduling can differ from @rq's cpu, - * so the context is set up there. The rq recorded in it keeps - * the dispatches targeting @rq. + * Tell the sched and its parent that the sched's effective caps + * on this cid changed. The invocations are equivalent to the + * dispatch path and may drop and re-acquire the rq lock + * temporarily while the rest of @batch is held privately, see + * scx_discard_ecaps_to_sync(). The dispatch kfuncs resolve + * their context on the executing cpu, which under core + * scheduling can differ from @rq's cpu, so the context is set + * up there. The rq recorded in it keeps the dispatches + * targeting @rq. + * + * Bypass propagates down the hierarchy, so a sched that isn't + * bypassing has no bypassing parent. The child's bypass state + * gates both deliveries. Both report the same before value, so + * one reported_ecaps covers them. */ - if (ecaps != pcpu->reported_ecaps && - SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) && - !scx_bypassing(pcpu->sch, cpu)) { - struct scx_dsp_ctx *dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx; + if (ecaps != pcpu->reported_ecaps && !scx_bypassing(pcpu->sch, cpu)) { + struct scx_sched *parent = scx_parent(pcpu->sch); + struct scx_dsp_ctx *dspc; - dspc->rq = rq; /* stash @prev so nested dispatches can access it */ rq->scx.sub_dispatch_prev = prev; - SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, scx_cpu_arg(cpu), - pcpu->reported_ecaps, ecaps); + if (SCX_HAS_OP(pcpu->sch, sub_ecaps_updated)) { + dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx; + dspc->rq = rq; + SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, + scx_cpu_arg(cpu), pcpu->reported_ecaps, ecaps); + scx_flush_dispatch_buf(pcpu->sch, rq); + } + if (SCX_HAS_OP(parent, sub_child_ecaps_updated)) { + dspc = &this_cpu_ptr(parent->pcpu)->dsp_ctx; + dspc->rq = rq; + SCX_CALL_OP(parent, sub_child_ecaps_updated, rq, + pcpu->sch->ops.sub_cgroup_id, scx_cpu_arg(cpu), + pcpu->reported_ecaps, ecaps); + scx_flush_dispatch_buf(parent, rq); + } rq->scx.sub_dispatch_prev = NULL; - scx_flush_dispatch_buf(pcpu->sch, rq); pcpu->reported_ecaps = ecaps; } @@ -1157,6 +1286,34 @@ void scx_offline_ecaps(struct rq *rq) } /* + * Clear every cap @sch holds. The pshard caps go first as they are the source a + * pending sync recomputes ecaps from. ecaps are then zeroed directly for the + * cap checks. + */ +static void clear_all_caps(struct scx_sched *sch) +{ + s32 si, cpu; + u32 cap_bit; + + /* enable may have failed before scx_alloc_pshards() */ + if (!sch->pshard) + return; + + for (si = 0; si < sch->nr_pshards; si++) { + struct scx_pshard *ps = sch->pshard[si]; + + guard(raw_spinlock_irqsave)(&ps->lock); + for (cap_bit = 0; cap_bit < __SCX_NR_CAPS; cap_bit++) + scx_cmask_clear(&ps->caps[cap_bit].cmask); + } + + for_each_possible_cpu(cpu) { + guard(rq_lock_irqsave)(cpu_rq(cpu)); + WRITE_ONCE(per_cpu_ptr(sch->pcpu, cpu)->ecaps, 0); + } +} + +/* * @pcpu's sched was unhashed before the grace period, so nothing re-queues its * sync node. Remove the node from @rq's pending list so the pcpu can be freed. */ @@ -1255,7 +1412,9 @@ static void scx_rehome_task(struct scx_sched *to, struct task_struct *p) lockdep_assert_held(&p->pi_lock); lockdep_assert_rq_held(task_rq(p)); - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_prepare_task_sched_change(p); + scoped_guard (sched_change, p, + DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { scx_disable_and_exit_task(scx_task_sched(p), p); scx_set_task_state(p, SCX_TASK_INIT_BEGIN); scx_set_task_state(p, SCX_TASK_INIT); @@ -1285,7 +1444,9 @@ static void scx_punt_task(struct scx_sched *to, struct task_struct *p) lockdep_assert_rq_held(task_rq(p)); WARN_ON_ONCE(!READ_ONCE(to->bypass_depth)); - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_prepare_task_sched_change(p); + scoped_guard (sched_change, p, + DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { scx_disable_and_exit_task(scx_task_sched(p), p); scx_set_task_sched(p, to); } @@ -1361,7 +1522,7 @@ static s32 scx_cgroup_claim_subtree(struct scx_sched *sch) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, - .sched_idle = tg->scx.idle, + .sched_idle = tg->scx.sched_idle, }; if (tg->scx.sched != parent || @@ -1465,7 +1626,7 @@ static void scx_cgroup_return_subtree(struct scx_sched *sch) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, - .sched_idle = tg->scx.idle, + .sched_idle = tg->scx.sched_idle, }; /* the first pass must have transferred everything */ @@ -1619,6 +1780,8 @@ dump: scx_cgroup_unlock(); percpu_up_write(&scx_fork_rwsem); + scx_ops_cid_sched_updated_disable(sch); + /* * All tasks are moved off of @sch but there may still be on-going * operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use @@ -1631,6 +1794,12 @@ dump: scx_unlink_sched(sch); + /* + * The parent restores what it delegated in ops.sub_detach(). @sch must + * hold no caps by then. + */ + clear_all_caps(sch); + mutex_unlock(&scx_enable_mutex); /* @@ -1790,6 +1959,15 @@ void scx_sub_enable_workfn(struct kthread_work *work) if (ret) goto err_disable; + /* + * Bypass before @sch is linked and grants can reach it. The parent's + * delivery advances reported_ecaps, so a sync consumed before @sch's + * ops are registered would reach the parent and never be replayed to + * @sch. While bypassed, the syncs are consumed without a delivery and + * replayed to both at unbypass. + */ + scx_bypass(sch, true); + ret = scx_link_sched(sch); if (ret) goto err_disable; @@ -1835,12 +2013,12 @@ void scx_sub_enable_workfn(struct kthread_work *work) } sch->sub_attached = true; - scx_bypass(sch, true); - for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op); + scx_ops_cid_sched_updated_enable(sch); + percpu_down_write(&scx_fork_rwsem); scx_cgroup_lock(); @@ -1943,7 +2121,9 @@ void scx_sub_enable_workfn(struct kthread_work *work) if (!(p->scx.flags & SCX_TASK_SUB_INIT)) continue; - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_prepare_task_sched_change(p); + scoped_guard (sched_change, p, + DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { /* * $p must be either READY or ENABLED. If ENABLED, * __scx_disabled_and_exit_task() first disables and @@ -2269,6 +2449,9 @@ static s32 sub_cap_preamble(u64 cgroup_id, u64 caps, const struct bpf_prog_aux * if (unlikely(!parent)) return -ENODEV; + if (!scx_kf_allowed_ctx(parent)) + return -EDEADLK; + if (!scx_is_cid_type()) { scx_error(parent, "sub-cap kfuncs require a cid-form scheduler"); return -EOPNOTSUPP; @@ -2309,8 +2492,8 @@ static s32 sub_cap_preamble(u64 cgroup_id, u64 caps, const struct bpf_prog_aux * * All-or-nothing keeps the caller-visible result binary per cid, so the denied * mask is one mask to interpret rather than a per-cap matrix. * - * Return 0 on full success, -EPERM if any cid was refused, or a negative - * errno on other failures. + * Return 0 on full success, -EPERM if any cid was refused, -ENODEV if the child + * is gone or being disabled, or a negative errno on other failures. */ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask *cmask__arena, @@ -2364,6 +2547,12 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, scoped_guard (raw_spinlock, &pps->lock) { guard(raw_spinlock_nested)(&cps->lock); + /* the child is being disabled, see clear_all_caps() */ + if (unlikely(READ_ONCE(child->aborting))) { + ret = -ENODEV; + goto out; + } + /* * Narrow granted_cids to cids the parent holds every * requested cap on. All-or-nothing per cid. @@ -2416,9 +2605,10 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, } } + ret = any_denied ? -EPERM : 0; +out: caps_updated_deliver(&to_deliver); - - return any_denied ? -EPERM : 0; + return ret; } /** diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index f7bcdfda8dd8..b76d750463f0 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -16,6 +16,9 @@ struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root); struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root); +void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch); +void scx_ops_cid_sched_updated_enable(struct scx_sched *sch); +void scx_ops_cid_sched_updated_disable(struct scx_sched *sch); void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch); struct cgroup *sch_cgroup(struct scx_sched *sch); void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch); @@ -28,16 +31,15 @@ bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux); void scx_free_pshards(struct scx_sched *sch); s32 scx_alloc_pshards(struct scx_sched *sch); void scx_init_root_caps(struct scx_sched *sch); -void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); +void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch); void scx_online_ecaps(struct rq *rq); void scx_offline_ecaps(struct rq *rq); void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu); void scx_discard_stale_ecaps_syncs(void); -struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 *enq_flags); +struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags); bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p); -void scx_reenq_reject(struct rq *rq); void scx_rescue_charge(struct rq *rq, s64 delta_exec); void scx_rescue_end(struct rq *rq); bool scx_rescue_keep(struct rq *rq, struct task_struct *p); @@ -72,10 +74,32 @@ static inline void scx_dec_has_subs(struct scx_sched *sch) static_branch_dec(&__scx_has_subs); } +/* hot-path hooks, gated inline so that a root-only system skips the calls */ +static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) +{ + lockdep_assert_rq_held(rq); + + if (scx_has_subs()) + __scx_process_sync_ecaps(rq, prev); +} + +static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, + struct rq *rq, + struct task_struct *p, + u64 *enq_flags) +{ + if (!scx_has_subs()) + return &rq->scx.local_dsq; + return __scx_resolve_local_dsq(sch, rq, p, enq_flags); +} + #else /* CONFIG_EXT_SUB_SCHED */ static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } static inline struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root) { return NULL; } +static inline void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch) {} +static inline void scx_ops_cid_sched_updated_enable(struct scx_sched *sch) {} +static inline void scx_ops_cid_sched_updated_disable(struct scx_sched *sch) {} static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } static inline const char *sch_cgrp_path(struct scx_sched *sch) { return "/"; } @@ -93,7 +117,6 @@ static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcp static inline void scx_discard_stale_ecaps_syncs(void) {} static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; } static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; } -static inline void scx_reenq_reject(struct rq *rq) {} static inline void scx_rescue_charge(struct rq *rq, s64 delta_exec) {} static inline void scx_rescue_end(struct rq *rq) {} static inline bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { return false; } @@ -169,17 +192,18 @@ static inline u64 scx_caps_for_task(struct task_struct *p) return SCX_CAP_ENQ; } -/* the cap @sch needs to preempt @rq's current task, 0 if none */ -static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) +/* the cap @sch needs to preempt @rq's current scheduling context, 0 if none */ +static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, + u64 enq_flags) { - struct task_struct *curr = rq->curr; + struct task_struct *donor = rq->donor; /* a kernel-forced placement preempts regardless of caps */ if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS)) return 0; /* a non-ext task can't be preempted by ext, own-subtree needs no cap */ - if (curr->sched_class != &ext_sched_class || - scx_is_descendant(scx_task_sched(curr), sch)) + if (donor->sched_class != &ext_sched_class || + scx_is_descendant(scx_task_sched(donor), sch)) return 0; return SCX_CAP_PREEMPT; } @@ -199,6 +223,8 @@ static inline u64 scx_caps_implied(u64 cap) /* may @p keep running on @rq's cpu? requires baseline cpu access */ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { + s32 cpu = cpu_of(rq); + if (!scx_has_subs()) return true; @@ -206,7 +232,9 @@ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p if (unlikely(is_migration_disabled(p))) return true; - return likely(!scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE)); + /* check caps against the sched running a bypassed task */ + return likely(!scx_missing_caps(scx_exec_sched(scx_task_sched(p), cpu), cpu, + SCX_CAP_BASE)); } /* the task admitted for rescue on @rq, NULL if none */ diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index 943d8d429a2c..38514d6bf149 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -59,17 +59,35 @@ enum scx_consts { }; /* - * Per-cid topology info. For each topology level (core, LLC, node) and shard, - * records the first cid in the unit and its global index. Global indices are - * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges over - * [0, nr_cores_at_init) with no gaps. No-topo cids have core/LLC/node fields - * set to -1 but always have valid shard assignments. + * Per-cid topology info. For each topology level (core, cluster, LLC, node) and + * shard, records the first cid in the unit and its global index. Global indices + * are consecutive integers assigned in cid-walk order, so e.g. core_idx ranges + * over [0, nr_cores_at_init) with no gaps. No-topo cids have core/cluster/LLC/ + * node fields set to -1 but always have valid shard assignments. + * + * A cluster is the cache-sharing level between the core and the LLC. Where + * this level is absent, each core forms a cluster of its own. Where the cache + * level spans the LLC, the whole LLC forms one cluster. Each cluster has a + * unique cluster_cid and a dense cluster_idx, and its cids form a contiguous + * range. + * + * Every cid therefore has a cluster, so cluster_cid alone does not say whether + * the machine has the level: the first cluster of an LLC starts at the LLC's + * own base cid and a core-wide one at the core's. * * Shards are contiguous CID ranges used as scalable locking/work domains for * sub-scheduler operations. By default each LLC becomes one shard, split into * smaller shards if the LLC exceeds the target size. No-topo cids are packed * into their own max-sized shards. * + * Shards are cut on core boundaries only, so a shard doesn't need to contain a + * whole cluster: a cluster may straddle two shards. Only core, cluster, LLC + * and node are nested, in that order. + * + * New fields are appended, never inserted: scx_bpf_cid_topo() copies this + * struct out sized by the program's own layout, and an older program's copy + * must stay a prefix of the kernel's. + * * @core_cid: first cid of this cid's core (smt-sibling group) * @core_idx: global index of that core, in [0, nr_cores_at_init) * @llc_cid: first cid of this cid's LLC @@ -78,6 +96,8 @@ enum scx_consts { * @node_idx: global index of that node, in [0, nr_nodes_at_init) * @shard_cid: first cid of this cid's shard * @shard_idx: global index of that shard, in [0, scx_nr_cid_shards) + * @cluster_cid: first cid of this cid's cluster + * @cluster_idx: global index of that cluster, in [0, nr_clusters_at_init) */ struct scx_cid_topo { s32 core_cid; @@ -88,6 +108,8 @@ struct scx_cid_topo { s32 node_idx; s32 shard_cid; s32 shard_idx; + s32 cluster_cid; + s32 cluster_idx; }; enum scx_cid_consts { diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 7455a83a6a99..b5546cd27dba 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -24,6 +24,7 @@ #include <linux/mmap_lock.h> #include <linux/hugetlb_inline.h> #include <linux/jiffies.h> +#include <linux/math.h> #include <linux/mm_api.h> #include <linux/highmem.h> #include <linux/hrtimer.h> @@ -820,12 +821,6 @@ static u64 ineligible_vruntime(struct cfs_rq *cfs_rq) if (curr && !curr->on_rq) curr = NULL; - /* - * This is called from set_next_task_fair(.first=true) / - * set_protect_slice() so curr had better be set and on_rq. - */ - WARN_ON_ONCE(!curr); - if (weight) { s64 runtime = cfs_rq->sum_w_vruntime; @@ -1004,71 +999,17 @@ static inline bool __entity_less(struct rb_node *a, const struct rb_node *b) return entity_before(__node_2_se(a), __node_2_se(b)); } -static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node *node) -{ - if (node) { - struct sched_entity *rse = __node_2_se(node); - - if (vruntime_cmp(se->min_vruntime, ">", rse->min_vruntime)) - se->min_vruntime = rse->min_vruntime; - } -} - -static inline void __min_slice_update(struct sched_entity *se, struct rb_node *node) +/* min() for wrapping vruntimes */ +static inline u64 __min_vruntime(u64 a, u64 b) { - if (node) { - struct sched_entity *rse = __node_2_se(node); - if (rse->min_slice < se->min_slice) - se->min_slice = rse->min_slice; - } -} - -static inline void __max_slice_update(struct sched_entity *se, struct rb_node *node) -{ - if (node) { - struct sched_entity *rse = __node_2_se(node); - if (rse->max_slice > se->max_slice) - se->max_slice = rse->max_slice; - } + return vruntime_cmp(a, "<", b) ? a : b; } -static inline void min_vruntime_copy(struct sched_entity *new, struct sched_entity *old) -{ - new->min_vruntime = old->min_vruntime; - new->min_slice = old->min_slice; - new->max_slice = old->max_slice; -} - -/* - * se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime) - */ -static inline bool min_vruntime_update(struct sched_entity *se, bool exit) -{ - u64 old_min_vruntime = se->min_vruntime; - u64 old_min_slice = se->min_slice; - u64 old_max_slice = se->max_slice; - struct rb_node *node = &se->run_node; - - se->min_vruntime = se->vruntime; - __min_vruntime_update(se, node->rb_right); - __min_vruntime_update(se, node->rb_left); - - se->min_slice = se->slice; - __min_slice_update(se, node->rb_right); - __min_slice_update(se, node->rb_left); - - se->max_slice = se->slice; - __max_slice_update(se, node->rb_right); - __max_slice_update(se, node->rb_left); - - return se->min_vruntime == old_min_vruntime && - se->min_slice == old_min_slice && - se->max_slice == old_max_slice; -} - - -RB_DECLARE_CALLBACKS_MULTI(static, min_vruntime_cb, struct sched_entity, - run_node, min_vruntime_copy, min_vruntime_update); +RB_DECLARE_CALLBACKS(static, min_vruntime_cb, + struct sched_entity, run_node, + RB_AUG(vruntime, min_vruntime, __min_vruntime), + RB_AUG(slice, min_slice, min), + RB_AUG(slice, max_slice, max)); /* * Enqueue an entity into the rb-tree: @@ -1137,10 +1078,9 @@ static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity /* If there are shorter slices than se's one */ if (slice != se->slice) { + vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se)); if (sched_feat(PREEMPT_SHORT)) vprot = min_vruntime(vprot, ineligible_vruntime(cfs_rq)); - else - vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se)); } se->vprot = vprot; @@ -1148,10 +1088,14 @@ static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) { - u64 slice = cfs_rq_min_slice(cfs_rq); u64 vruntime = min_vruntime(se->vruntime, avg_vruntime(cfs_rq)); + u64 slice = cfs_rq_min_slice(cfs_rq); + u64 vprot = min_vruntime(se->vprot, vruntime + calc_delta_fair(slice, se)); + + if (sched_feat(PREEMPT_SHORT) && slice != se->slice) + vprot = min_vruntime(vprot, ineligible_vruntime(cfs_rq)); - se->vprot = min_vruntime(se->vprot, vruntime + calc_delta_fair(slice, se)); + se->vprot = vprot; } static inline bool protect_slice(struct sched_entity *se) @@ -1478,7 +1422,7 @@ static inline int get_sched_cache_scale(int mul) return (1 + (tol - 1) * mul); } -static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) +static bool exceed_llc_capacity(struct sched_cache_group *grp, int cpu) { #ifdef CONFIG_NUMA_BALANCING unsigned long llc, footprint; @@ -1497,7 +1441,7 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) * excluded. */ llc = sd->llc_bytes; - footprint = READ_ONCE(mm->sc_stat.footprint); + footprint = READ_ONCE(grp->footprint); /* * Scale the LLC size by 256*llc_aggr_tolerance @@ -1526,7 +1470,7 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) return false; } -static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, +static bool invalid_llc_nr(struct sched_cache_group *grp, struct task_struct *p, int cpu) { int scale; @@ -1542,10 +1486,32 @@ static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, if (scale == INT_MAX) return false; - return !fits_capacity((mm->sc_stat.nr_running_avg * cpu_smt_num_threads), + return !fits_capacity((READ_ONCE(grp->nr_running_avg) * cpu_smt_num_threads), (scale * per_cpu(sd_llc_size, cpu))); } +/* + * A task counts in nr_pref_llc_running while it is queued on its preferred + * LLC (pref_llc_queued) and runnable (!sched_delayed), keeping the counter in + * the runnable domain so alb_break_llc() can compare it with h_nr_runnable. + */ +static bool task_pref_llc_runnable(struct task_struct *p) +{ + return p->pref_llc_queued && !p->se.sched_delayed; +} + +static void pref_llc_running_inc(struct rq *rq, struct task_struct *p) +{ + if (task_pref_llc_runnable(p)) + rq->nr_pref_llc_running++; +} + +static void pref_llc_running_dec(struct rq *rq, struct task_struct *p) +{ + if (task_pref_llc_runnable(p)) + rq->nr_pref_llc_running--; +} + static void account_llc_enqueue(struct rq *rq, struct task_struct *p) { int pref_llc, pref_llc_queued; @@ -1557,7 +1523,6 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p) pref_llc_queued = (pref_llc == task_llc(p)); rq->nr_llc_running++; - rq->nr_pref_llc_running += pref_llc_queued; /* * Record whether p is enqueued on its preferred @@ -1575,6 +1540,9 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p) */ p->pref_llc_queued = pref_llc_queued; + /* Skipped while delayed; clear_delayed() adds it back on wake. */ + pref_llc_running_inc(rq, p); + sd = rcu_dereference_all(rq->sd); if (sd && (unsigned int)pref_llc < sd->llc_max) sd->llc_counts[pref_llc]++; @@ -1591,7 +1559,12 @@ static void account_llc_dequeue(struct rq *rq, struct task_struct *p) rq->nr_llc_running--; if (p->pref_llc_queued) { - rq->nr_pref_llc_running--; + /* + * Skipped if still delayed (set_delayed() already removed it); + * clearing pref_llc_queued below also stops clear_delayed() + * from re-adding it. + */ + pref_llc_running_dec(rq, p); /* * Update the status in case * other logic might query @@ -1619,12 +1592,20 @@ static void account_llc_dequeue(struct rq *rq, struct task_struct *p) } } -void mm_init_sched(struct mm_struct *mm, - struct sched_cache_time __percpu *_pcpu_sched) +int mm_init_sched(struct mm_struct *mm, + struct sched_cache_time __percpu *_pcpu_sched) { + struct sched_cache_group *grp; unsigned long epoch = 0; int i; + grp = kzalloc_obj(*grp); + if (!grp) { + free_percpu(_pcpu_sched); + mm->sched_cache_grp = NULL; + return -ENOMEM; + } + for_each_possible_cpu(i) { struct sched_cache_time *pcpu_sched = per_cpu_ptr(_pcpu_sched, i); struct rq *rq = cpu_rq(i); @@ -1635,18 +1616,141 @@ void mm_init_sched(struct mm_struct *mm, epoch = rq->cpu_epoch; } - raw_spin_lock_init(&mm->sc_stat.lock); - mm->sc_stat.epoch = epoch; - mm->sc_stat.cpu = -1; - mm->sc_stat.next_scan = jiffies; - mm->sc_stat.nr_running_avg = 0; - mm->sc_stat.footprint = 0; + raw_spin_lock_init(&grp->lock); + grp->epoch = epoch; + grp->cpu = -1; + grp->next_scan = jiffies; + grp->nr_running_avg = 0; + grp->footprint = 0; + refcount_set(&grp->refcnt, 1); /* - * The update to mm->sc_stat should not be reordered - * before initialization to mm's other fields, in case + * The update to grp->pcpu_sched should not be reordered + * before initialization to grp's other fields, in case * the readers may get invalid mm_sched_epoch, etc. */ - smp_store_release(&mm->sc_stat.pcpu_sched, _pcpu_sched); + smp_store_release(&grp->pcpu_sched, _pcpu_sched); + /* + * Publish the group last. Not every reader qualifies it by + * grp->pcpu_sched - can_migrate_llc_task() only checks that the + * pointer is non-NULL before reading grp->footprint and + * grp->nr_running_avg - so a reachable group must already be + * fully initialized. + */ + smp_store_release(&mm->sched_cache_grp, grp); + return 0; +} + +static void sched_cache_group_free_rcu(struct rcu_head *rcu) +{ + struct sched_cache_group *grp = + container_of(rcu, struct sched_cache_group, rcu); + + free_percpu(grp->pcpu_sched); + kfree(grp); +} + +static void sched_cache_group_put(struct sched_cache_group *grp) +{ + if (!grp || !refcount_dec_and_test(&grp->refcnt)) + return; + + call_rcu(&grp->rcu, sched_cache_group_free_rcu); +} + +DEFINE_FREE(sched_cache_group_put, struct sched_cache_group *, + sched_cache_group_put(_T)); + +#define rcu_deref_sched_cache_grp(tsk) \ + rcu_dereference_check((tsk)->sched_cache_grp, (tsk) == current) + +static struct sched_cache_group *sched_cache_replace_grp(struct task_struct *p, + struct sched_cache_group *new) +{ + struct sched_cache_group *old; + + old = rcu_deref_sched_cache_grp(p); + rcu_assign_pointer(p->sched_cache_grp, new); + + return old; +} + +struct sched_cache_group *sched_cache_group_get(struct sched_cache_group *grp) +{ + /* + * refcount_inc_not_zero() is the acquire primitive for lockless + * (RCU) lookups; plain refcount_inc() would scribble the count if + * it already reached zero. Return NULL in that case. + */ + if (grp && !refcount_inc_not_zero(&grp->refcnt)) + grp = NULL; + + return grp; +} + +struct sched_cache_group *task_cache_group_get(struct task_struct *p) +{ + guard(rcu)(); + return sched_cache_group_get(rcu_dereference(p->sched_cache_grp)); +} + +void sched_cache_fork(struct task_struct *p) +{ + /* + * The child takes its own reference on the mm's cache group, separate + * from the reference held by the mm. @p is not yet visible to readers, + * so a plain initializing store is enough. + */ + RCU_INIT_POINTER(p->sched_cache_grp, + sched_cache_group_get(p->mm->sched_cache_grp)); +} + +void sched_cache_fork_cleanup(struct task_struct *p) +{ + /* + * A fork that fails after sched_cache_fork() never reaches exit_mm(), + * so drop the reference here. @p never became visible, so there are no + * concurrent readers and the reference we hold keeps the group alive. + */ + sched_cache_group_put(rcu_access_pointer(p->sched_cache_grp)); + RCU_INIT_POINTER(p->sched_cache_grp, NULL); +} + +void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm) +{ + struct sched_cache_group *old; + + /* + * Acquire the new reference before publishing the pointer, then drop + * the old one. @p is current and the only writer of its own pointer. + */ + old = sched_cache_replace_grp(p, sched_cache_group_get(mm->sched_cache_grp)); + sched_cache_group_put(old); +} + +void sched_cache_exit_mm(struct task_struct *p) +{ + struct sched_cache_group *grp = sched_cache_replace_grp(p, NULL); + +#ifdef CONFIG_NUMA_BALANCING + /* + * Subtract this task's footprint from the group before dropping the + * reference, so the group footprint converges as its threads exit. + * Unlocked for performance; clamp to avoid underflow. + */ + if (grp && p->total_numa_faults) { + unsigned long fp = READ_ONCE(grp->footprint); + unsigned long sub = min(fp, p->total_numa_faults); + + WRITE_ONCE(grp->footprint, fp - sub); + } +#endif + sched_cache_group_put(grp); +} + +void mm_destroy_sched(struct mm_struct *mm) +{ + sched_cache_group_put(mm->sched_cache_grp); + mm->sched_cache_grp = NULL; } /* because why would C be fully specified */ @@ -1697,14 +1801,14 @@ static unsigned long fraction_mm_sched(struct rq *rq, return div64_u64(NICE_0_LOAD * pcpu_sched->runtime, rq->cpu_runtime + 1); } -static int get_pref_llc(struct task_struct *p, struct mm_struct *mm) +static int get_pref_llc(struct task_struct *p, struct sched_cache_group *grp) { int mm_sched_llc = -1, mm_sched_cpu; - if (!mm) + if (!grp) return -1; - mm_sched_cpu = READ_ONCE(mm->sc_stat.cpu); + mm_sched_cpu = READ_ONCE(grp->cpu); if (mm_sched_cpu != -1) { mm_sched_llc = llc_id(mm_sched_cpu); @@ -1734,8 +1838,8 @@ static unsigned int task_running_on_cpu(int cpu, struct task_struct *p); static inline void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) { + struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp); struct sched_cache_time *pcpu_sched; - struct mm_struct *mm = p->mm; int mm_sched_llc = -1; unsigned long epoch; @@ -1746,12 +1850,18 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) return; /* * init_task, kthreads and user thread created - * by user_mode_thread() don't have mm. + * by user_mode_thread() don't have a cache group. + * In theory a kernel thread does not have any valid + * cache group, because sched_cache_fork() is not + * invoked for a kernel thread - !grp should gate the + * kernel thread. Use the PF_KTHREAD check explicitly + * here for safety reasons, to guard against future + * modifications and to pair with task_tick_cache(). */ - if (!mm || !mm->sc_stat.pcpu_sched) + if (p->flags & PF_KTHREAD || !grp || !grp->pcpu_sched) return; - pcpu_sched = per_cpu_ptr(mm->sc_stat.pcpu_sched, cpu_of(rq)); + pcpu_sched = per_cpu_ptr(grp->pcpu_sched, cpu_of(rq)); scoped_guard (raw_spinlock, &rq->cpu_epoch_lock) { __update_mm_sched(rq, pcpu_sched); @@ -1764,14 +1874,14 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) * If this process hasn't hit task_cache_work() for a while invalidate * its preferred state. */ - if ((long)(epoch - READ_ONCE(mm->sc_stat.epoch)) > llc_epoch_affinity_timeout || - invalid_llc_nr(mm, p, cpu_of(rq)) || - exceed_llc_capacity(mm, cpu_of(rq))) { - if (READ_ONCE(mm->sc_stat.cpu) != -1) - WRITE_ONCE(mm->sc_stat.cpu, -1); + if ((long)(epoch - READ_ONCE(grp->epoch)) > llc_epoch_affinity_timeout || + invalid_llc_nr(grp, p, cpu_of(rq)) || + exceed_llc_capacity(grp, cpu_of(rq))) { + if (READ_ONCE(grp->cpu) != -1) + WRITE_ONCE(grp->cpu, -1); } - mm_sched_llc = get_pref_llc(p, mm); + mm_sched_llc = get_pref_llc(p, grp); /* task not on rq accounted later in account_entity_enqueue() */ if (task_running_on_cpu(rq->cpu, p) && @@ -1784,31 +1894,32 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) static void task_tick_cache(struct rq *rq, struct task_struct *p) { + struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp); struct callback_head *work = &p->cache_work; - struct mm_struct *mm = p->mm; unsigned long epoch; if (!sched_cache_enabled()) return; - if (!mm || p->flags & PF_KTHREAD || - !mm->sc_stat.pcpu_sched) + if (!grp || p->flags & PF_KTHREAD || + !grp->pcpu_sched) return; epoch = rq->cpu_epoch; /* avoid moving backwards */ - if (time_after_eq(mm->sc_stat.epoch, epoch)) + if (time_after_eq(grp->epoch, epoch)) return; - guard(raw_spinlock)(&mm->sc_stat.lock); + guard(raw_spinlock)(&grp->lock); if (work->next == work) { task_work_add(p, work, TWA_RESUME); - WRITE_ONCE(mm->sc_stat.epoch, epoch); + WRITE_ONCE(grp->epoch, epoch); } } -static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p) +static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p, + struct sched_cache_group *grp) { #ifdef CONFIG_NUMA_BALANCING int cpu, curr_cpu, nid, pref_nid; @@ -1816,7 +1927,7 @@ static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p) if (!static_branch_likely(&sched_numa_balancing)) goto out; - cpu = READ_ONCE(p->mm->sc_stat.cpu); + cpu = READ_ONCE(grp->cpu); if (cpu != -1) nid = cpu_to_node(cpu); curr_cpu = task_cpu(p); @@ -1873,13 +1984,13 @@ static inline void update_avg_scale(u64 *avg, u64 sample) static void task_cache_work(struct callback_head *work) { + struct sched_cache_group *grp __free(sched_cache_group_put) = NULL; + cpumask_var_t cpus __free(free_cpumask_var) = CPUMASK_VAR_NULL; int cpu, m_a_cpu = -1, nr_running = 0, curr_cpu; unsigned long next_scan, now = jiffies; struct task_struct *p = current, *cur; unsigned long curr_m_a_occ = 0; - struct mm_struct *mm = p->mm; unsigned long m_a_occ = 0; - cpumask_var_t cpus; WARN_ON_ONCE(work != &p->cache_work); @@ -1888,21 +1999,30 @@ static void task_cache_work(struct callback_head *work) if (p->flags & PF_EXITING) return; - next_scan = READ_ONCE(mm->sc_stat.next_scan); + /* + * A reference makes sure grp is not released by others. The rcu + * lock can not be held till after zalloc_cpumask_var() below, + * because the latter might sleep. + */ + grp = task_cache_group_get(p); + if (!grp) + return; + + next_scan = READ_ONCE(grp->next_scan); if (time_before(now, next_scan)) return; /* only 1 thread is allowed to scan */ - if (!try_cmpxchg(&mm->sc_stat.next_scan, &next_scan, + if (!try_cmpxchg(&grp->next_scan, &next_scan, now + max_t(unsigned long, READ_ONCE(llc_epoch_period), 1))) return; curr_cpu = task_cpu(p); - if (invalid_llc_nr(mm, p, curr_cpu) || - exceed_llc_capacity(mm, curr_cpu)) { - if (READ_ONCE(mm->sc_stat.cpu) != -1) - WRITE_ONCE(mm->sc_stat.cpu, -1); + if (invalid_llc_nr(grp, p, curr_cpu) || + exceed_llc_capacity(grp, curr_cpu)) { + if (READ_ONCE(grp->cpu) != -1) + WRITE_ONCE(grp->cpu, -1); return; } @@ -1913,7 +2033,7 @@ static void task_cache_work(struct callback_head *work) scoped_guard (cpus_read_lock) { guard(rcu)(); - get_scan_cpumasks(cpus, p); + get_scan_cpumasks(cpus, p, grp); for_each_cpu(cpu, cpus) { /* XXX sched_cluster_active */ @@ -1926,16 +2046,20 @@ static void task_cache_work(struct callback_head *work) for_each_cpu(i, sched_domain_span(sd)) { occ = fraction_mm_sched(cpu_rq(i), - per_cpu_ptr(mm->sc_stat.pcpu_sched, i)); + per_cpu_ptr(grp->pcpu_sched, i)); a_occ += occ; if (occ > m_occ) { m_occ = occ; m_cpu = i; } + /* + * rcu_access_pointer() is used because the + * pointer is only compared, never dereferenced. + */ cur = rcu_dereference_all(cpu_rq(i)->curr); if (cur && !(cur->flags & (PF_EXITING | PF_KTHREAD)) && - cur->mm == mm) + rcu_access_pointer(cur->sched_cache_grp) == grp) nr_running++; } @@ -1959,7 +2083,7 @@ static void task_cache_work(struct callback_head *work) m_a_cpu = m_cpu; } - if (llc_id(cpu) == llc_id(READ_ONCE(mm->sc_stat.cpu))) + if (llc_id(cpu) == llc_id(READ_ONCE(grp->cpu))) curr_m_a_occ = a_occ; cpumask_andnot(cpus, cpus, sched_domain_span(sd)); @@ -1968,7 +2092,7 @@ static void task_cache_work(struct callback_head *work) if (m_a_occ > (2 * curr_m_a_occ)) { /* - * Avoid switching sc_stat.cpu too fast. + * Avoid switching sched_cache_grp->cpu too fast. * The reason to choose 2X is because: * 1. It is better to keep the preferred LLC stable, * rather than changing it frequently and cause migrations @@ -1977,11 +2101,10 @@ static void task_cache_work(struct callback_head *work) * 3. 2X is chosen based on test results, as it delivers * the optimal performance gain so far. */ - WRITE_ONCE(mm->sc_stat.cpu, m_a_cpu); + WRITE_ONCE(grp->cpu, m_a_cpu); } - update_avg_scale(&mm->sc_stat.nr_running_avg, nr_running); - free_cpumask_var(cpus); + update_avg_scale(&grp->nr_running_avg, nr_running); } void init_sched_mm(struct task_struct *p) @@ -1991,10 +2114,18 @@ void init_sched_mm(struct task_struct *p) init_task_work(work, task_cache_work); work->next = work; /* + * dup_task_struct() copies the parent's task_struct, including its + * sched_cache_grp, for which the child holds no reference. Clear it + * here - before copy_mm() runs - so the child never carries a + * borrowed pointer that the fork error path would put. + */ + RCU_INIT_POINTER(p->sched_cache_grp, NULL); + /* * Reset new task's preference to avoid * polluting account_llc_enqueue(). */ p->preferred_llc = -1; + p->pref_llc_queued = 0; } #else /* CONFIG_SCHED_CACHE */ @@ -2016,6 +2147,10 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p) {} static void account_llc_dequeue(struct rq *rq, struct task_struct *p) {} +static void pref_llc_running_inc(struct rq *rq, struct task_struct *p) {} + +static void pref_llc_running_dec(struct rq *rq, struct task_struct *p) {} + #endif /* CONFIG_SCHED_CACHE */ /* @@ -3522,7 +3657,7 @@ static void update_task_scan_period(struct task_struct *p, p->mm->numa_next_scan = jiffies + msecs_to_jiffies(p->numa_scan_period); - return; + goto out; } /* @@ -3566,7 +3701,10 @@ static void update_task_scan_period(struct task_struct *p, p->numa_scan_period = clamp(p->numa_scan_period + diff, task_scan_min(p), task_scan_max(p)); - memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); + +out: + memset(p->numa_faults_locality, 0, + sizeof(p->numa_faults_locality)); } /* @@ -3692,6 +3830,7 @@ static int preferred_group_nid(struct task_struct *p, int nid) static void task_numa_placement(struct task_struct *p) __context_unsafe(/* conditional locking */) { + struct sched_cache_group __maybe_unused *grp; int seq, nid, max_nid = NUMA_NO_NODE; unsigned long max_faults = 0; unsigned long fault_types[2] = { 0, 0 }; @@ -3784,19 +3923,24 @@ static void task_numa_placement(struct task_struct *p) * heuristic and occasional lost updates are tolerable. * * If a task exits, its corresponding footprint must - * be subtracted from the mm->sc_stat.footprint, otherwise - * the mm->sc_stat.footprint will not converge: - * the exiting thread's footprint remains unchanged/undecayed - * in mm->sc_stat.footprint. See exit_mm(). + * be subtracted from p->sched_cache_grp->footprint, + * otherwise the footprint will not converge: the + * exiting thread's footprint remains unchanged/undecayed. + * See exit_mm(). * * Lost updates and unsynchronized subtraction * in exit_mm() can cause footprint + diff to * go negative. Clamp to zero to prevent the * unsigned footprint from wrapping. */ - new_fp = (long)READ_ONCE(p->mm->sc_stat.footprint) + diff; - WRITE_ONCE(p->mm->sc_stat.footprint, - max(new_fp, 0L)); + scoped_guard(rcu) { + grp = rcu_dereference(p->sched_cache_grp); + + if (grp) { + new_fp = (long)READ_ONCE(grp->footprint) + diff; + WRITE_ONCE(grp->footprint, max(new_fp, 0L)); + } + } #endif } @@ -6266,12 +6410,20 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) load = cfs_rq->sum_weight; if (curr && curr->on_rq) load += avg_vruntime_weight(cfs_rq, curr->h_load.weight); - - weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); - lag *= load + weight; if (WARN_ON_ONCE(!load)) load = 1; - lag = div64_long(lag, load); + + weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); + + /* + * Rewrite, because the product (W + w_i)*vl'_i might + * overflow s64: + * + * vl_i = (W + w_i)*vl'_i / W + * = (W*vl'_i + w_i*vl'_i) / W + * = vl'_i + w_i*vl'_i/W + */ + lag += div64_long(lag * weight, load); /* * A heavy entity (relative to the tree) will pull the @@ -6390,15 +6542,27 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); static void set_delayed(struct sched_entity *se) { - se->sched_delayed = 1; - /* * Delayed se of cfs_rq have no tasks queued on them. * Do not adjust h_nr_runnable since __dequeue_task() * will account it for blocked tasks. + * + * This check can be removed because when flat pick + * patches get merged as only task can get delayed, + * same for clear_delayed(). */ - if (!entity_is_task(se)) + if (!entity_is_task(se)) { + se->sched_delayed = 1; return; + } + + /* + * Drop a task leaving the runnable set. + * Needs to be called before sched_delayed is set. + * clear_delayed() mirrors this after clearing the flag. + */ + pref_llc_running_dec(rq_of(cfs_rq_of(se)), task_of(se)); + se->sched_delayed = 1; for_each_sched_entity(se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); @@ -6420,6 +6584,13 @@ static void clear_delayed(struct sched_entity *se) if (!entity_is_task(se)) return; + /* + * Re-add on wake, after sched_delayed is cleared. On a final delayed + * dequeue account_llc_dequeue() already cleared pref_llc_queued, so + * this does nothing. + */ + pref_llc_running_inc(rq_of(cfs_rq_of(se)), task_of(se)); + for_each_sched_entity(se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); @@ -8022,10 +8193,12 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (p->in_iowait) cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); + curr = (cfs_rq->curr == se); /* - * XXX comment on the curr thing + * This can actually happen with CFS_BANDWIDTH, a task can be throttled + * and dequeued, unthrottled from another CPU and then enqueued again + * before schedule(). */ - curr = (cfs_rq->curr == se); if (curr) place_entity(cfs_rq, se, flags); @@ -8458,8 +8631,8 @@ static int sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) { unsigned long load, min_load = ULONG_MAX; - unsigned int min_exit_latency = UINT_MAX; - u64 latest_idle_timestamp = 0; + u64 min_exit_latency = U64_MAX; + unsigned int nr_candidates = 0; int least_loaded_cpu = this_cpu; int shallowest_idle_cpu = -1; int i; @@ -8480,24 +8653,16 @@ sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct * if (available_idle_cpu(i)) { struct cpuidle_state *idle = idle_get_state(rq); - if (idle && idle->exit_latency < min_exit_latency) { - /* - * We give priority to a CPU whose idle state - * has the smallest exit latency irrespective - * of any idle timestamp. - */ - min_exit_latency = idle->exit_latency; - latest_idle_timestamp = rq->idle_stamp; - shallowest_idle_cpu = i; - } else if ((!idle || idle->exit_latency == min_exit_latency) && - rq->idle_stamp > latest_idle_timestamp) { - /* - * If equal or no active idle state, then - * the most recently idled CPU might have - * a warmer cache. - */ - latest_idle_timestamp = rq->idle_stamp; + u64 exit_latency = idle ? idle->exit_latency : U64_MAX; + + if (shallowest_idle_cpu == -1 || exit_latency < min_exit_latency) { + min_exit_latency = exit_latency; shallowest_idle_cpu = i; + nr_candidates = 1; + } else if (exit_latency == min_exit_latency) { + nr_candidates++; + if (!reciprocal_scale(sched_rng(), nr_candidates)) + shallowest_idle_cpu = i; } } else if (shallowest_idle_cpu == -1) { load = cpu_load(cpu_rq(i)); @@ -8596,6 +8761,35 @@ static inline bool test_idle_cores(int cpu) } /* + * Redirect a CPU to a higher-priority available sibling in its SMT domain, + * subject to task affinity. + */ +static inline int select_idle_smt_cpu(struct task_struct *p, int cpu) +{ + struct sched_domain *sd; + int best = cpu; + int sibling; + + if (!sched_smt_active()) + return cpu; + + sd = rcu_dereference_all(cpu_rq(cpu)->sd); + if (!sd || !(sd->flags & SD_SHARE_CPUCAPACITY) || + !(sd->flags & SD_ASYM_PACKING)) + return cpu; + + for_each_cpu_and(sibling, sched_domain_span(sd), p->cpus_ptr) { + if (sibling == best || !choose_idle_cpu(sibling, p)) + continue; + + if (sched_asym_prefer(sibling, best)) + best = sibling; + } + + return best; +} + +/* * Scans the local SMT mask to see if the entire core is idle, and records this * information in sd_balance_shared->has_idle_cores. * @@ -8979,7 +9173,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) if (choose_idle_cpu(target, p) && asym_fits_cpu(task_util, util_min, util_max, target)) - return target; + goto select_smt_priority; /* * If the previous CPU is cache affine and idle, don't be stupid: @@ -8989,8 +9183,10 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) asym_fits_cpu(task_util, util_min, util_max, prev)) { if (!static_branch_unlikely(&sched_cluster_active) || - cpus_share_resources(prev, target)) - return prev; + cpus_share_resources(prev, target)) { + target = prev; + goto select_smt_priority; + } prev_aff = prev; } @@ -9008,7 +9204,8 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) prev == smp_processor_id() && this_rq()->nr_running <= 1 && asym_fits_cpu(task_util, util_min, util_max, prev)) { - return prev; + target = prev; + goto select_smt_priority; } /* Check a recently used CPU as a potential idle candidate: */ @@ -9022,8 +9219,10 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) asym_fits_cpu(task_util, util_min, util_max, recent_used_cpu)) { if (!static_branch_unlikely(&sched_cluster_active) || - cpus_share_resources(recent_used_cpu, target)) - return recent_used_cpu; + cpus_share_resources(recent_used_cpu, target)) { + target = recent_used_cpu; + goto select_smt_priority; + } } else { recent_used_cpu = -1; @@ -9045,7 +9244,11 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) */ if (sd) { i = select_idle_capacity(p, sd, target); - return ((unsigned)i < nr_cpumask_bits) ? i : target; + if ((unsigned int)i < nr_cpumask_bits) { + target = i; + goto select_smt_priority; + } + return target; } } @@ -9058,14 +9261,18 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) if (!has_idle_core && cpus_share_cache(prev, target)) { i = select_idle_smt(p, sd, prev); - if ((unsigned int)i < nr_cpumask_bits) - return i; + if ((unsigned int)i < nr_cpumask_bits) { + target = i; + goto select_smt_priority; + } } } i = select_idle_cpu(p, sd, has_idle_core, target); - if ((unsigned)i < nr_cpumask_bits) - return i; + if ((unsigned int)i < nr_cpumask_bits) { + target = i; + goto select_smt_priority; + } /* * For cluster machines which have lower sharing cache like L2 or @@ -9073,12 +9280,19 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) * first. But prev_cpu or recent_used_cpu may also be a good candidate, * use them if possible when no idle CPU found in select_idle_cpu(). */ - if ((unsigned int)prev_aff < nr_cpumask_bits) - return prev_aff; - if ((unsigned int)recent_used_cpu < nr_cpumask_bits) - return recent_used_cpu; + if ((unsigned int)prev_aff < nr_cpumask_bits) { + target = prev_aff; + goto select_smt_priority; + } + if ((unsigned int)recent_used_cpu < nr_cpumask_bits) { + target = recent_used_cpu; + goto select_smt_priority; + } return target; + +select_smt_priority: + return select_idle_smt_cpu(p, target); } /** @@ -9755,8 +9969,10 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) } /* Slow path */ - if (unlikely(sd)) - return sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag); + if (unlikely(sd)) { + new_cpu = sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag); + return select_idle_smt_cpu(p, new_cpu); + } /* Fast path */ if (wake_flags & WF_TTWU) @@ -9856,8 +10072,14 @@ static inline bool set_preempt_buddy(struct cfs_rq *cfs_rq, struct sched_entity static inline bool set_short_buddy(struct cfs_rq *cfs_rq, struct sched_entity *pse) { - if (cfs_rq->next && cfs_rq->next->slice < pse->slice) - return false; + if (cfs_rq->next) { + if (cfs_rq->next->slice < pse->slice) + return false; + + if (cfs_rq->next->slice == pse->slice && + entity_before(cfs_rq->next, pse)) + return false; + } set_next_buddy(cfs_rq, pse); return true; @@ -10395,6 +10617,7 @@ enum migration_type { #define LBF_SOME_PINNED 0x08 #define LBF_ACTIVE_LB 0x10 #define LBF_LLC_PINNED 0x20 +#define LBF_ACTIVE_LB_LLC 0x40 struct lb_env { struct sched_domain *sd; @@ -10724,7 +10947,7 @@ static inline bool task_misfits_asym_cpu(struct lb_env *env, struct task_struct static enum llc_mig can_migrate_llc_task(struct lb_env *env, struct task_struct *p) { - struct mm_struct *mm; + struct sched_cache_group *grp; bool to_pref; int cpu, src_cpu, dst_cpu; @@ -10733,19 +10956,19 @@ static enum llc_mig can_migrate_llc_task(struct lb_env *env, src_cpu = env->src_cpu; dst_cpu = env->dst_cpu; - mm = p->mm; - if (!mm) + grp = rcu_dereference_all(p->sched_cache_grp); + if (!grp) return mig_unrestricted; - cpu = READ_ONCE(mm->sc_stat.cpu); + cpu = READ_ONCE(grp->cpu); if (cpu < 0 || cpus_share_cache(src_cpu, dst_cpu)) return mig_unrestricted; /* skip cache aware load balance for too many threads */ - if (invalid_llc_nr(mm, p, dst_cpu) || - exceed_llc_capacity(mm, dst_cpu)) { - if (READ_ONCE(mm->sc_stat.cpu) != -1) - WRITE_ONCE(mm->sc_stat.cpu, -1); + if (invalid_llc_nr(grp, p, dst_cpu) || + exceed_llc_capacity(grp, dst_cpu)) { + if (READ_ONCE(grp->cpu) != -1) + WRITE_ONCE(grp->cpu, -1); return mig_unrestricted; } @@ -10814,6 +11037,21 @@ alb_break_llc(struct lb_env *env) } /* + * Returns true if p's preferred LLC does not match the destination CPU + * under migrate_llc_task semantics. Passive LB passes migrate_llc_task + * in env->migration_type, while active LB carries LBF_ACTIVE_LB_LLC in + * env->flags to avoid overwriting env->migration_type. + */ +static inline bool +migrate_llc_task_wrong_dst(struct task_struct *p, struct lb_env *env) +{ + return sched_cache_enabled() && + (env->migration_type == migrate_llc_task || + env->flags & LBF_ACTIVE_LB_LLC) && + READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu); +} + +/* * Check if migrating task p from env->src_cpu to * env->dst_cpu breaks LLC localiy. */ @@ -10841,8 +11079,7 @@ static bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env) * run on env->dst_cpu, skip the tasks do not prefer * env->dst_cpu, and find the one that prefers. */ - if (env->migration_type == migrate_llc_task && - READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu)) + if (migrate_llc_task_wrong_dst(p, env)) return true; if (can_migrate_llc_task(env, p) != mig_forbid) @@ -10865,6 +11102,12 @@ alb_break_llc(struct lb_env *env) } static inline bool +migrate_llc_task_wrong_dst(struct task_struct *p, struct lb_env *env) +{ + return false; +} + +static inline bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env) { return false; @@ -10963,7 +11206,7 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) * 4) too many balance attempts have failed. */ if (env->flags & LBF_ACTIVE_LB) - return 1; + return !migrate_llc_task_wrong_dst(p, env); degrades = migrate_degrades_locality(p, env); if (!degrades) { @@ -11202,21 +11445,7 @@ next: */ static void attach_tasks(struct lb_env *env) { - struct list_head *tasks = &env->tasks; - struct task_struct *p; - struct rq_flags rf; - - rq_lock(env->dst_rq, &rf); - update_rq_clock(env->dst_rq); - - while (!list_empty(tasks)) { - p = list_first_entry(tasks, struct task_struct, se.group_node); - list_del_init(&p->se.group_node); - - attach_task(env->dst_rq, p); - } - - rq_unlock(env->dst_rq, &rf); + __attach_tasks(env->dst_rq, &env->tasks); } #ifdef CONFIG_NO_HZ_COMMON @@ -11875,6 +12104,14 @@ static void record_sg_llc_stats(struct lb_env *env, return; /* + * A redo repeats stats published on the first pass. Hotplug can + * still leave env->cpus partial, so util would not match capacity. + */ + if ((env->flags & LBF_ALL_PINNED) || + !cpumask_subset(sched_group_span(group), env->cpus)) + return; + + /* * At this point we know this group spans a LLC domain. * Record the statistic of this group in its corresponding * shared LLC domain. @@ -13362,6 +13599,20 @@ static int need_active_balance(struct lb_env *env) } static int active_load_balance_cpu_stop(void *data); +static int active_load_balance_llc_cpu_stop(void *data); + +/* + * migration_type is checked elsewhere to decide migration policy, so + * it shouldn't be repurposed just to flag an LLC-directed active + * balance across the stopper. Pick the callback here instead. + */ +static inline cpu_stop_fn_t alb_stop_fn(struct lb_env *env) +{ + if (env->migration_type == migrate_llc_task) + return active_load_balance_llc_cpu_stop; + + return active_load_balance_cpu_stop; +} static int should_we_balance(struct lb_env *env) { @@ -13495,7 +13746,7 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq, }; bool need_unlock = false; - cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask); + cpumask_and(cpus, sched_domain_span(sd), cpu_preferred_mask); schedstat_inc(sd->lb_count[idle]); @@ -13707,7 +13958,7 @@ more_balance: } if (active_balance) { stop_one_cpu_nowait(cpu_of(busiest), - active_load_balance_cpu_stop, busiest, + alb_stop_fn(&env), busiest, &busiest->active_balance_work); } preempt_enable(); @@ -13812,7 +14063,7 @@ update_next_balance(struct sched_domain *sd, unsigned long *next_balance) * least 1 task to be running on each physical CPU where possible, and * avoids physical / logical imbalances. */ -static int active_load_balance_cpu_stop(void *data) +static int __active_load_balance_cpu_stop(void *data, unsigned int lb_flags) { struct rq *busiest_rq = data; int busiest_cpu = cpu_of(busiest_rq); @@ -13862,7 +14113,7 @@ static int active_load_balance_cpu_stop(void *data) .src_cpu = busiest_rq->cpu, .src_rq = busiest_rq, .idle = CPU_IDLE, - .flags = LBF_ACTIVE_LB, + .flags = LBF_ACTIVE_LB | lb_flags, }; schedstat_inc(sd->alb_count); @@ -13890,6 +14141,16 @@ out_unlock: return 0; } +static int active_load_balance_cpu_stop(void *data) +{ + return __active_load_balance_cpu_stop(data, 0); +} + +static int active_load_balance_llc_cpu_stop(void *data) +{ + return __active_load_balance_cpu_stop(data, LBF_ACTIVE_LB_LLC); +} + /* * Scale the max sched_balance_rq interval with the number of CPUs in the system. * This trades load-balance latency on larger machines for less cross talk. @@ -14610,10 +14871,8 @@ static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) */ this_rq->idle_stamp = rq_clock(this_rq); - /* - * Do not pull tasks towards !active CPUs... - */ - if (!cpu_active(this_cpu)) + /* Do not pull tasks towards !preferred CPUs */ + if (!cpu_preferred(this_cpu)) return 0; /* @@ -15253,14 +15512,18 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p) } } -static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) +static void set_next_task_fair(struct rq *rq, struct task_struct *p, enum snt_e type) { struct sched_entity *se = &p->se; - bool throttled = false; struct cfs_rq *cfs_rq = &rq->cfs; unsigned long weight = NICE_0_LOAD; + bool first = type == SNT_PICK; + bool throttled = false; bool on_rq = se->on_rq; + if (type == SNT_REPICK) + goto repick; + clear_buddies(cfs_rq, se); if (on_rq) @@ -15304,11 +15567,18 @@ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) WARN_ON_ONCE(se->sched_delayed); - if (hrtick_enabled_fair(rq)) - hrtick_start_fair(rq, p); - update_misfit_status(p, rq); sched_fair_update_stop_tick(rq, p); + +repick: + /* + * A same-task repick skips put_prev_task_fair(), but + * pick_task_fair() refreshed the entity hrtick_start_fair() reads + * before selecting it again. rq->cfs.curr identifies that entity, + * including with group scheduling. + */ + if (hrtick_enabled_fair(rq)) + hrtick_start_fair(rq, p); } void init_cfs_rq(struct cfs_rq *cfs_rq) diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index eb73b65ce6c4..76f3c84ca684 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -487,8 +487,11 @@ static void put_prev_task_idle(struct rq *rq, struct task_struct *prev, struct t update_rq_avg_idle(rq); } -static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) +static void set_next_task_idle(struct rq *rq, struct task_struct *next, enum snt_e type) { + if (type == SNT_REPICK) + return; + update_idle_core(rq); scx_update_idle(rq, true, true); schedstat_inc(rq->sched_goidle); diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index 4e152410653d..7c5381423334 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -626,8 +626,6 @@ static void init_rtpoll_triggers(struct psi_group *group, u64 now) static void psi_schedule_rtpoll_work(struct psi_group *group, unsigned long delay, bool force) { - struct task_struct *task; - /* * atomic_xchg should be called even when !force to provide a * full memory barrier (see the comment inside psi_rtpoll_work). @@ -635,19 +633,16 @@ static void psi_schedule_rtpoll_work(struct psi_group *group, unsigned long dela if (atomic_xchg(&group->rtpoll_scheduled, 1) && !force) return; - rcu_read_lock(); - - task = rcu_dereference(group->rtpoll_task); /* - * kworker might be NULL in case psi_trigger_destroy races with - * psi_task_change (hotpath) which can't use locks + * Only test whether a worker is installed; do not dereference the task. + * A racing trigger destruction may leave the timer armed. + * psi_cgroup_free() shuts it down before freeing the group. + * The system PSI group is static. */ - if (likely(task)) + if (likely(rcu_access_pointer(group->rtpoll_task))) mod_timer(&group->rtpoll_timer, jiffies + delay); else atomic_set(&group->rtpoll_scheduled, 0); - - rcu_read_unlock(); } static void psi_rtpoll_work(struct psi_group *group) @@ -1489,21 +1484,11 @@ void psi_trigger_destroy(struct psi_trigger *t) } /* - * Wait for psi_schedule_rtpoll_work RCU to complete its read-side - * critical section before destroying the trigger and optionally the - * rtpoll_task. - */ - synchronize_rcu(); - /* * Stop kthread 'psimon' after releasing rtpoll_trigger_lock to prevent * a deadlock while waiting for psi_rtpoll_work to acquire * rtpoll_trigger_lock */ if (task_to_destroy) { - /* - * After the RCU grace period has expired, the worker - * can no longer be found through group->rtpoll_task. - */ kthread_stop(task_to_destroy); atomic_set(&group->rtpoll_scheduled, 0); } diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 85303add726d..1535046a23ff 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1654,11 +1654,14 @@ static void wakeup_preempt_rt(struct rq *rq, struct task_struct *p, int flags) check_preempt_equal_prio(rq, p); } -static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool first) +static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, enum snt_e type) { struct sched_rt_entity *rt_se = &p->rt; struct rt_rq *rt_rq = &rq->rt; + if (type == SNT_REPICK) + return; + p->se.exec_start = rq_clock_task(rq); if (on_rt_rq(&p->rt)) update_stats_wait_end_rt(rt_rq, rt_se); @@ -1666,7 +1669,7 @@ static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool f /* The running task is never eligible for pushing */ dequeue_pushable_task(rq, p); - if (!first) + if (type != SNT_PICK) return; /* diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index e656c7059bf8..87c5335add37 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -791,6 +791,10 @@ enum scx_rq_flags { SCX_RQ_BAL_CB_PENDING = 1 << 6, /* must queue a cb after dispatching */ SCX_RQ_SUB_IDLE_RENOTIFY = 1 << 7, /* sub-scheds are owed update_idle() */ SCX_RQ_ROOT_IDLE_RENOTIFY = 1 << 8, /* the root is owed update_idle() */ + SCX_RQ_PROXY_RETRY = 1 << 9, /* proxy-rejected tasks need retry */ + SCX_RQ_PROXY_TICK = 1 << 10, /* proxy execution requires the tick */ + SCX_RQ_PROXY_PENDING = 1 << 11, /* blocked donor awaits proxy resolution */ + SCX_RQ_PROXY_BLOCKING = 1 << 12, /* donor is being removed for sched change */ SCX_RQ_IN_WAKEUP = 1 << 16, SCX_RQ_IN_DISPATCH = 1 << 17, @@ -810,13 +814,12 @@ struct scx_rq_rescue { struct scx_rq { struct scx_dispatch_q local_dsq; + struct scx_dispatch_q reject_dsq; /* staging for rejected tasks */ #ifdef CONFIG_EXT_SUB_SCHED - struct scx_dispatch_q reject_dsq; /* staging for cap-rejected tasks */ struct scx_rq_rescue rescue; #endif struct list_head runnable_list; /* runnable tasks on this rq */ struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */ - unsigned long ops_qseq; /* both stashed across the activate_task() in move_remote_task_to_local_dsq() */ u64 remote_activate_enq_flags; struct scx_sched *remote_activate_sch; @@ -831,6 +834,8 @@ struct scx_rq { #endif u64 clock; /* current per-rq clock -- see scx_bpf_now() */ #ifdef CONFIG_EXT_SUB_SCHED + /* sched of the open running session, see scx_cid_sched_update() */ + struct scx_sched *sched; struct llist_head ecaps_to_sync; /* pending ecaps syncs */ struct task_struct *sub_dispatch_prev; #endif @@ -845,6 +850,9 @@ struct scx_rq { struct list_head deferred_reenq_users; /* user DSQs requesting reenq */ struct balance_callback deferred_bal_cb; struct balance_callback kick_sync_bal_cb; +#ifdef CONFIG_NO_HZ_FULL + struct balance_callback proxy_tick_bal_cb; +#endif struct irq_work deferred_irq_work; struct irq_work kick_cpus_irq_work; }; @@ -1326,6 +1334,9 @@ struct rq { #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING u64 prev_steal_time_rq; #endif +#ifdef CONFIG_PREFERRED_CPU + bool npc_push_work_pending; +#endif /* calc_load related fields */ unsigned long calc_load_update; @@ -1371,7 +1382,6 @@ struct rq { struct task_struct *core_pick; struct sched_dl_entity *core_dl_server; unsigned int core_enabled; - unsigned int core_sched_seq; struct rb_root core_tree; /* shared state -- careful with sched_core_cpu_deactivate() */ @@ -2447,16 +2457,25 @@ extern __read_mostly unsigned int sysctl_sched_features; #ifdef CONFIG_JUMP_LABEL -#define SCHED_FEAT(name, enabled) \ -static __always_inline bool static_branch_##name(struct static_key *key) \ -{ \ - return static_key_##enabled(key); \ +union sched_feat_key { + struct static_key_true key_true; + struct static_key_false key_false; +}; + +#define sched_feat_branch_true(key) static_branch_likely(&(key)->key_true) +#define sched_feat_branch_false(key) static_branch_unlikely(&(key)->key_false) + +#define SCHED_FEAT(name, enabled) \ +static __always_inline bool \ +static_branch_##name(union sched_feat_key *key) \ +{ \ + return sched_feat_branch_##enabled(key); \ } #include "features.h" #undef SCHED_FEAT -extern struct static_key sched_feat_keys[__SCHED_FEAT_NR]; +extern union sched_feat_key sched_feat_keys[__SCHED_FEAT_NR]; #define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x])) #else /* !CONFIG_JUMP_LABEL: */ @@ -2508,6 +2527,12 @@ static inline bool task_is_blocked(struct task_struct *p) return !!p->blocked_on; } +#ifdef CONFIG_SCHED_PROXY_EXEC +void sched_proxy_block_task(struct rq *rq, struct task_struct *p); +#else +static inline void sched_proxy_block_task(struct rq *rq, struct task_struct *p) {} +#endif + static inline int task_on_cpu(struct rq *rq, struct task_struct *p) { return p->on_cpu; @@ -2527,11 +2552,17 @@ static inline int task_on_rq_migrating(struct task_struct *p) #define WF_EXEC 0x02 /* Wakeup after exec; maps to SD_BALANCE_EXEC */ #define WF_FORK 0x04 /* Wakeup after fork; maps to SD_BALANCE_FORK */ #define WF_TTWU 0x08 /* Wakeup; maps to SD_BALANCE_WAKE */ - -#define WF_SYNC 0x10 /* Waker goes to sleep after wakeup */ +/* + * Hint that the caller expects the waker to sleep soon. + * Scheduler classes may use it for placement or preemption. + * Callers must not rely on it to prevent migration, + * preserve CPU locality or make the wakee run next. + */ +#define WF_SYNC 0x10 #define WF_MIGRATED 0x20 /* Internal use, task got migrated */ #define WF_CURRENT_CPU 0x40 /* Prefer to move the wakee to the current CPU. */ #define WF_RQ_SELECTED 0x80 /* ->select_task_rq() was called */ +#define WF_TTWU_RQ 0x100 /* Wakeup completed through ttwu_runnable() */ static_assert(WF_EXEC == SD_BALANCE_EXEC); static_assert(WF_FORK == SD_BALANCE_FORK); @@ -2621,6 +2652,12 @@ struct affinity_context { extern s64 update_curr_common(struct rq *rq); +enum snt_e { + SNT_NORMAL, /* set_next_task() */ + SNT_PICK, /* put_prev_set_next_task(): prev != next */ + SNT_REPICK, /* put_prev_set_next_task(): prev == next */ +}; + struct sched_class { #ifdef CONFIG_UCLAMP_TASK @@ -2678,7 +2715,7 @@ struct sched_class { * __schedule: rq->lock */ void (*put_prev_task)(struct rq *rq, struct task_struct *p, struct task_struct *next); - void (*set_next_task)(struct rq *rq, struct task_struct *p, bool first); + void (*set_next_task)(struct rq *rq, struct task_struct *p, enum snt_e type); /* * select_task_rq: p->pi_lock @@ -2781,7 +2818,7 @@ static inline void put_prev_task(struct rq *rq, struct task_struct *prev) static inline void set_next_task(struct rq *rq, struct task_struct *next) { - next->sched_class->set_next_task(rq, next, false); + next->sched_class->set_next_task(rq, next, SNT_NORMAL); } static inline void @@ -2802,11 +2839,13 @@ static inline void put_prev_set_next_task(struct rq *rq, __put_prev_set_next_dl_server(rq, prev, next); - if (next == prev) + if (next == prev) { + next->sched_class->set_next_task(rq, next, SNT_REPICK); return; + } prev->sched_class->put_prev_task(rq, prev, next); - next->sched_class->set_next_task(rq, next, true); + next->sched_class->set_next_task(rq, next, SNT_PICK); } /* @@ -3139,6 +3178,25 @@ static inline void attach_one_task(struct rq *rq, struct task_struct *p) attach_task(rq, p); } +/* + * __attach_tasks() - attaches a list of tasks (using se.group_node) to + * the new rq + */ +static inline void __attach_tasks(struct rq *rq, struct list_head *tasks) +{ + guard(rq_lock)(rq); + update_rq_clock(rq); + + while (!list_empty(tasks)) { + struct task_struct *p; + + p = list_first_entry(tasks, struct task_struct, se.group_node); + list_del_init(&p->se.group_node); + + attach_task(rq, p); + } +} + #ifdef CONFIG_PREEMPT_RT # define SCHED_NR_MIGRATE_BREAK 8 #else @@ -4222,8 +4280,9 @@ extern void balance_callbacks(struct rq *rq, struct balance_callback *head); * after which it is enqueued again. * * Typically this must be called while holding task_rq_lock, since most/all - * properties are serialized under those locks. There is currently one - * exception to this rule in sched/ext which only holds rq->lock. + * properties are serialized under those locks. There are currently two + * exceptions to this rule in sched/ext which only hold rq->lock: scx_bypass() + * and rq_offline_scx(). */ /* @@ -4252,4 +4311,10 @@ DEFINE_CLASS_IS_UNCONDITIONAL(sched_change) #include "ext/ext.h" +#ifdef CONFIG_PREFERRED_CPU +void sched_push_current_non_preferred_cpu(struct rq *rq); +#else /* !CONFIG_PREFERRED_CPU */ +static inline void sched_push_current_non_preferred_cpu(struct rq *rq) { } +#endif + #endif /* _KERNEL_SCHED_SCHED_H */ diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c index c909ca0d8c87..1e0109ec36b3 100644 --- a/kernel/sched/stop_task.c +++ b/kernel/sched/stop_task.c @@ -27,8 +27,11 @@ wakeup_preempt_stop(struct rq *rq, struct task_struct *p, int flags) /* we're never preempted */ } -static void set_next_task_stop(struct rq *rq, struct task_struct *stop, bool first) +static void set_next_task_stop(struct rq *rq, struct task_struct *stop, enum snt_e type) { + if (type == SNT_REPICK) + return; + stop->se.exec_start = rq_clock_task(rq); } diff --git a/kernel/sched/syscalls.c b/kernel/sched/syscalls.c index b215b0ead9a6..45a127deeadd 100644 --- a/kernel/sched/syscalls.c +++ b/kernel/sched/syscalls.c @@ -678,6 +678,8 @@ change: if (prev_class != next_class) queue_flags |= DEQUEUE_CLASS; + scx_prepare_setscheduler(p, policy); + scoped_guard (sched_change, p, queue_flags) { if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 0248227d983a..919d0fb00bd9 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -32,6 +32,20 @@ static int __init sched_debug_setup(char *str) } early_param("sched_verbose", sched_debug_setup); +#ifdef CONFIG_SCHED_SMT +static bool sched_smt_asym_packing __read_mostly; + +static int __init setup_sched_smt_asym_packing(char *str) +{ + if (strcmp(str, "on")) + return 0; + + sched_smt_asym_packing = true; + return 1; +} +__setup("sched_smt_asym_packing=", setup_sched_smt_asym_packing); +#endif + static inline bool sched_debug(void) { return sched_debug_verbose; @@ -985,8 +999,8 @@ void sched_cache_active_set(void) } /* - * Update the bottom sched_domain's llc_bytes for @cpu and all its - * LLC siblings. Called from cacheinfo_cpu_online() or + * Update the bottom sched_domain's llc_bytes for @cpus sharing a physical + * LLC. Called from cacheinfo_cpu_online() or * cacheinfo_cpu_pre_down() with cpu hotplug lock held. * * Note: get_effective_llc_bytes() returns 0 on PowerPC. @@ -996,17 +1010,13 @@ void sched_cache_active_set(void) * and does not populates the per-CPU struct cpu_cacheinfo array * that get_cpu_cacheinfo_llc() reads. */ -void sched_update_llc_bytes(unsigned int cpu) +void sched_update_llc_bytes(const struct cpumask *cpus) { struct sched_domain *sd, *sdp; unsigned int i; sched_domains_mutex_lock(); - sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, cpu)); - if (!sdp) - goto unlock; - /* * ci->shared_cpu_map is built incrementally as CPUs come * online, so the first CPU in an LLC initially sees @@ -1014,14 +1024,22 @@ void sched_update_llc_bytes(unsigned int cpu) * get_effective_llc_bytes(). Re-evaluating every LLC * sibling on each online event corrects this once the full * shared_cpu_map is known. + * + * The departing CPU's domains have already been detached when + * cacheinfo removes it. Use the surviving cache siblings instead. + * They may belong to different cpuset partitions, so use each CPU's + * own LLC domain to scale its share of the physical cache. */ - for_each_cpu(i, sched_domain_span(sdp)) { + for_each_cpu(i, cpus) { + sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, i)); + if (!sdp) + continue; + sd = rcu_dereference_sched_domain(cpu_rq(i)->sd); if (sd) sd->llc_bytes = get_effective_llc_bytes(i, sdp); } -unlock: sched_domains_mutex_unlock(); } @@ -1950,6 +1968,10 @@ sd_init(struct sched_domain_topology_level *tl, if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS, "wrong sd_flags in topology description\n")) sd_flags &= TOPOLOGY_SD_FLAGS; +#ifdef CONFIG_SCHED_SMT + if (sched_smt_asym_packing && (sd_flags & SD_SHARE_CPUCAPACITY)) + sd_flags |= SD_ASYM_PACKING; +#endif sd_flags |= asym_cpu_capacity_classify(sd_span, cpu_map); *sd = (struct sched_domain){ diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c index d033f600f48c..f8d6d8b9d03e 100644 --- a/kernel/sched/wait.c +++ b/kernel/sched/wait.c @@ -54,14 +54,13 @@ int add_wait_queue_priority_exclusive(struct wait_queue_head *wq_head, { struct list_head *head = &wq_head->head; - wq_entry->flags |= WQ_FLAG_EXCLUSIVE | WQ_FLAG_PRIORITY; - guard(spinlock_irqsave)(&wq_head->lock); if (!list_empty(head) && (list_first_entry(head, typeof(*wq_entry), entry)->flags & WQ_FLAG_PRIORITY)) return -EBUSY; + wq_entry->flags |= WQ_FLAG_EXCLUSIVE | WQ_FLAG_PRIORITY; list_add(&wq_entry->entry, head); return 0; } @@ -174,15 +173,11 @@ EXPORT_SYMBOL_GPL(__wake_up_locked_key); * @mode: which threads * @key: opaque value to be passed to wakeup targets * - * The sync wakeup differs that the waker knows that it will schedule - * away soon, so while the target thread will be woken up, it will not - * be migrated to another CPU - ie. the two threads are 'synchronized' - * with each other. This can prevent needless bouncing between CPUs. - * - * On UP it can prevent extra preemption. + * Passes WF_SYNC to waitqueue wake functions. The default wake function + * forwards it to the scheduler; see WF_SYNC for the hint's semantics. * - * If this function wakes up a task, it executes a full memory barrier before - * accessing the task state. + * If this function wakes up a task, it executes a full memory barrier + * before accessing the task state. */ void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key) @@ -200,15 +195,7 @@ EXPORT_SYMBOL_GPL(__wake_up_sync_key); * @mode: which threads * @key: opaque value to be passed to wakeup targets * - * The sync wakeup differs in that the waker knows that it will schedule - * away soon, so while the target thread will be woken up, it will not - * be migrated to another CPU - ie. the two threads are 'synchronized' - * with each other. This can prevent needless bouncing between CPUs. - * - * On UP it can prevent extra preemption. - * - * If this function wakes up a task, it executes a full memory barrier before - * accessing the task state. + * Same as __wake_up_sync_key(), but called with @wq_head->lock held. */ void __wake_up_locked_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key) diff --git a/kernel/seccomp.c b/kernel/seccomp.c index 86cf4460d69e..3eecba5ccdef 100644 --- a/kernel/seccomp.c +++ b/kernel/seccomp.c @@ -205,6 +205,7 @@ static inline void seccomp_cache_prepare(struct seccomp_filter *sfilter) * @log: true if all actions except for SECCOMP_RET_ALLOW should be logged * @wait_killable_recv: Put notifying process in killable state once the * notification is received by the userspace listener. + * @restart_before_recv: Restart interrupted syscalls before notification receipt. * @prev: points to a previously installed, or inherited, filter * @prog: the BPF program to evaluate * @notif: the struct that holds all notification related information @@ -226,6 +227,7 @@ struct seccomp_filter { refcount_t users; bool log; bool wait_killable_recv; + bool restart_before_recv; struct action_cache cache; struct seccomp_filter *prev; struct bpf_prog *prog; @@ -953,6 +955,8 @@ static long seccomp_attach_filter(unsigned int flags, /* Set wait killable flag, if present. */ if (flags & SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV) filter->wait_killable_recv = true; + if (flags & SECCOMP_FILTER_FLAG_RESTART_BEFORE_RECV) + filter->restart_before_recv = true; /* * If there is an existing filter, make it the prev and don't drop its @@ -1208,8 +1212,12 @@ static int seccomp_do_user_notification(int this_syscall, * Check to see whether we should switch to wait * killable. Only return the interrupted error if not. */ - if (!(!wait_killable && should_sleep_killable(match, &n))) + if (!(!wait_killable && should_sleep_killable(match, &n))) { + if (err == -ERESTARTSYS && match->restart_before_recv && + n.state == SECCOMP_NOTIFY_INIT) + err = -ERESTARTNOINTR; goto interrupted; + } } addfd = list_first_entry_or_null(&n.addfd, @@ -1735,7 +1743,8 @@ static long seccomp_notify_addfd(struct seccomp_filter *filter, if (addfd.newfd && !(addfd.flags & SECCOMP_ADDFD_FLAG_SETFD)) return -EINVAL; - kaddfd.file = fget(addfd.srcfd); + /* Allow O_PATH files, as SCM_RIGHTS and pidfd_getfd() do. */ + kaddfd.file = fget_raw(addfd.srcfd); if (!kaddfd.file) return -EBADF; @@ -1808,10 +1817,13 @@ static long seccomp_notify_addfd(struct seccomp_filter *filter, * We need to check again if the addfd request has been handled, * and if not, we will remove it from the queue. */ - if (list_empty(&kaddfd.list)) + if (list_empty(&kaddfd.list)) { ret = kaddfd.ret; - else + } else { list_del(&kaddfd.list); + if (addfd.flags & SECCOMP_ADDFD_FLAG_SEND) + knotif->state = SECCOMP_NOTIFY_SENT; + } out_unlock: mutex_unlock(&filter->notify_lock); @@ -1977,10 +1989,10 @@ static long seccomp_set_mode_filter(unsigned int flags, return -EINVAL; /* - * The SECCOMP_FILTER_FLAG_WAIT_KILLABLE_SENT flag doesn't make sense - * without the SECCOMP_FILTER_FLAG_NEW_LISTENER flag. + * Notification wait flags require a userspace listener. */ - if ((flags & SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV) && + if ((flags & (SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV | + SECCOMP_FILTER_FLAG_RESTART_BEFORE_RECV)) && ((flags & SECCOMP_FILTER_FLAG_NEW_LISTENER) == 0)) return -EINVAL; diff --git a/kernel/signal.c b/kernel/signal.c index d31ebcb6ed4d..cb6ec7054eee 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -3170,6 +3170,10 @@ static void retarget_shared_pending(struct task_struct *tsk, sigset_t *which) sigset_t retarget; struct task_struct *t; + /* Nobody dequeues them in a dying group, see get_signal(). */ + if (tsk->signal->flags & SIGNAL_GROUP_EXIT) + return; + sigandsets(&retarget, &tsk->signal->shared_pending.signal, which); if (sigisemptyset(&retarget)) return; @@ -3255,6 +3259,16 @@ long do_no_restart_syscall(struct restart_block *param) static void __set_task_blocked(struct task_struct *tsk, const sigset_t *newset) { + sigset_t floor, floored; + + /* A user worker never unblocks anything but SIGKILL and SIGSTOP. */ + if (unlikely(tsk->flags & PF_USER_WORKER)) { + siginitsetinv(&floor, SIG_KERNEL_ONLY_MASK); + sigorsets(&floored, newset, &floor); + WARN_ON_ONCE(!sigequalsets(&floored, newset)); + newset = &floored; + } + if (task_sigpending(tsk) && !thread_group_empty(tsk)) { sigset_t newblocked; /* A set of now blocked but previously unblocked signals. */ @@ -4539,11 +4553,12 @@ int restore_altstack(const stack_t __user *uss) int __save_altstack(stack_t __user *uss, unsigned long sp) { - struct task_struct *t = current; - int err = __put_user((void __user *)t->sas_ss_sp, &uss->ss_sp) | - __put_user(t->sas_ss_flags, &uss->ss_flags) | - __put_user(t->sas_ss_size, &uss->ss_size); - return err; + scoped_user_write_access(uss, Efault) + unsafe_save_altstack(uss, sp, Efault); + return 0; + +Efault: + return -EFAULT; } #ifdef CONFIG_COMPAT @@ -4592,13 +4607,12 @@ int compat_restore_altstack(const compat_stack_t __user *uss) int __compat_save_altstack(compat_stack_t __user *uss, unsigned long sp) { - int err; - struct task_struct *t = current; - err = __put_user(ptr_to_compat((void __user *)t->sas_ss_sp), - &uss->ss_sp) | - __put_user(t->sas_ss_flags, &uss->ss_flags) | - __put_user(t->sas_ss_size, &uss->ss_size); - return err; + scoped_user_write_access(uss, Efault) + unsafe_compat_save_altstack(uss, sp, Efault); + return 0; + +Efault: + return -EFAULT; } #endif @@ -4776,12 +4790,12 @@ SYSCALL_DEFINE3(sigaction, int, sig, if (act) { old_sigset_t mask; - if (!access_ok(act, sizeof(*act)) || - __get_user(new_ka.sa.sa_handler, &act->sa_handler) || - __get_user(new_ka.sa.sa_restorer, &act->sa_restorer) || - __get_user(new_ka.sa.sa_flags, &act->sa_flags) || - __get_user(mask, &act->sa_mask)) - return -EFAULT; + scoped_user_read_access(act, Efault) { + unsafe_get_user(new_ka.sa.sa_handler, &act->sa_handler, Efault); + unsafe_get_user(new_ka.sa.sa_restorer, &act->sa_restorer, Efault); + unsafe_get_user(new_ka.sa.sa_flags, &act->sa_flags, Efault); + unsafe_get_user(mask, &act->sa_mask, Efault); + } #ifdef __ARCH_HAS_KA_RESTORER new_ka.ka_restorer = NULL; #endif @@ -4791,15 +4805,18 @@ SYSCALL_DEFINE3(sigaction, int, sig, ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL); if (!ret && oact) { - if (!access_ok(oact, sizeof(*oact)) || - __put_user(old_ka.sa.sa_handler, &oact->sa_handler) || - __put_user(old_ka.sa.sa_restorer, &oact->sa_restorer) || - __put_user(old_ka.sa.sa_flags, &oact->sa_flags) || - __put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask)) - return -EFAULT; + scoped_user_write_access(oact, Efault) { + unsafe_put_user(old_ka.sa.sa_handler, &oact->sa_handler, Efault); + unsafe_put_user(old_ka.sa.sa_restorer, &oact->sa_restorer, Efault); + unsafe_put_user(old_ka.sa.sa_flags, &oact->sa_flags, Efault); + unsafe_put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask, Efault); + } } return ret; + +Efault: + return -EFAULT; } #endif #ifdef CONFIG_COMPAT_OLD_SIGACTION @@ -4813,12 +4830,12 @@ COMPAT_SYSCALL_DEFINE3(sigaction, int, sig, compat_uptr_t handler, restorer; if (act) { - if (!access_ok(act, sizeof(*act)) || - __get_user(handler, &act->sa_handler) || - __get_user(restorer, &act->sa_restorer) || - __get_user(new_ka.sa.sa_flags, &act->sa_flags) || - __get_user(mask, &act->sa_mask)) - return -EFAULT; + scoped_user_read_access(act, Efault) { + unsafe_get_user(handler, &act->sa_handler, Efault); + unsafe_get_user(restorer, &act->sa_restorer, Efault); + unsafe_get_user(new_ka.sa.sa_flags, &act->sa_flags, Efault); + unsafe_get_user(mask, &act->sa_mask, Efault); + } #ifdef __ARCH_HAS_KA_RESTORER new_ka.ka_restorer = NULL; @@ -4831,16 +4848,19 @@ COMPAT_SYSCALL_DEFINE3(sigaction, int, sig, ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL); if (!ret && oact) { - if (!access_ok(oact, sizeof(*oact)) || - __put_user(ptr_to_compat(old_ka.sa.sa_handler), - &oact->sa_handler) || - __put_user(ptr_to_compat(old_ka.sa.sa_restorer), - &oact->sa_restorer) || - __put_user(old_ka.sa.sa_flags, &oact->sa_flags) || - __put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask)) - return -EFAULT; + scoped_user_write_access(oact, Efault) { + unsafe_put_user(ptr_to_compat(old_ka.sa.sa_handler), + &oact->sa_handler, Efault); + unsafe_put_user(ptr_to_compat(old_ka.sa.sa_restorer), + &oact->sa_restorer, Efault); + unsafe_put_user(old_ka.sa.sa_flags, &oact->sa_flags, Efault); + unsafe_put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask, Efault); + } } return ret; + +Efault: + return -EFAULT; } #endif diff --git a/kernel/smp.c b/kernel/smp.c index b696bcc60c08..89a30d831ef5 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -87,6 +87,7 @@ int smpcfd_dead_cpu(unsigned int cpu) free_cpumask_var(cfd->cpumask); free_cpumask_var(cfd->cpumask_ipi); + irq_work_run_cpu(cpu); return 0; } @@ -236,50 +237,76 @@ bool csd_lock_is_stuck(void) return !!atomic_read(&n_csd_lock_stuck); } +/* State that csd_lock_wait_toolong() carries across the __csd_lock_wait() loop. */ +struct csd_wait_state { + u64 ts_start; /* When the wait began. */ + u64 ts_report; /* When the last complaint was printed. */ + u64 ts_resend; /* When the last IPI was re-sent, 0 if never. */ + int bug_id; + unsigned long nmessages; +}; + +/* + * Report a CSD lock that came back, @ts_unstuck being when the release was + * noticed. Only mention the re-send delta if an IPI was actually re-sent. + */ +static void csd_lock_print_unstuck(struct csd_wait_state *state, int cpu, u64 ts_unstuck) +{ + if (state->ts_resend) + pr_alert("csd: CSD lock (#%d) got unstuck on CPU#%02d, CPU#%02d released the lock after %lld ns, %lld ns after the last IPI re-send.\n", + state->bug_id, raw_smp_processor_id(), cpu, + (s64)(ts_unstuck - state->ts_start), + (s64)(ts_unstuck - state->ts_resend)); + else + pr_alert("csd: CSD lock (#%d) got unstuck on CPU#%02d, CPU#%02d released the lock after %lld ns.\n", + state->bug_id, raw_smp_processor_id(), cpu, + (s64)(ts_unstuck - state->ts_start)); +} + /* * Complain if too much time spent waiting. Note that only * the CSD_TYPE_SYNC/ASYNC types provide the destination CPU, * so waiting on other types gets much less information. */ -static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, int *bug_id, unsigned long *nmessages) +static bool csd_lock_wait_toolong(call_single_data_t *csd, struct csd_wait_state *state) { int cpu = -1; int cpux; bool firsttime; - u64 ts2, ts_delta; + u64 ts_now, ts_delta; call_single_data_t *cpu_cur_csd; unsigned int flags = READ_ONCE(csd->node.u_flags); unsigned long long csd_lock_timeout_ns = csd_lock_timeout * NSEC_PER_MSEC; if (!(flags & CSD_FLAG_LOCK)) { - if (!unlikely(*bug_id)) + if (!unlikely(state->bug_id)) return true; + ts_now = ktime_get_mono_fast_ns(); cpu = csd_lock_wait_getcpu(csd); - pr_alert("csd: CSD lock (#%d) got unstuck on CPU#%02d, CPU#%02d released the lock.\n", - *bug_id, raw_smp_processor_id(), cpu); + csd_lock_print_unstuck(state, cpu, ts_now); atomic_dec(&n_csd_lock_stuck); return true; } - ts2 = ktime_get_mono_fast_ns(); + ts_now = ktime_get_mono_fast_ns(); /* How long since we last checked for a stuck CSD lock.*/ - ts_delta = ts2 - *ts1; - if (likely(ts_delta <= csd_lock_timeout_ns * (*nmessages + 1) * - (!*nmessages ? 1 : (ilog2(num_online_cpus()) / 2 + 1)) || + ts_delta = ts_now - state->ts_report; + if (likely(ts_delta <= csd_lock_timeout_ns * (state->nmessages + 1) * + (!state->nmessages ? 1 : (ilog2(num_online_cpus()) / 2 + 1)) || csd_lock_timeout_ns == 0)) return false; - if (ts0 > ts2) { + if (state->ts_start > ts_now) { /* Our own sched_clock went backward; don't blame another CPU. */ - ts_delta = ts0 - ts2; + ts_delta = state->ts_start - ts_now; pr_alert("sched_clock on CPU %d went backward by %llu ns\n", raw_smp_processor_id(), ts_delta); - *ts1 = ts2; + state->ts_report = ts_now; return false; } - firsttime = !*bug_id; + firsttime = !state->bug_id; if (firsttime) - *bug_id = atomic_inc_return(&csd_bug_count); + state->bug_id = atomic_inc_return(&csd_bug_count); cpu = csd_lock_wait_getcpu(csd); if (WARN_ONCE(cpu < 0 || cpu >= nr_cpu_ids, "%s: cpu = %d\n", __func__, cpu)) cpux = 0; @@ -293,11 +320,11 @@ static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, in */ cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); /* How long since this CSD lock was stuck. */ - ts_delta = ts2 - ts0; + ts_delta = ts_now - state->ts_start; pr_alert("csd: %s non-responsive CSD lock (#%d) on CPU#%d, waiting %lld ns for CPU#%02d %pS(%ps).\n", - firsttime ? "Detected" : "Continued", *bug_id, raw_smp_processor_id(), (s64)ts_delta, + firsttime ? "Detected" : "Continued", state->bug_id, raw_smp_processor_id(), (s64)ts_delta, cpu, csd->func, csd->info); - (*nmessages)++; + state->nmessages++; if (firsttime) atomic_inc(&n_csd_lock_stuck); /* @@ -308,23 +335,24 @@ static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, in BUG_ON(panic_on_ipistall > 0 && (s64)ts_delta > ((s64)panic_on_ipistall * NSEC_PER_MSEC)); if (cpu_cur_csd && csd != cpu_cur_csd) { pr_alert("\tcsd: CSD lock (#%d) handling prior %pS(%ps) request.\n", - *bug_id, READ_ONCE(per_cpu(cur_csd_func, cpux)), + state->bug_id, READ_ONCE(per_cpu(cur_csd_func, cpux)), READ_ONCE(per_cpu(cur_csd_info, cpux))); } else { pr_alert("\tcsd: CSD lock (#%d) %s.\n", - *bug_id, !cpu_cur_csd ? "unresponsive" : "handling this request"); + state->bug_id, !cpu_cur_csd ? "unresponsive" : "handling this request"); } if (cpu >= 0) { if (atomic_cmpxchg_acquire(&per_cpu(trigger_backtrace, cpu), 1, 0)) dump_cpu_task(cpu); if (!cpu_cur_csd) { - pr_alert("csd: Re-sending CSD lock (#%d) IPI from CPU#%02d to CPU#%02d\n", *bug_id, raw_smp_processor_id(), cpu); + pr_alert("csd: Re-sending CSD lock (#%d) IPI from CPU#%02d to CPU#%02d\n", state->bug_id, raw_smp_processor_id(), cpu); arch_send_call_function_single_ipi(cpu); + state->ts_resend = ktime_get_mono_fast_ns(); } } if (firsttime) dump_stack(); - *ts1 = ts2; + state->ts_report = ts_now; return false; } @@ -338,15 +366,13 @@ static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, in */ static void __csd_lock_wait(call_single_data_t *csd) { - unsigned long nmessages = 0; - int bug_id = 0; - u64 ts0, ts1; + struct csd_wait_state state = {}; guard(preempt)(); - ts1 = ts0 = ktime_get_mono_fast_ns(); + state.ts_report = state.ts_start = ktime_get_mono_fast_ns(); for (;;) { - if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id, &nmessages)) + if (csd_lock_wait_toolong(csd, &state)) break; cpu_relax(); } diff --git a/kernel/smpboot.c b/kernel/smpboot.c index 4503b60ce9bd..3f60e8c6dd30 100644 --- a/kernel/smpboot.c +++ b/kernel/smpboot.c @@ -103,6 +103,7 @@ static int smpboot_thread_fn(void *data) { struct smpboot_thread_data *td = data; struct smp_hotplug_thread *ht = td->ht; + bool should_run; while (1) { set_current_state(TASK_INTERRUPTIBLE); @@ -117,7 +118,8 @@ static int smpboot_thread_fn(void *data) return 0; } - if (kthread_should_park()) { + should_run = td->status == HP_THREAD_ACTIVE && ht->thread_should_run(td->cpu); + if (kthread_should_park() && !should_run) { __set_current_state(TASK_RUNNING); preempt_enable(); if (ht->park && td->status == HP_THREAD_ACTIVE) { @@ -151,7 +153,7 @@ static int smpboot_thread_fn(void *data) continue; } - if (!ht->thread_should_run(td->cpu)) { + if (!should_run) { preempt_enable_no_resched(); schedule(); } else { diff --git a/kernel/softirq.c b/kernel/softirq.c index 5d02c36c40e3..7fc2200392cd 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -350,8 +350,8 @@ static inline void ksoftirqd_run_end(void) local_irq_enable(); } -static inline void softirq_handle_begin(void) { } -static inline void softirq_handle_end(void) { } +static inline bool softirq_handle_begin(void) { return false; } +static inline void softirq_handle_end(bool from_irq_exit) { } static inline bool should_wake_ksoftirqd(void) { @@ -481,15 +481,38 @@ void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) } EXPORT_SYMBOL(__local_bh_enable_ip); -static inline void softirq_handle_begin(void) +static inline bool softirq_handle_begin(void) { - __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET); + if (!in_hardirq()) { + __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET); + return false; + } + + /* + * Only reached from irq_exit(), with HARDIRQ_OFFSET still set. + * Replace it with SOFTIRQ_OFFSET before handle_softirqs() enables + * interrupts. Use the raw operation to preserve the preemption + * disable location recorded by irq_enter_rcu(), and update lockdep + * directly. + */ + __preempt_count_sub(HARDIRQ_OFFSET - SOFTIRQ_OFFSET); + lockdep_softirqs_off(_RET_IP_); + WARN_ON_ONCE(irq_count() != SOFTIRQ_OFFSET); + + return true; } -static inline void softirq_handle_end(void) +static inline void softirq_handle_end(bool from_irq_exit) { - __local_bh_enable(SOFTIRQ_OFFSET); - WARN_ON_ONCE(in_interrupt()); + if (!from_irq_exit) { + __local_bh_enable(SOFTIRQ_OFFSET); + WARN_ON_ONCE(in_interrupt()); + return; + } + + lockdep_softirqs_on(_RET_IP_); + __preempt_count_add(HARDIRQ_OFFSET - SOFTIRQ_OFFSET); + WARN_ON_ONCE(irq_count() != HARDIRQ_OFFSET); } static inline void ksoftirqd_run_begin(void) @@ -605,6 +628,7 @@ static void handle_softirqs(bool ksirqd) unsigned long old_flags = current->flags; int max_restart = MAX_SOFTIRQ_RESTART; struct softirq_action *h; + bool from_irq_exit; bool in_hardirq; __u32 pending; int softirq_bit; @@ -618,7 +642,7 @@ static void handle_softirqs(bool ksirqd) pending = local_softirq_pending(); - softirq_handle_begin(); + from_irq_exit = softirq_handle_begin(); in_hardirq = lockdep_softirq_start(); account_softirq_enter(current); @@ -645,8 +669,8 @@ restart: h->action(); trace_softirq_exit(vec_nr); if (unlikely(prev_count != preempt_count())) { - pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", - vec_nr, softirq_to_name[vec_nr], h->action, + pr_err("huh, entered softirq %u %s with preempt_count %08x, exited with %08x?\n", + vec_nr, softirq_to_name[vec_nr], prev_count, preempt_count()); preempt_count_set(prev_count); } @@ -670,7 +694,7 @@ restart: account_softirq_exit(current); lockdep_softirq_end(in_hardirq); - softirq_handle_end(); + softirq_handle_end(from_irq_exit); current_restore_flags(old_flags, PF_MEMALLOC); } @@ -716,7 +740,7 @@ static inline void tick_irq_exit(void) /* Make sure that timer wheel updates are propagated */ if ((sched_core_idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) { - if (!in_hardirq()) + if (hardirq_count() == HARDIRQ_OFFSET) tick_nohz_irq_exit(); } #endif @@ -748,8 +772,12 @@ static inline void __irq_exit_rcu(void) lockdep_assert_irqs_disabled(); #endif account_hardirq_exit(current); - preempt_count_sub(HARDIRQ_OFFSET); - if (!in_interrupt() && local_softirq_pending()) { + + /* + * HARDIRQ_OFFSET is still set. Only the outermost interrupt handles + * softirqs, and only if it did not hit a softirq or BH disabled section. + */ + if (irq_count() == HARDIRQ_OFFSET && local_softirq_pending()) { /* * If we left hrtimers unarmed, make sure to arm them now, * before enabling interrupts to run softirq. @@ -758,11 +786,17 @@ static inline void __irq_exit_rcu(void) invoke_softirq(); } + /* + * Wake the timer thread even if the interrupt hit a softirq or a + * section with BHs disabled. Only the outermost interrupt can wake it. + */ if (IS_ENABLED(CONFIG_IRQ_FORCED_THREADING) && force_irqthreads() && - local_timers_pending_force_th() && !(in_nmi() | in_hardirq())) + local_timers_pending_force_th() && + hardirq_count() == HARDIRQ_OFFSET) wake_timersd(); tick_irq_exit(); + preempt_count_sub(HARDIRQ_OFFSET); } /** diff --git a/kernel/sysctl.c b/kernel/sysctl.c index f7b75985d542..f5145b32f6be 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -29,14 +29,12 @@ EXPORT_SYMBOL(sysctl_vals); const unsigned long sysctl_long_vals[] = { 0, 1, LONG_MAX }; EXPORT_SYMBOL_GPL(sysctl_long_vals); -#if defined(CONFIG_SYSCTL) +#ifdef CONFIG_SYSCTL /* Constants used for minimum and maximum */ static const int ngroups_max = NGROUPS_MAX; static const int cap_last_cap = CAP_LAST_CAP; -#ifdef CONFIG_SYSCTL - /** * enum sysctl_writes_mode - supported sysctl write modes * @@ -64,14 +62,6 @@ enum sysctl_writes_mode { }; static enum sysctl_writes_mode sysctl_writes_strict = SYSCTL_WRITES_STRICT; -#endif /* CONFIG_SYSCTL */ -#endif /* CONFIG_SYSCTL */ - -/* - * /proc/sys support - */ - -#ifdef CONFIG_SYSCTL static int _proc_do_string(char *data, int maxlen, int dir, char *buffer, size_t *lenp, loff_t *ppos) @@ -483,7 +473,7 @@ int proc_int_k2u_conv_kop(ulong *u_ptr, const int *k_ptr, bool *negp, if (val < 0) { *negp = true; - *u_ptr = k_ptr_op ? -k_ptr_op((ulong)val) : -(ulong)val; + *u_ptr = k_ptr_op ? k_ptr_op(-(ulong)val) : -(ulong)val; } else { *negp = false; *u_ptr = k_ptr_op ? k_ptr_op((ulong)val) : (ulong) val; @@ -595,7 +585,7 @@ static int do_proc_int_conv_minmax(bool *negp, unsigned long *u_ptr, int *k_ptr, static const char proc_wspace_sep[] = { ' ', '\t', '\n' }; /* - * Element type processed by do_proc_vec(). The tag selects the element size + * Element type processed by proc_vec(). The tag selects the element size * and signedness, and it selects which member of union proc_vec_conv is live. */ enum proc_vec_type { @@ -605,7 +595,7 @@ enum proc_vec_type { }; /* - * Converter passed to do_proc_vec(). Only the member matching the + * Converter passed to proc_vec(). Only the member matching the * enum proc_vec_type tag is ever read, so every dispatch stays fully typed and * no void * converter pointer is needed. */ @@ -637,101 +627,177 @@ static int proc_vec_conv(enum proc_vec_type type, union proc_vec_conv conv, return -EINVAL; } -/* - * Read/write a vector of @type elements. The element size and signedness are - * derived from @type, so a single runtime function replaces the per-type - * variants. table->data is walked as raw bytes (@i) advanced by @size; the - * converter performs the actual typed load/store. - */ -static int do_proc_vec(const struct ctl_table *table, int dir, - void *buffer, size_t *lenp, loff_t *ppos, - enum proc_vec_type type, union proc_vec_conv conv) +static int commit_conv_vec(const enum proc_vec_type data_type, void *dst, + const void *src, size_t nr) { - int vleft, first = 1, err = 0; - size_t left, size; - bool is_unsigned; - char *i, *p; + size_t i; - switch (type) { + switch (data_type) { case PROC_VEC_INT: - size = sizeof(int); - is_unsigned = false; - break; - case PROC_VEC_UINT: - size = sizeof(uint); - is_unsigned = true; - break; + for (i = 0; i < nr; i++) + WRITE_ONCE(((int *)dst)[i], ((const int *)src)[i]); + return 0; + case PROC_VEC_ULONG: - size = sizeof(ulong); - is_unsigned = true; - break; + for (i = 0; i < nr; i++) + WRITE_ONCE(((ulong *)dst)[i], ((const ulong *)src)[i]); + return 0; default: return -EINVAL; } +} - if (!table->data || !table->maxlen || !*lenp || - (*ppos && SYSCTL_KERN_TO_USER(dir))) { - *lenp = 0; - return 0; - } - - i = table->data; - vleft = table->maxlen / size; - left = *lenp; - - /* uint arrays are not supported, *Do not* add support for them. */ - if (type == PROC_VEC_UINT && vleft != 1) - return -EINVAL; - - if (SYSCTL_USER_TO_KERN(dir)) { - if (proc_first_pos_non_zero_ignore(ppos, table)) - goto out; - - if (left > PAGE_SIZE - 1) - left = PAGE_SIZE - 1; - p = buffer; +/** + * apply_conv_on_vec - Apply converter function on data vector + * + * @conv: The converter to be applied + * @table: The sysctl table + * @data_type: Type used in converter appliation (INT, UINT or ULONG) + * @data_size: Number of data elements. + * @conv_dir: %TRUE if this is a write to the sysctl file + * @buf_nbyte: Number of bytes for buf + * @buf: The user buffer + * @buf_left_final: Number of outstanding (non converted) bytes. + * + * Element signedness is derived from @data_type. table->data is walked + * as raw bytes (@data) advanced by @data_size; the converter performs + * the actual typed load/store. Sets buf_left_final to the number of + * bytes that where left outstanding after conversion. Can be > 0. + * + * Returns: %0 on success. Non-zero on error. + */ +static int apply_conv_on_vec(const union proc_vec_conv conv, + const struct ctl_table *table, + const enum proc_vec_type data_type, + const size_t data_size, const int conv_dir, + const size_t buf_nbyte, void *buf, + size_t *buf_left_final) +{ + int vec_left, err = 0; + size_t buf_left, nr_conv = 0; + char *data, *data_stage = NULL, *p; + bool is_unsigned = data_type == PROC_VEC_UINT || data_type == PROC_VEC_ULONG; + + buf_left = buf_nbyte; + + data = table->data; + if (SYSCTL_USER_TO_KERN(conv_dir)) { + if (buf_left > PAGE_SIZE - 1) + buf_left = PAGE_SIZE - 1; + p = buf; + + if (table->maxlen > data_size) { + data_stage = kmemdup(table->data, table->maxlen, GFP_KERNEL); + if (!data_stage) { + err = -ENOMEM; + goto out; + } + data = data_stage; + } } - for (; left && vleft--; i += size, first = 0) { + vec_left = table->maxlen / data_size; + for (; buf_left && vec_left--; data += data_size, nr_conv++) { unsigned long lval; bool neg = false; - if (SYSCTL_USER_TO_KERN(dir)) { - proc_skip_spaces(&p, &left); + if (SYSCTL_USER_TO_KERN(conv_dir)) { + proc_skip_spaces(&p, &buf_left); - if (!left) + if (!buf_left) break; - err = proc_get_long(&p, &left, &lval, &neg, + err = proc_get_long(&p, &buf_left, &lval, &neg, proc_wspace_sep, sizeof(proc_wspace_sep), NULL); if (!err && neg && is_unsigned) err = -EINVAL; if (err) break; - if (proc_vec_conv(type, conv, &neg, &lval, i, dir, table)) { + if (proc_vec_conv(data_type, conv, &neg, &lval, data, conv_dir, table)) { err = -EINVAL; break; } } else { - if (proc_vec_conv(type, conv, &neg, &lval, i, dir, table)) { + if (proc_vec_conv(data_type, conv, &neg, &lval, data, conv_dir, table)) { err = -EINVAL; break; } - if (!first) - proc_put_char(&buffer, &left, '\t'); - proc_put_long(&buffer, &left, lval, neg); + if (nr_conv) + proc_put_char(&buf, &buf_left, '\t'); + proc_put_long(&buf, &buf_left, lval, neg); } } - if (SYSCTL_KERN_TO_USER(dir) && !first && left && !err) - proc_put_char(&buffer, &left, '\n'); - if (SYSCTL_USER_TO_KERN(dir) && !err && left) - proc_skip_spaces(&p, &left); - if (SYSCTL_USER_TO_KERN(dir) && first) - return err ? : -EINVAL; - *lenp -= left; + if (SYSCTL_USER_TO_KERN(conv_dir)) { + if (!err && buf_left) + proc_skip_spaces(&p, &buf_left); + if (!nr_conv) { + err = err ? : -EINVAL; + goto out; + } + if (!err && data_stage) + err = commit_conv_vec(data_type, table->data, data_stage, nr_conv); + } else { + if (nr_conv && buf_left && !err) + proc_put_char(&buf, &buf_left, '\n'); + } + + *buf_left_final = buf_left; + +out: + kfree(data_stage); + return err; +} + +/* Read/write a vector of @type elements. */ +static int proc_vec(const struct ctl_table *table, int dir, void *buffer, + size_t *lenp, loff_t *ppos, enum proc_vec_type type, + union proc_vec_conv conv) +{ + int err = 0; + size_t data_size, left_nbyte = SIZE_MAX; + + switch (type) { + case PROC_VEC_INT: + data_size = sizeof(int); + break; + case PROC_VEC_UINT: + data_size = sizeof(uint); + break; + case PROC_VEC_ULONG: + data_size = sizeof(ulong); + break; + default: + return -EINVAL; + } + + if (!table->data || !table->maxlen || !*lenp || + (*ppos && SYSCTL_KERN_TO_USER(dir))) { + *lenp = 0; + return 0; + } + + if (SYSCTL_USER_TO_KERN(dir)) { + if (proc_first_pos_non_zero_ignore(ppos, table)) + goto out; + } + + err = apply_conv_on_vec(conv, table, type, data_size, dir, *lenp, buffer, + &left_nbyte); + + /* + * An unchanged left_nbyte signals a write with no parsed element; which + * is an error. Using SIZE_MAX to detect this error is possible because: + * 1. lenp is bounded by KMALLOC_MAX_SIZE in proc_sys_call_handler + * 2. lenp could never be SIZE_MAX as it is a "ridiculous" (exabyte) allocation. + */ + if (left_nbyte == SIZE_MAX) + return err; + + *lenp -= left_nbyte; out: *ppos += *lenp; + return err; } @@ -756,12 +822,15 @@ int proc_douintvec_conv(const struct ctl_table *table, int dir, void *buffer, int (*conv)(bool *negp, ulong *u_ptr, uint *k_ptr, int dir, const struct ctl_table *table)) { + /* uint arrays are not supported, *Do not* add support for them. */ + if (table->maxlen && (table->maxlen / sizeof(uint)) != 1) + return -EINVAL; if (!conv) conv = do_proc_uint_conv; - return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_UINT, - (union proc_vec_conv){ .uint_conv = conv }); + return proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_UINT, + (union proc_vec_conv){ .uint_conv = conv }); } /** @@ -820,8 +889,8 @@ int proc_dobool(const struct ctl_table *table, int dir, void *buffer, int proc_dointvec(const struct ctl_table *table, int dir, void *buffer, size_t *lenp, loff_t *ppos) { - return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_INT, - (union proc_vec_conv){ .int_conv = do_proc_int_conv }); + return proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_INT, + (union proc_vec_conv){ .int_conv = do_proc_int_conv }); } /** @@ -840,8 +909,7 @@ int proc_dointvec(const struct ctl_table *table, int dir, void *buffer, int proc_douintvec(const struct ctl_table *table, int dir, void *buffer, size_t *lenp, loff_t *ppos) { - return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_UINT, - (union proc_vec_conv){ .uint_conv = do_proc_uint_conv }); + return proc_douintvec_conv(table, dir, buffer, lenp, ppos, do_proc_uint_conv); } /** @@ -864,8 +932,8 @@ int proc_douintvec(const struct ctl_table *table, int dir, void *buffer, int proc_dointvec_minmax(const struct ctl_table *table, int dir, void *buffer, size_t *lenp, loff_t *ppos) { - return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_INT, - (union proc_vec_conv){ .int_conv = do_proc_int_conv_minmax }); + return proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_INT, + (union proc_vec_conv){ .int_conv = do_proc_int_conv_minmax }); } /** @@ -891,8 +959,8 @@ int proc_dointvec_minmax(const struct ctl_table *table, int dir, int proc_douintvec_minmax(const struct ctl_table *table, int dir, void *buffer, size_t *lenp, loff_t *ppos) { - return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_UINT, - (union proc_vec_conv){ .uint_conv = do_proc_uint_conv_minmax }); + return proc_douintvec_conv(table, dir, buffer, lenp, ppos, + do_proc_uint_conv_minmax); } /** @@ -935,8 +1003,7 @@ int proc_dou8vec_minmax(const struct ctl_table *table, int dir, tmp.extra2 = (unsigned int *) &max; val = READ_ONCE(*data); - res = do_proc_vec(&tmp, dir, buffer, lenp, ppos, PROC_VEC_UINT, - (union proc_vec_conv){ .uint_conv = do_proc_uint_conv_minmax }); + res = proc_douintvec_minmax(&tmp, dir, buffer, lenp, ppos); if (res) return res; if (SYSCTL_USER_TO_KERN(dir)) @@ -1066,8 +1133,8 @@ int proc_doulongvec_conv(const struct ctl_table *table, int dir, int (*conv)(bool *negp, ulong *u_ptr, ulong *k_ptr, int dir, const struct ctl_table *table)) { - return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_ULONG, - (union proc_vec_conv){ .ulong_conv = conv }); + return proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_ULONG, + (union proc_vec_conv){ .ulong_conv = conv }); } /** @@ -1087,10 +1154,10 @@ int proc_doulongvec_conv(const struct ctl_table *table, int dir, * Returns: %0 on success. */ int proc_doulongvec_minmax(const struct ctl_table *table, int dir, - void *buffer, size_t *lenp, loff_t *ppos) + void *buffer, size_t *lenp, loff_t *ppos) { - return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_ULONG, - (union proc_vec_conv){ .ulong_conv = do_proc_ulong_conv }); + return proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_ULONG, + (union proc_vec_conv){ .ulong_conv = do_proc_ulong_conv }); } /** @@ -1114,8 +1181,8 @@ int proc_dointvec_conv(const struct ctl_table *table, int dir, void *buffer, { if (!conv) conv = do_proc_int_conv; - return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_INT, - (union proc_vec_conv){ .int_conv = conv }); + return proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_INT, + (union proc_vec_conv){ .int_conv = conv }); } /** @@ -1265,6 +1332,87 @@ int proc_do_large_bitmap(const struct ctl_table *table, int dir, return err; } +int proc_do_static_key(const struct ctl_table *table, int dir, + void *buffer, size_t *lenp, loff_t *ppos) +{ + struct static_key *key = (struct static_key *)table->data; + static DEFINE_MUTEX(static_key_mutex); + int val, ret; + struct ctl_table tmp = { + .data = &val, + .maxlen = sizeof(val), + .mode = table->mode, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }; + + if (SYSCTL_USER_TO_KERN(dir) && !capable(CAP_SYS_ADMIN)) + return -EPERM; + + mutex_lock(&static_key_mutex); + val = static_key_enabled(key); + ret = proc_dointvec_minmax(&tmp, dir, buffer, lenp, ppos); + if (SYSCTL_USER_TO_KERN(dir) && !ret) { + if (val) + static_key_enable(key); + else + static_key_disable(key); + } + mutex_unlock(&static_key_mutex); + return ret; +} + +static const struct ctl_table sysctl_subsys_table[] = { + { + .procname = "sysctl_writes_strict", + .data = &sysctl_writes_strict, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_NEG_ONE, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "ngroups_max", + .data = (void *)&ngroups_max, + .maxlen = sizeof (int), + .mode = 0444, + .proc_handler = proc_dointvec, + }, + { + .procname = "cap_last_cap", + .data = (void *)&cap_last_cap, + .maxlen = sizeof(int), + .mode = 0444, + .proc_handler = proc_dointvec, + }, +#ifdef CONFIG_SYSCTL_ARCH_UNALIGN_ALLOW + { + .procname = "unaligned-trap", + .data = &unaligned_enabled, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif +#ifdef CONFIG_SYSCTL_ARCH_UNALIGN_NO_WARN + { + .procname = "ignore-unaligned-usertrap", + .data = &no_unaligned_warning, + .maxlen = sizeof (int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif +}; + +int __init sysctl_init_bases(void) +{ + register_sysctl_init("kernel", sysctl_subsys_table); + + return 0; +} + #else /* CONFIG_SYSCTL */ int proc_dostring(const struct ctl_table *table, int dir, @@ -1364,91 +1512,12 @@ int proc_do_large_bitmap(const struct ctl_table *table, int dir, return -ENOSYS; } -#endif /* CONFIG_SYSCTL */ - -#if defined(CONFIG_SYSCTL) int proc_do_static_key(const struct ctl_table *table, int dir, void *buffer, size_t *lenp, loff_t *ppos) { - struct static_key *key = (struct static_key *)table->data; - static DEFINE_MUTEX(static_key_mutex); - int val, ret; - struct ctl_table tmp = { - .data = &val, - .maxlen = sizeof(val), - .mode = table->mode, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }; - - if (SYSCTL_USER_TO_KERN(dir) && !capable(CAP_SYS_ADMIN)) - return -EPERM; - - mutex_lock(&static_key_mutex); - val = static_key_enabled(key); - ret = proc_dointvec_minmax(&tmp, dir, buffer, lenp, ppos); - if (SYSCTL_USER_TO_KERN(dir) && !ret) { - if (val) - static_key_enable(key); - else - static_key_disable(key); - } - mutex_unlock(&static_key_mutex); - return ret; + return -ENOSYS; } -static const struct ctl_table sysctl_subsys_table[] = { -#ifdef CONFIG_SYSCTL - { - .procname = "sysctl_writes_strict", - .data = &sysctl_writes_strict, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_NEG_ONE, - .extra2 = SYSCTL_ONE, - }, -#endif - { - .procname = "ngroups_max", - .data = (void *)&ngroups_max, - .maxlen = sizeof (int), - .mode = 0444, - .proc_handler = proc_dointvec, - }, - { - .procname = "cap_last_cap", - .data = (void *)&cap_last_cap, - .maxlen = sizeof(int), - .mode = 0444, - .proc_handler = proc_dointvec, - }, -#ifdef CONFIG_SYSCTL_ARCH_UNALIGN_ALLOW - { - .procname = "unaligned-trap", - .data = &unaligned_enabled, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, -#endif -#ifdef CONFIG_SYSCTL_ARCH_UNALIGN_NO_WARN - { - .procname = "ignore-unaligned-usertrap", - .data = &no_unaligned_warning, - .maxlen = sizeof (int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, -#endif -}; - -int __init sysctl_init_bases(void) -{ - register_sysctl_init("kernel", sysctl_subsys_table); - - return 0; -} #endif /* CONFIG_SYSCTL */ /* * No sense putting this after each symbol definition, twice, diff --git a/kernel/tests/.kunitconfig b/kernel/tests/.kunitconfig new file mode 100644 index 000000000000..b3d1206fd81a --- /dev/null +++ b/kernel/tests/.kunitconfig @@ -0,0 +1,4 @@ +CONFIG_KUNIT=y +CONFIG_NAMESPACES=y +CONFIG_USER_NS=y +CONFIG_USER_NS_MAP_KUNIT_TEST=y diff --git a/kernel/tests/user_ns_map_kunit.c b/kernel/tests/user_ns_map_kunit.c new file mode 100644 index 000000000000..033dccc6a535 --- /dev/null +++ b/kernel/tests/user_ns_map_kunit.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit test for user namespace map insertion and sorting. + */ + +#define pr_fmt(fmt) "user_namespace: " fmt + +#include <kunit/test.h> +#include <linux/user_namespace.h> + +#define NR_EXTENTS (UID_GID_MAP_MAX_BASE_EXTENTS + 5) + +static void user_ns_map_insert(struct kunit *test) +{ + struct uid_gid_map map; + struct uid_gid_extent extent; + int i, ret; + + memset(&map, 0, sizeof(map)); + + /* Insert up to UID_GID_MAP_MAX_BASE_EXTENTS elements */ + for (i = 0; i < UID_GID_MAP_MAX_BASE_EXTENTS; i++) { + extent.first = i * 10; + extent.lower_first = i * 100; + extent.count = 5; + + ret = uid_gid_map_insert_extent(&map, &extent); + KUNIT_ASSERT_EQ(test, ret, 0); + } + + KUNIT_EXPECT_EQ(test, map.nr_extents, UID_GID_MAP_MAX_BASE_EXTENTS); + + /* Verify the elements ended up in the 'extent' array */ + for (i = 0; i < UID_GID_MAP_MAX_BASE_EXTENTS; i++) { + KUNIT_EXPECT_EQ(test, map.extent[i].first, i * 10); + KUNIT_EXPECT_EQ(test, map.extent[i].lower_first, i * 100); + KUNIT_EXPECT_EQ(test, map.extent[i].count, 5); + } +} + +static void user_ns_map_insert_extended(struct kunit *test) +{ + struct uid_gid_map map; + struct uid_gid_extent extent; + int i, ret; + + memset(&map, 0, sizeof(map)); + + /* Insert more than UID_GID_MAP_MAX_BASE_EXTENTS elements */ + for (i = 0; i < NR_EXTENTS; i++) { + int value = 9 - i; + + extent.first = value * 10; + extent.lower_first = value * 100; + extent.count = 5; + + ret = uid_gid_map_insert_extent(&map, &extent); + KUNIT_ASSERT_EQ(test, ret, 0); + } + + KUNIT_EXPECT_EQ(test, map.nr_extents, NR_EXTENTS); + + /* Now sort the map to set up reverse mapping */ + ret = uid_gid_map_sort(&map); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, map.reverse); + + /* Verify the elements are in 'forward' and that sorting is correct */ + for (i = 0; i < map.nr_extents; i++) { + KUNIT_EXPECT_EQ(test, map.forward[i].first, i * 10); + KUNIT_EXPECT_EQ(test, map.forward[i].lower_first, i * 100); + KUNIT_EXPECT_EQ(test, map.forward[i].count, 5); + + KUNIT_EXPECT_EQ(test, map.reverse[i].first, i * 10); + KUNIT_EXPECT_EQ(test, map.reverse[i].lower_first, i * 100); + KUNIT_EXPECT_EQ(test, map.reverse[i].count, 5); + } + + kfree(map.forward); + kfree(map.reverse); +} + +static struct kunit_case user_ns_map_test_cases[] = { + KUNIT_CASE(user_ns_map_insert), + KUNIT_CASE(user_ns_map_insert_extended), + {} +}; + +static struct kunit_suite user_ns_map_test_suite = { + .name = "user_ns_map", + .test_cases = user_ns_map_test_cases, +}; + +kunit_test_suite(user_ns_map_test_suite); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("KUnit test for user namespace map insertion"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index cbf1693c86b3..17dd38a6cee7 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -780,7 +780,8 @@ static void hrtimer_switch_to_hres(void) return; } base->hres_active = true; - hrtimer_resolution = HIGH_RES_NSEC; + if (hrtimer_resolution != HIGH_RES_NSEC) + hrtimer_resolution = HIGH_RES_NSEC; tick_setup_sched_timer(true); /* "Retrigger" the interrupt to get things going */ @@ -2003,7 +2004,7 @@ bool hrtimer_active(const struct hrtimer *timer) base = READ_ONCE(timer->base); seq = raw_read_seqcount_begin(&base->seq); - if (timer->is_queued || base->running == timer) + if (timer->is_queued || READ_ONCE(base->running) == timer) return true; } while (read_seqcount_retry(&base->seq, seq) || base != READ_ONCE(timer->base)); @@ -2040,7 +2041,7 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, struct hrtimer_cloc lockdep_assert_held(&cpu_base->lock); debug_hrtimer_deactivate(timer); - base->running = timer; + WRITE_ONCE(base->running, timer); /* * Separate the ->running assignment from the ->is_queued assignment. @@ -2099,7 +2100,7 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, struct hrtimer_cloc raw_write_seqcount_barrier(&base->seq); WARN_ON_ONCE(base->running != timer); - base->running = NULL; + WRITE_ONCE(base->running, NULL); } static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now, @@ -2323,9 +2324,9 @@ void hrtimer_run_queues(void) static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer) { struct hrtimer_sleeper *t = container_of(timer, struct hrtimer_sleeper, timer); - struct task_struct *task = t->task; + struct task_struct *task = hrtimer_sleeper_task_get(t); - t->task = NULL; + hrtimer_sleeper_task_set(t, NULL); if (task) wake_up_process(task); @@ -2354,7 +2355,7 @@ void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl, enum hrtimer_mode /* If already expired, clear the task pointer and set current state to running */ if (!hrtimer_start_expires_user(&sl->timer, mode)) { - sl->task = NULL; + hrtimer_sleeper_task_set(sl, NULL); __set_current_state(TASK_RUNNING); } } @@ -2388,7 +2389,7 @@ static void __hrtimer_setup_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_ } __hrtimer_setup(&sl->timer, hrtimer_wakeup, clock_id, mode); - sl->task = current; + hrtimer_sleeper_task_set(sl, current); } /** @@ -2432,17 +2433,17 @@ static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mod set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); hrtimer_sleeper_start_expires(t, mode); - if (likely(t->task)) + if (likely(hrtimer_sleeper_task_get(t))) schedule(); hrtimer_cancel(&t->timer); mode = HRTIMER_MODE_ABS; - } while (t->task && !signal_pending(current)); + } while (hrtimer_sleeper_task_get(t) && !signal_pending(current)); __set_current_state(TASK_RUNNING); - if (!t->task) + if (!hrtimer_sleeper_task_get(t)) return 0; restart = ¤t->restart_block; diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c index 80c354811538..af1c69bf0c23 100644 --- a/kernel/time/jiffies.c +++ b/kernel/time/jiffies.c @@ -8,6 +8,7 @@ #include <linux/jiffies.h> #include <linux/module.h> #include <linux/init.h> +#include <linux/sysctl.h> #include "timekeeping.h" #include "tick-internal.h" @@ -101,6 +102,8 @@ void __init register_refined_jiffies(long cycles_per_second) #ifdef CONFIG_SYSCTL static ulong mult_hz(const ulong val) { + if (val >= ULONG_MAX / HZ) + return ULONG_MAX; return val * HZ; } diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 0bf4fcd969c8..cd75d4bb5b64 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -439,6 +439,38 @@ static void trigger_base_recalc_expires(struct k_itimer *timer, base->nextevt = 0; } +static inline bool cpu_timer_enqueue(struct timerqueue_head *head, + struct cpu_timer *ctmr) +{ + ctmr->head = head; + return timerqueue_add(head, &ctmr->node); +} + +static inline bool cpu_timer_queued(struct cpu_timer *ctmr) +{ + return !!ctmr->head; +} + +static inline bool cpu_timer_dequeue(struct cpu_timer *ctmr) +{ + if (cpu_timer_queued(ctmr)) { + timerqueue_del(ctmr->head, &ctmr->node); + ctmr->head = NULL; + return true; + } + return false; +} + +static inline u64 cpu_timer_getexpires(struct cpu_timer *ctmr) +{ + return ctmr->node.expires; +} + +static inline void cpu_timer_setexpires(struct cpu_timer *ctmr, u64 exp) +{ + ctmr->node.expires = exp; +} + /* * Dequeue the timer and reset the base if it was its earliest expiration. * It makes sure the next tick recalculates the base next expiration so we @@ -607,6 +639,7 @@ static int posix_cpu_timer_del(struct k_itimer *timer) } if (!ret) { + WARN_ON_ONCE(cpu_timer_queued(&timer->it.cpu)); put_pid(timer->it.cpu.pid); timer->it_status = POSIX_TIMER_DISARMED; } @@ -639,18 +672,50 @@ static void cleanup_timers(struct posix_cputimers *pct) cleanup_timerqueue(&pct->bases[CPUCLOCK_SCHED].tqhead); } +static inline void posix_cpu_timers_exit_work(void); + /* - * These are both called with the siglock held, when the current thread - * is being reaped. When the final (leader) thread in the group is reaped, - * posix_cpu_timers_exit_group will be called after posix_cpu_timers_exit. + * Invoked from posixtimer_exit_task() after PF_EXITING was set in tsk::flags or + * from posixtimer_exec_cleanup(). */ -void posix_cpu_timers_exit(struct task_struct *tsk) +void posix_cpu_timers_exit_task(void) { - cleanup_timers(&tsk->posix_cputimers); + posix_cpu_timers_exit_work(); + + guard(spinlock_irq)(¤t->sighand->siglock); + cleanup_timers(¤t->posix_cputimers); } -void posix_cpu_timers_exit_group(struct task_struct *tsk) + +/* + * Invoked from posixtimer_exit_group() after PF_EXITING was set in tsk::flags. + */ +void posix_cpu_timers_exit_group(void) { - cleanup_timers(&tsk->signal->posix_cputimers); + posix_cpu_timers_exit_task(); + + guard(spinlock_irq)(¤t->sighand->siglock); + cleanup_timers(¤t->signal->posix_cputimers); +} + +/* + * This function validates that POSIX CPU timers can be safely enqueued on the + * target task. + * + * Enqueue is allowed when PF_EXITING is not set. If set then it is only allowed + * for process shared timers (type = PIDTYPE_TGID) as long as tsk::signal::flags + * does not have SIGNAL_GROUP_EXIT set. PIDTYPE_PID targets are not allowed at + * all when the task has PF_EXITING set. + * + * This guarantees that after the POSIX timer cleanup in posixtimer_exit() no + * POSIX CPU timers are queued on the task or in case of a group exit on the + * process. + */ +static inline bool task_can_enqueue_timer(struct task_struct *tsk, enum pid_type type) +{ + if (likely(!(tsk->flags & PF_EXITING))) + return true; + + return type == PIDTYPE_TGID && !(tsk->signal->flags & SIGNAL_GROUP_EXIT); } /* @@ -663,7 +728,13 @@ static void arm_timer(struct k_itimer *timer, struct task_struct *p) struct cpu_timer *ctmr = &timer->it.cpu; u64 newexp = cpu_timer_getexpires(ctmr); + lockdep_assert_held(&p->sighand->siglock); + timer->it_status = POSIX_TIMER_ARMED; + + if (unlikely(!task_can_enqueue_timer(p, clock_pid_type(timer->it_clock)))) + return; + if (!cpu_timer_enqueue(&base->tqhead, ctmr)) return; @@ -1201,6 +1272,20 @@ static void posix_cpu_timers_work(struct callback_head *work) mutex_unlock(&cw->mutex); } +static inline void posix_cpu_timers_exit_work(void) +{ + /* Canceling the work is only valid for exit() but not for exec() */ + if (!(current->flags & PF_EXITING)) + return; + /* + * current->flags has PF_EXITING set so this can be done lockless and + * with interrupts enabled as PF_EXITING prevents the interrupt from + * scheduling the work. + */ + if (current->posix_cputimers_work.scheduled) + task_work_cancel(current, ¤t->posix_cputimers_work.work); +} + /* * Invoked from the posix-timer core when a cancel operation failed because * the timer is marked firing. The caller holds rcu_read_lock(), which @@ -1331,6 +1416,8 @@ static inline void __run_posix_cpu_timers(struct task_struct *tsk) lockdep_posixtimer_exit(); } +static inline void posix_cpu_timers_exit_work(void) { } + static void posix_cpu_timer_wait_running(struct k_itimer *timr) { cpu_relax(); @@ -1477,7 +1564,7 @@ void run_posix_cpu_timers(void) * posix_cpu_timer_del() may fail to lock_task_sighand(tsk) and * miss timer->it.cpu.firing != 0. */ - if (tsk->exit_state) + if (tsk->flags & PF_EXITING) return; /* diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index 436ba794cc0b..188dbedbffca 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -1077,13 +1077,9 @@ SYSCALL_DEFINE1(timer_delete, timer_t, timer_id) return 0; } -/* - * Invoked from do_exit() when the last thread of a thread group exits. - * At that point no other task can access the timers of the dying - * task anymore. - */ -void exit_itimers(struct task_struct *tsk) +static void posixtimer_delete_timers(void) { + struct task_struct *tsk = current; struct hlist_head timers; struct hlist_node *next; struct k_itimer *timer; @@ -1120,6 +1116,24 @@ void exit_itimers(struct task_struct *tsk) } } +void posixtimer_exit(bool group_dead) +{ + if (group_dead) { + hrtimer_cancel(¤t->signal->real_timer); + posix_cpu_timers_exit_group(); + posixtimer_delete_timers(); + } else { + posix_cpu_timers_exit_task(); + } +} + +void posixtimer_exec(void) +{ + posix_cpu_timers_exit_task(); + posixtimer_delete_timers(); + flush_itimer_signals(); +} + SYSCALL_DEFINE2(clock_settime, const clockid_t, which_clock, const struct __kernel_timespec __user *, tp) { diff --git a/kernel/time/posix-timers.h b/kernel/time/posix-timers.h index 4ea9611dd716..79fd7ea71046 100644 --- a/kernel/time/posix-timers.h +++ b/kernel/time/posix-timers.h @@ -51,3 +51,6 @@ int common_timer_set(struct k_itimer *timr, int flags, struct itimerspec64 *old_setting); void posix_timer_set_common(struct k_itimer *timer, struct itimerspec64 *new_setting); int common_timer_del(struct k_itimer *timer); + +void posix_cpu_timers_exit_task(void); +void posix_cpu_timers_exit_group(void); diff --git a/kernel/time/sleep_timeout.c b/kernel/time/sleep_timeout.c index 3c90574bd904..ad8c415851ae 100644 --- a/kernel/time/sleep_timeout.c +++ b/kernel/time/sleep_timeout.c @@ -212,7 +212,7 @@ int __sched schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta, hrtimer_set_expires_range_ns(&t.timer, *expires, delta); hrtimer_sleeper_start_expires(&t, mode); - if (likely(t.task)) + if (likely(hrtimer_sleeper_task_get(&t))) schedule(); hrtimer_cancel(&t.timer); @@ -220,7 +220,7 @@ int __sched schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta, __set_current_state(TASK_RUNNING); - return !t.task ? 0 : -EINTR; + return !hrtimer_sleeper_task_get(&t) ? 0 : -EINTR; } EXPORT_SYMBOL_GPL(schedule_hrtimeout_range_clock); diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 6c3fea386713..5d53ab18b31a 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -738,14 +738,11 @@ bool tick_nohz_tick_stopped_cpu(int cpu) */ static void tick_nohz_update_jiffies(ktime_t now) { - unsigned long flags; + /* Reached only from irq_enter_rcu(), i.e. hard interrupt entry. */ + lockdep_assert_irqs_disabled(); __this_cpu_write(tick_cpu_sched.idle_waketime, now); - - local_irq_save(flags); tick_do_update_jiffies64(now); - local_irq_restore(flags); - touch_softlockup_watchdog_sched(); } @@ -763,12 +760,18 @@ static ktime_t tick_forward_now(ktime_t expires, ktime_t now) return expires + TICK_NSEC; } +static void tick_nohz_forward_last_tick(struct tick_sched *ts, ktime_t now) +{ + if (now >= ts->last_tick) + ts->last_tick = tick_forward_now(ts->last_tick, now); +} + static void tick_nohz_restart(struct tick_sched *ts, ktime_t now) { - ktime_t expires = ts->last_tick; + ktime_t expires; - if (now >= expires) - expires = tick_forward_now(expires, now); + tick_nohz_forward_last_tick(ts, now); + expires = ts->last_tick; if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES)) { hrtimer_start(&ts->sched_timer, expires, HRTIMER_MODE_ABS_PINNED_HARD); @@ -819,7 +822,7 @@ u64 get_jiffies_update(unsigned long *basej) */ static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu) { - u64 basemono, next_tick, delta, expires; + u64 basemono, next_tick, expires; unsigned long basejiff; int tick_cpu; @@ -859,8 +862,7 @@ static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu) * If the tick is due in the next period, keep it ticking or * force prod the timer. */ - delta = next_tick - basemono; - if (delta <= (u64)TICK_NSEC) { + if (next_tick - basemono <= (u64)TICK_NSEC) { /* * We've not stopped the tick yet, and there's a timer in the * next period, so no point in stopping it either, bail. @@ -876,17 +878,19 @@ static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu) * the sleep time to the timekeeping 'max_deferment' value. * Otherwise we can sleep as long as we want. */ - delta = timekeeping_max_deferment(); tick_cpu = READ_ONCE(tick_do_timer_cpu); if (tick_cpu != cpu && - (tick_cpu != TICK_DO_TIMER_NONE || !tick_sched_flag_test(ts, TS_FLAG_DO_TIMER_LAST))) - delta = KTIME_MAX; - - /* Calculate the next expiry time */ - if (delta < (KTIME_MAX - basemono)) - expires = basemono + delta; - else + (tick_cpu != TICK_DO_TIMER_NONE || !tick_sched_flag_test(ts, TS_FLAG_DO_TIMER_LAST))) { expires = KTIME_MAX; + } else { + expires = timekeeping_max_deferment(); + + /* Calculate the next expiry time */ + if (expires < (KTIME_MAX - basemono)) + expires += basemono; + else + expires = KTIME_MAX; + } ts->timer_expires = min_t(u64, expires, next_tick); @@ -1329,6 +1333,12 @@ unsigned long tick_nohz_get_idle_calls_cpu(int cpu) return ts->idle_calls; } +static void tick_nohz_dyntick_stop(struct tick_sched *ts, ktime_t now) +{ + tick_nohz_forward_last_tick(ts, now); + kcpustat_dyntick_stop(now, ts->last_tick - TICK_NSEC); +} + void tick_nohz_idle_restart_tick(void) { struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); @@ -1341,7 +1351,7 @@ void tick_nohz_idle_restart_tick(void) * no tiny amount of idle time is accounted twice. */ ts->idle_entrytime = ktime_get(); - kcpustat_dyntick_stop(ts->idle_entrytime); + tick_nohz_dyntick_stop(ts, ts->idle_entrytime); tick_nohz_restart_sched_tick(ts, ts->idle_entrytime); } } @@ -1385,7 +1395,7 @@ void tick_nohz_idle_exit(void) if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) { now = ktime_get(); - kcpustat_dyntick_stop(now); + tick_nohz_dyntick_stop(ts, now); tick_nohz_idle_update_tick(ts, now); } diff --git a/kernel/time/time_test.c b/kernel/time/time_test.c index 1b99180da288..8b718767b3ba 100644 --- a/kernel/time/time_test.c +++ b/kernel/time/time_test.c @@ -87,8 +87,24 @@ static void time64_to_tm_test_date_range(struct kunit *test) } } +static void time64_to_tm_test_wide_day_count(struct kunit *test) +{ + /* 2^31 days: the first count that does not fit in a 32-bit long. */ + time64_t timestamp = (1LL << 31) * 86400; + struct tm result; + + time64_to_tm(timestamp, 0, &result); + + KUNIT_EXPECT_EQ(test, result.tm_year, 5879680); + KUNIT_EXPECT_EQ(test, result.tm_mon, 6); + KUNIT_EXPECT_EQ(test, result.tm_mday, 12); + KUNIT_EXPECT_EQ(test, result.tm_yday, 193); + KUNIT_EXPECT_EQ(test, result.tm_wday, 6); +} + static struct kunit_case time_test_cases[] = { KUNIT_CASE_SLOW(time64_to_tm_test_date_range), + KUNIT_CASE(time64_to_tm_test_wide_day_count), {} }; diff --git a/kernel/time/timeconv.c b/kernel/time/timeconv.c index 59b922c826e7..aed3af950fa0 100644 --- a/kernel/time/timeconv.c +++ b/kernel/time/timeconv.c @@ -49,8 +49,9 @@ void time64_to_tm(time64_t totalsecs, int offset, struct tm *result) u32 u32tmp, day_of_century, year_of_century, day_of_year, month, day; u64 u64tmp, udays, century, year; bool is_Jan_or_Feb, is_leap_year; - long days, rem; int remainder; + long rem; + s64 days; days = div_s64_rem(totalsecs, SECS_PER_DAY, &remainder); rem = remainder; @@ -70,7 +71,8 @@ void time64_to_tm(time64_t totalsecs, int offset, struct tm *result) result->tm_sec = rem % 60; /* January 1, 1970 was a Thursday. */ - result->tm_wday = (4 + days) % 7; + div_s64_rem(days + 4, 7, &remainder); + result->tm_wday = remainder; if (result->tm_wday < 0) result->tm_wday += 7; diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index ea2e6e55f37b..d54c4d303db6 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -861,8 +861,10 @@ static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int act * * Write xtime_sec first so that even if the memcpy() tears the store * data integrity is provided for ktime_get_real_seconds(). + * The same goes for ktime_sec and ktime_get_seconds(). */ WRITE_ONCE(tkd->timekeeper.xtime_sec, tk->xtime_sec); + WRITE_ONCE(tkd->timekeeper.ktime_sec, tk->ktime_sec); memcpy(&tkd->timekeeper, tk, sizeof(*tk)); write_seqcount_end(&tkd->seq); } @@ -1169,7 +1171,7 @@ time64_t ktime_get_seconds(void) struct timekeeper *tk = &tk_core.timekeeper; WARN_ON(timekeeping_suspended); - return tk->ktime_sec; + return READ_ONCE(tk->ktime_sec); } EXPORT_SYMBOL_GPL(ktime_get_seconds); diff --git a/kernel/time/timer.c b/kernel/time/timer.c index ae9abf14688e..42afdcb229d8 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -890,7 +890,7 @@ static inline void detach_timer(struct timer_list *timer, bool clear_pending) __hlist_del(entry); if (clear_pending) - entry->pprev = NULL; + WRITE_ONCE(entry->pprev, NULL); entry->next = LIST_POISON2; } diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c index 059d43355e65..f920e73fff51 100644 --- a/kernel/time/timer_migration.c +++ b/kernel/time/timer_migration.c @@ -715,7 +715,7 @@ static void __tmigr_cpu_activate(struct tmigr_cpu *tmc) trace_tmigr_cpu_active(tmc); - tmc->cpuevt.ignore = true; + WRITE_ONCE(tmc->cpuevt.ignore, true); WRITE_ONCE(tmc->wakeup, KTIME_MAX); walk_groups(&tmigr_active_up, &data, tmc); @@ -1258,7 +1258,7 @@ u64 tmigr_cpu_new_timer(u64 nextexp) ret = READ_ONCE(tmc->wakeup); if (nextexp != KTIME_MAX) { if (nextexp != tmc->cpuevt.nextevt.expires || - tmc->cpuevt.ignore) { + READ_ONCE(tmc->cpuevt.ignore)) { ret = tmigr_new_timer(tmc, nextexp); /* * Make sure the reevaluation of timers in idle path @@ -1362,7 +1362,7 @@ static u64 __tmigr_cpu_deactivate(struct tmigr_cpu *tmc, u64 nextexp) * or CPU goes offline. */ if (nextexp != KTIME_MAX) - tmc->cpuevt.ignore = false; + WRITE_ONCE(tmc->cpuevt.ignore, false); walk_groups(&tmigr_inactive_up, &data, tmc); return data.firstexp; diff --git a/kernel/time/vsyscall.c b/kernel/time/vsyscall.c index aa59919b8f2c..658553fd910b 100644 --- a/kernel/time/vsyscall.c +++ b/kernel/time/vsyscall.c @@ -9,12 +9,25 @@ #include <linux/hrtimer.h> #include <linux/timekeeper_internal.h> +#include <linux/vdso_time.h> #include <vdso/datapage.h> #include <vdso/helpers.h> #include <vdso/vsyscall.h> #include "timekeeping_internal.h" +#ifndef __arch_update_vdso_clock +static __always_inline void __arch_update_vdso_clock(struct vdso_clock *vc) +{ +} +#endif /* __arch_update_vdso_clock */ + +#ifndef __arch_sync_vdso_time_data +static __always_inline void __arch_sync_vdso_time_data(struct vdso_time_data *vdata) +{ +} +#endif /* __arch_sync_vdso_time_data */ + static inline void fill_clock_configuration(struct vdso_clock *vc, const struct tk_read_base *base) { vc->cycle_last = base->cycle_last; @@ -41,14 +54,12 @@ static inline void update_vdso_time_data(struct vdso_time_data *vdata, struct ti nsec = tk->tkr_mono.xtime_nsec; nsec += ((u64)tk->wall_to_monotonic.tv_nsec << tk->tkr_mono.shift); - while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) { - nsec -= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift); - vdso_ts->sec++; - } - vdso_ts->nsec = nsec; + vdso_ts->sec += __iter_div64_u64_rem(nsec, (u64)NSEC_PER_SEC << tk->tkr_mono.shift, + &vdso_ts->nsec); /* Copy MONOTONIC time for BOOTTIME */ sec = vdso_ts->sec; + nsec = vdso_ts->nsec; /* Add the boot offset */ sec += tk->monotonic_to_boot.tv_sec; nsec += (u64)tk->monotonic_to_boot.tv_nsec << tk->tkr_mono.shift; @@ -56,12 +67,8 @@ static inline void update_vdso_time_data(struct vdso_time_data *vdata, struct ti /* CLOCK_BOOTTIME */ vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_BOOTTIME]; vdso_ts->sec = sec; - - while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) { - nsec -= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift); - vdso_ts->sec++; - } - vdso_ts->nsec = nsec; + vdso_ts->sec += __iter_div64_u64_rem(nsec, (u64)NSEC_PER_SEC << tk->tkr_mono.shift, + &vdso_ts->nsec); /* CLOCK_MONOTONIC_RAW */ vdso_ts = &vc[CS_RAW].basetime[CLOCK_MONOTONIC_RAW]; @@ -161,11 +168,11 @@ void vdso_time_update_aux(struct timekeeper *tk) vdso_ts->sec = tk->xtime_sec + tk->monotonic_to_aux.tv_sec; - nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift; - nsec += tk->monotonic_to_aux.tv_nsec; - vdso_ts->sec += __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec); - nsec = nsec << tk->tkr_mono.shift; - vdso_ts->nsec = nsec; + nsec = tk->tkr_mono.xtime_nsec; + nsec += (u64)tk->monotonic_to_aux.tv_nsec << tk->tkr_mono.shift; + vdso_ts->sec += __iter_div64_u64_rem(nsec, + (u64)NSEC_PER_SEC << tk->tkr_mono.shift, + &vdso_ts->nsec); } __arch_update_vdso_clock(vc); @@ -177,7 +184,7 @@ void vdso_time_update_aux(struct timekeeper *tk) #endif /** - * vdso_update_begin - Start of a VDSO update section + * vdso_time_update_begin - Start of a VDSO update section * * Allows architecture code to safely update the architecture specific VDSO * data. Disables interrupts, acquires timekeeper lock to serialize against @@ -186,9 +193,9 @@ void vdso_time_update_aux(struct timekeeper *tk) * inconsistent data. * * Returns: Saved interrupt flags which need to be handed in to - * vdso_update_end(). + * vdso_time_update_end(). */ -unsigned long vdso_update_begin(void) +unsigned long vdso_time_update_begin(void) { struct vdso_time_data *vdata = vdso_k_time_data; unsigned long flags = timekeeper_lock_irqsave(); @@ -198,14 +205,14 @@ unsigned long vdso_update_begin(void) } /** - * vdso_update_end - End of a VDSO update section - * @flags: Interrupt flags as returned from vdso_update_begin() + * vdso_time_update_end - End of a VDSO update section + * @flags: Interrupt flags as returned from vdso_time_update_begin() * - * Pairs with vdso_update_begin(). Marks vdso data consistent, invokes data + * Pairs with vdso_time_update_begin(). Marks vdso data consistent, invokes data * synchronization if the architecture requires it, drops timekeeper lock * and restores interrupt flags. */ -void vdso_update_end(unsigned long flags) +void vdso_time_update_end(unsigned long flags) { struct vdso_time_data *vdata = vdso_k_time_data; diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 0ab5916575a9..d8faeb659cfc 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -770,7 +770,7 @@ config FPROBE_EVENTS config PROBE_EVENTS_BTF_ARGS depends on HAVE_FUNCTION_ARG_ACCESS_API depends on FPROBE_EVENTS || KPROBE_EVENTS - depends on DEBUG_INFO_BTF && BPF_SYSCALL + depends on DEBUG_INFO_BTF bool "Support BTF function arguments for probe events" default y help diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile index f934ff586bd4..1f925cd1b9f8 100644 --- a/kernel/trace/Makefile +++ b/kernel/trace/Makefile @@ -119,7 +119,7 @@ obj-$(CONFIG_KGDB_KDB) += trace_kdb.o endif obj-$(CONFIG_DYNAMIC_EVENTS) += trace_dynevent.o obj-$(CONFIG_PROBE_EVENTS) += trace_probe.o -obj-$(CONFIG_PROBE_EVENTS_BTF_ARGS) += trace_btf.o +obj-$(CONFIG_DEBUG_INFO_BTF) += trace_btf.o obj-$(CONFIG_UPROBE_EVENTS) += trace_uprobe.o obj-$(CONFIG_BOOTTIME_TRACING) += trace_boot.o obj-$(CONFIG_FTRACE_RECORD_RECURSION) += trace_recursion_record.o @@ -146,11 +146,12 @@ KASAN_SANITIZE_undefsyms_base.o := y UNDEFINED_ALLOWLIST = __asan __gcov __kasan __kcsan __hwasan __sancov __sanitizer __tsan __ubsan __msan \ __aeabi_unwind_cpp __s390_indirect_jump __x86_indirect_thunk simple_ring_buffer \ - $(shell $(NM) -u $(obj)/undefsyms_base.o 2>/dev/null | awk '{print $$2}') + current_thread_info_reg \ + $(shell $(NM) -u $(obj)/undefsyms_base.o 2>/dev/null | awk '$$2 {print $$2}') quiet_cmd_check_undefined = NM $< cmd_check_undefined = \ - undefsyms=$$($(NM) -u $< | grep -v $(addprefix -e , $(UNDEFINED_ALLOWLIST)) || true); \ + undefsyms=$$($(NM) -u $< | awk '$$2 {print $$2}' | grep -v $(addprefix -e , $(UNDEFINED_ALLOWLIST)) || true); \ if [ -n "$$undefsyms" ]; then \ echo "Unexpected symbols in $<:" >&2; \ echo "$$undefsyms" >&2; \ diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 29260951aa87..dfac5e1b8f40 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -823,6 +823,7 @@ const struct bpf_func_proto bpf_task_pt_regs_proto = { struct send_signal_irq_work { struct irq_work irq_work; + /* The claim: NULL marks the slot free for the next caller. */ struct task_struct *task; u32 sig; enum pid_type type; @@ -842,6 +843,8 @@ static void do_bpf_send_signal(struct irq_work *entry) group_send_sig_info(work->sig, siginfo, work->task, work->type); put_task_struct(work->task); + /* Release once the fields are consumed. */ + smp_store_release(&work->task, NULL); } static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struct *task, u64 value) @@ -885,14 +888,14 @@ static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struc return -EINVAL; work = this_cpu_ptr(&send_signal_work); - if (irq_work_is_busy(&work->irq_work)) + if (cmpxchg(&work->task, NULL, task)) return -EBUSY; /* Add the current task, which is the target of sending signal, * to the irq_work. The current task may change when queued * irq works get executed. */ - work->task = get_task_struct(task); + get_task_struct(task); work->has_siginfo = siginfo == &info; if (work->has_siginfo) copy_siginfo(&work->info, &info); @@ -995,7 +998,7 @@ static const struct bpf_func_proto bpf_d_path_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_d_path_btf_ids[0], - .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, + .arg2_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE, .arg3_type = ARG_MEM_SIZE_OR_ZERO, .allowed = bpf_d_path_allowed, }; @@ -1052,7 +1055,7 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { .func = bpf_snprintf_btf, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM | MEM_WRITE, + .arg1_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE, .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_MEM_SIZE, @@ -1565,7 +1568,7 @@ static const struct bpf_func_proto bpf_read_branch_records_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, + .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_UNINIT | MEM_WRITE, .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -2206,6 +2209,7 @@ int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link * return tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, link); } +/* On success the caller releases *buf with kfree_const() */ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, u32 *fd_type, const char **buf, u64 *probe_offset, u64 *probe_addr, @@ -2215,6 +2219,12 @@ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, struct bpf_prog *prog; int flags, err = 0; + /* + * The list of dynamic events is walked, and a probe which has siblings + * can be deleted, with its strings, while its event is in use. + */ + guard(mutex)(&event_mutex); + prog = event->prog; if (!prog) return -ENOENT; @@ -2255,6 +2265,12 @@ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, #endif } + if (!err && *buf) { + *buf = kstrdup_const(*buf, GFP_KERNEL); + if (!*buf) + err = -ENOMEM; + } + return err; } @@ -2388,7 +2404,7 @@ static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 goto error; for (p = buf, i = 0; i < cnt; i++) { - if (__get_user(usymbol, usyms + i)) { + if (get_user(usymbol, usyms + i)) { err = -EFAULT; goto error; } @@ -3295,8 +3311,8 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr /* * All uoffsets/uref_ctr_offsets/ucookies arrays have the same value - * size, we need to check their address range is safe for __get_user - * calls. + * size. Check their address ranges up front, so that a bad pointer + * fails before anything is looked up or allocated. */ size = sizeof(*uoffsets) * cnt; if (!access_ok(uoffsets, size) || !access_ok(uref_ctr_offsets, size) || @@ -3331,7 +3347,7 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr goto error_free; for (i = 0; i < cnt; i++) { - if (__get_user(uprobes[i].offset, uoffsets + i)) { + if (get_user(uprobes[i].offset, uoffsets + i)) { err = -EFAULT; goto error_free; } @@ -3339,11 +3355,11 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr err = -EINVAL; goto error_free; } - if (uref_ctr_offsets && __get_user(uprobes[i].ref_ctr_offset, uref_ctr_offsets + i)) { + if (uref_ctr_offsets && get_user(uprobes[i].ref_ctr_offset, uref_ctr_offsets + i)) { err = -EFAULT; goto error_free; } - if (ucookies && __get_user(uprobes[i].cookie, ucookies + i)) { + if (ucookies && get_user(uprobes[i].cookie, ucookies + i)) { err = -EFAULT; goto error_free; } diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index 1e9b00997ff2..da286619c5d8 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -47,6 +47,10 @@ static struct rhltable fprobe_ip_table; static DEFINE_MUTEX(fprobe_mutex); static struct fgraph_ops fprobe_graph_ops; +DEFINE_LOCK_GUARD_0(rcu_sched_notrace, + rcu_read_lock_sched_notrace(), + rcu_read_unlock_sched_notrace()) + static u32 fprobe_node_hashfn(const void *data, u32 len, u32 seed) { return hash_ptr(*(unsigned long **)data, 32); @@ -171,6 +175,11 @@ static inline bool write_fprobe_header(unsigned long *stack, static inline void read_fprobe_header(unsigned long *stack, struct fprobe **fp, unsigned int *size_words) { + if (!*stack) { + *fp = NULL; + *size_words = 0; + return; + } *fp = arch_decode_fprobe_header_fp(*stack); *size_words = arch_decode_fprobe_header_size(*stack); } @@ -203,6 +212,12 @@ static inline void read_fprobe_header(unsigned long *stack, { struct __fprobe_header *fph = (struct __fprobe_header *)stack; + if (!*stack) { + *fp = NULL; + *size_words = 0; + return; + } + *fp = fph->fp; *size_words = fph->size_words; } @@ -318,16 +333,14 @@ static void fprobe_ftrace_entry(unsigned long ip, unsigned long parent_ip, struct fprobe *fp; int bit; + if (!rcu_is_watching()) + return; + bit = ftrace_test_recursion_trylock(ip, parent_ip); if (bit < 0) return; - /* - * ftrace_test_recursion_trylock() disables preemption, but - * rhltable_lookup() checks whether rcu_read_lcok is held. - * So we take rcu_read_lock() here. - */ - rcu_read_lock(); + guard(rcu_sched_notrace)(); head = rhltable_lookup(&fprobe_ip_table, &ip, fprobe_rht_params); rhl_for_each_entry_rcu(node, pos, head, hlist) { @@ -342,7 +355,6 @@ static void fprobe_ftrace_entry(unsigned long ip, unsigned long parent_ip, else __fprobe_handler(ip, parent_ip, fp, fregs, NULL); } - rcu_read_unlock(); ftrace_test_recursion_unlock(bit); } NOKPROBE_SYMBOL(fprobe_ftrace_entry); @@ -556,10 +568,13 @@ static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops struct fprobe *fp; int used, ret; + if (!rcu_is_watching()) + return 0; + if (WARN_ON_ONCE(!fregs)) return 0; - guard(rcu)(); + guard(rcu_sched_notrace)(); head = rhltable_lookup(&fprobe_ip_table, &func, fprobe_rht_params); reserved_words = 0; rhl_for_each_entry_rcu(node, pos, head, hlist) { @@ -635,6 +650,10 @@ static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops } } + /* Terminate the list, fgraph_reserve_data() does not clear it. */ + if (used && used < reserved_words) + fgraph_data[used] = 0; + /* If any exit_handler is set, data must be used. */ return used != 0; } @@ -650,13 +669,16 @@ static void fprobe_return(struct ftrace_graph_ret *trace, int size, curr; int size_words; + if (!rcu_is_watching()) + return; + fgraph_data = (unsigned long *)fgraph_retrieve_data(gops->idx, &size); if (WARN_ON_ONCE(!fgraph_data)) return; size_words = SIZE_IN_LONG(size); ret_ip = ftrace_regs_get_instruction_pointer(fregs); - preempt_disable_notrace(); + guard(rcu_sched_notrace)(); curr = 0; while (size_words > curr) { @@ -672,7 +694,6 @@ static void fprobe_return(struct ftrace_graph_ret *trace, } curr += size; } - preempt_enable_notrace(); } NOKPROBE_SYMBOL(fprobe_return); diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 673a54fdf392..c7d74b33d36d 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -428,6 +428,7 @@ struct ftrace_profile { #ifdef CONFIG_FUNCTION_GRAPH_TRACER unsigned long long time; unsigned long long time_squared; + unsigned long long time_max; #endif }; @@ -530,9 +531,9 @@ static int function_stat_headers(struct seq_file *m) { #ifdef CONFIG_FUNCTION_GRAPH_TRACER seq_puts(m, " Function " - "Hit Time Avg s^2\n" + "Hit Time Avg s^2 max\n" " -------- " - "--- ---- --- ---\n"); + "--- ---- --- --- ---\n"); #else seq_puts(m, " Function Hit\n" " -------- ---\n"); @@ -613,6 +614,8 @@ static int function_stat_show(struct seq_file *m, void *v) trace_print_graph_duration(avg, &s); trace_seq_puts(&s, " "); trace_print_graph_duration(stddev, &s); + trace_seq_puts(&s, " "); + trace_print_graph_duration(rec->time_max, &s); trace_print_seq(m, &s); #endif seq_putc(m, '\n'); @@ -920,6 +923,8 @@ static void profile_graph_return(struct ftrace_graph_ret *trace, if (rec) { rec->time += calltime; rec->time_squared += calltime * calltime; + if (calltime > rec->time_max) + rec->time_max = calltime; } } @@ -4426,9 +4431,9 @@ subsys_initcall(ftrace_check_for_weak_functions); static int print_rec(struct seq_file *m, unsigned long ip) { - unsigned long offset; + unsigned long offset = 0; char str[KSYM_SYMBOL_LEN]; - char *modname; + char *modname = NULL; const char *ret; ret = kallsyms_lookup(ip, NULL, &offset, &modname, str); @@ -7702,6 +7707,11 @@ static int ftrace_process_locs(struct module *mod, } addr = ftrace_call_adjust(addr); + /* The architecture may return 0 to reject a patch site */ + if (!addr) { + skipped++; + continue; + } end_offset = (pg->index+1) * sizeof(pg->records[0]); if (end_offset > PAGE_SIZE << pg->order) { @@ -7715,6 +7725,20 @@ static int ftrace_process_locs(struct module *mod, rec->ip = addr; } + /* + * ftrace_release_mod() finds a module's page groups by their first + * record. If every entry was skipped there is none, so unlink the + * new page groups and free them now. + */ + if (mod && !start_pg->index) { + ftrace_pages->next = NULL; + mutex_unlock(&ftrace_lock); + /* Need to synchronize with ftrace_location_range() */ + synchronize_rcu(); + ftrace_free_pages(start_pg); + return 0; + } + if (pg->next) { pg_unuse = pg->next; pg->next = NULL; diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 04bb94c29f58..b52f51db9045 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -642,6 +642,11 @@ static __always_inline unsigned int rb_subbuf_max_data_size(struct trace_buffer return rb_subbuf_capacity(buffer) - RB_EVNT_HDR_SIZE - sizeof(event->array[0]); } +static unsigned long rb_calculate_nr_pages(struct trace_buffer *buffer, unsigned long size) +{ + return max(DIV_ROUND_UP(size, rb_subbuf_capacity(buffer)), RING_BUFFER_MIN_NR_PAGES); +} + /** * rb_subbuf_start - Get the start address of a subbuffer * @buffer: A trace buffer @@ -2380,6 +2385,9 @@ int ring_buffer_meta_seq_init(struct file *file, struct trace_buffer *buffer, in struct seq_file *m; int ret; + if (!cpumask_test_cpu(cpu, buffer->cpumask)) + return -ENODEV; + ret = seq_open(file, &rb_meta_seq_ops); if (ret) return ret; @@ -2440,22 +2448,33 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned long nr_pages, struct list_head *pages) { struct trace_buffer *buffer = cpu_buffer->buffer; + unsigned int subbuf_order = buffer->subbuf_order; struct ring_buffer_cpu_meta *meta = NULL; struct buffer_page *bpage, *tmp; - bool user_thread = current->mm != NULL; struct ring_buffer_desc *desc = NULL; + bool user_thread = false; long i; - /* - * Check if the available memory is there first. - * Note, si_mem_available() only gives us a rough estimate of available - * memory. It may not be accurate. But we don't care, we just want - * to prevent doing any allocation when it is obvious that it is - * not going to succeed. - */ - i = si_mem_available(); - if (i < nr_pages) - return -ENOMEM; + if (buffer->range_addr_start) { + meta = rb_range_meta(buffer, nr_pages, cpu_buffer->cpu); + } else if (buffer->remote) { + desc = ring_buffer_desc(buffer->remote->desc, cpu_buffer->cpu); + if (!desc || WARN_ON(desc->nr_page_va != (nr_pages + 1))) + return -EINVAL; + } else { + /* + * Check if the available memory is there first. + * Note, si_mem_available() only gives us a rough estimate of available + * memory. It may not be accurate. But we don't care, we just want + * to prevent doing any allocation when it is obvious that it is + * not going to succeed. + */ + i = si_mem_available(); + if (i < (nr_pages << subbuf_order)) + return -ENOMEM; + + user_thread = current->mm != NULL; + } /* * If a user thread allocates too much, and si_mem_available() @@ -2469,15 +2488,6 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, if (user_thread) set_current_oom_origin(); - if (buffer->range_addr_start) - meta = rb_range_meta(buffer, nr_pages, cpu_buffer->cpu); - - if (buffer->remote) { - desc = ring_buffer_desc(buffer->remote->desc, cpu_buffer->cpu); - if (!desc || WARN_ON(desc->nr_page_va != (nr_pages + 1))) - return -EINVAL; - } - for (i = 0; i < nr_pages; i++) { bpage = alloc_cpu_page(cpu_buffer->cpu); @@ -2513,12 +2523,11 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, bpage->id = i + 1; cpu_buffer->subbuf_ids[i + 1] = bpage; } else { - bpage->page = alloc_cpu_data(cpu_buffer->cpu, - cpu_buffer->buffer->subbuf_order); + bpage->page = alloc_cpu_data(cpu_buffer->cpu, subbuf_order); if (!bpage->page) goto free_pages; } - bpage->order = cpu_buffer->buffer->subbuf_order; + bpage->order = subbuf_order; if (user_thread && fatal_signal_pending(current)) goto free_pages; @@ -2864,7 +2873,7 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, goto fail_free_buffers; /* Need at least two pages plus the reader page */ - if (nr_pages < 3) + if (nr_pages < RING_BUFFER_MIN_NR_PAGES + 1) goto fail_free_buffers; again: @@ -2876,7 +2885,7 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, ptr += subbuf_size * nr_pages; } if (ptr > end) { - if (nr_pages <= 3) + if (nr_pages <= RING_BUFFER_MIN_NR_PAGES + 1) goto fail_free_buffers; nr_pages--; goto again; @@ -2899,13 +2908,10 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, if (nr_pages > rb_static_max_pages()) goto fail_free_buffers; - if (nr_pages < 2) + if (nr_pages < RING_BUFFER_MIN_NR_PAGES) goto fail_free_buffers; } else { - /* need at least two pages */ - nr_pages = DIV_ROUND_UP(size, rb_subbuf_capacity(buffer)); - if (nr_pages < 2) - nr_pages = 2; + nr_pages = rb_calculate_nr_pages(buffer, size); } cpumask_set_cpu(cpu, buffer->cpumask); @@ -3280,6 +3286,21 @@ static void update_pages_handler(struct work_struct *work) complete(&cpu_buffer->update_done); } +static bool rb_resize_disabled(struct trace_buffer *buffer, int cpu) +{ + lockdep_assert_held(&buffer->mutex); + + if (cpu != RING_BUFFER_ALL_CPUS) + return atomic_read(&buffer->buffers[cpu]->resize_disabled); + + for_each_buffer_cpu(buffer, cpu) { + if (atomic_read(&buffer->buffers[cpu]->resize_disabled)) + return true; + } + + return false; +} + /** * ring_buffer_resize - resize the ring buffer * @buffer: the buffer to resize. @@ -3318,26 +3339,19 @@ int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, mutex_lock(&buffer->mutex); atomic_inc(&buffer->resizing); - nr_pages = DIV_ROUND_UP(size, rb_subbuf_capacity(buffer)); + nr_pages = rb_calculate_nr_pages(buffer, size); - /* we need a minimum of two pages */ - if (nr_pages < 2) - nr_pages = 2; + /* + * Don't succeed if resizing is disabled, as a reader might be + * manipulating the ring buffer and is expecting a sane state while + * this is true. + */ + if (rb_resize_disabled(buffer, cpu_id)) { + err = -EBUSY; + goto out_err_unlock; + } if (cpu_id == RING_BUFFER_ALL_CPUS) { - /* - * Don't succeed if resizing is disabled, as a reader might be - * manipulating the ring buffer and is expecting a sane state while - * this is true. - */ - for_each_buffer_cpu(buffer, cpu) { - cpu_buffer = buffer->buffers[cpu]; - if (atomic_read(&cpu_buffer->resize_disabled)) { - err = -EBUSY; - goto out_err_unlock; - } - } - /* calculate the pages to update */ for_each_buffer_cpu(buffer, cpu) { cpu_buffer = buffer->buffers[cpu]; @@ -3409,16 +3423,6 @@ int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, if (nr_pages == cpu_buffer->nr_pages) goto out; - /* - * Don't succeed if resizing is disabled, as a reader might be - * manipulating the ring buffer and is expecting a sane state while - * this is true. - */ - if (atomic_read(&cpu_buffer->resize_disabled)) { - err = -EBUSY; - goto out_err_unlock; - } - cpu_buffer->nr_pages_to_update = nr_pages - cpu_buffer->nr_pages; @@ -3627,10 +3631,8 @@ static void rb_inc_iter(struct ring_buffer_iter *iter) /* Return the index into the sub-buffers for a given sub-buffer */ static int rb_meta_subbuf_idx(struct ring_buffer_cpu_meta *meta, void *subbuf) { - void *subbuf_array; + void *subbuf_array = rb_subbufs_from_meta(meta); - subbuf_array = (void *)meta + sizeof(int) * meta->nr_subbufs; - subbuf_array = (void *)ALIGN((unsigned long)subbuf_array, meta->subbuf_size); return (subbuf - subbuf_array) / meta->subbuf_size; } @@ -7464,6 +7466,8 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) if (psize > RB_WRITE_MASK + 1) return -EINVAL; + guard(cpus_read_lock)(); + /* prevent another thread from changing buffer sizes */ guard(mutex)(&buffer->mutex); @@ -7473,13 +7477,9 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) old_capacity = rb_subbuf_capacity(buffer); - /* The mmap fast path reads subbuf_order without buffer->mutex. */ - for_each_buffer_cpu(buffer, cpu) { - if (!cpumask_test_cpu(cpu, buffer->cpumask)) - continue; - if (atomic_read(&buffer->buffers[cpu]->resize_disabled)) - return -EBUSY; - } + /* Check it is resizable before we touch subbuf_order */ + if (rb_resize_disabled(buffer, RING_BUFFER_ALL_CPUS)) + return -EBUSY; atomic_inc(&buffer->record_disabled); @@ -7490,24 +7490,11 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) /* Make sure all new buffers are allocated, before deleting the old ones */ for_each_buffer_cpu(buffer, cpu) { - - if (!cpumask_test_cpu(cpu, buffer->cpumask)) - continue; - cpu_buffer = buffer->buffers[cpu]; - if (atomic_read(&cpu_buffer->resize_disabled)) { - err = -EBUSY; - goto error; - } - /* Update the number of pages to match the new size */ - nr_pages = old_capacity * buffer->buffers[cpu]->nr_pages; - nr_pages = DIV_ROUND_UP(nr_pages, rb_subbuf_capacity(buffer)); - - /* we need a minimum of two pages */ - if (nr_pages < 2) - nr_pages = 2; + nr_pages = rb_calculate_nr_pages(buffer, + old_capacity * buffer->buffers[cpu]->nr_pages); cpu_buffer->nr_pages_to_update = nr_pages; @@ -7529,9 +7516,6 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) struct list_head old_pages; unsigned long flags; - if (!cpumask_test_cpu(cpu, buffer->cpumask)) - continue; - cpu_buffer = buffer->buffers[cpu]; raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); @@ -7997,12 +7981,8 @@ consume: goto out; } - /* Did the reader catch up with the writer? */ - if (cpu_buffer->reader_page == cpu_buffer->commit_page) - goto out; - reader = rb_get_reader_page(cpu_buffer); - if (WARN_ON(!reader)) + if (!reader) goto out; /* Check if any events were dropped */ @@ -8102,7 +8082,7 @@ int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node) } /* allocate minimum pages, user can later expand it */ if (!nr_pages_same) - nr_pages = 2; + nr_pages = RING_BUFFER_MIN_NR_PAGES; buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, nr_pages, cpu); if (!buffer->buffers[cpu]) { diff --git a/kernel/trace/rv/Kconfig b/kernel/trace/rv/Kconfig index efa930f94ea4..9bfd429ffdea 100644 --- a/kernel/trace/rv/Kconfig +++ b/kernel/trace/rv/Kconfig @@ -113,6 +113,18 @@ config RV_REACT_PANIC Enables the panic reactor. The panic reactor emits a printk() message if an exception is found and panic()s the system. +config RV_REACTORS_KUNIT + tristate "KUnit tests for RV reactors" if !KUNIT_ALL_TESTS + depends on KUNIT + depends on RV_REACTORS + default KUNIT_ALL_TESTS + help + Enable KUnit tests for RV reactor registration and dispatch. + These tests verify the register/unregister lifecycle, duplicate + rejection, and that rv_react() correctly invokes callbacks. + + If unsure, say N. + config RV_MONITORS_KUNIT_TEST tristate "KUnit tests for RV monitors" if !KUNIT_ALL_TESTS depends on KUNIT && RV && RV_REACTORS diff --git a/kernel/trace/rv/Makefile b/kernel/trace/rv/Makefile index cdbf68c84f5a..c895d81dfdad 100644 --- a/kernel/trace/rv/Makefile +++ b/kernel/trace/rv/Makefile @@ -25,4 +25,5 @@ obj-$(CONFIG_RV_MON_WAKEUP) += monitors/wakeup/wakeup.o obj-$(CONFIG_RV_REACTORS) += rv_reactors.o obj-$(CONFIG_RV_REACT_PRINTK) += reactor_printk.o obj-$(CONFIG_RV_REACT_PANIC) += reactor_panic.o +obj-$(CONFIG_RV_REACTORS_KUNIT) += rv_reactors_kunit.o obj-$(CONFIG_RV_MONITORS_KUNIT_TEST) += rv_monitors_test.o diff --git a/kernel/trace/rv/monitors/wakeup/wakeup.c b/kernel/trace/rv/monitors/wakeup/wakeup.c index 01b47416f24e..2e803e79181e 100644 --- a/kernel/trace/rv/monitors/wakeup/wakeup.c +++ b/kernel/trace/rv/monitors/wakeup/wakeup.c @@ -128,7 +128,7 @@ static void disable_wakeup(void) ltl_monitor_destroy(); } -static struct rv_monitor rv_wakeup = { +static struct rv_monitor rv_this = { .name = "wakeup", .description = "Monitor that real-time tasks are not woken by lower-priority tasks", .enable = enable_wakeup, @@ -137,12 +137,12 @@ static struct rv_monitor rv_wakeup = { static int __init register_wakeup(void) { - return rv_register_monitor(&rv_wakeup, &rv_rtapp); + return rv_register_monitor(&rv_this, &rv_rtapp); } static void __exit unregister_wakeup(void) { - rv_unregister_monitor(&rv_wakeup); + rv_unregister_monitor(&rv_this); } module_init(register_wakeup); diff --git a/kernel/trace/rv/reactor_panic.c b/kernel/trace/rv/reactor_panic.c index 76537b8a4343..db7116ceafff 100644 --- a/kernel/trace/rv/reactor_panic.c +++ b/kernel/trace/rv/reactor_panic.c @@ -26,8 +26,7 @@ static struct rv_reactor rv_panic = { static int __init register_react_panic(void) { - rv_register_reactor(&rv_panic); - return 0; + return rv_register_reactor(&rv_panic); } static void __exit unregister_react_panic(void) diff --git a/kernel/trace/rv/reactor_printk.c b/kernel/trace/rv/reactor_printk.c index 48c934e315b3..002a10f6aa7b 100644 --- a/kernel/trace/rv/reactor_printk.c +++ b/kernel/trace/rv/reactor_printk.c @@ -25,8 +25,7 @@ static struct rv_reactor rv_printk = { static int __init register_react_printk(void) { - rv_register_reactor(&rv_printk); - return 0; + return rv_register_reactor(&rv_printk); } static void __exit unregister_react_printk(void) diff --git a/kernel/trace/rv/rv_reactors.c b/kernel/trace/rv/rv_reactors.c index 2f5fc8d18dea..8e57a95d446b 100644 --- a/kernel/trace/rv/rv_reactors.c +++ b/kernel/trace/rv/rv_reactors.c @@ -314,6 +314,7 @@ int rv_register_reactor(struct rv_reactor *reactor) guard(mutex)(&rv_interface_lock); return __rv_register_reactor(reactor); } +EXPORT_SYMBOL_GPL(rv_register_reactor); /** * rv_unregister_reactor - unregister a rv reactor. @@ -327,6 +328,7 @@ int rv_unregister_reactor(struct rv_reactor *reactor) list_del(&reactor->list); return 0; } +EXPORT_SYMBOL_GPL(rv_unregister_reactor); /* * reacting_on interface. @@ -465,7 +467,15 @@ int init_rv_reactors(struct dentry *root_dir) void rv_react(struct rv_monitor *monitor, const char *msg, ...) { - static DEFINE_WAIT_OVERRIDE_MAP(rv_react_map, LD_WAIT_FREE); + /* + * Reactors must not explicitly take locks, so they should be + * LD_WAIT_FREE. However, reactor callbacks can run with preemption + * enabled, meaning the preempting code (e.g. the scheduler taking + * rq->__lock at LD_WAIT_SPIN) may violate that constraint. Use + * LD_WAIT_SPIN to avoid false-positive lockdep reports. + * But you should still NOT be using locks in reactors. + */ + static DEFINE_WAIT_OVERRIDE_MAP(rv_react_map, LD_WAIT_SPIN); va_list args; if (!rv_reacting_on() || !monitor->react) diff --git a/kernel/trace/rv/rv_reactors_kunit.c b/kernel/trace/rv/rv_reactors_kunit.c new file mode 100644 index 000000000000..3850c6dda288 --- /dev/null +++ b/kernel/trace/rv/rv_reactors_kunit.c @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for RV reactor registration and dispatch. + * + * The dispatch tests rely on reacting_on being enabled, since rv_react() + * returns early when it is off. It is on by default when the suites run + * built-in; as a module, re-enable it if disabled via + * /sys/kernel/tracing/rv/reacting_on. + */ + +#include <kunit/test.h> +#include <linux/build_bug.h> +#include <linux/rv.h> +#include <linux/delay.h> +#include "rv.h" + +static struct rv_reactor test_reactor = { + .name = "kunit_test_reactor", + .description = "KUnit test reactor", +}; + +static void reactor_teardown(void *arg) +{ + rv_unregister_reactor(&test_reactor); +} + +static void register_test_reactor(struct kunit *test) +{ + KUNIT_ASSERT_EQ(test, rv_register_reactor(&test_reactor), 0); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, reactor_teardown, NULL), 0); +} + +static void test_double_register(struct kunit *test) +{ + register_test_reactor(test); + KUNIT_EXPECT_EQ(test, rv_register_reactor(&test_reactor), -EINVAL); +} + +static const char long_reactor_name[] = "kunit_reactor_name_too_long_xxx_"; +static_assert(sizeof(long_reactor_name) - 1 >= MAX_RV_REACTOR_NAME_SIZE, + "long_reactor_name must be at least MAX_RV_REACTOR_NAME_SIZE chars"); + +static void test_name_too_long(struct kunit *test) +{ + static struct rv_reactor long_reactor = { + .name = long_reactor_name, + }; + + KUNIT_EXPECT_EQ(test, rv_register_reactor(&long_reactor), -EINVAL); +} + +static struct kunit_case rv_reactor_registration_cases[] = { + KUNIT_CASE(test_double_register), + KUNIT_CASE(test_name_too_long), + {} +}; + +static struct kunit_suite rv_reactor_registration_suite = { + .name = "rv_reactor_registration", + .test_cases = rv_reactor_registration_cases, +}; + +static int react_call_count; + +__printf(1, 0) static void mock_react(const char *msg, va_list args) +{ + react_call_count++; + /* Busy-wait so a timer interrupt fires inside rv_react(). */ + mdelay(20); +} + +static void test_react_no_callback(struct kunit *test) +{ + struct rv_monitor monitor = { + .name = "kunit_null_react", + }; + + react_call_count = 0; + rv_react(&monitor, "no callback"); + + KUNIT_EXPECT_EQ(test, react_call_count, 0); +} + +static void test_react_callback_invoked(struct kunit *test) +{ + struct rv_monitor monitor = { + .name = "kunit_dispatch_monitor", + .react = mock_react, + }; + + react_call_count = 0; + rv_react(&monitor, "callback invocation test"); + KUNIT_EXPECT_EQ(test, react_call_count, 1); +} + +static struct kunit_case rv_react_dispatch_cases[] = { + KUNIT_CASE(test_react_no_callback), + KUNIT_CASE(test_react_callback_invoked), + {} +}; + +static struct kunit_suite rv_react_dispatch_suite = { + .name = "rv_react_dispatch", + .test_cases = rv_react_dispatch_cases, +}; + +kunit_test_suites(&rv_reactor_registration_suite, &rv_react_dispatch_suite); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("KUnit tests for RV reactor registration and dispatch"); diff --git a/kernel/trace/simple_ring_buffer.c b/kernel/trace/simple_ring_buffer.c index 49913bb0057a..abfaf9bee12e 100644 --- a/kernel/trace/simple_ring_buffer.c +++ b/kernel/trace/simple_ring_buffer.c @@ -383,8 +383,8 @@ int simple_ring_buffer_init_mm(struct simple_rb_per_cpu *cpu_buffer, void *page; int i; - /* At least 1 reader page and two pages in the ring-buffer */ - if (desc->nr_page_va < 3) + /* At least RING_BUFFER_MIN_NR_PAGES and 1 reader page */ + if (desc->nr_page_va < RING_BUFFER_MIN_NR_PAGES + 1) return -EINVAL; memset(cpu_buffer, 0, sizeof(*cpu_buffer)); diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index e4a490d3d08c..97a058623f94 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -100,6 +100,12 @@ struct tracers { }; /* + * Tracers that do not define their own flags use these empty flags, + * as tr->current_trace_flags must never be NULL. + */ +static struct tracer_flags no_tracer_flags; + +/* * To prevent the comm cache from being overwritten when no * tracing is active, only save the comm when a trace event * occurred. @@ -1318,7 +1324,12 @@ static int run_tracer_selftest(struct tracer *type) tracing_reset_online_cpus(&tr->array_buffer); tr->current_trace = type; - tr->current_trace_flags = type->flags ? : type->default_flags; + if (type->flags) + tr->current_trace_flags = type->flags; + else if (type->default_flags) + tr->current_trace_flags = type->default_flags; + else + tr->current_trace_flags = &no_tracer_flags; #ifdef CONFIG_TRACER_MAX_TRACE if (tracer_uses_snapshot(type)) { @@ -5089,7 +5100,12 @@ int tracing_set_tracer(struct trace_array *tr, const char *buf) return ret; } - tr->current_trace_flags = t->flags ? : t->tracer->flags; + if (t->flags) + tr->current_trace_flags = t->flags; + else if (t->tracer->flags) + tr->current_trace_flags = t->tracer->flags; + else + tr->current_trace_flags = &no_tracer_flags; if (trace->init) { ret = tracer_init(trace, tr); @@ -5881,7 +5897,7 @@ static int tracing_buffer_meta_open(struct inode *inode, struct file *filp) ret = ring_buffer_meta_seq_init(filp, tr->array_buffer.buffer, cpu); if (ret < 0) - __trace_array_put(tr); + trace_array_put(tr); return ret; } @@ -9287,6 +9303,12 @@ static struct notifier_block trace_module_nb = { static __init void tracer_init_tracefs_work_func(struct work_struct *work) { + /* + * Do not attempt to initialize tracefs if tracing was disabled or + * failed to allocate + */ + if (tracing_disabled) + return; event_trace_init(); diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 3749485a7d85..9044f03fe2d9 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -1006,9 +1006,6 @@ struct ftrace_hash { struct rcu_head rcu; }; -struct ftrace_func_entry * -ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip); - static __always_inline bool ftrace_hash_empty(struct ftrace_hash *hash) { return !hash || !(hash->count || (hash->flags & FTRACE_HASH_FL_MOD)); diff --git a/kernel/trace/trace_btf.c b/kernel/trace/trace_btf.c index ee7a04886bf6..f0e0c04d58d9 100644 --- a/kernel/trace/trace_btf.c +++ b/kernel/trace/trace_btf.c @@ -2,6 +2,7 @@ #include <linux/btf.h> #include <linux/kernel.h> #include <linux/slab.h> +#include <linux/kallsyms.h> #include "trace_btf.h" @@ -15,6 +16,9 @@ const struct btf_type *btf_find_func_proto(const char *func_name, struct btf **b const struct btf_type *t; s32 id; + if (unlikely(in_atomic() || irqs_disabled())) + return NULL; + id = bpf_find_btf_id(func_name, BTF_KIND_FUNC, btf_p); if (id < 0) return NULL; @@ -59,19 +63,33 @@ struct btf_anon_stack { u32 offset; }; -/* - * Find a member of data structure/union by name and return it. - * Return NULL if not found, or ERR_PTR(-EINVAL) if parameter is invalid. - * If the member is a member of an anonymous union/structure, the bit offset - * of that anonymous union/structure is stored into @anon_offset. - * If @member_type is non-NULL, the actual containing structure/union type - * of the found member is stored into @member_type. +/** + * btf_find_struct_member() - Find a member of data structure/union by name + * @btf: Pointer to BTF object. + * @type: Pointer to the struct/union BTF type to search within. + * @member_name: Name of the member to find. + * @anon_offset: Pointer to store bit offset of enclosing anonymous union/struct. + * @container_type: Pointer to store the actual containing struct/union type. + * + * Find a member of data structure/union by name, traversing into anonymous + * unions and structures if necessary. + * + * If the member is inside an anonymous union/structure, the bit offset + * of that anonymous union/structure from @type is stored into @anon_offset. + * If @container_type is non-NULL, the actual struct/union BTF type that + * directly contains the found member is stored into @container_type. + * + * Return: + * * Pointer to &struct btf_member on success. + * * %NULL if the member was not found. + * * %ERR_PTR(-EINVAL) if @type is not a struct/union. + * * %ERR_PTR(-ENOMEM) if memory allocation failed. */ const struct btf_member *btf_find_struct_member(struct btf *btf, const struct btf_type *type, const char *member_name, u32 *anon_offset, - const struct btf_type **member_type) + const struct btf_type **container_type) { struct btf_anon_stack *anon_stack; const struct btf_member *member; @@ -103,8 +121,8 @@ retry: if (name && !strcmp(member_name, name)) { if (anon_offset) *anon_offset = cur_offset; - if (member_type) - *member_type = type; + if (container_type) + *container_type = type; goto out; } } @@ -123,3 +141,85 @@ out: return member; } +void btf_trim_retval(unsigned long func, unsigned long *retval, bool *print_retval, + int *fmt, bool hex) +{ + const struct btf_type *t; + char name[KSYM_NAME_LEN]; + struct btf *btf; + u32 v, msb; + bool signed_type; + int kind; + + if (lookup_symbol_name(func, name)) + return; + + t = btf_find_func_proto(name, &btf); + if (IS_ERR_OR_NULL(t)) + return; + + t = btf_type_skip_modifiers(btf, t->type, NULL); + kind = t ? BTF_INFO_KIND(t->info) : BTF_KIND_UNKN; + switch (kind) { + case BTF_KIND_UNKN: + *print_retval = false; + break; + case BTF_KIND_STRUCT: + case BTF_KIND_UNION: + case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: + if (kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION) { + *fmt = RETVAL_FMT_HEX; + signed_type = false; + } else { + *fmt = RETVAL_FMT_DEC; + signed_type = btf_type_kflag(t); + if (!signed_type) + *fmt |= RETVAL_FMT_UNSIGNED; + } + + if (t->size > sizeof(unsigned long)) { + *fmt |= RETVAL_FMT_TRUNC; + msb = BITS_PER_LONG - 1; + } else { + msb = min_t(u32, BITS_PER_BYTE * t->size - 1, + BITS_PER_LONG - 1); + *retval &= GENMASK(msb, 0); + } + if (signed_type && t->size && !hex) + *retval = sign_extend64(*retval, msb); + break; + case BTF_KIND_INT: + v = *(u32 *)(t + 1); + signed_type = false; + if (BTF_INT_ENCODING(v) == BTF_INT_BOOL) { + *fmt = RETVAL_FMT_BOOL; + msb = 0; + } else { + signed_type = BTF_INT_ENCODING(v) & BTF_INT_SIGNED; + if (signed_type) + *fmt = RETVAL_FMT_DEC; + else + *fmt = RETVAL_FMT_HEX; + + if (t->size > sizeof(unsigned long)) { + *fmt |= RETVAL_FMT_TRUNC; + msb = BITS_PER_LONG - 1; + } else { + msb = min_t(u32, BTF_INT_BITS(v) - 1, + BITS_PER_LONG - 1); + } + } + *retval &= GENMASK(msb, 0); + if (signed_type && !hex) + *retval = sign_extend64(*retval, msb); + break; + default: + *fmt = RETVAL_FMT_HEX; + break; + } + + if (*print_retval) + *fmt |= RETVAL_FMT_BTF; + btf_put(btf); +} diff --git a/kernel/trace/trace_btf.h b/kernel/trace/trace_btf.h index 4bd26bceae23..b3d408e6a7f6 100644 --- a/kernel/trace/trace_btf.h +++ b/kernel/trace/trace_btf.h @@ -1,6 +1,15 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include <linux/btf.h> +enum { + RETVAL_FMT_HEX = BIT(0), + RETVAL_FMT_DEC = BIT(1), + RETVAL_FMT_BOOL = BIT(2), + RETVAL_FMT_TRUNC = BIT(3), + RETVAL_FMT_BTF = BIT(4), + RETVAL_FMT_UNSIGNED = BIT(5), +}; + const struct btf_type *btf_find_func_proto(const char *func_name, struct btf **btf_p); const struct btf_param *btf_get_func_param(const struct btf_type *func_proto, @@ -9,4 +18,14 @@ const struct btf_member *btf_find_struct_member(struct btf *btf, const struct btf_type *type, const char *member_name, u32 *anon_offset, - const struct btf_type **member_type); + const struct btf_type **container_type); + +#ifdef CONFIG_DEBUG_INFO_BTF +void btf_trim_retval(unsigned long func, unsigned long *retval, bool *print_retval, + int *fmt, bool hex); +#else +static inline void btf_trim_retval(unsigned long func, unsigned long *retval, + bool *print_retval, int *fmt, bool hex) +{ +} +#endif diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 30c0ddf90887..6428861102a1 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -4908,7 +4908,7 @@ static __init int event_trace_enable(void) */ __trace_early_add_events(tr); - seq_buf_str(&bootup_event_seq); + seq_buf_terminate(&bootup_event_seq); early_enable_events(tr, bootup_event_buf, false); trace_printk_start_comm(); @@ -4937,7 +4937,7 @@ static __init int event_trace_enable_again(void) if (!tr) return -ENODEV; - seq_buf_str(&bootup_event_seq); + seq_buf_terminate(&bootup_event_seq); early_enable_events(tr, bootup_event_buf, true); return 0; diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 2b46ca536045..127880ebe83f 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -482,18 +482,17 @@ predicate_parse(const char *str, int nr_parens, int nr_preds, int len; int N = 0; int i; - - nr_preds += 2; /* For TRUE and FALSE */ + int nr_preds_alloc = nr_preds + 2; /* For TRUE and FALSE */ op_stack = kmalloc_objs(*op_stack, nr_parens); if (!op_stack) return ERR_PTR(-ENOMEM); - prog_stack = kzalloc_objs(*prog_stack, nr_preds); + prog_stack = kzalloc_objs(*prog_stack, nr_preds_alloc); if (!prog_stack) { parse_error(pe, -ENOMEM, 0); goto out_free; } - inverts = kmalloc_array(nr_preds, sizeof(*inverts), GFP_KERNEL); + inverts = kmalloc_array(nr_preds_alloc, sizeof(*inverts), GFP_KERNEL); if (!inverts) { parse_error(pe, -ENOMEM, 0); goto out_free; @@ -512,7 +511,7 @@ predicate_parse(const char *str, int nr_parens, int nr_preds, switch (*next) { case '(': /* #2 */ - if (top - op_stack > nr_parens) { + if (top - op_stack >= nr_parens - 1) { ret = -EINVAL; goto out_free; } diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 8af97fd4ee2d..5c07fee3a328 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -289,7 +289,7 @@ static u64 hist_field_log2(struct hist_field *hist_field, u64 val = hist_fn_call(operand, elt, buffer, rbe, event); - return (u64) ilog2(roundup_pow_of_two(val)); + return order_base_2(val); } static u64 hist_field_bucket(struct hist_field *hist_field, @@ -873,7 +873,7 @@ static u64 hist_field_timestamp(struct hist_field *hist_field, struct hist_trigger_data *hist_data = hist_field->hist_data; struct trace_array *tr = hist_data->event_file->tr; - u64 ts = ring_buffer_event_time_stamp(buffer, rbe); + u64 ts = tracing_event_time_stamp(buffer, rbe); if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr)) ts = ns2usecs(ts); @@ -2121,7 +2121,8 @@ static int init_var_ref(struct hist_field *ref_field, ref_field->size = var_field->size; ref_field->is_signed = var_field->is_signed; ref_field->flags |= var_field->flags & - (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); + (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS | + HIST_FIELD_FL_STACKTRACE); if (system) { ref_field->system = kstrdup(system, GFP_KERNEL); @@ -2984,8 +2985,7 @@ find_synthetic_field_var(struct hist_trigger_data *target_hist_data, seq_buf_init(&s, synthetic_name, MAX_FILTER_STR_VAL); seq_buf_printf(&s, "synthetic_%s", field_name); - /* Terminate synthetic_name with a NUL. */ - seq_buf_str(&s); + seq_buf_terminate(&s); if (seq_buf_has_overflowed(&s)) { kfree(synthetic_name); @@ -3090,7 +3090,7 @@ create_field_var_hist(struct hist_trigger_data *target_hist_data, key_field = hist_data->fields[i]; if (!first) seq_buf_putc(&s, ','); - seq_buf_puts(&s, key_field->field->name); + expr_field_str(key_field, &s); first = false; } @@ -3102,8 +3102,7 @@ create_field_var_hist(struct hist_trigger_data *target_hist_data, if (saved_filter) seq_buf_printf(&s, " if %s", saved_filter); - /* Terminate cmd with a NUL. */ - seq_buf_str(&s); + seq_buf_terminate(&s); if (seq_buf_has_overflowed(&s)) { kfree(cmd); @@ -3318,6 +3317,7 @@ static struct field_var *create_field_var(struct hist_trigger_data *hist_data, ret = PTR_ERR(var); goto err; } + var->flags |= val->flags & HIST_FIELD_FL_STACKTRACE; field_var = kzalloc_obj(struct field_var); if (!field_var) { @@ -3946,7 +3946,10 @@ static int check_synth_field(struct synth_event *event, && field->is_dynamic) return 0; - if (strstr(hist_field->type, "long[") && field->is_stack) + if (!!(hist_field->flags & HIST_FIELD_FL_STACKTRACE) != field->is_stack) + return -EINVAL; + + if (field->is_stack) return 0; if (strcmp(field->type, hist_field->type) != 0) { @@ -4487,7 +4490,8 @@ static int create_var_field(struct hist_trigger_data *hist_data, if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME) update_var_execname(hist_data->fields[val_idx]); - if (!ret && hist_data->fields[val_idx]->flags & + if (!ret && !(hist_data->fields[val_idx]->flags & HIST_FIELD_FL_ALIAS) && + hist_data->fields[val_idx]->flags & (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++; @@ -5228,7 +5232,8 @@ static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, if (hist_field->flags & HIST_FIELD_FL_VAR) { var_idx = hist_field->var.idx; - if (hist_field->flags & + if (!(hist_field->flags & HIST_FIELD_FL_ALIAS) && + hist_field->flags & (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) { unsigned int str_start, var_str_idx, idx; char *str, *val_str; diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index c879d43a5fbb..acaba990aaaa 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -121,6 +121,7 @@ static ftrace_func_t select_trace_function(u32 flags_val) case TRACE_FUNC_OPT_ARGS: return function_args_trace_call; case TRACE_FUNC_OPT_STACK: + case TRACE_FUNC_OPT_STACK | TRACE_FUNC_OPT_ARGS: return function_stack_trace_call; case TRACE_FUNC_OPT_NO_REPEATS: return function_no_repeats_trace_call; @@ -295,7 +296,9 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip, if (likely(disabled == 1)) { trace_ctx = tracing_gen_ctx_flags(flags); - trace_function(tr, ip, parent_ip, trace_ctx, NULL); + if (!(tr->current_trace_flags->val & TRACE_FUNC_OPT_ARGS)) + fregs = NULL; + trace_function(tr, ip, parent_ip, trace_ctx, fregs); #ifdef CONFIG_UNWINDER_FRAME_POINTER if (ftrace_pids_enabled(op)) skip++; diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index c5befd0c5b9a..78a608315dd3 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -15,6 +15,7 @@ #include "trace.h" #include "trace_output.h" +#include "trace_btf.h" /* When set, irq functions might be ignored */ static int ftrace_graph_skip_irqs; @@ -876,9 +877,10 @@ static void print_graph_retval(struct trace_seq *s, struct ftrace_graph_ent_entr { unsigned long err_code = 0; unsigned long retval = 0; + bool hex_format; bool print_retaddr = false; bool print_retval = false; - bool hex_format = !!(opt_flags & TRACE_GRAPH_PRINT_RETVAL_HEX); + int retval_fmt = 0; #ifdef CONFIG_FUNCTION_GRAPH_RETVAL retval = graph_ret->retval; @@ -889,17 +891,38 @@ static void print_graph_retval(struct trace_seq *s, struct ftrace_graph_ent_entr print_retaddr = !!(opt_flags & TRACE_GRAPH_PRINT_RETADDR); #endif - if (print_retval && retval && !hex_format) { - /* Check if the return value matches the negative format */ - if (IS_ENABLED(CONFIG_64BIT) && (retval & BIT(31)) && - (((u64)retval) >> 32) == 0) { - err_code = sign_extend64(retval, 31); - } else { - err_code = retval; + if (print_retval) { + int fmt = RETVAL_FMT_HEX; + + hex_format = !!(opt_flags & TRACE_GRAPH_PRINT_RETVAL_HEX); + btf_trim_retval((unsigned long)func, &retval, &print_retval, &fmt, + hex_format); + if (print_retval) { + if (hex_format) + retval_fmt = RETVAL_FMT_HEX; + + if (retval && retval_fmt != RETVAL_FMT_HEX && + !(fmt & RETVAL_FMT_BTF)) { + /* Check if the return value matches the negative format */ + if (IS_ENABLED(CONFIG_64BIT) && (retval & BIT(31)) && + (((u64)retval) >> 32) == 0) { + err_code = sign_extend64(retval, 31); + } else { + err_code = retval; + } + + if (!IS_ERR_VALUE(err_code)) + err_code = 0; + } + + if (retval_fmt == RETVAL_FMT_HEX) { + retval_fmt |= (fmt & RETVAL_FMT_TRUNC); + } else { + if (err_code && fmt & RETVAL_FMT_HEX) + fmt = (fmt & ~RETVAL_FMT_HEX) | RETVAL_FMT_DEC; + retval_fmt = fmt; + } } - - if (!IS_ERR_VALUE(err_code)) - err_code = 0; } if (entry) { @@ -926,10 +949,17 @@ static void print_graph_retval(struct trace_seq *s, struct ftrace_graph_ent_entr trace_flags, false); if (print_retval) { - if (hex_format || (err_code == 0)) + if (retval_fmt & RETVAL_FMT_HEX) trace_seq_printf(s, " ret=0x%lx", retval); + else if (retval_fmt & RETVAL_FMT_BOOL) + trace_seq_printf(s, " ret=%s", retval ? "true" : "false"); + else if (retval_fmt & RETVAL_FMT_UNSIGNED) + trace_seq_printf(s, " ret=%lu", retval); else - trace_seq_printf(s, " ret=%ld", err_code); + trace_seq_printf(s, " ret=%ld", err_code ?: retval); + + if (retval_fmt & RETVAL_FMT_TRUNC) + trace_seq_printf(s, "(trunc)"); } if (!entry || print_retval || print_retaddr) diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index 804442b2f7d2..a880a90642d7 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -625,7 +625,7 @@ static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **pty { const struct btf_type *type = *ptype; const struct btf_member *field; - const struct btf_type *mtype; + const struct btf_type *container_type; struct btf *btf = ctx_btf(ctx); char *fieldname = *pfieldname; int bitoffs = 0; @@ -641,7 +641,7 @@ static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **pty anon_offs = 0; field = btf_find_struct_member(btf, type, fieldname, - &anon_offs, &mtype); + &anon_offs, &container_type); if (IS_ERR(field)) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return PTR_ERR(field); @@ -654,7 +654,7 @@ static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **pty bitoffs += anon_offs; /* Accumulate the bit-offsets of the dot-connected fields */ - if (btf_type_kflag(mtype)) { + if (btf_type_kflag(container_type)) { bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset); ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset); } else { @@ -2287,7 +2287,6 @@ const char **traceprobe_expand_meta_args(int argc, const char *argv[], int *new_argc, char *buf, int bufsize, struct traceprobe_parse_context *ctx) { - const struct btf_param *params = NULL; int i, j, n, used, ret, args_idx = -1; const char **new_argv __free(kfree) = NULL; @@ -2305,7 +2304,7 @@ const char **traceprobe_expand_meta_args(int argc, const char *argv[], if (args_idx != -1) { /* $arg* requires BTF info */ trace_probe_log_err(0, NOSUP_BTFARG); - return (const char **)params; + return ERR_PTR(-EOPNOTSUPP); } *new_argc = argc; return NULL; @@ -2316,7 +2315,7 @@ const char **traceprobe_expand_meta_args(int argc, const char *argv[], else *new_argc = argc; - new_argv = kcalloc(*new_argc, sizeof(char *), GFP_KERNEL); + new_argv = kcalloc(*new_argc, sizeof(*new_argv), GFP_KERNEL); if (!new_argv) return ERR_PTR(-ENOMEM); diff --git a/kernel/trace/trace_snapshot.c b/kernel/trace/trace_snapshot.c index 07b43c9863a2..4731ef7325a5 100644 --- a/kernel/trace/trace_snapshot.c +++ b/kernel/trace/trace_snapshot.c @@ -675,9 +675,12 @@ tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt, if (iter->cpu_file != RING_BUFFER_ALL_CPUS) return -EINVAL; #endif - if (tr->allocated_snapshot) + if (tr->allocated_snapshot) { ret = resize_buffer_duplicate_size(&tr->snapshot_buffer, &tr->array_buffer, iter->cpu_file); + if (ret < 0) + return ret; + } ret = tracing_arm_snapshot_locked(tr); if (ret) diff --git a/kernel/trace/undefsyms_base.c b/kernel/trace/undefsyms_base.c index e65baf58e6ff..37310baf706b 100644 --- a/kernel/trace/undefsyms_base.c +++ b/kernel/trace/undefsyms_base.c @@ -9,6 +9,7 @@ #include <linux/atomic.h> #include <linux/string.h> +#include <asm/local.h> #include <asm/page.h> void undefsyms_base(void *p, int n); @@ -18,11 +19,13 @@ static char page[PAGE_SIZE] __aligned(PAGE_SIZE); void undefsyms_base(void *p, int n) { char buffer[256] = { 0 }; - u32 u = 0; + memset((char * volatile)page, 8, PAGE_SIZE); memset((char * volatile)buffer, 8, sizeof(buffer)); memcpy((void * volatile)p, buffer, sizeof(buffer)); cmpxchg((u32 * volatile)&u, 0, 8); + cmpxchg((unsigned long *)p, 0, 8); + local_set((local_t *)p, 8); WARN_ON(n == 0xdeadbeef); } diff --git a/kernel/umh.c b/kernel/umh.c index 6e2c7bb315c6..512dc018a50b 100644 --- a/kernel/umh.c +++ b/kernel/umh.c @@ -8,7 +8,7 @@ #include <linux/binfmts.h> #include <linux/syscalls.h> #include <linux/unistd.h> -#include <linux/kmod.h> +#include <linux/umh.h> #include <linux/slab.h> #include <linux/completion.h> #include <linux/cred.h> diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c index 0bed462e9b2a..50c374ed4687 100644 --- a/kernel/user_namespace.c +++ b/kernel/user_namespace.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only +#include <kunit/visibility.h> #include <linux/export.h> #include <linux/nsproxy.h> #include <linux/slab.h> @@ -162,9 +163,6 @@ int create_user_ns(struct cred *new) ns_tree_add(ns); return 0; fail_keyring: -#ifdef CONFIG_PERSISTENT_KEYRINGS - key_put(ns->persistent_keyring_register); -#endif ns_common_free(ns); fail_free: kmem_cache_free(user_ns_cachep, ns); @@ -278,8 +276,8 @@ static int cmp_map_id(const void *k, const void *e) * map_id_range_down_max - Find idmap via binary search in ordered idmap array. * Can only be called if number of mappings exceeds UID_GID_MAP_MAX_BASE_EXTENTS. */ -static struct uid_gid_extent * -map_id_range_down_max(unsigned extents, struct uid_gid_map *map, u32 id, u32 count) +static const struct uid_gid_extent * +map_id_range_down_max(unsigned extents, const struct uid_gid_map *map, u32 id, u32 count) { struct idmap_key key; @@ -296,8 +294,8 @@ map_id_range_down_max(unsigned extents, struct uid_gid_map *map, u32 id, u32 cou * Can only be called if number of mappings is equal or less than * UID_GID_MAP_MAX_BASE_EXTENTS. */ -static struct uid_gid_extent * -map_id_range_down_base(unsigned extents, struct uid_gid_map *map, u32 id, u32 count) +static const struct uid_gid_extent * +map_id_range_down_base(unsigned extents, const struct uid_gid_map *map, u32 id, u32 count) { unsigned idx; u32 first, last, id2; @@ -315,9 +313,9 @@ map_id_range_down_base(unsigned extents, struct uid_gid_map *map, u32 id, u32 co return NULL; } -static u32 map_id_range_down(struct uid_gid_map *map, u32 id, u32 count) +static u32 map_id_range_down(const struct uid_gid_map *map, u32 id, u32 count) { - struct uid_gid_extent *extent; + const struct uid_gid_extent *extent; unsigned extents = map->nr_extents; smp_rmb(); @@ -335,7 +333,7 @@ static u32 map_id_range_down(struct uid_gid_map *map, u32 id, u32 count) return id; } -u32 map_id_down(struct uid_gid_map *map, u32 id) +u32 map_id_down(const struct uid_gid_map *map, u32 id) { return map_id_range_down(map, id, 1); } @@ -345,8 +343,8 @@ u32 map_id_down(struct uid_gid_map *map, u32 id) * Can only be called if number of mappings is equal or less than * UID_GID_MAP_MAX_BASE_EXTENTS. */ -static struct uid_gid_extent * -map_id_range_up_base(unsigned extents, struct uid_gid_map *map, u32 id, u32 count) +static const struct uid_gid_extent * +map_id_range_up_base(unsigned extents, const struct uid_gid_map *map, u32 id, u32 count) { unsigned idx; u32 first, last, id2; @@ -368,8 +366,8 @@ map_id_range_up_base(unsigned extents, struct uid_gid_map *map, u32 id, u32 coun * map_id_up_max - Find idmap via binary search in ordered idmap array. * Can only be called if number of mappings exceeds UID_GID_MAP_MAX_BASE_EXTENTS. */ -static struct uid_gid_extent * -map_id_range_up_max(unsigned extents, struct uid_gid_map *map, u32 id, u32 count) +static const struct uid_gid_extent * +map_id_range_up_max(unsigned extents, const struct uid_gid_map *map, u32 id, u32 count) { struct idmap_key key; @@ -381,9 +379,9 @@ map_id_range_up_max(unsigned extents, struct uid_gid_map *map, u32 id, u32 count sizeof(struct uid_gid_extent), cmp_map_id); } -u32 map_id_range_up(struct uid_gid_map *map, u32 id, u32 count) +u32 map_id_range_up(const struct uid_gid_map *map, u32 id, u32 count) { - struct uid_gid_extent *extent; + const struct uid_gid_extent *extent; unsigned extents = map->nr_extents; smp_rmb(); @@ -401,7 +399,7 @@ u32 map_id_range_up(struct uid_gid_map *map, u32 id, u32 count) return id; } -u32 map_id_up(struct uid_gid_map *map, u32 id) +u32 map_id_up(const struct uid_gid_map *map, u32 id) { return map_id_range_up(map, id, 1); } @@ -782,11 +780,13 @@ static bool mappings_overlap(struct uid_gid_map *new_map, } /* - * insert_extent - Safely insert a new idmap extent into struct uid_gid_map. + * uid_gid_map_insert_extent - Safely insert a new idmap extent into + * struct uid_gid_map. * Takes care to allocate a 4K block of memory if the number of mappings exceeds * UID_GID_MAP_MAX_BASE_EXTENTS. */ -static int insert_extent(struct uid_gid_map *map, struct uid_gid_extent *extent) +VISIBLE_IF_KUNIT int uid_gid_map_insert_extent(struct uid_gid_map *map, + struct uid_gid_extent *extent) { struct uid_gid_extent *dest; @@ -809,15 +809,20 @@ static int insert_extent(struct uid_gid_map *map, struct uid_gid_extent *extent) map->reverse = NULL; } - if (map->nr_extents < UID_GID_MAP_MAX_BASE_EXTENTS) - dest = &map->extent[map->nr_extents]; + /* + * nr_extents must be updated before the extent and forward arrays are + * accessed, otherwise KSAN will assert an out-of-bounds error. + */ + map->nr_extents++; + if (map->nr_extents <= UID_GID_MAP_MAX_BASE_EXTENTS) + dest = &map->extent[map->nr_extents - 1]; else - dest = &map->forward[map->nr_extents]; + dest = &map->forward[map->nr_extents - 1]; *dest = *extent; - map->nr_extents++; return 0; } +EXPORT_SYMBOL_IF_KUNIT(uid_gid_map_insert_extent); /* cmp function to sort() forward mappings */ static int cmp_extents_forward(const void *a, const void *b) @@ -850,10 +855,10 @@ static int cmp_extents_reverse(const void *a, const void *b) } /* - * sort_idmaps - Sorts an array of idmap entries. + * uid_gid_map_sort - Sorts an array of idmap entries. * Can only be called if number of mappings exceeds UID_GID_MAP_MAX_BASE_EXTENTS. */ -static int sort_idmaps(struct uid_gid_map *map) +VISIBLE_IF_KUNIT int uid_gid_map_sort(struct uid_gid_map *map) { if (map->nr_extents <= UID_GID_MAP_MAX_BASE_EXTENTS) return 0; @@ -874,6 +879,7 @@ static int sort_idmaps(struct uid_gid_map *map) return 0; } +EXPORT_SYMBOL_IF_KUNIT(uid_gid_map_sort); /** * verify_root_map() - check the uid 0 mapping @@ -929,6 +935,26 @@ static bool verify_root_map(const struct file *file, return true; } +/* + * Parse the decimal u32 at *pos, up to the next whitespace or NUL, and + * leave *pos pointing at that character. + */ +static int map_parse_u32(char **pos, u32 *res) +{ + char *end = *pos; + char c; + int ret; + + while (*end && !isspace(*end)) + end++; + c = *end; + *end = '\0'; + ret = kstrtou32(*pos, 10, res); + *end = c; + *pos = end; + return ret; +} + static ssize_t map_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos, int cap_setid, @@ -1001,17 +1027,20 @@ static ssize_t map_write(struct file *file, const char __user *buf, } pos = skip_spaces(pos); - extent.first = simple_strtoul(pos, &pos, 10); + if (map_parse_u32(&pos, &extent.first)) + goto out; if (!isspace(*pos)) goto out; pos = skip_spaces(pos); - extent.lower_first = simple_strtoul(pos, &pos, 10); + if (map_parse_u32(&pos, &extent.lower_first)) + goto out; if (!isspace(*pos)) goto out; pos = skip_spaces(pos); - extent.count = simple_strtoul(pos, &pos, 10); + if (map_parse_u32(&pos, &extent.count)) + goto out; if (*pos && !isspace(*pos)) goto out; @@ -1042,7 +1071,7 @@ static ssize_t map_write(struct file *file, const char __user *buf, (next_line != NULL)) goto out; - ret = insert_extent(&new_map, &extent); + ret = uid_gid_map_insert_extent(&new_map, &extent); if (ret < 0) goto out; ret = -EINVAL; @@ -1086,7 +1115,7 @@ static ssize_t map_write(struct file *file, const char __user *buf, * If we want to use binary search for lookup, this clones the extent * array and sorts both copies. */ - ret = sort_idmaps(&new_map); + ret = uid_gid_map_sort(&new_map); if (ret < 0) goto out; diff --git a/kernel/utsname.c b/kernel/utsname.c index ebbfc578a9d3..1ebf87e24607 100644 --- a/kernel/utsname.c +++ b/kernel/utsname.c @@ -81,7 +81,6 @@ struct uts_namespace *copy_utsname(u64 flags, { struct uts_namespace *new_ns; - BUG_ON(!old_ns); get_uts_ns(old_ns); if (!(flags & CLONE_NEWUTS)) diff --git a/kernel/vmcore_info.c b/kernel/vmcore_info.c index 8614430ca212..5a417f8a922a 100644 --- a/kernel/vmcore_info.c +++ b/kernel/vmcore_info.c @@ -216,7 +216,6 @@ static int __init crash_save_vmcoreinfo_init(void) VMCOREINFO_LENGTH(free_area.free_list, MIGRATE_TYPES); VMCOREINFO_NUMBER(NR_FREE_PAGES); VMCOREINFO_NUMBER(PG_lru); - VMCOREINFO_NUMBER(PG_private); VMCOREINFO_NUMBER(PG_swapcache); VMCOREINFO_NUMBER(PG_swapbacked); #define PAGE_SLAB_MAPCOUNT_VALUE (PGTY_slab << 24) diff --git a/kernel/watch_queue.c b/kernel/watch_queue.c index 538520861e8b..932a88ac1f39 100644 --- a/kernel/watch_queue.c +++ b/kernel/watch_queue.c @@ -101,7 +101,7 @@ static bool post_one_notification(struct watch_queue *wqueue, struct pipe_inode_info *pipe = wqueue->pipe; struct pipe_buffer *buf; struct page *page; - unsigned int head, tail, note, offset, len; + unsigned int head, tail, note, offset, len, nr_notes; bool done = false; spin_lock_irq(&pipe->rd_wait.lock); @@ -111,8 +111,10 @@ static bool post_one_notification(struct watch_queue *wqueue, if (pipe_full(head, tail, pipe->ring_size)) goto lost; - note = find_first_bit(wqueue->notes_bitmap, wqueue->nr_notes); - if (note >= wqueue->nr_notes) + /* Pairs with the smp_store_release() in watch_queue_set_size(). */ + nr_notes = smp_load_acquire(&wqueue->nr_notes); + note = find_first_bit(wqueue->notes_bitmap, nr_notes); + if (note >= nr_notes) goto lost; page = wqueue->notes[note / WATCH_QUEUE_NOTES_PER_PAGE]; @@ -297,7 +299,8 @@ long watch_queue_set_size(struct pipe_inode_info *pipe, unsigned int nr_notes) wqueue->notes = pages; wqueue->notes_bitmap = bitmap; wqueue->nr_pages = nr_pages; - wqueue->nr_notes = nr_notes; + /* Pairs with the smp_load_acquire() in post_one_notification(). */ + smp_store_release(&wqueue->nr_notes, nr_notes); return 0; error_p: diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 1ae3732a2c51..f2e8071a6f4e 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -56,6 +56,7 @@ #include <linux/kvm_para.h> #include <linux/delay.h> #include <linux/irq_work.h> +#include <linux/math64.h> #include "workqueue_internal.h" @@ -161,6 +162,9 @@ struct llc_shard_layout { * LN: pool->lock and wq_node_nr_active->lock protected for writes. Either for * reads. * + * FN: wq_flush_pnode->lock protected. list_empty(&pwq->flush_node) can also be + * tested with only pool->lock held. + * * K: Only modified by worker while holding pool->lock. Can be safely read by * self, while holding pool->lock or from IRQ context if %current is the * kworker. @@ -208,7 +212,7 @@ struct worker_pool { * but w/ pool->lock held. The readers grab pool->lock and are * guaranteed to see if the counter reached zero. */ - int nr_running; + int __data_racy nr_running; struct list_head worklist; /* L: list of pending works */ @@ -274,8 +278,10 @@ struct pool_workqueue { int work_color; /* L: current color */ int flush_color; /* L: flushing color */ int refcnt; /* L: reference count */ + int total_in_flight; /* L: sum of nr_in_flight[] */ int nr_in_flight[WORK_NR_COLORS]; /* L: nr of in_flight works */ + struct list_head flush_node; /* FN: node on wq_flush_pnode->pwqs */ bool plugged; /* L: execution suspended */ /* @@ -345,6 +351,21 @@ struct wq_node_nr_active { }; /* + * Per-node list of pwqs active since the last flush so that flushes visit only + * those. One per possible node plus the NUMA_NO_NODE fallback at nr_node_ids, + * each allocated on its node. A pwq is added when a work item is queued to it + * while off the list and removed by a flush which finds it with nothing in + * flight. ->work_color mirrors wq->work_color. Reading it and adding the pwq in + * one ->lock section keeps queueing coherent against flushing: a work item is + * either stamped with the new color or its pwq is visible to the flusher. + */ +struct wq_flush_pnode { + raw_spinlock_t lock; /* nests inside pool locks */ + int work_color; /* FN: mirrors wq->work_color */ + struct list_head pwqs; /* FN: pwqs active since last flush */ +}; + +/* * The externally visible workqueue. It relays the issued work items to * the appropriate worker_pool through its pool_workqueues. */ @@ -359,6 +380,7 @@ struct workqueue_struct { struct wq_flusher *first_flusher; /* WQ: first flusher */ struct list_head flusher_queue; /* WQ: flush waiters */ struct list_head flusher_overflow; /* WQ: flush overflow list */ + struct wq_flush_pnode **flush_pnodes; /* I: per-node flush membership */ struct list_head maydays; /* MD: pwqs requesting rescue */ struct worker *rescuer; /* MD: rescue worker */ @@ -367,6 +389,7 @@ struct workqueue_struct { /* See alloc_workqueue() function comment for info on min/max_active */ int max_active; /* WO: max active works */ + int percpu_max_active; /* WO: max active works per cpu */ int min_active; /* WO: min active works */ int saved_max_active; /* WQ: saved max_active */ int saved_min_active; /* WQ: saved min_active */ @@ -510,12 +533,6 @@ static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */ /* PL: hash of all unbound pools keyed by pool->attrs */ static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER); -/* I: attributes used when instantiating standard unbound pools on demand */ -static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS]; - -/* I: attributes used when instantiating ordered pools on demand */ -static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS]; - /* * I: kthread_worker to release pwq's. pwq release needs to be bounced to a * process context while holding a pool lock. Bounce to a dedicated kthread @@ -619,6 +636,26 @@ static void show_one_worker_pool(struct worker_pool *pool); list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \ lockdep_is_held(&(wq->mutex))) +/* + * Per-node arrays in this file carry an extra slot at nr_node_ids serving + * NUMA_NO_NODE. Return the slot after all nodes. + */ +static int next_node_with_fallback(int node) +{ + if (node >= nr_node_ids) + return nr_node_ids + 1; + + node = next_node(node, node_possible_map); + if (node >= nr_node_ids) + return nr_node_ids; + return node; +} + +#define for_each_node_with_fallback(node) \ + for ((node) = first_node(node_possible_map); \ + (node) <= nr_node_ids; \ + (node) = next_node_with_fallback(node)) + #ifdef CONFIG_DEBUG_OBJECTS_WORK static const struct debug_obj_descr work_debug_descr; @@ -739,7 +776,7 @@ static int worker_pool_assign_id(struct worker_pool *pool) } static struct pool_workqueue __rcu ** -unbound_pwq_slot(struct workqueue_struct *wq, int cpu) +pwq_slot(struct workqueue_struct *wq, int cpu) { if (cpu >= 0) return per_cpu_ptr(wq->cpu_pwq, cpu); @@ -748,9 +785,9 @@ unbound_pwq_slot(struct workqueue_struct *wq, int cpu) } /* @cpu < 0 for dfl_pwq */ -static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu) +static struct pool_workqueue *installed_pwq(struct workqueue_struct *wq, int cpu) { - return rcu_dereference_check(*unbound_pwq_slot(wq, cpu), + return rcu_dereference_check(*pwq_slot(wq, cpu), lockdep_is_held(&wq_pool_mutex) || lockdep_is_held(&wq->mutex)); } @@ -765,7 +802,7 @@ static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu) */ static struct cpumask *unbound_effective_cpumask(struct workqueue_struct *wq) { - return unbound_pwq(wq, -1)->pool->attrs->__pod_cpumask; + return installed_pwq(wq, -1)->pool->attrs->__pod_cpumask; } static unsigned int work_color_to_flags(int color) @@ -1532,6 +1569,7 @@ void wq_worker_tick(struct task_struct *task) struct worker *worker = kthread_data(task); struct pool_workqueue *pwq = worker->current_pwq; struct worker_pool *pool = worker->pool; + u64 delta; if (!pwq) return; @@ -1572,6 +1610,11 @@ void wq_worker_tick(struct task_struct *task) pwq->stats[PWQ_STAT_CM_WAKEUP]++; raw_spin_unlock(&pool->lock); + + delta = READ_ONCE(worker->task->se.sum_exec_runtime) - worker->current_at; + trace_workqueue_cpu_intensive(pwq, worker->current_work, + worker->current_func, + div_u64(delta, NSEC_PER_USEC)); } /** @@ -1635,6 +1678,22 @@ static struct wq_node_nr_active *wq_node_nr_active(struct workqueue_struct *wq, } /** + * wq_flush_pnode - Determine wq_flush_pnode to use + * @wq: workqueue of interest + * @node: NUMA node, can be %NUMA_NO_NODE + * + * Return @wq's wq_flush_pnode for @node, the nr_node_ids fallback if @node is + * %NUMA_NO_NODE. + */ +static struct wq_flush_pnode *wq_flush_pnode(struct workqueue_struct *wq, int node) +{ + if (node == NUMA_NO_NODE) + node = nr_node_ids; + + return wq->flush_pnodes[node]; +} + +/** * wq_update_node_max_active - Update per-node max_actives to use * @wq: workqueue to update * @off_cpu: CPU that's going down, -1 if a CPU is not going down @@ -1712,6 +1771,14 @@ static void put_pwq(struct pool_workqueue *pwq) lockdep_assert_held(&pwq->pool->lock); if (likely(--pwq->refcnt)) return; + + /* + * pwq_release_worker is only created in workqueue_init(). Getting + * here before that means an early workqueue allocation failed, which + * is fatal anyway. + */ + BUG_ON(!pwq_release_worker); + /* * @pwq can't be released under pool->lock, bounce to a dedicated * kthread_worker to avoid A-A deadlocks. @@ -1788,10 +1855,10 @@ static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill) /* * A concurrency-managed per-cpu pool accounts nr_active per pwq, so - * pwq->nr_active against wq->max_active is sufficient. + * pwq->nr_active against wq->percpu_max_active is sufficient. */ if (is_percpu_pool(pool)) { - obtained = pwq->nr_active < READ_ONCE(wq->max_active); + obtained = pwq->nr_active < READ_ONCE(wq->percpu_max_active); goto out; } @@ -2060,6 +2127,22 @@ static void pwq_dec_nr_active(struct pool_workqueue *pwq) } /** + * pwq_inc_nr_in_flight - increment pwq's nr_in_flight + * @pwq: pwq of interest + * @work_color: color of the work item being queued + * + * A work item or a barrier with @work_color is being queued to @pwq. + * + * CONTEXT: + * raw_spin_lock_irq(pool->lock). + */ +static void pwq_inc_nr_in_flight(struct pool_workqueue *pwq, int work_color) +{ + pwq->nr_in_flight[work_color]++; + pwq->total_in_flight++; +} + +/** * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight * @pwq: pwq of interest * @work_data: work_data of work which left the queue @@ -2083,6 +2166,7 @@ static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_ pwq_dec_nr_active(pwq); pwq->nr_in_flight[color]--; + pwq->total_in_flight--; /* is flush in progress and are we at the flushing tip? */ if (likely(pwq->flush_color != color)) @@ -2290,7 +2374,7 @@ static bool is_chained_work(struct workqueue_struct *wq) * Return %true iff I'm a worker executing a work item on @wq. If * I'm @worker, it's safe to dereference it without locking. */ - return worker && worker->current_pwq->wq == wq; + return worker && worker->current_pwq && worker->current_pwq->wq == wq; } /* @@ -2432,7 +2516,21 @@ retry: if (WARN_ON(!list_empty(&work->entry))) goto out; - pwq->nr_in_flight[pwq->work_color]++; + /* + * Add @pwq to its wq_flush_pnode list if off it. Syncing ->work_color + * in the same fpn->lock section is what keeps a concurrent flush from + * missing @work, see wq_flush_pnode. + */ + if (unlikely(list_empty(&pwq->flush_node))) { + struct wq_flush_pnode *fpn = wq_flush_pnode(wq, pool->node); + + raw_spin_lock(&fpn->lock); + pwq->work_color = fpn->work_color; + list_add_tail(&pwq->flush_node, &fpn->pwqs); + raw_spin_unlock(&fpn->lock); + } + + pwq_inc_nr_in_flight(pwq, pwq->work_color); work_flags = work_color_to_flags(pwq->work_color); /* @@ -3117,6 +3215,7 @@ static void send_mayday(struct pool_workqueue *pwq) list_add_tail(&pwq->mayday_node, &wq->maydays); wake_up_process(wq->rescuer->task); pwq->stats[PWQ_STAT_MAYDAY]++; + trace_workqueue_mayday(pwq); } } @@ -3623,6 +3722,7 @@ static bool assign_rescuer_work(struct pool_workqueue *pwq, struct worker *rescu list_for_each_entry_safe_from(work, n, &pool->worklist, entry) { if (get_work_pwq(work) == pwq && assign_work(work, rescuer, &n)) { pwq->stats[PWQ_STAT_RESCUED]++; + trace_workqueue_rescued(pwq, work, work->func); /* put the cursor for next search */ list_move_tail(&cursor->entry, &n->entry); return true; @@ -3758,6 +3858,9 @@ static void bh_worker(struct worker *worker) struct worker_pool *pool = worker->pool; int nr_restarts = BH_WORKER_RESTARTS; unsigned long end = jiffies + BH_WORKER_JIFFIES; + bool budget_exhausted = false; + bool timeout = false; + int restarts = 0; worker_lock_callback(pool); raw_spin_lock_irq(&pool->lock); @@ -3780,8 +3883,23 @@ static void bh_worker(struct worker *worker) if (assign_work(work, worker, NULL)) process_scheduled_works(worker); - } while (keep_working(pool) && - --nr_restarts && time_before(jiffies, end)); + + if (!keep_working(pool)) + break; + + if (!--nr_restarts) { + budget_exhausted = true; + break; + } + + if (!time_before(jiffies, end)) { + budget_exhausted = true; + timeout = true; + break; + } + + restarts++; + } while (1); worker_set_flags(worker, WORKER_PREP); done: @@ -3789,6 +3907,10 @@ done: kick_pool(pool); raw_spin_unlock_irq(&pool->lock); worker_unlock_callback(pool); + + if (budget_exhausted) + trace_workqueue_bh_budget_yield(pool, restarts, timeout, + pool->attrs->nice == HIGHPRI_NICE_LEVEL); } /* @@ -3918,7 +4040,7 @@ static void check_flush_dependency(struct workqueue_struct *target_wq, WARN_ONCE(current->flags & PF_MEMALLOC, "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps", current->pid, current->comm, target_wq->name, target_func); - WARN_ONCE(worker && ((worker->current_pwq->wq->flags & + WARN_ONCE(worker && worker->current_pwq && ((worker->current_pwq->wq->flags & (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM), "workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps", worker->current_pwq->wq->name, worker->current_func, @@ -4008,7 +4130,15 @@ static void insert_wq_barrier(struct pool_workqueue *pwq, __set_bit(WORK_STRUCT_LINKED_BIT, bits); } - pwq->nr_in_flight[work_color]++; + /* + * Flushes must wait for barriers too as a barrier pins @pwq and a + * worker until it runs and destroy_workqueue()'s drain relies on that. + * @target being in flight keeps @pwq on its wq_flush_pnode list, so no + * need to add it here. + */ + WARN_ON_ONCE(list_empty(&pwq->flush_node)); + + pwq_inc_nr_in_flight(pwq, work_color); work_flags |= work_color_to_flags(work_color); insert_work(pwq, &barr->work, head, work_flags); @@ -4020,23 +4150,23 @@ static void insert_wq_barrier(struct pool_workqueue *pwq, * @flush_color: new flush color, < 0 for no-op * @work_color: new work color, < 0 for no-op * - * Prepare pwqs for workqueue flushing. + * Prepare pwqs for workqueue flushing. Only the pwqs on @wq's wq_flush_pnode + * lists, which have been active since the last flush, are visited. * - * If @flush_color is non-negative, flush_color on all pwqs should be - * -1. If no pwq has in-flight commands at the specified color, all - * pwq->flush_color's stay at -1 and %false is returned. If any pwq - * has in flight commands, its pwq->flush_color is set to - * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq - * wakeup logic is armed and %true is returned. + * If @flush_color >= 0, flush_color on all visited pwqs should be -1. If no + * visited pwq has in-flight work items at the specified color, all + * pwq->flush_color's stay at -1 and return %false. If any visited pwq has + * in-flight work items, set its pwq->flush_color to @flush_color, update + * @wq->nr_pwqs_to_flush accordingly, arm pwq wakeup logic and return %true. * - * The caller should have initialized @wq->first_flusher prior to - * calling this function with non-negative @flush_color. If - * @flush_color is negative, no flush color update is done and %false - * is returned. + * The caller should have initialized @wq->first_flusher prior to calling this + * function with non-negative @flush_color. If @flush_color < 0, no flush color + * update is done and %false is returned. * - * If @work_color is non-negative, all pwqs should have the same - * work_color which is previous to @work_color and all will be - * advanced to @work_color. + * If @work_color >= 0, all visited pwqs and every wq_flush_pnode's work color + * mirror are advanced to @work_color. The mirrors of nodes with no active pwqs + * must be advanced too as a pwq added to a list later stamps the color it reads + * from the mirror. * * CONTEXT: * mutex_lock(wq->mutex). @@ -4048,50 +4178,81 @@ static void insert_wq_barrier(struct pool_workqueue *pwq, static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq, int flush_color, int work_color) { + struct pool_workqueue *pwq, *next; bool wait = false; - struct pool_workqueue *pwq; - struct worker_pool *current_pool = NULL; + int node; if (flush_color >= 0) { WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush)); atomic_set(&wq->nr_pwqs_to_flush, 1); } - /* - * For unbound workqueue, pwqs will map to only a few pools. - * Most of the time, pwqs within the same pool will be linked - * sequentially to wq->pwqs by cpu index. So in the majority - * of pwq iters, the pool is the same, only doing lock/unlock - * if the pool has changed. This can largely reduce expensive - * lock operations. - */ - for_each_pwq(pwq, wq) { - if (current_pool != pwq->pool) { - if (likely(current_pool)) - raw_spin_unlock_irq(¤t_pool->lock); - current_pool = pwq->pool; - raw_spin_lock_irq(¤t_pool->lock); + for_each_node_with_fallback(node) { + struct wq_flush_pnode *fpn = wq->flush_pnodes[node]; + struct worker_pool *current_pool = NULL; + LIST_HEAD(to_visit); + LIST_HEAD(to_keep); + + /* + * Advance the node's color mirror and take its active pwqs. + * Work items queued to off-list pwqs afterwards carry the new + * color and don't need visiting. + */ + raw_spin_lock_irq(&fpn->lock); + if (work_color >= 0) { + WARN_ON_ONCE(work_color != work_next_color(fpn->work_color)); + fpn->work_color = work_color; } + list_splice_init(&fpn->pwqs, &to_visit); + raw_spin_unlock_irq(&fpn->lock); + + /* + * Unbound workqueues map multiple pwqs to one pool. Only cycle + * pool->lock when the pool changes. + */ + list_for_each_entry_safe(pwq, next, &to_visit, flush_node) { + if (current_pool != pwq->pool) { + if (likely(current_pool)) + raw_spin_unlock_irq(¤t_pool->lock); + current_pool = pwq->pool; + raw_spin_lock_irq(¤t_pool->lock); + } - if (flush_color >= 0) { - WARN_ON_ONCE(pwq->flush_color != -1); + if (flush_color >= 0) { + WARN_ON_ONCE(pwq->flush_color != -1); - if (pwq->nr_in_flight[flush_color]) { - pwq->flush_color = flush_color; - atomic_inc(&wq->nr_pwqs_to_flush); - wait = true; + if (pwq->nr_in_flight[flush_color]) { + pwq->flush_color = flush_color; + atomic_inc(&wq->nr_pwqs_to_flush); + wait = true; + } } - } - if (work_color >= 0) { - WARN_ON_ONCE(work_color != work_next_color(pwq->work_color)); - pwq->work_color = work_color; + if (work_color >= 0) { + WARN_ON_ONCE(work_color != work_next_color(pwq->work_color)); + pwq->work_color = work_color; + } + + /* + * A cascade can arm a color retired several advances + * ago, so keep a pwq listed while any color has work + * items in flight. + */ + if (pwq->total_in_flight) + list_move_tail(&pwq->flush_node, &to_keep); + else + list_del_init(&pwq->flush_node); } - } + if (current_pool) + raw_spin_unlock_irq(¤t_pool->lock); - if (current_pool) - raw_spin_unlock_irq(¤t_pool->lock); + if (!list_empty(&to_keep)) { + raw_spin_lock_irq(&fpn->lock); + list_splice_tail(&to_keep, &fpn->pwqs); + raw_spin_unlock_irq(&fpn->lock); + } + } if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush)) complete(&wq->first_flusher->done); @@ -5049,13 +5210,10 @@ static void free_node_nr_active(struct wq_node_nr_active **nna_ar) { int node; - for_each_node(node) { + for_each_node_with_fallback(node) { kfree(nna_ar[node]); nna_ar[node] = NULL; } - - kfree(nna_ar[nr_node_ids]); - nna_ar[nr_node_ids] = NULL; } static void init_node_nr_active(struct wq_node_nr_active *nna) @@ -5075,21 +5233,15 @@ static int alloc_node_nr_active(struct wq_node_nr_active **nna_ar) struct wq_node_nr_active *nna; int node; - for_each_node(node) { - nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, node); + for_each_node_with_fallback(node) { + nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, + node < nr_node_ids ? node : NUMA_NO_NODE); if (!nna) goto err_free; init_node_nr_active(nna); nna_ar[node] = nna; } - /* [nr_node_ids] is used as the fallback */ - nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, NUMA_NO_NODE); - if (!nna) - goto err_free; - init_node_nr_active(nna); - nna_ar[nr_node_ids] = nna; - return 0; err_free: @@ -5097,6 +5249,47 @@ err_free: return -ENOMEM; } +static void free_flush_pnodes(struct workqueue_struct *wq) +{ + int node; + + if (!wq->flush_pnodes) + return; + + for_each_node_with_fallback(node) + kfree(wq->flush_pnodes[node]); + kfree(wq->flush_pnodes); + wq->flush_pnodes = NULL; +} + +static int alloc_flush_pnodes(struct workqueue_struct *wq) +{ + struct wq_flush_pnode *fpn; + int node; + + wq->flush_pnodes = kcalloc(nr_node_ids + 1, sizeof(*wq->flush_pnodes), GFP_KERNEL); + if (!wq->flush_pnodes) + return -ENOMEM; + + for_each_node_with_fallback(node) { + fpn = kzalloc_node(sizeof(*fpn), GFP_KERNEL, + node < nr_node_ids ? node : NUMA_NO_NODE); + if (!fpn) + goto err_free; + + raw_spin_lock_init(&fpn->lock); + fpn->work_color = wq->work_color; + INIT_LIST_HEAD(&fpn->pwqs); + wq->flush_pnodes[node] = fpn; + } + + return 0; + +err_free: + free_flush_pnodes(wq); + return -ENOMEM; +} + static void rcu_free_wq(struct rcu_head *rcu) { struct workqueue_struct *wq = @@ -5105,6 +5298,7 @@ static void rcu_free_wq(struct rcu_head *rcu) if (wq->flags & WQ_UNBOUND) free_node_nr_active(wq->node_nr_active); + free_flush_pnodes(wq); wq_free_lockdep(wq); free_percpu(wq->cpu_pwq); free_workqueue_attrs(wq->attrs); @@ -5286,6 +5480,19 @@ static void pwq_release_workfn(struct kthread_work *work) is_last = list_empty(&wq->pwqs); /* + * An idle pwq can linger on its wq_flush_pnode list until the + * next flush. wq->mutex excludes flushers, which never drop it + * with pwqs on their private lists. + */ + if (!list_empty(&pwq->flush_node)) { + struct wq_flush_pnode *fpn = wq_flush_pnode(wq, pool->node); + + raw_spin_lock_irq(&fpn->lock); + list_del_init(&pwq->flush_node); + raw_spin_unlock_irq(&fpn->lock); + } + + /* * For ordered workqueue with a plugged dfl_pwq, restart it now. */ if (!is_last && (wq->flags & __WQ_ORDERED)) @@ -5335,6 +5542,7 @@ static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq, pwq->refcnt = 1; INIT_LIST_HEAD(&pwq->inactive_works); INIT_LIST_HEAD(&pwq->pending_node); + INIT_LIST_HEAD(&pwq->flush_node); INIT_LIST_HEAD(&pwq->pwqs_node); INIT_LIST_HEAD(&pwq->mayday_node); kthread_init_work(&pwq->release_work, pwq_release_workfn); @@ -5385,22 +5593,34 @@ static struct worker_pool *get_percpu_pool(struct workqueue_struct *wq, int cpu) return &per_cpu_ptr(pools, cpu)[highpri]; } -/* obtain a pool matching @attr and create a pwq associating the pool and @wq */ +/* + * Obtain the pool backing @wq on @cpu and create a pwq associating the two. + * A WQ_PERCPU workqueue is backed by the static per-cpu pool of @cpu, + * everything else by a pool matching @attrs. @cpu < 0 is always unbound. + */ static struct pool_workqueue *alloc_pwq(struct workqueue_struct *wq, - const struct workqueue_attrs *attrs) + const struct workqueue_attrs *attrs, + int cpu) { struct worker_pool *pool; struct pool_workqueue *pwq; lockdep_assert_held(&wq_pool_mutex); - pool = get_unbound_pool(attrs); - if (!pool) - return NULL; + WARN_ON_ONCE((wq->flags & WQ_PERCPU) && cpu < 0); + + if (cpu >= 0 && (wq->flags & WQ_PERCPU)) { + pool = get_percpu_pool(wq, cpu); + } else { + pool = get_unbound_pool(attrs); + if (!pool) + return NULL; + } pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node); if (!pwq) { - put_unbound_pool(pool); + if (!is_percpu_pool(pool)) + put_unbound_pool(pool); return NULL; } @@ -5408,6 +5628,13 @@ static struct pool_workqueue *alloc_pwq(struct workqueue_struct *wq, return pwq; } +/* create a pwq backed by an unbound pool matching @attrs */ +static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq, + const struct workqueue_attrs *attrs) +{ + return alloc_pwq(wq, attrs, -1); +} + /** * wq_calc_pod_cpumask - calculate a wq_attrs' cpumask for a pod * @attrs: the wq_attrs of the default pwq of the target workqueue @@ -5437,10 +5664,10 @@ static void wq_calc_pod_cpumask(struct workqueue_attrs *attrs, int cpu) } /* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */ -static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq, - int cpu, struct pool_workqueue *pwq) +static struct pool_workqueue *install_pwq(struct workqueue_struct *wq, + int cpu, struct pool_workqueue *pwq) { - struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu); + struct pool_workqueue __rcu **slot = pwq_slot(wq, cpu); struct pool_workqueue *old_pwq; lockdep_assert_held(&wq_pool_mutex); @@ -5503,15 +5730,18 @@ apply_wqattrs_prepare(struct workqueue_struct *wq, /* * If something goes wrong during CPU up/down, we'll fall back to - * the default pwq covering whole @attrs->cpumask. Always create - * it even if we don't use it immediately. + * the default pwq covering whole @attrs->cpumask. Create it even + * if we don't use it immediately. A percpu workqueue has a pwq on + * every possible CPU and never falls back, so it has no default. */ copy_workqueue_attrs(new_attrs, attrs); wqattrs_actualize_cpumask(new_attrs, unbound_cpumask); cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask); - ctx->dfl_pwq = alloc_pwq(wq, new_attrs); - if (!ctx->dfl_pwq) - goto out_free; + if (!(wq->flags & WQ_PERCPU)) { + ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs); + if (!ctx->dfl_pwq) + goto out_free; + } for_each_possible_cpu(cpu) { if (new_attrs->ordered) { @@ -5519,7 +5749,7 @@ apply_wqattrs_prepare(struct workqueue_struct *wq, ctx->pwq_tbl[cpu] = ctx->dfl_pwq; } else { wq_calc_pod_cpumask(new_attrs, cpu); - ctx->pwq_tbl[cpu] = alloc_pwq(wq, new_attrs); + ctx->pwq_tbl[cpu] = alloc_pwq(wq, new_attrs, cpu); if (!ctx->pwq_tbl[cpu]) goto out_free; } @@ -5561,9 +5791,10 @@ static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx) /* save the previous pwqs and install the new ones */ for_each_possible_cpu(cpu) - ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu, - ctx->pwq_tbl[cpu]); - ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq); + ctx->pwq_tbl[cpu] = install_pwq(ctx->wq, cpu, + ctx->pwq_tbl[cpu]); + if (ctx->dfl_pwq) + ctx->dfl_pwq = install_pwq(ctx->wq, -1, ctx->dfl_pwq); /* update node_nr_active->max, which only unbound workqueues have */ if (ctx->wq->flags & WQ_UNBOUND) @@ -5577,10 +5808,6 @@ static int apply_workqueue_attrs_locked(struct workqueue_struct *wq, { struct apply_wqattrs_ctx *ctx; - /* only unbound workqueues can change attributes */ - if (WARN_ON(!(wq->flags & WQ_UNBOUND))) - return -EINVAL; - ctx = apply_wqattrs_prepare(wq, attrs, wq_unbound_cpumask); if (IS_ERR(ctx)) return PTR_ERR(ctx); @@ -5612,6 +5839,10 @@ int apply_workqueue_attrs(struct workqueue_struct *wq, { int ret; + /* only unbound workqueues can change attributes */ + if (WARN_ON(!(wq->flags & WQ_UNBOUND))) + return -EINVAL; + mutex_lock(&wq_pool_mutex); ret = apply_workqueue_attrs_locked(wq, attrs); mutex_unlock(&wq_pool_mutex); @@ -5660,11 +5891,11 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) /* nothing to do if the target cpumask matches the current pwq */ wq_calc_pod_cpumask(target_attrs, cpu); - if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs)) + if (wqattrs_equal(target_attrs, installed_pwq(wq, cpu)->pool->attrs)) return; /* create a new pwq */ - pwq = alloc_pwq(wq, target_attrs); + pwq = alloc_unbound_pwq(wq, target_attrs); if (!pwq) { pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n", wq->name); @@ -5673,49 +5904,43 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) /* Install the new pwq. */ mutex_lock(&wq->mutex); - old_pwq = install_unbound_pwq(wq, cpu, pwq); + old_pwq = install_pwq(wq, cpu, pwq); goto out_unlock; use_dfl_pwq: mutex_lock(&wq->mutex); - pwq = unbound_pwq(wq, -1); + pwq = installed_pwq(wq, -1); raw_spin_lock_irq(&pwq->pool->lock); get_pwq(pwq); raw_spin_unlock_irq(&pwq->pool->lock); - old_pwq = install_unbound_pwq(wq, cpu, pwq); + old_pwq = install_pwq(wq, cpu, pwq); out_unlock: mutex_unlock(&wq->mutex); put_pwq_unlocked(old_pwq); } -static int alloc_and_link_percpu_pwqs(struct workqueue_struct *wq) +/* the attrs @wq asked for, as spelled by its flags */ +static struct workqueue_attrs *alloc_wq_std_attrs(struct workqueue_struct *wq) { - struct pool_workqueue *pwq; - int cpu; - - for_each_possible_cpu(cpu) { - struct worker_pool *pool = get_percpu_pool(wq, cpu); - - pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node); - if (!pwq) - return -ENOMEM; + struct workqueue_attrs *attrs; - init_pwq(pwq, wq, pool); + attrs = alloc_workqueue_attrs(); + if (!attrs) + return NULL; - mutex_lock(&wq->mutex); - link_pwq(pwq); - mutex_unlock(&wq->mutex); + if (wq->flags & WQ_HIGHPRI) + attrs->nice = HIGHPRI_NICE_LEVEL; - rcu_assign_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu), pwq); - } + if (wq->flags & __WQ_ORDERED) + attrs->ordered = true; - return 0; + return attrs; } static int alloc_and_link_pwqs(struct workqueue_struct *wq) { - bool highpri = wq->flags & WQ_HIGHPRI; - int cpu, ret; + struct workqueue_attrs *attrs; + int ret; lockdep_assert_held(&wq_pool_mutex); @@ -5723,47 +5948,29 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq) if (!wq->cpu_pwq) goto enomem; - if (!(wq->flags & WQ_UNBOUND)) { - ret = alloc_and_link_percpu_pwqs(wq); - } else if (wq->flags & __WQ_ORDERED) { - struct pool_workqueue *dfl_pwq; + attrs = alloc_wq_std_attrs(wq); + if (!attrs) + goto enomem; + + ret = apply_workqueue_attrs_locked(wq, attrs); + free_workqueue_attrs(attrs); + if (ret) + goto enomem; + + if (wq->flags & __WQ_ORDERED) { + struct pool_workqueue *dfl_pwq = rcu_access_pointer(wq->dfl_pwq); - ret = apply_workqueue_attrs_locked(wq, ordered_wq_attrs[highpri]); /* there should only be single pwq for ordering guarantee */ - dfl_pwq = rcu_access_pointer(wq->dfl_pwq); - WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node || - wq->pwqs.prev != &dfl_pwq->pwqs_node), + WARN(wq->pwqs.next != &dfl_pwq->pwqs_node || + wq->pwqs.prev != &dfl_pwq->pwqs_node, "ordering guarantee broken for workqueue %s\n", wq->name); - } else { - ret = apply_workqueue_attrs_locked(wq, unbound_std_wq_attrs[highpri]); } - if (ret) - goto enomem; return 0; enomem: - if (wq->cpu_pwq) { - for_each_possible_cpu(cpu) { - struct pool_workqueue __rcu **slot; - struct pool_workqueue *pwq; - - slot = per_cpu_ptr(wq->cpu_pwq, cpu); - pwq = rcu_access_pointer(*slot); - if (pwq) { - /* - * Unlink pwq from wq->pwqs since link_pwq() - * may have already added it. wq->mutex is not - * needed as the wq has not been published yet. - */ - if (!list_empty(&pwq->pwqs_node)) - list_del_rcu(&pwq->pwqs_node); - kmem_cache_free(pwq_cache, pwq); - } - } - free_percpu(wq->cpu_pwq); - wq->cpu_pwq = NULL; - } + free_percpu(wq->cpu_pwq); + wq->cpu_pwq = NULL; return -ENOMEM; } @@ -5828,9 +6035,9 @@ static int init_rescuer(struct workqueue_struct *wq) * wq_adjust_max_active - update a wq's max_active to the current setting * @wq: target workqueue * - * If @wq isn't freezing, set @wq->max_active to the saved_max_active and - * activate inactive work items accordingly. If @wq is freezing, clear - * @wq->max_active to zero. + * If @wq isn't freezing, set the limit that applies to @wq's backing to the + * saved_max_active and activate inactive work items accordingly. If @wq is + * freezing, clear it to zero. */ static void wq_adjust_max_active(struct workqueue_struct *wq) { @@ -5847,20 +6054,25 @@ static void wq_adjust_max_active(struct workqueue_struct *wq) new_min = wq->saved_min_active; } - if (wq->max_active == new_max && wq->min_active == new_min) - return; - /* - * Update @wq->max/min_active and then kick inactive work items if more - * active work items are allowed. This doesn't break work item ordering + * Update the limit and then kick inactive work items if more active + * work items are allowed. This doesn't break work item ordering * because new work items are always queued behind existing inactive * work items if there are any. */ - WRITE_ONCE(wq->max_active, new_max); - WRITE_ONCE(wq->min_active, new_min); + if (wq->flags & WQ_UNBOUND) { + if (wq->max_active == new_max && wq->min_active == new_min) + return; - if (wq->flags & WQ_UNBOUND) + WRITE_ONCE(wq->max_active, new_max); + WRITE_ONCE(wq->min_active, new_min); wq_update_node_max_active(wq, -1); + } else { + if (wq->percpu_max_active == new_max) + return; + + WRITE_ONCE(wq->percpu_max_active, new_max); + } if (new_max == 0) return; @@ -5957,10 +6169,14 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, /* init wq */ wq->flags = flags; - wq->max_active = max_active; - wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE); - wq->saved_max_active = wq->max_active; - wq->saved_min_active = wq->min_active; + if (flags & WQ_UNBOUND) { + wq->max_active = max_active; + wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE); + wq->saved_min_active = wq->min_active; + } else { + wq->percpu_max_active = max_active; + } + wq->saved_max_active = max_active; mutex_init(&wq->mutex); atomic_set(&wq->nr_pwqs_to_flush, 0); INIT_LIST_HEAD(&wq->pwqs); @@ -5970,6 +6186,9 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, INIT_LIST_HEAD(&wq->list); + if (alloc_flush_pnodes(wq) < 0) + goto err_free_wq; + if (flags & WQ_UNBOUND) { if (alloc_node_nr_active(wq->node_nr_active) < 0) goto err_free_wq; @@ -6007,12 +6226,13 @@ err_unlock_free_node_nr_active: * flushing the pwq_release_worker ensures that the pwq_release_workfn() * completes before calling kfree(wq). */ - if (wq->flags & WQ_UNBOUND) { + if (pwq_release_worker) kthread_flush_worker(pwq_release_worker); + if (wq->flags & WQ_UNBOUND) free_node_nr_active(wq->node_nr_active); - } err_free_wq: free_workqueue_attrs(wq->attrs); + free_flush_pnodes(wq); kfree(wq); return NULL; err_unlock_destroy: @@ -6104,11 +6324,8 @@ EXPORT_SYMBOL_GPL(alloc_workqueue_lockdep_map); static bool pwq_busy(struct pool_workqueue *pwq) { - int i; - - for (i = 0; i < WORK_NR_COLORS; i++) - if (pwq->nr_in_flight[i]) - return true; + if (pwq->total_in_flight) + return true; if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1)) return true; @@ -6199,12 +6416,12 @@ void destroy_workqueue(struct workqueue_struct *wq) rcu_read_lock(); for_each_possible_cpu(cpu) { - put_pwq_unlocked(unbound_pwq(wq, cpu)); - RCU_INIT_POINTER(*unbound_pwq_slot(wq, cpu), NULL); + put_pwq_unlocked(installed_pwq(wq, cpu)); + RCU_INIT_POINTER(*pwq_slot(wq, cpu), NULL); } - put_pwq_unlocked(unbound_pwq(wq, -1)); - RCU_INIT_POINTER(*unbound_pwq_slot(wq, -1), NULL); + put_pwq_unlocked(installed_pwq(wq, -1)); + RCU_INIT_POINTER(*pwq_slot(wq, -1), NULL); rcu_read_unlock(); } @@ -6320,7 +6537,7 @@ bool current_is_workqueue_mem_reclaim(void) { struct worker *worker = current_wq_worker(); - return worker && + return worker && worker->current_pwq && ((worker->current_pwq->wq->flags & (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM); } @@ -8203,24 +8420,6 @@ void __init workqueue_init_early(void) init_cpu_worker_pool(pool, cpu, std_nice[i++]); } - /* create default unbound and ordered wq attrs */ - for (i = 0; i < NR_STD_WORKER_POOLS; i++) { - struct workqueue_attrs *attrs; - - BUG_ON(!(attrs = alloc_workqueue_attrs())); - attrs->nice = std_nice[i]; - unbound_std_wq_attrs[i] = attrs; - - /* - * An ordered wq should have only one pwq as ordering is - * guaranteed by max_active which is enforced by pwqs. - */ - BUG_ON(!(attrs = alloc_workqueue_attrs())); - attrs->nice = std_nice[i]; - attrs->ordered = true; - ordered_wq_attrs[i] = attrs; - } - system_wq = alloc_workqueue("events", WQ_PERCPU | __WQ_DEPRECATED, 0); system_percpu_wq = alloc_workqueue("events", WQ_PERCPU, 0); system_highpri_wq = alloc_workqueue("events_highpri", @@ -8264,7 +8463,7 @@ static void __init wq_cpu_intensive_thresh_init(void) * lot slower CPUs including microcontrollers where the threshold is way * too low. * - * Let's scale up the threshold upto 1 second if BogoMips is below 4000. + * Let's scale up the threshold up to 1 second if BogoMips is below 4000. * This is by no means accurate but it doesn't have to be. The mechanism * is still useful even when the threshold is fully scaled up. Also, as * the reports would usually be applicable to everyone, some machines |