diff options
Diffstat (limited to 'include')
| -rw-r--r-- | include/asm-generic/vmlinux.lds.h | 31 | ||||
| -rw-r--r-- | include/linux/bpf-cgroup-defs.h | 1 | ||||
| -rw-r--r-- | include/linux/bpf-cgroup.h | 64 | ||||
| -rw-r--r-- | include/linux/bpf.h | 230 | ||||
| -rw-r--r-- | include/linux/bpf_compiler_types.h | 21 | ||||
| -rw-r--r-- | include/linux/bpf_crypto.h | 24 | ||||
| -rw-r--r-- | include/linux/bpf_local_storage.h | 13 | ||||
| -rw-r--r-- | include/linux/bpf_lsm.h | 11 | ||||
| -rw-r--r-- | include/linux/bpf_verifier.h | 564 | ||||
| -rw-r--r-- | include/linux/btf.h | 65 | ||||
| -rw-r--r-- | include/linux/cnum.h | 3 | ||||
| -rw-r--r-- | include/linux/filter.h | 83 | ||||
| -rw-r--r-- | include/linux/ftrace.h | 1 | ||||
| -rw-r--r-- | include/linux/key.h | 2 | ||||
| -rw-r--r-- | include/linux/mm.h | 8 | ||||
| -rw-r--r-- | include/linux/module.h | 4 | ||||
| -rw-r--r-- | include/linux/tcp.h | 7 | ||||
| -rw-r--r-- | include/linux/tnum.h | 2 | ||||
| -rw-r--r-- | include/net/tcp.h | 248 | ||||
| -rw-r--r-- | include/uapi/linux/bpf.h | 126 | ||||
| -rw-r--r-- | include/uapi/linux/btf.h | 83 | ||||
| -rw-r--r-- | include/uapi/linux/keyctl.h | 1 |
22 files changed, 1325 insertions, 267 deletions
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 7a3e9f00d628..3eb732942d69 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -675,12 +675,43 @@ /* * .BTF */ +#ifdef CONFIG_DEBUG_INFO_BTF_INLINE +#define BTF_INLINE \ + . = ALIGN(PAGE_SIZE); \ + .BTF.inline : AT(ADDR(.BTF.inline) - LOAD_OFFSET) { \ + BOUNDED_SECTION_BY(.BTF.inline, _BTF_inline) \ + } +#else +#define BTF_INLINE +#endif + +#ifdef CONFIG_DEBUG_INFO_BTF_INLINE_MODULE +/* __MODULE_NAME_LEN + SHA-256 digest + u32 BTF size */ +#ifdef CONFIG_64BIT +#define BTF_LINK_SIZE (64 - 8 + 32 + 4) +#else +#define BTF_LINK_SIZE (64 - 4 + 32 + 4) +#endif +#define BTF_INLINE_LINK \ + . = ALIGN(PAGE_SIZE); \ + .BTF.inline.link : AT(ADDR(.BTF.inline.link) - LOAD_OFFSET) { \ + __start_BTF_inline_link = .; \ + BYTE(0) \ + . += BTF_LINK_SIZE - 1; \ + __stop_BTF_inline_link = .; \ + } +#else +#define BTF_INLINE_LINK +#endif + #ifdef CONFIG_DEBUG_INFO_BTF #define BTF \ . = ALIGN(PAGE_SIZE); \ .BTF : AT(ADDR(.BTF) - LOAD_OFFSET) { \ BOUNDED_SECTION_BY(.BTF, _BTF) \ } \ + BTF_INLINE \ + BTF_INLINE_LINK \ . = ALIGN(PAGE_SIZE); \ .BTF_ids : AT(ADDR(.BTF_ids) - LOAD_OFFSET) { \ *(.BTF_ids) \ diff --git a/include/linux/bpf-cgroup-defs.h b/include/linux/bpf-cgroup-defs.h index c9e6b26abab6..0147b8bec973 100644 --- a/include/linux/bpf-cgroup-defs.h +++ b/include/linux/bpf-cgroup-defs.h @@ -47,6 +47,7 @@ enum cgroup_bpf_attach_type { CGROUP_INET6_GETSOCKNAME, CGROUP_UNIX_GETSOCKNAME, CGROUP_INET_SOCK_RELEASE, + CGROUP_TCP_SOCK_OPS, CGROUP_LSM_START, CGROUP_LSM_END = CGROUP_LSM_START + CGROUP_LSM_NUM - 1, MAX_CGROUP_BPF_ATTACH_TYPE diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h index 4d0cc65976a1..02ef7f899b3d 100644 --- a/include/linux/bpf-cgroup.h +++ b/include/linux/bpf-cgroup.h @@ -100,6 +100,8 @@ struct bpf_cgroup_storage { struct bpf_cgroup_link { struct bpf_link link; struct cgroup *cgroup; + struct bpf_map *map; + wait_queue_head_t wait_hup; }; struct bpf_prog_list { @@ -110,6 +112,18 @@ struct bpf_prog_list { u32 flags; }; +#define bpf_cgroup_struct_ops_foreach(var, item, cgrp, atype) \ + for (item = rcu_dereference((cgrp)->bpf.effective[atype])->items;\ + ((var) = READ_ONCE(item->kdata)); \ + item++) + +static inline bool cgroup_bpf_is_struct_ops_atype(enum cgroup_bpf_attach_type atype) +{ + return atype == CGROUP_TCP_SOCK_OPS; +} +void cgroup_bpf_struct_ops_register(int atype, u32 type_id, void *cfi_stubs, bool mult_trace); +int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr); + void __init cgroup_bpf_lifetime_notifier_init(void); int __cgroup_bpf_run_filter_skb(struct sock *sk, @@ -331,27 +345,19 @@ static inline bool cgroup_bpf_sock_enabled(struct sock *sk, * calling bpf_setsockopt on listener-sk will not make sense anyway, * so passing 'sock_ops->sk == req_sk' to the bpf prog is appropriate here. */ -#define BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk) \ -({ \ - int __ret = 0; \ - if (cgroup_bpf_enabled(CGROUP_SOCK_OPS)) \ - __ret = __cgroup_bpf_run_filter_sock_ops(sk, \ - sock_ops, \ - CGROUP_SOCK_OPS); \ - __ret; \ -}) - -#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \ -({ \ - int __ret = 0; \ - if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) && (sock_ops)->sk) { \ - typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \ - if (__sk && sk_fullsock(__sk)) \ - __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \ - sock_ops, \ - CGROUP_SOCK_OPS); \ - } \ - __ret; \ +#define __BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk) \ + __cgroup_bpf_run_filter_sock_ops(sk, sock_ops, \ + CGROUP_SOCK_OPS) + +#define __BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \ +({ \ + int __ret = 0; \ + typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \ + if (__sk && sk_fullsock(__sk)) \ + __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \ + sock_ops, \ + CGROUP_SOCK_OPS); \ + __ret; \ }) #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(atype, major, minor, access) \ @@ -479,6 +485,20 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, return 0; } +static inline bool cgroup_bpf_is_struct_ops_atype(int atype) +{ + return false; +} +static inline void cgroup_bpf_struct_ops_register(int atype, u32 type_id, void *cfi_stubs, + bool mult_trace) +{ +} +static inline int cgroup_bpf_struct_ops_attach(struct bpf_map *map, + const union bpf_attr *attr) +{ + return -EOPNOTSUPP; +} + #define cgroup_bpf_enabled(atype) (0) #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, uaddrlen, atype, t_ctx) ({ 0; }) #define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, uaddrlen, atype) ({ 0; }) @@ -501,7 +521,7 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, #define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; }) #define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; }) #define BPF_CGROUP_RUN_PROG_UNIX_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; }) -#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; }) +#define __BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; }) #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(atype, major, minor, access) ({ 0; }) #define BPF_CGROUP_RUN_PROG_SYSCTL(head,table,write,buf,count,pos) ({ 0; }) #define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \ diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 0ecb9418dfbd..580f89c450b5 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -17,6 +17,7 @@ #include <linux/numa.h> #include <linux/mm_types.h> #include <linux/wait.h> +#include <linux/irq_work_types.h> #include <linux/refcount.h> #include <linux/mutex.h> #include <linux/module.h> @@ -90,6 +91,7 @@ struct bpf_map_ops { struct bpf_map *(*map_alloc)(union bpf_attr *attr); void (*map_release)(struct bpf_map *map, struct file *map_file); void (*map_free)(struct bpf_map *map); + void (*map_free_pre_rcu)(struct bpf_map *map); int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); void (*map_release_uref)(struct bpf_map *map); void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key); @@ -214,6 +216,7 @@ enum btf_field_type { BPF_UPTR = (1 << 11), BPF_RES_SPIN_LOCK = (1 << 12), BPF_TASK_WORK = (1 << 13), + BPF_RCU_HEAD = (1 << 14), }; enum bpf_cgroup_storage_type { @@ -268,6 +271,7 @@ struct btf_record { int wq_off; int refcount_off; int task_work_off; + int rcu_head_off; struct btf_field fields[]; }; @@ -337,12 +341,23 @@ struct bpf_map { bool frozen; /* write-once; write-protected by freeze_mutex */ bool free_after_mult_rcu_gp; bool free_after_rcu_gp; + bool was_inner_map; atomic64_t sleepable_refcnt; s64 __percpu *elem_count; u64 cookie; /* write-once */ char *excl_prog_sha; + /* + * Which programs use the map, see bpf_map_claim(): 0 - none so far, + * aux of the program - only that one, the same with BPF_MAP_USER_PATCHED + * set - only that one and it stored the addresses of its functions into + * the map, BPF_MAP_USER_MANY - more than one. + */ + unsigned long user; }; +#define BPF_MAP_USER_MANY 1UL +#define BPF_MAP_USER_PATCHED 1UL + static inline const char *btf_field_type_name(enum btf_field_type type) { switch (type) { @@ -373,6 +388,8 @@ static inline const char *btf_field_type_name(enum btf_field_type type) return "bpf_refcount"; case BPF_TASK_WORK: return "bpf_task_work"; + case BPF_RCU_HEAD: + return "bpf_rcu_head"; default: WARN_ON_ONCE(1); return "unknown"; @@ -413,6 +430,8 @@ static inline u32 btf_field_type_size(enum btf_field_type type) return sizeof(struct bpf_refcount); case BPF_TASK_WORK: return sizeof(struct bpf_task_work); + case BPF_RCU_HEAD: + return sizeof(struct bpf_rcu_head); default: WARN_ON_ONCE(1); return 0; @@ -447,6 +466,8 @@ static inline u32 btf_field_type_align(enum btf_field_type type) return __alignof__(struct bpf_refcount); case BPF_TASK_WORK: return __alignof__(struct bpf_task_work); + case BPF_RCU_HEAD: + return __alignof__(struct bpf_rcu_head); default: WARN_ON_ONCE(1); return 0; @@ -479,6 +500,7 @@ static inline void bpf_obj_init_field(const struct btf_field *field, void *addr) case BPF_KPTR_PERCPU: case BPF_UPTR: case BPF_TASK_WORK: + case BPF_RCU_HEAD: break; default: WARN_ON_ONCE(1); @@ -572,7 +594,7 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, if (long_memcpy) bpf_long_memcpy(dst, src, size); else - memcpy(dst, src, size); + data_race(memcpy(dst, src, size)); return; } @@ -580,10 +602,10 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, u32 next_off = rec->fields[i].offset; u32 sz = next_off - curr_off; - memcpy(dst + curr_off, src + curr_off, sz); + data_race(memcpy(dst + curr_off, src + curr_off, sz)); curr_off += rec->fields[i].size + sz; } - memcpy(dst + curr_off, src + curr_off, size - curr_off); + data_race(memcpy(dst + curr_off, src + curr_off, size - curr_off)); } static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) @@ -780,7 +802,10 @@ enum bpf_type_flag { */ PTR_UNTRUSTED = BIT(6 + BPF_BASE_TYPE_BITS), - /* MEM can be uninitialized. */ + /* + * MEM can be uninitialized. Generic memory outputs need not be fully + * initialized by the callee. + */ MEM_UNINIT = BIT(7 + BPF_BASE_TYPE_BITS), /* DYNPTR points to memory local to the bpf program. */ @@ -874,7 +899,7 @@ enum bpf_type_flag { /* function argument constraints */ enum bpf_arg_type { - ARG_DONTCARE = 0, /* unused argument in helper function */ + ARG_UNUSED = 0, /* unused argument; terminates argument iteration */ /* the following constraints used to prototype * bpf_map_lookup/update/delete_elem() functions @@ -909,6 +934,24 @@ enum bpf_arg_type { ARG_PTR_TO_TIMER, /* pointer to bpf_timer */ ARG_KPTR_XCHG_DEST, /* pointer to destination that kptrs are bpf_kptr_xchg'd into */ ARG_PTR_TO_DYNPTR, /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */ + + ARG_CONST_SCALAR, /* scalar known at verification time */ + ARG_CONST_MEM_SIZE, /* ARG_MEM_SIZE that must be constant */ + ARG_PTR_TO_ALLOC_BTF_ID, /* pointer to an allocated object */ + ARG_PTR_TO_REFCOUNTED_KPTR, /* pointer to a refcounted local kptr */ + ARG_PTR_TO_ITER, /* pointer to an iterator */ + ARG_PTR_TO_LIST_HEAD, /* pointer to bpf_list_head */ + ARG_PTR_TO_LIST_NODE, /* pointer to bpf_list_node */ + ARG_PTR_TO_RB_ROOT, /* pointer to bpf_rb_root */ + ARG_PTR_TO_RB_NODE, /* pointer to bpf_rb_node */ + ARG_PTR_TO_WORKQUEUE, /* pointer to bpf_wq */ + ARG_PTR_TO_TASK_WORK, /* pointer to bpf_task_work */ + ARG_PTR_TO_RCU_HEAD, /* pointer to bpf_rcu_head */ + ARG_PTR_TO_IRQ_FLAG, /* pointer to saved IRQ flags on the stack */ + ARG_PTR_TO_RES_SPIN_LOCK, /* pointer to bpf_res_spin_lock */ + ARG_PTR_TO_CTX_OUT, /* hook output argument passed through from ctx */ + ARG_PTR_TO_PROG_AUX, /* pointer to the caller's bpf_prog_aux */ + ARG_IGNORE, /* argument the verifier does not check at all */ __BPF_ARG_TYPE_MAX, /* Extended arg_types. */ @@ -977,6 +1020,13 @@ static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); */ #define MAX_BPF_FUNC_REG_ARGS 5 +/* A by-value argument takes two eightbytes at most, so the maximum number of + * argument slots of any function is 2 * MAX_BPF_FUNC_ARGS. A local array may + * need that size for processing, although eventually the maximum slots will + * be capped at MAX_BPF_FUNC_ARGS. + */ +#define MAX_BPF_FUNC_ARG_SLOTS (2 * MAX_BPF_FUNC_ARGS) + /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL * instructions after verifying @@ -1005,13 +1055,13 @@ struct bpf_func_proto { }; union { struct { - u32 *arg1_btf_id; - u32 *arg2_btf_id; - u32 *arg3_btf_id; - u32 *arg4_btf_id; - u32 *arg5_btf_id; + const u32 *arg1_btf_id; + const u32 *arg2_btf_id; + const u32 *arg3_btf_id; + const u32 *arg4_btf_id; + const u32 *arg5_btf_id; }; - u32 *arg_btf_id[MAX_BPF_FUNC_ARGS]; + const u32 *arg_btf_id[MAX_BPF_FUNC_ARGS]; struct { size_t arg1_size; size_t arg2_size; @@ -1114,6 +1164,7 @@ struct bpf_insn_access_aux { u32 ref_id; }; }; + u32 mem_size; struct bpf_verifier_log *log; /* for verbose logs */ bool is_retval; /* is accessing function return value ? */ }; @@ -1194,6 +1245,9 @@ struct bpf_prog_offload { u32 jited_len; }; +/* The argument is aligned to 16 bytes. */ +#define BTF_FMODEL_ALIGN16_ARG BIT(0) + /* The argument is signed. */ #define BTF_FMODEL_SIGNED_ARG BIT(1) @@ -1211,6 +1265,11 @@ struct btf_func_model { u8 arg_flags[MAX_BPF_FUNC_ARGS]; }; +static inline u32 btf_func_model_arg_slots(const struct btf_func_model *m, u32 arg) +{ + return (m->arg_size[arg] + sizeof(u64) - 1) / sizeof(u64); +} + /* Restore arguments before returning from trampoline to let original function * continue executing. This flag is used for fentry progs when there are no * fexit progs. @@ -1684,12 +1743,21 @@ static inline void bpf_trampoline_multi_detach(struct bpf_prog *prog, static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) {} #endif +/* The contexts that a global function is called in, and is verified for. */ +enum bpf_func_context { + BPF_FUNC_CTX_NON_SLEEPABLE, + BPF_FUNC_CTX_SLEEPABLE, + /* Sleepable code has disabled preemption or interrupts: no RCU read lock. */ + BPF_FUNC_CTX_NO_RCU_LOCK, + BPF_FUNC_CTX_CNT, +}; + struct bpf_func_info_aux { u16 linkage; bool unreliable; - /* Indexed by in_sleepable. */ - bool called[2]; - bool verified[2]; + bool called[BPF_FUNC_CTX_CNT]; + bool verified[BPF_FUNC_CTX_CNT]; + bool called_with_pkt_ptrs; }; enum bpf_jit_poke_reason { @@ -1721,6 +1789,7 @@ struct bpf_ctx_arg_aux { struct btf *btf; u32 btf_id; u32 ref_id; + u32 mem_size; bool refcounted; }; @@ -1749,12 +1818,18 @@ enum { }; struct bpf_stream { - atomic_t capacity; + refcount_t refcnt; + atomic_t capacity; /* bytes reserved against the stream limit */ + atomic_t readable; /* published bytes available to readers */ struct llist_head log; /* list of in-flight stream elements in LIFO order */ struct mutex lock; /* lock protecting backlog_{head,tail} */ struct llist_node *backlog_head; /* list of in-flight stream elements in FIFO order */ struct llist_node *backlog_tail; /* tail of the list above */ + wait_queue_head_t waitq; + struct irq_work notify_work; + bool notify_used; /* notify_work was queued at least once */ + bool dead; }; struct bpf_stream_stage { @@ -1773,6 +1848,7 @@ enum bpf_sig_keyring { BPF_SIG_KEYRING_SECONDARY, BPF_SIG_KEYRING_PLATFORM, BPF_SIG_KEYRING_USER, + BPF_SIG_KEYRING_BPF, }; struct bpf_prog_aux { @@ -1792,6 +1868,9 @@ struct bpf_prog_aux { u32 ctx_arg_info_size; u32 max_rdonly_access; u32 max_rdwr_access; + /* [min, max): the bytes of a rdwr buffer that may be written */ + u32 min_rdwr_write; + u32 max_rdwr_write; u32 subprog_start; struct btf *attach_btf; struct bpf_ctx_arg_aux *ctx_arg_info; @@ -1807,24 +1886,28 @@ struct bpf_prog_aux { bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ bool attach_tracing_prog; /* true if tracing another tracing program */ bool tramp_linked; /* true if it was ever linked to a trampoline */ + bool sleepable_async_cb; /* has a bpf_wq or bpf_task_work callback */ bool func_proto_unreliable; bool tail_call_reachable; bool xdp_has_frags; bool exception_cb; bool exception_boundary; - bool is_extended; /* true if extended by freplace program */ bool jits_use_priv_stack; bool priv_stack_requested; bool changes_pkt_data; + bool has_prologue; bool might_sleep; + bool can_unwind; bool kprobe_write_ctx; + /* has insns that work with the arena base, see bpf_prog_has_arena_insns() */ + bool uses_arena_insns; struct { s32 keyring_serial; u8 keyring_type; u8 verdict; } sig; u64 prog_array_member_cnt; /* counts how many times as member of prog_array */ - struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */ + struct mutex ext_mutex; /* mutex for freplace_link_cnt and prog_array_member_cnt */ struct bpf_arena *arena; void (*recursion_detected)(struct bpf_prog *prog); /* callback if recursion is detected */ /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ @@ -1856,6 +1939,7 @@ struct bpf_prog_aux { char name[BPF_OBJ_NAME_LEN]; u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64); u16 stack_arg_sp_adjust; + u16 freplace_link_cnt; /* counts freplace links extending this prog */ #ifdef CONFIG_SECURITY void *security; #endif @@ -1892,8 +1976,9 @@ struct bpf_prog_aux { union { struct work_struct work; struct rcu_head rcu; + struct rcu_work rwork; }; - struct bpf_stream stream[2]; + struct bpf_stream *stream[2]; struct mutex st_ops_assoc_mutex; struct bpf_map __rcu *st_ops_assoc; }; @@ -1940,6 +2025,15 @@ struct bpf_prog { }; }; +/* + * Does the program, or a callback of it, run under RCU Tasks Trace protection + * only, outside of any RCU read side critical section? + */ +static inline bool bpf_prog_runs_sleepable(const struct bpf_prog *prog) +{ + return prog->sleepable || prog->aux->sleepable_async_cb; +} + struct bpf_array_aux { /* Programs with direct jumps into programs part of this array. */ struct list_head poke_progs; @@ -2137,6 +2231,18 @@ struct btf_member; * unloaded while in use. * @name: The name of the struct bpf_struct_ops object. * @func_models: Func models + * @cgroup_atype: A value in enum cgroup_bpf_attach_type for cgroup attachment. + * 0 means the struct_ops type does not support cgroup attachment. + * If cgroup_atype is non-zero, the @reg and @unreg must be NULL + * because the attachment/detachment will be handled by the bpf core. + * @free_after_tasks_rcu_gp: Set to true if it needs the bpf core to wait for + * a tasks_rcu gp before freeing the struct_ops map + * and its progs. It is unnecessary if the @unreg + * has waited for the correct rcu gp or the @unreg + * has ensured all struct_ops prog has finished running. + * @free_after_mult_rcu_gp: Same as @free_after_tasks_rcu_gp but waiting for + * both tasks_trace_rcu and regular rcu grace period. + * It is usually needed if the struct_ops has sleepable prog. */ struct bpf_struct_ops { const struct bpf_verifier_ops *verifier_ops; @@ -2155,6 +2261,9 @@ struct bpf_struct_ops { struct module *owner; const char *name; struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; + int cgroup_atype; + bool free_after_tasks_rcu_gp; + bool free_after_mult_rcu_gp; }; /* Every member of a struct_ops type has an instance even a member is not @@ -2289,6 +2398,12 @@ u32 bpf_struct_ops_id(const void *kdata); int bpf_struct_ops_for_each_prog(const void *kdata, int (*cb)(struct bpf_prog *prog, void *data), void *data); +void *bpf_struct_ops_map_kdata(struct bpf_map *map); +void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map); +bool bpf_struct_ops_valid_to_reg(struct bpf_map *map); +int bpf_struct_ops_link_update_check(struct bpf_map *new_map, struct bpf_map *old_map, + struct bpf_map *expected_old_map); +int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map); #ifdef CONFIG_NET /* Define it here to avoid the use of forward declaration */ @@ -2353,6 +2468,32 @@ static inline void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struc static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc) { } +static inline u32 bpf_struct_ops_id(void *kdata) +{ + return 0; +} +static inline void *bpf_struct_ops_map_kdata(struct bpf_map *map) +{ + return NULL; +} +static inline int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map) +{ + return 0; +} +static inline void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map) +{ + return NULL; +} +static inline bool bpf_struct_ops_valid_to_reg(struct bpf_map *map) +{ + return false; +} +static inline int bpf_struct_ops_link_update_check(struct bpf_map *new_map, + struct bpf_map *old_map, + struct bpf_map *expected_old_map) +{ + return -EOPNOTSUPP; +} #endif @@ -2528,7 +2669,10 @@ u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, * since other cpus are walking the array of pointers in parallel. */ struct bpf_prog_array_item { - struct bpf_prog *prog; + union { + struct bpf_prog *prog; + void *kdata; + }; union { struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; u64 bpf_cookie; @@ -2570,6 +2714,7 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array, struct bpf_prog *include_prog, u64 bpf_cookie, struct bpf_prog_array **new_array); +struct bpf_prog *bpf_prog_dummy(void); struct bpf_run_ctx {}; @@ -2588,6 +2733,7 @@ struct bpf_trace_run_ctx { struct bpf_tramp_run_ctx { struct bpf_run_ctx run_ctx; u64 bpf_cookie; + int retval; struct bpf_run_ctx *saved_run_ctx; }; @@ -2821,8 +2967,10 @@ struct bpf_map *bpf_map_get_curr_or_next(u32 *id); struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id); -int bpf_map_alloc_pages(const struct bpf_map *map, int nid, - unsigned long nr_pages, struct page **page_array); +struct page *bpf_alloc_page(int nid, bool sleepable); +int bpf_alloc_pages(int nid, unsigned long nr_pages, + struct llist_head *pages, bool sleepable); +void bpf_free_pages(struct llist_head *pages); #ifdef CONFIG_MEMCG void bpf_map_memcg_enter(const struct bpf_map *map, struct mem_cgroup **old_memcg, struct mem_cgroup **new_memcg); @@ -3008,6 +3156,8 @@ typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog, union bpf_iter_link_info *linfo, struct bpf_iter_aux_info *aux); typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux); +typedef int (*bpf_iter_check_prog_t)(const struct bpf_prog *prog, + const struct bpf_iter_aux_info *aux); typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux, struct seq_file *seq); typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux, @@ -3017,7 +3167,8 @@ typedef const struct bpf_func_proto * const struct bpf_prog *prog); enum bpf_iter_feature { - BPF_ITER_RESCHED = BIT(0), + BPF_ITER_RESCHED = BIT(0), /* may cond_resched() between objects */ + BPF_ITER_NOSLEEP = BIT(1), /* ->show() can't run sleepable progs */ }; #define BPF_ITER_CTX_ARG_MAX 2 @@ -3025,6 +3176,8 @@ struct bpf_iter_reg { const char *target; bpf_iter_attach_target_t attach_target; bpf_iter_detach_target_t detach_target; + /* may the program run on what the link is attached to? */ + bpf_iter_check_prog_t check_prog; bpf_iter_show_fdinfo_t show_fdinfo; bpf_iter_fill_link_info_t fill_link_info; bpf_iter_get_func_proto_t get_func_proto; @@ -3061,6 +3214,8 @@ void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux, struct seq_file *seq); int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux, struct bpf_link_info *info); +int bpf_iter_map_check_value_write(const struct bpf_prog *prog, + const struct bpf_map *map); int map_set_for_each_callback_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, @@ -3218,7 +3373,7 @@ const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn); int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, - u16 btf_fd_idx, u8 **func_addr); + u16 desc_idx, u8 **func_addr); struct bpf_core_ctx { struct bpf_verifier_log *log; @@ -3561,7 +3716,7 @@ bpf_jit_find_kfunc_model(const struct bpf_prog *prog, static inline int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, - u16 btf_fd_idx, u8 **func_addr) + u16 desc_idx, u8 **func_addr) { return -ENOTSUPP; } @@ -3667,6 +3822,7 @@ int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log, void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id); int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr); int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog); +void bpf_prog_dev_bound_moved(struct bpf_prog *prog); void bpf_dev_bound_netdev_unregister(struct net_device *dev); static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) @@ -3728,6 +3884,10 @@ static inline int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, return -EOPNOTSUPP; } +static inline void bpf_prog_dev_bound_moved(struct bpf_prog *prog) +{ +} + static inline void bpf_dev_bound_netdev_unregister(struct net_device *dev) { } @@ -3858,6 +4018,8 @@ struct bpf_key { #if defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL) struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags); struct bpf_key *bpf_lookup_system_key(u64 id); +struct bpf_key *bpf_lookup_keyring(void); +bool bpf_keyring_enforced(void); void bpf_key_put(struct bpf_key *bkey); int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p, const struct bpf_dynptr *sig_p, @@ -3878,6 +4040,16 @@ static inline struct bpf_key *bpf_lookup_system_key(u64 id) return NULL; } +static inline struct bpf_key *bpf_lookup_keyring(void) +{ + return NULL; +} + +static inline bool bpf_keyring_enforced(void) +{ + return false; +} + static inline void bpf_key_put(struct bpf_key *bkey) { } @@ -4138,9 +4310,10 @@ void bpf_bprintf_cleanup(struct bpf_bprintf_data *data); int bpf_try_get_buffers(struct bpf_bprintf_buffers **bufs); void bpf_put_buffers(void); -void bpf_prog_stream_init(struct bpf_prog *prog); +int bpf_prog_stream_init(struct bpf_prog *prog, gfp_t gfp_extra_flags); void bpf_prog_stream_free(struct bpf_prog *prog); -int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len); +int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len); +int bpf_prog_stream_new_fd(struct bpf_prog *prog, enum bpf_stream_id stream_id, u32 flags); void bpf_stream_stage_init(struct bpf_stream_stage *ss); void bpf_stream_stage_free(struct bpf_stream_stage *ss); __printf(2, 3) @@ -4203,7 +4376,7 @@ struct bpf_prog *bpf_prog_find_from_stack(void); int bpf_insn_array_init(struct bpf_map *map, const struct bpf_prog *prog); int bpf_insn_array_ready(struct bpf_map *map); void bpf_insn_array_release(struct bpf_map *map); -void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len); +void bpf_insn_array_adjust(struct bpf_map *map, u32 first, u32 len); void bpf_insn_array_adjust_after_remove(struct bpf_map *map, u32 off, u32 len); #ifdef CONFIG_BPF_SYSCALL @@ -4228,6 +4401,9 @@ static inline bool bpf_map_is_percpu_map(enum bpf_map_type map_type) } } +#define BPF_EXIST_FLAGS (BPF_EXIST | BPF_NOEXIST) +#define BPF_CPU_FLAGS (BPF_F_CPU | BPF_F_ALL_CPUS) + static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 allowed_flags) { u32 cpu; diff --git a/include/linux/bpf_compiler_types.h b/include/linux/bpf_compiler_types.h new file mode 100644 index 000000000000..5c9e71acfdac --- /dev/null +++ b/include/linux/bpf_compiler_types.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_BPF_COMPILER_TYPES_H +#define __LINUX_BPF_COMPILER_TYPES_H + +#include <linux/compiler_types.h> + +/* + * Qualifiers for the BPF verifier rather than address spaces: __arena marks a + * pointer into a BPF arena, __nullable a pointer that may be NULL. They reach + * BPF programs as BTF type tags on the members of kernel structs shared with + * them; to kernel code and to sparse the pointers are ordinary. + */ +#ifdef __CHECKER__ +# define __arena +# define __nullable +#else +# define __arena BTF_TYPE_TAG(arena) +# define __nullable BTF_TYPE_TAG(nullable) +#endif + +#endif /* __LINUX_BPF_COMPILER_TYPES_H */ diff --git a/include/linux/bpf_crypto.h b/include/linux/bpf_crypto.h deleted file mode 100644 index a41e71d4e2d9..000000000000 --- a/include/linux/bpf_crypto.h +++ /dev/null @@ -1,24 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ -#ifndef _BPF_CRYPTO_H -#define _BPF_CRYPTO_H - -struct bpf_crypto_type { - void *(*alloc_tfm)(const char *algo); - void (*free_tfm)(void *tfm); - int (*has_algo)(const char *algo); - int (*setkey)(void *tfm, const u8 *key, unsigned int keylen); - int (*setauthsize)(void *tfm, unsigned int authsize); - int (*encrypt)(void *tfm, const u8 *src, u8 *dst, unsigned int len, u8 *iv); - int (*decrypt)(void *tfm, const u8 *src, u8 *dst, unsigned int len, u8 *iv); - unsigned int (*ivsize)(void *tfm); - unsigned int (*statesize)(void *tfm); - u32 (*get_flags)(void *tfm); - struct module *owner; - char name[14]; -}; - -int bpf_crypto_register_type(const struct bpf_crypto_type *type); -int bpf_crypto_unregister_type(const struct bpf_crypto_type *type); - -#endif /* _BPF_CRYPTO_H */ diff --git a/include/linux/bpf_local_storage.h b/include/linux/bpf_local_storage.h index 9e4f5c45c974..6c643ac51079 100644 --- a/include/linux/bpf_local_storage.h +++ b/include/linux/bpf_local_storage.h @@ -104,6 +104,19 @@ struct bpf_local_storage { refcount_t owner_refcnt;/* Used to pin owner when map_free is uncharging */ }; +/* + * Drop a reference got by refcount_inc_not_zero(&local_storage->owner_refcnt). + * bpf_local_storage_destroy() may have dropped the owner's reference meanwhile + * and wait for the count to reach zero, so this one can be the last. + */ +static inline void +bpf_local_storage_unpin_owner(struct bpf_local_storage *local_storage) +{ + bool last __maybe_unused; + + last = refcount_dec_and_test(&local_storage->owner_refcnt); +} + /* U16_MAX is much more than enough for sk local storage * considering a tcp_sock is ~2k. */ diff --git a/include/linux/bpf_lsm.h b/include/linux/bpf_lsm.h index dda272d78f01..eb4a38eef87e 100644 --- a/include/linux/bpf_lsm.h +++ b/include/linux/bpf_lsm.h @@ -21,6 +21,13 @@ extern bool bpf_lsm_initialized __ro_after_init; #include <linux/lsm_hook_defs.h> #undef LSM_HOOK +/* + * Number of xattr slots the BPF LSM reserves in the array handed to + * security_inode_init_security(), i.e. the maximum number of labels + * a policy may attach to an inode while it is being created. + */ +#define BPF_LSM_INODE_INIT_XATTRS 2 + struct bpf_storage_blob { struct bpf_local_storage __rcu *storage; }; @@ -28,7 +35,7 @@ struct bpf_storage_blob { extern struct lsm_blob_sizes bpf_lsm_blob_sizes; int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, - const struct bpf_prog *prog); + struct bpf_prog *prog); bool bpf_lsm_is_sleepable_hook(u32 btf_id); bool bpf_lsm_is_trusted(const struct bpf_prog *prog); @@ -71,7 +78,7 @@ static inline bool bpf_lsm_is_trusted(const struct bpf_prog *prog) } static inline int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, - const struct bpf_prog *prog) + struct bpf_prog *prog) { return -EOPNOTSUPP; } diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index b83ec99f1a13..5926f13ff43c 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -9,6 +9,20 @@ #include <linux/filter.h> /* for MAX_BPF_STACK */ #include <linux/tnum.h> #include <linux/cnum.h> +#include <linux/math64.h> /* for div_s64_rem() */ + +/* + * Mathematical modulo, the residue of 'v' modulo 'step', normalized to [0, step). + * This differs from C's '%' operator, which truncates the quotient toward zero + * and so returns a remainder with the sign of the dividend (e.g. -1 % 3 == -1, not 2). + */ +static inline u16 imod(s64 v, u16 step) +{ + s32 rem; + + div_s64_rem(v, step, &rem); + return rem < 0 ? rem + step : rem; +} /* Maximum variable offset umax_value permitted when resolving memory accesses. * In practice this is far bigger than any realistic pointer offset; this limit @@ -19,11 +33,12 @@ * that converting umax_value to int cannot overflow. */ #define BPF_MAX_VAR_SIZ (1 << 29) -/* size of tmp_str_buf in bpf_verifier. - * we need at least 306 bytes to fit full stack mask representation - * (in the "-8,-16,...,-512" form) +/* + * size of tmp_str_buf in bpf_verifier. + * we need at least 1399 bytes to fit full stack mask representation + * (in the "-8,-16,...,-2048" form) */ -#define TMP_STR_BUF_LEN 320 +#define TMP_STR_BUF_LEN 1408 /* Patch buffer size */ #define INSN_BUF_SIZE 32 @@ -124,6 +139,13 @@ struct bpf_reg_state { */ struct cnum64 r64; /* 64-bit range as circular number */ struct cnum32 r32; /* 32-bit range as circular number */ + /* + * The value described by this register, interpreted as s64, lies on + * a line described by a linear equation base + step * k. + * Invariant: base < step. + */ + u16 base; + u16 step; /* For PTR_TO_PACKET, used to find other pointers with the same variable * offset, so they can share range knowledge. * For PTR_TO_MAP_VALUE_OR_NULL this is used to share which map value we @@ -162,9 +184,9 @@ struct bpf_reg_state { * other registers. Kept outside the metadata union for ID remapping * during state comparisons. */ - u32 map_uid; + u32 map_uid:31; /* if (!precise && SCALAR_VALUE) min/max/tnum don't affect safety */ - bool precise; + u32 precise:1; }; static inline s64 reg_smin(const struct bpf_reg_state *reg) @@ -243,55 +265,18 @@ enum bpf_stack_slot_type { #define BPF_REG_SIZE 8 /* size of eBPF register in bytes */ +/* + * Largest number of BPF_REG_SIZE stack slots a single frame can have, sized + * for the largest stack budget any JIT supports. A frame may use any part of + * its program's budget; check_max_stack_depth() enforces the budget on the + * combined depth of frames sharing the kernel stack and on each frame using + * a private stack. + */ +#define MAX_BPF_STACK_SLOTS (MAX_BPF_STACK_JIT / BPF_REG_SIZE) + /* 4-byte stack slot granularity for liveness analysis */ #define BPF_HALF_REG_SIZE 4 #define STACK_SLOT_SZ 4 -#define STACK_SLOTS (MAX_BPF_STACK / BPF_HALF_REG_SIZE) /* 128 */ - -typedef struct { - u64 v[2]; -} spis_t; - -#define SPIS_ZERO ((spis_t){}) -#define SPIS_ALL ((spis_t){{ U64_MAX, U64_MAX }}) - -static inline bool spis_is_zero(spis_t s) -{ - return s.v[0] == 0 && s.v[1] == 0; -} - -static inline bool spis_equal(spis_t a, spis_t b) -{ - return a.v[0] == b.v[0] && a.v[1] == b.v[1]; -} - -static inline spis_t spis_or(spis_t a, spis_t b) -{ - return (spis_t){{ a.v[0] | b.v[0], a.v[1] | b.v[1] }}; -} - -static inline spis_t spis_and(spis_t a, spis_t b) -{ - return (spis_t){{ a.v[0] & b.v[0], a.v[1] & b.v[1] }}; -} - -static inline spis_t spis_not(spis_t s) -{ - return (spis_t){{ ~s.v[0], ~s.v[1] }}; -} - -static inline bool spis_test_bit(spis_t s, u32 slot) -{ - return s.v[slot / 64] & BIT_ULL(slot % 64); -} - -static inline void spis_or_range(spis_t *mask, u32 lo, u32 hi) -{ - u32 w; - - for (w = lo; w <= hi && w < STACK_SLOTS; w++) - mask->v[w / 64] |= BIT_ULL(w % 64); -} #define BPF_REGMASK_ARGS ((1 << BPF_REG_1) | (1 << BPF_REG_2) | \ (1 << BPF_REG_3) | (1 << BPF_REG_4) | \ @@ -343,6 +328,25 @@ struct bpf_retval_range { bool return_32bit; }; +/* SCEV/widening register space: r0..r10 plus every stack slot of a frame. */ +#define BPF_SCEV_REGS_NUM (MAX_BPF_REG + MAX_BPF_STACK_SLOTS) + +struct bpf_loop_iters { + u32 min_header_count; /* min number of times header is executed */ + u32 max_header_count; /* max number of times header is executed */ + unsigned long precise_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)]; +}; + +struct loop_stack_entry { + struct bpf_verifier_state *entry_state; + u32 min_header_count; + u32 max_header_count; + u32 loop_id:31; + u32 terminates:1; +}; + +#define LOOP_STACK_SIZE 16 + /* state of the program: * type of all registers and stack info */ @@ -375,6 +379,11 @@ struct bpf_func_state { bool in_async_callback_fn; bool in_exception_callback_fn; bool no_stack_arg_load; + /* + * The helper invokes this callback under rcu_read_lock(), so the + * callback and its callees run in an RCU read-side critical section. + */ + bool in_rcu_callback_fn; /* For callback calling functions that limit number of possible * callback executions (e.g. bpf_loop) keeps track of current * simulated iteration number. @@ -388,6 +397,11 @@ struct bpf_func_state { u32 callback_depth; /* Instructions processed in this frame and callees on the current path. */ u32 insns_subtotal; + /* + * Control-flow loop nesting at the current insn within this frame's + * subprogram (loops never cross subprogram boundaries). + */ + u32 loop_stack_cnt; /* The following fields should be last. See copy_func_state() */ /* The state of the stack. Each element of the array describes BPF_REG_SIZE @@ -405,6 +419,7 @@ struct bpf_func_state { u16 out_stack_arg_cnt; /* Number of outgoing on-stack argument slots */ struct bpf_reg_state *stack_arg_regs; /* Outgoing on-stack arguments */ + struct loop_stack_entry *loop_stack; }; #define MAX_CALL_FRAMES 16 @@ -421,30 +436,37 @@ enum { INSN_F_STACK_ARG_ACCESS = BIT(3), }; +/* Registers linked to one jump condition that a history entry can record */ +#define BPF_LINKED_REGS_MAX 5 + struct bpf_jmp_history_entry { /* insn idx can't be bigger than 1 million */ u32 idx : 20; u32 frame : 4; /* stack access frame number */ - u32 spi : 6; /* stack slot index (0..63) */ - u32 : 2; - u32 prev_idx : 20; /* special INSN_F_xxx flags */ u32 flags : 4; - u32 : 8; + u32 : 4; + u32 prev_idx : 20; + u32 spi : 12; /* stack slot index */ /* - * additional registers that need precision tracking when this - * jump is backtracked, vector of five 11-bit records + * Scalar registers and spilled scalars linked to the condition of + * this jump, which need precision tracking together when the jump is + * backtracked. Each is packed as 4 bits of frame number, one bit + * telling a register from a stack slot and 11 bits of register or + * slot index, see linked_regs_pack(). */ - u64 linked_regs; + u16 linked_regs[BPF_LINKED_REGS_MAX]; + u8 linked_regs_cnt; }; static_assert(MAX_CALL_FRAMES <= (1 << 4)); -static_assert(MAX_BPF_STACK / 8 <= (1 << 6)); +static_assert(MAX_BPF_STACK_SLOTS <= (1 << 12)); /* Maximum number of bpf_reg_state objects that can exist at once */ #define MAX_STACK_ARG_SLOTS (MAX_BPF_FUNC_ARGS - MAX_BPF_FUNC_REG_ARGS) -#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE + \ - MAX_STACK_ARG_SLOTS) * MAX_CALL_FRAMES) +#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK_SLOTS + MAX_STACK_ARG_SLOTS) * \ + MAX_CALL_FRAMES) + struct bpf_verifier_state { /* call stack tracking */ struct bpf_func_state *frame[MAX_CALL_FRAMES]; @@ -510,8 +532,13 @@ struct bpf_verifier_state { bool speculative; bool in_sleepable; + /* The callee left with unwind_exit. Next insn is unwind_goto. */ + bool unwound; - /* first and last insn idx of this verifier state */ + /* + * First and last insn idx of this verifier state. + * last_insn_idx is -1 if no instructions have been executed yet. + */ u32 first_insn_idx; u32 last_insn_idx; /* if this state is a backedge state then equal_state @@ -530,12 +557,31 @@ struct bpf_verifier_state { u32 may_goto_depth; }; +/* Number of BPF_REG_SIZE stack slots tracked for the frame so far. */ +static inline u32 bpf_stack_nr_slots(const struct bpf_func_state *frame) +{ + return frame->allocated_stack / BPF_REG_SIZE; +} + +/* + * Stack slot @spi of @frame, covering bytes [fp - (spi + 1) * 8, fp - spi * 8). + * The caller must ensure spi < bpf_stack_nr_slots(frame), see bpf_grow_stack_state(). + */ +static inline struct bpf_stack_state *bpf_stack_slot(const struct bpf_func_state *frame, u32 spi) +{ + return &frame->stack[spi]; +} + static inline struct bpf_reg_state * bpf_get_spilled_reg(int slot, struct bpf_func_state *frame, u32 mask) { - if (slot < frame->allocated_stack / BPF_REG_SIZE && - (1 << frame->stack[slot].slot_type[BPF_REG_SIZE - 1]) & mask) - return &frame->stack[slot].spilled_ptr; + struct bpf_stack_state *ss; + + if (slot >= bpf_stack_nr_slots(frame)) + return NULL; + ss = bpf_stack_slot(frame, slot); + if ((1 << ss->slot_type[BPF_REG_SIZE - 1]) & mask) + return &ss->spilled_ptr; return NULL; } @@ -551,7 +597,7 @@ bpf_get_spilled_stack_arg(int slot, struct bpf_func_state *frame) /* Iterate over 'frame', setting 'reg' to either NULL or a spilled register. */ #define bpf_for_each_spilled_reg(iter, frame, reg, mask) \ for (iter = 0, reg = bpf_get_spilled_reg(iter, frame, mask); \ - iter < frame->allocated_stack / BPF_REG_SIZE; \ + iter < bpf_stack_nr_slots(frame); \ iter++, reg = bpf_get_spilled_reg(iter, frame, mask)) /* Iterate over 'frame', setting 'reg' to either NULL or a spilled stack arg. */ @@ -576,7 +622,7 @@ bpf_get_spilled_stack_arg(int slot, struct bpf_func_state *frame) bpf_for_each_spilled_reg(___j, __state, __reg, __mask) { \ if (!__reg) \ continue; \ - __stack = &__state->stack[___j]; \ + __stack = bpf_stack_slot(__state, ___j); \ (void)(__expr); \ } \ __stack = NULL; \ @@ -640,6 +686,52 @@ struct bpf_iarray { u32 items[]; }; +#define iarray_for_each(item, arr) \ + for (int ___idx = 0; \ + ___idx < (arr)->cnt && ({ item = (arr)->items[___idx]; 1; }); \ + ___idx++) + +#define MAX_BACKEDGES 16 +#define MAX_LOOP_EXITS 256 + +/* Number of exit conditions dominating a backedge. */ +#define MAX_LATCHES 4 + +struct bpf_backedge { + int from; + /* + * Conditional jumps within the loop that dominate 'from' and have + * a successor outside the loop, nearest first. latches[0] is the + * latch; the remaining entries are additional exit conditions. + * Each usable condition gives an upper bound on the iteration count, + * so keeping only a subset is sound. + */ + int latches[MAX_LATCHES]; + u8 latches_cnt; +}; + +struct bpf_loop_exit { + int from; /* instruction inside the loop */ + int to; /* instruction outside the loop */ +}; + +struct bpf_loop { + struct bpf_backedge backedges[MAX_BACKEDGES]; + /* edges exiting from this loop, includes edges from nested loops */ + struct bpf_loop_exit *exits; + int backedges_cnt; + int exits_cnt; + bool irreducible; + bool backedges_overflow; + bool exits_overflow; + /* + * Loop-entry registers used to compute stack addresses for loads, + * stores and calls requiring fixed stack offsets, in this and nested + * loops. + */ + unsigned long store_base_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)]; +}; + struct bpf_insn_aux_data { union { enum bpf_reg_type ptr_type; /* pointer type for load/store insns */ @@ -671,45 +763,64 @@ struct bpf_insn_aux_data { /* remember the offset of node field within type to rewrite */ u64 insert_off; }; - struct bpf_iarray *jt; /* jump table for gotox or bpf_tailcall call instruction */ struct btf_struct_meta *kptr_struct_meta; u64 map_key_state; /* constant (32 bit) key tracking for maps */ + /* + * Per-instruction summary of stack slots in the current frame + * that this instruction may write to. + */ + DECLARE_BITMAP(may_write_mask, MAX_BPF_STACK_SLOTS); + DECLARE_BITMAP(live_stack_before, MAX_BPF_STACK_SLOTS); + u16 stack_ptrs; /* bitmask of regs that may hold a frame pointer here (arg_track) */ int ctx_field_size; /* the ctx field size for load insn, maybe 0 */ u32 seen; /* this insn was processed by the verifier at env->pass_cnt */ - bool nospec; /* do not execute this instruction speculatively */ - bool nospec_result; /* result is unsafe under speculation, nospec must follow */ - bool zext_dst; /* this insn zero extends dst reg */ - bool needs_zext; /* alu op needs to clear upper bits */ - bool prevent_zext; /* alu op cannot be zext (already used with 64-bit scalars) */ - bool non_sleepable; /* helper/kfunc may be called from non-sleepable context */ - bool is_iter_next; /* bpf_iter_<type>_next() kfunc call */ - bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */ - u8 alu_state; /* used in combination with alu_limit */ + u64 nospec:1; /* do not execute this instruction speculatively */ + u64 nospec_result:1; /* result is unsafe under speculation, nospec must follow */ + u64 zext_dst:1; /* this insn zero extends dst reg */ + u64 needs_zext:1; /* alu op needs to clear upper bits */ + u64 prevent_zext:1; /* alu op cannot be zext (already used with 64-bit scalars) */ + u64 non_sleepable:1; /* helper/kfunc may be called from non-sleepable context */ + u64 is_iter_next:1; /* bpf_iter_<type>_next() kfunc call */ + u64 call_with_percpu_alloc_ptr:1; /* {this,per}_cpu_ptr() with prog percpu alloc */ + u64 arena_scalar:1; /* ldx/stx/st/atomic through a number, it's an address in arena */ + u64 bb_end:1; /* last instruction of a basic block */ + u64 need_scev:1; + u64 alu_state:5; /* used in combination with alu_limit */ /* true if STX or LDX instruction is a part of a spill/fill * pattern for a bpf_fastcall call. */ - u8 fastcall_pattern:1; + u64 fastcall_pattern:1; /* for CALL instructions, a number of spill/fill pairs in the * bpf_fastcall pattern. */ - u8 fastcall_spills_num:3; - u8 arg_prog:4; + u64 fastcall_spills_num:3; + u64 arg_prog:4; + /* + * What a load found, on any path: an __arena pointer member of a + * kernel struct, which arena_member_fixup() rewrites, one that is + * __nullable as well, or anything else through any pointer. + */ + u64 arena_member:1; + u64 arena_member_nullable:1; + u64 non_arena_load:1; - /* below fields are initialized once */ - unsigned int orig_idx; /* original instruction index */ - u32 jmp_point:1; - u32 prune_point:1; + /* below flags are initialized once */ + u64 jmp_point:1; + u64 prune_point:1; /* ensure we check state equivalence and save state checkpoint and * this instruction, regardless of any heuristics */ - u32 force_checkpoint:1; + u64 force_checkpoint:1; /* true if instruction is a call to a helper function that * accepts callback function as a parameter. */ - u32 calls_callback:1; - u32 indirect_target:1; /* if it is an indirect jump target */ + u64 calls_callback:1; + u64 indirect_target:1; /* if it is an indirect jump target */ + u64 non_stack_access:1; /* instruction can access non-stack memory */ /* true if some jump or call instruction targets this instruction */ - u32 jump_target:1; + u64 jump_target:1; + + unsigned int orig_idx; /* original instruction index, initialized once */ /* * CFG strongly connected component this instruction belongs to, * zero if it is a singleton SCC. @@ -729,12 +840,23 @@ struct bpf_insn_aux_data { u16 const_reg_map_mask; u16 const_reg_subprog_mask; u32 const_reg_vals[10]; + /* + * Header index of the innermost loop containing this instruction, -1 if none. + * For a loop header, identifies the parent loop instead. + */ + s32 loop_header; + /* additional information about the loop if this instruction is a loop header */ + struct bpf_loop *loop; }; #define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */ #define MAX_USED_BTFS 64 /* max number of BTFs accessed by one BPF program */ -#define BPF_VERIFIER_TMP_LOG_SIZE 1024 +/* + * Longest line the verifier log can carry: a full stack mask of + * MAX_BPF_STACK_SLOTS slots, see TMP_STR_BUF_LEN, plus its prefix. + */ +#define BPF_VERIFIER_TMP_LOG_SIZE 2048 struct bpf_verifier_log { /* Logical start and end positions of a "log window" of the verifier log. @@ -787,6 +909,21 @@ int bpf_log_attr_finalize(struct bpf_log_attr *attr, struct bpf_verifier_log *lo #define BPF_MAX_SUBPROGS 256 +/* + * A pointer to a static subprog in the value of a frozen read-only array map: + * a 64-bit value that is the offset in bytes of the first instruction of + * the subprog in the program. + */ +struct bpf_func_ptr { + struct bpf_map *map; + u32 map_off; /* offset of the pointer in the value of the map */ + u32 orig_off; /* what the map has: the first instruction of the subprog */ + u32 xlated_off; /* the same after instructions were patched and removed */ +}; + +/* the subprog that a bpf_func_ptr pointed to was removed as dead code */ +#define BPF_FUNC_PTR_DELETED ((u32)-1) + struct bpf_subprog_arg_info { enum bpf_arg_type arg_type; union { @@ -806,7 +943,12 @@ struct bpf_subprog_info { u32 start; /* insn idx of function entry point */ u32 linfo_idx; /* The idx to the main_prog->aux->linfo */ u32 postorder_start; /* The idx to the env->cfg.insn_postorder */ - u32 exit_idx; /* Index of one of the BPF_EXIT instructions in this subprogram */ + /* + * Index of one of the BPF_EXIT instructions in this subprogram, or + * U32_MAX when it has none. + */ + u32 exit_idx; + struct bpf_iarray *jt; /* jump table shared by all gotox of this subprogram */ u16 stack_depth; /* max. stack depth used by this function */ u16 stack_extra; u32 insns_total; @@ -817,17 +959,20 @@ struct bpf_subprog_info { s16 fastcall_stack_off; bool has_tail_call: 1; bool might_throw: 1; + bool can_unwind: 1; /* has unwind_exit */ bool tail_call_reachable: 1; bool has_ld_abs: 1; bool is_cb: 1; bool is_async_cb: 1; bool is_exception_cb: 1; bool args_cached: 1; + /* true if the return value is passed in the R0:R2 register pair */ + bool ret_reg_pair: 1; /* true if bpf_fastcall stack region is used by functions that can't be inlined */ bool keep_fastcall_stack: 1; bool changes_pkt_data: 1; bool might_sleep: 1; - u8 arg_cnt:4; + u8 arg_slot_cnt:4; enum priv_stack_mode priv_stack_mode; struct bpf_subprog_arg_info args[MAX_BPF_FUNC_ARGS]; @@ -837,8 +982,8 @@ struct bpf_subprog_info { static inline u16 bpf_in_stack_arg_cnt(const struct bpf_subprog_info *sub) { - if (sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS) - return sub->arg_cnt - MAX_BPF_FUNC_REG_ARGS; + if (sub->arg_slot_cnt > MAX_BPF_FUNC_REG_ARGS) + return sub->arg_slot_cnt - MAX_BPF_FUNC_REG_ARGS; return 0; } @@ -849,7 +994,7 @@ struct backtrack_state { struct bpf_verifier_env *env; u32 frame; u32 reg_masks[MAX_CALL_FRAMES]; - u64 stack_masks[MAX_CALL_FRAMES]; + unsigned long stack_masks[MAX_CALL_FRAMES][BITS_TO_LONGS(MAX_BPF_STACK_SLOTS)]; u8 stack_arg_masks[MAX_CALL_FRAMES]; }; @@ -905,6 +1050,7 @@ struct bpf_scc_info { }; struct bpf_liveness; +struct scev; struct bpf_fd_array { union { @@ -928,6 +1074,7 @@ struct bpf_verifier_env { bool strict_alignment; /* perform strict pointer alignment checks */ bool test_state_freq; /* test verifier with different pruning frequency */ bool test_reg_invariants; /* fail verification on register invariants violations */ + bool arena_scalar; /* load and store through a number is arena access */ struct bpf_verifier_state *cur_state; /* current verifier state */ /* Search pruning optimization, array of list_heads for * lists of struct bpf_verifier_state_list. @@ -954,6 +1101,8 @@ struct bpf_verifier_env { bool bypass_spec_v1; bool bypass_spec_v4; bool seen_direct_write; + /* a global function is verified for BPF_FUNC_CTX_NO_RCU_LOCK */ + bool no_rcu_read_lock; bool seen_exception; bool signature; u32 insn_aux_data_len; @@ -961,9 +1110,24 @@ struct bpf_verifier_env { const struct bpf_line_info *prev_linfo; struct bpf_verifier_log log; struct bpf_diag *diag; + struct bpf_func_proto bpf_subprog_scratch; struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */ /* subprog indices sorted in topological order: leaves first, callers last */ int subprog_topo_order[BPF_MAX_SUBPROGS + 2]; + /* + * Pointers to static subprogs found in frozen read-only maps of the + * program, see resolve_func_ptrs(). Sorted by map and map_off. + */ + struct bpf_func_ptr *func_ptrs; + u32 func_ptr_cnt; + bool has_callx; + /* + * Call graph edges created by callx instructions. A bitmap of + * subprog_cnt * subprog_cnt bits, where bit (caller * subprog_cnt + callee) + * is set when the main verification pass sees 'caller' calling 'callee' + * via callx. Allocated when the first such edge is recorded. + */ + unsigned long *callx_edges; union { struct bpf_idmap idmap_scratch; struct bpf_idset idset_scratch; @@ -976,14 +1140,17 @@ struct bpf_verifier_env { * see bpf_subprog_info->postorder_start. */ int *insn_postorder; + int *postorder_nums; int cur_stack; /* current position in the insn_postorder vector */ int cur_postorder; + u32 gotox_edges; + bool subprog_jts_ready; } cfg; struct backtrack_state bt; struct bpf_jmp_history_entry *cur_hist_ent; - /* Per-callsite copy of parent's converged at_stack_in for cross-frame fills. */ - struct arg_track **callsite_at_stack; + /* Per-callsite snapshot of the parent's tracked spill slots for cross-frame fills. */ + struct spill_snapshot **callsite_at_stack; u32 pass_cnt; /* number of times do_check() was called */ u32 subprog_cnt; /* number of instructions analyzed by the verifier */ @@ -992,6 +1159,8 @@ struct bpf_verifier_env { u32 prev_jmps_processed, jmps_processed; /* maximum combined stack depth */ u32 max_stack_depth; + /* stack budget of the program, see bpf_prog_stack_limit() */ + u32 stack_limit; /* total verification time */ u64 verification_time; /* maximum number of verifier states kept in 'branching' instructions */ @@ -1027,7 +1196,7 @@ struct bpf_verifier_env { */ u32 scratched_regs; /* Same as scratched_regs but for stack slots */ - u64 scratched_stack_slots; + DECLARE_BITMAP(scratched_stack_slots, MAX_BPF_STACK_SLOTS); u64 prev_log_pos, prev_insn_print_pos; /* buffer used to temporary hold constants as scalar registers */ struct bpf_reg_state fake_reg[1]; @@ -1047,6 +1216,8 @@ struct bpf_verifier_env { u32 scc_cnt; struct bpf_iarray *succ; struct bpf_iarray *gotox_tmp_buf; + int *idoms; + struct scev *scev; }; static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog) @@ -1059,8 +1230,13 @@ static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env return &env->subprog_info[subprog]; } +static inline bool bpf_ret_reg_pair(struct bpf_verifier_env *env, int subprog) +{ + return subprog_info(env, subprog)->ret_reg_pair; +} + struct bpf_call_summary { - u8 num_params; + u8 arg_slot_cnt; bool is_void; bool fastcall; }; @@ -1083,6 +1259,12 @@ static inline bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn) insn->src_reg == BPF_PSEUDO_KFUNC_CALL; } +/* callx: indirect call of a bpf subprog whose address is in insn->dst_reg */ +static inline bool bpf_is_callx(const struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_CALL | BPF_X); +} + __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, va_list args); __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, @@ -1097,6 +1279,7 @@ int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual); __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env, u32 insn_off, const char *prefix_fmt, ...); +void bpf_verbose_snum(struct bpf_verifier_env *env, s64 num); #define verifier_bug_if(cond, env, fmt, args...) \ ({ \ @@ -1209,11 +1392,14 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *tgt_prog, u32 btf_id, struct bpf_attach_target_info *tgt_info); +int bpf_check_ext_prologue(struct bpf_verifier_log *log, + const struct bpf_prog *tgt_prog, u32 btf_id); void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab); int mark_chain_precision(struct bpf_verifier_env *env, int regno); int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx); +int bpf_split_cur_state(struct bpf_verifier_env *env); int bpf_update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st); void bpf_clear_jmp_history(struct bpf_verifier_state *state); @@ -1223,11 +1409,14 @@ struct list_head *bpf_explored_state(struct bpf_verifier_env *env, int idx); void bpf_free_verifier_state(struct bpf_verifier_state *state, bool free_self); void bpf_free_backedges(struct bpf_scc_visit *visit); int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, - int insn_flags, int spi, int frame, u64 linked_regs); + int insn_flags, int spi, int frame, const u16 *linked_regs, + u8 linked_regs_cnt); void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist); void bpf_mark_reg_not_init(const struct bpf_verifier_env *env, struct bpf_reg_state *reg); +void bpf_mark_reg_known_scalar(struct bpf_reg_state *reg, u64 imm); void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg); +int bpf_grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size); void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env, struct bpf_verifier_state *st); void bpf_clear_singular_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st); @@ -1241,6 +1430,24 @@ static inline int bpf_get_spi(s32 off) } /* + * Stack a program may use in total: combined over the frames of a call + * chain on the kernel stack, or per frame on a private stack. Any single + * frame may reach that deep. Only a JIT that lays out such frames may go + * beyond MAX_BPF_STACK, the interpreter's frame size, and only one whose + * tail calls let the target set up its own frame: without subprogram + * tail calls, do_misc_fixups() gives every program with tail calls a + * MAX_BPF_STACK frame, which a deeper frame would overrun. + */ +static inline u32 bpf_prog_stack_limit(const struct bpf_prog *prog) +{ + /* an offloaded program never runs on the host JIT, whatever it supports */ + if (prog->jit_requested && !bpf_prog_is_offloaded(prog->aux) && + bpf_jit_supports_large_stack() && bpf_jit_supports_subprog_tailcalls()) + return MAX_BPF_STACK_JIT; + return MAX_BPF_STACK; +} + +/* * Return the function state a stack pointer register refers to. frameno * shares storage with other pointer metadata, so return NULL for any * other register type instead of indexing frame[] with aliased bytes. @@ -1273,6 +1480,15 @@ static inline bool bpf_is_spilled_reg(const struct bpf_stack_state *stack) return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL; } +static inline int bpf_spill_size(const struct bpf_stack_state *stack) +{ + int i, spill_size = 0; + + for (i = BPF_REG_SIZE - 1; i >= 0 && stack->slot_type[i] == STACK_SPILL; i--) + spill_size++; + return spill_size; +} + static inline bool bpf_is_spilled_scalar_reg(const struct bpf_stack_state *stack) { return bpf_is_spilled_reg(stack) && stack->spilled_ptr.type == SCALAR_VALUE; @@ -1290,12 +1506,7 @@ static inline void bpf_bt_set_frame_reg(struct backtrack_state *bt, u32 frame, u static inline void bpf_bt_set_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot) { - bt->stack_masks[frame] |= 1ull << slot; -} - -static inline void bpf_bt_set_frame_slot_mask(struct backtrack_state *bt, u32 frame, u64 mask) -{ - bt->stack_masks[frame] |= mask; + __set_bit(slot, bt->stack_masks[frame]); } static inline void bt_set_frame_stack_arg_slot(struct backtrack_state *bt, u32 frame, u32 slot) @@ -1310,10 +1521,17 @@ static inline bool bt_is_frame_reg_set(struct backtrack_state *bt, u32 frame, u3 static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot) { - return bt->stack_masks[frame] & (1ull << slot); + return test_bit(slot, bt->stack_masks[frame]); } bool bpf_map_is_rdonly(const struct bpf_map *map); +struct bpf_func_ptr *bpf_map_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, u32 *cnt); +struct bpf_func_ptr *bpf_map_range_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, + u64 off, u64 size, u32 *cnt); +void bpf_adjust_func_ptrs(struct bpf_verifier_env *env, u32 off, u32 len); +void bpf_adjust_func_ptrs_after_remove(struct bpf_verifier_env *env, u32 off, u32 len); int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, bool is_ldsx); @@ -1441,7 +1659,7 @@ static inline void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno) static inline void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi) { - env->scratched_stack_slots |= 1ULL << spi; + __set_bit(spi, env->scratched_stack_slots); } static inline bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) @@ -1449,27 +1667,28 @@ static inline bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) return (env->scratched_regs >> regno) & 1; } -static inline bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno) +static inline bool stack_slot_scratched(const struct bpf_verifier_env *env, u32 spi) { - return (env->scratched_stack_slots >> regno) & 1; + return test_bit(spi, env->scratched_stack_slots); } static inline bool verifier_state_scratched(const struct bpf_verifier_env *env) { - return env->scratched_regs || env->scratched_stack_slots; + return env->scratched_regs || + !bitmap_empty(env->scratched_stack_slots, MAX_BPF_STACK_SLOTS); } static inline void mark_verifier_state_clean(struct bpf_verifier_env *env) { env->scratched_regs = 0U; - env->scratched_stack_slots = 0ULL; + bitmap_zero(env->scratched_stack_slots, MAX_BPF_STACK_SLOTS); } /* Used for printing the entire verifier state. */ static inline void mark_verifier_state_scratched(struct bpf_verifier_env *env) { env->scratched_regs = ~0U; - env->scratched_stack_slots = ~0ULL; + bitmap_fill(env->scratched_stack_slots, MAX_BPF_STACK_SLOTS); } static inline bool bpf_stack_narrow_access_ok(int off, int fill_size, int spill_size) @@ -1504,14 +1723,25 @@ struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *en const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog); int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); -void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); +void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, const unsigned long *stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); +/* Kinds of member a by-value struct or union may be composed of. */ +enum btf_member_kind { + BTF_MEMBER_SCALAR = BIT(0), /* an int or an enum */ + BTF_MEMBER_ARENA_PTR = BIT(1), /* a pointer carrying the "arena" type tag */ +}; + +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, u32 member_kinds); +u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t); + int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); int bpf_compute_const_regs(struct bpf_verifier_env *env); int bpf_prune_dead_branches(struct bpf_verifier_env *env); int bpf_check_cfg(struct bpf_verifier_env *env); +void bpf_free_subprog_jts(struct bpf_verifier_env *env); int bpf_compute_postorder(struct bpf_verifier_env *env); int bpf_compute_scc(struct bpf_verifier_env *env); @@ -1540,12 +1770,15 @@ struct ref_obj_desc { }; /* - * A memory argument a call fills in. The verifier allows the stack to be uninitialized if - * the range is a known constant. Stack slots are marked as STACK_MISC by check_mem_access(). + * Generic MEM_UNINIT arguments, indexed by ABI slot. var_size_mask excludes + * variable-sized buffers from raw mode without losing the output annotation. + * size records constant ranges to mark initialized after checking all arguments, + * only when the caller is allowed to read uninitialized stack memory. */ struct arg_raw_mem_desc { - u8 regno; - int size; + u16 mask; + u16 var_size_mask; + int size[MAX_BPF_FUNC_ARGS]; }; /* Size of PTR_TO_MEM returned, taken from a constant allocation-size argument */ @@ -1565,6 +1798,7 @@ struct bpf_call_arg_meta { struct btf *btf; u32 func_id; const struct bpf_func_proto *fn; + const struct btf_type *func_proto; u8 release_regno; u32 ret_btf_id; u32 subprogno; @@ -1572,20 +1806,25 @@ struct bpf_call_arg_meta { struct bpf_dynptr_desc dynptr; struct ref_obj_desc ref_obj; struct ret_mem_desc ret_mem; + struct arg_raw_mem_desc arg_raw_mem; + + /* Only set by subprog */ + bool subprog_may_change_pkt; /* Only set by kfunc */ bool r0_rdonly; u32 kfunc_flags; - const struct btf_type *func_proto; const char *func_name; struct arg_constant_desc arg_constant; + /* The type of the dynptr that the kfunc writes through */ + enum bpf_dynptr_type dynptr_write_type; /* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling, * generally to pass info about user-defined local kptr types to later * verification logic * bpf_obj_drop/bpf_percpu_obj_drop * Record the local kptr type to be drop'd - * bpf_refcount_acquire (via KF_ARG_PTR_TO_REFCOUNTED_KPTR arg type) + * bpf_refcount_acquire (via ARG_PTR_TO_REFCOUNTED_KPTR arg type) * Record the local kptr type to be refcount_incr'd and use * arg_owning_ref to determine whether refcount_acquire should be * fallible @@ -1593,7 +1832,6 @@ struct bpf_call_arg_meta { struct btf *arg_btf; u32 arg_btf_id; bool arg_owning_ref; - bool arg_prog; struct { struct btf_field *field; @@ -1611,7 +1849,6 @@ struct bpf_call_arg_meta { s64 const_map_key; struct btf *ret_btf; struct btf_field *kptr_field; - struct arg_raw_mem_desc arg_raw_mem; }; int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, @@ -1638,17 +1875,37 @@ bool bpf_is_may_goto_insn(struct bpf_insn *insn); void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn); bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, struct bpf_call_summary *cs); -s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, - struct bpf_insn *insn, int arg, - int insn_idx); -s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, - struct bpf_insn *insn, int arg, - int insn_idx); +/* Stack effects used for state pruning; must_write implies may_write. */ +struct arg_access_info { + u32 size; /* Maximum extent; U32_MAX if unknown. */ + u8 may_read:1; /* Incoming contents or initialization may be needed. */ + u8 may_write:1; + u8 must_write:1; /* Prior verifier state is destroyed throughout size. */ +}; + +struct arg_access_info +bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, + struct bpf_insn *insn, int arg, int insn_idx); +struct arg_access_info +bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, + struct bpf_insn *insn, int arg, int insn_idx); +struct arg_access_info +bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env, + struct bpf_insn *insn, int arg, int insn_idx); int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env); +bool bpf_same_memory_origin(const struct bpf_reg_state *reg_a, + const struct bpf_reg_state *reg_b); +int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + struct cnum64 r64, struct cnum32 r32, u16 base, u16 step); +bool bpf_reg_union_compatible(const struct bpf_reg_state *a, const struct bpf_reg_state *b); +int bpf_reg_union(struct bpf_verifier_env *env, struct bpf_reg_state *acc, + const struct bpf_reg_state *src); int bpf_stack_liveness_init(struct bpf_verifier_env *env); void bpf_stack_liveness_free(struct bpf_verifier_env *env); int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st); +const unsigned long *bpf_may_write_mask(struct bpf_verifier_env *env, u32 insn_idx); +bool bpf_needs_fixed_stack_off(struct bpf_verifier_env *env, int insn_idx); bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 spi); int bpf_compute_live_registers(struct bpf_verifier_env *env); @@ -1680,6 +1937,21 @@ static inline u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); } +static inline bool bpf_is_mem_insn(struct bpf_insn *insn) +{ + if (BPF_CLASS(insn->code) != BPF_ST && + BPF_CLASS(insn->code) != BPF_STX && + BPF_CLASS(insn->code) != BPF_LDX) + return false; + + if (insn->code == (BPF_ST | BPF_NOSPEC)) + return false; + + return (BPF_MODE(insn->code) == BPF_MEM || + BPF_MODE(insn->code) == BPF_MEMSX || + BPF_MODE(insn->code) == BPF_ATOMIC); +} + #define MAX_PACKET_OFF 0xffff #define CALLER_SAVED_REGS 6 @@ -1690,30 +1962,34 @@ enum bpf_reg_arg_type { }; #define MAX_KFUNC_DESCS 256 +/* Each kfunc can have its canonical and one specialized call target. */ +#define MAX_KFUNC_CALL_DESCS (MAX_KFUNC_DESCS * 2) +static_assert(MAX_KFUNC_CALL_DESCS <= S16_MAX + 1); struct bpf_kfunc_desc { struct btf_func_model func_model; struct bpf_func_proto proto; u32 func_id; - s32 imm; u16 offset; unsigned long addr; }; struct bpf_kfunc_desc_tab { u32 nr_descs; + u32 nr_base_descs; /* Sorted by func_id (BTF ID) and offset (fd_array offset) during - * verification. JITs do lookups by bpf_insn, where func_id may not be - * available, therefore at the end of verification do_misc_fixups() - * sorts this by imm and offset. + * verification. JITs use the descriptor index stored in the finalized + * call's off field. The first nr_base_descs entries are the canonical + * descriptors used for verifier lookups. Call specialization may append + * immutable descriptors for additional targets. * - * Grown one entry at a time by bpf_add_kfunc_call(). + * Grown one entry at a time by bpf_add_kfunc_call() and during + * call specialization. */ struct bpf_kfunc_desc descs[]; }; /* Functions exported from verifier.c, used by fixups.c */ -void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len); void bpf_mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog); bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env); bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm); @@ -1726,6 +2002,7 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt struct bpf_attach_target_info *tgt_info); /* Functions in fixups.c, called from bpf_check() */ +void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len); int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env); int bpf_optimize_bpf_loop(struct bpf_verifier_env *env); void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env); @@ -1738,4 +2015,23 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env); int bpf_do_misc_fixups(struct bpf_verifier_env *env); int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn); +int bpf_flip_opcode(u32 opcode); +u8 bpf_rev_opcode(u8 opcode); + +int bpf_compute_idoms(struct bpf_verifier_env *env); +int bpf_compute_loops(struct bpf_verifier_env *env); +int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx); +bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header); + +int bpf_init_scev(struct bpf_verifier_env *env); +void bpf_free_scev(struct bpf_verifier_env *env); +int bpf_compute_scev(struct bpf_verifier_env *env); + +int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_state *st, + struct bpf_loop_iters *iters); +int bpf_widen_scev_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *st, + struct bpf_verifier_state *loop_entry, struct bpf_loop_iters *iters); +int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *st, u32 insn_idx, + struct bpf_verifier_state *entry_state, u32 max_header_count); + #endif /* _LINUX_BPF_VERIFIER_H */ diff --git a/include/linux/btf.h b/include/linux/btf.h index 7c62ea17b116..b66e20a18c54 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -5,6 +5,7 @@ #define _LINUX_BTF_H 1 #include <linux/types.h> +#include <linux/module.h> #include <linux/bpfptr.h> #include <linux/bsearch.h> #include <linux/btf_ids.h> @@ -118,6 +119,19 @@ struct btf_show; struct btf_id_set; struct bpf_prog; +#define BTF_LINK_SHA256_LEN 32 + +/* + * A .BTF.link or .BTF.inline.link section identifies the module which + * carries a vmlinux BTF section. The module name is NUL terminated and the + * field is padded to __MODULE_NAME_LEN. + */ +struct btf_link { + char module_name[__MODULE_NAME_LEN]; + u8 sha256[BTF_LINK_SHA256_LEN]; + u32 btf_size; +} __packed; + typedef int (*btf_kfunc_filter_t)(const struct bpf_prog *prog, u32 kfunc_id); struct btf_kfunc_id_set { @@ -236,6 +250,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type u32 field_mask, u32 value_size); int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec); bool btf_type_is_void(const struct btf_type *t); +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t); s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind); s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p); struct btf *btf_get_module_btf(const struct module *module); @@ -261,6 +276,11 @@ const char *btf_type_str(const struct btf_type *t); i < btf_type_vlen(datasec_type); \ i++, member++) +#define for_each_loc(i, locsec_type, member) \ + for (i = 0, member = btf_type_loc_secinfo(locsec_type); \ + i < btf_type_vlen(locsec_type); \ + i++, member++) + static inline bool btf_type_is_ptr(const struct btf_type *t) { return BTF_INFO_KIND(t->info) == BTF_KIND_PTR; @@ -329,6 +349,21 @@ static inline u64 btf_enum64_value(const struct btf_enum64 *e) return ((u64)e->val_hi32 << 32) | e->val_lo32; } +static inline struct btf_loc_param *btf_loc_param(const struct btf_type *t) +{ + return (struct btf_loc_param *)(t + 1); +} + +static inline __u32 *btf_loc_proto_params(const struct btf_type *t) +{ + return (__u32 *)(t + 1); +} + +static inline struct btf_loc *btf_type_loc_secinfo(const struct btf_type *t) +{ + return (struct btf_loc *)(t + 1); +} + static inline bool btf_is_composite(const struct btf_type *t) { u16 kind = btf_kind(t); @@ -559,7 +594,7 @@ struct btf_field_desc { /* member struct size, or zero, if no members */ int m_sz; /* repeated per-member offsets */ - int m_off_cnt, m_offs[1]; + int m_off_cnt, m_offs[2]; }; struct btf_field_iter { @@ -573,7 +608,8 @@ struct btf_field_iter { #ifdef CONFIG_BPF_SYSCALL const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id); void btf_set_base_btf(struct btf *btf, const struct btf *base_btf); -int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **map_ids); +int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **map_ids, + __u32 **map_strs); int btf_field_iter_init(struct btf_field_iter *it, struct btf_type *t, enum btf_field_iter_kind iter_kind); __u32 *btf_field_iter_next(struct btf_field_iter *it); @@ -603,6 +639,29 @@ bool btf_types_are_same(const struct btf *btf1, u32 id1, const struct btf *btf2, u32 id2); int btf_check_iter_arg(struct btf *btf, const struct btf_type *func, int arg_idx); +#if IS_ENABLED(CONFIG_SYSFS) +struct bin_attribute *sysfs_btf_add(const char *name, void *data, size_t data_size, + bool mmap, const char *lazy_module_name); +bool sysfs_btf_update(struct bin_attribute *attr, void *data); +void sysfs_btf_remove(struct bin_attribute *attr); +#else +static inline struct bin_attribute * +sysfs_btf_add(const char *name, void *data, size_t data_size, bool mmap, + const char *lazy_module_name) +{ + return NULL; +} + +static inline bool sysfs_btf_update(struct bin_attribute *attr, void *data) +{ + return false; +} + +static inline void sysfs_btf_remove(struct bin_attribute *attr) +{ +} +#endif + static inline bool btf_type_is_struct_ptr(struct btf *btf, const struct btf_type *t) { if (!btf_type_is_ptr(t)) @@ -624,7 +683,7 @@ static inline void btf_set_base_btf(struct btf *btf, const struct btf *base_btf) } static inline int btf_relocate(void *log, struct btf *btf, const struct btf *base_btf, - __u32 **map_ids) + __u32 **map_ids, __u32 **map_strs) { return -EOPNOTSUPP; } diff --git a/include/linux/cnum.h b/include/linux/cnum.h index 49b7d0c7645d..49160318adf9 100644 --- a/include/linux/cnum.h +++ b/include/linux/cnum.h @@ -39,6 +39,7 @@ u32 cnum32_umin(struct cnum32 cnum); u32 cnum32_umax(struct cnum32 cnum); s32 cnum32_smin(struct cnum32 cnum); s32 cnum32_smax(struct cnum32 cnum); +struct cnum32 cnum32_union(struct cnum32 a, struct cnum32 b); struct cnum32 cnum32_intersect(struct cnum32 a, struct cnum32 b); void cnum32_intersect_with(struct cnum32 *dst, struct cnum32 src); void cnum32_intersect_with_urange(struct cnum32 *dst, u32 min, u32 max); @@ -65,6 +66,7 @@ u64 cnum64_umin(struct cnum64 cnum); u64 cnum64_umax(struct cnum64 cnum); s64 cnum64_smin(struct cnum64 cnum); s64 cnum64_smax(struct cnum64 cnum); +struct cnum64 cnum64_union(struct cnum64 a, struct cnum64 b); struct cnum64 cnum64_intersect(struct cnum64 a, struct cnum64 b); void cnum64_intersect_with(struct cnum64 *dst, struct cnum64 src); void cnum64_intersect_with_urange(struct cnum64 *dst, u64 min, u64 max); @@ -78,5 +80,6 @@ bool cnum64_is_subset(struct cnum64 outer, struct cnum64 inner); struct cnum32 cnum32_from_cnum64(struct cnum64 cnum); struct cnum64 cnum64_cnum32_intersect(struct cnum64 a, struct cnum32 b); +struct cnum64 cnum64_intersect_linear(struct cnum64 in, u16 base, u16 step); #endif /* _LINUX_CNUM_H */ diff --git a/include/linux/filter.h b/include/linux/filter.h index 39decde7fc73..e0218e677a1d 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -98,6 +98,11 @@ struct ctl_table_header; /* BPF program can access up to 512 bytes of stack space. */ #define MAX_BPF_STACK 512 +/* + * Stack budget of a program on a JIT that lays out frames of that size. + * The interpreter and JITs without such support keep MAX_BPF_STACK. + */ +#define MAX_BPF_STACK_JIT 2048 /* Helper macros for filter block array initializers. */ @@ -249,6 +254,29 @@ static inline bool insn_is_mov_percpu_addr(const struct bpf_insn *insn) .off = 0, \ .imm = 1 }) +static inline bool bpf_is_unwind_goto(const struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_UNWIND_GOTO; +} + +static inline bool bpf_is_unwind_exit(const struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_EXIT) && insn->src_reg == BPF_UNWIND_EXIT; +} + +/* + * The caller of the main prog doesn't tell unwind_exit from exit. An extension + * is different: its caller is the caller of the subprog that it replaced. + * An exception callback returns from the frame of the main prog. After freplace + * or a tail call it returns to a prog that wasn't verified with the callback. + */ +static inline bool bpf_prog_can_unwind(const struct bpf_prog *prog) +{ + if (prog->aux->exception_cb) + return false; + return bpf_is_subprog(prog) || prog->type == BPF_PROG_TYPE_EXT; +} + static inline bool insn_is_zext(const struct bpf_insn *insn) { return insn->code == (BPF_ALU | BPF_MOV | BPF_X) && insn->imm == 1; @@ -1105,6 +1133,19 @@ static inline unsigned int bpf_prog_size(unsigned int proglen) offsetof(struct bpf_prog, insns[proglen])); } +bool bpf_prog_has_arena_insns(const struct bpf_prog *prog); + +/* + * Whether a JIT has to set up the arena base for @prog. A program bound to + * an arena need not touch it, libbpf binds the arena of an object to every + * program in it. The flag caches bpf_prog_has_arena_insns() for the function + * about to be JITed, see bpf_prog_jit_compile() and jit_subprogs(). + */ +static inline bool bpf_prog_uses_arena_insns(const struct bpf_prog *prog) +{ + return prog->aux->uses_arena_insns; +} + static inline bool bpf_prog_was_classic(const struct bpf_prog *prog) { /* When classic BPF programs have been loaded and the arch @@ -1237,16 +1278,53 @@ bool bpf_jit_inlines_helper_call(s32 imm); bool bpf_jit_supports_subprog_tailcalls(void); bool bpf_jit_supports_percpu_insn(void); bool bpf_jit_supports_kfunc_call(void); +bool bpf_jit_supports_callx(void); +bool bpf_jit_supports_kfunc_ret_reg_pair(void); +bool bpf_jit_supports_unwind(void); bool bpf_jit_supports_stack_args(void); bool bpf_jit_supports_arena_args(void); bool bpf_jit_supports_far_kfunc_call(void); bool bpf_jit_supports_exceptions(void); bool bpf_jit_supports_ptr_xchg(void); bool bpf_jit_supports_arena(void); +bool bpf_jit_supports_arena_scalar(void); bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena); bool bpf_jit_supports_private_stack(void); +bool bpf_jit_supports_large_stack(void); bool bpf_jit_supports_timed_may_goto(void); bool bpf_jit_supports_fsession(void); + +struct bpf_jit_arg_abi { + /* Argument registers of the kernel convention. */ + u8 nr_arg_regs; + /* Round the register number up to an even one for 16-byte alignment. */ + bool even_reg_align; + /* Round the stack slot up to an even one for 16-byte alignment. */ + bool even_stack_align; + /* An argument may straddle the last register and the stack. */ + bool split_at_boundary; + /* A later argument may reuse a register a stack-passed one skipped. */ + bool backfill_after_stack; +}; + +const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void); +u32 bpf_jit_place_args(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, u8 *pos_of_slot); + +/* The JIT's scratch register, in place of an argument slot. */ +#define BPF_JIT_ARG_TMP 0xff + +/* Every argument slot moves at most once, and the scratch goes out and back. */ +#define BPF_JIT_MAX_ARG_MOVES (MAX_BPF_FUNC_ARG_SLOTS + 2) + +struct bpf_jit_arg_move { + u8 dst; + u8 src; +}; + +u32 bpf_jit_plan_arg_moves(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, + struct bpf_jit_arg_move *moves); u64 bpf_arch_uaddress_limit(void); void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie); u64 arch_bpf_timed_may_goto(void); @@ -1897,6 +1975,11 @@ static __always_inline long __bpf_xdp_redirect_map(struct bpf_map *map, u64 inde return XDP_REDIRECT; } +int __bpf_sock_ops_load_hdr_opt(struct bpf_sock_ops_kern *bpf_sock, + void *search_res, u32 len, u64 flags); +int __bpf_sock_ops_store_hdr_opt(struct bpf_sock_ops_kern *bpf_sock, + const void *from, u32 len, u64 flags); + #ifdef CONFIG_NET int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len); int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index bd76a16a63af..b041d9142d2b 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -416,6 +416,7 @@ void free_ftrace_hash(struct ftrace_hash *hash); struct ftrace_func_entry *add_ftrace_hash_entry_direct(struct ftrace_hash *hash, unsigned long ip, unsigned long direct); void add_ftrace_hash_entry(struct ftrace_hash *hash, struct ftrace_func_entry *entry); +struct ftrace_func_entry *ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip); void ftrace_hash_remove(struct ftrace_hash *hash); /* The hash used to know what functions callbacks trace */ diff --git a/include/linux/key.h b/include/linux/key.h index e40b5267658f..dae5ba255ad1 100644 --- a/include/linux/key.h +++ b/include/linux/key.h @@ -445,6 +445,8 @@ extern key_ref_t keyring_search(key_ref_t keyring, extern int keyring_restrict(key_ref_t keyring, const char *type, const char *restriction); +extern void key_register_bpf_keyring(struct key *keyring); + extern struct key *key_lookup(key_serial_t id); static inline key_serial_t key_serial(const struct key *key) diff --git a/include/linux/mm.h b/include/linux/mm.h index 0a2a7fc4a442..6882ad400935 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3314,10 +3314,10 @@ extern int access_process_vm(struct task_struct *tsk, unsigned long addr, extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf, int len, unsigned int gup_flags); -#ifdef CONFIG_BPF_SYSCALL -extern int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, - void *buf, int len, unsigned int gup_flags); -#endif +int copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags); +int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, + void *buf, int len, unsigned int gup_flags); long get_user_pages_remote(struct mm_struct *mm, unsigned long start, unsigned long nr_pages, diff --git a/include/linux/module.h b/include/linux/module.h index 2c5a02621c19..76f4f3596477 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -503,6 +503,10 @@ struct module { void *btf_data; void *btf_base_data; #endif +#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE) + unsigned int btf_inline_data_size; + void *btf_inline_data; +#endif #ifdef CONFIG_JUMP_LABEL struct jump_entry *jump_entries; unsigned int num_jump_entries; diff --git a/include/linux/tcp.h b/include/linux/tcp.h index e4d1720f49f0..d18f7bc1e5d0 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -233,6 +233,13 @@ struct tcp_sock { is_sack_reneg:1, /* in recovery from loss with SACK reneg? */ is_cwnd_limited:1,/* forward progress limited by snd_cwnd? */ recvmsg_inq : 1;/* Indicate # of bytes in queue upon recvmsg */ +#ifdef CONFIG_BPF + u32 bpf_tcp_ops_flags; +#define BPF_TCP_OPS_TEST_FLAG(TP, ARG) \ + (READ_ONCE((TP)->bpf_tcp_ops_flags) & BPF_TCP_OPS_FLAG_ ## ARG) +#else +#define BPF_TCP_OPS_TEST_FLAG(TP, ARG) (0) +#endif __cacheline_group_end(tcp_sock_read_txrx); /* RX read-mostly hotpath cache lines */ diff --git a/include/linux/tnum.h b/include/linux/tnum.h index ca2cfec8de08..16682248f963 100644 --- a/include/linux/tnum.h +++ b/include/linux/tnum.h @@ -134,4 +134,6 @@ static inline bool tnum_subreg_is_const(struct tnum a) /* Returns the smallest member of t larger than z */ u64 tnum_step(struct tnum t, u64 z); +u32 tnum_alignment(struct tnum a); + #endif /* _LINUX_TNUM_H */ diff --git a/include/net/tcp.h b/include/net/tcp.h index 0265c27a5fd7..3560adddf821 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -512,6 +512,7 @@ void tcp_set_keepalive(struct sock *sk, int val); void tcp_syn_ack_timeout(const struct request_sock *req); int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags); +int __tcp_set_rcvlowat(struct sock *sk, int val, bool wakeup); int tcp_set_rcvlowat(struct sock *sk, int val); void tcp_set_rcvbuf(struct sock *sk, int val); int tcp_set_window_clamp(struct sock *sk, int val); @@ -2930,7 +2931,7 @@ static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops, * program loaded). */ #ifdef CONFIG_BPF -static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) +static inline int __tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) { struct bpf_sock_ops_kern sock_ops; int ret; @@ -2947,7 +2948,7 @@ static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) if (nargs > 0) memcpy(sock_ops.args, args, nargs * sizeof(*args)); - ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); + ret = __BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); if (ret == 0) ret = sock_ops.reply; else @@ -2955,42 +2956,37 @@ static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) return ret; } -static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) -{ - u32 args[2] = {arg1, arg2}; - - return tcp_call_bpf(sk, op, 2, args); -} - -static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, - u32 arg3) -{ - u32 args[3] = {arg1, arg2, arg3}; - - return tcp_call_bpf(sk, op, 3, args); -} +#define tcp_call_bpf(sk, op) \ +({ \ + int __ret = 0; \ + if (cgroup_bpf_enabled(CGROUP_SOCK_OPS)) { \ + __ret = __tcp_call_bpf(sk, op, 0, NULL); \ + } \ + __ret; \ +}) + +#define tcp_call_bpf_flag(sk, op, ...) \ +do { \ + if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) && \ + BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), op ## _FLAG)) { \ + u32 __args[] = { __VA_ARGS__ }; \ + __tcp_call_bpf(sk, op, ARRAY_SIZE(__args), __args); \ + } \ +} while (0) static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) { tcp_sk(sk)->bpf_sock_ops_cb_flags = 0; + WRITE_ONCE(tcp_sk(sk)->bpf_tcp_ops_flags, 0); } #else -static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) +static inline int tcp_call_bpf(struct sock *sk, int op) { return -EPERM; } -static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) -{ - return -EPERM; -} - -static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, - u32 arg3) -{ - return -EPERM; -} +#define tcp_call_bpf_flag(sk, op, ...) do { } while (0) static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) { @@ -2998,12 +2994,196 @@ static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) #endif +#if defined(CONFIG_BPF_JIT) && defined(CONFIG_CGROUP_BPF) + +struct bpf_tcp_ops { + /* Should return the initial SYN (active open) or SYN-ACK (passive open) + * retransmission timeout. Return the timeout in jiffies, or <= 0 for + * the kernel default. + * + * @req: request_sock on the passive (synack) path; NULL otherwise. + */ + int (*timeout_init)(struct sock *sk, struct request_sock *req); + + /* Should return the initial advertised receive window, in packets, + * or < 0 for the kernel default. @req as in timeout_init(). + */ + int (*rwnd_init)(struct sock *sk, struct request_sock *req); + + /* Called when an active connection becomes established. + * @skb is the SYNACK that completed the 3WHS, or NULL for a + * TCP_REPAIR socket (tcp_finish_connect() with no skb). + */ + void (*active_established)(struct sock *sk, struct sk_buff *skb__nullable); + + /* Called when a passive connection becomes established. + * @skb is the ACK that completed the 3WHS. + */ + void (*passive_established)(struct sock *sk, struct sk_buff *skb); + + /* Called when the retransmission timer fires. */ + void (*rto)(struct sock *sk); + + /* Called on every RTT sample if BPF_TCP_OPS_FLAG_RTT is enabled. + * @mrtt: the measured RTT, in microseconds. + * @srtt: the updated smoothed RTT. + */ + void (*rtt)(struct sock *sk, long mrtt, u32 srtt); + + /* Called when the connection changes TCP state. + * @state: the new state (one of the TCP_* states). + */ + void (*set_state)(struct sock *sk, int state); + + /* Called when an skb is retransmitted. + * @skb: the retransmitted skb. + * @err: tcp_transmit_skb() return value (0 on success). + */ + void (*retrans)(struct sock *sk, struct sk_buff *skb, int err); + + /* Called right before an active connection is initialized. */ + void (*connect)(struct sock *sk); + + /* Called on listen(2), right after the socket enters TCP_LISTEN. */ + void (*listen)(struct sock *sk); + + /* + * Parse the TCP header options of an incoming skb received on an + * established connection. Use bpf_dynptr_from_skb()/bpf_skb_load_bytes() + * to access the options. + * + * Called if BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_ALL is enabled, or if + * BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_UNKNOWN is enabled and an unknown + * option is received. + */ + void (*parse_hdr)(struct sock *sk, struct sk_buff *skb); + + /* + * Reserve space in the outgoing TCP header for options to be written + * later by write_hdr_opt(). Call bpf_reserve_hdr_opt() to reserve bytes. + * + * Called if BPF_TCP_OPS_FLAG_WRITE_HDR_OPT is enabled. + * + * @skb: outgoing packet. NULL when called from tcp_current_mss() + * (MSS sizing). + * @req: request_sock on the synack path; NULL otherwise. + * @syn_skb: incoming SYN on the synack path; NULL otherwise. + * @synack_type: TCP_SYNACK_COOKIE indicates a stateless syncookie. + * @remaining: pointer to the size of space still available; cast it + * using bpf_rdonly_cast() before dereferencing. + */ + void (*hdr_opt_len)(struct sock *sk, struct sk_buff *skb, + struct request_sock *req, struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, + unsigned int *remaining); + + /* + * Write header options into the space reserved earlier by hdr_opt_len(). + * Use bpf_store_hdr_opt() to write; it appends within the reserved window + * shared with legacy SOCKOPS. + * + * Called if BPF_TCP_OPS_FLAG_WRITE_HDR_OPT is enabled. + * + * @skb: outgoing packet. + * @req: request_sock on the synack path; NULL otherwise. + * @syn_skb: incoming SYN on the synack path; NULL otherwise. + * @synack_type: TCP_SYNACK_COOKIE indicates a stateless syncookie. + * @opt_off: offset in the outgoing @skb's TCP header where the + * bpf_tcp_ops portion of the reserved window begins, i.e. after + * the kernel and legacy options. + */ + void (*write_hdr_opt)(struct sock *sk, struct sk_buff *skb, + struct request_sock *req, struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, + u32 opt_off); + + /* + * Called when an incoming skb is enqueued to sk->sk_receive_queue + * if BPF_TCP_OPS_FLAG_RCVQ is enabled. + */ + void (*enqueue_rcvq)(struct sock *sk, struct sk_buff *skb); + + /* + * Called after data is dequeued from sk->sk_receive_queue + * if BPF_TCP_OPS_FLAG_RCVQ is enabled. + */ + void (*dequeue_rcvq)(struct sock *sk); +}; + +#define __bpf_tcp_ops_call(op, sk, ...) \ +do { \ + const struct bpf_prog_array_item *item; \ + const struct bpf_tcp_ops *tcp_ops; \ + struct cgroup *cgrp; \ + \ + cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); \ + rcu_read_lock_dont_migrate(); \ + bpf_cgroup_struct_ops_foreach(tcp_ops, item, cgrp, \ + CGROUP_TCP_SOCK_OPS) { \ + if (tcp_ops->op) \ + tcp_ops->op(sk, ##__VA_ARGS__); \ + } \ + rcu_read_unlock_migrate(); \ +} while (0) + +#define bpf_tcp_ops_call(op, sk, ...) \ +do { \ + if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) \ + __bpf_tcp_ops_call(op, sk, ##__VA_ARGS__); \ +} while (0) + +#define bpf_tcp_ops_call_flag(op, flag, sk, ...) \ +do { \ + if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS) && \ + BPF_TCP_OPS_TEST_FLAG(tcp_sk(sk), flag)) \ + __bpf_tcp_ops_call(op, sk, ##__VA_ARGS__); \ +} while (0) + +#define bpf_tcp_ops_call_int(op, init_retval, sk, ...) \ +({ \ + int __retval = (init_retval); \ + if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) { \ + const struct bpf_prog_array_item *item; \ + const struct bpf_tcp_ops *tcp_ops; \ + struct bpf_tramp_run_ctx run_ctx; \ + struct bpf_run_ctx *old_run_ctx; \ + struct sock *__sk = sk_to_full_sk(sk); \ + struct request_sock *req = NULL; \ + struct cgroup *cgrp; \ + \ + if (__sk) { \ + run_ctx.retval = (init_retval); \ + cgrp = sock_cgroup_ptr(&__sk->sk_cgrp_data); \ + if (!sk_fullsock(sk)) \ + req = (struct request_sock *)sk; \ + rcu_read_lock_dont_migrate(); \ + old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);\ + bpf_cgroup_struct_ops_foreach(tcp_ops, item, cgrp, \ + CGROUP_TCP_SOCK_OPS) { \ + if (tcp_ops->op) \ + run_ctx.retval = tcp_ops->op(__sk, req, ##__VA_ARGS__); \ + } \ + bpf_reset_run_ctx(old_run_ctx); \ + rcu_read_unlock_migrate(); \ + __retval = run_ctx.retval; \ + } \ + } \ + __retval; \ +}) + +#else +#define __bpf_tcp_ops_call(op, sk, ...) do { } while (0) +#define bpf_tcp_ops_call(op, sk, ...) do { } while (0) +#define bpf_tcp_ops_call_flag(op, flag, sk, ...) do { } while (0) +#define bpf_tcp_ops_call_int(op, init_retval, sk, ...) (init_retval) +#endif + static inline u32 tcp_timeout_init(struct sock *sk) { int timeout; - timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL); - + timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT); + timeout = bpf_tcp_ops_call_int(timeout_init, timeout, sk); if (timeout <= 0) timeout = TCP_TIMEOUT_INIT; return min_t(int, timeout, TCP_RTO_MAX); @@ -3013,8 +3193,8 @@ static inline u32 tcp_rwnd_init_bpf(struct sock *sk) { int rwnd; - rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL); - + rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT); + rwnd = bpf_tcp_ops_call_int(rwnd_init, rwnd, sk); if (rwnd < 0) rwnd = 0; return rwnd; @@ -3022,13 +3202,13 @@ static inline u32 tcp_rwnd_init_bpf(struct sock *sk) static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk) { - return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1); + return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN) == 1); } static inline void tcp_bpf_rtt(struct sock *sk, long mrtt, u32 srtt) { - if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG)) - tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt); + tcp_call_bpf_flag(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt); + bpf_tcp_ops_call_flag(rtt, RTT, sk, mrtt, srtt); } #if IS_ENABLED(CONFIG_SMC) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 732b35cc08d1..f29b0aad440c 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -56,6 +56,19 @@ enum bpf_cond_pseudo_jmp { BPF_MAY_GOTO = 0, + /* + * Follows a call of a bpf subprog. Taken when the subprog left with + * BPF_EXIT that has src_reg == BPF_UNWIND_EXIT. + */ + BPF_UNWIND_GOTO = 1, +}; + +enum bpf_pseudo_exit { + /* + * When bpf_exit->src_reg == BPF_UNWIND_EXIT, it's the same as bpf_exit, + * except that BPF_UNWIND_GOTO after the call is taken. + */ + BPF_UNWIND_EXIT = 1, }; /* Register numbers */ @@ -936,6 +949,33 @@ union bpf_iter_link_info { * 0 on success or -1 if an error occurred (in which case, * *errno* is set appropriately). * + * BPF_PROG_STREAM_OPEN + * Description + * Open a file descriptor for one of the BPF streams associated + * with the program identified by *prog_fd*. The stream is selected + * by *stream_id*. + * + * The returned file descriptor supports **read**\ (2) and + * **poll**\ (2). Reads block while the stream is empty unless + * **BPF_F_STREAM_NONBLOCK** is specified in *flags*. A non-blocking + * read of an empty stream fails with **EAGAIN**. + * + * **poll**\ (2) reports **POLLIN** when data is available and + * **POLLHUP** once the program has been freed, that is, after every + * reference to it, including links and other file descriptors, has + * been dropped. Hangup may lag the final release because program + * teardown is deferred. Buffered data remains readable after + * **POLLHUP** and a read returns zero after all such data has been + * consumed. + * + * The file descriptor is read-only and has the close-on-exec flag + * set. It is not seekable and **lseek**\ (2) fails with **ESPIPE**. + * *flags* may only contain **BPF_F_STREAM_NONBLOCK**. + * + * Return + * A new file descriptor (a nonnegative integer), or -1 if an + * error occurred (in which case, *errno* is set appropriately). + * * NOTES * eBPF objects (maps and programs) can be shared between processes. * @@ -993,6 +1033,7 @@ enum bpf_cmd { BPF_TOKEN_CREATE, BPF_PROG_STREAM_READ_BY_FD, BPF_PROG_ASSOC_STRUCT_OPS, + BPF_PROG_STREAM_OPEN, __MAX_BPF_CMD, BPF_COMMON_ATTRS = 1 << 16, /* Indicate carrying syscall common attrs. */ }; @@ -1316,6 +1357,12 @@ enum bpf_perf_event_type { /* The verifier internal test flag. Behavior is undefined */ #define BPF_F_TEST_REG_INVARIANTS (1U << 7) +/* + * Load and store through a number is an access to the arena of the program + * at the low 32 bits of the number. It's for programs written in Rust. + */ +#define BPF_F_ARENA_SCALAR (1U << 8) + /* link_create.kprobe_multi.flags used in LINK_CREATE command for * BPF_TRACE_KPROBE_MULTI attach type to create return probe. */ @@ -1524,6 +1571,11 @@ enum { BPF_STREAM_STDERR = 2, }; +/* flags for BPF_PROG_STREAM_OPEN command */ +enum { + BPF_F_STREAM_NONBLOCK = (1U << 0), +}; + union bpf_attr { struct { /* anonymous struct used by BPF_MAP_CREATE command */ __u32 map_type; /* one of enum bpf_map_type */ @@ -1756,7 +1808,7 @@ union bpf_attr { __u32 prog_cnt; __u32 count; }; - __u32 :32; + __u32 type_id; /* output: per-program attach_flags. * not allowed to be set during effective query. */ @@ -1950,6 +2002,12 @@ union bpf_attr { __u32 flags; } prog_assoc_struct_ops; + struct { + __u32 prog_fd; + __u32 stream_id; + __u32 flags; + } prog_stream_open; + } __attribute__((aligned(8))); /* The description below is an attempt at providing documentation to eBPF @@ -4867,15 +4925,18 @@ union bpf_attr { * The non-negative copied *buf* length equal to or less than * *size* on success, or a negative error in case of failure. * - * long bpf_load_hdr_opt(struct bpf_sock_ops *skops, void *searchby_res, u32 len, u64 flags) + * long bpf_load_hdr_opt(void *ctx, void *searchby_res, u32 len, u64 flags) * Description * Load header option. Support reading a particular TCP header - * option for bpf program (**BPF_PROG_TYPE_SOCK_OPS**). + * option for bpf program (**BPF_PROG_TYPE_SOCK_OPS**). For the + * **bpf_tcp_ops** struct_ops, this helper can be called from the + * **parse_hdr**\ () and **write_hdr_opt**\ () operators. * - * If *flags* is 0, it will search the option from the - * *skops*\ **->skb_data**. The comment in **struct bpf_sock_ops** - * has details on what skb_data contains under different - * *skops*\ **->op**. + * If *flags* is 0, it will search the option from the packet + * associated with the current operation. For + * **BPF_PROG_TYPE_SOCK_OPS**, the comment in + * **struct bpf_sock_ops** has details on what skb_data + * contains under different *op*. * * The first byte of the *searchby_res* specifies the * kind that it wants to search. @@ -4908,6 +4969,8 @@ union bpf_attr { * * * **BPF_LOAD_HDR_OPT_TCP_SYN** to search from the * saved_syn packet or the just-received syn packet. + * Not supported by the **bpf_tcp_ops** struct_ops, which + * rejects all flags. * * Return * > 0 when found, the header option is copied to *searchby_res*. @@ -4928,9 +4991,9 @@ union bpf_attr { * packet. * * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. + * operation. * - * long bpf_store_hdr_opt(struct bpf_sock_ops *skops, const void *from, u32 len, u64 flags) + * long bpf_store_hdr_opt(void *ctx, const void *from, u32 len, u64 flags) * Description * Store header option. The data will be copied * from buffer *from* with length *len* to the TCP header. @@ -4946,7 +5009,9 @@ union bpf_attr { * by searching the same option in the outgoing skb. * * This helper can only be called during - * **BPF_SOCK_OPS_WRITE_HDR_OPT_CB**. + * **BPF_SOCK_OPS_WRITE_HDR_OPT_CB**, or from the + * **write_hdr_opt**\ () operator of the **bpf_tcp_ops** + * struct_ops. * * Return * 0 on success, or negative error in case of failure: @@ -4961,9 +5026,9 @@ union bpf_attr { * **-EFAULT** on failure to parse the existing header options. * * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. + * operation. * - * long bpf_reserve_hdr_opt(struct bpf_sock_ops *skops, u32 len, u64 flags) + * long bpf_reserve_hdr_opt(void *ctx, u32 len, u64 flags) * Description * Reserve *len* bytes for the bpf header option. The * space will be used by **bpf_store_hdr_opt**\ () later in @@ -4973,7 +5038,9 @@ union bpf_attr { * the total number of bytes will be reserved. * * This helper can only be called during - * **BPF_SOCK_OPS_HDR_OPT_LEN_CB**. + * **BPF_SOCK_OPS_HDR_OPT_LEN_CB**, or from the + * **hdr_opt_len**\ () operator of the **bpf_tcp_ops** + * struct_ops. * * Return * 0 on success, or negative error in case of failure: @@ -4983,7 +5050,7 @@ union bpf_attr { * **-ENOSPC** if there is not enough space in the header. * * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. + * operation. * * void *bpf_inode_storage_get(struct bpf_map *map, void *inode, void *value, u64 flags) * Description @@ -6890,6 +6957,8 @@ struct bpf_link_info { } xdp; struct { __u32 map_id; + __u32 :32; + __u64 cgroup_id; } struct_ops; struct { __u32 pf; @@ -7284,6 +7353,31 @@ enum { */ }; +/* + * Most callbacks of struct bpf_tcp_ops are placed in the slow + * path (e.g., one-shot connection setup or unlikely events like + * timers) and are invoked simply by defining non-NULL callbacks. + * + * Callbacks in the fast path, however, would incur noticeable + * overhead even when set to NULL, so they are disabled by default + * and must be explicitly enabled per socket via bpf_tcp_ops_set_flags(). + * + * The flags are per-socket and shared by all effective bpf_tcp_ops + * programs. + */ +enum { + /* .rtt() */ + BPF_TCP_OPS_FLAG_RTT = (1 << 0), + /* .parse_hdr() */ + BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_ALL = (1 << 1), + BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_UNKNOWN = (1 << 2), + /* .hdr_opt_len() and .write_hdr_opt() */ + BPF_TCP_OPS_FLAG_WRITE_HDR_OPT = (1 << 3), + /* .enqueue_rcvq() and .dequeue_rcvq() */ + BPF_TCP_OPS_FLAG_RCVQ = (1 << 4), + BPF_TCP_OPS_FLAG_ALL = (1 << 5) - 1, +}; + /* List of TCP states. There is a build check in net/ipv4/tcp.c to detect * changes between the TCP and BPF versions. Ideally this should never happen. * If it does, we need to add code to convert them before calling @@ -7600,6 +7694,10 @@ struct bpf_task_work { __u64 __opaque; } __attribute__((aligned(8))); +struct bpf_rcu_head { + __u64 __opaque[6]; +} __attribute__((aligned(8))); + struct bpf_wq { __u64 __opaque[2]; } __attribute__((aligned(8))); diff --git a/include/uapi/linux/btf.h b/include/uapi/linux/btf.h index 618167cab4e6..11d65871d163 100644 --- a/include/uapi/linux/btf.h +++ b/include/uapi/linux/btf.h @@ -54,8 +54,8 @@ struct btf_type { * decl_tag and type_tag */ __u32 info; - /* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64. - * "size" tells the size of the type it is describing. + /* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC, ENUM64 + * and LOC_PARAM. "size" tells the size of the type it is describing. * * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, * FUNC, FUNC_PROTO, VAR, DECL_TAG and TYPE_TAG. @@ -92,7 +92,9 @@ enum { BTF_KIND_DECL_TAG = 17, /* Decl Tag */ BTF_KIND_TYPE_TAG = 18, /* Type Tag */ BTF_KIND_ENUM64 = 19, /* Enumeration up to 64-bit values */ - + BTF_KIND_LOC_PARAM = 20, /* Location parameter information */ + BTF_KIND_LOC_PROTO = 21, /* Location prototype for site */ + BTF_KIND_LOCSEC = 22, /* Location section */ NR_BTF_KINDS, BTF_KIND_MAX = NR_BTF_KINDS - 1, }; @@ -212,4 +214,79 @@ struct btf_enum64 { __u32 val_hi32; }; +/* + * BTF_KIND_LOC_PARAM is followed by a single "struct btf_loc_param" + * that contains flags specifying the contents of the vlen-specified + * number of 4-byte values that follow. + */ +struct btf_loc_param { + __u32 flags; + __u32 values[]; +}; + +/* + * The combination of size, vlen and flags gives us the means to interpret + * the following vlen-specified set of 4-byte values: + * + * - a BTF_LOC_PARAM_CONST is a constant value; combination + * of size, vlen and _SIGNED flag determines it. If the value requires + * 64 bits it is stored in {lo,hi} order. + * - a BTF_LOC_PARAM_ADDR|BTF_LOC_PARAM_CONST is an address that should be + * normalized with respect to kernel/module base address. + * - a BTF_LOC_PARAM_REG with vlen 1 is a simple register number; + * with vlen 2 it is a multi-register parameter. Register numbers are + * numbers derived from DW_OP_reg values, i.e. 0 is DW_OP_reg0; since + * DW_OP_fbreg has its own special DW_OP_value and DW_OP_regx can refer + * to an arbitrary register number, we reserve BTF_LOC_PARAM_FBREG for + * the frame base register to avoid collisions. + * - a _REG | OFFSET describes an address without dereferencing it. + * - a _REG | DEREF with vlen 1 dereferences the value in the register + * number specified. + * - a REG | DEREF | OFFSET with vlen > 1 specifies the register number + * in the first 4-byte value and the offset in the remainder. + * In the case of REG and OFFSET combinations, the OFFSET has the width + * of the value words while the type size describes the represented parameter, + * so for example a REG | DEREF | OFFSET with size 8 and vlen 2 would be + * an 8-byte register dereference with signed 4-byte offset, since the vlen 2 + * values consist of a register value and the signed value. + */ +enum btf_loc_param_flags { + BTF_LOC_PARAM_SIGNED = 0x1, + BTF_LOC_PARAM_CONST = 0x2, + BTF_LOC_PARAM_ADDR = 0x4, + BTF_LOC_PARAM_REG = 0x8, + BTF_LOC_PARAM_DEREF = 0x10, + BTF_LOC_PARAM_OFFSET = 0x20, +}; + +enum { + BTF_LOC_PARAM_FBREG = 0xffffffff +}; + +/* + * BTF_KIND_LOC_PROTO specifies location prototypes; i.e. how locations relate + * to parameters; a struct btf_type of BTF_KIND_LOC_PROTO is followed by a + * vlen-specified number of __u32 BTF type ids which specify the associated + * BTF_KIND_LOC_PARAM for each function parameter associated with the + * location. The type should either be 0 (no location info) or point at + * a BTF_KIND_LOC_PARAM. + */ + +/* + * BTF_KIND_LOCSEC consists of vlen-specified number of "struct btf_loc" + * containing location site-specific information for a specific ELF section; + * for example locations in ".text" are in a LOCSEC named "inline.text". + * + * - function (func) + * - location prototype type id (loc_proto) + * - address offset (offset) relative to the runtime base address of the + * ELF section associated with the LOCSEC + */ + +struct btf_loc { + __u32 func; + __u32 loc_proto; + __u32 offset; +}; + #endif /* _UAPI__LINUX_BTF_H__ */ diff --git a/include/uapi/linux/keyctl.h b/include/uapi/linux/keyctl.h index 4c8884eea808..fa85b9760391 100644 --- a/include/uapi/linux/keyctl.h +++ b/include/uapi/linux/keyctl.h @@ -24,6 +24,7 @@ #define KEY_SPEC_GROUP_KEYRING -6 /* - key ID for GID-specific keyring */ #define KEY_SPEC_REQKEY_AUTH_KEY -7 /* - key ID for assumed request_key auth key */ #define KEY_SPEC_REQUESTOR_KEYRING -8 /* - key ID for request_key() dest keyring */ +#define KEY_SPEC_BPF_KEYRING -9 /* - key ID for the BPF-specific keyring */ /* request-key default keyrings */ #define KEY_REQKEY_DEFL_NO_CHANGE -1 |