summaryrefslogtreecommitdiff
path: root/include/linux/bpf.h
diff options
context:
space:
mode:
Diffstat (limited to 'include/linux/bpf.h')
-rw-r--r--include/linux/bpf.h415
1 files changed, 314 insertions, 101 deletions
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 05b34a6355b0..1d2676782d70 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -6,6 +6,7 @@
#include <uapi/linux/bpf.h>
#include <uapi/linux/filter.h>
+#include <linux/bpf_defs.h>
#include <crypto/sha2.h>
#include <linux/workqueue.h>
@@ -31,6 +32,9 @@
#include <linux/static_call.h>
#include <linux/memcontrol.h>
#include <linux/cfi.h>
+#include <linux/xattr.h>
+#include <linux/key.h>
+#include <linux/ftrace.h>
#include <asm/rqspinlock.h>
struct bpf_verifier_env;
@@ -110,7 +114,7 @@ struct bpf_map_ops {
long (*map_pop_elem)(struct bpf_map *map, void *value);
long (*map_peek_elem)(struct bpf_map *map, void *value);
void *(*map_lookup_percpu_elem)(struct bpf_map *map, void *key, u32 cpu);
- int (*map_get_hash)(struct bpf_map *map, u32 hash_buf_size, void *hash_buf);
+ int (*map_get_hash)(struct bpf_map *map);
/* funcs called by prog_array and perf_event_array map */
void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,
@@ -141,6 +145,7 @@ struct bpf_map_ops {
int (*map_direct_value_meta)(const struct bpf_map *map,
u64 imm, u32 *off);
int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma);
+ vm_fault_t (*map_mmap_fault)(struct bpf_map *map, struct vm_fault *vmf);
__poll_t (*map_poll)(struct bpf_map *map, struct file *filp,
struct poll_table_struct *pts);
unsigned long (*map_get_unmapped_area)(struct file *filep, unsigned long addr,
@@ -488,6 +493,35 @@ static inline bool btf_record_has_field(const struct btf_record *rec, enum btf_f
return rec->field_mask & type;
}
+static inline bool btf_field_is_nmi_safe(enum btf_field_type type)
+{
+ switch (type) {
+ case BPF_SPIN_LOCK:
+ case BPF_RES_SPIN_LOCK:
+ case BPF_TIMER:
+ case BPF_WORKQUEUE:
+ case BPF_TASK_WORK:
+ case BPF_KPTR_UNREF:
+ case BPF_REFCOUNT:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static inline bool btf_record_has_nmi_unsafe_fields(const struct btf_record *rec)
+{
+ int i;
+
+ if (IS_ERR_OR_NULL(rec))
+ return false;
+ for (i = 0; i < rec->cnt; i++) {
+ if (!btf_field_is_nmi_safe(rec->fields[i].type))
+ return true;
+ }
+ return false;
+}
+
static inline void bpf_obj_init(const struct btf_record *rec, void *obj)
{
int i;
@@ -536,7 +570,7 @@ static inline void bpf_obj_memcpy(struct btf_record *rec,
if (IS_ERR_OR_NULL(rec)) {
if (long_memcpy)
- bpf_long_memcpy(dst, src, round_up(size, 8));
+ bpf_long_memcpy(dst, src, size);
else
memcpy(dst, src, size);
return;
@@ -559,7 +593,7 @@ static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src)
{
- bpf_obj_memcpy(map->record, dst, src, map->value_size, true);
+ bpf_obj_memcpy(map->record, dst, src, round_up(map->value_size, 8), true);
}
static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src)
@@ -617,6 +651,8 @@ void bpf_rb_root_free(const struct btf_field *field, void *rb_root,
struct bpf_spin_lock *spin_lock);
u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena);
u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena);
+u64 bpf_arena_map_kern_vm_start(struct bpf_map *map);
+struct bpf_map *bpf_prog_arena(struct bpf_prog *prog);
int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size);
struct bpf_offload_dev;
@@ -678,6 +714,8 @@ int bpf_dynptr_from_file_sleepable(struct file *file, u32 flags,
void *bpf_arena_alloc_pages_non_sleepable(void *p__map, void *addr__ign, u32 page_cnt, int node_id,
u64 flags);
void bpf_arena_free_pages_non_sleepable(void *p__map, void *ptr__ign, u32 page_cnt);
+void *bpf_arena_alloc_pages_sleepable(void *p__map, void *addr__ign, u32 page_cnt, int node_id,
+ u64 flags);
#else
static inline void *bpf_arena_alloc_pages_non_sleepable(void *p__map, void *addr__ign, u32 page_cnt,
int node_id, u64 flags)
@@ -688,6 +726,12 @@ static inline void *bpf_arena_alloc_pages_non_sleepable(void *p__map, void *addr
static inline void bpf_arena_free_pages_non_sleepable(void *p__map, void *ptr__ign, u32 page_cnt)
{
}
+
+static inline void *bpf_arena_alloc_pages_sleepable(void *p__map, void *addr__ign, u32 page_cnt,
+ int node_id, u64 flags)
+{
+ return NULL;
+}
#endif
extern const struct bpf_map_ops bpf_map_offload_ops;
@@ -845,11 +889,12 @@ enum bpf_arg_type {
ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */
ARG_PTR_TO_ARENA,
- ARG_CONST_SIZE, /* number of bytes accessed from memory */
- ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */
+ ARG_MEM_SIZE, /* number of bytes accessed from memory */
+ ARG_MEM_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */
ARG_PTR_TO_CTX, /* pointer to context */
ARG_ANYTHING, /* any (initialized) argument is ok */
+ ARG_SCALAR, /* scalar argument */
ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */
ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */
ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */
@@ -917,6 +962,21 @@ enum bpf_return_type {
};
static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
+/* The longest tracepoint has 12 args.
+ * See include/trace/bpf_probe.h
+ *
+ * Also reuse this macro for maximum number of arguments a BPF function
+ * or a kfunc can have. Args 1-5 are passed in registers, args 6-12 via
+ * stack arg slots. The JIT may map some stack arg slots to registers based
+ * on the native calling convention (e.g., arg 6 to R9 on x86-64).
+ */
+#define MAX_BPF_FUNC_ARGS 12
+
+/* The maximum number of arguments passed through registers
+ * a single function may have.
+ */
+#define MAX_BPF_FUNC_REG_ARGS 5
+
/* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
* to in-kernel helper functions and for adjusting imm32 field in BPF_CALL
* instructions after verifying
@@ -941,7 +1001,7 @@ struct bpf_func_proto {
enum bpf_arg_type arg4_type;
enum bpf_arg_type arg5_type;
};
- enum bpf_arg_type arg_type[5];
+ enum bpf_arg_type arg_type[MAX_BPF_FUNC_ARGS];
};
union {
struct {
@@ -951,7 +1011,7 @@ struct bpf_func_proto {
u32 *arg4_btf_id;
u32 *arg5_btf_id;
};
- u32 *arg_btf_id[5];
+ u32 *arg_btf_id[MAX_BPF_FUNC_ARGS];
struct {
size_t arg1_size;
size_t arg2_size;
@@ -959,7 +1019,7 @@ struct bpf_func_proto {
size_t arg4_size;
size_t arg5_size;
};
- size_t arg_size[5];
+ size_t arg_size[MAX_BPF_FUNC_ARGS];
};
int *ret_btf_id; /* return value btf_id */
bool (*allowed)(const struct bpf_prog *prog);
@@ -1051,7 +1111,7 @@ struct bpf_insn_access_aux {
struct {
struct btf *btf;
u32 btf_id;
- u32 ref_obj_id;
+ u32 ref_id;
};
};
struct bpf_verifier_log *log; /* for verbose logs */
@@ -1074,21 +1134,6 @@ static inline bool bpf_pseudo_func(const struct bpf_insn *insn)
return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
}
-/* Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an
- * atomic load or store, and false if it is a read-modify-write instruction.
- */
-static inline bool
-bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn)
-{
- switch (atomic_insn->imm) {
- case BPF_LOAD_ACQ:
- case BPF_STORE_REL:
- return true;
- default:
- return false;
- }
-}
-
struct bpf_prog_ops {
int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
union bpf_attr __user *uattr);
@@ -1149,22 +1194,15 @@ struct bpf_prog_offload {
u32 jited_len;
};
-/* The longest tracepoint has 12 args.
- * See include/trace/bpf_probe.h
- */
-#define MAX_BPF_FUNC_ARGS 12
-
-/* The maximum number of arguments passed through registers
- * a single function may have.
- */
-#define MAX_BPF_FUNC_REG_ARGS 5
-
-/* The argument is a structure or a union. */
-#define BTF_FMODEL_STRUCT_ARG BIT(0)
-
/* The argument is signed. */
#define BTF_FMODEL_SIGNED_ARG BIT(1)
+/* The argument is an arena pointer. */
+#define BTF_FMODEL_ARENA_ARG BIT(2)
+
+/* The argument is nullable. */
+#define BTF_FMODEL_NULLABLE_ARG BIT(3)
+
struct btf_func_model {
u8 ret_size;
u8 ret_flags;
@@ -1233,11 +1271,20 @@ enum {
#define BPF_TRAMP_COOKIE_INDEX_SHIFT 8
#define BPF_TRAMP_IS_RETURN_SHIFT 63
-struct bpf_tramp_links {
- struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS];
- int nr_links;
+struct bpf_tramp_nodes {
+ struct bpf_tramp_node *nodes[BPF_MAX_TRAMP_LINKS];
+ int nr_nodes;
};
+/*
+ * The arena base against which a struct_ops trampoline converts the
+ * arguments marked with BTF_FMODEL_ARENA_ARG while saving them into the BPF
+ * ctx, ctx[arg] = (u32)(kaddr - kern_vm_start). Zero when the trampoline
+ * converts nothing.
+ */
+u64 bpf_tramp_arena_base(const struct btf_func_model *m,
+ struct bpf_tramp_nodes *tnodes, u32 flags);
+
struct bpf_tramp_run_ctx;
/* Different use cases for BPF trampoline:
@@ -1263,13 +1310,13 @@ struct bpf_tramp_run_ctx;
struct bpf_tramp_image;
int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
const struct btf_func_model *m, u32 flags,
- struct bpf_tramp_links *tlinks,
+ struct bpf_tramp_nodes *tnodes,
void *func_addr);
void *arch_alloc_bpf_trampoline(unsigned int size);
void arch_free_bpf_trampoline(void *image, unsigned int size);
int __must_check arch_protect_bpf_trampoline(void *image, unsigned int size);
int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
- struct bpf_tramp_links *tlinks, void *func_addr);
+ struct bpf_tramp_nodes *tnodes, void *func_addr);
u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
struct bpf_tramp_run_ctx *run_ctx);
@@ -1335,8 +1382,6 @@ struct bpf_trampoline {
/* hlist for trampoline_ip_table */
struct hlist_node hlist_ip;
struct ftrace_ops *fops;
- /* serializes access to fields of this trampoline */
- struct mutex mutex;
refcount_t refcnt;
u32 flags;
u64 key;
@@ -1357,6 +1402,11 @@ struct bpf_trampoline {
int progs_cnt[BPF_TRAMP_MAX];
/* Executable image of trampoline */
struct bpf_tramp_image *cur_image;
+ /* Used as temporary old image storage for multi_attach */
+ struct {
+ struct bpf_tramp_image *old_image;
+ u32 old_flags;
+ } multi_attach;
};
struct bpf_attach_target_info {
@@ -1454,11 +1504,13 @@ static inline int bpf_dynptr_check_off_len(const struct bpf_dynptr_kern *ptr, u6
return 0;
}
+struct bpf_tracing_multi_link;
+
#ifdef CONFIG_BPF_JIT
-int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
+int bpf_trampoline_link_prog(struct bpf_tramp_node *node,
struct bpf_trampoline *tr,
struct bpf_prog *tgt_prog);
-int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
+int bpf_trampoline_unlink_prog(struct bpf_tramp_node *node,
struct bpf_trampoline *tr,
struct bpf_prog *tgt_prog);
struct bpf_trampoline *bpf_trampoline_get(u64 key,
@@ -1466,6 +1518,12 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key,
void bpf_trampoline_put(struct bpf_trampoline *tr);
int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs);
+int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids,
+ struct bpf_tracing_multi_link *link);
+void bpf_trampoline_multi_detach(struct bpf_prog *prog,
+ struct bpf_tracing_multi_link *link);
+void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags);
+
/*
* When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn
* indirection with a direct call to the bpf program. If the architecture does
@@ -1541,14 +1599,17 @@ bool bpf_has_frame_pointer(unsigned long ip);
int bpf_jit_charge_modmem(u32 size);
void bpf_jit_uncharge_modmem(u32 size);
bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
+bool bpf_insn_is_indirect_target(const struct bpf_verifier_env *env, const struct bpf_prog *prog,
+ int insn_idx);
+u16 bpf_out_stack_arg_cnt(const struct bpf_verifier_env *env, const struct bpf_prog *prog);
#else
-static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
+static inline int bpf_trampoline_link_prog(struct bpf_tramp_node *node,
struct bpf_trampoline *tr,
struct bpf_prog *tgt_prog)
{
return -ENOTSUPP;
}
-static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
+static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_node *node,
struct bpf_trampoline *tr,
struct bpf_prog *tgt_prog)
{
@@ -1575,13 +1636,24 @@ static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
{
return false;
}
+static inline int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids,
+ struct bpf_tracing_multi_link *link)
+{
+ return -ENOTSUPP;
+}
+static inline void bpf_trampoline_multi_detach(struct bpf_prog *prog,
+ struct bpf_tracing_multi_link *link)
+{
+}
+static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) {}
#endif
struct bpf_func_info_aux {
u16 linkage;
bool unreliable;
- bool called : 1;
- bool verified : 1;
+ /* Indexed by in_sleepable. */
+ bool called[2];
+ bool verified[2];
};
enum bpf_jit_poke_reason {
@@ -1612,7 +1684,7 @@ struct bpf_ctx_arg_aux {
enum bpf_reg_type reg_type;
struct btf *btf;
u32 btf_id;
- u32 ref_obj_id;
+ u32 ref_id;
bool refcounted;
};
@@ -1654,6 +1726,19 @@ struct bpf_stream_stage {
int len;
};
+enum bpf_sig_verdict {
+ BPF_SIG_UNSIGNED = 0,
+ BPF_SIG_VERIFIED,
+};
+
+enum bpf_sig_keyring {
+ BPF_SIG_KEYRING_NONE = 0,
+ BPF_SIG_KEYRING_BUILTIN,
+ BPF_SIG_KEYRING_SECONDARY,
+ BPF_SIG_KEYRING_PLATFORM,
+ BPF_SIG_KEYRING_USER,
+};
+
struct bpf_prog_aux {
atomic64_t refcnt;
u32 used_map_cnt;
@@ -1696,6 +1781,11 @@ struct bpf_prog_aux {
bool changes_pkt_data;
bool might_sleep;
bool kprobe_write_ctx;
+ struct {
+ s32 keyring_serial;
+ u8 keyring_type;
+ u8 verdict;
+ } sig;
u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */
struct bpf_arena *arena;
@@ -1728,6 +1818,7 @@ struct bpf_prog_aux {
struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
char name[BPF_OBJ_NAME_LEN];
u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64);
+ u16 stack_arg_sp_adjust;
#ifdef CONFIG_SECURITY
void *security;
#endif
@@ -1774,8 +1865,9 @@ struct bpf_prog_aux {
struct bpf_prog {
u16 pages; /* Number of allocated pages */
- u16 jited:1, /* Is our filter JIT'ed? */
+ u32 jited:1, /* Is our filter JIT'ed? */
jit_requested:1,/* archs need to JIT the prog */
+ jit_required:1, /* program strictly requires JIT compiler */
gpl_compatible:1, /* Is filter GPL compatible? */
cb_access:1, /* Is control block accessed? */
dst_needed:1, /* Do we need dst entry? */
@@ -1854,6 +1946,10 @@ struct bpf_link_ops {
* target hook is sleepable, we'll go through tasks trace RCU GP and
* then "classic" RCU GP; this need for chaining tasks trace and
* classic RCU GPs is designated by setting bpf_link->sleepable flag
+ *
+ * For non-sleepable tracepoint links we go through SRCU gp instead,
+ * since RCU is not used in that case. Sleepable tracepoints still
+ * follow the scheme above.
*/
void (*dealloc_deferred)(struct bpf_link *link);
int (*detach)(struct bpf_link *link);
@@ -1867,12 +1963,17 @@ struct bpf_link_ops {
__poll_t (*poll)(struct file *file, struct poll_table_struct *pts);
};
-struct bpf_tramp_link {
- struct bpf_link link;
+struct bpf_tramp_node {
+ struct bpf_link *link;
struct hlist_node tramp_hlist;
u64 cookie;
};
+struct bpf_tramp_link {
+ struct bpf_link link;
+ struct bpf_tramp_node node;
+};
+
struct bpf_shim_tramp_link {
struct bpf_tramp_link link;
struct bpf_trampoline *trampoline;
@@ -1880,13 +1981,31 @@ struct bpf_shim_tramp_link {
struct bpf_tracing_link {
struct bpf_tramp_link link;
+ struct bpf_tramp_node fexit;
struct bpf_trampoline *trampoline;
struct bpf_prog *tgt_prog;
};
-struct bpf_fsession_link {
- struct bpf_tracing_link link;
- struct bpf_tramp_link fexit;
+struct bpf_tracing_multi_node {
+ struct bpf_tramp_node node;
+ struct bpf_trampoline *trampoline;
+ struct ftrace_func_entry entry;
+};
+
+struct bpf_tracing_multi_data {
+ struct ftrace_hash *unreg;
+ struct ftrace_hash *modify;
+ struct ftrace_hash *reg;
+ struct ftrace_func_entry *entry;
+};
+
+struct bpf_tracing_multi_link {
+ struct bpf_link link;
+ struct bpf_tracing_multi_data data;
+ u64 *cookies;
+ struct bpf_tramp_node *fexits;
+ int nodes_cnt;
+ struct bpf_tracing_multi_node nodes[] __counted_by(nodes_cnt);
};
struct bpf_raw_tp_link {
@@ -1912,6 +2031,8 @@ struct bpf_mount_opts {
u64 delegate_maps;
u64 delegate_progs;
u64 delegate_attachs;
+
+ struct simple_xattr_cache xa_cache;
};
struct bpf_token {
@@ -2070,6 +2191,18 @@ static inline void bpf_prog_put_recursion_context(struct bpf_prog *prog)
#endif
}
+static inline bool is_tracing_multi(enum bpf_attach_type type)
+{
+ return type == BPF_TRACE_FENTRY_MULTI || type == BPF_TRACE_FEXIT_MULTI ||
+ type == BPF_TRACE_FSESSION_MULTI;
+}
+
+static inline bool is_struct_ops_tramp(const struct bpf_tramp_nodes *fentry_nodes)
+{
+ return fentry_nodes->nr_nodes == 1 &&
+ fentry_nodes->nodes[0]->link->type == BPF_LINK_TYPE_STRUCT_OPS;
+}
+
#if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
/* This macro helps developer to register a struct_ops type and generate
* type information correctly. Developers should use this macro to register
@@ -2090,8 +2223,8 @@ void bpf_struct_ops_put(const void *kdata);
int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff);
int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
void *value);
-int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
- struct bpf_tramp_link *link,
+int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_nodes *tnodes,
+ struct bpf_tramp_node *node,
const struct btf_func_model *model,
void *stub_func,
void **image, u32 *image_off,
@@ -2116,6 +2249,9 @@ int bpf_prog_assoc_struct_ops(struct bpf_prog *prog, struct bpf_map *map);
void bpf_prog_disassoc_struct_ops(struct bpf_prog *prog);
void *bpf_prog_get_assoc_struct_ops(const struct bpf_prog_aux *aux);
u32 bpf_struct_ops_id(const void *kdata);
+int bpf_struct_ops_for_each_prog(const void *kdata,
+ int (*cb)(struct bpf_prog *prog, void *data),
+ void *data);
#ifdef CONFIG_NET
/* Define it here to avoid the use of forward declaration */
@@ -2183,31 +2319,33 @@ static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_op
#endif
-static inline int bpf_fsession_cnt(struct bpf_tramp_links *links)
+static inline int bpf_fsession_cnt(struct bpf_tramp_nodes *nodes)
{
- struct bpf_tramp_links fentries = links[BPF_TRAMP_FENTRY];
+ struct bpf_tramp_nodes fentries = nodes[BPF_TRAMP_FENTRY];
int cnt = 0;
- for (int i = 0; i < links[BPF_TRAMP_FENTRY].nr_links; i++) {
- if (fentries.links[i]->link.prog->expected_attach_type == BPF_TRACE_FSESSION)
+ for (int i = 0; i < nodes[BPF_TRAMP_FENTRY].nr_nodes; i++) {
+ if (fentries.nodes[i]->link->prog->expected_attach_type == BPF_TRACE_FSESSION)
+ cnt++;
+ if (fentries.nodes[i]->link->prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI)
cnt++;
}
return cnt;
}
-static inline bool bpf_prog_calls_session_cookie(struct bpf_tramp_link *link)
+static inline bool bpf_prog_calls_session_cookie(struct bpf_tramp_node *node)
{
- return link->link.prog->call_session_cookie;
+ return node->link->prog->call_session_cookie;
}
-static inline int bpf_fsession_cookie_cnt(struct bpf_tramp_links *links)
+static inline int bpf_fsession_cookie_cnt(struct bpf_tramp_nodes *nodes)
{
- struct bpf_tramp_links fentries = links[BPF_TRAMP_FENTRY];
+ struct bpf_tramp_nodes fentries = nodes[BPF_TRAMP_FENTRY];
int cnt = 0;
- for (int i = 0; i < links[BPF_TRAMP_FENTRY].nr_links; i++) {
- if (bpf_prog_calls_session_cookie(fentries.links[i]))
+ for (int i = 0; i < nodes[BPF_TRAMP_FENTRY].nr_nodes; i++) {
+ if (bpf_prog_calls_session_cookie(fentries.nodes[i]))
cnt++;
}
@@ -2365,18 +2503,13 @@ struct bpf_prog_array {
struct bpf_prog_array_item items[];
};
-struct bpf_empty_prog_array {
- struct bpf_prog_array hdr;
- struct bpf_prog *null_prog;
-};
-
/* to avoid allocating empty bpf_prog_array for cgroups that
* don't have bpf program attached use one global 'bpf_empty_prog_array'
* It will not be modified the caller of bpf_prog_array_alloc()
* (since caller requested prog_cnt == 0)
* that pointer should be 'freed' by bpf_prog_array_free()
*/
-extern struct bpf_empty_prog_array bpf_empty_prog_array;
+extern struct bpf_prog_array bpf_empty_prog_array;
struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
void bpf_prog_array_free(struct bpf_prog_array *progs);
@@ -2594,6 +2727,7 @@ bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *r
void bpf_obj_free_timer(const struct btf_record *rec, void *obj);
void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj);
void bpf_obj_free_task_work(const struct btf_record *rec, void *obj);
+void bpf_obj_cancel_fields(struct bpf_map *map, void *obj);
void bpf_obj_free_fields(const struct btf_record *rec, void *obj);
void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu);
@@ -2760,6 +2894,9 @@ void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type,
const struct bpf_link_ops *ops, struct bpf_prog *prog,
enum bpf_attach_type attach_type, bool sleepable);
+void bpf_tramp_link_init(struct bpf_tramp_link *link, enum bpf_link_type type,
+ const struct bpf_link_ops *ops, struct bpf_prog *prog,
+ enum bpf_attach_type attach_type, u64 cookie);
int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer);
int bpf_link_settle(struct bpf_link_primer *primer);
void bpf_link_cleanup(struct bpf_link_primer *primer);
@@ -2913,10 +3050,18 @@ int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size,
size_t actual_size);
/* verify correctness of eBPF program */
-int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size);
+struct bpf_log_attr;
+int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr,
+ struct bpf_log_attr *attr_log);
#ifndef CONFIG_BPF_JIT_ALWAYS_ON
-void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
+int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
+s32 bpf_call_args_imm(s16 idx);
+#else
+static inline s32 bpf_call_args_imm(s16 idx)
+{
+ return 0;
+}
#endif
struct btf *bpf_get_btf_vmlinux(void);
@@ -3008,7 +3153,7 @@ int btf_struct_access(struct bpf_verifier_log *log,
bool btf_struct_ids_match(struct bpf_verifier_log *log,
const struct btf *btf, u32 id, int off,
const struct btf *need_btf, u32 need_type_id,
- bool strict);
+ bool strict, bool walk_flex_arrays);
int btf_distill_func_proto(struct bpf_verifier_log *log,
struct btf *btf,
@@ -3032,7 +3177,6 @@ const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id,
const struct bpf_prog *prog);
void bpf_task_storage_free(struct task_struct *task);
void bpf_cgrp_storage_free(struct cgroup *cgroup);
-bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);
const struct btf_func_model *
bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
const struct bpf_insn *insn);
@@ -3078,6 +3222,56 @@ void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);
void bpf_prog_report_arena_violation(bool write, unsigned long addr, unsigned long fault_ip);
+static __always_inline u32
+bpf_prog_run_array_sleepable(const struct bpf_prog_array *array,
+ const void *ctx, bpf_prog_run_fn run_prog)
+{
+ const struct bpf_prog_array_item *item;
+ struct bpf_prog *prog;
+ struct bpf_run_ctx *old_run_ctx;
+ struct bpf_trace_run_ctx run_ctx;
+ u32 ret = 1;
+
+ if (unlikely(!array))
+ return ret;
+
+ migrate_disable();
+
+ run_ctx.is_uprobe = false;
+
+ old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
+ item = &array->items[0];
+ while ((prog = READ_ONCE(item->prog))) {
+ /* Skip dummy_bpf_prog placeholder (len == 0) */
+ if (unlikely(!prog->len)) {
+ item++;
+ continue;
+ }
+
+ if (unlikely(!bpf_prog_get_recursion_context(prog))) {
+ bpf_prog_inc_misses_counter(prog);
+ bpf_prog_put_recursion_context(prog);
+ item++;
+ continue;
+ }
+
+ run_ctx.bpf_cookie = item->bpf_cookie;
+
+ if (!prog->sleepable) {
+ guard(rcu)();
+ ret &= run_prog(prog, ctx);
+ } else {
+ ret &= run_prog(prog, ctx);
+ }
+
+ bpf_prog_put_recursion_context(prog);
+ item++;
+ }
+ bpf_reset_run_ctx(old_run_ctx);
+ migrate_enable();
+ return ret;
+}
+
#else /* !CONFIG_BPF_SYSCALL */
static inline struct bpf_prog *bpf_prog_get(u32 ufd)
{
@@ -3125,6 +3319,12 @@ static inline void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_
{
}
+static inline void bpf_tramp_link_init(struct bpf_tramp_link *link, enum bpf_link_type type,
+ const struct bpf_link_ops *ops, struct bpf_prog *prog,
+ enum bpf_attach_type attach_type, u64 cookie)
+{
+}
+
static inline int bpf_link_prime(struct bpf_link *link,
struct bpf_link_primer *primer)
{
@@ -3315,11 +3515,6 @@ static inline void bpf_task_storage_free(struct task_struct *task)
{
}
-static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
-{
- return false;
-}
-
static inline const struct btf_func_model *
bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
const struct bpf_insn *insn)
@@ -3616,15 +3811,25 @@ static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map,
#endif /* CONFIG_BPF_SYSCALL */
#endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */
-#if defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL)
+#ifdef CONFIG_KEYS
+struct bpf_key {
+ struct key *key;
+ bool has_ref;
+};
+#endif /* CONFIG_KEYS */
+#if defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL)
struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags);
struct bpf_key *bpf_lookup_system_key(u64 id);
void bpf_key_put(struct bpf_key *bkey);
-int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_p,
- struct bpf_dynptr *sig_p,
+int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p,
+ const struct bpf_dynptr *sig_p,
struct bpf_key *trusted_keyring);
+static inline s32 bpf_key_serial(const struct bpf_key *key)
+{
+ return key->has_ref ? key->key->serial : 0;
+}
#else
static inline struct bpf_key *bpf_lookup_user_key(u32 serial, u64 flags)
{
@@ -3640,12 +3845,17 @@ static inline void bpf_key_put(struct bpf_key *bkey)
{
}
-static inline int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_p,
- struct bpf_dynptr *sig_p,
+static inline int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p,
+ const struct bpf_dynptr *sig_p,
struct bpf_key *trusted_keyring)
{
return -EOPNOTSUPP;
}
+
+static inline s32 bpf_key_serial(const struct bpf_key *key)
+{
+ return 0;
+}
#endif /* defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL) */
/* verifier prototypes for helper functions called from eBPF programs */
@@ -3724,6 +3934,7 @@ extern const struct bpf_func_proto bpf_for_each_map_elem_proto;
extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto;
extern const struct bpf_func_proto bpf_sk_setsockopt_proto;
extern const struct bpf_func_proto bpf_sk_getsockopt_proto;
+extern const struct bpf_func_proto bpf_sk_setsockopt_nodelay_proto;
extern const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto;
extern const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto;
extern const struct bpf_func_proto bpf_find_vma_proto;
@@ -3920,15 +4131,6 @@ static inline void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) {}
static inline void bpf_cgroup_atype_put(int cgroup_atype) {}
#endif /* CONFIG_BPF_LSM */
-struct key;
-
-#ifdef CONFIG_KEYS
-struct bpf_key {
- struct key *key;
- bool has_ref;
-};
-#endif /* CONFIG_KEYS */
-
static inline bool type_is_alloc(u32 type)
{
return type & MEM_ALLOC;
@@ -3946,6 +4148,17 @@ static inline bool bpf_is_subprog(const struct bpf_prog *prog)
return prog->aux->func_idx != 0;
}
+const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn_off);
+struct bpf_linfo_source {
+ const char *file;
+ const char *line;
+ u32 file_name_off;
+ int line_num;
+ int line_col;
+};
+
+void bpf_get_linfo_source(struct btf *btf, const struct bpf_line_info *linfo,
+ struct bpf_linfo_source *src);
int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep,
const char **linep, int *nump);
struct bpf_prog *bpf_prog_find_from_stack(void);
@@ -3965,7 +4178,7 @@ bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image)
}
#endif
-static inline bool bpf_map_supports_cpu_flags(enum bpf_map_type map_type)
+static inline bool bpf_map_is_percpu_map(enum bpf_map_type map_type)
{
switch (map_type) {
case BPF_MAP_TYPE_PERCPU_ARRAY:
@@ -3992,13 +4205,13 @@ static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 all
return -EINVAL;
if (flags & (BPF_F_CPU | BPF_F_ALL_CPUS)) {
- if (!bpf_map_supports_cpu_flags(map->map_type))
+ if (!bpf_map_is_percpu_map(map->map_type))
return -EINVAL;
if ((flags & BPF_F_CPU) && (flags & BPF_F_ALL_CPUS))
return -EINVAL;
cpu = flags >> 32;
- if ((flags & BPF_F_CPU) && cpu >= num_possible_cpus())
+ if ((flags & BPF_F_CPU) && (cpu >= nr_cpu_ids || !cpu_possible(cpu)))
return -ERANGE;
}
generated by cgit v1.2.3 (git 2.0.4) at 2026-09-28 01:23:10 +0000