summaryrefslogtreecommitdiff
path: root/kernel/bpf
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/bpf')
-rw-r--r--kernel/bpf/arena.c2
-rw-r--r--kernel/bpf/arraymap.c6
-rw-r--r--kernel/bpf/bpf_iter.c4
-rw-r--r--kernel/bpf/bpf_struct_ops.c13
-rw-r--r--kernel/bpf/btf.c35
-rw-r--r--kernel/bpf/cgroup.c4
-rw-r--r--kernel/bpf/core.c14
-rw-r--r--kernel/bpf/crypto.c4
-rw-r--r--kernel/bpf/helpers.c4
-rw-r--r--kernel/bpf/inode.c4
-rw-r--r--kernel/bpf/liveness.c9
-rw-r--r--kernel/bpf/lpm_trie.c6
-rw-r--r--kernel/bpf/net_namespace.c2
-rw-r--r--kernel/bpf/offload.c6
-rw-r--r--kernel/bpf/syscall.c13
-rw-r--r--kernel/bpf/tcx.c2
-rw-r--r--kernel/bpf/token.c2
-rw-r--r--kernel/bpf/trampoline.c10
-rw-r--r--kernel/bpf/verifier.c69
19 files changed, 107 insertions, 102 deletions
diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c
index 42fae0a9f314..5baea15cb07d 100644
--- a/kernel/bpf/arena.c
+++ b/kernel/bpf/arena.c
@@ -324,7 +324,7 @@ static int remember_vma(struct bpf_arena *arena, struct vm_area_struct *vma)
{
struct vma_list *vml;
- vml = kmalloc(sizeof(*vml), GFP_KERNEL);
+ vml = kmalloc_obj(*vml, GFP_KERNEL);
if (!vml)
return -ENOMEM;
refcount_set(&vml->mmap_count, 1);
diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c
index 67e9e811de3a..188b0e35f856 100644
--- a/kernel/bpf/arraymap.c
+++ b/kernel/bpf/arraymap.c
@@ -1061,7 +1061,7 @@ static int prog_array_map_poke_track(struct bpf_map *map,
goto out;
}
- elem = kmalloc(sizeof(*elem), GFP_KERNEL);
+ elem = kmalloc_obj(*elem, GFP_KERNEL);
if (!elem) {
ret = -ENOMEM;
goto out;
@@ -1174,7 +1174,7 @@ static struct bpf_map *prog_array_map_alloc(union bpf_attr *attr)
struct bpf_array_aux *aux;
struct bpf_map *map;
- aux = kzalloc(sizeof(*aux), GFP_KERNEL_ACCOUNT);
+ aux = kzalloc_obj(*aux, GFP_KERNEL_ACCOUNT);
if (!aux)
return ERR_PTR(-ENOMEM);
@@ -1237,7 +1237,7 @@ static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file,
{
struct bpf_event_entry *ee;
- ee = kzalloc(sizeof(*ee), GFP_KERNEL);
+ ee = kzalloc_obj(*ee, GFP_KERNEL);
if (ee) {
ee->event = perf_file->private_data;
ee->perf_file = perf_file;
diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c
index 4b58d56ecab1..b5d16050f7b3 100644
--- a/kernel/bpf/bpf_iter.c
+++ b/kernel/bpf/bpf_iter.c
@@ -295,7 +295,7 @@ int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info)
{
struct bpf_iter_target_info *tinfo;
- tinfo = kzalloc(sizeof(*tinfo), GFP_KERNEL);
+ tinfo = kzalloc_obj(*tinfo, GFP_KERNEL);
if (!tinfo)
return -ENOMEM;
@@ -548,7 +548,7 @@ int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr,
if (prog->sleepable && !bpf_iter_target_support_resched(tinfo))
return -EINVAL;
- link = kzalloc(sizeof(*link), GFP_USER | __GFP_NOWARN);
+ link = kzalloc_obj(*link, GFP_USER | __GFP_NOWARN);
if (!link)
return -ENOMEM;
diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c
index c43346cb3d76..1ff292a6f3ed 100644
--- a/kernel/bpf/bpf_struct_ops.c
+++ b/kernel/bpf/bpf_struct_ops.c
@@ -218,7 +218,7 @@ static int prepare_arg_info(struct btf *btf,
args = btf_params(func_proto);
stub_args = btf_params(stub_func_proto);
- info_buf = kcalloc(nargs, sizeof(*info_buf), GFP_KERNEL);
+ info_buf = kzalloc_objs(*info_buf, nargs, GFP_KERNEL);
if (!info_buf)
return -ENOMEM;
@@ -378,8 +378,7 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
if (!is_valid_value_type(btf, value_id, t, value_name))
return -EINVAL;
- arg_info = kcalloc(btf_type_vlen(t), sizeof(*arg_info),
- GFP_KERNEL);
+ arg_info = kzalloc_objs(*arg_info, btf_type_vlen(t), GFP_KERNEL);
if (!arg_info)
return -ENOMEM;
@@ -721,7 +720,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
if (uvalue->common.state || refcount_read(&uvalue->common.refcnt))
return -EINVAL;
- tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL);
+ tlinks = kzalloc_objs(*tlinks, BPF_TRAMP_MAX, GFP_KERNEL);
if (!tlinks)
return -ENOMEM;
@@ -815,7 +814,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
/* Poison pointer on error instead of return for backward compatibility */
bpf_prog_assoc_struct_ops(prog, &st_map->map);
- link = kzalloc(sizeof(*link), GFP_USER);
+ link = kzalloc_obj(*link, GFP_USER);
if (!link) {
bpf_prog_put(prog);
err = -ENOMEM;
@@ -825,7 +824,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
&bpf_struct_ops_link_lops, prog, prog->expected_attach_type);
*plink++ = &link->link;
- ksym = kzalloc(sizeof(*ksym), GFP_USER);
+ ksym = kzalloc_obj(*ksym, GFP_USER);
if (!ksym) {
err = -ENOMEM;
goto reset_unlock;
@@ -1376,7 +1375,7 @@ int bpf_struct_ops_link_create(union bpf_attr *attr)
goto err_out;
}
- link = kzalloc(sizeof(*link), GFP_USER);
+ link = kzalloc_obj(*link, GFP_USER);
if (!link) {
err = -ENOMEM;
goto err_out;
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 7708958e3fb8..ee9037aa9ab7 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -1729,8 +1729,8 @@ static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t)
new_size = min_t(u32, BTF_MAX_TYPE,
btf->types_size + expand_by);
- new_types = kvcalloc(new_size, sizeof(*new_types),
- GFP_KERNEL | __GFP_NOWARN);
+ new_types = kvzalloc_objs(*new_types, new_size,
+ GFP_KERNEL | __GFP_NOWARN);
if (!new_types)
return -ENOMEM;
@@ -4072,7 +4072,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type
/* This needs to be kzalloc to zero out padding and unused fields, see
* comment in btf_record_equal.
*/
- rec = kzalloc(struct_size(rec, fields, cnt), GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
+ rec = kzalloc_flex(*rec, fields, cnt, GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
if (!rec)
return ERR_PTR(-ENOMEM);
@@ -5687,7 +5687,7 @@ btf_parse_struct_metas(struct bpf_verifier_log *log, struct btf *btf)
BUILD_BUG_ON(offsetof(struct btf_id_set, cnt) != 0);
BUILD_BUG_ON(sizeof(struct btf_id_set) != sizeof(u32));
- aof = kmalloc(sizeof(*aof), GFP_KERNEL | __GFP_NOWARN);
+ aof = kmalloc_obj(*aof, GFP_KERNEL | __GFP_NOWARN);
if (!aof)
return ERR_PTR(-ENOMEM);
aof->cnt = 0;
@@ -5885,7 +5885,7 @@ static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, u32 uat
if (attr->btf_size > BTF_MAX_SIZE)
return ERR_PTR(-E2BIG);
- env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
+ env = kzalloc_obj(*env, GFP_KERNEL | __GFP_NOWARN);
if (!env)
return ERR_PTR(-ENOMEM);
@@ -5897,7 +5897,7 @@ static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, u32 uat
if (err)
goto errout_free;
- btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
+ btf = kzalloc_obj(*btf, GFP_KERNEL | __GFP_NOWARN);
if (!btf) {
err = -ENOMEM;
goto errout;
@@ -6314,7 +6314,7 @@ static struct btf *btf_parse_base(struct btf_verifier_env *env, const char *name
if (!IS_ENABLED(CONFIG_DEBUG_INFO_BTF))
return ERR_PTR(-ENOENT);
- btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
+ btf = kzalloc_obj(*btf, GFP_KERNEL | __GFP_NOWARN);
if (!btf) {
err = -ENOMEM;
goto errout;
@@ -6365,7 +6365,7 @@ struct btf *btf_parse_vmlinux(void)
struct btf *btf;
int err;
- env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
+ env = kzalloc_obj(*env, GFP_KERNEL | __GFP_NOWARN);
if (!env)
return ERR_PTR(-ENOMEM);
@@ -6415,7 +6415,7 @@ static struct btf *btf_parse_module(const char *module_name, const void *data,
if (!vmlinux_btf)
return ERR_PTR(-EINVAL);
- env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
+ env = kzalloc_obj(*env, GFP_KERNEL | __GFP_NOWARN);
if (!env)
return ERR_PTR(-ENOMEM);
@@ -6432,7 +6432,7 @@ static struct btf *btf_parse_module(const char *module_name, const void *data,
base_btf = vmlinux_btf;
}
- btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
+ btf = kzalloc_obj(*btf, GFP_KERNEL | __GFP_NOWARN);
if (!btf) {
err = -ENOMEM;
goto errout;
@@ -8306,7 +8306,7 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op,
switch (op) {
case MODULE_STATE_COMING:
- btf_mod = kzalloc(sizeof(*btf_mod), GFP_KERNEL);
+ btf_mod = kzalloc_obj(*btf_mod, GFP_KERNEL);
if (!btf_mod) {
err = -ENOMEM;
goto out;
@@ -8341,7 +8341,7 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op,
if (IS_ENABLED(CONFIG_SYSFS)) {
struct bin_attribute *attr;
- attr = kzalloc(sizeof(*attr), GFP_KERNEL);
+ attr = kzalloc_obj(*attr, GFP_KERNEL);
if (!attr)
goto out;
@@ -8689,7 +8689,7 @@ static int btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook,
}
if (!tab) {
- tab = kzalloc(sizeof(*tab), GFP_KERNEL | __GFP_NOWARN);
+ tab = kzalloc_obj(*tab, GFP_KERNEL | __GFP_NOWARN);
if (!tab)
return -ENOMEM;
btf->kfunc_set_tab = tab;
@@ -9439,7 +9439,7 @@ int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo,
/* ~4k of temp memory necessary to convert LLVM spec like "0:1:0:5"
* into arrays of btf_ids of struct fields and array indices.
*/
- specs = kcalloc(3, sizeof(*specs), GFP_KERNEL_ACCOUNT);
+ specs = kzalloc_objs(*specs, 3, GFP_KERNEL_ACCOUNT);
if (!specs)
return -ENOMEM;
@@ -9464,7 +9464,8 @@ int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo,
goto out;
}
if (cc->cnt) {
- cands.cands = kcalloc(cc->cnt, sizeof(*cands.cands), GFP_KERNEL_ACCOUNT);
+ cands.cands = kzalloc_objs(*cands.cands, cc->cnt,
+ GFP_KERNEL_ACCOUNT);
if (!cands.cands) {
err = -ENOMEM;
goto out;
@@ -9616,7 +9617,7 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops,
tab = btf->struct_ops_tab;
if (!tab) {
- tab = kzalloc(struct_size(tab, ops, 4), GFP_KERNEL);
+ tab = kzalloc_flex(*tab, ops, 4, GFP_KERNEL);
if (!tab)
return -ENOMEM;
tab->capacity = 4;
@@ -9705,7 +9706,7 @@ int __register_bpf_struct_ops(struct bpf_struct_ops *st_ops)
if (IS_ERR(btf))
return PTR_ERR(btf);
- log = kzalloc(sizeof(*log), GFP_KERNEL | __GFP_NOWARN);
+ log = kzalloc_obj(*log, GFP_KERNEL | __GFP_NOWARN);
if (!log) {
err = -ENOMEM;
goto errout;
diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index b029f0369ecf..5d7a35e476e9 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -845,7 +845,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp,
if (pl) {
old_prog = pl->prog;
} else {
- pl = kmalloc(sizeof(*pl), GFP_KERNEL);
+ pl = kmalloc_obj(*pl, GFP_KERNEL);
if (!pl) {
bpf_cgroup_storages_free(new_storage);
return -ENOMEM;
@@ -1488,7 +1488,7 @@ int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
if (IS_ERR(cgrp))
return PTR_ERR(cgrp);
- link = kzalloc(sizeof(*link), GFP_USER);
+ link = kzalloc_obj(*link, GFP_USER);
if (!link) {
err = -ENOMEM;
goto out_put_cgroup;
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 5ab6bace7d0d..80b3e94f3fe3 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -108,7 +108,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag
if (fp == NULL)
return NULL;
- aux = kzalloc(sizeof(*aux), bpf_memcg_flags(GFP_KERNEL | gfp_extra_flags));
+ aux = kzalloc_obj(*aux, bpf_memcg_flags(GFP_KERNEL | gfp_extra_flags));
if (aux == NULL) {
vfree(fp);
return NULL;
@@ -180,9 +180,9 @@ int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog)
if (!prog->aux->nr_linfo || !prog->jit_requested)
return 0;
- prog->aux->jited_linfo = kvcalloc(prog->aux->nr_linfo,
- sizeof(*prog->aux->jited_linfo),
- bpf_memcg_flags(GFP_KERNEL | __GFP_NOWARN));
+ prog->aux->jited_linfo = kvzalloc_objs(*prog->aux->jited_linfo,
+ prog->aux->nr_linfo,
+ bpf_memcg_flags(GFP_KERNEL | __GFP_NOWARN));
if (!prog->aux->jited_linfo)
return -ENOMEM;
@@ -910,8 +910,8 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins
struct bpf_prog_pack *pack;
int err;
- pack = kzalloc(struct_size(pack, bitmap, BITS_TO_LONGS(BPF_PROG_CHUNK_COUNT)),
- GFP_KERNEL);
+ pack = kzalloc_flex(*pack, bitmap, BITS_TO_LONGS(BPF_PROG_CHUNK_COUNT),
+ GFP_KERNEL);
if (!pack)
return NULL;
pack->ptr = bpf_jit_alloc_exec(BPF_PROG_PACK_SIZE);
@@ -2597,7 +2597,7 @@ struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags)
struct bpf_prog_array *p;
if (prog_cnt)
- p = kzalloc(struct_size(p, items, prog_cnt + 1), flags);
+ p = kzalloc_flex(*p, items, prog_cnt + 1, flags);
else
p = &bpf_empty_prog_array.hdr;
diff --git a/kernel/bpf/crypto.c b/kernel/bpf/crypto.c
index 7e75a1936256..2b0660c32c92 100644
--- a/kernel/bpf/crypto.c
+++ b/kernel/bpf/crypto.c
@@ -68,7 +68,7 @@ int bpf_crypto_register_type(const struct bpf_crypto_type *type)
goto unlock;
}
- node = kmalloc(sizeof(*node), GFP_KERNEL);
+ node = kmalloc_obj(*node, GFP_KERNEL);
err = -ENOMEM;
if (!node)
goto unlock;
@@ -176,7 +176,7 @@ bpf_crypto_ctx_create(const struct bpf_crypto_params *params, u32 params__sz,
goto err_module_put;
}
- ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+ ctx = kzalloc_obj(*ctx, GFP_KERNEL);
if (!ctx) {
*err = -ENOMEM;
goto err_module_put;
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 7ac32798eb04..42a692682f18 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -3993,7 +3993,7 @@ __bpf_kfunc struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags)
if (IS_ERR(key_ref))
return NULL;
- bkey = kmalloc(sizeof(*bkey), GFP_KERNEL);
+ bkey = kmalloc_obj(*bkey, GFP_KERNEL);
if (!bkey) {
key_put(key_ref_to_ptr(key_ref));
return NULL;
@@ -4033,7 +4033,7 @@ __bpf_kfunc struct bpf_key *bpf_lookup_system_key(u64 id)
if (system_keyring_id_check(id) < 0)
return NULL;
- bkey = kmalloc(sizeof(*bkey), GFP_ATOMIC);
+ bkey = kmalloc_obj(*bkey, GFP_ATOMIC);
if (!bkey)
return NULL;
diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c
index 005ea3a2cda7..a111b0e9214e 100644
--- a/kernel/bpf/inode.c
+++ b/kernel/bpf/inode.c
@@ -195,7 +195,7 @@ static struct map_iter *map_iter_alloc(struct bpf_map *map)
{
struct map_iter *iter;
- iter = kzalloc(sizeof(*iter), GFP_KERNEL | __GFP_NOWARN);
+ iter = kzalloc_obj(*iter, GFP_KERNEL | __GFP_NOWARN);
if (!iter)
goto error;
@@ -1044,7 +1044,7 @@ static int bpf_init_fs_context(struct fs_context *fc)
{
struct bpf_mount_opts *opts;
- opts = kzalloc(sizeof(struct bpf_mount_opts), GFP_KERNEL);
+ opts = kzalloc_obj(struct bpf_mount_opts, GFP_KERNEL);
if (!opts)
return -ENOMEM;
diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
index 60db5d655495..998986853c61 100644
--- a/kernel/bpf/liveness.c
+++ b/kernel/bpf/liveness.c
@@ -193,8 +193,8 @@ static struct func_instance *__lookup_instance(struct bpf_verifier_env *env,
result = kvzalloc(size, GFP_KERNEL_ACCOUNT);
if (!result)
return ERR_PTR(-ENOMEM);
- result->must_write_set = kvcalloc(subprog_sz, sizeof(*result->must_write_set),
- GFP_KERNEL_ACCOUNT);
+ result->must_write_set = kvzalloc_objs(*result->must_write_set,
+ subprog_sz, GFP_KERNEL_ACCOUNT);
if (!result->must_write_set) {
kvfree(result);
return ERR_PTR(-ENOMEM);
@@ -217,7 +217,7 @@ static struct func_instance *lookup_instance(struct bpf_verifier_env *env,
int bpf_stack_liveness_init(struct bpf_verifier_env *env)
{
- env->liveness = kvzalloc(sizeof(*env->liveness), GFP_KERNEL_ACCOUNT);
+ env->liveness = kvzalloc_obj(*env->liveness, GFP_KERNEL_ACCOUNT);
if (!env->liveness)
return -ENOMEM;
hash_init(env->liveness->func_instances);
@@ -266,7 +266,8 @@ static struct per_frame_masks *alloc_frame_masks(struct bpf_verifier_env *env,
struct per_frame_masks *arr;
if (!instance->frames[frame]) {
- arr = kvcalloc(instance->insn_cnt, sizeof(*arr), GFP_KERNEL_ACCOUNT);
+ arr = kvzalloc_objs(*arr, instance->insn_cnt,
+ GFP_KERNEL_ACCOUNT);
instance->frames[frame] = arr;
if (!arr)
return ERR_PTR(-ENOMEM);
diff --git a/kernel/bpf/lpm_trie.c b/kernel/bpf/lpm_trie.c
index be66d7e520e0..1adeb4d3b8cf 100644
--- a/kernel/bpf/lpm_trie.c
+++ b/kernel/bpf/lpm_trie.c
@@ -683,9 +683,9 @@ static int trie_get_next_key(struct bpf_map *map, void *_key, void *_next_key)
if (!key || key->prefixlen > trie->max_prefixlen)
goto find_leftmost;
- node_stack = kmalloc_array(trie->max_prefixlen + 1,
- sizeof(struct lpm_trie_node *),
- GFP_ATOMIC | __GFP_NOWARN);
+ node_stack = kmalloc_objs(struct lpm_trie_node *,
+ trie->max_prefixlen + 1,
+ GFP_ATOMIC | __GFP_NOWARN);
if (!node_stack)
return -ENOMEM;
diff --git a/kernel/bpf/net_namespace.c b/kernel/bpf/net_namespace.c
index 8e88201c98bf..25f30f9edaef 100644
--- a/kernel/bpf/net_namespace.c
+++ b/kernel/bpf/net_namespace.c
@@ -494,7 +494,7 @@ int netns_bpf_link_create(const union bpf_attr *attr, struct bpf_prog *prog)
if (IS_ERR(net))
return PTR_ERR(net);
- net_link = kzalloc(sizeof(*net_link), GFP_USER);
+ net_link = kzalloc_obj(*net_link, GFP_USER);
if (!net_link) {
err = -ENOMEM;
goto out_put_net;
diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c
index 227f9b5f388b..7fcbbe0ad925 100644
--- a/kernel/bpf/offload.c
+++ b/kernel/bpf/offload.c
@@ -72,7 +72,7 @@ static int __bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
struct bpf_offload_netdev *ondev;
int err;
- ondev = kzalloc(sizeof(*ondev), GFP_KERNEL);
+ ondev = kzalloc_obj(*ondev, GFP_KERNEL);
if (!ondev)
return -ENOMEM;
@@ -182,7 +182,7 @@ static int __bpf_prog_dev_bound_init(struct bpf_prog *prog, struct net_device *n
struct bpf_prog_offload *offload;
int err;
- offload = kzalloc(sizeof(*offload), GFP_USER);
+ offload = kzalloc_obj(*offload, GFP_USER);
if (!offload)
return -ENOMEM;
@@ -777,7 +777,7 @@ bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv)
{
struct bpf_offload_dev *offdev;
- offdev = kzalloc(sizeof(*offdev), GFP_KERNEL);
+ offdev = kzalloc_obj(*offdev, GFP_KERNEL);
if (!offdev)
return ERR_PTR(-ENOMEM);
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index dd89bf809772..2d14fb6d0ed0 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -3633,7 +3633,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
if (prog->expected_attach_type == BPF_TRACE_FSESSION) {
struct bpf_fsession_link *fslink;
- fslink = kzalloc(sizeof(*fslink), GFP_USER);
+ fslink = kzalloc_obj(*fslink, GFP_USER);
if (fslink) {
bpf_link_init(&fslink->fexit.link, BPF_LINK_TYPE_TRACING,
&bpf_tracing_link_lops, prog, attach_type);
@@ -3643,7 +3643,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
link = NULL;
}
} else {
- link = kzalloc(sizeof(*link), GFP_USER);
+ link = kzalloc_obj(*link, GFP_USER);
}
if (!link) {
err = -ENOMEM;
@@ -4183,7 +4183,7 @@ static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *pro
if (IS_ERR(perf_file))
return PTR_ERR(perf_file);
- link = kzalloc(sizeof(*link), GFP_USER);
+ link = kzalloc_obj(*link, GFP_USER);
if (!link) {
err = -ENOMEM;
goto out_put_file;
@@ -4261,7 +4261,7 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
if (!btp)
return -ENOENT;
- link = kzalloc(sizeof(*link), GFP_USER);
+ link = kzalloc_obj(*link, GFP_USER);
if (!link) {
err = -ENOMEM;
goto out_put_btp;
@@ -6076,9 +6076,8 @@ static int bpf_prog_bind_map(union bpf_attr *attr)
goto out_unlock;
}
- used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1,
- sizeof(used_maps_new[0]),
- GFP_KERNEL);
+ used_maps_new = kmalloc_objs(used_maps_new[0],
+ prog->aux->used_map_cnt + 1, GFP_KERNEL);
if (!used_maps_new) {
ret = -ENOMEM;
goto out_unlock;
diff --git a/kernel/bpf/tcx.c b/kernel/bpf/tcx.c
index efd987ea6872..02db0113b8e7 100644
--- a/kernel/bpf/tcx.c
+++ b/kernel/bpf/tcx.c
@@ -321,7 +321,7 @@ int tcx_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
ret = -ENODEV;
goto out;
}
- tcx = kzalloc(sizeof(*tcx), GFP_USER);
+ tcx = kzalloc_obj(*tcx, GFP_USER);
if (!tcx) {
ret = -ENOMEM;
goto out;
diff --git a/kernel/bpf/token.c b/kernel/bpf/token.c
index 7e4aa1e44b50..e85a179523f0 100644
--- a/kernel/bpf/token.c
+++ b/kernel/bpf/token.c
@@ -172,7 +172,7 @@ int bpf_token_create(union bpf_attr *attr)
if (fdf.err)
return fdf.err;
- token = kzalloc(sizeof(*token), GFP_USER);
+ token = kzalloc_obj(*token, GFP_USER);
if (!token)
return -ENOMEM;
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 952cd7932461..b94565843f77 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -256,7 +256,7 @@ static int direct_ops_mod(struct bpf_trampoline *tr, void *addr, bool lock_direc
*/
static int direct_ops_alloc(struct bpf_trampoline *tr)
{
- tr->fops = kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL);
+ tr->fops = kzalloc_obj(struct ftrace_ops, GFP_KERNEL);
if (!tr->fops)
return -ENOMEM;
tr->fops->private = tr;
@@ -342,7 +342,7 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip)
goto out;
}
}
- tr = kzalloc(sizeof(*tr), GFP_KERNEL);
+ tr = kzalloc_obj(*tr, GFP_KERNEL);
if (!tr)
goto out;
if (direct_ops_alloc(tr)) {
@@ -446,7 +446,7 @@ bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_a
int kind;
*total = 0;
- tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL);
+ tlinks = kzalloc_objs(*tlinks, BPF_TRAMP_MAX, GFP_KERNEL);
if (!tlinks)
return ERR_PTR(-ENOMEM);
@@ -569,7 +569,7 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size)
void *image;
int err = -ENOMEM;
- im = kzalloc(sizeof(*im), GFP_KERNEL);
+ im = kzalloc_obj(*im, GFP_KERNEL);
if (!im)
goto out;
@@ -928,7 +928,7 @@ static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog
struct bpf_shim_tramp_link *shim_link = NULL;
struct bpf_prog *p;
- shim_link = kzalloc(sizeof(*shim_link), GFP_USER);
+ shim_link = kzalloc_obj(*shim_link, GFP_USER);
if (!shim_link)
return NULL;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index dbaafb64d3bd..63f05d90e708 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1779,7 +1779,7 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state,
for (i = 0; i <= src->curframe; i++) {
dst = dst_state->frame[i];
if (!dst) {
- dst = kzalloc(sizeof(*dst), GFP_KERNEL_ACCOUNT);
+ dst = kzalloc_obj(*dst, GFP_KERNEL_ACCOUNT);
if (!dst)
return -ENOMEM;
dst_state->frame[i] = dst;
@@ -2127,7 +2127,7 @@ static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
struct bpf_verifier_stack_elem *elem;
int err;
- elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL_ACCOUNT);
+ elem = kzalloc_obj(struct bpf_verifier_stack_elem, GFP_KERNEL_ACCOUNT);
if (!elem)
return ERR_PTR(-ENOMEM);
@@ -2949,7 +2949,7 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,
struct bpf_verifier_stack_elem *elem;
struct bpf_func_state *frame;
- elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL_ACCOUNT);
+ elem = kzalloc_obj(struct bpf_verifier_stack_elem, GFP_KERNEL_ACCOUNT);
if (!elem)
return ERR_PTR(-ENOMEM);
@@ -2972,7 +2972,7 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,
*/
elem->st.branches = 1;
elem->st.in_sleepable = is_sleepable;
- frame = kzalloc(sizeof(*frame), GFP_KERNEL_ACCOUNT);
+ frame = kzalloc_obj(*frame, GFP_KERNEL_ACCOUNT);
if (!frame)
return ERR_PTR(-ENOMEM);
init_func_state(env, frame,
@@ -3410,7 +3410,7 @@ static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
return -EINVAL;
}
- tab = kzalloc(sizeof(*tab), GFP_KERNEL_ACCOUNT);
+ tab = kzalloc_obj(*tab, GFP_KERNEL_ACCOUNT);
if (!tab)
return -ENOMEM;
prog_aux->kfunc_tab = tab;
@@ -3426,7 +3426,7 @@ static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
return 0;
if (!btf_tab && offset) {
- btf_tab = kzalloc(sizeof(*btf_tab), GFP_KERNEL_ACCOUNT);
+ btf_tab = kzalloc_obj(*btf_tab, GFP_KERNEL_ACCOUNT);
if (!btf_tab)
return -ENOMEM;
prog_aux->kfunc_btf_tab = btf_tab;
@@ -10580,7 +10580,7 @@ static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int calls
}
caller = state->frame[state->curframe];
- callee = kzalloc(sizeof(*callee), GFP_KERNEL_ACCOUNT);
+ callee = kzalloc_obj(*callee, GFP_KERNEL_ACCOUNT);
if (!callee)
return -ENOMEM;
state->frame[state->curframe + 1] = callee;
@@ -18860,11 +18860,13 @@ static int check_cfg(struct bpf_verifier_env *env)
int *insn_stack, *insn_state;
int ex_insn_beg, i, ret = 0;
- insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
+ insn_state = env->cfg.insn_state = kvzalloc_objs(int, insn_cnt,
+ GFP_KERNEL_ACCOUNT);
if (!insn_state)
return -ENOMEM;
- insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
+ insn_stack = env->cfg.insn_stack = kvzalloc_objs(int, insn_cnt,
+ GFP_KERNEL_ACCOUNT);
if (!insn_stack) {
kvfree(insn_state);
return -ENOMEM;
@@ -18951,9 +18953,9 @@ static int compute_postorder(struct bpf_verifier_env *env)
int *stack = NULL, *postorder = NULL, *state = NULL;
struct bpf_iarray *succ;
- postorder = kvcalloc(env->prog->len, sizeof(int), GFP_KERNEL_ACCOUNT);
- state = kvcalloc(env->prog->len, sizeof(int), GFP_KERNEL_ACCOUNT);
- stack = kvcalloc(env->prog->len, sizeof(int), GFP_KERNEL_ACCOUNT);
+ postorder = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT);
+ state = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT);
+ stack = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT);
if (!postorder || !state || !stack) {
kvfree(postorder);
kvfree(state);
@@ -19147,7 +19149,8 @@ static int check_btf_func(struct bpf_verifier_env *env,
urecord = make_bpfptr(attr->func_info, uattr.is_kernel);
krecord = prog->aux->func_info;
- info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
+ info_aux = kzalloc_objs(*info_aux, nfuncs,
+ GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
if (!info_aux)
return -ENOMEM;
@@ -19232,8 +19235,8 @@ static int check_btf_line(struct bpf_verifier_env *env,
/* Need to zero it in case the userspace may
* pass in a smaller bpf_line_info object.
*/
- linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info),
- GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
+ linfo = kvzalloc_objs(struct bpf_line_info, nr_linfo,
+ GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
if (!linfo)
return -ENOMEM;
@@ -20619,7 +20622,8 @@ hit:
if (loop) {
struct bpf_scc_backedge *backedge;
- backedge = kzalloc(sizeof(*backedge), GFP_KERNEL_ACCOUNT);
+ backedge = kzalloc_obj(*backedge,
+ GFP_KERNEL_ACCOUNT);
if (!backedge)
return -ENOMEM;
err = copy_verifier_state(&backedge->state, cur);
@@ -20683,7 +20687,7 @@ miss:
* When looping the sl->state.branches will be > 0 and this state
* will not be considered for equivalence until branches == 0.
*/
- new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL_ACCOUNT);
+ new_sl = kzalloc_obj(struct bpf_verifier_state_list, GFP_KERNEL_ACCOUNT);
if (!new_sl)
return -ENOMEM;
env->total_states++;
@@ -22765,7 +22769,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
goto out_undo_insn;
err = -ENOMEM;
- func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL);
+ func = kzalloc_objs(prog, env->subprog_cnt, GFP_KERNEL);
if (!func)
goto out_undo_insn;
@@ -24472,14 +24476,14 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog)
env->prev_linfo = NULL;
env->pass_cnt++;
- state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL_ACCOUNT);
+ state = kzalloc_obj(struct bpf_verifier_state, GFP_KERNEL_ACCOUNT);
if (!state)
return -ENOMEM;
state->curframe = 0;
state->speculative = false;
state->branches = 1;
state->in_sleepable = env->prog->sleepable;
- state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL_ACCOUNT);
+ state->frame[0] = kzalloc_obj(struct bpf_func_state, GFP_KERNEL_ACCOUNT);
if (!state->frame[0]) {
kfree(state);
return -ENOMEM;
@@ -25600,7 +25604,7 @@ static int compute_live_registers(struct bpf_verifier_env *env)
* - repeat the computation while {in,out} fields changes for
* any instruction.
*/
- state = kvcalloc(insn_cnt, sizeof(*state), GFP_KERNEL_ACCOUNT);
+ state = kvzalloc_objs(*state, insn_cnt, GFP_KERNEL_ACCOUNT);
if (!state) {
err = -ENOMEM;
goto out;
@@ -25828,7 +25832,8 @@ dfs_continue:
dfs_sz--;
}
}
- env->scc_info = kvcalloc(next_scc_id, sizeof(*env->scc_info), GFP_KERNEL_ACCOUNT);
+ env->scc_info = kvzalloc_objs(*env->scc_info, next_scc_id,
+ GFP_KERNEL_ACCOUNT);
if (!env->scc_info) {
err = -ENOMEM;
goto exit;
@@ -25859,7 +25864,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
/* 'struct bpf_verifier_env' can be global, but since it's not small,
* allocate/free it every time bpf_check() is called
*/
- env = kvzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL_ACCOUNT);
+ env = kvzalloc_obj(struct bpf_verifier_env, GFP_KERNEL_ACCOUNT);
if (!env)
return -ENOMEM;
@@ -25923,9 +25928,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;
env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS;
- env->explored_states = kvcalloc(state_htab_size(env),
- sizeof(struct list_head),
- GFP_KERNEL_ACCOUNT);
+ env->explored_states = kvzalloc_objs(struct list_head,
+ state_htab_size(env),
+ GFP_KERNEL_ACCOUNT);
ret = -ENOMEM;
if (!env->explored_states)
goto skip_full_check;
@@ -26062,9 +26067,9 @@ skip_full_check:
if (env->used_map_cnt) {
/* if program passed verifier, update used_maps in bpf_prog_info */
- env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt,
- sizeof(env->used_maps[0]),
- GFP_KERNEL_ACCOUNT);
+ env->prog->aux->used_maps = kmalloc_objs(env->used_maps[0],
+ env->used_map_cnt,
+ GFP_KERNEL_ACCOUNT);
if (!env->prog->aux->used_maps) {
ret = -ENOMEM;
@@ -26077,9 +26082,9 @@ skip_full_check:
}
if (env->used_btf_cnt) {
/* if program passed verifier, update used_btfs in bpf_prog_aux */
- env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt,
- sizeof(env->used_btfs[0]),
- GFP_KERNEL_ACCOUNT);
+ env->prog->aux->used_btfs = kmalloc_objs(env->used_btfs[0],
+ env->used_btf_cnt,
+ GFP_KERNEL_ACCOUNT);
if (!env->prog->aux->used_btfs) {
ret = -ENOMEM;
goto err_release_maps;
generated by cgit v1.2.3 (git 2.0.4) at 2026-02-26 10:49:30 +0000