diff options
Diffstat (limited to 'kernel/bpf/trampoline.c')
| -rw-r--r-- | kernel/bpf/trampoline.c | 48 |
1 files changed, 27 insertions, 21 deletions
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index bf4cb3dd444d..21d460866dfe 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -475,18 +475,11 @@ static int register_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im void *data __maybe_unused) { void *new_addr = im->image; - void *ip = tr->func.addr; - unsigned long faddr; int ret; - faddr = ftrace_location((unsigned long)ip); - if (faddr) { + if (tr->func.ftrace_managed) { if (!tr->fops) return -ENOTSUPP; - tr->func.ftrace_managed = true; - } - - if (tr->func.ftrace_managed) { ret = direct_ops_add(tr, new_addr); } else { ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr); @@ -859,7 +852,7 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) } } -static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog) +static int bpf_freplace_link_tgt_prog(struct bpf_prog *tgt_prog) { struct bpf_prog_aux *aux = tgt_prog->aux; @@ -873,7 +866,7 @@ static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog) */ return -EBUSY; - aux->is_extended = true; + aux->freplace_link_cnt++; return 0; } @@ -1023,13 +1016,20 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node, /* Cannot attach extension if fentry/fexit are in use. */ if (cnt) return -EBUSY; - err = bpf_freplace_check_tgt_prog(tgt_prog); + err = bpf_freplace_link_tgt_prog(tgt_prog); if (err) return err; + err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_NOP, + BPF_MOD_JUMP, NULL, + node->link->prog->bpf_func); + if (err) { + /* Undo the claim from bpf_freplace_link_tgt_prog(). */ + guard(mutex)(&tgt_prog->aux->ext_mutex); + tgt_prog->aux->freplace_link_cnt--; + return err; + } tr->extension_prog = node->link->prog; - return bpf_arch_text_poke(tr->func.addr, BPF_MOD_NOP, - BPF_MOD_JUMP, NULL, - node->link->prog->bpf_func); + return 0; } err = bpf_trampoline_add_prog(tr, node, cnt); if (err) @@ -1069,7 +1069,7 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_node *node, tr->extension_prog->bpf_func, NULL); tr->extension_prog = NULL; guard(mutex)(&tgt_prog->aux->ext_mutex); - tgt_prog->aux->is_extended = false; + tgt_prog->aux->freplace_link_cnt--; return err; } bpf_trampoline_remove_prog(tr, node); @@ -1279,6 +1279,7 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key, memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel)); tr->func.addr = (void *)tgt_info->tgt_addr; + tr->func.ftrace_managed = tr->ip != 0; out: trampoline_unlock(tr); return tr; @@ -1628,13 +1629,20 @@ static void ftrace_hash_add(struct ftrace_hash *hash, struct ftrace_func_entry * static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *ptr) { unsigned long addr = (unsigned long) im->image; - unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; + unsigned long ip = tr->ip; + + /* + * Different BTF ids get different trampolines, but they can still + * resolve to the same address (aliases, functions of the same name). + * There's only one call site to attach to. + */ + if (ftrace_lookup_ip(data->reg, ip)) + return -EINVAL; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); - tr->func.ftrace_managed = true; ftrace_hash_add(data->reg, data->entry, ip, addr); tr->cur_image = im; return 0; @@ -1643,13 +1651,12 @@ static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_ima static int unregister_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, void *ptr) { unsigned long addr = (unsigned long) tr->cur_image->image; - unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); - ftrace_hash_add(data->unreg, data->entry, ip, addr); + ftrace_hash_add(data->unreg, data->entry, tr->ip, addr); tr->cur_image = NULL; return 0; } @@ -1658,13 +1665,12 @@ static int modify_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, struct bool lock_direct_mutex, void *ptr) { unsigned long addr = (unsigned long) im->image; - unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); - ftrace_hash_add(data->modify, data->entry, ip, addr); + ftrace_hash_add(data->modify, data->entry, tr->ip, addr); tr->cur_image = im; return 0; } |