summaryrefslogtreecommitdiff
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/trace/Kconfig12
-rw-r--r--kernel/trace/fprobe.c13
-rw-r--r--kernel/trace/trace.c8
-rw-r--r--kernel/trace/trace_boot.c97
-rw-r--r--kernel/trace/trace_dynevent.c5
-rw-r--r--kernel/trace/trace_eprobe.c2
-rw-r--r--kernel/trace/trace_fprobe.c12
-rw-r--r--kernel/trace/trace_kprobe.c27
-rw-r--r--kernel/trace/trace_probe.c1183
-rw-r--r--kernel/trace/trace_probe.h278
-rw-r--r--kernel/trace/trace_probe_kernel.h2
-rw-r--r--kernel/trace/trace_probe_tmpl.h27
-rw-r--r--kernel/trace/trace_uprobe.c5
13 files changed, 1201 insertions, 470 deletions
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index 084f34dc6c9f..0ab5916575a9 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -779,6 +779,18 @@ config PROBE_EVENTS_BTF_ARGS
kernel function entry or a tracepoint.
This is available only if BTF (BPF Type Format) support is enabled.
+config PROBE_EVENTS_DUMP_FETCHARG
+ bool "Dump of dynamic probe event fetch-arguments"
+ depends on PROBE_EVENTS
+ default n
+ help
+ This shows the dump of fetch-arguments of dynamic probe events
+ alongside their event definitions in the dynamic_events file
+ as comment lines. This is useful to debug the probe events.
+ Since this exposes the raw values in the dynamic_events file,
+ it might be a security risk. Only enable it if you need to debug
+ probe events themselves.
+
config KPROBE_EVENTS
depends on KPROBES
depends on HAVE_REGS_AND_STACK_ACCESS_API
diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c
index f681015413b8..ddb0b12a5c4a 100644
--- a/kernel/trace/fprobe.c
+++ b/kernel/trace/fprobe.c
@@ -181,7 +181,7 @@ static inline void read_fprobe_header(unsigned long *stack,
struct __fprobe_header {
struct fprobe *fp;
unsigned long size_words;
-} __packed;
+};
#define FPROBE_HEADER_SIZE_IN_LONG SIZE_IN_LONG(sizeof(struct __fprobe_header))
@@ -464,15 +464,8 @@ static bool fprobe_exists_on_hash(unsigned long ip, bool ftrace)
#ifdef CONFIG_MODULES
static void fprobe_remove_ips(unsigned long *ips, unsigned int cnt)
{
- if (!nr_fgraph_fprobes)
- __fprobe_graph_unregister();
- else if (cnt)
- ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, 1, 0);
-
- if (!nr_ftrace_fprobes)
- __fprobe_ftrace_unregister();
- else if (cnt)
- ftrace_set_filter_ips(&fprobe_ftrace_ops, ips, cnt, 1, 0);
+ fprobe_graph_remove_ips(ips, cnt);
+ fprobe_ftrace_remove_ips(ips, cnt);
}
#endif
#else
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 89dc1c0ebb90..3e0907aef172 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -4320,14 +4320,16 @@ static const char readme_msg[] =
"\t args: <name>=fetcharg[:type]\n"
"\t fetcharg: (%<register>|$<efield>), @<address>, @<symbol>[+|-<offset>],\n"
#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
- "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
+ "\t $stack<index>, $stack, $retval, $comm, $arg<N>, $current\n"
#ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
- "\t <argname>[->field[->field|.field...]],\n"
+ "\t [(structname[,field])]<argname>[->field[->field|.field...]],\n"
+ "\t [(structname[,field])](fetcharg)->field[->field|.field...],\n"
#endif
#else
- "\t $stack<index>, $stack, $retval, $comm,\n"
+ "\t $stack<index>, $stack, $retval, $comm, $current\n"
#endif
"\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
+ "\t this_cpu_read(<fetcharg>), this_cpu_ptr(<fetcharg>)\n"
"\t kernel return probes support: $retval, $arg<N>, $comm\n"
"\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, char, string, symbol,\n"
"\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
diff --git a/kernel/trace/trace_boot.c b/kernel/trace/trace_boot.c
index 2ca2541c8a58..5da0a8bbb110 100644
--- a/kernel/trace/trace_boot.c
+++ b/kernel/trace/trace_boot.c
@@ -18,6 +18,7 @@
#include <linux/trace_events.h>
#include "trace.h"
+#include "trace_dynevent.h"
#define MAX_BUF_LEN 256
@@ -172,6 +173,93 @@ trace_boot_add_synth_event(struct xbc_node *node, const char *event)
}
#endif
+#if defined(CONFIG_EPROBE_EVENTS) || defined(CONFIG_FPROBE_EVENTS)
+static int __init
+trace_boot_add_probe_event(struct xbc_node *node, const char *group,
+ const char *event, char type, const char *type_name)
+{
+ struct xbc_node *anode;
+ char buf[MAX_BUF_LEN];
+ const char *val;
+ int ret = 0;
+
+ xbc_node_for_each_array_value(node, "probes", anode, val) {
+ if (val[0] == type && (val[1] == ':' || isspace(val[1]))) {
+ ret = strscpy(buf, val, MAX_BUF_LEN);
+ if (ret < 0) {
+ pr_err("%s command is too long: %s\n", type_name, val);
+ break;
+ }
+ } else {
+ ret = snprintf(buf, MAX_BUF_LEN, "%c:%s/%s %s", type, group, event, val);
+ if (ret >= MAX_BUF_LEN || ret < 0) {
+ pr_err("%s command is too long: %c:%s/%s %s\n",
+ type_name, type, group, event, val);
+ ret = -E2BIG;
+ break;
+ }
+ }
+
+ ret = dyn_event_create(buf, NULL);
+ if (ret) {
+ pr_err("Failed to add %s: %s\n", type_name, buf);
+ break;
+ }
+ }
+
+ return ret;
+}
+#endif
+
+#ifdef CONFIG_EPROBE_EVENTS
+static inline int __init
+trace_boot_add_eprobe_event(struct xbc_node *node, const char *group,
+ const char *event)
+{
+ return trace_boot_add_probe_event(node, group, event, 'e', "eprobe");
+}
+#else
+static inline int __init
+trace_boot_add_eprobe_event(struct xbc_node *node, const char *group,
+ const char *event)
+{
+ pr_err("Event probe is not supported.\n");
+ return -EOPNOTSUPP;
+}
+#endif
+
+#ifdef CONFIG_FPROBE_EVENTS
+static inline int __init
+trace_boot_add_fprobe_event(struct xbc_node *node, const char *group,
+ const char *event)
+{
+ return trace_boot_add_probe_event(node, group, event, 'f', "fprobe");
+}
+
+static inline int __init
+trace_boot_add_tprobe_event(struct xbc_node *node, const char *group,
+ const char *event)
+{
+ return trace_boot_add_probe_event(node, group, event, 't', "tprobe");
+}
+#else
+static inline int __init
+trace_boot_add_fprobe_event(struct xbc_node *node, const char *group,
+ const char *event)
+{
+ pr_err("Fprobe event is not supported.\n");
+ return -EOPNOTSUPP;
+}
+
+static inline int __init
+trace_boot_add_tprobe_event(struct xbc_node *node, const char *group,
+ const char *event)
+{
+ pr_err("Tracepoint probe is not supported.\n");
+ return -EOPNOTSUPP;
+}
+#endif
+
#ifdef CONFIG_HIST_TRIGGERS
static int __init __printf(3, 4)
append_printf(char **bufp, char *end, const char *fmt, ...)
@@ -477,6 +565,15 @@ trace_boot_init_one_event(struct trace_array *tr, struct xbc_node *gnode,
if (!strcmp(group, "synthetic"))
if (trace_boot_add_synth_event(enode, event) < 0)
return;
+ if (!strcmp(group, "eprobes"))
+ if (trace_boot_add_eprobe_event(enode, group, event) < 0)
+ return;
+ if (!strcmp(group, "fprobes"))
+ if (trace_boot_add_fprobe_event(enode, group, event) < 0)
+ return;
+ if (!strcmp(group, "tracepoints") || !strcmp(group, "tprobes"))
+ if (trace_boot_add_tprobe_event(enode, group, event) < 0)
+ return;
mutex_lock(&event_mutex);
file = find_event_file(tr, group, event);
diff --git a/kernel/trace/trace_dynevent.c b/kernel/trace/trace_dynevent.c
index c4dfbc293bae..6f2e39b797e5 100644
--- a/kernel/trace/trace_dynevent.c
+++ b/kernel/trace/trace_dynevent.c
@@ -116,6 +116,8 @@ int dyn_event_release(const char *raw_command, struct dyn_event_operations *type
return ret;
}
+static int create_dyn_event(const char *raw_command);
+
/*
* Locked version of event creation. The event creation must be protected by
* dyn_event_ops_mutex because of protecting trace_probe_log.
@@ -124,6 +126,9 @@ int dyn_event_create(const char *raw_command, struct dyn_event_operations *type)
{
int ret;
+ if (!type)
+ return create_dyn_event(raw_command);
+
mutex_lock(&dyn_event_ops_mutex);
ret = type->create(raw_command);
mutex_unlock(&dyn_event_ops_mutex);
diff --git a/kernel/trace/trace_eprobe.c b/kernel/trace/trace_eprobe.c
index bcd97cb24ac9..78fa1cbda9ac 100644
--- a/kernel/trace/trace_eprobe.c
+++ b/kernel/trace/trace_eprobe.c
@@ -87,6 +87,8 @@ static int eprobe_dyn_event_show(struct seq_file *m, struct dyn_event *ev)
seq_printf(m, " %s=%s", ep->tp.args[i].name, ep->tp.args[i].comm);
seq_putc(m, '\n');
+ trace_probe_dump_args(m, &ep->tp);
+
return 0;
}
diff --git a/kernel/trace/trace_fprobe.c b/kernel/trace/trace_fprobe.c
index 3120403a5b60..4e9034ebc3fb 100644
--- a/kernel/trace/trace_fprobe.c
+++ b/kernel/trace/trace_fprobe.c
@@ -238,13 +238,10 @@ static bool trace_fprobe_is_busy(struct dyn_event *ev)
static bool trace_fprobe_match_command_head(struct trace_fprobe *tf,
int argc, const char **argv)
{
- char buf[MAX_ARGSTR_LEN + 1];
-
if (!argc)
return true;
- snprintf(buf, sizeof(buf), "%s", trace_fprobe_symbol(tf));
- if (strcmp(buf, argv[0]))
+ if (strcmp(trace_fprobe_symbol(tf), argv[0]))
return false;
argc--; argv++;
@@ -474,7 +471,6 @@ static int fentry_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,
regs = ftrace_fill_perf_regs(fregs, regs);
entry->ip = entry_ip;
- memset(&entry[1], 0, dsize);
store_trace_args(&entry[1], &tf->tp, fregs, NULL, sizeof(*entry), dsize);
perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
head, NULL);
@@ -764,7 +760,7 @@ static int unregister_fprobe_event(struct trace_fprobe *tf)
return trace_probe_unregister_event_call(&tf->tp);
}
-static int __regsiter_tracepoint_fprobe(struct trace_fprobe *tf)
+static int __register_tracepoint_fprobe(struct trace_fprobe *tf)
{
struct tracepoint_user *tuser __free(tuser_put) = NULL;
struct module *mod __free(module_put) = NULL;
@@ -836,7 +832,7 @@ static int __register_trace_fprobe(struct trace_fprobe *tf)
tf->fp.flags &= ~FPROBE_FL_DISABLED;
if (trace_fprobe_is_tracepoint(tf))
- return __regsiter_tracepoint_fprobe(tf);
+ return __register_tracepoint_fprobe(tf);
/* TODO: handle filter, nofilter or symbol list */
return register_fprobe(&tf->fp, tf->symbol, NULL);
@@ -1449,6 +1445,8 @@ static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev)
seq_printf(m, " %s=%s", tf->tp.args[i].name, tf->tp.args[i].comm);
seq_putc(m, '\n');
+ trace_probe_dump_args(m, &tf->tp);
+
return 0;
}
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index a8420e6abb56..e41a7c113646 100644
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -149,20 +149,28 @@ static bool trace_kprobe_is_busy(struct dyn_event *ev)
static bool trace_kprobe_match_command_head(struct trace_kprobe *tk,
int argc, const char **argv)
{
- char buf[MAX_ARGSTR_LEN + 1];
+ char buf[32];
+ int len;
if (!argc)
return true;
- if (!tk->symbol)
+ if (!tk->symbol) {
snprintf(buf, sizeof(buf), "0x%p", tk->rp.kp.addr);
- else if (tk->rp.kp.offset)
- snprintf(buf, sizeof(buf), "%s+%u",
- trace_kprobe_symbol(tk), tk->rp.kp.offset);
- else
- snprintf(buf, sizeof(buf), "%s", trace_kprobe_symbol(tk));
- if (strcmp(buf, argv[0]))
+ if (strcmp(buf, argv[0]))
+ return false;
+ } else if (tk->rp.kp.offset) {
+ len = strlen(trace_kprobe_symbol(tk));
+ if (strncmp(trace_kprobe_symbol(tk), argv[0], len) ||
+ argv[0][len] != '+')
+ return false;
+
+ snprintf(buf, sizeof(buf), "%u", tk->rp.kp.offset);
+ if (strcmp(buf, &argv[0][len + 1]))
+ return false;
+ } else if (strcmp(trace_kprobe_symbol(tk), argv[0]))
return false;
+
argc--; argv++;
return trace_probe_match_command_args(&tk->tp, argc, argv);
@@ -1320,6 +1328,8 @@ static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev)
seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm);
seq_putc(m, '\n');
+ trace_probe_dump_args(m, &tk->tp);
+
return 0;
}
@@ -1719,7 +1729,6 @@ kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
return 0;
entry->ip = (unsigned long)tk->rp.kp.addr;
- memset(&entry[1], 0, dsize);
store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize);
perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
head, NULL);
diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c
index c8fd9b946f44..c4163904ba74 100644
--- a/kernel/trace/trace_probe.c
+++ b/kernel/trace/trace_probe.c
@@ -345,6 +345,60 @@ static int parse_trace_event(char *arg, struct fetch_insn *code,
return -EINVAL;
}
+/* this_cpu_* parser */
+#define THIS_CPU_PTR_PREFIX "this_cpu_ptr("
+#define THIS_CPU_READ_PREFIX "this_cpu_read("
+#define THIS_CPU_PTR_LEN (sizeof(THIS_CPU_PTR_PREFIX) - 1)
+#define THIS_CPU_READ_LEN (sizeof(THIS_CPU_READ_PREFIX) - 1)
+
+static int
+parse_probe_arg(char *arg, const struct fetch_type *type,
+ struct fetch_insn **pcode, struct fetch_insn *end,
+ struct traceprobe_parse_context *ctx);
+
+static int parse_this_cpu(char *arg, struct traceprobe_parse_context *ctx)
+{
+ bool is_read = false;
+ char *tmp;
+
+ /*
+ * This is only for kernel probes, excluding eprobe, because per-cpu
+ * pointer should not be recorded by events.
+ */
+ if (!(ctx->flags & TPARG_FL_KERNEL) ||
+ (ctx->flags & TPARG_FL_TEVENT)) {
+ trace_probe_log_err(ctx->offset, NOSUP_PERCPU);
+ return -EINVAL;
+ }
+ if (str_has_prefix(arg, THIS_CPU_PTR_PREFIX)) {
+ arg += THIS_CPU_PTR_LEN;
+ ctx->offset += THIS_CPU_PTR_LEN;
+ } else if (str_has_prefix(arg, THIS_CPU_READ_PREFIX)) {
+ arg += THIS_CPU_READ_LEN;
+ ctx->offset += THIS_CPU_READ_LEN;
+ is_read = true;
+ } else {
+ trace_probe_log_err(ctx->offset, BAD_FETCH_ARG);
+ return -EINVAL;
+ }
+
+ tmp = strrchr(arg, ')');
+ if (!tmp) {
+ trace_probe_log_err(ctx->offset + strlen(arg),
+ DEREF_OPEN_BRACE);
+ return -EINVAL;
+ }
+ *tmp = '\0';
+
+ ctx->stack[ctx->depth].type = STATE_DEREF;
+ ctx->stack[ctx->depth].deref.deref = FETCH_OP_CPU_PTR;
+ ctx->stack[ctx->depth].deref.offset = 0;
+ ctx->stack[ctx->depth].deref.cur_offs = ctx->offset;
+ ctx->stack[ctx->depth].deref.inner_arg = arg;
+ ctx->stack[ctx->depth].deref.is_cpu_read = is_read;
+ return 0;
+}
+
#ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
static u32 btf_type_int(const struct btf_type *t)
@@ -356,9 +410,8 @@ static bool btf_type_is_char_ptr(struct btf *btf, const struct btf_type *type)
{
const struct btf_type *real_type;
u32 intdata;
- s32 tid;
- real_type = btf_type_skip_modifiers(btf, type->type, &tid);
+ real_type = btf_type_skip_modifiers(btf, type->type, NULL);
if (!real_type)
return false;
@@ -375,14 +428,13 @@ static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type)
const struct btf_type *real_type;
const struct btf_array *array;
u32 intdata;
- s32 tid;
if (BTF_INFO_KIND(type->info) != BTF_KIND_ARRAY)
return false;
array = (const struct btf_array *)(type + 1);
- real_type = btf_type_skip_modifiers(btf, array->type, &tid);
+ real_type = btf_type_skip_modifiers(btf, array->type, NULL);
intdata = btf_type_int(real_type);
return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
@@ -443,10 +495,7 @@ static const char *fetch_type_from_btf_type(struct btf *btf,
return "s64";
case BTF_KIND_PTR:
/* pointer will be converted to "x??" */
- if (IS_ENABLED(CONFIG_64BIT))
- return "x64";
- else
- return "x32";
+ return IS_ENABLED(CONFIG_64BIT) ? "x64" : "x32";
case BTF_KIND_INT:
intdata = btf_type_int(type);
if (BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) {
@@ -570,6 +619,64 @@ static int split_next_field(char *varname, char **next_field,
return ret;
}
+/* Inner loop for solving dot operator ('.'). Return bit-offset of the given field */
+static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **ptype,
+ struct traceprobe_parse_context *ctx)
+{
+ const struct btf_type *type = *ptype;
+ const struct btf_member *field;
+ struct btf *btf = ctx_btf(ctx);
+ char *fieldname = *pfieldname;
+ int bitoffs = 0;
+ u32 anon_offs;
+ char *next;
+ int is_ptr;
+
+ do {
+ next = NULL;
+ is_ptr = split_next_field(fieldname, &next, ctx);
+ if (is_ptr < 0)
+ return is_ptr;
+
+ anon_offs = 0;
+ field = btf_find_struct_member(btf, type, fieldname,
+ &anon_offs);
+ if (IS_ERR(field)) {
+ trace_probe_log_err(ctx->offset, BAD_BTF_TID);
+ return PTR_ERR(field);
+ }
+ if (!field) {
+ trace_probe_log_err(ctx->offset, NO_BTF_FIELD);
+ return -ENOENT;
+ }
+ /* Add anonymous structure/union offset */
+ bitoffs += anon_offs;
+
+ /* Accumulate the bit-offsets of the dot-connected fields */
+ if (btf_type_kflag(type)) {
+ bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset);
+ ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset);
+ } else {
+ bitoffs += field->offset;
+ ctx->last_bitsize = 0;
+ }
+
+ type = btf_type_skip_modifiers(btf, field->type, NULL);
+ if (!type) {
+ trace_probe_log_err(ctx->offset, BAD_BTF_TID);
+ return -EINVAL;
+ }
+
+ if (next)
+ ctx->offset += next - fieldname;
+ fieldname = next;
+ } while (!is_ptr && fieldname);
+
+ *pfieldname = fieldname;
+ *ptype = type;
+
+ return bitoffs;
+}
/*
* Parse the field of data structure. The @type must be a pointer type
* pointing the target data structure type.
@@ -579,16 +686,13 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type,
struct traceprobe_parse_context *ctx)
{
struct fetch_insn *code = *pcode;
- const struct btf_member *field;
- u32 bitoffs, anon_offs;
- bool is_struct = ctx->struct_btf != NULL;
struct btf *btf = ctx_btf(ctx);
- char *next;
- int is_ptr;
- s32 tid;
+ bool is_first_field = true;
+ int bitoffs;
do {
- if (!is_struct) {
+ /* For the first field of typecast, @type will be the target structure type. */
+ if (!(is_first_field && ctx->struct_btf)) {
/* Outer loop for solving arrow operator ('->') */
if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) {
trace_probe_log_err(ctx->offset, NO_PTR_STRCT);
@@ -596,66 +700,31 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type,
}
/* Convert a struct pointer type to a struct type */
- type = btf_type_skip_modifiers(btf, type->type, &tid);
+ type = btf_type_skip_modifiers(btf, type->type, NULL);
if (!type) {
trace_probe_log_err(ctx->offset, BAD_BTF_TID);
return -EINVAL;
}
}
- /* Only the first type can skip being a pointer */
- is_struct = false;
-
- bitoffs = 0;
- do {
- /* Inner loop for solving dot operator ('.') */
- next = NULL;
- is_ptr = split_next_field(fieldname, &next, ctx);
- if (is_ptr < 0)
- return is_ptr;
-
- anon_offs = 0;
- field = btf_find_struct_member(btf, type, fieldname,
- &anon_offs);
- if (IS_ERR(field)) {
- trace_probe_log_err(ctx->offset, BAD_BTF_TID);
- return PTR_ERR(field);
- }
- if (!field) {
- trace_probe_log_err(ctx->offset, NO_BTF_FIELD);
- return -ENOENT;
- }
- /* Add anonymous structure/union offset */
- bitoffs += anon_offs;
-
- /* Accumulate the bit-offsets of the dot-connected fields */
- if (btf_type_kflag(type)) {
- bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset);
- ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset);
- } else {
- bitoffs += field->offset;
- ctx->last_bitsize = 0;
- }
-
- type = btf_type_skip_modifiers(btf, field->type, &tid);
- if (!type) {
- trace_probe_log_err(ctx->offset, BAD_BTF_TID);
- return -EINVAL;
- }
-
- ctx->offset += next - fieldname;
- fieldname = next;
- } while (!is_ptr && fieldname);
+ bitoffs = get_bitoffset_of_field(&fieldname, &type, ctx);
+ if (bitoffs < 0)
+ return bitoffs;
if (++code == end) {
trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
return -EINVAL;
}
code->op = FETCH_OP_DEREF; /* TODO: user deref support */
code->offset = bitoffs / 8;
+ if (is_first_field && ctx->struct_btf) {
+ /* The first field can be typecasted with field option. */
+ code->offset -= ctx->prefix_byteoffs;
+ }
*pcode = code;
ctx->last_bitoffs = bitoffs % 8;
ctx->last_type = type;
+ is_first_field = false;
} while (fieldname);
return 0;
@@ -674,7 +743,9 @@ static int parse_btf_arg(char *varname,
int i, is_ptr, ret;
u32 tid;
- if (!ctx->funcname && !(ctx->flags & TPARG_FL_TEVENT))
+ /* Note: field is not separated at this point, so check prefix. */
+ if (!str_has_prefix(varname, "$current") &&
+ !ctx->funcname && !(ctx->flags & TPARG_FL_TEVENT))
return -EINVAL;
is_ptr = split_next_field(varname, &field, ctx);
@@ -687,22 +758,23 @@ static int parse_btf_arg(char *varname,
return -EOPNOTSUPP;
}
- if (ctx->flags & TPARG_FL_TEVENT) {
- ret = parse_trace_event(varname, code, ctx);
+ if (!strcmp(varname, "$current")) {
+ code->op = FETCH_OP_CURRENT;
+ /* If no typecast is specified for $current, use task_struct by default */
+ ret = bpf_find_btf_id("task_struct", BTF_KIND_STRUCT, &ctx->struct_btf);
if (ret < 0) {
- trace_probe_log_err(ctx->offset, BAD_ATTACH_ARG);
- return ret;
+ trace_probe_log_err(ctx->offset, NO_BTF_ENTRY);
+ return -ENOENT;
}
- /* TEVENT is only here via a typecast */
- if (WARN_ON_ONCE(ctx->struct_btf == NULL))
- return -EINVAL;
- type = ctx->last_struct;
+ tid = (u32)ret;
+ type = ctx->last_struct =
+ btf_type_skip_modifiers(ctx->struct_btf, tid, NULL);
goto found_type;
}
if (ctx->flags & TPARG_FL_RETURN && !strcmp(varname, "$retval")) {
code->op = FETCH_OP_RETVAL;
- /* Check whether the function return type is not void */
+ /* Check whether the function return type is not void, even with typecast. */
if (query_btf_context(ctx) == 0) {
if (ctx->proto->type == 0) {
trace_probe_log_err(ctx->offset, NO_RETVAL);
@@ -755,7 +827,7 @@ static int parse_btf_arg(char *varname,
return -ENOENT;
found:
- type = btf_type_skip_modifiers(ctx->btf, tid, &tid);
+ type = btf_type_skip_modifiers(ctx->btf, tid, NULL);
found_type:
if (!type) {
trace_probe_log_err(ctx->offset, BAD_BTF_TID);
@@ -825,39 +897,165 @@ static int query_btf_struct(const char *sname, struct traceprobe_parse_context *
return 0;
}
-static int handle_typecast(char *arg, struct fetch_insn **pcode,
- struct fetch_insn *end,
- struct traceprobe_parse_context *ctx)
+static int parse_btf_casttype(char *casttype, struct traceprobe_parse_context *ctx)
{
- char *tmp;
+ char *field;
int ret;
- /* Currently this only works for eprobes */
- if (!(ctx->flags & TPARG_FL_TEVENT)) {
- trace_probe_log_err(ctx->offset, TYPECAST_NOT_EVENT);
+ /* Field option - evaluated later. */
+ field = strchr(casttype, ',');
+ if (field)
+ *field++ = '\0';
+
+ ret = query_btf_struct(casttype, ctx);
+ if (ret < 0) {
+ trace_probe_log_err(ctx->offset, NO_PTR_STRCT);
return -EINVAL;
}
+ if (field) {
+ struct btf_type *type = (struct btf_type *)ctx->last_struct;
+
+ ctx->offset += field - casttype;
+ ret = get_bitoffset_of_field(&field, &ctx->last_struct, ctx);
+ if (ret < 0)
+ return ret;
+ if (ret % 8) {
+ trace_probe_log_err(ctx->offset, TYPECAST_NOT_ALIGNED);
+ return -EINVAL;
+ }
+ if (field != NULL) {
+ /* this means @field skips an arrow operator ("->"). */
+ trace_probe_log_err(ctx->offset - 2, TYPECAST_BAD_ARROW);
+ return -EINVAL;
+ }
+ ctx->prefix_byteoffs = ret / 8;
+ /* Restore the original struct type (overwritten by get_bitoffset_of_field) */
+ ctx->last_struct = type;
+ }
+
+ return ret;
+}
+
+/* Find the matching closing parenthesis for a given opening parenthesis. */
+static char *find_matched_close_paren(char *s)
+{
+ char *p = s;
+ int count = 0;
+
+ while (*p) {
+ if (*p == '(')
+ count++;
+ else if (*p == ')') {
+ if (--count == 0)
+ return p;
+ }
+ p++;
+ }
+ return NULL;
+}
+
+static int handle_typecast(char *arg, struct traceprobe_parse_context *ctx)
+{
+ int orig_offset = ctx->offset;
+ char *close;
+ char *tmp;
+ char *fieldname;
+
+ if (!(tparg_is_event_probe(ctx->flags) ||
+ tparg_is_function_entry(ctx->flags) ||
+ tparg_is_function_return(ctx->flags))) {
+ trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
+ return -EOPNOTSUPP;
+ }
+
+ /*
+ * Always consider the token after typecast as a nested call
+ * For example: (STRUCT)VAR->FIELD and (STRUCT)(VAR)->FIELD are same.
+ * VAR is solved in the nested call.
+ */
tmp = strchr(arg, ')');
if (!tmp) {
trace_probe_log_err(ctx->offset + strlen(arg),
DEREF_OPEN_BRACE);
return -EINVAL;
}
- *tmp = '\0';
- ret = query_btf_struct(arg + 1, ctx);
- *tmp = ')';
+ *tmp++ = '\0';
- if (ret < 0) {
- trace_probe_log_err(ctx->offset + 1, NO_PTR_STRCT);
- return -EINVAL;
- }
+ ctx->offset += tmp - arg;
+ if (*tmp == '(') {
+ close = find_matched_close_paren(tmp);
- tmp++;
+ if (!close) {
+ trace_probe_log_err(ctx->offset, DEREF_OPEN_BRACE);
+ return -EINVAL;
+ }
+ /* We expect a field access for typecast */
+ if (close[1] != '-' || close[2] != '>') {
+ trace_probe_log_err(ctx->offset + close - tmp + 1,
+ TYPECAST_REQ_FIELD);
+ return -EINVAL;
+ }
+ /* Skip '(' */
+ ctx->offset += 1;
+ tmp++;
+ } else if (*tmp == '+' || *tmp == '-' ||
+ str_has_prefix(tmp, THIS_CPU_PTR_PREFIX) ||
+ str_has_prefix(tmp, THIS_CPU_READ_PREFIX)) {
+ /* Dereference can have another field access inside it. */
+ char *open = strchr(tmp + 1, '(');
+
+ if (!open) {
+ trace_probe_log_err(ctx->offset,
+ DEREF_NEED_BRACE);
+ return -EINVAL;
+ }
+ close = find_matched_close_paren(open);
+ if (!close) {
+ trace_probe_log_err(ctx->offset + strlen(tmp),
+ DEREF_OPEN_BRACE);
+ return -EINVAL;
+ }
+ close++;
+ /* We expect a field access for typecast */
+ if (close[0] != '-' || close[1] != '>') {
+ trace_probe_log_err(ctx->offset + close - tmp,
+ TYPECAST_REQ_FIELD);
+ return -EINVAL;
+ }
+ } else {
+ if (tmp[0] == '@') {
+ /* @sym+offset is not allowed without parenthesized */
+ close = strpbrk(tmp, "+-");
+ if (close && isdigit(close[1])) {
+ trace_probe_log_err(ctx->offset,
+ TYPECAST_SYM_OFFSET);
+ return -EINVAL;
+ }
+ }
+ /* Inner variable name */
+ close = strchr(tmp, '-');
+ if (!close || close[1] != '>') {
+ trace_probe_log_err(ctx->offset + strlen(tmp),
+ TYPECAST_REQ_FIELD);
+ return -EINVAL;
+ }
+ }
+ *close = '\0';
- ctx->offset += tmp - arg;
- ret = parse_btf_arg(tmp, pcode, end, ctx);
- return ret;
+ /* Let fieldname point the field name. */
+ if (close[1] == '-')
+ fieldname = close + 3; /* Skip "->" after closing parenthesis */
+ else
+ fieldname = close + 2; /* Skip ">" after inner variable name */
+
+ ctx->stack[ctx->depth].type = STATE_TYPECAST;
+ ctx->stack[ctx->depth].typecast.casttype = arg + 1;
+ ctx->stack[ctx->depth].typecast.fieldname = fieldname;
+ ctx->stack[ctx->depth].typecast.orig_offset = orig_offset;
+ ctx->stack[ctx->depth].typecast.field_offset_diff = fieldname - arg;
+ ctx->stack[ctx->depth].typecast.inner_arg = tmp;
+ return 0;
}
#else /* !CONFIG_PROBE_EVENTS_BTF_ARGS */
@@ -902,10 +1100,21 @@ static int check_prepare_btf_string_fetch(char *typename,
return 0;
}
-static int handle_typecast(char *arg, struct fetch_insn **pcode,
- struct fetch_insn *end,
+static int parse_btf_casttype(char *casttype,
+ struct traceprobe_parse_context *ctx)
+{
+ return -EOPNOTSUPP;
+}
+
+static int parse_btf_field(char *fieldname, const struct btf_type *type,
+ struct fetch_insn **pcode, struct fetch_insn *end,
struct traceprobe_parse_context *ctx)
{
+ return -EOPNOTSUPP;
+}
+
+static int handle_typecast(char *arg, struct traceprobe_parse_context *ctx)
+{
trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
return -EOPNOTSUPP;
}
@@ -1034,7 +1243,7 @@ void store_trace_entry_data(void *edata, struct trace_probe *tp, struct pt_regs
val = regs_get_kernel_argument(regs, code->param);
break;
case FETCH_OP_ST_EDATA:
- *(unsigned long *)((unsigned long)edata + code->offset) = val;
+ *(unsigned long *)((u8 *)edata + code->offset) = val;
break;
case FETCH_OP_END:
goto end;
@@ -1050,67 +1259,147 @@ NOKPROBE_SYMBOL(store_trace_entry_data)
#define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
-/* Parse $vars. @orig_arg points '$', which syncs to @ctx->offset */
-static int parse_probe_vars(char *orig_arg, const struct fetch_type *t,
- struct fetch_insn **pcode,
- struct fetch_insn *end,
- struct traceprobe_parse_context *ctx)
+static int parse_probe_var_retval(char *orig_arg,
+ struct fetch_insn **pcode,
+ struct fetch_insn *end,
+ struct traceprobe_parse_context *ctx)
{
struct fetch_insn *code = *pcode;
- int err = TP_ERR_BAD_VAR;
- char *arg = orig_arg + 1;
+
+ if (!(ctx->flags & TPARG_FL_RETURN)) {
+ trace_probe_log_err(ctx->offset, RETVAL_ON_PROBE);
+ return -EINVAL;
+ }
+ if (!(ctx->flags & TPARG_FL_KERNEL) ||
+ !IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
+ code->op = FETCH_OP_RETVAL;
+ return 0;
+ }
+ return parse_btf_arg(orig_arg, pcode, end, ctx);
+}
+
+static int parse_probe_var_stack(char *arg, int len, struct fetch_insn *code,
+ struct traceprobe_parse_context *ctx)
+{
unsigned long param;
- int ret = 0;
- int len;
+ int ret;
- if (ctx->flags & TPARG_FL_TEVENT) {
- if (parse_trace_event(arg, code, ctx) < 0) {
- /* 'comm' should be checked after field parsing. */
- if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
- code->op = FETCH_OP_COMM;
- return 0;
- }
- goto inval;
- }
+ if (arg[len] == '\0') {
+ code->op = FETCH_OP_STACKP;
return 0;
}
- if (str_has_prefix(arg, "retval")) {
- if (!(ctx->flags & TPARG_FL_RETURN)) {
- err = TP_ERR_RETVAL_ON_PROBE;
- goto inval;
+ if (isdigit(arg[len])) {
+ ret = kstrtoul(arg + len, 10, &param);
+ if (ret) {
+ trace_probe_log_err(ctx->offset, BAD_VAR);
+ return ret;
}
- if (!(ctx->flags & TPARG_FL_KERNEL) ||
- !IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
- code->op = FETCH_OP_RETVAL;
- return 0;
+
+ if ((ctx->flags & TPARG_FL_KERNEL) &&
+ param > PARAM_MAX_STACK) {
+ trace_probe_log_err(ctx->offset, BAD_STACK_NUM);
+ return -EINVAL;
}
+ code->op = FETCH_OP_STACK;
+ code->param = (unsigned int)param;
+ return 0;
+ }
+
+ trace_probe_log_err(ctx->offset, BAD_VAR);
+ return -EINVAL;
+}
+
+static int parse_probe_var_current(char *orig_arg, char *arg,
+ struct fetch_insn **pcode,
+ struct fetch_insn *end,
+ struct traceprobe_parse_context *ctx)
+{
+ struct fetch_insn *code = *pcode;
+
+ /* $current is only supported by kernel probe. */
+ if (!(ctx->flags & TPARG_FL_KERNEL)) {
+ trace_probe_log_err(ctx->offset, BAD_VAR);
+ return -EINVAL;
+ }
+ arg += strlen("current");
+ if (*arg == '-' && IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS))
return parse_btf_arg(orig_arg, pcode, end, ctx);
+
+ if (*arg != '\0') {
+ trace_probe_log_err(ctx->offset, BAD_VAR);
+ return -EINVAL;
}
- len = str_has_prefix(arg, "stack");
- if (len) {
+ code->op = FETCH_OP_CURRENT;
+ return 0;
+}
- if (arg[len] == '\0') {
- code->op = FETCH_OP_STACKP;
- return 0;
- }
+#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
+static int parse_probe_var_arg(char *arg, int len, struct fetch_insn *code,
+ struct traceprobe_parse_context *ctx)
+{
+ unsigned long param;
+ int ret;
- if (isdigit(arg[len])) {
- ret = kstrtoul(arg + len, 10, &param);
- if (ret)
- goto inval;
+ ret = kstrtoul(arg + len, 10, &param);
+ if (ret) {
+ trace_probe_log_err(ctx->offset, BAD_VAR);
+ return ret;
+ }
- if ((ctx->flags & TPARG_FL_KERNEL) &&
- param > PARAM_MAX_STACK) {
- err = TP_ERR_BAD_STACK_NUM;
- goto inval;
- }
- code->op = FETCH_OP_STACK;
- code->param = (unsigned int)param;
+ if (!param || param > PARAM_MAX_STACK) {
+ trace_probe_log_err(ctx->offset, BAD_ARG_NUM);
+ return -EINVAL;
+ }
+ param--; /* argN starts from 1, but internal arg[N] starts from 0 */
+
+ if (tparg_is_function_entry(ctx->flags)) {
+ code->op = FETCH_OP_ARG;
+ code->param = (unsigned int)param;
+ /*
+ * The tracepoint probe will probe a stub function, and the
+ * first parameter of the stub is a dummy and should be ignored.
+ */
+ if (ctx->flags & TPARG_FL_TPOINT)
+ code->param++;
+ } else if (tparg_is_function_return(ctx->flags)) {
+ /* function entry argument access from return probe */
+ ret = __store_entry_arg(ctx->tp, param);
+ if (ret < 0) /* This error should be an internal error */
+ return ret;
+
+ code->op = FETCH_OP_EDATA;
+ code->offset = ret;
+ } else {
+ trace_probe_log_err(ctx->offset, NOFENTRY_ARGS);
+ return -EINVAL;
+ }
+ return 0;
+}
+#else
+static int parse_probe_var_arg(char *arg, int len, struct fetch_insn *code,
+ struct traceprobe_parse_context *ctx)
+{
+ trace_probe_log_err(ctx->offset, BAD_VAR);
+ return -EINVAL;
+}
+#endif
+
+/* Parse $vars. @orig_arg points '$', which syncs to @ctx->offset */
+static int parse_probe_vars(char *orig_arg, const struct fetch_type *t,
+ struct fetch_insn **pcode,
+ struct fetch_insn *end,
+ struct traceprobe_parse_context *ctx)
+{
+ struct fetch_insn *code = *pcode;
+ char *arg = orig_arg + 1;
+ int len, ret;
+
+ if (ctx->flags & TPARG_FL_TEVENT) {
+ ret = parse_trace_event(arg, code, ctx);
+ if (!ret)
return 0;
- }
- goto inval;
}
if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
@@ -1118,46 +1407,27 @@ static int parse_probe_vars(char *orig_arg, const struct fetch_type *t,
return 0;
}
-#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
- len = str_has_prefix(arg, "arg");
- if (len) {
- ret = kstrtoul(arg + len, 10, &param);
- if (ret)
- goto inval;
+ /* eprobe only support event fields or '$comm'. */
+ if (ctx->flags & TPARG_FL_TEVENT)
+ goto inval;
- if (!param || param > PARAM_MAX_STACK) {
- err = TP_ERR_BAD_ARG_NUM;
- goto inval;
- }
- param--; /* argN starts from 1, but internal arg[N] starts from 0 */
+ if (str_has_prefix(arg, "retval"))
+ return parse_probe_var_retval(orig_arg, pcode, end, ctx);
- if (tparg_is_function_entry(ctx->flags)) {
- code->op = FETCH_OP_ARG;
- code->param = (unsigned int)param;
- /*
- * The tracepoint probe will probe a stub function, and the
- * first parameter of the stub is a dummy and should be ignored.
- */
- if (ctx->flags & TPARG_FL_TPOINT)
- code->param++;
- } else if (tparg_is_function_return(ctx->flags)) {
- /* function entry argument access from return probe */
- ret = __store_entry_arg(ctx->tp, param);
- if (ret < 0) /* This error should be an internal error */
- return ret;
+ len = str_has_prefix(arg, "stack");
+ if (len)
+ return parse_probe_var_stack(arg, len, code, ctx);
- code->op = FETCH_OP_EDATA;
- code->offset = ret;
- } else {
- err = TP_ERR_NOFENTRY_ARGS;
- goto inval;
- }
- return 0;
- }
-#endif
+ /* $current returns the address of the current task_struct. */
+ if (str_has_prefix(arg, "current"))
+ return parse_probe_var_current(orig_arg, arg, pcode, end, ctx);
+
+ len = str_has_prefix(arg, "arg");
+ if (len)
+ return parse_probe_var_arg(arg, len, code, ctx);
inval:
- __trace_probe_log_err(ctx->offset, err);
+ trace_probe_log_err(ctx->offset, BAD_VAR);
return -EINVAL;
}
@@ -1186,180 +1456,353 @@ static int __parse_imm_string(char *str, char **pbuf, int offs)
return 0;
}
-/* Recursive argument parser */
-static int
-parse_probe_arg(char *arg, const struct fetch_type *type,
- struct fetch_insn **pcode, struct fetch_insn *end,
- struct traceprobe_parse_context *ctx)
+static int parse_probe_arg_register(char *arg, struct fetch_insn *code,
+ struct traceprobe_parse_context *ctx)
+{
+ int ret;
+
+ if (ctx->flags & (TPARG_FL_TEVENT | TPARG_FL_FPROBE)) {
+ /* eprobe and fprobe do not handle registers */
+ trace_probe_log_err(ctx->offset, BAD_VAR);
+ return -EINVAL;
+ }
+ ret = regs_query_register_offset(arg + 1);
+ if (ret >= 0) {
+ code->op = FETCH_OP_REG;
+ code->param = (unsigned int)ret;
+ return 0;
+ }
+ trace_probe_log_err(ctx->offset, BAD_REG_NAME);
+ return -EINVAL;
+}
+
+static int parse_probe_arg_mem_symbol(char *arg, struct fetch_insn **pcode,
+ struct fetch_insn *end,
+ struct traceprobe_parse_context *ctx)
{
struct fetch_insn *code = *pcode;
unsigned long param;
+ long offset = 0;
+ int ret;
+
+ if (isdigit(arg[1])) {
+ ret = kstrtoul(arg + 1, 0, &param);
+ if (ret) {
+ trace_probe_log_err(ctx->offset, BAD_MEM_ADDR);
+ return ret;
+ }
+ /* load address */
+ code->op = FETCH_OP_IMM;
+ code->immediate = param;
+ } else if (arg[1] == '+') {
+ /* Kernel probes do not support file offsets */
+ if (ctx->flags & TPARG_FL_KERNEL) {
+ trace_probe_log_err(ctx->offset, FILE_ON_KPROBE);
+ return -EINVAL;
+ }
+ ret = kstrtol(arg + 2, 0, &offset);
+ if (ret) {
+ trace_probe_log_err(ctx->offset, BAD_FILE_OFFS);
+ return ret;
+ }
+
+ code->op = FETCH_OP_FOFFS;
+ code->immediate = (unsigned long)offset;
+ offset = 0;
+ } else {
+ /* uprobes don't support symbols */
+ if (!(ctx->flags & TPARG_FL_KERNEL)) {
+ trace_probe_log_err(ctx->offset, SYM_ON_UPROBE);
+ return -EINVAL;
+ }
+ /* Preserve symbol for updating */
+ code->op = FETCH_NOP_SYMBOL;
+ code->data = kstrdup(arg + 1, GFP_KERNEL);
+ if (!code->data)
+ return -ENOMEM;
+ if (++code == end) {
+ trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
+ return -EINVAL;
+ }
+ code->op = FETCH_OP_IMM;
+ code->immediate = 0;
+ }
+ /* These are fetching from memory */
+ if (++code == end) {
+ trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
+ return -EINVAL;
+ }
+ *pcode = code;
+ code->op = FETCH_OP_DEREF;
+ code->offset = offset;
+ return 0;
+}
+
+static int parse_probe_arg_deref(char *arg, struct traceprobe_parse_context *ctx)
+{
int deref = FETCH_OP_DEREF;
long offset = 0;
char *tmp;
- int ret = 0;
+ int ret;
- switch (arg[0]) {
- case '$':
- ret = parse_probe_vars(arg, type, pcode, end, ctx);
- break;
+ if (arg[1] == 'u') {
+ deref = FETCH_OP_UDEREF;
+ arg[1] = arg[0];
+ arg++;
+ }
+ if (arg[0] == '+')
+ arg++; /* Skip '+', because kstrtol() rejects it. */
+ tmp = strchr(arg, '(');
+ if (!tmp) {
+ trace_probe_log_err(ctx->offset, DEREF_NEED_BRACE);
+ return -EINVAL;
+ }
+ *tmp = '\0';
+ ret = kstrtol(arg, 0, &offset);
+ if (ret) {
+ trace_probe_log_err(ctx->offset, BAD_DEREF_OFFS);
+ return ret;
+ }
+ ctx->offset += (tmp + 1 - arg) + (arg[0] != '-' ? 1 : 0);
+ arg = tmp + 1;
- case '%': /* named register */
- if (ctx->flags & (TPARG_FL_TEVENT | TPARG_FL_FPROBE)) {
- /* eprobe and fprobe do not handle registers */
- trace_probe_log_err(ctx->offset, BAD_VAR);
- break;
+ tmp = strrchr(arg, ')');
+ if (!tmp) {
+ trace_probe_log_err(ctx->offset + strlen(arg),
+ DEREF_OPEN_BRACE);
+ return -EINVAL;
+ }
+ *tmp = '\0';
+
+ ctx->stack[ctx->depth].type = STATE_DEREF;
+ ctx->stack[ctx->depth].deref.deref = deref;
+ ctx->stack[ctx->depth].deref.offset = offset;
+ ctx->stack[ctx->depth].deref.cur_offs = ctx->offset;
+ ctx->stack[ctx->depth].deref.inner_arg = arg;
+ ctx->stack[ctx->depth].deref.is_cpu_read = false;
+ return 0;
+}
+
+static int parse_probe_arg_imm(char *arg, struct fetch_insn *code,
+ struct traceprobe_parse_context *ctx)
+{
+ char *tmp;
+ int ret;
+
+ if (arg[1] == '"') { /* Immediate string */
+ ret = __parse_imm_string(arg + 2, &tmp, ctx->offset + 2);
+ if (ret)
+ return ret;
+ code->op = FETCH_OP_IMMSTR;
+ code->data = tmp;
+ } else {
+ ret = str_to_immediate(arg + 1, &code->immediate);
+ if (ret) {
+ trace_probe_log_err(ctx->offset + 1, BAD_IMM);
+ return ret;
}
- ret = regs_query_register_offset(arg + 1);
- if (ret >= 0) {
- code->op = FETCH_OP_REG;
- code->param = (unsigned int)ret;
- ret = 0;
- } else
- trace_probe_log_err(ctx->offset, BAD_REG_NAME);
- break;
+ code->op = FETCH_OP_IMM;
+ }
+ return 0;
+}
- case '@': /* memory, file-offset or symbol */
- if (isdigit(arg[1])) {
- ret = kstrtoul(arg + 1, 0, &param);
- if (ret) {
- trace_probe_log_err(ctx->offset, BAD_MEM_ADDR);
- break;
- }
- /* load address */
- code->op = FETCH_OP_IMM;
- code->immediate = param;
- } else if (arg[1] == '+') {
- /* kprobes don't support file offsets */
- if (ctx->flags & TPARG_FL_KERNEL) {
- trace_probe_log_err(ctx->offset, FILE_ON_KPROBE);
- return -EINVAL;
- }
- ret = kstrtol(arg + 2, 0, &offset);
- if (ret) {
- trace_probe_log_err(ctx->offset, BAD_FILE_OFFS);
- break;
- }
+static int parse_probe_arg_default(char *arg, struct fetch_insn **pcode,
+ struct fetch_insn *end,
+ struct traceprobe_parse_context *ctx)
+{
+ int ret;
- code->op = FETCH_OP_FOFFS;
- code->immediate = (unsigned long)offset; // imm64?
- offset = 0;
- } else {
- /* uprobes don't support symbols */
- if (!(ctx->flags & TPARG_FL_KERNEL)) {
- trace_probe_log_err(ctx->offset, SYM_ON_UPROBE);
+ if (isalpha(arg[0]) || arg[0] == '_') {
+ /* BTF variable or event field */
+ if (ctx->flags & TPARG_FL_TEVENT) {
+ ret = parse_trace_event(arg, *pcode, ctx);
+ if (ret < 0) {
+ trace_probe_log_err(ctx->offset, NO_EVENT_FIELD);
return -EINVAL;
}
- /* Preserve symbol for updating */
- code->op = FETCH_NOP_SYMBOL;
- code->data = kstrdup(arg + 1, GFP_KERNEL);
- if (!code->data)
- return -ENOMEM;
- if (++code == end) {
- trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
- return -EINVAL;
- }
- code->op = FETCH_OP_IMM;
- code->immediate = 0;
+ return 0;
}
- /* These are fetching from memory */
- if (++code == end) {
- trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
+ if (!tparg_is_function_entry(ctx->flags) &&
+ !tparg_is_function_return(ctx->flags)) {
+ trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
return -EINVAL;
}
- *pcode = code;
- code->op = FETCH_OP_DEREF;
- code->offset = offset;
- break;
+ return parse_btf_arg(arg, pcode, end, ctx);
+ }
- case '+': /* deref memory */
- case '-':
- if (arg[1] == 'u') {
- deref = FETCH_OP_UDEREF;
- arg[1] = arg[0];
- arg++;
- }
- if (arg[0] == '+')
- arg++; /* Skip '+', because kstrtol() rejects it. */
- tmp = strchr(arg, '(');
- if (!tmp) {
- trace_probe_log_err(ctx->offset, DEREF_NEED_BRACE);
- return -EINVAL;
- }
- *tmp = '\0';
- ret = kstrtol(arg, 0, &offset);
- if (ret) {
- trace_probe_log_err(ctx->offset, BAD_DEREF_OFFS);
+ return 0;
+}
+
+static int parse_probe_arg_nested(char **parg, struct traceprobe_parse_context *ctx)
+{
+ char *arg = *parg;
+ int ret;
+
+ while (true) {
+ /* Determine if this is a nested argument */
+ if (arg[0] != '+' && arg[0] != '-' && arg[0] != '(' &&
+ !str_has_prefix(arg, THIS_CPU_PTR_PREFIX) &&
+ !str_has_prefix(arg, THIS_CPU_READ_PREFIX))
break;
+
+ /* If nested, check the maximum depth limit */
+ if (ctx->depth >= TRACEPROBE_MAX_NESTED_LEVEL) {
+ trace_probe_log_err(ctx->offset, TOO_MANY_NESTED);
+ return -E2BIG;
}
- ctx->offset += (tmp + 1 - arg) + (arg[0] != '-' ? 1 : 0);
- arg = tmp + 1;
- tmp = strrchr(arg, ')');
- if (!tmp) {
- trace_probe_log_err(ctx->offset + strlen(arg),
- DEREF_OPEN_BRACE);
- return -EINVAL;
- } else {
- const struct fetch_type *t2 = find_fetch_type(NULL, ctx->flags);
- int cur_offs = ctx->offset;
- *tmp = '\0';
- ret = parse_probe_arg(arg, t2, &code, end, ctx);
+ /* Perform the actual parsing subroutine calls */
+ switch (arg[0]) {
+ case '+':
+ case '-':
+ ret = parse_probe_arg_deref(arg, ctx);
if (ret)
- break;
- ctx->offset = cur_offs;
- if (code->op == FETCH_OP_COMM ||
- code->op == FETCH_OP_DATA) {
- trace_probe_log_err(ctx->offset, COMM_CANT_DEREF);
- return -EINVAL;
- }
- if (++code == end) {
- trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
- return -EINVAL;
- }
- *pcode = code;
-
- code->op = deref;
- code->offset = offset;
- /* Reset the last type if used */
- ctx->last_type = NULL;
- }
- break;
- case '\\': /* Immediate value */
- if (arg[1] == '"') { /* Immediate string */
- ret = __parse_imm_string(arg + 2, &tmp, ctx->offset + 2);
+ return ret;
+ arg = ctx->stack[ctx->depth].deref.inner_arg;
+ break;
+ case '(':
+ ret = handle_typecast(arg, ctx);
if (ret)
- break;
- code->op = FETCH_OP_DATA;
- code->data = tmp;
- } else {
- ret = str_to_immediate(arg + 1, &code->immediate);
+ return ret;
+ arg = ctx->stack[ctx->depth].typecast.inner_arg;
+ break;
+ default:
+ ret = parse_this_cpu(arg, ctx);
if (ret)
- trace_probe_log_err(ctx->offset + 1, BAD_IMM);
- else
- code->op = FETCH_OP_IMM;
+ return ret;
+ arg = ctx->stack[ctx->depth].deref.inner_arg;
+ break;
}
+ ctx->depth++;
+ }
+
+ *parg = arg;
+ return 0;
+}
+
+static int parse_probe_arg_leaf(char *arg, const struct fetch_type *type,
+ struct fetch_insn **pcode, struct fetch_insn *end,
+ struct traceprobe_parse_context *ctx)
+{
+ struct fetch_insn *code = *pcode;
+ int ret;
+
+ switch (arg[0]) {
+ case '$':
+ ret = parse_probe_vars(arg, type, pcode, end, ctx);
+ break;
+ case '%': /* named register */
+ ret = parse_probe_arg_register(arg, code, ctx);
+ break;
+ case '@': /* memory, file-offset or symbol */
+ ret = parse_probe_arg_mem_symbol(arg, pcode, end, ctx);
break;
- case '(':
- ret = handle_typecast(arg, pcode, end, ctx);
+ case '\\': /* Immediate value */
+ ret = parse_probe_arg_imm(arg, code, ctx);
break;
default:
- if (isalpha(arg[0]) || arg[0] == '_') { /* BTF variable */
- if (!tparg_is_function_entry(ctx->flags) &&
- !tparg_is_function_return(ctx->flags)) {
- trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
- return -EINVAL;
- }
- ret = parse_btf_arg(arg, pcode, end, ctx);
- break;
- }
+ ret = parse_probe_arg_default(arg, pcode, end, ctx);
+ break;
}
- if (!ret && code->op == FETCH_OP_NOP) {
+
+ if (ret)
+ return ret;
+
+ if (code->op == FETCH_OP_NOP) {
/* Parsed, but do not find fetch method */
trace_probe_log_err(ctx->offset, BAD_FETCH_ARG);
- ret = -EINVAL;
+ return -EINVAL;
}
- return ret;
+
+ return 0;
+}
+
+static int unwind_parse_states(struct fetch_insn **pcode, struct fetch_insn *end,
+ struct traceprobe_parse_context *ctx)
+{
+ struct parse_state *state;
+ struct fetch_insn *code;
+ int ret;
+
+ while (ctx->depth > 0) {
+ ctx->depth--;
+ state = &ctx->stack[ctx->depth];
+
+ if (state->type == STATE_DEREF) {
+ code = *pcode;
+ ctx->offset = state->deref.cur_offs;
+ if (code->op == FETCH_OP_COMM || code->op == FETCH_OP_IMMSTR) {
+ trace_probe_log_err(ctx->offset, COMM_CANT_DEREF);
+ return -EINVAL;
+ }
+
+ if (!(state->deref.deref == FETCH_OP_CPU_PTR &&
+ *state->deref.inner_arg == '@')) {
+ code++;
+ if (code == end) {
+ trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
+ return -EINVAL;
+ }
+ }
+ *pcode = code;
+
+ code->op = state->deref.deref;
+ code->offset = state->deref.offset;
+ ctx->last_type = NULL;
+
+ if (state->deref.is_cpu_read) {
+ code = *pcode;
+ code++;
+ if (code == end) {
+ trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
+ return -EINVAL;
+ }
+ code->op = FETCH_OP_DEREF;
+ code->offset = 0;
+ *pcode = code;
+ }
+ } else if (state->type == STATE_TYPECAST) {
+ clear_struct_btf(ctx);
+
+ /* resolve the typecast struct name */
+ ctx->offset = state->typecast.orig_offset + 1; /* for the '(' */
+ ret = parse_btf_casttype(state->typecast.casttype, ctx);
+ if (ret < 0)
+ return ret;
+
+ ctx->offset = state->typecast.orig_offset +
+ state->typecast.field_offset_diff;
+ ret = parse_btf_field(state->typecast.fieldname,
+ ctx->last_struct, pcode,
+ end, ctx);
+ ctx->prefix_byteoffs = 0;
+ if (ret < 0)
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+/* Loop-based (non-recursive) argument parser */
+static int
+parse_probe_arg(char *arg, const struct fetch_type *type,
+ struct fetch_insn **pcode, struct fetch_insn *end,
+ struct traceprobe_parse_context *ctx)
+{
+ int ret;
+
+ ctx->depth = 0;
+
+ ret = parse_probe_arg_nested(&arg, ctx);
+ if (ret)
+ return ret;
+
+ ret = parse_probe_arg_leaf(arg, type, pcode, end, ctx);
+ if (ret)
+ return ret;
+
+ return unwind_parse_states(pcode, end, ctx);
}
/* Bitfield type needs to be parsed into a fetch function */
@@ -1485,7 +1928,7 @@ static int finalize_fetch_insn(struct fetch_insn *code,
} else {
if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_UDEREF &&
code->op != FETCH_OP_IMM && code->op != FETCH_OP_COMM &&
- code->op != FETCH_OP_DATA && code->op != FETCH_OP_TP_ARG) {
+ code->op != FETCH_OP_IMMSTR && code->op != FETCH_OP_TP_ARG) {
trace_probe_log_err(ctx->offset + type_offset,
BAD_STRING);
return -EINVAL;
@@ -1494,7 +1937,7 @@ static int finalize_fetch_insn(struct fetch_insn *code,
if (!strcmp(parg->type->name, "symstr") ||
(code->op == FETCH_OP_IMM || code->op == FETCH_OP_COMM ||
- code->op == FETCH_OP_DATA) || code->op == FETCH_OP_TP_ARG ||
+ code->op == FETCH_OP_IMMSTR) || code->op == FETCH_OP_TP_ARG ||
parg->count) {
/*
* IMM, DATA and COMM is pointing actual address, those
@@ -1620,7 +2063,6 @@ static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size,
ctx);
if (ret < 0)
goto fail;
-
/* Update storing type if BTF is available */
if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
ctx->last_type) {
@@ -1670,7 +2112,7 @@ fail:
if (ret < 0) {
for (code = tmp; code < tmp + FETCH_INSN_MAX; code++)
if (code->op == FETCH_NOP_SYMBOL ||
- code->op == FETCH_OP_DATA)
+ code->op == FETCH_OP_IMMSTR)
kfree(code->data);
}
kfree(tmp);
@@ -1769,7 +2211,7 @@ void traceprobe_free_probe_arg(struct probe_arg *arg)
while (code && code->op != FETCH_OP_END) {
if (code->op == FETCH_NOP_SYMBOL ||
- code->op == FETCH_OP_DATA)
+ code->op == FETCH_OP_IMMSTR)
kfree(code->data);
code++;
}
@@ -2329,10 +2771,9 @@ int trace_probe_compare_arg_type(struct trace_probe *a, struct trace_probe *b)
return b->nr_args + 1;
for (i = 0; i < a->nr_args; i++) {
- if ((b->nr_args <= i) ||
- ((a->args[i].type != b->args[i].type) ||
- (a->args[i].count != b->args[i].count) ||
- strcmp(a->args[i].name, b->args[i].name)))
+ if ((a->args[i].type != b->args[i].type) ||
+ (a->args[i].count != b->args[i].count) ||
+ strcmp(a->args[i].name, b->args[i].name))
return i + 1;
}
@@ -2401,3 +2842,99 @@ int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_a
}
return 0;
}
+
+#ifdef CONFIG_PROBE_EVENTS_DUMP_FETCHARG
+
+struct fetch_op_decode {
+ const char *name;
+ void (*decode)(struct seq_file *m, struct fetch_insn *insn);
+};
+
+static const struct fetch_op_decode fetch_op_decode[];
+
+static void fetcharg_decode_none(struct seq_file *m, struct fetch_insn *insn)
+{
+ seq_puts(m, fetch_op_decode[insn->op].name);
+}
+
+static void fetcharg_decode_param(struct seq_file *m, struct fetch_insn *insn)
+{
+ seq_printf(m, "%s(%u)", fetch_op_decode[insn->op].name, insn->param);
+}
+
+static void fetcharg_decode_imm(struct seq_file *m, struct fetch_insn *insn)
+{
+ seq_printf(m, "%s(0x%lx)", fetch_op_decode[insn->op].name, insn->immediate);
+}
+
+static void fetcharg_decode_string(struct seq_file *m, struct fetch_insn *insn)
+{
+ seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, (char *)insn->data);
+}
+
+static void fetcharg_decode_symbol(struct seq_file *m, struct fetch_insn *insn)
+{
+ seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, (char *)insn->data);
+}
+
+static void fetcharg_decode_offset(struct seq_file *m, struct fetch_insn *insn)
+{
+ seq_printf(m, "%s(offset=%d)", fetch_op_decode[insn->op].name, insn->offset);
+}
+
+static void fetcharg_decode_store(struct seq_file *m, struct fetch_insn *insn)
+{
+ if (insn->op == FETCH_OP_ST_RAW)
+ seq_printf(m, "%s(size=%u)", fetch_op_decode[insn->op].name, insn->size);
+ else
+ seq_printf(m, "%s(offset=%d,size=%u)", fetch_op_decode[insn->op].name,
+ insn->offset, insn->size);
+}
+
+static void fetcharg_decode_bf(struct seq_file *m, struct fetch_insn *insn)
+{
+ seq_printf(m, "%s(basesize=%u,lshift=%u,rshift=%u)",
+ fetch_op_decode[insn->op].name, insn->basesize, insn->lshift, insn->rshift);
+}
+
+static void fetcharg_decode_tp_arg(struct seq_file *m, struct fetch_insn *insn)
+{
+ struct ftrace_event_field *field = insn->data;
+
+ seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, field->name);
+}
+
+#define FETCH_OP(opname, decode_fn) \
+ [FETCH_OP_##opname] = { .name = #opname, .decode = fetcharg_decode_##decode_fn }
+
+static const struct fetch_op_decode fetch_op_decode[] = FETCH_OP_LIST;
+#undef FETCH_OP
+
+static void trace_probe_dump_arg(struct seq_file *m, struct probe_arg *parg)
+{
+ int i;
+
+ seq_printf(m, "# %s: ", parg->name);
+ for (i = 0; i < FETCH_INSN_MAX; i++) {
+ struct fetch_insn *insn = parg->code + i;
+
+ if (insn->op >= ARRAY_SIZE(fetch_op_decode) || !fetch_op_decode[insn->op].decode)
+ seq_printf(m, "unknown(%d)", insn->op);
+ else
+ fetch_op_decode[insn->op].decode(m, insn);
+
+ if (insn->op == FETCH_OP_END)
+ break;
+ seq_puts(m, " -> ");
+ }
+ seq_putc(m, '\n');
+}
+
+void trace_probe_dump_args(struct seq_file *m, struct trace_probe *tp)
+{
+ int i;
+
+ for (i = 0; i < tp->nr_args; i++)
+ trace_probe_dump_arg(m, &tp->args[i]);
+}
+#endif /* CONFIG_PROBE_EVENTS_DUMP_FETCHARG */
diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h
index 0f09f7aaf93f..fba1af092a9b 100644
--- a/kernel/trace/trace_probe.h
+++ b/kernel/trace/trace_probe.h
@@ -32,7 +32,7 @@
#include "trace_output.h"
#define MAX_TRACE_ARGS 128
-#define MAX_ARGSTR_LEN 63
+#define MAX_ARGSTR_LEN 255
#define MAX_ARRAY_LEN 64
#define MAX_ARG_NAME_LEN 32
#define MAX_BTF_ARGS_LEN 128
@@ -83,38 +83,48 @@ static nokprobe_inline u32 update_data_loc(u32 loc, int consumed)
/* Printing function type */
typedef int (*print_type_func_t)(struct trace_seq *, void *, void *);
-enum fetch_op {
- FETCH_OP_NOP = 0,
- // Stage 1 (load) ops
- FETCH_OP_REG, /* Register : .param = offset */
- FETCH_OP_STACK, /* Stack : .param = index */
- FETCH_OP_STACKP, /* Stack pointer */
- FETCH_OP_RETVAL, /* Return value */
- FETCH_OP_IMM, /* Immediate : .immediate */
- FETCH_OP_COMM, /* Current comm */
- FETCH_OP_ARG, /* Function argument : .param */
- FETCH_OP_FOFFS, /* File offset: .immediate */
- FETCH_OP_DATA, /* Allocated data: .data */
- FETCH_OP_EDATA, /* Entry data: .offset */
- // Stage 2 (dereference) op
- FETCH_OP_DEREF, /* Dereference: .offset */
- FETCH_OP_UDEREF, /* User-space Dereference: .offset */
- // Stage 3 (store) ops
- FETCH_OP_ST_RAW, /* Raw: .size */
- FETCH_OP_ST_MEM, /* Mem: .offset, .size */
- FETCH_OP_ST_UMEM, /* Mem: .offset, .size */
- FETCH_OP_ST_STRING, /* String: .offset, .size */
- FETCH_OP_ST_USTRING, /* User String: .offset, .size */
- FETCH_OP_ST_SYMSTR, /* Kernel Symbol String: .offset, .size */
- FETCH_OP_ST_EDATA, /* Store Entry Data: .offset */
- // Stage 4 (modify) op
- FETCH_OP_MOD_BF, /* Bitfield: .basesize, .lshift, .rshift */
- // Stage 5 (loop) op
- FETCH_OP_LP_ARRAY, /* Array: .param = loop count */
- FETCH_OP_TP_ARG, /* Trace Point argument */
- FETCH_OP_END,
- FETCH_NOP_SYMBOL, /* Unresolved Symbol holder */
-};
+#define FETCH_OP_LIST { \
+ /* Stage 1 (load) ops */ \
+ FETCH_OP(NOP, none), /* NOP */ \
+ FETCH_OP(REG, param), /* Register: .param = offset */ \
+ FETCH_OP(STACK, param), /* Stack: .param = index */ \
+ FETCH_OP(STACKP, none), /* Stack pointer */ \
+ FETCH_OP(RETVAL, none), /* Return value */ \
+ FETCH_OP(IMM, imm), /* Immediate: .immediate */ \
+ FETCH_OP(COMM, none), /* Current comm */ \
+ FETCH_OP(CURRENT, none), /* Current task_struct address */\
+ FETCH_OP(ARG, param), /* Argument: .param = index */ \
+ FETCH_OP(FOFFS, imm), /* File offset: .immediate */ \
+ FETCH_OP(IMMSTR, string), /* Allocated string: .data */ \
+ FETCH_OP(EDATA, offset), /* Entry data: .offset */ \
+ FETCH_OP(TP_ARG, tp_arg), /* Tracepoint argument: .data */\
+ /* Stage 2 (dereference) ops */ \
+ FETCH_OP(DEREF, offset), /* Dereference: .offset */ \
+ FETCH_OP(UDEREF, offset), /* User-space dereference: .offset */\
+ FETCH_OP(CPU_PTR, none), /* Per-CPU pointer: .offset */ \
+ /* Stage 3 (store) ops */ \
+ FETCH_OP(ST_RAW, store), /* Raw value: .size */ \
+ FETCH_OP(ST_MEM, store), /* Memory: .offset, .size */ \
+ FETCH_OP(ST_UMEM, store), /* User memory: .offset, .size */\
+ FETCH_OP(ST_STRING, store), /* String: .offset, .size */ \
+ FETCH_OP(ST_USTRING, store), /* User string: .offset, .size */\
+ FETCH_OP(ST_SYMSTR, store), /* Symbol name: .offset, .size */\
+ FETCH_OP(ST_EDATA, offset), /* Entry data: .offset */ \
+ /* Stage 4 (modify) op */ \
+ FETCH_OP(MOD_BF, bf), /* Bitfield: .basesize, .lshift, .rshift*/\
+ /* Stage 5 (loop) op */ \
+ FETCH_OP(LP_ARRAY, param), /* Loop array: .param = count */\
+ /* End */ \
+ FETCH_OP(END, none), \
+ /* Unresolved Symbol holder */ \
+ FETCH_OP(NOP_SYMBOL, symbol), /* Non loaded symbol: .data = symbol name */\
+}
+
+#define FETCH_OP(opname, decode_fn) FETCH_OP_##opname
+enum fetch_op FETCH_OP_LIST;
+#undef FETCH_OP
+
+#define FETCH_NOP_SYMBOL FETCH_OP_NOP_SYMBOL
struct fetch_insn {
enum fetch_op op;
@@ -209,7 +219,6 @@ DECLARE_BASIC_PRINT_TYPE_FUNC(symbol);
_ASSIGN_FETCH_TYPE(#ptype, ptype, ftype, sizeof(ftype), sign, atype)
#define ASSIGN_FETCH_TYPE_END {}
-#define MAX_ARRAY_LEN 64
#ifdef CONFIG_KPROBE_EVENTS
bool trace_kprobe_on_func_entry(struct trace_event_call *call);
@@ -350,7 +359,7 @@ static inline int trace_probe_unregister_event_call(struct trace_probe *tp)
static inline bool trace_probe_has_single_file(struct trace_probe *tp)
{
- return !!list_is_singular(&tp->event->files);
+ return list_is_singular(&tp->event->files);
}
int trace_probe_init(struct trace_probe *tp, const char *event,
@@ -370,6 +379,13 @@ bool trace_probe_match_command_args(struct trace_probe *tp,
int trace_probe_create(const char *raw_command, int (*createfn)(int, const char **));
int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
u8 *data, void *field);
+#ifdef CONFIG_PROBE_EVENTS_DUMP_FETCHARG
+void trace_probe_dump_args(struct seq_file *m, struct trace_probe *tp);
+#else
+static inline void trace_probe_dump_args(struct seq_file *m, struct trace_probe *tp)
+{
+}
+#endif
#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
int traceprobe_get_entry_data_size(struct trace_probe *tp);
@@ -414,6 +430,39 @@ static inline bool tparg_is_function_return(unsigned int flags)
return (flags & TPARG_FL_LOC_MASK) == (TPARG_FL_KERNEL | TPARG_FL_RETURN);
}
+static inline bool tparg_is_event_probe(unsigned int flags)
+{
+ return !!(flags & TPARG_FL_TEVENT);
+}
+
+/* Each typecast consumes nested level. So the max number of typecast is 8. */
+#define TRACEPROBE_MAX_NESTED_LEVEL 8
+
+enum parse_state_type {
+ STATE_DEREF,
+ STATE_TYPECAST,
+};
+
+struct parse_state {
+ int type;
+ union {
+ struct {
+ int deref;
+ long offset;
+ int cur_offs;
+ char *inner_arg;
+ bool is_cpu_read;
+ } deref;
+ struct {
+ char *casttype;
+ char *fieldname;
+ int orig_offset;
+ int field_offset_diff;
+ char *inner_arg;
+ } typecast;
+ };
+};
+
struct traceprobe_parse_context {
struct trace_event_call *event;
/* BTF related parameters */
@@ -430,8 +479,12 @@ struct traceprobe_parse_context {
struct trace_probe *tp;
unsigned int flags;
int offset;
+ int prefix_byteoffs; /* The byte offset of the prefix field of typecast */
+ struct parse_state stack[TRACEPROBE_MAX_NESTED_LEVEL + 1];
+ int depth;
};
+
extern int traceprobe_parse_probe_arg(struct trace_probe *tp, int i,
const char *argv,
struct traceprobe_parse_context *ctx);
@@ -485,88 +538,95 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call,
#undef ERRORS
#define ERRORS \
- C(FILE_NOT_FOUND, "Failed to find the given file"), \
- C(NO_REGULAR_FILE, "Not a regular file"), \
- C(BAD_REFCNT, "Invalid reference counter offset"), \
- C(REFCNT_OPEN_BRACE, "Reference counter brace is not closed"), \
- C(BAD_REFCNT_SUFFIX, "Reference counter has wrong suffix"), \
- C(BAD_UPROBE_OFFS, "Invalid uprobe offset"), \
- C(BAD_MAXACT_TYPE, "Maxactive is only for function exit"), \
- C(BAD_MAXACT, "Invalid maxactive number"), \
- C(MAXACT_TOO_BIG, "Maxactive is too big"), \
- C(BAD_PROBE_ADDR, "Invalid probed address or symbol"), \
- C(NON_UNIQ_SYMBOL, "The symbol is not unique"), \
- C(BAD_RETPROBE, "Retprobe address must be an function entry"), \
- C(NO_TRACEPOINT, "Tracepoint is not found"), \
- C(BAD_TP_NAME, "Invalid character in tracepoint name"),\
- C(BAD_ADDR_SUFFIX, "Invalid probed address suffix"), \
- C(NO_GROUP_NAME, "Group name is not specified"), \
- C(GROUP_TOO_LONG, "Group name is too long"), \
- C(BAD_GROUP_NAME, "Group name must follow the same rules as C identifiers"), \
- C(NO_EVENT_NAME, "Event name is not specified"), \
- C(EVENT_TOO_LONG, "Event name is too long"), \
- C(BAD_EVENT_NAME, "Event name must follow the same rules as C identifiers"), \
- C(EVENT_EXIST, "Given group/event name is already used by another event"), \
- C(RETVAL_ON_PROBE, "$retval is not available on probe"), \
- C(NO_RETVAL, "This function returns 'void' type"), \
- C(BAD_STACK_NUM, "Invalid stack number"), \
+ C(ARGIDX_2BIG, "$argN index is too big"), \
+ C(ARGS_2LONG, "$arg* failed because the argument list is too long"), \
+ C(ARG_NAME_TOO_LONG, "Argument name is too long"), \
+ C(ARG_TOO_LONG, "Argument expression is too long"), \
+ C(ARRAY_NO_CLOSE, "Array is not closed"), \
+ C(ARRAY_TOO_BIG, "Array number is too big"), \
+ C(BAD_ADDR_SUFFIX, "Invalid probed address suffix"), \
+ C(BAD_ARG_NAME, "Argument name must follow the same rules as C identifiers"), \
C(BAD_ARG_NUM, "Invalid argument number"), \
- C(BAD_VAR, "Invalid $-variable specified"), \
- C(BAD_REG_NAME, "Invalid register name"), \
- C(BAD_MEM_ADDR, "Invalid memory address"), \
- C(BAD_IMM, "Invalid immediate value"), \
- C(IMMSTR_NO_CLOSE, "String is not closed with '\"'"), \
- C(FILE_ON_KPROBE, "File offset is not available with kprobe"), \
- C(BAD_FILE_OFFS, "Invalid file offset value"), \
- C(SYM_ON_UPROBE, "Symbol is not available with uprobe"), \
- C(TOO_MANY_OPS, "Dereference is too much nested"), \
- C(DEREF_NEED_BRACE, "Dereference needs a brace"), \
+ C(BAD_ARRAY_NUM, "Invalid array size"), \
+ C(BAD_ARRAY_SUFFIX, "Array has wrong suffix"), \
+ C(BAD_ATTACH_ARG, "Attached event does not have this field"), \
+ C(BAD_ATTACH_EVENT, "Attached event does not exist"), \
+ C(BAD_BITFIELD, "Invalid bitfield"), \
+ C(BAD_BTF_TID, "Failed to get BTF type info."), \
C(BAD_DEREF_OFFS, "Invalid dereference offset"), \
- C(DEREF_OPEN_BRACE, "Dereference brace is not closed"), \
- C(COMM_CANT_DEREF, "$comm can not be dereferenced"), \
+ C(BAD_EVENT_NAME, "Event name must follow the same rules as C identifiers"), \
C(BAD_FETCH_ARG, "Invalid fetch argument"), \
- C(ARRAY_NO_CLOSE, "Array is not closed"), \
- C(BAD_ARRAY_SUFFIX, "Array has wrong suffix"), \
- C(BAD_ARRAY_NUM, "Invalid array size"), \
- C(ARRAY_TOO_BIG, "Array number is too big"), \
- C(BAD_TYPE, "Unknown type is specified"), \
- C(BAD_STRING, "String accepts only memory argument"), \
+ C(BAD_FILE_OFFS, "Invalid file offset value"), \
+ C(BAD_GROUP_NAME, "Group name must follow the same rules as C identifiers"), \
+ C(BAD_HYPHEN, "Failed to parse single hyphen. Forgot '>'?"), \
+ C(BAD_IMM, "Invalid immediate value"), \
+ C(BAD_INSN_BNDRY, "Probe point is not an instruction boundary"), \
+ C(BAD_MAXACT, "Invalid maxactive number"), \
+ C(BAD_MAXACT_TYPE, "Maxactive is only for function exit"), \
+ C(BAD_MEM_ADDR, "Invalid memory address"), \
+ C(BAD_PROBE_ADDR, "Invalid probed address or symbol"), \
+ C(BAD_REFCNT, "Invalid reference counter offset"), \
+ C(BAD_REFCNT_SUFFIX, "Reference counter has wrong suffix"), \
+ C(BAD_REG_NAME, "Invalid register name"), \
+ C(BAD_RETPROBE, "Retprobe address must be an function entry"), \
+ C(BAD_STACK_NUM, "Invalid stack number"), \
+ C(BAD_STRING, "String accepts only memory argument"), \
C(BAD_SYMSTRING, "Symbol String doesn't accept data/userdata"), \
- C(BAD_BITFIELD, "Invalid bitfield"), \
- C(ARG_NAME_TOO_LONG, "Argument name is too long"), \
- C(NO_ARG_NAME, "Argument name is not specified"), \
- C(BAD_ARG_NAME, "Argument name must follow the same rules as C identifiers"), \
- C(USED_ARG_NAME, "This argument name is already used"), \
- C(ARG_TOO_LONG, "Argument expression is too long"), \
+ C(BAD_TP_NAME, "Invalid character in tracepoint name"), \
+ C(BAD_TYPE, "Unknown type is specified"), \
+ C(BAD_TYPE4STR, "This type does not fit for string."), \
+ C(BAD_UPROBE_OFFS, "Invalid uprobe offset"), \
+ C(BAD_VAR, "Invalid $-variable specified"), \
+ C(BAD_VAR_ARGS, "$arg* must be an independent parameter without name etc."), \
+ C(COMM_CANT_DEREF, "$comm can not be dereferenced"), \
+ C(DEREF_NEED_BRACE, "Dereference needs a brace"), \
+ C(DEREF_OPEN_BRACE, "Dereference brace is not closed"), \
+ C(DIFF_ARG_TYPE, "Argument type or name is different from existing probe"), \
+ C(DIFF_PROBE_TYPE, "Probe type is different from existing probe"), \
+ C(DOUBLE_ARGS, "$arg* can be used only once in the parameters"), \
+ C(EVENT_EXIST, "Given group/event name is already used by another event"), \
+ C(EVENT_TOO_BIG, "Event too big (too many fields?)"), \
+ C(EVENT_TOO_LONG, "Event name is too long"), \
+ C(FAIL_REG_PROBE, "Failed to register probe event"), \
+ C(FILE_NOT_FOUND, "Failed to find the given file"), \
+ C(FILE_ON_KPROBE, "File offset is not available for kernel probes"), \
+ C(GROUP_TOO_LONG, "Group name is too long"), \
+ C(IMMSTR_NO_CLOSE, "String is not closed with '\"'"), \
+ C(MAXACT_TOO_BIG, "Maxactive is too big"), \
+ C(NEED_STRING_TYPE, "$comm and immediate-string only accepts string type"), \
+ C(NOFENTRY_ARGS, "$arg* can be used only on function entry or exit"), \
+ C(NON_UNIQ_SYMBOL, "The symbol is not unique"), \
+ C(NOSUP_BTFARG, "BTF is not available or not supported"), \
+ C(NOSUP_DAT_ARG, "Non pointer structure/union argument is not supported."), \
+ C(NOSUP_PERCPU, "Per-cpu variable access is only for kernel probes"), \
C(NO_ARG_BODY, "No argument expression"), \
- C(BAD_INSN_BNDRY, "Probe point is not an instruction boundary"),\
- C(FAIL_REG_PROBE, "Failed to register probe event"),\
- C(DIFF_PROBE_TYPE, "Probe type is different from existing probe"),\
- C(DIFF_ARG_TYPE, "Argument type or name is different from existing probe"),\
- C(SAME_PROBE, "There is already the exact same probe event"),\
- C(NO_EVENT_INFO, "This requires both group and event name to attach"),\
- C(BAD_ATTACH_EVENT, "Attached event does not exist"),\
- C(BAD_ATTACH_ARG, "Attached event does not have this field"),\
+ C(NO_ARG_NAME, "Argument name is not specified"), \
+ C(NO_BTFARG, "This variable is not found at this probe point"), \
+ C(NO_BTF_ENTRY, "No BTF entry for this probe point"), \
+ C(NO_BTF_FIELD, "This field is not found."), \
C(NO_EP_FILTER, "No filter rule after 'if'"), \
- C(NOSUP_BTFARG, "BTF is not available or not supported"), \
- C(NO_BTFARG, "This variable is not found at this probe point"),\
- C(NO_BTF_ENTRY, "No BTF entry for this probe point"), \
- C(BAD_VAR_ARGS, "$arg* must be an independent parameter without name etc."),\
- C(NOFENTRY_ARGS, "$arg* can be used only on function entry or exit"), \
- C(DOUBLE_ARGS, "$arg* can be used only once in the parameters"), \
- C(ARGS_2LONG, "$arg* failed because the argument list is too long"), \
- C(ARGIDX_2BIG, "$argN index is too big"), \
+ C(NO_EVENT_FIELD, "This event field is not found."), \
+ C(NO_EVENT_INFO, "This requires both group and event name to attach"), \
+ C(NO_EVENT_NAME, "Event name is not specified"), \
+ C(NO_GROUP_NAME, "Group name is not specified"), \
C(NO_PTR_STRCT, "This is not a pointer to union/structure."), \
- C(NOSUP_DAT_ARG, "Non pointer structure/union argument is not supported."),\
- C(BAD_HYPHEN, "Failed to parse single hyphen. Forgot '>'?"), \
- C(NO_BTF_FIELD, "This field is not found."), \
- C(BAD_BTF_TID, "Failed to get BTF type info."),\
- C(BAD_TYPE4STR, "This type does not fit for string."),\
- C(NEED_STRING_TYPE, "$comm and immediate-string only accepts string type"),\
- C(TOO_MANY_ARGS, "Too many arguments are specified"), \
+ C(NO_REGULAR_FILE, "Not a regular file"), \
+ C(NO_RETVAL, "This function returns 'void' type"), \
+ C(NO_TRACEPOINT, "Tracepoint is not found"), \
+ C(REFCNT_OPEN_BRACE, "Reference counter brace is not closed"), \
+ C(RETVAL_ON_PROBE, "$retval is not available on probe"), \
+ C(SAME_PROBE, "There is already the exact same probe event"), \
+ C(SYM_ON_UPROBE, "Symbol is not available with uprobe"), \
+ C(TOO_MANY_ARGS, "Too many arguments are specified"), \
C(TOO_MANY_EARGS, "Too many entry arguments specified"), \
- C(EVENT_TOO_BIG, "Event too big (too many fields?)"), \
- C(TYPECAST_NOT_EVENT, "Typecasts are only for eprobe fields"),
+ C(TOO_MANY_NESTED, "Too many nested typecasts/dereferences"), \
+ C(TOO_MANY_OPS, "Dereference is too much nested"), \
+ C(TYPECAST_BAD_ARROW, "Typecast field option does not support -> operator"), \
+ C(TYPECAST_NOT_ALIGNED, "Typecast field option is not byte-aligned"), \
+ C(TYPECAST_NOT_EVENT, "Typecasts are only for eprobe fields"), \
+ C(TYPECAST_REQ_FIELD, "Typecast requires a field access"), \
+ C(TYPECAST_SYM_OFFSET, "@SYM+/-OFFSET with typecast needs parentheses"), \
+ C(USED_ARG_NAME, "This argument name is already used"),
#undef C
#define C(a, b) TP_ERR_##a
diff --git a/kernel/trace/trace_probe_kernel.h b/kernel/trace/trace_probe_kernel.h
index bb723eefd7b7..4eb4acfa09ba 100644
--- a/kernel/trace/trace_probe_kernel.h
+++ b/kernel/trace/trace_probe_kernel.h
@@ -12,7 +12,7 @@
static nokprobe_inline int
fetch_store_strlen_user(unsigned long addr)
{
- const void __user *uaddr = (__force const void __user *)addr;
+ const void __user *uaddr = (__force const void __user *)addr;
return strnlen_user_nofault(uaddr, MAX_STRING_SIZE);
}
diff --git a/kernel/trace/trace_probe_tmpl.h b/kernel/trace/trace_probe_tmpl.h
index f39b37fcdb3b..8db12f758fda 100644
--- a/kernel/trace/trace_probe_tmpl.h
+++ b/kernel/trace/trace_probe_tmpl.h
@@ -109,9 +109,12 @@ process_common_fetch_insn(struct fetch_insn *code, unsigned long *val)
case FETCH_OP_COMM:
*val = (unsigned long)current->comm;
break;
- case FETCH_OP_DATA:
+ case FETCH_OP_IMMSTR:
*val = (unsigned long)code->data;
break;
+ case FETCH_OP_CURRENT:
+ *val = (unsigned long)current;
+ break;
default:
return -EILSEQ;
}
@@ -126,25 +129,35 @@ process_fetch_insn_bottom(struct fetch_insn *code, unsigned long val,
struct fetch_insn *s3 = NULL;
int total = 0, ret = 0, i = 0;
u32 loc = 0;
- unsigned long lval = val;
+ unsigned long lval, llval = val;
stage2:
/* 2nd stage: dereference memory if needed */
do {
- if (code->op == FETCH_OP_DEREF) {
- lval = val;
+ lval = val;
+ switch (code->op) {
+ case FETCH_OP_DEREF:
ret = probe_mem_read(&val, (void *)val + code->offset,
sizeof(val));
- } else if (code->op == FETCH_OP_UDEREF) {
- lval = val;
+ break;
+ case FETCH_OP_UDEREF:
ret = probe_mem_read_user(&val,
(void *)val + code->offset, sizeof(val));
- } else
break;
+ case FETCH_OP_CPU_PTR:
+ val = (unsigned long)this_cpu_ptr((void __percpu *)val);
+ ret = 0;
+ break;
+ default:
+ lval = llval;
+ goto out;
+ }
if (ret)
return ret;
+ llval = lval;
code++;
} while (1);
+out:
s3 = code;
stage3:
diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c
index c274346853d1..861d857adadb 100644
--- a/kernel/trace/trace_uprobe.c
+++ b/kernel/trace/trace_uprobe.c
@@ -281,7 +281,7 @@ static bool trace_uprobe_is_busy(struct dyn_event *ev)
static bool trace_uprobe_match_command_head(struct trace_uprobe *tu,
int argc, const char **argv)
{
- char buf[MAX_ARGSTR_LEN + 1];
+ char buf[64];
int len;
if (!argc)
@@ -765,6 +765,9 @@ static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev)
seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm);
seq_putc(m, '\n');
+
+ trace_probe_dump_args(m, &tu->tp);
+
return 0;
}
generated by cgit v1.2.3 (git 2.0.4) at 2026-10-19 00:01:47 +0000