Skip to content

Commit 5330592

Browse files
mhiramatrostedt
mhiramat
authored and
rostedt
committed
tracing: probeevent: Introduce new argument fetching code
Replace {k,u}probe event argument fetching framework with switch-case based. Currently that is implemented with structures, macros and chain of function-pointers, which is more complicated than necessary and may get a performance penalty by retpoline. This simplify that with an array of "fetch_insn" (opcode and oprands), and make process_fetch_insn() just interprets it. No function pointers are used. Link: http://lkml.kernel.org/r/152465868340.26224.2551120475197839464.stgit@devbox Signed-off-by: Masami Hiramatsu <mhiramat@kernel.org> Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
1 parent 7bfbc63 commit 5330592

File tree

5 files changed

+491
-678
lines changed

5 files changed

+491
-678
lines changed

‎kernel/trace/trace_kprobe.c

Lines changed: 131 additions & 160 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
#include "trace_kprobe_selftest.h"
1616
#include "trace_probe.h"
17+
#include "trace_probe_tmpl.h"
1718

1819
#define KPROBE_EVENT_SYSTEM "kprobes"
1920
#define KRETPROBE_MAXACTIVE_MAX 4096
@@ -120,160 +121,6 @@ static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
120121
static int kretprobe_dispatcher(struct kretprobe_instance *ri,
121122
struct pt_regs *regs);
122123

123-
/* Memory fetching by symbol */
124-
struct symbol_cache {
125-
char *symbol;
126-
long offset;
127-
unsigned long addr;
128-
};
129-
130-
unsigned long update_symbol_cache(struct symbol_cache *sc)
131-
{
132-
sc->addr = (unsigned long)kallsyms_lookup_name(sc->symbol);
133-
134-
if (sc->addr)
135-
sc->addr += sc->offset;
136-
137-
return sc->addr;
138-
}
139-
140-
void free_symbol_cache(struct symbol_cache *sc)
141-
{
142-
kfree(sc->symbol);
143-
kfree(sc);
144-
}
145-
146-
struct symbol_cache *alloc_symbol_cache(const char *sym, long offset)
147-
{
148-
struct symbol_cache *sc;
149-
150-
if (!sym || strlen(sym) == 0)
151-
return NULL;
152-
153-
sc = kzalloc(sizeof(struct symbol_cache), GFP_KERNEL);
154-
if (!sc)
155-
return NULL;
156-
157-
sc->symbol = kstrdup(sym, GFP_KERNEL);
158-
if (!sc->symbol) {
159-
kfree(sc);
160-
return NULL;
161-
}
162-
sc->offset = offset;
163-
update_symbol_cache(sc);
164-
165-
return sc;
166-
}
167-
168-
/*
169-
* Kprobes-specific fetch functions
170-
*/
171-
#define DEFINE_FETCH_stack(type) \
172-
static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \
173-
void *offset, void *dest) \
174-
{ \
175-
*(type *)dest = (type)regs_get_kernel_stack_nth(regs, \
176-
(unsigned int)((unsigned long)offset)); \
177-
} \
178-
NOKPROBE_SYMBOL(FETCH_FUNC_NAME(stack, type));
179-
180-
DEFINE_BASIC_FETCH_FUNCS(stack)
181-
/* No string on the stack entry */
182-
#define fetch_stack_string NULL
183-
#define fetch_stack_string_size NULL
184-
185-
#define DEFINE_FETCH_memory(type) \
186-
static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \
187-
void *addr, void *dest) \
188-
{ \
189-
type retval; \
190-
if (probe_kernel_address(addr, retval)) \
191-
*(type *)dest = 0; \
192-
else \
193-
*(type *)dest = retval; \
194-
} \
195-
NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, type));
196-
197-
DEFINE_BASIC_FETCH_FUNCS(memory)
198-
/*
199-
* Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
200-
* length and relative data location.
201-
*/
202-
static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
203-
void *addr, void *dest)
204-
{
205-
int maxlen = get_rloc_len(*(u32 *)dest);
206-
u8 *dst = get_rloc_data(dest);
207-
long ret;
208-
209-
if (!maxlen)
210-
return;
211-
212-
/*
213-
* Try to get string again, since the string can be changed while
214-
* probing.
215-
*/
216-
ret = strncpy_from_unsafe(dst, addr, maxlen);
217-
218-
if (ret < 0) { /* Failed to fetch string */
219-
dst[0] = '\0';
220-
*(u32 *)dest = make_data_rloc(0, get_rloc_offs(*(u32 *)dest));
221-
} else {
222-
*(u32 *)dest = make_data_rloc(ret, get_rloc_offs(*(u32 *)dest));
223-
}
224-
}
225-
NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string));
226-
227-
/* Return the length of string -- including null terminal byte */
228-
static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
229-
void *addr, void *dest)
230-
{
231-
mm_segment_t old_fs;
232-
int ret, len = 0;
233-
u8 c;
234-
235-
old_fs = get_fs();
236-
set_fs(KERNEL_DS);
237-
pagefault_disable();
238-
239-
do {
240-
ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1);
241-
len++;
242-
} while (c && ret == 0 && len < MAX_STRING_SIZE);
243-
244-
pagefault_enable();
245-
set_fs(old_fs);
246-
247-
if (ret < 0) /* Failed to check the length */
248-
*(u32 *)dest = 0;
249-
else
250-
*(u32 *)dest = len;
251-
}
252-
NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string_size));
253-
254-
#define DEFINE_FETCH_symbol(type) \
255-
void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs, void *data, void *dest)\
256-
{ \
257-
struct symbol_cache *sc = data; \
258-
if (sc->addr) \
259-
fetch_memory_##type(regs, (void *)sc->addr, dest); \
260-
else \
261-
*(type *)dest = 0; \
262-
} \
263-
NOKPROBE_SYMBOL(FETCH_FUNC_NAME(symbol, type));
264-
265-
DEFINE_BASIC_FETCH_FUNCS(symbol)
266-
DEFINE_FETCH_symbol(string)
267-
DEFINE_FETCH_symbol(string_size)
268-
269-
/* kprobes don't support file_offset fetch methods */
270-
#define fetch_file_offset_u8 NULL
271-
#define fetch_file_offset_u16 NULL
272-
#define fetch_file_offset_u32 NULL
273-
#define fetch_file_offset_u64 NULL
274-
#define fetch_file_offset_string NULL
275-
#define fetch_file_offset_string_size NULL
276-
277124
/* Fetch type information table */
278125
static const struct fetch_type kprobes_fetch_type_table[] = {
279126
/* Special types */
@@ -529,7 +376,7 @@ static bool within_notrace_func(struct trace_kprobe *tk)
529376
/* Internal register function - just handle k*probes and flags */
530377
static int __register_trace_kprobe(struct trace_kprobe *tk)
531378
{
532-
int i, ret;
379+
int ret;
533380

534381
if (trace_probe_is_registered(&tk->tp))
535382
return -EINVAL;
@@ -540,9 +387,6 @@ static int __register_trace_kprobe(struct trace_kprobe *tk)
540387
return -EINVAL;
541388
}
542389

543-
for (i = 0; i < tk->tp.nr_args; i++)
544-
traceprobe_update_arg(&tk->tp.args[i]);
545-
546390
/* Set/clear disabled flag according to tp->flag */
547391
if (trace_probe_is_enabled(&tk->tp))
548392
tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;
@@ -876,8 +720,8 @@ static int create_trace_kprobe(int argc, char **argv)
876720

877721
/* Parse fetch argument */
878722
ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg,
879-
is_return, true,
880-
kprobes_fetch_type_table);
723+
is_return, true,
724+
kprobes_fetch_type_table);
881725
if (ret) {
882726
pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
883727
goto error;
@@ -1031,6 +875,133 @@ static const struct file_operations kprobe_profile_ops = {
1031875
.release = seq_release,
1032876
};
1033877

878+
/* Kprobe specific fetch functions */
879+
880+
/* Return the length of string -- including null terminal byte */
881+
static nokprobe_inline void
882+
fetch_store_strlen(unsigned long addr, void *dest)
883+
{
884+
mm_segment_t old_fs;
885+
int ret, len = 0;
886+
u8 c;
887+
888+
old_fs = get_fs();
889+
set_fs(KERNEL_DS);
890+
pagefault_disable();
891+
892+
do {
893+
ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1);
894+
len++;
895+
} while (c && ret == 0 && len < MAX_STRING_SIZE);
896+
897+
pagefault_enable();
898+
set_fs(old_fs);
899+
900+
if (ret < 0) /* Failed to check the length */
901+
*(u32 *)dest = 0;
902+
else
903+
*(u32 *)dest = len;
904+
}
905+
906+
/*
907+
* Fetch a null-terminated string. Caller MUST set *(u32 *)buf with max
908+
* length and relative data location.
909+
*/
910+
static nokprobe_inline void
911+
fetch_store_string(unsigned long addr, void *dest)
912+
{
913+
int maxlen = get_rloc_len(*(u32 *)dest);
914+
u8 *dst = get_rloc_data(dest);
915+
long ret;
916+
917+
if (!maxlen)
918+
return;
919+
920+
/*
921+
* Try to get string again, since the string can be changed while
922+
* probing.
923+
*/
924+
ret = strncpy_from_unsafe(dst, (void *)addr, maxlen);
925+
926+
if (ret < 0) { /* Failed to fetch string */
927+
dst[0] = '\0';
928+
*(u32 *)dest = make_data_rloc(0, get_rloc_offs(*(u32 *)dest));
929+
} else {
930+
*(u32 *)dest = make_data_rloc(ret, get_rloc_offs(*(u32 *)dest));
931+
}
932+
}
933+
934+
/* Note that we don't verify it, since the code does not come from user space */
935+
static int
936+
process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs, void *dest,
937+
bool pre)
938+
{
939+
unsigned long val;
940+
int ret;
941+
942+
/* 1st stage: get value from context */
943+
switch (code->op) {
944+
case FETCH_OP_REG:
945+
val = regs_get_register(regs, code->param);
946+
break;
947+
case FETCH_OP_STACK:
948+
val = regs_get_kernel_stack_nth(regs, code->param);
949+
break;
950+
case FETCH_OP_STACKP:
951+
val = kernel_stack_pointer(regs);
952+
break;
953+
case FETCH_OP_RETVAL:
954+
val = regs_return_value(regs);
955+
break;
956+
case FETCH_OP_IMM:
957+
val = code->immediate;
958+
break;
959+
case FETCH_OP_COMM:
960+
val = (unsigned long)current->comm;
961+
break;
962+
default:
963+
return -EILSEQ;
964+
}
965+
code++;
966+
967+
/* 2nd stage: dereference memory if needed */
968+
while (code->op == FETCH_OP_DEREF) {
969+
ret = probe_kernel_read(&val, (void *)val + code->offset,
970+
sizeof(val));
971+
if (ret)
972+
return ret;
973+
code++;
974+
}
975+
976+
/* 3rd stage: store value to buffer */
977+
switch (code->op) {
978+
case FETCH_OP_ST_RAW:
979+
fetch_store_raw(val, code, dest);
980+
break;
981+
case FETCH_OP_ST_MEM:
982+
probe_kernel_read(dest, (void *)val + code->offset, code->size);
983+
break;
984+
case FETCH_OP_ST_STRING:
985+
if (pre)
986+
fetch_store_strlen(val + code->offset, dest);
987+
else
988+
fetch_store_string(val + code->offset, dest);
989+
break;
990+
default:
991+
return -EILSEQ;
992+
}
993+
code++;
994+
995+
/* 4th stage: modify stored value if needed */
996+
if (code->op == FETCH_OP_MOD_BF) {
997+
fetch_apply_bitfield(code, dest);
998+
code++;
999+
}
1000+
1001+
return code->op == FETCH_OP_END ? 0 : -EILSEQ;
1002+
}
1003+
NOKPROBE_SYMBOL(process_fetch_insn)
1004+
10341005
/* Kprobe handler */
10351006
static nokprobe_inline void
10361007
__kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,

0 commit comments

Comments
 (0)