Skip to content

Commit 1e1dcd9

Browse files
4astdavem330
4ast
authored and
davem330
committed
perf: split perf_trace_buf_prepare into alloc and update parts
split allows to move expensive update of 'struct trace_entry' to later phase. Repurpose unused 1st argument of perf_tp_event() to indicate event type. While splitting use temp variable 'rctx' instead of '*rctx' to avoid unnecessary loads done by the compiler due to -fno-strict-aliasing Signed-off-by: Alexei Starovoitov <ast@kernel.org> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent e93735b commit 1e1dcd9

File tree

8 files changed

+49
-42
lines changed

8 files changed

+49
-42
lines changed

‎include/linux/perf_event.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1016,7 +1016,7 @@ static inline bool perf_paranoid_kernel(void)
10161016
}
10171017

10181018
extern void perf_event_init(void);
1019-
extern void perf_tp_event(u64 addr, u64 count, void *record,
1019+
extern void perf_tp_event(u16 event_type, u64 count, void *record,
10201020
int entry_size, struct pt_regs *regs,
10211021
struct hlist_head *head, int rctx,
10221022
struct task_struct *task);

‎include/linux/trace_events.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -605,15 +605,15 @@ extern void perf_trace_del(struct perf_event *event, int flags);
605605
extern int ftrace_profile_set_filter(struct perf_event *event, int event_id,
606606
char *filter_str);
607607
extern void ftrace_profile_free_filter(struct perf_event *event);
608-
extern void *perf_trace_buf_prepare(int size, unsigned short type,
609-
struct pt_regs **regs, int *rctxp);
608+
void perf_trace_buf_update(void *record, u16 type);
609+
void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp);
610610

611611
static inline void
612-
perf_trace_buf_submit(void *raw_data, int size, int rctx, u64 addr,
612+
perf_trace_buf_submit(void *raw_data, int size, int rctx, u16 type,
613613
u64 count, struct pt_regs *regs, void *head,
614614
struct task_struct *task)
615615
{
616-
perf_tp_event(addr, count, raw_data, size, regs, head, rctx, task);
616+
perf_tp_event(type, count, raw_data, size, regs, head, rctx, task);
617617
}
618618
#endif
619619

‎include/trace/perf.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,7 @@ perf_trace_##call(void *__data, proto) \
5353
sizeof(u64)); \
5454
__entry_size -= sizeof(u32); \
5555
\
56-
entry = perf_trace_buf_prepare(__entry_size, \
57-
event_call->event.type, &__regs, &rctx); \
56+
entry = perf_trace_buf_alloc(__entry_size, &__regs, &rctx); \
5857
if (!entry) \
5958
return; \
6059
\
@@ -64,8 +63,9 @@ perf_trace_##call(void *__data, proto) \
6463
\
6564
{ assign; } \
6665
\
67-
perf_trace_buf_submit(entry, __entry_size, rctx, 0, \
68-
__count, __regs, head, __task); \
66+
perf_trace_buf_submit(entry, __entry_size, rctx, \
67+
event_call->event.type, __count, __regs, \
68+
head, __task); \
6969
}
7070

7171
/*

‎kernel/events/core.c

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6987,7 +6987,7 @@ static int perf_tp_event_match(struct perf_event *event,
69876987
return 1;
69886988
}
69896989

6990-
void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
6990+
void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size,
69916991
struct pt_regs *regs, struct hlist_head *head, int rctx,
69926992
struct task_struct *task)
69936993
{
@@ -6999,9 +6999,11 @@ void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
69996999
.data = record,
70007000
};
70017001

7002-
perf_sample_data_init(&data, addr, 0);
7002+
perf_sample_data_init(&data, 0, 0);
70037003
data.raw = &raw;
70047004

7005+
perf_trace_buf_update(record, event_type);
7006+
70057007
hlist_for_each_entry_rcu(event, head, hlist_entry) {
70067008
if (perf_tp_event_match(event, &data, regs))
70077009
perf_swevent_event(event, count, &data, regs);

‎kernel/trace/trace_event_perf.c

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -260,42 +260,43 @@ void perf_trace_del(struct perf_event *p_event, int flags)
260260
tp_event->class->reg(tp_event, TRACE_REG_PERF_DEL, p_event);
261261
}
262262

263-
void *perf_trace_buf_prepare(int size, unsigned short type,
264-
struct pt_regs **regs, int *rctxp)
263+
void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp)
265264
{
266-
struct trace_entry *entry;
267-
unsigned long flags;
268265
char *raw_data;
269-
int pc;
266+
int rctx;
270267

271268
BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long));
272269

273270
if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
274-
"perf buffer not large enough"))
271+
"perf buffer not large enough"))
275272
return NULL;
276273

277-
pc = preempt_count();
278-
279-
*rctxp = perf_swevent_get_recursion_context();
280-
if (*rctxp < 0)
274+
*rctxp = rctx = perf_swevent_get_recursion_context();
275+
if (rctx < 0)
281276
return NULL;
282277

283278
if (regs)
284-
*regs = this_cpu_ptr(&__perf_regs[*rctxp]);
285-
raw_data = this_cpu_ptr(perf_trace_buf[*rctxp]);
279+
*regs = this_cpu_ptr(&__perf_regs[rctx]);
280+
raw_data = this_cpu_ptr(perf_trace_buf[rctx]);
286281

287282
/* zero the dead bytes from align to not leak stack to user */
288283
memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
284+
return raw_data;
285+
}
286+
EXPORT_SYMBOL_GPL(perf_trace_buf_alloc);
287+
NOKPROBE_SYMBOL(perf_trace_buf_alloc);
288+
289+
void perf_trace_buf_update(void *record, u16 type)
290+
{
291+
struct trace_entry *entry = record;
292+
int pc = preempt_count();
293+
unsigned long flags;
289294

290-
entry = (struct trace_entry *)raw_data;
291295
local_save_flags(flags);
292296
tracing_generic_entry_update(entry, flags, pc);
293297
entry->type = type;
294-
295-
return raw_data;
296298
}
297-
EXPORT_SYMBOL_GPL(perf_trace_buf_prepare);
298-
NOKPROBE_SYMBOL(perf_trace_buf_prepare);
299+
NOKPROBE_SYMBOL(perf_trace_buf_update);
299300

300301
#ifdef CONFIG_FUNCTION_TRACER
301302
static void
@@ -319,13 +320,13 @@ perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip,
319320
memset(&regs, 0, sizeof(regs));
320321
perf_fetch_caller_regs(&regs);
321322

322-
entry = perf_trace_buf_prepare(ENTRY_SIZE, TRACE_FN, NULL, &rctx);
323+
entry = perf_trace_buf_alloc(ENTRY_SIZE, NULL, &rctx);
323324
if (!entry)
324325
return;
325326

326327
entry->ip = ip;
327328
entry->parent_ip = parent_ip;
328-
perf_trace_buf_submit(entry, ENTRY_SIZE, rctx, 0,
329+
perf_trace_buf_submit(entry, ENTRY_SIZE, rctx, TRACE_FN,
329330
1, &regs, head, NULL);
330331

331332
#undef ENTRY_SIZE

‎kernel/trace/trace_kprobe.c

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1149,14 +1149,15 @@ kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
11491149
size = ALIGN(__size + sizeof(u32), sizeof(u64));
11501150
size -= sizeof(u32);
11511151

1152-
entry = perf_trace_buf_prepare(size, call->event.type, NULL, &rctx);
1152+
entry = perf_trace_buf_alloc(size, NULL, &rctx);
11531153
if (!entry)
11541154
return;
11551155

11561156
entry->ip = (unsigned long)tk->rp.kp.addr;
11571157
memset(&entry[1], 0, dsize);
11581158
store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1159-
perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
1159+
perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1160+
head, NULL);
11601161
}
11611162
NOKPROBE_SYMBOL(kprobe_perf_func);
11621163

@@ -1184,14 +1185,15 @@ kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
11841185
size = ALIGN(__size + sizeof(u32), sizeof(u64));
11851186
size -= sizeof(u32);
11861187

1187-
entry = perf_trace_buf_prepare(size, call->event.type, NULL, &rctx);
1188+
entry = perf_trace_buf_alloc(size, NULL, &rctx);
11881189
if (!entry)
11891190
return;
11901191

11911192
entry->func = (unsigned long)tk->rp.kp.addr;
11921193
entry->ret_ip = (unsigned long)ri->ret_addr;
11931194
store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1194-
perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
1195+
perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1196+
head, NULL);
11951197
}
11961198
NOKPROBE_SYMBOL(kretprobe_perf_func);
11971199
#endif /* CONFIG_PERF_EVENTS */

‎kernel/trace/trace_syscalls.c

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -587,15 +587,16 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
587587
size = ALIGN(size + sizeof(u32), sizeof(u64));
588588
size -= sizeof(u32);
589589

590-
rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
591-
sys_data->enter_event->event.type, NULL, &rctx);
590+
rec = perf_trace_buf_alloc(size, NULL, &rctx);
592591
if (!rec)
593592
return;
594593

595594
rec->nr = syscall_nr;
596595
syscall_get_arguments(current, regs, 0, sys_data->nb_args,
597596
(unsigned long *)&rec->args);
598-
perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL);
597+
perf_trace_buf_submit(rec, size, rctx,
598+
sys_data->enter_event->event.type, 1, regs,
599+
head, NULL);
599600
}
600601

601602
static int perf_sysenter_enable(struct trace_event_call *call)
@@ -660,14 +661,14 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
660661
size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
661662
size -= sizeof(u32);
662663

663-
rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
664-
sys_data->exit_event->event.type, NULL, &rctx);
664+
rec = perf_trace_buf_alloc(size, NULL, &rctx);
665665
if (!rec)
666666
return;
667667

668668
rec->nr = syscall_nr;
669669
rec->ret = syscall_get_return_value(current, regs);
670-
perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL);
670+
perf_trace_buf_submit(rec, size, rctx, sys_data->exit_event->event.type,
671+
1, regs, head, NULL);
671672
}
672673

673674
static int perf_sysexit_enable(struct trace_event_call *call)

‎kernel/trace/trace_uprobe.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1131,7 +1131,7 @@ static void __uprobe_perf_func(struct trace_uprobe *tu,
11311131
if (hlist_empty(head))
11321132
goto out;
11331133

1134-
entry = perf_trace_buf_prepare(size, call->event.type, NULL, &rctx);
1134+
entry = perf_trace_buf_alloc(size, NULL, &rctx);
11351135
if (!entry)
11361136
goto out;
11371137

@@ -1152,7 +1152,8 @@ static void __uprobe_perf_func(struct trace_uprobe *tu,
11521152
memset(data + len, 0, size - esize - len);
11531153
}
11541154

1155-
perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
1155+
perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1156+
head, NULL);
11561157
out:
11571158
preempt_enable();
11581159
}

0 commit comments

Comments
 (0)