Skip to content

Commit 555e0c1

Browse files
fweisbecfweisbec
committed
perf: Migrate perf to use new tick dependency mask model
Instead of providing asynchronous checks for the nohz subsystem to verify perf event tick dependency, migrate perf to the new mask. Perf needs the tick for two situations: 1) Freq events. We could set the tick dependency when those are installed on a CPU context. But setting a global dependency on top of the global freq events accounting is much easier. If people want that to be optimized, we can still refine that on the per-CPU tick dependency level. This patch dooesn't change the current behaviour anyway. 2) Throttled events: this is a per-cpu dependency. Reviewed-by: Chris Metcalf <cmetcalf@ezchip.com> Cc: Christoph Lameter <cl@linux.com> Cc: Chris Metcalf <cmetcalf@ezchip.com> Cc: Ingo Molnar <mingo@kernel.org> Cc: Luiz Capitulino <lcapitulino@redhat.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Rik van Riel <riel@redhat.com> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Viresh Kumar <viresh.kumar@linaro.org> Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
1 parent e6e6cc2 commit 555e0c1

File tree

4 files changed

+49
-32
lines changed

4 files changed

+49
-32
lines changed

‎include/linux/perf_event.h

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1108,12 +1108,6 @@ static inline void perf_event_task_tick(void) { }
11081108
static inline int perf_event_release_kernel(struct perf_event *event) { return 0; }
11091109
#endif
11101110

1111-
#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_NO_HZ_FULL)
1112-
extern bool perf_event_can_stop_tick(void);
1113-
#else
1114-
static inline bool perf_event_can_stop_tick(void) { return true; }
1115-
#endif
1116-
11171111
#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL)
11181112
extern void perf_restore_debug_store(void);
11191113
#else

‎include/linux/tick.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,6 @@ static inline void tick_dep_clear_signal(struct signal_struct *signal,
233233
tick_nohz_dep_clear_signal(signal, bit);
234234
}
235235

236-
extern void tick_nohz_full_kick(void);
237236
extern void tick_nohz_full_kick_cpu(int cpu);
238237
extern void tick_nohz_full_kick_all(void);
239238
extern void __tick_nohz_task_switch(void);
@@ -260,7 +259,6 @@ static inline void tick_dep_clear_signal(struct signal_struct *signal,
260259
enum tick_dep_bits bit) { }
261260

262261
static inline void tick_nohz_full_kick_cpu(int cpu) { }
263-
static inline void tick_nohz_full_kick(void) { }
264262
static inline void tick_nohz_full_kick_all(void) { }
265263
static inline void __tick_nohz_task_switch(void) { }
266264
#endif

‎kernel/events/core.c

Lines changed: 48 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3060,17 +3060,6 @@ static int perf_rotate_context(struct perf_cpu_context *cpuctx)
30603060
return rotate;
30613061
}
30623062

3063-
#ifdef CONFIG_NO_HZ_FULL
3064-
bool perf_event_can_stop_tick(void)
3065-
{
3066-
if (atomic_read(&nr_freq_events) ||
3067-
__this_cpu_read(perf_throttled_count))
3068-
return false;
3069-
else
3070-
return true;
3071-
}
3072-
#endif
3073-
30743063
void perf_event_task_tick(void)
30753064
{
30763065
struct list_head *head = this_cpu_ptr(&active_ctx_list);
@@ -3081,6 +3070,7 @@ void perf_event_task_tick(void)
30813070

30823071
__this_cpu_inc(perf_throttled_seq);
30833072
throttled = __this_cpu_xchg(perf_throttled_count, 0);
3073+
tick_dep_clear_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
30843074

30853075
list_for_each_entry_safe(ctx, tmp, head, active_ctx_list)
30863076
perf_adjust_freq_unthr_context(ctx, throttled);
@@ -3511,6 +3501,28 @@ static void unaccount_event_cpu(struct perf_event *event, int cpu)
35113501
atomic_dec(&per_cpu(perf_cgroup_events, cpu));
35123502
}
35133503

3504+
#ifdef CONFIG_NO_HZ_FULL
3505+
static DEFINE_SPINLOCK(nr_freq_lock);
3506+
#endif
3507+
3508+
static void unaccount_freq_event_nohz(void)
3509+
{
3510+
#ifdef CONFIG_NO_HZ_FULL
3511+
spin_lock(&nr_freq_lock);
3512+
if (atomic_dec_and_test(&nr_freq_events))
3513+
tick_nohz_dep_clear(TICK_DEP_BIT_PERF_EVENTS);
3514+
spin_unlock(&nr_freq_lock);
3515+
#endif
3516+
}
3517+
3518+
static void unaccount_freq_event(void)
3519+
{
3520+
if (tick_nohz_full_enabled())
3521+
unaccount_freq_event_nohz();
3522+
else
3523+
atomic_dec(&nr_freq_events);
3524+
}
3525+
35143526
static void unaccount_event(struct perf_event *event)
35153527
{
35163528
bool dec = false;
@@ -3527,7 +3539,7 @@ static void unaccount_event(struct perf_event *event)
35273539
if (event->attr.task)
35283540
atomic_dec(&nr_task_events);
35293541
if (event->attr.freq)
3530-
atomic_dec(&nr_freq_events);
3542+
unaccount_freq_event();
35313543
if (event->attr.context_switch) {
35323544
dec = true;
35333545
atomic_dec(&nr_switch_events);
@@ -6349,9 +6361,9 @@ static int __perf_event_overflow(struct perf_event *event,
63496361
if (unlikely(throttle
63506362
&& hwc->interrupts >= max_samples_per_tick)) {
63516363
__this_cpu_inc(perf_throttled_count);
6364+
tick_dep_set_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
63526365
hwc->interrupts = MAX_INTERRUPTS;
63536366
perf_log_throttle(event, 0);
6354-
tick_nohz_full_kick();
63556367
ret = 1;
63566368
}
63576369
}
@@ -7741,6 +7753,27 @@ static void account_event_cpu(struct perf_event *event, int cpu)
77417753
atomic_inc(&per_cpu(perf_cgroup_events, cpu));
77427754
}
77437755

7756+
/* Freq events need the tick to stay alive (see perf_event_task_tick). */
7757+
static void account_freq_event_nohz(void)
7758+
{
7759+
#ifdef CONFIG_NO_HZ_FULL
7760+
/* Lock so we don't race with concurrent unaccount */
7761+
spin_lock(&nr_freq_lock);
7762+
if (atomic_inc_return(&nr_freq_events) == 1)
7763+
tick_nohz_dep_set(TICK_DEP_BIT_PERF_EVENTS);
7764+
spin_unlock(&nr_freq_lock);
7765+
#endif
7766+
}
7767+
7768+
static void account_freq_event(void)
7769+
{
7770+
if (tick_nohz_full_enabled())
7771+
account_freq_event_nohz();
7772+
else
7773+
atomic_inc(&nr_freq_events);
7774+
}
7775+
7776+
77447777
static void account_event(struct perf_event *event)
77457778
{
77467779
bool inc = false;
@@ -7756,10 +7789,8 @@ static void account_event(struct perf_event *event)
77567789
atomic_inc(&nr_comm_events);
77577790
if (event->attr.task)
77587791
atomic_inc(&nr_task_events);
7759-
if (event->attr.freq) {
7760-
if (atomic_inc_return(&nr_freq_events) == 1)
7761-
tick_nohz_full_kick_all();
7762-
}
7792+
if (event->attr.freq)
7793+
account_freq_event();
77637794
if (event->attr.context_switch) {
77647795
atomic_inc(&nr_switch_events);
77657796
inc = true;

‎kernel/time/tick-sched.c

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
#include <linux/module.h>
2323
#include <linux/irq_work.h>
2424
#include <linux/posix-timers.h>
25-
#include <linux/perf_event.h>
2625
#include <linux/context_tracking.h>
2726

2827
#include <asm/irq_regs.h>
@@ -215,11 +214,6 @@ static bool can_stop_full_tick(struct tick_sched *ts)
215214
return false;
216215
}
217216

218-
if (!perf_event_can_stop_tick()) {
219-
trace_tick_stop(0, TICK_DEP_MASK_PERF_EVENTS);
220-
return false;
221-
}
222-
223217
#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
224218
/*
225219
* sched_clock_tick() needs us?
@@ -257,7 +251,7 @@ static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = {
257251
* This kick, unlike tick_nohz_full_kick_cpu() and tick_nohz_full_kick_all(),
258252
* is NMI safe.
259253
*/
260-
void tick_nohz_full_kick(void)
254+
static void tick_nohz_full_kick(void)
261255
{
262256
if (!tick_nohz_full_cpu(smp_processor_id()))
263257
return;

0 commit comments

Comments
 (0)