sched: Use jump labels to reduce overhead when bandwidth control is inactive

paulturner · Ingo Molnar · commit 56f570e512ee · 2011-11-16T08:48:18.000+01:00
Now that the linkage of jump-labels has been fixed they show a measurable improvement in overhead for the enabled-but-unused case. Workload is: 'taskset -c 0 perf stat --repeat 50 -e instructions,cycles,branches bash -c "for ((i=0;i<5;i++)); do $(dirname $0)/pipe-test 20000; done"' There's a speedup for all situations: instructions cycles branches ------------------------------------------------------------------------- Intel Westmere base 806611770 745895590 146765378 +jumplabel 803090165 (-0.44%) 713381840 (-4.36%) 144561130 AMD Barcelona base 824657415 740055589 148855354 +jumplabel 821056910 (-0.44%) 737558389 (-0.34%) 146635229 Signed-off-by: Paul Turner <pjt@google.com> Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Link: http://lkml.kernel.org/r/20111108042736.560831357@google.com Signed-off-by: Ingo Molnar <mingo@elte.hu>
diff --git a/kernel/sched.c b/kernel/sched.c
@@ -72,6 +72,7 @@
 #include <linux/ftrace.h>
 #include <linux/slab.h>
 #include <linux/init_task.h>
+#include <linux/jump_label.h>
 
 #include <asm/tlb.h>
 #include <asm/irq_regs.h>
@@ -503,7 +504,32 @@ static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
 	hrtimer_cancel(&cfs_b->period_timer);
 	hrtimer_cancel(&cfs_b->slack_timer);
 }
-#else
+
+#ifdef HAVE_JUMP_LABEL
+static struct jump_label_key __cfs_bandwidth_used;
+
+static inline bool cfs_bandwidth_used(void)
+{
+	return static_branch(&__cfs_bandwidth_used);
+}
+
+static void account_cfs_bandwidth_used(int enabled, int was_enabled)
+{
+	/* only need to count groups transitioning between enabled/!enabled */
+	if (enabled && !was_enabled)
+		jump_label_inc(&__cfs_bandwidth_used);
+	else if (!enabled && was_enabled)
+		jump_label_dec(&__cfs_bandwidth_used);
+}
+#else /* !HAVE_JUMP_LABEL */
+/* static_branch doesn't help unless supported */
+static int cfs_bandwidth_used(void)
+{
+	return 1;
+}
+static void account_cfs_bandwidth_used(int enabled, int was_enabled) {}
+#endif /* HAVE_JUMP_LABEL */
+#else /* !CONFIG_CFS_BANDWIDTH */
 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
 static void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
@@ -9203,7 +9229,7 @@ static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);
 
 static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
 {
-	int i, ret = 0, runtime_enabled;
+	int i, ret = 0, runtime_enabled, runtime_was_enabled;
 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
 
 	if (tg == &root_task_group)
@@ -9231,6 +9257,9 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
 		goto out_unlock;
 
 	runtime_enabled = quota != RUNTIME_INF;
+	runtime_was_enabled = cfs_b->quota != RUNTIME_INF;
+	account_cfs_bandwidth_used(runtime_enabled, runtime_was_enabled);
+
 	raw_spin_lock_irq(&cfs_b->lock);
 	cfs_b->period = ns_to_ktime(period);
 	cfs_b->quota = quota;
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
@@ -1421,21 +1421,21 @@ static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
 static __always_inline void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
 						   unsigned long delta_exec)
 {
-	if (!cfs_rq->runtime_enabled)
+	if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled)
 		return;
 
 	__account_cfs_rq_runtime(cfs_rq, delta_exec);
 }
 
 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
 {
-	return cfs_rq->throttled;
+	return cfs_bandwidth_used() && cfs_rq->throttled;
 }
 
 /* check whether cfs_rq, or any parent, is throttled */
 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
 {
-	return cfs_rq->throttle_count;
+	return cfs_bandwidth_used() && cfs_rq->throttle_count;
 }
 
 /*
@@ -1756,6 +1756,9 @@ static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
 
 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
 {
+	if (!cfs_bandwidth_used())
+		return;
+
 	if (!cfs_rq->runtime_enabled || cfs_rq->nr_running)
 		return;
 
@@ -1801,6 +1804,9 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
  */
 static void check_enqueue_throttle(struct cfs_rq *cfs_rq)
 {
+	if (!cfs_bandwidth_used())
+		return;
+
 	/* an active group must be handled by the update_curr()->put() path */
 	if (!cfs_rq->runtime_enabled || cfs_rq->curr)
 		return;
@@ -1818,6 +1824,9 @@ static void check_enqueue_throttle(struct cfs_rq *cfs_rq)
 /* conditionally throttle active cfs_rq's from put_prev_entity() */
 static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq)
 {
+	if (!cfs_bandwidth_used())
+		return;
+
 	if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0))
 		return;
 

Original file line number	Diff line number	Diff line change
`@@ -1421,21 +1421,21 @@ static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq,`
`1421`	`1421`	`static __always_inline void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,`
`1422`	`1422`	`unsigned long delta_exec)`
`1423`	`1423`	`{`
`1424`		`- if (!cfs_rq->runtime_enabled)`
	`1424`	`+ if (!cfs_bandwidth_used() \|\| !cfs_rq->runtime_enabled)`
`1425`	`1425`	`return;`
`1426`	`1426`
`1427`	`1427`	`__account_cfs_rq_runtime(cfs_rq, delta_exec);`
`1428`	`1428`	`}`
`1429`	`1429`
`1430`	`1430`	`static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)`
`1431`	`1431`	`{`
`1432`		`- return cfs_rq->throttled;`
	`1432`	`+ return cfs_bandwidth_used() && cfs_rq->throttled;`
`1433`	`1433`	`}`
`1434`	`1434`
`1435`	`1435`	`/* check whether cfs_rq, or any parent, is throttled */`
`1436`	`1436`	`static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)`
`1437`	`1437`	`{`
`1438`		`- return cfs_rq->throttle_count;`
	`1438`	`+ return cfs_bandwidth_used() && cfs_rq->throttle_count;`
`1439`	`1439`	`}`
`1440`	`1440`
`1441`	`1441`	`/*`
`@@ -1756,6 +1756,9 @@ static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq)`
`1756`	`1756`
`1757`	`1757`	`static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)`
`1758`	`1758`	`{`
	`1759`	`+ if (!cfs_bandwidth_used())`
	`1760`	`+ return;`
	`1761`	`+`
`1759`	`1762`	`if (!cfs_rq->runtime_enabled \|\| cfs_rq->nr_running)`
`1760`	`1763`	`return;`
`1761`	`1764`
`@@ -1801,6 +1804,9 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)`
`1801`	`1804`	`*/`
`1802`	`1805`	`static void check_enqueue_throttle(struct cfs_rq *cfs_rq)`
`1803`	`1806`	`{`
	`1807`	`+ if (!cfs_bandwidth_used())`
	`1808`	`+ return;`
	`1809`	`+`
`1804`	`1810`	`/* an active group must be handled by the update_curr()->put() path */`
`1805`	`1811`	`if (!cfs_rq->runtime_enabled \|\| cfs_rq->curr)`
`1806`	`1812`	`return;`
`@@ -1818,6 +1824,9 @@ static void check_enqueue_throttle(struct cfs_rq *cfs_rq)`
`1818`	`1824`	`/* conditionally throttle active cfs_rq's from put_prev_entity() */`
`1819`	`1825`	`static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq)`
`1820`	`1826`	`{`
	`1827`	`+ if (!cfs_bandwidth_used())`
	`1828`	`+ return;`
	`1829`	`+`
`1821`	`1830`	`if (likely(!cfs_rq->runtime_enabled \|\| cfs_rq->runtime_remaining > 0))`
`1822`	`1831`	`return;`
`1823`	`1832`