sched/fair: Mask UTIL_AVG_UNCHANGED usages

derkling · Ingo Molnar · commit 92a801e5d5b7 · 2018-11-12T06:17:52.000+01:00
The _task_util_est() is mainly used to add/remove the task contribution to/from the rq's estimated utilization at task enqueue/dequeue time. In both cases we ensure the UTIL_AVG_UNCHANGED flag is set to keep consistency between enqueue and dequeue time while still being transparent to update_load_avg calls which will eventually reset the flag. Let's move the flag forcing within _task_util_est() itself so that we can simplify calling code by hiding that estimated utilization implementation detail into one of its internal functions. This will affect also the "public" API task_util_est() but we know that the flag will (eventually) impact just on the LSB of the estimated utilization, thus it's certainly acceptable. Signed-off-by: Patrick Bellasi <patrick.bellasi@arm.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Dietmar Eggemann <dietmar.eggemann@arm.com> Cc: Juri Lelli <juri.lelli@redhat.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Morten Rasmussen <morten.rasmussen@arm.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Quentin Perret <quentin.perret@arm.com> Cc: Steve Muckle <smuckle@google.com> Cc: Suren Baghdasaryan <surenb@google.com> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Todd Kjos <tkjos@google.com> Cc: Vincent Guittot <vincent.guittot@linaro.org> Link: http://lkml.kernel.org/r/20181105145400.935-3-patrick.bellasi@arm.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
@@ -3604,7 +3604,7 @@ static inline unsigned long _task_util_est(struct task_struct *p)
 {
 	struct util_est ue = READ_ONCE(p->se.avg.util_est);
 
-	return max(ue.ewma, ue.enqueued);
+	return (max(ue.ewma, ue.enqueued) | UTIL_AVG_UNCHANGED);
 }
 
 static inline unsigned long task_util_est(struct task_struct *p)
@@ -3622,7 +3622,7 @@ static inline void util_est_enqueue(struct cfs_rq *cfs_rq,
 
 	/* Update root cfs_rq's estimated utilization */
 	enqueued  = cfs_rq->avg.util_est.enqueued;
-	enqueued += (_task_util_est(p) | UTIL_AVG_UNCHANGED);
+	enqueued += _task_util_est(p);
 	WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued);
 }
 
@@ -3650,8 +3650,7 @@ util_est_dequeue(struct cfs_rq *cfs_rq, struct task_struct *p, bool task_sleep)
 
 	/* Update root cfs_rq's estimated utilization */
 	ue.enqueued  = cfs_rq->avg.util_est.enqueued;
-	ue.enqueued -= min_t(unsigned int, ue.enqueued,
-			     (_task_util_est(p) | UTIL_AVG_UNCHANGED));
+	ue.enqueued -= min_t(unsigned int, ue.enqueued, _task_util_est(p));
 	WRITE_ONCE(cfs_rq->avg.util_est.enqueued, ue.enqueued);
 
 	/*
@@ -6292,7 +6291,7 @@ static unsigned long cpu_util_without(int cpu, struct task_struct *p)
 		 */
 		if (unlikely(task_on_rq_queued(p) || current == p)) {
 			estimated -= min_t(unsigned int, estimated,
-					   (_task_util_est(p) | UTIL_AVG_UNCHANGED));
+					   _task_util_est(p));
 		}
 		util = max(util, estimated);
 	}

Original file line number	Diff line number	Diff line change
`@@ -3604,7 +3604,7 @@ static inline unsigned long _task_util_est(struct task_struct *p)`
`3604`	`3604`	`{`
`3605`	`3605`	`struct util_est ue = READ_ONCE(p->se.avg.util_est);`
`3606`	`3606`
`3607`		`- return max(ue.ewma, ue.enqueued);`
	`3607`	`+ return (max(ue.ewma, ue.enqueued) \| UTIL_AVG_UNCHANGED);`
`3608`	`3608`	`}`
`3609`	`3609`
`3610`	`3610`	`static inline unsigned long task_util_est(struct task_struct *p)`
`@@ -3622,7 +3622,7 @@ static inline void util_est_enqueue(struct cfs_rq *cfs_rq,`
`3622`	`3622`
`3623`	`3623`	`/* Update root cfs_rq's estimated utilization */`
`3624`	`3624`	`enqueued = cfs_rq->avg.util_est.enqueued;`
`3625`		`- enqueued += (_task_util_est(p) \| UTIL_AVG_UNCHANGED);`
	`3625`	`+ enqueued += _task_util_est(p);`
`3626`	`3626`	`WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued);`
`3627`	`3627`	`}`
`3628`	`3628`
`@@ -3650,8 +3650,7 @@ util_est_dequeue(struct cfs_rq cfs_rq, struct task_struct p, bool task_sleep)`
`3650`	`3650`
`3651`	`3651`	`/* Update root cfs_rq's estimated utilization */`
`3652`	`3652`	`ue.enqueued = cfs_rq->avg.util_est.enqueued;`
`3653`		`- ue.enqueued -= min_t(unsigned int, ue.enqueued,`
`3654`		`- (_task_util_est(p) \| UTIL_AVG_UNCHANGED));`
	`3653`	`+ ue.enqueued -= min_t(unsigned int, ue.enqueued, _task_util_est(p));`
`3655`	`3654`	`WRITE_ONCE(cfs_rq->avg.util_est.enqueued, ue.enqueued);`
`3656`	`3655`
`3657`	`3656`	`/*`
`@@ -6292,7 +6291,7 @@ static unsigned long cpu_util_without(int cpu, struct task_struct *p)`
`6292`	`6291`	`*/`
`6293`	`6292`	`if (unlikely(task_on_rq_queued(p) \|\| current == p)) {`
`6294`	`6293`	`estimated -= min_t(unsigned int, estimated,`
`6295`		`- (_task_util_est(p) \| UTIL_AVG_UNCHANGED));`
	`6294`	`+ _task_util_est(p));`
`6296`	`6295`	`}`
`6297`	`6296`	`util = max(util, estimated);`
`6298`	`6297`	`}`