Skip to content

Commit b5179ac

Browse files
Peter ZijlstraIngo Molnar
Peter Zijlstra
authored and
Ingo Molnar
committed
sched/fair: Prepare to fix fairness problems on migration
Mike reported that our recent attempt to fix migration problems: 3a47d51 ("sched/fair: Fix fairness issue on migration") broke interactivity and the signal starve test. We reverted that commit and now let's try it again more carefully, with some other underlying problems fixed first. One problem is that I assumed ENQUEUE_WAKING was only set when we do a cross-cpu wakeup (migration), which isn't true. This means we now destroy the vruntime history of tasks and wakeup-preemption suffers. Cure this by making my assumption true, only call sched_class::task_waking() when we do a cross-cpu wakeup. This avoids the indirect call in the case we do a local wakeup. Reported-by: Mike Galbraith <mgalbraith@suse.de> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Andrew Hunter <ahh@google.com> Cc: Ben Segall <bsegall@google.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Matt Fleming <matt@codeblueprint.co.uk> Cc: Mike Galbraith <efault@gmx.de> Cc: Morten Rasmussen <morten.rasmussen@arm.com> Cc: Paul Turner <pjt@google.com> Cc: Pavan Kondeti <pkondeti@codeaurora.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: byungchul.park@lge.com Cc: linux-kernel@vger.kernel.org Fixes: 3a47d51 ("sched/fair: Fix fairness issue on migration") Signed-off-by: Ingo Molnar <mingo@kernel.org>
1 parent c58d25f commit b5179ac

File tree

2 files changed

+58
-8
lines changed

2 files changed

+58
-8
lines changed

‎kernel/sched/core.c

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1709,14 +1709,22 @@ static void
17091709
ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
17101710
struct pin_cookie cookie)
17111711
{
1712+
int en_flags = ENQUEUE_WAKEUP;
1713+
17121714
lockdep_assert_held(&rq->lock);
17131715

17141716
#ifdef CONFIG_SMP
17151717
if (p->sched_contributes_to_load)
17161718
rq->nr_uninterruptible--;
1719+
1720+
/*
1721+
* If we migrated; we must have called sched_class::task_waking().
1722+
*/
1723+
if (wake_flags & WF_MIGRATED)
1724+
en_flags |= ENQUEUE_WAKING;
17171725
#endif
17181726

1719-
ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING);
1727+
ttwu_activate(rq, p, en_flags);
17201728
ttwu_do_wakeup(rq, p, wake_flags, cookie);
17211729
}
17221730

@@ -1762,7 +1770,11 @@ void sched_ttwu_pending(void)
17621770
while (llist) {
17631771
p = llist_entry(llist, struct task_struct, wake_entry);
17641772
llist = llist_next(llist);
1765-
ttwu_do_activate(rq, p, 0, cookie);
1773+
/*
1774+
* See ttwu_queue(); we only call ttwu_queue_remote() when
1775+
* its a x-cpu wakeup.
1776+
*/
1777+
ttwu_do_activate(rq, p, WF_MIGRATED, cookie);
17661778
}
17671779

17681780
lockdep_unpin_lock(&rq->lock, cookie);
@@ -1849,7 +1861,7 @@ bool cpus_share_cache(int this_cpu, int that_cpu)
18491861
}
18501862
#endif /* CONFIG_SMP */
18511863

1852-
static void ttwu_queue(struct task_struct *p, int cpu)
1864+
static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags)
18531865
{
18541866
struct rq *rq = cpu_rq(cpu);
18551867
struct pin_cookie cookie;
@@ -1864,7 +1876,7 @@ static void ttwu_queue(struct task_struct *p, int cpu)
18641876

18651877
raw_spin_lock(&rq->lock);
18661878
cookie = lockdep_pin_lock(&rq->lock);
1867-
ttwu_do_activate(rq, p, 0, cookie);
1879+
ttwu_do_activate(rq, p, wake_flags, cookie);
18681880
lockdep_unpin_lock(&rq->lock, cookie);
18691881
raw_spin_unlock(&rq->lock);
18701882
}
@@ -2034,17 +2046,18 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
20342046
p->sched_contributes_to_load = !!task_contributes_to_load(p);
20352047
p->state = TASK_WAKING;
20362048

2037-
if (p->sched_class->task_waking)
2038-
p->sched_class->task_waking(p);
2039-
20402049
cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags);
20412050
if (task_cpu(p) != cpu) {
20422051
wake_flags |= WF_MIGRATED;
2052+
2053+
if (p->sched_class->task_waking)
2054+
p->sched_class->task_waking(p);
2055+
20432056
set_task_cpu(p, cpu);
20442057
}
20452058
#endif /* CONFIG_SMP */
20462059

2047-
ttwu_queue(p, cpu);
2060+
ttwu_queue(p, cpu, wake_flags);
20482061
stat:
20492062
if (schedstat_enabled())
20502063
ttwu_stat(p, cpu, wake_flags);

‎kernel/sched/fair.c

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3254,6 +3254,37 @@ static inline void check_schedstat_required(void)
32543254
#endif
32553255
}
32563256

3257+
3258+
/*
3259+
* MIGRATION
3260+
*
3261+
* dequeue
3262+
* update_curr()
3263+
* update_min_vruntime()
3264+
* vruntime -= min_vruntime
3265+
*
3266+
* enqueue
3267+
* update_curr()
3268+
* update_min_vruntime()
3269+
* vruntime += min_vruntime
3270+
*
3271+
* this way the vruntime transition between RQs is done when both
3272+
* min_vruntime are up-to-date.
3273+
*
3274+
* WAKEUP (remote)
3275+
*
3276+
* ->task_waking_fair()
3277+
* vruntime -= min_vruntime
3278+
*
3279+
* enqueue
3280+
* update_curr()
3281+
* update_min_vruntime()
3282+
* vruntime += min_vruntime
3283+
*
3284+
* this way we don't have the most up-to-date min_vruntime on the originating
3285+
* CPU and an up-to-date min_vruntime on the destination CPU.
3286+
*/
3287+
32573288
static void
32583289
enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
32593290
{
@@ -4810,6 +4841,12 @@ static unsigned long cpu_avg_load_per_task(int cpu)
48104841
return 0;
48114842
}
48124843

4844+
/*
4845+
* Called to migrate a waking task; as blocked tasks retain absolute vruntime
4846+
* the migration needs to deal with this by subtracting the old and adding the
4847+
* new min_vruntime -- the latter is done by enqueue_entity() when placing
4848+
* the task on the new runqueue.
4849+
*/
48134850
static void task_waking_fair(struct task_struct *p)
48144851
{
48154852
struct sched_entity *se = &p->se;

0 commit comments

Comments
 (0)