Skip to content

Commit 59e1678

Browse files
author
Ingo Molnar
committed
Merge branch 'sched/urgent' into sched/core, to pick up dependent fixes
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2 parents ff1cdc9 + c469933 commit 59e1678

File tree

1 file changed

+50
-16
lines changed

1 file changed

+50
-16
lines changed

‎kernel/sched/fair.c

Lines changed: 50 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2400,8 +2400,8 @@ void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags)
24002400
local = 1;
24012401

24022402
/*
2403-
* Retry task to preferred node migration periodically, in case it
2404-
* case it previously failed, or the scheduler moved us.
2403+
* Retry to migrate task to preferred node periodically, in case it
2404+
* previously failed, or the scheduler moved us.
24052405
*/
24062406
if (time_after(jiffies, p->numa_migrate_retry)) {
24072407
task_numa_placement(p);
@@ -5674,11 +5674,11 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p,
56745674
return target;
56755675
}
56765676

5677-
static unsigned long cpu_util_wake(int cpu, struct task_struct *p);
5677+
static unsigned long cpu_util_without(int cpu, struct task_struct *p);
56785678

5679-
static unsigned long capacity_spare_wake(int cpu, struct task_struct *p)
5679+
static unsigned long capacity_spare_without(int cpu, struct task_struct *p)
56805680
{
5681-
return max_t(long, capacity_of(cpu) - cpu_util_wake(cpu, p), 0);
5681+
return max_t(long, capacity_of(cpu) - cpu_util_without(cpu, p), 0);
56825682
}
56835683

56845684
/*
@@ -5738,7 +5738,7 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p,
57385738

57395739
avg_load += cfs_rq_load_avg(&cpu_rq(i)->cfs);
57405740

5741-
spare_cap = capacity_spare_wake(i, p);
5741+
spare_cap = capacity_spare_without(i, p);
57425742

57435743
if (spare_cap > max_spare_cap)
57445744
max_spare_cap = spare_cap;
@@ -5889,8 +5889,8 @@ static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p
58895889
return prev_cpu;
58905890

58915891
/*
5892-
* We need task's util for capacity_spare_wake, sync it up to prev_cpu's
5893-
* last_update_time.
5892+
* We need task's util for capacity_spare_without, sync it up to
5893+
* prev_cpu's last_update_time.
58945894
*/
58955895
if (!(sd_flag & SD_BALANCE_FORK))
58965896
sync_entity_load_avg(&p->se);
@@ -6216,10 +6216,19 @@ static inline unsigned long cpu_util(int cpu)
62166216
}
62176217

62186218
/*
6219-
* cpu_util_wake: Compute CPU utilization with any contributions from
6220-
* the waking task p removed.
6219+
* cpu_util_without: compute cpu utilization without any contributions from *p
6220+
* @cpu: the CPU which utilization is requested
6221+
* @p: the task which utilization should be discounted
6222+
*
6223+
* The utilization of a CPU is defined by the utilization of tasks currently
6224+
* enqueued on that CPU as well as tasks which are currently sleeping after an
6225+
* execution on that CPU.
6226+
*
6227+
* This method returns the utilization of the specified CPU by discounting the
6228+
* utilization of the specified task, whenever the task is currently
6229+
* contributing to the CPU utilization.
62216230
*/
6222-
static unsigned long cpu_util_wake(int cpu, struct task_struct *p)
6231+
static unsigned long cpu_util_without(int cpu, struct task_struct *p)
62236232
{
62246233
struct cfs_rq *cfs_rq;
62256234
unsigned int util;
@@ -6231,7 +6240,7 @@ static unsigned long cpu_util_wake(int cpu, struct task_struct *p)
62316240
cfs_rq = &cpu_rq(cpu)->cfs;
62326241
util = READ_ONCE(cfs_rq->avg.util_avg);
62336242

6234-
/* Discount task's blocked util from CPU's util */
6243+
/* Discount task's util from CPU's util */
62356244
util -= min_t(unsigned int, util, task_util(p));
62366245

62376246
/*
@@ -6240,14 +6249,14 @@ static unsigned long cpu_util_wake(int cpu, struct task_struct *p)
62406249
* a) if *p is the only task sleeping on this CPU, then:
62416250
* cpu_util (== task_util) > util_est (== 0)
62426251
* and thus we return:
6243-
* cpu_util_wake = (cpu_util - task_util) = 0
6252+
* cpu_util_without = (cpu_util - task_util) = 0
62446253
*
62456254
* b) if other tasks are SLEEPING on this CPU, which is now exiting
62466255
* IDLE, then:
62476256
* cpu_util >= task_util
62486257
* cpu_util > util_est (== 0)
62496258
* and thus we discount *p's blocked utilization to return:
6250-
* cpu_util_wake = (cpu_util - task_util) >= 0
6259+
* cpu_util_without = (cpu_util - task_util) >= 0
62516260
*
62526261
* c) if other tasks are RUNNABLE on that CPU and
62536262
* util_est > cpu_util
@@ -6260,8 +6269,33 @@ static unsigned long cpu_util_wake(int cpu, struct task_struct *p)
62606269
* covered by the following code when estimated utilization is
62616270
* enabled.
62626271
*/
6263-
if (sched_feat(UTIL_EST))
6264-
util = max(util, READ_ONCE(cfs_rq->avg.util_est.enqueued));
6272+
if (sched_feat(UTIL_EST)) {
6273+
unsigned int estimated =
6274+
READ_ONCE(cfs_rq->avg.util_est.enqueued);
6275+
6276+
/*
6277+
* Despite the following checks we still have a small window
6278+
* for a possible race, when an execl's select_task_rq_fair()
6279+
* races with LB's detach_task():
6280+
*
6281+
* detach_task()
6282+
* p->on_rq = TASK_ON_RQ_MIGRATING;
6283+
* ---------------------------------- A
6284+
* deactivate_task() \
6285+
* dequeue_task() + RaceTime
6286+
* util_est_dequeue() /
6287+
* ---------------------------------- B
6288+
*
6289+
* The additional check on "current == p" it's required to
6290+
* properly fix the execl regression and it helps in further
6291+
* reducing the chances for the above race.
6292+
*/
6293+
if (unlikely(task_on_rq_queued(p) || current == p)) {
6294+
estimated -= min_t(unsigned int, estimated,
6295+
(_task_util_est(p) | UTIL_AVG_UNCHANGED));
6296+
}
6297+
util = max(util, estimated);
6298+
}
62656299

62666300
/*
62676301
* Utilization (estimated) can exceed the CPU capacity, thus let's

0 commit comments

Comments
 (0)