Skip to content

Commit 7110744

Browse files
Jason LowIngo Molnar
Jason Low
authored and
Ingo Molnar
committed
sched, timer: Use the atomic task_cputime in thread_group_cputimer
Recent optimizations were made to thread_group_cputimer to improve its scalability by keeping track of cputime stats without a lock. However, the values were open coded to the structure, causing them to be at a different abstraction level from the regular task_cputime structure. Furthermore, any subsequent similar optimizations would not be able to share the new code, since they are specific to thread_group_cputimer. This patch adds the new task_cputime_atomic data structure (introduced in the previous patch in the series) to thread_group_cputimer for keeping track of the cputime atomically, which also helps generalize the code. Suggested-by: Ingo Molnar <mingo@kernel.org> Signed-off-by: Jason Low <jason.low2@hp.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Acked-by: Thomas Gleixner <tglx@linutronix.de> Acked-by: Rik van Riel <riel@redhat.com> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Aswin Chandramouleeswaran <aswin@hp.com> Cc: Borislav Petkov <bp@alien8.de> Cc: Davidlohr Bueso <dave@stgolabs.net> Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Mel Gorman <mgorman@suse.de> Cc: Mike Galbraith <umgwanakikbuti@gmail.com> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Preeti U Murthy <preeti@linux.vnet.ibm.com> Cc: Scott J Norton <scott.norton@hp.com> Cc: Steven Rostedt <rostedt@goodmis.org> Cc: Waiman Long <Waiman.Long@hp.com> Link: http://lkml.kernel.org/r/1430251224-5764-6-git-send-email-jason.low2@hp.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
1 parent 971e8a9 commit 7110744

File tree

4 files changed

+19
-23
lines changed

4 files changed

+19
-23
lines changed

‎include/linux/init_task.h

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,8 @@ extern struct fs_struct init_fs;
5050
.cpu_timers = INIT_CPU_TIMERS(sig.cpu_timers), \
5151
.rlim = INIT_RLIMITS, \
5252
.cputimer = { \
53-
.utime = ATOMIC64_INIT(0), \
54-
.stime = ATOMIC64_INIT(0), \
55-
.sum_exec_runtime = ATOMIC64_INIT(0), \
56-
.running = 0 \
53+
.cputime_atomic = INIT_CPUTIME_ATOMIC, \
54+
.running = 0, \
5755
}, \
5856
.cred_guard_mutex = \
5957
__MUTEX_INITIALIZER(sig.cred_guard_mutex), \

‎include/linux/sched.h

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -615,9 +615,7 @@ struct task_cputime_atomic {
615615
* used for thread group CPU timer calculations.
616616
*/
617617
struct thread_group_cputimer {
618-
atomic64_t utime;
619-
atomic64_t stime;
620-
atomic64_t sum_exec_runtime;
618+
struct task_cputime_atomic cputime_atomic;
621619
int running;
622620
};
623621

‎kernel/sched/stats.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,7 @@ static inline void account_group_user_time(struct task_struct *tsk,
216216
if (!cputimer_running(tsk))
217217
return;
218218

219-
atomic64_add(cputime, &cputimer->utime);
219+
atomic64_add(cputime, &cputimer->cputime_atomic.utime);
220220
}
221221

222222
/**
@@ -237,7 +237,7 @@ static inline void account_group_system_time(struct task_struct *tsk,
237237
if (!cputimer_running(tsk))
238238
return;
239239

240-
atomic64_add(cputime, &cputimer->stime);
240+
atomic64_add(cputime, &cputimer->cputime_atomic.stime);
241241
}
242242

243243
/**
@@ -258,5 +258,5 @@ static inline void account_group_exec_runtime(struct task_struct *tsk,
258258
if (!cputimer_running(tsk))
259259
return;
260260

261-
atomic64_add(ns, &cputimer->sum_exec_runtime);
261+
atomic64_add(ns, &cputimer->cputime_atomic.sum_exec_runtime);
262262
}

‎kernel/time/posix-cpu-timers.c

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -211,20 +211,20 @@ static inline void __update_gt_cputime(atomic64_t *cputime, u64 sum_cputime)
211211
}
212212
}
213213

214-
static void update_gt_cputime(struct thread_group_cputimer *cputimer, struct task_cputime *sum)
214+
static void update_gt_cputime(struct task_cputime_atomic *cputime_atomic, struct task_cputime *sum)
215215
{
216-
__update_gt_cputime(&cputimer->utime, sum->utime);
217-
__update_gt_cputime(&cputimer->stime, sum->stime);
218-
__update_gt_cputime(&cputimer->sum_exec_runtime, sum->sum_exec_runtime);
216+
__update_gt_cputime(&cputime_atomic->utime, sum->utime);
217+
__update_gt_cputime(&cputime_atomic->stime, sum->stime);
218+
__update_gt_cputime(&cputime_atomic->sum_exec_runtime, sum->sum_exec_runtime);
219219
}
220220

221-
/* Sample thread_group_cputimer values in "cputimer", store results in "times". */
222-
static inline void sample_group_cputimer(struct task_cputime *times,
223-
struct thread_group_cputimer *cputimer)
221+
/* Sample task_cputime_atomic values in "atomic_timers", store results in "times". */
222+
static inline void sample_cputime_atomic(struct task_cputime *times,
223+
struct task_cputime_atomic *atomic_times)
224224
{
225-
times->utime = atomic64_read(&cputimer->utime);
226-
times->stime = atomic64_read(&cputimer->stime);
227-
times->sum_exec_runtime = atomic64_read(&cputimer->sum_exec_runtime);
225+
times->utime = atomic64_read(&atomic_times->utime);
226+
times->stime = atomic64_read(&atomic_times->stime);
227+
times->sum_exec_runtime = atomic64_read(&atomic_times->sum_exec_runtime);
228228
}
229229

230230
void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times)
@@ -240,7 +240,7 @@ void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times)
240240
* to synchronize the timer to the clock every time we start it.
241241
*/
242242
thread_group_cputime(tsk, &sum);
243-
update_gt_cputime(cputimer, &sum);
243+
update_gt_cputime(&cputimer->cputime_atomic, &sum);
244244

245245
/*
246246
* We're setting cputimer->running without a lock. Ensure
@@ -251,7 +251,7 @@ void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times)
251251
*/
252252
WRITE_ONCE(cputimer->running, 1);
253253
}
254-
sample_group_cputimer(times, cputimer);
254+
sample_cputime_atomic(times, &cputimer->cputime_atomic);
255255
}
256256

257257
/*
@@ -1137,7 +1137,7 @@ static inline int fastpath_timer_check(struct task_struct *tsk)
11371137
if (READ_ONCE(sig->cputimer.running)) {
11381138
struct task_cputime group_sample;
11391139

1140-
sample_group_cputimer(&group_sample, &sig->cputimer);
1140+
sample_cputime_atomic(&group_sample, &sig->cputimer.cputime_atomic);
11411141

11421142
if (task_cputime_expired(&group_sample, &sig->cputime_expires))
11431143
return 1;

0 commit comments

Comments
 (0)