Skip to content

Commit 316c160

Browse files
Jason LowIngo Molnar
Jason Low
authored and
Ingo Molnar
committed
sched, timer: Convert usages of ACCESS_ONCE() in the scheduler to READ_ONCE()/WRITE_ONCE()
ACCESS_ONCE doesn't work reliably on non-scalar types. This patch removes the rest of the existing usages of ACCESS_ONCE() in the scheduler, and use the new READ_ONCE() and WRITE_ONCE() APIs as appropriate. Signed-off-by: Jason Low <jason.low2@hp.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Acked-by: Thomas Gleixner <tglx@linutronix.de> Acked-by: Rik van Riel <riel@redhat.com> Acked-by: Waiman Long <Waiman.Long@hp.com> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Aswin Chandramouleeswaran <aswin@hp.com> Cc: Borislav Petkov <bp@alien8.de> Cc: Davidlohr Bueso <dave@stgolabs.net> Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Mel Gorman <mgorman@suse.de> Cc: Mike Galbraith <umgwanakikbuti@gmail.com> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Preeti U Murthy <preeti@linux.vnet.ibm.com> Cc: Scott J Norton <scott.norton@hp.com> Cc: Steven Rostedt <rostedt@goodmis.org> Link: http://lkml.kernel.org/r/1430251224-5764-2-git-send-email-jason.low2@hp.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
1 parent ce2f5fe commit 316c160

File tree

12 files changed

+26
-26
lines changed

12 files changed

+26
-26
lines changed

‎include/linux/sched.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3085,13 +3085,13 @@ static inline void mm_update_next_owner(struct mm_struct *mm)
30853085
static inline unsigned long task_rlimit(const struct task_struct *tsk,
30863086
unsigned int limit)
30873087
{
3088-
return ACCESS_ONCE(tsk->signal->rlim[limit].rlim_cur);
3088+
return READ_ONCE(tsk->signal->rlim[limit].rlim_cur);
30893089
}
30903090

30913091
static inline unsigned long task_rlimit_max(const struct task_struct *tsk,
30923092
unsigned int limit)
30933093
{
3094-
return ACCESS_ONCE(tsk->signal->rlim[limit].rlim_max);
3094+
return READ_ONCE(tsk->signal->rlim[limit].rlim_max);
30953095
}
30963096

30973097
static inline unsigned long rlimit(unsigned int limit)

‎kernel/fork.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1094,7 +1094,7 @@ static void posix_cpu_timers_init_group(struct signal_struct *sig)
10941094
/* Thread group counters. */
10951095
thread_group_cputime_init(sig);
10961096

1097-
cpu_limit = ACCESS_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
1097+
cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
10981098
if (cpu_limit != RLIM_INFINITY) {
10991099
sig->cputime_expires.prof_exp = secs_to_cputime(cpu_limit);
11001100
sig->cputimer.running = 1;

‎kernel/sched/auto_group.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ autogroup_move_group(struct task_struct *p, struct autogroup *ag)
139139

140140
p->signal->autogroup = autogroup_kref_get(ag);
141141

142-
if (!ACCESS_ONCE(sysctl_sched_autogroup_enabled))
142+
if (!READ_ONCE(sysctl_sched_autogroup_enabled))
143143
goto out;
144144

145145
for_each_thread(p, t)

‎kernel/sched/auto_group.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ extern bool task_wants_autogroup(struct task_struct *p, struct task_group *tg);
2929
static inline struct task_group *
3030
autogroup_task_group(struct task_struct *p, struct task_group *tg)
3131
{
32-
int enabled = ACCESS_ONCE(sysctl_sched_autogroup_enabled);
32+
int enabled = READ_ONCE(sysctl_sched_autogroup_enabled);
3333

3434
if (enabled && task_wants_autogroup(p, tg))
3535
return p->signal->autogroup->tg;

‎kernel/sched/core.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -511,7 +511,7 @@ static bool set_nr_and_not_polling(struct task_struct *p)
511511
static bool set_nr_if_polling(struct task_struct *p)
512512
{
513513
struct thread_info *ti = task_thread_info(p);
514-
typeof(ti->flags) old, val = ACCESS_ONCE(ti->flags);
514+
typeof(ti->flags) old, val = READ_ONCE(ti->flags);
515515

516516
for (;;) {
517517
if (!(val & _TIF_POLLING_NRFLAG))
@@ -2526,7 +2526,7 @@ void scheduler_tick(void)
25262526
u64 scheduler_tick_max_deferment(void)
25272527
{
25282528
struct rq *rq = this_rq();
2529-
unsigned long next, now = ACCESS_ONCE(jiffies);
2529+
unsigned long next, now = READ_ONCE(jiffies);
25302530

25312531
next = rq->last_sched_tick + HZ;
25322532

‎kernel/sched/cputime.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -567,7 +567,7 @@ static void cputime_advance(cputime_t *counter, cputime_t new)
567567
{
568568
cputime_t old;
569569

570-
while (new > (old = ACCESS_ONCE(*counter)))
570+
while (new > (old = READ_ONCE(*counter)))
571571
cmpxchg_cputime(counter, old, new);
572572
}
573573

‎kernel/sched/deadline.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -995,7 +995,7 @@ select_task_rq_dl(struct task_struct *p, int cpu, int sd_flag, int flags)
995995
rq = cpu_rq(cpu);
996996

997997
rcu_read_lock();
998-
curr = ACCESS_ONCE(rq->curr); /* unlocked access */
998+
curr = READ_ONCE(rq->curr); /* unlocked access */
999999

10001000
/*
10011001
* If we are dealing with a -deadline task, we must

‎kernel/sched/fair.c

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -834,7 +834,7 @@ static unsigned int task_nr_scan_windows(struct task_struct *p)
834834

835835
static unsigned int task_scan_min(struct task_struct *p)
836836
{
837-
unsigned int scan_size = ACCESS_ONCE(sysctl_numa_balancing_scan_size);
837+
unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size);
838838
unsigned int scan, floor;
839839
unsigned int windows = 1;
840840

@@ -1794,7 +1794,7 @@ static void task_numa_placement(struct task_struct *p)
17941794
u64 runtime, period;
17951795
spinlock_t *group_lock = NULL;
17961796

1797-
seq = ACCESS_ONCE(p->mm->numa_scan_seq);
1797+
seq = READ_ONCE(p->mm->numa_scan_seq);
17981798
if (p->numa_scan_seq == seq)
17991799
return;
18001800
p->numa_scan_seq = seq;
@@ -1938,7 +1938,7 @@ static void task_numa_group(struct task_struct *p, int cpupid, int flags,
19381938
}
19391939

19401940
rcu_read_lock();
1941-
tsk = ACCESS_ONCE(cpu_rq(cpu)->curr);
1941+
tsk = READ_ONCE(cpu_rq(cpu)->curr);
19421942

19431943
if (!cpupid_match_pid(tsk, cpupid))
19441944
goto no_join;
@@ -2107,7 +2107,7 @@ void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags)
21072107

21082108
static void reset_ptenuma_scan(struct task_struct *p)
21092109
{
2110-
ACCESS_ONCE(p->mm->numa_scan_seq)++;
2110+
WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1);
21112111
p->mm->numa_scan_offset = 0;
21122112
}
21132113

@@ -4451,7 +4451,7 @@ static void __update_cpu_load(struct rq *this_rq, unsigned long this_load,
44514451
*/
44524452
static void update_idle_cpu_load(struct rq *this_rq)
44534453
{
4454-
unsigned long curr_jiffies = ACCESS_ONCE(jiffies);
4454+
unsigned long curr_jiffies = READ_ONCE(jiffies);
44554455
unsigned long load = this_rq->cfs.runnable_load_avg;
44564456
unsigned long pending_updates;
44574457

@@ -4473,7 +4473,7 @@ static void update_idle_cpu_load(struct rq *this_rq)
44734473
void update_cpu_load_nohz(void)
44744474
{
44754475
struct rq *this_rq = this_rq();
4476-
unsigned long curr_jiffies = ACCESS_ONCE(jiffies);
4476+
unsigned long curr_jiffies = READ_ONCE(jiffies);
44774477
unsigned long pending_updates;
44784478

44794479
if (curr_jiffies == this_rq->last_load_update_tick)
@@ -4558,7 +4558,7 @@ static unsigned long capacity_orig_of(int cpu)
45584558
static unsigned long cpu_avg_load_per_task(int cpu)
45594559
{
45604560
struct rq *rq = cpu_rq(cpu);
4561-
unsigned long nr_running = ACCESS_ONCE(rq->cfs.h_nr_running);
4561+
unsigned long nr_running = READ_ONCE(rq->cfs.h_nr_running);
45624562
unsigned long load_avg = rq->cfs.runnable_load_avg;
45634563

45644564
if (nr_running)
@@ -6220,8 +6220,8 @@ static unsigned long scale_rt_capacity(int cpu)
62206220
* Since we're reading these variables without serialization make sure
62216221
* we read them once before doing sanity checks on them.
62226222
*/
6223-
age_stamp = ACCESS_ONCE(rq->age_stamp);
6224-
avg = ACCESS_ONCE(rq->rt_avg);
6223+
age_stamp = READ_ONCE(rq->age_stamp);
6224+
avg = READ_ONCE(rq->rt_avg);
62256225
delta = __rq_clock_broken(rq) - age_stamp;
62266226

62276227
if (unlikely(delta < 0))

‎kernel/sched/rt.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1323,7 +1323,7 @@ select_task_rq_rt(struct task_struct *p, int cpu, int sd_flag, int flags)
13231323
rq = cpu_rq(cpu);
13241324

13251325
rcu_read_lock();
1326-
curr = ACCESS_ONCE(rq->curr); /* unlocked access */
1326+
curr = READ_ONCE(rq->curr); /* unlocked access */
13271327

13281328
/*
13291329
* If the current task on @p's runqueue is an RT task, then

‎kernel/sched/sched.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -713,7 +713,7 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
713713

714714
static inline u64 __rq_clock_broken(struct rq *rq)
715715
{
716-
return ACCESS_ONCE(rq->clock);
716+
return READ_ONCE(rq->clock);
717717
}
718718

719719
static inline u64 rq_clock(struct rq *rq)

‎kernel/sched/wait.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -601,7 +601,7 @@ EXPORT_SYMBOL(bit_wait_io);
601601

602602
__sched int bit_wait_timeout(struct wait_bit_key *word)
603603
{
604-
unsigned long now = ACCESS_ONCE(jiffies);
604+
unsigned long now = READ_ONCE(jiffies);
605605
if (signal_pending_state(current->state, current))
606606
return 1;
607607
if (time_after_eq(now, word->timeout))
@@ -613,7 +613,7 @@ EXPORT_SYMBOL_GPL(bit_wait_timeout);
613613

614614
__sched int bit_wait_io_timeout(struct wait_bit_key *word)
615615
{
616-
unsigned long now = ACCESS_ONCE(jiffies);
616+
unsigned long now = READ_ONCE(jiffies);
617617
if (signal_pending_state(current->state, current))
618618
return 1;
619619
if (time_after_eq(now, word->timeout))

‎kernel/time/posix-cpu-timers.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -852,10 +852,10 @@ static void check_thread_timers(struct task_struct *tsk,
852852
/*
853853
* Check for the special case thread timers.
854854
*/
855-
soft = ACCESS_ONCE(sig->rlim[RLIMIT_RTTIME].rlim_cur);
855+
soft = READ_ONCE(sig->rlim[RLIMIT_RTTIME].rlim_cur);
856856
if (soft != RLIM_INFINITY) {
857857
unsigned long hard =
858-
ACCESS_ONCE(sig->rlim[RLIMIT_RTTIME].rlim_max);
858+
READ_ONCE(sig->rlim[RLIMIT_RTTIME].rlim_max);
859859

860860
if (hard != RLIM_INFINITY &&
861861
tsk->rt.timeout > DIV_ROUND_UP(hard, USEC_PER_SEC/HZ)) {
@@ -958,11 +958,11 @@ static void check_process_timers(struct task_struct *tsk,
958958
SIGPROF);
959959
check_cpu_itimer(tsk, &sig->it[CPUCLOCK_VIRT], &virt_expires, utime,
960960
SIGVTALRM);
961-
soft = ACCESS_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
961+
soft = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
962962
if (soft != RLIM_INFINITY) {
963963
unsigned long psecs = cputime_to_secs(ptime);
964964
unsigned long hard =
965-
ACCESS_ONCE(sig->rlim[RLIMIT_CPU].rlim_max);
965+
READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_max);
966966
cputime_t x;
967967
if (psecs >= hard) {
968968
/*

0 commit comments

Comments
 (0)