Skip to content

Commit 69111ba

Browse files
Christoph Lametermpe
Christoph Lameter
authored and
mpe
committed
powerpc: Replace __get_cpu_var uses
This still has not been merged and now powerpc is the only arch that does not have this change. Sorry about missing linuxppc-dev before. V2->V2 - Fix up to work against 3.18-rc1 __get_cpu_var() is used for multiple purposes in the kernel source. One of them is address calculation via the form &__get_cpu_var(x). This calculates the address for the instance of the percpu variable of the current processor based on an offset. Other use cases are for storing and retrieving data from the current processors percpu area. __get_cpu_var() can be used as an lvalue when writing data or on the right side of an assignment. __get_cpu_var() is defined as : __get_cpu_var() always only does an address determination. However, store and retrieve operations could use a segment prefix (or global register on other platforms) to avoid the address calculation. this_cpu_write() and this_cpu_read() can directly take an offset into a percpu area and use optimized assembly code to read and write per cpu variables. This patch converts __get_cpu_var into either an explicit address calculation using this_cpu_ptr() or into a use of this_cpu operations that use the offset. Thereby address calculations are avoided and less registers are used when code is generated. At the end of the patch set all uses of __get_cpu_var have been removed so the macro is removed too. The patch set includes passes over all arches as well. Once these operations are used throughout then specialized macros can be defined in non -x86 arches as well in order to optimize per cpu access by f.e. using a global register that may be set to the per cpu base. Transformations done to __get_cpu_var() 1. Determine the address of the percpu instance of the current processor. DEFINE_PER_CPU(int, y); int *x = &__get_cpu_var(y); Converts to int *x = this_cpu_ptr(&y); 2. Same as #1 but this time an array structure is involved. DEFINE_PER_CPU(int, y[20]); int *x = __get_cpu_var(y); Converts to int *x = this_cpu_ptr(y); 3. Retrieve the content of the current processors instance of a per cpu variable. DEFINE_PER_CPU(int, y); int x = __get_cpu_var(y) Converts to int x = __this_cpu_read(y); 4. Retrieve the content of a percpu struct DEFINE_PER_CPU(struct mystruct, y); struct mystruct x = __get_cpu_var(y); Converts to memcpy(&x, this_cpu_ptr(&y), sizeof(x)); 5. Assignment to a per cpu variable DEFINE_PER_CPU(int, y) __get_cpu_var(y) = x; Converts to __this_cpu_write(y, x); 6. Increment/Decrement etc of a per cpu variable DEFINE_PER_CPU(int, y); __get_cpu_var(y)++ Converts to __this_cpu_inc(y) Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> CC: Paul Mackerras <paulus@samba.org> Signed-off-by: Christoph Lameter <cl@linux.com> [mpe: Fix build errors caused by set/or_softirq_pending(), and rework assignment in __set_breakpoint() to use memcpy().] Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
1 parent 0df1f24 commit 69111ba

32 files changed

+108
-103
lines changed

‎arch/powerpc/include/asm/hardirq.h

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,12 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
2121

2222
#define __ARCH_IRQ_STAT
2323

24-
#define local_softirq_pending() __get_cpu_var(irq_stat).__softirq_pending
24+
#define local_softirq_pending() __this_cpu_read(irq_stat.__softirq_pending)
25+
26+
#define __ARCH_SET_SOFTIRQ_PENDING
27+
28+
#define set_softirq_pending(x) __this_cpu_write(irq_stat.__softirq_pending, (x))
29+
#define or_softirq_pending(x) __this_cpu_or(irq_stat.__softirq_pending, (x))
2530

2631
static inline void ack_bad_irq(unsigned int irq)
2732
{

‎arch/powerpc/include/asm/tlbflush.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,14 +107,14 @@ extern void __flush_tlb_pending(struct ppc64_tlb_batch *batch);
107107

108108
static inline void arch_enter_lazy_mmu_mode(void)
109109
{
110-
struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
110+
struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch);
111111

112112
batch->active = 1;
113113
}
114114

115115
static inline void arch_leave_lazy_mmu_mode(void)
116116
{
117-
struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
117+
struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch);
118118

119119
if (batch->index)
120120
__flush_tlb_pending(batch);

‎arch/powerpc/include/asm/xics.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ DECLARE_PER_CPU(struct xics_cppr, xics_cppr);
9898

9999
static inline void xics_push_cppr(unsigned int vec)
100100
{
101-
struct xics_cppr *os_cppr = &__get_cpu_var(xics_cppr);
101+
struct xics_cppr *os_cppr = this_cpu_ptr(&xics_cppr);
102102

103103
if (WARN_ON(os_cppr->index >= MAX_NUM_PRIORITIES - 1))
104104
return;
@@ -111,7 +111,7 @@ static inline void xics_push_cppr(unsigned int vec)
111111

112112
static inline unsigned char xics_pop_cppr(void)
113113
{
114-
struct xics_cppr *os_cppr = &__get_cpu_var(xics_cppr);
114+
struct xics_cppr *os_cppr = this_cpu_ptr(&xics_cppr);
115115

116116
if (WARN_ON(os_cppr->index < 1))
117117
return LOWEST_PRIORITY;
@@ -121,7 +121,7 @@ static inline unsigned char xics_pop_cppr(void)
121121

122122
static inline void xics_set_base_cppr(unsigned char cppr)
123123
{
124-
struct xics_cppr *os_cppr = &__get_cpu_var(xics_cppr);
124+
struct xics_cppr *os_cppr = this_cpu_ptr(&xics_cppr);
125125

126126
/* we only really want to set the priority when there's
127127
* just one cppr value on the stack
@@ -133,7 +133,7 @@ static inline void xics_set_base_cppr(unsigned char cppr)
133133

134134
static inline unsigned char xics_cppr_top(void)
135135
{
136-
struct xics_cppr *os_cppr = &__get_cpu_var(xics_cppr);
136+
struct xics_cppr *os_cppr = this_cpu_ptr(&xics_cppr);
137137

138138
return os_cppr->stack[os_cppr->index];
139139
}

‎arch/powerpc/kernel/dbell.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ void doorbell_exception(struct pt_regs *regs)
4141

4242
may_hard_irq_enable();
4343

44-
__get_cpu_var(irq_stat).doorbell_irqs++;
44+
__this_cpu_inc(irq_stat.doorbell_irqs);
4545

4646
smp_ipi_demux();
4747

‎arch/powerpc/kernel/hw_breakpoint.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ int hw_breakpoint_slots(int type)
6363
int arch_install_hw_breakpoint(struct perf_event *bp)
6464
{
6565
struct arch_hw_breakpoint *info = counter_arch_bp(bp);
66-
struct perf_event **slot = &__get_cpu_var(bp_per_reg);
66+
struct perf_event **slot = this_cpu_ptr(&bp_per_reg);
6767

6868
*slot = bp;
6969

@@ -88,7 +88,7 @@ int arch_install_hw_breakpoint(struct perf_event *bp)
8888
*/
8989
void arch_uninstall_hw_breakpoint(struct perf_event *bp)
9090
{
91-
struct perf_event **slot = &__get_cpu_var(bp_per_reg);
91+
struct perf_event **slot = this_cpu_ptr(&bp_per_reg);
9292

9393
if (*slot != bp) {
9494
WARN_ONCE(1, "Can't find the breakpoint");
@@ -226,7 +226,7 @@ int __kprobes hw_breakpoint_handler(struct die_args *args)
226226
*/
227227
rcu_read_lock();
228228

229-
bp = __get_cpu_var(bp_per_reg);
229+
bp = __this_cpu_read(bp_per_reg);
230230
if (!bp)
231231
goto out;
232232
info = counter_arch_bp(bp);

‎arch/powerpc/kernel/iommu.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,7 @@ static unsigned long iommu_range_alloc(struct device *dev,
208208
* We don't need to disable preemption here because any CPU can
209209
* safely use any IOMMU pool.
210210
*/
211-
pool_nr = __raw_get_cpu_var(iommu_pool_hash) & (tbl->nr_pools - 1);
211+
pool_nr = __this_cpu_read(iommu_pool_hash) & (tbl->nr_pools - 1);
212212

213213
if (largealloc)
214214
pool = &(tbl->large_pool);

‎arch/powerpc/kernel/irq.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ static inline notrace void set_soft_enabled(unsigned long enable)
114114
static inline notrace int decrementer_check_overflow(void)
115115
{
116116
u64 now = get_tb_or_rtc();
117-
u64 *next_tb = &__get_cpu_var(decrementers_next_tb);
117+
u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
118118

119119
return now >= *next_tb;
120120
}
@@ -499,7 +499,7 @@ void __do_irq(struct pt_regs *regs)
499499

500500
/* And finally process it */
501501
if (unlikely(irq == NO_IRQ))
502-
__get_cpu_var(irq_stat).spurious_irqs++;
502+
__this_cpu_inc(irq_stat.spurious_irqs);
503503
else
504504
generic_handle_irq(irq);
505505

‎arch/powerpc/kernel/kgdb.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ static int kgdb_singlestep(struct pt_regs *regs)
155155
{
156156
struct thread_info *thread_info, *exception_thread_info;
157157
struct thread_info *backup_current_thread_info =
158-
&__get_cpu_var(kgdb_thread_info);
158+
this_cpu_ptr(&kgdb_thread_info);
159159

160160
if (user_mode(regs))
161161
return 0;

‎arch/powerpc/kernel/kprobes.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -119,15 +119,15 @@ static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
119119

120120
static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
121121
{
122-
__get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp;
122+
__this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
123123
kcb->kprobe_status = kcb->prev_kprobe.status;
124124
kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr;
125125
}
126126

127127
static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
128128
struct kprobe_ctlblk *kcb)
129129
{
130-
__get_cpu_var(current_kprobe) = p;
130+
__this_cpu_write(current_kprobe, p);
131131
kcb->kprobe_saved_msr = regs->msr;
132132
}
133133

@@ -192,7 +192,7 @@ static int __kprobes kprobe_handler(struct pt_regs *regs)
192192
ret = 1;
193193
goto no_kprobe;
194194
}
195-
p = __get_cpu_var(current_kprobe);
195+
p = __this_cpu_read(current_kprobe);
196196
if (p->break_handler && p->break_handler(p, regs)) {
197197
goto ss_probe;
198198
}

‎arch/powerpc/kernel/mce.c

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,8 @@ void save_mce_event(struct pt_regs *regs, long handled,
7373
uint64_t nip, uint64_t addr)
7474
{
7575
uint64_t srr1;
76-
int index = __get_cpu_var(mce_nest_count)++;
77-
struct machine_check_event *mce = &__get_cpu_var(mce_event[index]);
76+
int index = __this_cpu_inc_return(mce_nest_count);
77+
struct machine_check_event *mce = this_cpu_ptr(&mce_event[index]);
7878

7979
/*
8080
* Return if we don't have enough space to log mce event.
@@ -143,7 +143,7 @@ void save_mce_event(struct pt_regs *regs, long handled,
143143
*/
144144
int get_mce_event(struct machine_check_event *mce, bool release)
145145
{
146-
int index = __get_cpu_var(mce_nest_count) - 1;
146+
int index = __this_cpu_read(mce_nest_count) - 1;
147147
struct machine_check_event *mc_evt;
148148
int ret = 0;
149149

@@ -153,7 +153,7 @@ int get_mce_event(struct machine_check_event *mce, bool release)
153153

154154
/* Check if we have MCE info to process. */
155155
if (index < MAX_MC_EVT) {
156-
mc_evt = &__get_cpu_var(mce_event[index]);
156+
mc_evt = this_cpu_ptr(&mce_event[index]);
157157
/* Copy the event structure and release the original */
158158
if (mce)
159159
*mce = *mc_evt;
@@ -163,7 +163,7 @@ int get_mce_event(struct machine_check_event *mce, bool release)
163163
}
164164
/* Decrement the count to free the slot. */
165165
if (release)
166-
__get_cpu_var(mce_nest_count)--;
166+
__this_cpu_dec(mce_nest_count);
167167

168168
return ret;
169169
}
@@ -184,13 +184,13 @@ void machine_check_queue_event(void)
184184
if (!get_mce_event(&evt, MCE_EVENT_RELEASE))
185185
return;
186186

187-
index = __get_cpu_var(mce_queue_count)++;
187+
index = __this_cpu_inc_return(mce_queue_count);
188188
/* If queue is full, just return for now. */
189189
if (index >= MAX_MC_EVT) {
190-
__get_cpu_var(mce_queue_count)--;
190+
__this_cpu_dec(mce_queue_count);
191191
return;
192192
}
193-
__get_cpu_var(mce_event_queue[index]) = evt;
193+
memcpy(this_cpu_ptr(&mce_event_queue[index]), &evt, sizeof(evt));
194194

195195
/* Queue irq work to process this event later. */
196196
irq_work_queue(&mce_event_process_work);
@@ -208,11 +208,11 @@ static void machine_check_process_queued_event(struct irq_work *work)
208208
* For now just print it to console.
209209
* TODO: log this error event to FSP or nvram.
210210
*/
211-
while (__get_cpu_var(mce_queue_count) > 0) {
212-
index = __get_cpu_var(mce_queue_count) - 1;
211+
while (__this_cpu_read(mce_queue_count) > 0) {
212+
index = __this_cpu_read(mce_queue_count) - 1;
213213
machine_check_print_event_info(
214-
&__get_cpu_var(mce_event_queue[index]));
215-
__get_cpu_var(mce_queue_count)--;
214+
this_cpu_ptr(&mce_event_queue[index]));
215+
__this_cpu_dec(mce_queue_count);
216216
}
217217
}
218218

‎arch/powerpc/kernel/process.c

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -499,7 +499,7 @@ static inline int set_dawr(struct arch_hw_breakpoint *brk)
499499

500500
void __set_breakpoint(struct arch_hw_breakpoint *brk)
501501
{
502-
__get_cpu_var(current_brk) = *brk;
502+
memcpy(this_cpu_ptr(&current_brk), brk, sizeof(*brk));
503503

504504
if (cpu_has_feature(CPU_FTR_DAWR))
505505
set_dawr(brk);
@@ -842,7 +842,7 @@ struct task_struct *__switch_to(struct task_struct *prev,
842842
* schedule DABR
843843
*/
844844
#ifndef CONFIG_HAVE_HW_BREAKPOINT
845-
if (unlikely(!hw_brk_match(&__get_cpu_var(current_brk), &new->thread.hw_brk)))
845+
if (unlikely(!hw_brk_match(this_cpu_ptr(&current_brk), &new->thread.hw_brk)))
846846
__set_breakpoint(&new->thread.hw_brk);
847847
#endif /* CONFIG_HAVE_HW_BREAKPOINT */
848848
#endif
@@ -856,7 +856,7 @@ struct task_struct *__switch_to(struct task_struct *prev,
856856
* Collect processor utilization data per process
857857
*/
858858
if (firmware_has_feature(FW_FEATURE_SPLPAR)) {
859-
struct cpu_usage *cu = &__get_cpu_var(cpu_usage_array);
859+
struct cpu_usage *cu = this_cpu_ptr(&cpu_usage_array);
860860
long unsigned start_tb, current_tb;
861861
start_tb = old_thread->start_tb;
862862
cu->current_tb = current_tb = mfspr(SPRN_PURR);
@@ -866,7 +866,7 @@ struct task_struct *__switch_to(struct task_struct *prev,
866866
#endif /* CONFIG_PPC64 */
867867

868868
#ifdef CONFIG_PPC_BOOK3S_64
869-
batch = &__get_cpu_var(ppc64_tlb_batch);
869+
batch = this_cpu_ptr(&ppc64_tlb_batch);
870870
if (batch->active) {
871871
current_thread_info()->local_flags |= _TLF_LAZY_MMU;
872872
if (batch->index)
@@ -889,7 +889,7 @@ struct task_struct *__switch_to(struct task_struct *prev,
889889
#ifdef CONFIG_PPC_BOOK3S_64
890890
if (current_thread_info()->local_flags & _TLF_LAZY_MMU) {
891891
current_thread_info()->local_flags &= ~_TLF_LAZY_MMU;
892-
batch = &__get_cpu_var(ppc64_tlb_batch);
892+
batch = this_cpu_ptr(&ppc64_tlb_batch);
893893
batch->active = 1;
894894
}
895895
#endif /* CONFIG_PPC_BOOK3S_64 */

‎arch/powerpc/kernel/smp.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ void smp_muxed_ipi_message_pass(int cpu, int msg)
243243

244244
irqreturn_t smp_ipi_demux(void)
245245
{
246-
struct cpu_messages *info = &__get_cpu_var(ipi_message);
246+
struct cpu_messages *info = this_cpu_ptr(&ipi_message);
247247
unsigned int all;
248248

249249
mb(); /* order any irq clear */
@@ -442,9 +442,9 @@ void generic_mach_cpu_die(void)
442442
idle_task_exit();
443443
cpu = smp_processor_id();
444444
printk(KERN_DEBUG "CPU%d offline\n", cpu);
445-
__get_cpu_var(cpu_state) = CPU_DEAD;
445+
__this_cpu_write(cpu_state, CPU_DEAD);
446446
smp_wmb();
447-
while (__get_cpu_var(cpu_state) != CPU_UP_PREPARE)
447+
while (__this_cpu_read(cpu_state) != CPU_UP_PREPARE)
448448
cpu_relax();
449449
}
450450

‎arch/powerpc/kernel/sysfs.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -394,10 +394,10 @@ void ppc_enable_pmcs(void)
394394
ppc_set_pmu_inuse(1);
395395

396396
/* Only need to enable them once */
397-
if (__get_cpu_var(pmcs_enabled))
397+
if (__this_cpu_read(pmcs_enabled))
398398
return;
399399

400-
__get_cpu_var(pmcs_enabled) = 1;
400+
__this_cpu_write(pmcs_enabled, 1);
401401

402402
if (ppc_md.enable_pmcs)
403403
ppc_md.enable_pmcs();

‎arch/powerpc/kernel/time.c

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -458,9 +458,9 @@ static inline void clear_irq_work_pending(void)
458458

459459
DEFINE_PER_CPU(u8, irq_work_pending);
460460

461-
#define set_irq_work_pending_flag() __get_cpu_var(irq_work_pending) = 1
462-
#define test_irq_work_pending() __get_cpu_var(irq_work_pending)
463-
#define clear_irq_work_pending() __get_cpu_var(irq_work_pending) = 0
461+
#define set_irq_work_pending_flag() __this_cpu_write(irq_work_pending, 1)
462+
#define test_irq_work_pending() __this_cpu_read(irq_work_pending)
463+
#define clear_irq_work_pending() __this_cpu_write(irq_work_pending, 0)
464464

465465
#endif /* 32 vs 64 bit */
466466

@@ -482,8 +482,8 @@ void arch_irq_work_raise(void)
482482
static void __timer_interrupt(void)
483483
{
484484
struct pt_regs *regs = get_irq_regs();
485-
u64 *next_tb = &__get_cpu_var(decrementers_next_tb);
486-
struct clock_event_device *evt = &__get_cpu_var(decrementers);
485+
u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
486+
struct clock_event_device *evt = this_cpu_ptr(&decrementers);
487487
u64 now;
488488

489489
trace_timer_interrupt_entry(regs);
@@ -498,21 +498,21 @@ static void __timer_interrupt(void)
498498
*next_tb = ~(u64)0;
499499
if (evt->event_handler)
500500
evt->event_handler(evt);
501-
__get_cpu_var(irq_stat).timer_irqs_event++;
501+
__this_cpu_inc(irq_stat.timer_irqs_event);
502502
} else {
503503
now = *next_tb - now;
504504
if (now <= DECREMENTER_MAX)
505505
set_dec((int)now);
506506
/* We may have raced with new irq work */
507507
if (test_irq_work_pending())
508508
set_dec(1);
509-
__get_cpu_var(irq_stat).timer_irqs_others++;
509+
__this_cpu_inc(irq_stat.timer_irqs_others);
510510
}
511511

512512
#ifdef CONFIG_PPC64
513513
/* collect purr register values often, for accurate calculations */
514514
if (firmware_has_feature(FW_FEATURE_SPLPAR)) {
515-
struct cpu_usage *cu = &__get_cpu_var(cpu_usage_array);
515+
struct cpu_usage *cu = this_cpu_ptr(&cpu_usage_array);
516516
cu->current_tb = mfspr(SPRN_PURR);
517517
}
518518
#endif
@@ -527,7 +527,7 @@ static void __timer_interrupt(void)
527527
void timer_interrupt(struct pt_regs * regs)
528528
{
529529
struct pt_regs *old_regs;
530-
u64 *next_tb = &__get_cpu_var(decrementers_next_tb);
530+
u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
531531

532532
/* Ensure a positive value is written to the decrementer, or else
533533
* some CPUs will continue to take decrementer exceptions.
@@ -813,7 +813,7 @@ static void __init clocksource_init(void)
813813
static int decrementer_set_next_event(unsigned long evt,
814814
struct clock_event_device *dev)
815815
{
816-
__get_cpu_var(decrementers_next_tb) = get_tb_or_rtc() + evt;
816+
__this_cpu_write(decrementers_next_tb, get_tb_or_rtc() + evt);
817817
set_dec(evt);
818818

819819
/* We may have raced with new irq work */
@@ -833,7 +833,7 @@ static void decrementer_set_mode(enum clock_event_mode mode,
833833
/* Interrupt handler for the timer broadcast IPI */
834834
void tick_broadcast_ipi_handler(void)
835835
{
836-
u64 *next_tb = &__get_cpu_var(decrementers_next_tb);
836+
u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
837837

838838
*next_tb = get_tb_or_rtc();
839839
__timer_interrupt();

0 commit comments

Comments
 (0)