Skip to content

Commit 7c3576d

Browse files
jsgfAndi Kleen
jsgf
authored and
Andi Kleen
committed
[PATCH] i386: Convert PDA into the percpu section
Currently x86 (similar to x84-64) has a special per-cpu structure called "i386_pda" which can be easily and efficiently referenced via the %fs register. An ELF section is more flexible than a structure, allowing any piece of code to use this area. Indeed, such a section already exists: the per-cpu area. So this patch: (1) Removes the PDA and uses per-cpu variables for each current member. (2) Replaces the __KERNEL_PDA segment with __KERNEL_PERCPU. (3) Creates a per-cpu mirror of __per_cpu_offset called this_cpu_off, which can be used to calculate addresses for this CPU's variables. (4) Simplifies startup, because %fs doesn't need to be loaded with a special segment at early boot; it can be deferred until the first percpu area is allocated (or never for UP). The result is less code and one less x86-specific concept. Signed-off-by: Rusty Russell <rusty@rustcorp.com.au> Signed-off-by: Jeremy Fitzhardinge <jeremy@xensource.com> Signed-off-by: Andi Kleen <ak@suse.de> Cc: Andi Kleen <ak@suse.de>
1 parent 7a61d35 commit 7c3576d

File tree

17 files changed

+177
-195
lines changed

17 files changed

+177
-195
lines changed

‎arch/i386/kernel/asm-offsets.c

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
#include <asm/processor.h>
1616
#include <asm/thread_info.h>
1717
#include <asm/elf.h>
18-
#include <asm/pda.h>
1918

2019
#define DEFINE(sym, val) \
2120
asm volatile("\n->" #sym " %0 " #val : : "i" (val))
@@ -101,10 +100,6 @@ void foo(void)
101100

102101
OFFSET(crypto_tfm_ctx_offset, crypto_tfm, __crt_ctx);
103102

104-
BLANK();
105-
OFFSET(PDA_cpu, i386_pda, cpu_number);
106-
OFFSET(PDA_pcurrent, i386_pda, pcurrent);
107-
108103
#ifdef CONFIG_PARAVIRT
109104
BLANK();
110105
OFFSET(PARAVIRT_enabled, paravirt_ops, paravirt_enabled);

‎arch/i386/kernel/cpu/common.c

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
#include <asm/apic.h>
1919
#include <mach_apic.h>
2020
#endif
21-
#include <asm/pda.h>
2221

2322
#include "cpu.h"
2423

@@ -47,13 +46,10 @@ DEFINE_PER_CPU(struct gdt_page, gdt_page) = { .gdt = {
4746
[GDT_ENTRY_APMBIOS_BASE+2] = { 0x0000ffff, 0x00409200 }, /* data */
4847

4948
[GDT_ENTRY_ESPFIX_SS] = { 0x00000000, 0x00c09200 },
50-
[GDT_ENTRY_PDA] = { 0x00000000, 0x00c09200 }, /* set in setup_pda */
49+
[GDT_ENTRY_PERCPU] = { 0x00000000, 0x00000000 },
5150
} };
5251
EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
5352

54-
DEFINE_PER_CPU(struct i386_pda, _cpu_pda);
55-
EXPORT_PER_CPU_SYMBOL(_cpu_pda);
56-
5753
static int cachesize_override __cpuinitdata = -1;
5854
static int disable_x86_fxsr __cpuinitdata;
5955
static int disable_x86_serial_nr __cpuinitdata = 1;
@@ -634,21 +630,14 @@ void __init early_cpu_init(void)
634630
#endif
635631
}
636632

637-
/* Make sure %gs is initialized properly in idle threads */
633+
/* Make sure %fs is initialized properly in idle threads */
638634
struct pt_regs * __devinit idle_regs(struct pt_regs *regs)
639635
{
640636
memset(regs, 0, sizeof(struct pt_regs));
641-
regs->xfs = __KERNEL_PDA;
637+
regs->xfs = __KERNEL_PERCPU;
642638
return regs;
643639
}
644640

645-
/* Initial PDA used by boot CPU */
646-
struct i386_pda boot_pda = {
647-
._pda = &boot_pda,
648-
.cpu_number = 0,
649-
.pcurrent = &init_task,
650-
};
651-
652641
/*
653642
* cpu_init() initializes state that is per-CPU. Some data is already
654643
* initialized (naturally) in the bootstrap process, such as the GDT

‎arch/i386/kernel/entry.S

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ VM_MASK = 0x00020000
132132
movl $(__USER_DS), %edx; \
133133
movl %edx, %ds; \
134134
movl %edx, %es; \
135-
movl $(__KERNEL_PDA), %edx; \
135+
movl $(__KERNEL_PERCPU), %edx; \
136136
movl %edx, %fs
137137

138138
#define RESTORE_INT_REGS \
@@ -556,7 +556,6 @@ END(syscall_badsys)
556556

557557
#define FIXUP_ESPFIX_STACK \
558558
/* since we are on a wrong stack, we cant make it a C code :( */ \
559-
movl %fs:PDA_cpu, %ebx; \
560559
PER_CPU(gdt_page, %ebx); \
561560
GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \
562561
addl %esp, %eax; \
@@ -681,7 +680,7 @@ error_code:
681680
pushl %fs
682681
CFI_ADJUST_CFA_OFFSET 4
683682
/*CFI_REL_OFFSET fs, 0*/
684-
movl $(__KERNEL_PDA), %ecx
683+
movl $(__KERNEL_PERCPU), %ecx
685684
movl %ecx, %fs
686685
UNWIND_ESPFIX_STACK
687686
popl %ecx

‎arch/i386/kernel/head.S

Lines changed: 6 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -317,12 +317,12 @@ is386: movl $2,%ecx # set MP
317317
movl %eax,%cr0
318318

319319
call check_x87
320-
call setup_pda
321320
lgdt early_gdt_descr
322321
lidt idt_descr
323322
ljmp $(__KERNEL_CS),$1f
324323
1: movl $(__KERNEL_DS),%eax # reload all the segment registers
325324
movl %eax,%ss # after changing gdt.
325+
movl %eax,%fs # gets reset once there's real percpu
326326

327327
movl $(__USER_DS),%eax # DS/ES contains default USER segment
328328
movl %eax,%ds
@@ -332,16 +332,17 @@ is386: movl $2,%ecx # set MP
332332
movl %eax,%gs
333333
lldt %ax
334334

335-
movl $(__KERNEL_PDA),%eax
336-
mov %eax,%fs
337-
338335
cld # gcc2 wants the direction flag cleared at all times
339336
pushl $0 # fake return address for unwinder
340337
#ifdef CONFIG_SMP
341338
movb ready, %cl
342339
movb $1, ready
343340
cmpb $0,%cl # the first CPU calls start_kernel
344-
jne initialize_secondary # all other CPUs call initialize_secondary
341+
je 1f
342+
movl $(__KERNEL_PERCPU), %eax
343+
movl %eax,%fs # set this cpu's percpu
344+
jmp initialize_secondary # all other CPUs call initialize_secondary
345+
1:
345346
#endif /* CONFIG_SMP */
346347
jmp start_kernel
347348

@@ -364,23 +365,6 @@ check_x87:
364365
.byte 0xDB,0xE4 /* fsetpm for 287, ignored by 387 */
365366
ret
366367

367-
/*
368-
* Point the GDT at this CPU's PDA. On boot this will be
369-
* cpu_gdt_table and boot_pda; for secondary CPUs, these will be
370-
* that CPU's GDT and PDA.
371-
*/
372-
ENTRY(setup_pda)
373-
/* get the PDA pointer */
374-
movl start_pda, %eax
375-
376-
/* slot the PDA address into the GDT */
377-
mov early_gdt_descr+2, %ecx
378-
mov %ax, (__KERNEL_PDA+0+2)(%ecx) /* base & 0x0000ffff */
379-
shr $16, %eax
380-
mov %al, (__KERNEL_PDA+4+0)(%ecx) /* base & 0x00ff0000 */
381-
mov %ah, (__KERNEL_PDA+4+3)(%ecx) /* base & 0xff000000 */
382-
ret
383-
384368
/*
385369
* setup_idt
386370
*
@@ -553,9 +537,6 @@ ENTRY(empty_zero_page)
553537
* This starts the data section.
554538
*/
555539
.data
556-
ENTRY(start_pda)
557-
.long boot_pda
558-
559540
ENTRY(stack_start)
560541
.long init_thread_union+THREAD_SIZE
561542
.long __BOOT_DS

‎arch/i386/kernel/i386_ksyms.c

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,5 +28,3 @@ EXPORT_SYMBOL(__read_lock_failed);
2828
#endif
2929

3030
EXPORT_SYMBOL(csum_partial);
31-
32-
EXPORT_SYMBOL(_proxy_pda);

‎arch/i386/kernel/irq.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@
2424
DEFINE_PER_CPU(irq_cpustat_t, irq_stat) ____cacheline_internodealigned_in_smp;
2525
EXPORT_PER_CPU_SYMBOL(irq_stat);
2626

27+
DEFINE_PER_CPU(struct pt_regs *, irq_regs);
28+
EXPORT_PER_CPU_SYMBOL(irq_regs);
29+
2730
/*
2831
* 'what should we do if we get a hw irq event on an illegal vector'.
2932
* each architecture has to answer this themselves.

‎arch/i386/kernel/process.c

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
#include <linux/random.h>
4040
#include <linux/personality.h>
4141
#include <linux/tick.h>
42+
#include <linux/percpu.h>
4243

4344
#include <asm/uaccess.h>
4445
#include <asm/pgtable.h>
@@ -57,7 +58,6 @@
5758

5859
#include <asm/tlbflush.h>
5960
#include <asm/cpu.h>
60-
#include <asm/pda.h>
6161

6262
asmlinkage void ret_from_fork(void) __asm__("ret_from_fork");
6363

@@ -66,6 +66,12 @@ static int hlt_counter;
6666
unsigned long boot_option_idle_override = 0;
6767
EXPORT_SYMBOL(boot_option_idle_override);
6868

69+
DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
70+
EXPORT_PER_CPU_SYMBOL(current_task);
71+
72+
DEFINE_PER_CPU(int, cpu_number);
73+
EXPORT_PER_CPU_SYMBOL(cpu_number);
74+
6975
/*
7076
* Return saved PC of a blocked thread.
7177
*/
@@ -342,7 +348,7 @@ int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
342348

343349
regs.xds = __USER_DS;
344350
regs.xes = __USER_DS;
345-
regs.xfs = __KERNEL_PDA;
351+
regs.xfs = __KERNEL_PERCPU;
346352
regs.orig_eax = -1;
347353
regs.eip = (unsigned long) kernel_thread_helper;
348354
regs.xcs = __KERNEL_CS | get_kernel_rpl();
@@ -711,7 +717,7 @@ struct task_struct fastcall * __switch_to(struct task_struct *prev_p, struct tas
711717
if (prev->gs | next->gs)
712718
loadsegment(gs, next->gs);
713719

714-
write_pda(pcurrent, next_p);
720+
x86_write_percpu(current_task, next_p);
715721

716722
return prev_p;
717723
}

‎arch/i386/kernel/smpboot.c

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,6 @@
5353
#include <asm/desc.h>
5454
#include <asm/arch_hooks.h>
5555
#include <asm/nmi.h>
56-
#include <asm/pda.h>
5756

5857
#include <mach_apic.h>
5958
#include <mach_wakecpu.h>
@@ -99,6 +98,9 @@ EXPORT_SYMBOL(x86_cpu_to_apicid);
9998

10099
u8 apicid_2_node[MAX_APICID];
101100

101+
DEFINE_PER_CPU(unsigned long, this_cpu_off);
102+
EXPORT_PER_CPU_SYMBOL(this_cpu_off);
103+
102104
/*
103105
* Trampoline 80x86 program as an array.
104106
*/
@@ -456,7 +458,6 @@ extern struct {
456458
void * esp;
457459
unsigned short ss;
458460
} stack_start;
459-
extern struct i386_pda *start_pda;
460461

461462
#ifdef CONFIG_NUMA
462463

@@ -784,20 +785,17 @@ static inline struct task_struct * alloc_idle_task(int cpu)
784785
/* Initialize the CPU's GDT. This is either the boot CPU doing itself
785786
(still using the master per-cpu area), or a CPU doing it for a
786787
secondary which will soon come up. */
787-
static __cpuinit void init_gdt(int cpu, struct task_struct *idle)
788+
static __cpuinit void init_gdt(int cpu)
788789
{
789790
struct desc_struct *gdt = get_cpu_gdt_table(cpu);
790-
struct i386_pda *pda = &per_cpu(_cpu_pda, cpu);
791791

792-
pack_descriptor((u32 *)&gdt[GDT_ENTRY_PDA].a,
793-
(u32 *)&gdt[GDT_ENTRY_PDA].b,
794-
(unsigned long)pda, sizeof(*pda) - 1,
795-
0x80 | DESCTYPE_S | 0x2, 0); /* present read-write data segment */
792+
pack_descriptor((u32 *)&gdt[GDT_ENTRY_PERCPU].a,
793+
(u32 *)&gdt[GDT_ENTRY_PERCPU].b,
794+
__per_cpu_offset[cpu], 0xFFFFF,
795+
0x80 | DESCTYPE_S | 0x2, 0x8);
796796

797-
memset(pda, 0, sizeof(*pda));
798-
pda->_pda = pda;
799-
pda->cpu_number = cpu;
800-
pda->pcurrent = idle;
797+
per_cpu(this_cpu_off, cpu) = __per_cpu_offset[cpu];
798+
per_cpu(cpu_number, cpu) = cpu;
801799
}
802800

803801
/* Defined in head.S */
@@ -824,9 +822,9 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu)
824822
if (IS_ERR(idle))
825823
panic("failed fork for CPU %d", cpu);
826824

827-
init_gdt(cpu, idle);
825+
init_gdt(cpu);
826+
per_cpu(current_task, cpu) = idle;
828827
early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
829-
start_pda = cpu_pda(cpu);
830828

831829
idle->thread.eip = (unsigned long) start_secondary;
832830
/* start_eip had better be page-aligned! */
@@ -1188,14 +1186,14 @@ static inline void switch_to_new_gdt(void)
11881186
gdt_descr.address = (long)get_cpu_gdt_table(smp_processor_id());
11891187
gdt_descr.size = GDT_SIZE - 1;
11901188
load_gdt(&gdt_descr);
1191-
asm volatile ("mov %0, %%fs" : : "r" (__KERNEL_PDA) : "memory");
1189+
asm("mov %0, %%fs" : : "r" (__KERNEL_PERCPU) : "memory");
11921190
}
11931191

11941192
void __init native_smp_prepare_boot_cpu(void)
11951193
{
11961194
unsigned int cpu = smp_processor_id();
11971195

1198-
init_gdt(cpu, current);
1196+
init_gdt(cpu);
11991197
switch_to_new_gdt();
12001198

12011199
cpu_set(cpu, cpu_online_map);

‎arch/i386/kernel/vmi.c

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -504,8 +504,6 @@ static void vmi_pmd_clear(pmd_t *pmd)
504504
#endif
505505

506506
#ifdef CONFIG_SMP
507-
extern void setup_pda(void);
508-
509507
static void __devinit
510508
vmi_startup_ipi_hook(int phys_apicid, unsigned long start_eip,
511509
unsigned long start_esp)
@@ -530,13 +528,11 @@ vmi_startup_ipi_hook(int phys_apicid, unsigned long start_eip,
530528

531529
ap.ds = __USER_DS;
532530
ap.es = __USER_DS;
533-
ap.fs = __KERNEL_PDA;
531+
ap.fs = __KERNEL_PERCPU;
534532
ap.gs = 0;
535533

536534
ap.eflags = 0;
537535

538-
setup_pda();
539-
540536
#ifdef CONFIG_X86_PAE
541537
/* efer should match BSP efer. */
542538
if (cpu_has_nx) {

‎arch/i386/kernel/vmlinux.lds.S

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ OUTPUT_FORMAT("elf32-i386", "elf32-i386", "elf32-i386")
2626
OUTPUT_ARCH(i386)
2727
ENTRY(phys_startup_32)
2828
jiffies = jiffies_64;
29-
_proxy_pda = 1;
3029

3130
PHDRS {
3231
text PT_LOAD FLAGS(5); /* R_E */

‎include/asm-i386/current.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
11
#ifndef _I386_CURRENT_H
22
#define _I386_CURRENT_H
33

4-
#include <asm/pda.h>
54
#include <linux/compiler.h>
5+
#include <asm/percpu.h>
66

77
struct task_struct;
88

9+
DECLARE_PER_CPU(struct task_struct *, current_task);
910
static __always_inline struct task_struct *get_current(void)
1011
{
11-
return read_pda(pcurrent);
12+
return x86_read_percpu(current_task);
1213
}
1314

1415
#define current get_current()

‎include/asm-i386/irq_regs.h

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,27 @@
11
/*
22
* Per-cpu current frame pointer - the location of the last exception frame on
3-
* the stack, stored in the PDA.
3+
* the stack, stored in the per-cpu area.
44
*
55
* Jeremy Fitzhardinge <jeremy@goop.org>
66
*/
77
#ifndef _ASM_I386_IRQ_REGS_H
88
#define _ASM_I386_IRQ_REGS_H
99

10-
#include <asm/pda.h>
10+
#include <asm/percpu.h>
11+
12+
DECLARE_PER_CPU(struct pt_regs *, irq_regs);
1113

1214
static inline struct pt_regs *get_irq_regs(void)
1315
{
14-
return read_pda(irq_regs);
16+
return x86_read_percpu(irq_regs);
1517
}
1618

1719
static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
1820
{
1921
struct pt_regs *old_regs;
2022

21-
old_regs = read_pda(irq_regs);
22-
write_pda(irq_regs, new_regs);
23+
old_regs = get_irq_regs();
24+
x86_write_percpu(irq_regs, new_regs);
2325

2426
return old_regs;
2527
}

0 commit comments

Comments
 (0)