Skip to content

Commit ada3fa1

Browse files
torvaldstorvalds
committed
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu
* 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu: (46 commits) powerpc64: convert to dynamic percpu allocator sparc64: use embedding percpu first chunk allocator percpu: kill lpage first chunk allocator x86,percpu: use embedding for 64bit NUMA and page for 32bit NUMA percpu: update embedding first chunk allocator to handle sparse units percpu: use group information to allocate vmap areas sparsely vmalloc: implement pcpu_get_vm_areas() vmalloc: separate out insert_vmalloc_vm() percpu: add chunk->base_addr percpu: add pcpu_unit_offsets[] percpu: introduce pcpu_alloc_info and pcpu_group_info percpu: move pcpu_lpage_build_unit_map() and pcpul_lpage_dump_cfg() upward percpu: add @align to pcpu_fc_alloc_fn_t percpu: make @dyn_size mandatory for pcpu_setup_first_chunk() percpu: drop @static_size from first chunk allocators percpu: generalize first chunk allocator selection percpu: build first chunk allocators selectively percpu: rename 4k first chunk allocator to page percpu: improve boot messages percpu: fix pcpu_reclaim() locking ... Fix trivial conflict as by Tejun Heo in kernel/sched.c
2 parents 2f82af0 + 5579fd7 commit ada3fa1

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

80 files changed

+1910
-1228
lines changed

‎Documentation/kernel-parameters.txt

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1971,11 +1971,12 @@ and is between 256 and 4096 characters. It is defined in the file
19711971
Format: { 0 | 1 }
19721972
See arch/parisc/kernel/pdc_chassis.c
19731973

1974-
percpu_alloc= [X86] Select which percpu first chunk allocator to use.
1975-
Allowed values are one of "lpage", "embed" and "4k".
1976-
See comments in arch/x86/kernel/setup_percpu.c for
1977-
details on each allocator. This parameter is primarily
1978-
for debugging and performance comparison.
1974+
percpu_alloc= Select which percpu first chunk allocator to use.
1975+
Currently supported values are "embed" and "page".
1976+
Archs may support subset or none of the selections.
1977+
See comments in mm/percpu.c for details on each
1978+
allocator. This parameter is primarily for debugging
1979+
and performance comparison.
19791980

19801981
pf. [PARIDE]
19811982
See Documentation/blockdev/paride.txt.

‎Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -325,7 +325,7 @@ CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
325325
MODFLAGS = -DMODULE
326326
CFLAGS_MODULE = $(MODFLAGS)
327327
AFLAGS_MODULE = $(MODFLAGS)
328-
LDFLAGS_MODULE =
328+
LDFLAGS_MODULE = -T $(srctree)/scripts/module-common.lds
329329
CFLAGS_KERNEL =
330330
AFLAGS_KERNEL =
331331
CFLAGS_GCOV = -fprofile-arcs -ftest-coverage

‎arch/alpha/include/asm/percpu.h

Lines changed: 8 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -1,102 +1,18 @@
11
#ifndef __ALPHA_PERCPU_H
22
#define __ALPHA_PERCPU_H
33

4-
#include <linux/compiler.h>
5-
#include <linux/threads.h>
6-
#include <linux/percpu-defs.h>
7-
8-
/*
9-
* Determine the real variable name from the name visible in the
10-
* kernel sources.
11-
*/
12-
#define per_cpu_var(var) per_cpu__##var
13-
14-
#ifdef CONFIG_SMP
15-
16-
/*
17-
* per_cpu_offset() is the offset that has to be added to a
18-
* percpu variable to get to the instance for a certain processor.
19-
*/
20-
extern unsigned long __per_cpu_offset[NR_CPUS];
21-
22-
#define per_cpu_offset(x) (__per_cpu_offset[x])
23-
24-
#define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
25-
#ifdef CONFIG_DEBUG_PREEMPT
26-
#define my_cpu_offset per_cpu_offset(smp_processor_id())
27-
#else
28-
#define my_cpu_offset __my_cpu_offset
29-
#endif
30-
31-
#ifndef MODULE
32-
#define SHIFT_PERCPU_PTR(var, offset) RELOC_HIDE(&per_cpu_var(var), (offset))
33-
#define PER_CPU_DEF_ATTRIBUTES
34-
#else
354
/*
36-
* To calculate addresses of locally defined variables, GCC uses 32-bit
37-
* displacement from the GP. Which doesn't work for per cpu variables in
38-
* modules, as an offset to the kernel per cpu area is way above 4G.
5+
* To calculate addresses of locally defined variables, GCC uses
6+
* 32-bit displacement from the GP. Which doesn't work for per cpu
7+
* variables in modules, as an offset to the kernel per cpu area is
8+
* way above 4G.
399
*
40-
* This forces allocation of a GOT entry for per cpu variable using
41-
* ldq instruction with a 'literal' relocation.
42-
*/
43-
#define SHIFT_PERCPU_PTR(var, offset) ({ \
44-
extern int simple_identifier_##var(void); \
45-
unsigned long __ptr, tmp_gp; \
46-
asm ( "br %1, 1f \n\
47-
1: ldgp %1, 0(%1) \n\
48-
ldq %0, per_cpu__" #var"(%1)\t!literal" \
49-
: "=&r"(__ptr), "=&r"(tmp_gp)); \
50-
(typeof(&per_cpu_var(var)))(__ptr + (offset)); })
51-
52-
#define PER_CPU_DEF_ATTRIBUTES __used
53-
54-
#endif /* MODULE */
55-
56-
/*
57-
* A percpu variable may point to a discarded regions. The following are
58-
* established ways to produce a usable pointer from the percpu variable
59-
* offset.
10+
* Always use weak definitions for percpu variables in modules.
6011
*/
61-
#define per_cpu(var, cpu) \
62-
(*SHIFT_PERCPU_PTR(var, per_cpu_offset(cpu)))
63-
#define __get_cpu_var(var) \
64-
(*SHIFT_PERCPU_PTR(var, my_cpu_offset))
65-
#define __raw_get_cpu_var(var) \
66-
(*SHIFT_PERCPU_PTR(var, __my_cpu_offset))
67-
68-
#else /* ! SMP */
69-
70-
#define per_cpu(var, cpu) (*((void)(cpu), &per_cpu_var(var)))
71-
#define __get_cpu_var(var) per_cpu_var(var)
72-
#define __raw_get_cpu_var(var) per_cpu_var(var)
73-
74-
#define PER_CPU_DEF_ATTRIBUTES
75-
76-
#endif /* SMP */
77-
78-
#ifdef CONFIG_SMP
79-
#define PER_CPU_BASE_SECTION ".data.percpu"
80-
#else
81-
#define PER_CPU_BASE_SECTION ".data"
82-
#endif
83-
84-
#ifdef CONFIG_SMP
85-
86-
#ifdef MODULE
87-
#define PER_CPU_SHARED_ALIGNED_SECTION ""
88-
#else
89-
#define PER_CPU_SHARED_ALIGNED_SECTION ".shared_aligned"
90-
#endif
91-
#define PER_CPU_FIRST_SECTION ".first"
92-
93-
#else
94-
95-
#define PER_CPU_SHARED_ALIGNED_SECTION ""
96-
#define PER_CPU_FIRST_SECTION ""
97-
12+
#if defined(MODULE) && defined(CONFIG_SMP)
13+
#define ARCH_NEEDS_WEAK_PER_CPU
9814
#endif
9915

100-
#define PER_CPU_ATTRIBUTES
16+
#include <asm-generic/percpu.h>
10117

10218
#endif /* __ALPHA_PERCPU_H */

‎arch/alpha/include/asm/tlbflush.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#define _ALPHA_TLBFLUSH_H
33

44
#include <linux/mm.h>
5+
#include <linux/sched.h>
56
#include <asm/compiler.h>
67
#include <asm/pgalloc.h>
78

‎arch/alpha/kernel/vmlinux.lds.S

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -134,13 +134,6 @@ SECTIONS
134134
__bss_stop = .;
135135
_end = .;
136136

137-
/* Sections to be discarded */
138-
/DISCARD/ : {
139-
EXIT_TEXT
140-
EXIT_DATA
141-
*(.exitcall.exit)
142-
}
143-
144137
.mdebug 0 : {
145138
*(.mdebug)
146139
}
@@ -150,4 +143,6 @@ SECTIONS
150143

151144
STABS_DEBUG
152145
DWARF_DEBUG
146+
147+
DISCARDS
153148
}

‎arch/arm/kernel/vmlinux.lds.S

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ SECTIONS
8383
EXIT_TEXT
8484
EXIT_DATA
8585
*(.exitcall.exit)
86+
*(.discard)
8687
*(.ARM.exidx.exit.text)
8788
*(.ARM.extab.exit.text)
8889
#ifndef CONFIG_HOTPLUG_CPU

‎arch/avr32/kernel/vmlinux.lds.S

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -124,14 +124,11 @@ SECTIONS
124124
_end = .;
125125
}
126126

127+
DWARF_DEBUG
128+
127129
/* When something in the kernel is NOT compiled as a module, the module
128130
* cleanup code and data are put into these segments. Both can then be
129131
* thrown away, as cleanup code is never called unless it's a module.
130132
*/
131-
/DISCARD/ : {
132-
EXIT_DATA
133-
*(.exitcall.exit)
134-
}
135-
136-
DWARF_DEBUG
133+
DISCARDS
137134
}

‎arch/blackfin/kernel/vmlinux.lds.S

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -277,8 +277,5 @@ SECTIONS
277277

278278
DWARF_DEBUG
279279

280-
/DISCARD/ :
281-
{
282-
*(.exitcall.exit)
283-
}
280+
DISCARDS
284281
}

‎arch/blackfin/mm/sram-alloc.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,9 @@
4242
#include <asm/mem_map.h>
4343
#include "blackfin_sram.h"
4444

45-
static DEFINE_PER_CPU(spinlock_t, l1sram_lock) ____cacheline_aligned_in_smp;
46-
static DEFINE_PER_CPU(spinlock_t, l1_data_sram_lock) ____cacheline_aligned_in_smp;
47-
static DEFINE_PER_CPU(spinlock_t, l1_inst_sram_lock) ____cacheline_aligned_in_smp;
45+
static DEFINE_PER_CPU_SHARED_ALIGNED(spinlock_t, l1sram_lock);
46+
static DEFINE_PER_CPU_SHARED_ALIGNED(spinlock_t, l1_data_sram_lock);
47+
static DEFINE_PER_CPU_SHARED_ALIGNED(spinlock_t, l1_inst_sram_lock);
4848
static spinlock_t l2_sram_lock ____cacheline_aligned_in_smp;
4949

5050
/* the data structure for L1 scratchpad and DATA SRAM */

‎arch/cris/include/asm/mmu_context.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@ extern void switch_mm(struct mm_struct *prev, struct mm_struct *next,
1717
* registers like cr3 on the i386
1818
*/
1919

20-
extern volatile DEFINE_PER_CPU(pgd_t *,current_pgd); /* defined in arch/cris/mm/fault.c */
20+
/* defined in arch/cris/mm/fault.c */
21+
DECLARE_PER_CPU(pgd_t *, current_pgd);
2122

2223
static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
2324
{

‎arch/cris/kernel/vmlinux.lds.S

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -140,12 +140,7 @@ SECTIONS
140140
_end = .;
141141
__end = .;
142142

143-
/* Sections to be discarded */
144-
/DISCARD/ : {
145-
EXIT_TEXT
146-
EXIT_DATA
147-
*(.exitcall.exit)
148-
}
149-
150143
dram_end = dram_start + (CONFIG_ETRAX_DRAM_SIZE - __CONFIG_ETRAX_VMEM_SIZE)*1024*1024;
144+
145+
DISCARDS
151146
}

‎arch/cris/mm/fault.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ extern void die_if_kernel(const char *, struct pt_regs *, long);
2929

3030
/* current active page directory */
3131

32-
volatile DEFINE_PER_CPU(pgd_t *,current_pgd);
32+
DEFINE_PER_CPU(pgd_t *, current_pgd);
3333
unsigned long cris_signal_return_page;
3434

3535
/*

‎arch/frv/kernel/vmlinux.lds.S

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,8 @@ SECTIONS
177177
.debug_ranges 0 : { *(.debug_ranges) }
178178

179179
.comment 0 : { *(.comment) }
180+
181+
DISCARDS
180182
}
181183

182184
__kernel_image_size_no_bss = __bss_start - __kernel_image_start;

‎arch/h8300/kernel/vmlinux.lds.S

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -152,9 +152,6 @@ SECTIONS
152152
__end = . ;
153153
__ramstart = .;
154154
}
155-
/DISCARD/ : {
156-
*(.exitcall.exit)
157-
}
158155
.romfs :
159156
{
160157
*(.romfs*)
@@ -165,4 +162,6 @@ SECTIONS
165162
COMMAND_START = . - 0x200 ;
166163
__ramend = . ;
167164
}
165+
166+
DISCARDS
168167
}

‎arch/ia64/Kconfig

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,9 @@ config GENERIC_TIME_VSYSCALL
8989
bool
9090
default y
9191

92+
config HAVE_LEGACY_PER_CPU_AREA
93+
def_bool y
94+
9295
config HAVE_SETUP_PER_CPU_AREA
9396
def_bool y
9497

‎arch/ia64/kernel/setup.c

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -855,11 +855,17 @@ identify_cpu (struct cpuinfo_ia64 *c)
855855
c->unimpl_pa_mask = ~((1L<<63) | ((1L << phys_addr_size) - 1));
856856
}
857857

858+
/*
859+
* In UP configuration, setup_per_cpu_areas() is defined in
860+
* include/linux/percpu.h
861+
*/
862+
#ifdef CONFIG_SMP
858863
void __init
859864
setup_per_cpu_areas (void)
860865
{
861866
/* start_kernel() requires this... */
862867
}
868+
#endif
863869

864870
/*
865871
* Do the following calculations:

‎arch/ia64/kernel/smp.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,8 @@ static struct local_tlb_flush_counts {
5858
unsigned int count;
5959
} __attribute__((__aligned__(32))) local_tlb_flush_counts[NR_CPUS];
6060

61-
static DEFINE_PER_CPU(unsigned short, shadow_flush_counts[NR_CPUS]) ____cacheline_aligned;
61+
static DEFINE_PER_CPU_SHARED_ALIGNED(unsigned short [NR_CPUS],
62+
shadow_flush_counts);
6263

6364
#define IPI_CALL_FUNC 0
6465
#define IPI_CPU_STOP 1

‎arch/ia64/kernel/vmlinux.lds.S

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,14 @@ PHDRS {
2424
}
2525
SECTIONS
2626
{
27-
/* Sections to be discarded */
27+
/* unwind exit sections must be discarded before the rest of the
28+
sections get included. */
2829
/DISCARD/ : {
29-
EXIT_TEXT
30-
EXIT_DATA
31-
*(.exitcall.exit)
3230
*(.IA_64.unwind.exit.text)
3331
*(.IA_64.unwind_info.exit.text)
34-
}
32+
*(.comment)
33+
*(.note)
34+
}
3535

3636
v = PAGE_OFFSET; /* this symbol is here to make debugging easier... */
3737
phys_start = _start - LOAD_OFFSET;
@@ -316,7 +316,7 @@ SECTIONS
316316
.debug_funcnames 0 : { *(.debug_funcnames) }
317317
.debug_typenames 0 : { *(.debug_typenames) }
318318
.debug_varnames 0 : { *(.debug_varnames) }
319-
/* These must appear regardless of . */
320-
/DISCARD/ : { *(.comment) }
321-
/DISCARD/ : { *(.note) }
319+
320+
/* Default discards */
321+
DISCARDS
322322
}

‎arch/ia64/sn/kernel/setup.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ EXPORT_SYMBOL(sn_rtc_cycles_per_second);
7171
DEFINE_PER_CPU(struct sn_hub_info_s, __sn_hub_info);
7272
EXPORT_PER_CPU_SYMBOL(__sn_hub_info);
7373

74-
DEFINE_PER_CPU(short, __sn_cnodeid_to_nasid[MAX_COMPACT_NODES]);
74+
DEFINE_PER_CPU(short [MAX_COMPACT_NODES], __sn_cnodeid_to_nasid);
7575
EXPORT_PER_CPU_SYMBOL(__sn_cnodeid_to_nasid);
7676

7777
DEFINE_PER_CPU(struct nodepda_s *, __sn_nodepda);

‎arch/m32r/kernel/vmlinux.lds.S

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -120,13 +120,6 @@ SECTIONS
120120

121121
_end = . ;
122122

123-
/* Sections to be discarded */
124-
/DISCARD/ : {
125-
EXIT_TEXT
126-
EXIT_DATA
127-
*(.exitcall.exit)
128-
}
129-
130123
/* Stabs debugging sections. */
131124
.stab 0 : { *(.stab) }
132125
.stabstr 0 : { *(.stabstr) }
@@ -135,4 +128,7 @@ SECTIONS
135128
.stab.index 0 : { *(.stab.index) }
136129
.stab.indexstr 0 : { *(.stab.indexstr) }
137130
.comment 0 : { *(.comment) }
131+
132+
/* Sections to be discarded */
133+
DISCARDS
138134
}

‎arch/m68k/kernel/vmlinux-std.lds

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -82,13 +82,6 @@ SECTIONS
8282

8383
_end = . ;
8484

85-
/* Sections to be discarded */
86-
/DISCARD/ : {
87-
EXIT_TEXT
88-
EXIT_DATA
89-
*(.exitcall.exit)
90-
}
91-
9285
/* Stabs debugging sections. */
9386
.stab 0 : { *(.stab) }
9487
.stabstr 0 : { *(.stabstr) }
@@ -97,4 +90,7 @@ SECTIONS
9790
.stab.index 0 : { *(.stab.index) }
9891
.stab.indexstr 0 : { *(.stab.indexstr) }
9992
.comment 0 : { *(.comment) }
93+
94+
/* Sections to be discarded */
95+
DISCARDS
10096
}

0 commit comments

Comments
 (0)