Skip to content

Commit 7b73d97

Browse files
Christoph Hellwigdjbw
Christoph Hellwig
authored and
djbw
committed
mm: pass the vmem_altmap to vmemmap_populate
We can just pass this on instead of having to do a radix tree lookup without proper locking a few levels into the callchain. Signed-off-by: Christoph Hellwig <hch@lst.de> Signed-off-by: Dan Williams <dan.j.williams@intel.com>
1 parent 24e6d5a commit 7b73d97

File tree

11 files changed

+39
-29
lines changed

11 files changed

+39
-29
lines changed

‎arch/arm64/mm/mmu.c

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -654,12 +654,14 @@ int kern_addr_valid(unsigned long addr)
654654
}
655655
#ifdef CONFIG_SPARSEMEM_VMEMMAP
656656
#if !ARM64_SWAPPER_USES_SECTION_MAPS
657-
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
657+
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
658+
struct vmem_altmap *altmap)
658659
{
659660
return vmemmap_populate_basepages(start, end, node);
660661
}
661662
#else /* !ARM64_SWAPPER_USES_SECTION_MAPS */
662-
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
663+
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
664+
struct vmem_altmap *altmap)
663665
{
664666
unsigned long addr = start;
665667
unsigned long next;

‎arch/ia64/mm/discontig.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -754,7 +754,8 @@ void arch_refresh_nodedata(int update_node, pg_data_t *update_pgdat)
754754
#endif
755755

756756
#ifdef CONFIG_SPARSEMEM_VMEMMAP
757-
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
757+
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
758+
struct vmem_altmap *altmap)
758759
{
759760
return vmemmap_populate_basepages(start, end, node);
760761
}

‎arch/powerpc/mm/init_64.c

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,8 @@ static __meminit void vmemmap_list_populate(unsigned long phys,
183183
vmemmap_list = vmem_back;
184184
}
185185

186-
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
186+
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
187+
struct vmem_altmap *altmap)
187188
{
188189
unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift;
189190

@@ -193,16 +194,12 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
193194
pr_debug("vmemmap_populate %lx..%lx, node %d\n", start, end, node);
194195

195196
for (; start < end; start += page_size) {
196-
struct vmem_altmap *altmap;
197197
void *p;
198198
int rc;
199199

200200
if (vmemmap_populated(start, page_size))
201201
continue;
202202

203-
/* altmap lookups only work at section boundaries */
204-
altmap = to_vmem_altmap(SECTION_ALIGN_DOWN(start));
205-
206203
p = __vmemmap_alloc_block_buf(page_size, node, altmap);
207204
if (!p)
208205
return -ENOMEM;

‎arch/s390/mm/vmem.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,8 @@ static void vmem_remove_range(unsigned long start, unsigned long size)
211211
/*
212212
* Add a backed mem_map array to the virtual mem_map array.
213213
*/
214-
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
214+
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
215+
struct vmem_altmap *altmap)
215216
{
216217
unsigned long pgt_prot, sgt_prot;
217218
unsigned long address = start;

‎arch/sparc/mm/init_64.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2628,7 +2628,7 @@ EXPORT_SYMBOL(_PAGE_CACHE);
26282628

26292629
#ifdef CONFIG_SPARSEMEM_VMEMMAP
26302630
int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
2631-
int node)
2631+
int node, struct vmem_altmap *altmap)
26322632
{
26332633
unsigned long pte_base;
26342634

‎arch/x86/mm/init_64.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1411,9 +1411,9 @@ static int __meminit vmemmap_populate_hugepages(unsigned long start,
14111411
return 0;
14121412
}
14131413

1414-
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
1414+
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
1415+
struct vmem_altmap *altmap)
14151416
{
1416-
struct vmem_altmap *altmap = to_vmem_altmap(start);
14171417
int err;
14181418

14191419
if (boot_cpu_has(X86_FEATURE_PSE))

‎include/linux/memory_hotplug.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -327,7 +327,8 @@ extern void move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn,
327327
extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages);
328328
extern bool is_memblock_offlined(struct memory_block *mem);
329329
extern void remove_memory(int nid, u64 start, u64 size);
330-
extern int sparse_add_one_section(struct pglist_data *pgdat, unsigned long start_pfn);
330+
extern int sparse_add_one_section(struct pglist_data *pgdat,
331+
unsigned long start_pfn, struct vmem_altmap *altmap);
331332
extern void sparse_remove_one_section(struct zone *zone, struct mem_section *ms,
332333
unsigned long map_offset);
333334
extern struct page *sparse_decode_mem_map(unsigned long coded_mem_map,

‎include/linux/mm.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2538,7 +2538,8 @@ void sparse_mem_maps_populate_node(struct page **map_map,
25382538
unsigned long map_count,
25392539
int nodeid);
25402540

2541-
struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
2541+
struct page *sparse_mem_map_populate(unsigned long pnum, int nid,
2542+
struct vmem_altmap *altmap);
25422543
pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
25432544
p4d_t *vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node);
25442545
pud_t *vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node);
@@ -2556,7 +2557,8 @@ static inline void *vmemmap_alloc_block_buf(unsigned long size, int node)
25562557
void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
25572558
int vmemmap_populate_basepages(unsigned long start, unsigned long end,
25582559
int node);
2559-
int vmemmap_populate(unsigned long start, unsigned long end, int node);
2560+
int vmemmap_populate(unsigned long start, unsigned long end, int node,
2561+
struct vmem_altmap *altmap);
25602562
void vmemmap_populate_print_last(void);
25612563
#ifdef CONFIG_MEMORY_HOTPLUG
25622564
void vmemmap_free(unsigned long start, unsigned long end);

‎mm/memory_hotplug.c

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -250,15 +250,15 @@ void __init register_page_bootmem_info_node(struct pglist_data *pgdat)
250250
#endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */
251251

252252
static int __meminit __add_section(int nid, unsigned long phys_start_pfn,
253-
bool want_memblock)
253+
struct vmem_altmap *altmap, bool want_memblock)
254254
{
255255
int ret;
256256
int i;
257257

258258
if (pfn_valid(phys_start_pfn))
259259
return -EEXIST;
260260

261-
ret = sparse_add_one_section(NODE_DATA(nid), phys_start_pfn);
261+
ret = sparse_add_one_section(NODE_DATA(nid), phys_start_pfn, altmap);
262262
if (ret < 0)
263263
return ret;
264264

@@ -317,7 +317,8 @@ int __ref __add_pages(int nid, unsigned long phys_start_pfn,
317317
}
318318

319319
for (i = start_sec; i <= end_sec; i++) {
320-
err = __add_section(nid, section_nr_to_pfn(i), want_memblock);
320+
err = __add_section(nid, section_nr_to_pfn(i), altmap,
321+
want_memblock);
321322

322323
/*
323324
* EEXIST is finally dealt with by ioresource collision

‎mm/sparse-vmemmap.c

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,8 @@ int __meminit vmemmap_populate_basepages(unsigned long start,
278278
return 0;
279279
}
280280

281-
struct page * __meminit sparse_mem_map_populate(unsigned long pnum, int nid)
281+
struct page * __meminit sparse_mem_map_populate(unsigned long pnum, int nid,
282+
struct vmem_altmap *altmap)
282283
{
283284
unsigned long start;
284285
unsigned long end;
@@ -288,7 +289,7 @@ struct page * __meminit sparse_mem_map_populate(unsigned long pnum, int nid)
288289
start = (unsigned long)map;
289290
end = (unsigned long)(map + PAGES_PER_SECTION);
290291

291-
if (vmemmap_populate(start, end, nid))
292+
if (vmemmap_populate(start, end, nid, altmap))
292293
return NULL;
293294

294295
return map;
@@ -318,7 +319,7 @@ void __init sparse_mem_maps_populate_node(struct page **map_map,
318319
if (!present_section_nr(pnum))
319320
continue;
320321

321-
map_map[pnum] = sparse_mem_map_populate(pnum, nodeid);
322+
map_map[pnum] = sparse_mem_map_populate(pnum, nodeid, NULL);
322323
if (map_map[pnum])
323324
continue;
324325
ms = __nr_to_section(pnum);

‎mm/sparse.c

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -417,7 +417,8 @@ static void __init sparse_early_usemaps_alloc_node(void *data,
417417
}
418418

419419
#ifndef CONFIG_SPARSEMEM_VMEMMAP
420-
struct page __init *sparse_mem_map_populate(unsigned long pnum, int nid)
420+
struct page __init *sparse_mem_map_populate(unsigned long pnum, int nid,
421+
struct vmem_altmap *altmap)
421422
{
422423
struct page *map;
423424
unsigned long size;
@@ -472,7 +473,7 @@ void __init sparse_mem_maps_populate_node(struct page **map_map,
472473

473474
if (!present_section_nr(pnum))
474475
continue;
475-
map_map[pnum] = sparse_mem_map_populate(pnum, nodeid);
476+
map_map[pnum] = sparse_mem_map_populate(pnum, nodeid, NULL);
476477
if (map_map[pnum])
477478
continue;
478479
ms = __nr_to_section(pnum);
@@ -500,7 +501,7 @@ static struct page __init *sparse_early_mem_map_alloc(unsigned long pnum)
500501
struct mem_section *ms = __nr_to_section(pnum);
501502
int nid = sparse_early_nid(ms);
502503

503-
map = sparse_mem_map_populate(pnum, nid);
504+
map = sparse_mem_map_populate(pnum, nid, NULL);
504505
if (map)
505506
return map;
506507

@@ -678,10 +679,11 @@ void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn)
678679
#endif
679680

680681
#ifdef CONFIG_SPARSEMEM_VMEMMAP
681-
static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid)
682+
static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid,
683+
struct vmem_altmap *altmap)
682684
{
683685
/* This will make the necessary allocations eventually. */
684-
return sparse_mem_map_populate(pnum, nid);
686+
return sparse_mem_map_populate(pnum, nid, altmap);
685687
}
686688
static void __kfree_section_memmap(struct page *memmap)
687689
{
@@ -721,7 +723,8 @@ static struct page *__kmalloc_section_memmap(void)
721723
return ret;
722724
}
723725

724-
static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid)
726+
static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid,
727+
struct vmem_altmap *altmap)
725728
{
726729
return __kmalloc_section_memmap();
727730
}
@@ -773,7 +776,8 @@ static void free_map_bootmem(struct page *memmap)
773776
* set. If this is <=0, then that means that the passed-in
774777
* map was not consumed and must be freed.
775778
*/
776-
int __meminit sparse_add_one_section(struct pglist_data *pgdat, unsigned long start_pfn)
779+
int __meminit sparse_add_one_section(struct pglist_data *pgdat,
780+
unsigned long start_pfn, struct vmem_altmap *altmap)
777781
{
778782
unsigned long section_nr = pfn_to_section_nr(start_pfn);
779783
struct mem_section *ms;
@@ -789,7 +793,7 @@ int __meminit sparse_add_one_section(struct pglist_data *pgdat, unsigned long st
789793
ret = sparse_index_init(section_nr, pgdat->node_id);
790794
if (ret < 0 && ret != -EEXIST)
791795
return ret;
792-
memmap = kmalloc_section_memmap(section_nr, pgdat->node_id);
796+
memmap = kmalloc_section_memmap(section_nr, pgdat->node_id, altmap);
793797
if (!memmap)
794798
return -ENOMEM;
795799
usemap = __kmalloc_section_usemap();

0 commit comments

Comments
 (0)