Skip to content

Commit a9cd410

Browse files
Arun KStorvalds
Arun KS
authored and
torvalds
committed
mm/page_alloc.c: memory hotplug: free pages as higher order
When freeing pages are done with higher order, time spent on coalescing pages by buddy allocator can be reduced. With section size of 256MB, hot add latency of a single section shows improvement from 50-60 ms to less than 1 ms, hence improving the hot add latency by 60 times. Modify external providers of online callback to align with the change. [arunks@codeaurora.org: v11] Link: http://lkml.kernel.org/r/1547792588-18032-1-git-send-email-arunks@codeaurora.org [akpm@linux-foundation.org: remove unused local, per Arun] [akpm@linux-foundation.org: avoid return of void-returning __free_pages_core(), per Oscar] [akpm@linux-foundation.org: fix it for mm-convert-totalram_pages-and-totalhigh_pages-variables-to-atomic.patch] [arunks@codeaurora.org: v8] Link: http://lkml.kernel.org/r/1547032395-24582-1-git-send-email-arunks@codeaurora.org [arunks@codeaurora.org: v9] Link: http://lkml.kernel.org/r/1547098543-26452-1-git-send-email-arunks@codeaurora.org Link: http://lkml.kernel.org/r/1538727006-5727-1-git-send-email-arunks@codeaurora.org Signed-off-by: Arun KS <arunks@codeaurora.org> Reviewed-by: Andrew Morton <akpm@linux-foundation.org> Acked-by: Michal Hocko <mhocko@suse.com> Reviewed-by: Oscar Salvador <osalvador@suse.de> Reviewed-by: Alexander Duyck <alexander.h.duyck@linux.intel.com> Cc: K. Y. Srinivasan <kys@microsoft.com> Cc: Haiyang Zhang <haiyangz@microsoft.com> Cc: Stephen Hemminger <sthemmin@microsoft.com> Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com> Cc: Juergen Gross <jgross@suse.com> Cc: Dan Williams <dan.j.williams@intel.com> Cc: Vlastimil Babka <vbabka@suse.cz> Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com> Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org> Cc: Mathieu Malaterre <malat@debian.org> Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> Cc: Souptick Joarder <jrdr.linux@gmail.com> Cc: Mel Gorman <mgorman@techsingularity.net> Cc: Aaron Lu <aaron.lu@intel.com> Cc: Srivatsa Vaddagiri <vatsa@codeaurora.org> Cc: Vinayak Menon <vinmenon@codeaurora.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
1 parent 278d775 commit a9cd410

File tree

6 files changed

+45
-25
lines changed

6 files changed

+45
-25
lines changed

‎drivers/hv/hv_balloon.c

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -771,7 +771,7 @@ static void hv_mem_hot_add(unsigned long start, unsigned long size,
771771
}
772772
}
773773

774-
static void hv_online_page(struct page *pg)
774+
static void hv_online_page(struct page *pg, unsigned int order)
775775
{
776776
struct hv_hotadd_state *has;
777777
unsigned long flags;
@@ -780,10 +780,11 @@ static void hv_online_page(struct page *pg)
780780
spin_lock_irqsave(&dm_device.ha_lock, flags);
781781
list_for_each_entry(has, &dm_device.ha_region_list, list) {
782782
/* The page belongs to a different HAS. */
783-
if ((pfn < has->start_pfn) || (pfn >= has->end_pfn))
783+
if ((pfn < has->start_pfn) ||
784+
(pfn + (1UL << order) > has->end_pfn))
784785
continue;
785786

786-
hv_page_online_one(has, pg);
787+
hv_bring_pgs_online(has, pfn, 1UL << order);
787788
break;
788789
}
789790
spin_unlock_irqrestore(&dm_device.ha_lock, flags);

‎drivers/xen/balloon.c

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -369,14 +369,19 @@ static enum bp_state reserve_additional_memory(void)
369369
return BP_ECANCELED;
370370
}
371371

372-
static void xen_online_page(struct page *page)
372+
static void xen_online_page(struct page *page, unsigned int order)
373373
{
374-
__online_page_set_limits(page);
374+
unsigned long i, size = (1 << order);
375+
unsigned long start_pfn = page_to_pfn(page);
376+
struct page *p;
375377

378+
pr_debug("Online %lu pages starting at pfn 0x%lx\n", size, start_pfn);
376379
mutex_lock(&balloon_mutex);
377-
378-
__balloon_append(page);
379-
380+
for (i = 0; i < size; i++) {
381+
p = pfn_to_page(start_pfn + i);
382+
__online_page_set_limits(p);
383+
__balloon_append(p);
384+
}
380385
mutex_unlock(&balloon_mutex);
381386
}
382387

‎include/linux/memory_hotplug.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ extern int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn,
8989
unsigned long *valid_start, unsigned long *valid_end);
9090
extern void __offline_isolated_pages(unsigned long, unsigned long);
9191

92-
typedef void (*online_page_callback_t)(struct page *page);
92+
typedef void (*online_page_callback_t)(struct page *page, unsigned int order);
9393

9494
extern int set_online_page_callback(online_page_callback_t callback);
9595
extern int restore_online_page_callback(online_page_callback_t callback);

‎mm/internal.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,7 @@ static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn,
163163
extern int __isolate_free_page(struct page *page, unsigned int order);
164164
extern void memblock_free_pages(struct page *page, unsigned long pfn,
165165
unsigned int order);
166+
extern void __free_pages_core(struct page *page, unsigned int order);
166167
extern void prep_compound_page(struct page *page, unsigned int order);
167168
extern void post_alloc_hook(struct page *page, unsigned int order,
168169
gfp_t gfp_flags);

‎mm/memory_hotplug.c

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@
4747
* and restore_online_page_callback() for generic callback restore.
4848
*/
4949

50-
static void generic_online_page(struct page *page);
50+
static void generic_online_page(struct page *page, unsigned int order);
5151

5252
static online_page_callback_t online_page_callback = generic_online_page;
5353
static DEFINE_MUTEX(online_page_callback_lock);
@@ -656,26 +656,39 @@ void __online_page_free(struct page *page)
656656
}
657657
EXPORT_SYMBOL_GPL(__online_page_free);
658658

659-
static void generic_online_page(struct page *page)
659+
static void generic_online_page(struct page *page, unsigned int order)
660660
{
661-
__online_page_set_limits(page);
662-
__online_page_increment_counters(page);
663-
__online_page_free(page);
661+
__free_pages_core(page, order);
662+
totalram_pages_add(1UL << order);
663+
#ifdef CONFIG_HIGHMEM
664+
if (PageHighMem(page))
665+
totalhigh_pages_add(1UL << order);
666+
#endif
667+
}
668+
669+
static int online_pages_blocks(unsigned long start, unsigned long nr_pages)
670+
{
671+
unsigned long end = start + nr_pages;
672+
int order, onlined_pages = 0;
673+
674+
while (start < end) {
675+
order = min(MAX_ORDER - 1,
676+
get_order(PFN_PHYS(end) - PFN_PHYS(start)));
677+
(*online_page_callback)(pfn_to_page(start), order);
678+
679+
onlined_pages += (1UL << order);
680+
start += (1UL << order);
681+
}
682+
return onlined_pages;
664683
}
665684

666685
static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages,
667686
void *arg)
668687
{
669-
unsigned long i;
670688
unsigned long onlined_pages = *(unsigned long *)arg;
671-
struct page *page;
672689

673690
if (PageReserved(pfn_to_page(start_pfn)))
674-
for (i = 0; i < nr_pages; i++) {
675-
page = pfn_to_page(start_pfn + i);
676-
(*online_page_callback)(page);
677-
onlined_pages++;
678-
}
691+
onlined_pages += online_pages_blocks(start_pfn, nr_pages);
679692

680693
online_mem_sections(start_pfn, start_pfn + nr_pages);
681694

‎mm/page_alloc.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1303,7 +1303,7 @@ static void __free_pages_ok(struct page *page, unsigned int order)
13031303
local_irq_restore(flags);
13041304
}
13051305

1306-
static void __init __free_pages_boot_core(struct page *page, unsigned int order)
1306+
void __free_pages_core(struct page *page, unsigned int order)
13071307
{
13081308
unsigned int nr_pages = 1 << order;
13091309
struct page *p = page;
@@ -1382,7 +1382,7 @@ void __init memblock_free_pages(struct page *page, unsigned long pfn,
13821382
{
13831383
if (early_page_uninitialised(pfn))
13841384
return;
1385-
return __free_pages_boot_core(page, order);
1385+
__free_pages_core(page, order);
13861386
}
13871387

13881388
/*
@@ -1472,14 +1472,14 @@ static void __init deferred_free_range(unsigned long pfn,
14721472
if (nr_pages == pageblock_nr_pages &&
14731473
(pfn & (pageblock_nr_pages - 1)) == 0) {
14741474
set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1475-
__free_pages_boot_core(page, pageblock_order);
1475+
__free_pages_core(page, pageblock_order);
14761476
return;
14771477
}
14781478

14791479
for (i = 0; i < nr_pages; i++, page++, pfn++) {
14801480
if ((pfn & (pageblock_nr_pages - 1)) == 0)
14811481
set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1482-
__free_pages_boot_core(page, 0);
1482+
__free_pages_core(page, 0);
14831483
}
14841484
}
14851485

0 commit comments

Comments
 (0)