Skip to content

Commit 8a6c697

Browse files
kvaneeshmpe
kvaneesh
authored and
mpe
committed
powerpc/mm: Implement helpers for pagetable fragment support at PMD level
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com> Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
1 parent 0c4d268 commit 8a6c697

File tree

11 files changed

+144
-6
lines changed

11 files changed

+144
-6
lines changed

‎arch/powerpc/include/asm/book3s/64/hash-4k.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@
4242
/* 8 bytes per each pte entry */
4343
#define H_PTE_FRAG_SIZE_SHIFT (H_PTE_INDEX_SIZE + 3)
4444
#define H_PTE_FRAG_NR (PAGE_SIZE >> H_PTE_FRAG_SIZE_SHIFT)
45+
#define H_PMD_FRAG_SIZE_SHIFT (H_PMD_INDEX_SIZE + 3)
46+
#define H_PMD_FRAG_NR (PAGE_SIZE >> H_PMD_FRAG_SIZE_SHIFT)
4547

4648
/* memory key bits, only 8 keys supported */
4749
#define H_PTE_PKEY_BIT0 0

‎arch/powerpc/include/asm/book3s/64/hash-64k.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,13 @@
4646
#define H_PTE_FRAG_SIZE_SHIFT (H_PTE_INDEX_SIZE + 3 + 1)
4747
#define H_PTE_FRAG_NR (PAGE_SIZE >> H_PTE_FRAG_SIZE_SHIFT)
4848

49+
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE)
50+
#define H_PMD_FRAG_SIZE_SHIFT (H_PMD_INDEX_SIZE + 3 + 1)
51+
#else
52+
#define H_PMD_FRAG_SIZE_SHIFT (H_PMD_INDEX_SIZE + 3)
53+
#endif
54+
#define H_PMD_FRAG_NR (PAGE_SIZE >> H_PMD_FRAG_SIZE_SHIFT)
55+
4956
#ifndef __ASSEMBLY__
5057
#include <asm/errno.h>
5158

‎arch/powerpc/include/asm/book3s/64/mmu.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ typedef struct {
138138
* pagetable fragment support
139139
*/
140140
void *pte_frag;
141+
void *pmd_frag;
141142
#ifdef CONFIG_SPAPR_TCE_IOMMU
142143
struct list_head iommu_group_mem_list;
143144
#endif

‎arch/powerpc/include/asm/book3s/64/pgalloc.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,9 @@ extern struct kmem_cache *pgtable_cache[];
4242
})
4343

4444
extern pte_t *pte_fragment_alloc(struct mm_struct *, unsigned long, int);
45+
extern pmd_t *pmd_fragment_alloc(struct mm_struct *, unsigned long);
4546
extern void pte_fragment_free(unsigned long *, int);
47+
extern void pmd_fragment_free(unsigned long *);
4648
extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift);
4749
#ifdef CONFIG_SMP
4850
extern void __tlb_remove_table(void *_table);

‎arch/powerpc/include/asm/book3s/64/pgtable.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,12 @@ extern unsigned long __pte_frag_size_shift;
246246
#define PTE_FRAG_SIZE_SHIFT __pte_frag_size_shift
247247
#define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT)
248248

249+
extern unsigned long __pmd_frag_nr;
250+
#define PMD_FRAG_NR __pmd_frag_nr
251+
extern unsigned long __pmd_frag_size_shift;
252+
#define PMD_FRAG_SIZE_SHIFT __pmd_frag_size_shift
253+
#define PMD_FRAG_SIZE (1UL << PMD_FRAG_SIZE_SHIFT)
254+
249255
#define PTRS_PER_PTE (1 << PTE_INDEX_SIZE)
250256
#define PTRS_PER_PMD (1 << PMD_INDEX_SIZE)
251257
#define PTRS_PER_PUD (1 << PUD_INDEX_SIZE)

‎arch/powerpc/include/asm/book3s/64/radix-4k.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,7 @@
1515
#define RADIX_PTE_FRAG_SIZE_SHIFT (RADIX_PTE_INDEX_SIZE + 3)
1616
#define RADIX_PTE_FRAG_NR (PAGE_SIZE >> RADIX_PTE_FRAG_SIZE_SHIFT)
1717

18+
#define RADIX_PMD_FRAG_SIZE_SHIFT (RADIX_PMD_INDEX_SIZE + 3)
19+
#define RADIX_PMD_FRAG_NR (PAGE_SIZE >> RADIX_PMD_FRAG_SIZE_SHIFT)
20+
1821
#endif /* _ASM_POWERPC_PGTABLE_RADIX_4K_H */

‎arch/powerpc/include/asm/book3s/64/radix-64k.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,8 @@
1616
*/
1717
#define RADIX_PTE_FRAG_SIZE_SHIFT (RADIX_PTE_INDEX_SIZE + 3)
1818
#define RADIX_PTE_FRAG_NR (PAGE_SIZE >> RADIX_PTE_FRAG_SIZE_SHIFT)
19+
20+
#define RADIX_PMD_FRAG_SIZE_SHIFT (RADIX_PMD_INDEX_SIZE + 3)
21+
#define RADIX_PMD_FRAG_NR (PAGE_SIZE >> RADIX_PMD_FRAG_SIZE_SHIFT)
22+
1923
#endif /* _ASM_POWERPC_PGTABLE_RADIX_64K_H */

‎arch/powerpc/mm/hash_utils_64.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1012,6 +1012,8 @@ void __init hash__early_init_mmu(void)
10121012
*/
10131013
__pte_frag_nr = H_PTE_FRAG_NR;
10141014
__pte_frag_size_shift = H_PTE_FRAG_SIZE_SHIFT;
1015+
__pmd_frag_nr = H_PMD_FRAG_NR;
1016+
__pmd_frag_size_shift = H_PMD_FRAG_SIZE_SHIFT;
10151017

10161018
__pte_index_size = H_PTE_INDEX_SIZE;
10171019
__pmd_index_size = H_PMD_INDEX_SIZE;

‎arch/powerpc/mm/mmu_context_book3s64.c

Lines changed: 31 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
160160
mm->context.id = index;
161161

162162
mm->context.pte_frag = NULL;
163+
mm->context.pmd_frag = NULL;
163164
#ifdef CONFIG_SPAPR_TCE_IOMMU
164165
mm_iommu_init(mm);
165166
#endif
@@ -190,16 +191,11 @@ static void destroy_contexts(mm_context_t *ctx)
190191
spin_unlock(&mmu_context_lock);
191192
}
192193

193-
static void destroy_pagetable_page(struct mm_struct *mm)
194+
static void pte_frag_destroy(void *pte_frag)
194195
{
195196
int count;
196-
void *pte_frag;
197197
struct page *page;
198198

199-
pte_frag = mm->context.pte_frag;
200-
if (!pte_frag)
201-
return;
202-
203199
page = virt_to_page(pte_frag);
204200
/* drop all the pending references */
205201
count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT;
@@ -210,6 +206,35 @@ static void destroy_pagetable_page(struct mm_struct *mm)
210206
}
211207
}
212208

209+
static void pmd_frag_destroy(void *pmd_frag)
210+
{
211+
int count;
212+
struct page *page;
213+
214+
page = virt_to_page(pmd_frag);
215+
/* drop all the pending references */
216+
count = ((unsigned long)pmd_frag & ~PAGE_MASK) >> PMD_FRAG_SIZE_SHIFT;
217+
/* We allow PTE_FRAG_NR fragments from a PTE page */
218+
if (page_ref_sub_and_test(page, PMD_FRAG_NR - count)) {
219+
pgtable_pmd_page_dtor(page);
220+
free_unref_page(page);
221+
}
222+
}
223+
224+
static void destroy_pagetable_page(struct mm_struct *mm)
225+
{
226+
void *frag;
227+
228+
frag = mm->context.pte_frag;
229+
if (frag)
230+
pte_frag_destroy(frag);
231+
232+
frag = mm->context.pmd_frag;
233+
if (frag)
234+
pmd_frag_destroy(frag);
235+
return;
236+
}
237+
213238
void destroy_context(struct mm_struct *mm)
214239
{
215240
#ifdef CONFIG_SPAPR_TCE_IOMMU

‎arch/powerpc/mm/pgtable-book3s64.c

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,11 @@
2020
#include "mmu_decl.h"
2121
#include <trace/events/thp.h>
2222

23+
unsigned long __pmd_frag_nr;
24+
EXPORT_SYMBOL(__pmd_frag_nr);
25+
unsigned long __pmd_frag_size_shift;
26+
EXPORT_SYMBOL(__pmd_frag_size_shift);
27+
2328
int (*register_process_table)(unsigned long base, unsigned long page_size,
2429
unsigned long tbl_size);
2530

@@ -226,6 +231,85 @@ void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
226231
}
227232
EXPORT_SYMBOL_GPL(mmu_partition_table_set_entry);
228233

234+
static pmd_t *get_pmd_from_cache(struct mm_struct *mm)
235+
{
236+
void *pmd_frag, *ret;
237+
238+
spin_lock(&mm->page_table_lock);
239+
ret = mm->context.pmd_frag;
240+
if (ret) {
241+
pmd_frag = ret + PMD_FRAG_SIZE;
242+
/*
243+
* If we have taken up all the fragments mark PTE page NULL
244+
*/
245+
if (((unsigned long)pmd_frag & ~PAGE_MASK) == 0)
246+
pmd_frag = NULL;
247+
mm->context.pmd_frag = pmd_frag;
248+
}
249+
spin_unlock(&mm->page_table_lock);
250+
return (pmd_t *)ret;
251+
}
252+
253+
static pmd_t *__alloc_for_pmdcache(struct mm_struct *mm)
254+
{
255+
void *ret = NULL;
256+
struct page *page;
257+
gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO;
258+
259+
if (mm == &init_mm)
260+
gfp &= ~__GFP_ACCOUNT;
261+
page = alloc_page(gfp);
262+
if (!page)
263+
return NULL;
264+
if (!pgtable_pmd_page_ctor(page)) {
265+
__free_pages(page, 0);
266+
return NULL;
267+
}
268+
269+
ret = page_address(page);
270+
/*
271+
* if we support only one fragment just return the
272+
* allocated page.
273+
*/
274+
if (PMD_FRAG_NR == 1)
275+
return ret;
276+
277+
spin_lock(&mm->page_table_lock);
278+
/*
279+
* If we find pgtable_page set, we return
280+
* the allocated page with single fragement
281+
* count.
282+
*/
283+
if (likely(!mm->context.pmd_frag)) {
284+
set_page_count(page, PMD_FRAG_NR);
285+
mm->context.pmd_frag = ret + PMD_FRAG_SIZE;
286+
}
287+
spin_unlock(&mm->page_table_lock);
288+
289+
return (pmd_t *)ret;
290+
}
291+
292+
pmd_t *pmd_fragment_alloc(struct mm_struct *mm, unsigned long vmaddr)
293+
{
294+
pmd_t *pmd;
295+
296+
pmd = get_pmd_from_cache(mm);
297+
if (pmd)
298+
return pmd;
299+
300+
return __alloc_for_pmdcache(mm);
301+
}
302+
303+
void pmd_fragment_free(unsigned long *pmd)
304+
{
305+
struct page *page = virt_to_page(pmd);
306+
307+
if (put_page_testzero(page)) {
308+
pgtable_pmd_page_dtor(page);
309+
free_unref_page(page);
310+
}
311+
}
312+
229313
static pte_t *get_pte_from_cache(struct mm_struct *mm)
230314
{
231315
void *pte_frag, *ret;

‎arch/powerpc/mm/pgtable-radix.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -640,6 +640,8 @@ void __init radix__early_init_mmu(void)
640640
#endif
641641
__pte_frag_nr = RADIX_PTE_FRAG_NR;
642642
__pte_frag_size_shift = RADIX_PTE_FRAG_SIZE_SHIFT;
643+
__pmd_frag_nr = RADIX_PMD_FRAG_NR;
644+
__pmd_frag_size_shift = RADIX_PMD_FRAG_SIZE_SHIFT;
643645

644646
if (!firmware_has_feature(FW_FEATURE_LPAR)) {
645647
radix_init_native();

0 commit comments

Comments
 (0)