powerpc/book3s64/mm: Simplify the rcu callback for page table free

kvaneesh · mpe · commit 0c4d268029bf · 2018-05-15T22:29:11.000+10:00
Instead of encoding shift in the table address, use an enumerated index value.
This allow us to do different things in the callback for pte and pmd.

Signed-off-by: Aneesh Kumar K.V &lt;aneesh.kumar@linux.vnet.ibm.com&gt;
Signed-off-by: Michael Ellerman &lt;mpe@ellerman.id.au&gt;
diff --git a/arch/powerpc/include/asm/book3s/64/pgalloc.h b/arch/powerpc/include/asm/book3s/64/pgalloc.h
@@ -124,14 +124,14 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
 }
 
 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
-                                  unsigned long address)
+				  unsigned long address)
 {
 	/*
 	 * By now all the pud entries should be none entries. So go
 	 * ahead and flush the page walk cache
 	 */
 	flush_tlb_pgtable(tlb, address);
-	pgtable_free_tlb(tlb, pud, PUD_CACHE_INDEX);
+	pgtable_free_tlb(tlb, pud, PUD_INDEX);
 }
 
 static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
@@ -146,14 +146,14 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
 }
 
 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
-                                  unsigned long address)
+				  unsigned long address)
 {
 	/*
 	 * By now all the pud entries should be none entries. So go
 	 * ahead and flush the page walk cache
 	 */
 	flush_tlb_pgtable(tlb, address);
-        return pgtable_free_tlb(tlb, pmd, PMD_CACHE_INDEX);
+	return pgtable_free_tlb(tlb, pmd, PMD_INDEX);
 }
 
 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
@@ -203,7 +203,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
 	 * ahead and flush the page walk cache
 	 */
 	flush_tlb_pgtable(tlb, address);
-	pgtable_free_tlb(tlb, table, 0);
+	pgtable_free_tlb(tlb, table, PTE_INDEX);
 }
 
 #define check_pgt_cache()	do { } while (0)
diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h
@@ -273,6 +273,16 @@ extern unsigned long __pte_frag_size_shift;
 /* Bits to mask out from a PGD to get to the PUD page */
 #define PGD_MASKED_BITS		0xc0000000000000ffUL
 
+/*
+ * Used as an indicator for rcu callback functions
+ */
+enum pgtable_index {
+	PTE_INDEX = 0,
+	PMD_INDEX,
+	PUD_INDEX,
+	PGD_INDEX,
+};
+
 extern unsigned long __vmalloc_start;
 extern unsigned long __vmalloc_end;
 #define VMALLOC_START	__vmalloc_start
diff --git a/arch/powerpc/mm/pgtable-book3s64.c b/arch/powerpc/mm/pgtable-book3s64.c
@@ -309,38 +309,45 @@ void pte_fragment_free(unsigned long *table, int kernel)
 	}
 }
 
+static inline void pgtable_free(void *table, int index)
+{
+	switch (index) {
+	case PTE_INDEX:
+		pte_fragment_free(table, 0);
+		break;
+	case PMD_INDEX:
+		kmem_cache_free(PGT_CACHE(PMD_CACHE_INDEX), table);
+		break;
+	case PUD_INDEX:
+		kmem_cache_free(PGT_CACHE(PUD_CACHE_INDEX), table);
+		break;
+		/* We don't free pgd table via RCU callback */
+	default:
+		BUG();
+	}
+}
+
 #ifdef CONFIG_SMP
-void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
+void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int index)
 {
 	unsigned long pgf = (unsigned long)table;
 
-	BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
-	pgf |= shift;
+	BUG_ON(index > MAX_PGTABLE_INDEX_SIZE);
+	pgf |= index;
 	tlb_remove_table(tlb, (void *)pgf);
 }
 
 void __tlb_remove_table(void *_table)
 {
 	void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);
-	unsigned int shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
+	unsigned int index = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
 
-	if (!shift)
-		/* PTE page needs special handling */
-		pte_fragment_free(table, 0);
-	else {
-		BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
-		kmem_cache_free(PGT_CACHE(shift), table);
-	}
+	return pgtable_free(table, index);
 }
 #else
-void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
+void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int index)
 {
-	if (!shift) {
-		/* PTE page needs special handling */
-		pte_fragment_free(table, 0);
-	} else {
-		BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
-		kmem_cache_free(PGT_CACHE(shift), table);
-	}
+
+	return pgtable_free(table, index);
 }
 #endif

Original file line number	Diff line number	Diff line change
`@@ -124,14 +124,14 @@ static inline void pud_populate(struct mm_struct mm, pud_t pud, pmd_t *pmd)`
`124`	`124`	`}`
`125`	`125`
`126`	`126`	`static inline void __pud_free_tlb(struct mmu_gather tlb, pud_t pud,`
`127`		`- unsigned long address)`
	`127`	`+ unsigned long address)`
`128`	`128`	`{`
`129`	`129`	`/*`
`130`	`130`	`* By now all the pud entries should be none entries. So go`
`131`	`131`	`* ahead and flush the page walk cache`
`132`	`132`	`*/`
`133`	`133`	`flush_tlb_pgtable(tlb, address);`
`134`		`- pgtable_free_tlb(tlb, pud, PUD_CACHE_INDEX);`
	`134`	`+ pgtable_free_tlb(tlb, pud, PUD_INDEX);`
`135`	`135`	`}`
`136`	`136`
`137`	`137`	`static inline pmd_t pmd_alloc_one(struct mm_struct mm, unsigned long addr)`
`@@ -146,14 +146,14 @@ static inline void pmd_free(struct mm_struct mm, pmd_t pmd)`
`146`	`146`	`}`
`147`	`147`
`148`	`148`	`static inline void __pmd_free_tlb(struct mmu_gather tlb, pmd_t pmd,`
`149`		`- unsigned long address)`
	`149`	`+ unsigned long address)`
`150`	`150`	`{`
`151`	`151`	`/*`
`152`	`152`	`* By now all the pud entries should be none entries. So go`
`153`	`153`	`* ahead and flush the page walk cache`
`154`	`154`	`*/`
`155`	`155`	`flush_tlb_pgtable(tlb, address);`
`156`		`- return pgtable_free_tlb(tlb, pmd, PMD_CACHE_INDEX);`
	`156`	`+ return pgtable_free_tlb(tlb, pmd, PMD_INDEX);`
`157`	`157`	`}`
`158`	`158`
`159`	`159`	`static inline void pmd_populate_kernel(struct mm_struct mm, pmd_t pmd,`
`@@ -203,7 +203,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,`
`203`	`203`	`* ahead and flush the page walk cache`
`204`	`204`	`*/`
`205`	`205`	`flush_tlb_pgtable(tlb, address);`
`206`		`- pgtable_free_tlb(tlb, table, 0);`
	`206`	`+ pgtable_free_tlb(tlb, table, PTE_INDEX);`
`207`	`207`	`}`
`208`	`208`
`209`	`209`	`#define check_pgt_cache() do { } while (0)`