Kernel/linux-next
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git synced 2025-01-16 05:26:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

mm: pgtable: introduce pagetable_dtor()

Browse Source 
The pagetable_p*_dtor() are exactly the same except for the handling of
ptlock.  If we make ptlock_free() handle the case where ptdesc->ptl is
NULL and remove VM_BUG_ON_PAGE() from pmd_ptlock_free(), we can unify
pagetable_p*_dtor() into one function.  Let's introduce pagetable_dtor()
to do this.

Later, pagetable_dtor() will be moved to tlb_remove_ptdesc(), so that
ptlock and page table pages can be freed together (regardless of whether
RCU is used).  This prevents the use-after-free problem where the ptlock
is freed immediately but the page table pages is freed later via RCU.

Link: https://lkml.kernel.org/r/47f44fff9dc68d9d9e9a0d6c036df275f820598a.1736317725.git.zhengqi.arch@bytedance.com
Signed-off-by: Qi Zheng <zhengqi.arch@bytedance.com>
Originally-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Kevin Brodsky <kevin.brodsky@arm.com>
Acked-by: Alexander Gordeev <agordeev@linux.ibm.com>	[s390]
Cc: Alexandre Ghiti <alex@ghiti.fr>
Cc: Alexandre Ghiti <alexghiti@rivosinc.com>
Cc: Andreas Larsson <andreas@gaisler.com>
Cc: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jann Horn <jannh@google.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Mike Rapoport (Microsoft) <rppt@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vishal Moola (Oracle) <vishal.moola@gmail.com>
Cc: Will Deacon <will@kernel.org>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Qi Zheng 2025-01-08 14:57:23 +08:00 committed by Andrew Morton
parent 78966b5502
commit 2dac349644
28 changed files with 62 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Documentation/mm/split_page_table_lock.rst
View File

@ -62,7 +62,7 @@ Support of split page table lock by an architecture
===================================================
There's no need in special enabling of PTE split page table lock: everything
required is done by pagetable_pte_ctor() and pagetable_pte_dtor(), which
required is done by pagetable_pte_ctor() and pagetable_dtor(), which
must be called on PTE table allocation / freeing.
Make sure the architecture doesn't use slab allocator for page table
@ -73,7 +73,7 @@ PMD split lock only makes sense if you have more than two page table
levels.
PMD split lock enabling requires pagetable_pmd_ctor() call on PMD table
allocation and pagetable_pmd_dtor() on freeing.
allocation and pagetable_dtor() on freeing.
Allocation usually happens in pmd_alloc_one(), freeing in pmd_free() and
pmd_free_tlb(), but make sure you cover all PMD table allocation / freeing

4
arch/arm/include/asm/tlb.h
View File

@ -41,7 +41,7 @@ __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long addr)
{
struct ptdesc *ptdesc = page_ptdesc(pte);
pagetable_pte_dtor(ptdesc);
pagetable_dtor(ptdesc);
#ifndef CONFIG_ARM_LPAE
/*
@ -61,7 +61,7 @@ __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr)
#ifdef CONFIG_ARM_LPAE
struct ptdesc *ptdesc = virt_to_ptdesc(pmdp);
pagetable_pmd_dtor(ptdesc);
pagetable_dtor(ptdesc);
tlb_remove_ptdesc(tlb, ptdesc);
#endif
}

8
arch/arm64/include/asm/tlb.h
View File

@ -82,7 +82,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
{
struct ptdesc *ptdesc = page_ptdesc(pte);
pagetable_pte_dtor(ptdesc);
pagetable_dtor(ptdesc);
tlb_remove_ptdesc(tlb, ptdesc);
}
@ -92,7 +92,7 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
{
struct ptdesc *ptdesc = virt_to_ptdesc(pmdp);
pagetable_pmd_dtor(ptdesc);
pagetable_dtor(ptdesc);
tlb_remove_ptdesc(tlb, ptdesc);
}
#endif
@ -106,7 +106,7 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp,
if (!pgtable_l4_enabled())
return;
pagetable_pud_dtor(ptdesc);
pagetable_dtor(ptdesc);
tlb_remove_ptdesc(tlb, ptdesc);
}
#endif
@ -120,7 +120,7 @@ static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4dp,
if (!pgtable_l5_enabled())
return;
pagetable_p4d_dtor(ptdesc);
pagetable_dtor(ptdesc);
tlb_remove_ptdesc(tlb, ptdesc);
}
#endif

2
arch/csky/include/asm/pgalloc.h
View File

@ -63,7 +63,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)
#define __pte_free_tlb(tlb, pte, address) \
do { \
pagetable_pte_dtor(page_ptdesc(pte)); \
pagetable_dtor(page_ptdesc(pte)); \
tlb_remove_page_ptdesc(tlb, page_ptdesc(pte)); \
} while (0)

2
arch/hexagon/include/asm/pgalloc.h
View File

@ -89,7 +89,7 @@ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
#define __pte_free_tlb(tlb, pte, addr) \
do { \
pagetable_pte_dtor((page_ptdesc(pte))); \
pagetable_dtor((page_ptdesc(pte))); \
tlb_remove_page_ptdesc((tlb), (page_ptdesc(pte))); \
} while (0)

2
arch/loongarch/include/asm/pgalloc.h
View File

@ -57,7 +57,7 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
#define __pte_free_tlb(tlb, pte, address) \
do { \
pagetable_pte_dtor(page_ptdesc(pte)); \
pagetable_dtor(page_ptdesc(pte)); \
tlb_remove_page_ptdesc((tlb), page_ptdesc(pte)); \
} while (0)

4
arch/m68k/include/asm/mcf_pgalloc.h
View File

@ -37,7 +37,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pgtable,
{
struct ptdesc *ptdesc = virt_to_ptdesc(pgtable);
pagetable_pte_dtor(ptdesc);
pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
@ -61,7 +61,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pgtable)
{
struct ptdesc *ptdesc = virt_to_ptdesc(pgtable);
pagetable_pte_dtor(ptdesc);
pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}

2
arch/m68k/include/asm/sun3_pgalloc.h
View File

@ -19,7 +19,7 @@ extern const char bad_pmd_string[];
#define __pte_free_tlb(tlb, pte, addr) \
do { \
pagetable_pte_dtor(page_ptdesc(pte)); \
pagetable_dtor(page_ptdesc(pte)); \
tlb_remove_page_ptdesc((tlb), page_ptdesc(pte)); \
} while (0)

2
arch/m68k/mm/motorola.c
View File

@ -201,7 +201,7 @@ int free_pointer_table(void *table, int type)
list_del(dp);
mmu_page_dtor((void *)page);
if (type == TABLE_PTE)
pagetable_pte_dtor(virt_to_ptdesc((void *)page));
pagetable_dtor(virt_to_ptdesc((void *)page));
free_page (page);
return 1;
} else if (ptable_list[type].next != dp) {

2
arch/mips/include/asm/pgalloc.h
View File

@ -56,7 +56,7 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
#define __pte_free_tlb(tlb, pte, address) \
do { \
pagetable_pte_dtor(page_ptdesc(pte)); \
pagetable_dtor(page_ptdesc(pte)); \
tlb_remove_page_ptdesc((tlb), page_ptdesc(pte)); \
} while (0)

2
arch/nios2/include/asm/pgalloc.h
View File

@ -30,7 +30,7 @@ extern pgd_t *pgd_alloc(struct mm_struct *mm);
#define __pte_free_tlb(tlb, pte, addr) \
do { \
pagetable_pte_dtor(page_ptdesc(pte)); \
pagetable_dtor(page_ptdesc(pte)); \
tlb_remove_page_ptdesc((tlb), (page_ptdesc(pte))); \
} while (0)

2
arch/openrisc/include/asm/pgalloc.h
View File

@ -68,7 +68,7 @@ extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm);
#define __pte_free_tlb(tlb, pte, addr) \
do { \
pagetable_pte_dtor(page_ptdesc(pte)); \
pagetable_dtor(page_ptdesc(pte)); \
tlb_remove_page_ptdesc((tlb), (page_ptdesc(pte))); \
} while (0)

2
arch/powerpc/mm/book3s64/mmu_context.c
View File

@ -253,7 +253,7 @@ static void pmd_frag_destroy(void *pmd_frag)
count = ((unsigned long)pmd_frag & ~PAGE_MASK) >> PMD_FRAG_SIZE_SHIFT;
/* We allow PTE_FRAG_NR fragments from a PTE page */
if (atomic_sub_and_test(PMD_FRAG_NR - count, &ptdesc->pt_frag_refcount)) {
pagetable_pmd_dtor(ptdesc);
pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
}

2
arch/powerpc/mm/book3s64/pgtable.c
View File

@ -477,7 +477,7 @@ void pmd_fragment_free(unsigned long *pmd)
BUG_ON(atomic_read(&ptdesc->pt_frag_refcount) <= 0);
if (atomic_dec_and_test(&ptdesc->pt_frag_refcount)) {
pagetable_pmd_dtor(ptdesc);
pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
}

4
arch/powerpc/mm/pgtable-frag.c
View File

@ -25,7 +25,7 @@ void pte_frag_destroy(void *pte_frag)
count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT;
/* We allow PTE_FRAG_NR fragments from a PTE page */
if (atomic_sub_and_test(PTE_FRAG_NR - count, &ptdesc->pt_frag_refcount)) {
pagetable_pte_dtor(ptdesc);
pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
}
@ -111,7 +111,7 @@ static void pte_free_now(struct rcu_head *head)
struct ptdesc *ptdesc;
ptdesc = container_of(head, struct ptdesc, pt_rcu_head);
pagetable_pte_dtor(ptdesc);
pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}

8
arch/riscv/include/asm/pgalloc.h
View File

@ -100,7 +100,7 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
if (pgtable_l4_enabled) {
struct ptdesc *ptdesc = virt_to_ptdesc(pud);
pagetable_pud_dtor(ptdesc);
pagetable_dtor(ptdesc);
riscv_tlb_remove_ptdesc(tlb, ptdesc);
}
}
@ -111,7 +111,7 @@ static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d,
if (pgtable_l5_enabled) {
struct ptdesc *ptdesc = virt_to_ptdesc(p4d);
pagetable_p4d_dtor(ptdesc);
pagetable_dtor(ptdesc);
riscv_tlb_remove_ptdesc(tlb, virt_to_ptdesc(p4d));
}
}
@ -144,7 +144,7 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
{
struct ptdesc *ptdesc = virt_to_ptdesc(pmd);
pagetable_pmd_dtor(ptdesc);
pagetable_dtor(ptdesc);
riscv_tlb_remove_ptdesc(tlb, ptdesc);
}
@ -155,7 +155,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
{
struct ptdesc *ptdesc = page_ptdesc(pte);
pagetable_pte_dtor(ptdesc);
pagetable_dtor(ptdesc);
riscv_tlb_remove_ptdesc(tlb, ptdesc);
}
#endif /* CONFIG_MMU */

4
arch/riscv/mm/init.c
View File

@ -1558,7 +1558,7 @@ static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd)
return;
}
pagetable_pte_dtor(ptdesc);
pagetable_dtor(ptdesc);
if (PageReserved(page))
free_reserved_page(page);
else
@ -1580,7 +1580,7 @@ static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud, bool is_vmemm
}
if (!is_vmemmap)
pagetable_pmd_dtor(ptdesc);
pagetable_dtor(ptdesc);
if (PageReserved(page))
free_reserved_page(page);
else

6
arch/s390/include/asm/pgalloc.h
View File

@ -66,7 +66,7 @@ static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)
if (mm_p4d_folded(mm))
return;
pagetable_p4d_dtor(virt_to_ptdesc(p4d));
pagetable_dtor(virt_to_ptdesc(p4d));
crst_table_free(mm, (unsigned long *) p4d);
}
@ -87,7 +87,7 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud)
if (mm_pud_folded(mm))
return;
pagetable_pud_dtor(virt_to_ptdesc(pud));
pagetable_dtor(virt_to_ptdesc(pud));
crst_table_free(mm, (unsigned long *) pud);
}
@ -109,7 +109,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
{
if (mm_pmd_folded(mm))
return;
pagetable_pmd_dtor(virt_to_ptdesc(pmd));
pagetable_dtor(virt_to_ptdesc(pmd));
crst_table_free(mm, (unsigned long *) pmd);
}

6
arch/s390/include/asm/tlb.h
View File

@ -102,7 +102,7 @@ static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
{
if (mm_pmd_folded(tlb->mm))
return;
pagetable_pmd_dtor(virt_to_ptdesc(pmd));
pagetable_dtor(virt_to_ptdesc(pmd));
__tlb_adjust_range(tlb, address, PAGE_SIZE);
tlb->mm->context.flush_mm = 1;
tlb->freed_tables = 1;
@ -122,7 +122,7 @@ static inline void p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d,
{
if (mm_p4d_folded(tlb->mm))
return;
pagetable_p4d_dtor(virt_to_ptdesc(p4d));
pagetable_dtor(virt_to_ptdesc(p4d));
__tlb_adjust_range(tlb, address, PAGE_SIZE);
tlb->mm->context.flush_mm = 1;
tlb->freed_tables = 1;
@ -141,7 +141,7 @@ static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
{
if (mm_pud_folded(tlb->mm))
return;
pagetable_pud_dtor(virt_to_ptdesc(pud));
pagetable_dtor(virt_to_ptdesc(pud));
tlb->mm->context.flush_mm = 1;
tlb->freed_tables = 1;
tlb->cleared_p4ds = 1;

2
arch/s390/mm/pgalloc.c
View File

@ -182,7 +182,7 @@ unsigned long *page_table_alloc(struct mm_struct *mm)
static void pagetable_pte_dtor_free(struct ptdesc *ptdesc)
{
pagetable_pte_dtor(ptdesc);
pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}

2
arch/sh/include/asm/pgalloc.h
View File

@ -34,7 +34,7 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
#define __pte_free_tlb(tlb, pte, addr) \
do { \
pagetable_pte_dtor(page_ptdesc(pte)); \
pagetable_dtor(page_ptdesc(pte)); \
tlb_remove_page_ptdesc((tlb), (page_ptdesc(pte))); \
} while (0)

2
arch/sparc/mm/init_64.c
View File

@ -2915,7 +2915,7 @@ static void __pte_free(pgtable_t pte)
{
struct ptdesc *ptdesc = virt_to_ptdesc(pte);
pagetable_pte_dtor(ptdesc);
pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}

2
arch/sparc/mm/srmmu.c
View File

@ -372,7 +372,7 @@ void pte_free(struct mm_struct *mm, pgtable_t ptep)
page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT);
spin_lock(&mm->page_table_lock);
if (page_ref_dec_return(page) == 1)
pagetable_pte_dtor(page_ptdesc(page));
pagetable_dtor(page_ptdesc(page));
spin_unlock(&mm->page_table_lock);
srmmu_free_nocache(ptep, SRMMU_PTE_TABLE_SIZE);

6
arch/um/include/asm/pgalloc.h
View File

@ -27,7 +27,7 @@ extern pgd_t *pgd_alloc(struct mm_struct *);
#define __pte_free_tlb(tlb, pte, address) \
do { \
pagetable_pte_dtor(page_ptdesc(pte)); \
pagetable_dtor(page_ptdesc(pte)); \
tlb_remove_page_ptdesc((tlb), (page_ptdesc(pte))); \
} while (0)
@ -35,7 +35,7 @@ do { \
#define __pmd_free_tlb(tlb, pmd, address) \
do { \
pagetable_pmd_dtor(virt_to_ptdesc(pmd)); \
pagetable_dtor(virt_to_ptdesc(pmd)); \
tlb_remove_page_ptdesc((tlb), virt_to_ptdesc(pmd)); \
} while (0)
@ -43,7 +43,7 @@ do { \
#define __pud_free_tlb(tlb, pud, address) \
do { \
pagetable_pud_dtor(virt_to_ptdesc(pud)); \
pagetable_dtor(virt_to_ptdesc(pud)); \
tlb_remove_page_ptdesc((tlb), virt_to_ptdesc(pud)); \
} while (0)

12
arch/x86/mm/pgtable.c
View File

@ -60,7 +60,7 @@ early_param("userpte", setup_userpte);
void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
{
pagetable_pte_dtor(page_ptdesc(pte));
pagetable_dtor(page_ptdesc(pte));
paravirt_release_pte(page_to_pfn(pte));
paravirt_tlb_remove_table(tlb, pte);
}
@ -77,7 +77,7 @@ void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
#ifdef CONFIG_X86_PAE
tlb->need_flush_all = 1;
#endif
pagetable_pmd_dtor(ptdesc);
pagetable_dtor(ptdesc);
paravirt_tlb_remove_table(tlb, ptdesc_page(ptdesc));
}
@ -86,7 +86,7 @@ void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
{
struct ptdesc *ptdesc = virt_to_ptdesc(pud);
pagetable_pud_dtor(ptdesc);
pagetable_dtor(ptdesc);
paravirt_release_pud(__pa(pud) >> PAGE_SHIFT);
paravirt_tlb_remove_table(tlb, virt_to_page(pud));
}
@ -96,7 +96,7 @@ void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d)
{
struct ptdesc *ptdesc = virt_to_ptdesc(p4d);
pagetable_p4d_dtor(ptdesc);
pagetable_dtor(ptdesc);
paravirt_release_p4d(__pa(p4d) >> PAGE_SHIFT);
paravirt_tlb_remove_table(tlb, virt_to_page(p4d));
}
@ -233,7 +233,7 @@ static void free_pmds(struct mm_struct *mm, pmd_t *pmds[], int count)
if (pmds[i]) {
ptdesc = virt_to_ptdesc(pmds[i]);
pagetable_pmd_dtor(ptdesc);
pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
mm_dec_nr_pmds(mm);
}
@ -867,7 +867,7 @@ int pud_free_pmd_page(pud_t *pud, unsigned long addr)
free_page((unsigned long)pmd_sv);
pagetable_pmd_dtor(virt_to_ptdesc(pmd));
pagetable_dtor(virt_to_ptdesc(pmd));
free_page((unsigned long)pmd);
return 1;

8
include/asm-generic/pgalloc.h
View File

@ -109,7 +109,7 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte_page)
{
struct ptdesc *ptdesc = page_ptdesc(pte_page);
pagetable_pte_dtor(ptdesc);
pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
@ -153,7 +153,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
struct ptdesc *ptdesc = virt_to_ptdesc(pmd);
BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
pagetable_pmd_dtor(ptdesc);
pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
#endif
@ -202,7 +202,7 @@ static inline void __pud_free(struct mm_struct *mm, pud_t *pud)
struct ptdesc *ptdesc = virt_to_ptdesc(pud);
BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
pagetable_pud_dtor(ptdesc);
pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
@ -248,7 +248,7 @@ static inline void __p4d_free(struct mm_struct *mm, p4d_t *p4d)
struct ptdesc *ptdesc = virt_to_ptdesc(p4d);
BUG_ON((unsigned long)p4d & (PAGE_SIZE-1));
pagetable_p4d_dtor(ptdesc);
pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}

52
include/linux/mm.h
View File

@ -2992,6 +2992,15 @@ static inline bool ptlock_init(struct ptdesc *ptdesc) { return true; }
static inline void ptlock_free(struct ptdesc *ptdesc) {}
#endif /* defined(CONFIG_SPLIT_PTE_PTLOCKS) */
static inline void pagetable_dtor(struct ptdesc *ptdesc)
{
struct folio *folio = ptdesc_folio(ptdesc);
ptlock_free(ptdesc);
__folio_clear_pgtable(folio);
lruvec_stat_sub_folio(folio, NR_PAGETABLE);
}
static inline bool pagetable_pte_ctor(struct ptdesc *ptdesc)
{
struct folio *folio = ptdesc_folio(ptdesc);
@ -3003,15 +3012,6 @@ static inline bool pagetable_pte_ctor(struct ptdesc *ptdesc)
return true;
}
static inline void pagetable_pte_dtor(struct ptdesc *ptdesc)
{
struct folio *folio = ptdesc_folio(ptdesc);
ptlock_free(ptdesc);
__folio_clear_pgtable(folio);
lruvec_stat_sub_folio(folio, NR_PAGETABLE);
}
pte_t *___pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp);
static inline pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr,
pmd_t *pmdvalp)
@ -3088,14 +3088,6 @@ static inline bool pmd_ptlock_init(struct ptdesc *ptdesc)
return ptlock_init(ptdesc);
}
static inline void pmd_ptlock_free(struct ptdesc *ptdesc)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
VM_BUG_ON_PAGE(ptdesc->pmd_huge_pte, ptdesc_page(ptdesc));
#endif
ptlock_free(ptdesc);
}
#define pmd_huge_pte(mm, pmd) (pmd_ptdesc(pmd)->pmd_huge_pte)
#else
@ -3106,7 +3098,6 @@ static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
}
static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) { return true; }
static inline void pmd_ptlock_free(struct ptdesc *ptdesc) {}
#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
@ -3131,15 +3122,6 @@ static inline bool pagetable_pmd_ctor(struct ptdesc *ptdesc)
return true;
}
static inline void pagetable_pmd_dtor(struct ptdesc *ptdesc)
{
struct folio *folio = ptdesc_folio(ptdesc);
pmd_ptlock_free(ptdesc);
__folio_clear_pgtable(folio);
lruvec_stat_sub_folio(folio, NR_PAGETABLE);
}
/*
* No scalability reason to split PUD locks yet, but follow the same pattern
* as the PMD locks to make it easier if we decide to. The VM should not be
@ -3167,14 +3149,6 @@ static inline void pagetable_pud_ctor(struct ptdesc *ptdesc)
lruvec_stat_add_folio(folio, NR_PAGETABLE);
}
static inline void pagetable_pud_dtor(struct ptdesc *ptdesc)
{
struct folio *folio = ptdesc_folio(ptdesc);
__folio_clear_pgtable(folio);
lruvec_stat_sub_folio(folio, NR_PAGETABLE);
}
static inline void pagetable_p4d_ctor(struct ptdesc *ptdesc)
{
struct folio *folio = ptdesc_folio(ptdesc);
@ -3183,14 +3157,6 @@ static inline void pagetable_p4d_ctor(struct ptdesc *ptdesc)
lruvec_stat_add_folio(folio, NR_PAGETABLE);
}
static inline void pagetable_p4d_dtor(struct ptdesc *ptdesc)
{
struct folio *folio = ptdesc_folio(ptdesc);
__folio_clear_pgtable(folio);
lruvec_stat_sub_folio(folio, NR_PAGETABLE);
}
extern void __init pagecache_init(void);
extern void free_initmem(void);

3
mm/memory.c
View File

@ -7031,7 +7031,8 @@ bool ptlock_alloc(struct ptdesc *ptdesc)
void ptlock_free(struct ptdesc *ptdesc)
{
kmem_cache_free(page_ptl_cachep, ptdesc->ptl);
if (ptdesc->ptl)
kmem_cache_free(page_ptl_cachep, ptdesc->ptl);
}
#endif