Kernel/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-01-04 04:04:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

mm/damon/ops-common: atomically test and clear young on ptes and pmds

Browse Source 
It is racy to non-atomically read a pte, then clear the young bit, then
write it back as this could discard dirty information.  Further, it is bad
practice to directly set a pte entry within a table.  Instead clearing
young must go through the arch-provided helper,
ptep_test_and_clear_young() to ensure it is modified atomically and to
give the arch code visibility and allow it to check (and potentially
modify) the operation.

Link: https://lkml.kernel.org/r/20230602092949.545577-3-ryan.roberts@arm.com
Fixes: 3f49584b26 ("mm/damon: implement primitives for the virtual memory address spaces").
Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
Reviewed-by: Zi Yan <ziy@nvidia.com>
Reviewed-by: SeongJae Park <sj@kernel.org>
Reviewed-by: Mike Rapoport (IBM) <rppt@kernel.org>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Lorenzo Stoakes <lstoakes@gmail.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Uladzislau Rezki (Sony) <urezki@gmail.com>
Cc: Yu Zhao <yuzhao@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Ryan Roberts 2023-06-02 10:29:47 +01:00 committed by Andrew Morton
parent b3f78e7498
commit c11d34fa13
4 changed files with 12 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
mm/damon/ops-common.c
View File

@ -37,7 +37,7 @@ struct folio *damon_get_folio(unsigned long pfn)
return folio; return folio;
} }
void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr) void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr)
{ {
bool referenced = false; bool referenced = false;
struct folio *folio = damon_get_folio(pte_pfn(*pte)); struct folio *folio = damon_get_folio(pte_pfn(*pte));
@ -45,13 +45,11 @@ void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr)
if (!folio) if (!folio)
return; return;
if (pte_young(*pte)) { if (ptep_test_and_clear_young(vma, addr, pte))
referenced = true; referenced = true;
*pte = pte_mkold(*pte);
}
#ifdef CONFIG_MMU_NOTIFIER #ifdef CONFIG_MMU_NOTIFIER
if (mmu_notifier_clear_young(mm, addr, addr + PAGE_SIZE)) if (mmu_notifier_clear_young(vma->vm_mm, addr, addr + PAGE_SIZE))
referenced = true; referenced = true;
#endif /* CONFIG_MMU_NOTIFIER */ #endif /* CONFIG_MMU_NOTIFIER */
@ -62,7 +60,7 @@ void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr)
folio_put(folio); folio_put(folio);
} }
void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr) void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr)
{ {
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGEPAGE
bool referenced = false; bool referenced = false;
@ -71,13 +69,11 @@ void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr)
if (!folio) if (!folio)
return; return;
if (pmd_young(*pmd)) { if (pmdp_test_and_clear_young(vma, addr, pmd))
referenced = true; referenced = true;
*pmd = pmd_mkold(*pmd);
}
#ifdef CONFIG_MMU_NOTIFIER #ifdef CONFIG_MMU_NOTIFIER
if (mmu_notifier_clear_young(mm, addr, addr + HPAGE_PMD_SIZE)) if (mmu_notifier_clear_young(vma->vm_mm, addr, addr + HPAGE_PMD_SIZE))
referenced = true; referenced = true;
#endif /* CONFIG_MMU_NOTIFIER */ #endif /* CONFIG_MMU_NOTIFIER */

4
mm/damon/ops-common.h
View File

@ -9,8 +9,8 @@
struct folio *damon_get_folio(unsigned long pfn); struct folio *damon_get_folio(unsigned long pfn);
void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr); void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr);
void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr); void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr);
int damon_cold_score(struct damon_ctx *c, struct damon_region *r, int damon_cold_score(struct damon_ctx *c, struct damon_region *r,
struct damos *s); struct damos *s);

4
mm/damon/paddr.c
View File

@ -24,9 +24,9 @@ static bool __damon_pa_mkold(struct folio *folio, struct vm_area_struct *vma,
while (page_vma_mapped_walk(&pvmw)) { while (page_vma_mapped_walk(&pvmw)) {
addr = pvmw.address; addr = pvmw.address;
if (pvmw.pte) if (pvmw.pte)
damon_ptep_mkold(pvmw.pte, vma->vm_mm, addr); damon_ptep_mkold(pvmw.pte, vma, addr);
else else
damon_pmdp_mkold(pvmw.pmd, vma->vm_mm, addr); damon_pmdp_mkold(pvmw.pmd, vma, addr);
} }
return true; return true;
} }

4
mm/damon/vaddr.c
View File

@ -311,7 +311,7 @@ static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr,
} }
if (pmd_trans_huge(*pmd)) { if (pmd_trans_huge(*pmd)) {
damon_pmdp_mkold(pmd, walk->mm, addr); damon_pmdp_mkold(pmd, walk->vma, addr);
spin_unlock(ptl); spin_unlock(ptl);
return 0; return 0;
} }
@ -323,7 +323,7 @@ static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr,
pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
if (!pte_present(*pte)) if (!pte_present(*pte))
goto out; goto out;
damon_ptep_mkold(pte, walk->mm, addr); damon_ptep_mkold(pte, walk->vma, addr);
out: out:
pte_unmap_unlock(pte, ptl); pte_unmap_unlock(pte, ptl);
return 0; return 0;