mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git
synced 2024-12-29 09:12:07 +00:00
mm/gup: consistently name GUP-fast functions
Patch series "mm/gup: consistently call it GUP-fast".
Some cleanups around function names, comments and the config option of
"GUP-fast" -- GUP without "lock" safety belts on.
With this cleanup it's easy to judge which functions are GUP-fast
specific. We now consistently call it "GUP-fast", avoiding mixing it with
"fast GUP", "lockless", or simply "gup" (which I always considered
confusing in the ode).
So the magic now happens in functions that contain "gup_fast", whereby
gup_fast() is the entry point into that magic. Comments consistently
reference either "GUP-fast" or "gup_fast()".
This patch (of 3):
Let's consistently call the "fast-only" part of GUP "GUP-fast" and rename
all relevant internal functions to start with "gup_fast", to make it
clearer that this is not ordinary GUP. The current mixture of "lockless",
"gup" and "gup_fast" is confusing.
Further, avoid the term "huge" when talking about a "leaf" -- for example,
we nowadays check pmd_leaf() because pmd_huge() is gone. For the
"hugepd"/"hugepte" stuff, it's part of the name ("is_hugepd"), so that
stays.
What remains is the "external" interface:
* get_user_pages_fast_only()
* get_user_pages_fast()
* pin_user_pages_fast()
The high-level internal functions for GUP-fast (+slow fallback) are now:
* internal_get_user_pages_fast() -> gup_fast_fallback()
* lockless_pages_from_mm() -> gup_fast()
The basic GUP-fast walker functions:
* gup_pgd_range() -> gup_fast_pgd_range()
* gup_p4d_range() -> gup_fast_p4d_range()
* gup_pud_range() -> gup_fast_pud_range()
* gup_pmd_range() -> gup_fast_pmd_range()
* gup_pte_range() -> gup_fast_pte_range()
* gup_huge_pgd() -> gup_fast_pgd_leaf()
* gup_huge_pud() -> gup_fast_pud_leaf()
* gup_huge_pmd() -> gup_fast_pmd_leaf()
The weird hugepd stuff:
* gup_huge_pd() -> gup_fast_hugepd()
* gup_hugepte() -> gup_fast_hugepte()
The weird devmap stuff:
* __gup_device_huge_pud() -> gup_fast_devmap_pud_leaf()
* __gup_device_huge_pmd -> gup_fast_devmap_pmd_leaf()
* __gup_device_huge() -> gup_fast_devmap_leaf()
* undo_dev_pagemap() -> gup_fast_undo_dev_pagemap()
Helper functions:
* unpin_user_pages_lockless() -> gup_fast_unpin_user_pages()
* gup_fast_folio_allowed() is already properly named
* gup_fast_permitted() is already properly named
With "gup_fast()", we now even have a function that is referred to in
comment in mm/mmu_gather.c.
Link: https://lkml.kernel.org/r/20240402125516.223131-1-david@redhat.com
Link: https://lkml.kernel.org/r/20240402125516.223131-2-david@redhat.com
Signed-off-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Mike Rapoport (IBM) <rppt@kernel.org>
Reviewed-by: John Hubbard <jhubbard@nvidia.com>
Cc: Peter Xu <peterx@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
David Hildenbrand
Andrew Morton
parent
f6a8dd98a2
commit
23babe1934
205
mm/gup.c
205
mm/gup.c
@ -440,7 +440,7 @@ void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages,
|
||||
}
|
||||
EXPORT_SYMBOL(unpin_user_page_range_dirty_lock);
|
||||
|
||||
static void unpin_user_pages_lockless(struct page **pages, unsigned long npages)
|
||||
static void gup_fast_unpin_user_pages(struct page **pages, unsigned long npages)
|
||||
{
|
||||
unsigned long i;
|
||||
struct folio *folio;
|
||||
@ -525,9 +525,9 @@ static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
|
||||
return (__boundary - 1 < end - 1) ? __boundary : end;
|
||||
}
|
||||
|
||||
static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
|
||||
unsigned long end, unsigned int flags,
|
||||
struct page **pages, int *nr)
|
||||
static int gup_fast_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
|
||||
unsigned long end, unsigned int flags, struct page **pages,
|
||||
int *nr)
|
||||
{
|
||||
unsigned long pte_end;
|
||||
struct page *page;
|
||||
@ -577,7 +577,7 @@ static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
|
||||
* of the other folios. See writable_file_mapping_allowed() and
|
||||
* gup_fast_folio_allowed() for more information.
|
||||
*/
|
||||
static int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
|
||||
static int gup_fast_hugepd(hugepd_t hugepd, unsigned long addr,
|
||||
unsigned int pdshift, unsigned long end, unsigned int flags,
|
||||
struct page **pages, int *nr)
|
||||
{
|
||||
@ -588,7 +588,7 @@ static int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
|
||||
ptep = hugepte_offset(hugepd, addr, pdshift);
|
||||
do {
|
||||
next = hugepte_addr_end(addr, end, sz);
|
||||
if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr))
|
||||
if (!gup_fast_hugepte(ptep, sz, addr, end, flags, pages, nr))
|
||||
return 0;
|
||||
} while (ptep++, addr = next, addr != end);
|
||||
|
||||
@ -613,8 +613,8 @@ static struct page *follow_hugepd(struct vm_area_struct *vma, hugepd_t hugepd,
|
||||
h = hstate_vma(vma);
|
||||
ptep = hugepte_offset(hugepd, addr, pdshift);
|
||||
ptl = huge_pte_lock(h, vma->vm_mm, ptep);
|
||||
ret = gup_huge_pd(hugepd, addr, pdshift, addr + PAGE_SIZE,
|
||||
flags, &page, &nr);
|
||||
ret = gup_fast_hugepd(hugepd, addr, pdshift, addr + PAGE_SIZE,
|
||||
flags, &page, &nr);
|
||||
spin_unlock(ptl);
|
||||
|
||||
if (ret) {
|
||||
@ -626,7 +626,7 @@ static struct page *follow_hugepd(struct vm_area_struct *vma, hugepd_t hugepd,
|
||||
return NULL;
|
||||
}
|
||||
#else /* CONFIG_ARCH_HAS_HUGEPD */
|
||||
static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
|
||||
static inline int gup_fast_hugepd(hugepd_t hugepd, unsigned long addr,
|
||||
unsigned int pdshift, unsigned long end, unsigned int flags,
|
||||
struct page **pages, int *nr)
|
||||
{
|
||||
@ -2750,7 +2750,7 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
|
||||
EXPORT_SYMBOL(get_user_pages_unlocked);
|
||||
|
||||
/*
|
||||
* Fast GUP
|
||||
* GUP-fast
|
||||
*
|
||||
* get_user_pages_fast attempts to pin user pages by walking the page
|
||||
* tables directly and avoids taking locks. Thus the walker needs to be
|
||||
@ -2764,7 +2764,7 @@ EXPORT_SYMBOL(get_user_pages_unlocked);
|
||||
*
|
||||
* Another way to achieve this is to batch up page table containing pages
|
||||
* belonging to more than one mm_user, then rcu_sched a callback to free those
|
||||
* pages. Disabling interrupts will allow the fast_gup walker to both block
|
||||
* pages. Disabling interrupts will allow the gup_fast() walker to both block
|
||||
* the rcu_sched callback, and an IPI that we broadcast for splitting THPs
|
||||
* (which is a relatively rare event). The code below adopts this strategy.
|
||||
*
|
||||
@ -2873,9 +2873,8 @@ static bool gup_fast_folio_allowed(struct folio *folio, unsigned int flags)
|
||||
return !reject_file_backed || shmem_mapping(mapping);
|
||||
}
|
||||
|
||||
static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start,
|
||||
unsigned int flags,
|
||||
struct page **pages)
|
||||
static void __maybe_unused gup_fast_undo_dev_pagemap(int *nr, int nr_start,
|
||||
unsigned int flags, struct page **pages)
|
||||
{
|
||||
while ((*nr) - nr_start) {
|
||||
struct page *page = pages[--(*nr)];
|
||||
@ -2890,27 +2889,27 @@ static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start,
|
||||
|
||||
#ifdef CONFIG_ARCH_HAS_PTE_SPECIAL
|
||||
/*
|
||||
* Fast-gup relies on pte change detection to avoid concurrent pgtable
|
||||
* GUP-fast relies on pte change detection to avoid concurrent pgtable
|
||||
* operations.
|
||||
*
|
||||
* To pin the page, fast-gup needs to do below in order:
|
||||
* To pin the page, GUP-fast needs to do below in order:
|
||||
* (1) pin the page (by prefetching pte), then (2) check pte not changed.
|
||||
*
|
||||
* For the rest of pgtable operations where pgtable updates can be racy
|
||||
* with fast-gup, we need to do (1) clear pte, then (2) check whether page
|
||||
* with GUP-fast, we need to do (1) clear pte, then (2) check whether page
|
||||
* is pinned.
|
||||
*
|
||||
* Above will work for all pte-level operations, including THP split.
|
||||
*
|
||||
* For THP collapse, it's a bit more complicated because fast-gup may be
|
||||
* For THP collapse, it's a bit more complicated because GUP-fast may be
|
||||
* walking a pgtable page that is being freed (pte is still valid but pmd
|
||||
* can be cleared already). To avoid race in such condition, we need to
|
||||
* also check pmd here to make sure pmd doesn't change (corresponds to
|
||||
* pmdp_collapse_flush() in the THP collapse code path).
|
||||
*/
|
||||
static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
|
||||
unsigned long end, unsigned int flags,
|
||||
struct page **pages, int *nr)
|
||||
static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
|
||||
unsigned long end, unsigned int flags, struct page **pages,
|
||||
int *nr)
|
||||
{
|
||||
struct dev_pagemap *pgmap = NULL;
|
||||
int nr_start = *nr, ret = 0;
|
||||
@ -2943,7 +2942,7 @@ static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
|
||||
|
||||
pgmap = get_dev_pagemap(pte_pfn(pte), pgmap);
|
||||
if (unlikely(!pgmap)) {
|
||||
undo_dev_pagemap(nr, nr_start, flags, pages);
|
||||
gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages);
|
||||
goto pte_unmap;
|
||||
}
|
||||
} else if (pte_special(pte))
|
||||
@ -3007,20 +3006,19 @@ static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
|
||||
*
|
||||
* For a futex to be placed on a THP tail page, get_futex_key requires a
|
||||
* get_user_pages_fast_only implementation that can pin pages. Thus it's still
|
||||
* useful to have gup_huge_pmd even if we can't operate on ptes.
|
||||
* useful to have gup_fast_pmd_leaf even if we can't operate on ptes.
|
||||
*/
|
||||
static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
|
||||
unsigned long end, unsigned int flags,
|
||||
struct page **pages, int *nr)
|
||||
static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
|
||||
unsigned long end, unsigned int flags, struct page **pages,
|
||||
int *nr)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
#endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */
|
||||
|
||||
#if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
|
||||
static int __gup_device_huge(unsigned long pfn, unsigned long addr,
|
||||
unsigned long end, unsigned int flags,
|
||||
struct page **pages, int *nr)
|
||||
static int gup_fast_devmap_leaf(unsigned long pfn, unsigned long addr,
|
||||
unsigned long end, unsigned int flags, struct page **pages, int *nr)
|
||||
{
|
||||
int nr_start = *nr;
|
||||
struct dev_pagemap *pgmap = NULL;
|
||||
@ -3030,19 +3028,19 @@ static int __gup_device_huge(unsigned long pfn, unsigned long addr,
|
||||
|
||||
pgmap = get_dev_pagemap(pfn, pgmap);
|
||||
if (unlikely(!pgmap)) {
|
||||
undo_dev_pagemap(nr, nr_start, flags, pages);
|
||||
gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages);
|
||||
break;
|
||||
}
|
||||
|
||||
if (!(flags & FOLL_PCI_P2PDMA) && is_pci_p2pdma_page(page)) {
|
||||
undo_dev_pagemap(nr, nr_start, flags, pages);
|
||||
gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages);
|
||||
break;
|
||||
}
|
||||
|
||||
SetPageReferenced(page);
|
||||
pages[*nr] = page;
|
||||
if (unlikely(try_grab_page(page, flags))) {
|
||||
undo_dev_pagemap(nr, nr_start, flags, pages);
|
||||
gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages);
|
||||
break;
|
||||
}
|
||||
(*nr)++;
|
||||
@ -3053,62 +3051,62 @@ static int __gup_device_huge(unsigned long pfn, unsigned long addr,
|
||||
return addr == end;
|
||||
}
|
||||
|
||||
static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
|
||||
unsigned long end, unsigned int flags,
|
||||
struct page **pages, int *nr)
|
||||
static int gup_fast_devmap_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr,
|
||||
unsigned long end, unsigned int flags, struct page **pages,
|
||||
int *nr)
|
||||
{
|
||||
unsigned long fault_pfn;
|
||||
int nr_start = *nr;
|
||||
|
||||
fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
|
||||
if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
|
||||
if (!gup_fast_devmap_leaf(fault_pfn, addr, end, flags, pages, nr))
|
||||
return 0;
|
||||
|
||||
if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
|
||||
undo_dev_pagemap(nr, nr_start, flags, pages);
|
||||
gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages);
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
|
||||
unsigned long end, unsigned int flags,
|
||||
struct page **pages, int *nr)
|
||||
static int gup_fast_devmap_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr,
|
||||
unsigned long end, unsigned int flags, struct page **pages,
|
||||
int *nr)
|
||||
{
|
||||
unsigned long fault_pfn;
|
||||
int nr_start = *nr;
|
||||
|
||||
fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
|
||||
if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
|
||||
if (!gup_fast_devmap_leaf(fault_pfn, addr, end, flags, pages, nr))
|
||||
return 0;
|
||||
|
||||
if (unlikely(pud_val(orig) != pud_val(*pudp))) {
|
||||
undo_dev_pagemap(nr, nr_start, flags, pages);
|
||||
gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages);
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
#else
|
||||
static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
|
||||
unsigned long end, unsigned int flags,
|
||||
struct page **pages, int *nr)
|
||||
static int gup_fast_devmap_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr,
|
||||
unsigned long end, unsigned int flags, struct page **pages,
|
||||
int *nr)
|
||||
{
|
||||
BUILD_BUG();
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,
|
||||
unsigned long end, unsigned int flags,
|
||||
struct page **pages, int *nr)
|
||||
static int gup_fast_devmap_pud_leaf(pud_t pud, pud_t *pudp, unsigned long addr,
|
||||
unsigned long end, unsigned int flags, struct page **pages,
|
||||
int *nr)
|
||||
{
|
||||
BUILD_BUG();
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
|
||||
unsigned long end, unsigned int flags,
|
||||
struct page **pages, int *nr)
|
||||
static int gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr,
|
||||
unsigned long end, unsigned int flags, struct page **pages,
|
||||
int *nr)
|
||||
{
|
||||
struct page *page;
|
||||
struct folio *folio;
|
||||
@ -3120,8 +3118,8 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
|
||||
if (pmd_devmap(orig)) {
|
||||
if (unlikely(flags & FOLL_LONGTERM))
|
||||
return 0;
|
||||
return __gup_device_huge_pmd(orig, pmdp, addr, end, flags,
|
||||
pages, nr);
|
||||
return gup_fast_devmap_pmd_leaf(orig, pmdp, addr, end, flags,
|
||||
pages, nr);
|
||||
}
|
||||
|
||||
page = pmd_page(orig);
|
||||
@ -3150,9 +3148,9 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
|
||||
unsigned long end, unsigned int flags,
|
||||
struct page **pages, int *nr)
|
||||
static int gup_fast_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr,
|
||||
unsigned long end, unsigned int flags, struct page **pages,
|
||||
int *nr)
|
||||
{
|
||||
struct page *page;
|
||||
struct folio *folio;
|
||||
@ -3164,8 +3162,8 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
|
||||
if (pud_devmap(orig)) {
|
||||
if (unlikely(flags & FOLL_LONGTERM))
|
||||
return 0;
|
||||
return __gup_device_huge_pud(orig, pudp, addr, end, flags,
|
||||
pages, nr);
|
||||
return gup_fast_devmap_pud_leaf(orig, pudp, addr, end, flags,
|
||||
pages, nr);
|
||||
}
|
||||
|
||||
page = pud_page(orig);
|
||||
@ -3195,9 +3193,9 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
|
||||
unsigned long end, unsigned int flags,
|
||||
struct page **pages, int *nr)
|
||||
static int gup_fast_pgd_leaf(pgd_t orig, pgd_t *pgdp, unsigned long addr,
|
||||
unsigned long end, unsigned int flags, struct page **pages,
|
||||
int *nr)
|
||||
{
|
||||
int refs;
|
||||
struct page *page;
|
||||
@ -3235,8 +3233,9 @@ static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned long end,
|
||||
unsigned int flags, struct page **pages, int *nr)
|
||||
static int gup_fast_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr,
|
||||
unsigned long end, unsigned int flags, struct page **pages,
|
||||
int *nr)
|
||||
{
|
||||
unsigned long next;
|
||||
pmd_t *pmdp;
|
||||
@ -3250,11 +3249,11 @@ static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned lo
|
||||
return 0;
|
||||
|
||||
if (unlikely(pmd_leaf(pmd))) {
|
||||
/* See gup_pte_range() */
|
||||
/* See gup_fast_pte_range() */
|
||||
if (pmd_protnone(pmd))
|
||||
return 0;
|
||||
|
||||
if (!gup_huge_pmd(pmd, pmdp, addr, next, flags,
|
||||
if (!gup_fast_pmd_leaf(pmd, pmdp, addr, next, flags,
|
||||
pages, nr))
|
||||
return 0;
|
||||
|
||||
@ -3263,18 +3262,20 @@ static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned lo
|
||||
* architecture have different format for hugetlbfs
|
||||
* pmd format and THP pmd format
|
||||
*/
|
||||
if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
|
||||
PMD_SHIFT, next, flags, pages, nr))
|
||||
if (!gup_fast_hugepd(__hugepd(pmd_val(pmd)), addr,
|
||||
PMD_SHIFT, next, flags, pages, nr))
|
||||
return 0;
|
||||
} else if (!gup_pte_range(pmd, pmdp, addr, next, flags, pages, nr))
|
||||
} else if (!gup_fast_pte_range(pmd, pmdp, addr, next, flags,
|
||||
pages, nr))
|
||||
return 0;
|
||||
} while (pmdp++, addr = next, addr != end);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, unsigned long end,
|
||||
unsigned int flags, struct page **pages, int *nr)
|
||||
static int gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr,
|
||||
unsigned long end, unsigned int flags, struct page **pages,
|
||||
int *nr)
|
||||
{
|
||||
unsigned long next;
|
||||
pud_t *pudp;
|
||||
@ -3287,22 +3288,24 @@ static int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, unsigned lo
|
||||
if (unlikely(!pud_present(pud)))
|
||||
return 0;
|
||||
if (unlikely(pud_leaf(pud))) {
|
||||
if (!gup_huge_pud(pud, pudp, addr, next, flags,
|
||||
pages, nr))
|
||||
if (!gup_fast_pud_leaf(pud, pudp, addr, next, flags,
|
||||
pages, nr))
|
||||
return 0;
|
||||
} else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) {
|
||||
if (!gup_huge_pd(__hugepd(pud_val(pud)), addr,
|
||||
PUD_SHIFT, next, flags, pages, nr))
|
||||
if (!gup_fast_hugepd(__hugepd(pud_val(pud)), addr,
|
||||
PUD_SHIFT, next, flags, pages, nr))
|
||||
return 0;
|
||||
} else if (!gup_pmd_range(pudp, pud, addr, next, flags, pages, nr))
|
||||
} else if (!gup_fast_pmd_range(pudp, pud, addr, next, flags,
|
||||
pages, nr))
|
||||
return 0;
|
||||
} while (pudp++, addr = next, addr != end);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned long end,
|
||||
unsigned int flags, struct page **pages, int *nr)
|
||||
static int gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr,
|
||||
unsigned long end, unsigned int flags, struct page **pages,
|
||||
int *nr)
|
||||
{
|
||||
unsigned long next;
|
||||
p4d_t *p4dp;
|
||||
@ -3316,17 +3319,18 @@ static int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned lo
|
||||
return 0;
|
||||
BUILD_BUG_ON(p4d_leaf(p4d));
|
||||
if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) {
|
||||
if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,
|
||||
P4D_SHIFT, next, flags, pages, nr))
|
||||
if (!gup_fast_hugepd(__hugepd(p4d_val(p4d)), addr,
|
||||
P4D_SHIFT, next, flags, pages, nr))
|
||||
return 0;
|
||||
} else if (!gup_pud_range(p4dp, p4d, addr, next, flags, pages, nr))
|
||||
} else if (!gup_fast_pud_range(p4dp, p4d, addr, next, flags,
|
||||
pages, nr))
|
||||
return 0;
|
||||
} while (p4dp++, addr = next, addr != end);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static void gup_pgd_range(unsigned long addr, unsigned long end,
|
||||
static void gup_fast_pgd_range(unsigned long addr, unsigned long end,
|
||||
unsigned int flags, struct page **pages, int *nr)
|
||||
{
|
||||
unsigned long next;
|
||||
@ -3340,19 +3344,20 @@ static void gup_pgd_range(unsigned long addr, unsigned long end,
|
||||
if (pgd_none(pgd))
|
||||
return;
|
||||
if (unlikely(pgd_leaf(pgd))) {
|
||||
if (!gup_huge_pgd(pgd, pgdp, addr, next, flags,
|
||||
pages, nr))
|
||||
if (!gup_fast_pgd_leaf(pgd, pgdp, addr, next, flags,
|
||||
pages, nr))
|
||||
return;
|
||||
} else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) {
|
||||
if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
|
||||
PGDIR_SHIFT, next, flags, pages, nr))
|
||||
if (!gup_fast_hugepd(__hugepd(pgd_val(pgd)), addr,
|
||||
PGDIR_SHIFT, next, flags, pages, nr))
|
||||
return;
|
||||
} else if (!gup_p4d_range(pgdp, pgd, addr, next, flags, pages, nr))
|
||||
} else if (!gup_fast_p4d_range(pgdp, pgd, addr, next, flags,
|
||||
pages, nr))
|
||||
return;
|
||||
} while (pgdp++, addr = next, addr != end);
|
||||
}
|
||||
#else
|
||||
static inline void gup_pgd_range(unsigned long addr, unsigned long end,
|
||||
static inline void gup_fast_pgd_range(unsigned long addr, unsigned long end,
|
||||
unsigned int flags, struct page **pages, int *nr)
|
||||
{
|
||||
}
|
||||
@ -3369,10 +3374,8 @@ static bool gup_fast_permitted(unsigned long start, unsigned long end)
|
||||
}
|
||||
#endif
|
||||
|
||||
static unsigned long lockless_pages_from_mm(unsigned long start,
|
||||
unsigned long end,
|
||||
unsigned int gup_flags,
|
||||
struct page **pages)
|
||||
static unsigned long gup_fast(unsigned long start, unsigned long end,
|
||||
unsigned int gup_flags, struct page **pages)
|
||||
{
|
||||
unsigned long flags;
|
||||
int nr_pinned = 0;
|
||||
@ -3400,16 +3403,16 @@ static unsigned long lockless_pages_from_mm(unsigned long start,
|
||||
* that come from THPs splitting.
|
||||
*/
|
||||
local_irq_save(flags);
|
||||
gup_pgd_range(start, end, gup_flags, pages, &nr_pinned);
|
||||
gup_fast_pgd_range(start, end, gup_flags, pages, &nr_pinned);
|
||||
local_irq_restore(flags);
|
||||
|
||||
/*
|
||||
* When pinning pages for DMA there could be a concurrent write protect
|
||||
* from fork() via copy_page_range(), in this case always fail fast GUP.
|
||||
* from fork() via copy_page_range(), in this case always fail GUP-fast.
|
||||
*/
|
||||
if (gup_flags & FOLL_PIN) {
|
||||
if (read_seqcount_retry(¤t->mm->write_protect_seq, seq)) {
|
||||
unpin_user_pages_lockless(pages, nr_pinned);
|
||||
gup_fast_unpin_user_pages(pages, nr_pinned);
|
||||
return 0;
|
||||
} else {
|
||||
sanity_check_pinned_pages(pages, nr_pinned);
|
||||
@ -3418,10 +3421,8 @@ static unsigned long lockless_pages_from_mm(unsigned long start,
|
||||
return nr_pinned;
|
||||
}
|
||||
|
||||
static int internal_get_user_pages_fast(unsigned long start,
|
||||
unsigned long nr_pages,
|
||||
unsigned int gup_flags,
|
||||
struct page **pages)
|
||||
static int gup_fast_fallback(unsigned long start, unsigned long nr_pages,
|
||||
unsigned int gup_flags, struct page **pages)
|
||||
{
|
||||
unsigned long len, end;
|
||||
unsigned long nr_pinned;
|
||||
@ -3449,7 +3450,7 @@ static int internal_get_user_pages_fast(unsigned long start,
|
||||
if (unlikely(!access_ok((void __user *)start, len)))
|
||||
return -EFAULT;
|
||||
|
||||
nr_pinned = lockless_pages_from_mm(start, end, gup_flags, pages);
|
||||
nr_pinned = gup_fast(start, end, gup_flags, pages);
|
||||
if (nr_pinned == nr_pages || gup_flags & FOLL_FAST_ONLY)
|
||||
return nr_pinned;
|
||||
|
||||
@ -3503,7 +3504,7 @@ int get_user_pages_fast_only(unsigned long start, int nr_pages,
|
||||
FOLL_GET | FOLL_FAST_ONLY))
|
||||
return -EINVAL;
|
||||
|
||||
return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
|
||||
return gup_fast_fallback(start, nr_pages, gup_flags, pages);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(get_user_pages_fast_only);
|
||||
|
||||
@ -3534,7 +3535,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages,
|
||||
*/
|
||||
if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_GET))
|
||||
return -EINVAL;
|
||||
return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
|
||||
return gup_fast_fallback(start, nr_pages, gup_flags, pages);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(get_user_pages_fast);
|
||||
|
||||
@ -3562,7 +3563,7 @@ int pin_user_pages_fast(unsigned long start, int nr_pages,
|
||||
{
|
||||
if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN))
|
||||
return -EINVAL;
|
||||
return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
|
||||
return gup_fast_fallback(start, nr_pages, gup_flags, pages);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(pin_user_pages_fast);
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user