blob: 5168665da584be7f4d454d3702cb23661af6ce76 [file] [log] [blame]
Thomas Gleixner457c8992019-05-19 13:08:55 +01001// SPDX-License-Identifier: GPL-2.0-only
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07002#include <linux/kernel.h>
3#include <linux/errno.h>
4#include <linux/err.h>
5#include <linux/spinlock.h>
6
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07007#include <linux/mm.h>
Dan Williams3565fce2016-01-15 16:56:55 -08008#include <linux/memremap.h>
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07009#include <linux/pagemap.h>
10#include <linux/rmap.h>
11#include <linux/swap.h>
12#include <linux/swapops.h>
13
Ingo Molnar174cd4b2017-02-02 19:15:33 +010014#include <linux/sched/signal.h>
Steve Capper2667f502014-10-09 15:29:14 -070015#include <linux/rwsem.h>
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +053016#include <linux/hugetlb.h>
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -080017#include <linux/migrate.h>
18#include <linux/mm_inline.h>
19#include <linux/sched/mm.h>
Kirill A. Shutemov1027e442015-09-04 15:47:55 -070020
Dave Hansen33a709b2016-02-12 13:02:19 -080021#include <asm/mmu_context.h>
Steve Capper2667f502014-10-09 15:29:14 -070022#include <asm/pgtable.h>
Kirill A. Shutemov1027e442015-09-04 15:47:55 -070023#include <asm/tlbflush.h>
Steve Capper2667f502014-10-09 15:29:14 -070024
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -070025#include "internal.h"
26
Keith Buschdf06b372018-10-26 15:10:28 -070027struct follow_page_context {
28 struct dev_pagemap *pgmap;
29 unsigned int page_mask;
30};
31
John Hubbard47e29d32020-04-01 21:05:33 -070032static void hpage_pincount_add(struct page *page, int refs)
33{
34 VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
35 VM_BUG_ON_PAGE(page != compound_head(page), page);
36
37 atomic_add(refs, compound_pincount_ptr(page));
38}
39
40static void hpage_pincount_sub(struct page *page, int refs)
41{
42 VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
43 VM_BUG_ON_PAGE(page != compound_head(page), page);
44
45 atomic_sub(refs, compound_pincount_ptr(page));
46}
47
John Hubbarda707cdd2020-01-30 22:12:21 -080048/*
49 * Return the compound head page with ref appropriately incremented,
50 * or NULL if that failed.
51 */
52static inline struct page *try_get_compound_head(struct page *page, int refs)
53{
54 struct page *head = compound_head(page);
55
56 if (WARN_ON_ONCE(page_ref_count(head) < 0))
57 return NULL;
58 if (unlikely(!page_cache_add_speculative(head, refs)))
59 return NULL;
60 return head;
61}
62
John Hubbard3faa52c2020-04-01 21:05:29 -070063/*
64 * try_grab_compound_head() - attempt to elevate a page's refcount, by a
65 * flags-dependent amount.
66 *
67 * "grab" names in this file mean, "look at flags to decide whether to use
68 * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.
69 *
70 * Either FOLL_PIN or FOLL_GET (or neither) must be set, but not both at the
71 * same time. (That's true throughout the get_user_pages*() and
72 * pin_user_pages*() APIs.) Cases:
73 *
74 * FOLL_GET: page's refcount will be incremented by 1.
75 * FOLL_PIN: page's refcount will be incremented by GUP_PIN_COUNTING_BIAS.
76 *
77 * Return: head page (with refcount appropriately incremented) for success, or
78 * NULL upon failure. If neither FOLL_GET nor FOLL_PIN was set, that's
79 * considered failure, and furthermore, a likely bug in the caller, so a warning
80 * is also emitted.
81 */
82static __maybe_unused struct page *try_grab_compound_head(struct page *page,
83 int refs,
84 unsigned int flags)
85{
86 if (flags & FOLL_GET)
87 return try_get_compound_head(page, refs);
88 else if (flags & FOLL_PIN) {
John Hubbard1970dc62020-04-01 21:05:37 -070089 int orig_refs = refs;
90
John Hubbard47e29d32020-04-01 21:05:33 -070091 /*
Pingfan Liudf3a0a22020-04-01 21:06:04 -070092 * Can't do FOLL_LONGTERM + FOLL_PIN with CMA in the gup fast
93 * path, so fail and let the caller fall back to the slow path.
94 */
95 if (unlikely(flags & FOLL_LONGTERM) &&
96 is_migrate_cma_page(page))
97 return NULL;
98
99 /*
John Hubbard47e29d32020-04-01 21:05:33 -0700100 * When pinning a compound page of order > 1 (which is what
101 * hpage_pincount_available() checks for), use an exact count to
102 * track it, via hpage_pincount_add/_sub().
103 *
104 * However, be sure to *also* increment the normal page refcount
105 * field at least once, so that the page really is pinned.
106 */
107 if (!hpage_pincount_available(page))
108 refs *= GUP_PIN_COUNTING_BIAS;
109
110 page = try_get_compound_head(page, refs);
111 if (!page)
112 return NULL;
113
114 if (hpage_pincount_available(page))
115 hpage_pincount_add(page, refs);
116
John Hubbard1970dc62020-04-01 21:05:37 -0700117 mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED,
118 orig_refs);
119
John Hubbard47e29d32020-04-01 21:05:33 -0700120 return page;
John Hubbard3faa52c2020-04-01 21:05:29 -0700121 }
122
123 WARN_ON_ONCE(1);
124 return NULL;
125}
126
127/**
128 * try_grab_page() - elevate a page's refcount by a flag-dependent amount
129 *
130 * This might not do anything at all, depending on the flags argument.
131 *
132 * "grab" names in this file mean, "look at flags to decide whether to use
133 * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.
134 *
135 * @page: pointer to page to be grabbed
136 * @flags: gup flags: these are the FOLL_* flag values.
137 *
138 * Either FOLL_PIN or FOLL_GET (or neither) may be set, but not both at the same
139 * time. Cases:
140 *
141 * FOLL_GET: page's refcount will be incremented by 1.
142 * FOLL_PIN: page's refcount will be incremented by GUP_PIN_COUNTING_BIAS.
143 *
144 * Return: true for success, or if no action was required (if neither FOLL_PIN
145 * nor FOLL_GET was set, nothing is done). False for failure: FOLL_GET or
146 * FOLL_PIN was set, but the page could not be grabbed.
147 */
148bool __must_check try_grab_page(struct page *page, unsigned int flags)
149{
150 WARN_ON_ONCE((flags & (FOLL_GET | FOLL_PIN)) == (FOLL_GET | FOLL_PIN));
151
152 if (flags & FOLL_GET)
153 return try_get_page(page);
154 else if (flags & FOLL_PIN) {
John Hubbard47e29d32020-04-01 21:05:33 -0700155 int refs = 1;
156
John Hubbard3faa52c2020-04-01 21:05:29 -0700157 page = compound_head(page);
158
159 if (WARN_ON_ONCE(page_ref_count(page) <= 0))
160 return false;
161
John Hubbard47e29d32020-04-01 21:05:33 -0700162 if (hpage_pincount_available(page))
163 hpage_pincount_add(page, 1);
164 else
165 refs = GUP_PIN_COUNTING_BIAS;
166
167 /*
168 * Similar to try_grab_compound_head(): even if using the
169 * hpage_pincount_add/_sub() routines, be sure to
170 * *also* increment the normal page refcount field at least
171 * once, so that the page really is pinned.
172 */
173 page_ref_add(page, refs);
John Hubbard1970dc62020-04-01 21:05:37 -0700174
175 mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED, 1);
John Hubbard3faa52c2020-04-01 21:05:29 -0700176 }
177
178 return true;
179}
180
181#ifdef CONFIG_DEV_PAGEMAP_OPS
182static bool __unpin_devmap_managed_user_page(struct page *page)
183{
John Hubbard47e29d32020-04-01 21:05:33 -0700184 int count, refs = 1;
John Hubbard3faa52c2020-04-01 21:05:29 -0700185
186 if (!page_is_devmap_managed(page))
187 return false;
188
John Hubbard47e29d32020-04-01 21:05:33 -0700189 if (hpage_pincount_available(page))
190 hpage_pincount_sub(page, 1);
191 else
192 refs = GUP_PIN_COUNTING_BIAS;
193
194 count = page_ref_sub_return(page, refs);
John Hubbard3faa52c2020-04-01 21:05:29 -0700195
John Hubbard1970dc62020-04-01 21:05:37 -0700196 mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_RELEASED, 1);
John Hubbard3faa52c2020-04-01 21:05:29 -0700197 /*
198 * devmap page refcounts are 1-based, rather than 0-based: if
199 * refcount is 1, then the page is free and the refcount is
200 * stable because nobody holds a reference on the page.
201 */
202 if (count == 1)
203 free_devmap_managed_page(page);
204 else if (!count)
205 __put_page(page);
206
207 return true;
208}
209#else
210static bool __unpin_devmap_managed_user_page(struct page *page)
211{
212 return false;
213}
214#endif /* CONFIG_DEV_PAGEMAP_OPS */
215
216/**
217 * unpin_user_page() - release a dma-pinned page
218 * @page: pointer to page to be released
219 *
220 * Pages that were pinned via pin_user_pages*() must be released via either
221 * unpin_user_page(), or one of the unpin_user_pages*() routines. This is so
222 * that such pages can be separately tracked and uniquely handled. In
223 * particular, interactions with RDMA and filesystems need special handling.
224 */
225void unpin_user_page(struct page *page)
226{
John Hubbard47e29d32020-04-01 21:05:33 -0700227 int refs = 1;
228
John Hubbard3faa52c2020-04-01 21:05:29 -0700229 page = compound_head(page);
230
231 /*
232 * For devmap managed pages we need to catch refcount transition from
233 * GUP_PIN_COUNTING_BIAS to 1, when refcount reach one it means the
234 * page is free and we need to inform the device driver through
235 * callback. See include/linux/memremap.h and HMM for details.
236 */
237 if (__unpin_devmap_managed_user_page(page))
238 return;
239
John Hubbard47e29d32020-04-01 21:05:33 -0700240 if (hpage_pincount_available(page))
241 hpage_pincount_sub(page, 1);
242 else
243 refs = GUP_PIN_COUNTING_BIAS;
244
245 if (page_ref_sub_and_test(page, refs))
John Hubbard3faa52c2020-04-01 21:05:29 -0700246 __put_page(page);
John Hubbard1970dc62020-04-01 21:05:37 -0700247
248 mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_RELEASED, 1);
John Hubbard3faa52c2020-04-01 21:05:29 -0700249}
250EXPORT_SYMBOL(unpin_user_page);
251
John Hubbardfc1d8e72019-05-13 17:19:08 -0700252/**
John Hubbardf1f6a7d2020-01-30 22:13:35 -0800253 * unpin_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages
akpm@linux-foundation.org2d15eb32019-09-23 15:35:04 -0700254 * @pages: array of pages to be maybe marked dirty, and definitely released.
John Hubbardfc1d8e72019-05-13 17:19:08 -0700255 * @npages: number of pages in the @pages array.
akpm@linux-foundation.org2d15eb32019-09-23 15:35:04 -0700256 * @make_dirty: whether to mark the pages dirty
John Hubbardfc1d8e72019-05-13 17:19:08 -0700257 *
258 * "gup-pinned page" refers to a page that has had one of the get_user_pages()
259 * variants called on that page.
260 *
261 * For each page in the @pages array, make that page (or its head page, if a
akpm@linux-foundation.org2d15eb32019-09-23 15:35:04 -0700262 * compound page) dirty, if @make_dirty is true, and if the page was previously
John Hubbardf1f6a7d2020-01-30 22:13:35 -0800263 * listed as clean. In any case, releases all pages using unpin_user_page(),
264 * possibly via unpin_user_pages(), for the non-dirty case.
John Hubbardfc1d8e72019-05-13 17:19:08 -0700265 *
John Hubbardf1f6a7d2020-01-30 22:13:35 -0800266 * Please see the unpin_user_page() documentation for details.
John Hubbardfc1d8e72019-05-13 17:19:08 -0700267 *
akpm@linux-foundation.org2d15eb32019-09-23 15:35:04 -0700268 * set_page_dirty_lock() is used internally. If instead, set_page_dirty() is
269 * required, then the caller should a) verify that this is really correct,
270 * because _lock() is usually required, and b) hand code it:
John Hubbardf1f6a7d2020-01-30 22:13:35 -0800271 * set_page_dirty_lock(), unpin_user_page().
John Hubbardfc1d8e72019-05-13 17:19:08 -0700272 *
273 */
John Hubbardf1f6a7d2020-01-30 22:13:35 -0800274void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
275 bool make_dirty)
John Hubbardfc1d8e72019-05-13 17:19:08 -0700276{
akpm@linux-foundation.org2d15eb32019-09-23 15:35:04 -0700277 unsigned long index;
John Hubbardfc1d8e72019-05-13 17:19:08 -0700278
akpm@linux-foundation.org2d15eb32019-09-23 15:35:04 -0700279 /*
280 * TODO: this can be optimized for huge pages: if a series of pages is
281 * physically contiguous and part of the same compound page, then a
282 * single operation to the head page should suffice.
283 */
284
285 if (!make_dirty) {
John Hubbardf1f6a7d2020-01-30 22:13:35 -0800286 unpin_user_pages(pages, npages);
akpm@linux-foundation.org2d15eb32019-09-23 15:35:04 -0700287 return;
288 }
289
290 for (index = 0; index < npages; index++) {
291 struct page *page = compound_head(pages[index]);
292 /*
293 * Checking PageDirty at this point may race with
294 * clear_page_dirty_for_io(), but that's OK. Two key
295 * cases:
296 *
297 * 1) This code sees the page as already dirty, so it
298 * skips the call to set_page_dirty(). That could happen
299 * because clear_page_dirty_for_io() called
300 * page_mkclean(), followed by set_page_dirty().
301 * However, now the page is going to get written back,
302 * which meets the original intention of setting it
303 * dirty, so all is well: clear_page_dirty_for_io() goes
304 * on to call TestClearPageDirty(), and write the page
305 * back.
306 *
307 * 2) This code sees the page as clean, so it calls
308 * set_page_dirty(). The page stays dirty, despite being
309 * written back, so it gets written back again in the
310 * next writeback cycle. This is harmless.
311 */
312 if (!PageDirty(page))
313 set_page_dirty_lock(page);
John Hubbardf1f6a7d2020-01-30 22:13:35 -0800314 unpin_user_page(page);
akpm@linux-foundation.org2d15eb32019-09-23 15:35:04 -0700315 }
John Hubbardfc1d8e72019-05-13 17:19:08 -0700316}
John Hubbardf1f6a7d2020-01-30 22:13:35 -0800317EXPORT_SYMBOL(unpin_user_pages_dirty_lock);
John Hubbardfc1d8e72019-05-13 17:19:08 -0700318
319/**
John Hubbardf1f6a7d2020-01-30 22:13:35 -0800320 * unpin_user_pages() - release an array of gup-pinned pages.
John Hubbardfc1d8e72019-05-13 17:19:08 -0700321 * @pages: array of pages to be marked dirty and released.
322 * @npages: number of pages in the @pages array.
323 *
John Hubbardf1f6a7d2020-01-30 22:13:35 -0800324 * For each page in the @pages array, release the page using unpin_user_page().
John Hubbardfc1d8e72019-05-13 17:19:08 -0700325 *
John Hubbardf1f6a7d2020-01-30 22:13:35 -0800326 * Please see the unpin_user_page() documentation for details.
John Hubbardfc1d8e72019-05-13 17:19:08 -0700327 */
John Hubbardf1f6a7d2020-01-30 22:13:35 -0800328void unpin_user_pages(struct page **pages, unsigned long npages)
John Hubbardfc1d8e72019-05-13 17:19:08 -0700329{
330 unsigned long index;
331
332 /*
333 * TODO: this can be optimized for huge pages: if a series of pages is
334 * physically contiguous and part of the same compound page, then a
335 * single operation to the head page should suffice.
336 */
337 for (index = 0; index < npages; index++)
John Hubbardf1f6a7d2020-01-30 22:13:35 -0800338 unpin_user_page(pages[index]);
John Hubbardfc1d8e72019-05-13 17:19:08 -0700339}
John Hubbardf1f6a7d2020-01-30 22:13:35 -0800340EXPORT_SYMBOL(unpin_user_pages);
John Hubbardfc1d8e72019-05-13 17:19:08 -0700341
Christoph Hellwig050a9ad2019-07-11 20:57:21 -0700342#ifdef CONFIG_MMU
Kirill A. Shutemov69e68b42014-06-04 16:08:11 -0700343static struct page *no_page_table(struct vm_area_struct *vma,
344 unsigned int flags)
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700345{
Kirill A. Shutemov69e68b42014-06-04 16:08:11 -0700346 /*
347 * When core dumping an enormous anonymous area that nobody
348 * has touched so far, we don't want to allocate unnecessary pages or
349 * page tables. Return error instead of NULL to skip handle_mm_fault,
350 * then get_dump_page() will return NULL to leave a hole in the dump.
351 * But we can only make this optimization where a hole would surely
352 * be zero-filled if handle_mm_fault() actually did handle it.
353 */
354 if ((flags & FOLL_DUMP) && (!vma->vm_ops || !vma->vm_ops->fault))
355 return ERR_PTR(-EFAULT);
356 return NULL;
357}
358
Kirill A. Shutemov1027e442015-09-04 15:47:55 -0700359static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address,
360 pte_t *pte, unsigned int flags)
361{
362 /* No page to get reference */
363 if (flags & FOLL_GET)
364 return -EFAULT;
365
366 if (flags & FOLL_TOUCH) {
367 pte_t entry = *pte;
368
369 if (flags & FOLL_WRITE)
370 entry = pte_mkdirty(entry);
371 entry = pte_mkyoung(entry);
372
373 if (!pte_same(*pte, entry)) {
374 set_pte_at(vma->vm_mm, address, pte, entry);
375 update_mmu_cache(vma, address, pte);
376 }
377 }
378
379 /* Proper page table entry exists, but no corresponding struct page */
380 return -EEXIST;
381}
382
Linus Torvalds19be0ea2016-10-13 13:07:36 -0700383/*
384 * FOLL_FORCE can write to even unwritable pte's, but only
385 * after we've gone through a COW cycle and they are dirty.
386 */
387static inline bool can_follow_write_pte(pte_t pte, unsigned int flags)
388{
Linus Torvaldsf6f37322017-12-15 18:53:22 -0800389 return pte_write(pte) ||
Linus Torvalds19be0ea2016-10-13 13:07:36 -0700390 ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pte_dirty(pte));
391}
392
Kirill A. Shutemov69e68b42014-06-04 16:08:11 -0700393static struct page *follow_page_pte(struct vm_area_struct *vma,
Keith Buschdf06b372018-10-26 15:10:28 -0700394 unsigned long address, pmd_t *pmd, unsigned int flags,
395 struct dev_pagemap **pgmap)
Kirill A. Shutemov69e68b42014-06-04 16:08:11 -0700396{
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700397 struct mm_struct *mm = vma->vm_mm;
Kirill A. Shutemov69e68b42014-06-04 16:08:11 -0700398 struct page *page;
399 spinlock_t *ptl;
400 pte_t *ptep, pte;
Claudio Imbrendaf28d4362020-04-01 21:05:56 -0700401 int ret;
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700402
John Hubbardeddb1c22020-01-30 22:12:54 -0800403 /* FOLL_GET and FOLL_PIN are mutually exclusive. */
404 if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) ==
405 (FOLL_PIN | FOLL_GET)))
406 return ERR_PTR(-EINVAL);
Kirill A. Shutemov69e68b42014-06-04 16:08:11 -0700407retry:
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700408 if (unlikely(pmd_bad(*pmd)))
Kirill A. Shutemov69e68b42014-06-04 16:08:11 -0700409 return no_page_table(vma, flags);
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700410
411 ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700412 pte = *ptep;
413 if (!pte_present(pte)) {
414 swp_entry_t entry;
415 /*
416 * KSM's break_ksm() relies upon recognizing a ksm page
417 * even while it is being migrated, so for that case we
418 * need migration_entry_wait().
419 */
420 if (likely(!(flags & FOLL_MIGRATION)))
421 goto no_page;
Kirill A. Shutemov0661a332015-02-10 14:10:04 -0800422 if (pte_none(pte))
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700423 goto no_page;
424 entry = pte_to_swp_entry(pte);
425 if (!is_migration_entry(entry))
426 goto no_page;
427 pte_unmap_unlock(ptep, ptl);
428 migration_entry_wait(mm, pmd, address);
Kirill A. Shutemov69e68b42014-06-04 16:08:11 -0700429 goto retry;
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700430 }
Mel Gorman8a0516e2015-02-12 14:58:22 -0800431 if ((flags & FOLL_NUMA) && pte_protnone(pte))
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700432 goto no_page;
Linus Torvalds19be0ea2016-10-13 13:07:36 -0700433 if ((flags & FOLL_WRITE) && !can_follow_write_pte(pte, flags)) {
Kirill A. Shutemov69e68b42014-06-04 16:08:11 -0700434 pte_unmap_unlock(ptep, ptl);
435 return NULL;
436 }
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700437
438 page = vm_normal_page(vma, address, pte);
John Hubbard3faa52c2020-04-01 21:05:29 -0700439 if (!page && pte_devmap(pte) && (flags & (FOLL_GET | FOLL_PIN))) {
Dan Williams3565fce2016-01-15 16:56:55 -0800440 /*
John Hubbard3faa52c2020-04-01 21:05:29 -0700441 * Only return device mapping pages in the FOLL_GET or FOLL_PIN
442 * case since they are only valid while holding the pgmap
443 * reference.
Dan Williams3565fce2016-01-15 16:56:55 -0800444 */
Keith Buschdf06b372018-10-26 15:10:28 -0700445 *pgmap = get_dev_pagemap(pte_pfn(pte), *pgmap);
446 if (*pgmap)
Dan Williams3565fce2016-01-15 16:56:55 -0800447 page = pte_page(pte);
448 else
449 goto no_page;
450 } else if (unlikely(!page)) {
Kirill A. Shutemov1027e442015-09-04 15:47:55 -0700451 if (flags & FOLL_DUMP) {
452 /* Avoid special (like zero) pages in core dumps */
453 page = ERR_PTR(-EFAULT);
454 goto out;
455 }
456
457 if (is_zero_pfn(pte_pfn(pte))) {
458 page = pte_page(pte);
459 } else {
Kirill A. Shutemov1027e442015-09-04 15:47:55 -0700460 ret = follow_pfn_pte(vma, address, ptep, flags);
461 page = ERR_PTR(ret);
462 goto out;
463 }
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700464 }
465
Kirill A. Shutemov6742d292016-01-15 16:52:28 -0800466 if (flags & FOLL_SPLIT && PageTransCompound(page)) {
Kirill A. Shutemov6742d292016-01-15 16:52:28 -0800467 get_page(page);
468 pte_unmap_unlock(ptep, ptl);
469 lock_page(page);
470 ret = split_huge_page(page);
471 unlock_page(page);
472 put_page(page);
473 if (ret)
474 return ERR_PTR(ret);
475 goto retry;
476 }
477
John Hubbard3faa52c2020-04-01 21:05:29 -0700478 /* try_grab_page() does nothing unless FOLL_GET or FOLL_PIN is set. */
479 if (unlikely(!try_grab_page(page, flags))) {
480 page = ERR_PTR(-ENOMEM);
481 goto out;
Linus Torvalds8fde12c2019-04-11 10:49:19 -0700482 }
Claudio Imbrendaf28d4362020-04-01 21:05:56 -0700483 /*
484 * We need to make the page accessible if and only if we are going
485 * to access its content (the FOLL_PIN case). Please see
486 * Documentation/core-api/pin_user_pages.rst for details.
487 */
488 if (flags & FOLL_PIN) {
489 ret = arch_make_page_accessible(page);
490 if (ret) {
491 unpin_user_page(page);
492 page = ERR_PTR(ret);
493 goto out;
494 }
495 }
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700496 if (flags & FOLL_TOUCH) {
497 if ((flags & FOLL_WRITE) &&
498 !pte_dirty(pte) && !PageDirty(page))
499 set_page_dirty(page);
500 /*
501 * pte_mkyoung() would be more correct here, but atomic care
502 * is needed to avoid losing the dirty bit: it is easier to use
503 * mark_page_accessed().
504 */
505 mark_page_accessed(page);
506 }
Eric B Munsonde60f5f2015-11-05 18:51:36 -0800507 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
Kirill A. Shutemove90309c2016-01-15 16:54:33 -0800508 /* Do not mlock pte-mapped THP */
509 if (PageTransCompound(page))
510 goto out;
511
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700512 /*
513 * The preliminary mapping check is mainly to avoid the
514 * pointless overhead of lock_page on the ZERO_PAGE
515 * which might bounce very badly if there is contention.
516 *
517 * If the page is already locked, we don't need to
518 * handle it now - vmscan will handle it later if and
519 * when it attempts to reclaim the page.
520 */
521 if (page->mapping && trylock_page(page)) {
522 lru_add_drain(); /* push cached pages to LRU */
523 /*
524 * Because we lock page here, and migration is
525 * blocked by the pte's page reference, and we
526 * know the page is still mapped, we don't even
527 * need to check for file-cache page truncation.
528 */
529 mlock_vma_page(page);
530 unlock_page(page);
531 }
532 }
Kirill A. Shutemov1027e442015-09-04 15:47:55 -0700533out:
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700534 pte_unmap_unlock(ptep, ptl);
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700535 return page;
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700536no_page:
537 pte_unmap_unlock(ptep, ptl);
538 if (!pte_none(pte))
Kirill A. Shutemov69e68b42014-06-04 16:08:11 -0700539 return NULL;
540 return no_page_table(vma, flags);
541}
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700542
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700543static struct page *follow_pmd_mask(struct vm_area_struct *vma,
544 unsigned long address, pud_t *pudp,
Keith Buschdf06b372018-10-26 15:10:28 -0700545 unsigned int flags,
546 struct follow_page_context *ctx)
Kirill A. Shutemov69e68b42014-06-04 16:08:11 -0700547{
Huang Ying68827282018-06-07 17:06:34 -0700548 pmd_t *pmd, pmdval;
Kirill A. Shutemov69e68b42014-06-04 16:08:11 -0700549 spinlock_t *ptl;
550 struct page *page;
551 struct mm_struct *mm = vma->vm_mm;
552
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700553 pmd = pmd_offset(pudp, address);
Huang Ying68827282018-06-07 17:06:34 -0700554 /*
555 * The READ_ONCE() will stabilize the pmdval in a register or
556 * on the stack so that it will stop changing under the code.
557 */
558 pmdval = READ_ONCE(*pmd);
559 if (pmd_none(pmdval))
Kirill A. Shutemov69e68b42014-06-04 16:08:11 -0700560 return no_page_table(vma, flags);
Wei Yangbe9d3042020-01-30 22:12:14 -0800561 if (pmd_huge(pmdval) && is_vm_hugetlb_page(vma)) {
Naoya Horiguchie66f17f2015-02-11 15:25:22 -0800562 page = follow_huge_pmd(mm, address, pmd, flags);
563 if (page)
564 return page;
565 return no_page_table(vma, flags);
Kirill A. Shutemov69e68b42014-06-04 16:08:11 -0700566 }
Huang Ying68827282018-06-07 17:06:34 -0700567 if (is_hugepd(__hugepd(pmd_val(pmdval)))) {
Aneesh Kumar K.V4dc71452017-07-06 15:38:56 -0700568 page = follow_huge_pd(vma, address,
Huang Ying68827282018-06-07 17:06:34 -0700569 __hugepd(pmd_val(pmdval)), flags,
Aneesh Kumar K.V4dc71452017-07-06 15:38:56 -0700570 PMD_SHIFT);
571 if (page)
572 return page;
573 return no_page_table(vma, flags);
574 }
Zi Yan84c3fc42017-09-08 16:11:01 -0700575retry:
Huang Ying68827282018-06-07 17:06:34 -0700576 if (!pmd_present(pmdval)) {
Zi Yan84c3fc42017-09-08 16:11:01 -0700577 if (likely(!(flags & FOLL_MIGRATION)))
578 return no_page_table(vma, flags);
579 VM_BUG_ON(thp_migration_supported() &&
Huang Ying68827282018-06-07 17:06:34 -0700580 !is_pmd_migration_entry(pmdval));
581 if (is_pmd_migration_entry(pmdval))
Zi Yan84c3fc42017-09-08 16:11:01 -0700582 pmd_migration_entry_wait(mm, pmd);
Huang Ying68827282018-06-07 17:06:34 -0700583 pmdval = READ_ONCE(*pmd);
584 /*
585 * MADV_DONTNEED may convert the pmd to null because
586 * mmap_sem is held in read mode
587 */
588 if (pmd_none(pmdval))
589 return no_page_table(vma, flags);
Zi Yan84c3fc42017-09-08 16:11:01 -0700590 goto retry;
591 }
Huang Ying68827282018-06-07 17:06:34 -0700592 if (pmd_devmap(pmdval)) {
Dan Williams3565fce2016-01-15 16:56:55 -0800593 ptl = pmd_lock(mm, pmd);
Keith Buschdf06b372018-10-26 15:10:28 -0700594 page = follow_devmap_pmd(vma, address, pmd, flags, &ctx->pgmap);
Dan Williams3565fce2016-01-15 16:56:55 -0800595 spin_unlock(ptl);
596 if (page)
597 return page;
598 }
Huang Ying68827282018-06-07 17:06:34 -0700599 if (likely(!pmd_trans_huge(pmdval)))
Keith Buschdf06b372018-10-26 15:10:28 -0700600 return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
Kirill A. Shutemov6742d292016-01-15 16:52:28 -0800601
Huang Ying68827282018-06-07 17:06:34 -0700602 if ((flags & FOLL_NUMA) && pmd_protnone(pmdval))
Aneesh Kumar K.Vdb08f202017-02-24 14:59:53 -0800603 return no_page_table(vma, flags);
604
Zi Yan84c3fc42017-09-08 16:11:01 -0700605retry_locked:
Kirill A. Shutemov6742d292016-01-15 16:52:28 -0800606 ptl = pmd_lock(mm, pmd);
Huang Ying68827282018-06-07 17:06:34 -0700607 if (unlikely(pmd_none(*pmd))) {
608 spin_unlock(ptl);
609 return no_page_table(vma, flags);
610 }
Zi Yan84c3fc42017-09-08 16:11:01 -0700611 if (unlikely(!pmd_present(*pmd))) {
612 spin_unlock(ptl);
613 if (likely(!(flags & FOLL_MIGRATION)))
614 return no_page_table(vma, flags);
615 pmd_migration_entry_wait(mm, pmd);
616 goto retry_locked;
617 }
Kirill A. Shutemov6742d292016-01-15 16:52:28 -0800618 if (unlikely(!pmd_trans_huge(*pmd))) {
619 spin_unlock(ptl);
Keith Buschdf06b372018-10-26 15:10:28 -0700620 return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
Kirill A. Shutemov69e68b42014-06-04 16:08:11 -0700621 }
Song Liubfe7b002019-09-23 15:38:25 -0700622 if (flags & (FOLL_SPLIT | FOLL_SPLIT_PMD)) {
Kirill A. Shutemov6742d292016-01-15 16:52:28 -0800623 int ret;
624 page = pmd_page(*pmd);
625 if (is_huge_zero_page(page)) {
626 spin_unlock(ptl);
627 ret = 0;
Kirill A. Shutemov78ddc532016-01-15 16:52:42 -0800628 split_huge_pmd(vma, pmd, address);
Naoya Horiguchi337d9ab2016-07-26 15:24:03 -0700629 if (pmd_trans_unstable(pmd))
630 ret = -EBUSY;
Song Liubfe7b002019-09-23 15:38:25 -0700631 } else if (flags & FOLL_SPLIT) {
Linus Torvalds8fde12c2019-04-11 10:49:19 -0700632 if (unlikely(!try_get_page(page))) {
633 spin_unlock(ptl);
634 return ERR_PTR(-ENOMEM);
635 }
Kirill A. Shutemov6742d292016-01-15 16:52:28 -0800636 spin_unlock(ptl);
637 lock_page(page);
638 ret = split_huge_page(page);
639 unlock_page(page);
640 put_page(page);
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -0700641 if (pmd_none(*pmd))
642 return no_page_table(vma, flags);
Song Liubfe7b002019-09-23 15:38:25 -0700643 } else { /* flags & FOLL_SPLIT_PMD */
644 spin_unlock(ptl);
645 split_huge_pmd(vma, pmd, address);
646 ret = pte_alloc(mm, pmd) ? -ENOMEM : 0;
Kirill A. Shutemov6742d292016-01-15 16:52:28 -0800647 }
648
649 return ret ? ERR_PTR(ret) :
Keith Buschdf06b372018-10-26 15:10:28 -0700650 follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
Kirill A. Shutemov6742d292016-01-15 16:52:28 -0800651 }
Kirill A. Shutemov6742d292016-01-15 16:52:28 -0800652 page = follow_trans_huge_pmd(vma, address, pmd, flags);
653 spin_unlock(ptl);
Keith Buschdf06b372018-10-26 15:10:28 -0700654 ctx->page_mask = HPAGE_PMD_NR - 1;
Kirill A. Shutemov6742d292016-01-15 16:52:28 -0800655 return page;
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700656}
657
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700658static struct page *follow_pud_mask(struct vm_area_struct *vma,
659 unsigned long address, p4d_t *p4dp,
Keith Buschdf06b372018-10-26 15:10:28 -0700660 unsigned int flags,
661 struct follow_page_context *ctx)
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700662{
663 pud_t *pud;
664 spinlock_t *ptl;
665 struct page *page;
666 struct mm_struct *mm = vma->vm_mm;
667
668 pud = pud_offset(p4dp, address);
669 if (pud_none(*pud))
670 return no_page_table(vma, flags);
Wei Yangbe9d3042020-01-30 22:12:14 -0800671 if (pud_huge(*pud) && is_vm_hugetlb_page(vma)) {
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700672 page = follow_huge_pud(mm, address, pud, flags);
673 if (page)
674 return page;
675 return no_page_table(vma, flags);
676 }
Aneesh Kumar K.V4dc71452017-07-06 15:38:56 -0700677 if (is_hugepd(__hugepd(pud_val(*pud)))) {
678 page = follow_huge_pd(vma, address,
679 __hugepd(pud_val(*pud)), flags,
680 PUD_SHIFT);
681 if (page)
682 return page;
683 return no_page_table(vma, flags);
684 }
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700685 if (pud_devmap(*pud)) {
686 ptl = pud_lock(mm, pud);
Keith Buschdf06b372018-10-26 15:10:28 -0700687 page = follow_devmap_pud(vma, address, pud, flags, &ctx->pgmap);
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700688 spin_unlock(ptl);
689 if (page)
690 return page;
691 }
692 if (unlikely(pud_bad(*pud)))
693 return no_page_table(vma, flags);
694
Keith Buschdf06b372018-10-26 15:10:28 -0700695 return follow_pmd_mask(vma, address, pud, flags, ctx);
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700696}
697
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700698static struct page *follow_p4d_mask(struct vm_area_struct *vma,
699 unsigned long address, pgd_t *pgdp,
Keith Buschdf06b372018-10-26 15:10:28 -0700700 unsigned int flags,
701 struct follow_page_context *ctx)
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700702{
703 p4d_t *p4d;
Aneesh Kumar K.V4dc71452017-07-06 15:38:56 -0700704 struct page *page;
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700705
706 p4d = p4d_offset(pgdp, address);
707 if (p4d_none(*p4d))
708 return no_page_table(vma, flags);
709 BUILD_BUG_ON(p4d_huge(*p4d));
710 if (unlikely(p4d_bad(*p4d)))
711 return no_page_table(vma, flags);
712
Aneesh Kumar K.V4dc71452017-07-06 15:38:56 -0700713 if (is_hugepd(__hugepd(p4d_val(*p4d)))) {
714 page = follow_huge_pd(vma, address,
715 __hugepd(p4d_val(*p4d)), flags,
716 P4D_SHIFT);
717 if (page)
718 return page;
719 return no_page_table(vma, flags);
720 }
Keith Buschdf06b372018-10-26 15:10:28 -0700721 return follow_pud_mask(vma, address, p4d, flags, ctx);
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700722}
723
724/**
725 * follow_page_mask - look up a page descriptor from a user-virtual address
726 * @vma: vm_area_struct mapping @address
727 * @address: virtual address to look up
728 * @flags: flags modifying lookup behaviour
Mike Rapoport78179552018-11-16 15:08:29 -0800729 * @ctx: contains dev_pagemap for %ZONE_DEVICE memory pinning and a
730 * pointer to output page_mask
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700731 *
732 * @flags can have FOLL_ flags set, defined in <linux/mm.h>
733 *
Mike Rapoport78179552018-11-16 15:08:29 -0800734 * When getting pages from ZONE_DEVICE memory, the @ctx->pgmap caches
735 * the device's dev_pagemap metadata to avoid repeating expensive lookups.
736 *
737 * On output, the @ctx->page_mask is set according to the size of the page.
738 *
739 * Return: the mapped (struct page *), %NULL if no mapping exists, or
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700740 * an error pointer if there is a mapping to something not represented
741 * by a page descriptor (see also vm_normal_page()).
742 */
Bharath Vedarthama7030ae2019-07-11 20:54:34 -0700743static struct page *follow_page_mask(struct vm_area_struct *vma,
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700744 unsigned long address, unsigned int flags,
Keith Buschdf06b372018-10-26 15:10:28 -0700745 struct follow_page_context *ctx)
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700746{
747 pgd_t *pgd;
748 struct page *page;
749 struct mm_struct *mm = vma->vm_mm;
750
Keith Buschdf06b372018-10-26 15:10:28 -0700751 ctx->page_mask = 0;
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700752
753 /* make this handle hugepd */
754 page = follow_huge_addr(mm, address, flags & FOLL_WRITE);
755 if (!IS_ERR(page)) {
John Hubbard3faa52c2020-04-01 21:05:29 -0700756 WARN_ON_ONCE(flags & (FOLL_GET | FOLL_PIN));
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700757 return page;
758 }
759
760 pgd = pgd_offset(mm, address);
761
762 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
763 return no_page_table(vma, flags);
764
Anshuman Khandualfaaa5b62017-07-06 15:38:50 -0700765 if (pgd_huge(*pgd)) {
766 page = follow_huge_pgd(mm, address, pgd, flags);
767 if (page)
768 return page;
769 return no_page_table(vma, flags);
770 }
Aneesh Kumar K.V4dc71452017-07-06 15:38:56 -0700771 if (is_hugepd(__hugepd(pgd_val(*pgd)))) {
772 page = follow_huge_pd(vma, address,
773 __hugepd(pgd_val(*pgd)), flags,
774 PGDIR_SHIFT);
775 if (page)
776 return page;
777 return no_page_table(vma, flags);
778 }
Anshuman Khandualfaaa5b62017-07-06 15:38:50 -0700779
Keith Buschdf06b372018-10-26 15:10:28 -0700780 return follow_p4d_mask(vma, address, pgd, flags, ctx);
781}
782
783struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
784 unsigned int foll_flags)
785{
786 struct follow_page_context ctx = { NULL };
787 struct page *page;
788
789 page = follow_page_mask(vma, address, foll_flags, &ctx);
790 if (ctx.pgmap)
791 put_dev_pagemap(ctx.pgmap);
792 return page;
Aneesh Kumar K.V080dbb62017-07-06 15:38:44 -0700793}
794
Kirill A. Shutemovf2b495c2014-06-04 16:08:11 -0700795static int get_gate_page(struct mm_struct *mm, unsigned long address,
796 unsigned int gup_flags, struct vm_area_struct **vma,
797 struct page **page)
798{
799 pgd_t *pgd;
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +0300800 p4d_t *p4d;
Kirill A. Shutemovf2b495c2014-06-04 16:08:11 -0700801 pud_t *pud;
802 pmd_t *pmd;
803 pte_t *pte;
804 int ret = -EFAULT;
805
806 /* user gate pages are read-only */
807 if (gup_flags & FOLL_WRITE)
808 return -EFAULT;
809 if (address > TASK_SIZE)
810 pgd = pgd_offset_k(address);
811 else
812 pgd = pgd_offset_gate(mm, address);
Andy Lutomirskib5d1c392019-07-11 20:57:43 -0700813 if (pgd_none(*pgd))
814 return -EFAULT;
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +0300815 p4d = p4d_offset(pgd, address);
Andy Lutomirskib5d1c392019-07-11 20:57:43 -0700816 if (p4d_none(*p4d))
817 return -EFAULT;
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +0300818 pud = pud_offset(p4d, address);
Andy Lutomirskib5d1c392019-07-11 20:57:43 -0700819 if (pud_none(*pud))
820 return -EFAULT;
Kirill A. Shutemovf2b495c2014-06-04 16:08:11 -0700821 pmd = pmd_offset(pud, address);
Zi Yan84c3fc42017-09-08 16:11:01 -0700822 if (!pmd_present(*pmd))
Kirill A. Shutemovf2b495c2014-06-04 16:08:11 -0700823 return -EFAULT;
824 VM_BUG_ON(pmd_trans_huge(*pmd));
825 pte = pte_offset_map(pmd, address);
826 if (pte_none(*pte))
827 goto unmap;
828 *vma = get_gate_vma(mm);
829 if (!page)
830 goto out;
831 *page = vm_normal_page(*vma, address, *pte);
832 if (!*page) {
833 if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(*pte)))
834 goto unmap;
835 *page = pte_page(*pte);
836 }
Linus Torvalds8fde12c2019-04-11 10:49:19 -0700837 if (unlikely(!try_get_page(*page))) {
838 ret = -ENOMEM;
839 goto unmap;
840 }
Kirill A. Shutemovf2b495c2014-06-04 16:08:11 -0700841out:
842 ret = 0;
843unmap:
844 pte_unmap(pte);
845 return ret;
846}
847
Paul Cassella9a95f3c2014-08-06 16:07:24 -0700848/*
849 * mmap_sem must be held on entry. If @nonblocking != NULL and
850 * *@flags does not include FOLL_NOWAIT, the mmap_sem may be released.
851 * If it is, *@nonblocking will be set to 0 and -EBUSY returned.
852 */
Kirill A. Shutemov16744482014-06-04 16:08:12 -0700853static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma,
854 unsigned long address, unsigned int *flags, int *nonblocking)
855{
Kirill A. Shutemov16744482014-06-04 16:08:12 -0700856 unsigned int fault_flags = 0;
Souptick Joarder2b740302018-08-23 17:01:36 -0700857 vm_fault_t ret;
Kirill A. Shutemov16744482014-06-04 16:08:12 -0700858
Eric B Munsonde60f5f2015-11-05 18:51:36 -0800859 /* mlock all present pages, but do not fault in new pages */
860 if ((*flags & (FOLL_POPULATE | FOLL_MLOCK)) == FOLL_MLOCK)
861 return -ENOENT;
Kirill A. Shutemov16744482014-06-04 16:08:12 -0700862 if (*flags & FOLL_WRITE)
863 fault_flags |= FAULT_FLAG_WRITE;
Dave Hansen1b2ee122016-02-12 13:02:21 -0800864 if (*flags & FOLL_REMOTE)
865 fault_flags |= FAULT_FLAG_REMOTE;
Kirill A. Shutemov16744482014-06-04 16:08:12 -0700866 if (nonblocking)
867 fault_flags |= FAULT_FLAG_ALLOW_RETRY;
868 if (*flags & FOLL_NOWAIT)
869 fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT;
Andres Lagar-Cavilla234b2392014-09-17 10:51:48 -0700870 if (*flags & FOLL_TRIED) {
871 VM_WARN_ON_ONCE(fault_flags & FAULT_FLAG_ALLOW_RETRY);
872 fault_flags |= FAULT_FLAG_TRIED;
873 }
Kirill A. Shutemov16744482014-06-04 16:08:12 -0700874
Kirill A. Shutemovdcddffd2016-07-26 15:25:18 -0700875 ret = handle_mm_fault(vma, address, fault_flags);
Kirill A. Shutemov16744482014-06-04 16:08:12 -0700876 if (ret & VM_FAULT_ERROR) {
James Morse9a291a72017-06-02 14:46:46 -0700877 int err = vm_fault_to_errno(ret, *flags);
878
879 if (err)
880 return err;
Kirill A. Shutemov16744482014-06-04 16:08:12 -0700881 BUG();
882 }
883
884 if (tsk) {
885 if (ret & VM_FAULT_MAJOR)
886 tsk->maj_flt++;
887 else
888 tsk->min_flt++;
889 }
890
891 if (ret & VM_FAULT_RETRY) {
Andrea Arcangeli96312e62018-03-09 15:51:06 -0800892 if (nonblocking && !(fault_flags & FAULT_FLAG_RETRY_NOWAIT))
Kirill A. Shutemov16744482014-06-04 16:08:12 -0700893 *nonblocking = 0;
894 return -EBUSY;
895 }
896
897 /*
898 * The VM_FAULT_WRITE bit tells us that do_wp_page has broken COW when
899 * necessary, even if maybe_mkwrite decided not to set pte_write. We
900 * can thus safely do subsequent page lookups as if they were reads.
901 * But only do so when looping for pte_write is futile: in some cases
902 * userspace may also be wanting to write to the gotten user page,
903 * which a read fault here might prevent (a readonly page might get
904 * reCOWed by userspace write).
905 */
906 if ((ret & VM_FAULT_WRITE) && !(vma->vm_flags & VM_WRITE))
Mario Leinweber29231172018-04-05 16:24:18 -0700907 *flags |= FOLL_COW;
Kirill A. Shutemov16744482014-06-04 16:08:12 -0700908 return 0;
909}
910
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -0700911static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
912{
913 vm_flags_t vm_flags = vma->vm_flags;
Dave Hansen1b2ee122016-02-12 13:02:21 -0800914 int write = (gup_flags & FOLL_WRITE);
915 int foreign = (gup_flags & FOLL_REMOTE);
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -0700916
917 if (vm_flags & (VM_IO | VM_PFNMAP))
918 return -EFAULT;
919
Willy Tarreau7f7ccc22018-05-11 08:11:44 +0200920 if (gup_flags & FOLL_ANON && !vma_is_anonymous(vma))
921 return -EFAULT;
922
Dave Hansen1b2ee122016-02-12 13:02:21 -0800923 if (write) {
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -0700924 if (!(vm_flags & VM_WRITE)) {
925 if (!(gup_flags & FOLL_FORCE))
926 return -EFAULT;
927 /*
928 * We used to let the write,force case do COW in a
929 * VM_MAYWRITE VM_SHARED !VM_WRITE vma, so ptrace could
930 * set a breakpoint in a read-only mapping of an
931 * executable, without corrupting the file (yet only
932 * when that file had been opened for writing!).
933 * Anon pages in shared mappings are surprising: now
934 * just reject it.
935 */
Hugh Dickins46435362016-01-30 18:03:16 -0800936 if (!is_cow_mapping(vm_flags))
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -0700937 return -EFAULT;
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -0700938 }
939 } else if (!(vm_flags & VM_READ)) {
940 if (!(gup_flags & FOLL_FORCE))
941 return -EFAULT;
942 /*
943 * Is there actually any vma we can reach here which does not
944 * have VM_MAYREAD set?
945 */
946 if (!(vm_flags & VM_MAYREAD))
947 return -EFAULT;
948 }
Dave Hansend61172b2016-02-12 13:02:24 -0800949 /*
950 * gups are always data accesses, not instruction
951 * fetches, so execute=false here
952 */
953 if (!arch_vma_access_permitted(vma, write, false, foreign))
Dave Hansen33a709b2016-02-12 13:02:19 -0800954 return -EFAULT;
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -0700955 return 0;
956}
957
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700958/**
959 * __get_user_pages() - pin user pages in memory
960 * @tsk: task_struct of target task
961 * @mm: mm_struct of target mm
962 * @start: starting user address
963 * @nr_pages: number of pages from start to pin
964 * @gup_flags: flags modifying pin behaviour
965 * @pages: array that receives pointers to the pages pinned.
966 * Should be at least nr_pages long. Or NULL, if caller
967 * only intends to ensure the pages are faulted in.
968 * @vmas: array of pointers to vmas corresponding to each page.
969 * Or NULL if the caller does not require them.
970 * @nonblocking: whether waiting for disk IO or mmap_sem contention
971 *
Liu Xiangd2dfbe42019-11-30 17:49:53 -0800972 * Returns either number of pages pinned (which may be less than the
973 * number requested), or an error. Details about the return value:
974 *
975 * -- If nr_pages is 0, returns 0.
976 * -- If nr_pages is >0, but no pages were pinned, returns -errno.
977 * -- If nr_pages is >0, and some pages were pinned, returns the number of
978 * pages pinned. Again, this may be less than nr_pages.
979 *
980 * The caller is responsible for releasing returned @pages, via put_page().
981 *
982 * @vmas are valid only as long as mmap_sem is held.
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700983 *
Paul Cassella9a95f3c2014-08-06 16:07:24 -0700984 * Must be called with mmap_sem held. It may be released. See below.
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -0700985 *
986 * __get_user_pages walks a process's page tables and takes a reference to
987 * each struct page that each user address corresponds to at a given
988 * instant. That is, it takes the page that would be accessed if a user
989 * thread accesses the given user virtual address at that instant.
990 *
991 * This does not guarantee that the page exists in the user mappings when
992 * __get_user_pages returns, and there may even be a completely different
993 * page there in some cases (eg. if mmapped pagecache has been invalidated
994 * and subsequently re faulted). However it does guarantee that the page
995 * won't be freed completely. And mostly callers simply care that the page
996 * contains data that was valid *at some point in time*. Typically, an IO
997 * or similar operation cannot guarantee anything stronger anyway because
998 * locks can't be held over the syscall boundary.
999 *
1000 * If @gup_flags & FOLL_WRITE == 0, the page must not be written to. If
1001 * the page is written to, set_page_dirty (or set_page_dirty_lock, as
1002 * appropriate) must be called after the page is finished with, and
1003 * before put_page is called.
1004 *
1005 * If @nonblocking != NULL, __get_user_pages will not wait for disk IO
1006 * or mmap_sem contention, and if waiting is needed to pin all pages,
Paul Cassella9a95f3c2014-08-06 16:07:24 -07001007 * *@nonblocking will be set to 0. Further, if @gup_flags does not
1008 * include FOLL_NOWAIT, the mmap_sem will be released via up_read() in
1009 * this case.
1010 *
1011 * A caller using such a combination of @nonblocking and @gup_flags
1012 * must therefore hold the mmap_sem for reading only, and recognize
1013 * when it's been released. Otherwise, it must be held for either
1014 * reading or writing and will not be released.
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001015 *
1016 * In most cases, get_user_pages or get_user_pages_fast should be used
1017 * instead of __get_user_pages. __get_user_pages should be used only if
1018 * you need some special @gup_flags.
1019 */
Lorenzo Stoakes0d731752016-10-24 10:57:25 +01001020static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001021 unsigned long start, unsigned long nr_pages,
1022 unsigned int gup_flags, struct page **pages,
1023 struct vm_area_struct **vmas, int *nonblocking)
1024{
Keith Buschdf06b372018-10-26 15:10:28 -07001025 long ret = 0, i = 0;
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -07001026 struct vm_area_struct *vma = NULL;
Keith Buschdf06b372018-10-26 15:10:28 -07001027 struct follow_page_context ctx = { NULL };
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001028
1029 if (!nr_pages)
1030 return 0;
1031
Andrey Konovalovf9652592019-09-25 16:48:34 -07001032 start = untagged_addr(start);
1033
John Hubbardeddb1c22020-01-30 22:12:54 -08001034 VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN)));
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001035
1036 /*
1037 * If FOLL_FORCE is set then do not force a full fault as the hinting
1038 * fault information is unrelated to the reference behaviour of a task
1039 * using the address space
1040 */
1041 if (!(gup_flags & FOLL_FORCE))
1042 gup_flags |= FOLL_NUMA;
1043
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001044 do {
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -07001045 struct page *page;
1046 unsigned int foll_flags = gup_flags;
1047 unsigned int page_increm;
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001048
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -07001049 /* first iteration or cross vma bound */
1050 if (!vma || start >= vma->vm_end) {
1051 vma = find_extend_vma(mm, start);
1052 if (!vma && in_gate_area(mm, start)) {
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -07001053 ret = get_gate_page(mm, start & PAGE_MASK,
1054 gup_flags, &vma,
1055 pages ? &pages[i] : NULL);
1056 if (ret)
John Hubbard08be37b2018-11-30 14:08:53 -08001057 goto out;
Keith Buschdf06b372018-10-26 15:10:28 -07001058 ctx.page_mask = 0;
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -07001059 goto next_page;
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001060 }
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001061
Keith Buschdf06b372018-10-26 15:10:28 -07001062 if (!vma || check_vma_flags(vma, gup_flags)) {
1063 ret = -EFAULT;
1064 goto out;
1065 }
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -07001066 if (is_vm_hugetlb_page(vma)) {
1067 i = follow_hugetlb_page(mm, vma, pages, vmas,
1068 &start, &nr_pages, i,
Andrea Arcangeli87ffc112017-02-22 15:43:13 -08001069 gup_flags, nonblocking);
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -07001070 continue;
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001071 }
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -07001072 }
1073retry:
1074 /*
1075 * If we have a pending SIGKILL, don't keep faulting pages and
1076 * potentially allocating memory.
1077 */
Davidlohr Buesofa45f112019-01-03 15:28:55 -08001078 if (fatal_signal_pending(current)) {
Keith Buschdf06b372018-10-26 15:10:28 -07001079 ret = -ERESTARTSYS;
1080 goto out;
1081 }
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -07001082 cond_resched();
Keith Buschdf06b372018-10-26 15:10:28 -07001083
1084 page = follow_page_mask(vma, start, foll_flags, &ctx);
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -07001085 if (!page) {
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -07001086 ret = faultin_page(tsk, vma, start, &foll_flags,
1087 nonblocking);
1088 switch (ret) {
1089 case 0:
1090 goto retry;
Keith Buschdf06b372018-10-26 15:10:28 -07001091 case -EBUSY:
1092 ret = 0;
1093 /* FALLTHRU */
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -07001094 case -EFAULT:
1095 case -ENOMEM:
1096 case -EHWPOISON:
Keith Buschdf06b372018-10-26 15:10:28 -07001097 goto out;
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -07001098 case -ENOENT:
1099 goto next_page;
1100 }
1101 BUG();
Kirill A. Shutemov1027e442015-09-04 15:47:55 -07001102 } else if (PTR_ERR(page) == -EEXIST) {
1103 /*
1104 * Proper page table entry exists, but no corresponding
1105 * struct page.
1106 */
1107 goto next_page;
1108 } else if (IS_ERR(page)) {
Keith Buschdf06b372018-10-26 15:10:28 -07001109 ret = PTR_ERR(page);
1110 goto out;
Kirill A. Shutemov1027e442015-09-04 15:47:55 -07001111 }
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -07001112 if (pages) {
1113 pages[i] = page;
1114 flush_anon_page(vma, page, start);
1115 flush_dcache_page(page);
Keith Buschdf06b372018-10-26 15:10:28 -07001116 ctx.page_mask = 0;
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -07001117 }
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001118next_page:
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -07001119 if (vmas) {
1120 vmas[i] = vma;
Keith Buschdf06b372018-10-26 15:10:28 -07001121 ctx.page_mask = 0;
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -07001122 }
Keith Buschdf06b372018-10-26 15:10:28 -07001123 page_increm = 1 + (~(start >> PAGE_SHIFT) & ctx.page_mask);
Kirill A. Shutemovfa5bb202014-06-04 16:08:13 -07001124 if (page_increm > nr_pages)
1125 page_increm = nr_pages;
1126 i += page_increm;
1127 start += page_increm * PAGE_SIZE;
1128 nr_pages -= page_increm;
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001129 } while (nr_pages);
Keith Buschdf06b372018-10-26 15:10:28 -07001130out:
1131 if (ctx.pgmap)
1132 put_dev_pagemap(ctx.pgmap);
1133 return i ? i : ret;
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001134}
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001135
Tobias Klauser771ab432016-12-12 16:41:53 -08001136static bool vma_permits_fault(struct vm_area_struct *vma,
1137 unsigned int fault_flags)
Dave Hansend4925e02016-02-12 13:02:16 -08001138{
Dave Hansen1b2ee122016-02-12 13:02:21 -08001139 bool write = !!(fault_flags & FAULT_FLAG_WRITE);
1140 bool foreign = !!(fault_flags & FAULT_FLAG_REMOTE);
Dave Hansen33a709b2016-02-12 13:02:19 -08001141 vm_flags_t vm_flags = write ? VM_WRITE : VM_READ;
Dave Hansend4925e02016-02-12 13:02:16 -08001142
1143 if (!(vm_flags & vma->vm_flags))
1144 return false;
1145
Dave Hansen33a709b2016-02-12 13:02:19 -08001146 /*
1147 * The architecture might have a hardware protection
Dave Hansen1b2ee122016-02-12 13:02:21 -08001148 * mechanism other than read/write that can deny access.
Dave Hansend61172b2016-02-12 13:02:24 -08001149 *
1150 * gup always represents data access, not instruction
1151 * fetches, so execute=false here:
Dave Hansen33a709b2016-02-12 13:02:19 -08001152 */
Dave Hansend61172b2016-02-12 13:02:24 -08001153 if (!arch_vma_access_permitted(vma, write, false, foreign))
Dave Hansen33a709b2016-02-12 13:02:19 -08001154 return false;
1155
Dave Hansend4925e02016-02-12 13:02:16 -08001156 return true;
1157}
1158
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001159/*
1160 * fixup_user_fault() - manually resolve a user page fault
1161 * @tsk: the task_struct to use for page fault accounting, or
1162 * NULL if faults are not to be recorded.
1163 * @mm: mm_struct of target mm
1164 * @address: user address
1165 * @fault_flags:flags to pass down to handle_mm_fault()
Dominik Dingel4a9e1cd2016-01-15 16:57:04 -08001166 * @unlocked: did we unlock the mmap_sem while retrying, maybe NULL if caller
1167 * does not allow retry
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001168 *
1169 * This is meant to be called in the specific scenario where for locking reasons
1170 * we try to access user memory in atomic context (within a pagefault_disable()
1171 * section), this returns -EFAULT, and we want to resolve the user fault before
1172 * trying again.
1173 *
1174 * Typically this is meant to be used by the futex code.
1175 *
1176 * The main difference with get_user_pages() is that this function will
1177 * unconditionally call handle_mm_fault() which will in turn perform all the
1178 * necessary SW fixup of the dirty and young bits in the PTE, while
Dominik Dingel4a9e1cd2016-01-15 16:57:04 -08001179 * get_user_pages() only guarantees to update these in the struct page.
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001180 *
1181 * This is important for some architectures where those bits also gate the
1182 * access permission to the page because they are maintained in software. On
1183 * such architectures, gup() will not be enough to make a subsequent access
1184 * succeed.
1185 *
Dominik Dingel4a9e1cd2016-01-15 16:57:04 -08001186 * This function will not return with an unlocked mmap_sem. So it has not the
1187 * same semantics wrt the @mm->mmap_sem as does filemap_fault().
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001188 */
1189int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
Dominik Dingel4a9e1cd2016-01-15 16:57:04 -08001190 unsigned long address, unsigned int fault_flags,
1191 bool *unlocked)
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001192{
1193 struct vm_area_struct *vma;
Souptick Joarder2b740302018-08-23 17:01:36 -07001194 vm_fault_t ret, major = 0;
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001195
Andrey Konovalovf9652592019-09-25 16:48:34 -07001196 address = untagged_addr(address);
1197
Dominik Dingel4a9e1cd2016-01-15 16:57:04 -08001198 if (unlocked)
1199 fault_flags |= FAULT_FLAG_ALLOW_RETRY;
1200
1201retry:
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001202 vma = find_extend_vma(mm, address);
1203 if (!vma || address < vma->vm_start)
1204 return -EFAULT;
1205
Dave Hansend4925e02016-02-12 13:02:16 -08001206 if (!vma_permits_fault(vma, fault_flags))
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001207 return -EFAULT;
1208
Kirill A. Shutemovdcddffd2016-07-26 15:25:18 -07001209 ret = handle_mm_fault(vma, address, fault_flags);
Dominik Dingel4a9e1cd2016-01-15 16:57:04 -08001210 major |= ret & VM_FAULT_MAJOR;
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001211 if (ret & VM_FAULT_ERROR) {
James Morse9a291a72017-06-02 14:46:46 -07001212 int err = vm_fault_to_errno(ret, 0);
1213
1214 if (err)
1215 return err;
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001216 BUG();
1217 }
Dominik Dingel4a9e1cd2016-01-15 16:57:04 -08001218
1219 if (ret & VM_FAULT_RETRY) {
1220 down_read(&mm->mmap_sem);
1221 if (!(fault_flags & FAULT_FLAG_TRIED)) {
1222 *unlocked = true;
1223 fault_flags &= ~FAULT_FLAG_ALLOW_RETRY;
1224 fault_flags |= FAULT_FLAG_TRIED;
1225 goto retry;
1226 }
1227 }
1228
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001229 if (tsk) {
Dominik Dingel4a9e1cd2016-01-15 16:57:04 -08001230 if (major)
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001231 tsk->maj_flt++;
1232 else
1233 tsk->min_flt++;
1234 }
1235 return 0;
1236}
Paolo Bonziniadd6a0c2016-06-07 17:51:18 +02001237EXPORT_SYMBOL_GPL(fixup_user_fault);
Kirill A. Shutemov4bbd4c72014-06-04 16:08:10 -07001238
Andrea Arcangelif0818f42015-02-11 15:27:17 -08001239static __always_inline long __get_user_pages_locked(struct task_struct *tsk,
1240 struct mm_struct *mm,
1241 unsigned long start,
1242 unsigned long nr_pages,
Andrea Arcangelif0818f42015-02-11 15:27:17 -08001243 struct page **pages,
1244 struct vm_area_struct **vmas,
Al Viroe7167122017-11-19 11:32:05 -05001245 int *locked,
Andrea Arcangeli0fd71a52015-02-11 15:27:20 -08001246 unsigned int flags)
Andrea Arcangelif0818f42015-02-11 15:27:17 -08001247{
Andrea Arcangelif0818f42015-02-11 15:27:17 -08001248 long ret, pages_done;
1249 bool lock_dropped;
1250
1251 if (locked) {
1252 /* if VM_FAULT_RETRY can be returned, vmas become invalid */
1253 BUG_ON(vmas);
1254 /* check caller initialized locked */
1255 BUG_ON(*locked != 1);
1256 }
1257
John Hubbardeddb1c22020-01-30 22:12:54 -08001258 /*
1259 * FOLL_PIN and FOLL_GET are mutually exclusive. Traditional behavior
1260 * is to set FOLL_GET if the caller wants pages[] filled in (but has
1261 * carelessly failed to specify FOLL_GET), so keep doing that, but only
1262 * for FOLL_GET, not for the newer FOLL_PIN.
1263 *
1264 * FOLL_PIN always expects pages to be non-null, but no need to assert
1265 * that here, as any failures will be obvious enough.
1266 */
1267 if (pages && !(flags & FOLL_PIN))
Andrea Arcangelif0818f42015-02-11 15:27:17 -08001268 flags |= FOLL_GET;
Andrea Arcangelif0818f42015-02-11 15:27:17 -08001269
1270 pages_done = 0;
1271 lock_dropped = false;
1272 for (;;) {
1273 ret = __get_user_pages(tsk, mm, start, nr_pages, flags, pages,
1274 vmas, locked);
1275 if (!locked)
1276 /* VM_FAULT_RETRY couldn't trigger, bypass */
1277 return ret;
1278
1279 /* VM_FAULT_RETRY cannot return errors */
1280 if (!*locked) {
1281 BUG_ON(ret < 0);
1282 BUG_ON(ret >= nr_pages);
1283 }
1284
Andrea Arcangelif0818f42015-02-11 15:27:17 -08001285 if (ret > 0) {
1286 nr_pages -= ret;
1287 pages_done += ret;
1288 if (!nr_pages)
1289 break;
1290 }
1291 if (*locked) {
Andrea Arcangeli96312e62018-03-09 15:51:06 -08001292 /*
1293 * VM_FAULT_RETRY didn't trigger or it was a
1294 * FOLL_NOWAIT.
1295 */
Andrea Arcangelif0818f42015-02-11 15:27:17 -08001296 if (!pages_done)
1297 pages_done = ret;
1298 break;
1299 }
Mike Rapoportdf172772019-05-31 22:30:33 -07001300 /*
1301 * VM_FAULT_RETRY triggered, so seek to the faulting offset.
1302 * For the prefault case (!pages) we only update counts.
1303 */
1304 if (likely(pages))
1305 pages += ret;
Andrea Arcangelif0818f42015-02-11 15:27:17 -08001306 start += ret << PAGE_SHIFT;
1307
1308 /*
1309 * Repeat on the address that fired VM_FAULT_RETRY
1310 * without FAULT_FLAG_ALLOW_RETRY but with
1311 * FAULT_FLAG_TRIED.
1312 */
1313 *locked = 1;
1314 lock_dropped = true;
1315 down_read(&mm->mmap_sem);
1316 ret = __get_user_pages(tsk, mm, start, 1, flags | FOLL_TRIED,
1317 pages, NULL, NULL);
1318 if (ret != 1) {
1319 BUG_ON(ret > 1);
1320 if (!pages_done)
1321 pages_done = ret;
1322 break;
1323 }
1324 nr_pages--;
1325 pages_done++;
1326 if (!nr_pages)
1327 break;
Mike Rapoportdf172772019-05-31 22:30:33 -07001328 if (likely(pages))
1329 pages++;
Andrea Arcangelif0818f42015-02-11 15:27:17 -08001330 start += PAGE_SIZE;
1331 }
Al Viroe7167122017-11-19 11:32:05 -05001332 if (lock_dropped && *locked) {
Andrea Arcangelif0818f42015-02-11 15:27:17 -08001333 /*
1334 * We must let the caller know we temporarily dropped the lock
1335 * and so the critical section protected by it was lost.
1336 */
1337 up_read(&mm->mmap_sem);
1338 *locked = 0;
1339 }
1340 return pages_done;
1341}
1342
Christoph Hellwigd3649f62019-07-11 20:57:18 -07001343/**
1344 * populate_vma_page_range() - populate a range of pages in the vma.
1345 * @vma: target vma
1346 * @start: start address
1347 * @end: end address
1348 * @nonblocking:
1349 *
1350 * This takes care of mlocking the pages too if VM_LOCKED is set.
1351 *
1352 * return 0 on success, negative error code on error.
1353 *
1354 * vma->vm_mm->mmap_sem must be held.
1355 *
1356 * If @nonblocking is NULL, it may be held for read or write and will
1357 * be unperturbed.
1358 *
1359 * If @nonblocking is non-NULL, it must held for read only and may be
1360 * released. If it's released, *@nonblocking will be set to 0.
1361 */
1362long populate_vma_page_range(struct vm_area_struct *vma,
1363 unsigned long start, unsigned long end, int *nonblocking)
1364{
1365 struct mm_struct *mm = vma->vm_mm;
1366 unsigned long nr_pages = (end - start) / PAGE_SIZE;
1367 int gup_flags;
1368
1369 VM_BUG_ON(start & ~PAGE_MASK);
1370 VM_BUG_ON(end & ~PAGE_MASK);
1371 VM_BUG_ON_VMA(start < vma->vm_start, vma);
1372 VM_BUG_ON_VMA(end > vma->vm_end, vma);
1373 VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_sem), mm);
1374
1375 gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK;
1376 if (vma->vm_flags & VM_LOCKONFAULT)
1377 gup_flags &= ~FOLL_POPULATE;
1378 /*
1379 * We want to touch writable mappings with a write fault in order
1380 * to break COW, except for shared mappings because these don't COW
1381 * and we would not want to dirty them for nothing.
1382 */
1383 if ((vma->vm_flags & (VM_WRITE | VM_SHARED)) == VM_WRITE)
1384 gup_flags |= FOLL_WRITE;
1385
1386 /*
1387 * We want mlock to succeed for regions that have any permissions
1388 * other than PROT_NONE.
1389 */
1390 if (vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC))
1391 gup_flags |= FOLL_FORCE;
1392
1393 /*
1394 * We made sure addr is within a VMA, so the following will
1395 * not result in a stack expansion that recurses back here.
1396 */
1397 return __get_user_pages(current, mm, start, nr_pages, gup_flags,
1398 NULL, NULL, nonblocking);
1399}
1400
1401/*
1402 * __mm_populate - populate and/or mlock pages within a range of address space.
1403 *
1404 * This is used to implement mlock() and the MAP_POPULATE / MAP_LOCKED mmap
1405 * flags. VMAs must be already marked with the desired vm_flags, and
1406 * mmap_sem must not be held.
1407 */
1408int __mm_populate(unsigned long start, unsigned long len, int ignore_errors)
1409{
1410 struct mm_struct *mm = current->mm;
1411 unsigned long end, nstart, nend;
1412 struct vm_area_struct *vma = NULL;
1413 int locked = 0;
1414 long ret = 0;
1415
1416 end = start + len;
1417
1418 for (nstart = start; nstart < end; nstart = nend) {
1419 /*
1420 * We want to fault in pages for [nstart; end) address range.
1421 * Find first corresponding VMA.
1422 */
1423 if (!locked) {
1424 locked = 1;
1425 down_read(&mm->mmap_sem);
1426 vma = find_vma(mm, nstart);
1427 } else if (nstart >= vma->vm_end)
1428 vma = vma->vm_next;
1429 if (!vma || vma->vm_start >= end)
1430 break;
1431 /*
1432 * Set [nstart; nend) to intersection of desired address
1433 * range with the first VMA. Also, skip undesirable VMA types.
1434 */
1435 nend = min(end, vma->vm_end);
1436 if (vma->vm_flags & (VM_IO | VM_PFNMAP))
1437 continue;
1438 if (nstart < vma->vm_start)
1439 nstart = vma->vm_start;
1440 /*
1441 * Now fault in a range of pages. populate_vma_page_range()
1442 * double checks the vma flags, so that it won't mlock pages
1443 * if the vma was already munlocked.
1444 */
1445 ret = populate_vma_page_range(vma, nstart, nend, &locked);
1446 if (ret < 0) {
1447 if (ignore_errors) {
1448 ret = 0;
1449 continue; /* continue at next VMA */
1450 }
1451 break;
1452 }
1453 nend = nstart + ret * PAGE_SIZE;
1454 ret = 0;
1455 }
1456 if (locked)
1457 up_read(&mm->mmap_sem);
1458 return ret; /* 0 or negative error code */
1459}
1460
1461/**
1462 * get_dump_page() - pin user page in memory while writing it to core dump
1463 * @addr: user address
1464 *
1465 * Returns struct page pointer of user page pinned for dump,
1466 * to be freed afterwards by put_page().
1467 *
1468 * Returns NULL on any kind of failure - a hole must then be inserted into
1469 * the corefile, to preserve alignment with its headers; and also returns
1470 * NULL wherever the ZERO_PAGE, or an anonymous pte_none, has been found -
1471 * allowing a hole to be left in the corefile to save diskspace.
1472 *
1473 * Called without mmap_sem, but after all other threads have been killed.
1474 */
1475#ifdef CONFIG_ELF_CORE
1476struct page *get_dump_page(unsigned long addr)
1477{
1478 struct vm_area_struct *vma;
1479 struct page *page;
1480
1481 if (__get_user_pages(current, current->mm, addr, 1,
1482 FOLL_FORCE | FOLL_DUMP | FOLL_GET, &page, &vma,
1483 NULL) < 1)
1484 return NULL;
1485 flush_cache_page(vma, addr, page_to_pfn(page));
1486 return page;
1487}
1488#endif /* CONFIG_ELF_CORE */
Christoph Hellwig050a9ad2019-07-11 20:57:21 -07001489#else /* CONFIG_MMU */
1490static long __get_user_pages_locked(struct task_struct *tsk,
1491 struct mm_struct *mm, unsigned long start,
1492 unsigned long nr_pages, struct page **pages,
1493 struct vm_area_struct **vmas, int *locked,
1494 unsigned int foll_flags)
1495{
1496 struct vm_area_struct *vma;
1497 unsigned long vm_flags;
1498 int i;
1499
1500 /* calculate required read or write permissions.
1501 * If FOLL_FORCE is set, we only require the "MAY" flags.
1502 */
1503 vm_flags = (foll_flags & FOLL_WRITE) ?
1504 (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
1505 vm_flags &= (foll_flags & FOLL_FORCE) ?
1506 (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
1507
1508 for (i = 0; i < nr_pages; i++) {
1509 vma = find_vma(mm, start);
1510 if (!vma)
1511 goto finish_or_fault;
1512
1513 /* protect what we can, including chardevs */
1514 if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
1515 !(vm_flags & vma->vm_flags))
1516 goto finish_or_fault;
1517
1518 if (pages) {
1519 pages[i] = virt_to_page(start);
1520 if (pages[i])
1521 get_page(pages[i]);
1522 }
1523 if (vmas)
1524 vmas[i] = vma;
1525 start = (start + PAGE_SIZE) & PAGE_MASK;
1526 }
1527
1528 return i;
1529
1530finish_or_fault:
1531 return i ? : -EFAULT;
1532}
1533#endif /* !CONFIG_MMU */
Christoph Hellwigd3649f62019-07-11 20:57:18 -07001534
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -08001535#if defined(CONFIG_FS_DAX) || defined (CONFIG_CMA)
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -08001536static bool check_dax_vmas(struct vm_area_struct **vmas, long nr_pages)
1537{
1538 long i;
1539 struct vm_area_struct *vma_prev = NULL;
1540
1541 for (i = 0; i < nr_pages; i++) {
1542 struct vm_area_struct *vma = vmas[i];
1543
1544 if (vma == vma_prev)
1545 continue;
1546
1547 vma_prev = vma;
1548
1549 if (vma_is_fsdax(vma))
1550 return true;
1551 }
1552 return false;
1553}
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -08001554
1555#ifdef CONFIG_CMA
1556static struct page *new_non_cma_page(struct page *page, unsigned long private)
1557{
1558 /*
1559 * We want to make sure we allocate the new page from the same node
1560 * as the source page.
1561 */
1562 int nid = page_to_nid(page);
1563 /*
1564 * Trying to allocate a page for migration. Ignore allocation
1565 * failure warnings. We don't force __GFP_THISNODE here because
1566 * this node here is the node where we have CMA reservation and
1567 * in some case these nodes will have really less non movable
1568 * allocation memory.
1569 */
1570 gfp_t gfp_mask = GFP_USER | __GFP_NOWARN;
1571
1572 if (PageHighMem(page))
1573 gfp_mask |= __GFP_HIGHMEM;
1574
1575#ifdef CONFIG_HUGETLB_PAGE
1576 if (PageHuge(page)) {
1577 struct hstate *h = page_hstate(page);
1578 /*
1579 * We don't want to dequeue from the pool because pool pages will
1580 * mostly be from the CMA region.
1581 */
1582 return alloc_migrate_huge_page(h, gfp_mask, nid, NULL);
1583 }
1584#endif
1585 if (PageTransHuge(page)) {
1586 struct page *thp;
1587 /*
1588 * ignore allocation failure warnings
1589 */
1590 gfp_t thp_gfpmask = GFP_TRANSHUGE | __GFP_NOWARN;
1591
1592 /*
1593 * Remove the movable mask so that we don't allocate from
1594 * CMA area again.
1595 */
1596 thp_gfpmask &= ~__GFP_MOVABLE;
1597 thp = __alloc_pages_node(nid, thp_gfpmask, HPAGE_PMD_ORDER);
1598 if (!thp)
1599 return NULL;
1600 prep_transhuge_page(thp);
1601 return thp;
1602 }
1603
1604 return __alloc_pages_node(nid, gfp_mask, 0);
1605}
1606
Ira Weiny932f4a62019-05-13 17:17:03 -07001607static long check_and_migrate_cma_pages(struct task_struct *tsk,
1608 struct mm_struct *mm,
1609 unsigned long start,
1610 unsigned long nr_pages,
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -08001611 struct page **pages,
Ira Weiny932f4a62019-05-13 17:17:03 -07001612 struct vm_area_struct **vmas,
1613 unsigned int gup_flags)
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -08001614{
Pingfan Liuaa712392019-07-11 20:57:39 -07001615 unsigned long i;
1616 unsigned long step;
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -08001617 bool drain_allow = true;
1618 bool migrate_allow = true;
1619 LIST_HEAD(cma_page_list);
zhong jiangb96cc652019-11-30 17:49:50 -08001620 long ret = nr_pages;
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -08001621
1622check_again:
Pingfan Liuaa712392019-07-11 20:57:39 -07001623 for (i = 0; i < nr_pages;) {
1624
1625 struct page *head = compound_head(pages[i]);
1626
1627 /*
1628 * gup may start from a tail page. Advance step by the left
1629 * part.
1630 */
Matthew Wilcox (Oracle)d8c65462019-09-23 15:34:30 -07001631 step = compound_nr(head) - (pages[i] - head);
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -08001632 /*
1633 * If we get a page from the CMA zone, since we are going to
1634 * be pinning these entries, we might as well move them out
1635 * of the CMA zone if possible.
1636 */
Pingfan Liuaa712392019-07-11 20:57:39 -07001637 if (is_migrate_cma_page(head)) {
1638 if (PageHuge(head))
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -08001639 isolate_huge_page(head, &cma_page_list);
Pingfan Liuaa712392019-07-11 20:57:39 -07001640 else {
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -08001641 if (!PageLRU(head) && drain_allow) {
1642 lru_add_drain_all();
1643 drain_allow = false;
1644 }
1645
1646 if (!isolate_lru_page(head)) {
1647 list_add_tail(&head->lru, &cma_page_list);
1648 mod_node_page_state(page_pgdat(head),
1649 NR_ISOLATED_ANON +
1650 page_is_file_cache(head),
1651 hpage_nr_pages(head));
1652 }
1653 }
1654 }
Pingfan Liuaa712392019-07-11 20:57:39 -07001655
1656 i += step;
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -08001657 }
1658
1659 if (!list_empty(&cma_page_list)) {
1660 /*
1661 * drop the above get_user_pages reference.
1662 */
1663 for (i = 0; i < nr_pages; i++)
1664 put_page(pages[i]);
1665
1666 if (migrate_pages(&cma_page_list, new_non_cma_page,
1667 NULL, 0, MIGRATE_SYNC, MR_CONTIG_RANGE)) {
1668 /*
1669 * some of the pages failed migration. Do get_user_pages
1670 * without migration.
1671 */
1672 migrate_allow = false;
1673
1674 if (!list_empty(&cma_page_list))
1675 putback_movable_pages(&cma_page_list);
1676 }
1677 /*
Ira Weiny932f4a62019-05-13 17:17:03 -07001678 * We did migrate all the pages, Try to get the page references
1679 * again migrating any new CMA pages which we failed to isolate
1680 * earlier.
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -08001681 */
zhong jiangb96cc652019-11-30 17:49:50 -08001682 ret = __get_user_pages_locked(tsk, mm, start, nr_pages,
Ira Weiny932f4a62019-05-13 17:17:03 -07001683 pages, vmas, NULL,
1684 gup_flags);
1685
zhong jiangb96cc652019-11-30 17:49:50 -08001686 if ((ret > 0) && migrate_allow) {
1687 nr_pages = ret;
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -08001688 drain_allow = true;
1689 goto check_again;
1690 }
1691 }
1692
zhong jiangb96cc652019-11-30 17:49:50 -08001693 return ret;
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -08001694}
1695#else
Ira Weiny932f4a62019-05-13 17:17:03 -07001696static long check_and_migrate_cma_pages(struct task_struct *tsk,
1697 struct mm_struct *mm,
1698 unsigned long start,
1699 unsigned long nr_pages,
1700 struct page **pages,
1701 struct vm_area_struct **vmas,
1702 unsigned int gup_flags)
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -08001703{
1704 return nr_pages;
1705}
Christoph Hellwig050a9ad2019-07-11 20:57:21 -07001706#endif /* CONFIG_CMA */
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -08001707
Dan Williams2bb6d282017-11-29 16:10:35 -08001708/*
Ira Weiny932f4a62019-05-13 17:17:03 -07001709 * __gup_longterm_locked() is a wrapper for __get_user_pages_locked which
1710 * allows us to process the FOLL_LONGTERM flag.
Dan Williams2bb6d282017-11-29 16:10:35 -08001711 */
Ira Weiny932f4a62019-05-13 17:17:03 -07001712static long __gup_longterm_locked(struct task_struct *tsk,
1713 struct mm_struct *mm,
1714 unsigned long start,
1715 unsigned long nr_pages,
1716 struct page **pages,
1717 struct vm_area_struct **vmas,
1718 unsigned int gup_flags)
Dan Williams2bb6d282017-11-29 16:10:35 -08001719{
Ira Weiny932f4a62019-05-13 17:17:03 -07001720 struct vm_area_struct **vmas_tmp = vmas;
1721 unsigned long flags = 0;
Dan Williams2bb6d282017-11-29 16:10:35 -08001722 long rc, i;
1723
Ira Weiny932f4a62019-05-13 17:17:03 -07001724 if (gup_flags & FOLL_LONGTERM) {
1725 if (!pages)
1726 return -EINVAL;
Dan Williams2bb6d282017-11-29 16:10:35 -08001727
Ira Weiny932f4a62019-05-13 17:17:03 -07001728 if (!vmas_tmp) {
1729 vmas_tmp = kcalloc(nr_pages,
1730 sizeof(struct vm_area_struct *),
1731 GFP_KERNEL);
1732 if (!vmas_tmp)
1733 return -ENOMEM;
1734 }
1735 flags = memalloc_nocma_save();
Dan Williams2bb6d282017-11-29 16:10:35 -08001736 }
1737
Ira Weiny932f4a62019-05-13 17:17:03 -07001738 rc = __get_user_pages_locked(tsk, mm, start, nr_pages, pages,
1739 vmas_tmp, NULL, gup_flags);
Dan Williams2bb6d282017-11-29 16:10:35 -08001740
Ira Weiny932f4a62019-05-13 17:17:03 -07001741 if (gup_flags & FOLL_LONGTERM) {
1742 memalloc_nocma_restore(flags);
1743 if (rc < 0)
1744 goto out;
1745
1746 if (check_dax_vmas(vmas_tmp, rc)) {
1747 for (i = 0; i < rc; i++)
1748 put_page(pages[i]);
1749 rc = -EOPNOTSUPP;
1750 goto out;
1751 }
1752
1753 rc = check_and_migrate_cma_pages(tsk, mm, start, rc, pages,
1754 vmas_tmp, gup_flags);
Aneesh Kumar K.V9a4e9f32019-03-05 15:47:44 -08001755 }
1756
Dan Williams2bb6d282017-11-29 16:10:35 -08001757out:
Ira Weiny932f4a62019-05-13 17:17:03 -07001758 if (vmas_tmp != vmas)
1759 kfree(vmas_tmp);
Dan Williams2bb6d282017-11-29 16:10:35 -08001760 return rc;
1761}
Ira Weiny932f4a62019-05-13 17:17:03 -07001762#else /* !CONFIG_FS_DAX && !CONFIG_CMA */
1763static __always_inline long __gup_longterm_locked(struct task_struct *tsk,
1764 struct mm_struct *mm,
1765 unsigned long start,
1766 unsigned long nr_pages,
1767 struct page **pages,
1768 struct vm_area_struct **vmas,
1769 unsigned int flags)
1770{
1771 return __get_user_pages_locked(tsk, mm, start, nr_pages, pages, vmas,
1772 NULL, flags);
1773}
1774#endif /* CONFIG_FS_DAX || CONFIG_CMA */
1775
John Hubbard22bf29b2020-04-01 21:05:10 -07001776#ifdef CONFIG_MMU
1777static long __get_user_pages_remote(struct task_struct *tsk,
1778 struct mm_struct *mm,
1779 unsigned long start, unsigned long nr_pages,
1780 unsigned int gup_flags, struct page **pages,
1781 struct vm_area_struct **vmas, int *locked)
1782{
1783 /*
1784 * Parts of FOLL_LONGTERM behavior are incompatible with
1785 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
1786 * vmas. However, this only comes up if locked is set, and there are
1787 * callers that do request FOLL_LONGTERM, but do not set locked. So,
1788 * allow what we can.
1789 */
1790 if (gup_flags & FOLL_LONGTERM) {
1791 if (WARN_ON_ONCE(locked))
1792 return -EINVAL;
1793 /*
1794 * This will check the vmas (even if our vmas arg is NULL)
1795 * and return -ENOTSUPP if DAX isn't allowed in this case:
1796 */
1797 return __gup_longterm_locked(tsk, mm, start, nr_pages, pages,
1798 vmas, gup_flags | FOLL_TOUCH |
1799 FOLL_REMOTE);
1800 }
1801
1802 return __get_user_pages_locked(tsk, mm, start, nr_pages, pages, vmas,
1803 locked,
1804 gup_flags | FOLL_TOUCH | FOLL_REMOTE);
1805}
1806
Ira Weiny932f4a62019-05-13 17:17:03 -07001807/*
John Hubbardc4237f82020-01-30 22:12:36 -08001808 * get_user_pages_remote() - pin user pages in memory
1809 * @tsk: the task_struct to use for page fault accounting, or
1810 * NULL if faults are not to be recorded.
1811 * @mm: mm_struct of target mm
1812 * @start: starting user address
1813 * @nr_pages: number of pages from start to pin
1814 * @gup_flags: flags modifying lookup behaviour
1815 * @pages: array that receives pointers to the pages pinned.
1816 * Should be at least nr_pages long. Or NULL, if caller
1817 * only intends to ensure the pages are faulted in.
1818 * @vmas: array of pointers to vmas corresponding to each page.
1819 * Or NULL if the caller does not require them.
1820 * @locked: pointer to lock flag indicating whether lock is held and
1821 * subsequently whether VM_FAULT_RETRY functionality can be
1822 * utilised. Lock must initially be held.
1823 *
1824 * Returns either number of pages pinned (which may be less than the
1825 * number requested), or an error. Details about the return value:
1826 *
1827 * -- If nr_pages is 0, returns 0.
1828 * -- If nr_pages is >0, but no pages were pinned, returns -errno.
1829 * -- If nr_pages is >0, and some pages were pinned, returns the number of
1830 * pages pinned. Again, this may be less than nr_pages.
1831 *
1832 * The caller is responsible for releasing returned @pages, via put_page().
1833 *
1834 * @vmas are valid only as long as mmap_sem is held.
1835 *
1836 * Must be called with mmap_sem held for read or write.
1837 *
1838 * get_user_pages walks a process's page tables and takes a reference to
1839 * each struct page that each user address corresponds to at a given
1840 * instant. That is, it takes the page that would be accessed if a user
1841 * thread accesses the given user virtual address at that instant.
1842 *
1843 * This does not guarantee that the page exists in the user mappings when
1844 * get_user_pages returns, and there may even be a completely different
1845 * page there in some cases (eg. if mmapped pagecache has been invalidated
1846 * and subsequently re faulted). However it does guarantee that the page
1847 * won't be freed completely. And mostly callers simply care that the page
1848 * contains data that was valid *at some point in time*. Typically, an IO
1849 * or similar operation cannot guarantee anything stronger anyway because
1850 * locks can't be held over the syscall boundary.
1851 *
1852 * If gup_flags & FOLL_WRITE == 0, the page must not be written to. If the page
1853 * is written to, set_page_dirty (or set_page_dirty_lock, as appropriate) must
1854 * be called after the page is finished with, and before put_page is called.
1855 *
1856 * get_user_pages is typically used for fewer-copy IO operations, to get a
1857 * handle on the memory by some means other than accesses via the user virtual
1858 * addresses. The pages may be submitted for DMA to devices or accessed via
1859 * their kernel linear mapping (via the kmap APIs). Care should be taken to
1860 * use the correct cache flushing APIs.
1861 *
1862 * See also get_user_pages_fast, for performance critical applications.
1863 *
1864 * get_user_pages should be phased out in favor of
1865 * get_user_pages_locked|unlocked or get_user_pages_fast. Nothing
1866 * should use get_user_pages because it cannot pass
1867 * FAULT_FLAG_ALLOW_RETRY to handle_mm_fault.
1868 */
1869long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
1870 unsigned long start, unsigned long nr_pages,
1871 unsigned int gup_flags, struct page **pages,
1872 struct vm_area_struct **vmas, int *locked)
1873{
1874 /*
John Hubbardeddb1c22020-01-30 22:12:54 -08001875 * FOLL_PIN must only be set internally by the pin_user_pages*() APIs,
1876 * never directly by the caller, so enforce that with an assertion:
1877 */
1878 if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
1879 return -EINVAL;
1880
John Hubbard22bf29b2020-04-01 21:05:10 -07001881 return __get_user_pages_remote(tsk, mm, start, nr_pages, gup_flags,
1882 pages, vmas, locked);
John Hubbardc4237f82020-01-30 22:12:36 -08001883}
1884EXPORT_SYMBOL(get_user_pages_remote);
1885
John Hubbardeddb1c22020-01-30 22:12:54 -08001886#else /* CONFIG_MMU */
1887long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
1888 unsigned long start, unsigned long nr_pages,
1889 unsigned int gup_flags, struct page **pages,
1890 struct vm_area_struct **vmas, int *locked)
1891{
1892 return 0;
1893}
John Hubbard3faa52c2020-04-01 21:05:29 -07001894
1895static long __get_user_pages_remote(struct task_struct *tsk,
1896 struct mm_struct *mm,
1897 unsigned long start, unsigned long nr_pages,
1898 unsigned int gup_flags, struct page **pages,
1899 struct vm_area_struct **vmas, int *locked)
1900{
1901 return 0;
1902}
John Hubbardeddb1c22020-01-30 22:12:54 -08001903#endif /* !CONFIG_MMU */
1904
John Hubbardc4237f82020-01-30 22:12:36 -08001905/*
Ira Weiny932f4a62019-05-13 17:17:03 -07001906 * This is the same as get_user_pages_remote(), just with a
1907 * less-flexible calling convention where we assume that the task
1908 * and mm being operated on are the current task's and don't allow
1909 * passing of a locked parameter. We also obviously don't pass
1910 * FOLL_REMOTE in here.
1911 */
1912long get_user_pages(unsigned long start, unsigned long nr_pages,
1913 unsigned int gup_flags, struct page **pages,
1914 struct vm_area_struct **vmas)
1915{
John Hubbardeddb1c22020-01-30 22:12:54 -08001916 /*
1917 * FOLL_PIN must only be set internally by the pin_user_pages*() APIs,
1918 * never directly by the caller, so enforce that with an assertion:
1919 */
1920 if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
1921 return -EINVAL;
1922
Ira Weiny932f4a62019-05-13 17:17:03 -07001923 return __gup_longterm_locked(current, current->mm, start, nr_pages,
1924 pages, vmas, gup_flags | FOLL_TOUCH);
1925}
1926EXPORT_SYMBOL(get_user_pages);
Dan Williams2bb6d282017-11-29 16:10:35 -08001927
Christoph Hellwigd3649f62019-07-11 20:57:18 -07001928/*
1929 * We can leverage the VM_FAULT_RETRY functionality in the page fault
1930 * paths better by using either get_user_pages_locked() or
1931 * get_user_pages_unlocked().
Kirill A. Shutemovacc3c8d2015-04-14 15:44:45 -07001932 *
Christoph Hellwigd3649f62019-07-11 20:57:18 -07001933 * get_user_pages_locked() is suitable to replace the form:
Kirill A. Shutemovacc3c8d2015-04-14 15:44:45 -07001934 *
Christoph Hellwigd3649f62019-07-11 20:57:18 -07001935 * down_read(&mm->mmap_sem);
1936 * do_something()
1937 * get_user_pages(tsk, mm, ..., pages, NULL);
1938 * up_read(&mm->mmap_sem);
Kirill A. Shutemovacc3c8d2015-04-14 15:44:45 -07001939 *
Christoph Hellwigd3649f62019-07-11 20:57:18 -07001940 * to:
Kirill A. Shutemovacc3c8d2015-04-14 15:44:45 -07001941 *
Christoph Hellwigd3649f62019-07-11 20:57:18 -07001942 * int locked = 1;
1943 * down_read(&mm->mmap_sem);
1944 * do_something()
1945 * get_user_pages_locked(tsk, mm, ..., pages, &locked);
1946 * if (locked)
1947 * up_read(&mm->mmap_sem);
Kirill A. Shutemovacc3c8d2015-04-14 15:44:45 -07001948 */
Christoph Hellwigd3649f62019-07-11 20:57:18 -07001949long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
1950 unsigned int gup_flags, struct page **pages,
1951 int *locked)
Kirill A. Shutemovacc3c8d2015-04-14 15:44:45 -07001952{
Kirill A. Shutemovacc3c8d2015-04-14 15:44:45 -07001953 /*
Christoph Hellwigd3649f62019-07-11 20:57:18 -07001954 * FIXME: Current FOLL_LONGTERM behavior is incompatible with
1955 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
1956 * vmas. As there are no users of this flag in this call we simply
1957 * disallow this option for now.
Kirill A. Shutemovacc3c8d2015-04-14 15:44:45 -07001958 */
Christoph Hellwigd3649f62019-07-11 20:57:18 -07001959 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
1960 return -EINVAL;
Kirill A. Shutemovacc3c8d2015-04-14 15:44:45 -07001961
Christoph Hellwigd3649f62019-07-11 20:57:18 -07001962 return __get_user_pages_locked(current, current->mm, start, nr_pages,
1963 pages, NULL, locked,
1964 gup_flags | FOLL_TOUCH);
Kirill A. Shutemovacc3c8d2015-04-14 15:44:45 -07001965}
Christoph Hellwigd3649f62019-07-11 20:57:18 -07001966EXPORT_SYMBOL(get_user_pages_locked);
Kirill A. Shutemovacc3c8d2015-04-14 15:44:45 -07001967
1968/*
Christoph Hellwigd3649f62019-07-11 20:57:18 -07001969 * get_user_pages_unlocked() is suitable to replace the form:
Kirill A. Shutemovacc3c8d2015-04-14 15:44:45 -07001970 *
Christoph Hellwigd3649f62019-07-11 20:57:18 -07001971 * down_read(&mm->mmap_sem);
1972 * get_user_pages(tsk, mm, ..., pages, NULL);
1973 * up_read(&mm->mmap_sem);
1974 *
1975 * with:
1976 *
1977 * get_user_pages_unlocked(tsk, mm, ..., pages);
1978 *
1979 * It is functionally equivalent to get_user_pages_fast so
1980 * get_user_pages_fast should be used instead if specific gup_flags
1981 * (e.g. FOLL_FORCE) are not required.
Kirill A. Shutemovacc3c8d2015-04-14 15:44:45 -07001982 */
Christoph Hellwigd3649f62019-07-11 20:57:18 -07001983long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
1984 struct page **pages, unsigned int gup_flags)
Kirill A. Shutemovacc3c8d2015-04-14 15:44:45 -07001985{
1986 struct mm_struct *mm = current->mm;
Christoph Hellwigd3649f62019-07-11 20:57:18 -07001987 int locked = 1;
1988 long ret;
Kirill A. Shutemovacc3c8d2015-04-14 15:44:45 -07001989
Christoph Hellwigd3649f62019-07-11 20:57:18 -07001990 /*
1991 * FIXME: Current FOLL_LONGTERM behavior is incompatible with
1992 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
1993 * vmas. As there are no users of this flag in this call we simply
1994 * disallow this option for now.
1995 */
1996 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
1997 return -EINVAL;
Kirill A. Shutemovacc3c8d2015-04-14 15:44:45 -07001998
Christoph Hellwigd3649f62019-07-11 20:57:18 -07001999 down_read(&mm->mmap_sem);
2000 ret = __get_user_pages_locked(current, mm, start, nr_pages, pages, NULL,
2001 &locked, gup_flags | FOLL_TOUCH);
Kirill A. Shutemovacc3c8d2015-04-14 15:44:45 -07002002 if (locked)
2003 up_read(&mm->mmap_sem);
Christoph Hellwigd3649f62019-07-11 20:57:18 -07002004 return ret;
Kirill A. Shutemovacc3c8d2015-04-14 15:44:45 -07002005}
Christoph Hellwigd3649f62019-07-11 20:57:18 -07002006EXPORT_SYMBOL(get_user_pages_unlocked);
Steve Capper2667f502014-10-09 15:29:14 -07002007
2008/*
Christoph Hellwig67a929e2019-07-11 20:57:14 -07002009 * Fast GUP
Steve Capper2667f502014-10-09 15:29:14 -07002010 *
2011 * get_user_pages_fast attempts to pin user pages by walking the page
2012 * tables directly and avoids taking locks. Thus the walker needs to be
2013 * protected from page table pages being freed from under it, and should
2014 * block any THP splits.
2015 *
2016 * One way to achieve this is to have the walker disable interrupts, and
2017 * rely on IPIs from the TLB flushing code blocking before the page table
2018 * pages are freed. This is unsuitable for architectures that do not need
2019 * to broadcast an IPI when invalidating TLBs.
2020 *
2021 * Another way to achieve this is to batch up page table containing pages
2022 * belonging to more than one mm_user, then rcu_sched a callback to free those
2023 * pages. Disabling interrupts will allow the fast_gup walker to both block
2024 * the rcu_sched callback, and an IPI that we broadcast for splitting THPs
2025 * (which is a relatively rare event). The code below adopts this strategy.
2026 *
2027 * Before activating this code, please be aware that the following assumptions
2028 * are currently made:
2029 *
Peter Zijlstraff2e6d722020-02-03 17:37:02 -08002030 * *) Either MMU_GATHER_RCU_TABLE_FREE is enabled, and tlb_remove_table() is used to
Kirill A. Shutemove5855132017-06-06 14:31:20 +03002031 * free pages containing page tables or TLB flushing requires IPI broadcast.
Steve Capper2667f502014-10-09 15:29:14 -07002032 *
Steve Capper2667f502014-10-09 15:29:14 -07002033 * *) ptes can be read atomically by the architecture.
2034 *
2035 * *) access_ok is sufficient to validate userspace address ranges.
2036 *
2037 * The last two assumptions can be relaxed by the addition of helper functions.
2038 *
2039 * This code is based heavily on the PowerPC implementation by Nick Piggin.
2040 */
Christoph Hellwig67a929e2019-07-11 20:57:14 -07002041#ifdef CONFIG_HAVE_FAST_GUP
John Hubbard3faa52c2020-04-01 21:05:29 -07002042
2043static void put_compound_head(struct page *page, int refs, unsigned int flags)
2044{
John Hubbard47e29d32020-04-01 21:05:33 -07002045 if (flags & FOLL_PIN) {
John Hubbard1970dc62020-04-01 21:05:37 -07002046 mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_RELEASED,
2047 refs);
2048
John Hubbard47e29d32020-04-01 21:05:33 -07002049 if (hpage_pincount_available(page))
2050 hpage_pincount_sub(page, refs);
2051 else
2052 refs *= GUP_PIN_COUNTING_BIAS;
2053 }
John Hubbard3faa52c2020-04-01 21:05:29 -07002054
2055 VM_BUG_ON_PAGE(page_ref_count(page) < refs, page);
2056 /*
2057 * Calling put_page() for each ref is unnecessarily slow. Only the last
2058 * ref needs a put_page().
2059 */
2060 if (refs > 1)
2061 page_ref_sub(page, refs - 1);
2062 put_page(page);
2063}
2064
Christoph Hellwig39656e82019-07-11 20:56:49 -07002065#ifdef CONFIG_GUP_GET_PTE_LOW_HIGH
John Hubbard3faa52c2020-04-01 21:05:29 -07002066
Kirill A. Shutemov0005d202017-03-16 18:26:51 +03002067/*
Christoph Hellwig39656e82019-07-11 20:56:49 -07002068 * WARNING: only to be used in the get_user_pages_fast() implementation.
2069 *
2070 * With get_user_pages_fast(), we walk down the pagetables without taking any
2071 * locks. For this we would like to load the pointers atomically, but sometimes
2072 * that is not possible (e.g. without expensive cmpxchg8b on x86_32 PAE). What
2073 * we do have is the guarantee that a PTE will only either go from not present
2074 * to present, or present to not present or both -- it will not switch to a
2075 * completely different present page without a TLB flush in between; something
2076 * that we are blocking by holding interrupts off.
2077 *
2078 * Setting ptes from not present to present goes:
2079 *
2080 * ptep->pte_high = h;
2081 * smp_wmb();
2082 * ptep->pte_low = l;
2083 *
2084 * And present to not present goes:
2085 *
2086 * ptep->pte_low = 0;
2087 * smp_wmb();
2088 * ptep->pte_high = 0;
2089 *
2090 * We must ensure here that the load of pte_low sees 'l' IFF pte_high sees 'h'.
2091 * We load pte_high *after* loading pte_low, which ensures we don't see an older
2092 * value of pte_high. *Then* we recheck pte_low, which ensures that we haven't
2093 * picked up a changed pte high. We might have gotten rubbish values from
2094 * pte_low and pte_high, but we are guaranteed that pte_low will not have the
2095 * present bit set *unless* it is 'l'. Because get_user_pages_fast() only
2096 * operates on present ptes we're safe.
2097 */
2098static inline pte_t gup_get_pte(pte_t *ptep)
2099{
2100 pte_t pte;
2101
2102 do {
2103 pte.pte_low = ptep->pte_low;
2104 smp_rmb();
2105 pte.pte_high = ptep->pte_high;
2106 smp_rmb();
2107 } while (unlikely(pte.pte_low != ptep->pte_low));
2108
2109 return pte;
2110}
2111#else /* CONFIG_GUP_GET_PTE_LOW_HIGH */
2112/*
2113 * We require that the PTE can be read atomically.
Kirill A. Shutemov0005d202017-03-16 18:26:51 +03002114 */
2115static inline pte_t gup_get_pte(pte_t *ptep)
2116{
2117 return READ_ONCE(*ptep);
2118}
Christoph Hellwig39656e82019-07-11 20:56:49 -07002119#endif /* CONFIG_GUP_GET_PTE_LOW_HIGH */
Kirill A. Shutemov0005d202017-03-16 18:26:51 +03002120
Guenter Roeck790c7362019-07-11 20:57:46 -07002121static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start,
John Hubbard3b78d832020-04-01 21:05:22 -07002122 unsigned int flags,
Guenter Roeck790c7362019-07-11 20:57:46 -07002123 struct page **pages)
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002124{
2125 while ((*nr) - nr_start) {
2126 struct page *page = pages[--(*nr)];
2127
2128 ClearPageReferenced(page);
John Hubbard3faa52c2020-04-01 21:05:29 -07002129 if (flags & FOLL_PIN)
2130 unpin_user_page(page);
2131 else
2132 put_page(page);
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002133 }
2134}
2135
Laurent Dufour3010a5e2018-06-07 17:06:08 -07002136#ifdef CONFIG_ARCH_HAS_PTE_SPECIAL
Steve Capper2667f502014-10-09 15:29:14 -07002137static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
Ira Weinyb798bec2019-05-13 17:17:07 -07002138 unsigned int flags, struct page **pages, int *nr)
Steve Capper2667f502014-10-09 15:29:14 -07002139{
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002140 struct dev_pagemap *pgmap = NULL;
2141 int nr_start = *nr, ret = 0;
Steve Capper2667f502014-10-09 15:29:14 -07002142 pte_t *ptep, *ptem;
Steve Capper2667f502014-10-09 15:29:14 -07002143
2144 ptem = ptep = pte_offset_map(&pmd, addr);
2145 do {
Kirill A. Shutemov0005d202017-03-16 18:26:51 +03002146 pte_t pte = gup_get_pte(ptep);
Kirill A. Shutemov7aef4172016-01-15 16:52:32 -08002147 struct page *head, *page;
Steve Capper2667f502014-10-09 15:29:14 -07002148
2149 /*
2150 * Similar to the PMD case below, NUMA hinting must take slow
Mel Gorman8a0516e2015-02-12 14:58:22 -08002151 * path using the pte_protnone check.
Steve Capper2667f502014-10-09 15:29:14 -07002152 */
Kirill A. Shutemove7884f82017-03-16 18:26:50 +03002153 if (pte_protnone(pte))
2154 goto pte_unmap;
2155
Ira Weinyb798bec2019-05-13 17:17:07 -07002156 if (!pte_access_permitted(pte, flags & FOLL_WRITE))
Kirill A. Shutemove7884f82017-03-16 18:26:50 +03002157 goto pte_unmap;
2158
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002159 if (pte_devmap(pte)) {
Ira Weiny7af75562019-05-13 17:17:14 -07002160 if (unlikely(flags & FOLL_LONGTERM))
2161 goto pte_unmap;
2162
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002163 pgmap = get_dev_pagemap(pte_pfn(pte), pgmap);
2164 if (unlikely(!pgmap)) {
John Hubbard3b78d832020-04-01 21:05:22 -07002165 undo_dev_pagemap(nr, nr_start, flags, pages);
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002166 goto pte_unmap;
2167 }
2168 } else if (pte_special(pte))
Steve Capper2667f502014-10-09 15:29:14 -07002169 goto pte_unmap;
2170
2171 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
2172 page = pte_page(pte);
2173
John Hubbard3faa52c2020-04-01 21:05:29 -07002174 head = try_grab_compound_head(page, 1, flags);
Linus Torvalds8fde12c2019-04-11 10:49:19 -07002175 if (!head)
Steve Capper2667f502014-10-09 15:29:14 -07002176 goto pte_unmap;
2177
2178 if (unlikely(pte_val(pte) != pte_val(*ptep))) {
John Hubbard3faa52c2020-04-01 21:05:29 -07002179 put_compound_head(head, 1, flags);
Steve Capper2667f502014-10-09 15:29:14 -07002180 goto pte_unmap;
2181 }
2182
Kirill A. Shutemov7aef4172016-01-15 16:52:32 -08002183 VM_BUG_ON_PAGE(compound_head(page) != head, page);
Kirill A. Shutemove9348052017-03-16 18:26:52 +03002184
Claudio Imbrendaf28d4362020-04-01 21:05:56 -07002185 /*
2186 * We need to make the page accessible if and only if we are
2187 * going to access its content (the FOLL_PIN case). Please
2188 * see Documentation/core-api/pin_user_pages.rst for
2189 * details.
2190 */
2191 if (flags & FOLL_PIN) {
2192 ret = arch_make_page_accessible(page);
2193 if (ret) {
2194 unpin_user_page(page);
2195 goto pte_unmap;
2196 }
2197 }
Kirill A. Shutemove9348052017-03-16 18:26:52 +03002198 SetPageReferenced(page);
Steve Capper2667f502014-10-09 15:29:14 -07002199 pages[*nr] = page;
2200 (*nr)++;
2201
2202 } while (ptep++, addr += PAGE_SIZE, addr != end);
2203
2204 ret = 1;
2205
2206pte_unmap:
Christoph Hellwig832d7aa2017-12-29 08:54:01 +01002207 if (pgmap)
2208 put_dev_pagemap(pgmap);
Steve Capper2667f502014-10-09 15:29:14 -07002209 pte_unmap(ptem);
2210 return ret;
2211}
2212#else
2213
2214/*
2215 * If we can't determine whether or not a pte is special, then fail immediately
2216 * for ptes. Note, we can still pin HugeTLB and THP as these are guaranteed not
2217 * to be special.
2218 *
2219 * For a futex to be placed on a THP tail page, get_futex_key requires a
2220 * __get_user_pages_fast implementation that can pin pages. Thus it's still
2221 * useful to have gup_huge_pmd even if we can't operate on ptes.
2222 */
2223static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
Ira Weinyb798bec2019-05-13 17:17:07 -07002224 unsigned int flags, struct page **pages, int *nr)
Steve Capper2667f502014-10-09 15:29:14 -07002225{
2226 return 0;
2227}
Laurent Dufour3010a5e2018-06-07 17:06:08 -07002228#endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */
Steve Capper2667f502014-10-09 15:29:14 -07002229
Robin Murphy17596732019-07-16 16:30:47 -07002230#if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002231static int __gup_device_huge(unsigned long pfn, unsigned long addr,
John Hubbard86dfbed2020-04-01 21:05:14 -07002232 unsigned long end, unsigned int flags,
2233 struct page **pages, int *nr)
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002234{
2235 int nr_start = *nr;
2236 struct dev_pagemap *pgmap = NULL;
2237
2238 do {
2239 struct page *page = pfn_to_page(pfn);
2240
2241 pgmap = get_dev_pagemap(pfn, pgmap);
2242 if (unlikely(!pgmap)) {
John Hubbard3b78d832020-04-01 21:05:22 -07002243 undo_dev_pagemap(nr, nr_start, flags, pages);
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002244 return 0;
2245 }
2246 SetPageReferenced(page);
2247 pages[*nr] = page;
John Hubbard3faa52c2020-04-01 21:05:29 -07002248 if (unlikely(!try_grab_page(page, flags))) {
2249 undo_dev_pagemap(nr, nr_start, flags, pages);
2250 return 0;
2251 }
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002252 (*nr)++;
2253 pfn++;
2254 } while (addr += PAGE_SIZE, addr != end);
Christoph Hellwig832d7aa2017-12-29 08:54:01 +01002255
2256 if (pgmap)
2257 put_dev_pagemap(pgmap);
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002258 return 1;
2259}
2260
Dan Williamsa9b6de72018-04-19 21:32:19 -07002261static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
John Hubbard86dfbed2020-04-01 21:05:14 -07002262 unsigned long end, unsigned int flags,
2263 struct page **pages, int *nr)
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002264{
2265 unsigned long fault_pfn;
Dan Williamsa9b6de72018-04-19 21:32:19 -07002266 int nr_start = *nr;
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002267
Dan Williamsa9b6de72018-04-19 21:32:19 -07002268 fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
John Hubbard86dfbed2020-04-01 21:05:14 -07002269 if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
Dan Williamsa9b6de72018-04-19 21:32:19 -07002270 return 0;
2271
2272 if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
John Hubbard3b78d832020-04-01 21:05:22 -07002273 undo_dev_pagemap(nr, nr_start, flags, pages);
Dan Williamsa9b6de72018-04-19 21:32:19 -07002274 return 0;
2275 }
2276 return 1;
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002277}
2278
Dan Williamsa9b6de72018-04-19 21:32:19 -07002279static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
John Hubbard86dfbed2020-04-01 21:05:14 -07002280 unsigned long end, unsigned int flags,
2281 struct page **pages, int *nr)
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002282{
2283 unsigned long fault_pfn;
Dan Williamsa9b6de72018-04-19 21:32:19 -07002284 int nr_start = *nr;
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002285
Dan Williamsa9b6de72018-04-19 21:32:19 -07002286 fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
John Hubbard86dfbed2020-04-01 21:05:14 -07002287 if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
Dan Williamsa9b6de72018-04-19 21:32:19 -07002288 return 0;
2289
2290 if (unlikely(pud_val(orig) != pud_val(*pudp))) {
John Hubbard3b78d832020-04-01 21:05:22 -07002291 undo_dev_pagemap(nr, nr_start, flags, pages);
Dan Williamsa9b6de72018-04-19 21:32:19 -07002292 return 0;
2293 }
2294 return 1;
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002295}
2296#else
Dan Williamsa9b6de72018-04-19 21:32:19 -07002297static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
John Hubbard86dfbed2020-04-01 21:05:14 -07002298 unsigned long end, unsigned int flags,
2299 struct page **pages, int *nr)
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002300{
2301 BUILD_BUG();
2302 return 0;
2303}
2304
Dan Williamsa9b6de72018-04-19 21:32:19 -07002305static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,
John Hubbard86dfbed2020-04-01 21:05:14 -07002306 unsigned long end, unsigned int flags,
2307 struct page **pages, int *nr)
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002308{
2309 BUILD_BUG();
2310 return 0;
2311}
2312#endif
2313
John Hubbarda43e9822020-01-30 22:12:17 -08002314static int record_subpages(struct page *page, unsigned long addr,
2315 unsigned long end, struct page **pages)
2316{
2317 int nr;
2318
2319 for (nr = 0; addr != end; addr += PAGE_SIZE)
2320 pages[nr++] = page++;
2321
2322 return nr;
2323}
2324
Christoph Hellwigcbd34da2019-07-11 20:57:28 -07002325#ifdef CONFIG_ARCH_HAS_HUGEPD
2326static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
2327 unsigned long sz)
2328{
2329 unsigned long __boundary = (addr + sz) & ~(sz-1);
2330 return (__boundary - 1 < end - 1) ? __boundary : end;
2331}
2332
2333static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
John Hubbard0cd22af2019-10-18 20:19:53 -07002334 unsigned long end, unsigned int flags,
2335 struct page **pages, int *nr)
Christoph Hellwigcbd34da2019-07-11 20:57:28 -07002336{
2337 unsigned long pte_end;
2338 struct page *head, *page;
2339 pte_t pte;
2340 int refs;
2341
2342 pte_end = (addr + sz) & ~(sz-1);
2343 if (pte_end < end)
2344 end = pte_end;
2345
2346 pte = READ_ONCE(*ptep);
2347
John Hubbard0cd22af2019-10-18 20:19:53 -07002348 if (!pte_access_permitted(pte, flags & FOLL_WRITE))
Christoph Hellwigcbd34da2019-07-11 20:57:28 -07002349 return 0;
2350
2351 /* hugepages are never "special" */
2352 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
2353
Christoph Hellwigcbd34da2019-07-11 20:57:28 -07002354 head = pte_page(pte);
Christoph Hellwigcbd34da2019-07-11 20:57:28 -07002355 page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
John Hubbarda43e9822020-01-30 22:12:17 -08002356 refs = record_subpages(page, addr, end, pages + *nr);
Christoph Hellwigcbd34da2019-07-11 20:57:28 -07002357
John Hubbard3faa52c2020-04-01 21:05:29 -07002358 head = try_grab_compound_head(head, refs, flags);
John Hubbarda43e9822020-01-30 22:12:17 -08002359 if (!head)
Christoph Hellwigcbd34da2019-07-11 20:57:28 -07002360 return 0;
Christoph Hellwigcbd34da2019-07-11 20:57:28 -07002361
2362 if (unlikely(pte_val(pte) != pte_val(*ptep))) {
John Hubbard3b78d832020-04-01 21:05:22 -07002363 put_compound_head(head, refs, flags);
Christoph Hellwigcbd34da2019-07-11 20:57:28 -07002364 return 0;
2365 }
2366
John Hubbarda43e9822020-01-30 22:12:17 -08002367 *nr += refs;
Christoph Hellwig520b4a42019-07-11 20:57:36 -07002368 SetPageReferenced(head);
Christoph Hellwigcbd34da2019-07-11 20:57:28 -07002369 return 1;
2370}
2371
2372static int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
John Hubbard0cd22af2019-10-18 20:19:53 -07002373 unsigned int pdshift, unsigned long end, unsigned int flags,
Christoph Hellwigcbd34da2019-07-11 20:57:28 -07002374 struct page **pages, int *nr)
2375{
2376 pte_t *ptep;
2377 unsigned long sz = 1UL << hugepd_shift(hugepd);
2378 unsigned long next;
2379
2380 ptep = hugepte_offset(hugepd, addr, pdshift);
2381 do {
2382 next = hugepte_addr_end(addr, end, sz);
John Hubbard0cd22af2019-10-18 20:19:53 -07002383 if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr))
Christoph Hellwigcbd34da2019-07-11 20:57:28 -07002384 return 0;
2385 } while (ptep++, addr = next, addr != end);
2386
2387 return 1;
2388}
2389#else
2390static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
John Hubbard0cd22af2019-10-18 20:19:53 -07002391 unsigned int pdshift, unsigned long end, unsigned int flags,
Christoph Hellwigcbd34da2019-07-11 20:57:28 -07002392 struct page **pages, int *nr)
2393{
2394 return 0;
2395}
2396#endif /* CONFIG_ARCH_HAS_HUGEPD */
2397
Steve Capper2667f502014-10-09 15:29:14 -07002398static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
John Hubbard0cd22af2019-10-18 20:19:53 -07002399 unsigned long end, unsigned int flags,
2400 struct page **pages, int *nr)
Steve Capper2667f502014-10-09 15:29:14 -07002401{
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -08002402 struct page *head, *page;
Steve Capper2667f502014-10-09 15:29:14 -07002403 int refs;
2404
Ira Weinyb798bec2019-05-13 17:17:07 -07002405 if (!pmd_access_permitted(orig, flags & FOLL_WRITE))
Steve Capper2667f502014-10-09 15:29:14 -07002406 return 0;
2407
Ira Weiny7af75562019-05-13 17:17:14 -07002408 if (pmd_devmap(orig)) {
2409 if (unlikely(flags & FOLL_LONGTERM))
2410 return 0;
John Hubbard86dfbed2020-04-01 21:05:14 -07002411 return __gup_device_huge_pmd(orig, pmdp, addr, end, flags,
2412 pages, nr);
Ira Weiny7af75562019-05-13 17:17:14 -07002413 }
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002414
Punit Agrawald63206e2017-07-06 15:39:39 -07002415 page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
John Hubbarda43e9822020-01-30 22:12:17 -08002416 refs = record_subpages(page, addr, end, pages + *nr);
Steve Capper2667f502014-10-09 15:29:14 -07002417
John Hubbard3faa52c2020-04-01 21:05:29 -07002418 head = try_grab_compound_head(pmd_page(orig), refs, flags);
John Hubbarda43e9822020-01-30 22:12:17 -08002419 if (!head)
Steve Capper2667f502014-10-09 15:29:14 -07002420 return 0;
Steve Capper2667f502014-10-09 15:29:14 -07002421
2422 if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
John Hubbard3b78d832020-04-01 21:05:22 -07002423 put_compound_head(head, refs, flags);
Steve Capper2667f502014-10-09 15:29:14 -07002424 return 0;
2425 }
2426
John Hubbarda43e9822020-01-30 22:12:17 -08002427 *nr += refs;
Kirill A. Shutemove9348052017-03-16 18:26:52 +03002428 SetPageReferenced(head);
Steve Capper2667f502014-10-09 15:29:14 -07002429 return 1;
2430}
2431
2432static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
John Hubbard86dfbed2020-04-01 21:05:14 -07002433 unsigned long end, unsigned int flags,
2434 struct page **pages, int *nr)
Steve Capper2667f502014-10-09 15:29:14 -07002435{
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -08002436 struct page *head, *page;
Steve Capper2667f502014-10-09 15:29:14 -07002437 int refs;
2438
Ira Weinyb798bec2019-05-13 17:17:07 -07002439 if (!pud_access_permitted(orig, flags & FOLL_WRITE))
Steve Capper2667f502014-10-09 15:29:14 -07002440 return 0;
2441
Ira Weiny7af75562019-05-13 17:17:14 -07002442 if (pud_devmap(orig)) {
2443 if (unlikely(flags & FOLL_LONGTERM))
2444 return 0;
John Hubbard86dfbed2020-04-01 21:05:14 -07002445 return __gup_device_huge_pud(orig, pudp, addr, end, flags,
2446 pages, nr);
Ira Weiny7af75562019-05-13 17:17:14 -07002447 }
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002448
Punit Agrawald63206e2017-07-06 15:39:39 -07002449 page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
John Hubbarda43e9822020-01-30 22:12:17 -08002450 refs = record_subpages(page, addr, end, pages + *nr);
Steve Capper2667f502014-10-09 15:29:14 -07002451
John Hubbard3faa52c2020-04-01 21:05:29 -07002452 head = try_grab_compound_head(pud_page(orig), refs, flags);
John Hubbarda43e9822020-01-30 22:12:17 -08002453 if (!head)
Steve Capper2667f502014-10-09 15:29:14 -07002454 return 0;
Steve Capper2667f502014-10-09 15:29:14 -07002455
2456 if (unlikely(pud_val(orig) != pud_val(*pudp))) {
John Hubbard3b78d832020-04-01 21:05:22 -07002457 put_compound_head(head, refs, flags);
Steve Capper2667f502014-10-09 15:29:14 -07002458 return 0;
2459 }
2460
John Hubbarda43e9822020-01-30 22:12:17 -08002461 *nr += refs;
Kirill A. Shutemove9348052017-03-16 18:26:52 +03002462 SetPageReferenced(head);
Steve Capper2667f502014-10-09 15:29:14 -07002463 return 1;
2464}
2465
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +05302466static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
Ira Weinyb798bec2019-05-13 17:17:07 -07002467 unsigned long end, unsigned int flags,
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +05302468 struct page **pages, int *nr)
2469{
2470 int refs;
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -08002471 struct page *head, *page;
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +05302472
Ira Weinyb798bec2019-05-13 17:17:07 -07002473 if (!pgd_access_permitted(orig, flags & FOLL_WRITE))
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +05302474 return 0;
2475
Kirill A. Shutemovb59f65f2017-03-16 18:26:53 +03002476 BUILD_BUG_ON(pgd_devmap(orig));
John Hubbarda43e9822020-01-30 22:12:17 -08002477
Punit Agrawald63206e2017-07-06 15:39:39 -07002478 page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);
John Hubbarda43e9822020-01-30 22:12:17 -08002479 refs = record_subpages(page, addr, end, pages + *nr);
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +05302480
John Hubbard3faa52c2020-04-01 21:05:29 -07002481 head = try_grab_compound_head(pgd_page(orig), refs, flags);
John Hubbarda43e9822020-01-30 22:12:17 -08002482 if (!head)
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +05302483 return 0;
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +05302484
2485 if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) {
John Hubbard3b78d832020-04-01 21:05:22 -07002486 put_compound_head(head, refs, flags);
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +05302487 return 0;
2488 }
2489
John Hubbarda43e9822020-01-30 22:12:17 -08002490 *nr += refs;
Kirill A. Shutemove9348052017-03-16 18:26:52 +03002491 SetPageReferenced(head);
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +05302492 return 1;
2493}
2494
Steve Capper2667f502014-10-09 15:29:14 -07002495static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
Ira Weinyb798bec2019-05-13 17:17:07 -07002496 unsigned int flags, struct page **pages, int *nr)
Steve Capper2667f502014-10-09 15:29:14 -07002497{
2498 unsigned long next;
2499 pmd_t *pmdp;
2500
2501 pmdp = pmd_offset(&pud, addr);
2502 do {
Christian Borntraeger38c5ce92015-01-06 22:54:46 +01002503 pmd_t pmd = READ_ONCE(*pmdp);
Steve Capper2667f502014-10-09 15:29:14 -07002504
2505 next = pmd_addr_end(addr, end);
Zi Yan84c3fc42017-09-08 16:11:01 -07002506 if (!pmd_present(pmd))
Steve Capper2667f502014-10-09 15:29:14 -07002507 return 0;
2508
Yu Zhao414fd082019-02-12 15:35:58 -08002509 if (unlikely(pmd_trans_huge(pmd) || pmd_huge(pmd) ||
2510 pmd_devmap(pmd))) {
Steve Capper2667f502014-10-09 15:29:14 -07002511 /*
2512 * NUMA hinting faults need to be handled in the GUP
2513 * slowpath for accounting purposes and so that they
2514 * can be serialised against THP migration.
2515 */
Mel Gorman8a0516e2015-02-12 14:58:22 -08002516 if (pmd_protnone(pmd))
Steve Capper2667f502014-10-09 15:29:14 -07002517 return 0;
2518
Ira Weinyb798bec2019-05-13 17:17:07 -07002519 if (!gup_huge_pmd(pmd, pmdp, addr, next, flags,
Steve Capper2667f502014-10-09 15:29:14 -07002520 pages, nr))
2521 return 0;
2522
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +05302523 } else if (unlikely(is_hugepd(__hugepd(pmd_val(pmd))))) {
2524 /*
2525 * architecture have different format for hugetlbfs
2526 * pmd format and THP pmd format
2527 */
2528 if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
Ira Weinyb798bec2019-05-13 17:17:07 -07002529 PMD_SHIFT, next, flags, pages, nr))
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +05302530 return 0;
Ira Weinyb798bec2019-05-13 17:17:07 -07002531 } else if (!gup_pte_range(pmd, addr, next, flags, pages, nr))
Mario Leinweber29231172018-04-05 16:24:18 -07002532 return 0;
Steve Capper2667f502014-10-09 15:29:14 -07002533 } while (pmdp++, addr = next, addr != end);
2534
2535 return 1;
2536}
2537
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +03002538static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end,
Ira Weinyb798bec2019-05-13 17:17:07 -07002539 unsigned int flags, struct page **pages, int *nr)
Steve Capper2667f502014-10-09 15:29:14 -07002540{
2541 unsigned long next;
2542 pud_t *pudp;
2543
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +03002544 pudp = pud_offset(&p4d, addr);
Steve Capper2667f502014-10-09 15:29:14 -07002545 do {
Christian Borntraegere37c6982014-12-07 21:41:33 +01002546 pud_t pud = READ_ONCE(*pudp);
Steve Capper2667f502014-10-09 15:29:14 -07002547
2548 next = pud_addr_end(addr, end);
Qiujun Huang154945202020-01-30 22:12:10 -08002549 if (unlikely(!pud_present(pud)))
Steve Capper2667f502014-10-09 15:29:14 -07002550 return 0;
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +05302551 if (unlikely(pud_huge(pud))) {
Ira Weinyb798bec2019-05-13 17:17:07 -07002552 if (!gup_huge_pud(pud, pudp, addr, next, flags,
Aneesh Kumar K.Vf30c59e2014-11-05 21:57:40 +05302553 pages, nr))
2554 return 0;
2555 } else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) {
2556 if (!gup_huge_pd(__hugepd(pud_val(pud)), addr,
Ira Weinyb798bec2019-05-13 17:17:07 -07002557 PUD_SHIFT, next, flags, pages, nr))
Steve Capper2667f502014-10-09 15:29:14 -07002558 return 0;
Ira Weinyb798bec2019-05-13 17:17:07 -07002559 } else if (!gup_pmd_range(pud, addr, next, flags, pages, nr))
Steve Capper2667f502014-10-09 15:29:14 -07002560 return 0;
2561 } while (pudp++, addr = next, addr != end);
2562
2563 return 1;
2564}
2565
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +03002566static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,
Ira Weinyb798bec2019-05-13 17:17:07 -07002567 unsigned int flags, struct page **pages, int *nr)
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +03002568{
2569 unsigned long next;
2570 p4d_t *p4dp;
2571
2572 p4dp = p4d_offset(&pgd, addr);
2573 do {
2574 p4d_t p4d = READ_ONCE(*p4dp);
2575
2576 next = p4d_addr_end(addr, end);
2577 if (p4d_none(p4d))
2578 return 0;
2579 BUILD_BUG_ON(p4d_huge(p4d));
2580 if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) {
2581 if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,
Ira Weinyb798bec2019-05-13 17:17:07 -07002582 P4D_SHIFT, next, flags, pages, nr))
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +03002583 return 0;
Ira Weinyb798bec2019-05-13 17:17:07 -07002584 } else if (!gup_pud_range(p4d, addr, next, flags, pages, nr))
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +03002585 return 0;
2586 } while (p4dp++, addr = next, addr != end);
2587
2588 return 1;
2589}
2590
Kirill A. Shutemov5b65c4672017-09-09 00:56:03 +03002591static void gup_pgd_range(unsigned long addr, unsigned long end,
Ira Weinyb798bec2019-05-13 17:17:07 -07002592 unsigned int flags, struct page **pages, int *nr)
Kirill A. Shutemov5b65c4672017-09-09 00:56:03 +03002593{
2594 unsigned long next;
2595 pgd_t *pgdp;
2596
2597 pgdp = pgd_offset(current->mm, addr);
2598 do {
2599 pgd_t pgd = READ_ONCE(*pgdp);
2600
2601 next = pgd_addr_end(addr, end);
2602 if (pgd_none(pgd))
2603 return;
2604 if (unlikely(pgd_huge(pgd))) {
Ira Weinyb798bec2019-05-13 17:17:07 -07002605 if (!gup_huge_pgd(pgd, pgdp, addr, next, flags,
Kirill A. Shutemov5b65c4672017-09-09 00:56:03 +03002606 pages, nr))
2607 return;
2608 } else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) {
2609 if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
Ira Weinyb798bec2019-05-13 17:17:07 -07002610 PGDIR_SHIFT, next, flags, pages, nr))
Kirill A. Shutemov5b65c4672017-09-09 00:56:03 +03002611 return;
Ira Weinyb798bec2019-05-13 17:17:07 -07002612 } else if (!gup_p4d_range(pgd, addr, next, flags, pages, nr))
Kirill A. Shutemov5b65c4672017-09-09 00:56:03 +03002613 return;
2614 } while (pgdp++, addr = next, addr != end);
2615}
Christoph Hellwig050a9ad2019-07-11 20:57:21 -07002616#else
2617static inline void gup_pgd_range(unsigned long addr, unsigned long end,
2618 unsigned int flags, struct page **pages, int *nr)
2619{
2620}
2621#endif /* CONFIG_HAVE_FAST_GUP */
Kirill A. Shutemov5b65c4672017-09-09 00:56:03 +03002622
2623#ifndef gup_fast_permitted
2624/*
2625 * Check if it's allowed to use __get_user_pages_fast() for the range, or
2626 * we need to fall back to the slow version:
2627 */
Christoph Hellwig26f4c322019-07-11 20:56:45 -07002628static bool gup_fast_permitted(unsigned long start, unsigned long end)
Kirill A. Shutemov5b65c4672017-09-09 00:56:03 +03002629{
Christoph Hellwig26f4c322019-07-11 20:56:45 -07002630 return true;
Kirill A. Shutemov5b65c4672017-09-09 00:56:03 +03002631}
2632#endif
2633
Steve Capper2667f502014-10-09 15:29:14 -07002634/*
2635 * Like get_user_pages_fast() except it's IRQ-safe in that it won't fall back to
Michael S. Tsirkind0811072018-04-13 15:35:23 -07002636 * the regular GUP.
2637 * Note a difference with get_user_pages_fast: this always returns the
2638 * number of pages pinned, 0 if no pages were pinned.
Christoph Hellwig050a9ad2019-07-11 20:57:21 -07002639 *
2640 * If the architecture does not support this function, simply return with no
2641 * pages pinned.
Steve Capper2667f502014-10-09 15:29:14 -07002642 */
2643int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
2644 struct page **pages)
2645{
Wei Yangd4faa402018-10-26 15:07:55 -07002646 unsigned long len, end;
Kirill A. Shutemov5b65c4672017-09-09 00:56:03 +03002647 unsigned long flags;
Pingfan Liu4628b062020-04-01 21:06:00 -07002648 int nr_pinned = 0;
John Hubbard94202f12020-04-01 21:05:25 -07002649 /*
2650 * Internally (within mm/gup.c), gup fast variants must set FOLL_GET,
2651 * because gup fast is always a "pin with a +1 page refcount" request.
2652 */
2653 unsigned int gup_flags = FOLL_GET;
2654
2655 if (write)
2656 gup_flags |= FOLL_WRITE;
Steve Capper2667f502014-10-09 15:29:14 -07002657
Christoph Hellwigf455c8542019-07-11 20:56:41 -07002658 start = untagged_addr(start) & PAGE_MASK;
Steve Capper2667f502014-10-09 15:29:14 -07002659 len = (unsigned long) nr_pages << PAGE_SHIFT;
2660 end = start + len;
2661
Christoph Hellwig26f4c322019-07-11 20:56:45 -07002662 if (end <= start)
2663 return 0;
Linus Torvalds96d4f262019-01-03 18:57:57 -08002664 if (unlikely(!access_ok((void __user *)start, len)))
Steve Capper2667f502014-10-09 15:29:14 -07002665 return 0;
2666
2667 /*
2668 * Disable interrupts. We use the nested form as we can already have
2669 * interrupts disabled by get_futex_key.
2670 *
2671 * With interrupts disabled, we block page table pages from being
Fengguang Wu2ebe8222018-10-30 15:10:51 -07002672 * freed from under us. See struct mmu_table_batch comments in
2673 * include/asm-generic/tlb.h for more details.
Steve Capper2667f502014-10-09 15:29:14 -07002674 *
2675 * We do not adopt an rcu_read_lock(.) here as we also want to
2676 * block IPIs that come from THPs splitting.
2677 */
2678
Christoph Hellwig050a9ad2019-07-11 20:57:21 -07002679 if (IS_ENABLED(CONFIG_HAVE_FAST_GUP) &&
2680 gup_fast_permitted(start, end)) {
Kirill A. Shutemov5b65c4672017-09-09 00:56:03 +03002681 local_irq_save(flags);
Pingfan Liu4628b062020-04-01 21:06:00 -07002682 gup_pgd_range(start, end, gup_flags, pages, &nr_pinned);
Kirill A. Shutemov5b65c4672017-09-09 00:56:03 +03002683 local_irq_restore(flags);
2684 }
Steve Capper2667f502014-10-09 15:29:14 -07002685
Pingfan Liu4628b062020-04-01 21:06:00 -07002686 return nr_pinned;
Steve Capper2667f502014-10-09 15:29:14 -07002687}
Christoph Hellwig050a9ad2019-07-11 20:57:21 -07002688EXPORT_SYMBOL_GPL(__get_user_pages_fast);
Steve Capper2667f502014-10-09 15:29:14 -07002689
Ira Weiny7af75562019-05-13 17:17:14 -07002690static int __gup_longterm_unlocked(unsigned long start, int nr_pages,
2691 unsigned int gup_flags, struct page **pages)
2692{
2693 int ret;
2694
2695 /*
2696 * FIXME: FOLL_LONGTERM does not work with
2697 * get_user_pages_unlocked() (see comments in that function)
2698 */
2699 if (gup_flags & FOLL_LONGTERM) {
2700 down_read(&current->mm->mmap_sem);
2701 ret = __gup_longterm_locked(current, current->mm,
2702 start, nr_pages,
2703 pages, NULL, gup_flags);
2704 up_read(&current->mm->mmap_sem);
2705 } else {
2706 ret = get_user_pages_unlocked(start, nr_pages,
2707 pages, gup_flags);
2708 }
2709
2710 return ret;
2711}
2712
John Hubbardeddb1c22020-01-30 22:12:54 -08002713static int internal_get_user_pages_fast(unsigned long start, int nr_pages,
2714 unsigned int gup_flags,
2715 struct page **pages)
Steve Capper2667f502014-10-09 15:29:14 -07002716{
Kirill A. Shutemov5b65c4672017-09-09 00:56:03 +03002717 unsigned long addr, len, end;
Pingfan Liu4628b062020-04-01 21:06:00 -07002718 int nr_pinned = 0, ret = 0;
Steve Capper2667f502014-10-09 15:29:14 -07002719
John Hubbardf4000fd2020-01-30 22:12:43 -08002720 if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM |
John Hubbard94202f12020-04-01 21:05:25 -07002721 FOLL_FORCE | FOLL_PIN | FOLL_GET)))
Christoph Hellwig817be122019-07-11 20:57:25 -07002722 return -EINVAL;
2723
Christoph Hellwigf455c8542019-07-11 20:56:41 -07002724 start = untagged_addr(start) & PAGE_MASK;
Kirill A. Shutemov5b65c4672017-09-09 00:56:03 +03002725 addr = start;
2726 len = (unsigned long) nr_pages << PAGE_SHIFT;
2727 end = start + len;
2728
Christoph Hellwig26f4c322019-07-11 20:56:45 -07002729 if (end <= start)
Michael S. Tsirkinc61611f2018-04-13 15:35:20 -07002730 return 0;
Linus Torvalds96d4f262019-01-03 18:57:57 -08002731 if (unlikely(!access_ok((void __user *)start, len)))
Michael S. Tsirkinc61611f2018-04-13 15:35:20 -07002732 return -EFAULT;
Kirill A. Shutemov73e10a62017-03-16 18:26:54 +03002733
Christoph Hellwig050a9ad2019-07-11 20:57:21 -07002734 if (IS_ENABLED(CONFIG_HAVE_FAST_GUP) &&
2735 gup_fast_permitted(start, end)) {
Kirill A. Shutemov5b65c4672017-09-09 00:56:03 +03002736 local_irq_disable();
Pingfan Liu4628b062020-04-01 21:06:00 -07002737 gup_pgd_range(addr, end, gup_flags, pages, &nr_pinned);
Kirill A. Shutemov5b65c4672017-09-09 00:56:03 +03002738 local_irq_enable();
Pingfan Liu4628b062020-04-01 21:06:00 -07002739 ret = nr_pinned;
Kirill A. Shutemov73e10a62017-03-16 18:26:54 +03002740 }
Steve Capper2667f502014-10-09 15:29:14 -07002741
Pingfan Liu4628b062020-04-01 21:06:00 -07002742 if (nr_pinned < nr_pages) {
Steve Capper2667f502014-10-09 15:29:14 -07002743 /* Try to get the remaining pages with get_user_pages */
Pingfan Liu4628b062020-04-01 21:06:00 -07002744 start += nr_pinned << PAGE_SHIFT;
2745 pages += nr_pinned;
Steve Capper2667f502014-10-09 15:29:14 -07002746
Pingfan Liu4628b062020-04-01 21:06:00 -07002747 ret = __gup_longterm_unlocked(start, nr_pages - nr_pinned,
Ira Weiny7af75562019-05-13 17:17:14 -07002748 gup_flags, pages);
Steve Capper2667f502014-10-09 15:29:14 -07002749
2750 /* Have to be a bit careful with return values */
Pingfan Liu4628b062020-04-01 21:06:00 -07002751 if (nr_pinned > 0) {
Steve Capper2667f502014-10-09 15:29:14 -07002752 if (ret < 0)
Pingfan Liu4628b062020-04-01 21:06:00 -07002753 ret = nr_pinned;
Steve Capper2667f502014-10-09 15:29:14 -07002754 else
Pingfan Liu4628b062020-04-01 21:06:00 -07002755 ret += nr_pinned;
Steve Capper2667f502014-10-09 15:29:14 -07002756 }
2757 }
2758
2759 return ret;
2760}
John Hubbardeddb1c22020-01-30 22:12:54 -08002761
2762/**
2763 * get_user_pages_fast() - pin user pages in memory
John Hubbard3faa52c2020-04-01 21:05:29 -07002764 * @start: starting user address
2765 * @nr_pages: number of pages from start to pin
2766 * @gup_flags: flags modifying pin behaviour
2767 * @pages: array that receives pointers to the pages pinned.
2768 * Should be at least nr_pages long.
John Hubbardeddb1c22020-01-30 22:12:54 -08002769 *
2770 * Attempt to pin user pages in memory without taking mm->mmap_sem.
2771 * If not successful, it will fall back to taking the lock and
2772 * calling get_user_pages().
2773 *
2774 * Returns number of pages pinned. This may be fewer than the number requested.
2775 * If nr_pages is 0 or negative, returns 0. If no pages were pinned, returns
2776 * -errno.
2777 */
2778int get_user_pages_fast(unsigned long start, int nr_pages,
2779 unsigned int gup_flags, struct page **pages)
2780{
2781 /*
2782 * FOLL_PIN must only be set internally by the pin_user_pages*() APIs,
2783 * never directly by the caller, so enforce that:
2784 */
2785 if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
2786 return -EINVAL;
2787
John Hubbard94202f12020-04-01 21:05:25 -07002788 /*
2789 * The caller may or may not have explicitly set FOLL_GET; either way is
2790 * OK. However, internally (within mm/gup.c), gup fast variants must set
2791 * FOLL_GET, because gup fast is always a "pin with a +1 page refcount"
2792 * request.
2793 */
2794 gup_flags |= FOLL_GET;
John Hubbardeddb1c22020-01-30 22:12:54 -08002795 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
2796}
Christoph Hellwig050a9ad2019-07-11 20:57:21 -07002797EXPORT_SYMBOL_GPL(get_user_pages_fast);
John Hubbardeddb1c22020-01-30 22:12:54 -08002798
2799/**
2800 * pin_user_pages_fast() - pin user pages in memory without taking locks
2801 *
John Hubbard3faa52c2020-04-01 21:05:29 -07002802 * @start: starting user address
2803 * @nr_pages: number of pages from start to pin
2804 * @gup_flags: flags modifying pin behaviour
2805 * @pages: array that receives pointers to the pages pinned.
2806 * Should be at least nr_pages long.
2807 *
2808 * Nearly the same as get_user_pages_fast(), except that FOLL_PIN is set. See
2809 * get_user_pages_fast() for documentation on the function arguments, because
2810 * the arguments here are identical.
2811 *
2812 * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
2813 * see Documentation/vm/pin_user_pages.rst for further details.
John Hubbardeddb1c22020-01-30 22:12:54 -08002814 *
2815 * This is intended for Case 1 (DIO) in Documentation/vm/pin_user_pages.rst. It
2816 * is NOT intended for Case 2 (RDMA: long-term pins).
2817 */
2818int pin_user_pages_fast(unsigned long start, int nr_pages,
2819 unsigned int gup_flags, struct page **pages)
2820{
John Hubbard3faa52c2020-04-01 21:05:29 -07002821 /* FOLL_GET and FOLL_PIN are mutually exclusive. */
2822 if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2823 return -EINVAL;
2824
2825 gup_flags |= FOLL_PIN;
2826 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
John Hubbardeddb1c22020-01-30 22:12:54 -08002827}
2828EXPORT_SYMBOL_GPL(pin_user_pages_fast);
2829
2830/**
2831 * pin_user_pages_remote() - pin pages of a remote process (task != current)
2832 *
John Hubbard3faa52c2020-04-01 21:05:29 -07002833 * @tsk: the task_struct to use for page fault accounting, or
2834 * NULL if faults are not to be recorded.
2835 * @mm: mm_struct of target mm
2836 * @start: starting user address
2837 * @nr_pages: number of pages from start to pin
2838 * @gup_flags: flags modifying lookup behaviour
2839 * @pages: array that receives pointers to the pages pinned.
2840 * Should be at least nr_pages long. Or NULL, if caller
2841 * only intends to ensure the pages are faulted in.
2842 * @vmas: array of pointers to vmas corresponding to each page.
2843 * Or NULL if the caller does not require them.
2844 * @locked: pointer to lock flag indicating whether lock is held and
2845 * subsequently whether VM_FAULT_RETRY functionality can be
2846 * utilised. Lock must initially be held.
2847 *
2848 * Nearly the same as get_user_pages_remote(), except that FOLL_PIN is set. See
2849 * get_user_pages_remote() for documentation on the function arguments, because
2850 * the arguments here are identical.
2851 *
2852 * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
2853 * see Documentation/vm/pin_user_pages.rst for details.
John Hubbardeddb1c22020-01-30 22:12:54 -08002854 *
2855 * This is intended for Case 1 (DIO) in Documentation/vm/pin_user_pages.rst. It
2856 * is NOT intended for Case 2 (RDMA: long-term pins).
2857 */
2858long pin_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
2859 unsigned long start, unsigned long nr_pages,
2860 unsigned int gup_flags, struct page **pages,
2861 struct vm_area_struct **vmas, int *locked)
2862{
John Hubbard3faa52c2020-04-01 21:05:29 -07002863 /* FOLL_GET and FOLL_PIN are mutually exclusive. */
2864 if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2865 return -EINVAL;
2866
2867 gup_flags |= FOLL_PIN;
2868 return __get_user_pages_remote(tsk, mm, start, nr_pages, gup_flags,
2869 pages, vmas, locked);
John Hubbardeddb1c22020-01-30 22:12:54 -08002870}
2871EXPORT_SYMBOL(pin_user_pages_remote);
2872
2873/**
2874 * pin_user_pages() - pin user pages in memory for use by other devices
2875 *
John Hubbard3faa52c2020-04-01 21:05:29 -07002876 * @start: starting user address
2877 * @nr_pages: number of pages from start to pin
2878 * @gup_flags: flags modifying lookup behaviour
2879 * @pages: array that receives pointers to the pages pinned.
2880 * Should be at least nr_pages long. Or NULL, if caller
2881 * only intends to ensure the pages are faulted in.
2882 * @vmas: array of pointers to vmas corresponding to each page.
2883 * Or NULL if the caller does not require them.
2884 *
2885 * Nearly the same as get_user_pages(), except that FOLL_TOUCH is not set, and
2886 * FOLL_PIN is set.
2887 *
2888 * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
2889 * see Documentation/vm/pin_user_pages.rst for details.
John Hubbardeddb1c22020-01-30 22:12:54 -08002890 *
2891 * This is intended for Case 1 (DIO) in Documentation/vm/pin_user_pages.rst. It
2892 * is NOT intended for Case 2 (RDMA: long-term pins).
2893 */
2894long pin_user_pages(unsigned long start, unsigned long nr_pages,
2895 unsigned int gup_flags, struct page **pages,
2896 struct vm_area_struct **vmas)
2897{
John Hubbard3faa52c2020-04-01 21:05:29 -07002898 /* FOLL_GET and FOLL_PIN are mutually exclusive. */
2899 if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2900 return -EINVAL;
2901
2902 gup_flags |= FOLL_PIN;
2903 return __gup_longterm_locked(current, current->mm, start, nr_pages,
2904 pages, vmas, gup_flags);
John Hubbardeddb1c22020-01-30 22:12:54 -08002905}
2906EXPORT_SYMBOL(pin_user_pages);