blob: fc2745ca33088275095d88610b51c989dbd0d85d [file] [log] [blame]
Matthew Wilcoxd475c632015-02-16 15:58:56 -08001/*
2 * fs/dax.c - Direct Access filesystem code
3 * Copyright (c) 2013-2014 Intel Corporation
4 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com>
5 * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
15 */
16
17#include <linux/atomic.h>
18#include <linux/blkdev.h>
19#include <linux/buffer_head.h>
Ross Zwislerd77e92e2015-09-09 10:29:40 -060020#include <linux/dax.h>
Matthew Wilcoxd475c632015-02-16 15:58:56 -080021#include <linux/fs.h>
22#include <linux/genhd.h>
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -080023#include <linux/highmem.h>
24#include <linux/memcontrol.h>
25#include <linux/mm.h>
Matthew Wilcoxd475c632015-02-16 15:58:56 -080026#include <linux/mutex.h>
Ross Zwisler9973c982016-01-22 15:10:47 -080027#include <linux/pagevec.h>
Matthew Wilcox289c6ae2015-02-16 15:58:59 -080028#include <linux/sched.h>
Ingo Molnarf361bf42017-02-03 23:47:37 +010029#include <linux/sched/signal.h>
Matthew Wilcoxd475c632015-02-16 15:58:56 -080030#include <linux/uio.h>
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -080031#include <linux/vmstat.h>
Dan Williams34c0fd52016-01-15 16:56:14 -080032#include <linux/pfn_t.h>
Dan Williams0e749e52016-01-15 16:55:53 -080033#include <linux/sizes.h>
Jan Kara4b4bb462016-12-14 15:07:53 -080034#include <linux/mmu_notifier.h>
Christoph Hellwiga254e562016-09-19 11:24:49 +100035#include <linux/iomap.h>
36#include "internal.h"
Matthew Wilcoxd475c632015-02-16 15:58:56 -080037
Ross Zwisler282a8e02017-02-22 15:39:50 -080038#define CREATE_TRACE_POINTS
39#include <trace/events/fs_dax.h>
40
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -040041static inline unsigned int pe_order(enum page_entry_size pe_size)
42{
43 if (pe_size == PE_SIZE_PTE)
44 return PAGE_SHIFT - PAGE_SHIFT;
45 if (pe_size == PE_SIZE_PMD)
46 return PMD_SHIFT - PAGE_SHIFT;
47 if (pe_size == PE_SIZE_PUD)
48 return PUD_SHIFT - PAGE_SHIFT;
49 return ~0;
50}
51
Jan Karaac401cc2016-05-12 18:29:18 +020052/* We choose 4096 entries - same as per-zone page wait tables */
53#define DAX_WAIT_TABLE_BITS 12
54#define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)
55
Ross Zwisler917f3452017-09-06 16:18:58 -070056/* The 'colour' (ie low bits) within a PMD of a page offset. */
57#define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1)
Matthew Wilcox977fbdc2018-01-31 16:17:36 -080058#define PG_PMD_NR (PMD_SIZE >> PAGE_SHIFT)
Ross Zwisler917f3452017-09-06 16:18:58 -070059
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -040060/* The order of a PMD entry */
61#define PMD_ORDER (PMD_SHIFT - PAGE_SHIFT)
62
Ross Zwislerce95ab0f2016-11-08 11:31:44 +110063static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
Jan Karaac401cc2016-05-12 18:29:18 +020064
65static int __init init_dax_wait_table(void)
66{
67 int i;
68
69 for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++)
70 init_waitqueue_head(wait_table + i);
71 return 0;
72}
73fs_initcall(init_dax_wait_table);
74
Ross Zwisler527b19d2017-09-06 16:18:51 -070075/*
Matthew Wilcox3159f942017-11-03 13:30:42 -040076 * DAX pagecache entries use XArray value entries so they can't be mistaken
77 * for pages. We use one bit for locking, one bit for the entry size (PMD)
78 * and two more to tell us if the entry is a zero page or an empty entry that
79 * is just used for locking. In total four special bits.
Ross Zwisler527b19d2017-09-06 16:18:51 -070080 *
81 * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE
82 * and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem
83 * block allocation.
84 */
Matthew Wilcox3159f942017-11-03 13:30:42 -040085#define DAX_SHIFT (4)
86#define DAX_LOCKED (1UL << 0)
87#define DAX_PMD (1UL << 1)
88#define DAX_ZERO_PAGE (1UL << 2)
89#define DAX_EMPTY (1UL << 3)
Ross Zwisler527b19d2017-09-06 16:18:51 -070090
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -040091static unsigned long dax_to_pfn(void *entry)
Ross Zwisler527b19d2017-09-06 16:18:51 -070092{
Matthew Wilcox3159f942017-11-03 13:30:42 -040093 return xa_to_value(entry) >> DAX_SHIFT;
Ross Zwisler527b19d2017-09-06 16:18:51 -070094}
95
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -040096static void *dax_make_locked(unsigned long pfn, unsigned long flags)
Ross Zwisler527b19d2017-09-06 16:18:51 -070097{
Matthew Wilcox3159f942017-11-03 13:30:42 -040098 return xa_mk_value(flags | ((unsigned long)pfn << DAX_SHIFT) |
99 DAX_LOCKED);
Ross Zwisler527b19d2017-09-06 16:18:51 -0700100}
101
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400102static void *dax_make_entry(pfn_t pfn, unsigned long flags)
103{
104 return xa_mk_value(flags | (pfn_t_to_pfn(pfn) << DAX_SHIFT));
105}
106
107static void *dax_make_page_entry(struct page *page)
108{
109 pfn_t pfn = page_to_pfn_t(page);
110 return dax_make_entry(pfn, PageHead(page) ? DAX_PMD : 0);
111}
112
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -0400113static bool dax_is_locked(void *entry)
114{
115 return xa_to_value(entry) & DAX_LOCKED;
116}
117
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400118static unsigned int dax_entry_order(void *entry)
Ross Zwisler527b19d2017-09-06 16:18:51 -0700119{
Matthew Wilcox3159f942017-11-03 13:30:42 -0400120 if (xa_to_value(entry) & DAX_PMD)
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -0400121 return PMD_ORDER;
Ross Zwisler527b19d2017-09-06 16:18:51 -0700122 return 0;
123}
124
Ross Zwisler642261a2016-11-08 11:34:45 +1100125static int dax_is_pmd_entry(void *entry)
126{
Matthew Wilcox3159f942017-11-03 13:30:42 -0400127 return xa_to_value(entry) & DAX_PMD;
Ross Zwisler642261a2016-11-08 11:34:45 +1100128}
129
130static int dax_is_pte_entry(void *entry)
131{
Matthew Wilcox3159f942017-11-03 13:30:42 -0400132 return !(xa_to_value(entry) & DAX_PMD);
Ross Zwisler642261a2016-11-08 11:34:45 +1100133}
134
135static int dax_is_zero_entry(void *entry)
136{
Matthew Wilcox3159f942017-11-03 13:30:42 -0400137 return xa_to_value(entry) & DAX_ZERO_PAGE;
Ross Zwisler642261a2016-11-08 11:34:45 +1100138}
139
140static int dax_is_empty_entry(void *entry)
141{
Matthew Wilcox3159f942017-11-03 13:30:42 -0400142 return xa_to_value(entry) & DAX_EMPTY;
Ross Zwisler642261a2016-11-08 11:34:45 +1100143}
144
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800145/*
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400146 * DAX page cache entry locking
Jan Karaac401cc2016-05-12 18:29:18 +0200147 */
148struct exceptional_entry_key {
Matthew Wilcoxec4907f2018-03-28 11:01:43 -0400149 struct xarray *xa;
Ross Zwisler63e95b52016-11-08 11:32:20 +1100150 pgoff_t entry_start;
Jan Karaac401cc2016-05-12 18:29:18 +0200151};
152
153struct wait_exceptional_entry_queue {
Ingo Molnarac6424b2017-06-20 12:06:13 +0200154 wait_queue_entry_t wait;
Jan Karaac401cc2016-05-12 18:29:18 +0200155 struct exceptional_entry_key key;
156};
157
Matthew Wilcoxec4907f2018-03-28 11:01:43 -0400158static wait_queue_head_t *dax_entry_waitqueue(struct xarray *xa,
Ross Zwisler63e95b52016-11-08 11:32:20 +1100159 pgoff_t index, void *entry, struct exceptional_entry_key *key)
160{
161 unsigned long hash;
162
163 /*
164 * If 'entry' is a PMD, align the 'index' that we use for the wait
165 * queue to the start of that PMD. This ensures that all offsets in
166 * the range covered by the PMD map to the same bit lock.
167 */
Ross Zwisler642261a2016-11-08 11:34:45 +1100168 if (dax_is_pmd_entry(entry))
Ross Zwisler917f3452017-09-06 16:18:58 -0700169 index &= ~PG_PMD_COLOUR;
Ross Zwisler63e95b52016-11-08 11:32:20 +1100170
Matthew Wilcoxec4907f2018-03-28 11:01:43 -0400171 key->xa = xa;
Ross Zwisler63e95b52016-11-08 11:32:20 +1100172 key->entry_start = index;
173
Matthew Wilcoxec4907f2018-03-28 11:01:43 -0400174 hash = hash_long((unsigned long)xa ^ index, DAX_WAIT_TABLE_BITS);
Ross Zwisler63e95b52016-11-08 11:32:20 +1100175 return wait_table + hash;
176}
177
Matthew Wilcoxec4907f2018-03-28 11:01:43 -0400178static int wake_exceptional_entry_func(wait_queue_entry_t *wait,
179 unsigned int mode, int sync, void *keyp)
Jan Karaac401cc2016-05-12 18:29:18 +0200180{
181 struct exceptional_entry_key *key = keyp;
182 struct wait_exceptional_entry_queue *ewait =
183 container_of(wait, struct wait_exceptional_entry_queue, wait);
184
Matthew Wilcoxec4907f2018-03-28 11:01:43 -0400185 if (key->xa != ewait->key.xa ||
Ross Zwisler63e95b52016-11-08 11:32:20 +1100186 key->entry_start != ewait->key.entry_start)
Jan Karaac401cc2016-05-12 18:29:18 +0200187 return 0;
188 return autoremove_wake_function(wait, mode, sync, NULL);
189}
190
191/*
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700192 * @entry may no longer be the entry at the index in the mapping.
193 * The important information it's conveying is whether the entry at
194 * this index used to be a PMD entry.
Ross Zwislere30331f2017-09-06 16:18:39 -0700195 */
Matthew Wilcoxec4907f2018-03-28 11:01:43 -0400196static void dax_wake_mapping_entry_waiter(struct xarray *xa,
Ross Zwislere30331f2017-09-06 16:18:39 -0700197 pgoff_t index, void *entry, bool wake_all)
198{
199 struct exceptional_entry_key key;
200 wait_queue_head_t *wq;
201
Matthew Wilcoxec4907f2018-03-28 11:01:43 -0400202 wq = dax_entry_waitqueue(xa, index, entry, &key);
Ross Zwislere30331f2017-09-06 16:18:39 -0700203
204 /*
205 * Checking for locked entry and prepare_to_wait_exclusive() happens
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700206 * under the i_pages lock, ditto for entry handling in our callers.
Ross Zwislere30331f2017-09-06 16:18:39 -0700207 * So at this point all tasks that could have seen our entry locked
208 * must be in the waitqueue and the following check will see them.
209 */
210 if (waitqueue_active(wq))
211 __wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key);
212}
213
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -0400214static void dax_wake_entry(struct xa_state *xas, void *entry, bool wake_all)
215{
216 return dax_wake_mapping_entry_waiter(xas->xa, xas->xa_index, entry,
217 wake_all);
218}
219
220/*
221 * Look up entry in page cache, wait for it to become unlocked if it
222 * is a DAX entry and return it. The caller must subsequently call
223 * put_unlocked_entry() if it did not lock the entry or dax_unlock_entry()
224 * if it did.
225 *
226 * Must be called with the i_pages lock held.
227 */
228static void *get_unlocked_entry(struct xa_state *xas)
229{
230 void *entry;
231 struct wait_exceptional_entry_queue ewait;
232 wait_queue_head_t *wq;
233
234 init_wait(&ewait.wait);
235 ewait.wait.func = wake_exceptional_entry_func;
236
237 for (;;) {
238 entry = xas_load(xas);
239 if (!entry || xa_is_internal(entry) ||
240 WARN_ON_ONCE(!xa_is_value(entry)) ||
241 !dax_is_locked(entry))
242 return entry;
243
244 wq = dax_entry_waitqueue(xas->xa, xas->xa_index, entry,
245 &ewait.key);
246 prepare_to_wait_exclusive(wq, &ewait.wait,
247 TASK_UNINTERRUPTIBLE);
248 xas_unlock_irq(xas);
249 xas_reset(xas);
250 schedule();
251 finish_wait(wq, &ewait.wait);
252 xas_lock_irq(xas);
253 }
254}
255
256static void put_unlocked_entry(struct xa_state *xas, void *entry)
257{
258 /* If we were the only waiter woken, wake the next one */
259 if (entry)
260 dax_wake_entry(xas, entry, false);
261}
262
263/*
264 * We used the xa_state to get the entry, but then we locked the entry and
265 * dropped the xa_lock, so we know the xa_state is stale and must be reset
266 * before use.
267 */
268static void dax_unlock_entry(struct xa_state *xas, void *entry)
269{
270 void *old;
271
272 xas_reset(xas);
273 xas_lock_irq(xas);
274 old = xas_store(xas, entry);
275 xas_unlock_irq(xas);
276 BUG_ON(!dax_is_locked(old));
277 dax_wake_entry(xas, entry, false);
278}
279
280/*
281 * Return: The entry stored at this location before it was locked.
282 */
283static void *dax_lock_entry(struct xa_state *xas, void *entry)
284{
285 unsigned long v = xa_to_value(entry);
286 return xas_store(xas, xa_mk_value(v | DAX_LOCKED));
287}
288
Ross Zwislere30331f2017-09-06 16:18:39 -0700289/*
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700290 * Check whether the given slot is locked. Must be called with the i_pages
291 * lock held.
Jan Karaac401cc2016-05-12 18:29:18 +0200292 */
293static inline int slot_locked(struct address_space *mapping, void **slot)
294{
Matthew Wilcox3159f942017-11-03 13:30:42 -0400295 unsigned long entry = xa_to_value(
296 radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock));
297 return entry & DAX_LOCKED;
Jan Karaac401cc2016-05-12 18:29:18 +0200298}
299
300/*
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700301 * Mark the given slot as locked. Must be called with the i_pages lock held.
Jan Karaac401cc2016-05-12 18:29:18 +0200302 */
303static inline void *lock_slot(struct address_space *mapping, void **slot)
304{
Matthew Wilcox3159f942017-11-03 13:30:42 -0400305 unsigned long v = xa_to_value(
306 radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock));
307 void *entry = xa_mk_value(v | DAX_LOCKED);
308 radix_tree_replace_slot(&mapping->i_pages, slot, entry);
309 return entry;
Jan Karaac401cc2016-05-12 18:29:18 +0200310}
311
312/*
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700313 * Mark the given slot as unlocked. Must be called with the i_pages lock held.
Jan Karaac401cc2016-05-12 18:29:18 +0200314 */
315static inline void *unlock_slot(struct address_space *mapping, void **slot)
316{
Matthew Wilcox3159f942017-11-03 13:30:42 -0400317 unsigned long v = xa_to_value(
318 radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock));
319 void *entry = xa_mk_value(v & ~DAX_LOCKED);
320 radix_tree_replace_slot(&mapping->i_pages, slot, entry);
321 return entry;
Jan Karaac401cc2016-05-12 18:29:18 +0200322}
323
324/*
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400325 * Lookup entry in page cache, wait for it to become unlocked if it is
Matthew Wilcox3159f942017-11-03 13:30:42 -0400326 * a DAX entry and return it. The caller must call
Jan Karaac401cc2016-05-12 18:29:18 +0200327 * put_unlocked_mapping_entry() when he decided not to lock the entry or
328 * put_locked_mapping_entry() when he locked the entry and now wants to
329 * unlock it.
330 *
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700331 * Must be called with the i_pages lock held.
Jan Karaac401cc2016-05-12 18:29:18 +0200332 */
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700333static void *__get_unlocked_mapping_entry(struct address_space *mapping,
334 pgoff_t index, void ***slotp, bool (*wait_fn)(void))
Jan Karaac401cc2016-05-12 18:29:18 +0200335{
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100336 void *entry, **slot;
Jan Karaac401cc2016-05-12 18:29:18 +0200337 struct wait_exceptional_entry_queue ewait;
Ross Zwisler63e95b52016-11-08 11:32:20 +1100338 wait_queue_head_t *wq;
Jan Karaac401cc2016-05-12 18:29:18 +0200339
340 init_wait(&ewait.wait);
341 ewait.wait.func = wake_exceptional_entry_func;
Jan Karaac401cc2016-05-12 18:29:18 +0200342
343 for (;;) {
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700344 bool revalidate;
345
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700346 entry = __radix_tree_lookup(&mapping->i_pages, index, NULL,
Jan Karaac401cc2016-05-12 18:29:18 +0200347 &slot);
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700348 if (!entry ||
Matthew Wilcox3159f942017-11-03 13:30:42 -0400349 WARN_ON_ONCE(!xa_is_value(entry)) ||
Jan Karaac401cc2016-05-12 18:29:18 +0200350 !slot_locked(mapping, slot)) {
351 if (slotp)
352 *slotp = slot;
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100353 return entry;
Jan Karaac401cc2016-05-12 18:29:18 +0200354 }
Ross Zwisler63e95b52016-11-08 11:32:20 +1100355
Matthew Wilcoxec4907f2018-03-28 11:01:43 -0400356 wq = dax_entry_waitqueue(&mapping->i_pages, index, entry,
357 &ewait.key);
Jan Karaac401cc2016-05-12 18:29:18 +0200358 prepare_to_wait_exclusive(wq, &ewait.wait,
359 TASK_UNINTERRUPTIBLE);
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700360 xa_unlock_irq(&mapping->i_pages);
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700361 revalidate = wait_fn();
Jan Karaac401cc2016-05-12 18:29:18 +0200362 finish_wait(wq, &ewait.wait);
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700363 xa_lock_irq(&mapping->i_pages);
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700364 if (revalidate)
365 return ERR_PTR(-EAGAIN);
Jan Karaac401cc2016-05-12 18:29:18 +0200366 }
367}
368
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700369static bool entry_wait(void)
370{
371 schedule();
372 /*
373 * Never return an ERR_PTR() from
374 * __get_unlocked_mapping_entry(), just keep looping.
375 */
376 return false;
377}
378
379static void *get_unlocked_mapping_entry(struct address_space *mapping,
380 pgoff_t index, void ***slotp)
381{
382 return __get_unlocked_mapping_entry(mapping, index, slotp, entry_wait);
383}
384
385static void unlock_mapping_entry(struct address_space *mapping, pgoff_t index)
Jan Karab1aa8122016-12-14 15:07:24 -0800386{
387 void *entry, **slot;
388
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700389 xa_lock_irq(&mapping->i_pages);
390 entry = __radix_tree_lookup(&mapping->i_pages, index, NULL, &slot);
Matthew Wilcox3159f942017-11-03 13:30:42 -0400391 if (WARN_ON_ONCE(!entry || !xa_is_value(entry) ||
Jan Karab1aa8122016-12-14 15:07:24 -0800392 !slot_locked(mapping, slot))) {
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700393 xa_unlock_irq(&mapping->i_pages);
Jan Karab1aa8122016-12-14 15:07:24 -0800394 return;
395 }
396 unlock_slot(mapping, slot);
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700397 xa_unlock_irq(&mapping->i_pages);
Matthew Wilcoxec4907f2018-03-28 11:01:43 -0400398 dax_wake_mapping_entry_waiter(&mapping->i_pages, index, entry, false);
Jan Karab1aa8122016-12-14 15:07:24 -0800399}
400
Jan Karaac401cc2016-05-12 18:29:18 +0200401static void put_locked_mapping_entry(struct address_space *mapping,
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700402 pgoff_t index)
Jan Karaac401cc2016-05-12 18:29:18 +0200403{
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700404 unlock_mapping_entry(mapping, index);
Jan Karaac401cc2016-05-12 18:29:18 +0200405}
406
407/*
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400408 * Called when we are done with page cache entry we looked up via
Jan Karaac401cc2016-05-12 18:29:18 +0200409 * get_unlocked_mapping_entry() and which we didn't lock in the end.
410 */
411static void put_unlocked_mapping_entry(struct address_space *mapping,
412 pgoff_t index, void *entry)
413{
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700414 if (!entry)
Jan Karaac401cc2016-05-12 18:29:18 +0200415 return;
416
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400417 /* We have to wake up next waiter for the page cache entry lock */
Matthew Wilcoxec4907f2018-03-28 11:01:43 -0400418 dax_wake_mapping_entry_waiter(&mapping->i_pages, index, entry, false);
Ross Zwisler422476c2016-11-08 11:33:44 +1100419}
420
Dan Williamsd2c997c2017-12-22 22:02:48 -0800421static unsigned long dax_entry_size(void *entry)
422{
423 if (dax_is_zero_entry(entry))
424 return 0;
425 else if (dax_is_empty_entry(entry))
426 return 0;
427 else if (dax_is_pmd_entry(entry))
428 return PMD_SIZE;
429 else
430 return PAGE_SIZE;
431}
432
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400433static unsigned long dax_end_pfn(void *entry)
Dan Williamsd2c997c2017-12-22 22:02:48 -0800434{
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400435 return dax_to_pfn(entry) + dax_entry_size(entry) / PAGE_SIZE;
Dan Williamsd2c997c2017-12-22 22:02:48 -0800436}
437
438/*
439 * Iterate through all mapped pfns represented by an entry, i.e. skip
440 * 'empty' and 'zero' entries.
441 */
442#define for_each_mapped_pfn(entry, pfn) \
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400443 for (pfn = dax_to_pfn(entry); \
444 pfn < dax_end_pfn(entry); pfn++)
Dan Williamsd2c997c2017-12-22 22:02:48 -0800445
Dan Williams73449da2018-07-13 21:49:50 -0700446/*
447 * TODO: for reflink+dax we need a way to associate a single page with
448 * multiple address_space instances at different linear_page_index()
449 * offsets.
450 */
451static void dax_associate_entry(void *entry, struct address_space *mapping,
452 struct vm_area_struct *vma, unsigned long address)
Dan Williamsd2c997c2017-12-22 22:02:48 -0800453{
Dan Williams73449da2018-07-13 21:49:50 -0700454 unsigned long size = dax_entry_size(entry), pfn, index;
455 int i = 0;
Dan Williamsd2c997c2017-12-22 22:02:48 -0800456
457 if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
458 return;
459
Dan Williams73449da2018-07-13 21:49:50 -0700460 index = linear_page_index(vma, address & ~(size - 1));
Dan Williamsd2c997c2017-12-22 22:02:48 -0800461 for_each_mapped_pfn(entry, pfn) {
462 struct page *page = pfn_to_page(pfn);
463
464 WARN_ON_ONCE(page->mapping);
465 page->mapping = mapping;
Dan Williams73449da2018-07-13 21:49:50 -0700466 page->index = index + i++;
Dan Williamsd2c997c2017-12-22 22:02:48 -0800467 }
468}
469
470static void dax_disassociate_entry(void *entry, struct address_space *mapping,
471 bool trunc)
472{
473 unsigned long pfn;
474
475 if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
476 return;
477
478 for_each_mapped_pfn(entry, pfn) {
479 struct page *page = pfn_to_page(pfn);
480
481 WARN_ON_ONCE(trunc && page_ref_count(page) > 1);
482 WARN_ON_ONCE(page->mapping && page->mapping != mapping);
483 page->mapping = NULL;
Dan Williams73449da2018-07-13 21:49:50 -0700484 page->index = 0;
Dan Williamsd2c997c2017-12-22 22:02:48 -0800485 }
486}
487
Dan Williams5fac7402018-03-09 17:44:31 -0800488static struct page *dax_busy_page(void *entry)
489{
490 unsigned long pfn;
491
492 for_each_mapped_pfn(entry, pfn) {
493 struct page *page = pfn_to_page(pfn);
494
495 if (page_ref_count(page) > 1)
496 return page;
497 }
498 return NULL;
499}
500
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700501bool dax_lock_mapping_entry(struct page *page)
502{
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400503 XA_STATE(xas, NULL, 0);
504 void *entry;
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700505
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700506 for (;;) {
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400507 struct address_space *mapping = READ_ONCE(page->mapping);
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700508
509 if (!dax_mapping(mapping))
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400510 return false;
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700511
512 /*
513 * In the device-dax case there's no need to lock, a
514 * struct dev_pagemap pin is sufficient to keep the
515 * inode alive, and we assume we have dev_pagemap pin
516 * otherwise we would not have a valid pfn_to_page()
517 * translation.
518 */
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400519 if (S_ISCHR(mapping->host->i_mode))
520 return true;
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700521
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400522 xas.xa = &mapping->i_pages;
523 xas_lock_irq(&xas);
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700524 if (mapping != page->mapping) {
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400525 xas_unlock_irq(&xas);
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700526 continue;
527 }
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400528 xas_set(&xas, page->index);
529 entry = xas_load(&xas);
530 if (dax_is_locked(entry)) {
531 entry = get_unlocked_entry(&xas);
532 /* Did the page move while we slept? */
533 if (dax_to_pfn(entry) != page_to_pfn(page)) {
534 xas_unlock_irq(&xas);
535 continue;
536 }
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700537 }
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400538 dax_lock_entry(&xas, entry);
539 xas_unlock_irq(&xas);
540 return true;
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700541 }
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700542}
543
544void dax_unlock_mapping_entry(struct page *page)
545{
546 struct address_space *mapping = page->mapping;
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400547 XA_STATE(xas, &mapping->i_pages, page->index);
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700548
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400549 if (S_ISCHR(mapping->host->i_mode))
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700550 return;
551
Matthew Wilcox9f32d222018-06-12 09:46:30 -0400552 dax_unlock_entry(&xas, dax_make_page_entry(page));
Dan Williamsc2a7d2a2018-07-13 21:50:16 -0700553}
554
Jan Karaac401cc2016-05-12 18:29:18 +0200555/*
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400556 * Find page cache entry at given index. If it is a DAX entry, return it
557 * with the entry locked. If the page cache doesn't contain an entry at
558 * that index, add a locked empty entry.
Jan Karaac401cc2016-05-12 18:29:18 +0200559 *
Matthew Wilcox3159f942017-11-03 13:30:42 -0400560 * When requesting an entry with size DAX_PMD, grab_mapping_entry() will
Ross Zwisler642261a2016-11-08 11:34:45 +1100561 * either return that locked entry or will return an error. This error will
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700562 * happen if there are any 4k entries within the 2MiB range that we are
563 * requesting.
Ross Zwisler642261a2016-11-08 11:34:45 +1100564 *
565 * We always favor 4k entries over 2MiB entries. There isn't a flow where we
566 * evict 4k entries in order to 'upgrade' them to a 2MiB entry. A 2MiB
567 * insertion will fail if it finds any 4k entries already in the tree, and a
568 * 4k insertion will cause an existing 2MiB entry to be unmapped and
569 * downgraded to 4k entries. This happens for both 2MiB huge zero pages as
570 * well as 2MiB empty entries.
571 *
572 * The exception to this downgrade path is for 2MiB DAX PMD entries that have
573 * real storage backing them. We will leave these real 2MiB DAX entries in
574 * the tree, and PTE writes will simply dirty the entire 2MiB DAX entry.
575 *
Jan Karaac401cc2016-05-12 18:29:18 +0200576 * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For
577 * persistent memory the benefit is doubtful. We can add that later if we can
578 * show it helps.
579 */
Ross Zwisler642261a2016-11-08 11:34:45 +1100580static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index,
581 unsigned long size_flag)
Jan Karaac401cc2016-05-12 18:29:18 +0200582{
Ross Zwisler642261a2016-11-08 11:34:45 +1100583 bool pmd_downgrade = false; /* splitting 2MiB entry into 4k entries? */
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100584 void *entry, **slot;
Jan Karaac401cc2016-05-12 18:29:18 +0200585
586restart:
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700587 xa_lock_irq(&mapping->i_pages);
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100588 entry = get_unlocked_mapping_entry(mapping, index, &slot);
Ross Zwisler642261a2016-11-08 11:34:45 +1100589
Matthew Wilcox3159f942017-11-03 13:30:42 -0400590 if (WARN_ON_ONCE(entry && !xa_is_value(entry))) {
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700591 entry = ERR_PTR(-EIO);
592 goto out_unlock;
593 }
594
Ross Zwisler642261a2016-11-08 11:34:45 +1100595 if (entry) {
Matthew Wilcox3159f942017-11-03 13:30:42 -0400596 if (size_flag & DAX_PMD) {
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700597 if (dax_is_pte_entry(entry)) {
Ross Zwisler642261a2016-11-08 11:34:45 +1100598 put_unlocked_mapping_entry(mapping, index,
599 entry);
600 entry = ERR_PTR(-EEXIST);
601 goto out_unlock;
602 }
603 } else { /* trying to grab a PTE entry */
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700604 if (dax_is_pmd_entry(entry) &&
Ross Zwisler642261a2016-11-08 11:34:45 +1100605 (dax_is_zero_entry(entry) ||
606 dax_is_empty_entry(entry))) {
607 pmd_downgrade = true;
608 }
609 }
610 }
611
Jan Karaac401cc2016-05-12 18:29:18 +0200612 /* No entry for given index? Make sure radix tree is big enough. */
Ross Zwisler642261a2016-11-08 11:34:45 +1100613 if (!entry || pmd_downgrade) {
Jan Karaac401cc2016-05-12 18:29:18 +0200614 int err;
615
Ross Zwisler642261a2016-11-08 11:34:45 +1100616 if (pmd_downgrade) {
617 /*
618 * Make sure 'entry' remains valid while we drop
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700619 * the i_pages lock.
Ross Zwisler642261a2016-11-08 11:34:45 +1100620 */
621 entry = lock_slot(mapping, slot);
622 }
623
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700624 xa_unlock_irq(&mapping->i_pages);
Ross Zwisler642261a2016-11-08 11:34:45 +1100625 /*
626 * Besides huge zero pages the only other thing that gets
627 * downgraded are empty entries which don't need to be
628 * unmapped.
629 */
630 if (pmd_downgrade && dax_is_zero_entry(entry))
Matthew Wilcox977fbdc2018-01-31 16:17:36 -0800631 unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR,
632 PG_PMD_NR, false);
Ross Zwisler642261a2016-11-08 11:34:45 +1100633
Jan Kara0cb80b42016-12-12 21:34:12 -0500634 err = radix_tree_preload(
635 mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM);
636 if (err) {
637 if (pmd_downgrade)
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700638 put_locked_mapping_entry(mapping, index);
Jan Kara0cb80b42016-12-12 21:34:12 -0500639 return ERR_PTR(err);
640 }
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700641 xa_lock_irq(&mapping->i_pages);
Ross Zwisler642261a2016-11-08 11:34:45 +1100642
Ross Zwislere11f8b72017-04-07 16:04:57 -0700643 if (!entry) {
644 /*
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700645 * We needed to drop the i_pages lock while calling
Ross Zwislere11f8b72017-04-07 16:04:57 -0700646 * radix_tree_preload() and we didn't have an entry to
647 * lock. See if another thread inserted an entry at
648 * our index during this time.
649 */
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700650 entry = __radix_tree_lookup(&mapping->i_pages, index,
Ross Zwislere11f8b72017-04-07 16:04:57 -0700651 NULL, &slot);
652 if (entry) {
653 radix_tree_preload_end();
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700654 xa_unlock_irq(&mapping->i_pages);
Ross Zwislere11f8b72017-04-07 16:04:57 -0700655 goto restart;
656 }
657 }
658
Ross Zwisler642261a2016-11-08 11:34:45 +1100659 if (pmd_downgrade) {
Dan Williamsd2c997c2017-12-22 22:02:48 -0800660 dax_disassociate_entry(entry, mapping, false);
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700661 radix_tree_delete(&mapping->i_pages, index);
Ross Zwisler642261a2016-11-08 11:34:45 +1100662 mapping->nrexceptional--;
Matthew Wilcoxec4907f2018-03-28 11:01:43 -0400663 dax_wake_mapping_entry_waiter(&mapping->i_pages,
664 index, entry, true);
Ross Zwisler642261a2016-11-08 11:34:45 +1100665 }
666
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400667 entry = dax_make_locked(0, size_flag | DAX_EMPTY);
Ross Zwisler642261a2016-11-08 11:34:45 +1100668
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700669 err = __radix_tree_insert(&mapping->i_pages, index,
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400670 dax_entry_order(entry), entry);
Jan Karaac401cc2016-05-12 18:29:18 +0200671 radix_tree_preload_end();
672 if (err) {
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700673 xa_unlock_irq(&mapping->i_pages);
Ross Zwisler642261a2016-11-08 11:34:45 +1100674 /*
Ross Zwislere11f8b72017-04-07 16:04:57 -0700675 * Our insertion of a DAX entry failed, most likely
676 * because we were inserting a PMD entry and it
677 * collided with a PTE sized entry at a different
678 * index in the PMD range. We haven't inserted
679 * anything into the radix tree and have no waiters to
680 * wake.
Ross Zwisler642261a2016-11-08 11:34:45 +1100681 */
Jan Karaac401cc2016-05-12 18:29:18 +0200682 return ERR_PTR(err);
683 }
684 /* Good, we have inserted empty locked entry into the tree. */
685 mapping->nrexceptional++;
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700686 xa_unlock_irq(&mapping->i_pages);
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100687 return entry;
Jan Karaac401cc2016-05-12 18:29:18 +0200688 }
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100689 entry = lock_slot(mapping, slot);
Ross Zwisler642261a2016-11-08 11:34:45 +1100690 out_unlock:
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700691 xa_unlock_irq(&mapping->i_pages);
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100692 return entry;
Jan Karaac401cc2016-05-12 18:29:18 +0200693}
694
Dan Williams5fac7402018-03-09 17:44:31 -0800695/**
696 * dax_layout_busy_page - find first pinned page in @mapping
697 * @mapping: address space to scan for a page with ref count > 1
698 *
699 * DAX requires ZONE_DEVICE mapped pages. These pages are never
700 * 'onlined' to the page allocator so they are considered idle when
701 * page->count == 1. A filesystem uses this interface to determine if
702 * any page in the mapping is busy, i.e. for DMA, or other
703 * get_user_pages() usages.
704 *
705 * It is expected that the filesystem is holding locks to block the
706 * establishment of new mappings in this address_space. I.e. it expects
707 * to be able to run unmap_mapping_range() and subsequently not race
708 * mapping_mapped() becoming true.
709 */
710struct page *dax_layout_busy_page(struct address_space *mapping)
711{
Matthew Wilcox084a8992018-05-17 13:03:48 -0400712 XA_STATE(xas, &mapping->i_pages, 0);
713 void *entry;
714 unsigned int scanned = 0;
Dan Williams5fac7402018-03-09 17:44:31 -0800715 struct page *page = NULL;
Dan Williams5fac7402018-03-09 17:44:31 -0800716
717 /*
718 * In the 'limited' case get_user_pages() for dax is disabled.
719 */
720 if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
721 return NULL;
722
723 if (!dax_mapping(mapping) || !mapping_mapped(mapping))
724 return NULL;
725
Dan Williams5fac7402018-03-09 17:44:31 -0800726 /*
727 * If we race get_user_pages_fast() here either we'll see the
Matthew Wilcox084a8992018-05-17 13:03:48 -0400728 * elevated page count in the iteration and wait, or
Dan Williams5fac7402018-03-09 17:44:31 -0800729 * get_user_pages_fast() will see that the page it took a reference
730 * against is no longer mapped in the page tables and bail to the
731 * get_user_pages() slow path. The slow path is protected by
732 * pte_lock() and pmd_lock(). New references are not taken without
733 * holding those locks, and unmap_mapping_range() will not zero the
734 * pte or pmd without holding the respective lock, so we are
735 * guaranteed to either see new references or prevent new
736 * references from being established.
737 */
738 unmap_mapping_range(mapping, 0, 0, 1);
739
Matthew Wilcox084a8992018-05-17 13:03:48 -0400740 xas_lock_irq(&xas);
741 xas_for_each(&xas, entry, ULONG_MAX) {
742 if (WARN_ON_ONCE(!xa_is_value(entry)))
743 continue;
744 if (unlikely(dax_is_locked(entry)))
745 entry = get_unlocked_entry(&xas);
746 if (entry)
747 page = dax_busy_page(entry);
748 put_unlocked_entry(&xas, entry);
Dan Williams5fac7402018-03-09 17:44:31 -0800749 if (page)
750 break;
Matthew Wilcox084a8992018-05-17 13:03:48 -0400751 if (++scanned % XA_CHECK_SCHED)
752 continue;
753
754 xas_pause(&xas);
755 xas_unlock_irq(&xas);
756 cond_resched();
757 xas_lock_irq(&xas);
Dan Williams5fac7402018-03-09 17:44:31 -0800758 }
Matthew Wilcox084a8992018-05-17 13:03:48 -0400759 xas_unlock_irq(&xas);
Dan Williams5fac7402018-03-09 17:44:31 -0800760 return page;
761}
762EXPORT_SYMBOL_GPL(dax_layout_busy_page);
763
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400764static int __dax_invalidate_entry(struct address_space *mapping,
Jan Karac6dcf522016-08-10 17:22:44 +0200765 pgoff_t index, bool trunc)
766{
Matthew Wilcox07f2d892018-03-28 15:40:41 -0400767 XA_STATE(xas, &mapping->i_pages, index);
Jan Karac6dcf522016-08-10 17:22:44 +0200768 int ret = 0;
769 void *entry;
Jan Karac6dcf522016-08-10 17:22:44 +0200770
Matthew Wilcox07f2d892018-03-28 15:40:41 -0400771 xas_lock_irq(&xas);
772 entry = get_unlocked_entry(&xas);
Matthew Wilcox3159f942017-11-03 13:30:42 -0400773 if (!entry || WARN_ON_ONCE(!xa_is_value(entry)))
Jan Karac6dcf522016-08-10 17:22:44 +0200774 goto out;
775 if (!trunc &&
Matthew Wilcox07f2d892018-03-28 15:40:41 -0400776 (xas_get_mark(&xas, PAGECACHE_TAG_DIRTY) ||
777 xas_get_mark(&xas, PAGECACHE_TAG_TOWRITE)))
Jan Karac6dcf522016-08-10 17:22:44 +0200778 goto out;
Dan Williamsd2c997c2017-12-22 22:02:48 -0800779 dax_disassociate_entry(entry, mapping, trunc);
Matthew Wilcox07f2d892018-03-28 15:40:41 -0400780 xas_store(&xas, NULL);
Jan Karac6dcf522016-08-10 17:22:44 +0200781 mapping->nrexceptional--;
782 ret = 1;
783out:
Matthew Wilcox07f2d892018-03-28 15:40:41 -0400784 put_unlocked_entry(&xas, entry);
785 xas_unlock_irq(&xas);
Jan Karac6dcf522016-08-10 17:22:44 +0200786 return ret;
787}
Matthew Wilcox07f2d892018-03-28 15:40:41 -0400788
Jan Karaac401cc2016-05-12 18:29:18 +0200789/*
Matthew Wilcox3159f942017-11-03 13:30:42 -0400790 * Delete DAX entry at @index from @mapping. Wait for it
791 * to be unlocked before deleting it.
Jan Karaac401cc2016-05-12 18:29:18 +0200792 */
793int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)
794{
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400795 int ret = __dax_invalidate_entry(mapping, index, true);
Jan Karaac401cc2016-05-12 18:29:18 +0200796
Jan Karaac401cc2016-05-12 18:29:18 +0200797 /*
798 * This gets called from truncate / punch_hole path. As such, the caller
799 * must hold locks protecting against concurrent modifications of the
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400800 * page cache (usually fs-private i_mmap_sem for writing). Since the
Matthew Wilcox3159f942017-11-03 13:30:42 -0400801 * caller has seen a DAX entry for this index, we better find it
Jan Karaac401cc2016-05-12 18:29:18 +0200802 * at that index as well...
803 */
Jan Karac6dcf522016-08-10 17:22:44 +0200804 WARN_ON_ONCE(!ret);
805 return ret;
806}
Jan Karaac401cc2016-05-12 18:29:18 +0200807
Jan Karac6dcf522016-08-10 17:22:44 +0200808/*
Matthew Wilcox3159f942017-11-03 13:30:42 -0400809 * Invalidate DAX entry if it is clean.
Jan Karac6dcf522016-08-10 17:22:44 +0200810 */
811int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
812 pgoff_t index)
813{
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400814 return __dax_invalidate_entry(mapping, index, false);
Jan Karaac401cc2016-05-12 18:29:18 +0200815}
816
Dan Williamscccbce62017-01-27 13:31:42 -0800817static int copy_user_dax(struct block_device *bdev, struct dax_device *dax_dev,
818 sector_t sector, size_t size, struct page *to,
819 unsigned long vaddr)
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800820{
Dan Williamscccbce62017-01-27 13:31:42 -0800821 void *vto, *kaddr;
822 pgoff_t pgoff;
Dan Williamscccbce62017-01-27 13:31:42 -0800823 long rc;
824 int id;
Ross Zwislere2e05392015-08-18 13:55:41 -0600825
Dan Williamscccbce62017-01-27 13:31:42 -0800826 rc = bdev_dax_pgoff(bdev, sector, size, &pgoff);
827 if (rc)
828 return rc;
829
830 id = dax_read_lock();
Huaisheng Ye86ed9132018-07-30 15:15:48 +0800831 rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, NULL);
Dan Williamscccbce62017-01-27 13:31:42 -0800832 if (rc < 0) {
833 dax_read_unlock(id);
834 return rc;
835 }
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800836 vto = kmap_atomic(to);
Dan Williamscccbce62017-01-27 13:31:42 -0800837 copy_user_page(vto, (void __force *)kaddr, vaddr, to);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800838 kunmap_atomic(vto);
Dan Williamscccbce62017-01-27 13:31:42 -0800839 dax_read_unlock(id);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800840 return 0;
841}
842
Ross Zwisler642261a2016-11-08 11:34:45 +1100843/*
844 * By this point grab_mapping_entry() has ensured that we have a locked entry
845 * of the appropriate size so we don't have to worry about downgrading PMDs to
846 * PTEs. If we happen to be trying to insert a PTE and there is a PMD
847 * already in the tree, we will skip the insertion and just dirty the PMD as
848 * appropriate.
849 */
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400850static void *dax_insert_entry(struct address_space *mapping,
851 struct vm_fault *vmf,
852 void *entry, pfn_t pfn_t, unsigned long flags, bool dirty)
Ross Zwisler9973c982016-01-22 15:10:47 -0800853{
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700854 struct radix_tree_root *pages = &mapping->i_pages;
Dan Williams3fe07912017-10-14 17:13:45 -0700855 unsigned long pfn = pfn_t_to_pfn(pfn_t);
Jan Karaac401cc2016-05-12 18:29:18 +0200856 pgoff_t index = vmf->pgoff;
Dan Williams3fe07912017-10-14 17:13:45 -0700857 void *new_entry;
Ross Zwisler9973c982016-01-22 15:10:47 -0800858
Jan Karaf5b7b742017-11-01 16:36:40 +0100859 if (dirty)
Dmitry Monakhovd2b2a282016-02-05 15:36:55 -0800860 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
Ross Zwisler9973c982016-01-22 15:10:47 -0800861
Matthew Wilcox3159f942017-11-03 13:30:42 -0400862 if (dax_is_zero_entry(entry) && !(flags & DAX_ZERO_PAGE)) {
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700863 /* we are replacing a zero page with block mapping */
864 if (dax_is_pmd_entry(entry))
Matthew Wilcox977fbdc2018-01-31 16:17:36 -0800865 unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR,
866 PG_PMD_NR, false);
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700867 else /* pte entry */
Matthew Wilcox977fbdc2018-01-31 16:17:36 -0800868 unmap_mapping_pages(mapping, vmf->pgoff, 1, false);
Ross Zwisler9973c982016-01-22 15:10:47 -0800869 }
870
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700871 xa_lock_irq(pages);
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400872 new_entry = dax_make_locked(pfn, flags);
Dan Williamsd2c997c2017-12-22 22:02:48 -0800873 if (dax_entry_size(entry) != dax_entry_size(new_entry)) {
874 dax_disassociate_entry(entry, mapping, false);
Dan Williams73449da2018-07-13 21:49:50 -0700875 dax_associate_entry(new_entry, mapping, vmf->vma, vmf->address);
Dan Williamsd2c997c2017-12-22 22:02:48 -0800876 }
Ross Zwisler642261a2016-11-08 11:34:45 +1100877
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700878 if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) {
Ross Zwisler642261a2016-11-08 11:34:45 +1100879 /*
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400880 * Only swap our new entry into the page cache if the current
Ross Zwisler642261a2016-11-08 11:34:45 +1100881 * entry is a zero page or an empty entry. If a normal PTE or
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400882 * PMD entry is already in the cache, we leave it alone. This
Ross Zwisler642261a2016-11-08 11:34:45 +1100883 * means that if we are trying to insert a PTE and the
884 * existing entry is a PMD, we will just leave the PMD in the
885 * tree and dirty it if necessary.
886 */
Johannes Weinerf7942432016-12-12 16:43:41 -0800887 struct radix_tree_node *node;
Jan Karaac401cc2016-05-12 18:29:18 +0200888 void **slot;
889 void *ret;
Ross Zwisler9973c982016-01-22 15:10:47 -0800890
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700891 ret = __radix_tree_lookup(pages, index, &node, &slot);
Jan Karaac401cc2016-05-12 18:29:18 +0200892 WARN_ON_ONCE(ret != entry);
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700893 __radix_tree_replace(pages, node, slot,
Mel Gormanc7df8ad2017-11-15 17:37:41 -0800894 new_entry, NULL);
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700895 entry = new_entry;
Ross Zwisler9973c982016-01-22 15:10:47 -0800896 }
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700897
Jan Karaf5b7b742017-11-01 16:36:40 +0100898 if (dirty)
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700899 radix_tree_tag_set(pages, index, PAGECACHE_TAG_DIRTY);
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700900
Matthew Wilcoxb93b0162018-04-10 16:36:56 -0700901 xa_unlock_irq(pages);
Ross Zwisler91d25ba2017-09-06 16:18:43 -0700902 return entry;
Ross Zwisler9973c982016-01-22 15:10:47 -0800903}
904
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400905static inline
906unsigned long pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma)
Jan Kara4b4bb462016-12-14 15:07:53 -0800907{
908 unsigned long address;
909
910 address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
911 VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
912 return address;
913}
914
915/* Walk all mappings of a given index of a file and writeprotect them */
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -0400916static void dax_entry_mkclean(struct address_space *mapping, pgoff_t index,
917 unsigned long pfn)
Jan Kara4b4bb462016-12-14 15:07:53 -0800918{
919 struct vm_area_struct *vma;
Ross Zwislerf729c8c2017-01-10 16:57:24 -0800920 pte_t pte, *ptep = NULL;
921 pmd_t *pmdp = NULL;
Jan Kara4b4bb462016-12-14 15:07:53 -0800922 spinlock_t *ptl;
Jan Kara4b4bb462016-12-14 15:07:53 -0800923
924 i_mmap_lock_read(mapping);
925 vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) {
Jérôme Glissea4d1a882017-08-31 17:17:26 -0400926 unsigned long address, start, end;
Jan Kara4b4bb462016-12-14 15:07:53 -0800927
928 cond_resched();
929
930 if (!(vma->vm_flags & VM_SHARED))
931 continue;
932
933 address = pgoff_address(index, vma);
Jérôme Glissea4d1a882017-08-31 17:17:26 -0400934
935 /*
936 * Note because we provide start/end to follow_pte_pmd it will
937 * call mmu_notifier_invalidate_range_start() on our behalf
938 * before taking any lock.
939 */
940 if (follow_pte_pmd(vma->vm_mm, address, &start, &end, &ptep, &pmdp, &ptl))
Jan Kara4b4bb462016-12-14 15:07:53 -0800941 continue;
Jan Kara4b4bb462016-12-14 15:07:53 -0800942
Jérôme Glisse0f108512017-11-15 17:34:07 -0800943 /*
944 * No need to call mmu_notifier_invalidate_range() as we are
945 * downgrading page table protection not changing it to point
946 * to a new page.
947 *
Mike Rapoportad56b732018-03-21 21:22:47 +0200948 * See Documentation/vm/mmu_notifier.rst
Jérôme Glisse0f108512017-11-15 17:34:07 -0800949 */
Ross Zwislerf729c8c2017-01-10 16:57:24 -0800950 if (pmdp) {
951#ifdef CONFIG_FS_DAX_PMD
952 pmd_t pmd;
953
954 if (pfn != pmd_pfn(*pmdp))
955 goto unlock_pmd;
Linus Torvaldsf6f37322017-12-15 18:53:22 -0800956 if (!pmd_dirty(*pmdp) && !pmd_write(*pmdp))
Ross Zwislerf729c8c2017-01-10 16:57:24 -0800957 goto unlock_pmd;
958
959 flush_cache_page(vma, address, pfn);
960 pmd = pmdp_huge_clear_flush(vma, address, pmdp);
961 pmd = pmd_wrprotect(pmd);
962 pmd = pmd_mkclean(pmd);
963 set_pmd_at(vma->vm_mm, address, pmdp, pmd);
Ross Zwislerf729c8c2017-01-10 16:57:24 -0800964unlock_pmd:
Ross Zwislerf729c8c2017-01-10 16:57:24 -0800965#endif
Jan H. Schönherree190ca2018-01-31 16:14:04 -0800966 spin_unlock(ptl);
Ross Zwislerf729c8c2017-01-10 16:57:24 -0800967 } else {
968 if (pfn != pte_pfn(*ptep))
969 goto unlock_pte;
970 if (!pte_dirty(*ptep) && !pte_write(*ptep))
971 goto unlock_pte;
972
973 flush_cache_page(vma, address, pfn);
974 pte = ptep_clear_flush(vma, address, ptep);
975 pte = pte_wrprotect(pte);
976 pte = pte_mkclean(pte);
977 set_pte_at(vma->vm_mm, address, ptep, pte);
Ross Zwislerf729c8c2017-01-10 16:57:24 -0800978unlock_pte:
979 pte_unmap_unlock(ptep, ptl);
980 }
Jan Kara4b4bb462016-12-14 15:07:53 -0800981
Jérôme Glissea4d1a882017-08-31 17:17:26 -0400982 mmu_notifier_invalidate_range_end(vma->vm_mm, start, end);
Jan Kara4b4bb462016-12-14 15:07:53 -0800983 }
984 i_mmap_unlock_read(mapping);
985}
986
Matthew Wilcox9fc747f62018-03-28 16:03:45 -0400987static int dax_writeback_one(struct xa_state *xas, struct dax_device *dax_dev,
988 struct address_space *mapping, void *entry)
Ross Zwisler9973c982016-01-22 15:10:47 -0800989{
Dan Williams3fe07912017-10-14 17:13:45 -0700990 unsigned long pfn;
991 long ret = 0;
Dan Williamscccbce62017-01-27 13:31:42 -0800992 size_t size;
Ross Zwisler9973c982016-01-22 15:10:47 -0800993
Ross Zwisler9973c982016-01-22 15:10:47 -0800994 /*
Jan Karaa6abc2c2016-12-14 15:07:47 -0800995 * A page got tagged dirty in DAX mapping? Something is seriously
996 * wrong.
Ross Zwisler9973c982016-01-22 15:10:47 -0800997 */
Matthew Wilcox3159f942017-11-03 13:30:42 -0400998 if (WARN_ON(!xa_is_value(entry)))
Jan Karaa6abc2c2016-12-14 15:07:47 -0800999 return -EIO;
Ross Zwisler9973c982016-01-22 15:10:47 -08001000
Matthew Wilcox9fc747f62018-03-28 16:03:45 -04001001 if (unlikely(dax_is_locked(entry))) {
1002 void *old_entry = entry;
1003
1004 entry = get_unlocked_entry(xas);
1005
1006 /* Entry got punched out / reallocated? */
1007 if (!entry || WARN_ON_ONCE(!xa_is_value(entry)))
1008 goto put_unlocked;
1009 /*
1010 * Entry got reallocated elsewhere? No need to writeback.
1011 * We have to compare pfns as we must not bail out due to
1012 * difference in lockbit or entry type.
1013 */
1014 if (dax_to_pfn(old_entry) != dax_to_pfn(entry))
1015 goto put_unlocked;
1016 if (WARN_ON_ONCE(dax_is_empty_entry(entry) ||
1017 dax_is_zero_entry(entry))) {
1018 ret = -EIO;
1019 goto put_unlocked;
1020 }
1021
1022 /* Another fsync thread may have already done this entry */
1023 if (!xas_get_mark(xas, PAGECACHE_TAG_TOWRITE))
1024 goto put_unlocked;
Ross Zwisler9973c982016-01-22 15:10:47 -08001025 }
1026
Jan Karaa6abc2c2016-12-14 15:07:47 -08001027 /* Lock the entry to serialize with page faults */
Matthew Wilcox9fc747f62018-03-28 16:03:45 -04001028 dax_lock_entry(xas, entry);
1029
Jan Karaa6abc2c2016-12-14 15:07:47 -08001030 /*
1031 * We can clear the tag now but we have to be careful so that concurrent
1032 * dax_writeback_one() calls for the same index cannot finish before we
1033 * actually flush the caches. This is achieved as the calls will look
Matthew Wilcoxb93b0162018-04-10 16:36:56 -07001034 * at the entry only under the i_pages lock and once they do that
1035 * they will see the entry locked and wait for it to unlock.
Jan Karaa6abc2c2016-12-14 15:07:47 -08001036 */
Matthew Wilcox9fc747f62018-03-28 16:03:45 -04001037 xas_clear_mark(xas, PAGECACHE_TAG_TOWRITE);
1038 xas_unlock_irq(xas);
Jan Karaa6abc2c2016-12-14 15:07:47 -08001039
Ross Zwisler642261a2016-11-08 11:34:45 +11001040 /*
1041 * Even if dax_writeback_mapping_range() was given a wbc->range_start
1042 * in the middle of a PMD, the 'index' we are given will be aligned to
Dan Williams3fe07912017-10-14 17:13:45 -07001043 * the start index of the PMD, as will the pfn we pull from 'entry'.
1044 * This allows us to flush for PMD_SIZE and not have to worry about
1045 * partial PMD writebacks.
Ross Zwisler642261a2016-11-08 11:34:45 +11001046 */
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -04001047 pfn = dax_to_pfn(entry);
1048 size = PAGE_SIZE << dax_entry_order(entry);
Dan Williamscccbce62017-01-27 13:31:42 -08001049
Matthew Wilcox9fc747f62018-03-28 16:03:45 -04001050 dax_entry_mkclean(mapping, xas->xa_index, pfn);
Dan Williams3fe07912017-10-14 17:13:45 -07001051 dax_flush(dax_dev, page_address(pfn_to_page(pfn)), size);
Jan Kara4b4bb462016-12-14 15:07:53 -08001052 /*
1053 * After we have flushed the cache, we can clear the dirty tag. There
1054 * cannot be new dirty data in the pfn after the flush has completed as
1055 * the pfn mappings are writeprotected and fault waits for mapping
1056 * entry lock.
1057 */
Matthew Wilcox9fc747f62018-03-28 16:03:45 -04001058 xas_reset(xas);
1059 xas_lock_irq(xas);
1060 xas_store(xas, entry);
1061 xas_clear_mark(xas, PAGECACHE_TAG_DIRTY);
1062 dax_wake_entry(xas, entry, false);
1063
1064 trace_dax_writeback_one(mapping->host, xas->xa_index,
1065 size >> PAGE_SHIFT);
Ross Zwisler9973c982016-01-22 15:10:47 -08001066 return ret;
1067
Jan Karaa6abc2c2016-12-14 15:07:47 -08001068 put_unlocked:
Matthew Wilcox9fc747f62018-03-28 16:03:45 -04001069 put_unlocked_entry(xas, entry);
Ross Zwisler9973c982016-01-22 15:10:47 -08001070 return ret;
1071}
1072
1073/*
1074 * Flush the mapping to the persistent domain within the byte range of [start,
1075 * end]. This is required by data integrity operations to ensure file data is
1076 * on persistent storage prior to completion of the operation.
1077 */
Ross Zwisler7f6d5b52016-02-26 15:19:55 -08001078int dax_writeback_mapping_range(struct address_space *mapping,
1079 struct block_device *bdev, struct writeback_control *wbc)
Ross Zwisler9973c982016-01-22 15:10:47 -08001080{
Matthew Wilcox9fc747f62018-03-28 16:03:45 -04001081 XA_STATE(xas, &mapping->i_pages, wbc->range_start >> PAGE_SHIFT);
Ross Zwisler9973c982016-01-22 15:10:47 -08001082 struct inode *inode = mapping->host;
Matthew Wilcox9fc747f62018-03-28 16:03:45 -04001083 pgoff_t end_index = wbc->range_end >> PAGE_SHIFT;
Dan Williamscccbce62017-01-27 13:31:42 -08001084 struct dax_device *dax_dev;
Matthew Wilcox9fc747f62018-03-28 16:03:45 -04001085 void *entry;
1086 int ret = 0;
1087 unsigned int scanned = 0;
Ross Zwisler9973c982016-01-22 15:10:47 -08001088
1089 if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
1090 return -EIO;
1091
Ross Zwisler7f6d5b52016-02-26 15:19:55 -08001092 if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL)
1093 return 0;
1094
Dan Williamscccbce62017-01-27 13:31:42 -08001095 dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
1096 if (!dax_dev)
1097 return -EIO;
1098
Matthew Wilcox9fc747f62018-03-28 16:03:45 -04001099 trace_dax_writeback_range(inode, xas.xa_index, end_index);
Ross Zwisler9973c982016-01-22 15:10:47 -08001100
Matthew Wilcox9fc747f62018-03-28 16:03:45 -04001101 tag_pages_for_writeback(mapping, xas.xa_index, end_index);
Ross Zwislerd14a3f42017-05-08 16:00:10 -07001102
Matthew Wilcox9fc747f62018-03-28 16:03:45 -04001103 xas_lock_irq(&xas);
1104 xas_for_each_marked(&xas, entry, end_index, PAGECACHE_TAG_TOWRITE) {
1105 ret = dax_writeback_one(&xas, dax_dev, mapping, entry);
1106 if (ret < 0) {
1107 mapping_set_error(mapping, ret);
Ross Zwisler9973c982016-01-22 15:10:47 -08001108 break;
Ross Zwisler9973c982016-01-22 15:10:47 -08001109 }
Matthew Wilcox9fc747f62018-03-28 16:03:45 -04001110 if (++scanned % XA_CHECK_SCHED)
1111 continue;
1112
1113 xas_pause(&xas);
1114 xas_unlock_irq(&xas);
1115 cond_resched();
1116 xas_lock_irq(&xas);
Ross Zwisler9973c982016-01-22 15:10:47 -08001117 }
Matthew Wilcox9fc747f62018-03-28 16:03:45 -04001118 xas_unlock_irq(&xas);
Dan Williamscccbce62017-01-27 13:31:42 -08001119 put_dax(dax_dev);
Matthew Wilcox9fc747f62018-03-28 16:03:45 -04001120 trace_dax_writeback_range_done(inode, xas.xa_index, end_index);
1121 return ret;
Ross Zwisler9973c982016-01-22 15:10:47 -08001122}
1123EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);
1124
Jan Kara31a6f1a2017-11-01 16:36:32 +01001125static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos)
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -08001126{
Linus Torvaldsa3841f92017-11-17 09:51:57 -08001127 return (iomap->addr + (pos & PAGE_MASK) - iomap->offset) >> 9;
Jan Kara31a6f1a2017-11-01 16:36:32 +01001128}
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -08001129
Jan Kara5e161e42017-11-01 16:36:33 +01001130static int dax_iomap_pfn(struct iomap *iomap, loff_t pos, size_t size,
1131 pfn_t *pfnp)
1132{
1133 const sector_t sector = dax_iomap_sector(iomap, pos);
1134 pgoff_t pgoff;
Jan Kara5e161e42017-11-01 16:36:33 +01001135 int id, rc;
1136 long length;
1137
1138 rc = bdev_dax_pgoff(iomap->bdev, sector, size, &pgoff);
Dan Williamscccbce62017-01-27 13:31:42 -08001139 if (rc)
1140 return rc;
Dan Williamscccbce62017-01-27 13:31:42 -08001141 id = dax_read_lock();
Jan Kara5e161e42017-11-01 16:36:33 +01001142 length = dax_direct_access(iomap->dax_dev, pgoff, PHYS_PFN(size),
Huaisheng Ye86ed9132018-07-30 15:15:48 +08001143 NULL, pfnp);
Jan Kara5e161e42017-11-01 16:36:33 +01001144 if (length < 0) {
1145 rc = length;
1146 goto out;
Dan Williamscccbce62017-01-27 13:31:42 -08001147 }
Jan Kara5e161e42017-11-01 16:36:33 +01001148 rc = -EINVAL;
1149 if (PFN_PHYS(length) < size)
1150 goto out;
1151 if (pfn_t_to_pfn(*pfnp) & (PHYS_PFN(size)-1))
1152 goto out;
1153 /* For larger pages we need devmap */
1154 if (length > 1 && !pfn_t_devmap(*pfnp))
1155 goto out;
1156 rc = 0;
1157out:
Dan Williamscccbce62017-01-27 13:31:42 -08001158 dax_read_unlock(id);
Jan Kara5e161e42017-11-01 16:36:33 +01001159 return rc;
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -08001160}
1161
Ross Zwislere30331f2017-09-06 16:18:39 -07001162/*
Ross Zwisler91d25ba2017-09-06 16:18:43 -07001163 * The user has performed a load from a hole in the file. Allocating a new
1164 * page in the file would cause excessive storage usage for workloads with
1165 * sparse files. Instead we insert a read-only mapping of the 4k zero page.
1166 * If this page is ever written to we will re-fault and change the mapping to
1167 * point to real DAX storage instead.
Ross Zwislere30331f2017-09-06 16:18:39 -07001168 */
Souptick Joarderab77dab2018-06-07 17:04:29 -07001169static vm_fault_t dax_load_hole(struct address_space *mapping, void *entry,
Ross Zwislere30331f2017-09-06 16:18:39 -07001170 struct vm_fault *vmf)
1171{
1172 struct inode *inode = mapping->host;
Ross Zwisler91d25ba2017-09-06 16:18:43 -07001173 unsigned long vaddr = vmf->address;
Matthew Wilcoxb90ca5c2018-09-11 21:27:44 -07001174 pfn_t pfn = pfn_to_pfn_t(my_zero_pfn(vaddr));
1175 vm_fault_t ret;
Ross Zwislere30331f2017-09-06 16:18:39 -07001176
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -04001177 dax_insert_entry(mapping, vmf, entry, pfn,
Matthew Wilcox3159f942017-11-03 13:30:42 -04001178 DAX_ZERO_PAGE, false);
1179
Souptick Joarderab77dab2018-06-07 17:04:29 -07001180 ret = vmf_insert_mixed(vmf->vma, vaddr, pfn);
Ross Zwislere30331f2017-09-06 16:18:39 -07001181 trace_dax_load_hole(inode, vmf, ret);
1182 return ret;
1183}
1184
Vishal Verma4b0228f2016-04-21 15:13:46 -04001185static bool dax_range_is_aligned(struct block_device *bdev,
1186 unsigned int offset, unsigned int length)
1187{
1188 unsigned short sector_size = bdev_logical_block_size(bdev);
1189
1190 if (!IS_ALIGNED(offset, sector_size))
1191 return false;
1192 if (!IS_ALIGNED(length, sector_size))
1193 return false;
1194
1195 return true;
1196}
1197
Dan Williamscccbce62017-01-27 13:31:42 -08001198int __dax_zero_page_range(struct block_device *bdev,
1199 struct dax_device *dax_dev, sector_t sector,
1200 unsigned int offset, unsigned int size)
Christoph Hellwig679c8bd2016-05-09 10:47:04 +02001201{
Dan Williamscccbce62017-01-27 13:31:42 -08001202 if (dax_range_is_aligned(bdev, offset, size)) {
1203 sector_t start_sector = sector + (offset >> 9);
Vishal Verma4b0228f2016-04-21 15:13:46 -04001204
1205 return blkdev_issue_zeroout(bdev, start_sector,
Linus Torvalds53ef7d02017-05-05 18:49:20 -07001206 size >> 9, GFP_NOFS, 0);
Vishal Verma4b0228f2016-04-21 15:13:46 -04001207 } else {
Dan Williamscccbce62017-01-27 13:31:42 -08001208 pgoff_t pgoff;
1209 long rc, id;
1210 void *kaddr;
Dan Williamscccbce62017-01-27 13:31:42 -08001211
Dan Williamse84b83b2017-05-10 19:38:13 -07001212 rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff);
Dan Williamscccbce62017-01-27 13:31:42 -08001213 if (rc)
1214 return rc;
1215
1216 id = dax_read_lock();
Huaisheng Ye86ed9132018-07-30 15:15:48 +08001217 rc = dax_direct_access(dax_dev, pgoff, 1, &kaddr, NULL);
Dan Williamscccbce62017-01-27 13:31:42 -08001218 if (rc < 0) {
1219 dax_read_unlock(id);
1220 return rc;
1221 }
Dan Williams81f55872017-05-29 13:12:20 -07001222 memset(kaddr + offset, 0, size);
Mikulas Patockac3ca0152017-08-31 21:47:43 -04001223 dax_flush(dax_dev, kaddr + offset, size);
Dan Williamscccbce62017-01-27 13:31:42 -08001224 dax_read_unlock(id);
Vishal Verma4b0228f2016-04-21 15:13:46 -04001225 }
Christoph Hellwig679c8bd2016-05-09 10:47:04 +02001226 return 0;
1227}
1228EXPORT_SYMBOL_GPL(__dax_zero_page_range);
1229
Christoph Hellwiga254e562016-09-19 11:24:49 +10001230static loff_t
Ross Zwisler11c59c92016-11-08 11:32:46 +11001231dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
Christoph Hellwiga254e562016-09-19 11:24:49 +10001232 struct iomap *iomap)
1233{
Dan Williamscccbce62017-01-27 13:31:42 -08001234 struct block_device *bdev = iomap->bdev;
1235 struct dax_device *dax_dev = iomap->dax_dev;
Christoph Hellwiga254e562016-09-19 11:24:49 +10001236 struct iov_iter *iter = data;
1237 loff_t end = pos + length, done = 0;
1238 ssize_t ret = 0;
Dan Williamsa77d4782018-03-16 17:36:44 -07001239 size_t xfer;
Dan Williamscccbce62017-01-27 13:31:42 -08001240 int id;
Christoph Hellwiga254e562016-09-19 11:24:49 +10001241
1242 if (iov_iter_rw(iter) == READ) {
1243 end = min(end, i_size_read(inode));
1244 if (pos >= end)
1245 return 0;
1246
1247 if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN)
1248 return iov_iter_zero(min(length, end - pos), iter);
1249 }
1250
1251 if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED))
1252 return -EIO;
1253
Jan Karae3fce682016-08-10 17:10:28 +02001254 /*
1255 * Write can allocate block for an area which has a hole page mapped
1256 * into page tables. We have to tear down these mappings so that data
1257 * written by write(2) is visible in mmap.
1258 */
Jan Karacd656372017-05-12 15:46:50 -07001259 if (iomap->flags & IOMAP_F_NEW) {
Jan Karae3fce682016-08-10 17:10:28 +02001260 invalidate_inode_pages2_range(inode->i_mapping,
1261 pos >> PAGE_SHIFT,
1262 (end - 1) >> PAGE_SHIFT);
1263 }
1264
Dan Williamscccbce62017-01-27 13:31:42 -08001265 id = dax_read_lock();
Christoph Hellwiga254e562016-09-19 11:24:49 +10001266 while (pos < end) {
1267 unsigned offset = pos & (PAGE_SIZE - 1);
Dan Williamscccbce62017-01-27 13:31:42 -08001268 const size_t size = ALIGN(length + offset, PAGE_SIZE);
1269 const sector_t sector = dax_iomap_sector(iomap, pos);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001270 ssize_t map_len;
Dan Williamscccbce62017-01-27 13:31:42 -08001271 pgoff_t pgoff;
1272 void *kaddr;
Christoph Hellwiga254e562016-09-19 11:24:49 +10001273
Michal Hockod1908f52017-02-03 13:13:26 -08001274 if (fatal_signal_pending(current)) {
1275 ret = -EINTR;
1276 break;
1277 }
1278
Dan Williamscccbce62017-01-27 13:31:42 -08001279 ret = bdev_dax_pgoff(bdev, sector, size, &pgoff);
1280 if (ret)
1281 break;
1282
1283 map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size),
Huaisheng Ye86ed9132018-07-30 15:15:48 +08001284 &kaddr, NULL);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001285 if (map_len < 0) {
1286 ret = map_len;
1287 break;
1288 }
1289
Dan Williamscccbce62017-01-27 13:31:42 -08001290 map_len = PFN_PHYS(map_len);
1291 kaddr += offset;
Christoph Hellwiga254e562016-09-19 11:24:49 +10001292 map_len -= offset;
1293 if (map_len > end - pos)
1294 map_len = end - pos;
1295
Ross Zwislera2e050f2017-09-06 16:18:54 -07001296 /*
1297 * The userspace address for the memory copy has already been
1298 * validated via access_ok() in either vfs_read() or
1299 * vfs_write(), depending on which operation we are doing.
1300 */
Christoph Hellwiga254e562016-09-19 11:24:49 +10001301 if (iov_iter_rw(iter) == WRITE)
Dan Williamsa77d4782018-03-16 17:36:44 -07001302 xfer = dax_copy_from_iter(dax_dev, pgoff, kaddr,
Dan Williamsfec53772017-05-29 21:56:49 -07001303 map_len, iter);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001304 else
Dan Williamsa77d4782018-03-16 17:36:44 -07001305 xfer = dax_copy_to_iter(dax_dev, pgoff, kaddr,
Dan Williamsb3a9a0c2018-05-02 06:46:33 -07001306 map_len, iter);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001307
Dan Williamsa77d4782018-03-16 17:36:44 -07001308 pos += xfer;
1309 length -= xfer;
1310 done += xfer;
1311
1312 if (xfer == 0)
1313 ret = -EFAULT;
1314 if (xfer < map_len)
1315 break;
Christoph Hellwiga254e562016-09-19 11:24:49 +10001316 }
Dan Williamscccbce62017-01-27 13:31:42 -08001317 dax_read_unlock(id);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001318
1319 return done ? done : ret;
1320}
1321
1322/**
Ross Zwisler11c59c92016-11-08 11:32:46 +11001323 * dax_iomap_rw - Perform I/O to a DAX file
Christoph Hellwiga254e562016-09-19 11:24:49 +10001324 * @iocb: The control block for this I/O
1325 * @iter: The addresses to do I/O from or to
1326 * @ops: iomap ops passed from the file system
1327 *
1328 * This function performs read and write operations to directly mapped
1329 * persistent memory. The callers needs to take care of read/write exclusion
1330 * and evicting any page cache pages in the region under I/O.
1331 */
1332ssize_t
Ross Zwisler11c59c92016-11-08 11:32:46 +11001333dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
Christoph Hellwig8ff6daa2017-01-27 23:20:26 -08001334 const struct iomap_ops *ops)
Christoph Hellwiga254e562016-09-19 11:24:49 +10001335{
1336 struct address_space *mapping = iocb->ki_filp->f_mapping;
1337 struct inode *inode = mapping->host;
1338 loff_t pos = iocb->ki_pos, ret = 0, done = 0;
1339 unsigned flags = 0;
1340
Christoph Hellwig168316d2017-02-08 14:43:13 -05001341 if (iov_iter_rw(iter) == WRITE) {
1342 lockdep_assert_held_exclusive(&inode->i_rwsem);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001343 flags |= IOMAP_WRITE;
Christoph Hellwig168316d2017-02-08 14:43:13 -05001344 } else {
1345 lockdep_assert_held(&inode->i_rwsem);
1346 }
Christoph Hellwiga254e562016-09-19 11:24:49 +10001347
Christoph Hellwiga254e562016-09-19 11:24:49 +10001348 while (iov_iter_count(iter)) {
1349 ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops,
Ross Zwisler11c59c92016-11-08 11:32:46 +11001350 iter, dax_iomap_actor);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001351 if (ret <= 0)
1352 break;
1353 pos += ret;
1354 done += ret;
1355 }
1356
1357 iocb->ki_pos += done;
1358 return done ? done : ret;
1359}
Ross Zwisler11c59c92016-11-08 11:32:46 +11001360EXPORT_SYMBOL_GPL(dax_iomap_rw);
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001361
Souptick Joarderab77dab2018-06-07 17:04:29 -07001362static vm_fault_t dax_fault_return(int error)
Jan Kara9f141d62016-10-19 14:34:31 +02001363{
1364 if (error == 0)
1365 return VM_FAULT_NOPAGE;
1366 if (error == -ENOMEM)
1367 return VM_FAULT_OOM;
1368 return VM_FAULT_SIGBUS;
1369}
1370
Dan Williamsaaa422c2017-11-13 16:38:44 -08001371/*
1372 * MAP_SYNC on a dax mapping guarantees dirty metadata is
1373 * flushed on write-faults (non-cow), but not read-faults.
1374 */
1375static bool dax_fault_is_synchronous(unsigned long flags,
1376 struct vm_area_struct *vma, struct iomap *iomap)
1377{
1378 return (flags & IOMAP_WRITE) && (vma->vm_flags & VM_SYNC)
1379 && (iomap->flags & IOMAP_F_DIRTY);
1380}
1381
Souptick Joarderab77dab2018-06-07 17:04:29 -07001382static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp,
Jan Karac0b24622018-01-07 16:38:43 -05001383 int *iomap_errp, const struct iomap_ops *ops)
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001384{
Jan Karaa0987ad2017-11-01 16:36:34 +01001385 struct vm_area_struct *vma = vmf->vma;
1386 struct address_space *mapping = vma->vm_file->f_mapping;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001387 struct inode *inode = mapping->host;
Jan Kara1a29d852016-12-14 15:07:01 -08001388 unsigned long vaddr = vmf->address;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001389 loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001390 struct iomap iomap = { 0 };
Jan Kara9484ab12016-11-10 10:26:50 +11001391 unsigned flags = IOMAP_FAULT;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001392 int error, major = 0;
Jan Karad2c43ef2017-11-01 16:36:35 +01001393 bool write = vmf->flags & FAULT_FLAG_WRITE;
Jan Karacaa51d22017-11-01 16:36:42 +01001394 bool sync;
Souptick Joarderab77dab2018-06-07 17:04:29 -07001395 vm_fault_t ret = 0;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001396 void *entry;
Jan Kara1b5a1cb2017-11-01 16:36:36 +01001397 pfn_t pfn;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001398
Souptick Joarderab77dab2018-06-07 17:04:29 -07001399 trace_dax_pte_fault(inode, vmf, ret);
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001400 /*
1401 * Check whether offset isn't beyond end of file now. Caller is supposed
1402 * to hold locks serializing us with truncate / punch hole so this is
1403 * a reliable test.
1404 */
Ross Zwislera9c42b32017-05-08 16:00:00 -07001405 if (pos >= i_size_read(inode)) {
Souptick Joarderab77dab2018-06-07 17:04:29 -07001406 ret = VM_FAULT_SIGBUS;
Ross Zwislera9c42b32017-05-08 16:00:00 -07001407 goto out;
1408 }
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001409
Jan Karad2c43ef2017-11-01 16:36:35 +01001410 if (write && !vmf->cow_page)
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001411 flags |= IOMAP_WRITE;
1412
Jan Kara13e451f2017-05-12 15:46:57 -07001413 entry = grab_mapping_entry(mapping, vmf->pgoff, 0);
1414 if (IS_ERR(entry)) {
Souptick Joarderab77dab2018-06-07 17:04:29 -07001415 ret = dax_fault_return(PTR_ERR(entry));
Jan Kara13e451f2017-05-12 15:46:57 -07001416 goto out;
1417 }
1418
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001419 /*
Ross Zwislere2093922017-06-02 14:46:37 -07001420 * It is possible, particularly with mixed reads & writes to private
1421 * mappings, that we have raced with a PMD fault that overlaps with
1422 * the PTE we need to set up. If so just return and the fault will be
1423 * retried.
1424 */
1425 if (pmd_trans_huge(*vmf->pmd) || pmd_devmap(*vmf->pmd)) {
Souptick Joarderab77dab2018-06-07 17:04:29 -07001426 ret = VM_FAULT_NOPAGE;
Ross Zwislere2093922017-06-02 14:46:37 -07001427 goto unlock_entry;
1428 }
1429
1430 /*
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001431 * Note that we don't bother to use iomap_apply here: DAX required
1432 * the file system block size to be equal the page size, which means
1433 * that we never have to deal with more than a single extent here.
1434 */
1435 error = ops->iomap_begin(inode, pos, PAGE_SIZE, flags, &iomap);
Jan Karac0b24622018-01-07 16:38:43 -05001436 if (iomap_errp)
1437 *iomap_errp = error;
Ross Zwislera9c42b32017-05-08 16:00:00 -07001438 if (error) {
Souptick Joarderab77dab2018-06-07 17:04:29 -07001439 ret = dax_fault_return(error);
Jan Kara13e451f2017-05-12 15:46:57 -07001440 goto unlock_entry;
Ross Zwislera9c42b32017-05-08 16:00:00 -07001441 }
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001442 if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) {
Jan Kara13e451f2017-05-12 15:46:57 -07001443 error = -EIO; /* fs corruption? */
1444 goto error_finish_iomap;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001445 }
1446
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001447 if (vmf->cow_page) {
Jan Kara31a6f1a2017-11-01 16:36:32 +01001448 sector_t sector = dax_iomap_sector(&iomap, pos);
1449
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001450 switch (iomap.type) {
1451 case IOMAP_HOLE:
1452 case IOMAP_UNWRITTEN:
1453 clear_user_highpage(vmf->cow_page, vaddr);
1454 break;
1455 case IOMAP_MAPPED:
Dan Williamscccbce62017-01-27 13:31:42 -08001456 error = copy_user_dax(iomap.bdev, iomap.dax_dev,
1457 sector, PAGE_SIZE, vmf->cow_page, vaddr);
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001458 break;
1459 default:
1460 WARN_ON_ONCE(1);
1461 error = -EIO;
1462 break;
1463 }
1464
1465 if (error)
Jan Kara13e451f2017-05-12 15:46:57 -07001466 goto error_finish_iomap;
Jan Karab1aa8122016-12-14 15:07:24 -08001467
1468 __SetPageUptodate(vmf->cow_page);
Souptick Joarderab77dab2018-06-07 17:04:29 -07001469 ret = finish_fault(vmf);
1470 if (!ret)
1471 ret = VM_FAULT_DONE_COW;
Jan Kara13e451f2017-05-12 15:46:57 -07001472 goto finish_iomap;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001473 }
1474
Dan Williamsaaa422c2017-11-13 16:38:44 -08001475 sync = dax_fault_is_synchronous(flags, vma, &iomap);
Jan Karacaa51d22017-11-01 16:36:42 +01001476
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001477 switch (iomap.type) {
1478 case IOMAP_MAPPED:
1479 if (iomap.flags & IOMAP_F_NEW) {
1480 count_vm_event(PGMAJFAULT);
Jan Karaa0987ad2017-11-01 16:36:34 +01001481 count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001482 major = VM_FAULT_MAJOR;
1483 }
Jan Kara1b5a1cb2017-11-01 16:36:36 +01001484 error = dax_iomap_pfn(&iomap, pos, PAGE_SIZE, &pfn);
1485 if (error < 0)
1486 goto error_finish_iomap;
1487
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -04001488 entry = dax_insert_entry(mapping, vmf, entry, pfn,
Jan Karacaa51d22017-11-01 16:36:42 +01001489 0, write && !sync);
Jan Kara1b5a1cb2017-11-01 16:36:36 +01001490
Jan Karacaa51d22017-11-01 16:36:42 +01001491 /*
1492 * If we are doing synchronous page fault and inode needs fsync,
1493 * we can insert PTE into page tables only after that happens.
1494 * Skip insertion for now and return the pfn so that caller can
1495 * insert it after fsync is done.
1496 */
1497 if (sync) {
1498 if (WARN_ON_ONCE(!pfnp)) {
1499 error = -EIO;
1500 goto error_finish_iomap;
1501 }
1502 *pfnp = pfn;
Souptick Joarderab77dab2018-06-07 17:04:29 -07001503 ret = VM_FAULT_NEEDDSYNC | major;
Jan Karacaa51d22017-11-01 16:36:42 +01001504 goto finish_iomap;
1505 }
Jan Kara1b5a1cb2017-11-01 16:36:36 +01001506 trace_dax_insert_mapping(inode, vmf, entry);
1507 if (write)
Souptick Joarderab77dab2018-06-07 17:04:29 -07001508 ret = vmf_insert_mixed_mkwrite(vma, vaddr, pfn);
Jan Kara1b5a1cb2017-11-01 16:36:36 +01001509 else
Souptick Joarderab77dab2018-06-07 17:04:29 -07001510 ret = vmf_insert_mixed(vma, vaddr, pfn);
Jan Kara1b5a1cb2017-11-01 16:36:36 +01001511
Souptick Joarderab77dab2018-06-07 17:04:29 -07001512 goto finish_iomap;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001513 case IOMAP_UNWRITTEN:
1514 case IOMAP_HOLE:
Jan Karad2c43ef2017-11-01 16:36:35 +01001515 if (!write) {
Souptick Joarderab77dab2018-06-07 17:04:29 -07001516 ret = dax_load_hole(mapping, entry, vmf);
Jan Kara13e451f2017-05-12 15:46:57 -07001517 goto finish_iomap;
Ross Zwisler15502902016-11-08 11:33:26 +11001518 }
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001519 /*FALLTHRU*/
1520 default:
1521 WARN_ON_ONCE(1);
1522 error = -EIO;
1523 break;
1524 }
1525
Jan Kara13e451f2017-05-12 15:46:57 -07001526 error_finish_iomap:
Souptick Joarderab77dab2018-06-07 17:04:29 -07001527 ret = dax_fault_return(error);
Jan Kara9f141d62016-10-19 14:34:31 +02001528 finish_iomap:
1529 if (ops->iomap_end) {
1530 int copied = PAGE_SIZE;
1531
Souptick Joarderab77dab2018-06-07 17:04:29 -07001532 if (ret & VM_FAULT_ERROR)
Jan Kara9f141d62016-10-19 14:34:31 +02001533 copied = 0;
1534 /*
1535 * The fault is done by now and there's no way back (other
1536 * thread may be already happily using PTE we have installed).
1537 * Just ignore error from ->iomap_end since we cannot do much
1538 * with it.
1539 */
1540 ops->iomap_end(inode, pos, PAGE_SIZE, copied, flags, &iomap);
Ross Zwisler15502902016-11-08 11:33:26 +11001541 }
Jan Kara13e451f2017-05-12 15:46:57 -07001542 unlock_entry:
Ross Zwisler91d25ba2017-09-06 16:18:43 -07001543 put_locked_mapping_entry(mapping, vmf->pgoff);
Jan Kara13e451f2017-05-12 15:46:57 -07001544 out:
Souptick Joarderab77dab2018-06-07 17:04:29 -07001545 trace_dax_pte_fault_done(inode, vmf, ret);
1546 return ret | major;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001547}
Ross Zwisler642261a2016-11-08 11:34:45 +11001548
1549#ifdef CONFIG_FS_DAX_PMD
Souptick Joarderab77dab2018-06-07 17:04:29 -07001550static vm_fault_t dax_pmd_load_hole(struct vm_fault *vmf, struct iomap *iomap,
Ross Zwisler91d25ba2017-09-06 16:18:43 -07001551 void *entry)
Ross Zwisler642261a2016-11-08 11:34:45 +11001552{
Dave Jiangf4200392017-02-22 15:40:06 -08001553 struct address_space *mapping = vmf->vma->vm_file->f_mapping;
1554 unsigned long pmd_addr = vmf->address & PMD_MASK;
Ross Zwisler653b2ea2017-02-22 15:39:57 -08001555 struct inode *inode = mapping->host;
Ross Zwisler642261a2016-11-08 11:34:45 +11001556 struct page *zero_page;
Ross Zwisler653b2ea2017-02-22 15:39:57 -08001557 void *ret = NULL;
Ross Zwisler642261a2016-11-08 11:34:45 +11001558 spinlock_t *ptl;
1559 pmd_t pmd_entry;
Dan Williams3fe07912017-10-14 17:13:45 -07001560 pfn_t pfn;
Ross Zwisler642261a2016-11-08 11:34:45 +11001561
Dave Jiangf4200392017-02-22 15:40:06 -08001562 zero_page = mm_get_huge_zero_page(vmf->vma->vm_mm);
Ross Zwisler642261a2016-11-08 11:34:45 +11001563
1564 if (unlikely(!zero_page))
Ross Zwisler653b2ea2017-02-22 15:39:57 -08001565 goto fallback;
Ross Zwisler642261a2016-11-08 11:34:45 +11001566
Dan Williams3fe07912017-10-14 17:13:45 -07001567 pfn = page_to_pfn_t(zero_page);
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -04001568 ret = dax_insert_entry(mapping, vmf, entry, pfn,
Matthew Wilcox3159f942017-11-03 13:30:42 -04001569 DAX_PMD | DAX_ZERO_PAGE, false);
Ross Zwisler642261a2016-11-08 11:34:45 +11001570
Dave Jiangf4200392017-02-22 15:40:06 -08001571 ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);
1572 if (!pmd_none(*(vmf->pmd))) {
Ross Zwisler642261a2016-11-08 11:34:45 +11001573 spin_unlock(ptl);
Ross Zwisler653b2ea2017-02-22 15:39:57 -08001574 goto fallback;
Ross Zwisler642261a2016-11-08 11:34:45 +11001575 }
1576
Dave Jiangf4200392017-02-22 15:40:06 -08001577 pmd_entry = mk_pmd(zero_page, vmf->vma->vm_page_prot);
Ross Zwisler642261a2016-11-08 11:34:45 +11001578 pmd_entry = pmd_mkhuge(pmd_entry);
Dave Jiangf4200392017-02-22 15:40:06 -08001579 set_pmd_at(vmf->vma->vm_mm, pmd_addr, vmf->pmd, pmd_entry);
Ross Zwisler642261a2016-11-08 11:34:45 +11001580 spin_unlock(ptl);
Dave Jiangf4200392017-02-22 15:40:06 -08001581 trace_dax_pmd_load_hole(inode, vmf, zero_page, ret);
Ross Zwisler642261a2016-11-08 11:34:45 +11001582 return VM_FAULT_NOPAGE;
Ross Zwisler653b2ea2017-02-22 15:39:57 -08001583
1584fallback:
Dave Jiangf4200392017-02-22 15:40:06 -08001585 trace_dax_pmd_load_hole_fallback(inode, vmf, zero_page, ret);
Ross Zwisler653b2ea2017-02-22 15:39:57 -08001586 return VM_FAULT_FALLBACK;
Ross Zwisler642261a2016-11-08 11:34:45 +11001587}
1588
Souptick Joarderab77dab2018-06-07 17:04:29 -07001589static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
Dave Jianga2d58162017-02-24 14:56:59 -08001590 const struct iomap_ops *ops)
Ross Zwisler642261a2016-11-08 11:34:45 +11001591{
Dave Jiangf4200392017-02-22 15:40:06 -08001592 struct vm_area_struct *vma = vmf->vma;
Ross Zwisler642261a2016-11-08 11:34:45 +11001593 struct address_space *mapping = vma->vm_file->f_mapping;
Dave Jiangd8a849e2017-02-22 15:40:03 -08001594 unsigned long pmd_addr = vmf->address & PMD_MASK;
1595 bool write = vmf->flags & FAULT_FLAG_WRITE;
Jan Karacaa51d22017-11-01 16:36:42 +01001596 bool sync;
Jan Kara9484ab12016-11-10 10:26:50 +11001597 unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT;
Ross Zwisler642261a2016-11-08 11:34:45 +11001598 struct inode *inode = mapping->host;
Souptick Joarderab77dab2018-06-07 17:04:29 -07001599 vm_fault_t result = VM_FAULT_FALLBACK;
Ross Zwisler642261a2016-11-08 11:34:45 +11001600 struct iomap iomap = { 0 };
1601 pgoff_t max_pgoff, pgoff;
Ross Zwisler642261a2016-11-08 11:34:45 +11001602 void *entry;
1603 loff_t pos;
1604 int error;
Jan Kara302a5e32017-11-01 16:36:37 +01001605 pfn_t pfn;
Ross Zwisler642261a2016-11-08 11:34:45 +11001606
Ross Zwisler282a8e02017-02-22 15:39:50 -08001607 /*
1608 * Check whether offset isn't beyond end of file now. Caller is
1609 * supposed to hold locks serializing us with truncate / punch hole so
1610 * this is a reliable test.
1611 */
1612 pgoff = linear_page_index(vma, pmd_addr);
Jeff Moyer957ac8c2017-11-14 20:37:27 -05001613 max_pgoff = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
Ross Zwisler282a8e02017-02-22 15:39:50 -08001614
Dave Jiangf4200392017-02-22 15:40:06 -08001615 trace_dax_pmd_fault(inode, vmf, max_pgoff, 0);
Ross Zwisler282a8e02017-02-22 15:39:50 -08001616
Ross Zwislerfffa2812017-08-25 15:55:36 -07001617 /*
1618 * Make sure that the faulting address's PMD offset (color) matches
1619 * the PMD offset from the start of the file. This is necessary so
1620 * that a PMD range in the page table overlaps exactly with a PMD
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -04001621 * range in the page cache.
Ross Zwislerfffa2812017-08-25 15:55:36 -07001622 */
1623 if ((vmf->pgoff & PG_PMD_COLOUR) !=
1624 ((vmf->address >> PAGE_SHIFT) & PG_PMD_COLOUR))
1625 goto fallback;
1626
Ross Zwisler642261a2016-11-08 11:34:45 +11001627 /* Fall back to PTEs if we're going to COW */
1628 if (write && !(vma->vm_flags & VM_SHARED))
1629 goto fallback;
1630
1631 /* If the PMD would extend outside the VMA */
1632 if (pmd_addr < vma->vm_start)
1633 goto fallback;
1634 if ((pmd_addr + PMD_SIZE) > vma->vm_end)
1635 goto fallback;
1636
Jeff Moyer957ac8c2017-11-14 20:37:27 -05001637 if (pgoff >= max_pgoff) {
Ross Zwisler282a8e02017-02-22 15:39:50 -08001638 result = VM_FAULT_SIGBUS;
1639 goto out;
1640 }
Ross Zwisler642261a2016-11-08 11:34:45 +11001641
1642 /* If the PMD would extend beyond the file size */
Jeff Moyer957ac8c2017-11-14 20:37:27 -05001643 if ((pgoff | PG_PMD_COLOUR) >= max_pgoff)
Ross Zwisler642261a2016-11-08 11:34:45 +11001644 goto fallback;
1645
1646 /*
Ross Zwisler91d25ba2017-09-06 16:18:43 -07001647 * grab_mapping_entry() will make sure we get a 2MiB empty entry, a
1648 * 2MiB zero page entry or a DAX PMD. If it can't (because a 4k page
1649 * is already in the tree, for instance), it will return -EEXIST and
1650 * we just fall back to 4k entries.
Jan Kara9f141d62016-10-19 14:34:31 +02001651 */
Matthew Wilcox3159f942017-11-03 13:30:42 -04001652 entry = grab_mapping_entry(mapping, pgoff, DAX_PMD);
Jan Kara9f141d62016-10-19 14:34:31 +02001653 if (IS_ERR(entry))
Ross Zwisler876f2942017-05-12 15:47:00 -07001654 goto fallback;
1655
1656 /*
Ross Zwislere2093922017-06-02 14:46:37 -07001657 * It is possible, particularly with mixed reads & writes to private
1658 * mappings, that we have raced with a PTE fault that overlaps with
1659 * the PMD we need to set up. If so just return and the fault will be
1660 * retried.
1661 */
1662 if (!pmd_none(*vmf->pmd) && !pmd_trans_huge(*vmf->pmd) &&
1663 !pmd_devmap(*vmf->pmd)) {
1664 result = 0;
1665 goto unlock_entry;
1666 }
1667
1668 /*
Ross Zwisler876f2942017-05-12 15:47:00 -07001669 * Note that we don't use iomap_apply here. We aren't doing I/O, only
1670 * setting up a mapping, so really we're using iomap_begin() as a way
1671 * to look up our filesystem block.
1672 */
1673 pos = (loff_t)pgoff << PAGE_SHIFT;
1674 error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap);
1675 if (error)
1676 goto unlock_entry;
1677
1678 if (iomap.offset + iomap.length < pos + PMD_SIZE)
Jan Kara9f141d62016-10-19 14:34:31 +02001679 goto finish_iomap;
1680
Dan Williamsaaa422c2017-11-13 16:38:44 -08001681 sync = dax_fault_is_synchronous(iomap_flags, vma, &iomap);
Jan Karacaa51d22017-11-01 16:36:42 +01001682
Ross Zwisler642261a2016-11-08 11:34:45 +11001683 switch (iomap.type) {
1684 case IOMAP_MAPPED:
Jan Kara302a5e32017-11-01 16:36:37 +01001685 error = dax_iomap_pfn(&iomap, pos, PMD_SIZE, &pfn);
1686 if (error < 0)
1687 goto finish_iomap;
1688
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -04001689 entry = dax_insert_entry(mapping, vmf, entry, pfn,
Matthew Wilcox3159f942017-11-03 13:30:42 -04001690 DAX_PMD, write && !sync);
Jan Kara302a5e32017-11-01 16:36:37 +01001691
Jan Karacaa51d22017-11-01 16:36:42 +01001692 /*
1693 * If we are doing synchronous page fault and inode needs fsync,
1694 * we can insert PMD into page tables only after that happens.
1695 * Skip insertion for now and return the pfn so that caller can
1696 * insert it after fsync is done.
1697 */
1698 if (sync) {
1699 if (WARN_ON_ONCE(!pfnp))
1700 goto finish_iomap;
1701 *pfnp = pfn;
1702 result = VM_FAULT_NEEDDSYNC;
1703 goto finish_iomap;
1704 }
1705
Jan Kara302a5e32017-11-01 16:36:37 +01001706 trace_dax_pmd_insert_mapping(inode, vmf, PMD_SIZE, pfn, entry);
1707 result = vmf_insert_pfn_pmd(vma, vmf->address, vmf->pmd, pfn,
1708 write);
Ross Zwisler642261a2016-11-08 11:34:45 +11001709 break;
1710 case IOMAP_UNWRITTEN:
1711 case IOMAP_HOLE:
1712 if (WARN_ON_ONCE(write))
Ross Zwisler876f2942017-05-12 15:47:00 -07001713 break;
Ross Zwisler91d25ba2017-09-06 16:18:43 -07001714 result = dax_pmd_load_hole(vmf, &iomap, entry);
Ross Zwisler642261a2016-11-08 11:34:45 +11001715 break;
1716 default:
1717 WARN_ON_ONCE(1);
1718 break;
1719 }
1720
Jan Kara9f141d62016-10-19 14:34:31 +02001721 finish_iomap:
1722 if (ops->iomap_end) {
1723 int copied = PMD_SIZE;
1724
1725 if (result == VM_FAULT_FALLBACK)
1726 copied = 0;
1727 /*
1728 * The fault is done by now and there's no way back (other
1729 * thread may be already happily using PMD we have installed).
1730 * Just ignore error from ->iomap_end since we cannot do much
1731 * with it.
1732 */
1733 ops->iomap_end(inode, pos, PMD_SIZE, copied, iomap_flags,
1734 &iomap);
1735 }
Ross Zwisler876f2942017-05-12 15:47:00 -07001736 unlock_entry:
Ross Zwisler91d25ba2017-09-06 16:18:43 -07001737 put_locked_mapping_entry(mapping, pgoff);
Ross Zwisler642261a2016-11-08 11:34:45 +11001738 fallback:
1739 if (result == VM_FAULT_FALLBACK) {
Dave Jiangd8a849e2017-02-22 15:40:03 -08001740 split_huge_pmd(vma, vmf->pmd, vmf->address);
Ross Zwisler642261a2016-11-08 11:34:45 +11001741 count_vm_event(THP_FAULT_FALLBACK);
1742 }
Ross Zwisler282a8e02017-02-22 15:39:50 -08001743out:
Dave Jiangf4200392017-02-22 15:40:06 -08001744 trace_dax_pmd_fault_done(inode, vmf, max_pgoff, result);
Ross Zwisler642261a2016-11-08 11:34:45 +11001745 return result;
1746}
Dave Jianga2d58162017-02-24 14:56:59 -08001747#else
Souptick Joarderab77dab2018-06-07 17:04:29 -07001748static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
Arnd Bergmann01cddfe2017-02-27 14:26:44 -08001749 const struct iomap_ops *ops)
Dave Jianga2d58162017-02-24 14:56:59 -08001750{
1751 return VM_FAULT_FALLBACK;
1752}
Ross Zwisler642261a2016-11-08 11:34:45 +11001753#endif /* CONFIG_FS_DAX_PMD */
Dave Jianga2d58162017-02-24 14:56:59 -08001754
1755/**
1756 * dax_iomap_fault - handle a page fault on a DAX file
1757 * @vmf: The description of the fault
Jan Karacec04e82017-11-01 16:36:38 +01001758 * @pe_size: Size of the page to fault in
Jan Kara9a0dd422017-11-01 16:36:39 +01001759 * @pfnp: PFN to insert for synchronous faults if fsync is required
Jan Karac0b24622018-01-07 16:38:43 -05001760 * @iomap_errp: Storage for detailed error code in case of error
Jan Karacec04e82017-11-01 16:36:38 +01001761 * @ops: Iomap ops passed from the file system
Dave Jianga2d58162017-02-24 14:56:59 -08001762 *
1763 * When a page fault occurs, filesystems may call this helper in
1764 * their fault handler for DAX files. dax_iomap_fault() assumes the caller
1765 * has done all the necessary locking for page fault to proceed
1766 * successfully.
1767 */
Souptick Joarderab77dab2018-06-07 17:04:29 -07001768vm_fault_t dax_iomap_fault(struct vm_fault *vmf, enum page_entry_size pe_size,
Jan Karac0b24622018-01-07 16:38:43 -05001769 pfn_t *pfnp, int *iomap_errp, const struct iomap_ops *ops)
Dave Jianga2d58162017-02-24 14:56:59 -08001770{
Dave Jiangc791ace2017-02-24 14:57:08 -08001771 switch (pe_size) {
1772 case PE_SIZE_PTE:
Jan Karac0b24622018-01-07 16:38:43 -05001773 return dax_iomap_pte_fault(vmf, pfnp, iomap_errp, ops);
Dave Jiangc791ace2017-02-24 14:57:08 -08001774 case PE_SIZE_PMD:
Jan Kara9a0dd422017-11-01 16:36:39 +01001775 return dax_iomap_pmd_fault(vmf, pfnp, ops);
Dave Jianga2d58162017-02-24 14:56:59 -08001776 default:
1777 return VM_FAULT_FALLBACK;
1778 }
1779}
1780EXPORT_SYMBOL_GPL(dax_iomap_fault);
Jan Kara71eab6d2017-11-01 16:36:43 +01001781
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -04001782/*
Jan Kara71eab6d2017-11-01 16:36:43 +01001783 * dax_insert_pfn_mkwrite - insert PTE or PMD entry into page tables
1784 * @vmf: The description of the fault
Jan Kara71eab6d2017-11-01 16:36:43 +01001785 * @pfn: PFN to insert
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001786 * @order: Order of entry to insert.
Jan Kara71eab6d2017-11-01 16:36:43 +01001787 *
Matthew Wilcoxa77d19f2018-03-27 13:39:38 -04001788 * This function inserts a writeable PTE or PMD entry into the page tables
1789 * for an mmaped DAX file. It also marks the page cache entry as dirty.
Jan Kara71eab6d2017-11-01 16:36:43 +01001790 */
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001791static vm_fault_t
1792dax_insert_pfn_mkwrite(struct vm_fault *vmf, pfn_t pfn, unsigned int order)
Jan Kara71eab6d2017-11-01 16:36:43 +01001793{
1794 struct address_space *mapping = vmf->vma->vm_file->f_mapping;
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001795 XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, order);
1796 void *entry;
Souptick Joarderab77dab2018-06-07 17:04:29 -07001797 vm_fault_t ret;
Jan Kara71eab6d2017-11-01 16:36:43 +01001798
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001799 xas_lock_irq(&xas);
1800 entry = get_unlocked_entry(&xas);
Jan Kara71eab6d2017-11-01 16:36:43 +01001801 /* Did we race with someone splitting entry or so? */
1802 if (!entry ||
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001803 (order == 0 && !dax_is_pte_entry(entry)) ||
1804 (order == PMD_ORDER && (xa_is_internal(entry) ||
1805 !dax_is_pmd_entry(entry)))) {
1806 put_unlocked_entry(&xas, entry);
1807 xas_unlock_irq(&xas);
Jan Kara71eab6d2017-11-01 16:36:43 +01001808 trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf,
1809 VM_FAULT_NOPAGE);
1810 return VM_FAULT_NOPAGE;
1811 }
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001812 xas_set_mark(&xas, PAGECACHE_TAG_DIRTY);
1813 dax_lock_entry(&xas, entry);
1814 xas_unlock_irq(&xas);
1815 if (order == 0)
Souptick Joarderab77dab2018-06-07 17:04:29 -07001816 ret = vmf_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn);
Jan Kara71eab6d2017-11-01 16:36:43 +01001817#ifdef CONFIG_FS_DAX_PMD
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001818 else if (order == PMD_ORDER)
Souptick Joarderab77dab2018-06-07 17:04:29 -07001819 ret = vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd,
Jan Kara71eab6d2017-11-01 16:36:43 +01001820 pfn, true);
Jan Kara71eab6d2017-11-01 16:36:43 +01001821#endif
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001822 else
Souptick Joarderab77dab2018-06-07 17:04:29 -07001823 ret = VM_FAULT_FALLBACK;
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001824 dax_unlock_entry(&xas, entry);
Souptick Joarderab77dab2018-06-07 17:04:29 -07001825 trace_dax_insert_pfn_mkwrite(mapping->host, vmf, ret);
1826 return ret;
Jan Kara71eab6d2017-11-01 16:36:43 +01001827}
1828
1829/**
1830 * dax_finish_sync_fault - finish synchronous page fault
1831 * @vmf: The description of the fault
1832 * @pe_size: Size of entry to be inserted
1833 * @pfn: PFN to insert
1834 *
1835 * This function ensures that the file range touched by the page fault is
1836 * stored persistently on the media and handles inserting of appropriate page
1837 * table entry.
1838 */
Souptick Joarderab77dab2018-06-07 17:04:29 -07001839vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf,
1840 enum page_entry_size pe_size, pfn_t pfn)
Jan Kara71eab6d2017-11-01 16:36:43 +01001841{
1842 int err;
1843 loff_t start = ((loff_t)vmf->pgoff) << PAGE_SHIFT;
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001844 unsigned int order = pe_order(pe_size);
1845 size_t len = PAGE_SIZE << order;
Jan Kara71eab6d2017-11-01 16:36:43 +01001846
Jan Kara71eab6d2017-11-01 16:36:43 +01001847 err = vfs_fsync_range(vmf->vma->vm_file, start, start + len - 1, 1);
1848 if (err)
1849 return VM_FAULT_SIGBUS;
Matthew Wilcoxcfc93c62018-03-28 11:48:03 -04001850 return dax_insert_pfn_mkwrite(vmf, pfn, order);
Jan Kara71eab6d2017-11-01 16:36:43 +01001851}
1852EXPORT_SYMBOL_GPL(dax_finish_sync_fault);