dax.c 45.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * fs/dax.c - Direct Access filesystem code
 * Copyright (c) 2013-2014 Intel Corporation
 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com>
 * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

#include <linux/atomic.h>
#include <linux/blkdev.h>
#include <linux/buffer_head.h>
20
#include <linux/dax.h>
21 22
#include <linux/fs.h>
#include <linux/genhd.h>
23 24 25
#include <linux/highmem.h>
#include <linux/memcontrol.h>
#include <linux/mm.h>
26
#include <linux/mutex.h>
27
#include <linux/pagevec.h>
28
#include <linux/sched.h>
29
#include <linux/sched/signal.h>
30
#include <linux/uio.h>
31
#include <linux/vmstat.h>
32
#include <linux/pfn_t.h>
33
#include <linux/sizes.h>
34
#include <linux/mmu_notifier.h>
35 36
#include <linux/iomap.h>
#include "internal.h"
37

38 39 40
#define CREATE_TRACE_POINTS
#include <trace/events/fs_dax.h>

41 42 43 44 45 46 47 48 49 50 51
static inline unsigned int pe_order(enum page_entry_size pe_size)
{
	if (pe_size == PE_SIZE_PTE)
		return PAGE_SHIFT - PAGE_SHIFT;
	if (pe_size == PE_SIZE_PMD)
		return PMD_SHIFT - PAGE_SHIFT;
	if (pe_size == PE_SIZE_PUD)
		return PUD_SHIFT - PAGE_SHIFT;
	return ~0;
}

Jan Kara's avatar
Jan Kara committed
52 53 54 55
/* We choose 4096 entries - same as per-zone page wait tables */
#define DAX_WAIT_TABLE_BITS 12
#define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)

56 57
/* The 'colour' (ie low bits) within a PMD of a page offset.  */
#define PG_PMD_COLOUR	((PMD_SIZE >> PAGE_SHIFT) - 1)
58
#define PG_PMD_NR	(PMD_SIZE >> PAGE_SHIFT)
59

60 61 62
/* The order of a PMD entry */
#define PMD_ORDER	(PMD_SHIFT - PAGE_SHIFT)

63
static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
Jan Kara's avatar
Jan Kara committed
64 65 66 67 68 69 70 71 72 73 74

static int __init init_dax_wait_table(void)
{
	int i;

	for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++)
		init_waitqueue_head(wait_table + i);
	return 0;
}
fs_initcall(init_dax_wait_table);

75
/*
76 77 78 79
 * DAX pagecache entries use XArray value entries so they can't be mistaken
 * for pages.  We use one bit for locking, one bit for the entry size (PMD)
 * and two more to tell us if the entry is a zero page or an empty entry that
 * is just used for locking.  In total four special bits.
80 81 82 83 84
 *
 * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE
 * and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem
 * block allocation.
 */
85 86 87 88 89
#define DAX_SHIFT	(4)
#define DAX_LOCKED	(1UL << 0)
#define DAX_PMD		(1UL << 1)
#define DAX_ZERO_PAGE	(1UL << 2)
#define DAX_EMPTY	(1UL << 3)
90

91
static unsigned long dax_to_pfn(void *entry)
92
{
93
	return xa_to_value(entry) >> DAX_SHIFT;
94 95
}

96 97 98 99 100
static void *dax_make_entry(pfn_t pfn, unsigned long flags)
{
	return xa_mk_value(flags | (pfn_t_to_pfn(pfn) << DAX_SHIFT));
}

101 102 103 104 105
static bool dax_is_locked(void *entry)
{
	return xa_to_value(entry) & DAX_LOCKED;
}

106
static unsigned int dax_entry_order(void *entry)
107
{
108
	if (xa_to_value(entry) & DAX_PMD)
109
		return PMD_ORDER;
110 111 112
	return 0;
}

113
static unsigned long dax_is_pmd_entry(void *entry)
114
{
115
	return xa_to_value(entry) & DAX_PMD;
116 117
}

118
static bool dax_is_pte_entry(void *entry)
119
{
120
	return !(xa_to_value(entry) & DAX_PMD);
121 122
}

123
static int dax_is_zero_entry(void *entry)
124
{
125
	return xa_to_value(entry) & DAX_ZERO_PAGE;
126 127
}

128
static int dax_is_empty_entry(void *entry)
129
{
130
	return xa_to_value(entry) & DAX_EMPTY;
131 132
}

Jan Kara's avatar
Jan Kara committed
133
/*
134
 * DAX page cache entry locking
Jan Kara's avatar
Jan Kara committed
135 136
 */
struct exceptional_entry_key {
137
	struct xarray *xa;
138
	pgoff_t entry_start;
Jan Kara's avatar
Jan Kara committed
139 140 141
};

struct wait_exceptional_entry_queue {
142
	wait_queue_entry_t wait;
Jan Kara's avatar
Jan Kara committed
143 144 145
	struct exceptional_entry_key key;
};

146 147
static wait_queue_head_t *dax_entry_waitqueue(struct xa_state *xas,
		void *entry, struct exceptional_entry_key *key)
148 149
{
	unsigned long hash;
150
	unsigned long index = xas->xa_index;
151 152 153 154 155 156

	/*
	 * If 'entry' is a PMD, align the 'index' that we use for the wait
	 * queue to the start of that PMD.  This ensures that all offsets in
	 * the range covered by the PMD map to the same bit lock.
	 */
157
	if (dax_is_pmd_entry(entry))
158
		index &= ~PG_PMD_COLOUR;
159
	key->xa = xas->xa;
160 161
	key->entry_start = index;

162
	hash = hash_long((unsigned long)xas->xa ^ index, DAX_WAIT_TABLE_BITS);
163 164 165
	return wait_table + hash;
}

166 167
static int wake_exceptional_entry_func(wait_queue_entry_t *wait,
		unsigned int mode, int sync, void *keyp)
Jan Kara's avatar
Jan Kara committed
168 169 170 171 172
{
	struct exceptional_entry_key *key = keyp;
	struct wait_exceptional_entry_queue *ewait =
		container_of(wait, struct wait_exceptional_entry_queue, wait);

173
	if (key->xa != ewait->key.xa ||
174
	    key->entry_start != ewait->key.entry_start)
Jan Kara's avatar
Jan Kara committed
175 176 177 178
		return 0;
	return autoremove_wake_function(wait, mode, sync, NULL);
}

179
/*
Matthew Wilcox's avatar
Matthew Wilcox committed
180 181 182
 * @entry may no longer be the entry at the index in the mapping.
 * The important information it's conveying is whether the entry at
 * this index used to be a PMD entry.
183
 */
184
static void dax_wake_entry(struct xa_state *xas, void *entry, bool wake_all)
185 186 187 188
{
	struct exceptional_entry_key key;
	wait_queue_head_t *wq;

189
	wq = dax_entry_waitqueue(xas, entry, &key);
190 191 192

	/*
	 * Checking for locked entry and prepare_to_wait_exclusive() happens
Matthew Wilcox's avatar
Matthew Wilcox committed
193
	 * under the i_pages lock, ditto for entry handling in our callers.
194 195 196 197 198 199 200
	 * So at this point all tasks that could have seen our entry locked
	 * must be in the waitqueue and the following check will see them.
	 */
	if (waitqueue_active(wq))
		__wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key);
}

201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
/*
 * Look up entry in page cache, wait for it to become unlocked if it
 * is a DAX entry and return it.  The caller must subsequently call
 * put_unlocked_entry() if it did not lock the entry or dax_unlock_entry()
 * if it did.
 *
 * Must be called with the i_pages lock held.
 */
static void *get_unlocked_entry(struct xa_state *xas)
{
	void *entry;
	struct wait_exceptional_entry_queue ewait;
	wait_queue_head_t *wq;

	init_wait(&ewait.wait);
	ewait.wait.func = wake_exceptional_entry_func;

	for (;;) {
219 220
		entry = xas_find_conflict(xas);
		if (!entry || WARN_ON_ONCE(!xa_is_value(entry)) ||
221 222 223
				!dax_is_locked(entry))
			return entry;

224
		wq = dax_entry_waitqueue(xas, entry, &ewait.key);
225 226 227 228 229 230 231 232 233 234
		prepare_to_wait_exclusive(wq, &ewait.wait,
					  TASK_UNINTERRUPTIBLE);
		xas_unlock_irq(xas);
		xas_reset(xas);
		schedule();
		finish_wait(wq, &ewait.wait);
		xas_lock_irq(xas);
	}
}

235 236 237 238 239 240 241 242 243 244 245 246 247 248
/*
 * The only thing keeping the address space around is the i_pages lock
 * (it's cycled in clear_inode() after removing the entries from i_pages)
 * After we call xas_unlock_irq(), we cannot touch xas->xa.
 */
static void wait_entry_unlocked(struct xa_state *xas, void *entry)
{
	struct wait_exceptional_entry_queue ewait;
	wait_queue_head_t *wq;

	init_wait(&ewait.wait);
	ewait.wait.func = wake_exceptional_entry_func;

	wq = dax_entry_waitqueue(xas, entry, &ewait.key);
249 250 251 252 253 254 255
	/*
	 * Unlike get_unlocked_entry() there is no guarantee that this
	 * path ever successfully retrieves an unlocked entry before an
	 * inode dies. Perform a non-exclusive wait in case this path
	 * never successfully performs its own wake up.
	 */
	prepare_to_wait(wq, &ewait.wait, TASK_UNINTERRUPTIBLE);
256 257 258 259 260
	xas_unlock_irq(xas);
	schedule();
	finish_wait(wq, &ewait.wait);
}

261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276
static void put_unlocked_entry(struct xa_state *xas, void *entry)
{
	/* If we were the only waiter woken, wake the next one */
	if (entry)
		dax_wake_entry(xas, entry, false);
}

/*
 * We used the xa_state to get the entry, but then we locked the entry and
 * dropped the xa_lock, so we know the xa_state is stale and must be reset
 * before use.
 */
static void dax_unlock_entry(struct xa_state *xas, void *entry)
{
	void *old;

277
	BUG_ON(dax_is_locked(entry));
278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294
	xas_reset(xas);
	xas_lock_irq(xas);
	old = xas_store(xas, entry);
	xas_unlock_irq(xas);
	BUG_ON(!dax_is_locked(old));
	dax_wake_entry(xas, entry, false);
}

/*
 * Return: The entry stored at this location before it was locked.
 */
static void *dax_lock_entry(struct xa_state *xas, void *entry)
{
	unsigned long v = xa_to_value(entry);
	return xas_store(xas, xa_mk_value(v | DAX_LOCKED));
}

295 296 297 298 299 300 301 302 303 304 305 306
static unsigned long dax_entry_size(void *entry)
{
	if (dax_is_zero_entry(entry))
		return 0;
	else if (dax_is_empty_entry(entry))
		return 0;
	else if (dax_is_pmd_entry(entry))
		return PMD_SIZE;
	else
		return PAGE_SIZE;
}

307
static unsigned long dax_end_pfn(void *entry)
308
{
309
	return dax_to_pfn(entry) + dax_entry_size(entry) / PAGE_SIZE;
310 311 312 313 314 315 316
}

/*
 * Iterate through all mapped pfns represented by an entry, i.e. skip
 * 'empty' and 'zero' entries.
 */
#define for_each_mapped_pfn(entry, pfn) \
317 318
	for (pfn = dax_to_pfn(entry); \
			pfn < dax_end_pfn(entry); pfn++)
319

320 321 322 323 324 325 326
/*
 * TODO: for reflink+dax we need a way to associate a single page with
 * multiple address_space instances at different linear_page_index()
 * offsets.
 */
static void dax_associate_entry(void *entry, struct address_space *mapping,
		struct vm_area_struct *vma, unsigned long address)
327
{
328 329
	unsigned long size = dax_entry_size(entry), pfn, index;
	int i = 0;
330 331 332 333

	if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
		return;

334
	index = linear_page_index(vma, address & ~(size - 1));
335 336 337 338 339
	for_each_mapped_pfn(entry, pfn) {
		struct page *page = pfn_to_page(pfn);

		WARN_ON_ONCE(page->mapping);
		page->mapping = mapping;
340
		page->index = index + i++;
341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357
	}
}

static void dax_disassociate_entry(void *entry, struct address_space *mapping,
		bool trunc)
{
	unsigned long pfn;

	if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
		return;

	for_each_mapped_pfn(entry, pfn) {
		struct page *page = pfn_to_page(pfn);

		WARN_ON_ONCE(trunc && page_ref_count(page) > 1);
		WARN_ON_ONCE(page->mapping && page->mapping != mapping);
		page->mapping = NULL;
358
		page->index = 0;
359 360 361
	}
}

362 363 364 365 366 367 368 369 370 371 372 373 374
static struct page *dax_busy_page(void *entry)
{
	unsigned long pfn;

	for_each_mapped_pfn(entry, pfn) {
		struct page *page = pfn_to_page(pfn);

		if (page_ref_count(page) > 1)
			return page;
	}
	return NULL;
}

375 376 377 378 379
/*
 * dax_lock_mapping_entry - Lock the DAX entry corresponding to a page
 * @page: The page whose entry we want to lock
 *
 * Context: Process context.
380 381
 * Return: A cookie to pass to dax_unlock_page() or 0 if the entry could
 * not be locked.
382
 */
383
dax_entry_t dax_lock_page(struct page *page)
384
{
385 386
	XA_STATE(xas, NULL, 0);
	void *entry;
387

388 389
	/* Ensure page->mapping isn't freed while we look at it */
	rcu_read_lock();
390
	for (;;) {
391
		struct address_space *mapping = READ_ONCE(page->mapping);
392

393
		entry = NULL;
394
		if (!mapping || !dax_mapping(mapping))
395
			break;
396 397 398 399 400 401 402 403

		/*
		 * In the device-dax case there's no need to lock, a
		 * struct dev_pagemap pin is sufficient to keep the
		 * inode alive, and we assume we have dev_pagemap pin
		 * otherwise we would not have a valid pfn_to_page()
		 * translation.
		 */
404
		entry = (void *)~0UL;
405
		if (S_ISCHR(mapping->host->i_mode))
406
			break;
407

408 409
		xas.xa = &mapping->i_pages;
		xas_lock_irq(&xas);
410
		if (mapping != page->mapping) {
411
			xas_unlock_irq(&xas);
412 413
			continue;
		}
414 415 416
		xas_set(&xas, page->index);
		entry = xas_load(&xas);
		if (dax_is_locked(entry)) {
417
			rcu_read_unlock();
418
			wait_entry_unlocked(&xas, entry);
419
			rcu_read_lock();
420
			continue;
421
		}
422 423
		dax_lock_entry(&xas, entry);
		xas_unlock_irq(&xas);
424
		break;
425
	}
426
	rcu_read_unlock();
427
	return (dax_entry_t)entry;
428 429
}

430
void dax_unlock_page(struct page *page, dax_entry_t cookie)
431 432
{
	struct address_space *mapping = page->mapping;
433
	XA_STATE(xas, &mapping->i_pages, page->index);
434

435
	if (S_ISCHR(mapping->host->i_mode))
436 437
		return;

438
	dax_unlock_entry(&xas, (void *)cookie);
439 440
}

Jan Kara's avatar
Jan Kara committed
441
/*
442 443 444
 * Find page cache entry at given index. If it is a DAX entry, return it
 * with the entry locked. If the page cache doesn't contain an entry at
 * that index, add a locked empty entry.
Jan Kara's avatar
Jan Kara committed
445
 *
446
 * When requesting an entry with size DAX_PMD, grab_mapping_entry() will
447 448 449
 * either return that locked entry or will return VM_FAULT_FALLBACK.
 * This will happen if there are any PTE entries within the PMD range
 * that we are requesting.
450
 *
451 452 453 454 455 456
 * We always favor PTE entries over PMD entries. There isn't a flow where we
 * evict PTE entries in order to 'upgrade' them to a PMD entry.  A PMD
 * insertion will fail if it finds any PTE entries already in the tree, and a
 * PTE insertion will cause an existing PMD entry to be unmapped and
 * downgraded to PTE entries.  This happens for both PMD zero pages as
 * well as PMD empty entries.
457
 *
458 459 460
 * The exception to this downgrade path is for PMD entries that have
 * real storage backing them.  We will leave these real PMD entries in
 * the tree, and PTE writes will simply dirty the entire PMD entry.
461
 *
Jan Kara's avatar
Jan Kara committed
462 463 464
 * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For
 * persistent memory the benefit is doubtful. We can add that later if we can
 * show it helps.
465 466 467 468
 *
 * On error, this function does not return an ERR_PTR.  Instead it returns
 * a VM_FAULT code, encoded as an xarray internal entry.  The ERR_PTR values
 * overlap with xarray value entries.
Jan Kara's avatar
Jan Kara committed
469
 */
470 471
static void *grab_mapping_entry(struct xa_state *xas,
		struct address_space *mapping, unsigned long size_flag)
Jan Kara's avatar
Jan Kara committed
472
{
473 474 475
	unsigned long index = xas->xa_index;
	bool pmd_downgrade = false; /* splitting PMD entry into PTE entries? */
	void *entry;
476

477 478 479
retry:
	xas_lock_irq(xas);
	entry = get_unlocked_entry(xas);
480

481
	if (entry) {
482
		if (!xa_is_value(entry)) {
483 484 485 486
			xas_set_err(xas, EIO);
			goto out_unlock;
		}

487
		if (size_flag & DAX_PMD) {
488
			if (dax_is_pte_entry(entry)) {
489 490
				put_unlocked_entry(xas, entry);
				goto fallback;
491 492
			}
		} else { /* trying to grab a PTE entry */
493
			if (dax_is_pmd_entry(entry) &&
494 495 496 497 498 499 500
			    (dax_is_zero_entry(entry) ||
			     dax_is_empty_entry(entry))) {
				pmd_downgrade = true;
			}
		}
	}

501 502 503 504 505 506
	if (pmd_downgrade) {
		/*
		 * Make sure 'entry' remains valid while we drop
		 * the i_pages lock.
		 */
		dax_lock_entry(xas, entry);
507 508 509 510 511 512

		/*
		 * Besides huge zero pages the only other thing that gets
		 * downgraded are empty entries which don't need to be
		 * unmapped.
		 */
513 514 515 516 517 518 519
		if (dax_is_zero_entry(entry)) {
			xas_unlock_irq(xas);
			unmap_mapping_pages(mapping,
					xas->xa_index & ~PG_PMD_COLOUR,
					PG_PMD_NR, false);
			xas_reset(xas);
			xas_lock_irq(xas);
520 521
		}

522 523 524 525 526 527 528
		dax_disassociate_entry(entry, mapping, false);
		xas_store(xas, NULL);	/* undo the PMD join */
		dax_wake_entry(xas, entry, true);
		mapping->nrexceptional--;
		entry = NULL;
		xas_set(xas, index);
	}
529

530 531 532 533 534 535 536
	if (entry) {
		dax_lock_entry(xas, entry);
	} else {
		entry = dax_make_entry(pfn_to_pfn_t(0), size_flag | DAX_EMPTY);
		dax_lock_entry(xas, entry);
		if (xas_error(xas))
			goto out_unlock;
Jan Kara's avatar
Jan Kara committed
537 538
		mapping->nrexceptional++;
	}
539 540 541 542 543 544 545 546 547

out_unlock:
	xas_unlock_irq(xas);
	if (xas_nomem(xas, mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM))
		goto retry;
	if (xas->xa_node == XA_ERROR(-ENOMEM))
		return xa_mk_internal(VM_FAULT_OOM);
	if (xas_error(xas))
		return xa_mk_internal(VM_FAULT_SIGBUS);
548
	return entry;
549 550 551
fallback:
	xas_unlock_irq(xas);
	return xa_mk_internal(VM_FAULT_FALLBACK);
Jan Kara's avatar
Jan Kara committed
552 553
}

554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570
/**
 * dax_layout_busy_page - find first pinned page in @mapping
 * @mapping: address space to scan for a page with ref count > 1
 *
 * DAX requires ZONE_DEVICE mapped pages. These pages are never
 * 'onlined' to the page allocator so they are considered idle when
 * page->count == 1. A filesystem uses this interface to determine if
 * any page in the mapping is busy, i.e. for DMA, or other
 * get_user_pages() usages.
 *
 * It is expected that the filesystem is holding locks to block the
 * establishment of new mappings in this address_space. I.e. it expects
 * to be able to run unmap_mapping_range() and subsequently not race
 * mapping_mapped() becoming true.
 */
struct page *dax_layout_busy_page(struct address_space *mapping)
{
571 572 573
	XA_STATE(xas, &mapping->i_pages, 0);
	void *entry;
	unsigned int scanned = 0;
574 575 576 577 578 579 580 581 582 583 584 585 586
	struct page *page = NULL;

	/*
	 * In the 'limited' case get_user_pages() for dax is disabled.
	 */
	if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
		return NULL;

	if (!dax_mapping(mapping) || !mapping_mapped(mapping))
		return NULL;

	/*
	 * If we race get_user_pages_fast() here either we'll see the
587
	 * elevated page count in the iteration and wait, or
588 589 590 591 592 593 594 595 596 597 598
	 * get_user_pages_fast() will see that the page it took a reference
	 * against is no longer mapped in the page tables and bail to the
	 * get_user_pages() slow path.  The slow path is protected by
	 * pte_lock() and pmd_lock(). New references are not taken without
	 * holding those locks, and unmap_mapping_range() will not zero the
	 * pte or pmd without holding the respective lock, so we are
	 * guaranteed to either see new references or prevent new
	 * references from being established.
	 */
	unmap_mapping_range(mapping, 0, 0, 1);

599 600 601 602 603 604 605 606 607
	xas_lock_irq(&xas);
	xas_for_each(&xas, entry, ULONG_MAX) {
		if (WARN_ON_ONCE(!xa_is_value(entry)))
			continue;
		if (unlikely(dax_is_locked(entry)))
			entry = get_unlocked_entry(&xas);
		if (entry)
			page = dax_busy_page(entry);
		put_unlocked_entry(&xas, entry);
608 609
		if (page)
			break;
610 611 612 613 614 615 616
		if (++scanned % XA_CHECK_SCHED)
			continue;

		xas_pause(&xas);
		xas_unlock_irq(&xas);
		cond_resched();
		xas_lock_irq(&xas);
617
	}
618
	xas_unlock_irq(&xas);
619 620 621 622
	return page;
}
EXPORT_SYMBOL_GPL(dax_layout_busy_page);

623
static int __dax_invalidate_entry(struct address_space *mapping,
624 625
					  pgoff_t index, bool trunc)
{
626
	XA_STATE(xas, &mapping->i_pages, index);
627 628 629
	int ret = 0;
	void *entry;

630 631
	xas_lock_irq(&xas);
	entry = get_unlocked_entry(&xas);
632
	if (!entry || WARN_ON_ONCE(!xa_is_value(entry)))
633 634
		goto out;
	if (!trunc &&
635 636
	    (xas_get_mark(&xas, PAGECACHE_TAG_DIRTY) ||
	     xas_get_mark(&xas, PAGECACHE_TAG_TOWRITE)))
637
		goto out;
638
	dax_disassociate_entry(entry, mapping, trunc);
639
	xas_store(&xas, NULL);
640 641 642
	mapping->nrexceptional--;
	ret = 1;
out:
643 644
	put_unlocked_entry(&xas, entry);
	xas_unlock_irq(&xas);
645 646
	return ret;
}
647

Jan Kara's avatar
Jan Kara committed
648
/*
649 650
 * Delete DAX entry at @index from @mapping.  Wait for it
 * to be unlocked before deleting it.
Jan Kara's avatar
Jan Kara committed
651 652 653
 */
int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)
{
654
	int ret = __dax_invalidate_entry(mapping, index, true);
Jan Kara's avatar
Jan Kara committed
655 656 657 658

	/*
	 * This gets called from truncate / punch_hole path. As such, the caller
	 * must hold locks protecting against concurrent modifications of the
659
	 * page cache (usually fs-private i_mmap_sem for writing). Since the
660
	 * caller has seen a DAX entry for this index, we better find it
Jan Kara's avatar
Jan Kara committed
661 662
	 * at that index as well...
	 */
663 664 665 666 667
	WARN_ON_ONCE(!ret);
	return ret;
}

/*
668
 * Invalidate DAX entry if it is clean.
669 670 671 672
 */
int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
				      pgoff_t index)
{
673
	return __dax_invalidate_entry(mapping, index, false);
Jan Kara's avatar
Jan Kara committed
674 675
}

676 677 678
static int copy_user_dax(struct block_device *bdev, struct dax_device *dax_dev,
		sector_t sector, size_t size, struct page *to,
		unsigned long vaddr)
679
{
680 681 682 683 684 685 686 687 688 689
	void *vto, *kaddr;
	pgoff_t pgoff;
	long rc;
	int id;

	rc = bdev_dax_pgoff(bdev, sector, size, &pgoff);
	if (rc)
		return rc;

	id = dax_read_lock();
690
	rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, NULL);
691 692 693 694
	if (rc < 0) {
		dax_read_unlock(id);
		return rc;
	}
695
	vto = kmap_atomic(to);
696
	copy_user_page(vto, (void __force *)kaddr, vaddr, to);
697
	kunmap_atomic(vto);
698
	dax_read_unlock(id);
699 700 701
	return 0;
}

702 703 704 705 706 707 708
/*
 * By this point grab_mapping_entry() has ensured that we have a locked entry
 * of the appropriate size so we don't have to worry about downgrading PMDs to
 * PTEs.  If we happen to be trying to insert a PTE and there is a PMD
 * already in the tree, we will skip the insertion and just dirty the PMD as
 * appropriate.
 */
709 710 711
static void *dax_insert_entry(struct xa_state *xas,
		struct address_space *mapping, struct vm_fault *vmf,
		void *entry, pfn_t pfn, unsigned long flags, bool dirty)
712
{
713
	void *new_entry = dax_make_entry(pfn, flags);
714

715
	if (dirty)
716
		__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
717

718
	if (dax_is_zero_entry(entry) && !(flags & DAX_ZERO_PAGE)) {
719
		unsigned long index = xas->xa_index;
720 721
		/* we are replacing a zero page with block mapping */
		if (dax_is_pmd_entry(entry))
722
			unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR,
723
					PG_PMD_NR, false);
724
		else /* pte entry */
725
			unmap_mapping_pages(mapping, index, 1, false);
726 727
	}

728 729
	xas_reset(xas);
	xas_lock_irq(xas);
730 731
	if (dax_entry_size(entry) != dax_entry_size(new_entry)) {
		dax_disassociate_entry(entry, mapping, false);
732
		dax_associate_entry(new_entry, mapping, vmf->vma, vmf->address);
733
	}
734

735
	if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) {
736
		/*
737
		 * Only swap our new entry into the page cache if the current
738
		 * entry is a zero page or an empty entry.  If a normal PTE or
739
		 * PMD entry is already in the cache, we leave it alone.  This
740 741 742 743
		 * means that if we are trying to insert a PTE and the
		 * existing entry is a PMD, we will just leave the PMD in the
		 * tree and dirty it if necessary.
		 */
744 745 746
		void *old = dax_lock_entry(xas, new_entry);
		WARN_ON_ONCE(old != xa_mk_value(xa_to_value(entry) |
					DAX_LOCKED));
747
		entry = new_entry;
748 749
	} else {
		xas_load(xas);	/* Walk the xa_state */
750
	}
751

752
	if (dirty)
753
		xas_set_mark(xas, PAGECACHE_TAG_DIRTY);
754

755
	xas_unlock_irq(xas);
756
	return entry;
757 758
}

759 760
static inline
unsigned long pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma)
761 762 763 764 765 766 767 768 769
{
	unsigned long address;

	address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
	VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
	return address;
}

/* Walk all mappings of a given index of a file and writeprotect them */
770 771
static void dax_entry_mkclean(struct address_space *mapping, pgoff_t index,
		unsigned long pfn)
772 773
{
	struct vm_area_struct *vma;
774 775
	pte_t pte, *ptep = NULL;
	pmd_t *pmdp = NULL;
776 777 778 779
	spinlock_t *ptl;

	i_mmap_lock_read(mapping);
	vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) {
780 781
		struct mmu_notifier_range range;
		unsigned long address;
782 783 784 785 786 787 788

		cond_resched();

		if (!(vma->vm_flags & VM_SHARED))
			continue;

		address = pgoff_address(index, vma);
789 790 791 792 793 794

		/*
		 * Note because we provide start/end to follow_pte_pmd it will
		 * call mmu_notifier_invalidate_range_start() on our behalf
		 * before taking any lock.
		 */
795 796
		if (follow_pte_pmd(vma->vm_mm, address, &range,
				   &ptep, &pmdp, &ptl))
797 798
			continue;

799 800 801 802 803
		/*
		 * No need to call mmu_notifier_invalidate_range() as we are
		 * downgrading page table protection not changing it to point
		 * to a new page.
		 *
804
		 * See Documentation/vm/mmu_notifier.rst
805
		 */
806 807 808 809 810 811
		if (pmdp) {
#ifdef CONFIG_FS_DAX_PMD
			pmd_t pmd;

			if (pfn != pmd_pfn(*pmdp))
				goto unlock_pmd;
812
			if (!pmd_dirty(*pmdp) && !pmd_write(*pmdp))
813 814 815 816 817 818 819 820 821
				goto unlock_pmd;

			flush_cache_page(vma, address, pfn);
			pmd = pmdp_huge_clear_flush(vma, address, pmdp);
			pmd = pmd_wrprotect(pmd);
			pmd = pmd_mkclean(pmd);
			set_pmd_at(vma->vm_mm, address, pmdp, pmd);
unlock_pmd:
#endif
822
			spin_unlock(ptl);
823 824 825 826 827 828 829 830 831 832 833 834 835 836
		} else {
			if (pfn != pte_pfn(*ptep))
				goto unlock_pte;
			if (!pte_dirty(*ptep) && !pte_write(*ptep))
				goto unlock_pte;

			flush_cache_page(vma, address, pfn);
			pte = ptep_clear_flush(vma, address, ptep);
			pte = pte_wrprotect(pte);
			pte = pte_mkclean(pte);
			set_pte_at(vma->vm_mm, address, ptep, pte);
unlock_pte:
			pte_unmap_unlock(ptep, ptl);
		}
837

838
		mmu_notifier_invalidate_range_end(&range);
839 840 841 842
	}
	i_mmap_unlock_read(mapping);
}

843 844
static int dax_writeback_one(struct xa_state *xas, struct dax_device *dax_dev,
		struct address_space *mapping, void *entry)
845
{
846 847
	unsigned long pfn;
	long ret = 0;
848
	size_t size;
849 850

	/*
851 852
	 * A page got tagged dirty in DAX mapping? Something is seriously
	 * wrong.
853
	 */
854
	if (WARN_ON(!xa_is_value(entry)))
855
		return -EIO;
856

857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880
	if (unlikely(dax_is_locked(entry))) {
		void *old_entry = entry;

		entry = get_unlocked_entry(xas);

		/* Entry got punched out / reallocated? */
		if (!entry || WARN_ON_ONCE(!xa_is_value(entry)))
			goto put_unlocked;
		/*
		 * Entry got reallocated elsewhere? No need to writeback.
		 * We have to compare pfns as we must not bail out due to
		 * difference in lockbit or entry type.
		 */
		if (dax_to_pfn(old_entry) != dax_to_pfn(entry))
			goto put_unlocked;
		if (WARN_ON_ONCE(dax_is_empty_entry(entry) ||
					dax_is_zero_entry(entry))) {
			ret = -EIO;
			goto put_unlocked;
		}

		/* Another fsync thread may have already done this entry */
		if (!xas_get_mark(xas, PAGECACHE_TAG_TOWRITE))
			goto put_unlocked;
881 882
	}

883
	/* Lock the entry to serialize with page faults */
884 885
	dax_lock_entry(xas, entry);

886 887 888 889
	/*
	 * We can clear the tag now but we have to be careful so that concurrent
	 * dax_writeback_one() calls for the same index cannot finish before we
	 * actually flush the caches. This is achieved as the calls will look
Matthew Wilcox's avatar
Matthew Wilcox committed
890 891
	 * at the entry only under the i_pages lock and once they do that
	 * they will see the entry locked and wait for it to unlock.
892
	 */
893 894
	xas_clear_mark(xas, PAGECACHE_TAG_TOWRITE);
	xas_unlock_irq(xas);
895

896 897 898
	/*
	 * Even if dax_writeback_mapping_range() was given a wbc->range_start
	 * in the middle of a PMD, the 'index' we are given will be aligned to
899 900 901
	 * the start index of the PMD, as will the pfn we pull from 'entry'.
	 * This allows us to flush for PMD_SIZE and not have to worry about
	 * partial PMD writebacks.
902
	 */
903 904
	pfn = dax_to_pfn(entry);
	size = PAGE_SIZE << dax_entry_order(entry);
905

906
	dax_entry_mkclean(mapping, xas->xa_index, pfn);
907
	dax_flush(dax_dev, page_address(pfn_to_page(pfn)), size);
908 909 910 911 912 913
	/*
	 * After we have flushed the cache, we can clear the dirty tag. There
	 * cannot be new dirty data in the pfn after the flush has completed as
	 * the pfn mappings are writeprotected and fault waits for mapping
	 * entry lock.
	 */
914 915 916 917 918 919 920 921
	xas_reset(xas);
	xas_lock_irq(xas);
	xas_store(xas, entry);
	xas_clear_mark(xas, PAGECACHE_TAG_DIRTY);
	dax_wake_entry(xas, entry, false);

	trace_dax_writeback_one(mapping->host, xas->xa_index,
			size >> PAGE_SHIFT);
922 923
	return ret;

924
 put_unlocked:
925
	put_unlocked_entry(xas, entry);
926 927 928 929 930 931 932 933
	return ret;
}

/*
 * Flush the mapping to the persistent domain within the byte range of [start,
 * end]. This is required by data integrity operations to ensure file data is
 * on persistent storage prior to completion of the operation.
 */
934 935
int dax_writeback_mapping_range(struct address_space *mapping,
		struct block_device *bdev, struct writeback_control *wbc)
936
{
937
	XA_STATE(xas, &mapping->i_pages, wbc->range_start >> PAGE_SHIFT);
938
	struct inode *inode = mapping->host;
939
	pgoff_t end_index = wbc->range_end >> PAGE_SHIFT;
940
	struct dax_device *dax_dev;
941 942 943
	void *entry;
	int ret = 0;
	unsigned int scanned = 0;
944 945 946 947

	if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
		return -EIO;

948 949 950
	if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL)
		return 0;

951 952 953 954
	dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
	if (!dax_dev)
		return -EIO;

955
	trace_dax_writeback_range(inode, xas.xa_index, end_index);
956

957
	tag_pages_for_writeback(mapping, xas.xa_index, end_index);
958

959 960 961 962 963
	xas_lock_irq(&xas);
	xas_for_each_marked(&xas, entry, end_index, PAGECACHE_TAG_TOWRITE) {
		ret = dax_writeback_one(&xas, dax_dev, mapping, entry);
		if (ret < 0) {
			mapping_set_error(mapping, ret);
964 965
			break;
		}
966 967 968 969 970 971 972
		if (++scanned % XA_CHECK_SCHED)
			continue;

		xas_pause(&xas);
		xas_unlock_irq(&xas);
		cond_resched();
		xas_lock_irq(&xas);
973
	}
974
	xas_unlock_irq(&xas);
975
	put_dax(dax_dev);
976 977
	trace_dax_writeback_range_done(inode, xas.xa_index, end_index);
	return ret;
978 979 980
}
EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);

981
static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos)
982
{
983
	return (iomap->addr + (pos & PAGE_MASK) - iomap->offset) >> 9;
984 985
}

986 987
static int dax_iomap_pfn(struct iomap *iomap, loff_t pos, size_t size,
			 pfn_t *pfnp)
988
{
989
	const sector_t sector = dax_iomap_sector(iomap, pos);
990 991
	pgoff_t pgoff;
	int id, rc;
992
	long length;
993

994
	rc = bdev_dax_pgoff(iomap->bdev, sector, size, &pgoff);
995 996 997
	if (rc)
		return rc;
	id = dax_read_lock();
998
	length = dax_direct_access(iomap->dax_dev, pgoff, PHYS_PFN(size),
999
				   NULL, pfnp);
1000 1001 1002
	if (length < 0) {
		rc = length;
		goto out;
1003
	}
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013
	rc = -EINVAL;
	if (PFN_PHYS(length) < size)
		goto out;
	if (pfn_t_to_pfn(*pfnp) & (PHYS_PFN(size)-1))
		goto out;
	/* For larger pages we need devmap */
	if (length > 1 && !pfn_t_devmap(*pfnp))
		goto out;
	rc = 0;
out:
1014
	dax_read_unlock(id);
1015
	return rc;
1016 1017
}

1018
/*
1019 1020 1021 1022 1023
 * The user has performed a load from a hole in the file.  Allocating a new
 * page in the file would cause excessive storage usage for workloads with
 * sparse files.  Instead we insert a read-only mapping of the 4k zero page.
 * If this page is ever written to we will re-fault and change the mapping to
 * point to real DAX storage instead.
1024
 */
1025 1026 1027
static vm_fault_t dax_load_hole(struct xa_state *xas,
		struct address_space *mapping, void **entry,
		struct vm_fault *vmf)
1028 1029
{
	struct inode *inode = mapping->host;
1030
	unsigned long vaddr = vmf->address;
1031 1032
	pfn_t pfn = pfn_to_pfn_t(my_zero_pfn(vaddr));
	vm_fault_t ret;
1033

1034
	*entry = dax_insert_entry(xas, mapping, vmf, *entry, pfn,
1035 1036
			DAX_ZERO_PAGE, false);

1037
	ret = vmf_insert_mixed(vmf->vma, vaddr, pfn);
1038 1039 1040 1041
	trace_dax_load_hole(inode, vmf, ret);
	return ret;
}

1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054
static bool dax_range_is_aligned(struct block_device *bdev,
				 unsigned int offset, unsigned int length)
{
	unsigned short sector_size = bdev_logical_block_size(bdev);

	if (!IS_ALIGNED(offset, sector_size))
		return false;
	if (!IS_ALIGNED(length, sector_size))
		return false;

	return true;
}

1055 1056 1057
int __dax_zero_page_range(struct block_device *bdev,
		struct dax_device *dax_dev, sector_t sector,
		unsigned int offset, unsigned int size)
1058
{
1059 1060
	if (dax_range_is_aligned(bdev, offset, size)) {
		sector_t start_sector = sector + (offset >> 9);
1061 1062

		return blkdev_issue_zeroout(bdev, start_sector,
1063
				size >> 9, GFP_NOFS, 0);
1064
	} else {
1065 1066 1067 1068
		pgoff_t pgoff;
		long rc, id;
		void *kaddr;

1069
		rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff);
1070 1071 1072 1073
		if (rc)
			return rc;

		id = dax_read_lock();
1074
		rc = dax_direct_access(dax_dev, pgoff, 1, &kaddr, NULL);
1075 1076 1077 1078
		if (rc < 0) {
			dax_read_unlock(id);
			return rc;
		}
1079
		memset(kaddr + offset, 0, size);
1080
		dax_flush(dax_dev, kaddr + offset, size);
1081
		dax_read_unlock(id);
1082
	}
1083 1084 1085 1086
	return 0;
}
EXPORT_SYMBOL_GPL(__dax_zero_page_range);

1087
static loff_t
1088
dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
1089 1090
		struct iomap *iomap)
{
1091 1092
	struct block_device *bdev = iomap->bdev;
	struct dax_device *dax_dev = iomap->dax_dev;
1093 1094 1095
	struct iov_iter *iter = data;
	loff_t end = pos + length, done = 0;
	ssize_t ret = 0;
1096
	size_t xfer;
1097
	int id;
1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110

	if (iov_iter_rw(iter) == READ) {
		end = min(end, i_size_read(inode));
		if (pos >= end)
			return 0;

		if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN)
			return iov_iter_zero(min(length, end - pos), iter);
	}

	if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED))
		return -EIO;

1111 1112 1113 1114 1115
	/*
	 * Write can allocate block for an area which has a hole page mapped
	 * into page tables. We have to tear down these mappings so that data
	 * written by write(2) is visible in mmap.
	 */
1116
	if (iomap->flags & IOMAP_F_NEW) {
1117 1118 1119 1120 1121
		invalidate_inode_pages2_range(inode->i_mapping,
					      pos >> PAGE_SHIFT,
					      (end - 1) >> PAGE_SHIFT);
	}

1122
	id = dax_read_lock();
1123 1124
	while (pos < end) {
		unsigned offset = pos & (PAGE_SIZE - 1);
1125 1126
		const size_t size = ALIGN(length + offset, PAGE_SIZE);
		const sector_t sector = dax_iomap_sector(iomap, pos);
1127
		ssize_t map_len;
1128 1129
		pgoff_t pgoff;
		void *kaddr;
1130

1131 1132 1133 1134 1135
		if (fatal_signal_pending(current)) {
			ret = -EINTR;
			break;
		}

1136 1137 1138 1139 1140
		ret = bdev_dax_pgoff(bdev, sector, size, &pgoff);
		if (ret)
			break;

		map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size),
1141
				&kaddr, NULL);
1142 1143 1144 1145 1146
		if (map_len < 0) {
			ret = map_len;
			break;
		}

1147 1148
		map_len = PFN_PHYS(map_len);
		kaddr += offset;
1149 1150 1151 1152
		map_len -= offset;
		if (map_len > end - pos)
			map_len = end - pos;

1153 1154 1155 1156 1157
		/*
		 * The userspace address for the memory copy has already been
		 * validated via access_ok() in either vfs_read() or
		 * vfs_write(), depending on which operation we are doing.
		 */
1158
		if (iov_iter_rw(iter) == WRITE)
1159
			xfer = dax_copy_from_iter(dax_dev, pgoff, kaddr,
1160
					map_len, iter);
1161
		else
1162
			xfer = dax_copy_to_iter(dax_dev, pgoff, kaddr,
1163
					map_len, iter);
1164

1165 1166 1167 1168 1169 1170 1171 1172
		pos += xfer;
		length -= xfer;
		done += xfer;

		if (xfer == 0)
			ret = -EFAULT;
		if (xfer < map_len)
			break;
1173
	}
1174
	dax_read_unlock(id);
1175 1176 1177 1178 1179

	return done ? done : ret;
}

/**
1180
 * dax_iomap_rw - Perform I/O to a DAX file
1181 1182 1183 1184 1185 1186 1187 1188 1189
 * @iocb:	The control block for this I/O
 * @iter:	The addresses to do I/O from or to
 * @ops:	iomap ops passed from the file system
 *
 * This function performs read and write operations to directly mapped
 * persistent memory.  The callers needs to take care of read/write exclusion
 * and evicting any page cache pages in the region under I/O.
 */
ssize_t
1190
dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
1191
		const struct iomap_ops *ops)
1192 1193 1194 1195 1196 1197
{
	struct address_space *mapping = iocb->ki_filp->f_mapping;
	struct inode *inode = mapping->host;
	loff_t pos = iocb->ki_pos, ret = 0, done = 0;
	unsigned flags = 0;

1198 1199
	if (iov_iter_rw(iter) == WRITE) {
		lockdep_assert_held_exclusive(&inode->i_rwsem);
1200
		flags |= IOMAP_WRITE;
1201 1202 1203
	} else {
		lockdep_assert_held(&inode->i_rwsem);
	}
1204 1205 1206

	while (iov_iter_count(iter)) {
		ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops,
1207
				iter, dax_iomap_actor);
1208 1209 1210 1211 1212 1213 1214 1215 1216
		if (ret <= 0)
			break;
		pos += ret;
		done += ret;
	}

	iocb->ki_pos += done;
	return done ? done : ret;
}
1217
EXPORT_SYMBOL_GPL(dax_iomap_rw);
1218

1219
static vm_fault_t dax_fault_return(int error)
1220 1221 1222 1223 1224 1225 1226 1227
{
	if (error == 0)
		return VM_FAULT_NOPAGE;
	if (error == -ENOMEM)
		return VM_FAULT_OOM;
	return VM_FAULT_SIGBUS;
}

1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238
/*
 * MAP_SYNC on a dax mapping guarantees dirty metadata is
 * flushed on write-faults (non-cow), but not read-faults.
 */
static bool dax_fault_is_synchronous(unsigned long flags,
		struct vm_area_struct *vma, struct iomap *iomap)
{
	return (flags & IOMAP_WRITE) && (vma->vm_flags & VM_SYNC)
		&& (iomap->flags & IOMAP_F_DIRTY);
}

1239
static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp,
1240
			       int *iomap_errp, const struct iomap_ops *ops)
1241
{
1242 1243
	struct vm_area_struct *vma = vmf->vma;
	struct address_space *mapping = vma->vm_file->f_mapping;
1244
	XA_STATE(xas, &mapping->i_pages, vmf->pgoff);
1245
	struct inode *inode = mapping->host;
1246
	unsigned long vaddr = vmf->address;
1247 1248
	loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT;
	struct iomap iomap = { 0 };
1249
	unsigned flags = IOMAP_FAULT;
1250
	int error, major = 0;
1251
	bool write = vmf->flags & FAULT_FLAG_WRITE;
1252
	bool sync;
1253
	vm_fault_t ret = 0;
1254
	void *entry;
1255
	pfn_t pfn;
1256

1257
	trace_dax_pte_fault(inode, vmf, ret);
1258 1259 1260 1261 1262
	/*
	 * Check whether offset isn't beyond end of file now. Caller is supposed
	 * to hold locks serializing us with truncate / punch hole so this is
	 * a reliable test.
	 */
1263
	if (pos >= i_size_read(inode)) {
1264
		ret = VM_FAULT_SIGBUS;
1265 1266
		goto out;
	}
1267

1268
	if (write && !vmf->cow_page)
1269 1270
		flags |= IOMAP_WRITE;

1271 1272 1273
	entry = grab_mapping_entry(&xas, mapping, 0);
	if (xa_is_internal(entry)) {
		ret = xa_to_internal(entry);
1274 1275 1276
		goto out;
	}

1277 1278 1279 1280 1281 1282 1283
	/*
	 * It is possible, particularly with mixed reads & writes to private
	 * mappings, that we have raced with a PMD fault that overlaps with
	 * the PTE we need to set up.  If so just return and the fault will be
	 * retried.
	 */
	if (pmd_trans_huge(*vmf->pmd) || pmd_devmap(*vmf->pmd)) {
1284
		ret = VM_FAULT_NOPAGE;
1285 1286 1287
		goto unlock_entry;
	}

1288 1289 1290 1291 1292 1293
	/*
	 * Note that we don't bother to use iomap_apply here: DAX required
	 * the file system block size to be equal the page size, which means
	 * that we never have to deal with more than a single extent here.
	 */
	error = ops->iomap_begin(inode, pos, PAGE_SIZE, flags, &iomap);
1294 1295
	if (iomap_errp)
		*iomap_errp = error;
1296
	if (error) {
1297
		ret = dax_fault_return(error);
1298
		goto unlock_entry;
1299
	}
1300
	if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) {
1301 1302
		error = -EIO;	/* fs corruption? */
		goto error_finish_iomap;
1303 1304 1305
	}

	if (vmf->cow_page) {
1306 1307
		sector_t sector = dax_iomap_sector(&iomap, pos);

1308 1309 1310 1311 1312 1313
		switch (iomap.type) {
		case IOMAP_HOLE:
		case IOMAP_UNWRITTEN:
			clear_user_highpage(vmf->cow_page, vaddr);
			break;
		case IOMAP_MAPPED:
1314 1315
			error = copy_user_dax(iomap.bdev, iomap.dax_dev,
					sector, PAGE_SIZE, vmf->cow_page, vaddr);
1316 1317 1318 1319 1320 1321 1322 1323
			break;
		default:
			WARN_ON_ONCE(1);
			error = -EIO;
			break;
		}

		if (error)
1324
			goto error_finish_iomap;
1325 1326

		__SetPageUptodate(vmf->cow_page);
1327 1328 1329
		ret = finish_fault(vmf);
		if (!ret)
			ret = VM_FAULT_DONE_COW;
1330
		goto finish_iomap;
1331 1332
	}

1333
	sync = dax_fault_is_synchronous(flags, vma, &iomap);
1334

1335 1336 1337 1338
	switch (iomap.type) {
	case IOMAP_MAPPED:
		if (iomap.flags & IOMAP_F_NEW) {
			count_vm_event(PGMAJFAULT);
1339
			count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
1340 1341
			major = VM_FAULT_MAJOR;
		}
1342 1343 1344 1345
		error = dax_iomap_pfn(&iomap, pos, PAGE_SIZE, &pfn);
		if (error < 0)
			goto error_finish_iomap;

1346
		entry = dax_insert_entry(&xas, mapping, vmf, entry, pfn,
1347
						 0, write && !sync);
1348

1349 1350 1351 1352 1353 1354