dax.c 44.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * fs/dax.c - Direct Access filesystem code
 * Copyright (c) 2013-2014 Intel Corporation
 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com>
 * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

#include <linux/atomic.h>
#include <linux/blkdev.h>
#include <linux/buffer_head.h>
20
#include <linux/dax.h>
21 22
#include <linux/fs.h>
#include <linux/genhd.h>
23 24 25
#include <linux/highmem.h>
#include <linux/memcontrol.h>
#include <linux/mm.h>
26
#include <linux/mutex.h>
27
#include <linux/pagevec.h>
28
#include <linux/sched.h>
29
#include <linux/sched/signal.h>
30
#include <linux/uio.h>
31
#include <linux/vmstat.h>
32
#include <linux/pfn_t.h>
33
#include <linux/sizes.h>
34
#include <linux/mmu_notifier.h>
35 36
#include <linux/iomap.h>
#include "internal.h"
37

38 39 40
#define CREATE_TRACE_POINTS
#include <trace/events/fs_dax.h>

41 42 43 44 45 46 47 48 49 50 51
static inline unsigned int pe_order(enum page_entry_size pe_size)
{
	if (pe_size == PE_SIZE_PTE)
		return PAGE_SHIFT - PAGE_SHIFT;
	if (pe_size == PE_SIZE_PMD)
		return PMD_SHIFT - PAGE_SHIFT;
	if (pe_size == PE_SIZE_PUD)
		return PUD_SHIFT - PAGE_SHIFT;
	return ~0;
}

Jan Kara's avatar
Jan Kara committed
52 53 54 55
/* We choose 4096 entries - same as per-zone page wait tables */
#define DAX_WAIT_TABLE_BITS 12
#define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)

56 57
/* The 'colour' (ie low bits) within a PMD of a page offset.  */
#define PG_PMD_COLOUR	((PMD_SIZE >> PAGE_SHIFT) - 1)
58
#define PG_PMD_NR	(PMD_SIZE >> PAGE_SHIFT)
59

60 61 62
/* The order of a PMD entry */
#define PMD_ORDER	(PMD_SHIFT - PAGE_SHIFT)

63
static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
Jan Kara's avatar
Jan Kara committed
64 65 66 67 68 69 70 71 72 73 74

static int __init init_dax_wait_table(void)
{
	int i;

	for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++)
		init_waitqueue_head(wait_table + i);
	return 0;
}
fs_initcall(init_dax_wait_table);

75
/*
76 77 78 79
 * DAX pagecache entries use XArray value entries so they can't be mistaken
 * for pages.  We use one bit for locking, one bit for the entry size (PMD)
 * and two more to tell us if the entry is a zero page or an empty entry that
 * is just used for locking.  In total four special bits.
80 81 82 83 84
 *
 * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE
 * and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem
 * block allocation.
 */
85 86 87 88 89
#define DAX_SHIFT	(4)
#define DAX_LOCKED	(1UL << 0)
#define DAX_PMD		(1UL << 1)
#define DAX_ZERO_PAGE	(1UL << 2)
#define DAX_EMPTY	(1UL << 3)
90

Matthew Wilcox's avatar
Matthew Wilcox committed
91
static unsigned long dax_to_pfn(void *entry)
92
{
93
	return xa_to_value(entry) >> DAX_SHIFT;
94 95
}

96 97 98 99 100 101 102 103 104 105 106
static void *dax_make_entry(pfn_t pfn, unsigned long flags)
{
	return xa_mk_value(flags | (pfn_t_to_pfn(pfn) << DAX_SHIFT));
}

static void *dax_make_page_entry(struct page *page)
{
	pfn_t pfn = page_to_pfn_t(page);
	return dax_make_entry(pfn, PageHead(page) ? DAX_PMD : 0);
}

107 108 109 110 111
static bool dax_is_locked(void *entry)
{
	return xa_to_value(entry) & DAX_LOCKED;
}

Matthew Wilcox's avatar
Matthew Wilcox committed
112
static unsigned int dax_entry_order(void *entry)
113
{
114
	if (xa_to_value(entry) & DAX_PMD)
115
		return PMD_ORDER;
116 117 118
	return 0;
}

119
static int dax_is_pmd_entry(void *entry)
120
{
121
	return xa_to_value(entry) & DAX_PMD;
122 123
}

124
static int dax_is_pte_entry(void *entry)
125
{
126
	return !(xa_to_value(entry) & DAX_PMD);
127 128
}

129
static int dax_is_zero_entry(void *entry)
130
{
131
	return xa_to_value(entry) & DAX_ZERO_PAGE;
132 133
}

134
static int dax_is_empty_entry(void *entry)
135
{
136
	return xa_to_value(entry) & DAX_EMPTY;
137 138
}

Jan Kara's avatar
Jan Kara committed
139
/*
Matthew Wilcox's avatar
Matthew Wilcox committed
140
 * DAX page cache entry locking
Jan Kara's avatar
Jan Kara committed
141 142
 */
struct exceptional_entry_key {
143
	struct xarray *xa;
144
	pgoff_t entry_start;
Jan Kara's avatar
Jan Kara committed
145 146 147
};

struct wait_exceptional_entry_queue {
148
	wait_queue_entry_t wait;
Jan Kara's avatar
Jan Kara committed
149 150 151
	struct exceptional_entry_key key;
};

152 153
static wait_queue_head_t *dax_entry_waitqueue(struct xa_state *xas,
		void *entry, struct exceptional_entry_key *key)
154 155
{
	unsigned long hash;
156
	unsigned long index = xas->xa_index;
157 158 159 160 161 162

	/*
	 * If 'entry' is a PMD, align the 'index' that we use for the wait
	 * queue to the start of that PMD.  This ensures that all offsets in
	 * the range covered by the PMD map to the same bit lock.
	 */
163
	if (dax_is_pmd_entry(entry))
164
		index &= ~PG_PMD_COLOUR;
165
	key->xa = xas->xa;
166 167
	key->entry_start = index;

168
	hash = hash_long((unsigned long)xas->xa ^ index, DAX_WAIT_TABLE_BITS);
169 170 171
	return wait_table + hash;
}

172 173
static int wake_exceptional_entry_func(wait_queue_entry_t *wait,
		unsigned int mode, int sync, void *keyp)
Jan Kara's avatar
Jan Kara committed
174 175 176 177 178
{
	struct exceptional_entry_key *key = keyp;
	struct wait_exceptional_entry_queue *ewait =
		container_of(wait, struct wait_exceptional_entry_queue, wait);

179
	if (key->xa != ewait->key.xa ||
180
	    key->entry_start != ewait->key.entry_start)
Jan Kara's avatar
Jan Kara committed
181 182 183 184
		return 0;
	return autoremove_wake_function(wait, mode, sync, NULL);
}

185
/*
Matthew Wilcox's avatar
Matthew Wilcox committed
186 187 188
 * @entry may no longer be the entry at the index in the mapping.
 * The important information it's conveying is whether the entry at
 * this index used to be a PMD entry.
189
 */
190
static void dax_wake_entry(struct xa_state *xas, void *entry, bool wake_all)
191 192 193 194
{
	struct exceptional_entry_key key;
	wait_queue_head_t *wq;

195
	wq = dax_entry_waitqueue(xas, entry, &key);
196 197 198

	/*
	 * Checking for locked entry and prepare_to_wait_exclusive() happens
Matthew Wilcox's avatar
Matthew Wilcox committed
199
	 * under the i_pages lock, ditto for entry handling in our callers.
200 201 202 203 204 205 206
	 * So at this point all tasks that could have seen our entry locked
	 * must be in the waitqueue and the following check will see them.
	 */
	if (waitqueue_active(wq))
		__wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key);
}

207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230
/*
 * Look up entry in page cache, wait for it to become unlocked if it
 * is a DAX entry and return it.  The caller must subsequently call
 * put_unlocked_entry() if it did not lock the entry or dax_unlock_entry()
 * if it did.
 *
 * Must be called with the i_pages lock held.
 */
static void *get_unlocked_entry(struct xa_state *xas)
{
	void *entry;
	struct wait_exceptional_entry_queue ewait;
	wait_queue_head_t *wq;

	init_wait(&ewait.wait);
	ewait.wait.func = wake_exceptional_entry_func;

	for (;;) {
		entry = xas_load(xas);
		if (!entry || xa_is_internal(entry) ||
				WARN_ON_ONCE(!xa_is_value(entry)) ||
				!dax_is_locked(entry))
			return entry;

231
		wq = dax_entry_waitqueue(xas, entry, &ewait.key);
232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274
		prepare_to_wait_exclusive(wq, &ewait.wait,
					  TASK_UNINTERRUPTIBLE);
		xas_unlock_irq(xas);
		xas_reset(xas);
		schedule();
		finish_wait(wq, &ewait.wait);
		xas_lock_irq(xas);
	}
}

static void put_unlocked_entry(struct xa_state *xas, void *entry)
{
	/* If we were the only waiter woken, wake the next one */
	if (entry)
		dax_wake_entry(xas, entry, false);
}

/*
 * We used the xa_state to get the entry, but then we locked the entry and
 * dropped the xa_lock, so we know the xa_state is stale and must be reset
 * before use.
 */
static void dax_unlock_entry(struct xa_state *xas, void *entry)
{
	void *old;

	xas_reset(xas);
	xas_lock_irq(xas);
	old = xas_store(xas, entry);
	xas_unlock_irq(xas);
	BUG_ON(!dax_is_locked(old));
	dax_wake_entry(xas, entry, false);
}

/*
 * Return: The entry stored at this location before it was locked.
 */
static void *dax_lock_entry(struct xa_state *xas, void *entry)
{
	unsigned long v = xa_to_value(entry);
	return xas_store(xas, xa_mk_value(v | DAX_LOCKED));
}

275 276 277 278 279 280 281 282 283 284 285 286
static unsigned long dax_entry_size(void *entry)
{
	if (dax_is_zero_entry(entry))
		return 0;
	else if (dax_is_empty_entry(entry))
		return 0;
	else if (dax_is_pmd_entry(entry))
		return PMD_SIZE;
	else
		return PAGE_SIZE;
}

Matthew Wilcox's avatar
Matthew Wilcox committed
287
static unsigned long dax_end_pfn(void *entry)
288
{
Matthew Wilcox's avatar
Matthew Wilcox committed
289
	return dax_to_pfn(entry) + dax_entry_size(entry) / PAGE_SIZE;
290 291 292 293 294 295 296
}

/*
 * Iterate through all mapped pfns represented by an entry, i.e. skip
 * 'empty' and 'zero' entries.
 */
#define for_each_mapped_pfn(entry, pfn) \
Matthew Wilcox's avatar
Matthew Wilcox committed
297 298
	for (pfn = dax_to_pfn(entry); \
			pfn < dax_end_pfn(entry); pfn++)
299

300 301 302 303 304 305 306
/*
 * TODO: for reflink+dax we need a way to associate a single page with
 * multiple address_space instances at different linear_page_index()
 * offsets.
 */
static void dax_associate_entry(void *entry, struct address_space *mapping,
		struct vm_area_struct *vma, unsigned long address)
307
{
308 309
	unsigned long size = dax_entry_size(entry), pfn, index;
	int i = 0;
310 311 312 313

	if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
		return;

314
	index = linear_page_index(vma, address & ~(size - 1));
315 316 317 318 319
	for_each_mapped_pfn(entry, pfn) {
		struct page *page = pfn_to_page(pfn);

		WARN_ON_ONCE(page->mapping);
		page->mapping = mapping;
320
		page->index = index + i++;
321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337
	}
}

static void dax_disassociate_entry(void *entry, struct address_space *mapping,
		bool trunc)
{
	unsigned long pfn;

	if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
		return;

	for_each_mapped_pfn(entry, pfn) {
		struct page *page = pfn_to_page(pfn);

		WARN_ON_ONCE(trunc && page_ref_count(page) > 1);
		WARN_ON_ONCE(page->mapping && page->mapping != mapping);
		page->mapping = NULL;
338
		page->index = 0;
339 340 341
	}
}

342 343 344 345 346 347 348 349 350 351 352 353 354
static struct page *dax_busy_page(void *entry)
{
	unsigned long pfn;

	for_each_mapped_pfn(entry, pfn) {
		struct page *page = pfn_to_page(pfn);

		if (page_ref_count(page) > 1)
			return page;
	}
	return NULL;
}

355 356
bool dax_lock_mapping_entry(struct page *page)
{
357 358
	XA_STATE(xas, NULL, 0);
	void *entry;
359 360

	for (;;) {
361
		struct address_space *mapping = READ_ONCE(page->mapping);
362 363

		if (!dax_mapping(mapping))
364
			return false;
365 366 367 368 369 370 371 372

		/*
		 * In the device-dax case there's no need to lock, a
		 * struct dev_pagemap pin is sufficient to keep the
		 * inode alive, and we assume we have dev_pagemap pin
		 * otherwise we would not have a valid pfn_to_page()
		 * translation.
		 */
373 374
		if (S_ISCHR(mapping->host->i_mode))
			return true;
375

376 377
		xas.xa = &mapping->i_pages;
		xas_lock_irq(&xas);
378
		if (mapping != page->mapping) {
379
			xas_unlock_irq(&xas);
380 381
			continue;
		}
382 383 384 385 386 387 388 389 390
		xas_set(&xas, page->index);
		entry = xas_load(&xas);
		if (dax_is_locked(entry)) {
			entry = get_unlocked_entry(&xas);
			/* Did the page move while we slept? */
			if (dax_to_pfn(entry) != page_to_pfn(page)) {
				xas_unlock_irq(&xas);
				continue;
			}
391
		}
392 393 394
		dax_lock_entry(&xas, entry);
		xas_unlock_irq(&xas);
		return true;
395 396 397 398 399 400
	}
}

void dax_unlock_mapping_entry(struct page *page)
{
	struct address_space *mapping = page->mapping;
401
	XA_STATE(xas, &mapping->i_pages, page->index);
402

403
	if (S_ISCHR(mapping->host->i_mode))
404 405
		return;

406
	dax_unlock_entry(&xas, dax_make_page_entry(page));
407 408
}

Jan Kara's avatar
Jan Kara committed
409
/*
Matthew Wilcox's avatar
Matthew Wilcox committed
410 411 412
 * Find page cache entry at given index. If it is a DAX entry, return it
 * with the entry locked. If the page cache doesn't contain an entry at
 * that index, add a locked empty entry.
Jan Kara's avatar
Jan Kara committed
413
 *
414
 * When requesting an entry with size DAX_PMD, grab_mapping_entry() will
415 416 417
 * either return that locked entry or will return VM_FAULT_FALLBACK.
 * This will happen if there are any PTE entries within the PMD range
 * that we are requesting.
418
 *
419 420 421 422 423 424
 * We always favor PTE entries over PMD entries. There isn't a flow where we
 * evict PTE entries in order to 'upgrade' them to a PMD entry.  A PMD
 * insertion will fail if it finds any PTE entries already in the tree, and a
 * PTE insertion will cause an existing PMD entry to be unmapped and
 * downgraded to PTE entries.  This happens for both PMD zero pages as
 * well as PMD empty entries.
425
 *
426 427 428
 * The exception to this downgrade path is for PMD entries that have
 * real storage backing them.  We will leave these real PMD entries in
 * the tree, and PTE writes will simply dirty the entire PMD entry.
429
 *
Jan Kara's avatar
Jan Kara committed
430 431 432
 * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For
 * persistent memory the benefit is doubtful. We can add that later if we can
 * show it helps.
433 434 435 436
 *
 * On error, this function does not return an ERR_PTR.  Instead it returns
 * a VM_FAULT code, encoded as an xarray internal entry.  The ERR_PTR values
 * overlap with xarray value entries.
Jan Kara's avatar
Jan Kara committed
437
 */
438 439
static void *grab_mapping_entry(struct xa_state *xas,
		struct address_space *mapping, unsigned long size_flag)
Jan Kara's avatar
Jan Kara committed
440
{
441 442 443
	unsigned long index = xas->xa_index;
	bool pmd_downgrade = false; /* splitting PMD entry into PTE entries? */
	void *entry;
444

445 446 447 448 449
retry:
	xas_lock_irq(xas);
	entry = get_unlocked_entry(xas);
	if (xa_is_internal(entry))
		goto fallback;
450

451
	if (entry) {
452 453 454 455 456
		if (WARN_ON_ONCE(!xa_is_value(entry))) {
			xas_set_err(xas, EIO);
			goto out_unlock;
		}

457
		if (size_flag & DAX_PMD) {
458
			if (dax_is_pte_entry(entry)) {
459 460
				put_unlocked_entry(xas, entry);
				goto fallback;
461 462
			}
		} else { /* trying to grab a PTE entry */
463
			if (dax_is_pmd_entry(entry) &&
464 465 466 467 468 469 470
			    (dax_is_zero_entry(entry) ||
			     dax_is_empty_entry(entry))) {
				pmd_downgrade = true;
			}
		}
	}

471 472 473 474 475 476
	if (pmd_downgrade) {
		/*
		 * Make sure 'entry' remains valid while we drop
		 * the i_pages lock.
		 */
		dax_lock_entry(xas, entry);
477 478 479 480 481 482

		/*
		 * Besides huge zero pages the only other thing that gets
		 * downgraded are empty entries which don't need to be
		 * unmapped.
		 */
483 484 485 486 487 488 489
		if (dax_is_zero_entry(entry)) {
			xas_unlock_irq(xas);
			unmap_mapping_pages(mapping,
					xas->xa_index & ~PG_PMD_COLOUR,
					PG_PMD_NR, false);
			xas_reset(xas);
			xas_lock_irq(xas);
490 491
		}

492 493 494 495 496 497 498
		dax_disassociate_entry(entry, mapping, false);
		xas_store(xas, NULL);	/* undo the PMD join */
		dax_wake_entry(xas, entry, true);
		mapping->nrexceptional--;
		entry = NULL;
		xas_set(xas, index);
	}
499

500 501 502 503 504 505 506
	if (entry) {
		dax_lock_entry(xas, entry);
	} else {
		entry = dax_make_entry(pfn_to_pfn_t(0), size_flag | DAX_EMPTY);
		dax_lock_entry(xas, entry);
		if (xas_error(xas))
			goto out_unlock;
Jan Kara's avatar
Jan Kara committed
507 508
		mapping->nrexceptional++;
	}
509 510 511 512 513 514 515 516 517

out_unlock:
	xas_unlock_irq(xas);
	if (xas_nomem(xas, mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM))
		goto retry;
	if (xas->xa_node == XA_ERROR(-ENOMEM))
		return xa_mk_internal(VM_FAULT_OOM);
	if (xas_error(xas))
		return xa_mk_internal(VM_FAULT_SIGBUS);
518
	return entry;
519 520 521
fallback:
	xas_unlock_irq(xas);
	return xa_mk_internal(VM_FAULT_FALLBACK);
Jan Kara's avatar
Jan Kara committed
522 523
}

524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
/**
 * dax_layout_busy_page - find first pinned page in @mapping
 * @mapping: address space to scan for a page with ref count > 1
 *
 * DAX requires ZONE_DEVICE mapped pages. These pages are never
 * 'onlined' to the page allocator so they are considered idle when
 * page->count == 1. A filesystem uses this interface to determine if
 * any page in the mapping is busy, i.e. for DMA, or other
 * get_user_pages() usages.
 *
 * It is expected that the filesystem is holding locks to block the
 * establishment of new mappings in this address_space. I.e. it expects
 * to be able to run unmap_mapping_range() and subsequently not race
 * mapping_mapped() becoming true.
 */
struct page *dax_layout_busy_page(struct address_space *mapping)
{
541 542 543
	XA_STATE(xas, &mapping->i_pages, 0);
	void *entry;
	unsigned int scanned = 0;
544 545 546 547 548 549 550 551 552 553 554 555 556
	struct page *page = NULL;

	/*
	 * In the 'limited' case get_user_pages() for dax is disabled.
	 */
	if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
		return NULL;

	if (!dax_mapping(mapping) || !mapping_mapped(mapping))
		return NULL;

	/*
	 * If we race get_user_pages_fast() here either we'll see the
557
	 * elevated page count in the iteration and wait, or
558 559 560 561 562 563 564 565 566 567 568
	 * get_user_pages_fast() will see that the page it took a reference
	 * against is no longer mapped in the page tables and bail to the
	 * get_user_pages() slow path.  The slow path is protected by
	 * pte_lock() and pmd_lock(). New references are not taken without
	 * holding those locks, and unmap_mapping_range() will not zero the
	 * pte or pmd without holding the respective lock, so we are
	 * guaranteed to either see new references or prevent new
	 * references from being established.
	 */
	unmap_mapping_range(mapping, 0, 0, 1);

569 570 571 572 573 574 575 576 577
	xas_lock_irq(&xas);
	xas_for_each(&xas, entry, ULONG_MAX) {
		if (WARN_ON_ONCE(!xa_is_value(entry)))
			continue;
		if (unlikely(dax_is_locked(entry)))
			entry = get_unlocked_entry(&xas);
		if (entry)
			page = dax_busy_page(entry);
		put_unlocked_entry(&xas, entry);
578 579
		if (page)
			break;
580 581 582 583 584 585 586
		if (++scanned % XA_CHECK_SCHED)
			continue;

		xas_pause(&xas);
		xas_unlock_irq(&xas);
		cond_resched();
		xas_lock_irq(&xas);
587
	}
588
	xas_unlock_irq(&xas);
589 590 591 592
	return page;
}
EXPORT_SYMBOL_GPL(dax_layout_busy_page);

Matthew Wilcox's avatar
Matthew Wilcox committed
593
static int __dax_invalidate_entry(struct address_space *mapping,
594 595
					  pgoff_t index, bool trunc)
{
596
	XA_STATE(xas, &mapping->i_pages, index);
597 598 599
	int ret = 0;
	void *entry;

600 601
	xas_lock_irq(&xas);
	entry = get_unlocked_entry(&xas);
602
	if (!entry || WARN_ON_ONCE(!xa_is_value(entry)))
603 604
		goto out;
	if (!trunc &&
605 606
	    (xas_get_mark(&xas, PAGECACHE_TAG_DIRTY) ||
	     xas_get_mark(&xas, PAGECACHE_TAG_TOWRITE)))
607
		goto out;
608
	dax_disassociate_entry(entry, mapping, trunc);
609
	xas_store(&xas, NULL);
610 611 612
	mapping->nrexceptional--;
	ret = 1;
out:
613 614
	put_unlocked_entry(&xas, entry);
	xas_unlock_irq(&xas);
615 616
	return ret;
}
617

Jan Kara's avatar
Jan Kara committed
618
/*
619 620
 * Delete DAX entry at @index from @mapping.  Wait for it
 * to be unlocked before deleting it.
Jan Kara's avatar
Jan Kara committed
621 622 623
 */
int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)
{
Matthew Wilcox's avatar
Matthew Wilcox committed
624
	int ret = __dax_invalidate_entry(mapping, index, true);
Jan Kara's avatar
Jan Kara committed
625 626 627 628

	/*
	 * This gets called from truncate / punch_hole path. As such, the caller
	 * must hold locks protecting against concurrent modifications of the
Matthew Wilcox's avatar
Matthew Wilcox committed
629
	 * page cache (usually fs-private i_mmap_sem for writing). Since the
630
	 * caller has seen a DAX entry for this index, we better find it
Jan Kara's avatar
Jan Kara committed
631 632
	 * at that index as well...
	 */
633 634 635 636 637
	WARN_ON_ONCE(!ret);
	return ret;
}

/*
638
 * Invalidate DAX entry if it is clean.
639 640 641 642
 */
int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
				      pgoff_t index)
{
Matthew Wilcox's avatar
Matthew Wilcox committed
643
	return __dax_invalidate_entry(mapping, index, false);
Jan Kara's avatar
Jan Kara committed
644 645
}

646 647 648
static int copy_user_dax(struct block_device *bdev, struct dax_device *dax_dev,
		sector_t sector, size_t size, struct page *to,
		unsigned long vaddr)
649
{
650 651 652 653 654 655 656 657 658 659
	void *vto, *kaddr;
	pgoff_t pgoff;
	long rc;
	int id;

	rc = bdev_dax_pgoff(bdev, sector, size, &pgoff);
	if (rc)
		return rc;

	id = dax_read_lock();
660
	rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, NULL);
661 662 663 664
	if (rc < 0) {
		dax_read_unlock(id);
		return rc;
	}
665
	vto = kmap_atomic(to);
666
	copy_user_page(vto, (void __force *)kaddr, vaddr, to);
667
	kunmap_atomic(vto);
668
	dax_read_unlock(id);
669 670 671
	return 0;
}

672 673 674 675 676 677 678
/*
 * By this point grab_mapping_entry() has ensured that we have a locked entry
 * of the appropriate size so we don't have to worry about downgrading PMDs to
 * PTEs.  If we happen to be trying to insert a PTE and there is a PMD
 * already in the tree, we will skip the insertion and just dirty the PMD as
 * appropriate.
 */
679 680 681
static void *dax_insert_entry(struct xa_state *xas,
		struct address_space *mapping, struct vm_fault *vmf,
		void *entry, pfn_t pfn, unsigned long flags, bool dirty)
682
{
683
	void *new_entry = dax_make_entry(pfn, flags);
684

685
	if (dirty)
686
		__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
687

688
	if (dax_is_zero_entry(entry) && !(flags & DAX_ZERO_PAGE)) {
689
		unsigned long index = xas->xa_index;
690 691
		/* we are replacing a zero page with block mapping */
		if (dax_is_pmd_entry(entry))
692
			unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR,
693
					PG_PMD_NR, false);
694
		else /* pte entry */
695
			unmap_mapping_pages(mapping, index, 1, false);
696 697
	}

698 699
	xas_reset(xas);
	xas_lock_irq(xas);
700 701
	if (dax_entry_size(entry) != dax_entry_size(new_entry)) {
		dax_disassociate_entry(entry, mapping, false);
702
		dax_associate_entry(new_entry, mapping, vmf->vma, vmf->address);
703
	}
704

705
	if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) {
706
		/*
Matthew Wilcox's avatar
Matthew Wilcox committed
707
		 * Only swap our new entry into the page cache if the current
708
		 * entry is a zero page or an empty entry.  If a normal PTE or
Matthew Wilcox's avatar
Matthew Wilcox committed
709
		 * PMD entry is already in the cache, we leave it alone.  This
710 711 712 713
		 * means that if we are trying to insert a PTE and the
		 * existing entry is a PMD, we will just leave the PMD in the
		 * tree and dirty it if necessary.
		 */
714 715 716
		void *old = dax_lock_entry(xas, new_entry);
		WARN_ON_ONCE(old != xa_mk_value(xa_to_value(entry) |
					DAX_LOCKED));
717
		entry = new_entry;
718 719
	} else {
		xas_load(xas);	/* Walk the xa_state */
720
	}
721

722
	if (dirty)
723
		xas_set_mark(xas, PAGECACHE_TAG_DIRTY);
724

725
	xas_unlock_irq(xas);
726
	return entry;
727 728
}

Matthew Wilcox's avatar
Matthew Wilcox committed
729 730
static inline
unsigned long pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma)
731 732 733 734 735 736 737 738 739
{
	unsigned long address;

	address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
	VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
	return address;
}

/* Walk all mappings of a given index of a file and writeprotect them */
Matthew Wilcox's avatar
Matthew Wilcox committed
740 741
static void dax_entry_mkclean(struct address_space *mapping, pgoff_t index,
		unsigned long pfn)
742 743
{
	struct vm_area_struct *vma;
744 745
	pte_t pte, *ptep = NULL;
	pmd_t *pmdp = NULL;
746 747 748 749
	spinlock_t *ptl;

	i_mmap_lock_read(mapping);
	vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) {
750
		unsigned long address, start, end;
751 752 753 754 755 756 757

		cond_resched();

		if (!(vma->vm_flags & VM_SHARED))
			continue;

		address = pgoff_address(index, vma);
758 759 760 761 762 763 764

		/*
		 * Note because we provide start/end to follow_pte_pmd it will
		 * call mmu_notifier_invalidate_range_start() on our behalf
		 * before taking any lock.
		 */
		if (follow_pte_pmd(vma->vm_mm, address, &start, &end, &ptep, &pmdp, &ptl))
765 766
			continue;

767 768 769 770 771
		/*
		 * No need to call mmu_notifier_invalidate_range() as we are
		 * downgrading page table protection not changing it to point
		 * to a new page.
		 *
772
		 * See Documentation/vm/mmu_notifier.rst
773
		 */
774 775 776 777 778 779
		if (pmdp) {
#ifdef CONFIG_FS_DAX_PMD
			pmd_t pmd;

			if (pfn != pmd_pfn(*pmdp))
				goto unlock_pmd;
780
			if (!pmd_dirty(*pmdp) && !pmd_write(*pmdp))
781 782 783 784 785 786 787 788 789
				goto unlock_pmd;

			flush_cache_page(vma, address, pfn);
			pmd = pmdp_huge_clear_flush(vma, address, pmdp);
			pmd = pmd_wrprotect(pmd);
			pmd = pmd_mkclean(pmd);
			set_pmd_at(vma->vm_mm, address, pmdp, pmd);
unlock_pmd:
#endif
790
			spin_unlock(ptl);
791 792 793 794 795 796 797 798 799 800 801 802 803 804
		} else {
			if (pfn != pte_pfn(*ptep))
				goto unlock_pte;
			if (!pte_dirty(*ptep) && !pte_write(*ptep))
				goto unlock_pte;

			flush_cache_page(vma, address, pfn);
			pte = ptep_clear_flush(vma, address, ptep);
			pte = pte_wrprotect(pte);
			pte = pte_mkclean(pte);
			set_pte_at(vma->vm_mm, address, ptep, pte);
unlock_pte:
			pte_unmap_unlock(ptep, ptl);
		}
805

806
		mmu_notifier_invalidate_range_end(vma->vm_mm, start, end);
807 808 809 810
	}
	i_mmap_unlock_read(mapping);
}

811 812
static int dax_writeback_one(struct xa_state *xas, struct dax_device *dax_dev,
		struct address_space *mapping, void *entry)
813
{
Dan Williams's avatar
Dan Williams committed
814 815
	unsigned long pfn;
	long ret = 0;
816
	size_t size;
817 818

	/*
819 820
	 * A page got tagged dirty in DAX mapping? Something is seriously
	 * wrong.
821
	 */
822
	if (WARN_ON(!xa_is_value(entry)))
823
		return -EIO;
824

825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848
	if (unlikely(dax_is_locked(entry))) {
		void *old_entry = entry;

		entry = get_unlocked_entry(xas);

		/* Entry got punched out / reallocated? */
		if (!entry || WARN_ON_ONCE(!xa_is_value(entry)))
			goto put_unlocked;
		/*
		 * Entry got reallocated elsewhere? No need to writeback.
		 * We have to compare pfns as we must not bail out due to
		 * difference in lockbit or entry type.
		 */
		if (dax_to_pfn(old_entry) != dax_to_pfn(entry))
			goto put_unlocked;
		if (WARN_ON_ONCE(dax_is_empty_entry(entry) ||
					dax_is_zero_entry(entry))) {
			ret = -EIO;
			goto put_unlocked;
		}

		/* Another fsync thread may have already done this entry */
		if (!xas_get_mark(xas, PAGECACHE_TAG_TOWRITE))
			goto put_unlocked;
849 850
	}

851
	/* Lock the entry to serialize with page faults */
852 853
	dax_lock_entry(xas, entry);

854 855 856 857
	/*
	 * We can clear the tag now but we have to be careful so that concurrent
	 * dax_writeback_one() calls for the same index cannot finish before we
	 * actually flush the caches. This is achieved as the calls will look
Matthew Wilcox's avatar
Matthew Wilcox committed
858 859
	 * at the entry only under the i_pages lock and once they do that
	 * they will see the entry locked and wait for it to unlock.
860
	 */
861 862
	xas_clear_mark(xas, PAGECACHE_TAG_TOWRITE);
	xas_unlock_irq(xas);
863

864 865 866
	/*
	 * Even if dax_writeback_mapping_range() was given a wbc->range_start
	 * in the middle of a PMD, the 'index' we are given will be aligned to
Dan Williams's avatar
Dan Williams committed
867 868 869
	 * the start index of the PMD, as will the pfn we pull from 'entry'.
	 * This allows us to flush for PMD_SIZE and not have to worry about
	 * partial PMD writebacks.
870
	 */
Matthew Wilcox's avatar
Matthew Wilcox committed
871 872
	pfn = dax_to_pfn(entry);
	size = PAGE_SIZE << dax_entry_order(entry);
873

874
	dax_entry_mkclean(mapping, xas->xa_index, pfn);
Dan Williams's avatar
Dan Williams committed
875
	dax_flush(dax_dev, page_address(pfn_to_page(pfn)), size);
876 877 878 879 880 881
	/*
	 * After we have flushed the cache, we can clear the dirty tag. There
	 * cannot be new dirty data in the pfn after the flush has completed as
	 * the pfn mappings are writeprotected and fault waits for mapping
	 * entry lock.
	 */
882 883 884 885 886 887 888 889
	xas_reset(xas);
	xas_lock_irq(xas);
	xas_store(xas, entry);
	xas_clear_mark(xas, PAGECACHE_TAG_DIRTY);
	dax_wake_entry(xas, entry, false);

	trace_dax_writeback_one(mapping->host, xas->xa_index,
			size >> PAGE_SHIFT);
890 891
	return ret;

892
 put_unlocked:
893
	put_unlocked_entry(xas, entry);
894 895 896 897 898 899 900 901
	return ret;
}

/*
 * Flush the mapping to the persistent domain within the byte range of [start,
 * end]. This is required by data integrity operations to ensure file data is
 * on persistent storage prior to completion of the operation.
 */
902 903
int dax_writeback_mapping_range(struct address_space *mapping,
		struct block_device *bdev, struct writeback_control *wbc)
904
{
905
	XA_STATE(xas, &mapping->i_pages, wbc->range_start >> PAGE_SHIFT);
906
	struct inode *inode = mapping->host;
907
	pgoff_t end_index = wbc->range_end >> PAGE_SHIFT;
908
	struct dax_device *dax_dev;
909 910 911
	void *entry;
	int ret = 0;
	unsigned int scanned = 0;
912 913 914 915

	if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
		return -EIO;

916 917 918
	if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL)
		return 0;

919 920 921 922
	dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
	if (!dax_dev)
		return -EIO;

923
	trace_dax_writeback_range(inode, xas.xa_index, end_index);
924

925
	tag_pages_for_writeback(mapping, xas.xa_index, end_index);
926

927 928 929 930 931
	xas_lock_irq(&xas);
	xas_for_each_marked(&xas, entry, end_index, PAGECACHE_TAG_TOWRITE) {
		ret = dax_writeback_one(&xas, dax_dev, mapping, entry);
		if (ret < 0) {
			mapping_set_error(mapping, ret);
932 933
			break;
		}
934 935 936 937 938 939 940
		if (++scanned % XA_CHECK_SCHED)
			continue;

		xas_pause(&xas);
		xas_unlock_irq(&xas);
		cond_resched();
		xas_lock_irq(&xas);
941
	}
942
	xas_unlock_irq(&xas);
943
	put_dax(dax_dev);
944 945
	trace_dax_writeback_range_done(inode, xas.xa_index, end_index);
	return ret;
946 947 948
}
EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);

949
static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos)
950
{
951
	return (iomap->addr + (pos & PAGE_MASK) - iomap->offset) >> 9;
952 953
}

954 955
static int dax_iomap_pfn(struct iomap *iomap, loff_t pos, size_t size,
			 pfn_t *pfnp)
956
{
957
	const sector_t sector = dax_iomap_sector(iomap, pos);
958 959
	pgoff_t pgoff;
	int id, rc;
960
	long length;
961

962
	rc = bdev_dax_pgoff(iomap->bdev, sector, size, &pgoff);
963 964 965
	if (rc)
		return rc;
	id = dax_read_lock();
966
	length = dax_direct_access(iomap->dax_dev, pgoff, PHYS_PFN(size),
967
				   NULL, pfnp);
968 969 970
	if (length < 0) {
		rc = length;
		goto out;
971
	}
972 973 974 975 976 977 978 979 980 981
	rc = -EINVAL;
	if (PFN_PHYS(length) < size)
		goto out;
	if (pfn_t_to_pfn(*pfnp) & (PHYS_PFN(size)-1))
		goto out;
	/* For larger pages we need devmap */
	if (length > 1 && !pfn_t_devmap(*pfnp))
		goto out;
	rc = 0;
out:
982
	dax_read_unlock(id);
983
	return rc;
984 985
}

986
/*
987 988 989 990 991
 * The user has performed a load from a hole in the file.  Allocating a new
 * page in the file would cause excessive storage usage for workloads with
 * sparse files.  Instead we insert a read-only mapping of the 4k zero page.
 * If this page is ever written to we will re-fault and change the mapping to
 * point to real DAX storage instead.
992
 */
993 994 995
static vm_fault_t dax_load_hole(struct xa_state *xas,
		struct address_space *mapping, void **entry,
		struct vm_fault *vmf)
996 997
{
	struct inode *inode = mapping->host;
998
	unsigned long vaddr = vmf->address;
999 1000
	pfn_t pfn = pfn_to_pfn_t(my_zero_pfn(vaddr));
	vm_fault_t ret;
1001

1002
	*entry = dax_insert_entry(xas, mapping, vmf, *entry, pfn,
1003 1004
			DAX_ZERO_PAGE, false);

1005
	ret = vmf_insert_mixed(vmf->vma, vaddr, pfn);
1006 1007 1008 1009
	trace_dax_load_hole(inode, vmf, ret);
	return ret;
}

1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022
static bool dax_range_is_aligned(struct block_device *bdev,
				 unsigned int offset, unsigned int length)
{
	unsigned short sector_size = bdev_logical_block_size(bdev);

	if (!IS_ALIGNED(offset, sector_size))
		return false;
	if (!IS_ALIGNED(length, sector_size))
		return false;

	return true;
}

1023 1024 1025
int __dax_zero_page_range(struct block_device *bdev,
		struct dax_device *dax_dev, sector_t sector,
		unsigned int offset, unsigned int size)
1026
{
1027 1028
	if (dax_range_is_aligned(bdev, offset, size)) {
		sector_t start_sector = sector + (offset >> 9);
1029 1030

		return blkdev_issue_zeroout(bdev, start_sector,
1031
				size >> 9, GFP_NOFS, 0);
1032
	} else {
1033 1034 1035 1036
		pgoff_t pgoff;
		long rc, id;
		void *kaddr;

1037
		rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff);
1038 1039 1040 1041
		if (rc)
			return rc;

		id = dax_read_lock();
1042
		rc = dax_direct_access(dax_dev, pgoff, 1, &kaddr, NULL);
1043 1044 1045 1046
		if (rc < 0) {
			dax_read_unlock(id);
			return rc;
		}
1047
		memset(kaddr + offset, 0, size);
1048
		dax_flush(dax_dev, kaddr + offset, size);
1049
		dax_read_unlock(id);
1050
	}
1051 1052 1053 1054
	return 0;
}
EXPORT_SYMBOL_GPL(__dax_zero_page_range);

1055
static loff_t
1056
dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
1057 1058
		struct iomap *iomap)
{
1059 1060
	struct block_device *bdev = iomap->bdev;
	struct dax_device *dax_dev = iomap->dax_dev;
1061 1062 1063
	struct iov_iter *iter = data;
	loff_t end = pos + length, done = 0;
	ssize_t ret = 0;
1064
	size_t xfer;
1065
	int id;
1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078

	if (iov_iter_rw(iter) == READ) {
		end = min(end, i_size_read(inode));
		if (pos >= end)
			return 0;

		if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN)
			return iov_iter_zero(min(length, end - pos), iter);
	}

	if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED))
		return -EIO;

1079 1080 1081 1082 1083
	/*
	 * Write can allocate block for an area which has a hole page mapped
	 * into page tables. We have to tear down these mappings so that data
	 * written by write(2) is visible in mmap.
	 */
1084
	if (iomap->flags & IOMAP_F_NEW) {
1085 1086 1087 1088 1089
		invalidate_inode_pages2_range(inode->i_mapping,
					      pos >> PAGE_SHIFT,
					      (end - 1) >> PAGE_SHIFT);
	}

1090
	id = dax_read_lock();
1091 1092
	while (pos < end) {
		unsigned offset = pos & (PAGE_SIZE - 1);
1093 1094
		const size_t size = ALIGN(length + offset, PAGE_SIZE);
		const sector_t sector = dax_iomap_sector(iomap, pos);
1095
		ssize_t map_len;
1096 1097
		pgoff_t pgoff;
		void *kaddr;
1098

1099 1100 1101 1102 1103
		if (fatal_signal_pending(current)) {
			ret = -EINTR;
			break;
		}

1104 1105 1106 1107 1108
		ret = bdev_dax_pgoff(bdev, sector, size, &pgoff);
		if (ret)
			break;

		map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size),
1109
				&kaddr, NULL);
1110 1111 1112 1113 1114
		if (map_len < 0) {
			ret = map_len;
			break;
		}

1115 1116
		map_len = PFN_PHYS(map_len);
		kaddr += offset;
1117 1118 1119 1120
		map_len -= offset;
		if (map_len > end - pos)
			map_len = end - pos;

1121 1122 1123 1124 1125
		/*
		 * The userspace address for the memory copy has already been
		 * validated via access_ok() in either vfs_read() or
		 * vfs_write(), depending on which operation we are doing.
		 */
1126
		if (iov_iter_rw(iter) == WRITE)
1127
			xfer = dax_copy_from_iter(dax_dev, pgoff, kaddr,
1128
					map_len, iter);
1129
		else
1130
			xfer = dax_copy_to_iter(dax_dev, pgoff, kaddr,
1131
					map_len, iter);
1132

1133 1134 1135 1136 1137 1138 1139 1140
		pos += xfer;
		length -= xfer;
		done += xfer;

		if (xfer == 0)
			ret = -EFAULT;
		if (xfer < map_len)
			break;
1141
	}
1142
	dax_read_unlock(id);
1143 1144 1145 1146 1147

	return done ? done : ret;
}

/**
1148
 * dax_iomap_rw - Perform I/O to a DAX file
1149 1150 1151 1152 1153 1154 1155 1156 1157
 * @iocb:	The control block for this I/O
 * @iter:	The addresses to do I/O from or to
 * @ops:	iomap ops passed from the file system
 *
 * This function performs read and write operations to directly mapped
 * persistent memory.  The callers needs to take care of read/write exclusion
 * and evicting any page cache pages in the region under I/O.
 */
ssize_t
1158
dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
1159
		const struct iomap_ops *ops)
1160 1161 1162 1163 1164 1165
{
	struct address_space *mapping = iocb->ki_filp->f_mapping;
	struct inode *inode = mapping->host;
	loff_t pos = iocb->ki_pos, ret = 0, done = 0;
	unsigned flags = 0;

1166 1167
	if (iov_iter_rw(iter) == WRITE) {
		lockdep_assert_held_exclusive(&inode->i_rwsem);
1168
		flags |= IOMAP_WRITE;
1169 1170 1171
	} else {
		lockdep_assert_held(&inode->i_rwsem);
	}
1172 1173 1174

	while (iov_iter_count(iter)) {
		ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops,
1175
				iter, dax_iomap_actor);
1176 1177 1178 1179 1180 1181 1182 1183 1184
		if (ret <= 0)
			break;
		pos += ret;
		done += ret;
	}

	iocb->ki_pos += done;
	return done ? done : ret;
}
1185
EXPORT_SYMBOL_GPL(dax_iomap_rw);
1186

1187
static vm_fault_t dax_fault_return(int error)
1188 1189 1190 1191 1192 1193 1194 1195
{
	if (error == 0)
		return VM_FAULT_NOPAGE;
	if (error == -ENOMEM)
		return VM_FAULT_OOM;
	return VM_FAULT_SIGBUS;
}

1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206
/*
 * MAP_SYNC on a dax mapping guarantees dirty metadata is
 * flushed on write-faults (non-cow), but not read-faults.
 */
static bool dax_fault_is_synchronous(unsigned long flags,
		struct vm_area_struct *vma, struct iomap *iomap)
{
	return (flags & IOMAP_WRITE) && (vma->vm_flags & VM_SYNC)
		&& (iomap->flags & IOMAP_F_DIRTY);
}

1207
static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp,
1208
			       int *iomap_errp, const struct iomap_ops *ops)
1209
{
1210 1211
	struct vm_area_struct *vma = vmf->vma;
	struct address_space *mapping = vma->vm_file->f_mapping;
1212
	XA_STATE(xas, &mapping->i_pages, vmf->pgoff);
1213
	struct inode *inode = mapping->host;
1214
	unsigned long vaddr = vmf->address;