data.c 39.2 KB
Newer Older
Jaegeuk Kim's avatar
Jaegeuk Kim committed
1
/*
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
 * fs/f2fs/data.c
 *
 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
 *             http://www.samsung.com/
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
#include <linux/fs.h>
#include <linux/f2fs_fs.h>
#include <linux/buffer_head.h>
#include <linux/mpage.h>
#include <linux/writeback.h>
#include <linux/backing-dev.h>
Chao Yu's avatar
Chao Yu committed
17
#include <linux/pagevec.h>
18
19
#include <linux/blkdev.h>
#include <linux/bio.h>
20
#include <linux/prefetch.h>
21
#include <linux/uio.h>
22
#include <linux/cleancache.h>
23
24
25
26

#include "f2fs.h"
#include "node.h"
#include "segment.h"
Jaegeuk Kim's avatar
Jaegeuk Kim committed
27
#include "trace.h"
28
#include <trace/events/f2fs.h>
29

30
static void f2fs_read_end_io(struct bio *bio)
31
{
32
33
	struct bio_vec *bvec;
	int i;
34

35
	if (f2fs_bio_encrypted(bio)) {
36
		if (bio->bi_error) {
37
38
39
40
41
42
43
			f2fs_release_crypto_ctx(bio->bi_private);
		} else {
			f2fs_end_io_crypto_work(bio->bi_private, bio);
			return;
		}
	}

44
45
	bio_for_each_segment_all(bvec, bio, i) {
		struct page *page = bvec->bv_page;
46

47
		if (!bio->bi_error) {
48
49
50
51
52
53
54
55
56
57
			SetPageUptodate(page);
		} else {
			ClearPageUptodate(page);
			SetPageError(page);
		}
		unlock_page(page);
	}
	bio_put(bio);
}

58
static void f2fs_write_end_io(struct bio *bio)
59
{
60
	struct f2fs_sb_info *sbi = bio->bi_private;
61
62
	struct bio_vec *bvec;
	int i;
63

64
	bio_for_each_segment_all(bvec, bio, i) {
65
66
		struct page *page = bvec->bv_page;

67
68
		f2fs_restore_and_release_control_page(&page);

69
		if (unlikely(bio->bi_error)) {
70
			set_page_dirty(page);
71
			set_bit(AS_EIO, &page->mapping->flags);
72
			f2fs_stop_checkpoint(sbi);
73
74
75
		}
		end_page_writeback(page);
		dec_page_count(sbi, F2FS_WRITEBACK);
76
	}
77
78
79
80
81
82
83
84

	if (!get_pages(sbi, F2FS_WRITEBACK) &&
			!list_empty(&sbi->cp_wait.task_list))
		wake_up(&sbi->cp_wait);

	bio_put(bio);
}

85
86
87
88
89
90
91
92
/*
 * Low-level block read/write IO operations.
 */
static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
				int npages, bool is_read)
{
	struct bio *bio;

93
	bio = f2fs_bio_alloc(npages);
94
95

	bio->bi_bdev = sbi->sb->s_bdev;
Chao Yu's avatar
Chao Yu committed
96
	bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
97
	bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io;
98
	bio->bi_private = is_read ? NULL : sbi;
99
100
101
102

	return bio;
}

Jaegeuk Kim's avatar
Jaegeuk Kim committed
103
static void __submit_merged_bio(struct f2fs_bio_info *io)
104
{
Jaegeuk Kim's avatar
Jaegeuk Kim committed
105
	struct f2fs_io_info *fio = &io->fio;
106
107
108
109

	if (!io->bio)
		return;

110
	if (is_read_io(fio->rw))
111
		trace_f2fs_submit_read_bio(io->sbi->sb, fio, io->bio);
112
	else
113
		trace_f2fs_submit_write_bio(io->sbi->sb, fio, io->bio);
114

115
	submit_bio(fio->rw, io->bio);
116
117
118
119
	io->bio = NULL;
}

void f2fs_submit_merged_bio(struct f2fs_sb_info *sbi,
Jaegeuk Kim's avatar
Jaegeuk Kim committed
120
				enum page_type type, int rw)
121
122
123
124
125
126
{
	enum page_type btype = PAGE_TYPE_OF_BIO(type);
	struct f2fs_bio_info *io;

	io = is_read_io(rw) ? &sbi->read_io : &sbi->write_io[btype];

127
	down_write(&io->io_rwsem);
Jaegeuk Kim's avatar
Jaegeuk Kim committed
128
129
130
131

	/* change META to META_FLUSH in the checkpoint procedure */
	if (type >= META_FLUSH) {
		io->fio.type = META_FLUSH;
Jaegeuk Kim's avatar
Jaegeuk Kim committed
132
133
134
135
		if (test_opt(sbi, NOBARRIER))
			io->fio.rw = WRITE_FLUSH | REQ_META | REQ_PRIO;
		else
			io->fio.rw = WRITE_FLUSH_FUA | REQ_META | REQ_PRIO;
Jaegeuk Kim's avatar
Jaegeuk Kim committed
136
137
	}
	__submit_merged_bio(io);
138
	up_write(&io->io_rwsem);
139
140
141
142
143
144
}

/*
 * Fill the locked page with data located in the block address.
 * Return unlocked page.
 */
145
int f2fs_submit_page_bio(struct f2fs_io_info *fio)
146
147
{
	struct bio *bio;
148
	struct page *page = fio->encrypted_page ? fio->encrypted_page : fio->page;
149

150
	trace_f2fs_submit_page_bio(page, fio);
151
	f2fs_trace_ios(fio, 0);
152
153

	/* Allocate a new bio */
154
	bio = __bio_alloc(fio->sbi, fio->blk_addr, 1, is_read_io(fio->rw));
155
156
157
158
159
160

	if (bio_add_page(bio, page, PAGE_CACHE_SIZE, 0) < PAGE_CACHE_SIZE) {
		bio_put(bio);
		return -EFAULT;
	}

161
	submit_bio(fio->rw, bio);
162
163
164
	return 0;
}

165
void f2fs_submit_page_mbio(struct f2fs_io_info *fio)
166
{
167
	struct f2fs_sb_info *sbi = fio->sbi;
Jaegeuk Kim's avatar
Jaegeuk Kim committed
168
	enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
169
	struct f2fs_bio_info *io;
170
	bool is_read = is_read_io(fio->rw);
171
	struct page *bio_page;
172

173
	io = is_read ? &sbi->read_io : &sbi->write_io[btype];
174

175
	verify_block_addr(sbi, fio->blk_addr);
176

177
	down_write(&io->io_rwsem);
178

179
	if (!is_read)
180
181
		inc_page_count(sbi, F2FS_WRITEBACK);

182
	if (io->bio && (io->last_block_in_bio != fio->blk_addr - 1 ||
Jaegeuk Kim's avatar
Jaegeuk Kim committed
183
184
						io->fio.rw != fio->rw))
		__submit_merged_bio(io);
185
186
alloc_new:
	if (io->bio == NULL) {
Jaegeuk Kim's avatar
Jaegeuk Kim committed
187
		int bio_blocks = MAX_BIO_BLOCKS(sbi);
188

189
		io->bio = __bio_alloc(sbi, fio->blk_addr, bio_blocks, is_read);
Jaegeuk Kim's avatar
Jaegeuk Kim committed
190
		io->fio = *fio;
191
192
	}

193
194
195
	bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page;

	if (bio_add_page(io->bio, bio_page, PAGE_CACHE_SIZE, 0) <
196
							PAGE_CACHE_SIZE) {
Jaegeuk Kim's avatar
Jaegeuk Kim committed
197
		__submit_merged_bio(io);
198
199
200
		goto alloc_new;
	}

201
	io->last_block_in_bio = fio->blk_addr;
202
	f2fs_trace_ios(fio, 0);
203

204
	up_write(&io->io_rwsem);
205
	trace_f2fs_submit_page_mbio(fio->page, fio);
206
207
}

Jaegeuk Kim's avatar
Jaegeuk Kim committed
208
/*
209
210
211
212
213
 * Lock ordering for the change of data block address:
 * ->data_page
 *  ->node_page
 *    update block addresses in the node page
 */
214
void set_data_blkaddr(struct dnode_of_data *dn)
215
216
217
218
219
220
{
	struct f2fs_node *rn;
	__le32 *addr_array;
	struct page *node_page = dn->node_page;
	unsigned int ofs_in_node = dn->ofs_in_node;

221
	f2fs_wait_on_page_writeback(node_page, NODE);
222

223
	rn = F2FS_NODE(node_page);
224
225
226

	/* Get physical address of data block */
	addr_array = blkaddr_in_node(rn);
227
	addr_array[ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
228
229
230
231
232
	set_page_dirty(node_page);
}

int reserve_new_block(struct dnode_of_data *dn)
{
233
	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
234

235
	if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
236
		return -EPERM;
237
	if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))
238
239
		return -ENOSPC;

240
241
	trace_f2fs_reserve_new_block(dn->inode, dn->nid, dn->ofs_in_node);

242
	dn->data_blkaddr = NEW_ADDR;
243
	set_data_blkaddr(dn);
244
	mark_inode_dirty(dn->inode);
245
246
247
248
	sync_inode_page(dn);
	return 0;
}

249
250
251
252
253
254
255
256
int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
{
	bool need_put = dn->inode_page ? false : true;
	int err;

	err = get_dnode_of_data(dn, index, ALLOC_NODE);
	if (err)
		return err;
257

258
259
	if (dn->data_blkaddr == NULL_ADDR)
		err = reserve_new_block(dn);
260
	if (err || need_put)
261
262
263
264
		f2fs_put_dnode(dn);
	return err;
}

265
int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index)
266
{
267
	struct extent_info ei;
268
	struct inode *inode = dn->inode;
269

270
271
272
	if (f2fs_lookup_extent_cache(inode, index, &ei)) {
		dn->data_blkaddr = ei.blk + index - ei.fofs;
		return 0;
273
	}
274

275
	return f2fs_reserve_block(dn, index);
276
277
}

278
279
struct page *get_read_data_page(struct inode *inode, pgoff_t index,
						int rw, bool for_write)
280
281
282
283
{
	struct address_space *mapping = inode->i_mapping;
	struct dnode_of_data dn;
	struct page *page;
Chao Yu's avatar
Chao Yu committed
284
	struct extent_info ei;
285
	int err;
286
	struct f2fs_io_info fio = {
287
		.sbi = F2FS_I_SB(inode),
288
		.type = DATA,
289
		.rw = rw,
290
		.encrypted_page = NULL,
291
	};
292

293
294
295
	if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode))
		return read_mapping_page(mapping, index, NULL);

296
	page = f2fs_grab_cache_page(mapping, index, for_write);
297
298
299
	if (!page)
		return ERR_PTR(-ENOMEM);

Chao Yu's avatar
Chao Yu committed
300
301
302
303
304
	if (f2fs_lookup_extent_cache(inode, index, &ei)) {
		dn.data_blkaddr = ei.blk + index - ei.fofs;
		goto got_it;
	}

305
	set_new_dnode(&dn, inode, NULL, NULL, 0);
306
	err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
307
308
	if (err)
		goto put_err;
309
310
	f2fs_put_dnode(&dn);

311
	if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
312
313
		err = -ENOENT;
		goto put_err;
314
	}
Chao Yu's avatar
Chao Yu committed
315
got_it:
316
317
	if (PageUptodate(page)) {
		unlock_page(page);
318
		return page;
319
	}
320

Jaegeuk Kim's avatar
Jaegeuk Kim committed
321
322
323
324
325
326
327
328
329
	/*
	 * A new dentry page is allocated but not able to be written, since its
	 * new inode page couldn't be allocated due to -ENOSPC.
	 * In such the case, its blkaddr can be remained as NEW_ADDR.
	 * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata.
	 */
	if (dn.data_blkaddr == NEW_ADDR) {
		zero_user_segment(page, 0, PAGE_CACHE_SIZE);
		SetPageUptodate(page);
330
		unlock_page(page);
Jaegeuk Kim's avatar
Jaegeuk Kim committed
331
332
		return page;
	}
333

334
	fio.blk_addr = dn.data_blkaddr;
335
336
	fio.page = page;
	err = f2fs_submit_page_bio(&fio);
337
	if (err)
338
		goto put_err;
339
	return page;
340
341
342
343

put_err:
	f2fs_put_page(page, 1);
	return ERR_PTR(err);
344
345
346
347
348
349
350
351
352
353
354
355
}

struct page *find_data_page(struct inode *inode, pgoff_t index)
{
	struct address_space *mapping = inode->i_mapping;
	struct page *page;

	page = find_get_page(mapping, index);
	if (page && PageUptodate(page))
		return page;
	f2fs_put_page(page, 0);

356
	page = get_read_data_page(inode, index, READ_SYNC, false);
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
	if (IS_ERR(page))
		return page;

	if (PageUptodate(page))
		return page;

	wait_on_page_locked(page);
	if (unlikely(!PageUptodate(page))) {
		f2fs_put_page(page, 0);
		return ERR_PTR(-EIO);
	}
	return page;
}

/*
 * If it tries to access a hole, return an error.
 * Because, the callers, functions in dir.c and GC, should be able to know
 * whether this page exists or not.
 */
376
377
struct page *get_lock_data_page(struct inode *inode, pgoff_t index,
							bool for_write)
378
379
380
381
{
	struct address_space *mapping = inode->i_mapping;
	struct page *page;
repeat:
382
	page = get_read_data_page(inode, index, READ_SYNC, for_write);
383
384
	if (IS_ERR(page))
		return page;
385

386
	/* wait for read completion */
387
	lock_page(page);
388
	if (unlikely(!PageUptodate(page))) {
389
390
		f2fs_put_page(page, 1);
		return ERR_PTR(-EIO);
391
	}
392
	if (unlikely(page->mapping != mapping)) {
393
394
		f2fs_put_page(page, 1);
		goto repeat;
395
396
397
398
	}
	return page;
}

Jaegeuk Kim's avatar
Jaegeuk Kim committed
399
/*
400
401
 * Caller ensures that this data page is never allocated.
 * A new zero-filled data page is allocated in the page cache.
402
 *
Chao Yu's avatar
Chao Yu committed
403
404
 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
 * f2fs_unlock_op().
405
406
 * Note that, ipage is set only by make_empty_dir, and if any error occur,
 * ipage should be released by this function.
407
 */
408
struct page *get_new_data_page(struct inode *inode,
409
		struct page *ipage, pgoff_t index, bool new_i_size)
410
411
412
413
414
{
	struct address_space *mapping = inode->i_mapping;
	struct page *page;
	struct dnode_of_data dn;
	int err;
415
repeat:
416
	page = f2fs_grab_cache_page(mapping, index, true);
417
418
419
420
421
422
	if (!page) {
		/*
		 * before exiting, we should make sure ipage will be released
		 * if any error occur.
		 */
		f2fs_put_page(ipage, 1);
423
		return ERR_PTR(-ENOMEM);
424
	}
425

426
	set_new_dnode(&dn, inode, ipage, NULL, 0);
427
	err = f2fs_reserve_block(&dn, index);
428
429
	if (err) {
		f2fs_put_page(page, 1);
430
		return ERR_PTR(err);
431
	}
432
433
	if (!ipage)
		f2fs_put_dnode(&dn);
434
435

	if (PageUptodate(page))
436
		goto got_it;
437
438
439

	if (dn.data_blkaddr == NEW_ADDR) {
		zero_user_segment(page, 0, PAGE_CACHE_SIZE);
440
		SetPageUptodate(page);
441
	} else {
442
		f2fs_put_page(page, 1);
443

444
		page = get_read_data_page(inode, index, READ_SYNC, true);
445
		if (IS_ERR(page))
446
			goto repeat;
447
448
449

		/* wait for read completion */
		lock_page(page);
450
	}
451
got_it:
452
453
454
	if (new_i_size && i_size_read(inode) <
				((loff_t)(index + 1) << PAGE_CACHE_SHIFT)) {
		i_size_write(inode, ((loff_t)(index + 1) << PAGE_CACHE_SHIFT));
455
456
		/* Only the directory inode sets new_i_size */
		set_inode_flag(F2FS_I(inode), FI_UPDATE_DIR);
457
458
459
460
	}
	return page;
}

461
462
static int __allocate_data_block(struct dnode_of_data *dn)
{
463
	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
464
	struct f2fs_inode_info *fi = F2FS_I(dn->inode);
465
466
	struct f2fs_summary sum;
	struct node_info ni;
467
	int seg = CURSEG_WARM_DATA;
468
	pgoff_t fofs;
469
470
471

	if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
		return -EPERM;
472
473
474
475
476

	dn->data_blkaddr = datablock_addr(dn->node_page, dn->ofs_in_node);
	if (dn->data_blkaddr == NEW_ADDR)
		goto alloc;

477
478
479
	if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))
		return -ENOSPC;

480
alloc:
481
482
483
	get_node_info(sbi, dn->nid, &ni);
	set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);

484
485
486
	if (dn->ofs_in_node == 0 && dn->inode_page == dn->node_page)
		seg = CURSEG_DIRECT_IO;

487
488
	allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr,
								&sum, seg);
489
	set_data_blkaddr(dn);
490

491
492
493
	/* update i_size */
	fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) +
							dn->ofs_in_node;
494
495
496
	if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_CACHE_SHIFT))
		i_size_write(dn->inode,
				((loff_t)(fofs + 1) << PAGE_CACHE_SHIFT));
497

498
	/* direct IO doesn't use extent cache to maximize the performance */
499
	f2fs_drop_largest_extent(dn->inode, fofs);
500

501
502
503
	return 0;
}

504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
static void __allocate_data_blocks(struct inode *inode, loff_t offset,
							size_t count)
{
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
	struct dnode_of_data dn;
	u64 start = F2FS_BYTES_TO_BLK(offset);
	u64 len = F2FS_BYTES_TO_BLK(count);
	bool allocated;
	u64 end_offset;

	while (len) {
		f2fs_balance_fs(sbi);
		f2fs_lock_op(sbi);

		/* When reading holes, we need its node page */
		set_new_dnode(&dn, inode, NULL, NULL, 0);
		if (get_dnode_of_data(&dn, start, ALLOC_NODE))
			goto out;

		allocated = false;
		end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode));

		while (dn.ofs_in_node < end_offset && len) {
527
528
			block_t blkaddr;

529
530
531
			if (unlikely(f2fs_cp_error(sbi)))
				goto sync_out;

532
			blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node);
533
			if (blkaddr == NULL_ADDR || blkaddr == NEW_ADDR) {
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
				if (__allocate_data_block(&dn))
					goto sync_out;
				allocated = true;
			}
			len--;
			start++;
			dn.ofs_in_node++;
		}

		if (allocated)
			sync_inode_page(&dn);

		f2fs_put_dnode(&dn);
		f2fs_unlock_op(sbi);
	}
	return;

sync_out:
	if (allocated)
		sync_inode_page(&dn);
	f2fs_put_dnode(&dn);
out:
	f2fs_unlock_op(sbi);
	return;
}

Jaegeuk Kim's avatar
Jaegeuk Kim committed
560
/*
Jaegeuk Kim's avatar
Jaegeuk Kim committed
561
562
 * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with
 * f2fs_map_blocks structure.
Chao Yu's avatar
Chao Yu committed
563
564
565
566
567
 * If original data blocks are allocated, then give them to blockdev.
 * Otherwise,
 *     a. preallocate requested block addresses
 *     b. do not use extent cache for better performance
 *     c. give the block addresses to blockdev
568
 */
Jaegeuk Kim's avatar
Jaegeuk Kim committed
569
static int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
Chao Yu's avatar
Chao Yu committed
570
						int create, int flag)
571
{
Jaegeuk Kim's avatar
Jaegeuk Kim committed
572
	unsigned int maxblocks = map->m_len;
573
	struct dnode_of_data dn;
574
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
575
576
577
	int mode = create ? ALLOC_NODE : LOOKUP_NODE_RA;
	pgoff_t pgofs, end_offset;
	int err = 0, ofs = 1;
578
	struct extent_info ei;
579
	bool allocated = false;
580

Jaegeuk Kim's avatar
Jaegeuk Kim committed
581
582
583
584
585
	map->m_len = 0;
	map->m_flags = 0;

	/* it only supports block size == page size */
	pgofs =	(pgoff_t)map->m_lblk;
586

587
	if (f2fs_lookup_extent_cache(inode, pgofs, &ei)) {
Jaegeuk Kim's avatar
Jaegeuk Kim committed
588
589
590
		map->m_pblk = ei.blk + pgofs - ei.fofs;
		map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs);
		map->m_flags = F2FS_MAP_MAPPED;
591
		goto out;
592
	}
593

594
	if (create)
595
		f2fs_lock_op(F2FS_I_SB(inode));
596
597
598

	/* When reading holes, we need its node page */
	set_new_dnode(&dn, inode, NULL, NULL, 0);
599
	err = get_dnode_of_data(&dn, pgofs, mode);
600
	if (err) {
601
602
603
		if (err == -ENOENT)
			err = 0;
		goto unlock_out;
604
	}
Chao Yu's avatar
Chao Yu committed
605
606
607

	if (dn.data_blkaddr == NEW_ADDR || dn.data_blkaddr == NULL_ADDR) {
		if (create) {
608
609
610
611
			if (unlikely(f2fs_cp_error(sbi))) {
				err = -EIO;
				goto put_out;
			}
Chao Yu's avatar
Chao Yu committed
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
			err = __allocate_data_block(&dn);
			if (err)
				goto put_out;
			allocated = true;
			map->m_flags = F2FS_MAP_NEW;
		} else {
			if (flag != F2FS_GET_BLOCK_FIEMAP ||
						dn.data_blkaddr != NEW_ADDR) {
				if (flag == F2FS_GET_BLOCK_BMAP)
					err = -ENOENT;
				goto put_out;
			}

			/*
			 * preallocated unwritten block should be mapped
			 * for fiemap.
			 */
			if (dn.data_blkaddr == NEW_ADDR)
				map->m_flags = F2FS_MAP_UNWRITTEN;
Chao Yu's avatar
Chao Yu committed
631
632
		}
	}
633

Chao Yu's avatar
Chao Yu committed
634
635
636
	map->m_flags |= F2FS_MAP_MAPPED;
	map->m_pblk = dn.data_blkaddr;
	map->m_len = 1;
637

638
	end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode));
639
640
641
642
643
644
645
646
647
648
649
650
	dn.ofs_in_node++;
	pgofs++;

get_next:
	if (dn.ofs_in_node >= end_offset) {
		if (allocated)
			sync_inode_page(&dn);
		allocated = false;
		f2fs_put_dnode(&dn);

		set_new_dnode(&dn, inode, NULL, NULL, 0);
		err = get_dnode_of_data(&dn, pgofs, mode);
651
		if (err) {
652
653
654
655
			if (err == -ENOENT)
				err = 0;
			goto unlock_out;
		}
Chao Yu's avatar
Chao Yu committed
656

657
		end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode));
658
	}
659

Jaegeuk Kim's avatar
Jaegeuk Kim committed
660
	if (maxblocks > map->m_len) {
661
		block_t blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node);
Chao Yu's avatar
Chao Yu committed
662
663
664

		if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR) {
			if (create) {
665
666
667
668
				if (unlikely(f2fs_cp_error(sbi))) {
					err = -EIO;
					goto sync_out;
				}
Chao Yu's avatar
Chao Yu committed
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
				err = __allocate_data_block(&dn);
				if (err)
					goto sync_out;
				allocated = true;
				map->m_flags |= F2FS_MAP_NEW;
				blkaddr = dn.data_blkaddr;
			} else {
				/*
				 * we only merge preallocated unwritten blocks
				 * for fiemap.
				 */
				if (flag != F2FS_GET_BLOCK_FIEMAP ||
						blkaddr != NEW_ADDR)
					goto sync_out;
			}
684
		}
Chao Yu's avatar
Chao Yu committed
685

arter97's avatar
arter97 committed
686
		/* Give more consecutive addresses for the readahead */
687
688
689
690
		if ((map->m_pblk != NEW_ADDR &&
				blkaddr == (map->m_pblk + ofs)) ||
				(map->m_pblk == NEW_ADDR &&
				blkaddr == NEW_ADDR)) {
691
692
693
			ofs++;
			dn.ofs_in_node++;
			pgofs++;
Jaegeuk Kim's avatar
Jaegeuk Kim committed
694
			map->m_len++;
695
696
			goto get_next;
		}
697
	}
698
699
700
701
sync_out:
	if (allocated)
		sync_inode_page(&dn);
put_out:
702
	f2fs_put_dnode(&dn);
703
704
unlock_out:
	if (create)
705
		f2fs_unlock_op(F2FS_I_SB(inode));
706
out:
Jaegeuk Kim's avatar
Jaegeuk Kim committed
707
	trace_f2fs_map_blocks(inode, map, err);
708
	return err;
709
710
}

Jaegeuk Kim's avatar
Jaegeuk Kim committed
711
static int __get_data_block(struct inode *inode, sector_t iblock,
Chao Yu's avatar
Chao Yu committed
712
			struct buffer_head *bh, int create, int flag)
Jaegeuk Kim's avatar
Jaegeuk Kim committed
713
714
715
716
717
718
719
{
	struct f2fs_map_blocks map;
	int ret;

	map.m_lblk = iblock;
	map.m_len = bh->b_size >> inode->i_blkbits;

Chao Yu's avatar
Chao Yu committed
720
	ret = f2fs_map_blocks(inode, &map, create, flag);
Jaegeuk Kim's avatar
Jaegeuk Kim committed
721
722
723
724
725
726
727
728
	if (!ret) {
		map_bh(bh, inode->i_sb, map.m_pblk);
		bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags;
		bh->b_size = map.m_len << inode->i_blkbits;
	}
	return ret;
}

729
static int get_data_block(struct inode *inode, sector_t iblock,
Chao Yu's avatar
Chao Yu committed
730
731
732
733
734
735
			struct buffer_head *bh_result, int create, int flag)
{
	return __get_data_block(inode, iblock, bh_result, create, flag);
}

static int get_data_block_dio(struct inode *inode, sector_t iblock,
736
737
			struct buffer_head *bh_result, int create)
{
Chao Yu's avatar
Chao Yu committed
738
739
	return __get_data_block(inode, iblock, bh_result, create,
						F2FS_GET_BLOCK_DIO);
740
741
}

Chao Yu's avatar
Chao Yu committed
742
static int get_data_block_bmap(struct inode *inode, sector_t iblock,
743
744
			struct buffer_head *bh_result, int create)
{
Chao Yu's avatar
Chao Yu committed
745
746
	return __get_data_block(inode, iblock, bh_result, create,
						F2FS_GET_BLOCK_BMAP);
747
748
}

749
750
751
752
753
754
755
756
757
758
static inline sector_t logical_to_blk(struct inode *inode, loff_t offset)
{
	return (offset >> inode->i_blkbits);
}

static inline loff_t blk_to_logical(struct inode *inode, sector_t blk)
{
	return (blk << inode->i_blkbits);
}

Jaegeuk Kim's avatar
Jaegeuk Kim committed
759
760
761
int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
		u64 start, u64 len)
{
762
763
764
765
766
767
768
769
770
771
772
773
	struct buffer_head map_bh;
	sector_t start_blk, last_blk;
	loff_t isize = i_size_read(inode);
	u64 logical = 0, phys = 0, size = 0;
	u32 flags = 0;
	bool past_eof = false, whole_file = false;
	int ret = 0;

	ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC);
	if (ret)
		return ret;

774
775
776
777
778
779
	if (f2fs_has_inline_data(inode)) {
		ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
		if (ret != -EAGAIN)
			return ret;
	}

780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
	mutex_lock(&inode->i_mutex);

	if (len >= isize) {
		whole_file = true;
		len = isize;
	}

	if (logical_to_blk(inode, len) == 0)
		len = blk_to_logical(inode, 1);

	start_blk = logical_to_blk(inode, start);
	last_blk = logical_to_blk(inode, start + len - 1);
next:
	memset(&map_bh, 0, sizeof(struct buffer_head));
	map_bh.b_size = len;

Chao Yu's avatar
Chao Yu committed
796
797
	ret = get_data_block(inode, start_blk, &map_bh, 0,
					F2FS_GET_BLOCK_FIEMAP);
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
	if (ret)
		goto out;

	/* HOLE */
	if (!buffer_mapped(&map_bh)) {
		start_blk++;

		if (!past_eof && blk_to_logical(inode, start_blk) >= isize)
			past_eof = 1;

		if (past_eof && size) {
			flags |= FIEMAP_EXTENT_LAST;
			ret = fiemap_fill_next_extent(fieinfo, logical,
					phys, size, flags);
		} else if (size) {
			ret = fiemap_fill_next_extent(fieinfo, logical,
					phys, size, flags);
			size = 0;
		}

		/* if we have holes up to/past EOF then we're done */
		if (start_blk > last_blk || past_eof || ret)
			goto out;
	} else {
		if (start_blk > last_blk && !whole_file) {
			ret = fiemap_fill_next_extent(fieinfo, logical,
					phys, size, flags);
			goto out;
		}

		/*
		 * if size != 0 then we know we already have an extent
		 * to add, so add it.
		 */
		if (size) {
			ret = fiemap_fill_next_extent(fieinfo, logical,
					phys, size, flags);
			if (ret)
				goto out;
		}

		logical = blk_to_logical(inode, start_blk);
		phys = blk_to_logical(inode, map_bh.b_blocknr);
		size = map_bh.b_size;
		flags = 0;
		if (buffer_unwritten(&map_bh))
			flags = FIEMAP_EXTENT_UNWRITTEN;

		start_blk += logical_to_blk(inode, size);

		/*
		 * If we are past the EOF, then we need to make sure as
		 * soon as we find a hole that the last extent we found
		 * is marked with FIEMAP_EXTENT_LAST
		 */
		if (!past_eof && logical + size >= isize)
			past_eof = true;
	}
	cond_resched();
	if (fatal_signal_pending(current))
		ret = -EINTR;
	else
		goto next;
out:
	if (ret == 1)
		ret = 0;

	mutex_unlock(&inode->i_mutex);
	return ret;
Jaegeuk Kim's avatar
Jaegeuk Kim committed
867
868
}

869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
/*
 * This function was originally taken from fs/mpage.c, and customized for f2fs.
 * Major change was from block_size == page_size in f2fs by default.
 */
static int f2fs_mpage_readpages(struct address_space *mapping,
			struct list_head *pages, struct page *page,
			unsigned nr_pages)
{
	struct bio *bio = NULL;
	unsigned page_idx;
	sector_t last_block_in_bio = 0;
	struct inode *inode = mapping->host;
	const unsigned blkbits = inode->i_blkbits;
	const unsigned blocksize = 1 << blkbits;
	sector_t block_in_file;
	sector_t last_block;
	sector_t last_block_in_file;
	sector_t block_nr;
	struct block_device *bdev = inode->i_sb->s_bdev;
	struct f2fs_map_blocks map;

	map.m_pblk = 0;
	map.m_lblk = 0;
	map.m_len = 0;
	map.m_flags = 0;

	for (page_idx = 0; nr_pages; page_idx++, nr_pages--) {

		prefetchw(&page->flags);
		if (pages) {
			page = list_entry(pages->prev, struct page, lru);
			list_del(&page->lru);
			if (add_to_page_cache_lru(page, mapping,
						  page->index, GFP_KERNEL))
				goto next_page;
		}

		block_in_file = (sector_t)page->index;
		last_block = block_in_file + nr_pages;
		last_block_in_file = (i_size_read(inode) + blocksize - 1) >>
								blkbits;
		if (last_block > last_block_in_file)
			last_block = last_block_in_file;

		/*
		 * Map blocks using the previous result first.
		 */
		if ((map.m_flags & F2FS_MAP_MAPPED) &&
				block_in_file > map.m_lblk &&
				block_in_file < (map.m_lblk + map.m_len))
			goto got_it;

		/*
		 * Then do more f2fs_map_blocks() calls until we are
		 * done with this page.
		 */
		map.m_flags = 0;

		if (block_in_file < last_block) {
			map.m_lblk = block_in_file;
			map.m_len = last_block - block_in_file;

931
932
			if (f2fs_map_blocks(inode, &map, 0,
							F2FS_GET_BLOCK_READ))
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
				goto set_error_page;
		}
got_it:
		if ((map.m_flags & F2FS_MAP_MAPPED)) {
			block_nr = map.m_pblk + block_in_file - map.m_lblk;
			SetPageMappedToDisk(page);

			if (!PageUptodate(page) && !cleancache_get_page(page)) {
				SetPageUptodate(page);
				goto confused;
			}
		} else {
			zero_user_segment(page, 0, PAGE_CACHE_SIZE);
			SetPageUptodate(page);
			unlock_page(page);
			goto next_page;
		}

		/*
		 * This page will go to BIO.  Do we need to send this
		 * BIO off first?
		 */
		if (bio && (last_block_in_bio != block_nr - 1)) {
submit_and_realloc:
			submit_bio(READ, bio);
			bio = NULL;
		}
		if (bio == NULL) {
961
962
963
964
965
966
967
968
969
970
			struct f2fs_crypto_ctx *ctx = NULL;

			if (f2fs_encrypted_inode(inode) &&
					S_ISREG(inode->i_mode)) {

				ctx = f2fs_get_crypto_ctx(inode);
				if (IS_ERR(ctx))
					goto set_error_page;

				/* wait the page to be moved by cleaning */
971
972
				f2fs_wait_on_encrypted_page_writeback(
						F2FS_I_SB(inode), block_nr);
973
974
			}

975
			bio = bio_alloc(GFP_KERNEL,
976
				min_t(int, nr_pages, BIO_MAX_PAGES));
977
978
979
			if (!bio) {
				if (ctx)
					f2fs_release_crypto_ctx(ctx);
980
				goto set_error_page;
981
			}
982
983
			bio->bi_bdev = bdev;
			bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(block_nr);
984
			bio->bi_end_io = f2fs_read_end_io;
985
			bio->bi_private = ctx;
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
		}

		if (bio_add_page(bio, page, blocksize, 0) < blocksize)
			goto submit_and_realloc;

		last_block_in_bio = block_nr;
		goto next_page;
set_error_page:
		SetPageError(page);
		zero_user_segment(page, 0, PAGE_CACHE_SIZE);
		unlock_page(page);
		goto next_page;
confused:
		if (bio) {
			submit_bio(READ, bio);
			bio = NULL;
		}
		unlock_page(page);
next_page:
		if (pages)
			page_cache_release(page);
	}
	BUG_ON(pages && !list_empty(pages));
	if (bio)
		submit_bio(READ, bio);
	return 0;
}

1014
1015
static int f2fs_read_data_page(struct file *file, struct page *page)
{
1016
	struct inode *inode = page->mapping->host;
1017
	int ret = -EAGAIN;
1018

1019
1020
	trace_f2fs_readpage(page, DATA);

arter97's avatar
arter97 committed
1021
	/* If the file has inline data, try to read it directly */
1022
1023
	if (f2fs_has_inline_data(inode))
		ret = f2fs_read_inline_data(inode, page);
1024
	if (ret == -EAGAIN)
1025
		ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1);
1026
	return ret;
1027
1028
1029
1030
1031
1032
}

static int f2fs_read_data_pages(struct file *file,
			struct address_space *mapping,
			struct list_head *pages, unsigned nr_pages)
{
1033
	struct inode *inode = file->f_mapping->host;
1034
1035
1036
	struct page *page = list_entry(pages->prev, struct page, lru);

	trace_f2fs_readpages(inode, page, nr_pages);
1037
1038
1039
1040
1041

	/* If the file has inline data, skip readpages */
	if (f2fs_has_inline_data(inode))
		return 0;

1042
	return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages);
1043
1044
}

1045
int do_write_data_page(struct f2fs_io_info *fio)
1046
{
1047
	struct page *page = fio->page;
1048
1049
1050
1051
1052
	struct inode *inode = page->mapping->host;
	struct dnode_of_data dn;
	int err = 0;

	set_new_dnode(&dn, inode, NULL, NULL, 0);
1053
	err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
1054
1055
1056
	if (err)
		return err;

1057
	fio->blk_addr = dn.data_blkaddr;
1058
1059

	/* This page is already truncated */
1060
1061
	if (fio->blk_addr == NULL_ADDR) {
		ClearPageUptodate(page);
1062
		goto out_writepage;
1063
	}
1064

1065
	if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) {
1066
1067
1068
1069
1070

		/* wait for GCed encrypted page writeback */
		f2fs_wait_on_encrypted_page_writeback(F2FS_I_SB(inode),
							fio->blk_addr);

1071
1072
1073
1074
1075
1076
1077
		fio->encrypted_page = f2fs_encrypt(inode, fio->page);
		if (IS_ERR(fio->encrypted_page)) {
			err = PTR_ERR(fio->encrypted_page);
			goto out_writepage;
		}
	}

1078
1079
1080
1081
1082
1083
	set_page_writeback(page);

	/*
	 * If current allocation needs SSR,
	 * it had better in-place writes for updated data.
	 */
1084
	if (unlikely(fio->blk_addr != NEW_ADDR &&
1085
			!is_cold_data(page) &&
1086
			!IS_ATOMIC_WRITTEN_PAGE(page) &&
1087
			need_inplace_update(inode))) {
1088
		rewrite_data_page(fio);
1089
		set_inode_flag(F2FS_I(inode), FI_UPDATE_WRITE);
1090
		trace_f2fs_do_write_data_page(page, IPU);
1091
	} else {
1092
		write_data_page(&dn, fio);
1093
		set_data_blkaddr(&dn);
1094
		f2fs_update_extent_cache(&dn);
1095
		trace_f2fs_do_write_data_page(page, OPU);
1096
		set_inode_flag(F2FS_I(inode), FI_APPEND_WRITE);
1097
1098
		if (page->index == 0)
			set_inode_flag(F2FS_I(inode), FI_FIRST_BLOCK_WRITTEN);
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
	}
out_writepage:
	f2fs_put_dnode(&dn);
	return err;
}

static int f2fs_write_data_page(struct page *page,
					struct writeback_control *wbc)
{
	struct inode *inode = page->mapping->host;
1109
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1110
1111
1112
	loff_t i_size = i_size_read(inode);
	const pgoff_t end_index = ((unsigned long long) i_size)
							>> PAGE_CACHE_SHIFT;
1113
	unsigned offset = 0;
1114
	bool need_balance_fs = false;
1115
	int err = 0;
Jaegeuk Kim's avatar
Jaegeuk Kim committed
1116
	struct f2fs_io_info fio = {
1117
		.sbi = sbi,
Jaegeuk Kim's avatar
Jaegeuk Kim committed
1118
		.type = DATA,
Chris Fries's avatar
Chris Fries committed
1119
		.rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE,
1120
		.page = page,
1121
		.encrypted_page = NULL,
Jaegeuk Kim's avatar
Jaegeuk Kim committed
1122
	};
1123

1124
1125
	trace_f2fs_writepage(page, DATA);