blk-lib.c 10.2 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2 3 4 5 6 7 8 9 10 11 12
/*
 * Functions related to generic helpers functions
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/scatterlist.h>

#include "blk.h"

13
struct bio *blk_next_bio(struct bio *bio, unsigned int nr_pages, gfp_t gfp)
14
{
15 16 17 18
	struct bio *new = bio_alloc(gfp, nr_pages);

	if (bio) {
		bio_chain(bio, new);
19
		submit_bio(bio);
20
	}
21

22
	return new;
23 24
}

25
int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
26
		sector_t nr_sects, gfp_t gfp_mask, int flags,
27
		struct bio **biop)
28 29
{
	struct request_queue *q = bdev_get_queue(bdev);
30
	struct bio *bio = *biop;
31
	unsigned int op;
32
	sector_t bs_mask;
33 34 35

	if (!q)
		return -ENXIO;
36

37 38 39
	if (bdev_read_only(bdev))
		return -EPERM;

40 41 42 43 44 45 46 47 48
	if (flags & BLKDEV_DISCARD_SECURE) {
		if (!blk_queue_secure_erase(q))
			return -EOPNOTSUPP;
		op = REQ_OP_SECURE_ERASE;
	} else {
		if (!blk_queue_discard(q))
			return -EOPNOTSUPP;
		op = REQ_OP_DISCARD;
	}
49

50 51 52 53
	bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
	if ((sector | nr_sects) & bs_mask)
		return -EINVAL;

54 55
	if (!nr_sects)
		return -EINVAL;
56

57
	while (nr_sects) {
58
		sector_t req_sects = min_t(sector_t, nr_sects,
59
				bio_allowed_max_sectors(q));
60

61 62
		WARN_ON_ONCE((req_sects << 9) > UINT_MAX);

63
		bio = blk_next_bio(bio, 0, gfp_mask);
64
		bio->bi_iter.bi_sector = sector;
65
		bio_set_dev(bio, bdev);
66
		bio_set_op_attrs(bio, op, 0);
67

68
		bio->bi_iter.bi_size = req_sects << 9;
69
		sector += req_sects;
70
		nr_sects -= req_sects;
71

72 73 74 75 76 77 78
		/*
		 * We can loop for a long time in here, if someone does
		 * full device discards (like mkfs). Be nice and allow
		 * us to schedule out to avoid softlocking if preempt
		 * is disabled.
		 */
		cond_resched();
79
	}
80 81 82 83 84 85 86 87 88 89 90 91

	*biop = bio;
	return 0;
}
EXPORT_SYMBOL(__blkdev_issue_discard);

/**
 * blkdev_issue_discard - queue a discard
 * @bdev:	blockdev to issue discard for
 * @sector:	start sector
 * @nr_sects:	number of sectors to discard
 * @gfp_mask:	memory allocation flags (for bio_alloc)
92
 * @flags:	BLKDEV_DISCARD_* flags to control behaviour
93 94 95 96 97 98 99 100 101 102 103 104
 *
 * Description:
 *    Issue a discard request for the sectors in question.
 */
int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
		sector_t nr_sects, gfp_t gfp_mask, unsigned long flags)
{
	struct bio *bio = NULL;
	struct blk_plug plug;
	int ret;

	blk_start_plug(&plug);
105
	ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, flags,
106
			&bio);
107
	if (!ret && bio) {
108
		ret = submit_bio_wait(bio);
109
		if (ret == -EOPNOTSUPP)
110
			ret = 0;
111
		bio_put(bio);
112
	}
113
	blk_finish_plug(&plug);
114

115
	return ret;
116 117
}
EXPORT_SYMBOL(blkdev_issue_discard);
118

119
/**
120
 * __blkdev_issue_write_same - generate number of bios with same page
121 122 123 124 125
 * @bdev:	target blockdev
 * @sector:	start sector
 * @nr_sects:	number of sectors to write
 * @gfp_mask:	memory allocation flags (for bio_alloc)
 * @page:	page containing data to write
126
 * @biop:	pointer to anchor bio
127 128
 *
 * Description:
129
 *  Generate and issue number of bios(REQ_OP_WRITE_SAME) with same page.
130
 */
131 132 133
static int __blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
		sector_t nr_sects, gfp_t gfp_mask, struct page *page,
		struct bio **biop)
134 135 136
{
	struct request_queue *q = bdev_get_queue(bdev);
	unsigned int max_write_same_sectors;
137
	struct bio *bio = *biop;
138
	sector_t bs_mask;
139 140 141 142

	if (!q)
		return -ENXIO;

143 144 145
	if (bdev_read_only(bdev))
		return -EPERM;

146 147 148 149
	bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
	if ((sector | nr_sects) & bs_mask)
		return -EINVAL;

150 151 152
	if (!bdev_write_same(bdev))
		return -EOPNOTSUPP;

153
	/* Ensure that max_write_same_sectors doesn't overflow bi_size */
154
	max_write_same_sectors = bio_allowed_max_sectors(q);
155 156

	while (nr_sects) {
157
		bio = blk_next_bio(bio, 1, gfp_mask);
158
		bio->bi_iter.bi_sector = sector;
159
		bio_set_dev(bio, bdev);
160 161 162 163
		bio->bi_vcnt = 1;
		bio->bi_io_vec->bv_page = page;
		bio->bi_io_vec->bv_offset = 0;
		bio->bi_io_vec->bv_len = bdev_logical_block_size(bdev);
164
		bio_set_op_attrs(bio, REQ_OP_WRITE_SAME, 0);
165 166

		if (nr_sects > max_write_same_sectors) {
167
			bio->bi_iter.bi_size = max_write_same_sectors << 9;
168 169 170
			nr_sects -= max_write_same_sectors;
			sector += max_write_same_sectors;
		} else {
171
			bio->bi_iter.bi_size = nr_sects << 9;
172 173
			nr_sects = 0;
		}
174
		cond_resched();
175 176
	}

177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203
	*biop = bio;
	return 0;
}

/**
 * blkdev_issue_write_same - queue a write same operation
 * @bdev:	target blockdev
 * @sector:	start sector
 * @nr_sects:	number of sectors to write
 * @gfp_mask:	memory allocation flags (for bio_alloc)
 * @page:	page containing data
 *
 * Description:
 *    Issue a write same request for the sectors in question.
 */
int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
				sector_t nr_sects, gfp_t gfp_mask,
				struct page *page)
{
	struct bio *bio = NULL;
	struct blk_plug plug;
	int ret;

	blk_start_plug(&plug);
	ret = __blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask, page,
			&bio);
	if (ret == 0 && bio) {
204
		ret = submit_bio_wait(bio);
205 206
		bio_put(bio);
	}
207
	blk_finish_plug(&plug);
208
	return ret;
209 210 211
}
EXPORT_SYMBOL(blkdev_issue_write_same);

212 213
static int __blkdev_issue_write_zeroes(struct block_device *bdev,
		sector_t sector, sector_t nr_sects, gfp_t gfp_mask,
214
		struct bio **biop, unsigned flags)
215 216 217 218 219 220 221 222
{
	struct bio *bio = *biop;
	unsigned int max_write_zeroes_sectors;
	struct request_queue *q = bdev_get_queue(bdev);

	if (!q)
		return -ENXIO;

223 224 225
	if (bdev_read_only(bdev))
		return -EPERM;

226 227 228 229 230 231 232
	/* Ensure that max_write_zeroes_sectors doesn't overflow bi_size */
	max_write_zeroes_sectors = bdev_write_zeroes_sectors(bdev);

	if (max_write_zeroes_sectors == 0)
		return -EOPNOTSUPP;

	while (nr_sects) {
233
		bio = blk_next_bio(bio, 0, gfp_mask);
234
		bio->bi_iter.bi_sector = sector;
235
		bio_set_dev(bio, bdev);
236 237 238
		bio->bi_opf = REQ_OP_WRITE_ZEROES;
		if (flags & BLKDEV_ZERO_NOUNMAP)
			bio->bi_opf |= REQ_NOUNMAP;
239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254

		if (nr_sects > max_write_zeroes_sectors) {
			bio->bi_iter.bi_size = max_write_zeroes_sectors << 9;
			nr_sects -= max_write_zeroes_sectors;
			sector += max_write_zeroes_sectors;
		} else {
			bio->bi_iter.bi_size = nr_sects << 9;
			nr_sects = 0;
		}
		cond_resched();
	}

	*biop = bio;
	return 0;
}

255 256 257 258 259 260 261 262
/*
 * Convert a number of 512B sectors to a number of pages.
 * The result is limited to a number of pages that can fit into a BIO.
 * Also make sure that the result is always at least 1 (page) for the cases
 * where nr_sects is lower than the number of sectors in a page.
 */
static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects)
{
263
	sector_t pages = DIV_ROUND_UP_SECTOR_T(nr_sects, PAGE_SIZE / 512);
264

265
	return min(pages, (sector_t)BIO_MAX_PAGES);
266 267
}

268 269 270 271 272 273 274 275 276 277 278 279
static int __blkdev_issue_zero_pages(struct block_device *bdev,
		sector_t sector, sector_t nr_sects, gfp_t gfp_mask,
		struct bio **biop)
{
	struct request_queue *q = bdev_get_queue(bdev);
	struct bio *bio = *biop;
	int bi_size = 0;
	unsigned int sz;

	if (!q)
		return -ENXIO;

280 281 282
	if (bdev_read_only(bdev))
		return -EPERM;

283
	while (nr_sects != 0) {
284 285
		bio = blk_next_bio(bio, __blkdev_sectors_to_bio_pages(nr_sects),
				   gfp_mask);
286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
		bio->bi_iter.bi_sector = sector;
		bio_set_dev(bio, bdev);
		bio_set_op_attrs(bio, REQ_OP_WRITE, 0);

		while (nr_sects != 0) {
			sz = min((sector_t) PAGE_SIZE, nr_sects << 9);
			bi_size = bio_add_page(bio, ZERO_PAGE(0), sz, 0);
			nr_sects -= bi_size >> 9;
			sector += bi_size >> 9;
			if (bi_size < sz)
				break;
		}
		cond_resched();
	}

	*biop = bio;
	return 0;
}

305
/**
306
 * __blkdev_issue_zeroout - generate number of zero filed write bios
307 308 309 310
 * @bdev:	blockdev to issue
 * @sector:	start sector
 * @nr_sects:	number of sectors to write
 * @gfp_mask:	memory allocation flags (for bio_alloc)
311
 * @biop:	pointer to anchor bio
312
 * @flags:	controls detailed behavior
313 314
 *
 * Description:
315 316 317 318 319
 *  Zero-fill a block range, either using hardware offload or by explicitly
 *  writing zeroes to the device.
 *
 *  If a device is using logical block provisioning, the underlying space will
 *  not be released if %flags contains BLKDEV_ZERO_NOUNMAP.
320 321 322
 *
 *  If %flags contains BLKDEV_ZERO_NOFALLBACK, the function will return
 *  -EOPNOTSUPP if no explicit hardware offload for zeroing is provided.
323
 */
324 325
int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
		sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
326
		unsigned flags)
327
{
328
	int ret;
329 330 331 332 333
	sector_t bs_mask;

	bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
	if ((sector | nr_sects) & bs_mask)
		return -EINVAL;
334

335
	ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp_mask,
336
			biop, flags);
337
	if (ret != -EOPNOTSUPP || (flags & BLKDEV_ZERO_NOFALLBACK))
338
		return ret;
339

340 341
	return __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask,
					 biop);
342
}
343
EXPORT_SYMBOL(__blkdev_issue_zeroout);
344 345 346 347 348 349 350

/**
 * blkdev_issue_zeroout - zero-fill a block range
 * @bdev:	blockdev to write
 * @sector:	start sector
 * @nr_sects:	number of sectors to write
 * @gfp_mask:	memory allocation flags (for bio_alloc)
351
 * @flags:	controls detailed behavior
352 353
 *
 * Description:
354 355 356
 *  Zero-fill a block range, either using hardware offload or by explicitly
 *  writing zeroes to the device.  See __blkdev_issue_zeroout() for the
 *  valid values for %flags.
357 358
 */
int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
359
		sector_t nr_sects, gfp_t gfp_mask, unsigned flags)
360
{
361 362 363
	int ret = 0;
	sector_t bs_mask;
	struct bio *bio;
364
	struct blk_plug plug;
365
	bool try_write_zeroes = !!bdev_write_zeroes_sectors(bdev);
366

367 368 369 370 371 372
	bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
	if ((sector | nr_sects) & bs_mask)
		return -EINVAL;

retry:
	bio = NULL;
373
	blk_start_plug(&plug);
374 375 376 377 378 379 380 381 382 383
	if (try_write_zeroes) {
		ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects,
						  gfp_mask, &bio, flags);
	} else if (!(flags & BLKDEV_ZERO_NOFALLBACK)) {
		ret = __blkdev_issue_zero_pages(bdev, sector, nr_sects,
						gfp_mask, &bio);
	} else {
		/* No zeroing offload support */
		ret = -EOPNOTSUPP;
	}
384 385 386 387 388
	if (ret == 0 && bio) {
		ret = submit_bio_wait(bio);
		bio_put(bio);
	}
	blk_finish_plug(&plug);
389 390 391 392 393 394 395 396 397 398 399 400 401 402 403
	if (ret && try_write_zeroes) {
		if (!(flags & BLKDEV_ZERO_NOFALLBACK)) {
			try_write_zeroes = false;
			goto retry;
		}
		if (!bdev_write_zeroes_sectors(bdev)) {
			/*
			 * Zeroing offload support was indicated, but the
			 * device reported ILLEGAL REQUEST (for some devices
			 * there is no non-destructive way to verify whether
			 * WRITE ZEROES is actually supported).
			 */
			ret = -EOPNOTSUPP;
		}
	}
404

405
	return ret;
406
}
407
EXPORT_SYMBOL(blkdev_issue_zeroout);