loop.c 51.7 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
/*
 *  linux/drivers/block/loop.c
 *
 *  Written by Theodore Ts'o, 3/29/93
 *
 * Copyright 1993 by Theodore Ts'o.  Redistribution of this file is
 * permitted under the GNU General Public License.
 *
 * DES encryption plus some minor changes by Werner Almesberger, 30-MAY-1993
 * more DES encryption plus IDEA encryption by Nicholas J. Leon, June 20, 1996
 *
 * Modularized and updated for 1.1.16 kernel - Mitch Dsouza 28th May 1994
 * Adapted for 1.3.59 kernel - Andries Brouwer, 1 Feb 1996
 *
 * Fixed do_loop_request() re-entrancy - Vincent.Renardias@waw.com Mar 20, 1997
 *
 * Added devfs support - Richard Gooch <rgooch@atnf.csiro.au> 16-Jan-1998
 *
 * Handle sparse backing files correctly - Kenn Humborg, Jun 28, 1998
 *
 * Loadable modules and other fixes by AK, 1998
 *
 * Make real block number available to downstream transfer functions, enables
 * CBC (and relatives) mode encryption requiring unique IVs per data block.
 * Reed H. Petty, rhp@draper.net
 *
 * Maximum number of loop devices now dynamic via max_loop module parameter.
 * Russell Kroll <rkroll@exploits.org> 19990701
 *
 * Maximum number of loop devices when compiled-in now selectable by passing
 * max_loop=<1-255> to the kernel on boot.
32
 * Erik I. Bolsø, <eriki@himolde.no>, Oct 31, 1999
Linus Torvalds's avatar
Linus Torvalds committed
33 34 35 36 37 38 39 40 41 42
 *
 * Completely rewrite request handling to be make_request_fn style and
 * non blocking, pushing work to a helper thread. Lots of fixes from
 * Al Viro too.
 * Jens Axboe <axboe@suse.de>, Nov 2000
 *
 * Support up to 256 loop devices
 * Heinz Mauelshagen <mge@sistina.com>, Feb 2002
 *
 * Support for falling back on the write file operation when the address space
43
 * operations write_begin is not available on the backing filesystem.
Linus Torvalds's avatar
Linus Torvalds committed
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
 * Anton Altaparmakov, 16 Feb 2005
 *
 * Still To Fix:
 * - Advisory locking is ignored here.
 * - Should use an own CAP_* category instead of CAP_SYS_ADMIN
 *
 */

#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/stat.h>
#include <linux/errno.h>
#include <linux/major.h>
#include <linux/wait.h>
#include <linux/blkdev.h>
#include <linux/blkpg.h>
#include <linux/init.h>
#include <linux/swap.h>
#include <linux/slab.h>
66
#include <linux/compat.h>
Linus Torvalds's avatar
Linus Torvalds committed
67
#include <linux/suspend.h>
68
#include <linux/freezer.h>
69
#include <linux/mutex.h>
Linus Torvalds's avatar
Linus Torvalds committed
70 71 72
#include <linux/writeback.h>
#include <linux/completion.h>
#include <linux/highmem.h>
73
#include <linux/kthread.h>
74
#include <linux/splice.h>
75
#include <linux/sysfs.h>
76
#include <linux/miscdevice.h>
77
#include <linux/falloc.h>
78
#include <linux/uio.h>
79
#include "loop.h"
Linus Torvalds's avatar
Linus Torvalds committed
80

81
#include <linux/uaccess.h>
Linus Torvalds's avatar
Linus Torvalds committed
82

83 84
static DEFINE_IDR(loop_index_idr);
static DEFINE_MUTEX(loop_index_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
85

86 87 88
static int max_part;
static int part_shift;

Linus Torvalds's avatar
Linus Torvalds committed
89 90 91 92 93
static int transfer_xor(struct loop_device *lo, int cmd,
			struct page *raw_page, unsigned raw_off,
			struct page *loop_page, unsigned loop_off,
			int size, sector_t real_block)
{
94 95
	char *raw_buf = kmap_atomic(raw_page) + raw_off;
	char *loop_buf = kmap_atomic(loop_page) + loop_off;
Linus Torvalds's avatar
Linus Torvalds committed
96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111
	char *in, *out, *key;
	int i, keysize;

	if (cmd == READ) {
		in = raw_buf;
		out = loop_buf;
	} else {
		in = loop_buf;
		out = raw_buf;
	}

	key = lo->lo_encrypt_key;
	keysize = lo->lo_encrypt_key_size;
	for (i = 0; i < size; i++)
		*out++ = *in++ ^ key[(i & 511) % keysize];

112 113
	kunmap_atomic(loop_buf);
	kunmap_atomic(raw_buf);
Linus Torvalds's avatar
Linus Torvalds committed
114 115 116 117 118 119 120 121 122 123 124 125 126
	cond_resched();
	return 0;
}

static int xor_init(struct loop_device *lo, const struct loop_info64 *info)
{
	if (unlikely(info->lo_encrypt_key_size <= 0))
		return -EINVAL;
	return 0;
}

static struct loop_func_table none_funcs = {
	.number = LO_CRYPT_NONE,
127
}; 
Linus Torvalds's avatar
Linus Torvalds committed
128 129 130 131 132

static struct loop_func_table xor_funcs = {
	.number = LO_CRYPT_XOR,
	.transfer = transfer_xor,
	.init = xor_init
133
}; 
Linus Torvalds's avatar
Linus Torvalds committed
134 135 136 137 138 139 140

/* xfer_funcs[0] is special - its release function is never called */
static struct loop_func_table *xfer_funcs[MAX_LO_CRYPT] = {
	&none_funcs,
	&xor_funcs
};

141
static loff_t get_size(loff_t offset, loff_t sizelimit, struct file *file)
Linus Torvalds's avatar
Linus Torvalds committed
142
{
143
	loff_t loopsize;
Linus Torvalds's avatar
Linus Torvalds committed
144 145

	/* Compute loopsize in bytes */
146 147 148 149
	loopsize = i_size_read(file->f_mapping->host);
	if (offset > 0)
		loopsize -= offset;
	/* offset is beyond i_size, weird but possible */
150 151
	if (loopsize < 0)
		return 0;
Linus Torvalds's avatar
Linus Torvalds committed
152

153 154
	if (sizelimit > 0 && sizelimit < loopsize)
		loopsize = sizelimit;
Linus Torvalds's avatar
Linus Torvalds committed
155 156 157 158 159 160 161
	/*
	 * Unfortunately, if we want to do I/O on the device,
	 * the number of 512-byte sectors has to fit into a sector_t.
	 */
	return loopsize >> 9;
}

162 163 164 165 166
static loff_t get_loop_size(struct loop_device *lo, struct file *file)
{
	return get_size(lo->lo_offset, lo->lo_sizelimit, file);
}

167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
static void __loop_update_dio(struct loop_device *lo, bool dio)
{
	struct file *file = lo->lo_backing_file;
	struct address_space *mapping = file->f_mapping;
	struct inode *inode = mapping->host;
	unsigned short sb_bsize = 0;
	unsigned dio_align = 0;
	bool use_dio;

	if (inode->i_sb->s_bdev) {
		sb_bsize = bdev_logical_block_size(inode->i_sb->s_bdev);
		dio_align = sb_bsize - 1;
	}

	/*
	 * We support direct I/O only if lo_offset is aligned with the
	 * logical I/O size of backing device, and the logical block
	 * size of loop is bigger than the backing device's and the loop
	 * needn't transform transfer.
	 *
	 * TODO: the above condition may be loosed in the future, and
	 * direct I/O may be switched runtime at that time because most
189
	 * of requests in sane applications should be PAGE_SIZE aligned
190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215
	 */
	if (dio) {
		if (queue_logical_block_size(lo->lo_queue) >= sb_bsize &&
				!(lo->lo_offset & dio_align) &&
				mapping->a_ops->direct_IO &&
				!lo->transfer)
			use_dio = true;
		else
			use_dio = false;
	} else {
		use_dio = false;
	}

	if (lo->use_dio == use_dio)
		return;

	/* flush dirty pages before changing direct IO */
	vfs_fsync(file, 0);

	/*
	 * The flag of LO_FLAGS_DIRECT_IO is handled similarly with
	 * LO_FLAGS_READ_ONLY, both are set from kernel, and losetup
	 * will get updated by ioctl(LOOP_GET_STATUS)
	 */
	blk_mq_freeze_queue(lo->lo_queue);
	lo->use_dio = use_dio;
216 217
	if (use_dio) {
		queue_flag_clear_unlocked(QUEUE_FLAG_NOMERGES, lo->lo_queue);
218
		lo->lo_flags |= LO_FLAGS_DIRECT_IO;
219 220
	} else {
		queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, lo->lo_queue);
221
		lo->lo_flags &= ~LO_FLAGS_DIRECT_IO;
222
	}
223 224 225
	blk_mq_unfreeze_queue(lo->lo_queue);
}

Linus Torvalds's avatar
Linus Torvalds committed
226
static int
227
figure_loop_size(struct loop_device *lo, loff_t offset, loff_t sizelimit)
Linus Torvalds's avatar
Linus Torvalds committed
228
{
229
	loff_t size = get_size(offset, sizelimit, lo->lo_backing_file);
Linus Torvalds's avatar
Linus Torvalds committed
230
	sector_t x = (sector_t)size;
231
	struct block_device *bdev = lo->lo_device;
Linus Torvalds's avatar
Linus Torvalds committed
232 233 234

	if (unlikely((loff_t)x != size))
		return -EFBIG;
235 236 237 238
	if (lo->lo_offset != offset)
		lo->lo_offset = offset;
	if (lo->lo_sizelimit != sizelimit)
		lo->lo_sizelimit = sizelimit;
239
	set_capacity(lo->lo_disk, x);
240 241 242
	bd_set_size(bdev, (loff_t)get_capacity(bdev->bd_disk) << 9);
	/* let user-space know about the new size */
	kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
243
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
244 245 246 247 248 249 250 251
}

static inline int
lo_do_transfer(struct loop_device *lo, int cmd,
	       struct page *rpage, unsigned roffs,
	       struct page *lpage, unsigned loffs,
	       int size, sector_t rblock)
{
252 253 254 255
	int ret;

	ret = lo->transfer(lo, cmd, rpage, roffs, lpage, loffs, size, rblock);
	if (likely(!ret))
Linus Torvalds's avatar
Linus Torvalds committed
256 257
		return 0;

258 259 260 261
	printk_ratelimited(KERN_ERR
		"loop: Transfer error at byte offset %llu, length %i.\n",
		(unsigned long long)rblock << 9, size);
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
262 263
}

264
static int lo_write_bvec(struct file *file, struct bio_vec *bvec, loff_t *ppos)
Linus Torvalds's avatar
Linus Torvalds committed
265
{
266
	struct iov_iter i;
Linus Torvalds's avatar
Linus Torvalds committed
267
	ssize_t bw;
268

269
	iov_iter_bvec(&i, ITER_BVEC | WRITE, bvec, 1, bvec->bv_len);
Linus Torvalds's avatar
Linus Torvalds committed
270

271
	file_start_write(file);
272
	bw = vfs_iter_write(file, &i, ppos, 0);
273
	file_end_write(file);
274 275

	if (likely(bw ==  bvec->bv_len))
Linus Torvalds's avatar
Linus Torvalds committed
276
		return 0;
277 278 279 280

	printk_ratelimited(KERN_ERR
		"loop: Write error at byte offset %llu, length %i.\n",
		(unsigned long long)*ppos, bvec->bv_len);
Linus Torvalds's avatar
Linus Torvalds committed
281 282 283 284 285
	if (bw >= 0)
		bw = -EIO;
	return bw;
}

286 287
static int lo_write_simple(struct loop_device *lo, struct request *rq,
		loff_t pos)
Linus Torvalds's avatar
Linus Torvalds committed
288
{
289 290 291 292 293 294 295 296 297 298 299 300
	struct bio_vec bvec;
	struct req_iterator iter;
	int ret = 0;

	rq_for_each_segment(bvec, rq, iter) {
		ret = lo_write_bvec(lo->lo_backing_file, &bvec, &pos);
		if (ret < 0)
			break;
		cond_resched();
	}

	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
301 302
}

303
/*
304 305 306
 * This is the slow, transforming version that needs to double buffer the
 * data as it cannot do the transformations in place without having direct
 * access to the destination pages of the backing file.
Linus Torvalds's avatar
Linus Torvalds committed
307
 */
308 309
static int lo_write_transfer(struct loop_device *lo, struct request *rq,
		loff_t pos)
Linus Torvalds's avatar
Linus Torvalds committed
310
{
311
	struct bio_vec bvec, b;
312
	struct req_iterator iter;
313
	struct page *page;
314
	int ret = 0;
Linus Torvalds's avatar
Linus Torvalds committed
315

316 317 318
	page = alloc_page(GFP_NOIO);
	if (unlikely(!page))
		return -ENOMEM;
319

320
	rq_for_each_segment(bvec, rq, iter) {
321 322 323 324 325 326 327 328 329
		ret = lo_do_transfer(lo, WRITE, page, 0, bvec.bv_page,
			bvec.bv_offset, bvec.bv_len, pos >> 9);
		if (unlikely(ret))
			break;

		b.bv_page = page;
		b.bv_offset = 0;
		b.bv_len = bvec.bv_len;
		ret = lo_write_bvec(lo->lo_backing_file, &b, &pos);
Linus Torvalds's avatar
Linus Torvalds committed
330 331 332
		if (ret < 0)
			break;
	}
333 334

	__free_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
335 336 337
	return ret;
}

338 339
static int lo_read_simple(struct loop_device *lo, struct request *rq,
		loff_t pos)
Linus Torvalds's avatar
Linus Torvalds committed
340
{
341 342 343 344
	struct bio_vec bvec;
	struct req_iterator iter;
	struct iov_iter i;
	ssize_t len;
Linus Torvalds's avatar
Linus Torvalds committed
345

346 347
	rq_for_each_segment(bvec, rq, iter) {
		iov_iter_bvec(&i, ITER_BVEC, &bvec, 1, bvec.bv_len);
348
		len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0);
349 350
		if (len < 0)
			return len;
Linus Torvalds's avatar
Linus Torvalds committed
351

352
		flush_dcache_page(bvec.bv_page);
353

354 355
		if (len != bvec.bv_len) {
			struct bio *bio;
Linus Torvalds's avatar
Linus Torvalds committed
356

357 358 359 360 361 362 363 364
			__rq_for_each_bio(bio, rq)
				zero_fill_bio(bio);
			break;
		}
		cond_resched();
	}

	return 0;
365 366
}

367 368
static int lo_read_transfer(struct loop_device *lo, struct request *rq,
		loff_t pos)
Linus Torvalds's avatar
Linus Torvalds committed
369
{
370 371 372 373 374 375
	struct bio_vec bvec, b;
	struct req_iterator iter;
	struct iov_iter i;
	struct page *page;
	ssize_t len;
	int ret = 0;
Linus Torvalds's avatar
Linus Torvalds committed
376

377 378 379
	page = alloc_page(GFP_NOIO);
	if (unlikely(!page))
		return -ENOMEM;
380

381 382
	rq_for_each_segment(bvec, rq, iter) {
		loff_t offset = pos;
383

384 385 386
		b.bv_page = page;
		b.bv_offset = 0;
		b.bv_len = bvec.bv_len;
387

388
		iov_iter_bvec(&i, ITER_BVEC, &b, 1, b.bv_len);
389
		len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0);
390 391 392 393
		if (len < 0) {
			ret = len;
			goto out_free_page;
		}
Linus Torvalds's avatar
Linus Torvalds committed
394

395 396 397 398
		ret = lo_do_transfer(lo, READ, page, 0, bvec.bv_page,
			bvec.bv_offset, len, offset >> 9);
		if (ret)
			goto out_free_page;
Linus Torvalds's avatar
Linus Torvalds committed
399

400
		flush_dcache_page(bvec.bv_page);
401

402
		if (len != bvec.bv_len) {
403 404 405 406
			struct bio *bio;

			__rq_for_each_bio(bio, rq)
				zero_fill_bio(bio);
Linus Torvalds's avatar
Linus Torvalds committed
407
			break;
408
		}
Linus Torvalds's avatar
Linus Torvalds committed
409
	}
410 411 412 413 414

	ret = 0;
out_free_page:
	__free_page(page);
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
415 416
}

417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450
static int lo_discard(struct loop_device *lo, struct request *rq, loff_t pos)
{
	/*
	 * We use punch hole to reclaim the free space used by the
	 * image a.k.a. discard. However we do not support discard if
	 * encryption is enabled, because it may give an attacker
	 * useful information.
	 */
	struct file *file = lo->lo_backing_file;
	int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE;
	int ret;

	if ((!file->f_op->fallocate) || lo->lo_encrypt_key_size) {
		ret = -EOPNOTSUPP;
		goto out;
	}

	ret = file->f_op->fallocate(file, mode, pos, blk_rq_bytes(rq));
	if (unlikely(ret && ret != -EINVAL && ret != -EOPNOTSUPP))
		ret = -EIO;
 out:
	return ret;
}

static int lo_req_flush(struct loop_device *lo, struct request *rq)
{
	struct file *file = lo->lo_backing_file;
	int ret = vfs_fsync(file, 0);
	if (unlikely(ret && ret != -EINVAL))
		ret = -EIO;

	return ret;
}

451
static void lo_complete_rq(struct request *rq)
452
{
453
	struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
454

455 456
	if (unlikely(req_op(cmd->rq) == REQ_OP_READ && cmd->use_aio &&
		     cmd->ret >= 0 && cmd->ret < blk_rq_bytes(cmd->rq))) {
457 458
		struct bio *bio = cmd->rq->bio;

459
		bio_advance(bio, cmd->ret);
460 461
		zero_fill_bio(bio);
	}
462

463
	blk_mq_end_request(rq, cmd->ret < 0 ? BLK_STS_IOERR : BLK_STS_OK);
464 465
}

466 467 468 469 470 471 472 473 474
static void lo_rw_aio_do_completion(struct loop_cmd *cmd)
{
	if (!atomic_dec_and_test(&cmd->ref))
		return;
	kfree(cmd->bvec);
	cmd->bvec = NULL;
	blk_mq_complete_request(cmd->rq);
}

475 476 477 478
static void lo_rw_aio_complete(struct kiocb *iocb, long ret, long ret2)
{
	struct loop_cmd *cmd = container_of(iocb, struct loop_cmd, iocb);

479 480
	if (cmd->css)
		css_put(cmd->css);
481
	cmd->ret = ret;
482
	lo_rw_aio_do_completion(cmd);
483 484 485 486 487 488 489
}

static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd,
		     loff_t pos, bool rw)
{
	struct iov_iter iter;
	struct bio_vec *bvec;
490 491
	struct request *rq = cmd->rq;
	struct bio *bio = rq->bio;
492
	struct file *file = lo->lo_backing_file;
493 494
	unsigned int offset;
	int segments = 0;
495 496
	int ret;

497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529
	if (rq->bio != rq->biotail) {
		struct req_iterator iter;
		struct bio_vec tmp;

		__rq_for_each_bio(bio, rq)
			segments += bio_segments(bio);
		bvec = kmalloc(sizeof(struct bio_vec) * segments, GFP_NOIO);
		if (!bvec)
			return -EIO;
		cmd->bvec = bvec;

		/*
		 * The bios of the request may be started from the middle of
		 * the 'bvec' because of bio splitting, so we can't directly
		 * copy bio->bi_iov_vec to new bvec. The rq_for_each_segment
		 * API will take care of all details for us.
		 */
		rq_for_each_segment(tmp, rq, iter) {
			*bvec = tmp;
			bvec++;
		}
		bvec = cmd->bvec;
		offset = 0;
	} else {
		/*
		 * Same here, this bio may be started from the middle of the
		 * 'bvec' because of bio splitting, so offset from the bvec
		 * must be passed to iov iterator
		 */
		offset = bio->bi_iter.bi_bvec_done;
		bvec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
		segments = bio_segments(bio);
	}
530
	atomic_set(&cmd->ref, 2);
531 532

	iov_iter_bvec(&iter, ITER_BVEC | rw, bvec,
533 534
		      segments, blk_rq_bytes(rq));
	iter.iov_offset = offset;
535 536 537 538 539

	cmd->iocb.ki_pos = pos;
	cmd->iocb.ki_filp = file;
	cmd->iocb.ki_complete = lo_rw_aio_complete;
	cmd->iocb.ki_flags = IOCB_DIRECT;
540 541
	if (cmd->css)
		kthread_associate_blkcg(cmd->css);
542 543

	if (rw == WRITE)
544
		ret = call_write_iter(file, &cmd->iocb, &iter);
545
	else
546
		ret = call_read_iter(file, &cmd->iocb, &iter);
547

548
	lo_rw_aio_do_completion(cmd);
549
	kthread_associate_blkcg(NULL);
550

551 552 553 554 555
	if (ret != -EIOCBQUEUED)
		cmd->iocb.ki_complete(&cmd->iocb, ret, 0);
	return 0;
}

556
static int do_req_filebacked(struct loop_device *lo, struct request *rq)
557 558
{
	struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
559
	loff_t pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset;
560 561 562 563 564 565 566 567 568 569

	/*
	 * lo_write_simple and lo_read_simple should have been covered
	 * by io submit style function like lo_rw_aio(), one blocker
	 * is that lo_read_simple() need to call flush_dcache_page after
	 * the page is written from kernel, and it isn't easy to handle
	 * this in io submit style function which submits all segments
	 * of the req at one time. And direct read IO doesn't need to
	 * run flush_dcache_page().
	 */
570 571 572 573
	switch (req_op(rq)) {
	case REQ_OP_FLUSH:
		return lo_req_flush(lo, rq);
	case REQ_OP_DISCARD:
574
	case REQ_OP_WRITE_ZEROES:
575 576 577 578 579 580
		return lo_discard(lo, rq, pos);
	case REQ_OP_WRITE:
		if (lo->transfer)
			return lo_write_transfer(lo, rq, pos);
		else if (cmd->use_aio)
			return lo_rw_aio(lo, cmd, pos, WRITE);
581
		else
582 583
			return lo_write_simple(lo, rq, pos);
	case REQ_OP_READ:
584
		if (lo->transfer)
585 586 587
			return lo_read_transfer(lo, rq, pos);
		else if (cmd->use_aio)
			return lo_rw_aio(lo, cmd, pos, READ);
588
		else
589 590 591 592 593
			return lo_read_simple(lo, rq, pos);
	default:
		WARN_ON_ONCE(1);
		return -EIO;
		break;
594
	}
Linus Torvalds's avatar
Linus Torvalds committed
595 596
}

597 598 599 600 601 602
static inline void loop_update_dio(struct loop_device *lo)
{
	__loop_update_dio(lo, io_is_direct(lo->lo_backing_file) |
			lo->use_dio);
}

603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624
static void loop_reread_partitions(struct loop_device *lo,
				   struct block_device *bdev)
{
	int rc;

	/*
	 * bd_mutex has been held already in release path, so don't
	 * acquire it if this function is called in such case.
	 *
	 * If the reread partition isn't from release path, lo_refcnt
	 * must be at least one and it can only become zero when the
	 * current holder is released.
	 */
	if (!atomic_read(&lo->lo_refcnt))
		rc = __blkdev_reread_part(bdev);
	else
		rc = blkdev_reread_part(bdev);
	if (rc)
		pr_warn("%s: partition scan of loop%d (%s) failed (rc=%d)\n",
			__func__, lo->lo_number, lo->lo_file_name, rc);
}

Linus Torvalds's avatar
Linus Torvalds committed
625 626 627 628 629 630 631 632
/*
 * loop_change_fd switched the backing store of a loopback device to
 * a new file. This is useful for operating system installers to free up
 * the original file and in High Availability environments to switch to
 * an alternative location for the content in case of server meltdown.
 * This can only work if the loop device is used read-only, and if the
 * new backing store is the same size and type as the old backing store.
 */
Al Viro's avatar
Al Viro committed
633 634
static int loop_change_fd(struct loop_device *lo, struct block_device *bdev,
			  unsigned int arg)
Linus Torvalds's avatar
Linus Torvalds committed
635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666
{
	struct file	*file, *old_file;
	struct inode	*inode;
	int		error;

	error = -ENXIO;
	if (lo->lo_state != Lo_bound)
		goto out;

	/* the loop device has to be read-only */
	error = -EINVAL;
	if (!(lo->lo_flags & LO_FLAGS_READ_ONLY))
		goto out;

	error = -EBADF;
	file = fget(arg);
	if (!file)
		goto out;

	inode = file->f_mapping->host;
	old_file = lo->lo_backing_file;

	error = -EINVAL;

	if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode))
		goto out_putf;

	/* size of the new backing store needs to be the same */
	if (get_loop_size(lo, file) != get_loop_size(lo, old_file))
		goto out_putf;

	/* and ... switch */
667 668 669 670 671 672 673 674
	blk_mq_freeze_queue(lo->lo_queue);
	mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask);
	lo->lo_backing_file = file;
	lo->old_gfp_mask = mapping_gfp_mask(file->f_mapping);
	mapping_set_gfp_mask(file->f_mapping,
			     lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));
	loop_update_dio(lo);
	blk_mq_unfreeze_queue(lo->lo_queue);
Linus Torvalds's avatar
Linus Torvalds committed
675 676

	fput(old_file);
677
	if (lo->lo_flags & LO_FLAGS_PARTSCAN)
678
		loop_reread_partitions(lo, bdev);
Linus Torvalds's avatar
Linus Torvalds committed
679 680 681 682 683 684 685 686 687 688 689 690 691 692 693
	return 0;

 out_putf:
	fput(file);
 out:
	return error;
}

static inline int is_loop_device(struct file *file)
{
	struct inode *i = file->f_mapping->host;

	return i && S_ISBLK(i->i_mode) && MAJOR(i->i_rdev) == LOOP_MAJOR;
}

694 695 696 697 698
/* loop sysfs attributes */

static ssize_t loop_attr_show(struct device *dev, char *page,
			      ssize_t (*callback)(struct loop_device *, char *))
{
699 700
	struct gendisk *disk = dev_to_disk(dev);
	struct loop_device *lo = disk->private_data;
701

702
	return callback(lo, page);
703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719
}

#define LOOP_ATTR_RO(_name)						\
static ssize_t loop_attr_##_name##_show(struct loop_device *, char *);	\
static ssize_t loop_attr_do_show_##_name(struct device *d,		\
				struct device_attribute *attr, char *b)	\
{									\
	return loop_attr_show(d, b, loop_attr_##_name##_show);		\
}									\
static struct device_attribute loop_attr_##_name =			\
	__ATTR(_name, S_IRUGO, loop_attr_do_show_##_name, NULL);

static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf)
{
	ssize_t ret;
	char *p = NULL;

720
	spin_lock_irq(&lo->lo_lock);
721
	if (lo->lo_backing_file)
722
		p = file_path(lo->lo_backing_file, buf, PAGE_SIZE - 1);
723
	spin_unlock_irq(&lo->lo_lock);
724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753

	if (IS_ERR_OR_NULL(p))
		ret = PTR_ERR(p);
	else {
		ret = strlen(p);
		memmove(buf, p, ret);
		buf[ret++] = '\n';
		buf[ret] = 0;
	}

	return ret;
}

static ssize_t loop_attr_offset_show(struct loop_device *lo, char *buf)
{
	return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_offset);
}

static ssize_t loop_attr_sizelimit_show(struct loop_device *lo, char *buf)
{
	return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_sizelimit);
}

static ssize_t loop_attr_autoclear_show(struct loop_device *lo, char *buf)
{
	int autoclear = (lo->lo_flags & LO_FLAGS_AUTOCLEAR);

	return sprintf(buf, "%s\n", autoclear ? "1" : "0");
}

754 755 756 757 758 759 760
static ssize_t loop_attr_partscan_show(struct loop_device *lo, char *buf)
{
	int partscan = (lo->lo_flags & LO_FLAGS_PARTSCAN);

	return sprintf(buf, "%s\n", partscan ? "1" : "0");
}

761 762 763 764 765 766 767
static ssize_t loop_attr_dio_show(struct loop_device *lo, char *buf)
{
	int dio = (lo->lo_flags & LO_FLAGS_DIRECT_IO);

	return sprintf(buf, "%s\n", dio ? "1" : "0");
}

768 769 770 771
LOOP_ATTR_RO(backing_file);
LOOP_ATTR_RO(offset);
LOOP_ATTR_RO(sizelimit);
LOOP_ATTR_RO(autoclear);
772
LOOP_ATTR_RO(partscan);
773
LOOP_ATTR_RO(dio);
774 775 776 777 778 779

static struct attribute *loop_attrs[] = {
	&loop_attr_backing_file.attr,
	&loop_attr_offset.attr,
	&loop_attr_sizelimit.attr,
	&loop_attr_autoclear.attr,
780
	&loop_attr_partscan.attr,
781
	&loop_attr_dio.attr,
782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801
	NULL,
};

static struct attribute_group loop_attribute_group = {
	.name = "loop",
	.attrs= loop_attrs,
};

static int loop_sysfs_init(struct loop_device *lo)
{
	return sysfs_create_group(&disk_to_dev(lo->lo_disk)->kobj,
				  &loop_attribute_group);
}

static void loop_sysfs_exit(struct loop_device *lo)
{
	sysfs_remove_group(&disk_to_dev(lo->lo_disk)->kobj,
			   &loop_attribute_group);
}

802 803 804 805 806 807 808 809
static void loop_config_discard(struct loop_device *lo)
{
	struct file *file = lo->lo_backing_file;
	struct inode *inode = file->f_mapping->host;
	struct request_queue *q = lo->lo_queue;

	/*
	 * We use punch hole to reclaim the free space used by the
810
	 * image a.k.a. discard. However we do not support discard if
811 812 813 814 815 816 817
	 * encryption is enabled, because it may give an attacker
	 * useful information.
	 */
	if ((!file->f_op->fallocate) ||
	    lo->lo_encrypt_key_size) {
		q->limits.discard_granularity = 0;
		q->limits.discard_alignment = 0;
818
		blk_queue_max_discard_sectors(q, 0);
819
		blk_queue_max_write_zeroes_sectors(q, 0);
820 821 822 823 824
		queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q);
		return;
	}

	q->limits.discard_granularity = inode->i_sb->s_blocksize;
825
	q->limits.discard_alignment = 0;
826

827 828
	blk_queue_max_discard_sectors(q, UINT_MAX >> 9);
	blk_queue_max_write_zeroes_sectors(q, UINT_MAX >> 9);
829 830 831
	queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
}

Ming Lei's avatar
Ming Lei committed
832 833
static void loop_unprepare_queue(struct loop_device *lo)
{
834
	kthread_flush_worker(&lo->worker);
Ming Lei's avatar
Ming Lei committed
835 836 837
	kthread_stop(lo->worker_task);
}

838 839 840 841 842 843
static int loop_kthread_worker_fn(void *worker_ptr)
{
	current->flags |= PF_LESS_THROTTLE;
	return kthread_worker_fn(worker_ptr);
}

Ming Lei's avatar
Ming Lei committed
844 845
static int loop_prepare_queue(struct loop_device *lo)
{
846
	kthread_init_worker(&lo->worker);
847
	lo->worker_task = kthread_run(loop_kthread_worker_fn,
Ming Lei's avatar
Ming Lei committed
848 849 850 851 852 853 854
			&lo->worker, "loop%d", lo->lo_number);
	if (IS_ERR(lo->worker_task))
		return -ENOMEM;
	set_user_nice(lo->worker_task, MIN_NICE);
	return 0;
}

Al Viro's avatar
Al Viro committed
855
static int loop_set_fd(struct loop_device *lo, fmode_t mode,
Linus Torvalds's avatar
Linus Torvalds committed
856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881
		       struct block_device *bdev, unsigned int arg)
{
	struct file	*file, *f;
	struct inode	*inode;
	struct address_space *mapping;
	int		lo_flags = 0;
	int		error;
	loff_t		size;

	/* This is safe, since we have a reference from open(). */
	__module_get(THIS_MODULE);

	error = -EBADF;
	file = fget(arg);
	if (!file)
		goto out;

	error = -EBUSY;
	if (lo->lo_state != Lo_unbound)
		goto out_putf;

	/* Avoid recursion */
	f = file;
	while (is_loop_device(f)) {
		struct loop_device *l;

Al Viro's avatar
Al Viro committed
882
		if (f->f_mapping->host->i_bdev == bdev)
Linus Torvalds's avatar
Linus Torvalds committed
883 884 885 886 887 888 889 890 891 892 893 894 895 896
			goto out_putf;

		l = f->f_mapping->host->i_bdev->bd_disk->private_data;
		if (l->lo_state == Lo_unbound) {
			error = -EINVAL;
			goto out_putf;
		}
		f = l->lo_backing_file;
	}

	mapping = file->f_mapping;
	inode = mapping->host;

	error = -EINVAL;
897 898
	if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode))
		goto out_putf;
Linus Torvalds's avatar
Linus Torvalds committed
899

900
	if (!(file->f_mode & FMODE_WRITE) || !(mode & FMODE_WRITE) ||
901
	    !file->f_op->write_iter)
902
		lo_flags |= LO_FLAGS_READ_ONLY;
903

904
	error = -EFBIG;
Linus Torvalds's avatar
Linus Torvalds committed
905
	size = get_loop_size(lo, file);
906
	if ((loff_t)(sector_t)size != size)
Linus Torvalds's avatar
Linus Torvalds committed
907
		goto out_putf;
Ming Lei's avatar
Ming Lei committed
908 909
	error = loop_prepare_queue(lo);
	if (error)
910
		goto out_putf;
Linus Torvalds's avatar
Linus Torvalds committed
911

912
	error = 0;
Linus Torvalds's avatar
Linus Torvalds committed
913 914 915

	set_device_ro(bdev, (lo_flags & LO_FLAGS_READ_ONLY) != 0);

916
	lo->use_dio = false;
Linus Torvalds's avatar
Linus Torvalds committed
917 918 919
	lo->lo_device = bdev;
	lo->lo_flags = lo_flags;
	lo->lo_backing_file = file;
920
	lo->transfer = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
921 922 923 924 925
	lo->ioctl = NULL;
	lo->lo_sizelimit = 0;
	lo->old_gfp_mask = mapping_gfp_mask(mapping);
	mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));

926
	if (!(lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync)
927
		blk_queue_write_cache(lo->lo_queue, true, false);
928

929
	loop_update_dio(lo);
930
	set_capacity(lo->lo_disk, size);
Linus Torvalds's avatar
Linus Torvalds committed
931
	bd_set_size(bdev, size << 9);
932
	loop_sysfs_init(lo);
933 934
	/* let user-space know about the new size */
	kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
Linus Torvalds's avatar
Linus Torvalds committed
935

936 937
	set_blocksize(bdev, S_ISBLK(inode->i_mode) ?
		      block_size(inode->i_bdev) : PAGE_SIZE);
Linus Torvalds's avatar
Linus Torvalds committed
938

939
	lo->lo_state = Lo_bound;
940 941 942
	if (part_shift)
		lo->lo_flags |= LO_FLAGS_PARTSCAN;
	if (lo->lo_flags & LO_FLAGS_PARTSCAN)
943
		loop_reread_partitions(lo, bdev);
944 945 946 947 948

	/* Grab the block_device to prevent its destruction after we
	 * put /dev/loopXX inode. Later in loop_clr_fd() we bdput(bdev).
	 */
	bdgrab(bdev);
Linus Torvalds's avatar
Linus Torvalds committed
949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995
	return 0;

 out_putf:
	fput(file);
 out:
	/* This is safe: open() is still holding a reference. */
	module_put(THIS_MODULE);
	return error;
}

static int
loop_release_xfer(struct loop_device *lo)
{
	int err = 0;
	struct loop_func_table *xfer = lo->lo_encryption;

	if (xfer) {
		if (xfer->release)
			err = xfer->release(lo);
		lo->transfer = NULL;
		lo->lo_encryption = NULL;
		module_put(xfer->owner);
	}
	return err;
}

static int
loop_init_xfer(struct loop_device *lo, struct loop_func_table *xfer,
	       const struct loop_info64 *i)
{
	int err = 0;

	if (xfer) {
		struct module *owner = xfer->owner;

		if (!try_module_get(owner))
			return -EINVAL;
		if (xfer->init)
			err = xfer->init(lo, i);
		if (err)
			module_put(owner);
		else
			lo->lo_encryption = xfer;
	}
	return err;
}

996
static int loop_clr_fd(struct loop_device *lo)
Linus Torvalds's avatar
Linus Torvalds committed
997 998
{
	struct file *filp = lo->lo_backing_file;
999
	gfp_t gfp = lo->old_gfp_mask;
1000
	struct block_device *bdev = lo->lo_device;
Linus Torvalds's avatar
Linus Torvalds committed
1001 1002 1003 1004

	if (lo->lo_state != Lo_bound)
		return -ENXIO;

1005 1006 1007 1008 1009 1010 1011 1012 1013 1014
	/*
	 * If we've explicitly asked to tear down the loop device,
	 * and it has an elevated reference count, set it for auto-teardown when
	 * the last reference goes away. This stops $!~#$@ udev from
	 * preventing teardown because it decided that it needs to run blkid on
	 * the loopback device whenever they appear. xfstests is notorious for
	 * failing tests because blkid via udev races with a losetup
	 * <dev>/do something like mkfs/losetup -d <dev> causing the losetup -d
	 * command to fail with EBUSY.
	 */
1015
	if (atomic_read(&lo->lo_refcnt) > 1) {
1016 1017 1018 1019
		lo->lo_flags |= LO_FLAGS_AUTOCLEAR;
		mutex_unlock(&lo->lo_ctl_mutex);
		return 0;
	}
Linus Torvalds's avatar
Linus Torvalds committed
1020 1021 1022 1023

	if (filp == NULL)
		return -EINVAL;

1024 1025 1026
	/* freeze request queue during the transition */
	blk_mq_freeze_queue(lo->lo_queue);

Linus Torvalds's avatar
Linus Torvalds committed
1027 1028 1029
	spin_lock_irq(&lo->lo_lock);
	lo->lo_state = Lo_rundown;
	lo->lo_backing_file = NULL;
1030
	spin_unlock_irq(&lo->lo_lock);
Linus Torvalds's avatar
Linus Torvalds committed
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042

	loop_release_xfer(lo);
	lo->transfer = NULL;
	lo->ioctl = NULL;
	lo->lo_device = NULL;
	lo->lo_encryption = NULL;
	lo->lo_offset = 0;
	lo->lo_sizelimit = 0;
	lo->lo_encrypt_key_size = 0;
	memset(lo->lo_encrypt_key, 0, LO_KEY_SIZE);
	memset(lo->lo_crypt_name, 0, LO_NAME_SIZE);
	memset(lo->lo_file_name, 0, LO_NAME_SIZE);
1043
	blk_queue_logical_block_size(lo->lo_queue, 512);
1044 1045
	blk_queue_physical_block_size(lo->lo_queue, 512);
	blk_queue_io_min(lo->lo_queue, 512);
1046 1047
	if (bdev) {
		bdput(bdev);
Al Viro's avatar
Al Viro committed
1048
		invalidate_bdev(bdev);
1049
	}
1050
	set_capacity(lo->lo_disk, 0);
1051
	loop_sysfs_exit(lo);
1052
	if (bdev) {
Al Viro's avatar
Al Viro committed
1053
		bd_set_size(bdev, 0);
1054 1055 1056
		/* let user-space know about this change */
		kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
	}
Linus Torvalds's avatar
Linus Torvalds committed
1057 1058 1059 1060
	mapping_set_gfp_mask(filp->f_mapping, gfp);
	lo->lo_state = Lo_unbound;
	/* This is safe: open() is still holding a reference. */
	module_put(THIS_MODULE);
1061 1062
	blk_mq_unfreeze_queue(lo->lo_queue);

1063
	if (lo->lo_flags & LO_FLAGS_PARTSCAN && bdev)
1064
		loop_reread_partitions(lo, bdev);
1065 1066 1067
	lo->lo_flags = 0;
	if (!part_shift)
		lo->lo_disk->flags |= GENHD_FL_NO_PART_SCAN;
Ming Lei's avatar
Ming Lei committed
1068
	loop_unprepare_queue(lo);
1069 1070 1071 1072 1073 1074 1075 1076
	mutex_unlock(&lo->lo_ctl_mutex);
	/*
	 * Need not hold lo_ctl_mutex to fput backing file.
	 * Calling fput holding lo_ctl_mutex triggers a circular
	 * lock dependency possibility warning as fput can take
	 * bd_mutex which is usually taken before lo_ctl_mutex.
	 */
	fput(filp);
Linus Torvalds's avatar
Linus Torvalds committed
1077 1078 1079 1080 1081 1082 1083 1084
	return 0;
}

static int
loop_set_status(struct loop_device *lo, const struct loop_info64 *info)
{
	int err;
	struct loop_func_table *xfer;
1085
	kuid_t uid = current_uid();
Linus Torvalds's avatar
Linus Torvalds committed
1086

1087
	if (lo->lo_encrypt_key_size &&
1088
	    !uid_eq(lo->lo_key_owner, uid) &&
Linus Torvalds's avatar
Linus Torvalds committed
1089 1090 1091 1092 1093 1094 1095
	    !capable(CAP_SYS_ADMIN))
		return -EPERM;
	if (lo->lo_state != Lo_bound)
		return -ENXIO;
	if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE)
		return -EINVAL;

1096 1097 1098
	/* I/O need to be drained during transfer transition */
	blk_mq_freeze_queue(lo->lo_queue);

Linus Torvalds's avatar
Linus Torvalds committed
1099 1100
	err = loop_release_xfer(lo);
	if (err)
1101
		goto exit;
Linus Torvalds's avatar
Linus Torvalds committed
1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115

	if (info->lo_encrypt_type) {
		unsigned int type = info->lo_encrypt_type;

		if (type >= MAX_LO_CRYPT)
			return -EINVAL;
		xfer = xfer_funcs[type];
		if (xfer == NULL)
			return -EINVAL;
	} else
		xfer = NULL;

	err = loop_init_xfer(lo, xfer, info);
	if (err)
1116
		goto exit;
Linus Torvalds's avatar
Linus Torvalds committed
1117 1118

	if (lo->lo_offset != info->lo_offset ||
1119 1120
	    lo->lo_sizelimit != info->lo_sizelimit) {
		if (figure_loop_size(lo, info->lo_offset, info->lo_sizelimit)) {
1121 1122 1123
			err = -EFBIG;
			goto exit;
		}
1124
	}
1125

1126
	loop_config_discard(lo);
Linus Torvalds's avatar
Linus Torvalds committed
1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137

	memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE);
	memcpy(lo->lo_crypt_name, info->lo_crypt_name, LO_NAME_SIZE);
	lo->lo_file_name[LO_NAME_SIZE-1] = 0;
	lo->lo_crypt_name[LO_NAME_SIZE-1] = 0;

	if (!xfer)
		xfer = &none_funcs;
	lo->transfer = xfer->transfer;
	lo->ioctl = xfer->ioctl;

1138 1139 1140 1141
	if ((lo->lo_flags & LO_FLAGS_AUTOCLEAR) !=
	     (info->lo_flags & LO_FLAGS_AUTOCLEAR))
		lo->lo_flags ^= LO_FLAGS_AUTOCLEAR;

Linus Torvalds's avatar
Linus Torvalds committed
1142 1143 1144 1145 1146 1147
	lo->lo_encrypt_key_size = info->lo_encrypt_key_size;
	lo->lo_init[0] = info->lo_init[0];
	lo->lo_init[1] = info->lo_init[1];
	if (info->lo_encrypt_key_size) {
		memcpy(lo->lo_encrypt_key, info->lo_encrypt_key,
		       info->lo_encrypt_key_size);
1148
		lo->lo_key_owner = uid;
1149
	}
Linus Torvalds's avatar
Linus Torvalds committed
1150

1151 1152 1153
	/* update dio if lo_offset or transfer is changed */
	__loop_update_dio(lo, lo->use_dio);

1154 1155
 exit:
	blk_mq_unfreeze_queue(lo->lo_queue);
1156 1157 1158 1159 1160 1161 1162 1163

	if (!err && (info->lo_flags & LO_FLAGS_PARTSCAN) &&
	     !(lo->lo_flags & LO_FLAGS_PARTSCAN)) {
		lo->lo_flags |= LO_FLAGS_PARTSCAN;
		lo->lo_disk->flags &= ~GENHD_FL_NO_PART_SCAN;
		loop_reread_partitions(lo, lo->lo_device);
	}

1164
	return err;
Linus Torvalds's avatar
Linus Torvalds committed
1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175
}

static int
loop_get_status(struct loop_device *lo, struct loop_info64 *info)
{
	struct file *file = lo->lo_backing_file;
	struct kstat stat;
	int error;

	if (lo->lo_state != Lo_bound)
		return -ENXIO;
1176 1177
	error = vfs_getattr(&file->f_path, &stat,
			    STATX_INO, AT_STATX_SYNC_AS_STAT);
Linus Torvalds's avatar
Linus Torvalds committed
1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306
	if (error)
		return error;
	memset(info, 0, sizeof(*info));
	info->lo_number = lo->lo_number;
	info->lo_device = huge_encode_dev(stat.dev);
	info->lo_inode = stat.ino;
	info->lo_rdevice = huge_encode_dev(lo->lo_device ? stat.rdev : stat.dev);
	info->lo_offset = lo->lo_offset;
	info->lo_sizelimit = lo->lo_sizelimit;
	info->lo_flags = lo->lo_flags;
	memcpy(info->lo_file_name, lo->lo_file_name, LO_NAME_SIZE);
	memcpy(info->lo_crypt_name, lo->lo_crypt_name, LO_NAME_SIZE);
	info->lo_encrypt_type =
		lo->lo_encryption ? lo->lo_encryption->number : 0;
	if (lo->lo_encrypt_key_size && capable(CAP_SYS_ADMIN)) {
		info->lo_encrypt_key_size = lo->lo_encrypt_key_size;
		memcpy(info->lo_encrypt_key, lo->lo_encrypt_key,
		       lo->lo_encrypt_key_size);
	}
	return 0;
}

static void
loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64)
{
	memset(info64, 0, sizeof(*info64));
	info64->lo_number = info->lo_number;
	info64->lo_device = info->lo_device;
	info64->lo_inode = info->lo_inode;
	info64->lo_rdevice = info->lo_rdevice;
	info64->lo_offset = info->lo_offset;
	info64->lo_sizelimit = 0;
	info64->lo_encrypt_type = info->lo_encrypt_type;
	info64->lo_encrypt_key_size = info->lo_encrypt_key_size;
	info64->lo_flags = info->lo_flags;
	info64->lo_init[0] = info->lo_init[0];
	info64->lo_init[1] = info->lo_init[1];
	if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
		memcpy(info64->lo_crypt_name, info->lo_name, LO_NAME_SIZE);
	else
		memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE);
	memcpy(info64->lo_encrypt_key, info->lo_encrypt_key, LO_KEY_SIZE);
}

static int
loop_info64_to_old(const struct loop_info64 *info64, struct loop_info *info)
{
	memset(info, 0, sizeof(*info));
	info->lo_number = info64->lo_number;
	info->lo_device = info64->lo_device;
	info->lo_inode = info64->lo_inode;
	info->lo_rdevice = info64->lo_rdevice;
	info->lo_offset = info64->lo_offset;
	info->lo_encrypt_type = info64->lo_encrypt_type;
	info->lo_encrypt_key_size = info64->lo_encrypt_key_size;
	info->lo_flags = info64->lo_flags;
	info->lo_init[0] = info64->lo_init[0];
	info->lo_init[1] = info64->lo_init[1];
	if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
		memcpy(info->lo_name, info64->lo_crypt_name, LO_NAME_SIZE);
	else
		memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE);
	memcpy(info->lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE);

	/* error in case values were truncated */
	if (info->lo_device != info64->lo_device ||
	    info->lo_rdevice != info64->lo_rdevice ||
	    info->lo_inode != info64->lo_inode ||
	    info->lo_offset != info64->lo_offset)
		return -EOVERFLOW;

	return 0;
}

static int
loop_set_status_old(struct loop_device *lo, const struct loop_info __user *arg)
{
	struct loop_info info;
	struct loop_info64 info64;

	if (copy_from_user(&info, arg, sizeof (struct loop_info)))
		return -EFAULT;
	loop_info64_from_old(&info, &info64);
	return loop_set_status(lo, &info64);
}

static int
loop_set_status64(struct loop_device *lo, const struct loop_info64 __user *arg)
{
	struct loop_info64 info64;

	if (copy_from_user(&info64, arg, sizeof (struct loop_info64)))
		return -EFAULT;
	return loop_set_status(lo, &info64);
}

static int
loop_get_status_old(struct loop_device *lo, struct loop_info __user *arg) {
	struct loop_info info;
	struct loop_info64 info64;
	int err = 0;

	if (!arg)
		err = -EINVAL;
	if (!err)
		err = loop_get_status(lo, &info64);
	if (!err)
		err = loop_info64_to_old(&info64, &info);
	if (!err && copy_to_user(arg, &info, sizeof(info)))
		err = -EFAULT;

	return err;
}

static int
loop_get_status64(struct loop_device *lo, struct loop_info64 __user *arg) {
	struct loop_info64 info64;
	int err = 0;

	if (!arg)
		err = -EINVAL;
	if (!err)
		err = loop_get_status(lo, &info64);
	if (!err && copy_to_user(arg, &info64, sizeof(info64)))
		err = -EFAULT;

	return err;
}

1307
static int loop_set_capacity(struct loop_device *lo)
1308 1309
{
	if (unlikely(lo->lo_state != Lo_bound))
1310
		return -ENXIO;
1311

1312
	return figure_loop_size(lo, lo->lo_offset, lo->lo_sizelimit);
1313 1314
}

1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328
static int loop_set_dio(struct loop_device *lo, unsigned long arg)
{
	int error = -ENXIO;
	if (lo->lo_state != Lo_bound)
		goto out;

	__loop_update_dio(lo, !!arg);
	if (lo->use_dio == !!arg)
		return 0;
	error = -EINVAL;
 out:
	return error;
}

1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339
static int loop_set_block_size(struct loop_device *lo, unsigned long arg)
{
	if (lo->lo_state != Lo_bound)
		return -ENXIO;

	if (arg < 512 || arg > PAGE_SIZE || !is_power_of_2(arg))
		return -EINVAL;

	blk_mq_freeze_queue(lo->lo_queue);

	blk_queue_logical_block_size(lo->lo_queue, arg);
1340 1341
	blk_queue_physical_block_size(lo->lo_queue, arg);
	blk_queue_io_min(lo->lo_queue, arg);
1342 1343 1344 1345 1346 1347 1348
	loop_update_dio(lo);

	blk_mq_unfreeze_queue(lo->lo_queue);

	return 0;
}

Al Viro's avatar
Al Viro committed
1349
static int lo_ioctl(struct block_device *bdev, fmode_t mode,
Linus Torvalds's avatar
Linus Torvalds committed
1350 1351
	unsigned int cmd, unsigned long arg)
{
Al Viro's avatar
Al Viro committed
1352
	struct loop_device *lo = bdev->bd_disk->private_data;
Linus Torvalds's avatar
Linus Torvalds committed
1353 1354
	int err;

1355
	mutex_lock_nested(&lo->lo_ctl_mutex, 1);
Linus Torvalds's avatar
Linus Torvalds committed
1356 1357
	switch (cmd) {
	case LOOP_SET_FD:
Al Viro's avatar
Al Viro committed
1358
		err = loop_set_fd(lo, mode, bdev, arg);
Linus Torvalds's avatar
Linus Torvalds committed
1359 1360
		break;
	case LOOP_CHANGE_FD:
Al Viro's avatar
Al Viro committed
1361
		err = loop_change_fd(lo, bdev, arg);
Linus Torvalds's avatar
Linus Torvalds committed
1362 1363
		break;
	case LOOP_CLR_FD:
1364
		/* loop_clr_fd would have unlocked lo_ctl_mutex on success */
1365
		err = loop_clr_fd(lo);
1366 1367
		if (!err)
			goto out_unlocked;
Linus Torvalds's avatar
Linus Torvalds committed
1368 1369
		break;
	case LOOP_SET_STATUS:
1370 1371 1372 1373
		err = -EPERM;
		if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
			err = loop_set_status_old(lo,
					(struct loop_info __user *)arg);
Linus Torvalds's avatar
Linus Torvalds committed
1374 1375 1376 1377 1378
		break;
	case LOOP_GET_STATUS:
		err = loop_get_status_old(lo, (struct loop_info __user *) arg);
		break;
	case LOOP_SET_STATUS64:
1379 1380 1381 1382
		err = -EPERM;
		if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
			err = loop_set_status64(lo,
					(struct loop_info64 __user *) arg);
Linus Torvalds's avatar
Linus Torvalds committed
1383 1384 1385 1386
		break;
	case LOOP_GET_STATUS64:
		err = loop_get_status64(lo, (struct loop_info64 __user *) arg);
		break;
1387 1388 1389
	case LOOP_SET_CAPACITY:
		err = -EPERM;
		if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
1390
			err = loop_set_capacity(lo);
1391
		break;
1392 1393 1394 1395 1396
	case LOOP_SET_DIRECT_IO:
		err = -EPERM;
		if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
			err = loop_set_dio(lo, arg);
		break;
1397 1398 1399 1400 1401
	case LOOP_SET_BLOCK_SIZE:
		err = -EPERM;
		if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
			err = loop_set_block_size(lo, arg);
		break;
Linus Torvalds's avatar
Linus Torvalds committed
1402 1403 1404
	default:
		err = lo->ioctl ? lo->ioctl(lo, cmd, arg) : -EINVAL;
	}
1405
	mutex_unlock(&lo->lo_ctl_mutex);
1406 1407

out_unlocked:
Linus Torvalds's avatar
Linus Torvalds committed
1408 1409 1410
	return err;
}

1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431
#ifdef CONFIG_COMPAT
struct compat_loop_info {
	compat_int_t	lo_number;      /* ioctl r/o */
	compat_dev_t	lo_device;      /* ioctl r/o */
	compat_ulong_t	lo_inode;       /* ioctl r/o */
	compat_dev_t	lo_rdevice;     /* ioctl r/o */
	compat_int_t	lo_offset;
	compat_int_t	lo_encrypt_type;
	compat_int_t	lo_encrypt_key_size;    /* ioctl w/o */
	compat_int_t	lo_flags;       /* ioctl r/o */
	char		lo_name[LO_NAME_SIZE];
	unsigned char	lo_encrypt_key[LO_KEY_SIZE]; /* ioctl w/o */
	compat_ulong_t	lo_init[2];
	char		reserved[4];
};

/*
 * Transfer 32-bit compatibility structure in userspace to 64-bit loop info
 * - noinlined to reduce stack space usage in main part of driver
 */
static noinline int
1432
loop_info64_from_compat(const struct compat_loop_info __user *arg,
1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529
			struct loop_info64 *info64)
{
	struct compat_loop_info info;

	if (copy_from_user(&info, arg, sizeof(info)))
		return -EFAULT;

	memset(info64, 0, sizeof(*info64));
	info64->lo_number = info.lo_number;
	info64->lo_device = info.lo_device;
	info64->lo_inode = info.lo_inode;
	info64->lo_rdevice = info.lo_rdevice;
	info64->lo_offset = info.lo_offset;
	info64->lo_sizelimit = 0;
	info64->lo_encrypt_type = info.lo_encrypt_type;
	info64->lo_encrypt_key_size = info.lo_encrypt_key_size;
	info64->lo_flags = info.lo_flags;
	info64->lo_init[0] = info.lo_init[0];
	info64->lo_init[1] = info.lo_init[1];
	if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
		memcpy(info64->lo_crypt_name, info.lo_name, LO_NAME_SIZE);
	else
		memcpy(info64->lo_file_name, info.lo_name, LO_NAME_SIZE);
	memcpy(info64->lo_encrypt_key, info.lo_encrypt_key, LO_KEY_SIZE);
	return 0;
}

/*
 * Transfer 64-bit loop info to 32-bit compatibility structure in userspace
 * - noinlined to reduce stack space usage in main part of driver
 */
static noinline int
loop_info64_to_compat(const struct loop_info64 *info64,
		      struct compat_loop_info __user *arg)
{
	struct compat_loop_info info;

	memset(&info, 0, sizeof(info));
	info.lo_number = info64->lo_number;
	info.lo_device = info64->lo_device;
	info.lo_inode = info64->lo_inode;
	info.lo_rdevice = info64->lo_rdevice;
	info.lo_offset = info64->lo_offset;
	info.lo_encrypt_type = info64->lo_encrypt_type;
	info.lo_encrypt_key_size = info64->lo_encrypt_key_size;
	info.lo_flags = info64->lo_flags;
	info.lo_init[0] = info64->lo_init[0];
	info.lo_init[1] = info64->lo_init[1];
	if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
		memcpy(info.lo_name, info64->lo_crypt_name, LO_NAME_SIZE);
	else
		memcpy(info.lo_name, info64->lo_file_name, LO_NAME_SIZE);
	memcpy(info.lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE);

	/* error in case values were truncated */
	if (info.lo_device != info64->lo_device ||
	    info.lo_rdevice != info64->lo_rdevice ||
	    info.lo_inode != info64->lo_inode ||
	    info.lo_offset != info64->lo_offset ||
	    info.lo_init[0] != info64->lo_init[0] ||
	    info.lo_init[1] != info64->lo_init[1])
		return -EOVERFLOW;

	if (copy_to_user(arg, &info, sizeof(info)))
		return -EFAULT;
	return 0;
}

static int
loop_set_status_compat(struct loop_device *lo,
		       const struct compat_loop_info __user *arg)
{
	struct loop_info64 info64;
	int ret;

	ret = loop_info64_from_compat(arg, &info64);
	if (ret < 0)
		return ret;
	return loop_set_status(lo, &info64);
}

static int
loop_get_status_compat(struct loop_device *lo,
		       struct compat_loop_info __user *arg)
{
	struct loop_info64 info64;
	int err = 0;

	if (!arg)
		err = -EINVAL;
	if (!err)
		err = loop_get_status(lo, &info64);
	if (!err)
		err = loop_info64_to_compat(&info64, arg);
	return err;
}

Al Viro's avatar
Al Viro committed
1530 1531
static int lo_compat_ioctl(struct block_device *bdev, fmode_t mode,
			   unsigned int cmd, unsigned long arg)
1532
{
Al Viro's avatar
Al Viro committed
1533
	struct loop_device *lo = bdev->bd_disk->private_data;
1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548
	int err;

	switch(cmd) {
	case LOOP_SET_STATUS:
		mutex_lock(&lo->lo_ctl_mutex);
		err = loop_set_status_compat(
			lo, (const struct compat_loop_info __user *) arg);
		mutex_unlock(&lo->lo_ctl_mutex);
		break;
	case LOOP_GET_STATUS:
		mutex_lock(&lo->lo_ctl_mutex);
		err = loop_get_status_compat(
			lo, (struct compat_loop_info __user *) arg);
		mutex_unlock(&lo->lo_ctl_mutex);
		break;
1549
	case LOOP_SET_CAPACITY:
1550 1551 1552 1553 1554 1555
	case LOOP_CLR_FD:
	case LOOP_GET_STATUS64:
	case LOOP_SET_STATUS64:
		arg = (unsigned long) compat_ptr(arg);
	case LOOP_SET_FD:
	case LOOP_CHANGE_FD:
Al Viro's avatar
Al Viro committed
1556
		err = lo_ioctl(bdev, mode, cmd, arg);
1557 1558 1559 1560 1561 1562 1563 1564 1565
		break;
	default:
		err = -ENOIOCTLCMD;
		break;
	}
	return err;
}
#endif

Al Viro's avatar
Al Viro committed
1566
static int lo_open(struct block_device *bdev, fmode_t mode)
Linus Torvalds's avatar
Linus Torvalds committed
1567
{
1568 1569 1570 1571 1572 1573 1574 1575 1576
	struct loop_device *lo;
	int err = 0;

	mutex_lock(&loop_index_mutex);
	lo = bdev->bd_disk->private_data;
	if (!lo) {
		err = -ENXIO;
		goto out;
	}
Linus Torvalds's avatar
Linus Torvalds committed
1577

1578
	atomic_inc(&lo->lo_refcnt);
1579 1580 1581
out:
	mutex_unlock(&loop_index_mutex);
	return err;
Linus Torvalds's avatar
Linus Torvalds committed
1582 1583
}

1584
static void __lo_release(struct loop_device *lo)
Linus Torvalds's avatar
Linus Torvalds committed
1585
{
1586
	int err;
Linus Torvalds's avatar
Linus Torvalds committed
1587

1588 1589
	if (atomic_dec_return(&lo->lo_refcnt))
		return;
1590

1591
	mutex_lock(&lo->lo_ctl_mutex);
1592 1593 1594 1595 1596
	if (lo->lo_flags & LO_FLAGS_AUTOCLEAR) {
		/*
		 * In autoclear mode, stop the loop thread
		 * and remove configuration after last close.
		 */
1597
		err = loop_clr_fd(lo);
1598
		if (!err)
1599
			return;
1600
	} else if (lo->lo_state == Lo_bound) {
1601 1602 1603 1604
		/*
		 * Otherwise keep thread (if running) and config,
		 * but flush possible ongoing bios in thread.
		 */
1605 1606
		blk_mq_freeze_queue(lo->lo_queue);
		blk_mq_unfreeze_queue(lo->lo_queue);
1607
	}
1608

1609
	mutex_unlock(&lo->lo_ctl_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
1610 1611
}

1612 1613 1614 1615 1616 1617 1618
static void lo_release(struct gendisk *disk, fmode_t mode)
{
	mutex_lock(&loop_index_mutex);
	__lo_release(disk->private_data);
	mutex_unlock(&loop_index_mutex);
}

1619
static const struct block_device_operations lo_fops = {
Linus Torvalds's avatar
Linus Torvalds committed
1620
	.owner =	THIS_MODULE,
Al Viro's avatar
Al Viro committed
1621 1622 1623
	.open =		lo_open,
	.release =	lo_release,
	.ioctl =	lo_ioctl,
1624
#ifdef CONFIG_COMPAT
Al Viro's avatar
Al Viro committed
1625
	.compat_ioctl =	lo_compat_ioctl,
1626
#endif
Linus Torvalds's avatar
Linus Torvalds committed