blkdev.h 47.2 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3
#ifndef _LINUX_BLKDEV_H
#define _LINUX_BLKDEV_H

4 5
#include <linux/sched.h>

Jens Axboe's avatar
Jens Axboe committed
6 7
#ifdef CONFIG_BLOCK

Linus Torvalds's avatar
Linus Torvalds committed
8 9 10
#include <linux/major.h>
#include <linux/genhd.h>
#include <linux/list.h>
11
#include <linux/llist.h>
Linus Torvalds's avatar
Linus Torvalds committed
12 13 14 15 16 17 18 19
#include <linux/timer.h>
#include <linux/workqueue.h>
#include <linux/pagemap.h>
#include <linux/backing-dev.h>
#include <linux/wait.h>
#include <linux/mempool.h>
#include <linux/bio.h>
#include <linux/stringify.h>
20
#include <linux/gfp.h>
21
#include <linux/bsg.h>
22
#include <linux/smp.h>
Tejun Heo's avatar
Tejun Heo committed
23
#include <linux/rcupdate.h>
24
#include <linux/percpu-refcount.h>
Linus Torvalds's avatar
Linus Torvalds committed
25 26 27

#include <asm/scatterlist.h>

28
struct module;
29 30
struct scsi_ioctl_command;

Linus Torvalds's avatar
Linus Torvalds committed
31 32 33
struct request_queue;
struct elevator_queue;
struct request_pm_state;
34
struct blk_trace;
35 36
struct request;
struct sg_io_hdr;
Mike Christie's avatar
Mike Christie committed
37
struct bsg_job;
Tejun Heo's avatar
Tejun Heo committed
38
struct blkcg_gq;
39
struct blk_flush_queue;
Linus Torvalds's avatar
Linus Torvalds committed
40 41 42 43

#define BLKDEV_MIN_RQ	4
#define BLKDEV_MAX_RQ	128	/* Default maximum */

44 45 46 47 48 49
/*
 * Maximum number of blkcg policies allowed to be registered concurrently.
 * Defined here to simplify include dependency.
 */
#define BLKCG_MAX_POLS		2

Linus Torvalds's avatar
Linus Torvalds committed
50
struct request;
51
typedef void (rq_end_io_fn)(struct request *, int);
Linus Torvalds's avatar
Linus Torvalds committed
52

53 54 55
#define BLK_RL_SYNCFULL		(1U << 0)
#define BLK_RL_ASYNCFULL	(1U << 1)

Linus Torvalds's avatar
Linus Torvalds committed
56
struct request_list {
57
	struct request_queue	*q;	/* the queue this rl belongs to */
58 59 60
#ifdef CONFIG_BLK_CGROUP
	struct blkcg_gq		*blkg;	/* blkg this request pool belongs to */
#endif
61 62 63 64
	/*
	 * count[], starved[], and wait[] are indexed by
	 * BLK_RW_SYNC/BLK_RW_ASYNC
	 */
65 66 67 68
	int			count[2];
	int			starved[2];
	mempool_t		*rq_pool;
	wait_queue_head_t	wait[2];
69
	unsigned int		flags;
Linus Torvalds's avatar
Linus Torvalds committed
70 71
};

72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
/*
 * request command types
 */
enum rq_cmd_type_bits {
	REQ_TYPE_FS		= 1,	/* fs request */
	REQ_TYPE_BLOCK_PC,		/* scsi command */
	REQ_TYPE_SENSE,			/* sense request */
	REQ_TYPE_PM_SUSPEND,		/* suspend request */
	REQ_TYPE_PM_RESUME,		/* resume request */
	REQ_TYPE_PM_SHUTDOWN,		/* shutdown request */
	REQ_TYPE_SPECIAL,		/* driver defined type */
	/*
	 * for ATA/ATAPI devices. this really doesn't belong here, ide should
	 * use REQ_TYPE_SPECIAL and use rq->cmd[0] with the range of driver
	 * private REQ_LB opcodes to differentiate what type of request this is
	 */
	REQ_TYPE_ATA_TASKFILE,
89
	REQ_TYPE_ATA_PC,
90 91
};

Linus Torvalds's avatar
Linus Torvalds committed
92 93 94
#define BLK_MAX_CDB	16

/*
95 96 97 98
 * Try to put the fields that are referenced together in the same cacheline.
 *
 * If you modify this structure, make sure to update blk_rq_init() and
 * especially blk_mq_rq_ctx_init() to take care of the added fields.
Linus Torvalds's avatar
Linus Torvalds committed
99 100
 */
struct request {
101
	struct list_head queuelist;
102 103
	union {
		struct call_single_data csd;
104
		unsigned long fifo_time;
105
	};
106

107
	struct request_queue *q;
108
	struct blk_mq_ctx *mq_ctx;
109

110
	u64 cmd_flags;
111
	enum rq_cmd_type_bits cmd_type;
112
	unsigned long atomic_flags;
Linus Torvalds's avatar
Linus Torvalds committed
113

114 115
	int cpu;

116 117
	/* the following two fields are internal, NEVER access directly */
	unsigned int __data_len;	/* total data len */
118
	sector_t __sector;		/* sector cursor */
Linus Torvalds's avatar
Linus Torvalds committed
119 120 121 122

	struct bio *bio;
	struct bio *biotail;

123 124 125 126 127 128 129 130 131 132 133 134
	/*
	 * The hash is used inside the scheduler, and killed once the
	 * request reaches the dispatch list. The ipi_list is only used
	 * to queue the request for softirq completion, which is long
	 * after the request has been unhashed (and even removed from
	 * the dispatch list).
	 */
	union {
		struct hlist_node hash;	/* merge hash */
		struct list_head ipi_list;
	};

135 136 137
	/*
	 * The rb_node is only used inside the io scheduler, requests
	 * are pruned when moved to the dispatch queue. So let the
138
	 * completion_data share space with the rb_node.
139 140 141
	 */
	union {
		struct rb_node rb_node;	/* sort/lookup */
142
		void *completion_data;
143
	};
144

145
	/*
146
	 * Three pointers are available for the IO schedulers, if they need
147 148
	 * more they have to dynamically allocate it.  Flush requests are
	 * never put on the IO scheduler. So let the flush fields share
149
	 * space with the elevator data.
150
	 */
151
	union {
152 153 154 155 156
		struct {
			struct io_cq		*icq;
			void			*priv[2];
		} elv;

157 158 159
		struct {
			unsigned int		seq;
			struct list_head	list;
160
			rq_end_io_fn		*saved_end_io;
161 162
		} flush;
	};
163

164
	struct gendisk *rq_disk;
165
	struct hd_struct *part;
Linus Torvalds's avatar
Linus Torvalds committed
166
	unsigned long start_time;
167
#ifdef CONFIG_BLK_CGROUP
168
	struct request_list *rl;		/* rl this rq is alloced from */
169 170 171
	unsigned long long start_time_ns;
	unsigned long long io_start_time_ns;    /* when passed to hardware */
#endif
Linus Torvalds's avatar
Linus Torvalds committed
172 173 174 175
	/* Number of scatter-gather DMA addr+len pairs after
	 * physical address coalescing is performed.
	 */
	unsigned short nr_phys_segments;
176 177 178
#if defined(CONFIG_BLK_DEV_INTEGRITY)
	unsigned short nr_integrity_segments;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
179

180 181
	unsigned short ioprio;

Tejun Heo's avatar
Tejun Heo committed
182
	void *special;		/* opaque pointer available for LLD use */
Linus Torvalds's avatar
Linus Torvalds committed
183

184 185 186
	int tag;
	int errors;

Linus Torvalds's avatar
Linus Torvalds committed
187 188 189
	/*
	 * when request is used as a packet command carrier
	 */
190 191
	unsigned char __cmd[BLK_MAX_CDB];
	unsigned char *cmd;
192
	unsigned short cmd_len;
Linus Torvalds's avatar
Linus Torvalds committed
193

194
	unsigned int extra_len;	/* length of alignment and padding */
Linus Torvalds's avatar
Linus Torvalds committed
195
	unsigned int sense_len;
Tejun Heo's avatar
Tejun Heo committed
196
	unsigned int resid_len;	/* residual count */
Linus Torvalds's avatar
Linus Torvalds committed
197 198
	void *sense;

199 200
	unsigned long deadline;
	struct list_head timeout_list;
Linus Torvalds's avatar
Linus Torvalds committed
201
	unsigned int timeout;
202
	int retries;
Linus Torvalds's avatar
Linus Torvalds committed
203 204

	/*
205
	 * completion callback.
Linus Torvalds's avatar
Linus Torvalds committed
206 207 208
	 */
	rq_end_io_fn *end_io;
	void *end_io_data;
209 210 211

	/* for bidi */
	struct request *next_rq;
Linus Torvalds's avatar
Linus Torvalds committed
212 213
};

214 215 216 217 218
static inline unsigned short req_get_ioprio(struct request *req)
{
	return req->ioprio;
}

Linus Torvalds's avatar
Linus Torvalds committed
219
/*
220
 * State information carried for REQ_TYPE_PM_SUSPEND and REQ_TYPE_PM_RESUME
Linus Torvalds's avatar
Linus Torvalds committed
221 222 223 224 225 226 227 228 229 230 231 232 233
 * requests. Some step values could eventually be made generic.
 */
struct request_pm_state
{
	/* PM state machine step value, currently driver specific */
	int	pm_step;
	/* requested PM state value (S1, S2, S3, S4, ...) */
	u32	pm_state;
	void*	data;		/* for driver use */
};

#include <linux/elevator.h>

234 235
struct blk_queue_ctx;

236
typedef void (request_fn_proc) (struct request_queue *q);
237
typedef void (make_request_fn) (struct request_queue *q, struct bio *bio);
238
typedef int (prep_rq_fn) (struct request_queue *, struct request *);
239
typedef void (unprep_rq_fn) (struct request_queue *, struct request *);
Linus Torvalds's avatar
Linus Torvalds committed
240 241

struct bio_vec;
242 243 244 245 246 247 248 249
struct bvec_merge_data {
	struct block_device *bi_bdev;
	sector_t bi_sector;
	unsigned bi_size;
	unsigned long bi_rw;
};
typedef int (merge_bvec_fn) (struct request_queue *, struct bvec_merge_data *,
			     struct bio_vec *);
250
typedef void (softirq_done_fn)(struct request *);
251
typedef int (dma_drain_needed_fn)(struct request *);
252
typedef int (lld_busy_fn) (struct request_queue *q);
Mike Christie's avatar
Mike Christie committed
253
typedef int (bsg_job_fn) (struct bsg_job *);
Linus Torvalds's avatar
Linus Torvalds committed
254

255 256 257 258 259 260 261 262
enum blk_eh_timer_return {
	BLK_EH_NOT_HANDLED,
	BLK_EH_HANDLED,
	BLK_EH_RESET_TIMER,
};

typedef enum blk_eh_timer_return (rq_timed_out_fn)(struct request *);

Linus Torvalds's avatar
Linus Torvalds committed
263 264 265 266 267 268 269 270 271 272
enum blk_queue_state {
	Queue_down,
	Queue_up,
};

struct blk_queue_tag {
	struct request **tag_index;	/* map of busy tags */
	unsigned long *tag_map;		/* bit map of free/busy tags */
	int busy;			/* current depth */
	int max_depth;			/* what we will send to device */
273
	int real_max_depth;		/* what the array can hold */
Linus Torvalds's avatar
Linus Torvalds committed
274 275 276
	atomic_t refcnt;		/* map can be shared */
};

277 278 279
#define BLK_SCSI_MAX_CMDS	(256)
#define BLK_SCSI_CMD_PER_LONG	(BLK_SCSI_MAX_CMDS / (sizeof(long) * 8))

280 281 282 283 284
struct queue_limits {
	unsigned long		bounce_pfn;
	unsigned long		seg_boundary_mask;

	unsigned int		max_hw_sectors;
285
	unsigned int		chunk_sectors;
286 287
	unsigned int		max_sectors;
	unsigned int		max_segment_size;
288 289 290 291
	unsigned int		physical_block_size;
	unsigned int		alignment_offset;
	unsigned int		io_min;
	unsigned int		io_opt;
292
	unsigned int		max_discard_sectors;
293
	unsigned int		max_write_same_sectors;
294 295
	unsigned int		discard_granularity;
	unsigned int		discard_alignment;
296 297

	unsigned short		logical_block_size;
298
	unsigned short		max_segments;
299
	unsigned short		max_integrity_segments;
300

301
	unsigned char		misaligned;
302
	unsigned char		discard_misaligned;
303
	unsigned char		cluster;
304
	unsigned char		discard_zeroes_data;
305
	unsigned char		raid_partial_stripes_expensive;
306 307
};

308
struct request_queue {
Linus Torvalds's avatar
Linus Torvalds committed
309 310 311 312 313
	/*
	 * Together with queue_head for cacheline sharing
	 */
	struct list_head	queue_head;
	struct request		*last_merge;
314
	struct elevator_queue	*elevator;
315 316
	int			nr_rqs[2];	/* # allocated [a]sync rqs */
	int			nr_rqs_elvpriv;	/* # allocated rqs w/ elvpriv */
Linus Torvalds's avatar
Linus Torvalds committed
317 318

	/*
319 320 321 322
	 * If blkcg is not used, @q->root_rl serves all requests.  If blkcg
	 * is used, root blkg allocates from @q->root_rl and all other
	 * blkgs from their own blkg->rl.  Which one to use should be
	 * determined using bio_request_list().
Linus Torvalds's avatar
Linus Torvalds committed
323
	 */
324
	struct request_list	root_rl;
Linus Torvalds's avatar
Linus Torvalds committed
325 326 327 328

	request_fn_proc		*request_fn;
	make_request_fn		*make_request_fn;
	prep_rq_fn		*prep_rq_fn;
329
	unprep_rq_fn		*unprep_rq_fn;
Linus Torvalds's avatar
Linus Torvalds committed
330
	merge_bvec_fn		*merge_bvec_fn;
331
	softirq_done_fn		*softirq_done_fn;
332
	rq_timed_out_fn		*rq_timed_out_fn;
333
	dma_drain_needed_fn	*dma_drain_needed;
334
	lld_busy_fn		*lld_busy_fn;
Linus Torvalds's avatar
Linus Torvalds committed
335

336 337 338 339 340
	struct blk_mq_ops	*mq_ops;

	unsigned int		*mq_map;

	/* sw queues */
341
	struct blk_mq_ctx __percpu	*queue_ctx;
342 343 344 345 346 347
	unsigned int		nr_queues;

	/* hw dispatch queues */
	struct blk_mq_hw_ctx	**queue_hw_ctx;
	unsigned int		nr_hw_queues;

348 349 350
	/*
	 * Dispatch queue sorting
	 */
Jens Axboe's avatar
Jens Axboe committed
351
	sector_t		end_sector;
352 353
	struct request		*boundary_rq;

Linus Torvalds's avatar
Linus Torvalds committed
354
	/*
355
	 * Delayed queue handling
Linus Torvalds's avatar
Linus Torvalds committed
356
	 */
357
	struct delayed_work	delay_work;
Linus Torvalds's avatar
Linus Torvalds committed
358 359 360 361 362 363 364 365 366 367

	struct backing_dev_info	backing_dev_info;

	/*
	 * The queue owner gets to use this for whatever they like.
	 * ll_rw_blk doesn't touch it.
	 */
	void			*queuedata;

	/*
368
	 * various queue flags, see QUEUE_* below
Linus Torvalds's avatar
Linus Torvalds committed
369
	 */
370
	unsigned long		queue_flags;
Linus Torvalds's avatar
Linus Torvalds committed
371

372 373 374 375 376 377
	/*
	 * ida allocated id for this queue.  Used to index queues from
	 * ioctx.
	 */
	int			id;

Linus Torvalds's avatar
Linus Torvalds committed
378
	/*
379
	 * queue needs bounce pages for pages above this limit
Linus Torvalds's avatar
Linus Torvalds committed
380
	 */
381
	gfp_t			bounce_gfp;
Linus Torvalds's avatar
Linus Torvalds committed
382 383

	/*
's avatar
committed
384 385 386
	 * protects queue structures from reentrancy. ->__queue_lock should
	 * _never_ be used directly, it is queue private. always use
	 * ->queue_lock.
Linus Torvalds's avatar
Linus Torvalds committed
387
	 */
's avatar
committed
388
	spinlock_t		__queue_lock;
Linus Torvalds's avatar
Linus Torvalds committed
389 390 391 392 393 394 395
	spinlock_t		*queue_lock;

	/*
	 * queue kobject
	 */
	struct kobject kobj;

396 397 398 399 400
	/*
	 * mq queue kobject
	 */
	struct kobject mq_kobj;

Lin Ming's avatar
Lin Ming committed
401 402 403 404 405 406
#ifdef CONFIG_PM_RUNTIME
	struct device		*dev;
	int			rpm_status;
	unsigned int		nr_pending;
#endif

Linus Torvalds's avatar
Linus Torvalds committed
407 408 409 410 411 412 413 414
	/*
	 * queue settings
	 */
	unsigned long		nr_requests;	/* Max # of requests */
	unsigned int		nr_congestion_on;
	unsigned int		nr_congestion_off;
	unsigned int		nr_batching;

415
	unsigned int		dma_drain_size;
416
	void			*dma_drain_buffer;
417
	unsigned int		dma_pad_mask;
Linus Torvalds's avatar
Linus Torvalds committed
418 419 420
	unsigned int		dma_alignment;

	struct blk_queue_tag	*queue_tags;
421
	struct list_head	tag_busy_list;
Linus Torvalds's avatar
Linus Torvalds committed
422

423
	unsigned int		nr_sorted;
424
	unsigned int		in_flight[2];
425 426 427 428 429 430
	/*
	 * Number of active block driver functions for which blk_drain_queue()
	 * must wait. Must be incremented around functions that unlock the
	 * queue_lock internally, e.g. scsi_request_fn().
	 */
	unsigned int		request_fn_active;
Linus Torvalds's avatar
Linus Torvalds committed
431

432 433 434 435
	unsigned int		rq_timeout;
	struct timer_list	timeout;
	struct list_head	timeout_list;

436
	struct list_head	icq_list;
437
#ifdef CONFIG_BLK_CGROUP
438
	DECLARE_BITMAP		(blkcg_pols, BLKCG_MAX_POLS);
Tejun Heo's avatar
Tejun Heo committed
439
	struct blkcg_gq		*root_blkg;
440
	struct list_head	blkg_list;
441
#endif
442

443 444
	struct queue_limits	limits;

Linus Torvalds's avatar
Linus Torvalds committed
445 446 447 448 449
	/*
	 * sg stuff
	 */
	unsigned int		sg_timeout;
	unsigned int		sg_reserved_size;
450
	int			node;
451
#ifdef CONFIG_BLK_DEV_IO_TRACE
452
	struct blk_trace	*blk_trace;
453
#endif
Linus Torvalds's avatar
Linus Torvalds committed
454
	/*
455
	 * for flush operations
Linus Torvalds's avatar
Linus Torvalds committed
456
	 */
457
	unsigned int		flush_flags;
458
	unsigned int		flush_not_queueable:1;
459
	struct blk_flush_queue	*fq;
460

461 462 463 464
	struct list_head	requeue_list;
	spinlock_t		requeue_lock;
	struct work_struct	requeue_work;

465
	struct mutex		sysfs_lock;
466

467
	int			bypass_depth;
468
	int			mq_freeze_depth;
469

470
#if defined(CONFIG_BLK_DEV_BSG)
Mike Christie's avatar
Mike Christie committed
471 472
	bsg_job_fn		*bsg_job_fn;
	int			bsg_job_size;
473 474
	struct bsg_class_device bsg_dev;
#endif
475 476 477 478 479

#ifdef CONFIG_BLK_DEV_THROTTLING
	/* Throttle data */
	struct throtl_data *td;
#endif
Tejun Heo's avatar
Tejun Heo committed
480
	struct rcu_head		rcu_head;
481
	wait_queue_head_t	mq_freeze_wq;
482
	struct percpu_ref	mq_usage_counter;
483
	struct list_head	all_q_node;
484 485 486

	struct blk_mq_tag_set	*tag_set;
	struct list_head	tag_set_list;
Linus Torvalds's avatar
Linus Torvalds committed
487 488 489 490
};

#define QUEUE_FLAG_QUEUED	1	/* uses generic tag queueing */
#define QUEUE_FLAG_STOPPED	2	/* queue is stopped */
491 492
#define	QUEUE_FLAG_SYNCFULL	3	/* read queue has been filled */
#define QUEUE_FLAG_ASYNCFULL	4	/* write queue has been filled */
493
#define QUEUE_FLAG_DYING	5	/* queue being torn down */
494
#define QUEUE_FLAG_BYPASS	6	/* act as dumb FIFO queue */
495 496
#define QUEUE_FLAG_BIDI		7	/* queue supports bidi requests */
#define QUEUE_FLAG_NOMERGES     8	/* disable merge attempts */
Dan Williams's avatar
Dan Williams committed
497
#define QUEUE_FLAG_SAME_COMP	9	/* complete on same CPU-group */
498 499 500
#define QUEUE_FLAG_FAIL_IO     10	/* fake timeout */
#define QUEUE_FLAG_STACKABLE   11	/* supports request stacking */
#define QUEUE_FLAG_NONROT      12	/* non-rotational device (SSD) */
501
#define QUEUE_FLAG_VIRT        QUEUE_FLAG_NONROT /* paravirt device */
502 503 504 505 506
#define QUEUE_FLAG_IO_STAT     13	/* do IO stats */
#define QUEUE_FLAG_DISCARD     14	/* supports DISCARD */
#define QUEUE_FLAG_NOXMERGES   15	/* No extended merges */
#define QUEUE_FLAG_ADD_RANDOM  16	/* Contributes to random pool */
#define QUEUE_FLAG_SECDISCARD  17	/* supports SECDISCARD */
Dan Williams's avatar
Dan Williams committed
507
#define QUEUE_FLAG_SAME_FORCE  18	/* force complete on same CPU */
508
#define QUEUE_FLAG_DEAD        19	/* queue tear-down finished */
509
#define QUEUE_FLAG_INIT_DONE   20	/* queue is initialized */
510
#define QUEUE_FLAG_NO_SG_MERGE 21	/* don't attempt to merge SG segments*/
511
#define QUEUE_FLAG_SG_GAPS     22	/* queue doesn't support SG gaps */
512 513

#define QUEUE_FLAG_DEFAULT	((1 << QUEUE_FLAG_IO_STAT) |		\
514
				 (1 << QUEUE_FLAG_STACKABLE)	|	\
515 516
				 (1 << QUEUE_FLAG_SAME_COMP)	|	\
				 (1 << QUEUE_FLAG_ADD_RANDOM))
517

518 519 520
#define QUEUE_FLAG_MQ_DEFAULT	((1 << QUEUE_FLAG_IO_STAT) |		\
				 (1 << QUEUE_FLAG_SAME_COMP))

521
static inline void queue_lockdep_assert_held(struct request_queue *q)
522
{
523 524
	if (q->queue_lock)
		lockdep_assert_held(q->queue_lock);
525 526
}

527 528 529 530 531 532
static inline void queue_flag_set_unlocked(unsigned int flag,
					   struct request_queue *q)
{
	__set_bit(flag, &q->queue_flags);
}

Jens Axboe's avatar
Jens Axboe committed
533 534 535
static inline int queue_flag_test_and_clear(unsigned int flag,
					    struct request_queue *q)
{
536
	queue_lockdep_assert_held(q);
Jens Axboe's avatar
Jens Axboe committed
537 538 539 540 541 542 543 544 545 546 547 548

	if (test_bit(flag, &q->queue_flags)) {
		__clear_bit(flag, &q->queue_flags);
		return 1;
	}

	return 0;
}

static inline int queue_flag_test_and_set(unsigned int flag,
					  struct request_queue *q)
{
549
	queue_lockdep_assert_held(q);
Jens Axboe's avatar
Jens Axboe committed
550 551 552 553 554 555 556 557 558

	if (!test_bit(flag, &q->queue_flags)) {
		__set_bit(flag, &q->queue_flags);
		return 0;
	}

	return 1;
}

559 560
static inline void queue_flag_set(unsigned int flag, struct request_queue *q)
{
561
	queue_lockdep_assert_held(q);
562 563 564 565 566 567 568 569 570
	__set_bit(flag, &q->queue_flags);
}

static inline void queue_flag_clear_unlocked(unsigned int flag,
					     struct request_queue *q)
{
	__clear_bit(flag, &q->queue_flags);
}

571 572 573 574 575
static inline int queue_in_flight(struct request_queue *q)
{
	return q->in_flight[0] + q->in_flight[1];
}

576 577
static inline void queue_flag_clear(unsigned int flag, struct request_queue *q)
{
578
	queue_lockdep_assert_held(q);
579 580 581
	__clear_bit(flag, &q->queue_flags);
}

Linus Torvalds's avatar
Linus Torvalds committed
582 583
#define blk_queue_tagged(q)	test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags)
#define blk_queue_stopped(q)	test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
584
#define blk_queue_dying(q)	test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
585
#define blk_queue_dead(q)	test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags)
586
#define blk_queue_bypass(q)	test_bit(QUEUE_FLAG_BYPASS, &(q)->queue_flags)
587
#define blk_queue_init_done(q)	test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
588
#define blk_queue_nomerges(q)	test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
589 590
#define blk_queue_noxmerges(q)	\
	test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
591
#define blk_queue_nonrot(q)	test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
592
#define blk_queue_io_stat(q)	test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
593
#define blk_queue_add_random(q)	test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
594 595
#define blk_queue_stackable(q)	\
	test_bit(QUEUE_FLAG_STACKABLE, &(q)->queue_flags)
596
#define blk_queue_discard(q)	test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)
Adrian Hunter's avatar
Adrian Hunter committed
597 598
#define blk_queue_secdiscard(q)	(blk_queue_discard(q) && \
	test_bit(QUEUE_FLAG_SECDISCARD, &(q)->queue_flags))
Linus Torvalds's avatar
Linus Torvalds committed
599

600 601 602 603 604 605
#define blk_noretry_request(rq) \
	((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
			     REQ_FAILFAST_DRIVER))

#define blk_account_rq(rq) \
	(((rq)->cmd_flags & REQ_STARTED) && \
606
	 ((rq)->cmd_type == REQ_TYPE_FS))
607

Linus Torvalds's avatar
Linus Torvalds committed
608
#define blk_pm_request(rq)	\
609 610
	((rq)->cmd_type == REQ_TYPE_PM_SUSPEND || \
	 (rq)->cmd_type == REQ_TYPE_PM_RESUME)
Linus Torvalds's avatar
Linus Torvalds committed
611

612
#define blk_rq_cpu_valid(rq)	((rq)->cpu != -1)
613
#define blk_bidi_rq(rq)		((rq)->next_rq != NULL)
614 615
/* rq->queuelist of dequeued request must be list_empty() */
#define blk_queued_rq(rq)	(!list_empty(&(rq)->queuelist))
Linus Torvalds's avatar
Linus Torvalds committed
616 617 618

#define list_entry_rq(ptr)	list_entry((ptr), struct request, queuelist)

619
#define rq_data_dir(rq)		(((rq)->cmd_flags & 1) != 0)
Linus Torvalds's avatar
Linus Torvalds committed
620

621 622 623 624 625 626 627 628 629
/*
 * Driver can handle struct request, if it either has an old style
 * request_fn defined, or is blk-mq based.
 */
static inline bool queue_is_rq_based(struct request_queue *q)
{
	return q->request_fn || q->mq_ops;
}

630 631 632 633 634
static inline unsigned int blk_queue_cluster(struct request_queue *q)
{
	return q->limits.cluster;
}

635
/*
636
 * We regard a request as sync, if either a read or a sync write
637
 */
638 639
static inline bool rw_is_sync(unsigned int rw_flags)
{
640
	return !(rw_flags & REQ_WRITE) || (rw_flags & REQ_SYNC);
641 642 643 644 645 646 647
}

static inline bool rq_is_sync(struct request *rq)
{
	return rw_is_sync(rq->cmd_flags);
}

648
static inline bool blk_rl_full(struct request_list *rl, bool sync)
Linus Torvalds's avatar
Linus Torvalds committed
649
{
650 651 652
	unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;

	return rl->flags & flag;
Linus Torvalds's avatar
Linus Torvalds committed
653 654
}

655
static inline void blk_set_rl_full(struct request_list *rl, bool sync)
Linus Torvalds's avatar
Linus Torvalds committed
656
{
657 658 659
	unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;

	rl->flags |= flag;
Linus Torvalds's avatar
Linus Torvalds committed
660 661
}

662
static inline void blk_clear_rl_full(struct request_list *rl, bool sync)
Linus Torvalds's avatar
Linus Torvalds committed
663
{
664 665 666
	unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;

	rl->flags &= ~flag;
Linus Torvalds's avatar
Linus Torvalds committed
667 668
}

669 670 671 672
static inline bool rq_mergeable(struct request *rq)
{
	if (rq->cmd_type != REQ_TYPE_FS)
		return false;
Linus Torvalds's avatar
Linus Torvalds committed
673

674 675 676 677 678
	if (rq->cmd_flags & REQ_NOMERGE_FLAGS)
		return false;

	return true;
}
Linus Torvalds's avatar
Linus Torvalds committed
679

680 681 682 683 684 685 686 687 688
static inline bool blk_check_merge_flags(unsigned int flags1,
					 unsigned int flags2)
{
	if ((flags1 & REQ_DISCARD) != (flags2 & REQ_DISCARD))
		return false;

	if ((flags1 & REQ_SECURE) != (flags2 & REQ_SECURE))
		return false;

689 690 691
	if ((flags1 & REQ_WRITE_SAME) != (flags2 & REQ_WRITE_SAME))
		return false;

692 693 694
	return true;
}

695 696 697 698 699 700 701 702
static inline bool blk_write_same_mergeable(struct bio *a, struct bio *b)
{
	if (bio_data(a) == bio_data(b))
		return true;

	return false;
}

Linus Torvalds's avatar
Linus Torvalds committed
703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718
/*
 * q->prep_rq_fn return values
 */
#define BLKPREP_OK		0	/* serve it */
#define BLKPREP_KILL		1	/* fatal error, kill */
#define BLKPREP_DEFER		2	/* leave on queue */

extern unsigned long blk_max_low_pfn, blk_max_pfn;

/*
 * standard bounce addresses:
 *
 * BLK_BOUNCE_HIGH	: bounce all highmem pages
 * BLK_BOUNCE_ANY	: don't bounce anything
 * BLK_BOUNCE_ISA	: bounce pages above ISA DMA boundary
 */
719 720

#if BITS_PER_LONG == 32
Linus Torvalds's avatar
Linus Torvalds committed
721
#define BLK_BOUNCE_HIGH		((u64)blk_max_low_pfn << PAGE_SHIFT)
722 723 724 725
#else
#define BLK_BOUNCE_HIGH		-1ULL
#endif
#define BLK_BOUNCE_ANY		(-1ULL)
726
#define BLK_BOUNCE_ISA		(DMA_BIT_MASK(24))
Linus Torvalds's avatar
Linus Torvalds committed
727

728 729 730 731
/*
 * default timeout for SG_IO if none specified
 */
#define BLK_DEFAULT_SG_TIMEOUT	(60 * HZ)
732
#define BLK_MIN_SG_TIMEOUT	(7 * HZ)
733

734
#ifdef CONFIG_BOUNCE
Linus Torvalds's avatar
Linus Torvalds committed
735
extern int init_emergency_isa_pool(void);
736
extern void blk_queue_bounce(struct request_queue *q, struct bio **bio);
Linus Torvalds's avatar
Linus Torvalds committed
737 738 739 740 741
#else
static inline int init_emergency_isa_pool(void)
{
	return 0;
}
742
static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio)
Linus Torvalds's avatar
Linus Torvalds committed
743 744 745 746
{
}
#endif /* CONFIG_MMU */

747 748 749 750
struct rq_map_data {
	struct page **pages;
	int page_order;
	int nr_entries;
751
	unsigned long offset;
752
	int null_mapped;
753
	int from_user;
754 755
};

756
struct req_iterator {
757
	struct bvec_iter iter;
758 759 760 761
	struct bio *bio;
};

/* This should not be used directly - use rq_for_each_segment */
762 763
#define for_each_bio(_bio)		\
	for (; _bio; _bio = _bio->bi_next)
764
#define __rq_for_each_bio(_bio, rq)	\
Linus Torvalds's avatar
Linus Torvalds committed
765 766 767
	if ((rq->bio))			\
		for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)

768 769
#define rq_for_each_segment(bvl, _rq, _iter)			\
	__rq_for_each_bio(_iter.bio, _rq)			\
770
		bio_for_each_segment(bvl, _iter.bio, _iter.iter)
771

Kent Overstreet's avatar
Kent Overstreet committed
772
#define rq_iter_last(bvec, _iter)				\
773
		(_iter.bio->bi_next == NULL &&			\
Kent Overstreet's avatar
Kent Overstreet committed
774
		 bio_iter_last(bvec, _iter.iter))
775

776 777 778 779 780 781 782 783 784 785 786
#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
# error	"You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
#endif
#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
extern void rq_flush_dcache_pages(struct request *rq);
#else
static inline void rq_flush_dcache_pages(struct request *rq)
{
}
#endif

Linus Torvalds's avatar
Linus Torvalds committed
787 788 789
extern int blk_register_queue(struct gendisk *disk);
extern void blk_unregister_queue(struct gendisk *disk);
extern void generic_make_request(struct bio *bio);
790
extern void blk_rq_init(struct request_queue *q, struct request *rq);
Linus Torvalds's avatar
Linus Torvalds committed
791
extern void blk_put_request(struct request *);
792 793
extern void __blk_put_request(struct request_queue *, struct request *);
extern struct request *blk_get_request(struct request_queue *, int, gfp_t);
794 795
extern struct request *blk_make_request(struct request_queue *, struct bio *,
					gfp_t);
Jens Axboe's avatar
Jens Axboe committed
796
extern void blk_rq_set_block_pc(struct request *);
797
extern void blk_requeue_request(struct request_queue *, struct request *);
798 799
extern void blk_add_request_payload(struct request *rq, struct page *page,
		unsigned int len);
800
extern int blk_rq_check_limits(struct request_queue *q, struct request *rq);
801
extern int blk_lld_busy(struct request_queue *q);
802 803 804 805 806
extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
			     struct bio_set *bs, gfp_t gfp_mask,
			     int (*bio_ctr)(struct bio *, struct bio *, void *),
			     void *data);
extern void blk_rq_unprep_clone(struct request *rq);
807 808
extern int blk_insert_cloned_request(struct request_queue *q,
				     struct request *rq);
809
extern void blk_delay_queue(struct request_queue *, unsigned long);
810
extern void blk_recount_segments(struct request_queue *, struct bio *);
811
extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
812 813
extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,
			      unsigned int, void __user *);
814 815
extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t,
			  unsigned int, void __user *);
816 817
extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t,
			 struct scsi_ioctl_command __user *);
818

819
extern void blk_queue_bio(struct request_queue *q, struct bio *bio);
820

821 822 823 824 825
/*
 * A queue has just exitted congestion.  Note this in the global counter of
 * congested queues, and wake up anyone who was waiting for requests to be
 * put back.
 */
826
static inline void blk_clear_queue_congested(struct request_queue *q, int sync)
827
{
828
	clear_bdi_congested(&q->backing_dev_info, sync);
829 830 831 832 833 834
}

/*
 * A queue has just entered congestion.  Flag that in the queue's VM-visible
 * state flags and increment the global gounter of congested queues.
 */
835
static inline void blk_set_queue_congested(struct request_queue *q, int sync)
836
{
837
	set_bdi_congested(&q->backing_dev_info, sync);
838 839
}

840 841
extern void blk_start_queue(struct request_queue *q);
extern void blk_stop_queue(struct request_queue *q);
Linus Torvalds's avatar
Linus Torvalds committed
842
extern void blk_sync_queue(struct request_queue *q);
843
extern void __blk_stop_queue(struct request_queue *q);
844
extern void __blk_run_queue(struct request_queue *q);
845
extern void blk_run_queue(struct request_queue *);
846
extern void blk_run_queue_async(struct request_queue *q);
847
extern int blk_rq_map_user(struct request_queue *, struct request *,
848 849
			   struct rq_map_data *, void __user *, unsigned long,
			   gfp_t);
850
extern int blk_rq_unmap_user(struct bio *);
851 852
extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, unsigned int, gfp_t);
extern int blk_rq_map_user_iov(struct request_queue *, struct request *,
853 854
			       struct rq_map_data *, const struct sg_iovec *,
			       int, unsigned int, gfp_t);
855
extern int blk_execute_rq(struct request_queue *, struct gendisk *,
856
			  struct request *, int);
857
extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
858
				  struct request *, int, rq_end_io_fn *);
859

860
static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
Linus Torvalds's avatar
Linus Torvalds committed
861
{
862
	return bdev->bd_disk->queue;	/* this is never NULL */
Linus Torvalds's avatar
Linus Torvalds committed
863 864
}

865
/*
866 867 868 869 870 871
 * blk_rq_pos()			: the current sector
 * blk_rq_bytes()		: bytes left in the entire request
 * blk_rq_cur_bytes()		: bytes left in the current segment
 * blk_rq_err_bytes()		: bytes left till the next error boundary
 * blk_rq_sectors()		: sectors left in the entire request
 * blk_rq_cur_sectors()		: sectors left in the current segment
872
 */
873 874
static inline sector_t blk_rq_pos(const struct request *rq)
{
875
	return rq->__sector;
876 877 878 879
}

static inline unsigned int blk_rq_bytes(const struct request *rq)
{
880
	return rq->__data_len;
881 882
}

883 884 885 886
static inline int blk_rq_cur_bytes(const struct request *rq)
{
	return rq->bio ? bio_cur_bytes(rq->bio) : 0;
}
887

888 889
extern unsigned int blk_rq_err_bytes(const struct request *rq);

890 891
static inline unsigned int blk_rq_sectors(const struct request *rq)
{
892
	return blk_rq_bytes(rq) >> 9;
893 894 895 896
}

static inline unsigned int blk_rq_cur_sectors(const struct request *rq)
{
897
	return blk_rq_cur_bytes(rq) >> 9;
898 899
}