Commit 544ccc8d authored by Omar Sandoval's avatar Omar Sandoval Committed by Jens Axboe

block: get rid of struct blk_issue_stat

struct blk_issue_stat squashes three things into one u64:

- The time the driver started working on a request
- The original size of the request (for the io.low controller)
- Flags for writeback throttling

It turns out that on x86_64, we have a 4 byte hole in struct request
which we can fill with the non-timestamp fields from blk_issue_stat,
simplifying things quite a bit.
Signed-off-by: default avatarOmar Sandoval <osandov@fb.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 5238dcf4
......@@ -2991,7 +2991,10 @@ void blk_start_request(struct request *req)
blk_dequeue_request(req);
if (test_bit(QUEUE_FLAG_STATS, &req->q->queue_flags)) {
blk_stat_set_issue(&req->issue_stat, blk_rq_sectors(req));
req->io_start_time_ns = ktime_get_ns();
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
req->throtl_size = blk_rq_sectors(req);
#endif
req->rq_flags |= RQF_STATS;
wbt_issue(req->q->rq_wb, req);
}
......
......@@ -310,6 +310,7 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
rq->rq_disk = NULL;
rq->part = NULL;
rq->start_time = jiffies;
rq->io_start_time_ns = 0;
rq->nr_phys_segments = 0;
#if defined(CONFIG_BLK_DEV_INTEGRITY)
rq->nr_integrity_segments = 0;
......@@ -329,7 +330,7 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
#ifdef CONFIG_BLK_CGROUP
rq->rl = NULL;
set_start_time_ns(rq);
rq->io_start_time_ns = 0;
rq->cgroup_io_start_time_ns = 0;
#endif
data->ctx->rq_dispatched[op_is_sync(op)]++;
......@@ -669,7 +670,10 @@ void blk_mq_start_request(struct request *rq)
trace_block_rq_issue(q, rq);
if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
blk_stat_set_issue(&rq->issue_stat, blk_rq_sectors(rq));
rq->io_start_time_ns = ktime_get_ns();
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
rq->throtl_size = blk_rq_sectors(rq);
#endif
rq->rq_flags |= RQF_STATS;
wbt_issue(q->rq_wb, rq);
}
......
......@@ -55,11 +55,8 @@ void blk_stat_add(struct request *rq)
int bucket;
u64 now, value;
now = __blk_stat_time(ktime_to_ns(ktime_get()));
if (now < blk_stat_time(&rq->issue_stat))
return;
value = now - blk_stat_time(&rq->issue_stat);
now = ktime_get_ns();
value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0;
blk_throtl_stat_add(rq, value);
......
......@@ -8,21 +8,6 @@
#include <linux/rcupdate.h>
#include <linux/timer.h>
/*
* from upper:
* 4 bits: reserved for other usage
* 12 bits: size
* 48 bits: time
*/
#define BLK_STAT_RES_BITS 4
#define BLK_STAT_SIZE_BITS 12
#define BLK_STAT_RES_SHIFT (64 - BLK_STAT_RES_BITS)
#define BLK_STAT_SIZE_SHIFT (BLK_STAT_RES_SHIFT - BLK_STAT_SIZE_BITS)
#define BLK_STAT_TIME_MASK ((1ULL << BLK_STAT_SIZE_SHIFT) - 1)
#define BLK_STAT_SIZE_MASK \
(((1ULL << BLK_STAT_SIZE_BITS) - 1) << BLK_STAT_SIZE_SHIFT)
#define BLK_STAT_RES_MASK (~((1ULL << BLK_STAT_RES_SHIFT) - 1))
/**
* struct blk_stat_callback - Block statistics callback.
*
......@@ -82,34 +67,6 @@ void blk_free_queue_stats(struct blk_queue_stats *);
void blk_stat_add(struct request *);
static inline u64 __blk_stat_time(u64 time)
{
return time & BLK_STAT_TIME_MASK;
}
static inline u64 blk_stat_time(struct blk_issue_stat *stat)
{
return __blk_stat_time(stat->stat);
}
static inline sector_t blk_capped_size(sector_t size)
{
return size & ((1ULL << BLK_STAT_SIZE_BITS) - 1);
}
static inline sector_t blk_stat_size(struct blk_issue_stat *stat)
{
return (stat->stat & BLK_STAT_SIZE_MASK) >> BLK_STAT_SIZE_SHIFT;
}
static inline void blk_stat_set_issue(struct blk_issue_stat *stat,
sector_t size)
{
stat->stat = (stat->stat & BLK_STAT_RES_MASK) |
(ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK) |
(((u64)blk_capped_size(size)) << BLK_STAT_SIZE_SHIFT);
}
/* record time/size info in request but not add a callback */
void blk_stat_enable_accounting(struct request_queue *q);
......
......@@ -2279,8 +2279,7 @@ void blk_throtl_stat_add(struct request *rq, u64 time_ns)
struct request_queue *q = rq->q;
struct throtl_data *td = q->td;
throtl_track_latency(td, blk_stat_size(&rq->issue_stat),
req_op(rq), time_ns >> 10);
throtl_track_latency(td, rq->throtl_size, req_op(rq), time_ns >> 10);
}
void blk_throtl_bio_endio(struct bio *bio)
......
......@@ -31,22 +31,22 @@
static inline void wbt_clear_state(struct request *rq)
{
rq->issue_stat.stat &= ~BLK_STAT_RES_MASK;
rq->wbt_flags = 0;
}
static inline enum wbt_flags wbt_flags(struct request *rq)
{
return (rq->issue_stat.stat & BLK_STAT_RES_MASK) >> BLK_STAT_RES_SHIFT;
return rq->wbt_flags;
}
static inline bool wbt_is_tracked(struct request *rq)
{
return (rq->issue_stat.stat >> BLK_STAT_RES_SHIFT) & WBT_TRACKED;
return rq->wbt_flags & WBT_TRACKED;
}
static inline bool wbt_is_read(struct request *rq)
{
return (rq->issue_stat.stat >> BLK_STAT_RES_SHIFT) & WBT_READ;
return rq->wbt_flags & WBT_READ;
}
enum {
......@@ -657,7 +657,7 @@ void wbt_issue(struct rq_wb *rwb, struct request *rq)
*/
if (wbt_is_read(rq) && !rwb->sync_issue) {
rwb->sync_cookie = rq;
rwb->sync_issue = blk_stat_time(&rq->issue_stat);
rwb->sync_issue = rq->io_start_time_ns;
}
}
......@@ -746,8 +746,6 @@ int wbt_init(struct request_queue *q)
struct rq_wb *rwb;
int i;
BUILD_BUG_ON(WBT_NR_BITS > BLK_STAT_RES_BITS);
rwb = kzalloc(sizeof(*rwb), GFP_KERNEL);
if (!rwb)
return -ENOMEM;
......
......@@ -63,7 +63,7 @@ struct rq_wb {
struct blk_stat_callback *cb;
s64 sync_issue;
u64 sync_issue;
void *sync_cookie;
unsigned int wc;
......@@ -90,7 +90,7 @@ static inline unsigned int wbt_inflight(struct rq_wb *rwb)
static inline void wbt_track(struct request *rq, enum wbt_flags flags)
{
rq->issue_stat.stat |= ((u64)flags) << BLK_STAT_RES_SHIFT;
rq->wbt_flags |= flags;
}
void __wbt_done(struct rq_wb *, enum wbt_flags);
......
......@@ -485,11 +485,11 @@ static void kyber_completed_request(struct request *rq)
if (blk_stat_is_active(kqd->cb))
return;
now = __blk_stat_time(ktime_to_ns(ktime_get()));
if (now < blk_stat_time(&rq->issue_stat))
now = ktime_get_ns();
if (now < rq->io_start_time_ns)
return;
latency = now - blk_stat_time(&rq->issue_stat);
latency = now - rq->io_start_time_ns;
if (latency > target)
blk_stat_activate_msecs(kqd->cb, 10);
......
......@@ -91,10 +91,6 @@ static inline bool blk_path_error(blk_status_t error)
return true;
}
struct blk_issue_stat {
u64 stat;
};
/*
* From most significant bit:
* 1 bit: reserved for other usage, see below
......
......@@ -206,8 +206,18 @@ struct request {
struct gendisk *rq_disk;
struct hd_struct *part;
unsigned long start_time;
struct blk_issue_stat issue_stat;
/* Number of scatter-gather DMA addr+len pairs after
/* Time that I/O was submitted to the device. */
u64 io_start_time_ns;
#ifdef CONFIG_BLK_WBT
unsigned short wbt_flags;
#endif
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
unsigned short throtl_size;
#endif
/*
* Number of scatter-gather DMA addr+len pairs after
* physical address coalescing is performed.
*/
unsigned short nr_phys_segments;
......@@ -267,8 +277,8 @@ struct request {
#ifdef CONFIG_BLK_CGROUP
struct request_list *rl; /* rl this rq is alloced from */
unsigned long long start_time_ns;
unsigned long long io_start_time_ns; /* when passed to hardware */
unsigned long long cgroup_start_time_ns;
unsigned long long cgroup_io_start_time_ns; /* when passed to hardware */
#endif
};
......@@ -1797,25 +1807,25 @@ int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned lo
static inline void set_start_time_ns(struct request *req)
{
preempt_disable();
req->start_time_ns = sched_clock();
req->cgroup_start_time_ns = sched_clock();
preempt_enable();
}
static inline void set_io_start_time_ns(struct request *req)
{
preempt_disable();
req->io_start_time_ns = sched_clock();
req->cgroup_io_start_time_ns = sched_clock();
preempt_enable();
}
static inline uint64_t rq_start_time_ns(struct request *req)
{
return req->start_time_ns;
return req->cgroup_start_time_ns;
}
static inline uint64_t rq_io_start_time_ns(struct request *req)
{
return req->io_start_time_ns;
return req->cgroup_io_start_time_ns;
}
#else
static inline void set_start_time_ns(struct request *req) {}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment