blk-stat.c 4.72 KB
Newer Older
1 2 3 4 5 6
/*
 * Block stat tracking code
 *
 * Copyright (C) 2016 Jens Axboe
 */
#include <linux/kernel.h>
7
#include <linux/rculist.h>
8 9 10 11
#include <linux/blk-mq.h>

#include "blk-stat.h"
#include "blk-mq.h"
12
#include "blk.h"
13

14 15 16
struct blk_queue_stats {
	struct list_head callbacks;
	spinlock_t lock;
17
	bool enable_accounting;
18 19 20 21 22 23
};

static void blk_stat_init(struct blk_rq_stat *stat)
{
	stat->min = -1ULL;
	stat->max = stat->nr_samples = stat->mean = 0;
24
	stat->batch = 0;
25 26
}

27
/* src is a per-cpu stat, mean isn't initialized */
28 29 30 31 32 33 34 35
static void blk_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src)
{
	if (!src->nr_samples)
		return;

	dst->min = min(dst->min, src->min);
	dst->max = max(dst->max, src->max);

36 37 38
	dst->mean = div_u64(src->batch + dst->mean * dst->nr_samples,
				dst->nr_samples + src->nr_samples);

39 40 41
	dst->nr_samples += src->nr_samples;
}

42
static void __blk_stat_add(struct blk_rq_stat *stat, u64 value)
43
{
44 45 46
	stat->min = min(stat->min, value);
	stat->max = max(stat->max, value);
	stat->batch += value;
47
	stat->nr_samples++;
48 49
}

50
void blk_stat_add(struct request *rq, u64 now)
51
{
52 53 54 55
	struct request_queue *q = rq->q;
	struct blk_stat_callback *cb;
	struct blk_rq_stat *stat;
	int bucket;
56
	u64 value;
57

58
	value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0;
59

60 61
	blk_throtl_stat_add(rq, value);

62 63
	rcu_read_lock();
	list_for_each_entry_rcu(cb, &q->stats->callbacks, list) {
64 65 66 67 68 69 70 71 72 73
		if (!blk_stat_is_active(cb))
			continue;

		bucket = cb->bucket_fn(rq);
		if (bucket < 0)
			continue;

		stat = &get_cpu_ptr(cb->cpu_stat)[bucket];
		__blk_stat_add(stat, value);
		put_cpu_ptr(cb->cpu_stat);
74
	}
75
	rcu_read_unlock();
76 77
}

78
static void blk_stat_timer_fn(struct timer_list *t)
79
{
80
	struct blk_stat_callback *cb = from_timer(cb, t, timer);
81 82
	unsigned int bucket;
	int cpu;
83

84 85
	for (bucket = 0; bucket < cb->buckets; bucket++)
		blk_stat_init(&cb->stat[bucket]);
86

87 88
	for_each_online_cpu(cpu) {
		struct blk_rq_stat *cpu_stat;
89

90 91 92 93
		cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
		for (bucket = 0; bucket < cb->buckets; bucket++) {
			blk_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]);
			blk_stat_init(&cpu_stat[bucket]);
94
		}
95
	}
96

97
	cb->timer_fn(cb);
98 99
}

100 101
struct blk_stat_callback *
blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *),
102
			int (*bucket_fn)(const struct request *),
103
			unsigned int buckets, void *data)
104
{
105
	struct blk_stat_callback *cb;
106

107 108 109
	cb = kmalloc(sizeof(*cb), GFP_KERNEL);
	if (!cb)
		return NULL;
110

111 112 113 114 115 116 117 118 119 120 121 122 123
	cb->stat = kmalloc_array(buckets, sizeof(struct blk_rq_stat),
				 GFP_KERNEL);
	if (!cb->stat) {
		kfree(cb);
		return NULL;
	}
	cb->cpu_stat = __alloc_percpu(buckets * sizeof(struct blk_rq_stat),
				      __alignof__(struct blk_rq_stat));
	if (!cb->cpu_stat) {
		kfree(cb->stat);
		kfree(cb);
		return NULL;
	}
124

125 126 127 128
	cb->timer_fn = timer_fn;
	cb->bucket_fn = bucket_fn;
	cb->data = data;
	cb->buckets = buckets;
129
	timer_setup(&cb->timer, blk_stat_timer_fn, 0);
130 131

	return cb;
132
}
133
EXPORT_SYMBOL_GPL(blk_stat_alloc_callback);
134

135 136
void blk_stat_add_callback(struct request_queue *q,
			   struct blk_stat_callback *cb)
137
{
138 139
	unsigned int bucket;
	int cpu;
140

141 142
	for_each_possible_cpu(cpu) {
		struct blk_rq_stat *cpu_stat;
143

144 145 146 147
		cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
		for (bucket = 0; bucket < cb->buckets; bucket++)
			blk_stat_init(&cpu_stat[bucket]);
	}
148

149 150
	spin_lock(&q->stats->lock);
	list_add_tail_rcu(&cb->list, &q->stats->callbacks);
151
	blk_queue_flag_set(QUEUE_FLAG_STATS, q);
152 153 154
	spin_unlock(&q->stats->lock);
}
EXPORT_SYMBOL_GPL(blk_stat_add_callback);
155

156 157 158 159 160
void blk_stat_remove_callback(struct request_queue *q,
			      struct blk_stat_callback *cb)
{
	spin_lock(&q->stats->lock);
	list_del_rcu(&cb->list);
161
	if (list_empty(&q->stats->callbacks) && !q->stats->enable_accounting)
162
		blk_queue_flag_clear(QUEUE_FLAG_STATS, q);
163
	spin_unlock(&q->stats->lock);
164

165
	del_timer_sync(&cb->timer);
166
}
167
EXPORT_SYMBOL_GPL(blk_stat_remove_callback);
168

169
static void blk_stat_free_callback_rcu(struct rcu_head *head)
170
{
171 172 173 174 175 176
	struct blk_stat_callback *cb;

	cb = container_of(head, struct blk_stat_callback, rcu);
	free_percpu(cb->cpu_stat);
	kfree(cb->stat);
	kfree(cb);
177 178
}

179
void blk_stat_free_callback(struct blk_stat_callback *cb)
180
{
181 182
	if (cb)
		call_rcu(&cb->rcu, blk_stat_free_callback_rcu);
183
}
184
EXPORT_SYMBOL_GPL(blk_stat_free_callback);
185

186 187 188 189
void blk_stat_enable_accounting(struct request_queue *q)
{
	spin_lock(&q->stats->lock);
	q->stats->enable_accounting = true;
190
	blk_queue_flag_set(QUEUE_FLAG_STATS, q);
191 192 193
	spin_unlock(&q->stats->lock);
}

194
struct blk_queue_stats *blk_alloc_queue_stats(void)
195
{
196 197 198 199 200 201 202 203
	struct blk_queue_stats *stats;

	stats = kmalloc(sizeof(*stats), GFP_KERNEL);
	if (!stats)
		return NULL;

	INIT_LIST_HEAD(&stats->callbacks);
	spin_lock_init(&stats->lock);
204
	stats->enable_accounting = false;
205 206 207 208 209 210 211 212 213 214

	return stats;
}

void blk_free_queue_stats(struct blk_queue_stats *stats)
{
	if (!stats)
		return;

	WARN_ON(!list_empty(&stats->callbacks));
215

216
	kfree(stats);
217
}