blk-cgroup.c 38.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
 * Common Block IO controller cgroup interface
 *
 * Based on ideas and code from CFQ, CFS and BFQ:
 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
 *
 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
 *		      Paolo Valente <paolo.valente@unimore.it>
 *
 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
 * 	              Nauman Rafique <nauman@google.com>
12 13 14 15
 *
 * For policy-specific per-blkcg data:
 * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
 *                    Arianna Avanzini <avanzini.arianna@gmail.com>
16 17
 */
#include <linux/ioprio.h>
18
#include <linux/kdev_t.h>
19
#include <linux/module.h>
20
#include <linux/sched/signal.h>
21
#include <linux/err.h>
22
#include <linux/blkdev.h>
23
#include <linux/backing-dev.h>
24
#include <linux/slab.h>
25
#include <linux/genhd.h>
26
#include <linux/delay.h>
Tejun Heo's avatar
Tejun Heo committed
27
#include <linux/atomic.h>
28
#include <linux/ctype.h>
29
#include <linux/blk-cgroup.h>
30
#include "blk.h"
31

32 33
#define MAX_KEY_LEN 100

34 35 36 37 38 39 40 41
/*
 * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
 * blkcg_pol_register_mutex nests outside of it and synchronizes entire
 * policy [un]register operations including cgroup file additions /
 * removals.  Putting cgroup file registration outside blkcg_pol_mutex
 * allows grabbing it from cgroup callbacks.
 */
static DEFINE_MUTEX(blkcg_pol_register_mutex);
42
static DEFINE_MUTEX(blkcg_pol_mutex);
43

44
struct blkcg blkcg_root;
45
EXPORT_SYMBOL_GPL(blkcg_root);
46

Tejun Heo's avatar
Tejun Heo committed
47 48
struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;

49
static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
50

51 52
static LIST_HEAD(all_blkcgs);		/* protected by blkcg_pol_mutex */

53
static bool blkcg_policy_enabled(struct request_queue *q,
54
				 const struct blkcg_policy *pol)
55 56 57 58
{
	return pol && test_bit(pol->plid, q->blkcg_pols);
}

59 60 61 62 63 64
/**
 * blkg_free - free a blkg
 * @blkg: blkg to free
 *
 * Free @blkg which may be partially allocated.
 */
65
static void blkg_free(struct blkcg_gq *blkg)
66
{
67
	int i;
68 69 70 71

	if (!blkg)
		return;

72
	for (i = 0; i < BLKCG_MAX_POLS; i++)
73 74
		if (blkg->pd[i])
			blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
75

76
	if (blkg->blkcg != &blkcg_root)
77
		blk_exit_rl(blkg->q, &blkg->rl);
78 79 80

	blkg_rwstat_exit(&blkg->stat_ios);
	blkg_rwstat_exit(&blkg->stat_bytes);
81
	kfree(blkg);
82 83 84 85 86 87
}

/**
 * blkg_alloc - allocate a blkg
 * @blkcg: block cgroup the new blkg is associated with
 * @q: request_queue the new blkg is associated with
88
 * @gfp_mask: allocation mask to use
89
 *
90
 * Allocate a new blkg assocating @blkcg and @q.
91
 */
92 93
static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
				   gfp_t gfp_mask)
94
{
95
	struct blkcg_gq *blkg;
96
	int i;
97 98

	/* alloc and init base part */
99
	blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node);
100 101 102
	if (!blkg)
		return NULL;

103 104 105 106
	if (blkg_rwstat_init(&blkg->stat_bytes, gfp_mask) ||
	    blkg_rwstat_init(&blkg->stat_ios, gfp_mask))
		goto err_free;

107
	blkg->q = q;
108
	INIT_LIST_HEAD(&blkg->q_node);
109
	blkg->blkcg = blkcg;
110
	atomic_set(&blkg->refcnt, 1);
111

112 113 114 115 116 117 118
	/* root blkg uses @q->root_rl, init rl only for !root blkgs */
	if (blkcg != &blkcg_root) {
		if (blk_init_rl(&blkg->rl, q, gfp_mask))
			goto err_free;
		blkg->rl.blkg = blkg;
	}

119
	for (i = 0; i < BLKCG_MAX_POLS; i++) {
120
		struct blkcg_policy *pol = blkcg_policy[i];
121
		struct blkg_policy_data *pd;
122

123
		if (!blkcg_policy_enabled(q, pol))
124 125 126
			continue;

		/* alloc per-policy data and attach it to blkg */
127
		pd = pol->pd_alloc_fn(gfp_mask, q->node);
128 129
		if (!pd)
			goto err_free;
130

131 132
		blkg->pd[i] = pd;
		pd->blkg = blkg;
133
		pd->plid = i;
134 135
	}

136
	return blkg;
137 138 139 140

err_free:
	blkg_free(blkg);
	return NULL;
141 142
}

143 144
struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,
				      struct request_queue *q, bool update_hint)
145
{
146
	struct blkcg_gq *blkg;
147

148
	/*
149 150 151 152
	 * Hint didn't match.  Look up from the radix tree.  Note that the
	 * hint can only be updated under queue_lock as otherwise @blkg
	 * could have already been removed from blkg_tree.  The caller is
	 * responsible for grabbing queue_lock if @update_hint.
153 154
	 */
	blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);
155 156 157 158 159
	if (blkg && blkg->q == q) {
		if (update_hint) {
			lockdep_assert_held(q->queue_lock);
			rcu_assign_pointer(blkcg->blkg_hint, blkg);
		}
160
		return blkg;
161
	}
162

163 164
	return NULL;
}
165
EXPORT_SYMBOL_GPL(blkg_lookup_slowpath);
166

167
/*
168 169
 * If @new_blkg is %NULL, this function tries to allocate a new one as
 * necessary using %GFP_NOWAIT.  @new_blkg is always consumed on return.
170
 */
171
static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
172 173
				    struct request_queue *q,
				    struct blkcg_gq *new_blkg)
174
{
175
	struct blkcg_gq *blkg;
176
	struct bdi_writeback_congested *wb_congested;
177
	int i, ret;
178

179 180 181
	WARN_ON_ONCE(!rcu_read_lock_held());
	lockdep_assert_held(q->queue_lock);

182
	/* blkg holds a reference to blkcg */
183
	if (!css_tryget_online(&blkcg->css)) {
184
		ret = -ENODEV;
185
		goto err_free_blkg;
186
	}
187

188
	wb_congested = wb_congested_get_create(q->backing_dev_info,
189 190 191
					       blkcg->css.id,
					       GFP_NOWAIT | __GFP_NOWARN);
	if (!wb_congested) {
192
		ret = -ENOMEM;
193
		goto err_put_css;
194 195
	}

196 197 198 199 200 201
	/* allocate */
	if (!new_blkg) {
		new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT | __GFP_NOWARN);
		if (unlikely(!new_blkg)) {
			ret = -ENOMEM;
			goto err_put_congested;
202 203
		}
	}
204 205
	blkg = new_blkg;
	blkg->wb_congested = wb_congested;
206

207
	/* link parent */
208 209 210
	if (blkcg_parent(blkcg)) {
		blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false);
		if (WARN_ON_ONCE(!blkg->parent)) {
211
			ret = -ENODEV;
212
			goto err_put_congested;
213 214 215 216
		}
		blkg_get(blkg->parent);
	}

217 218 219 220 221
	/* invoke per-policy init */
	for (i = 0; i < BLKCG_MAX_POLS; i++) {
		struct blkcg_policy *pol = blkcg_policy[i];

		if (blkg->pd[i] && pol->pd_init_fn)
222
			pol->pd_init_fn(blkg->pd[i]);
223 224 225
	}

	/* insert */
226
	spin_lock(&blkcg->lock);
227 228 229 230
	ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);
	if (likely(!ret)) {
		hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
		list_add(&blkg->q_node, &q->blkg_list);
231 232 233 234 235

		for (i = 0; i < BLKCG_MAX_POLS; i++) {
			struct blkcg_policy *pol = blkcg_policy[i];

			if (blkg->pd[i] && pol->pd_online_fn)
236
				pol->pd_online_fn(blkg->pd[i]);
237
		}
238
	}
239
	blkg->online = true;
240
	spin_unlock(&blkcg->lock);
241

242
	if (!ret)
243
		return blkg;
244

245 246 247 248
	/* @blkg failed fully initialized, use the usual release path */
	blkg_put(blkg);
	return ERR_PTR(ret);

249 250 251
err_put_congested:
	wb_congested_put(wb_congested);
err_put_css:
252
	css_put(&blkcg->css);
253
err_free_blkg:
254
	blkg_free(new_blkg);
255
	return ERR_PTR(ret);
256
}
257

258
/**
259
 * blkg_lookup_create - lookup blkg, try to create one if not there
260 261 262 263
 * @blkcg: blkcg of interest
 * @q: request_queue of interest
 *
 * Lookup blkg for the @blkcg - @q pair.  If it doesn't exist, try to
264 265 266
 * create one.  blkg creation is performed recursively from blkcg_root such
 * that all non-root blkg's have access to the parent blkg.  This function
 * should be called under RCU read lock and @q->queue_lock.
267 268 269 270 271
 *
 * Returns pointer to the looked up or created blkg on success, ERR_PTR()
 * value on error.  If @q is dead, returns ERR_PTR(-EINVAL).  If @q is not
 * dead and bypassing, returns ERR_PTR(-EBUSY).
 */
272 273
struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
				    struct request_queue *q)
274
{
275 276 277 278 279
	struct blkcg_gq *blkg;

	WARN_ON_ONCE(!rcu_read_lock_held());
	lockdep_assert_held(q->queue_lock);

280 281 282 283 284 285 286
	/*
	 * This could be the first entry point of blkcg implementation and
	 * we shouldn't allow anything to go through for a bypassing queue.
	 */
	if (unlikely(blk_queue_bypass(q)))
		return ERR_PTR(blk_queue_dying(q) ? -ENODEV : -EBUSY);

287 288 289 290
	blkg = __blkg_lookup(blkcg, q, true);
	if (blkg)
		return blkg;

291 292 293 294 295 296 297 298 299 300 301 302 303
	/*
	 * Create blkgs walking down from blkcg_root to @blkcg, so that all
	 * non-root blkgs have access to their parents.
	 */
	while (true) {
		struct blkcg *pos = blkcg;
		struct blkcg *parent = blkcg_parent(blkcg);

		while (parent && !__blkg_lookup(parent, q, false)) {
			pos = parent;
			parent = blkcg_parent(parent);
		}

304
		blkg = blkg_create(pos, q, NULL);
305 306 307
		if (pos == blkcg || IS_ERR(blkg))
			return blkg;
	}
308
}
309

310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327
static void blkg_pd_offline(struct blkcg_gq *blkg)
{
	int i;

	lockdep_assert_held(blkg->q->queue_lock);
	lockdep_assert_held(&blkg->blkcg->lock);

	for (i = 0; i < BLKCG_MAX_POLS; i++) {
		struct blkcg_policy *pol = blkcg_policy[i];

		if (blkg->pd[i] && !blkg->pd[i]->offline &&
		    pol->pd_offline_fn) {
			pol->pd_offline_fn(blkg->pd[i]);
			blkg->pd[i]->offline = true;
		}
	}
}

328
static void blkg_destroy(struct blkcg_gq *blkg)
329
{
330
	struct blkcg *blkcg = blkg->blkcg;
331
	struct blkcg_gq *parent = blkg->parent;
332

333
	lockdep_assert_held(blkg->q->queue_lock);
334
	lockdep_assert_held(&blkcg->lock);
335 336

	/* Something wrong if we are trying to remove same group twice */
337
	WARN_ON_ONCE(list_empty(&blkg->q_node));
338
	WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
339

340 341 342 343 344
	if (parent) {
		blkg_rwstat_add_aux(&parent->stat_bytes, &blkg->stat_bytes);
		blkg_rwstat_add_aux(&parent->stat_ios, &blkg->stat_ios);
	}

345 346
	blkg->online = false;

347
	radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
348
	list_del_init(&blkg->q_node);
349
	hlist_del_init_rcu(&blkg->blkcg_node);
350

351 352 353 354 355
	/*
	 * Both setting lookup hint to and clearing it from @blkg are done
	 * under queue_lock.  If it's not pointing to @blkg now, it never
	 * will.  Hint assignment itself can race safely.
	 */
356
	if (rcu_access_pointer(blkcg->blkg_hint) == blkg)
357 358
		rcu_assign_pointer(blkcg->blkg_hint, NULL);

359 360 361 362 363 364 365
	/*
	 * Put the reference taken at the time of creation so that when all
	 * queues are gone, group can be destroyed.
	 */
	blkg_put(blkg);
}

366 367 368 369
/**
 * blkg_destroy_all - destroy all blkgs associated with a request_queue
 * @q: request_queue of interest
 *
370
 * Destroy all blkgs associated with @q.
371
 */
372
static void blkg_destroy_all(struct request_queue *q)
373
{
374
	struct blkcg_gq *blkg, *n;
375

376
	lockdep_assert_held(q->queue_lock);
377

378
	list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
379
		struct blkcg *blkcg = blkg->blkcg;
380

381
		spin_lock(&blkcg->lock);
382
		blkg_pd_offline(blkg);
383 384
		blkg_destroy(blkg);
		spin_unlock(&blkcg->lock);
385
	}
386 387 388

	q->root_blkg = NULL;
	q->root_rl.blkg = NULL;
389 390
}

391 392 393 394 395 396 397 398 399
/*
 * A group is RCU protected, but having an rcu lock does not mean that one
 * can access all the fields of blkg and assume these are valid.  For
 * example, don't try to follow throtl_data and request queue links.
 *
 * Having a reference to blkg under an rcu allows accesses to only values
 * local to groups like group stats and group rate limits.
 */
void __blkg_release_rcu(struct rcu_head *rcu_head)
400
{
401
	struct blkcg_gq *blkg = container_of(rcu_head, struct blkcg_gq, rcu_head);
402

403
	/* release the blkcg and parent blkg refs this blkg has been holding */
404
	css_put(&blkg->blkcg->css);
405
	if (blkg->parent)
406
		blkg_put(blkg->parent);
407

408 409
	wb_congested_put(blkg->wb_congested);

410
	blkg_free(blkg);
411
}
412
EXPORT_SYMBOL_GPL(__blkg_release_rcu);
413

414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429
/*
 * The next function used by blk_queue_for_each_rl().  It's a bit tricky
 * because the root blkg uses @q->root_rl instead of its own rl.
 */
struct request_list *__blk_queue_next_rl(struct request_list *rl,
					 struct request_queue *q)
{
	struct list_head *ent;
	struct blkcg_gq *blkg;

	/*
	 * Determine the current blkg list_head.  The first entry is
	 * root_rl which is off @q->blkg_list and mapped to the head.
	 */
	if (rl == &q->root_rl) {
		ent = &q->blkg_list;
430 431 432
		/* There are no more block groups, hence no request lists */
		if (list_empty(ent))
			return NULL;
433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448
	} else {
		blkg = container_of(rl, struct blkcg_gq, rl);
		ent = &blkg->q_node;
	}

	/* walk to the next list_head, skip root blkcg */
	ent = ent->next;
	if (ent == &q->root_blkg->q_node)
		ent = ent->next;
	if (ent == &q->blkg_list)
		return NULL;

	blkg = container_of(ent, struct blkcg_gq, q_node);
	return &blkg->rl;
}

449 450
static int blkcg_reset_stats(struct cgroup_subsys_state *css,
			     struct cftype *cftype, u64 val)
451
{
452
	struct blkcg *blkcg = css_to_blkcg(css);
453
	struct blkcg_gq *blkg;
454
	int i;
455

456
	mutex_lock(&blkcg_pol_mutex);
457
	spin_lock_irq(&blkcg->lock);
Tejun Heo's avatar
Tejun Heo committed
458 459 460 461 462 463

	/*
	 * Note that stat reset is racy - it doesn't synchronize against
	 * stat updates.  This is a debug feature which shouldn't exist
	 * anyway.  If you get hit by a race, retry.
	 */
464
	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
465 466 467
		blkg_rwstat_reset(&blkg->stat_bytes);
		blkg_rwstat_reset(&blkg->stat_ios);

468
		for (i = 0; i < BLKCG_MAX_POLS; i++) {
469
			struct blkcg_policy *pol = blkcg_policy[i];
470

471 472
			if (blkg->pd[i] && pol->pd_reset_stats_fn)
				pol->pd_reset_stats_fn(blkg->pd[i]);
473
		}
474
	}
475

476
	spin_unlock_irq(&blkcg->lock);
477
	mutex_unlock(&blkcg_pol_mutex);
478 479 480
	return 0;
}

481
const char *blkg_dev_name(struct blkcg_gq *blkg)
482
{
483
	/* some drivers (floppy) instantiate a queue w/o disk registered */
484 485
	if (blkg->q->backing_dev_info->dev)
		return dev_name(blkg->q->backing_dev_info->dev);
486
	return NULL;
487
}
488
EXPORT_SYMBOL_GPL(blkg_dev_name);
489

490 491 492 493 494 495 496 497 498 499 500
/**
 * blkcg_print_blkgs - helper for printing per-blkg data
 * @sf: seq_file to print to
 * @blkcg: blkcg of interest
 * @prfill: fill function to print out a blkg
 * @pol: policy in question
 * @data: data to be passed to @prfill
 * @show_total: to print out sum of prfill return values or not
 *
 * This function invokes @prfill on each blkg of @blkcg if pd for the
 * policy specified by @pol exists.  @prfill is invoked with @sf, the
501 502 503
 * policy data and @data and the matching queue lock held.  If @show_total
 * is %true, the sum of the return values from @prfill is printed with
 * "Total" label at the end.
504 505 506 507
 *
 * This is to be used to construct print functions for
 * cftype->read_seq_string method.
 */
508
void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
509 510
		       u64 (*prfill)(struct seq_file *,
				     struct blkg_policy_data *, int),
511
		       const struct blkcg_policy *pol, int data,
512
		       bool show_total)
513
{
514
	struct blkcg_gq *blkg;
515
	u64 total = 0;
516

517
	rcu_read_lock();
518
	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
519
		spin_lock_irq(blkg->q->queue_lock);
520
		if (blkcg_policy_enabled(blkg->q, pol))
521
			total += prfill(sf, blkg->pd[pol->plid], data);
522 523 524
		spin_unlock_irq(blkg->q->queue_lock);
	}
	rcu_read_unlock();
525 526 527 528

	if (show_total)
		seq_printf(sf, "Total %llu\n", (unsigned long long)total);
}
529
EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
530 531 532 533

/**
 * __blkg_prfill_u64 - prfill helper for a single u64 value
 * @sf: seq_file to print to
534
 * @pd: policy private data of interest
535 536
 * @v: value to print
 *
537
 * Print @v to @sf for the device assocaited with @pd.
538
 */
539
u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
540
{
541
	const char *dname = blkg_dev_name(pd->blkg);
542 543 544 545 546 547 548

	if (!dname)
		return 0;

	seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
	return v;
}
549
EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
550 551 552 553

/**
 * __blkg_prfill_rwstat - prfill helper for a blkg_rwstat
 * @sf: seq_file to print to
554
 * @pd: policy private data of interest
555 556
 * @rwstat: rwstat to print
 *
557
 * Print @rwstat to @sf for the device assocaited with @pd.
558
 */
559
u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
560
			 const struct blkg_rwstat *rwstat)
561 562 563 564 565 566 567
{
	static const char *rwstr[] = {
		[BLKG_RWSTAT_READ]	= "Read",
		[BLKG_RWSTAT_WRITE]	= "Write",
		[BLKG_RWSTAT_SYNC]	= "Sync",
		[BLKG_RWSTAT_ASYNC]	= "Async",
	};
568
	const char *dname = blkg_dev_name(pd->blkg);
569 570 571 572 573 574 575 576
	u64 v;
	int i;

	if (!dname)
		return 0;

	for (i = 0; i < BLKG_RWSTAT_NR; i++)
		seq_printf(sf, "%s %s %llu\n", dname, rwstr[i],
577
			   (unsigned long long)atomic64_read(&rwstat->aux_cnt[i]));
578

579 580
	v = atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_READ]) +
		atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_WRITE]);
581 582 583
	seq_printf(sf, "%s Total %llu\n", dname, (unsigned long long)v);
	return v;
}
584
EXPORT_SYMBOL_GPL(__blkg_prfill_rwstat);
585

586 587 588
/**
 * blkg_prfill_stat - prfill callback for blkg_stat
 * @sf: seq_file to print to
589 590
 * @pd: policy private data of interest
 * @off: offset to the blkg_stat in @pd
591 592 593
 *
 * prfill callback for printing a blkg_stat.
 */
594
u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off)
595
{
596
	return __blkg_prfill_u64(sf, pd, blkg_stat_read((void *)pd + off));
597
}
598
EXPORT_SYMBOL_GPL(blkg_prfill_stat);
599

600 601 602
/**
 * blkg_prfill_rwstat - prfill callback for blkg_rwstat
 * @sf: seq_file to print to
603 604
 * @pd: policy private data of interest
 * @off: offset to the blkg_rwstat in @pd
605 606 607
 *
 * prfill callback for printing a blkg_rwstat.
 */
608 609
u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
		       int off)
610
{
611
	struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd + off);
612

613
	return __blkg_prfill_rwstat(sf, pd, &rwstat);
614
}
615
EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
616

617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697
static u64 blkg_prfill_rwstat_field(struct seq_file *sf,
				    struct blkg_policy_data *pd, int off)
{
	struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd->blkg + off);

	return __blkg_prfill_rwstat(sf, pd, &rwstat);
}

/**
 * blkg_print_stat_bytes - seq_show callback for blkg->stat_bytes
 * @sf: seq_file to print to
 * @v: unused
 *
 * To be used as cftype->seq_show to print blkg->stat_bytes.
 * cftype->private must be set to the blkcg_policy.
 */
int blkg_print_stat_bytes(struct seq_file *sf, void *v)
{
	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
			  blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
			  offsetof(struct blkcg_gq, stat_bytes), true);
	return 0;
}
EXPORT_SYMBOL_GPL(blkg_print_stat_bytes);

/**
 * blkg_print_stat_bytes - seq_show callback for blkg->stat_ios
 * @sf: seq_file to print to
 * @v: unused
 *
 * To be used as cftype->seq_show to print blkg->stat_ios.  cftype->private
 * must be set to the blkcg_policy.
 */
int blkg_print_stat_ios(struct seq_file *sf, void *v)
{
	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
			  blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
			  offsetof(struct blkcg_gq, stat_ios), true);
	return 0;
}
EXPORT_SYMBOL_GPL(blkg_print_stat_ios);

static u64 blkg_prfill_rwstat_field_recursive(struct seq_file *sf,
					      struct blkg_policy_data *pd,
					      int off)
{
	struct blkg_rwstat rwstat = blkg_rwstat_recursive_sum(pd->blkg,
							      NULL, off);
	return __blkg_prfill_rwstat(sf, pd, &rwstat);
}

/**
 * blkg_print_stat_bytes_recursive - recursive version of blkg_print_stat_bytes
 * @sf: seq_file to print to
 * @v: unused
 */
int blkg_print_stat_bytes_recursive(struct seq_file *sf, void *v)
{
	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
			  blkg_prfill_rwstat_field_recursive,
			  (void *)seq_cft(sf)->private,
			  offsetof(struct blkcg_gq, stat_bytes), true);
	return 0;
}
EXPORT_SYMBOL_GPL(blkg_print_stat_bytes_recursive);

/**
 * blkg_print_stat_ios_recursive - recursive version of blkg_print_stat_ios
 * @sf: seq_file to print to
 * @v: unused
 */
int blkg_print_stat_ios_recursive(struct seq_file *sf, void *v)
{
	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
			  blkg_prfill_rwstat_field_recursive,
			  (void *)seq_cft(sf)->private,
			  offsetof(struct blkcg_gq, stat_ios), true);
	return 0;
}
EXPORT_SYMBOL_GPL(blkg_print_stat_ios_recursive);

698 699
/**
 * blkg_stat_recursive_sum - collect hierarchical blkg_stat
700 701 702
 * @blkg: blkg of interest
 * @pol: blkcg_policy which contains the blkg_stat
 * @off: offset to the blkg_stat in blkg_policy_data or @blkg
703
 *
704 705 706 707 708 709
 * Collect the blkg_stat specified by @blkg, @pol and @off and all its
 * online descendants and their aux counts.  The caller must be holding the
 * queue lock for online tests.
 *
 * If @pol is NULL, blkg_stat is at @off bytes into @blkg; otherwise, it is
 * at @off bytes into @blkg's blkg_policy_data of the policy.
710
 */
711 712
u64 blkg_stat_recursive_sum(struct blkcg_gq *blkg,
			    struct blkcg_policy *pol, int off)
713 714
{
	struct blkcg_gq *pos_blkg;
715
	struct cgroup_subsys_state *pos_css;
716
	u64 sum = 0;
717

718
	lockdep_assert_held(blkg->q->queue_lock);
719 720

	rcu_read_lock();
721 722 723 724 725
	blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
		struct blkg_stat *stat;

		if (!pos_blkg->online)
			continue;
726

727 728 729 730 731 732
		if (pol)
			stat = (void *)blkg_to_pd(pos_blkg, pol) + off;
		else
			stat = (void *)blkg + off;

		sum += blkg_stat_read(stat) + atomic64_read(&stat->aux_cnt);
733 734 735 736 737 738 739 740 741
	}
	rcu_read_unlock();

	return sum;
}
EXPORT_SYMBOL_GPL(blkg_stat_recursive_sum);

/**
 * blkg_rwstat_recursive_sum - collect hierarchical blkg_rwstat
742 743 744
 * @blkg: blkg of interest
 * @pol: blkcg_policy which contains the blkg_rwstat
 * @off: offset to the blkg_rwstat in blkg_policy_data or @blkg
745
 *
746 747 748 749 750 751
 * Collect the blkg_rwstat specified by @blkg, @pol and @off and all its
 * online descendants and their aux counts.  The caller must be holding the
 * queue lock for online tests.
 *
 * If @pol is NULL, blkg_rwstat is at @off bytes into @blkg; otherwise, it
 * is at @off bytes into @blkg's blkg_policy_data of the policy.
752
 */
753 754
struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg,
					     struct blkcg_policy *pol, int off)
755 756
{
	struct blkcg_gq *pos_blkg;
757
	struct cgroup_subsys_state *pos_css;
758
	struct blkg_rwstat sum = { };
759 760
	int i;

761
	lockdep_assert_held(blkg->q->queue_lock);
762 763

	rcu_read_lock();
764
	blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
765
		struct blkg_rwstat *rwstat;
766 767 768 769

		if (!pos_blkg->online)
			continue;

770 771 772 773 774
		if (pol)
			rwstat = (void *)blkg_to_pd(pos_blkg, pol) + off;
		else
			rwstat = (void *)pos_blkg + off;

775
		for (i = 0; i < BLKG_RWSTAT_NR; i++)
776 777 778
			atomic64_add(atomic64_read(&rwstat->aux_cnt[i]) +
				percpu_counter_sum_positive(&rwstat->cpu_cnt[i]),
				&sum.aux_cnt[i]);
779 780 781 782 783 784 785
	}
	rcu_read_unlock();

	return sum;
}
EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum);

786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806
/* Performs queue bypass and policy enabled checks then looks up blkg. */
static struct blkcg_gq *blkg_lookup_check(struct blkcg *blkcg,
					  const struct blkcg_policy *pol,
					  struct request_queue *q)
{
	WARN_ON_ONCE(!rcu_read_lock_held());
	lockdep_assert_held(q->queue_lock);

	if (!blkcg_policy_enabled(q, pol))
		return ERR_PTR(-EOPNOTSUPP);

	/*
	 * This could be the first entry point of blkcg implementation and
	 * we shouldn't allow anything to go through for a bypassing queue.
	 */
	if (unlikely(blk_queue_bypass(q)))
		return ERR_PTR(blk_queue_dying(q) ? -ENODEV : -EBUSY);

	return __blkg_lookup(blkcg, q, true /* update_hint */);
}

807 808 809
/**
 * blkg_conf_prep - parse and prepare for per-blkg config update
 * @blkcg: target block cgroup
810
 * @pol: target policy
811 812 813 814
 * @input: input string
 * @ctx: blkg_conf_ctx to be filled
 *
 * Parse per-blkg config update from @input and initialize @ctx with the
815 816 817
 * result.  @ctx->blkg points to the blkg to be updated and @ctx->body the
 * part of @input following MAJ:MIN.  This function returns with RCU read
 * lock and queue lock held and must be paired with blkg_conf_finish().
818
 */
819
int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
820
		   char *input, struct blkg_conf_ctx *ctx)
821
	__acquires(rcu) __acquires(disk->queue->queue_lock)
822
{
823
	struct gendisk *disk;
824
	struct request_queue *q;
825
	struct blkcg_gq *blkg;
826
	unsigned int major, minor;
827 828
	int key_len, part, ret;
	char *body;
829

830
	if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2)
831
		return -EINVAL;
832

833 834 835 836 837
	body = input + key_len;
	if (!isspace(*body))
		return -EINVAL;
	body = skip_spaces(body);

838
	disk = get_gendisk(MKDEV(major, minor), &part);
839
	if (!disk)
840
		return -ENODEV;
841
	if (part) {
842 843
		ret = -ENODEV;
		goto fail;
844
	}
845

846
	q = disk->queue;
847

848 849
	rcu_read_lock();
	spin_lock_irq(q->queue_lock);
850

851
	blkg = blkg_lookup_check(blkcg, pol, q);
852 853
	if (IS_ERR(blkg)) {
		ret = PTR_ERR(blkg);
854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876
		goto fail_unlock;
	}

	if (blkg)
		goto success;

	/*
	 * Create blkgs walking down from blkcg_root to @blkcg, so that all
	 * non-root blkgs have access to their parents.
	 */
	while (true) {
		struct blkcg *pos = blkcg;
		struct blkcg *parent;
		struct blkcg_gq *new_blkg;

		parent = blkcg_parent(blkcg);
		while (parent && !__blkg_lookup(parent, q, false)) {
			pos = parent;
			parent = blkcg_parent(parent);
		}

		/* Drop locks to do new blkg allocation with GFP_KERNEL. */
		spin_unlock_irq(q->queue_lock);
877
		rcu_read_unlock();
878 879 880 881 882

		new_blkg = blkg_alloc(pos, q, GFP_KERNEL);
		if (unlikely(!new_blkg)) {
			ret = -ENOMEM;
			goto fail;
883
		}
884

885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907
		rcu_read_lock();
		spin_lock_irq(q->queue_lock);

		blkg = blkg_lookup_check(pos, pol, q);
		if (IS_ERR(blkg)) {
			ret = PTR_ERR(blkg);
			goto fail_unlock;
		}

		if (blkg) {
			blkg_free(new_blkg);
		} else {
			blkg = blkg_create(pos, q, new_blkg);
			if (unlikely(IS_ERR(blkg))) {
				ret = PTR_ERR(blkg);
				goto fail_unlock;
			}
		}

		if (pos == blkcg)
			goto success;
	}
success:
908 909
	ctx->disk = disk;
	ctx->blkg = blkg;
910
	ctx->body = body;
911
	return 0;
912 913 914 915 916

fail_unlock:
	spin_unlock_irq(q->queue_lock);
	rcu_read_unlock();
fail:
917
	put_disk_and_module(disk);
918 919 920 921 922 923 924 925 926 927 928
	/*
	 * If queue was bypassing, we should retry.  Do so after a
	 * short msleep().  It isn't strictly necessary but queue
	 * can be bypassing for some time and it's always nice to
	 * avoid busy looping.
	 */
	if (ret == -EBUSY) {
		msleep(10);
		ret = restart_syscall();
	}
	return ret;
929
}
930
EXPORT_SYMBOL_GPL(blkg_conf_prep);
931

932 933 934 935 936 937 938
/**
 * blkg_conf_finish - finish up per-blkg config update
 * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep()
 *
 * Finish up after per-blkg config update.  This function must be paired
 * with blkg_conf_prep().
 */
939
void blkg_conf_finish(struct blkg_conf_ctx *ctx)
940
	__releases(ctx->disk->queue->queue_lock) __releases(rcu)
941
{
942
	spin_unlock_irq(ctx->disk->queue->queue_lock);
943
	rcu_read_unlock();
944
	put_disk_and_module(ctx->disk);
945
}
946
EXPORT_SYMBOL_GPL(blkg_conf_finish);
947

948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986
static int blkcg_print_stat(struct seq_file *sf, void *v)
{
	struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
	struct blkcg_gq *blkg;

	rcu_read_lock();

	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
		const char *dname;
		struct blkg_rwstat rwstat;
		u64 rbytes, wbytes, rios, wios;

		dname = blkg_dev_name(blkg);
		if (!dname)
			continue;

		spin_lock_irq(blkg->q->queue_lock);

		rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
					offsetof(struct blkcg_gq, stat_bytes));
		rbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
		wbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);

		rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
					offsetof(struct blkcg_gq, stat_ios));
		rios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
		wios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);

		spin_unlock_irq(blkg->q->queue_lock);

		if (rbytes || wbytes || rios || wios)
			seq_printf(sf, "%s rbytes=%llu wbytes=%llu rios=%llu wios=%llu\n",
				   dname, rbytes, wbytes, rios, wios);
	}

	rcu_read_unlock();
	return 0;
}

987
static struct cftype blkcg_files[] = {
988 989
	{
		.name = "stat",
990
		.flags = CFTYPE_NOT_ON_ROOT,
991 992 993 994 995
		.seq_show = blkcg_print_stat,
	},
	{ }	/* terminate */
};

996
static struct cftype blkcg_legacy_files[] = {
997 998
	{
		.name = "reset_stats",
999
		.write_u64 = blkcg_reset_stats,
1000
	},
1001
	{ }	/* terminate */
1002 1003
};

1004
/**
1005
 * blkcg_css_offline - cgroup css_offline callback
1006
 * @css: css of interest
1007
 *
1008
 * This function is called when @css is about to go away and responsible
1009 1010 1011 1012
 * for offlining all blkgs pd and killing all wbs associated with @css.
 * blkgs pd offline should be done while holding both q and blkcg locks.
 * As blkcg lock is nested inside q lock, this function performs reverse
 * double lock dancing.
1013 1014 1015
 *
 * This is the blkcg counterpart of ioc_release_fn().
 */
1016
static void blkcg_css_offline(struct cgroup_subsys_state *css)
1017
{
1018
	struct blkcg *blkcg = css_to_blkcg(css);
1019
	struct blkcg_gq *blkg;
1020

1021
	spin_lock_irq(&blkcg->lock);
1022

1023
	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
1024
		struct request_queue *q = blkg->q;
1025

1026
		if (spin_trylock(q->queue_lock)) {
1027
			blkg_pd_offline(blkg);
1028 1029 1030 1031
			spin_unlock(q->queue_lock);
		} else {
			spin_unlock_irq(&blkcg->lock);
			cpu_relax();
1032
			spin_lock_irq(&blkcg->lock);
1033
		}
1034
	}
1035

1036
	spin_unlock_irq(&blkcg->lock);
1037 1038

	wb_blkcg_offline(blkcg);
1039 1040
}

1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070
/**
 * blkcg_destroy_all_blkgs - destroy all blkgs associated with a blkcg
 * @blkcg: blkcg of interest
 *
 * This function is called when blkcg css is about to free and responsible for
 * destroying all blkgs associated with @blkcg.
 * blkgs should be removed while holding both q and blkcg locks. As blkcg lock
 * is nested inside q lock, this function performs reverse double lock dancing.
 */
static void blkcg_destroy_all_blkgs(struct blkcg *blkcg)
{
	spin_lock_irq(&blkcg->lock);
	while (!hlist_empty(&blkcg->blkg_list)) {
		struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,
						    struct blkcg_gq,
						    blkcg_node);
		struct request_queue *q = blkg->q;

		if (spin_trylock(q->queue_lock)) {
			blkg_destroy(blkg);
			spin_unlock(q->queue_lock);
		} else {
			spin_unlock_irq(&blkcg->lock);
			cpu_relax();
			spin_lock_irq(&blkcg->lock);
		}
	}
	spin_unlock_irq(&blkcg->lock);
}

1071
static void blkcg_css_free(struct cgroup_subsys_state *css)
1072
{
1073
	struct blkcg *blkcg = css_to_blkcg(css);
1074
	int i;
1075

1076 1077
	blkcg_destroy_all_blkgs(blkcg);

1078
	mutex_lock(&blkcg_pol_mutex);
1079

1080 1081
	list_del(&blkcg->all_blkcgs_node);

1082
	for (i = 0; i < BLKCG_MAX_POLS; i++)
1083 1084 1085 1086 1087
		if (blkcg->cpd[i])
			blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);

	mutex_unlock(&blkcg_pol_mutex);

1088
	kfree(blkcg);
1089 1090
}

1091 1092
static struct cgroup_subsys_state *
blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
1093
{
1094
	struct blkcg *blkcg;
1095 1096
	struct cgroup_subsys_state *ret;
	int i;
1097

1098 1099
	mutex_lock(&blkcg_pol_mutex);

1100
	if (!parent_css) {
1101
		blkcg = &blkcg_root;
1102 1103 1104 1105
	} else {
		blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
		if (!blkcg) {
			ret = ERR_PTR(-ENOMEM);
1106
			goto unlock;
1107
		}
1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119
	}

	for (i = 0; i < BLKCG_MAX_POLS ; i++) {
		struct blkcg_policy *pol = blkcg_policy[i];
		struct blkcg_policy_data *cpd;

		/*
		 * If the policy hasn't been attached yet, wait for it
		 * to be attached before doing anything else. Otherwise,
		 * check if the policy requires any specific per-cgroup
		 * data: if it does, allocate and initialize it.
		 */
1120
		if (!pol || !pol->cpd_alloc_fn)
1121 1122
			continue;

1123
		cpd = pol->cpd_alloc_fn(GFP_KERNEL);
1124 1125 1126 1127
		if (!cpd) {
			ret = ERR_PTR(-ENOMEM);
			goto free_pd_blkcg;
		}
1128 1129
		blkcg->cpd[i] = cpd;
		cpd->blkcg = blkcg;
1130
		cpd->plid = i;
1131 1132
		if (pol->cpd_init_fn)
			pol->cpd_init_fn(cpd);
1133
	}
1134 1135

	spin_lock_init(&blkcg->lock);
1136
	INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN);
1137
	INIT_HLIST_HEAD(&blkcg->blkg_list);
1138 1139 1140
#ifdef CONFIG_CGROUP_WRITEBACK
	INIT_LIST_HEAD(&blkcg->cgwb_list);
#endif
1141 1142 1143
	list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);

	mutex_unlock(&blkcg_pol_mutex);
1144
	return &blkcg->css;
1145 1146 1147

free_pd_blkcg:
	for (i--; i >= 0; i--)
1148 1149
		if (blkcg->cpd[i])
			blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1150 1151 1152 1153

	if (blkcg != &blkcg_root)
		kfree(blkcg);
unlock:
1154
	mutex_unlock(&blkcg_pol_mutex);
1155
	return ret;
1156 1157
}

1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169
/**
 * blkcg_init_queue - initialize blkcg part of request queue
 * @q: request_queue to initialize
 *
 * Called from blk_alloc_queue_node(). Responsible for initializing blkcg
 * part of new request_queue @q.
 *
 * RETURNS:
 * 0 on success, -errno on failure.
 */
int blkcg_init_queue(struct request_queue *q)
{
1170 1171
	struct blkcg_gq *new_blkg, *blkg;
	bool preloaded;
1172 1173
	int ret;

1174 1175 1176 1177 1178 1179
	new_blkg = blkg_alloc(&blkcg_root, q, GFP_KERNEL);
	if (!new_blkg)
		return -ENOMEM;

	preloaded = !radix_tree_preload(GFP_KERNEL);

1180
	/* Make sure the root blkg exists. */
1181 1182
	rcu_read_lock();
	spin_lock_irq(q->queue_lock);
1183
	blkg = blkg_create(&blkcg_root, q, new_blkg);
1184 1185 1186 1187
	if (IS_ERR(blkg))
		goto err_unlock;
	q->root_blkg = blkg;
	q->root_rl.blkg = blkg;
1188 1189 1190
	spin_unlock_irq(q->queue_lock);
	rcu_read_unlock();

1191 1192 1193
	if (preloaded)
		radix_tree_preload_end();

1194 1195 1196 1197 1198 1199 1200
	ret = blk_throtl_init(q);
	if (ret) {
		spin_lock_irq(q->queue_lock);
		blkg_destroy_all(q);
		spin_unlock_irq(q->queue_lock);
	}
	return ret;
1201 1202 1203 1204 1205 1206 1207

err_unlock:
	spin_unlock_irq(q->queue_lock);
	rcu_read_unlock();
	if (preloaded)
		radix_tree_preload_end();
	return PTR_ERR(blkg);
1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219
}

/**
 * blkcg_drain_queue - drain blkcg part of request_queue
 * @q: request_queue to drain
 *
 * Called from blk_drain_queue().  Responsible for draining blkcg part.
 */
void blkcg_drain_queue(struct request_queue *q)
{
	lockdep_assert_held(q->queue_lock);

1220 1221 1222 1223 1224 1225 1226
	/*
	 * @q could be exiting and already have destroyed all blkgs as
	 * indicated by NULL root_blkg.  If so, don't confuse policies.
	 */
	if (!q->root_blkg)
		return;

1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237
	blk_throtl_drain(q);
}

/**
 * blkcg_exit_queue - exit and release blkcg part of request_queue
 * @q: request_queue being released
 *
 * Called from blk_release_queue().  Responsible for exiting blkcg part.
 */
void blkcg_exit_queue(struct request_queue *q)
{
1238
	spin_lock_irq(q->queue_lock);
1239
	blkg_destroy_all(q);
1240 1241
	spin_unlock_irq(q->queue_lock);

1242 1243 1244
	blk_throtl_exit(q);
}

1245 1246 1247 1248 1249 1250
/*
 * We cannot support shared io contexts, as we have no mean to support
 * two tasks with the same ioc in two different groups without major rework
 * of the main cic data structures.  For now we allow a task to change
 * its cgroup only if it's the only owner of its ioc.
 */
1251
static int blkcg_can_attach(struct cgroup_taskset *tset)
1252
{
1253
	struct task_struct *task;
1254
	struct cgroup_subsys_state *dst_css;
1255 1256 1257 1258
	struct io_context *ioc;
	int ret = 0;

	/* task_lock() is needed to avoid races with exit_io_context() */
1259
	cgroup_taskset_for_each(task, dst_css, tset) {
1260 1261 1262 1263 1264 1265 1266 1267
		task_lock(task);
		ioc = task->io_context;
		if (ioc && atomic_read(&ioc->nr_tasks) > 1)
			ret = -EINVAL;
		task_unlock(task);
		if (ret)
			break;
	}
1268 1269 1270
	return ret;
}

1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290
static void blkcg_bind(struct cgroup_subsys_state *root_css)
{
	int i;

	mutex_lock(&blkcg_pol_mutex);

	for (i = 0; i < BLKCG_MAX_POLS; i++) {
		struct blkcg_policy *pol = blkcg_policy[i];
		struct blkcg *blkcg;

		if (!pol || !pol->cpd_bind_fn)
			continue;

		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node)
			if (blkcg->cpd[pol->plid])
				pol->cpd_bind_fn(blkcg->cpd[pol->plid]);
	}
	mutex_unlock(&blkcg_pol_mutex);
}

1291
struct cgroup_subsys io_cgrp_subsys = {
1292 1293 1294
	.css_alloc = blkcg_css_alloc,
	.css_offline = blkcg_css_offline,
	.css_free = blkcg_css_free,
1295
	.can_attach = blkcg_can_attach,
1296
	.bind = blkcg_bind,
1297
	.dfl_cftypes = blkcg_files,
1298
	.legacy_cftypes = blkcg_legacy_files,
1299
	.legacy_name = "blkio",
1300 1301 1302 1303 1304 1305 1306 1307
#ifdef CONFIG_MEMCG
	/*
	 * This ensures that, if available, memcg is automatically enabled
	 * together on the default hierarchy so that the owner cgroup can
	 * be retrieved from writeback pages.
	 */
	.depends_on = 1 << memory_cgrp_id,
#endif
1308
};