page-writeback.c 83.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
/*
2
 * mm/page-writeback.c
Linus Torvalds's avatar
Linus Torvalds committed
3 4
 *
 * Copyright (C) 2002, Linus Torvalds.
5
 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
Linus Torvalds's avatar
Linus Torvalds committed
6 7 8 9
 *
 * Contains functions related to writing back dirty pages at the
 * address_space level.
 *
10
 * 10Apr2002	Andrew Morton
Linus Torvalds's avatar
Linus Torvalds committed
11 12 13 14
 *		Initial version
 */

#include <linux/kernel.h>
15
#include <linux/export.h>
Linus Torvalds's avatar
Linus Torvalds committed
16 17 18 19 20 21 22 23 24
#include <linux/spinlock.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/slab.h>
#include <linux/pagemap.h>
#include <linux/writeback.h>
#include <linux/init.h>
#include <linux/backing-dev.h>
25
#include <linux/task_io_accounting_ops.h>
Linus Torvalds's avatar
Linus Torvalds committed
26 27
#include <linux/blkdev.h>
#include <linux/mpage.h>
28
#include <linux/rmap.h>
Linus Torvalds's avatar
Linus Torvalds committed
29 30 31 32 33
#include <linux/percpu.h>
#include <linux/smp.h>
#include <linux/sysctl.h>
#include <linux/cpu.h>
#include <linux/syscalls.h>
Al Viro's avatar
Al Viro committed
34
#include <linux/buffer_head.h> /* __set_page_dirty_buffers */
35
#include <linux/pagevec.h>
36
#include <linux/timer.h>
37
#include <linux/sched/rt.h>
38
#include <linux/sched/signal.h>
39
#include <linux/mm_inline.h>
40
#include <trace/events/writeback.h>
Linus Torvalds's avatar
Linus Torvalds committed
41

42 43
#include "internal.h"

44 45 46 47 48
/*
 * Sleep at most 200ms at a time in balance_dirty_pages().
 */
#define MAX_PAUSE		max(HZ/5, 1)

49 50 51 52 53 54
/*
 * Try to keep balance_dirty_pages() call intervals higher than this many pages
 * by raising pause time to max_pause when falls below it.
 */
#define DIRTY_POLL_THRESH	(128 >> (PAGE_SHIFT - 10))

55 56 57 58 59
/*
 * Estimate write bandwidth at 200ms intervals.
 */
#define BANDWIDTH_INTERVAL	max(HZ/5, 1)

60 61
#define RATELIMIT_CALC_SHIFT	10

Linus Torvalds's avatar
Linus Torvalds committed
62 63 64 65 66 67 68 69 70
/*
 * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited
 * will look to see if it needs to force writeback or throttling.
 */
static long ratelimit_pages = 32;

/* The following parameters are exported via /proc/sys/vm */

/*
71
 * Start background writeback (via writeback threads) at this percentage
Linus Torvalds's avatar
Linus Torvalds committed
72
 */
73
int dirty_background_ratio = 10;
Linus Torvalds's avatar
Linus Torvalds committed
74

75 76 77 78 79 80
/*
 * dirty_background_bytes starts at 0 (disabled) so that it is a function of
 * dirty_background_ratio * the amount of dirtyable memory
 */
unsigned long dirty_background_bytes;

81 82 83 84 85 86
/*
 * free highmem will not be subtracted from the total free memory
 * for calculating free ratios if vm_highmem_is_dirtyable is true
 */
int vm_highmem_is_dirtyable;

Linus Torvalds's avatar
Linus Torvalds committed
87 88 89
/*
 * The generator of dirty data starts writeback at this percentage
 */
90
int vm_dirty_ratio = 20;
Linus Torvalds's avatar
Linus Torvalds committed
91

92 93 94 95 96 97
/*
 * vm_dirty_bytes starts at 0 (disabled) so that it is a function of
 * vm_dirty_ratio * the amount of dirtyable memory
 */
unsigned long vm_dirty_bytes;

Linus Torvalds's avatar
Linus Torvalds committed
98
/*
99
 * The interval between `kupdate'-style writebacks
Linus Torvalds's avatar
Linus Torvalds committed
100
 */
101
unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */
Linus Torvalds's avatar
Linus Torvalds committed
102

103 104
EXPORT_SYMBOL_GPL(dirty_writeback_interval);

Linus Torvalds's avatar
Linus Torvalds committed
105
/*
106
 * The longest time for which data is allowed to remain dirty
Linus Torvalds's avatar
Linus Torvalds committed
107
 */
108
unsigned int dirty_expire_interval = 30 * 100; /* centiseconds */
Linus Torvalds's avatar
Linus Torvalds committed
109 110 111 112 113 114 115

/*
 * Flag that makes the machine dump writes/reads and block dirtyings.
 */
int block_dump;

/*
116 117
 * Flag that puts the machine in "laptop mode". Doubles as a timeout in jiffies:
 * a full sync is triggered after this time elapses without any disk activity.
Linus Torvalds's avatar
Linus Torvalds committed
118 119 120 121 122 123 124
 */
int laptop_mode;

EXPORT_SYMBOL(laptop_mode);

/* End of sysctl-exported parameters */

125
struct wb_domain global_wb_domain;
Linus Torvalds's avatar
Linus Torvalds committed
126

127 128
/* consolidated parameters for balance_dirty_pages() and its subroutines */
struct dirty_throttle_control {
129 130
#ifdef CONFIG_CGROUP_WRITEBACK
	struct wb_domain	*dom;
131
	struct dirty_throttle_control *gdtc;	/* only set in memcg dtc's */
132
#endif
133
	struct bdi_writeback	*wb;
134
	struct fprop_local_percpu *wb_completions;
135

136
	unsigned long		avail;		/* dirtyable */
137 138 139 140 141 142
	unsigned long		dirty;		/* file_dirty + write + nfs */
	unsigned long		thresh;		/* dirty threshold */
	unsigned long		bg_thresh;	/* dirty background threshold */

	unsigned long		wb_dirty;	/* per-wb counterparts */
	unsigned long		wb_thresh;
143
	unsigned long		wb_bg_thresh;
144 145

	unsigned long		pos_ratio;
146 147
};

148 149 150 151 152 153
/*
 * Length of period for aging writeout fractions of bdis. This is an
 * arbitrarily chosen number. The longer the period, the slower fractions will
 * reflect changes in current writeout rate.
 */
#define VM_COMPLETIONS_PERIOD_LEN (3*HZ)
154

155 156
#ifdef CONFIG_CGROUP_WRITEBACK

157 158 159 160
#define GDTC_INIT(__wb)		.wb = (__wb),				\
				.dom = &global_wb_domain,		\
				.wb_completions = &(__wb)->completions

161
#define GDTC_INIT_NO_WB		.dom = &global_wb_domain
162 163 164 165 166

#define MDTC_INIT(__wb, __gdtc)	.wb = (__wb),				\
				.dom = mem_cgroup_wb_domain(__wb),	\
				.wb_completions = &(__wb)->memcg_completions, \
				.gdtc = __gdtc
167 168 169 170 171

static bool mdtc_valid(struct dirty_throttle_control *dtc)
{
	return dtc->dom;
}
172 173 174 175 176 177

static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc)
{
	return dtc->dom;
}

178 179 180 181 182
static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc)
{
	return mdtc->gdtc;
}

183 184 185 186 187
static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb)
{
	return &wb->memcg_completions;
}

188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216
static void wb_min_max_ratio(struct bdi_writeback *wb,
			     unsigned long *minp, unsigned long *maxp)
{
	unsigned long this_bw = wb->avg_write_bandwidth;
	unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth);
	unsigned long long min = wb->bdi->min_ratio;
	unsigned long long max = wb->bdi->max_ratio;

	/*
	 * @wb may already be clean by the time control reaches here and
	 * the total may not include its bw.
	 */
	if (this_bw < tot_bw) {
		if (min) {
			min *= this_bw;
			do_div(min, tot_bw);
		}
		if (max < 100) {
			max *= this_bw;
			do_div(max, tot_bw);
		}
	}

	*minp = min;
	*maxp = max;
}

#else	/* CONFIG_CGROUP_WRITEBACK */

217 218
#define GDTC_INIT(__wb)		.wb = (__wb),                           \
				.wb_completions = &(__wb)->completions
219
#define GDTC_INIT_NO_WB
220 221 222 223 224 225
#define MDTC_INIT(__wb, __gdtc)

static bool mdtc_valid(struct dirty_throttle_control *dtc)
{
	return false;
}
226 227 228 229 230 231

static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc)
{
	return &global_wb_domain;
}

232 233 234 235 236
static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc)
{
	return NULL;
}

237 238 239 240 241
static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb)
{
	return NULL;
}

242 243 244 245 246 247 248 249 250
static void wb_min_max_ratio(struct bdi_writeback *wb,
			     unsigned long *minp, unsigned long *maxp)
{
	*minp = wb->bdi->min_ratio;
	*maxp = wb->bdi->max_ratio;
}

#endif	/* CONFIG_CGROUP_WRITEBACK */

251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268
/*
 * In a memory zone, there is a certain amount of pages we consider
 * available for the page cache, which is essentially the number of
 * free and reclaimable pages, minus some zone reserves to protect
 * lowmem and the ability to uphold the zone's watermarks without
 * requiring writeback.
 *
 * This number of dirtyable pages is the base value of which the
 * user-configurable dirty ratio is the effictive number of pages that
 * are allowed to be actually dirtied.  Per individual zone, or
 * globally by using the sum of dirtyable pages over all zones.
 *
 * Because the user is allowed to specify the dirty limit globally as
 * absolute number of bytes, calculating the per-zone dirty limit can
 * require translating the configured limit into a percentage of
 * global dirtyable memory first.
 */

269
/**
270 271
 * node_dirtyable_memory - number of dirtyable pages in a node
 * @pgdat: the node
272
 *
273 274
 * Returns the node's number of pages potentially available for dirty
 * page cache.  This is the base value for the per-node dirty limits.
275
 */
276
static unsigned long node_dirtyable_memory(struct pglist_data *pgdat)
277
{
278 279 280 281 282 283 284 285 286 287 288
	unsigned long nr_pages = 0;
	int z;

	for (z = 0; z < MAX_NR_ZONES; z++) {
		struct zone *zone = pgdat->node_zones + z;

		if (!populated_zone(zone))
			continue;

		nr_pages += zone_page_state(zone, NR_FREE_PAGES);
	}
289

290 291 292 293 294
	/*
	 * Pages reserved for the kernel should not be considered
	 * dirtyable, to prevent a situation where reclaim has to
	 * clean pages in order to balance the zones.
	 */
295
	nr_pages -= min(nr_pages, pgdat->totalreserve_pages);
296

297 298
	nr_pages += node_page_state(pgdat, NR_INACTIVE_FILE);
	nr_pages += node_page_state(pgdat, NR_ACTIVE_FILE);
299 300 301 302

	return nr_pages;
}

303 304 305 306
static unsigned long highmem_dirtyable_memory(unsigned long total)
{
#ifdef CONFIG_HIGHMEM
	int node;
307
	unsigned long x = 0;
308
	int i;
309 310

	for_each_node_state(node, N_HIGH_MEMORY) {
311 312
		for (i = ZONE_NORMAL + 1; i < MAX_NR_ZONES; i++) {
			struct zone *z;
313
			unsigned long nr_pages;
314 315 316 317 318

			if (!is_highmem_idx(i))
				continue;

			z = &NODE_DATA(node)->node_zones[i];
319 320
			if (!populated_zone(z))
				continue;
321

322
			nr_pages = zone_page_state(z, NR_FREE_PAGES);
323
			/* watch for underflows */
324
			nr_pages -= min(nr_pages, high_wmark_pages(z));
325 326 327
			nr_pages += zone_page_state(z, NR_ZONE_INACTIVE_FILE);
			nr_pages += zone_page_state(z, NR_ZONE_ACTIVE_FILE);
			x += nr_pages;
328
		}
329
	}
330

331 332 333 334 335 336 337 338 339 340 341 342
	/*
	 * Unreclaimable memory (kernel memory or anonymous memory
	 * without swap) can bring down the dirtyable pages below
	 * the zone's dirty balance reserve and the above calculation
	 * will underflow.  However we still want to add in nodes
	 * which are below threshold (negative values) to get a more
	 * accurate calculation but make sure that the total never
	 * underflows.
	 */
	if ((long)x < 0)
		x = 0;

343 344 345 346 347 348 349 350 351 352 353 354 355
	/*
	 * Make sure that the number of highmem pages is never larger
	 * than the number of the total dirtyable memory. This can only
	 * occur in very strange VM situations but we want to make sure
	 * that this does not occur.
	 */
	return min(x, total);
#else
	return 0;
#endif
}

/**
356
 * global_dirtyable_memory - number of globally dirtyable pages
357
 *
358 359
 * Returns the global number of pages potentially available for dirty
 * page cache.  This is the base value for the global dirty limits.
360
 */
361
static unsigned long global_dirtyable_memory(void)
362 363 364
{
	unsigned long x;

365
	x = global_zone_page_state(NR_FREE_PAGES);
366 367 368 369 370 371
	/*
	 * Pages reserved for the kernel should not be considered
	 * dirtyable, to prevent a situation where reclaim has to
	 * clean pages in order to balance the zones.
	 */
	x -= min(x, totalreserve_pages);
372

373 374
	x += global_node_page_state(NR_INACTIVE_FILE);
	x += global_node_page_state(NR_ACTIVE_FILE);
375

376 377 378 379 380 381
	if (!vm_highmem_is_dirtyable)
		x -= highmem_dirtyable_memory(x);

	return x + 1;	/* Ensure that we never return 0 */
}

382 383 384
/**
 * domain_dirty_limits - calculate thresh and bg_thresh for a wb_domain
 * @dtc: dirty_throttle_control of interest
385
 *
386 387 388 389
 * Calculate @dtc->thresh and ->bg_thresh considering
 * vm_dirty_{bytes|ratio} and dirty_background_{bytes|ratio}.  The caller
 * must ensure that @dtc->avail is set before calling this function.  The
 * dirty limits will be lifted by 1/4 for PF_LESS_THROTTLE (ie. nfsd) and
390 391
 * real-time tasks.
 */
392
static void domain_dirty_limits(struct dirty_throttle_control *dtc)
393
{
394 395 396 397
	const unsigned long available_memory = dtc->avail;
	struct dirty_throttle_control *gdtc = mdtc_gdtc(dtc);
	unsigned long bytes = vm_dirty_bytes;
	unsigned long bg_bytes = dirty_background_bytes;
398 399 400
	/* convert ratios to per-PAGE_SIZE for higher precision */
	unsigned long ratio = (vm_dirty_ratio * PAGE_SIZE) / 100;
	unsigned long bg_ratio = (dirty_background_ratio * PAGE_SIZE) / 100;
401 402
	unsigned long thresh;
	unsigned long bg_thresh;
403 404
	struct task_struct *tsk;

405 406 407 408 409 410 411
	/* gdtc is !NULL iff @dtc is for memcg domain */
	if (gdtc) {
		unsigned long global_avail = gdtc->avail;

		/*
		 * The byte settings can't be applied directly to memcg
		 * domains.  Convert them to ratios by scaling against
412 413 414
		 * globally available memory.  As the ratios are in
		 * per-PAGE_SIZE, they can be obtained by dividing bytes by
		 * number of pages.
415 416
		 */
		if (bytes)
417 418
			ratio = min(DIV_ROUND_UP(bytes, global_avail),
				    PAGE_SIZE);
419
		if (bg_bytes)
420 421
			bg_ratio = min(DIV_ROUND_UP(bg_bytes, global_avail),
				       PAGE_SIZE);
422 423 424 425 426
		bytes = bg_bytes = 0;
	}

	if (bytes)
		thresh = DIV_ROUND_UP(bytes, PAGE_SIZE);
427
	else
428
		thresh = (ratio * available_memory) / PAGE_SIZE;
429

430 431
	if (bg_bytes)
		bg_thresh = DIV_ROUND_UP(bg_bytes, PAGE_SIZE);
432
	else
433
		bg_thresh = (bg_ratio * available_memory) / PAGE_SIZE;
434

435
	if (bg_thresh >= thresh)
436
		bg_thresh = thresh / 2;
437 438
	tsk = current;
	if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) {
439 440
		bg_thresh += bg_thresh / 4 + global_wb_domain.dirty_limit / 32;
		thresh += thresh / 4 + global_wb_domain.dirty_limit / 32;
441
	}
442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466
	dtc->thresh = thresh;
	dtc->bg_thresh = bg_thresh;

	/* we should eventually report the domain in the TP */
	if (!gdtc)
		trace_global_dirty_state(bg_thresh, thresh);
}

/**
 * global_dirty_limits - background-writeback and dirty-throttling thresholds
 * @pbackground: out parameter for bg_thresh
 * @pdirty: out parameter for thresh
 *
 * Calculate bg_thresh and thresh for global_wb_domain.  See
 * domain_dirty_limits() for details.
 */
void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty)
{
	struct dirty_throttle_control gdtc = { GDTC_INIT_NO_WB };

	gdtc.avail = global_dirtyable_memory();
	domain_dirty_limits(&gdtc);

	*pbackground = gdtc.bg_thresh;
	*pdirty = gdtc.thresh;
467 468
}

469
/**
470 471
 * node_dirty_limit - maximum number of dirty pages allowed in a node
 * @pgdat: the node
472
 *
473 474
 * Returns the maximum number of dirty pages allowed in a node, based
 * on the node's dirtyable memory.
475
 */
476
static unsigned long node_dirty_limit(struct pglist_data *pgdat)
477
{
478
	unsigned long node_memory = node_dirtyable_memory(pgdat);
479 480 481 482 483
	struct task_struct *tsk = current;
	unsigned long dirty;

	if (vm_dirty_bytes)
		dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE) *
484
			node_memory / global_dirtyable_memory();
485
	else
486
		dirty = vm_dirty_ratio * node_memory / 100;
487 488 489 490 491 492 493 494

	if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk))
		dirty += dirty / 4;

	return dirty;
}

/**
495 496
 * node_dirty_ok - tells whether a node is within its dirty limits
 * @pgdat: the node to check
497
 *
498
 * Returns %true when the dirty pages in @pgdat are within the node's
499 500
 * dirty limit, %false if the limit is exceeded.
 */
501
bool node_dirty_ok(struct pglist_data *pgdat)
502
{
503 504 505
	unsigned long limit = node_dirty_limit(pgdat);
	unsigned long nr_pages = 0;

506 507 508
	nr_pages += node_page_state(pgdat, NR_FILE_DIRTY);
	nr_pages += node_page_state(pgdat, NR_UNSTABLE_NFS);
	nr_pages += node_page_state(pgdat, NR_WRITEBACK);
509

510
	return nr_pages <= limit;
511 512
}

513
int dirty_background_ratio_handler(struct ctl_table *table, int write,
514
		void __user *buffer, size_t *lenp,
515 516 517 518
		loff_t *ppos)
{
	int ret;

519
	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
520 521 522 523 524 525
	if (ret == 0 && write)
		dirty_background_bytes = 0;
	return ret;
}

int dirty_background_bytes_handler(struct ctl_table *table, int write,
526
		void __user *buffer, size_t *lenp,
527 528 529 530
		loff_t *ppos)
{
	int ret;

531
	ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
532 533 534 535 536
	if (ret == 0 && write)
		dirty_background_ratio = 0;
	return ret;
}

537
int dirty_ratio_handler(struct ctl_table *table, int write,
538
		void __user *buffer, size_t *lenp,
539 540 541
		loff_t *ppos)
{
	int old_ratio = vm_dirty_ratio;
542 543
	int ret;

544
	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
545
	if (ret == 0 && write && vm_dirty_ratio != old_ratio) {
546
		writeback_set_ratelimit();
547 548 549 550 551 552
		vm_dirty_bytes = 0;
	}
	return ret;
}

int dirty_bytes_handler(struct ctl_table *table, int write,
553
		void __user *buffer, size_t *lenp,
554 555
		loff_t *ppos)
{
556
	unsigned long old_bytes = vm_dirty_bytes;
557 558
	int ret;

559
	ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
560
	if (ret == 0 && write && vm_dirty_bytes != old_bytes) {
561
		writeback_set_ratelimit();
562
		vm_dirty_ratio = 0;
563 564 565 566
	}
	return ret;
}

567 568 569 570 571 572 573 574 575
static unsigned long wp_next_time(unsigned long cur_time)
{
	cur_time += VM_COMPLETIONS_PERIOD_LEN;
	/* 0 has a special meaning... */
	if (!cur_time)
		return 1;
	return cur_time;
}

576 577 578
static void wb_domain_writeout_inc(struct wb_domain *dom,
				   struct fprop_local_percpu *completions,
				   unsigned int max_prop_frac)
579
{
580 581
	__fprop_inc_percpu_max(&dom->completions, completions,
			       max_prop_frac);
582
	/* First event after period switching was turned off? */
583
	if (unlikely(!dom->period_time)) {
584 585 586 587 588 589
		/*
		 * We can race with other __bdi_writeout_inc calls here but
		 * it does not cause any harm since the resulting time when
		 * timer will fire and what is in writeout_period_time will be
		 * roughly the same.
		 */
Tejun Heo's avatar
Tejun Heo committed
590 591
		dom->period_time = wp_next_time(jiffies);
		mod_timer(&dom->period_timer, dom->period_time);
592
	}
593 594
}

595 596 597 598 599
/*
 * Increment @wb's writeout completion count and the global writeout
 * completion count. Called from test_clear_page_writeback().
 */
static inline void __wb_writeout_inc(struct bdi_writeback *wb)
600
{
601
	struct wb_domain *cgdom;
602

603
	inc_wb_stat(wb, WB_WRITTEN);
604 605
	wb_domain_writeout_inc(&global_wb_domain, &wb->completions,
			       wb->bdi->max_prop_frac);
606 607 608 609 610

	cgdom = mem_cgroup_wb_domain(wb);
	if (cgdom)
		wb_domain_writeout_inc(cgdom, wb_memcg_completions(wb),
				       wb->bdi->max_prop_frac);
611 612
}

613
void wb_writeout_inc(struct bdi_writeback *wb)
614
{
615 616 617
	unsigned long flags;

	local_irq_save(flags);
618
	__wb_writeout_inc(wb);
619
	local_irq_restore(flags);
620
}
621
EXPORT_SYMBOL_GPL(wb_writeout_inc);
622

623 624 625 626
/*
 * On idle system, we can be called long after we scheduled because we use
 * deferred timers so count with missed periods.
 */
627
static void writeout_period(struct timer_list *t)
628
{
629
	struct wb_domain *dom = from_timer(dom, t, period_timer);
Tejun Heo's avatar
Tejun Heo committed
630
	int miss_periods = (jiffies - dom->period_time) /
631 632
						 VM_COMPLETIONS_PERIOD_LEN;

Tejun Heo's avatar
Tejun Heo committed
633 634
	if (fprop_new_period(&dom->completions, miss_periods + 1)) {
		dom->period_time = wp_next_time(dom->period_time +
635
				miss_periods * VM_COMPLETIONS_PERIOD_LEN);
Tejun Heo's avatar
Tejun Heo committed
636
		mod_timer(&dom->period_timer, dom->period_time);
637 638 639 640 641
	} else {
		/*
		 * Aging has zeroed all fractions. Stop wasting CPU on period
		 * updates.
		 */
Tejun Heo's avatar
Tejun Heo committed
642
		dom->period_time = 0;
643 644 645
	}
}

Tejun Heo's avatar
Tejun Heo committed
646 647 648
int wb_domain_init(struct wb_domain *dom, gfp_t gfp)
{
	memset(dom, 0, sizeof(*dom));
649 650 651

	spin_lock_init(&dom->lock);

652
	timer_setup(&dom->period_timer, writeout_period, TIMER_DEFERRABLE);
653 654 655

	dom->dirty_limit_tstamp = jiffies;

Tejun Heo's avatar
Tejun Heo committed
656 657 658
	return fprop_global_init(&dom->completions, gfp);
}

659 660 661 662 663 664 665 666
#ifdef CONFIG_CGROUP_WRITEBACK
void wb_domain_exit(struct wb_domain *dom)
{
	del_timer_sync(&dom->period_timer);
	fprop_global_destroy(&dom->completions);
}
#endif

667
/*
668 669 670
 * bdi_min_ratio keeps the sum of the minimum dirty shares of all
 * registered backing devices, which, for obvious reasons, can not
 * exceed 100%.
671 672 673 674 675 676 677
 */
static unsigned int bdi_min_ratio;

int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)
{
	int ret = 0;

678
	spin_lock_bh(&bdi_lock);
679
	if (min_ratio > bdi->max_ratio) {
680
		ret = -EINVAL;
681 682 683 684 685 686 687 688 689
	} else {
		min_ratio -= bdi->min_ratio;
		if (bdi_min_ratio + min_ratio < 100) {
			bdi_min_ratio += min_ratio;
			bdi->min_ratio += min_ratio;
		} else {
			ret = -EINVAL;
		}
	}
690
	spin_unlock_bh(&bdi_lock);
691 692 693 694 695 696 697 698 699 700 701

	return ret;
}

int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned max_ratio)
{
	int ret = 0;

	if (max_ratio > 100)
		return -EINVAL;

702
	spin_lock_bh(&bdi_lock);
703 704 705 706
	if (bdi->min_ratio > max_ratio) {
		ret = -EINVAL;
	} else {
		bdi->max_ratio = max_ratio;
707
		bdi->max_prop_frac = (FPROP_FRAC_BASE * max_ratio) / 100;
708
	}
709
	spin_unlock_bh(&bdi_lock);
710 711 712

	return ret;
}
713
EXPORT_SYMBOL(bdi_set_max_ratio);
714

715 716 717 718 719 720
static unsigned long dirty_freerun_ceiling(unsigned long thresh,
					   unsigned long bg_thresh)
{
	return (thresh + bg_thresh) / 2;
}

721 722
static unsigned long hard_dirty_limit(struct wb_domain *dom,
				      unsigned long thresh)
723
{
724
	return max(thresh, dom->dirty_limit);
725 726
}

727 728 729 730 731 732
/*
 * Memory which can be further allocated to a memcg domain is capped by
 * system-wide clean memory excluding the amount being used in the domain.
 */
static void mdtc_calc_avail(struct dirty_throttle_control *mdtc,
			    unsigned long filepages, unsigned long headroom)
733 734
{
	struct dirty_throttle_control *gdtc = mdtc_gdtc(mdtc);
735 736 737
	unsigned long clean = filepages - min(filepages, mdtc->dirty);
	unsigned long global_clean = gdtc->avail - min(gdtc->avail, gdtc->dirty);
	unsigned long other_clean = global_clean - min(global_clean, clean);
738

739
	mdtc->avail = filepages + min(headroom, other_clean);
740 741
}

742
/**
743 744
 * __wb_calc_thresh - @wb's share of dirty throttling threshold
 * @dtc: dirty_throttle_context of interest
745
 *
746
 * Returns @wb's dirty limit in pages. The term "dirty" in the context of
747
 * dirty balancing includes all PG_dirty, PG_writeback and NFS unstable pages.
748 749 750 751 752 753
 *
 * Note that balance_dirty_pages() will only seriously take it as a hard limit
 * when sleeping max_pause per page is not enough to keep the dirty pages under
 * control. For example, when the device is completely stalled due to some error
 * conditions, or when there are 1000 dd tasks writing to a slow 10MB/s USB key.
 * In the other normal situations, it acts more gently by throttling the tasks
754
 * more (rather than completely block them) when the wb dirty pages go high.
755
 *
756
 * It allocates high/low dirty limits to fast/slow devices, in order to prevent
757 758 759
 * - starving fast devices
 * - piling up dirty pages (that will take long time to sync) on slow devices
 *
760
 * The wb's share of dirty limit will be adapting to its throughput and
761 762
 * bounded by the bdi->min_ratio and/or bdi->max_ratio parameters, if set.
 */
763
static unsigned long __wb_calc_thresh(struct dirty_throttle_control *dtc)
764
{
765
	struct wb_domain *dom = dtc_dom(dtc);
766
	unsigned long thresh = dtc->thresh;
767
	u64 wb_thresh;
768
	long numerator, denominator;
769
	unsigned long wb_min_ratio, wb_max_ratio;
770

771
	/*
772
	 * Calculate this BDI's share of the thresh ratio.
773
	 */
774
	fprop_fraction_percpu(&dom->completions, dtc->wb_completions,
Tejun Heo's avatar
Tejun Heo committed
775
			      &numerator, &denominator);
776

777 778 779
	wb_thresh = (thresh * (100 - bdi_min_ratio)) / 100;
	wb_thresh *= numerator;
	do_div(wb_thresh, denominator);
780

781
	wb_min_max_ratio(dtc->wb, &wb_min_ratio, &wb_max_ratio);
782

783 784 785
	wb_thresh += (thresh * wb_min_ratio) / 100;
	if (wb_thresh > (thresh * wb_max_ratio) / 100)
		wb_thresh = thresh * wb_max_ratio / 100;
786

787
	return wb_thresh;
Linus Torvalds's avatar
Linus Torvalds committed
788 789
}

790 791 792 793 794
unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh)
{
	struct dirty_throttle_control gdtc = { GDTC_INIT(wb),
					       .thresh = thresh };
	return __wb_calc_thresh(&gdtc);
Linus Torvalds's avatar
Linus Torvalds committed
795 796
}

797 798 799 800 801 802 803 804 805 806 807 808 809 810
/*
 *                           setpoint - dirty 3
 *        f(dirty) := 1.0 + (----------------)
 *                           limit - setpoint
 *
 * it's a 3rd order polynomial that subjects to
 *
 * (1) f(freerun)  = 2.0 => rampup dirty_ratelimit reasonably fast
 * (2) f(setpoint) = 1.0 => the balance point
 * (3) f(limit)    = 0   => the hard limit
 * (4) df/dx      <= 0	 => negative feedback control
 * (5) the closer to setpoint, the smaller |df/dx| (and the reverse)
 *     => fast response on large errors; small oscillation near setpoint
 */
811
static long long pos_ratio_polynom(unsigned long setpoint,
812 813 814 815 816 817
					  unsigned long dirty,
					  unsigned long limit)
{
	long long pos_ratio;
	long x;

818
	x = div64_s64(((s64)setpoint - (s64)dirty) << RATELIMIT_CALC_SHIFT,
819
		      (limit - setpoint) | 1);
820 821 822 823 824 825 826 827
	pos_ratio = x;
	pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
	pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
	pos_ratio += 1 << RATELIMIT_CALC_SHIFT;

	return clamp(pos_ratio, 0LL, 2LL << RATELIMIT_CALC_SHIFT);
}

828 829 830 831 832
/*
 * Dirty position control.
 *
 * (o) global/bdi setpoints
 *
833
 * We want the dirty pages be balanced around the global/wb setpoints.
834 835 836 837 838 839 840 841 842
 * When the number of dirty pages is higher/lower than the setpoint, the
 * dirty position control ratio (and hence task dirty ratelimit) will be
 * decreased/increased to bring the dirty pages back to the setpoint.
 *
 *     pos_ratio = 1 << RATELIMIT_CALC_SHIFT
 *
 *     if (dirty < setpoint) scale up   pos_ratio
 *     if (dirty > setpoint) scale down pos_ratio
 *
843 844
 *     if (wb_dirty < wb_setpoint) scale up   pos_ratio
 *     if (wb_dirty > wb_setpoint) scale down pos_ratio
845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868
 *
 *     task_ratelimit = dirty_ratelimit * pos_ratio >> RATELIMIT_CALC_SHIFT
 *
 * (o) global control line
 *
 *     ^ pos_ratio
 *     |
 *     |            |<===== global dirty control scope ======>|
 * 2.0 .............*
 *     |            .*
 *     |            . *
 *     |            .   *
 *     |            .     *
 *     |            .        *
 *     |            .            *
 * 1.0 ................................*
 *     |            .                  .     *
 *     |            .                  .          *
 *     |            .                  .              *
 *     |            .                  .                 *
 *     |            .                  .                    *
 *   0 +------------.------------------.----------------------*------------->
 *           freerun^          setpoint^                 limit^   dirty pages
 *
869
 * (o) wb control line
870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894
 *
 *     ^ pos_ratio
 *     |
 *     |            *
 *     |              *
 *     |                *
 *     |                  *
 *     |                    * |<=========== span ============>|
 * 1.0 .......................*
 *     |                      . *
 *     |                      .   *
 *     |                      .     *
 *     |                      .       *
 *     |                      .         *
 *     |                      .           *
 *     |                      .             *
 *     |                      .               *
 *     |                      .                 *
 *     |                      .                   *
 *     |                      .                     *
 * 1/4 ...............................................* * * * * * * * * * * *
 *     |                      .                         .
 *     |                      .                           .
 *     |                      .                             .
 *   0 +----------------------.-------------------------------.------------->
895
 *                wb_setpoint^                    x_intercept^
896
 *
897
 * The wb control line won't drop below pos_ratio=1/4, so that wb_dirty can
898 899
 * be smoothly throttled down to normal if it starts high in situations like
 * - start writing to a slow SD card and a fast disk at the same time. The SD
900 901
 *   card's wb_dirty may rush to many times higher than wb_setpoint.
 * - the wb dirty thresh drops quickly due to change of JBOD workload
902
 */
903
static void wb_position_ratio(struct dirty_throttle_control *dtc)
904
{
905
	struct bdi_writeback *wb = dtc->wb;
906
	unsigned long write_bw = wb->avg_write_bandwidth;
907
	unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh);
908
	unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh);
909
	unsigned long wb_thresh = dtc->wb_thresh;
910 911
	unsigned long x_intercept;
	unsigned long setpoint;		/* dirty pages' target balance point */
912
	unsigned long wb_setpoint;
913 914 915 916
	unsigned long span;
	long long pos_ratio;		/* for scaling up/down the rate limit */
	long x;

917 918
	dtc->pos_ratio = 0;

919
	if (unlikely(dtc->dirty >= limit))
920
		return;
921 922 923 924

	/*
	 * global setpoint
	 *
925 926 927
	 * See comment for pos_ratio_polynom().
	 */
	setpoint = (freerun + limit) / 2;
928
	pos_ratio = pos_ratio_polynom(setpoint, dtc->dirty, limit);
929 930 931 932

	/*
	 * The strictlimit feature is a tool preventing mistrusted filesystems
	 * from growing a large number of dirty pages before throttling. For
933 934
	 * such filesystems balance_dirty_pages always checks wb counters
	 * against wb limits. Even if global "nr_dirty" is under "freerun".
935 936 937 938
	 * This is especially important for fuse which sets bdi->max_ratio to
	 * 1% by default. Without strictlimit feature, fuse writeback may
	 * consume arbitrary amount of RAM because it is accounted in
	 * NR_WRITEBACK_TEMP which is not involved in calculating "nr_dirty".
939
	 *
940
	 * Here, in wb_position_ratio(), we calculate pos_ratio based on
941
	 * two values: wb_dirty and wb_thresh. Let's consider an example:
942 943
	 * total amount of RAM is 16GB, bdi->max_ratio is equal to 1%, global
	 * limits are set by default to 10% and 20% (background and throttle).
944
	 * Then wb_thresh is 1% of 20% of 16GB. This amounts to ~8K pages.
945
	 * wb_calc_thresh(wb, bg_thresh) is about ~4K pages. wb_setpoint is
946
	 * about ~6K pages (as the average of background and throttle wb
947
	 * limits). The 3rd order polynomial will provide positive feedback if
948
	 * wb_dirty is under wb_setpoint and vice versa.
949
	 *
950
	 * Note, that we cannot use global counters in these calculations
951
	 * because we want to throttle process writing to a strictlimit wb
952 953
	 * much earlier than global "freerun" is reached (~23MB vs. ~2.3GB
	 * in the example above).
954
	 */
955
	if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) {
956
		long long wb_pos_ratio;
957

958 959 960 961 962
		if (dtc->wb_dirty < 8) {
			dtc->pos_ratio = min_t(long long, pos_ratio * 2,
					   2 << RATELIMIT_CALC_SHIFT);
			return;
		}
963

964
		if (dtc->wb_dirty >= wb_thresh)
965
			return;
966

967 968
		wb_setpoint = dirty_freerun_ceiling(wb_thresh,
						    dtc->wb_bg_thresh);
969

970
		if (wb_setpoint == 0 || wb_setpoint == wb_thresh)
971
			return;
972

973
		wb_pos_ratio = pos_ratio_polynom(wb_setpoint, dtc->wb_dirty,
974
						 wb_thresh);
975 976

		/*
977 978
		 * Typically, for strictlimit case, wb_setpoint << setpoint
		 * and pos_ratio >> wb_pos_ratio. In the other words global
979
		 * state ("dirty") is not limiting factor and we have to
980
		 * make decision based on wb counters. But there is an
981 982
		 * important case when global pos_ratio should get precedence:
		 * global limits are exceeded (e.g. due to activities on other
983
		 * wb's) while given strictlimit wb is below limit.
984
		 *
985
		 * "pos_ratio * wb_pos_ratio" would work for the case above,
986
		 * but it would look too non-natural for the case of all
987
		 * activity in the system coming from a single strictlimit wb
988 989 990 991
		 * with bdi->max_ratio == 100%.
		 *
		 * Note that min() below somewhat changes the dynamics of the
		 * control system. Normally, pos_ratio value can be well over 3
992
		 * (when globally we are at freerun and wb is well below wb
993 994 995 996
		 * setpoint). Now the maximum pos_ratio in the same situation
		 * is 2. We might want to tweak this if we observe the control
		 * system is too slow to adapt.
		 */
997 998
		dtc->pos_ratio = min(pos_ratio, wb_pos_ratio);
		return;
999
	}
1000 1001 1002

	/*
	 * We have computed basic pos_ratio above based on global situation. If
1003
	 * the wb is over/under its share of dirty pages, we want to scale
1004 1005 1006 1007
	 * pos_ratio further down/up. That is done by the following mechanism.
	 */

	/*
1008
	 * wb setpoint
1009
	 *
1010
	 *        f(wb_dirty) := 1.0 + k * (wb_dirty - wb_setpoint)
1011
	 *
1012
	 *                        x_intercept - wb_dirty
1013
	 *                     := --------------------------
1014
	 *                        x_intercept - wb_setpoint
1015
	 *
1016
	 * The main wb control line is a linear function that subjects to
1017
	 *
1018 1019 1020
	 * (1) f(wb_setpoint) = 1.0
	 * (2) k = - 1 / (8 * write_bw)  (in single wb case)
	 *     or equally: x_intercept = wb_setpoint + 8 * write_bw
1021
	 *
1022
	 * For single wb case, the dirty pages are observed to fluctuate
1023
	 * regularly within range
1024
	 *        [wb_setpoint - write_bw/2, wb_setpoint + write_bw/2]
1025 1026 1027
	 * for various filesystems, where (2) can yield in a reasonable 12.5%
	 * fluctuation range for pos_ratio.
	 *
1028
	 * For JBOD case, wb_thresh (not wb_dirty!) could fluctuate up to its
1029
	 * own size, so move the slope over accordingly and choose a slope that
1030
	 * yields 100% pos_ratio fluctuation on suddenly doubled wb_thresh.
1031
	 */
1032 1033
	if (unlikely(wb_thresh > dtc->thresh))
		wb_thresh = dtc->thresh;
1034
	/*
1035
	 * It's very possible that wb_thresh is close to 0 not because the
1036 1037 1038 1039 1040
	 * device is slow, but that it has remained inactive for long time.
	 * Honour such devices a reasonable good (hopefully IO efficient)
	 * threshold, so that the occasional writes won't be blocked and active
	 * writes can rampup the threshold quickly.
	 */
1041
	wb_thresh = max(wb_thresh, (limit - dtc->dirty) / 8);
1042
	/*
1043 1044
	 * scale global setpoint to wb's:
	 *	wb_setpoint = setpoint * wb_thresh / thresh
1045
	 */
1046
	x = div_u64((u64)wb_thresh << 16, dtc->thresh | 1);
1047
	wb_setpoint = setpoint * (u64)x >> 16;
1048
	/*
1049 1050
	 * Use span=(8*write_bw) in single wb case as indicated by
	 * (thresh - wb_thresh ~= 0) and transit to wb_thresh in JBOD case.
1051
	 *
1052 1053 1054
	 *        wb_thresh                    thresh - wb_thresh
	 * span = --------- * (8 * write_bw) + ------------------ * wb_thresh
	 *         thresh                           thresh
1055
	 */
1056
	span = (dtc->thresh - wb_thresh + 8 * write_bw) * (u64)x >> 16;
1057
	x_intercept = wb_setpoint + span;
1058

1059 1060
	if (dtc->wb_dirty < x_intercept - span / 4) {
		pos_ratio = div64_u64(pos_ratio * (x_intercept - dtc->wb_dirty),
1061
				      (x_intercept - wb_setpoint) | 1);
1062 1063 1064
	} else
		pos_ratio /= 4;

1065
	/*
1066
	 * wb reserve area, safeguard against dirty pool underrun and disk idle
1067 1068 1069
	 * It may push the desired control point of global dirty pages higher
	 * than setpoint.
	 */
1070
	x_intercept = wb_thresh / 2;
1071 1072 1073 1074
	if (dtc->wb_dirty < x_intercept) {
		if (dtc->wb_dirty > x_intercept / 8)
			pos_ratio = div_u64(pos_ratio * x_intercept,
					    dtc->wb_dirty);
1075
		else
1076 1077 1078
			pos_ratio *= 8;
	}

1079
	dtc->pos_ratio = pos_ratio;
1080 1081
}

1082 1083 1084
static void wb_update_write_bandwidth(struct bdi_writeback *wb,
				      unsigned long elapsed,
				      unsigned long written)
1085 1086
{
	const unsigned long period = roundup_pow_of_two(3 * HZ);
1087 1088
	unsigned long avg = wb->avg_write_bandwidth;
	unsigned long old = wb->write_bandwidth;
1089 1090 1091 1092 1093 1094 1095 1096
	u64 bw;

	/*
	 * bw = written * HZ / elapsed
	 *
	 *                   bw * elapsed + write_bandwidth * (period - elapsed)
	 * write_bandwidth = ---------------------------------------------------
	 *                                          period
1097 1098 1099
	 *
	 * @written may have decreased due to account_page_redirty().
	 * Avoid underflowing @bw calculation.
1100
	 */
1101
	bw = written - min(written, wb->written_stamp);
1102 1103 1104 1105 1106 1107
	bw *= HZ;
	if (unlikely(elapsed > period)) {
		do_div(bw, elapsed);
		avg = bw;
		goto out;
	}
1108
	bw += (u64)wb->write_bandwidth * (period - elapsed);
1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120
	bw >>= ilog2(period);

	/*
	 * one more level of smoothing, for filtering out sudden spikes
	 */
	if (avg > old && old >= (unsigned long)bw)
		avg -= (avg - old) >> 3;

	if (avg < old && old <= (unsigned long)bw)
		avg += (old - avg) >> 3;

out:
1121 1122 1123 1124 1125 1126 1127
	/* keep avg > 0 to guarantee that tot > 0 if there are dirty wbs */
	avg = max(avg, 1LU);
	if (wb_has_dirty_io(wb)) {
		long delta = avg - wb->avg_write_bandwidth;
		WARN_ON_ONCE(atomic_long_add_return(delta,
					&wb->bdi->tot_write_bandwidth) <= 0);
	}
1128 1129
	wb->write_bandwidth = bw;
	wb->avg_write_bandwidth = avg;
1130 1131
}

1132
static void update_dirty_limit(struct dirty_throttle_control *dtc)
1133
{
1134
	struct wb_domain *dom = dtc_dom(dtc);
1135
	unsigned long thresh = dtc->thresh;
1136
	unsigned long limit = dom->dirty_limit;
1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148

	/*
	 * Follow