domain.c 49.6 KB
Newer Older
1
2
3
4
5
6
7
8
/*
 * drivers/base/power/domain.c - Common code related to device power domains.
 *
 * Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp.
 *
 * This file is released under the GPLv2.
 */

9
#include <linux/delay.h>
10
11
#include <linux/kernel.h>
#include <linux/io.h>
12
#include <linux/platform_device.h>
13
14
#include <linux/pm_runtime.h>
#include <linux/pm_domain.h>
15
#include <linux/pm_qos.h>
16
#include <linux/pm_clock.h>
17
18
#include <linux/slab.h>
#include <linux/err.h>
19
20
#include <linux/sched.h>
#include <linux/suspend.h>
21
22
#include <linux/export.h>

23
24
#define GENPD_RETRY_MAX_MS	250		/* Approximate */

25
26
27
28
29
30
31
32
33
34
35
#define GENPD_DEV_CALLBACK(genpd, type, callback, dev)		\
({								\
	type (*__routine)(struct device *__d); 			\
	type __ret = (type)0;					\
								\
	__routine = genpd->dev_ops.callback; 			\
	if (__routine) {					\
		__ret = __routine(dev); 			\
	}							\
	__ret;							\
})
36

37
38
39
static LIST_HEAD(gpd_list);
static DEFINE_MUTEX(gpd_list_lock);

40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
/*
 * Get the generic PM domain for a particular struct device.
 * This validates the struct device pointer, the PM domain pointer,
 * and checks that the PM domain pointer is a real generic PM domain.
 * Any failure results in NULL being returned.
 */
struct generic_pm_domain *pm_genpd_lookup_dev(struct device *dev)
{
	struct generic_pm_domain *genpd = NULL, *gpd;

	if (IS_ERR_OR_NULL(dev) || IS_ERR_OR_NULL(dev->pm_domain))
		return NULL;

	mutex_lock(&gpd_list_lock);
	list_for_each_entry(gpd, &gpd_list, gpd_list_node) {
		if (&gpd->domain == dev->pm_domain) {
			genpd = gpd;
			break;
		}
	}
	mutex_unlock(&gpd_list_lock);

	return genpd;
}

/*
 * This should only be used where we are certain that the pm_domain
 * attached to the device is a genpd domain.
 */
static struct generic_pm_domain *dev_to_genpd(struct device *dev)
70
71
72
73
{
	if (IS_ERR_OR_NULL(dev->pm_domain))
		return ERR_PTR(-EINVAL);

74
	return pd_to_genpd(dev->pm_domain);
75
}
76

77
static int genpd_stop_dev(struct generic_pm_domain *genpd, struct device *dev)
78
{
79
	return GENPD_DEV_CALLBACK(genpd, int, stop, dev);
80
81
}

82
static int genpd_start_dev(struct generic_pm_domain *genpd, struct device *dev)
83
{
84
	return GENPD_DEV_CALLBACK(genpd, int, start, dev);
85
86
}

87
static bool genpd_sd_counter_dec(struct generic_pm_domain *genpd)
88
{
89
90
91
92
93
94
95
96
97
98
99
	bool ret = false;

	if (!WARN_ON(atomic_read(&genpd->sd_count) == 0))
		ret = !!atomic_dec_and_test(&genpd->sd_count);

	return ret;
}

static void genpd_sd_counter_inc(struct generic_pm_domain *genpd)
{
	atomic_inc(&genpd->sd_count);
100
	smp_mb__after_atomic();
101
102
}

103
static int genpd_power_on(struct generic_pm_domain *genpd, bool timed)
104
105
106
107
108
109
110
111
{
	ktime_t time_start;
	s64 elapsed_ns;
	int ret;

	if (!genpd->power_on)
		return 0;

112
113
114
	if (!timed)
		return genpd->power_on(genpd);

115
116
117
118
119
120
121
122
123
124
125
	time_start = ktime_get();
	ret = genpd->power_on(genpd);
	if (ret)
		return ret;

	elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start));
	if (elapsed_ns <= genpd->power_on_latency_ns)
		return ret;

	genpd->power_on_latency_ns = elapsed_ns;
	genpd->max_off_time_changed = true;
126
127
	pr_debug("%s: Power-%s latency exceeded, new value %lld ns\n",
		 genpd->name, "on", elapsed_ns);
128
129
130
131

	return ret;
}

132
static int genpd_power_off(struct generic_pm_domain *genpd, bool timed)
133
134
135
136
137
138
139
140
{
	ktime_t time_start;
	s64 elapsed_ns;
	int ret;

	if (!genpd->power_off)
		return 0;

141
142
143
	if (!timed)
		return genpd->power_off(genpd);

144
145
146
147
148
149
150
151
152
153
154
	time_start = ktime_get();
	ret = genpd->power_off(genpd);
	if (ret == -EBUSY)
		return ret;

	elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start));
	if (elapsed_ns <= genpd->power_off_latency_ns)
		return ret;

	genpd->power_off_latency_ns = elapsed_ns;
	genpd->max_off_time_changed = true;
155
156
	pr_debug("%s: Power-%s latency exceeded, new value %lld ns\n",
		 genpd->name, "off", elapsed_ns);
157
158
159
160

	return ret;
}

161
/**
162
 * genpd_queue_power_off_work - Queue up the execution of genpd_poweroff().
163
164
 * @genpd: PM domait to power off.
 *
165
 * Queue up the execution of genpd_poweroff() unless it's already been done
166
167
168
169
170
171
172
 * before.
 */
static void genpd_queue_power_off_work(struct generic_pm_domain *genpd)
{
	queue_work(pm_wq, &genpd->power_off_work);
}

173
static int genpd_poweron(struct generic_pm_domain *genpd);
174

175
/**
176
 * __genpd_poweron - Restore power to a given PM domain and its masters.
177
178
 * @genpd: PM domain to power up.
 *
179
 * Restore power to @genpd and all of its masters so that it is possible to
180
181
 * resume a device belonging to it.
 */
182
static int __genpd_poweron(struct generic_pm_domain *genpd)
183
{
184
	struct gpd_link *link;
185
186
	int ret = 0;

187
	if (genpd->status == GPD_STATE_ACTIVE
188
	    || (genpd->prepared_count > 0 && genpd->suspend_power_off))
189
		return 0;
190

191
192
193
194
195
196
197
	/*
	 * The list is guaranteed not to change while the loop below is being
	 * executed, unless one of the masters' .power_on() callbacks fiddles
	 * with it.
	 */
	list_for_each_entry(link, &genpd->slave_links, slave_node) {
		genpd_sd_counter_inc(link->master);
198

199
		ret = genpd_poweron(link->master);
200
201
		if (ret) {
			genpd_sd_counter_dec(link->master);
202
			goto err;
203
		}
204
205
	}

206
	ret = genpd_power_on(genpd, true);
207
208
	if (ret)
		goto err;
209

210
	genpd->status = GPD_STATE_ACTIVE;
211
	return 0;
212
213

 err:
214
215
216
	list_for_each_entry_continue_reverse(link,
					&genpd->slave_links,
					slave_node) {
217
		genpd_sd_counter_dec(link->master);
218
219
		genpd_queue_power_off_work(link->master);
	}
220

221
222
223
224
	return ret;
}

/**
225
 * genpd_poweron - Restore power to a given PM domain and its masters.
226
227
 * @genpd: PM domain to power up.
 */
228
static int genpd_poweron(struct generic_pm_domain *genpd)
229
230
231
232
{
	int ret;

	mutex_lock(&genpd->lock);
233
	ret = __genpd_poweron(genpd);
234
235
	mutex_unlock(&genpd->lock);
	return ret;
236
237
}

238
239
static int genpd_save_dev(struct generic_pm_domain *genpd, struct device *dev)
{
240
	return GENPD_DEV_CALLBACK(genpd, int, save_state, dev);
241
242
}

243
static int genpd_restore_dev(struct generic_pm_domain *genpd,
244
			struct device *dev)
245
{
246
	return GENPD_DEV_CALLBACK(genpd, int, restore_state, dev);
247
248
}

249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
static int genpd_dev_pm_qos_notifier(struct notifier_block *nb,
				     unsigned long val, void *ptr)
{
	struct generic_pm_domain_data *gpd_data;
	struct device *dev;

	gpd_data = container_of(nb, struct generic_pm_domain_data, nb);
	dev = gpd_data->base.dev;

	for (;;) {
		struct generic_pm_domain *genpd;
		struct pm_domain_data *pdd;

		spin_lock_irq(&dev->power.lock);

		pdd = dev->power.subsys_data ?
				dev->power.subsys_data->domain_data : NULL;
266
		if (pdd && pdd->dev) {
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
			to_gpd_data(pdd)->td.constraint_changed = true;
			genpd = dev_to_genpd(dev);
		} else {
			genpd = ERR_PTR(-ENODATA);
		}

		spin_unlock_irq(&dev->power.lock);

		if (!IS_ERR(genpd)) {
			mutex_lock(&genpd->lock);
			genpd->max_off_time_changed = true;
			mutex_unlock(&genpd->lock);
		}

		dev = dev->parent;
		if (!dev || dev->power.ignore_children)
			break;
	}

	return NOTIFY_DONE;
}

289
/**
290
 * genpd_poweroff - Remove power from a given PM domain.
291
 * @genpd: PM domain to power down.
292
 * @is_async: PM domain is powered down from a scheduled work
293
294
 *
 * If all of the @genpd's devices have been suspended and all of its subdomains
295
 * have been powered down, remove power from @genpd.
296
 */
297
static int genpd_poweroff(struct generic_pm_domain *genpd, bool is_async)
298
{
299
	struct pm_domain_data *pdd;
300
	struct gpd_link *link;
301
	unsigned int not_suspended = 0;
302

303
304
305
	/*
	 * Do not try to power off the domain in the following situations:
	 * (1) The domain is already in the "power off" state.
306
	 * (2) System suspend is in progress.
307
	 */
308
	if (genpd->status == GPD_STATE_POWER_OFF
309
	    || genpd->prepared_count > 0)
310
311
		return 0;

312
	if (atomic_read(&genpd->sd_count) > 0)
313
314
		return -EBUSY;

315
316
317
318
319
320
321
322
323
	list_for_each_entry(pdd, &genpd->dev_list, list_node) {
		enum pm_qos_flags_status stat;

		stat = dev_pm_qos_flags(pdd->dev,
					PM_QOS_FLAG_NO_POWER_OFF
						| PM_QOS_FLAG_REMOTE_WAKEUP);
		if (stat > PM_QOS_FLAGS_NONE)
			return -EBUSY;

324
		if (!pm_runtime_suspended(pdd->dev) || pdd->dev->power.irq_safe)
325
			not_suspended++;
326
	}
327

328
	if (not_suspended > 1 || (not_suspended == 1 && is_async))
329
330
331
332
333
334
335
		return -EBUSY;

	if (genpd->gov && genpd->gov->power_down_ok) {
		if (!genpd->gov->power_down_ok(&genpd->domain))
			return -EAGAIN;
	}

336
	if (genpd->power_off) {
337
338
339
340
		int ret;

		if (atomic_read(&genpd->sd_count) > 0)
			return -EBUSY;
341

342
		/*
343
		 * If sd_count > 0 at this point, one of the subdomains hasn't
344
345
		 * managed to call genpd_poweron() for the master yet after
		 * incrementing it.  In that case genpd_poweron() will wait
346
		 * for us to drop the lock, so we can call .power_off() and let
347
		 * the genpd_poweron() restore power for us (this shouldn't
348
349
		 * happen very often).
		 */
350
		ret = genpd_power_off(genpd, true);
351
352
		if (ret)
			return ret;
353
	}
354

355
	genpd->status = GPD_STATE_POWER_OFF;
356

357
358
359
360
	list_for_each_entry(link, &genpd->slave_links, slave_node) {
		genpd_sd_counter_dec(link->master);
		genpd_queue_power_off_work(link->master);
	}
361

362
	return 0;
363
364
365
366
367
368
369
370
371
372
373
374
}

/**
 * genpd_power_off_work_fn - Power off PM domain whose subdomain count is 0.
 * @work: Work structure used for scheduling the execution of this function.
 */
static void genpd_power_off_work_fn(struct work_struct *work)
{
	struct generic_pm_domain *genpd;

	genpd = container_of(work, struct generic_pm_domain, power_off_work);

375
	mutex_lock(&genpd->lock);
376
	genpd_poweroff(genpd, true);
377
	mutex_unlock(&genpd->lock);
378
379
380
381
382
383
384
385
386
387
388
389
390
}

/**
 * pm_genpd_runtime_suspend - Suspend a device belonging to I/O PM domain.
 * @dev: Device to suspend.
 *
 * Carry out a runtime suspend of a device under the assumption that its
 * pm_domain field points to the domain member of an object of type
 * struct generic_pm_domain representing a PM domain consisting of I/O devices.
 */
static int pm_genpd_runtime_suspend(struct device *dev)
{
	struct generic_pm_domain *genpd;
391
	bool (*stop_ok)(struct device *__dev);
392
393
394
	struct gpd_timing_data *td = &dev_gpd_data(dev)->td;
	ktime_t time_start;
	s64 elapsed_ns;
395
	int ret;
396
397
398

	dev_dbg(dev, "%s()\n", __func__);

399
400
	genpd = dev_to_genpd(dev);
	if (IS_ERR(genpd))
401
402
		return -EINVAL;

403
404
405
406
	stop_ok = genpd->gov ? genpd->gov->stop_ok : NULL;
	if (stop_ok && !stop_ok(dev))
		return -EBUSY;

407
408
409
	/* Measure suspend latency. */
	time_start = ktime_get();

410
	ret = genpd_save_dev(genpd, dev);
411
412
	if (ret)
		return ret;
413

414
	ret = genpd_stop_dev(genpd, dev);
415
	if (ret) {
416
		genpd_restore_dev(genpd, dev);
417
418
419
		return ret;
	}

420
421
422
423
424
425
426
427
428
429
	/* Update suspend latency value if the measured time exceeds it. */
	elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start));
	if (elapsed_ns > td->suspend_latency_ns) {
		td->suspend_latency_ns = elapsed_ns;
		dev_dbg(dev, "suspend latency exceeded, %lld ns\n",
			elapsed_ns);
		genpd->max_off_time_changed = true;
		td->constraint_changed = true;
	}

430
431
432
433
434
435
436
	/*
	 * If power.irq_safe is set, this routine will be run with interrupts
	 * off, so it can't use mutexes.
	 */
	if (dev->power.irq_safe)
		return 0;

437
	mutex_lock(&genpd->lock);
438
	genpd_poweroff(genpd, false);
439
	mutex_unlock(&genpd->lock);
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454

	return 0;
}

/**
 * pm_genpd_runtime_resume - Resume a device belonging to I/O PM domain.
 * @dev: Device to resume.
 *
 * Carry out a runtime resume of a device under the assumption that its
 * pm_domain field points to the domain member of an object of type
 * struct generic_pm_domain representing a PM domain consisting of I/O devices.
 */
static int pm_genpd_runtime_resume(struct device *dev)
{
	struct generic_pm_domain *genpd;
455
456
457
	struct gpd_timing_data *td = &dev_gpd_data(dev)->td;
	ktime_t time_start;
	s64 elapsed_ns;
458
	int ret;
459
	bool timed = true;
460
461
462

	dev_dbg(dev, "%s()\n", __func__);

463
464
	genpd = dev_to_genpd(dev);
	if (IS_ERR(genpd))
465
466
		return -EINVAL;

467
	/* If power.irq_safe, the PM domain is never powered off. */
468
469
470
471
	if (dev->power.irq_safe) {
		timed = false;
		goto out;
	}
472

473
	mutex_lock(&genpd->lock);
474
	ret = __genpd_poweron(genpd);
475
	mutex_unlock(&genpd->lock);
476

477
478
	if (ret)
		return ret;
479

480
 out:
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
	/* Measure resume latency. */
	if (timed)
		time_start = ktime_get();

	genpd_start_dev(genpd, dev);
	genpd_restore_dev(genpd, dev);

	/* Update resume latency value if the measured time exceeds it. */
	if (timed) {
		elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start));
		if (elapsed_ns > td->resume_latency_ns) {
			td->resume_latency_ns = elapsed_ns;
			dev_dbg(dev, "resume latency exceeded, %lld ns\n",
				elapsed_ns);
			genpd->max_off_time_changed = true;
			td->constraint_changed = true;
		}
	}
499

500
501
502
	return 0;
}

503
504
505
506
507
508
509
510
static bool pd_ignore_unused;
static int __init pd_ignore_unused_setup(char *__unused)
{
	pd_ignore_unused = true;
	return 1;
}
__setup("pd_ignore_unused", pd_ignore_unused_setup);

511
/**
512
 * genpd_poweroff_unused - Power off all PM domains with no devices in use.
513
 */
514
static int __init genpd_poweroff_unused(void)
515
516
517
{
	struct generic_pm_domain *genpd;

518
519
	if (pd_ignore_unused) {
		pr_warn("genpd: Not disabling unused power domains\n");
520
		return 0;
521
522
	}

523
524
525
526
527
528
529
	mutex_lock(&gpd_list_lock);

	list_for_each_entry(genpd, &gpd_list, gpd_list_node)
		genpd_queue_power_off_work(genpd);

	mutex_unlock(&gpd_list_lock);

530
531
532
533
	return 0;
}
late_initcall(genpd_poweroff_unused);

534
535
#ifdef CONFIG_PM_SLEEP

536
537
538
539
/**
 * pm_genpd_present - Check if the given PM domain has been initialized.
 * @genpd: PM domain to check.
 */
540
static bool pm_genpd_present(const struct generic_pm_domain *genpd)
541
{
542
	const struct generic_pm_domain *gpd;
543
544
545
546
547
548
549
550
551
552
553

	if (IS_ERR_OR_NULL(genpd))
		return false;

	list_for_each_entry(gpd, &gpd_list, gpd_list_node)
		if (gpd == genpd)
			return true;

	return false;
}

554
555
556
557
558
559
static bool genpd_dev_active_wakeup(struct generic_pm_domain *genpd,
				    struct device *dev)
{
	return GENPD_DEV_CALLBACK(genpd, bool, active_wakeup, dev);
}

560
/**
561
 * pm_genpd_sync_poweroff - Synchronously power off a PM domain and its masters.
562
 * @genpd: PM domain to power off, if possible.
563
 * @timed: True if latency measurements are allowed.
564
565
 *
 * Check if the given PM domain can be powered off (during system suspend or
566
 * hibernation) and do that if so.  Also, in that case propagate to its masters.
567
 *
568
569
570
571
 * This function is only called in "noirq" and "syscore" stages of system power
 * transitions, so it need not acquire locks (all of the "noirq" callbacks are
 * executed sequentially, so it is guaranteed that it will never run twice in
 * parallel).
572
 */
573
574
static void pm_genpd_sync_poweroff(struct generic_pm_domain *genpd,
				   bool timed)
575
{
576
	struct gpd_link *link;
577

578
	if (genpd->status == GPD_STATE_POWER_OFF)
579
580
		return;

581
582
	if (genpd->suspended_count != genpd->device_count
	    || atomic_read(&genpd->sd_count) > 0)
583
584
		return;

585
	genpd_power_off(genpd, timed);
586

587
	genpd->status = GPD_STATE_POWER_OFF;
588
589
590

	list_for_each_entry(link, &genpd->slave_links, slave_node) {
		genpd_sd_counter_dec(link->master);
591
		pm_genpd_sync_poweroff(link->master, timed);
592
593
594
	}
}

595
596
597
/**
 * pm_genpd_sync_poweron - Synchronously power on a PM domain and its masters.
 * @genpd: PM domain to power on.
598
 * @timed: True if latency measurements are allowed.
599
 *
600
601
602
603
 * This function is only called in "noirq" and "syscore" stages of system power
 * transitions, so it need not acquire locks (all of the "noirq" callbacks are
 * executed sequentially, so it is guaranteed that it will never run twice in
 * parallel).
604
 */
605
606
static void pm_genpd_sync_poweron(struct generic_pm_domain *genpd,
				  bool timed)
607
608
609
{
	struct gpd_link *link;

610
	if (genpd->status == GPD_STATE_ACTIVE)
611
612
613
		return;

	list_for_each_entry(link, &genpd->slave_links, slave_node) {
614
		pm_genpd_sync_poweron(link->master, timed);
615
616
617
		genpd_sd_counter_inc(link->master);
	}

618
	genpd_power_on(genpd, timed);
619
620
621
622

	genpd->status = GPD_STATE_ACTIVE;
}

623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
/**
 * resume_needed - Check whether to resume a device before system suspend.
 * @dev: Device to check.
 * @genpd: PM domain the device belongs to.
 *
 * There are two cases in which a device that can wake up the system from sleep
 * states should be resumed by pm_genpd_prepare(): (1) if the device is enabled
 * to wake up the system and it has to remain active for this purpose while the
 * system is in the sleep state and (2) if the device is not enabled to wake up
 * the system from sleep states and it generally doesn't generate wakeup signals
 * by itself (those signals are generated on its behalf by other parts of the
 * system).  In the latter case it may be necessary to reconfigure the device's
 * wakeup settings during system suspend, because it may have been set up to
 * signal remote wakeup from the system's working state as needed by runtime PM.
 * Return 'true' in either of the above cases.
 */
static bool resume_needed(struct device *dev, struct generic_pm_domain *genpd)
{
	bool active_wakeup;

	if (!device_can_wakeup(dev))
		return false;

646
	active_wakeup = genpd_dev_active_wakeup(genpd, dev);
647
648
649
	return device_may_wakeup(dev) ? active_wakeup : !active_wakeup;
}

650
651
652
653
654
655
656
657
658
659
660
661
/**
 * pm_genpd_prepare - Start power transition of a device in a PM domain.
 * @dev: Device to start the transition of.
 *
 * Start a power transition of a device (during a system-wide power transition)
 * under the assumption that its pm_domain field points to the domain member of
 * an object of type struct generic_pm_domain representing a PM domain
 * consisting of I/O devices.
 */
static int pm_genpd_prepare(struct device *dev)
{
	struct generic_pm_domain *genpd;
662
	int ret;
663
664
665
666
667
668
669

	dev_dbg(dev, "%s()\n", __func__);

	genpd = dev_to_genpd(dev);
	if (IS_ERR(genpd))
		return -EINVAL;

670
671
672
673
674
675
676
677
678
679
	/*
	 * If a wakeup request is pending for the device, it should be woken up
	 * at this point and a system wakeup event should be reported if it's
	 * set up to wake up the system from sleep states.
	 */
	pm_runtime_get_noresume(dev);
	if (pm_runtime_barrier(dev) && device_may_wakeup(dev))
		pm_wakeup_event(dev, 0);

	if (pm_wakeup_pending()) {
680
		pm_runtime_put(dev);
681
682
683
		return -EBUSY;
	}

684
685
686
	if (resume_needed(dev, genpd))
		pm_runtime_resume(dev);

687
	mutex_lock(&genpd->lock);
688

689
690
	if (genpd->prepared_count++ == 0) {
		genpd->suspended_count = 0;
691
		genpd->suspend_power_off = genpd->status == GPD_STATE_POWER_OFF;
692
	}
693

694
	mutex_unlock(&genpd->lock);
695
696

	if (genpd->suspend_power_off) {
697
		pm_runtime_put_noidle(dev);
698
699
700
701
		return 0;
	}

	/*
702
	 * The PM domain must be in the GPD_STATE_ACTIVE state at this point,
703
	 * so genpd_poweron() will return immediately, but if the device
704
	 * is suspended (e.g. it's been stopped by genpd_stop_dev()), we need
705
	 * to make it operational.
706
	 */
707
	pm_runtime_resume(dev);
708
709
	__pm_runtime_disable(dev, false);

710
711
712
713
714
715
716
717
	ret = pm_generic_prepare(dev);
	if (ret) {
		mutex_lock(&genpd->lock);

		if (--genpd->prepared_count == 0)
			genpd->suspend_power_off = false;

		mutex_unlock(&genpd->lock);
718
		pm_runtime_enable(dev);
719
	}
720

721
	pm_runtime_put(dev);
722
	return ret;
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
}

/**
 * pm_genpd_suspend - Suspend a device belonging to an I/O PM domain.
 * @dev: Device to suspend.
 *
 * Suspend a device under the assumption that its pm_domain field points to the
 * domain member of an object of type struct generic_pm_domain representing
 * a PM domain consisting of I/O devices.
 */
static int pm_genpd_suspend(struct device *dev)
{
	struct generic_pm_domain *genpd;

	dev_dbg(dev, "%s()\n", __func__);

	genpd = dev_to_genpd(dev);
	if (IS_ERR(genpd))
		return -EINVAL;

743
	return genpd->suspend_power_off ? 0 : pm_generic_suspend(dev);
744
745
746
}

/**
747
 * pm_genpd_suspend_late - Late suspend of a device from an I/O PM domain.
748
749
750
751
752
753
 * @dev: Device to suspend.
 *
 * Carry out a late suspend of a device under the assumption that its
 * pm_domain field points to the domain member of an object of type
 * struct generic_pm_domain representing a PM domain consisting of I/O devices.
 */
754
static int pm_genpd_suspend_late(struct device *dev)
755
756
757
758
759
760
761
762
763
{
	struct generic_pm_domain *genpd;

	dev_dbg(dev, "%s()\n", __func__);

	genpd = dev_to_genpd(dev);
	if (IS_ERR(genpd))
		return -EINVAL;

764
	return genpd->suspend_power_off ? 0 : pm_generic_suspend_late(dev);
765
}
766

767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
/**
 * pm_genpd_suspend_noirq - Completion of suspend of device in an I/O PM domain.
 * @dev: Device to suspend.
 *
 * Stop the device and remove power from the domain if all devices in it have
 * been stopped.
 */
static int pm_genpd_suspend_noirq(struct device *dev)
{
	struct generic_pm_domain *genpd;

	dev_dbg(dev, "%s()\n", __func__);

	genpd = dev_to_genpd(dev);
	if (IS_ERR(genpd))
		return -EINVAL;
783

784
	if (genpd->suspend_power_off
785
	    || (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev)))
786
787
		return 0;

788
	genpd_stop_dev(genpd, dev);
789
790
791
792
793
794
795

	/*
	 * Since all of the "noirq" callbacks are executed sequentially, it is
	 * guaranteed that this function will never run twice in parallel for
	 * the same PM domain, so it is not necessary to use locking here.
	 */
	genpd->suspended_count++;
796
	pm_genpd_sync_poweroff(genpd, true);
797
798
799
800
801

	return 0;
}

/**
802
 * pm_genpd_resume_noirq - Start of resume of device in an I/O PM domain.
803
804
 * @dev: Device to resume.
 *
805
 * Restore power to the device's PM domain, if necessary, and start the device.
806
807
808
809
810
811
812
813
814
815
816
 */
static int pm_genpd_resume_noirq(struct device *dev)
{
	struct generic_pm_domain *genpd;

	dev_dbg(dev, "%s()\n", __func__);

	genpd = dev_to_genpd(dev);
	if (IS_ERR(genpd))
		return -EINVAL;

817
	if (genpd->suspend_power_off
818
	    || (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev)))
819
820
821
822
823
824
825
		return 0;

	/*
	 * Since all of the "noirq" callbacks are executed sequentially, it is
	 * guaranteed that this function will never run twice in parallel for
	 * the same PM domain, so it is not necessary to use locking here.
	 */
826
	pm_genpd_sync_poweron(genpd, true);
827
828
	genpd->suspended_count--;

829
	return genpd_start_dev(genpd, dev);
830
831
832
}

/**
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
 * pm_genpd_resume_early - Early resume of a device in an I/O PM domain.
 * @dev: Device to resume.
 *
 * Carry out an early resume of a device under the assumption that its
 * pm_domain field points to the domain member of an object of type
 * struct generic_pm_domain representing a power domain consisting of I/O
 * devices.
 */
static int pm_genpd_resume_early(struct device *dev)
{
	struct generic_pm_domain *genpd;

	dev_dbg(dev, "%s()\n", __func__);

	genpd = dev_to_genpd(dev);
	if (IS_ERR(genpd))
		return -EINVAL;

851
	return genpd->suspend_power_off ? 0 : pm_generic_resume_early(dev);
852
853
854
855
}

/**
 * pm_genpd_resume - Resume of device in an I/O PM domain.
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
 * @dev: Device to resume.
 *
 * Resume a device under the assumption that its pm_domain field points to the
 * domain member of an object of type struct generic_pm_domain representing
 * a power domain consisting of I/O devices.
 */
static int pm_genpd_resume(struct device *dev)
{
	struct generic_pm_domain *genpd;

	dev_dbg(dev, "%s()\n", __func__);

	genpd = dev_to_genpd(dev);
	if (IS_ERR(genpd))
		return -EINVAL;

872
	return genpd->suspend_power_off ? 0 : pm_generic_resume(dev);
873
874
875
}

/**
876
 * pm_genpd_freeze - Freezing a device in an I/O PM domain.
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
 * @dev: Device to freeze.
 *
 * Freeze a device under the assumption that its pm_domain field points to the
 * domain member of an object of type struct generic_pm_domain representing
 * a power domain consisting of I/O devices.
 */
static int pm_genpd_freeze(struct device *dev)
{
	struct generic_pm_domain *genpd;

	dev_dbg(dev, "%s()\n", __func__);

	genpd = dev_to_genpd(dev);
	if (IS_ERR(genpd))
		return -EINVAL;

893
	return genpd->suspend_power_off ? 0 : pm_generic_freeze(dev);
894
895
896
}

/**
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
 * pm_genpd_freeze_late - Late freeze of a device in an I/O PM domain.
 * @dev: Device to freeze.
 *
 * Carry out a late freeze of a device under the assumption that its
 * pm_domain field points to the domain member of an object of type
 * struct generic_pm_domain representing a power domain consisting of I/O
 * devices.
 */
static int pm_genpd_freeze_late(struct device *dev)
{
	struct generic_pm_domain *genpd;

	dev_dbg(dev, "%s()\n", __func__);

	genpd = dev_to_genpd(dev);
	if (IS_ERR(genpd))
		return -EINVAL;

915
	return genpd->suspend_power_off ? 0 : pm_generic_freeze_late(dev);
916
917
918
919
}

/**
 * pm_genpd_freeze_noirq - Completion of freezing a device in an I/O PM domain.
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
 * @dev: Device to freeze.
 *
 * Carry out a late freeze of a device under the assumption that its
 * pm_domain field points to the domain member of an object of type
 * struct generic_pm_domain representing a power domain consisting of I/O
 * devices.
 */
static int pm_genpd_freeze_noirq(struct device *dev)
{
	struct generic_pm_domain *genpd;

	dev_dbg(dev, "%s()\n", __func__);

	genpd = dev_to_genpd(dev);
	if (IS_ERR(genpd))
		return -EINVAL;

937
	return genpd->suspend_power_off ? 0 : genpd_stop_dev(genpd, dev);
938
}
939

940
941
942
943
944
945
946
947
948
949
/**
 * pm_genpd_thaw_noirq - Early thaw of device in an I/O PM domain.
 * @dev: Device to thaw.
 *
 * Start the device, unless power has been removed from the domain already
 * before the system transition.
 */
static int pm_genpd_thaw_noirq(struct device *dev)
{
	struct generic_pm_domain *genpd;
950

951
	dev_dbg(dev, "%s()\n", __func__);
952

953
954
955
956
	genpd = dev_to_genpd(dev);
	if (IS_ERR(genpd))
		return -EINVAL;

957
	return genpd->suspend_power_off ?
958
		0 : genpd_start_dev(genpd, dev);
959
960
961
}

/**
962
 * pm_genpd_thaw_early - Early thaw of device in an I/O PM domain.
963
964
965
966
967
968
969
 * @dev: Device to thaw.
 *
 * Carry out an early thaw of a device under the assumption that its
 * pm_domain field points to the domain member of an object of type
 * struct generic_pm_domain representing a power domain consisting of I/O
 * devices.
 */
970
static int pm_genpd_thaw_early(struct device *dev)
971
972
973
974
975
976
977
978
979
{
	struct generic_pm_domain *genpd;

	dev_dbg(dev, "%s()\n", __func__);

	genpd = dev_to_genpd(dev);
	if (IS_ERR(genpd))
		return -EINVAL;

980
	return genpd->suspend_power_off ? 0 : pm_generic_thaw_early(dev);
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
}

/**
 * pm_genpd_thaw - Thaw a device belonging to an I/O power domain.
 * @dev: Device to thaw.
 *
 * Thaw a device under the assumption that its pm_domain field points to the
 * domain member of an object of type struct generic_pm_domain representing
 * a power domain consisting of I/O devices.
 */
static int pm_genpd_thaw(struct device *dev)
{
	struct generic_pm_domain *genpd;

	dev_dbg(dev, "%s()\n", __func__);

	genpd = dev_to_genpd(dev);
	if (IS_ERR(genpd))
		return -EINVAL;

1001
	return genpd->suspend_power_off ? 0 : pm_generic_thaw(dev);
1002
1003
1004
}

/**
1005
 * pm_genpd_restore_noirq - Start of restore of device in an I/O PM domain.
1006
1007
 * @dev: Device to resume.
 *
1008
1009
 * Make sure the domain will be in the same power state as before the
 * hibernation the system is resuming from and start the device if necessary.
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
 */
static int pm_genpd_restore_noirq(struct device *dev)
{
	struct generic_pm_domain *genpd;

	dev_dbg(dev, "%s()\n", __func__);

	genpd = dev_to_genpd(dev);
	if (IS_ERR(genpd))
		return -EINVAL;

	/*
	 * Since all of the "noirq" callbacks are executed sequentially, it is
	 * guaranteed that this function will never run twice in parallel for
	 * the same PM domain, so it is not necessary to use locking here.
1025
1026
1027
	 *
	 * At this point suspended_count == 0 means we are being run for the
	 * first time for the given domain in the present cycle.
1028
	 */
1029
	if (genpd->suspended_count++ == 0) {
1030
		/*
1031
		 * The boot kernel might put the domain into arbitrary state,
1032
1033
		 * so make it appear as powered off to pm_genpd_sync_poweron(),
		 * so that it tries to power it on in case it was really off.
1034
		 */
1035
1036
1037
1038
1039
1040
		genpd->status = GPD_STATE_POWER_OFF;
		if (genpd->suspend_power_off) {
			/*
			 * If the domain was off before the hibernation, make
			 * sure it will be off going forward.
			 */
1041
			genpd_power_off(genpd, true);
1042
1043
1044

			return 0;
		}
1045
1046
	}

1047
1048
1049
	if (genpd->suspend_power_off)
		return 0;

1050
	pm_genpd_sync_poweron(genpd, true);
1051

1052
	return genpd_start_dev(genpd, dev);
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
}

/**
 * pm_genpd_complete - Complete power transition of a device in a power domain.
 * @dev: Device to complete the transition of.
 *
 * Complete a power transition of a device (during a system-wide power
 * transition) under the assumption that its pm_domain field points to the
 * domain member of an object of type struct generic_pm_domain representing
 * a power domain consisting of I/O devices.
 */
static void pm_genpd_complete(struct device *dev)
{
	struct generic_pm_domain *genpd;
	bool run_complete;

	dev_dbg(dev, "%s()\n", __func__);

	genpd = dev_to_genpd(dev);
	if (IS_ERR(genpd))
		return;

	mutex_lock(&genpd->lock);

	run_complete = !genpd->suspend_power_off;
	if (--genpd->prepared_count == 0)
		genpd->suspend_power_off = false;

	mutex_unlock(&genpd->lock);

	if (run_complete) {
		pm_generic_complete(dev);
1085
		pm_runtime_set_active(dev);
1086
		pm_runtime_enable(dev);
1087
		pm_request_idle(dev);
1088
1089
1090
	}
}

1091
/**
1092
 * genpd_syscore_switch - Switch power during system core suspend or resume.
1093
1094
1095
1096
1097
 * @dev: Device that normally is marked as "always on" to switch power for.
 *
 * This routine may only be called during the system core (syscore) suspend or
 * resume phase for devices whose "always on" flags are set.
 */
1098
static void genpd_syscore_switch(struct device *dev, bool suspend)
1099
1100
1101
1102
1103
1104
1105
1106
1107
{
	struct generic_pm_domain *genpd;

	genpd = dev_to_genpd(dev);
	if (!pm_genpd_present(genpd))
		return;

	if (suspend) {
		genpd->suspended_count++;
1108
		pm_genpd_sync_poweroff(genpd, false);
1109
	} else {
1110
		pm_genpd_sync_poweron(genpd, false);
1111
1112
1113
		genpd->suspended_count--;
	}
}
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125

void pm_genpd_syscore_poweroff(struct device *dev)
{
	genpd_syscore_switch(dev, true);
}
EXPORT_SYMBOL_GPL(pm_genpd_syscore_poweroff);

void pm_genpd_syscore_poweron(struct device *dev)
{
	genpd_syscore_switch(dev, false);
}
EXPORT_SYMBOL_GPL(pm_genpd_syscore_poweron);
1126

1127
#else /* !CONFIG_PM_SLEEP */
1128
1129
1130

#define pm_genpd_prepare		NULL
#define pm_genpd_suspend		NULL
1131
#define pm_genpd_suspend_late		NULL
1132
#define pm_genpd_suspend_noirq		NULL
1133
#define pm_genpd_resume_early		NULL
1134
1135
1136
#define pm_genpd_resume_noirq		NULL
#define pm_genpd_resume			NULL
#define pm_genpd_freeze			NULL
1137
#define pm_genpd_freeze_late		NULL
1138
#define pm_genpd_freeze_noirq		NULL
1139
#define pm_genpd_thaw_early		NULL
1140
1141
1142
1143
1144
1145
1146
#define pm_genpd_thaw_noirq		NULL
#define pm_genpd_thaw			NULL
#define pm_genpd_restore_noirq		NULL
#define pm_genpd_complete		NULL

#endif /* CONFIG_PM_SLEEP */

1147
1148
1149
static struct generic_pm_domain_data *genpd_alloc_dev_data(struct device *dev,
					struct generic_pm_domain *genpd,
					struct gpd_timing_data *td)
1150
1151
{
	struct generic_pm_domain_data *gpd_data;
1152
1153
1154
1155
1156
	int ret;

	ret = dev_pm_get_subsys_data(dev);
	if (ret)
		return ERR_PTR(ret);
1157
1158

	gpd_data = kzalloc(sizeof(*gpd_data), GFP_KERNEL);
1159
1160
1161
1162
	if (!gpd_data) {
		ret = -ENOMEM;
		goto err_put;
	}
1163

1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
	if (td)
		gpd_data->td = *td;

	gpd_data->base.dev = dev;
	gpd_data->td.constraint_changed = true;
	gpd_data->td.effective_constraint_ns = -1;
	gpd_data->nb.notifier_call = genpd_dev_pm_qos_notifier;

	spin_lock_irq(&dev->power.lock);

	if (dev->power.subsys_data->domain_data) {
		ret = -EINVAL;
		goto err_free;
	}

	dev->power.subsys_data->domain_data = &gpd_data->base;
	dev->pm_domain = &genpd->domain;

	spin_unlock_irq(&dev->power.lock);

1184
	return gpd_data;
1185

1186
1187
1188
 err_free:
	spin_unlock_irq(&dev->power.lock);
	kfree(gpd_data);
1189
1190
1191
 err_put:
	dev_pm_put_subsys_data(dev);
	return ERR_PTR(ret);
1192
1193
}

1194
1195
static void genpd_free_dev_data(struct device *dev,
				struct generic_pm_domain_data *gpd_data)
1196
{
1197
1198
1199
1200
1201
1202
1203
	spin_lock_irq(&dev->power.lock);

	dev->pm_domain = NULL;
	dev->power.subsys_data->domain_data = NULL;

	spin_unlock_irq(&dev->power.lock);

1204
	kfree(gpd_data);
1205
	dev_pm_put_subsys_data(dev);
1206
1207
}

1208
/**
1209
 * __pm_genpd_add_device - Add a device to an I/O PM domain.
1210
1211
 * @genpd: PM domain to add the device to.
 * @dev: Device to be added.
1212
 * @td: Set of PM QoS timing parameters to attach to the device.
1213
 */
1214
1215
int __pm_genpd_add_device(struct generic_pm_domain *genpd, struct device *dev,
			  struct gpd_timing_data *td)
1216
{