async.c 10 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51
/*
 * async.c: Asynchronous function calls for boot performance
 *
 * (C) Copyright 2009 Intel Corporation
 * Author: Arjan van de Ven <arjan@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; version 2
 * of the License.
 */


/*

Goals and Theory of Operation

The primary goal of this feature is to reduce the kernel boot time,
by doing various independent hardware delays and discovery operations
decoupled and not strictly serialized.

More specifically, the asynchronous function call concept allows
certain operations (primarily during system boot) to happen
asynchronously, out of order, while these operations still
have their externally visible parts happen sequentially and in-order.
(not unlike how out-of-order CPUs retire their instructions in order)

Key to the asynchronous function call implementation is the concept of
a "sequence cookie" (which, although it has an abstracted type, can be
thought of as a monotonically incrementing number).

The async core will assign each scheduled event such a sequence cookie and
pass this to the called functions.

The asynchronously called function should before doing a globally visible
operation, such as registering device numbers, call the
async_synchronize_cookie() function and pass in its own cookie. The
async_synchronize_cookie() function will make sure that all asynchronous
operations that were scheduled prior to the operation corresponding with the
cookie have completed.

Subsystem/driver initialization code that scheduled asynchronous probe
functions, but which shares global resources with other drivers/subsystems
that do not use the asynchronous call feature, need to do a full
synchronization with the async_synchronize_full() function, before returning
from their init function. This is to maintain strict ordering between the
asynchronous and synchronous parts of the kernel.

*/

#include <linux/async.h>
52 53
#include <linux/atomic.h>
#include <linux/ktime.h>
54
#include <linux/export.h>
55 56
#include <linux/wait.h>
#include <linux/sched.h>
57
#include <linux/slab.h>
58

59 60
#include "workqueue_internal.h"

61 62
static async_cookie_t next_cookie = 1;

63 64
#define MAX_WORK		32768
#define ASYNC_COOKIE_MAX	ULLONG_MAX	/* infinity cookie */
65

66
static LIST_HEAD(async_global_pending);	/* pending from all registered doms */
67
static ASYNC_DOMAIN(async_dfl_domain);
68 69 70
static DEFINE_SPINLOCK(async_lock);

struct async_entry {
71 72
	struct list_head	domain_list;
	struct list_head	global_list;
73 74
	struct work_struct	work;
	async_cookie_t		cookie;
75
	async_func_t		func;
76
	void			*data;
77
	struct async_domain	*domain;
78 79 80 81 82 83
};

static DECLARE_WAIT_QUEUE_HEAD(async_done);

static atomic_t entry_count;

84
static async_cookie_t lowest_in_progress(struct async_domain *domain)
85
{
86
	struct async_entry *first = NULL;
87
	async_cookie_t ret = ASYNC_COOKIE_MAX;
88 89 90
	unsigned long flags;

	spin_lock_irqsave(&async_lock, flags);
91

92 93 94 95 96 97 98 99 100
	if (domain) {
		if (!list_empty(&domain->pending))
			first = list_first_entry(&domain->pending,
					struct async_entry, domain_list);
	} else {
		if (!list_empty(&async_global_pending))
			first = list_first_entry(&async_global_pending,
					struct async_entry, global_list);
	}
101

102 103
	if (first)
		ret = first->cookie;
104

105 106 107
	spin_unlock_irqrestore(&async_lock, flags);
	return ret;
}
108

109 110 111
/*
 * pick the first pending entry and run it
 */
112
static void async_run_entry_fn(struct work_struct *work)
113
{
114 115
	struct async_entry *entry =
		container_of(work, struct async_entry, work);
116
	unsigned long flags;
117
	ktime_t uninitialized_var(calltime), delta, rettime;
118

119
	/* 1) run (and print duration) */
120
	if (initcall_debug && system_state < SYSTEM_RUNNING) {
121
		pr_debug("calling  %lli_%pF @ %i\n",
122
			(long long)entry->cookie,
123
			entry->func, task_pid_nr(current));
124 125 126
		calltime = ktime_get();
	}
	entry->func(entry->data, entry->cookie);
127
	if (initcall_debug && system_state < SYSTEM_RUNNING) {
128 129
		rettime = ktime_get();
		delta = ktime_sub(rettime, calltime);
130
		pr_debug("initcall %lli_%pF returned 0 after %lld usecs\n",
131 132 133
			(long long)entry->cookie,
			entry->func,
			(long long)ktime_to_ns(delta) >> 10);
134 135
	}

136
	/* 2) remove self from the pending queues */
137
	spin_lock_irqsave(&async_lock, flags);
138 139
	list_del_init(&entry->domain_list);
	list_del_init(&entry->global_list);
140

141
	/* 3) free the entry */
142 143 144 145 146
	kfree(entry);
	atomic_dec(&entry_count);

	spin_unlock_irqrestore(&async_lock, flags);

147
	/* 4) wake up any waiters */
148 149 150
	wake_up(&async_done);
}

151
static async_cookie_t __async_schedule(async_func_t func, void *data, struct async_domain *domain)
152 153 154 155 156 157 158 159 160 161 162 163
{
	struct async_entry *entry;
	unsigned long flags;
	async_cookie_t newcookie;

	/* allow irq-off callers */
	entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC);

	/*
	 * If we're out of memory or if there's too much work
	 * pending already, we execute synchronously.
	 */
164
	if (!entry || atomic_read(&entry_count) > MAX_WORK) {
165 166 167 168 169 170
		kfree(entry);
		spin_lock_irqsave(&async_lock, flags);
		newcookie = next_cookie++;
		spin_unlock_irqrestore(&async_lock, flags);

		/* low on memory.. run synchronously */
171
		func(data, newcookie);
172 173
		return newcookie;
	}
174 175
	INIT_LIST_HEAD(&entry->domain_list);
	INIT_LIST_HEAD(&entry->global_list);
176
	INIT_WORK(&entry->work, async_run_entry_fn);
177
	entry->func = func;
178
	entry->data = data;
179
	entry->domain = domain;
180 181

	spin_lock_irqsave(&async_lock, flags);
182 183

	/* allocate cookie and queue */
184
	newcookie = entry->cookie = next_cookie++;
185 186 187 188 189

	list_add_tail(&entry->domain_list, &domain->pending);
	if (domain->registered)
		list_add_tail(&entry->global_list, &async_global_pending);

190 191
	atomic_inc(&entry_count);
	spin_unlock_irqrestore(&async_lock, flags);
192

193 194 195
	/* mark that this task has queued an async job, used by module init */
	current->flags |= PF_USED_ASYNC;

196 197 198
	/* schedule for execution */
	queue_work(system_unbound_wq, &entry->work);

199 200 201
	return newcookie;
}

Cornelia Huck's avatar
Cornelia Huck committed
202 203
/**
 * async_schedule - schedule a function for asynchronous execution
204
 * @func: function to execute asynchronously
Cornelia Huck's avatar
Cornelia Huck committed
205 206 207 208 209
 * @data: data pointer to pass to the function
 *
 * Returns an async_cookie_t that may be used for checkpointing later.
 * Note: This function may be called from atomic or non-atomic contexts.
 */
210
async_cookie_t async_schedule(async_func_t func, void *data)
211
{
212
	return __async_schedule(func, data, &async_dfl_domain);
213 214 215
}
EXPORT_SYMBOL_GPL(async_schedule);

Cornelia Huck's avatar
Cornelia Huck committed
216
/**
217
 * async_schedule_domain - schedule a function for asynchronous execution within a certain domain
218
 * @func: function to execute asynchronously
Cornelia Huck's avatar
Cornelia Huck committed
219
 * @data: data pointer to pass to the function
220
 * @domain: the domain
Cornelia Huck's avatar
Cornelia Huck committed
221 222
 *
 * Returns an async_cookie_t that may be used for checkpointing later.
223 224 225 226
 * @domain may be used in the async_synchronize_*_domain() functions to
 * wait within a certain synchronization domain rather than globally.  A
 * synchronization domain is specified via @domain.  Note: This function
 * may be called from atomic or non-atomic contexts.
Cornelia Huck's avatar
Cornelia Huck committed
227
 */
228
async_cookie_t async_schedule_domain(async_func_t func, void *data,
229
				     struct async_domain *domain)
230
{
231
	return __async_schedule(func, data, domain);
232
}
233
EXPORT_SYMBOL_GPL(async_schedule_domain);
234

Cornelia Huck's avatar
Cornelia Huck committed
235 236 237 238 239
/**
 * async_synchronize_full - synchronize all asynchronous function calls
 *
 * This function waits until all asynchronous function calls have been done.
 */
240 241
void async_synchronize_full(void)
{
242
	async_synchronize_full_domain(NULL);
243 244 245
}
EXPORT_SYMBOL_GPL(async_synchronize_full);

246 247 248 249 250 251 252 253 254 255 256 257
/**
 * async_unregister_domain - ensure no more anonymous waiters on this domain
 * @domain: idle domain to flush out of any async_synchronize_full instances
 *
 * async_synchronize_{cookie|full}_domain() are not flushed since callers
 * of these routines should know the lifetime of @domain
 *
 * Prefer ASYNC_DOMAIN_EXCLUSIVE() declarations over flushing
 */
void async_unregister_domain(struct async_domain *domain)
{
	spin_lock_irq(&async_lock);
258
	WARN_ON(!domain->registered || !list_empty(&domain->pending));
259 260 261 262 263
	domain->registered = 0;
	spin_unlock_irq(&async_lock);
}
EXPORT_SYMBOL_GPL(async_unregister_domain);

Cornelia Huck's avatar
Cornelia Huck committed
264
/**
265
 * async_synchronize_full_domain - synchronize all asynchronous function within a certain domain
266
 * @domain: the domain to synchronize
Cornelia Huck's avatar
Cornelia Huck committed
267
 *
268
 * This function waits until all asynchronous function calls for the
269
 * synchronization domain specified by @domain have been done.
Cornelia Huck's avatar
Cornelia Huck committed
270
 */
271
void async_synchronize_full_domain(struct async_domain *domain)
272
{
273
	async_synchronize_cookie_domain(ASYNC_COOKIE_MAX, domain);
274
}
275
EXPORT_SYMBOL_GPL(async_synchronize_full_domain);
276

Cornelia Huck's avatar
Cornelia Huck committed
277
/**
278
 * async_synchronize_cookie_domain - synchronize asynchronous function calls within a certain domain with cookie checkpointing
Cornelia Huck's avatar
Cornelia Huck committed
279
 * @cookie: async_cookie_t to use as checkpoint
280
 * @domain: the domain to synchronize (%NULL for all registered domains)
Cornelia Huck's avatar
Cornelia Huck committed
281
 *
282
 * This function waits until all asynchronous function calls for the
283 284
 * synchronization domain specified by @domain submitted prior to @cookie
 * have been done.
Cornelia Huck's avatar
Cornelia Huck committed
285
 */
286
void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *domain)
287
{
288
	ktime_t uninitialized_var(starttime), delta, endtime;
289

290
	if (initcall_debug && system_state < SYSTEM_RUNNING) {
291
		pr_debug("async_waiting @ %i\n", task_pid_nr(current));
292 293 294
		starttime = ktime_get();
	}

295
	wait_event(async_done, lowest_in_progress(domain) >= cookie);
296

297
	if (initcall_debug && system_state < SYSTEM_RUNNING) {
298 299 300
		endtime = ktime_get();
		delta = ktime_sub(endtime, starttime);

301
		pr_debug("async_continuing @ %i after %lli usec\n",
302 303
			task_pid_nr(current),
			(long long)ktime_to_ns(delta) >> 10);
304 305
	}
}
306
EXPORT_SYMBOL_GPL(async_synchronize_cookie_domain);
307

Cornelia Huck's avatar
Cornelia Huck committed
308 309 310 311 312 313 314
/**
 * async_synchronize_cookie - synchronize asynchronous function calls with cookie checkpointing
 * @cookie: async_cookie_t to use as checkpoint
 *
 * This function waits until all asynchronous function calls prior to @cookie
 * have been done.
 */
315 316
void async_synchronize_cookie(async_cookie_t cookie)
{
317
	async_synchronize_cookie_domain(cookie, &async_dfl_domain);
318 319
}
EXPORT_SYMBOL_GPL(async_synchronize_cookie);
320 321 322 323 324 325 326 327 328 329 330 331

/**
 * current_is_async - is %current an async worker task?
 *
 * Returns %true if %current is an async worker task.
 */
bool current_is_async(void)
{
	struct worker *worker = current_wq_worker();

	return worker && worker->current_func == async_run_entry_fn;
}
332
EXPORT_SYMBOL_GPL(current_is_async);