stop_machine.c 4.55 KB
Newer Older
Rusty Russell's avatar
Rusty Russell committed
1
/* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation.
2 3
 * GPL v2 and any later version.
 */
Linus Torvalds's avatar
Linus Torvalds committed
4 5
#include <linux/cpu.h>
#include <linux/err.h>
6 7 8 9
#include <linux/kthread.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/stop_machine.h>
Linus Torvalds's avatar
Linus Torvalds committed
10
#include <linux/syscalls.h>
11 12
#include <linux/interrupt.h>

Linus Torvalds's avatar
Linus Torvalds committed
13 14 15
#include <asm/atomic.h>
#include <asm/uaccess.h>

Rusty Russell's avatar
Rusty Russell committed
16
/* This controls the threads on each CPU. */
Linus Torvalds's avatar
Linus Torvalds committed
17
enum stopmachine_state {
Rusty Russell's avatar
Rusty Russell committed
18 19 20
	/* Dummy starting state for thread. */
	STOPMACHINE_NONE,
	/* Awaiting everyone to be scheduled. */
Linus Torvalds's avatar
Linus Torvalds committed
21
	STOPMACHINE_PREPARE,
Rusty Russell's avatar
Rusty Russell committed
22
	/* Disable interrupts. */
Linus Torvalds's avatar
Linus Torvalds committed
23
	STOPMACHINE_DISABLE_IRQ,
Rusty Russell's avatar
Rusty Russell committed
24
	/* Run the function */
25
	STOPMACHINE_RUN,
Rusty Russell's avatar
Rusty Russell committed
26
	/* Exit */
Linus Torvalds's avatar
Linus Torvalds committed
27 28
	STOPMACHINE_EXIT,
};
Rusty Russell's avatar
Rusty Russell committed
29
static enum stopmachine_state state;
Linus Torvalds's avatar
Linus Torvalds committed
30

31 32 33
struct stop_machine_data {
	int (*fn)(void *);
	void *data;
Rusty Russell's avatar
Rusty Russell committed
34 35
	int fnret;
};
36

Rusty Russell's avatar
Rusty Russell committed
37 38 39 40
/* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
static unsigned int num_threads;
static atomic_t thread_ack;
static DEFINE_MUTEX(lock);
41 42 43 44
/* setup_lock protects refcount, stop_machine_wq and stop_machine_work. */
static DEFINE_MUTEX(setup_lock);
/* Users of stop_machine. */
static int refcount;
45 46
static struct workqueue_struct *stop_machine_wq;
static struct stop_machine_data active, idle;
47
static const struct cpumask *active_cpus;
48
static void __percpu *stop_machine_work;
49

Rusty Russell's avatar
Rusty Russell committed
50
static void set_state(enum stopmachine_state newstate)
Linus Torvalds's avatar
Linus Torvalds committed
51
{
Rusty Russell's avatar
Rusty Russell committed
52 53 54 55
	/* Reset ack counter. */
	atomic_set(&thread_ack, num_threads);
	smp_wmb();
	state = newstate;
Linus Torvalds's avatar
Linus Torvalds committed
56 57
}

Rusty Russell's avatar
Rusty Russell committed
58 59
/* Last one to ack a state moves to the next state. */
static void ack_state(void)
Linus Torvalds's avatar
Linus Torvalds committed
60
{
61 62
	if (atomic_dec_and_test(&thread_ack))
		set_state(state + 1);
Linus Torvalds's avatar
Linus Torvalds committed
63 64
}

65 66 67
/* This is the actual function which stops the CPU. It runs
 * in the context of a dedicated stopmachine workqueue. */
static void stop_cpu(struct work_struct *unused)
Linus Torvalds's avatar
Linus Torvalds committed
68
{
Rusty Russell's avatar
Rusty Russell committed
69
	enum stopmachine_state curstate = STOPMACHINE_NONE;
70 71
	struct stop_machine_data *smdata = &idle;
	int cpu = smp_processor_id();
72
	int err;
73 74

	if (!active_cpus) {
75
		if (cpu == cpumask_first(cpu_online_mask))
76 77
			smdata = &active;
	} else {
78
		if (cpumask_test_cpu(cpu, active_cpus))
79 80
			smdata = &active;
	}
Rusty Russell's avatar
Rusty Russell committed
81 82 83
	/* Simple state machine */
	do {
		/* Chill out and ensure we re-read stopmachine_state. */
84
		cpu_relax();
Rusty Russell's avatar
Rusty Russell committed
85 86 87 88 89 90 91 92
		if (state != curstate) {
			curstate = state;
			switch (curstate) {
			case STOPMACHINE_DISABLE_IRQ:
				local_irq_disable();
				hard_irq_disable();
				break;
			case STOPMACHINE_RUN:
93 94 95 96 97
				/* On multiple CPUs only a single error code
				 * is needed to tell that something failed. */
				err = smdata->fn(smdata->data);
				if (err)
					smdata->fnret = err;
Rusty Russell's avatar
Rusty Russell committed
98 99 100 101 102 103 104
				break;
			default:
				break;
			}
			ack_state();
		}
	} while (curstate != STOPMACHINE_EXIT);
Linus Torvalds's avatar
Linus Torvalds committed
105 106 107 108

	local_irq_enable();
}

Rusty Russell's avatar
Rusty Russell committed
109 110
/* Callback for CPUs which aren't supposed to do anything. */
static int chill(void *unused)
111
{
Rusty Russell's avatar
Rusty Russell committed
112
	return 0;
113
}
Linus Torvalds's avatar
Linus Torvalds committed
114

115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151
int stop_machine_create(void)
{
	mutex_lock(&setup_lock);
	if (refcount)
		goto done;
	stop_machine_wq = create_rt_workqueue("kstop");
	if (!stop_machine_wq)
		goto err_out;
	stop_machine_work = alloc_percpu(struct work_struct);
	if (!stop_machine_work)
		goto err_out;
done:
	refcount++;
	mutex_unlock(&setup_lock);
	return 0;

err_out:
	if (stop_machine_wq)
		destroy_workqueue(stop_machine_wq);
	mutex_unlock(&setup_lock);
	return -ENOMEM;
}
EXPORT_SYMBOL_GPL(stop_machine_create);

void stop_machine_destroy(void)
{
	mutex_lock(&setup_lock);
	refcount--;
	if (refcount)
		goto done;
	destroy_workqueue(stop_machine_wq);
	free_percpu(stop_machine_work);
done:
	mutex_unlock(&setup_lock);
}
EXPORT_SYMBOL_GPL(stop_machine_destroy);

152
int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
Linus Torvalds's avatar
Linus Torvalds committed
153
{
154
	struct work_struct *sm_work;
155
	int i, ret;
Rusty Russell's avatar
Rusty Russell committed
156

157 158 159 160
	/* Set up initial state. */
	mutex_lock(&lock);
	num_threads = num_online_cpus();
	active_cpus = cpus;
Rusty Russell's avatar
Rusty Russell committed
161 162 163 164 165 166 167
	active.fn = fn;
	active.data = data;
	active.fnret = 0;
	idle.fn = chill;
	idle.data = NULL;

	set_state(STOPMACHINE_PREPARE);
Linus Torvalds's avatar
Linus Torvalds committed
168

169
	/* Schedule the stop_cpu work on all cpus: hold this CPU so one
Rusty Russell's avatar
Rusty Russell committed
170
	 * doesn't hit this CPU until we're ready. */
171
	get_cpu();
172
	for_each_online_cpu(i) {
173
		sm_work = per_cpu_ptr(stop_machine_work, i);
174 175 176
		INIT_WORK(sm_work, stop_cpu);
		queue_work_on(i, stop_machine_wq, sm_work);
	}
Rusty Russell's avatar
Rusty Russell committed
177 178
	/* This will release the thread on our CPU. */
	put_cpu();
179
	flush_workqueue(stop_machine_wq);
180
	ret = active.fnret;
Rusty Russell's avatar
Rusty Russell committed
181
	mutex_unlock(&lock);
182
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
183 184
}

185
int stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
Linus Torvalds's avatar
Linus Torvalds committed
186 187 188
{
	int ret;

189 190 191
	ret = stop_machine_create();
	if (ret)
		return ret;
Linus Torvalds's avatar
Linus Torvalds committed
192
	/* No CPUs can come up or down during this. */
193
	get_online_cpus();
194
	ret = __stop_machine(fn, data, cpus);
195
	put_online_cpus();
196
	stop_machine_destroy();
Linus Torvalds's avatar
Linus Torvalds committed
197 198
	return ret;
}
199
EXPORT_SYMBOL_GPL(stop_machine);