nmi.c 16 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
2 3 4
/*
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *  Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
5
 *  Copyright (C) 2011	Don Zickus Red Hat, Inc.
6 7 8 9 10 11 12 13 14 15 16
 *
 *  Pentium III FXSR, SSE support
 *	Gareth Hughes <gareth@valinux.com>, May 2000
 */

/*
 * Handle hardware traps and faults.
 */
#include <linux/spinlock.h>
#include <linux/kprobes.h>
#include <linux/kdebug.h>
17
#include <linux/sched/debug.h>
18
#include <linux/nmi.h>
19
#include <linux/debugfs.h>
20 21
#include <linux/delay.h>
#include <linux/hardirq.h>
22
#include <linux/ratelimit.h>
23
#include <linux/slab.h>
24
#include <linux/export.h>
25
#include <linux/atomic.h>
26
#include <linux/sched/clock.h>
27 28 29 30 31

#if defined(CONFIG_EDAC)
#include <linux/edac.h>
#endif

32
#include <asm/cpu_entry_area.h>
33 34
#include <asm/traps.h>
#include <asm/mach_traps.h>
35
#include <asm/nmi.h>
36
#include <asm/x86_init.h>
37
#include <asm/reboot.h>
38
#include <asm/cache.h>
39
#include <asm/nospec-branch.h>
40

41 42 43
#define CREATE_TRACE_POINTS
#include <trace/events/nmi.h>

44
struct nmi_desc {
Scott Wood's avatar
Scott Wood committed
45
	raw_spinlock_t lock;
46 47 48 49 50 51
	struct list_head head;
};

static struct nmi_desc nmi_desc[NMI_MAX] = 
{
	{
Scott Wood's avatar
Scott Wood committed
52
		.lock = __RAW_SPIN_LOCK_UNLOCKED(&nmi_desc[0].lock),
53 54 55
		.head = LIST_HEAD_INIT(nmi_desc[0].head),
	},
	{
Scott Wood's avatar
Scott Wood committed
56
		.lock = __RAW_SPIN_LOCK_UNLOCKED(&nmi_desc[1].lock),
57 58
		.head = LIST_HEAD_INIT(nmi_desc[1].head),
	},
59
	{
Scott Wood's avatar
Scott Wood committed
60
		.lock = __RAW_SPIN_LOCK_UNLOCKED(&nmi_desc[2].lock),
61 62 63
		.head = LIST_HEAD_INIT(nmi_desc[2].head),
	},
	{
Scott Wood's avatar
Scott Wood committed
64
		.lock = __RAW_SPIN_LOCK_UNLOCKED(&nmi_desc[3].lock),
65 66
		.head = LIST_HEAD_INIT(nmi_desc[3].head),
	},
67 68

};
69

Don Zickus's avatar
Don Zickus committed
70 71 72 73 74 75 76 77 78
struct nmi_stats {
	unsigned int normal;
	unsigned int unknown;
	unsigned int external;
	unsigned int swallow;
};

static DEFINE_PER_CPU(struct nmi_stats, nmi_stats);

79
static int ignore_nmis __read_mostly;
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94

int unknown_nmi_panic;
/*
 * Prevent NMI reason port (0x61) being accessed simultaneously, can
 * only be used in NMI handler.
 */
static DEFINE_RAW_SPINLOCK(nmi_reason_lock);

static int __init setup_unknown_nmi_panic(char *str)
{
	unknown_nmi_panic = 1;
	return 1;
}
__setup("unknown_nmi_panic", setup_unknown_nmi_panic);

95 96
#define nmi_to_desc(type) (&nmi_desc[type])

97
static u64 nmi_longest_ns = 1 * NSEC_PER_MSEC;
98

99 100 101 102 103 104 105 106
static int __init nmi_warning_debugfs(void)
{
	debugfs_create_u64("nmi_longest_ns", 0644,
			arch_debugfs_dir, &nmi_longest_ns);
	return 0;
}
fs_initcall(nmi_warning_debugfs);

107 108 109 110
static void nmi_max_handler(struct irq_work *w)
{
	struct nmiaction *a = container_of(w, struct nmiaction, irq_work);
	int remainder_ns, decimal_msecs;
111
	u64 whole_msecs = READ_ONCE(a->max_duration);
112 113 114 115 116 117 118 119 120

	remainder_ns = do_div(whole_msecs, (1000 * 1000));
	decimal_msecs = remainder_ns / 1000;

	printk_ratelimited(KERN_INFO
		"INFO: NMI handler (%ps) took too long to run: %lld.%03d msecs\n",
		a->handler, whole_msecs, decimal_msecs);
}

121
static int nmi_handle(unsigned int type, struct pt_regs *regs)
122 123 124 125 126 127 128 129 130 131 132 133 134
{
	struct nmi_desc *desc = nmi_to_desc(type);
	struct nmiaction *a;
	int handled=0;

	rcu_read_lock();

	/*
	 * NMIs are edge-triggered, which means if you have enough
	 * of them concurrently, you can lose some because only one
	 * can be latched at any given time.  Walk the whole list
	 * to handle those situations.
	 */
135
	list_for_each_entry_rcu(a, &desc->head, list) {
136 137
		int thishandled;
		u64 delta;
138

139
		delta = sched_clock();
140 141
		thishandled = a->handler(type, regs);
		handled += thishandled;
142
		delta = sched_clock() - delta;
143
		trace_nmi_handler(a->handler, (int)delta, thishandled);
144

145
		if (delta < nmi_longest_ns || delta < a->max_duration)
146 147
			continue;

148 149
		a->max_duration = delta;
		irq_work_queue(&a->irq_work);
150
	}
151 152 153 154 155 156

	rcu_read_unlock();

	/* return total number of NMI events handled */
	return handled;
}
157
NOKPROBE_SYMBOL(nmi_handle);
158

159
int __register_nmi_handler(unsigned int type, struct nmiaction *action)
160 161 162 163
{
	struct nmi_desc *desc = nmi_to_desc(type);
	unsigned long flags;

164 165 166
	if (!action->handler)
		return -EINVAL;

167 168
	init_irq_work(&action->irq_work, nmi_max_handler);

Scott Wood's avatar
Scott Wood committed
169
	raw_spin_lock_irqsave(&desc->lock, flags);
170

171
	/*
172 173
	 * Indicate if there are multiple registrations on the
	 * internal NMI handler call chains (SERR and IO_CHECK).
174
	 */
175 176
	WARN_ON_ONCE(type == NMI_SERR && !list_empty(&desc->head));
	WARN_ON_ONCE(type == NMI_IO_CHECK && !list_empty(&desc->head));
177

178 179 180 181 182 183 184 185 186
	/*
	 * some handlers need to be executed first otherwise a fake
	 * event confuses some handlers (kdump uses this flag)
	 */
	if (action->flags & NMI_FLAG_FIRST)
		list_add_rcu(&action->list, &desc->head);
	else
		list_add_tail_rcu(&action->list, &desc->head);
	
Scott Wood's avatar
Scott Wood committed
187
	raw_spin_unlock_irqrestore(&desc->lock, flags);
188 189
	return 0;
}
190
EXPORT_SYMBOL(__register_nmi_handler);
191

192
void unregister_nmi_handler(unsigned int type, const char *name)
193 194 195 196 197
{
	struct nmi_desc *desc = nmi_to_desc(type);
	struct nmiaction *n;
	unsigned long flags;

Scott Wood's avatar
Scott Wood committed
198
	raw_spin_lock_irqsave(&desc->lock, flags);
199 200 201 202 203 204 205 206 207 208 209 210 211 212

	list_for_each_entry_rcu(n, &desc->head, list) {
		/*
		 * the name passed in to describe the nmi handler
		 * is used as the lookup key
		 */
		if (!strcmp(n->name, name)) {
			WARN(in_nmi(),
				"Trying to free NMI (%s) from NMI context!\n", n->name);
			list_del_rcu(&n->list);
			break;
		}
	}

Scott Wood's avatar
Scott Wood committed
213
	raw_spin_unlock_irqrestore(&desc->lock, flags);
214 215 216 217
	synchronize_rcu();
}
EXPORT_SYMBOL_GPL(unregister_nmi_handler);

218
static void
219 220
pci_serr_error(unsigned char reason, struct pt_regs *regs)
{
221
	/* check to see if anyone registered against these types of errors */
222
	if (nmi_handle(NMI_SERR, regs))
223 224
		return;

225 226 227 228
	pr_emerg("NMI: PCI system error (SERR) for reason %02x on CPU %d.\n",
		 reason, smp_processor_id());

	if (panic_on_unrecovered_nmi)
229
		nmi_panic(regs, "NMI: Not continuing");
230 231 232 233 234 235 236

	pr_emerg("Dazed and confused, but trying to continue\n");

	/* Clear and disable the PCI SERR error line. */
	reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR;
	outb(reason, NMI_REASON_PORT);
}
237
NOKPROBE_SYMBOL(pci_serr_error);
238

239
static void
240 241 242 243
io_check_error(unsigned char reason, struct pt_regs *regs)
{
	unsigned long i;

244
	/* check to see if anyone registered against these types of errors */
245
	if (nmi_handle(NMI_IO_CHECK, regs))
246 247
		return;

248 249 250
	pr_emerg(
	"NMI: IOCK error (debug interrupt?) for reason %02x on CPU %d.\n",
		 reason, smp_processor_id());
251
	show_regs(regs);
252

253
	if (panic_on_io_nmi) {
254
		nmi_panic(regs, "NMI IOCK error: Not continuing");
255 256 257 258 259 260 261 262

		/*
		 * If we end up here, it means we have received an NMI while
		 * processing panic(). Simply return without delaying and
		 * re-enabling NMIs.
		 */
		return;
	}
263 264 265 266 267 268 269 270 271 272 273 274 275 276

	/* Re-enable the IOCK line, wait for a few seconds */
	reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_IOCHK;
	outb(reason, NMI_REASON_PORT);

	i = 20000;
	while (--i) {
		touch_nmi_watchdog();
		udelay(100);
	}

	reason &= ~NMI_REASON_CLEAR_IOCHK;
	outb(reason, NMI_REASON_PORT);
}
277
NOKPROBE_SYMBOL(io_check_error);
278

279
static void
280 281
unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
{
282 283
	int handled;

284 285 286 287 288 289
	/*
	 * Use 'false' as back-to-back NMIs are dealt with one level up.
	 * Of course this makes having multiple 'unknown' handlers useless
	 * as only the first one is ever run (unless it can actually determine
	 * if it caused the NMI)
	 */
290
	handled = nmi_handle(NMI_UNKNOWN, regs);
Don Zickus's avatar
Don Zickus committed
291 292
	if (handled) {
		__this_cpu_add(nmi_stats.unknown, handled);
293
		return;
Don Zickus's avatar
Don Zickus committed
294 295 296 297
	}

	__this_cpu_add(nmi_stats.unknown, 1);

298 299 300 301 302
	pr_emerg("Uhhuh. NMI received for unknown reason %02x on CPU %d.\n",
		 reason, smp_processor_id());

	pr_emerg("Do you have a strange power saving mode enabled?\n");
	if (unknown_nmi_panic || panic_on_unrecovered_nmi)
303
		nmi_panic(regs, "NMI: Not continuing");
304 305 306

	pr_emerg("Dazed and confused, but trying to continue\n");
}
307
NOKPROBE_SYMBOL(unknown_nmi_error);
308

309 310 311
static DEFINE_PER_CPU(bool, swallow_nmi);
static DEFINE_PER_CPU(unsigned long, last_nmi_rip);

312
static void default_do_nmi(struct pt_regs *regs)
313 314
{
	unsigned char reason = 0;
315
	int handled;
316
	bool b2b = false;
317 318 319 320 321 322

	/*
	 * CPU-specific NMI must be processed before non-CPU-specific
	 * NMI, otherwise we may lose it, because the CPU-specific
	 * NMI can not be detected/processed on other CPUs.
	 */
323 324 325 326 327 328 329 330 331 332 333 334 335 336 337

	/*
	 * Back-to-back NMIs are interesting because they can either
	 * be two NMI or more than two NMIs (any thing over two is dropped
	 * due to NMI being edge-triggered).  If this is the second half
	 * of the back-to-back NMI, assume we dropped things and process
	 * more handlers.  Otherwise reset the 'swallow' NMI behaviour
	 */
	if (regs->ip == __this_cpu_read(last_nmi_rip))
		b2b = true;
	else
		__this_cpu_write(swallow_nmi, false);

	__this_cpu_write(last_nmi_rip, regs->ip);

338
	handled = nmi_handle(NMI_LOCAL, regs);
Don Zickus's avatar
Don Zickus committed
339
	__this_cpu_add(nmi_stats.normal, handled);
340 341 342 343 344 345 346 347 348 349 350
	if (handled) {
		/*
		 * There are cases when a NMI handler handles multiple
		 * events in the current NMI.  One of these events may
		 * be queued for in the next NMI.  Because the event is
		 * already handled, the next NMI will result in an unknown
		 * NMI.  Instead lets flag this for a potential NMI to
		 * swallow.
		 */
		if (handled > 1)
			__this_cpu_write(swallow_nmi, true);
351
		return;
352
	}
353

354 355 356 357 358 359 360 361 362 363 364 365 366
	/*
	 * Non-CPU-specific NMI: NMI sources can be processed on any CPU.
	 *
	 * Another CPU may be processing panic routines while holding
	 * nmi_reason_lock. Check if the CPU issued the IPI for crash dumping,
	 * and if so, call its callback directly.  If there is no CPU preparing
	 * crash dump, we simply loop here.
	 */
	while (!raw_spin_trylock(&nmi_reason_lock)) {
		run_crash_ipi_callback(regs);
		cpu_relax();
	}

367
	reason = x86_platform.get_nmi_reason();
368 369 370 371 372 373 374 375 376 377 378 379 380

	if (reason & NMI_REASON_MASK) {
		if (reason & NMI_REASON_SERR)
			pci_serr_error(reason, regs);
		else if (reason & NMI_REASON_IOCHK)
			io_check_error(reason, regs);
#ifdef CONFIG_X86_32
		/*
		 * Reassert NMI in case it became active
		 * meanwhile as it's edge-triggered:
		 */
		reassert_nmi();
#endif
Don Zickus's avatar
Don Zickus committed
381
		__this_cpu_add(nmi_stats.external, 1);
382 383 384 385 386
		raw_spin_unlock(&nmi_reason_lock);
		return;
	}
	raw_spin_unlock(&nmi_reason_lock);

387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417
	/*
	 * Only one NMI can be latched at a time.  To handle
	 * this we may process multiple nmi handlers at once to
	 * cover the case where an NMI is dropped.  The downside
	 * to this approach is we may process an NMI prematurely,
	 * while its real NMI is sitting latched.  This will cause
	 * an unknown NMI on the next run of the NMI processing.
	 *
	 * We tried to flag that condition above, by setting the
	 * swallow_nmi flag when we process more than one event.
	 * This condition is also only present on the second half
	 * of a back-to-back NMI, so we flag that condition too.
	 *
	 * If both are true, we assume we already processed this
	 * NMI previously and we swallow it.  Otherwise we reset
	 * the logic.
	 *
	 * There are scenarios where we may accidentally swallow
	 * a 'real' unknown NMI.  For example, while processing
	 * a perf NMI another perf NMI comes in along with a
	 * 'real' unknown NMI.  These two NMIs get combined into
	 * one (as descibed above).  When the next NMI gets
	 * processed, it will be flagged by perf as handled, but
	 * noone will know that there was a 'real' unknown NMI sent
	 * also.  As a result it gets swallowed.  Or if the first
	 * perf NMI returns two events handled then the second
	 * NMI will get eaten by the logic below, again losing a
	 * 'real' unknown NMI.  But this is the best we can do
	 * for now.
	 */
	if (b2b && __this_cpu_read(swallow_nmi))
Don Zickus's avatar
Don Zickus committed
418
		__this_cpu_add(nmi_stats.swallow, 1);
419 420
	else
		unknown_nmi_error(reason, regs);
421
}
422
NOKPROBE_SYMBOL(default_do_nmi);
423

424
/*
425 426
 * NMIs can page fault or hit breakpoints which will cause it to lose
 * its NMI context with the CPU when the breakpoint or page fault does an IRET.
427 428 429 430 431 432 433
 *
 * As a result, NMIs can nest if NMIs get unmasked due an IRET during
 * NMI processing.  On x86_64, the asm glue protects us from nested NMIs
 * if the outer NMI came from kernel mode, but we can still nest if the
 * outer NMI came from user mode.
 *
 * To handle these nested NMIs, we have three states:
434 435 436 437 438 439 440 441 442 443 444 445 446
 *
 *  1) not running
 *  2) executing
 *  3) latched
 *
 * When no NMI is in progress, it is in the "not running" state.
 * When an NMI comes in, it goes into the "executing" state.
 * Normally, if another NMI is triggered, it does not interrupt
 * the running NMI and the HW will simply latch it so that when
 * the first NMI finishes, it will restart the second NMI.
 * (Note, the latch is binary, thus multiple NMIs triggering,
 *  when one is running, are ignored. Only one NMI is restarted.)
 *
447 448 449 450 451 452 453 454
 * If an NMI executes an iret, another NMI can preempt it. We do not
 * want to allow this new NMI to run, but we want to execute it when the
 * first one finishes.  We set the state to "latched", and the exit of
 * the first NMI will perform a dec_return, if the result is zero
 * (NOT_RUNNING), then it will simply exit the NMI handler. If not, the
 * dec_return would have set the state to NMI_EXECUTING (what we want it
 * to be when we are running). In this case, we simply jump back to
 * rerun the NMI handler again, and restart the 'latched' NMI.
455 456 457 458 459
 *
 * No trap (breakpoint or page fault) should be hit before nmi_restart,
 * thus there is no race between the first check of state for NOT_RUNNING
 * and setting it to NMI_EXECUTING. The HW will prevent nested NMIs
 * at this point.
460 461 462 463 464 465 466 467
 *
 * In case the NMI takes a page fault, we need to save off the CR2
 * because the NMI could have preempted another page fault and corrupt
 * the CR2 that is about to be read. As nested NMIs must be restarted
 * and they can not take breakpoints or page faults, the update of the
 * CR2 must be done before converting the nmi state back to NOT_RUNNING.
 * Otherwise, there would be a race of another nested NMI coming in
 * after setting state to NOT_RUNNING but before updating the nmi_cr2.
468 469
 */
enum nmi_states {
470
	NMI_NOT_RUNNING = 0,
471 472 473 474
	NMI_EXECUTING,
	NMI_LATCHED,
};
static DEFINE_PER_CPU(enum nmi_states, nmi_state);
475
static DEFINE_PER_CPU(unsigned long, nmi_cr2);
476

477
#ifdef CONFIG_X86_64
478
/*
479 480 481
 * In x86_64, we need to handle breakpoint -> NMI -> breakpoint.  Without
 * some care, the inner breakpoint will clobber the outer breakpoint's
 * stack.
482
 *
483 484 485 486 487 488 489 490
 * If a breakpoint is being processed, and the debug stack is being
 * used, if an NMI comes in and also hits a breakpoint, the stack
 * pointer will be set to the same fixed address as the breakpoint that
 * was interrupted, causing that stack to be corrupted. To handle this
 * case, check if the stack that was interrupted is the debug stack, and
 * if so, change the IDT so that new breakpoints will use the current
 * stack and not switch to the fixed address. On return of the NMI,
 * switch back to the original IDT.
491 492
 */
static DEFINE_PER_CPU(int, update_debug_stack);
493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509

static bool notrace is_debug_stack(unsigned long addr)
{
	struct cea_exception_stacks *cs = __this_cpu_read(cea_exception_stacks);
	unsigned long top = CEA_ESTACK_TOP(cs, DB);
	unsigned long bot = CEA_ESTACK_BOT(cs, DB1);

	if (__this_cpu_read(debug_stack_usage))
		return true;
	/*
	 * Note, this covers the guard page between DB and DB1 as well to
	 * avoid two checks. But by all means @addr can never point into
	 * the guard page.
	 */
	return addr >= bot && addr < top;
}
NOKPROBE_SYMBOL(is_debug_stack);
510
#endif
511

512 513
dotraplinkage notrace void
do_nmi(struct pt_regs *regs, long error_code)
514
{
515 516 517 518 519 520 521 522 523
	if (this_cpu_read(nmi_state) != NMI_NOT_RUNNING) {
		this_cpu_write(nmi_state, NMI_LATCHED);
		return;
	}
	this_cpu_write(nmi_state, NMI_EXECUTING);
	this_cpu_write(nmi_cr2, read_cr2());
nmi_restart:

#ifdef CONFIG_X86_64
524 525 526 527 528 529 530 531
	/*
	 * If we interrupted a breakpoint, it is possible that
	 * the nmi handler will have breakpoints too. We need to
	 * change the IDT such that breakpoints that happen here
	 * continue to use the NMI stack.
	 */
	if (unlikely(is_debug_stack(regs->sp))) {
		debug_stack_set_zero();
532
		this_cpu_write(update_debug_stack, 1);
533
	}
534 535
#endif

536 537 538 539 540 541 542 543
	nmi_enter();

	inc_irq_stat(__nmi_count);

	if (!ignore_nmis)
		default_do_nmi(regs);

	nmi_exit();
544

545 546 547 548 549 550 551 552 553 554 555
#ifdef CONFIG_X86_64
	if (unlikely(this_cpu_read(update_debug_stack))) {
		debug_stack_reset();
		this_cpu_write(update_debug_stack, 0);
	}
#endif

	if (unlikely(this_cpu_read(nmi_cr2) != read_cr2()))
		write_cr2(this_cpu_read(nmi_cr2));
	if (this_cpu_dec_return(nmi_state))
		goto nmi_restart;
556 557 558

	if (user_mode(regs))
		mds_user_clear_cpu_buffers();
559
}
560
NOKPROBE_SYMBOL(do_nmi);
561 562 563 564 565 566 567 568 569 570

void stop_nmi(void)
{
	ignore_nmis++;
}

void restart_nmi(void)
{
	ignore_nmis--;
}
571 572 573 574 575 576

/* reset the back-to-back NMI logic */
void local_touch_nmi(void)
{
	__this_cpu_write(last_nmi_rip, 0);
}
577
EXPORT_SYMBOL_GPL(local_touch_nmi);