processor.h 24.9 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
2 3
#ifndef _ASM_X86_PROCESSOR_H
#define _ASM_X86_PROCESSOR_H
4

5 6
#include <asm/processor-flags.h>

7 8 9
/* Forward declaration, a strange C thing */
struct task_struct;
struct mm_struct;
10
struct vm86;
11

12 13 14
#include <asm/math_emu.h>
#include <asm/segment.h>
#include <asm/types.h>
15
#include <uapi/asm/sigcontext.h>
16
#include <asm/current.h>
17
#include <asm/cpufeatures.h>
18
#include <asm/page.h>
19
#include <asm/pgtable_types.h>
20
#include <asm/percpu.h>
21 22
#include <asm/msr.h>
#include <asm/desc_defs.h>
23
#include <asm/nops.h>
24
#include <asm/special_insns.h>
25
#include <asm/fpu/types.h>
26
#include <asm/unwind_hints.h>
27

28
#include <linux/personality.h>
29
#include <linux/cache.h>
30
#include <linux/threads.h>
31
#include <linux/math64.h>
32
#include <linux/err.h>
33
#include <linux/irqflags.h>
34
#include <linux/mem_encrypt.h>
35 36 37 38 39 40 41 42

/*
 * We handle most unaligned accesses in hardware.  On the other hand
 * unaligned DMA can be quite expensive on some Nehalem processors.
 *
 * Based on this we disable the IP header alignment in network drivers.
 */
#define NET_IP_ALIGN	0
43

44
#define HBP_NUM 4
45 46 47 48 49 50 51
/*
 * Default implementation of macro that returns current
 * instruction pointer ("program counter").
 */
static inline void *current_text_addr(void)
{
	void *pc;
52 53 54

	asm volatile("mov $1f, %0; 1:":"=r" (pc));

55 56 57
	return pc;
}

58 59 60 61 62
/*
 * These alignment constraints are for performance in the vSMP case,
 * but in the task_struct case we must also meet hardware imposed
 * alignment requirements of the FPU state:
 */
63
#ifdef CONFIG_X86_VSMP
64 65
# define ARCH_MIN_TASKALIGN		(1 << INTERNODE_CACHE_SHIFT)
# define ARCH_MIN_MMSTRUCT_ALIGN	(1 << INTERNODE_CACHE_SHIFT)
66
#else
67
# define ARCH_MIN_TASKALIGN		__alignof__(union fpregs_state)
68
# define ARCH_MIN_MMSTRUCT_ALIGN	0
69 70
#endif

71 72 73 74 75 76 77 78 79 80 81
enum tlb_infos {
	ENTRIES,
	NR_INFO
};

extern u16 __read_mostly tlb_lli_4k[NR_INFO];
extern u16 __read_mostly tlb_lli_2m[NR_INFO];
extern u16 __read_mostly tlb_lli_4m[NR_INFO];
extern u16 __read_mostly tlb_lld_4k[NR_INFO];
extern u16 __read_mostly tlb_lld_2m[NR_INFO];
extern u16 __read_mostly tlb_lld_4m[NR_INFO];
82
extern u16 __read_mostly tlb_lld_1g[NR_INFO];
83

84 85
/*
 *  CPU type and hardware bug flags. Kept separately for each CPU.
86
 *  Members of this structure are referenced in head_32.S, so think twice
87 88 89 90
 *  before touching them. [mj]
 */

struct cpuinfo_x86 {
91 92 93
	__u8			x86;		/* CPU family */
	__u8			x86_vendor;	/* CPU vendor */
	__u8			x86_model;
94
	__u8			x86_stepping;
95
#ifdef CONFIG_X86_64
96
	/* Number of 4K pages in DTLB/ITLB combined(in pages): */
97
	int			x86_tlbsize;
98
#endif
99 100 101 102
	__u8			x86_virt_bits;
	__u8			x86_phys_bits;
	/* CPUID returned core id bits: */
	__u8			x86_coreid_bits;
103
	__u8			cu_id;
104 105 106 107
	/* Max extended CPUID function supported: */
	__u32			extended_cpuid_level;
	/* Maximum supported CPUID level, -1=no CPUID: */
	int			cpuid_level;
108
	__u32			x86_capability[NCAPINTS + NBUGINTS];
109 110 111
	char			x86_vendor_id[16];
	char			x86_model_id[64];
	/* in KB - valid for CPUS which support this call: */
112
	unsigned int		x86_cache_size;
113
	int			x86_cache_alignment;	/* In bytes */
114 115 116
	/* Cache QoS architectural values: */
	int			x86_cache_max_rmid;	/* max index */
	int			x86_cache_occ_scale;	/* scale to bytes */
117 118 119 120 121
	int			x86_power;
	unsigned long		loops_per_jiffy;
	/* cpuid returned max cores value: */
	u16			 x86_max_cores;
	u16			apicid;
Yinghai Lu's avatar
Yinghai Lu committed
122
	u16			initial_apicid;
123 124 125 126 127
	u16			x86_clflush_size;
	/* number of cores as seen by the OS: */
	u16			booted_cores;
	/* Physical processor id: */
	u16			phys_proc_id;
128 129
	/* Logical processor id: */
	u16			logical_proc_id;
130 131 132 133
	/* Core id: */
	u16			cpu_core_id;
	/* Index into per_cpu list: */
	u16			cpu_index;
134
	u32			microcode;
135
	unsigned		initialized : 1;
136
} __randomize_layout;
137

138 139 140 141 142 143 144 145 146 147 148
struct cpuid_regs {
	u32 eax, ebx, ecx, edx;
};

enum cpuid_regs_idx {
	CPUID_EAX = 0,
	CPUID_EBX,
	CPUID_ECX,
	CPUID_EDX,
};

149 150 151 152 153 154 155 156 157 158
#define X86_VENDOR_INTEL	0
#define X86_VENDOR_CYRIX	1
#define X86_VENDOR_AMD		2
#define X86_VENDOR_UMC		3
#define X86_VENDOR_CENTAUR	5
#define X86_VENDOR_TRANSMETA	7
#define X86_VENDOR_NSC		8
#define X86_VENDOR_NUM		9

#define X86_VENDOR_UNKNOWN	0xff
159

160 161 162
/*
 * capabilities of CPUs
 */
163 164 165
extern struct cpuinfo_x86	boot_cpu_data;
extern struct cpuinfo_x86	new_cpu_data;

166
extern struct x86_hw_tss	doublefault_tss;
167 168
extern __u32			cpu_caps_cleared[NCAPINTS + NBUGINTS];
extern __u32			cpu_caps_set[NCAPINTS + NBUGINTS];
169 170

#ifdef CONFIG_SMP
171
DECLARE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
172 173
#define cpu_data(cpu)		per_cpu(cpu_info, cpu)
#else
174
#define cpu_info		boot_cpu_data
175 176 177
#define cpu_data(cpu)		boot_cpu_data
#endif

178 179
extern const struct seq_operations cpuinfo_op;

180 181 182
#define cache_line_size()	(boot_cpu_data.x86_cache_alignment)

extern void cpu_detect(struct cpuinfo_x86 *c);
183

184 185 186 187 188
static inline unsigned long l1tf_pfn_limit(void)
{
	return BIT(boot_cpu_data.x86_phys_bits - 1 - PAGE_SHIFT) - 1;
}

189
extern void early_cpu_init(void);
190 191
extern void identify_boot_cpu(void);
extern void identify_secondary_cpu(struct cpuinfo_x86 *);
192
extern void print_cpu_info(struct cpuinfo_x86 *);
193
void print_cpu_msr(struct cpuinfo_x86 *);
194

195 196 197 198 199 200 201 202
#ifdef CONFIG_X86_32
extern int have_cpuid_p(void);
#else
static inline int have_cpuid_p(void)
{
	return 1;
}
#endif
203
static inline void native_cpuid(unsigned int *eax, unsigned int *ebx,
204
				unsigned int *ecx, unsigned int *edx)
205 206
{
	/* ecx is often an input as well as an output. */
207
	asm volatile("cpuid"
208 209 210 211
	    : "=a" (*eax),
	      "=b" (*ebx),
	      "=c" (*ecx),
	      "=d" (*edx)
212 213
	    : "0" (*eax), "2" (*ecx)
	    : "memory");
214 215
}

216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233
#define native_cpuid_reg(reg)					\
static inline unsigned int native_cpuid_##reg(unsigned int op)	\
{								\
	unsigned int eax = op, ebx, ecx = 0, edx;		\
								\
	native_cpuid(&eax, &ebx, &ecx, &edx);			\
								\
	return reg;						\
}

/*
 * Native CPUID functions returning a single datum.
 */
native_cpuid_reg(eax)
native_cpuid_reg(ebx)
native_cpuid_reg(ecx)
native_cpuid_reg(edx)

234 235 236 237 238 239 240 241
/*
 * Friendlier CR3 helpers.
 */
static inline unsigned long read_cr3_pa(void)
{
	return __read_cr3() & CR3_ADDR_MASK;
}

242 243 244 245 246
static inline unsigned long native_read_cr3_pa(void)
{
	return __native_read_cr3() & CR3_ADDR_MASK;
}

247 248
static inline void load_cr3(pgd_t *pgdir)
{
249
	write_cr3(__sme_pa(pgdir));
250
}
251

252 253 254 255 256
/*
 * Note that while the legacy 'TSS' name comes from 'Task State Segment',
 * on modern x86 CPUs the TSS also holds information important to 64-bit mode,
 * unrelated to the task-switch mechanism:
 */
257 258 259
#ifdef CONFIG_X86_32
/* This is the TSS defined by the hardware. */
struct x86_hw_tss {
260 261 262
	unsigned short		back_link, __blh;
	unsigned long		sp0;
	unsigned short		ss0, __ss0h;
263
	unsigned long		sp1;
264 265

	/*
266 267 268 269 270 271
	 * We don't use ring 1, so ss1 is a convenient scratch space in
	 * the same cacheline as sp0.  We use ss1 to cache the value in
	 * MSR_IA32_SYSENTER_CS.  When we context switch
	 * MSR_IA32_SYSENTER_CS, we first check if the new value being
	 * written matches ss1, and, if it's not, then we wrmsr the new
	 * value and update ss1.
272
	 *
273 274 275 276
	 * The only reason we context switch MSR_IA32_SYSENTER_CS is
	 * that we set it to zero in vm86 tasks to avoid corrupting the
	 * stack if we were to go through the sysenter path from vm86
	 * mode.
277 278 279 280
	 */
	unsigned short		ss1;	/* MSR_IA32_SYSENTER_CS */

	unsigned short		__ss1h;
281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
	unsigned long		sp2;
	unsigned short		ss2, __ss2h;
	unsigned long		__cr3;
	unsigned long		ip;
	unsigned long		flags;
	unsigned long		ax;
	unsigned long		cx;
	unsigned long		dx;
	unsigned long		bx;
	unsigned long		sp;
	unsigned long		bp;
	unsigned long		si;
	unsigned long		di;
	unsigned short		es, __esh;
	unsigned short		cs, __csh;
	unsigned short		ss, __ssh;
	unsigned short		ds, __dsh;
	unsigned short		fs, __fsh;
	unsigned short		gs, __gsh;
	unsigned short		ldt, __ldth;
	unsigned short		trace;
	unsigned short		io_bitmap_base;

304 305 306
} __attribute__((packed));
#else
struct x86_hw_tss {
307 308
	u32			reserved1;
	u64			sp0;
309 310 311 312 313

	/*
	 * We store cpu_current_top_of_stack in sp1 so it's always accessible.
	 * Linux does not use ring 1, so sp1 is not otherwise needed.
	 */
314
	u64			sp1;
315

316 317 318 319 320 321 322 323
	u64			sp2;
	u64			reserved2;
	u64			ist[7];
	u32			reserved3;
	u32			reserved4;
	u16			reserved5;
	u16			io_bitmap_base;

324
} __attribute__((packed));
325 326 327
#endif

/*
328
 * IO-bitmap sizes:
329
 */
330 331 332
#define IO_BITMAP_BITS			65536
#define IO_BITMAP_BYTES			(IO_BITMAP_BITS/8)
#define IO_BITMAP_LONGS			(IO_BITMAP_BYTES/sizeof(long))
333
#define IO_BITMAP_OFFSET		(offsetof(struct tss_struct, io_bitmap) - offsetof(struct tss_struct, x86_tss))
334
#define INVALID_IO_BITMAP_OFFSET	0x8000
335

336
struct entry_stack {
337 338 339
	unsigned long		words[64];
};

340 341
struct entry_stack_page {
	struct entry_stack stack;
342
} __aligned(PAGE_SIZE);
343

344
struct tss_struct {
345
	/*
346 347 348
	 * The fixed hardware portion.  This must not cross a page boundary
	 * at risk of violating the SDM's advice and potentially triggering
	 * errata.
349 350
	 */
	struct x86_hw_tss	x86_tss;
351 352 353 354 355 356 357

	/*
	 * The extra 1 is there because the CPU will access an
	 * additional byte beyond the end of the IO permission
	 * bitmap. The extra byte must be all 1 bits, and must
	 * be within the limit.
	 */
358
	unsigned long		io_bitmap[IO_BITMAP_LONGS + 1];
359
} __aligned(PAGE_SIZE);
360

361
DECLARE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss_rw);
362

363 364 365 366 367 368 369 370 371 372
/*
 * sizeof(unsigned long) coming from an extra "long" at the end
 * of the iobitmap.
 *
 * -1? seg base+limit should be pointing to the address of the
 * last valid byte
 */
#define __KERNEL_TSS_LIMIT	\
	(IO_BITMAP_OFFSET + IO_BITMAP_BYTES + sizeof(unsigned long) - 1)

373 374
#ifdef CONFIG_X86_32
DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack);
375
#else
376 377
/* The RO copy can't be accessed with this_cpu_xyz(), so use the RW copy. */
#define cpu_current_top_of_stack cpu_tss_rw.x86_tss.sp1
378 379
#endif

380 381 382
/*
 * Save the original ist values for checking stack pointers during debugging
 */
383
struct orig_ist {
384
	unsigned long		ist[7];
385 386
};

387
#ifdef CONFIG_X86_64
388
DECLARE_PER_CPU(struct orig_ist, orig_ist);
389

390 391 392 393 394 395 396 397 398 399 400 401 402
union irq_stack_union {
	char irq_stack[IRQ_STACK_SIZE];
	/*
	 * GCC hardcodes the stack canary as %gs:40.  Since the
	 * irq_stack is the object at %gs:0, we reserve the bottom
	 * 48 bytes of the irq stack for the canary.
	 */
	struct {
		char gs_base[40];
		unsigned long stack_canary;
	};
};

403
DECLARE_PER_CPU_FIRST(union irq_stack_union, irq_stack_union) __visible;
404 405
DECLARE_INIT_PER_CPU(irq_stack_union);

406 407 408 409 410
static inline unsigned long cpu_kernelmode_gs_base(int cpu)
{
	return (unsigned long)per_cpu(irq_stack_union.gs_base, cpu);
}

411
DECLARE_PER_CPU(char *, irq_stack_ptr);
412 413
DECLARE_PER_CPU(unsigned int, irq_count);
extern asmlinkage void ignore_sysret(void);
414 415 416 417 418

#if IS_ENABLED(CONFIG_KVM)
/* Save actual FS/GS selectors and bases to current->thread */
void save_fsgs_for_kvm(void);
#endif
419
#else	/* X86_64 */
420
#ifdef CONFIG_STACKPROTECTOR
421 422 423 424 425 426 427 428 429 430
/*
 * Make sure stack canary segment base is cached-aligned:
 *   "For Intel Atom processors, avoid non zero segment base address
 *    that is not aligned to cache line boundary at all cost."
 * (Optim Ref Manual Assembly/Compiler Coding Rule 15.)
 */
struct stack_canary {
	char __pad[20];		/* canary at %gs:20 */
	unsigned long canary;
};
431
DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
432
#endif
433 434 435 436 437 438 439 440 441
/*
 * per-CPU IRQ handling stacks
 */
struct irq_stack {
	u32                     stack[THREAD_SIZE/sizeof(u32)];
} __aligned(THREAD_SIZE);

DECLARE_PER_CPU(struct irq_stack *, hardirq_stack);
DECLARE_PER_CPU(struct irq_stack *, softirq_stack);
442
#endif	/* X86_64 */
443

444
extern unsigned int fpu_kernel_xstate_size;
445
extern unsigned int fpu_user_xstate_size;
446

447 448
struct perf_event;

449 450 451 452
typedef struct {
	unsigned long		seg;
} mm_segment_t;

453
struct thread_struct {
454 455
	/* Cached TLS descriptors: */
	struct desc_struct	tls_array[GDT_ENTRY_TLS_ENTRIES];
456
#ifdef CONFIG_X86_32
457
	unsigned long		sp0;
458
#endif
459
	unsigned long		sp;
460
#ifdef CONFIG_X86_32
461
	unsigned long		sysenter_cs;
462
#else
463 464 465 466
	unsigned short		es;
	unsigned short		ds;
	unsigned short		fsindex;
	unsigned short		gsindex;
467
#endif
468

469
#ifdef CONFIG_X86_64
470 471 472 473 474 475 476 477 478
	unsigned long		fsbase;
	unsigned long		gsbase;
#else
	/*
	 * XXX: this could presumably be unsigned short.  Alternatively,
	 * 32-bit kernels could be taught to use fsindex instead.
	 */
	unsigned long fs;
	unsigned long gs;
479
#endif
480

481 482 483 484
	/* Save middle states of ptrace breakpoints */
	struct perf_event	*ptrace_bps[HBP_NUM];
	/* Debug status used for traps, single steps, etc... */
	unsigned long           debugreg6;
485 486
	/* Keep track of the exact dr7 value set by the user */
	unsigned long           ptrace_dr7;
487 488
	/* Fault info: */
	unsigned long		cr2;
489
	unsigned long		trap_nr;
490
	unsigned long		error_code;
491
#ifdef CONFIG_VM86
492
	/* Virtual 86 mode info */
493
	struct vm86		*vm86;
494
#endif
495 496 497 498 499
	/* IO permissions: */
	unsigned long		*io_bitmap_ptr;
	unsigned long		iopl;
	/* Max allowed port in the bitmap, in bytes: */
	unsigned		io_bitmap_max;
500

501 502
	mm_segment_t		addr_limit;

503
	unsigned int		sig_on_uaccess_err:1;
504 505
	unsigned int		uaccess_err:1;	/* uaccess failed */

506 507 508 509 510 511
	/* Floating point and extended processor state */
	struct fpu		fpu;
	/*
	 * WARNING: 'fpu' is dynamically-sized.  It *MUST* be at
	 * the end.
	 */
512 513
};

514 515 516 517 518 519 520 521
/* Whitelist the FPU state from the task_struct for hardened usercopy. */
static inline void arch_thread_struct_whitelist(unsigned long *offset,
						unsigned long *size)
{
	*offset = offsetof(struct thread_struct, fpu.state);
	*size = fpu_kernel_xstate_size;
}

522 523 524 525 526 527 528 529 530
/*
 * Thread-synchronous status.
 *
 * This is different from the flags in that nobody else
 * ever touches our thread-synchronous status, so we don't
 * have to worry about atomic accesses.
 */
#define TS_COMPAT		0x0002	/* 32bit syscall active (64BIT)*/

531 532 533 534 535 536 537
/*
 * Set IOPL bits in EFLAGS from given mask
 */
static inline void native_set_iopl_mask(unsigned mask)
{
#ifdef CONFIG_X86_32
	unsigned int reg;
538

539 540 541 542 543 544 545 546
	asm volatile ("pushfl;"
		      "popl %0;"
		      "andl %1, %0;"
		      "orl %2, %0;"
		      "pushl %0;"
		      "popfl"
		      : "=&r" (reg)
		      : "i" (~X86_EFLAGS_IOPL), "r" (mask));
547 548 549
#endif
}

550
static inline void
551
native_load_sp0(unsigned long sp0)
552
{
553
	this_cpu_write(cpu_tss_rw.x86_tss.sp0, sp0);
554
}
555

556 557 558 559 560 561 562
static inline void native_swapgs(void)
{
#ifdef CONFIG_X86_64
	asm volatile("swapgs" ::: "memory");
#endif
}

563
static inline unsigned long current_top_of_stack(void)
564
{
565 566 567 568 569
	/*
	 *  We can't read directly from tss.sp0: sp0 on x86_32 is special in
	 *  and around vm86 mode and sp0 on x86_64 is special because of the
	 *  entry trampoline.
	 */
570
	return this_cpu_read_stable(cpu_current_top_of_stack);
571 572
}

573 574 575 576 577 578
static inline bool on_thread_stack(void)
{
	return (unsigned long)(current_top_of_stack() -
			       current_stack_pointer) < THREAD_SIZE;
}

579 580 581
#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#else
582
#define __cpuid			native_cpuid
583

584
static inline void load_sp0(unsigned long sp0)
585
{
586
	native_load_sp0(sp0);
587 588
}

589
#define set_iopl_mask native_set_iopl_mask
590 591
#endif /* CONFIG_PARAVIRT */

592 593 594 595
/* Free all resources held by a thread. */
extern void release_thread(struct task_struct *);

unsigned long get_wchan(struct task_struct *p);
596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628

/*
 * Generic CPUID function
 * clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx
 * resulting in stale register contents being returned.
 */
static inline void cpuid(unsigned int op,
			 unsigned int *eax, unsigned int *ebx,
			 unsigned int *ecx, unsigned int *edx)
{
	*eax = op;
	*ecx = 0;
	__cpuid(eax, ebx, ecx, edx);
}

/* Some CPUID calls want 'count' to be placed in ecx */
static inline void cpuid_count(unsigned int op, int count,
			       unsigned int *eax, unsigned int *ebx,
			       unsigned int *ecx, unsigned int *edx)
{
	*eax = op;
	*ecx = count;
	__cpuid(eax, ebx, ecx, edx);
}

/*
 * CPUID functions returning a single datum
 */
static inline unsigned int cpuid_eax(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
629

630 631
	return eax;
}
632

633 634 635 636 637
static inline unsigned int cpuid_ebx(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
638

639 640
	return ebx;
}
641

642 643 644 645 646
static inline unsigned int cpuid_ecx(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
647

648 649
	return ecx;
}
650

651 652 653 654 655
static inline unsigned int cpuid_edx(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
656

657 658 659
	return edx;
}

660
/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */
661
static __always_inline void rep_nop(void)
662
{
663
	asm volatile("rep; nop" ::: "memory");
664 665
}

666
static __always_inline void cpu_relax(void)
667 668 669 670
{
	rep_nop();
}

671 672 673 674 675 676 677 678 679 680 681 682 683 684
/*
 * This function forces the icache and prefetched instruction stream to
 * catch up with reality in two very specific cases:
 *
 *  a) Text was modified using one virtual address and is about to be executed
 *     from the same physical page at a different virtual address.
 *
 *  b) Text was modified on a different CPU, may subsequently be
 *     executed on this CPU, and you want to make sure the new version
 *     gets executed.  This generally means you're calling this in a IPI.
 *
 * If you're calling this for a different reason, you're probably doing
 * it wrong.
 */
685 686
static inline void sync_core(void)
{
687
	/*
688 689 690 691 692 693 694 695 696 697 698 699
	 * There are quite a few ways to do this.  IRET-to-self is nice
	 * because it works on every CPU, at any CPL (so it's compatible
	 * with paravirtualization), and it never exits to a hypervisor.
	 * The only down sides are that it's a bit slow (it seems to be
	 * a bit more than 2x slower than the fastest options) and that
	 * it unmasks NMIs.  The "push %cs" is needed because, in
	 * paravirtual environments, __KERNEL_CS may not be a valid CS
	 * value when we do IRET directly.
	 *
	 * In case NMI unmasking or performance ever becomes a problem,
	 * the next best option appears to be MOV-to-CR2 and an
	 * unconditional jump.  That sequence also works on all CPUs,
700
	 * but it will fault at CPL3 (i.e. Xen PV).
701 702 703 704 705 706 707
	 *
	 * CPUID is the conventional way, but it's nasty: it doesn't
	 * exist on some 486-like CPUs, and it usually exits to a
	 * hypervisor.
	 *
	 * Like all of Linux's memory ordering operations, this is a
	 * compiler barrier as well.
708
	 */
709 710 711 712 713 714 715
#ifdef CONFIG_X86_32
	asm volatile (
		"pushfl\n\t"
		"pushl %%cs\n\t"
		"pushl $1f\n\t"
		"iret\n\t"
		"1:"
716
		: ASM_CALL_CONSTRAINT : : "memory");
717
#else
718 719 720
	unsigned int tmp;

	asm volatile (
721
		UNWIND_HINT_SAVE
722 723 724 725 726 727 728 729 730
		"mov %%ss, %0\n\t"
		"pushq %q0\n\t"
		"pushq %%rsp\n\t"
		"addq $8, (%%rsp)\n\t"
		"pushfq\n\t"
		"mov %%cs, %0\n\t"
		"pushq %q0\n\t"
		"pushq $1f\n\t"
		"iretq\n\t"
731
		UNWIND_HINT_RESTORE
732
		"1:"
733
		: "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc", "memory");
734
#endif
735 736 737
}

extern void select_idle_routine(const struct cpuinfo_x86 *c);
738
extern void amd_e400_c1e_apic_setup(void);
739

740
extern unsigned long		boot_option_idle_override;
741

742
enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT,
743
			 IDLE_POLL};
744

745 746 747
extern void enable_sep_cpu(void);
extern int sysenter_setup(void);

748
void early_trap_pf_init(void);
749

750
/* Defined in head.S */
751
extern struct desc_ptr		early_gdt_descr;
752

753
extern void switch_to_new_gdt(int);
754
extern void load_direct_gdt(int);
755
extern void load_fixmap_gdt(int);
756
extern void load_percpu_segment(int);
757 758
extern void cpu_init(void);

759 760
static inline unsigned long get_debugctlmsr(void)
{
Peter Zijlstra's avatar
Peter Zijlstra committed
761
	unsigned long debugctlmsr = 0;
762 763 764 765 766 767 768

#ifndef CONFIG_X86_DEBUGCTLMSR
	if (boot_cpu_data.x86 < 6)
		return 0;
#endif
	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);

Peter Zijlstra's avatar
Peter Zijlstra committed
769
	return debugctlmsr;
770 771
}

772 773 774 775 776 777 778 779 780
static inline void update_debugctlmsr(unsigned long debugctlmsr)
{
#ifndef CONFIG_X86_DEBUGCTLMSR
	if (boot_cpu_data.x86 < 6)
		return;
#endif
	wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
}

781 782
extern void set_task_blockstep(struct task_struct *task, bool on);

783 784
/* Boot loader type from the setup header: */
extern int			bootloader_type;
785
extern int			bootloader_version;
786

787
extern char			ignore_fpu_irq;
788 789 790 791 792

#define HAVE_ARCH_PICK_MMAP_LAYOUT 1
#define ARCH_HAS_PREFETCHW
#define ARCH_HAS_SPINLOCK_PREFETCH

793
#ifdef CONFIG_X86_32
794
# define BASE_PREFETCH		""
795
# define ARCH_HAS_PREFETCH
796
#else
797
# define BASE_PREFETCH		"prefetcht0 %P1"
798 799
#endif

800 801 802 803 804 805
/*
 * Prefetch instructions for Pentium III (+) and AMD Athlon (+)
 *
 * It's not worth to care about 3dnow prefetches for the K6
 * because they are microcoded there and very slow.
 */
806 807
static inline void prefetch(const void *x)
{
808
	alternative_input(BASE_PREFETCH, "prefetchnta %P1",
809
			  X86_FEATURE_XMM,
810
			  "m" (*(const char *)x));
811 812
}

813 814 815 816 817
/*
 * 3dnow prefetch to get an exclusive cache line.
 * Useful for spinlocks to avoid one state transition in the
 * cache coherency protocol:
 */
818 819
static inline void prefetchw(const void *x)
{
820 821 822
	alternative_input(BASE_PREFETCH, "prefetchw %P1",
			  X86_FEATURE_3DNOWPREFETCH,
			  "m" (*(const char *)x));
823 824
}

825 826 827 828 829
static inline void spin_lock_prefetch(const void *x)
{
	prefetchw(x);
}

830 831 832
#define TOP_OF_INIT_STACK ((unsigned long)&init_stack + sizeof(init_stack) - \
			   TOP_OF_KERNEL_STACK_PADDING)

833 834
#define task_top_of_stack(task) ((unsigned long)(task_pt_regs(task) + 1))

835 836 837 838 839 840 841
#define task_pt_regs(task) \
({									\
	unsigned long __ptr = (unsigned long)task_stack_page(task);	\
	__ptr += THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;		\
	((struct pt_regs *)__ptr) - 1;					\
})

842 843 844 845
#ifdef CONFIG_X86_32
/*
 * User space process size: 3GB (default).
 */
846
#define IA32_PAGE_OFFSET	PAGE_OFFSET
847
#define TASK_SIZE		PAGE_OFFSET
848
#define TASK_SIZE_LOW		TASK_SIZE
849
#define TASK_SIZE_MAX		TASK_SIZE
850
#define DEFAULT_MAP_WINDOW	TASK_SIZE
851 852 853 854
#define STACK_TOP		TASK_SIZE
#define STACK_TOP_MAX		STACK_TOP

#define INIT_THREAD  {							  \
855
	.sp0			= TOP_OF_INIT_STACK,			  \
856 857
	.sysenter_cs		= __KERNEL_CS,				  \
	.io_bitmap_ptr		= NULL,					  \
858
	.addr_limit		= KERNEL_DS,				  \
859 860
}

861
#define KSTK_ESP(task)		(task_pt_regs(task)->sp)
862 863 864

#else
/*
865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880
 * User space process size.  This is the first address outside the user range.
 * There are a few constraints that determine this:
 *
 * On Intel CPUs, if a SYSCALL instruction is at the highest canonical
 * address, then that syscall will enter the kernel with a
 * non-canonical return address, and SYSRET will explode dangerously.
 * We avoid this particular problem by preventing anything executable
 * from being mapped at the maximum canonical address.
 *
 * On AMD CPUs in the Ryzen family, there's a nasty bug in which the
 * CPUs malfunction if they execute code from the highest canonical page.
 * They'll speculate right off the end of the canonical space, and
 * bad things happen.  This is worked around in the same way as the
 * Intel problem.
 *
 * With page table isolation enabled, we map the LDT in ... [stay tuned]
881
 */
882
#define TASK_SIZE_MAX	((1UL << __VIRTUAL_MASK_SHIFT) - PAGE_SIZE)
883

884
#define DEFAULT_MAP_WINDOW	((1UL << 47) - PAGE_SIZE)
885 886 887 888

/* This decides where the kernel will search for a free chunk of vm
 * space during mmap's.
 */
889 890
#define IA32_PAGE_OFFSET	((current->personality & ADDR_LIMIT_3GB) ? \
					0xc0000000 : 0xFFFFe000)
891

892 893
#define TASK_SIZE_LOW		(test_thread_flag(TIF_ADDR32) ? \
					IA32_PAGE_OFFSET : DEFAULT_MAP_WINDOW)
894
#define TASK_SIZE		(test_thread_flag(TIF_ADDR32) ? \
895
					IA32_PAGE_OFFSET : TASK_SIZE_MAX)
896
#define TASK_SIZE_OF(child)	((test_tsk_thread_flag(child, TIF_ADDR32)) ? \
897
					IA32_PAGE_OFFSET : TASK_SIZE_MAX)
898

899
#define STACK_TOP		TASK_SIZE_LOW
900
#define STACK_TOP_MAX		TASK_SIZE_MAX
901

902 903
#define INIT_THREAD  {						\
	.addr_limit		= KERNEL_DS,			\
904 905
}

906
extern unsigned long KSTK_ESP(struct task_struct *task);
907

908 909
#endif /* CONFIG_X86_64 */

Ingo Molnar's avatar
Ingo Molnar committed
910 911 912
extern void start_thread(struct pt_regs *regs, unsigned long new_ip,
					       unsigned long new_sp);

913 914
/*
 * This decides where the kernel will search for a free chunk of vm
915 916
 * space during mmap's.
 */
917
#define __TASK_UNMAPPED_BASE(task_size)	(PAGE_ALIGN(task_size / 3))
918
#define TASK_UNMAPPED_BASE		__TASK_UNMAPPED_BASE(TASK_SIZE_LOW)
919

920
#define KSTK_EIP(task)		(task_pt_regs(task)->ip)
921

922 923 924 925 926 927 928
/* Get/set a process' ability to use the timestamp counter instruction */
#define GET_TSC_CTL(adr)	get_tsc_mode((adr))
#define SET_TSC_CTL(val)	set_tsc_mode((val))

extern int get_tsc_mode(unsigned long adr);
extern int set_tsc_mode(unsigned int val);

929 930
DECLARE_PER_CPU(u64, msr_misc_features_shadow);

931
/* Register/unregister a process' MPX related resource */
932 933
#define MPX_ENABLE_MANAGEMENT()	mpx_enable_management()
#define MPX_DISABLE_MANAGEMENT()	mpx_disable_management()
934 935

#ifdef CONFIG_X86_INTEL_MPX
936 937
extern int mpx_enable_management(void);
extern int mpx_disable_management(void);
938
#else
939
static inline int mpx_enable_management(void)
940 941 942
{
	return -EINVAL;
}
943
static inline int mpx_disable_management(void)
944 945 946 947 948
{
	return -EINVAL;
}
#endif /* CONFIG_X86_INTEL_MPX */

949
#ifdef CONFIG_CPU_SUP_AMD
950
extern u16 amd_get_nb_id(int cpu);
951
extern u32 amd_get_nodes_per_socket