common.c 51.2 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
2 3 4
/* cpu_feature_enabled() cannot be used this early */
#define USE_EARLY_PGTABLE_L5

5
#include <linux/memblock.h>
6
#include <linux/linkage.h>
7
#include <linux/bitops.h>
8
#include <linux/kernel.h>
9
#include <linux/export.h>
10 11
#include <linux/percpu.h>
#include <linux/string.h>
12
#include <linux/ctype.h>
Linus Torvalds's avatar
Linus Torvalds committed
13
#include <linux/delay.h>
14
#include <linux/sched/mm.h>
15
#include <linux/sched/clock.h>
16
#include <linux/sched/task.h>
17
#include <linux/sched/smt.h>
18
#include <linux/init.h>
19
#include <linux/kprobes.h>
20
#include <linux/kgdb.h>
Linus Torvalds's avatar
Linus Torvalds committed
21
#include <linux/smp.h>
22
#include <linux/io.h>
23
#include <linux/syscore_ops.h>
24 25

#include <asm/stackprotector.h>
26
#include <asm/perf_event.h>
Linus Torvalds's avatar
Linus Torvalds committed
27
#include <asm/mmu_context.h>
28
#include <asm/doublefault.h>
29
#include <asm/archrandom.h>
30 31
#include <asm/hypervisor.h>
#include <asm/processor.h>
32
#include <asm/tlbflush.h>
33
#include <asm/debugreg.h>
34
#include <asm/sections.h>
35
#include <asm/vsyscall.h>
36 37
#include <linux/topology.h>
#include <linux/cpumask.h>
38
#include <asm/pgtable.h>
Arun Sharma's avatar
Arun Sharma committed
39
#include <linux/atomic.h>
40 41 42 43
#include <asm/proto.h>
#include <asm/setup.h>
#include <asm/apic.h>
#include <asm/desc.h>
44
#include <asm/fpu/internal.h>
45
#include <asm/mtrr.h>
46
#include <asm/hwcap2.h>
47
#include <linux/numa.h>
48
#include <asm/asm.h>
49
#include <asm/bugs.h>
50
#include <asm/cpu.h>
51
#include <asm/mce.h>
52
#include <asm/msr.h>
53
#include <asm/memtype.h>
54 55
#include <asm/microcode.h>
#include <asm/microcode_intel.h>
56 57
#include <asm/intel-family.h>
#include <asm/cpu_device_id.h>
Tejun Heo's avatar
Tejun Heo committed
58
#include <asm/uv/uv.h>
59
#include <asm/resctrl_sched.h>
Linus Torvalds's avatar
Linus Torvalds committed
60 61 62

#include "cpu.h"

63 64
u32 elf_hwcap2 __read_mostly;

65 66
/* all of these masks are initialized in setup_cpu_local_masks() */
cpumask_var_t cpu_initialized_mask;
67 68
cpumask_var_t cpu_callout_mask;
cpumask_var_t cpu_callin_mask;
69 70 71 72

/* representing cpus for which sibling maps can be computed */
cpumask_var_t cpu_sibling_setup_mask;

73 74 75 76 77 78 79
/* Number of siblings per CPU package */
int smp_num_siblings = 1;
EXPORT_SYMBOL(smp_num_siblings);

/* Last level cache ID of each logical CPU */
DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;

80
/* correctly size the local cpu masks */
81
void __init setup_cpu_local_masks(void)
82 83 84 85 86 87 88
{
	alloc_bootmem_cpumask_var(&cpu_initialized_mask);
	alloc_bootmem_cpumask_var(&cpu_callin_mask);
	alloc_bootmem_cpumask_var(&cpu_callout_mask);
	alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
}

89
static void default_init(struct cpuinfo_x86 *c)
90 91
{
#ifdef CONFIG_X86_64
92
	cpu_detect_cache_sizes(c);
93 94 95 96 97 98 99 100 101 102 103 104 105
#else
	/* Not much we can do here... */
	/* Check if at least it has cpuid */
	if (c->cpuid_level == -1) {
		/* No cpuid. It must be an ancient CPU */
		if (c->x86 == 4)
			strcpy(c->x86_model_id, "486");
		else if (c->x86 == 3)
			strcpy(c->x86_model_id, "386");
	}
#endif
}

106
static const struct cpu_dev default_cpu = {
107 108 109 110 111
	.c_init		= default_init,
	.c_vendor	= "Unknown",
	.c_x86_vendor	= X86_VENDOR_UNKNOWN,
};

112
static const struct cpu_dev *this_cpu = &default_cpu;
113

114
DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
Yinghai Lu's avatar
Yinghai Lu committed
115
#ifdef CONFIG_X86_64
116 117 118 119 120
	/*
	 * We need valid kernel segments for data and code in long mode too
	 * IRET will check the segment types  kkeil 2000/10/28
	 * Also sysret mandates a special GDT layout
	 *
121
	 * TLS descriptors are currently at a different place compared to i386.
122 123
	 * Hopefully nobody expects them at a fixed place (Wine?)
	 */
Akinobu Mita's avatar
Akinobu Mita committed
124 125 126 127 128 129
	[GDT_ENTRY_KERNEL32_CS]		= GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc093, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER32_CS]	= GDT_ENTRY_INIT(0xc0fb, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f3, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xa0fb, 0, 0xfffff),
Yinghai Lu's avatar
Yinghai Lu committed
130
#else
Akinobu Mita's avatar
Akinobu Mita committed
131 132 133 134
	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xc09a, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xc0fa, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f2, 0, 0xfffff),
135 136 137 138 139
	/*
	 * Segments used for calling PnP BIOS have byte granularity.
	 * They code segments and data segments have fixed 64k limits,
	 * the transfer segment sizes are set at run time.
	 */
140
	/* 32-bit code */
Akinobu Mita's avatar
Akinobu Mita committed
141
	[GDT_ENTRY_PNPBIOS_CS32]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
142
	/* 16-bit code */
Akinobu Mita's avatar
Akinobu Mita committed
143
	[GDT_ENTRY_PNPBIOS_CS16]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
144
	/* 16-bit data */
Akinobu Mita's avatar
Akinobu Mita committed
145
	[GDT_ENTRY_PNPBIOS_DS]		= GDT_ENTRY_INIT(0x0092, 0, 0xffff),
146
	/* 16-bit data */
Akinobu Mita's avatar
Akinobu Mita committed
147
	[GDT_ENTRY_PNPBIOS_TS1]		= GDT_ENTRY_INIT(0x0092, 0, 0),
148
	/* 16-bit data */
Akinobu Mita's avatar
Akinobu Mita committed
149
	[GDT_ENTRY_PNPBIOS_TS2]		= GDT_ENTRY_INIT(0x0092, 0, 0),
150 151 152 153
	/*
	 * The APM segments have byte granularity and their bases
	 * are set at run time.  All have 64k limits.
	 */
154
	/* 32-bit code */
Akinobu Mita's avatar
Akinobu Mita committed
155
	[GDT_ENTRY_APMBIOS_BASE]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
156
	/* 16-bit code */
Akinobu Mita's avatar
Akinobu Mita committed
157
	[GDT_ENTRY_APMBIOS_BASE+1]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
158
	/* data */
159
	[GDT_ENTRY_APMBIOS_BASE+2]	= GDT_ENTRY_INIT(0x4092, 0, 0xffff),
160

Akinobu Mita's avatar
Akinobu Mita committed
161 162
	[GDT_ENTRY_ESPFIX_SS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
	[GDT_ENTRY_PERCPU]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
163
	GDT_STACK_CANARY_INIT
Yinghai Lu's avatar
Yinghai Lu committed
164
#endif
165
} };
166
EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
167

168
#ifdef CONFIG_X86_64
169
static int __init x86_nopcid_setup(char *s)
170
{
171 172 173
	/* nopcid doesn't accept parameters */
	if (s)
		return -EINVAL;
174 175 176

	/* do not emit a message if the feature is not present */
	if (!boot_cpu_has(X86_FEATURE_PCID))
177
		return 0;
178 179 180

	setup_clear_cpu_cap(X86_FEATURE_PCID);
	pr_info("nopcid: PCID feature disabled\n");
181
	return 0;
182
}
183
early_param("nopcid", x86_nopcid_setup);
184 185
#endif

186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201
static int __init x86_noinvpcid_setup(char *s)
{
	/* noinvpcid doesn't accept parameters */
	if (s)
		return -EINVAL;

	/* do not emit a message if the feature is not present */
	if (!boot_cpu_has(X86_FEATURE_INVPCID))
		return 0;

	setup_clear_cpu_cap(X86_FEATURE_INVPCID);
	pr_info("noinvpcid: INVPCID feature disabled\n");
	return 0;
}
early_param("noinvpcid", x86_noinvpcid_setup);

202
#ifdef CONFIG_X86_32
203 204
static int cachesize_override = -1;
static int disable_x86_serial_nr = 1;
Linus Torvalds's avatar
Linus Torvalds committed
205

206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224
static int __init cachesize_setup(char *str)
{
	get_option(&str, &cachesize_override);
	return 1;
}
__setup("cachesize=", cachesize_setup);

static int __init x86_sep_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_SEP);
	return 1;
}
__setup("nosep", x86_sep_setup);

/* Standard macro to see if a specific flag is changeable */
static inline int flag_is_changeable_p(u32 flag)
{
	u32 f1, f2;

225 226 227 228 229 230 231
	/*
	 * Cyrix and IDT cpus allow disabling of CPUID
	 * so the code below may return different results
	 * when it is executed before and after enabling
	 * the CPUID. Add "volatile" to not allow gcc to
	 * optimize the subsequent calls to this function.
	 */
Ingo Molnar's avatar
Ingo Molnar committed
232 233 234 235 236 237 238 239 240 241 242
	asm volatile ("pushfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "movl %0, %1	\n\t"
		      "xorl %2, %0	\n\t"
		      "pushl %0		\n\t"
		      "popfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "popfl		\n\t"

243 244
		      : "=&r" (f1), "=&r" (f2)
		      : "ir" (flag));
245 246 247 248 249

	return ((f1^f2) & flag) != 0;
}

/* Probe for the CPUID instruction */
250
int have_cpuid_p(void)
251 252 253 254
{
	return flag_is_changeable_p(X86_EFLAGS_ID);
}

255
static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
256
{
Ingo Molnar's avatar
Ingo Molnar committed
257 258 259 260 261 262 263 264 265 266 267
	unsigned long lo, hi;

	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
		return;

	/* Disable processor serial number: */

	rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
	lo |= 0x200000;
	wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);

268
	pr_notice("CPU serial number disabled.\n");
Ingo Molnar's avatar
Ingo Molnar committed
269 270 271 272
	clear_cpu_cap(c, X86_FEATURE_PN);

	/* Disabling the serial number may affect the cpuid level */
	c->cpuid_level = cpuid_eax(0);
273 274 275 276 277 278 279 280
}

static int __init x86_serial_nr_setup(char *s)
{
	disable_x86_serial_nr = 0;
	return 1;
}
__setup("serialnumber", x86_serial_nr_setup);
281
#else
282 283 284 285 286 287 288
static inline int flag_is_changeable_p(u32 flag)
{
	return 1;
}
static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
{
}
289
#endif
290

291 292
static __init int setup_disable_smep(char *arg)
{
293
	setup_clear_cpu_cap(X86_FEATURE_SMEP);
294 295 296 297
	return 1;
}
__setup("nosmep", setup_disable_smep);

298
static __always_inline void setup_smep(struct cpuinfo_x86 *c)
299
{
300
	if (cpu_has(c, X86_FEATURE_SMEP))
301
		cr4_set_bits(X86_CR4_SMEP);
302 303
}

304 305
static __init int setup_disable_smap(char *arg)
{
306
	setup_clear_cpu_cap(X86_FEATURE_SMAP);
307 308 309 310
	return 1;
}
__setup("nosmap", setup_disable_smap);

311 312
static __always_inline void setup_smap(struct cpuinfo_x86 *c)
{
313
	unsigned long eflags = native_save_fl();
314 315 316 317

	/* This should have been cleared long ago */
	BUG_ON(eflags & X86_EFLAGS_AC);

318 319
	if (cpu_has(c, X86_FEATURE_SMAP)) {
#ifdef CONFIG_X86_SMAP
320
		cr4_set_bits(X86_CR4_SMAP);
321
#else
322
		cr4_clear_bits(X86_CR4_SMAP);
323 324
#endif
	}
325 326
}

327 328 329 330 331 332 333 334 335 336 337 338
static __always_inline void setup_umip(struct cpuinfo_x86 *c)
{
	/* Check the boot processor, plus build option for UMIP. */
	if (!cpu_feature_enabled(X86_FEATURE_UMIP))
		goto out;

	/* Check the current processor's cpuid bits. */
	if (!cpu_has(c, X86_FEATURE_UMIP))
		goto out;

	cr4_set_bits(X86_CR4_UMIP);

339
	pr_info_once("x86/cpu: User Mode Instruction Prevention (UMIP) activated\n");
340

341 342 343 344 345 346 347 348 349 350
	return;

out:
	/*
	 * Make sure UMIP is disabled in case it was enabled in a
	 * previous boot (e.g., via kexec).
	 */
	cr4_clear_bits(X86_CR4_UMIP);
}

351 352 353
/* These bits should not change their value after CPU init is finished. */
static const unsigned long cr4_pinned_mask =
	X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | X86_CR4_FSGSBASE;
354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377
static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
static unsigned long cr4_pinned_bits __ro_after_init;

void native_write_cr0(unsigned long val)
{
	unsigned long bits_missing = 0;

set_register:
	asm volatile("mov %0,%%cr0": "+r" (val), "+m" (__force_order));

	if (static_branch_likely(&cr_pinning)) {
		if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) {
			bits_missing = X86_CR0_WP;
			val |= bits_missing;
			goto set_register;
		}
		/* Warn after we've set the missing bits. */
		WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n");
	}
}
EXPORT_SYMBOL(native_write_cr0);

void native_write_cr4(unsigned long val)
{
378
	unsigned long bits_changed = 0;
379 380 381 382 383

set_register:
	asm volatile("mov %0,%%cr4": "+r" (val), "+m" (cr4_pinned_bits));

	if (static_branch_likely(&cr_pinning)) {
384 385 386
		if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) {
			bits_changed = (val & cr4_pinned_mask) ^ cr4_pinned_bits;
			val = (val & ~cr4_pinned_mask) | cr4_pinned_bits;
387 388
			goto set_register;
		}
389 390 391
		/* Warn after we've corrected the changed bits. */
		WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n",
			  bits_changed);
392 393 394 395 396 397 398 399 400 401 402
	}
}
EXPORT_SYMBOL(native_write_cr4);

void cr4_init(void)
{
	unsigned long cr4 = __read_cr4();

	if (boot_cpu_has(X86_FEATURE_PCID))
		cr4 |= X86_CR4_PCIDE;
	if (static_branch_likely(&cr_pinning))
403
		cr4 = (cr4 & ~cr4_pinned_mask) | cr4_pinned_bits;
404 405 406 407 408 409

	__write_cr4(cr4);

	/* Initialize cr4 shadow for this CPU. */
	this_cpu_write(cpu_tlbstate.cr4, cr4);
}
Kees Cook's avatar
Kees Cook committed
410 411 412 413 414 415 416 417

/*
 * Once CPU feature detection is finished (and boot params have been
 * parsed), record any of the sensitive CR bits that are set, and
 * enable CR pinning.
 */
static void __init setup_cr_pinning(void)
{
418
	cr4_pinned_bits = this_cpu_read(cpu_tlbstate.cr4) & cr4_pinned_mask;
Kees Cook's avatar
Kees Cook committed
419 420 421
	static_key_enable(&cr_pinning.key);
}

422 423 424 425 426 427 428
/*
 * Protection Keys are not available in 32-bit mode.
 */
static bool pku_disabled;

static __always_inline void setup_pku(struct cpuinfo_x86 *c)
{
429 430
	struct pkru_state *pk;

431 432 433 434
	/* check the boot processor, plus compile options for PKU: */
	if (!cpu_feature_enabled(X86_FEATURE_PKU))
		return;
	/* checks the actual processor's cpuid bits: */
435 436 437 438 439 440
	if (!cpu_has(c, X86_FEATURE_PKU))
		return;
	if (pku_disabled)
		return;

	cr4_set_bits(X86_CR4_PKE);
441 442 443
	pk = get_xsave_addr(&init_fpstate.xsave, XFEATURE_PKRU);
	if (pk)
		pk->pkru = init_pkru_value;
444 445 446 447 448
	/*
	 * Seting X86_CR4_PKE will cause the X86_FEATURE_OSPKE
	 * cpuid bit to be set.  We need to ensure that we
	 * update that bit in this CPU's "cpu_info".
	 */
449
	set_cpu_cap(c, X86_FEATURE_OSPKE);
450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472
}

#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
static __init int setup_disable_pku(char *arg)
{
	/*
	 * Do not clear the X86_FEATURE_PKU bit.  All of the
	 * runtime checks are against OSPKE so clearing the
	 * bit does nothing.
	 *
	 * This way, we will see "pku" in cpuinfo, but not
	 * "ospke", which is exactly what we want.  It shows
	 * that the CPU has PKU, but the OS has not enabled it.
	 * This happens to be exactly how a system would look
	 * if we disabled the config option.
	 */
	pr_info("x86: 'nopku' specified, disabling Memory Protection Keys\n");
	pku_disabled = true;
	return 1;
}
__setup("nopku", setup_disable_pku);
#endif /* CONFIG_X86_64 */

473 474 475 476 477 478 479 480 481
/*
 * Some CPU features depend on higher CPUID levels, which may not always
 * be available due to CPUID level capping or broken virtualization
 * software.  Add those features to this table to auto-disable them.
 */
struct cpuid_dependent_feature {
	u32 feature;
	u32 level;
};
Ingo Molnar's avatar
Ingo Molnar committed
482

483
static const struct cpuid_dependent_feature
484 485 486 487 488 489 490
cpuid_dependent_features[] = {
	{ X86_FEATURE_MWAIT,		0x00000005 },
	{ X86_FEATURE_DCA,		0x00000009 },
	{ X86_FEATURE_XSAVE,		0x0000000d },
	{ 0, 0 }
};

491
static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
492 493
{
	const struct cpuid_dependent_feature *df;
494

495
	for (df = cpuid_dependent_features; df->feature; df++) {
Ingo Molnar's avatar
Ingo Molnar committed
496 497 498

		if (!cpu_has(c, df->feature))
			continue;
499 500 501 502 503 504 505
		/*
		 * Note: cpuid_level is set to -1 if unavailable, but
		 * extended_extended_level is set to 0 if unavailable
		 * and the legitimate extended levels are all negative
		 * when signed; hence the weird messing around with
		 * signs here...
		 */
Ingo Molnar's avatar
Ingo Molnar committed
506
		if (!((s32)df->level < 0 ?
507
		     (u32)df->level > (u32)c->extended_cpuid_level :
Ingo Molnar's avatar
Ingo Molnar committed
508 509 510 511 512 513 514
		     (s32)df->level > (s32)c->cpuid_level))
			continue;

		clear_cpu_cap(c, df->feature);
		if (!warn)
			continue;

515 516
		pr_warn("CPU: CPU feature " X86_CAP_FMT " disabled, no CPUID level 0x%x\n",
			x86_cap_flag(df->feature), df->level);
517
	}
518
}
519

520 521 522
/*
 * Naming convention should be: <Name> [(<Codename>)]
 * This table only is used unless init_<vendor>() below doesn't set it;
Ingo Molnar's avatar
Ingo Molnar committed
523 524
 * in particular, if CPUID levels 0x80000002..4 are supported, this
 * isn't used
525 526 527
 */

/* Look up CPU names by table lookup. */
528
static const char *table_lookup_model(struct cpuinfo_x86 *c)
529
{
530 531
#ifdef CONFIG_X86_32
	const struct legacy_cpu_model_info *info;
532 533 534 535 536 537 538

	if (c->x86_model >= 16)
		return NULL;	/* Range check */

	if (!this_cpu)
		return NULL;

539
	info = this_cpu->legacy_models;
540

541
	while (info->family) {
542 543 544 545
		if (info->family == c->x86)
			return info->model_names[c->x86_model];
		info++;
	}
546
#endif
547 548 549
	return NULL;		/* Not found */
}

550 551 552
/* Aligned to unsigned long to avoid split lock in atomic bitmap ops */
__u32 cpu_caps_cleared[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
__u32 cpu_caps_set[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
553

554 555 556 557 558
void load_percpu_segment(int cpu)
{
#ifdef CONFIG_X86_32
	loadsegment(fs, __KERNEL_PERCPU);
#else
559
	__loadsegment_simple(gs, 0);
560
	wrmsrl(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu));
561
#endif
562
	load_stack_canary_segment();
563 564
}

565 566 567 568 569
#ifdef CONFIG_X86_32
/* The 32-bit entry code needs to find cpu_entry_area. */
DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
#endif

570 571 572 573 574 575 576 577 578 579 580
/* Load the original GDT from the per-cpu structure */
void load_direct_gdt(int cpu)
{
	struct desc_ptr gdt_descr;

	gdt_descr.address = (long)get_cpu_gdt_rw(cpu);
	gdt_descr.size = GDT_SIZE - 1;
	load_gdt(&gdt_descr);
}
EXPORT_SYMBOL_GPL(load_direct_gdt);

581 582 583 584 585 586 587 588 589
/* Load a fixmap remapping of the per-cpu GDT */
void load_fixmap_gdt(int cpu)
{
	struct desc_ptr gdt_descr;

	gdt_descr.address = (long)get_cpu_gdt_ro(cpu);
	gdt_descr.size = GDT_SIZE - 1;
	load_gdt(&gdt_descr);
}
590
EXPORT_SYMBOL_GPL(load_fixmap_gdt);
591

Ingo Molnar's avatar
Ingo Molnar committed
592 593 594 595
/*
 * Current gdt points %fs at the "master" per-cpu area: after this,
 * it's on the real one.
 */
596
void switch_to_new_gdt(int cpu)
597
{
598 599
	/* Load the original GDT */
	load_direct_gdt(cpu);
600
	/* Reload the per-cpu base */
601
	load_percpu_segment(cpu);
602 603
}

604
static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
Linus Torvalds's avatar
Linus Torvalds committed
605

606
static void get_model_name(struct cpuinfo_x86 *c)
Linus Torvalds's avatar
Linus Torvalds committed
607 608
{
	unsigned int *v;
609
	char *p, *q, *s;
Linus Torvalds's avatar
Linus Torvalds committed
610

611
	if (c->extended_cpuid_level < 0x80000004)
612
		return;
Linus Torvalds's avatar
Linus Torvalds committed
613

Ingo Molnar's avatar
Ingo Molnar committed
614
	v = (unsigned int *)c->x86_model_id;
Linus Torvalds's avatar
Linus Torvalds committed
615 616 617 618 619
	cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
	cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
	cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
	c->x86_model_id[48] = 0;

620 621 622 623 624 625 626 627 628 629 630 631 632 633 634
	/* Trim whitespace */
	p = q = s = &c->x86_model_id[0];

	while (*p == ' ')
		p++;

	while (*p) {
		/* Note the last non-whitespace index */
		if (!isspace(*p))
			s = q;

		*q++ = *p++;
	}

	*(s + 1) = '\0';
Linus Torvalds's avatar
Linus Torvalds committed
635 636
}

637
void detect_num_cpu_cores(struct cpuinfo_x86 *c)
638 639 640
{
	unsigned int eax, ebx, ecx, edx;

641
	c->x86_max_cores = 1;
642
	if (!IS_ENABLED(CONFIG_SMP) || c->cpuid_level < 4)
643
		return;
644 645 646

	cpuid_count(4, 0, &eax, &ebx, &ecx, &edx);
	if (eax & 0x1f)
647
		c->x86_max_cores = (eax >> 26) + 1;
648 649
}

650
void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
Linus Torvalds's avatar
Linus Torvalds committed
651
{
652
	unsigned int n, dummy, ebx, ecx, edx, l2size;
Linus Torvalds's avatar
Linus Torvalds committed
653

654
	n = c->extended_cpuid_level;
Linus Torvalds's avatar
Linus Torvalds committed
655 656

	if (n >= 0x80000005) {
657 658
		cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
		c->x86_cache_size = (ecx>>24) + (edx>>24);
659 660 661 662
#ifdef CONFIG_X86_64
		/* On K8 L1 TLB is inclusive, so don't count it */
		c->x86_tlbsize = 0;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
663 664 665 666 667
	}

	if (n < 0x80000006)	/* Some chips just has a large L1. */
		return;

668
	cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
Linus Torvalds's avatar
Linus Torvalds committed
669
	l2size = ecx >> 16;
670

671 672 673
#ifdef CONFIG_X86_64
	c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
#else
Linus Torvalds's avatar
Linus Torvalds committed
674
	/* do processor-specific cache resizing */
675 676
	if (this_cpu->legacy_cache_size)
		l2size = this_cpu->legacy_cache_size(c, l2size);
Linus Torvalds's avatar
Linus Torvalds committed
677 678 679 680 681

	/* Allow user to override all this if necessary. */
	if (cachesize_override != -1)
		l2size = cachesize_override;

682
	if (l2size == 0)
Linus Torvalds's avatar
Linus Torvalds committed
683
		return;		/* Again, no L2 cache is possible */
684
#endif
Linus Torvalds's avatar
Linus Torvalds committed
685 686 687 688

	c->x86_cache_size = l2size;
}

689 690 691 692 693 694
u16 __read_mostly tlb_lli_4k[NR_INFO];
u16 __read_mostly tlb_lli_2m[NR_INFO];
u16 __read_mostly tlb_lli_4m[NR_INFO];
u16 __read_mostly tlb_lld_4k[NR_INFO];
u16 __read_mostly tlb_lld_2m[NR_INFO];
u16 __read_mostly tlb_lld_4m[NR_INFO];
695
u16 __read_mostly tlb_lld_1g[NR_INFO];
696

697
static void cpu_detect_tlb(struct cpuinfo_x86 *c)
698 699 700 701
{
	if (this_cpu->c_detect_tlb)
		this_cpu->c_detect_tlb(c);

702
	pr_info("Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
703
		tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
704 705 706 707 708
		tlb_lli_4m[ENTRIES]);

	pr_info("Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n",
		tlb_lld_4k[ENTRIES], tlb_lld_2m[ENTRIES],
		tlb_lld_4m[ENTRIES], tlb_lld_1g[ENTRIES]);
709 710
}

711
int detect_ht_early(struct cpuinfo_x86 *c)
Linus Torvalds's avatar
Linus Torvalds committed
712
{
Borislav Petkov's avatar
Borislav Petkov committed
713
#ifdef CONFIG_SMP
714
	u32 eax, ebx, ecx, edx;
Linus Torvalds's avatar
Linus Torvalds committed
715

716
	if (!cpu_has(c, X86_FEATURE_HT))
717
		return -1;
Linus Torvalds's avatar
Linus Torvalds committed
718

719
	if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
720
		return -1;
Linus Torvalds's avatar
Linus Torvalds committed
721

722
	if (cpu_has(c, X86_FEATURE_XTOPOLOGY))
723
		return -1;
Linus Torvalds's avatar
Linus Torvalds committed
724

725
	cpuid(1, &eax, &ebx, &ecx, &edx);
Linus Torvalds's avatar
Linus Torvalds committed
726

727
	smp_num_siblings = (ebx & 0xff0000) >> 16;
728
	if (smp_num_siblings == 1)
729
		pr_info_once("CPU0: Hyper-Threading is disabled\n");
730 731 732
#endif
	return 0;
}
733

734 735 736 737
void detect_ht(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_SMP
	int index_msb, core_bits;
738

739
	if (detect_ht_early(c) < 0)
740
		return;
741

Ingo Molnar's avatar
Ingo Molnar committed
742 743
	index_msb = get_count_order(smp_num_siblings);
	c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, index_msb);
744

Ingo Molnar's avatar
Ingo Molnar committed
745
	smp_num_siblings = smp_num_siblings / c->x86_max_cores;
746

Ingo Molnar's avatar
Ingo Molnar committed
747
	index_msb = get_count_order(smp_num_siblings);
748

Ingo Molnar's avatar
Ingo Molnar committed
749
	core_bits = get_count_order(c->x86_max_cores);
750

Ingo Molnar's avatar
Ingo Molnar committed
751 752
	c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, index_msb) &
				       ((1 << core_bits) - 1);
753
#endif
754
}
Linus Torvalds's avatar
Linus Torvalds committed
755

756
static void get_cpu_vendor(struct cpuinfo_x86 *c)
Linus Torvalds's avatar
Linus Torvalds committed
757 758
{
	char *v = c->x86_vendor_id;
Ingo Molnar's avatar
Ingo Molnar committed
759
	int i;
Linus Torvalds's avatar
Linus Torvalds committed
760 761

	for (i = 0; i < X86_VENDOR_NUM; i++) {
Yinghai Lu's avatar
Yinghai Lu committed
762 763 764 765 766 767
		if (!cpu_devs[i])
			break;

		if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
		    (cpu_devs[i]->c_ident[1] &&
		     !strcmp(v, cpu_devs[i]->c_ident[1]))) {
Ingo Molnar's avatar
Ingo Molnar committed
768

Yinghai Lu's avatar
Yinghai Lu committed
769 770 771
			this_cpu = cpu_devs[i];
			c->x86_vendor = this_cpu->c_x86_vendor;
			return;
Linus Torvalds's avatar
Linus Torvalds committed
772 773
		}
	}
Yinghai Lu's avatar
Yinghai Lu committed
774

775 776
	pr_err_once("CPU: vendor_id '%s' unknown, using generic init.\n" \
		    "CPU: Your system may be unstable.\n", v);
Yinghai Lu's avatar
Yinghai Lu committed
777

778 779
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	this_cpu = &default_cpu;
Linus Torvalds's avatar
Linus Torvalds committed
780 781
}

782
void cpu_detect(struct cpuinfo_x86 *c)
Linus Torvalds's avatar
Linus Torvalds committed
783 784
{
	/* Get vendor name */
785 786 787 788
	cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
	      (unsigned int *)&c->x86_vendor_id[0],
	      (unsigned int *)&c->x86_vendor_id[8],
	      (unsigned int *)&c->x86_vendor_id[4]);
Linus Torvalds's avatar
Linus Torvalds committed
789 790

	c->x86 = 4;
791
	/* Intel-defined flags: level 0x00000001 */
Linus Torvalds's avatar
Linus Torvalds committed
792 793
	if (c->cpuid_level >= 0x00000001) {
		u32 junk, tfms, cap0, misc;
Ingo Molnar's avatar
Ingo Molnar committed
794

Linus Torvalds's avatar
Linus Torvalds committed
795
		cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
796 797
		c->x86		= x86_family(tfms);
		c->x86_model	= x86_model(tfms);
798
		c->x86_stepping	= x86_stepping(tfms);
Ingo Molnar's avatar
Ingo Molnar committed
799

800 801
		if (cap0 & (1<<19)) {
			c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
802
			c->x86_cache_alignment = c->x86_clflush_size;
803
		}
Linus Torvalds's avatar
Linus Torvalds committed
804 805
	}
}
806

807 808 809 810
static void apply_forced_caps(struct cpuinfo_x86 *c)
{
	int i;

811
	for (i = 0; i < NCAPINTS + NBUGINTS; i++) {
812 813 814 815 816
		c->x86_capability[i] &= ~cpu_caps_cleared[i];
		c->x86_capability[i] |= cpu_caps_set[i];
	}
}

817 818 819 820 821 822 823 824 825 826 827
static void init_speculation_control(struct cpuinfo_x86 *c)
{
	/*
	 * The Intel SPEC_CTRL CPUID bit implies IBRS and IBPB support,
	 * and they also have a different bit for STIBP support. Also,
	 * a hypervisor might have set the individual AMD bits even on
	 * Intel CPUs, for finer-grained selection of what's available.
	 */
	if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
		set_cpu_cap(c, X86_FEATURE_IBRS);
		set_cpu_cap(c, X86_FEATURE_IBPB);
828
		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
829
	}
830

831 832
	if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
		set_cpu_cap(c, X86_FEATURE_STIBP);
833

834 835
	if (cpu_has(c, X86_FEATURE_SPEC_CTRL_SSBD) ||
	    cpu_has(c, X86_FEATURE_VIRT_SSBD))
836 837
		set_cpu_cap(c, X86_FEATURE_SSBD);

838
	if (cpu_has(c, X86_FEATURE_AMD_IBRS)) {
839
		set_cpu_cap(c, X86_FEATURE_IBRS);
840 841
		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
	}
842 843 844 845

	if (cpu_has(c, X86_FEATURE_AMD_IBPB))
		set_cpu_cap(c, X86_FEATURE_IBPB);

846
	if (cpu_has(c, X86_FEATURE_AMD_STIBP)) {
847
		set_cpu_cap(c, X86_FEATURE_STIBP);
848 849
		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
	}
850 851 852 853 854 855

	if (cpu_has(c, X86_FEATURE_AMD_SSBD)) {
		set_cpu_cap(c, X86_FEATURE_SSBD);
		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
		clear_cpu_cap(c, X86_FEATURE_VIRT_SSBD);
	}
856 857
}

858
void get_cpu_cap(struct cpuinfo_x86 *c)
859
{
860
	u32 eax, ebx, ecx, edx;
861

862 863
	/* Intel-defined flags: level 0x00000001 */
	if (c->cpuid_level >= 0x00000001) {
864
		cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
Ingo Molnar's avatar
Ingo Molnar committed
865

866 867
		c->x86_capability[CPUID_1_ECX] = ecx;
		c->x86_capability[CPUID_1_EDX] = edx;
868
	}
869

870 871 872 873
	/* Thermal and Power Management Leaf: level 0x00000006 (eax) */
	if (c->cpuid_level >= 0x00000006)
		c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006);

874 875 876
	/* Additional Intel-defined flags: level 0x00000007 */
	if (c->cpuid_level >= 0x00000007) {
		cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
877
		c->x86_capability[CPUID_7_0_EBX] = ebx;
878
		c->x86_capability[CPUID_7_ECX] = ecx;
879
		c->x86_capability[CPUID_7_EDX] = edx;
880 881 882 883 884 885

		/* Check valid sub-leaf index before accessing it */
		if (eax >= 1) {
			cpuid_count(0x00000007, 1, &eax, &ebx, &ecx, &edx);
			c->x86_capability[CPUID_7_1_EAX] = eax;
		}
886 887
	}

888 889 890 891
	/* Extended state features: level 0x0000000d */
	if (c->cpuid_level >= 0x0000000d) {
		cpuid_count(0x0000000d, 1, &eax, &ebx, &ecx, &edx);

892
		c->x86_capability[CPUID_D_1_EAX] = eax;
893 894
	}

895
	/* AMD-defined flags: level 0x80000001 */
896 897 898 899 900 901
	eax = cpuid_eax(0x80000000);
	c->extended_cpuid_level = eax;

	if ((eax & 0xffff0000) == 0x80000000) {
		if (eax >= 0x80000001) {
			cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
Ingo Molnar's avatar
Ingo Molnar committed
902

903 904
			c->x86_capability[CPUID_8000_0001_ECX] = ecx;
			c->x86_capability[CPUID_8000_0001_EDX] = edx;
905 906 907
		}
	}

908 909 910 911 912 913 914
	if (c->extended_cpuid_level >= 0x80000007