init.c 26.3 KB
Newer Older
1
#include <linux/gfp.h>
2
#include <linux/initrd.h>
3
#include <linux/ioport.h>
4
#include <linux/swap.h>
5
#include <linux/memblock.h>
6
#include <linux/bootmem.h>	/* for max_low_pfn */
7 8
#include <linux/swapfile.h>
#include <linux/swapops.h>
9

Laura Abbott's avatar
Laura Abbott committed
10
#include <asm/set_memory.h>
11
#include <asm/e820/api.h>
12
#include <asm/init.h>
13
#include <asm/page.h>
14
#include <asm/page_types.h>
15
#include <asm/sections.h>
16
#include <asm/setup.h>
17
#include <asm/tlbflush.h>
18
#include <asm/tlb.h>
19
#include <asm/proto.h>
20
#include <asm/dma.h>		/* for MAX_DMA_PFN */
21
#include <asm/microcode.h>
22
#include <asm/kaslr.h>
23
#include <asm/hypervisor.h>
24
#include <asm/cpufeature.h>
25
#include <asm/pti.h>
26

27 28 29 30 31 32 33
/*
 * We need to define the tracepoints somewhere, and tlb.c
 * is only compied when SMP=y.
 */
#define CREATE_TRACE_POINTS
#include <trace/events/tlb.h>

34 35
#include "mm_internal.h"

36 37
/*
 * Tables translating between page_cache_type_t and pte encoding.
38
 *
39 40 41 42 43
 * The default values are defined statically as minimal supported mode;
 * WC and WT fall back to UC-.  pat_init() updates these values to support
 * more cache modes, WC and WT, when it is safe to do so.  See pat_init()
 * for the details.  Note, __early_ioremap() used during early boot-time
 * takes pgprot_t (pte encoding) and does not use these tables.
44 45 46 47 48
 *
 *   Index into __cachemode2pte_tbl[] is the cachemode.
 *
 *   Index into __pte2cachemode_tbl[] are the caching attribute bits of the pte
 *   (_PAGE_PWT, _PAGE_PCD, _PAGE_PAT) at index bit positions 0, 1, 2.
49 50
 */
uint16_t __cachemode2pte_tbl[_PAGE_CACHE_MODE_NUM] = {
51
	[_PAGE_CACHE_MODE_WB      ]	= 0         | 0        ,
52
	[_PAGE_CACHE_MODE_WC      ]	= 0         | _PAGE_PCD,
53 54 55 56
	[_PAGE_CACHE_MODE_UC_MINUS]	= 0         | _PAGE_PCD,
	[_PAGE_CACHE_MODE_UC      ]	= _PAGE_PWT | _PAGE_PCD,
	[_PAGE_CACHE_MODE_WT      ]	= 0         | _PAGE_PCD,
	[_PAGE_CACHE_MODE_WP      ]	= 0         | _PAGE_PCD,
57
};
58
EXPORT_SYMBOL(__cachemode2pte_tbl);
59

60
uint8_t __pte2cachemode_tbl[8] = {
61
	[__pte2cm_idx( 0        | 0         | 0        )] = _PAGE_CACHE_MODE_WB,
62
	[__pte2cm_idx(_PAGE_PWT | 0         | 0        )] = _PAGE_CACHE_MODE_UC_MINUS,
63 64 65
	[__pte2cm_idx( 0        | _PAGE_PCD | 0        )] = _PAGE_CACHE_MODE_UC_MINUS,
	[__pte2cm_idx(_PAGE_PWT | _PAGE_PCD | 0        )] = _PAGE_CACHE_MODE_UC,
	[__pte2cm_idx( 0        | 0         | _PAGE_PAT)] = _PAGE_CACHE_MODE_WB,
66
	[__pte2cm_idx(_PAGE_PWT | 0         | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC_MINUS,
67
	[__pte2cm_idx(0         | _PAGE_PCD | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC_MINUS,
68 69
	[__pte2cm_idx(_PAGE_PWT | _PAGE_PCD | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC,
};
70
EXPORT_SYMBOL(__pte2cachemode_tbl);
71

72 73 74
static unsigned long __initdata pgt_buf_start;
static unsigned long __initdata pgt_buf_end;
static unsigned long __initdata pgt_buf_top;
75

76 77
static unsigned long min_pfn_mapped;

78 79
static bool __initdata can_use_brk_pgt = true;

80 81 82 83 84 85 86 87 88
/*
 * Pages returned are already directly mapped.
 *
 * Changing that is likely to break Xen, see commit:
 *
 *    279b706 x86,xen: introduce x86_init.mapping.pagetable_reserve
 *
 * for detailed information.
 */
Yinghai Lu's avatar
Yinghai Lu committed
89
__ref void *alloc_low_pages(unsigned int num)
90 91
{
	unsigned long pfn;
Yinghai Lu's avatar
Yinghai Lu committed
92
	int i;
93 94

	if (after_bootmem) {
Yinghai Lu's avatar
Yinghai Lu committed
95
		unsigned int order;
96

Yinghai Lu's avatar
Yinghai Lu committed
97
		order = get_order((unsigned long)num << PAGE_SHIFT);
98
		return (void *)__get_free_pages(GFP_ATOMIC | __GFP_ZERO, order);
99 100
	}

101
	if ((pgt_buf_end + num) > pgt_buf_top || !can_use_brk_pgt) {
102 103
		unsigned long ret;
		if (min_pfn_mapped >= max_pfn_mapped)
104
			panic("alloc_low_pages: ran out of memory");
105 106
		ret = memblock_find_in_range(min_pfn_mapped << PAGE_SHIFT,
					max_pfn_mapped << PAGE_SHIFT,
Yinghai Lu's avatar
Yinghai Lu committed
107
					PAGE_SIZE * num , PAGE_SIZE);
108
		if (!ret)
109
			panic("alloc_low_pages: can not alloc memory");
Yinghai Lu's avatar
Yinghai Lu committed
110
		memblock_reserve(ret, PAGE_SIZE * num);
111
		pfn = ret >> PAGE_SHIFT;
Yinghai Lu's avatar
Yinghai Lu committed
112 113 114
	} else {
		pfn = pgt_buf_end;
		pgt_buf_end += num;
115 116
		printk(KERN_DEBUG "BRK [%#010lx, %#010lx] PGTABLE\n",
			pfn << PAGE_SHIFT, (pgt_buf_end << PAGE_SHIFT) - 1);
Yinghai Lu's avatar
Yinghai Lu committed
117 118 119 120 121 122 123 124
	}

	for (i = 0; i < num; i++) {
		void *adr;

		adr = __va((pfn + i) << PAGE_SHIFT);
		clear_page(adr);
	}
125

Yinghai Lu's avatar
Yinghai Lu committed
126
	return __va(pfn << PAGE_SHIFT);
127 128
}

129 130 131 132 133 134 135 136 137 138 139 140
/*
 * By default need 3 4k for initial PMD_SIZE,  3 4k for 0-ISA_END_ADDRESS.
 * With KASLR memory randomization, depending on the machine e820 memory
 * and the PUD alignment. We may need twice more pages when KASLR memory
 * randomization is enabled.
 */
#ifndef CONFIG_RANDOMIZE_MEMORY
#define INIT_PGD_PAGE_COUNT      6
#else
#define INIT_PGD_PAGE_COUNT      12
#endif
#define INIT_PGT_BUF_SIZE	(INIT_PGD_PAGE_COUNT * PAGE_SIZE)
141 142 143 144 145 146 147 148 149 150 151 152 153
RESERVE_BRK(early_pgt_alloc, INIT_PGT_BUF_SIZE);
void  __init early_alloc_pgt_buf(void)
{
	unsigned long tables = INIT_PGT_BUF_SIZE;
	phys_addr_t base;

	base = __pa(extend_brk(tables, PAGE_SIZE));

	pgt_buf_start = base >> PAGE_SHIFT;
	pgt_buf_end = pgt_buf_start;
	pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT);
}

154 155
int after_bootmem;

156
early_param_on_off("gbpages", "nogbpages", direct_gbpages, CONFIG_X86_DIRECT_GBPAGES);
157

158 159 160 161 162 163
struct map_range {
	unsigned long start;
	unsigned long end;
	unsigned page_size_mask;
};

164
static int page_size_mask;
165

166
static void __init probe_page_size_mask(void)
167 168
{
	/*
169
	 * For pagealloc debugging, identity mapping will use small pages.
170 171 172
	 * This will simplify cpa(), which otherwise needs to support splitting
	 * large pages into small in interrupt context, etc.
	 */
173
	if (boot_cpu_has(X86_FEATURE_PSE) && !debug_pagealloc_enabled())
174
		page_size_mask |= 1 << PG_LEVEL_2M;
175 176
	else
		direct_gbpages = 0;
177 178

	/* Enable PSE if available */
179
	if (boot_cpu_has(X86_FEATURE_PSE))
180
		cr4_set_bits_and_update_boot(X86_CR4_PSE);
181 182

	/* Enable PGE if available */
183
	__supported_pte_mask &= ~_PAGE_GLOBAL;
184
	if (boot_cpu_has(X86_FEATURE_PGE)) {
185
		cr4_set_bits_and_update_boot(X86_CR4_PGE);
186
		__supported_pte_mask |= _PAGE_GLOBAL;
187
	}
188

189 190 191 192 193 194
	/* By the default is everything supported: */
	__default_kernel_pte_mask = __supported_pte_mask;
	/* Except when with PTI where the kernel is mostly non-Global: */
	if (cpu_feature_enabled(X86_FEATURE_PTI))
		__default_kernel_pte_mask &= ~_PAGE_GLOBAL;

195
	/* Enable 1 GB linear kernel mappings if available: */
196
	if (direct_gbpages && boot_cpu_has(X86_FEATURE_GBPAGES)) {
197 198 199 200 201
		printk(KERN_INFO "Using GB pages for direct mapping\n");
		page_size_mask |= 1 << PG_LEVEL_1G;
	} else {
		direct_gbpages = 0;
	}
202
}
203

204 205
static void setup_pcid(void)
{
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
	if (!IS_ENABLED(CONFIG_X86_64))
		return;

	if (!boot_cpu_has(X86_FEATURE_PCID))
		return;

	if (boot_cpu_has(X86_FEATURE_PGE)) {
		/*
		 * This can't be cr4_set_bits_and_update_boot() -- the
		 * trampoline code can't handle CR4.PCIDE and it wouldn't
		 * do any good anyway.  Despite the name,
		 * cr4_set_bits_and_update_boot() doesn't actually cause
		 * the bits in question to remain set all the way through
		 * the secondary boot asm.
		 *
		 * Instead, we brute-force it and set CR4.PCIDE manually in
		 * start_secondary().
		 */
		cr4_set_bits(X86_CR4_PCIDE);

		/*
		 * INVPCID's single-context modes (2/3) only work if we set
		 * X86_CR4_PCIDE, *and* we INVPCID support.  It's unusable
		 * on systems that have X86_CR4_PCIDE clear, or that have
		 * no INVPCID support at all.
		 */
		if (boot_cpu_has(X86_FEATURE_INVPCID))
			setup_force_cpu_cap(X86_FEATURE_INVPCID_SINGLE);
	} else {
		/*
		 * flush_tlb_all(), as currently implemented, won't work if
		 * PCID is on but PGE is not.  Since that combination
		 * doesn't exist on real hardware, there's no reason to try
		 * to fully support it, but it's polite to avoid corrupting
		 * data if we're on an improperly configured VM.
		 */
		setup_clear_cpu_cap(X86_FEATURE_PCID);
243 244 245
	}
}

246 247 248 249 250 251
#ifdef CONFIG_X86_32
#define NR_RANGE_MR 3
#else /* CONFIG_X86_64 */
#define NR_RANGE_MR 5
#endif

252 253 254
static int __meminit save_mr(struct map_range *mr, int nr_range,
			     unsigned long start_pfn, unsigned long end_pfn,
			     unsigned long page_size_mask)
255 256 257 258 259 260 261 262 263 264 265 266 267
{
	if (start_pfn < end_pfn) {
		if (nr_range >= NR_RANGE_MR)
			panic("run out of range for init_memory_mapping\n");
		mr[nr_range].start = start_pfn<<PAGE_SHIFT;
		mr[nr_range].end   = end_pfn<<PAGE_SHIFT;
		mr[nr_range].page_size_mask = page_size_mask;
		nr_range++;
	}

	return nr_range;
}

268 269 270 271
/*
 * adjust the page_size_mask for small range to go with
 *	big page size instead small one if nearby are ram too.
 */
272
static void __ref adjust_range_page_size_mask(struct map_range *mr,
273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
							 int nr_range)
{
	int i;

	for (i = 0; i < nr_range; i++) {
		if ((page_size_mask & (1<<PG_LEVEL_2M)) &&
		    !(mr[i].page_size_mask & (1<<PG_LEVEL_2M))) {
			unsigned long start = round_down(mr[i].start, PMD_SIZE);
			unsigned long end = round_up(mr[i].end, PMD_SIZE);

#ifdef CONFIG_X86_32
			if ((end >> PAGE_SHIFT) > max_low_pfn)
				continue;
#endif

			if (memblock_is_region_memory(start, end - start))
				mr[i].page_size_mask |= 1<<PG_LEVEL_2M;
		}
		if ((page_size_mask & (1<<PG_LEVEL_1G)) &&
		    !(mr[i].page_size_mask & (1<<PG_LEVEL_1G))) {
			unsigned long start = round_down(mr[i].start, PUD_SIZE);
			unsigned long end = round_up(mr[i].end, PUD_SIZE);

			if (memblock_is_region_memory(start, end - start))
				mr[i].page_size_mask |= 1<<PG_LEVEL_1G;
		}
	}
}

302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326
static const char *page_size_string(struct map_range *mr)
{
	static const char str_1g[] = "1G";
	static const char str_2m[] = "2M";
	static const char str_4m[] = "4M";
	static const char str_4k[] = "4k";

	if (mr->page_size_mask & (1<<PG_LEVEL_1G))
		return str_1g;
	/*
	 * 32-bit without PAE has a 4M large page size.
	 * PG_LEVEL_2M is misnamed, but we can at least
	 * print out the right size in the string.
	 */
	if (IS_ENABLED(CONFIG_X86_32) &&
	    !IS_ENABLED(CONFIG_X86_PAE) &&
	    mr->page_size_mask & (1<<PG_LEVEL_2M))
		return str_4m;

	if (mr->page_size_mask & (1<<PG_LEVEL_2M))
		return str_2m;

	return str_4k;
}

327 328 329
static int __meminit split_mem_range(struct map_range *mr, int nr_range,
				     unsigned long start,
				     unsigned long end)
330
{
331
	unsigned long start_pfn, end_pfn, limit_pfn;
332
	unsigned long pfn;
333
	int i;
334

335 336
	limit_pfn = PFN_DOWN(end);

337
	/* head if not big page alignment ? */
338
	pfn = start_pfn = PFN_DOWN(start);
339 340 341 342 343 344 345
#ifdef CONFIG_X86_32
	/*
	 * Don't use a large page for the first 2/4MB of memory
	 * because there are often fixed size MTRRs in there
	 * and overlapping MTRRs into large pages can cause
	 * slowdowns.
	 */
346
	if (pfn == 0)
347
		end_pfn = PFN_DOWN(PMD_SIZE);
348
	else
349
		end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
350
#else /* CONFIG_X86_64 */
351
	end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
352
#endif
353 354
	if (end_pfn > limit_pfn)
		end_pfn = limit_pfn;
355 356
	if (start_pfn < end_pfn) {
		nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
357
		pfn = end_pfn;
358 359 360
	}

	/* big page (2M) range */
361
	start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
362
#ifdef CONFIG_X86_32
363
	end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
364
#else /* CONFIG_X86_64 */
365
	end_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE));
366 367
	if (end_pfn > round_down(limit_pfn, PFN_DOWN(PMD_SIZE)))
		end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
368 369 370 371 372
#endif

	if (start_pfn < end_pfn) {
		nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
				page_size_mask & (1<<PG_LEVEL_2M));
373
		pfn = end_pfn;
374 375 376 377
	}

#ifdef CONFIG_X86_64
	/* big page (1G) range */
378
	start_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE));
379
	end_pfn = round_down(limit_pfn, PFN_DOWN(PUD_SIZE));
380 381 382 383
	if (start_pfn < end_pfn) {
		nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
				page_size_mask &
				 ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G)));
384
		pfn = end_pfn;
385 386 387
	}

	/* tail is not big page (1G) alignment */
388
	start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
389
	end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
390 391 392
	if (start_pfn < end_pfn) {
		nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
				page_size_mask & (1<<PG_LEVEL_2M));
393
		pfn = end_pfn;
394 395 396 397
	}
#endif

	/* tail is not big page (2M) alignment */
398
	start_pfn = pfn;
399
	end_pfn = limit_pfn;
400 401
	nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);

402 403 404
	if (!after_bootmem)
		adjust_range_page_size_mask(mr, nr_range);

405 406 407 408 409 410 411 412 413 414 415 416 417 418 419
	/* try to merge same page size and continuous */
	for (i = 0; nr_range > 1 && i < nr_range - 1; i++) {
		unsigned long old_start;
		if (mr[i].end != mr[i+1].start ||
		    mr[i].page_size_mask != mr[i+1].page_size_mask)
			continue;
		/* move it */
		old_start = mr[i].start;
		memmove(&mr[i], &mr[i+1],
			(nr_range - 1 - i) * sizeof(struct map_range));
		mr[i--].start = old_start;
		nr_range--;
	}

	for (i = 0; i < nr_range; i++)
420
		pr_debug(" [mem %#010lx-%#010lx] page %s\n",
421
				mr[i].start, mr[i].end - 1,
422
				page_size_string(&mr[i]));
423

424 425 426
	return nr_range;
}

427
struct range pfn_mapped[E820_MAX_ENTRIES];
428
int nr_pfn_mapped;
429 430 431

static void add_pfn_range_mapped(unsigned long start_pfn, unsigned long end_pfn)
{
432
	nr_pfn_mapped = add_range_with_merge(pfn_mapped, E820_MAX_ENTRIES,
433
					     nr_pfn_mapped, start_pfn, end_pfn);
434
	nr_pfn_mapped = clean_sort_range(pfn_mapped, E820_MAX_ENTRIES);
435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454

	max_pfn_mapped = max(max_pfn_mapped, end_pfn);

	if (start_pfn < (1UL<<(32-PAGE_SHIFT)))
		max_low_pfn_mapped = max(max_low_pfn_mapped,
					 min(end_pfn, 1UL<<(32-PAGE_SHIFT)));
}

bool pfn_range_is_mapped(unsigned long start_pfn, unsigned long end_pfn)
{
	int i;

	for (i = 0; i < nr_pfn_mapped; i++)
		if ((start_pfn >= pfn_mapped[i].start) &&
		    (end_pfn <= pfn_mapped[i].end))
			return true;

	return false;
}

455 456 457 458 459
/*
 * Setup the direct mapping of the physical memory at PAGE_OFFSET.
 * This runs before bootmem is initialized and gets pages directly from
 * the physical memory. To access them they are temporarily mapped.
 */
460
unsigned long __ref init_memory_mapping(unsigned long start,
461 462 463 464 465 466
					       unsigned long end)
{
	struct map_range mr[NR_RANGE_MR];
	unsigned long ret = 0;
	int nr_range, i;

467
	pr_debug("init_memory_mapping: [mem %#010lx-%#010lx]\n",
468 469 470 471 472
	       start, end - 1);

	memset(mr, 0, sizeof(mr));
	nr_range = split_mem_range(mr, 0, start, end);

473 474 475 476
	for (i = 0; i < nr_range; i++)
		ret = kernel_physical_mapping_init(mr[i].start, mr[i].end,
						   mr[i].page_size_mask);

477 478
	add_pfn_range_mapped(start >> PAGE_SHIFT, ret >> PAGE_SHIFT);

479 480 481
	return ret >> PAGE_SHIFT;
}

482
/*
483
 * We need to iterate through the E820 memory map and create direct mappings
484
 * for only E820_TYPE_RAM and E820_KERN_RESERVED regions. We cannot simply
485 486 487 488 489 490 491 492 493
 * create direct mappings for all pfns from [0 to max_low_pfn) and
 * [4GB to max_pfn) because of possible memory holes in high addresses
 * that cannot be marked as UC by fixed/variable range MTRRs.
 * Depending on the alignment of E820 ranges, this may possibly result
 * in using smaller size (i.e. 4K instead of 2M or 1G) page tables.
 *
 * init_mem_mapping() calls init_range_memory_mapping() with big range.
 * That range would have hole in the middle or ends, and only ram parts
 * will be mapped in init_range_memory_mapping().
494
 */
495
static unsigned long __init init_range_memory_mapping(
496 497
					   unsigned long r_start,
					   unsigned long r_end)
498 499
{
	unsigned long start_pfn, end_pfn;
500
	unsigned long mapped_ram_size = 0;
501 502 503
	int i;

	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) {
504 505 506
		u64 start = clamp_val(PFN_PHYS(start_pfn), r_start, r_end);
		u64 end = clamp_val(PFN_PHYS(end_pfn), r_start, r_end);
		if (start >= end)
507 508
			continue;

509 510 511 512 513 514
		/*
		 * if it is overlapping with brk pgt, we need to
		 * alloc pgt buf from memblock instead.
		 */
		can_use_brk_pgt = max(start, (u64)pgt_buf_end<<PAGE_SHIFT) >=
				    min(end, (u64)pgt_buf_top<<PAGE_SHIFT);
515
		init_memory_mapping(start, end);
516
		mapped_ram_size += end - start;
517
		can_use_brk_pgt = true;
518
	}
519 520

	return mapped_ram_size;
521 522
}

523 524 525
static unsigned long __init get_new_step_size(unsigned long step_size)
{
	/*
526
	 * Initial mapped size is PMD_SIZE (2M).
527 528 529
	 * We can not set step_size to be PUD_SIZE (1G) yet.
	 * In worse case, when we cross the 1G boundary, and
	 * PG_LEVEL_2M is not set, we will need 1+1+512 pages (2M + 8k)
530 531
	 * to map 1G range with PTE. Hence we use one less than the
	 * difference of page table level shifts.
532
	 *
533 534 535 536 537
	 * Don't need to worry about overflow in the top-down case, on 32bit,
	 * when step_size is 0, round_down() returns 0 for start, and that
	 * turns it into 0x100000000ULL.
	 * In the bottom-up case, round_up(x, 0) returns 0 though too, which
	 * needs to be taken into consideration by the code below.
538
	 */
539
	return step_size << (PMD_SHIFT - PAGE_SHIFT - 1);
540 541
}

542 543 544 545 546 547 548 549 550 551 552 553
/**
 * memory_map_top_down - Map [map_start, map_end) top down
 * @map_start: start address of the target memory range
 * @map_end: end address of the target memory range
 *
 * This function will setup direct mapping for memory range
 * [map_start, map_end) in top-down. That said, the page tables
 * will be allocated at the end of the memory, and we map the
 * memory in top-down.
 */
static void __init memory_map_top_down(unsigned long map_start,
				       unsigned long map_end)
554
{
555
	unsigned long real_end, start, last_start;
556 557 558
	unsigned long step_size;
	unsigned long addr;
	unsigned long mapped_ram_size = 0;
559

560
	/* xen has big range in reserved near end of ram, skip it at first.*/
561
	addr = memblock_find_in_range(map_start, map_end, PMD_SIZE, PMD_SIZE);
562 563 564 565 566 567 568
	real_end = addr + PMD_SIZE;

	/* step_size need to be small so pgt_buf from BRK could cover it */
	step_size = PMD_SIZE;
	max_pfn_mapped = 0; /* will get exact value next */
	min_pfn_mapped = real_end >> PAGE_SHIFT;
	last_start = start = real_end;
569 570 571 572 573 574 575

	/*
	 * We start from the top (end of memory) and go to the bottom.
	 * The memblock_find_in_range() gets us a block of RAM from the
	 * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages
	 * for page table.
	 */
576
	while (last_start > map_start) {
577 578
		if (last_start > step_size) {
			start = round_down(last_start - 1, step_size);
579 580
			if (start < map_start)
				start = map_start;
581
		} else
582
			start = map_start;
583
		mapped_ram_size += init_range_memory_mapping(start,
584 585 586
							last_start);
		last_start = start;
		min_pfn_mapped = last_start >> PAGE_SHIFT;
587
		if (mapped_ram_size >= step_size)
588
			step_size = get_new_step_size(step_size);
589 590
	}

591 592 593 594
	if (real_end < map_end)
		init_range_memory_mapping(real_end, map_end);
}

595 596 597 598 599 600 601 602 603 604 605 606 607 608
/**
 * memory_map_bottom_up - Map [map_start, map_end) bottom up
 * @map_start: start address of the target memory range
 * @map_end: end address of the target memory range
 *
 * This function will setup direct mapping for memory range
 * [map_start, map_end) in bottom-up. Since we have limited the
 * bottom-up allocation above the kernel, the page tables will
 * be allocated just above the kernel and we map the memory
 * in [map_start, map_end) in bottom-up.
 */
static void __init memory_map_bottom_up(unsigned long map_start,
					unsigned long map_end)
{
609
	unsigned long next, start;
610 611 612 613 614 615 616 617 618 619 620 621 622 623
	unsigned long mapped_ram_size = 0;
	/* step_size need to be small so pgt_buf from BRK could cover it */
	unsigned long step_size = PMD_SIZE;

	start = map_start;
	min_pfn_mapped = start >> PAGE_SHIFT;

	/*
	 * We start from the bottom (@map_start) and go to the top (@map_end).
	 * The memblock_find_in_range() gets us a block of RAM from the
	 * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages
	 * for page table.
	 */
	while (start < map_end) {
624
		if (step_size && map_end - start > step_size) {
625 626 627
			next = round_up(start + 1, step_size);
			if (next > map_end)
				next = map_end;
628
		} else {
629
			next = map_end;
630
		}
631

632
		mapped_ram_size += init_range_memory_mapping(start, next);
633 634
		start = next;

635
		if (mapped_ram_size >= step_size)
636 637 638 639
			step_size = get_new_step_size(step_size);
	}
}

640 641 642 643
void __init init_mem_mapping(void)
{
	unsigned long end;

644
	pti_check_boottime_disable();
645
	probe_page_size_mask();
646
	setup_pcid();
647 648 649 650 651 652 653 654 655 656

#ifdef CONFIG_X86_64
	end = max_pfn << PAGE_SHIFT;
#else
	end = max_low_pfn << PAGE_SHIFT;
#endif

	/* the ISA range is always mapped regardless of memory holes */
	init_memory_mapping(0, ISA_END_ADDRESS);

657 658 659
	/* Init the trampoline, possibly with KASLR memory offset */
	init_trampoline();

660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678
	/*
	 * If the allocation is in bottom-up direction, we setup direct mapping
	 * in bottom-up, otherwise we setup direct mapping in top-down.
	 */
	if (memblock_bottom_up()) {
		unsigned long kernel_end = __pa_symbol(_end);

		/*
		 * we need two separate calls here. This is because we want to
		 * allocate page tables above the kernel. So we first map
		 * [kernel_end, end) to make memory above the kernel be mapped
		 * as soon as possible. And then use page tables allocated above
		 * the kernel to map [ISA_END_ADDRESS, kernel_end).
		 */
		memory_map_bottom_up(kernel_end, end);
		memory_map_bottom_up(ISA_END_ADDRESS, kernel_end);
	} else {
		memory_map_top_down(ISA_END_ADDRESS, end);
	}
679

680 681 682 683 684
#ifdef CONFIG_X86_64
	if (max_pfn > max_low_pfn) {
		/* can we preseve max_low_pfn ?*/
		max_low_pfn = max_pfn;
	}
685 686
#else
	early_ioremap_page_table_range_init();
687 688
#endif

689 690 691
	load_cr3(swapper_pg_dir);
	__flush_tlb_all();

692
	x86_init.hyper.init_mem_mapping();
693

694
	early_memtest(0, max_pfn_mapped << PAGE_SHIFT);
695
}
696

697 698 699 700
/*
 * devmem_is_allowed() checks to see if /dev/mem access to a certain address
 * is valid. The argument is a physical page number.
 *
701 702 703 704 705 706 707
 * On x86, access has to be given to the first megabyte of RAM because that
 * area traditionally contains BIOS code and data regions used by X, dosemu,
 * and similar apps. Since they map the entire memory range, the whole range
 * must be allowed (for mapping), but any areas that would otherwise be
 * disallowed are flagged as being "zero filled" instead of rejected.
 * Access has to be given to non-kernel-ram areas as well, these contain the
 * PCI mmio resources as well as potential bios/acpi data regions.
708 709 710
 */
int devmem_is_allowed(unsigned long pagenr)
{
711 712 713
	if (region_intersects(PFN_PHYS(pagenr), PAGE_SIZE,
				IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE)
			!= REGION_DISJOINT) {
714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732
		/*
		 * For disallowed memory regions in the low 1MB range,
		 * request that the page be shown as all zeros.
		 */
		if (pagenr < 256)
			return 2;

		return 0;
	}

	/*
	 * This must follow RAM test, since System RAM is considered a
	 * restricted resource under CONFIG_STRICT_IOMEM.
	 */
	if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) {
		/* Low 1MB bypasses iomem restrictions. */
		if (pagenr < 256)
			return 1;

733
		return 0;
734 735 736
	}

	return 1;
737 738
}

739 740
void free_init_pages(char *what, unsigned long begin, unsigned long end)
{
741
	unsigned long begin_aligned, end_aligned;
742

743 744 745 746 747 748 749 750 751 752
	/* Make sure boundaries are page aligned */
	begin_aligned = PAGE_ALIGN(begin);
	end_aligned   = end & PAGE_MASK;

	if (WARN_ON(begin_aligned != begin || end_aligned != end)) {
		begin = begin_aligned;
		end   = end_aligned;
	}

	if (begin >= end)
753 754 755 756 757 758 759
		return;

	/*
	 * If debugging page accesses then do not free this memory but
	 * mark them not present - any buggy init-section access will
	 * create a kernel page fault:
	 */
760 761 762 763 764 765 766 767 768 769 770 771
	if (debug_pagealloc_enabled()) {
		pr_info("debug: unmapping init [mem %#010lx-%#010lx]\n",
			begin, end - 1);
		set_memory_np(begin, (end - begin) >> PAGE_SHIFT);
	} else {
		/*
		 * We just marked the kernel text read only above, now that
		 * we are going to free part of that, we need to make that
		 * writeable and non-executable first.
		 */
		set_memory_nx(begin, (end - begin) >> PAGE_SHIFT);
		set_memory_rw(begin, (end - begin) >> PAGE_SHIFT);
772

773 774 775
		free_reserved_area((void *)begin, (void *)end,
				   POISON_FREE_INITMEM, what);
	}
776 777
}

778 779 780 781 782 783 784 785 786 787 788
/*
 * begin/end can be in the direct map or the "high kernel mapping"
 * used for the kernel image only.  free_init_pages() will do the
 * right thing for either kind of address.
 */
void free_kernel_image_pages(void *begin, void *end)
{
	free_init_pages("unused kernel image",
			(unsigned long)begin, (unsigned long)end);
}

789
void __ref free_initmem(void)
790
{
791
	e820__reallocate_tables();
792

793
	free_kernel_image_pages(&__init_begin, &__init_end);
794
}
795 796

#ifdef CONFIG_BLK_DEV_INITRD
797
void __init free_initrd_mem(unsigned long start, unsigned long end)
798
{
799 800 801 802 803 804 805 806 807
	/*
	 * end could be not aligned, and We can not align that,
	 * decompresser could be confused by aligned initrd_end
	 * We already reserve the end partial page before in
	 *   - i386_start_kernel()
	 *   - x86_64_start_kernel()
	 *   - relocate_initrd()
	 * So here We can do PAGE_ALIGN() safely to get partial page to be freed
	 */
808
	free_init_pages("initrd", start, PAGE_ALIGN(end));
809 810
}
#endif
811

812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858
/*
 * Calculate the precise size of the DMA zone (first 16 MB of RAM),
 * and pass it to the MM layer - to help it set zone watermarks more
 * accurately.
 *
 * Done on 64-bit systems only for the time being, although 32-bit systems
 * might benefit from this as well.
 */
void __init memblock_find_dma_reserve(void)
{
#ifdef CONFIG_X86_64
	u64 nr_pages = 0, nr_free_pages = 0;
	unsigned long start_pfn, end_pfn;
	phys_addr_t start_addr, end_addr;
	int i;
	u64 u;

	/*
	 * Iterate over all memory ranges (free and reserved ones alike),
	 * to calculate the total number of pages in the first 16 MB of RAM:
	 */
	nr_pages = 0;
	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) {
		start_pfn = min(start_pfn, MAX_DMA_PFN);
		end_pfn   = min(end_pfn,   MAX_DMA_PFN);

		nr_pages += end_pfn - start_pfn;
	}

	/*
	 * Iterate over free memory ranges to calculate the number of free
	 * pages in the DMA zone, while not counting potential partial
	 * pages at the beginning or the end of the range:
	 */
	nr_free_pages = 0;
	for_each_free_mem_range(u, NUMA_NO_NODE, MEMBLOCK_NONE, &start_addr, &end_addr, NULL) {
		start_pfn = min_t(unsigned long, PFN_UP(start_addr), MAX_DMA_PFN);
		end_pfn   = min_t(unsigned long, PFN_DOWN(end_addr), MAX_DMA_PFN);

		if (start_pfn < end_pfn)
			nr_free_pages += end_pfn - start_pfn;
	}

	set_dma_reserve(nr_pages - nr_free_pages);
#endif
}

859 860 861 862 863 864 865
void __init zone_sizes_init(void)
{
	unsigned long max_zone_pfns[MAX_NR_ZONES];

	memset(max_zone_pfns, 0, sizeof(max_zone_pfns));

#ifdef CONFIG_ZONE_DMA
866
	max_zone_pfns[ZONE_DMA]		= min(MAX_DMA_PFN, max_low_pfn);
867 868
#endif
#ifdef CONFIG_ZONE_DMA32
869
	max_zone_pfns[ZONE_DMA32]	= min(MAX_DMA32_PFN, max_low_pfn);
870 871 872 873 874 875 876 877 878
#endif
	max_zone_pfns[ZONE_NORMAL]	= max_low_pfn;
#ifdef CONFIG_HIGHMEM
	max_zone_pfns[ZONE_HIGHMEM]	= max_pfn;
#endif

	free_area_init_nodes(max_zone_pfns);
}

879
__visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate) = {
880
	.loaded_mm = &init_mm,
881
	.next_asid = 1,
882 883
	.cr4 = ~0UL,	/* fail hard if we screw up cr4 shadow initialization */
};
884
EXPORT_PER_CPU_SYMBOL(cpu_tlbstate);
885

886 887 888 889 890 891 892 893
void update_cache_mode_entry(unsigned entry, enum page_cache_mode cache)
{
	/* entry 0 MUST be WB (hardwired to speed up translations) */
	BUG_ON(!entry && cache != _PAGE_CACHE_MODE_WB);

	__cachemode2pte_tbl[cache] = __cm_idx2pte(entry);
	__pte2cachemode_tbl[entry] = cache;
}
894

895
#ifdef CONFIG_SWAP
896 897 898 899 900 901 902 903
unsigned long max_swapfile_size(void)
{
	unsigned long pages;

	pages = generic_max_swapfile_size();

	if (boot_cpu_has_bug(X86_BUG_L1TF)) {
		/* Limit the swap file size to MAX_PA/2 for L1TF workaround */
904 905 906 907 908
		unsigned long l1tf_limit = l1tf_pfn_limit() + 1;
		/*
		 * We encode swap offsets also with 3 bits below those for pfn
		 * which makes the usable limit higher.
		 */
909
#if CONFIG_PGTABLE_LEVELS > 2
910 911 912
		l1tf_limit <<= PAGE_SHIFT - SWP_OFFSET_FIRST_BIT;
#endif
		pages = min_t(unsigned long, l1tf_limit, pages);
913 914 915
	}
	return pages;
}
916
#endif