init_32.c 25 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
Linus Torvalds's avatar
Linus Torvalds committed
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
/*
 *
 *  Copyright (C) 1995  Linus Torvalds
 *
 *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
 */

#include <linux/signal.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/ptrace.h>
#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/swap.h>
#include <linux/smp.h>
#include <linux/init.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
24
#include <linux/pci.h>
25
#include <linux/pfn.h>
26
#include <linux/poison.h>
27
#include <linux/memblock.h>
Linus Torvalds's avatar
Linus Torvalds committed
28
#include <linux/proc_fs.h>
29
#include <linux/memory_hotplug.h>
30
#include <linux/initrd.h>
31
#include <linux/cpumask.h>
32
#include <linux/gfp.h>
Linus Torvalds's avatar
Linus Torvalds committed
33

34
#include <asm/asm.h>
35
#include <asm/bios_ebda.h>
Linus Torvalds's avatar
Linus Torvalds committed
36
#include <asm/processor.h>
37
#include <linux/uaccess.h>
Linus Torvalds's avatar
Linus Torvalds committed
38 39 40
#include <asm/pgtable.h>
#include <asm/dma.h>
#include <asm/fixmap.h>
41
#include <asm/e820/api.h>
Linus Torvalds's avatar
Linus Torvalds committed
42
#include <asm/apic.h>
43
#include <asm/bugs.h>
Linus Torvalds's avatar
Linus Torvalds committed
44 45
#include <asm/tlb.h>
#include <asm/tlbflush.h>
46
#include <asm/olpc_ofw.h>
47
#include <asm/pgalloc.h>
Linus Torvalds's avatar
Linus Torvalds committed
48
#include <asm/sections.h>
49
#include <asm/paravirt.h>
50
#include <asm/setup.h>
Laura Abbott's avatar
Laura Abbott committed
51
#include <asm/set_memory.h>
52
#include <asm/page_types.h>
53
#include <asm/cpu_entry_area.h>
54
#include <asm/init.h>
Linus Torvalds's avatar
Linus Torvalds committed
55

56 57
#include "mm_internal.h"

Linus Torvalds's avatar
Linus Torvalds committed
58 59
unsigned long highstart_pfn, highend_pfn;

60
bool __read_mostly __vmalloc_start_set = false;
61

Linus Torvalds's avatar
Linus Torvalds committed
62 63 64 65 66 67 68
/*
 * Creates a middle page table and puts a pointer to it in the
 * given global directory entry. This only returns the gd entry
 * in non-PAE compilation mode, since the middle layer is folded.
 */
static pmd_t * __init one_md_table_init(pgd_t *pgd)
{
69
	p4d_t *p4d;
Linus Torvalds's avatar
Linus Torvalds committed
70 71
	pud_t *pud;
	pmd_t *pmd_table;
72

Linus Torvalds's avatar
Linus Torvalds committed
73
#ifdef CONFIG_X86_PAE
74
	if (!(pgd_val(*pgd) & _PAGE_PRESENT)) {
75
		pmd_table = (pmd_t *)alloc_low_page();
76
		paravirt_alloc_pmd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT);
77
		set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));
78 79
		p4d = p4d_offset(pgd, 0);
		pud = pud_offset(p4d, 0);
80
		BUG_ON(pmd_table != pmd_offset(pud, 0));
81 82

		return pmd_table;
83 84
	}
#endif
85 86
	p4d = p4d_offset(pgd, 0);
	pud = pud_offset(p4d, 0);
Linus Torvalds's avatar
Linus Torvalds committed
87
	pmd_table = pmd_offset(pud, 0);
88

Linus Torvalds's avatar
Linus Torvalds committed
89 90 91 92 93
	return pmd_table;
}

/*
 * Create a page table and place a pointer to it in a middle page
94
 * directory entry:
Linus Torvalds's avatar
Linus Torvalds committed
95 96 97
 */
static pte_t * __init one_page_table_init(pmd_t *pmd)
{
98
	if (!(pmd_val(*pmd) & _PAGE_PRESENT)) {
99
		pte_t *page_table = (pte_t *)alloc_low_page();
100

101
		paravirt_alloc_pte(&init_mm, __pa(page_table) >> PAGE_SHIFT);
Linus Torvalds's avatar
Linus Torvalds committed
102
		set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE));
103
		BUG_ON(page_table != pte_offset_kernel(pmd, 0));
Linus Torvalds's avatar
Linus Torvalds committed
104
	}
105

Linus Torvalds's avatar
Linus Torvalds committed
106 107 108
	return pte_offset_kernel(pmd, 0);
}

109
pmd_t * __init populate_extra_pmd(unsigned long vaddr)
110 111 112
{
	int pgd_idx = pgd_index(vaddr);
	int pmd_idx = pmd_index(vaddr);
113 114 115 116 117 118 119

	return one_md_table_init(swapper_pg_dir + pgd_idx) + pmd_idx;
}

pte_t * __init populate_extra_pte(unsigned long vaddr)
{
	int pte_idx = pte_index(vaddr);
120 121
	pmd_t *pmd;

122 123
	pmd = populate_extra_pmd(vaddr);
	return one_page_table_init(pmd) + pte_idx;
124 125
}

126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
static unsigned long __init
page_table_range_init_count(unsigned long start, unsigned long end)
{
	unsigned long count = 0;
#ifdef CONFIG_HIGHMEM
	int pmd_idx_kmap_begin = fix_to_virt(FIX_KMAP_END) >> PMD_SHIFT;
	int pmd_idx_kmap_end = fix_to_virt(FIX_KMAP_BEGIN) >> PMD_SHIFT;
	int pgd_idx, pmd_idx;
	unsigned long vaddr;

	if (pmd_idx_kmap_begin == pmd_idx_kmap_end)
		return 0;

	vaddr = start;
	pgd_idx = pgd_index(vaddr);
141
	pmd_idx = pmd_index(vaddr);
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156

	for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd_idx++) {
		for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end);
							pmd_idx++) {
			if ((vaddr >> PMD_SHIFT) >= pmd_idx_kmap_begin &&
			    (vaddr >> PMD_SHIFT) <= pmd_idx_kmap_end)
				count++;
			vaddr += PMD_SIZE;
		}
		pmd_idx = 0;
	}
#endif
	return count;
}

157
static pte_t *__init page_table_kmap_check(pte_t *pte, pmd_t *pmd,
158 159
					   unsigned long vaddr, pte_t *lastpte,
					   void **adr)
160 161 162 163 164 165 166 167 168 169 170 171 172
{
#ifdef CONFIG_HIGHMEM
	/*
	 * Something (early fixmap) may already have put a pte
	 * page here, which causes the page table allocation
	 * to become nonlinear. Attempt to fix it, and if it
	 * is still nonlinear then we have to bug.
	 */
	int pmd_idx_kmap_begin = fix_to_virt(FIX_KMAP_END) >> PMD_SHIFT;
	int pmd_idx_kmap_end = fix_to_virt(FIX_KMAP_BEGIN) >> PMD_SHIFT;

	if (pmd_idx_kmap_begin != pmd_idx_kmap_end
	    && (vaddr >> PMD_SHIFT) >= pmd_idx_kmap_begin
173
	    && (vaddr >> PMD_SHIFT) <= pmd_idx_kmap_end) {
174 175 176
		pte_t *newpte;
		int i;

177
		BUG_ON(after_bootmem);
178
		newpte = *adr;
179 180
		for (i = 0; i < PTRS_PER_PTE; i++)
			set_pte(newpte + i, pte[i]);
181
		*adr = (void *)(((unsigned long)(*adr)) + PAGE_SIZE);
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197

		paravirt_alloc_pte(&init_mm, __pa(newpte) >> PAGE_SHIFT);
		set_pmd(pmd, __pmd(__pa(newpte)|_PAGE_TABLE));
		BUG_ON(newpte != pte_offset_kernel(pmd, 0));
		__flush_tlb_all();

		paravirt_release_pte(__pa(pte) >> PAGE_SHIFT);
		pte = newpte;
	}
	BUG_ON(vaddr < fix_to_virt(FIX_KMAP_BEGIN - 1)
	       && vaddr > fix_to_virt(FIX_KMAP_END)
	       && lastpte && lastpte + PTRS_PER_PTE != pte);
#endif
	return pte;
}

Linus Torvalds's avatar
Linus Torvalds committed
198
/*
199
 * This function initializes a certain range of kernel virtual memory
Linus Torvalds's avatar
Linus Torvalds committed
200 201
 * with new bootmem page tables, everywhere page tables are missing in
 * the given range.
202 203 204
 *
 * NOTE: The pagetables are allocated contiguous on the physical space
 * so we can cache the place of the first one and move around without
Linus Torvalds's avatar
Linus Torvalds committed
205 206
 * checking the pgd every time.
 */
207 208
static void __init
page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base)
Linus Torvalds's avatar
Linus Torvalds committed
209 210 211
{
	int pgd_idx, pmd_idx;
	unsigned long vaddr;
212 213
	pgd_t *pgd;
	pmd_t *pmd;
214
	pte_t *pte = NULL;
215 216 217 218 219
	unsigned long count = page_table_range_init_count(start, end);
	void *adr = NULL;

	if (count)
		adr = alloc_low_pages(count);
Linus Torvalds's avatar
Linus Torvalds committed
220 221 222 223 224 225 226

	vaddr = start;
	pgd_idx = pgd_index(vaddr);
	pmd_idx = pmd_index(vaddr);
	pgd = pgd_base + pgd_idx;

	for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) {
227 228
		pmd = one_md_table_init(pgd);
		pmd = pmd + pmd_index(vaddr);
229 230
		for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end);
							pmd++, pmd_idx++) {
231
			pte = page_table_kmap_check(one_page_table_init(pmd),
232
						    pmd, vaddr, pte, &adr);
Linus Torvalds's avatar
Linus Torvalds committed
233 234 235 236 237 238 239 240 241

			vaddr += PMD_SIZE;
		}
		pmd_idx = 0;
	}
}

static inline int is_kernel_text(unsigned long addr)
{
242
	if (addr >= (unsigned long)_text && addr <= (unsigned long)__init_end)
Linus Torvalds's avatar
Linus Torvalds committed
243 244 245 246 247
		return 1;
	return 0;
}

/*
248 249 250
 * This maps the physical memory to kernel virtual address space, a total
 * of max_low_pfn pages, by creating page tables starting from address
 * PAGE_OFFSET:
Linus Torvalds's avatar
Linus Torvalds committed
251
 */
252 253 254 255
unsigned long __init
kernel_physical_mapping_init(unsigned long start,
			     unsigned long end,
			     unsigned long page_size_mask)
Linus Torvalds's avatar
Linus Torvalds committed
256
{
257
	int use_pse = page_size_mask == (1<<PG_LEVEL_2M);
258
	unsigned long last_map_addr = end;
259
	unsigned long start_pfn, end_pfn;
260
	pgd_t *pgd_base = swapper_pg_dir;
261
	int pgd_idx, pmd_idx, pte_ofs;
Linus Torvalds's avatar
Linus Torvalds committed
262 263 264 265
	unsigned long pfn;
	pgd_t *pgd;
	pmd_t *pmd;
	pte_t *pte;
266 267 268
	unsigned pages_2m, pages_4k;
	int mapping_iter;

269 270 271
	start_pfn = start >> PAGE_SHIFT;
	end_pfn = end >> PAGE_SHIFT;

272 273 274 275 276 277 278 279 280 281 282 283 284 285 286
	/*
	 * First iteration will setup identity mapping using large/small pages
	 * based on use_pse, with other attributes same as set by
	 * the early code in head_32.S
	 *
	 * Second iteration will setup the appropriate attributes (NX, GLOBAL..)
	 * as desired for the kernel identity mapping.
	 *
	 * This two pass mechanism conforms to the TLB app note which says:
	 *
	 *     "Software should not write to a paging-structure entry in a way
	 *      that would change, for any linear address, both the page size
	 *      and either the page frame or attributes."
	 */
	mapping_iter = 1;
Linus Torvalds's avatar
Linus Torvalds committed
287

288
	if (!boot_cpu_has(X86_FEATURE_PSE))
289
		use_pse = 0;
Linus Torvalds's avatar
Linus Torvalds committed
290

291 292
repeat:
	pages_2m = pages_4k = 0;
293 294 295
	pfn = start_pfn;
	pgd_idx = pgd_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET);
	pgd = pgd_base + pgd_idx;
Linus Torvalds's avatar
Linus Torvalds committed
296 297
	for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) {
		pmd = one_md_table_init(pgd);
298

299 300 301 302 303 304 305 306 307
		if (pfn >= end_pfn)
			continue;
#ifdef CONFIG_X86_PAE
		pmd_idx = pmd_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET);
		pmd += pmd_idx;
#else
		pmd_idx = 0;
#endif
		for (; pmd_idx < PTRS_PER_PMD && pfn < end_pfn;
308
		     pmd++, pmd_idx++) {
309
			unsigned int addr = pfn * PAGE_SIZE + PAGE_OFFSET;
Linus Torvalds's avatar
Linus Torvalds committed
310

311 312 313 314
			/*
			 * Map with big pages if possible, otherwise
			 * create normal page tables:
			 */
315
			if (use_pse) {
316
				unsigned int addr2;
317
				pgprot_t prot = PAGE_KERNEL_LARGE;
318 319 320 321 322 323 324
				/*
				 * first pass will use the same initial
				 * identity mapping attribute + _PAGE_PSE.
				 */
				pgprot_t init_prot =
					__pgprot(PTE_IDENT_ATTR |
						 _PAGE_PSE);
325

326
				pfn &= PMD_MASK >> PAGE_SHIFT;
327
				addr2 = (pfn + PTRS_PER_PTE-1) * PAGE_SIZE +
328 329
					PAGE_OFFSET + PAGE_SIZE-1;

330 331
				if (is_kernel_text(addr) ||
				    is_kernel_text(addr2))
332 333
					prot = PAGE_KERNEL_LARGE_EXEC;

334
				pages_2m++;
335 336 337 338
				if (mapping_iter == 1)
					set_pmd(pmd, pfn_pmd(pfn, init_prot));
				else
					set_pmd(pmd, pfn_pmd(pfn, prot));
339

Linus Torvalds's avatar
Linus Torvalds committed
340
				pfn += PTRS_PER_PTE;
341 342 343
				continue;
			}
			pte = one_page_table_init(pmd);
Linus Torvalds's avatar
Linus Torvalds committed
344

345 346 347
			pte_ofs = pte_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET);
			pte += pte_ofs;
			for (; pte_ofs < PTRS_PER_PTE && pfn < end_pfn;
348 349
			     pte++, pfn++, pte_ofs++, addr += PAGE_SIZE) {
				pgprot_t prot = PAGE_KERNEL;
350 351 352 353 354
				/*
				 * first pass will use the same initial
				 * identity mapping attribute.
				 */
				pgprot_t init_prot = __pgprot(PTE_IDENT_ATTR);
355

356 357
				if (is_kernel_text(addr))
					prot = PAGE_KERNEL_EXEC;
358

359
				pages_4k++;
360
				if (mapping_iter == 1) {
361
					set_pte(pte, pfn_pte(pfn, init_prot));
362 363
					last_map_addr = (pfn << PAGE_SHIFT) + PAGE_SIZE;
				} else
364
					set_pte(pte, pfn_pte(pfn, prot));
Linus Torvalds's avatar
Linus Torvalds committed
365 366 367
			}
		}
	}
368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387
	if (mapping_iter == 1) {
		/*
		 * update direct mapping page count only in the first
		 * iteration.
		 */
		update_page_count(PG_LEVEL_2M, pages_2m);
		update_page_count(PG_LEVEL_4K, pages_4k);

		/*
		 * local global flush tlb, which will flush the previous
		 * mappings present in both small and large page TLB's.
		 */
		__flush_tlb_all();

		/*
		 * Second iteration will set the actual desired PTE attributes.
		 */
		mapping_iter = 2;
		goto repeat;
	}
388
	return last_map_addr;
389 390
}

Linus Torvalds's avatar
Linus Torvalds committed
391 392
pte_t *kmap_pte;

393 394
static inline pte_t *kmap_get_fixmap_pte(unsigned long vaddr)
{
395 396 397 398 399
	pgd_t *pgd = pgd_offset_k(vaddr);
	p4d_t *p4d = p4d_offset(pgd, vaddr);
	pud_t *pud = pud_offset(p4d, vaddr);
	pmd_t *pmd = pmd_offset(pud, vaddr);
	return pte_offset_kernel(pmd, vaddr);
400
}
Linus Torvalds's avatar
Linus Torvalds committed
401 402 403 404 405

static void __init kmap_init(void)
{
	unsigned long kmap_vstart;

406 407 408
	/*
	 * Cache the first kmap pte:
	 */
Linus Torvalds's avatar
Linus Torvalds committed
409 410 411 412
	kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN);
	kmap_pte = kmap_get_fixmap_pte(kmap_vstart);
}

413
#ifdef CONFIG_HIGHMEM
Linus Torvalds's avatar
Linus Torvalds committed
414 415
static void __init permanent_kmaps_init(pgd_t *pgd_base)
{
416
	unsigned long vaddr;
Linus Torvalds's avatar
Linus Torvalds committed
417
	pgd_t *pgd;
418
	p4d_t *p4d;
Linus Torvalds's avatar
Linus Torvalds committed
419 420 421 422 423 424 425 426
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	vaddr = PKMAP_BASE;
	page_table_range_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base);

	pgd = swapper_pg_dir + pgd_index(vaddr);
427 428
	p4d = p4d_offset(pgd, vaddr);
	pud = pud_offset(p4d, vaddr);
Linus Torvalds's avatar
Linus Torvalds committed
429 430
	pmd = pmd_offset(pud, vaddr);
	pte = pte_offset_kernel(pmd, vaddr);
431
	pkmap_page_table = pte;
Linus Torvalds's avatar
Linus Torvalds committed
432 433
}

434 435
void __init add_highpages_with_active_regions(int nid,
			 unsigned long start_pfn, unsigned long end_pfn)
Linus Torvalds's avatar
Linus Torvalds committed
436
{
437 438 439
	phys_addr_t start, end;
	u64 i;

440
	for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &start, &end, NULL) {
441 442 443 444 445 446
		unsigned long pfn = clamp_t(unsigned long, PFN_UP(start),
					    start_pfn, end_pfn);
		unsigned long e_pfn = clamp_t(unsigned long, PFN_DOWN(end),
					      start_pfn, end_pfn);
		for ( ; pfn < e_pfn; pfn++)
			if (pfn_valid(pfn))
447
				free_highmem_page(pfn_to_page(pfn));
448
	}
449
}
Linus Torvalds's avatar
Linus Torvalds committed
450
#else
451 452 453
static inline void permanent_kmaps_init(pgd_t *pgd_base)
{
}
Linus Torvalds's avatar
Linus Torvalds committed
454 455
#endif /* CONFIG_HIGHMEM */

456 457 458 459 460 461 462 463 464 465 466 467 468 469 470
void __init sync_initial_page_table(void)
{
	clone_pgd_range(initial_page_table + KERNEL_PGD_BOUNDARY,
			swapper_pg_dir     + KERNEL_PGD_BOUNDARY,
			KERNEL_PGD_PTRS);

	/*
	 * sync back low identity map too.  It is used for example
	 * in the 32-bit EFI stub.
	 */
	clone_pgd_range(initial_page_table,
			swapper_pg_dir     + KERNEL_PGD_BOUNDARY,
			min(KERNEL_PGD_PTRS, KERNEL_PGD_BOUNDARY));
}

471
void __init native_pagetable_init(void)
Linus Torvalds's avatar
Linus Torvalds committed
472
{
473
	unsigned long pfn, va;
474
	pgd_t *pgd, *base = swapper_pg_dir;
475
	p4d_t *p4d;
476 477 478
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;
479 480

	/*
481
	 * Remove any mappings which extend past the end of physical
482 483 484 485 486 487
	 * memory from the boot time page table.
	 * In virtual address space, we should have at least two pages
	 * from VMALLOC_END to pkmap or fixmap according to VMALLOC_END
	 * definition. And max_low_pfn is set to VMALLOC_END physical
	 * address. If initial memory mapping is doing right job, we
	 * should have pte used near max_low_pfn or one pmd is not present.
488
	 */
489
	for (pfn = max_low_pfn; pfn < 1<<(32-PAGE_SHIFT); pfn++) {
490 491 492 493 494
		va = PAGE_OFFSET + (pfn<<PAGE_SHIFT);
		pgd = base + pgd_index(va);
		if (!pgd_present(*pgd))
			break;

495 496
		p4d = p4d_offset(pgd, va);
		pud = pud_offset(p4d, va);
497 498 499 500
		pmd = pmd_offset(pud, va);
		if (!pmd_present(*pmd))
			break;

501 502 503 504 505 506 507
		/* should not be large page here */
		if (pmd_large(*pmd)) {
			pr_warn("try to clear pte for ram above max_low_pfn: pfn: %lx pmd: %p pmd phys: %lx, but pmd is big page and is not using pte !\n",
				pfn, pmd, __pa(pmd));
			BUG_ON(1);
		}

508 509 510 511
		pte = pte_offset_kernel(pmd, va);
		if (!pte_present(*pte))
			break;

512 513
		printk(KERN_DEBUG "clearing pte for ram above max_low_pfn: pfn: %lx pmd: %p pmd phys: %lx pte: %p pte phys: %lx\n",
				pfn, pmd, __pa(pmd), pte, __pa(pte));
514 515
		pte_clear(NULL, va, pte);
	}
516
	paravirt_alloc_pmd(&init_mm, __pa(base) >> PAGE_SHIFT);
517
	paging_init();
518 519 520 521 522 523 524 525
}

/*
 * Build a proper pagetable for the kernel mappings.  Up until this
 * point, we've been running on some set of pagetables constructed by
 * the boot process.
 *
 * If we're booting on native hardware, this will be a pagetable
526 527
 * constructed in arch/x86/kernel/head_32.S.  The root of the
 * pagetable will be swapper_pg_dir.
528 529 530 531
 *
 * If we're booting paravirtualized under a hypervisor, then there are
 * more options: we may already be running PAE, and the pagetable may
 * or may not be based in swapper_pg_dir.  In any case,
532
 * paravirt_pagetable_init() will set up swapper_pg_dir
533 534 535 536 537 538
 * appropriately for the rest of the initialization to work.
 *
 * In general, pagetable_init() assumes that the pagetable may already
 * be partially populated, and so it avoids stomping on any existing
 * mappings.
 */
539
void __init early_ioremap_page_table_range_init(void)
540
{
541
	pgd_t *pgd_base = swapper_pg_dir;
542
	unsigned long vaddr, end;
543

Linus Torvalds's avatar
Linus Torvalds committed
544 545 546 547 548
	/*
	 * Fixed mappings, only the page table structure has to be
	 * created - mappings will be set by set_fixmap():
	 */
	vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
549 550
	end = (FIXADDR_TOP + PMD_SIZE - 1) & PMD_MASK;
	page_table_range_init(vaddr, end, pgd_base);
551
	early_ioremap_reset();
552 553 554 555 556 557
}

static void __init pagetable_init(void)
{
	pgd_t *pgd_base = swapper_pg_dir;

Linus Torvalds's avatar
Linus Torvalds committed
558 559 560
	permanent_kmaps_init(pgd_base);
}

561 562 563 564 565
#define DEFAULT_PTE_MASK ~(_PAGE_NX | _PAGE_GLOBAL)
/* Bits supported by the hardware: */
pteval_t __supported_pte_mask __read_mostly = DEFAULT_PTE_MASK;
/* Bits allowed in normal kernel mappings: */
pteval_t __default_kernel_pte_mask __read_mostly = DEFAULT_PTE_MASK;
566
EXPORT_SYMBOL_GPL(__supported_pte_mask);
567 568
/* Used in PAGE_KERNEL_* macros which are reasonably used out-of-tree: */
EXPORT_SYMBOL(__default_kernel_pte_mask);
569

570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
/* user-defined highmem size */
static unsigned int highmem_pages = -1;

/*
 * highmem=size forces highmem to be exactly 'size' bytes.
 * This works even on boxes that have no highmem otherwise.
 * This also works to reduce highmem size on bigger boxes.
 */
static int __init parse_highmem(char *arg)
{
	if (!arg)
		return -EINVAL;

	highmem_pages = memparse(arg, &arg) >> PAGE_SHIFT;
	return 0;
}
early_param("highmem", parse_highmem);

588 589 590 591 592
#define MSG_HIGHMEM_TOO_BIG \
	"highmem size (%luMB) is bigger than pages available (%luMB)!\n"

#define MSG_LOWMEM_TOO_SMALL \
	"highmem size (%luMB) results in <64MB lowmem, ignoring it!\n"
593
/*
594 595 596
 * All of RAM fits into lowmem - but if user wants highmem
 * artificially via the highmem=x boot parameter then create
 * it:
597
 */
598
static void __init lowmem_pfn_init(void)
599
{
Yinghai Lu's avatar
Yinghai Lu committed
600
	/* max_low_pfn is 0, we already have early_res support */
601
	max_low_pfn = max_pfn;
602

603 604 605 606 607 608 609 610 611 612 613
	if (highmem_pages == -1)
		highmem_pages = 0;
#ifdef CONFIG_HIGHMEM
	if (highmem_pages >= max_pfn) {
		printk(KERN_ERR MSG_HIGHMEM_TOO_BIG,
			pages_to_mb(highmem_pages), pages_to_mb(max_pfn));
		highmem_pages = 0;
	}
	if (highmem_pages) {
		if (max_low_pfn - highmem_pages < 64*1024*1024/PAGE_SIZE) {
			printk(KERN_ERR MSG_LOWMEM_TOO_SMALL,
614 615 616
				pages_to_mb(highmem_pages));
			highmem_pages = 0;
		}
617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633
		max_low_pfn -= highmem_pages;
	}
#else
	if (highmem_pages)
		printk(KERN_ERR "ignoring highmem size on non-highmem kernel!\n");
#endif
}

#define MSG_HIGHMEM_TOO_SMALL \
	"only %luMB highmem pages available, ignoring highmem size of %luMB!\n"

#define MSG_HIGHMEM_TRIMMED \
	"Warning: only 4GB will be used. Use a HIGHMEM64G enabled kernel!\n"
/*
 * We have more RAM than fits into lowmem - we try to put it into
 * highmem, also taking the highmem=x boot parameter into account:
 */
634
static void __init highmem_pfn_init(void)
635
{
636 637
	max_low_pfn = MAXMEM_PFN;

638 639 640 641 642 643 644 645 646 647 648 649
	if (highmem_pages == -1)
		highmem_pages = max_pfn - MAXMEM_PFN;

	if (highmem_pages + MAXMEM_PFN < max_pfn)
		max_pfn = MAXMEM_PFN + highmem_pages;

	if (highmem_pages + MAXMEM_PFN > max_pfn) {
		printk(KERN_WARNING MSG_HIGHMEM_TOO_SMALL,
			pages_to_mb(max_pfn - MAXMEM_PFN),
			pages_to_mb(highmem_pages));
		highmem_pages = 0;
	}
650
#ifndef CONFIG_HIGHMEM
651 652 653 654 655 656 657
	/* Maximum memory usable is what is directly addressable */
	printk(KERN_WARNING "Warning only %ldMB will be used.\n", MAXMEM>>20);
	if (max_pfn > MAX_NONPAE_PFN)
		printk(KERN_WARNING "Use a HIGHMEM64G enabled kernel.\n");
	else
		printk(KERN_WARNING "Use a HIGHMEM enabled kernel.\n");
	max_pfn = MAXMEM_PFN;
658 659
#else /* !CONFIG_HIGHMEM */
#ifndef CONFIG_HIGHMEM64G
660 661 662 663
	if (max_pfn > MAX_NONPAE_PFN) {
		max_pfn = MAX_NONPAE_PFN;
		printk(KERN_WARNING MSG_HIGHMEM_TRIMMED);
	}
664 665
#endif /* !CONFIG_HIGHMEM64G */
#endif /* !CONFIG_HIGHMEM */
666 667 668 669 670 671 672 673 674
}

/*
 * Determine low and high memory ranges:
 */
void __init find_low_pfn_range(void)
{
	/* it could update max_pfn */

675
	if (max_pfn <= MAXMEM_PFN)
676
		lowmem_pfn_init();
677 678
	else
		highmem_pfn_init();
679 680
}

681
#ifndef CONFIG_NEED_MULTIPLE_NODES
682
void __init initmem_init(void)
683 684 685 686 687 688 689 690 691 692 693
{
#ifdef CONFIG_HIGHMEM
	highstart_pfn = highend_pfn = max_pfn;
	if (max_pfn > max_low_pfn)
		highstart_pfn = max_low_pfn;
	printk(KERN_NOTICE "%ldMB HIGHMEM available.\n",
		pages_to_mb(highend_pfn - highstart_pfn));
	high_memory = (void *) __va(highstart_pfn * PAGE_SIZE - 1) + 1;
#else
	high_memory = (void *) __va(max_low_pfn * PAGE_SIZE - 1) + 1;
#endif
Tejun Heo's avatar
Tejun Heo committed
694

695
	memblock_set_node(0, PHYS_ADDR_MAX, &memblock.memory, 0);
Tejun Heo's avatar
Tejun Heo committed
696 697
	sparse_memory_present_with_active_regions(0);

698
#ifdef CONFIG_FLATMEM
699
	max_mapnr = IS_ENABLED(CONFIG_HIGHMEM) ? highend_pfn : max_low_pfn;
700
#endif
701 702
	__vmalloc_start_set = true;

703 704 705 706 707
	printk(KERN_NOTICE "%ldMB LOWMEM available.\n",
			pages_to_mb(max_low_pfn));

	setup_bootmem_allocator();
}
708
#endif /* !CONFIG_NEED_MULTIPLE_NODES */
709 710 711 712 713

void __init setup_bootmem_allocator(void)
{
	printk(KERN_INFO "  mapped low ram: 0 - %08lx\n",
		 max_pfn_mapped<<PAGE_SHIFT);
714
	printk(KERN_INFO "  low ram: 0 - %08lx\n", max_low_pfn<<PAGE_SHIFT);
715 716
}

Linus Torvalds's avatar
Linus Torvalds committed
717 718 719 720 721 722 723 724 725 726 727 728 729 730
/*
 * paging_init() sets up the page tables - note that the first 8MB are
 * already mapped by head.S.
 *
 * This routines also unmaps the page at virtual kernel address 0, so
 * that we can trap those pesky NULL-reference errors in the kernel.
 */
void __init paging_init(void)
{
	pagetable_init();

	__flush_tlb_all();

	kmap_init();
731 732 733 734

	/*
	 * NOTE: at this point the bootmem allocator is fully available.
	 */
735
	olpc_dt_build_devicetree();
736
	sparse_memory_present_with_active_regions(MAX_NUMNODES);
737 738
	sparse_init();
	zone_sizes_init();
Linus Torvalds's avatar
Linus Torvalds committed
739 740 741 742
}

/*
 * Test if the WP bit works in supervisor mode. It isn't supported on 386's
Dmitri Vorobiev's avatar
Dmitri Vorobiev committed
743 744 745
 * and also on some strange 486's. All 586+'s are OK. This used to involve
 * black magic jumps to work around some nasty CPU bugs, but fortunately the
 * switch to using exceptions got rid of all that.
Linus Torvalds's avatar
Linus Torvalds committed
746 747 748
 */
static void __init test_wp_bit(void)
{
749
	char z = 0;
750

751
	printk(KERN_INFO "Checking if this processor honours the WP bit even in supervisor mode...");
Linus Torvalds's avatar
Linus Torvalds committed
752

753
	__set_fixmap(FIX_WP_TEST, __pa_symbol(empty_zero_page), PAGE_KERNEL_RO);
Linus Torvalds's avatar
Linus Torvalds committed
754

755 756 757 758
	if (probe_kernel_write((char *)fix_to_virt(FIX_WP_TEST), &z, 1)) {
		clear_fixmap(FIX_WP_TEST);
		printk(KERN_CONT "Ok.\n");
		return;
Linus Torvalds's avatar
Linus Torvalds committed
759
	}
760

761 762
	printk(KERN_CONT "No.\n");
	panic("Linux doesn't support CPUs with broken WP.");
Linus Torvalds's avatar
Linus Torvalds committed
763 764 765 766
}

void __init mem_init(void)
{
767 768
	pci_iommu_alloc();

769
#ifdef CONFIG_FLATMEM
Eric Sesterhenn's avatar
Eric Sesterhenn committed
770
	BUG_ON(!mem_map);
Linus Torvalds's avatar
Linus Torvalds committed
771
#endif
772 773
	/*
	 * With CONFIG_DEBUG_PAGEALLOC initialization of highmem pages has to
774
	 * be done before memblock_free_all(). Memblock use free low memory for
775 776 777 778 779 780 781 782
	 * temporary data (see find_range_array()) and for this purpose can use
	 * pages that was already passed to the buddy allocator, hence marked as
	 * not accessible in the page tables when compiled with
	 * CONFIG_DEBUG_PAGEALLOC. Otherwise order of initialization is not
	 * important here.
	 */
	set_highmem_pages_init();

Linus Torvalds's avatar
Linus Torvalds committed
783
	/* this will put all low memory onto the freelists */
784
	memblock_free_all();
Linus Torvalds's avatar
Linus Torvalds committed
785

786
	after_bootmem = 1;
787
	x86_init.hyper.init_after_bootmem();
788

789
	mem_init_print_info(NULL);
790
	printk(KERN_INFO "virtual kernel memory layout:\n"
791
		"    fixmap  : 0x%08lx - 0x%08lx   (%4ld kB)\n"
792
		"  cpu_entry : 0x%08lx - 0x%08lx   (%4ld kB)\n"
793
#ifdef CONFIG_HIGHMEM
794
		"    pkmap   : 0x%08lx - 0x%08lx   (%4ld kB)\n"
795
#endif
796 797 798 799 800 801 802
		"    vmalloc : 0x%08lx - 0x%08lx   (%4ld MB)\n"
		"    lowmem  : 0x%08lx - 0x%08lx   (%4ld MB)\n"
		"      .init : 0x%08lx - 0x%08lx   (%4ld kB)\n"
		"      .data : 0x%08lx - 0x%08lx   (%4ld kB)\n"
		"      .text : 0x%08lx - 0x%08lx   (%4ld kB)\n",
		FIXADDR_START, FIXADDR_TOP,
		(FIXADDR_TOP - FIXADDR_START) >> 10,
803

804 805 806 807
		CPU_ENTRY_AREA_BASE,
		CPU_ENTRY_AREA_BASE + CPU_ENTRY_AREA_MAP_SIZE,
		CPU_ENTRY_AREA_MAP_SIZE >> 10,

808
#ifdef CONFIG_HIGHMEM
809 810
		PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE,
		(LAST_PKMAP*PAGE_SIZE) >> 10,
811 812
#endif

813 814
		VMALLOC_START, VMALLOC_END,
		(VMALLOC_END - VMALLOC_START) >> 20,
815

816 817
		(unsigned long)__va(0), (unsigned long)high_memory,
		((unsigned long)high_memory - (unsigned long)__va(0)) >> 20,
818

819 820 821
		(unsigned long)&__init_begin, (unsigned long)&__init_end,
		((unsigned long)&__init_end -
		 (unsigned long)&__init_begin) >> 10,
822

823 824
		(unsigned long)&_etext, (unsigned long)&_edata,
		((unsigned long)&_edata - (unsigned long)&_etext) >> 10,
825

826 827
		(unsigned long)&_text, (unsigned long)&_etext,
		((unsigned long)&_etext - (unsigned long)&_text) >> 10);
828

829 830 831 832 833 834 835 836 837 838 839 840 841 842
	/*
	 * Check boundaries twice: Some fundamental inconsistencies can
	 * be detected at build time already.
	 */
#define __FIXADDR_TOP (-PAGE_SIZE)
#ifdef CONFIG_HIGHMEM
	BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP*PAGE_SIZE	> FIXADDR_START);
	BUILD_BUG_ON(VMALLOC_END			> PKMAP_BASE);
#endif
#define high_memory (-128UL << 20)
	BUILD_BUG_ON(VMALLOC_START			>= VMALLOC_END);
#undef high_memory
#undef __FIXADDR_TOP

843
#ifdef CONFIG_HIGHMEM
844 845
	BUG_ON(PKMAP_BASE + LAST_PKMAP*PAGE_SIZE	> FIXADDR_START);
	BUG_ON(VMALLOC_END				> PKMAP_BASE);
846
#endif
847
	BUG_ON(VMALLOC_START				>= VMALLOC_END);
848
	BUG_ON((unsigned long)high_memory		> VMALLOC_START);
849

850
	test_wp_bit();
Linus Torvalds's avatar
Linus Torvalds committed
851 852
}

853
#ifdef CONFIG_MEMORY_HOTPLUG
854 855
int arch_add_memory(int nid, u64 start, u64 size,
			struct mhp_restrictions *restrictions)
856 857 858 859
{
	unsigned long start_pfn = start >> PAGE_SHIFT;
	unsigned long nr_pages = size >> PAGE_SHIFT;

860
	return __add_pages(nid, start_pfn, nr_pages, restrictions);
861
}
862 863

#ifdef CONFIG_MEMORY_HOTREMOVE
864 865
void arch_remove_memory(int nid, u64 start, u64 size,
			struct vmem_altmap *altmap)
866 867 868 869 870 871
{
	unsigned long start_pfn = start >> PAGE_SHIFT;
	unsigned long nr_pages = size >> PAGE_SHIFT;
	struct zone *zone;

	zone = page_zone(pfn_to_page(start_pfn));
872
	__remove_pages(zone, start_pfn, nr_pages, altmap);
873 874
}
#endif
875
#endif
876

877
int kernel_set_to_readonly __read_mostly;
878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906

void set_kernel_text_rw(void)
{
	unsigned long start = PFN_ALIGN(_text);
	unsigned long size = PFN_ALIGN(_etext) - start;

	if (!kernel_set_to_readonly)
		return;

	pr_debug("Set kernel text: %lx - %lx for read write\n",
		 start, start+size);

	set_pages_rw(virt_to_page(start), size >> PAGE_SHIFT);
}

void set_kernel_text_ro(void)
{
	unsigned long start = PFN_ALIGN(_text);
	unsigned long size = PFN_ALIGN(_etext) - start;

	if (!kernel_set_to_readonly)
		return;

	pr_debug("Set kernel text: %lx - %lx for read only\n",
		 start, start+size);

	set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
}

907 908 909 910
static void mark_nxdata_nx(void)
{
	/*
	 * When this called, init has already been executed and released,
Lucas De Marchi's avatar
Lucas De Marchi committed
911
	 * so everything past _etext should be NX.
912 913 914 915 916 917 918 919 920 921 922 923
	 */
	unsigned long start = PFN_ALIGN(_etext);
	/*
	 * This comes from is_kernel_text upper limit. Also HPAGE where used:
	 */
	unsigned long size = (((unsigned long)__init_end + HPAGE_SIZE) & HPAGE_MASK) - start;

	if (__supported_pte_mask & _PAGE_NX)
		printk(KERN_INFO "NX-protecting the kernel data: %luk\n", size >> 10);
	set_pages_nx(virt_to_page(start), size >> PAGE_SHIFT);
}

924 925
void mark_rodata_ro(void)
{
926
	unsigned long start = PFN_ALIGN(_text);
927
	unsigned long size = (unsigned long)__end_rodata - start;
928

929
	set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
930
	pr_info("Write protecting kernel text and read-only data: %luk\n",
931
		size >> 10);
932

933 934
	kernel_set_to_readonly = 1;

935
#ifdef CONFIG_CPA_DEBUG
936
	pr_info("Testing CPA: Reverting %lx-%lx\n", start, start + size);
937
	set_pages_rw(virt_to_page(start), size >> PAGE_SHIFT);
938

939
	pr_info("Testing CPA: write protecting again\n");
940
	set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
941
#endif
942
	mark_nxdata_nx();
Stephen Smalley's avatar
Stephen Smalley committed
943 944
	if (__supported_pte_mask & _PAGE_NX)
		debug_checkwx();
945
}