pgtable.c 7.64 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 *  linux/arch/i386/mm/pgtable.c
 */

#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/smp.h>
#include <linux/highmem.h>
#include <linux/slab.h>
#include <linux/pagemap.h>
#include <linux/spinlock.h>
15
#include <linux/module.h>
Linus Torvalds's avatar
Linus Torvalds committed
16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32

#include <asm/system.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/fixmap.h>
#include <asm/e820.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>

void show_mem(void)
{
	int total = 0, reserved = 0;
	int shared = 0, cached = 0;
	int highmem = 0;
	struct page *page;
	pg_data_t *pgdat;
	unsigned long i;
33
	unsigned long flags;
Linus Torvalds's avatar
Linus Torvalds committed
34

35
	printk(KERN_INFO "Mem-info:\n");
Linus Torvalds's avatar
Linus Torvalds committed
36
	show_free_areas();
37
	printk(KERN_INFO "Free swap:       %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
38
	for_each_online_pgdat(pgdat) {
39
		pgdat_resize_lock(pgdat, &flags);
Linus Torvalds's avatar
Linus Torvalds committed
40
		for (i = 0; i < pgdat->node_spanned_pages; ++i) {
41
			page = pgdat_page_nr(pgdat, i);
Linus Torvalds's avatar
Linus Torvalds committed
42 43 44 45 46 47 48 49 50 51
			total++;
			if (PageHighMem(page))
				highmem++;
			if (PageReserved(page))
				reserved++;
			else if (PageSwapCache(page))
				cached++;
			else if (page_count(page))
				shared += page_count(page) - 1;
		}
52
		pgdat_resize_unlock(pgdat, &flags);
Linus Torvalds's avatar
Linus Torvalds committed
53
	}
54 55 56 57 58
	printk(KERN_INFO "%d pages of RAM\n", total);
	printk(KERN_INFO "%d pages of HIGHMEM\n", highmem);
	printk(KERN_INFO "%d reserved pages\n", reserved);
	printk(KERN_INFO "%d pages shared\n", shared);
	printk(KERN_INFO "%d pages swap cached\n", cached);
59

60
	printk(KERN_INFO "%lu pages dirty\n", global_page_state(NR_FILE_DIRTY));
61 62
	printk(KERN_INFO "%lu pages writeback\n",
					global_page_state(NR_WRITEBACK));
63
	printk(KERN_INFO "%lu pages mapped\n", global_page_state(NR_FILE_MAPPED));
64 65 66
	printk(KERN_INFO "%lu pages slab\n",
		global_page_state(NR_SLAB_RECLAIMABLE) +
		global_page_state(NR_SLAB_UNRECLAIMABLE));
67 68
	printk(KERN_INFO "%lu pages pagetables\n",
					global_page_state(NR_PAGETABLE));
Linus Torvalds's avatar
Linus Torvalds committed
69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120
}

/*
 * Associate a virtual page frame with a given physical page frame 
 * and protection flags for that frame.
 */ 
static void set_pte_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	pgd = swapper_pg_dir + pgd_index(vaddr);
	if (pgd_none(*pgd)) {
		BUG();
		return;
	}
	pud = pud_offset(pgd, vaddr);
	if (pud_none(*pud)) {
		BUG();
		return;
	}
	pmd = pmd_offset(pud, vaddr);
	if (pmd_none(*pmd)) {
		BUG();
		return;
	}
	pte = pte_offset_kernel(pmd, vaddr);
	/* <pfn,flags> stored as-is, to permit clearing entries */
	set_pte(pte, pfn_pte(pfn, flags));

	/*
	 * It's enough to flush this one mapping.
	 * (PGE mappings get flushed as well)
	 */
	__flush_tlb_one(vaddr);
}

/*
 * Associate a large virtual page frame with a given physical page frame 
 * and protection flags for that frame. pfn is for the base of the page,
 * vaddr is what the page gets mapped to - both must be properly aligned. 
 * The pmd must already be instantiated. Assumes PAE mode.
 */ 
void set_pmd_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;

	if (vaddr & (PMD_SIZE-1)) {		/* vaddr is misaligned */
121
		printk(KERN_WARNING "set_pmd_pfn: vaddr misaligned\n");
Linus Torvalds's avatar
Linus Torvalds committed
122 123 124
		return; /* BUG(); */
	}
	if (pfn & (PTRS_PER_PTE-1)) {		/* pfn is misaligned */
125
		printk(KERN_WARNING "set_pmd_pfn: pfn misaligned\n");
Linus Torvalds's avatar
Linus Torvalds committed
126 127 128 129
		return; /* BUG(); */
	}
	pgd = swapper_pg_dir + pgd_index(vaddr);
	if (pgd_none(*pgd)) {
130
		printk(KERN_WARNING "set_pmd_pfn: pgd_none\n");
Linus Torvalds's avatar
Linus Torvalds committed
131 132 133 134 135 136 137 138 139 140 141 142
		return; /* BUG(); */
	}
	pud = pud_offset(pgd, vaddr);
	pmd = pmd_offset(pud, vaddr);
	set_pmd(pmd, pfn_pmd(pfn, flags));
	/*
	 * It's enough to flush this one mapping.
	 * (PGE mappings get flushed as well)
	 */
	__flush_tlb_one(vaddr);
}

143 144 145 146 147 148
static int fixmaps;
#ifndef CONFIG_COMPAT_VDSO
unsigned long __FIXADDR_TOP = 0xfffff000;
EXPORT_SYMBOL(__FIXADDR_TOP);
#endif

Linus Torvalds's avatar
Linus Torvalds committed
149 150 151 152 153 154 155 156 157
void __set_fixmap (enum fixed_addresses idx, unsigned long phys, pgprot_t flags)
{
	unsigned long address = __fix_to_virt(idx);

	if (idx >= __end_of_fixed_addresses) {
		BUG();
		return;
	}
	set_pte_pfn(address, phys >> PAGE_SHIFT, flags);
158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
	fixmaps++;
}

/**
 * reserve_top_address - reserves a hole in the top of kernel address space
 * @reserve - size of hole to reserve
 *
 * Can be used to relocate the fixmap area and poke a hole in the top
 * of kernel address space to make room for a hypervisor.
 */
void reserve_top_address(unsigned long reserve)
{
	BUG_ON(fixmaps > 0);
#ifdef CONFIG_COMPAT_VDSO
	BUG_ON(reserve != 0);
#else
	__FIXADDR_TOP = -reserve - PAGE_SIZE;
	__VMALLOC_RESERVE += reserve;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195
}

pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
{
	return (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
}

struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
	struct page *pte;

#ifdef CONFIG_HIGHPTE
	pte = alloc_pages(GFP_KERNEL|__GFP_HIGHMEM|__GFP_REPEAT|__GFP_ZERO, 0);
#else
	pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
#endif
	return pte;
}

196
void pmd_ctor(void *pmd, struct kmem_cache *cache, unsigned long flags)
Linus Torvalds's avatar
Linus Torvalds committed
197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220
{
	memset(pmd, 0, PTRS_PER_PMD*sizeof(pmd_t));
}

/*
 * List of all pgd's needed for non-PAE so it can invalidate entries
 * in both cached and uncached pgd's; not needed for PAE since the
 * kernel pmd is shared. If PAE were not to share the pmd a similar
 * tactic would be needed. This is essentially codepath-based locking
 * against pageattr.c; it is the unique case in which a valid change
 * of kernel pagetables can't be lazily synchronized by vmalloc faults.
 * vmalloc faults work because attached pagetables are never freed.
 * The locking scheme was chosen on the basis of manfred's
 * recommendations and having no core impact whatsoever.
 * -- wli
 */
DEFINE_SPINLOCK(pgd_lock);
struct page *pgd_list;

static inline void pgd_list_add(pgd_t *pgd)
{
	struct page *page = virt_to_page(pgd);
	page->index = (unsigned long)pgd_list;
	if (pgd_list)
221
		set_page_private(pgd_list, (unsigned long)&page->index);
Linus Torvalds's avatar
Linus Torvalds committed
222
	pgd_list = page;
223
	set_page_private(page, (unsigned long)&pgd_list);
Linus Torvalds's avatar
Linus Torvalds committed
224 225 226 227 228 229
}

static inline void pgd_list_del(pgd_t *pgd)
{
	struct page *next, **pprev, *page = virt_to_page(pgd);
	next = (struct page *)page->index;
230
	pprev = (struct page **)page_private(page);
Linus Torvalds's avatar
Linus Torvalds committed
231 232
	*pprev = next;
	if (next)
233
		set_page_private(next, (unsigned long)pprev);
Linus Torvalds's avatar
Linus Torvalds committed
234 235
}

236
void pgd_ctor(void *pgd, struct kmem_cache *cache, unsigned long unused)
Linus Torvalds's avatar
Linus Torvalds committed
237 238 239
{
	unsigned long flags;

240 241
	if (PTRS_PER_PMD == 1) {
		memset(pgd, 0, USER_PTRS_PER_PGD*sizeof(pgd_t));
Linus Torvalds's avatar
Linus Torvalds committed
242
		spin_lock_irqsave(&pgd_lock, flags);
243
	}
Linus Torvalds's avatar
Linus Torvalds committed
244

245
	clone_pgd_range((pgd_t *)pgd + USER_PTRS_PER_PGD,
Linus Torvalds's avatar
Linus Torvalds committed
246
			swapper_pg_dir + USER_PTRS_PER_PGD,
247
			KERNEL_PGD_PTRS);
Linus Torvalds's avatar
Linus Torvalds committed
248 249 250 251 252 253 254 255
	if (PTRS_PER_PMD > 1)
		return;

	pgd_list_add(pgd);
	spin_unlock_irqrestore(&pgd_lock, flags);
}

/* never called when PTRS_PER_PMD > 1 */
256
void pgd_dtor(void *pgd, struct kmem_cache *cache, unsigned long unused)
Linus Torvalds's avatar
Linus Torvalds committed
257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295
{
	unsigned long flags; /* can be called from interrupt context */

	spin_lock_irqsave(&pgd_lock, flags);
	pgd_list_del(pgd);
	spin_unlock_irqrestore(&pgd_lock, flags);
}

pgd_t *pgd_alloc(struct mm_struct *mm)
{
	int i;
	pgd_t *pgd = kmem_cache_alloc(pgd_cache, GFP_KERNEL);

	if (PTRS_PER_PMD == 1 || !pgd)
		return pgd;

	for (i = 0; i < USER_PTRS_PER_PGD; ++i) {
		pmd_t *pmd = kmem_cache_alloc(pmd_cache, GFP_KERNEL);
		if (!pmd)
			goto out_oom;
		set_pgd(&pgd[i], __pgd(1 + __pa(pmd)));
	}
	return pgd;

out_oom:
	for (i--; i >= 0; i--)
		kmem_cache_free(pmd_cache, (void *)__va(pgd_val(pgd[i])-1));
	kmem_cache_free(pgd_cache, pgd);
	return NULL;
}

void pgd_free(pgd_t *pgd)
{
	int i;

	/* in the PAE case user pgd entries are overwritten before usage */
	if (PTRS_PER_PMD > 1)
		for (i = 0; i < USER_PTRS_PER_PGD; ++i)
			kmem_cache_free(pmd_cache, (void *)__va(pgd_val(pgd[i])-1));
296
	/* in the non-PAE case, free_pgtables() clears user pgd entries */
Linus Torvalds's avatar
Linus Torvalds committed
297 298
	kmem_cache_free(pgd_cache, pgd);
}