mlock.c 22.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7
/*
 *	linux/mm/mlock.c
 *
 *  (C) Copyright 1995 Linus Torvalds
 *  (C) Copyright 2002 Christoph Hellwig
 */

8
#include <linux/capability.h>
Linus Torvalds's avatar
Linus Torvalds committed
9 10
#include <linux/mman.h>
#include <linux/mm.h>
11 12 13
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/pagemap.h>
14
#include <linux/pagevec.h>
Linus Torvalds's avatar
Linus Torvalds committed
15 16
#include <linux/mempolicy.h>
#include <linux/syscalls.h>
17
#include <linux/sched.h>
18
#include <linux/export.h>
19 20 21
#include <linux/rmap.h>
#include <linux/mmzone.h>
#include <linux/hugetlb.h>
22 23
#include <linux/memcontrol.h>
#include <linux/mm_inline.h>
24 25

#include "internal.h"
Linus Torvalds's avatar
Linus Torvalds committed
26

27 28
int can_do_mlock(void)
{
Jiri Slaby's avatar
Jiri Slaby committed
29
	if (rlimit(RLIMIT_MEMLOCK) != 0)
30
		return 1;
31 32
	if (capable(CAP_IPC_LOCK))
		return 1;
33 34 35
	return 0;
}
EXPORT_SYMBOL(can_do_mlock);
Linus Torvalds's avatar
Linus Torvalds committed
36

37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56
/*
 * Mlocked pages are marked with PageMlocked() flag for efficient testing
 * in vmscan and, possibly, the fault path; and to support semi-accurate
 * statistics.
 *
 * An mlocked page [PageMlocked(page)] is unevictable.  As such, it will
 * be placed on the LRU "unevictable" list, rather than the [in]active lists.
 * The unevictable list is an LRU sibling list to the [in]active lists.
 * PageUnevictable is set to indicate the unevictable state.
 *
 * When lazy mlocking via vmscan, it is important to ensure that the
 * vma's VM_LOCKED status is not concurrently being modified, otherwise we
 * may have mlocked a page that is being munlocked. So lazy mlock must take
 * the mmap_sem for read, and verify that the vma really is locked
 * (see mm/rmap.c).
 */

/*
 *  LRU accounting for clear_page_mlock()
 */
57
void clear_page_mlock(struct page *page)
58
{
59
	if (!TestClearPageMlocked(page))
60 61
		return;

62 63
	mod_zone_page_state(page_zone(page), NR_MLOCK,
			    -hpage_nr_pages(page));
64
	count_vm_event(UNEVICTABLE_PGCLEARED);
65 66 67 68
	if (!isolate_lru_page(page)) {
		putback_lru_page(page);
	} else {
		/*
69
		 * We lost the race. the page already moved to evictable list.
70
		 */
71
		if (PageUnevictable(page))
72
			count_vm_event(UNEVICTABLE_PGSTRANDED);
73 74 75 76 77 78 79 80 81
	}
}

/*
 * Mark page as mlocked if not already.
 * If page on LRU, isolate and putback to move to unevictable list.
 */
void mlock_vma_page(struct page *page)
{
82
	/* Serialize with page migration */
83 84
	BUG_ON(!PageLocked(page));

85
	if (!TestSetPageMlocked(page)) {
86 87
		mod_zone_page_state(page_zone(page), NR_MLOCK,
				    hpage_nr_pages(page));
88 89 90 91
		count_vm_event(UNEVICTABLE_PGMLOCKED);
		if (!isolate_lru_page(page))
			putback_lru_page(page);
	}
92 93
}

94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
/*
 * Isolate a page from LRU with optional get_page() pin.
 * Assumes lru_lock already held and page already pinned.
 */
static bool __munlock_isolate_lru_page(struct page *page, bool getpage)
{
	if (PageLRU(page)) {
		struct lruvec *lruvec;

		lruvec = mem_cgroup_page_lruvec(page, page_zone(page));
		if (getpage)
			get_page(page);
		ClearPageLRU(page);
		del_page_from_lru_list(page, lruvec, page_lru(page));
		return true;
	}

	return false;
}

114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
/*
 * Finish munlock after successful page isolation
 *
 * Page must be locked. This is a wrapper for try_to_munlock()
 * and putback_lru_page() with munlock accounting.
 */
static void __munlock_isolated_page(struct page *page)
{
	int ret = SWAP_AGAIN;

	/*
	 * Optimization: if the page was mapped just once, that's our mapping
	 * and we don't need to check all the other vmas.
	 */
	if (page_mapcount(page) > 1)
		ret = try_to_munlock(page);

	/* Did try_to_unlock() succeed or punt? */
	if (ret != SWAP_MLOCK)
		count_vm_event(UNEVICTABLE_PGMUNLOCKED);

	putback_lru_page(page);
}

/*
 * Accounting for page isolation fail during munlock
 *
 * Performs accounting when page isolation fails in munlock. There is nothing
 * else to do because it means some other task has already removed the page
 * from the LRU. putback_lru_page() will take care of removing the page from
 * the unevictable list, if necessary. vmscan [page_referenced()] will move
 * the page back to the unevictable list if some other vma has it mlocked.
 */
static void __munlock_isolation_failed(struct page *page)
{
	if (PageUnevictable(page))
150
		__count_vm_event(UNEVICTABLE_PGSTRANDED);
151
	else
152
		__count_vm_event(UNEVICTABLE_PGMUNLOCKED);
153 154
}

155 156
/**
 * munlock_vma_page - munlock a vma page
157 158 159 160
 * @page - page to be unlocked, either a normal page or THP page head
 *
 * returns the size of the page as a page mask (0 for normal page,
 *         HPAGE_PMD_NR - 1 for THP head page)
161
 *
162 163 164 165 166 167 168 169 170 171
 * called from munlock()/munmap() path with page supposedly on the LRU.
 * When we munlock a page, because the vma where we found the page is being
 * munlock()ed or munmap()ed, we want to check whether other vmas hold the
 * page locked so that we can leave it on the unevictable lru list and not
 * bother vmscan with it.  However, to walk the page's rmap list in
 * try_to_munlock() we must isolate the page from the LRU.  If some other
 * task has removed the page from the LRU, we won't be able to do that.
 * So we clear the PageMlocked as we might not get another chance.  If we
 * can't isolate the page, we leave it for putback_lru_page() and vmscan
 * [page_referenced()/try_to_unmap()] to deal with.
172
 */
173
unsigned int munlock_vma_page(struct page *page)
174
{
175
	unsigned int nr_pages;
176
	struct zone *zone = page_zone(page);
177

178
	/* For try_to_munlock() and to serialize with page migration */
179 180
	BUG_ON(!PageLocked(page));

181
	/*
182 183 184
	 * Serialize with any parallel __split_huge_page_refcount() which
	 * might otherwise copy PageMlocked to part of the tail pages before
	 * we clear it in the head page. It also stabilizes hpage_nr_pages().
185
	 */
186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204
	spin_lock_irq(&zone->lru_lock);

	nr_pages = hpage_nr_pages(page);
	if (!TestClearPageMlocked(page))
		goto unlock_out;

	__mod_zone_page_state(zone, NR_MLOCK, -nr_pages);

	if (__munlock_isolate_lru_page(page, true)) {
		spin_unlock_irq(&zone->lru_lock);
		__munlock_isolated_page(page);
		goto out;
	}
	__munlock_isolation_failed(page);

unlock_out:
	spin_unlock_irq(&zone->lru_lock);

out:
205
	return nr_pages - 1;
206 207
}

208
/**
209
 * populate_vma_page_range() -  populate a range of pages in the vma.
210 211 212
 * @vma:   target vma
 * @start: start address
 * @end:   end address
213
 * @nonblocking:
214
 *
215
 * This takes care of mlocking the pages too if VM_LOCKED is set.
216
 *
217
 * return 0 on success, negative error code on error.
218
 *
219 220 221 222 223 224 225
 * vma->vm_mm->mmap_sem must be held.
 *
 * If @nonblocking is NULL, it may be held for read or write and will
 * be unperturbed.
 *
 * If @nonblocking is non-NULL, it must held for read only and may be
 * released.  If it's released, *@nonblocking will be set to 0.
226
 */
227
long populate_vma_page_range(struct vm_area_struct *vma,
228
		unsigned long start, unsigned long end, int *nonblocking)
229 230
{
	struct mm_struct *mm = vma->vm_mm;
231
	unsigned long nr_pages = (end - start) / PAGE_SIZE;
232
	int gup_flags;
233 234 235

	VM_BUG_ON(start & ~PAGE_MASK);
	VM_BUG_ON(end   & ~PAGE_MASK);
236 237
	VM_BUG_ON_VMA(start < vma->vm_start, vma);
	VM_BUG_ON_VMA(end   > vma->vm_end, vma);
238
	VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_sem), mm);
239

240
	gup_flags = FOLL_TOUCH | FOLL_POPULATE;
241 242 243 244 245 246
	/*
	 * We want to touch writable mappings with a write fault in order
	 * to break COW, except for shared mappings because these don't COW
	 * and we would not want to dirty them for nothing.
	 */
	if ((vma->vm_flags & (VM_WRITE | VM_SHARED)) == VM_WRITE)
247
		gup_flags |= FOLL_WRITE;
248

249 250 251 252 253 254 255
	/*
	 * We want mlock to succeed for regions that have any permissions
	 * other than PROT_NONE.
	 */
	if (vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC))
		gup_flags |= FOLL_FORCE;

256 257 258 259
	/*
	 * We made sure addr is within a VMA, so the following will
	 * not result in a stack expansion that recurses back here.
	 */
260
	return __get_user_pages(current, mm, start, nr_pages, gup_flags,
261
				NULL, NULL, nonblocking);
262 263 264 265 266 267 268 269 270 271 272 273
}

/*
 * convert get_user_pages() return value to posix mlock() error
 */
static int __mlock_posix_error_return(long retval)
{
	if (retval == -EFAULT)
		retval = -ENOMEM;
	else if (retval == -ENOMEM)
		retval = -EAGAIN;
	return retval;
274 275
}

276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
/*
 * Prepare page for fast batched LRU putback via putback_lru_evictable_pagevec()
 *
 * The fast path is available only for evictable pages with single mapping.
 * Then we can bypass the per-cpu pvec and get better performance.
 * when mapcount > 1 we need try_to_munlock() which can fail.
 * when !page_evictable(), we need the full redo logic of putback_lru_page to
 * avoid leaving evictable page in unevictable list.
 *
 * In case of success, @page is added to @pvec and @pgrescued is incremented
 * in case that the page was previously unevictable. @page is also unlocked.
 */
static bool __putback_lru_fast_prepare(struct page *page, struct pagevec *pvec,
		int *pgrescued)
{
291 292
	VM_BUG_ON_PAGE(PageLRU(page), page);
	VM_BUG_ON_PAGE(!PageLocked(page), page);
293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321

	if (page_mapcount(page) <= 1 && page_evictable(page)) {
		pagevec_add(pvec, page);
		if (TestClearPageUnevictable(page))
			(*pgrescued)++;
		unlock_page(page);
		return true;
	}

	return false;
}

/*
 * Putback multiple evictable pages to the LRU
 *
 * Batched putback of evictable pages that bypasses the per-cpu pvec. Some of
 * the pages might have meanwhile become unevictable but that is OK.
 */
static void __putback_lru_fast(struct pagevec *pvec, int pgrescued)
{
	count_vm_events(UNEVICTABLE_PGMUNLOCKED, pagevec_count(pvec));
	/*
	 *__pagevec_lru_add() calls release_pages() so we don't call
	 * put_page() explicitly
	 */
	__pagevec_lru_add(pvec);
	count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
}

322 323 324 325 326 327 328 329
/*
 * Munlock a batch of pages from the same zone
 *
 * The work is split to two main phases. First phase clears the Mlocked flag
 * and attempts to isolate the pages, all under a single zone lru lock.
 * The second phase finishes the munlock only for pages where isolation
 * succeeded.
 *
330
 * Note that the pagevec may be modified during the process.
331 332 333 334 335
 */
static void __munlock_pagevec(struct pagevec *pvec, struct zone *zone)
{
	int i;
	int nr = pagevec_count(pvec);
336
	int delta_munlocked;
337 338
	struct pagevec pvec_putback;
	int pgrescued = 0;
339

340 341
	pagevec_init(&pvec_putback, 0);

342 343 344 345 346 347 348
	/* Phase 1: page isolation */
	spin_lock_irq(&zone->lru_lock);
	for (i = 0; i < nr; i++) {
		struct page *page = pvec->pages[i];

		if (TestClearPageMlocked(page)) {
			/*
349 350
			 * We already have pin from follow_page_mask()
			 * so we can spare the get_page() here.
351
			 */
352 353 354 355
			if (__munlock_isolate_lru_page(page, false))
				continue;
			else
				__munlock_isolation_failed(page);
356
		}
357 358 359 360 361 362 363 364 365

		/*
		 * We won't be munlocking this page in the next phase
		 * but we still need to release the follow_page_mask()
		 * pin. We cannot do it under lru_lock however. If it's
		 * the last pin, __page_cache_release() would deadlock.
		 */
		pagevec_add(&pvec_putback, pvec->pages[i]);
		pvec->pages[i] = NULL;
366
	}
367
	delta_munlocked = -nr + pagevec_count(&pvec_putback);
368
	__mod_zone_page_state(zone, NR_MLOCK, delta_munlocked);
369 370
	spin_unlock_irq(&zone->lru_lock);

371 372 373
	/* Now we can release pins of pages that we are not munlocking */
	pagevec_release(&pvec_putback);

374
	/* Phase 2: page munlock */
375 376 377 378 379
	for (i = 0; i < nr; i++) {
		struct page *page = pvec->pages[i];

		if (page) {
			lock_page(page);
380 381
			if (!__putback_lru_fast_prepare(page, &pvec_putback,
					&pgrescued)) {
382 383 384 385 386
				/*
				 * Slow path. We don't want to lose the last
				 * pin before unlock_page()
				 */
				get_page(page); /* for putback_lru_page() */
387 388
				__munlock_isolated_page(page);
				unlock_page(page);
389
				put_page(page); /* from follow_page_mask() */
390
			}
391 392
		}
	}
393

394 395 396 397
	/*
	 * Phase 3: page putback for pages that qualified for the fast path
	 * This will also call put_page() to return pin from follow_page_mask()
	 */
398 399
	if (pagevec_count(&pvec_putback))
		__putback_lru_fast(&pvec_putback, pgrescued);
400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423
}

/*
 * Fill up pagevec for __munlock_pagevec using pte walk
 *
 * The function expects that the struct page corresponding to @start address is
 * a non-TPH page already pinned and in the @pvec, and that it belongs to @zone.
 *
 * The rest of @pvec is filled by subsequent pages within the same pmd and same
 * zone, as long as the pte's are present and vm_normal_page() succeeds. These
 * pages also get pinned.
 *
 * Returns the address of the next page that should be scanned. This equals
 * @start + PAGE_SIZE when no page could be added by the pte walk.
 */
static unsigned long __munlock_pagevec_fill(struct pagevec *pvec,
		struct vm_area_struct *vma, int zoneid,	unsigned long start,
		unsigned long end)
{
	pte_t *pte;
	spinlock_t *ptl;

	/*
	 * Initialize pte walk starting at the already pinned page where we
424 425
	 * are sure that there is a pte, as it was pinned under the same
	 * mmap_sem write op.
426 427
	 */
	pte = get_locked_pte(vma->vm_mm, start,	&ptl);
428 429 430 431
	/* Make sure we do not cross the page table boundary */
	end = pgd_addr_end(start, end);
	end = pud_addr_end(start, end);
	end = pmd_addr_end(start, end);
432 433 434 435 436 437 438 439 440 441 442 443 444 445

	/* The page next to the pinned page is the first we will try to get */
	start += PAGE_SIZE;
	while (start < end) {
		struct page *page = NULL;
		pte++;
		if (pte_present(*pte))
			page = vm_normal_page(vma, start, *pte);
		/*
		 * Break if page could not be obtained or the page's node+zone does not
		 * match
		 */
		if (!page || page_zone_id(page) != zoneid)
			break;
446

447 448 449 450 451 452 453 454 455 456 457
		get_page(page);
		/*
		 * Increase the address that will be returned *before* the
		 * eventual break due to pvec becoming full by adding the page
		 */
		start += PAGE_SIZE;
		if (pagevec_add(pvec, page) == 0)
			break;
	}
	pte_unmap_unlock(pte, ptl);
	return start;
458 459
}

460
/*
461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476
 * munlock_vma_pages_range() - munlock all pages in the vma range.'
 * @vma - vma containing range to be munlock()ed.
 * @start - start address in @vma of the range
 * @end - end of range in @vma.
 *
 *  For mremap(), munmap() and exit().
 *
 * Called with @vma VM_LOCKED.
 *
 * Returns with VM_LOCKED cleared.  Callers must be prepared to
 * deal with this.
 *
 * We don't save and restore VM_LOCKED here because pages are
 * still on lru.  In unmap path, pages might be scanned by reclaim
 * and re-mlocked by try_to_{munlock|unmap} before we unmap and
 * free them.  This will result in freeing mlocked pages.
477
 */
478
void munlock_vma_pages_range(struct vm_area_struct *vma,
479
			     unsigned long start, unsigned long end)
480 481
{
	vma->vm_flags &= ~VM_LOCKED;
482

483
	while (start < end) {
484
		struct page *page = NULL;
485 486
		unsigned int page_mask;
		unsigned long page_increm;
487 488 489
		struct pagevec pvec;
		struct zone *zone;
		int zoneid;
490

491
		pagevec_init(&pvec, 0);
492 493 494 495 496 497 498
		/*
		 * Although FOLL_DUMP is intended for get_dump_page(),
		 * it just so happens that its special treatment of the
		 * ZERO_PAGE (returning an error instead of doing get_page)
		 * suits munlock very well (and if somehow an abnormal page
		 * has sneaked into the range, we won't oops here: great).
		 */
499
		page = follow_page_mask(vma, start, FOLL_GET | FOLL_DUMP,
500 501
				&page_mask);

502
		if (page && !IS_ERR(page)) {
503 504 505 506 507 508 509 510 511 512 513 514 515
			if (PageTransHuge(page)) {
				lock_page(page);
				/*
				 * Any THP page found by follow_page_mask() may
				 * have gotten split before reaching
				 * munlock_vma_page(), so we need to recompute
				 * the page_mask here.
				 */
				page_mask = munlock_vma_page(page);
				unlock_page(page);
				put_page(page); /* follow_page_mask() */
			} else {
				/*
516 517 518 519 520 521 522 523 524 525 526 527 528
				 * Non-huge pages are handled in batches via
				 * pagevec. The pin from follow_page_mask()
				 * prevents them from collapsing by THP.
				 */
				pagevec_add(&pvec, page);
				zone = page_zone(page);
				zoneid = page_zone_id(page);

				/*
				 * Try to fill the rest of pagevec using fast
				 * pte walk. This will also update start to
				 * the next page to process. Then munlock the
				 * pagevec.
529
				 */
530 531 532 533
				start = __munlock_pagevec_fill(&pvec, vma,
						zoneid, start, end);
				__munlock_pagevec(&pvec, zone);
				goto next;
534
			}
535
		}
536 537 538
		/* It's a bug to munlock in the middle of a THP page */
		VM_BUG_ON((start >> PAGE_SHIFT) & page_mask);
		page_increm = 1 + page_mask;
539
		start += page_increm * PAGE_SIZE;
540
next:
541 542
		cond_resched();
	}
543 544 545 546 547 548 549
}

/*
 * mlock_fixup  - handle mlock[all]/munlock[all] requests.
 *
 * Filters out "special" vmas -- VM_LOCKED never gets set for these, and
 * munlock is a no-op.  However, for some special vmas, we go ahead and
550
 * populate the ptes.
551 552 553
 *
 * For vmas that pass the filters, merge/split as appropriate.
 */
Linus Torvalds's avatar
Linus Torvalds committed
554
static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev,
555
	unsigned long start, unsigned long end, vm_flags_t newflags)
Linus Torvalds's avatar
Linus Torvalds committed
556
{
557
	struct mm_struct *mm = vma->vm_mm;
Linus Torvalds's avatar
Linus Torvalds committed
558
	pgoff_t pgoff;
559
	int nr_pages;
Linus Torvalds's avatar
Linus Torvalds committed
560
	int ret = 0;
561
	int lock = !!(newflags & VM_LOCKED);
Linus Torvalds's avatar
Linus Torvalds committed
562

563
	if (newflags == vma->vm_flags || (vma->vm_flags & VM_SPECIAL) ||
564
	    is_vm_hugetlb_page(vma) || vma == get_gate_vma(current->mm))
565 566
		goto out;	/* don't set VM_LOCKED,  don't count */

Linus Torvalds's avatar
Linus Torvalds committed
567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
	pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
	*prev = vma_merge(mm, *prev, start, end, newflags, vma->anon_vma,
			  vma->vm_file, pgoff, vma_policy(vma));
	if (*prev) {
		vma = *prev;
		goto success;
	}

	if (start != vma->vm_start) {
		ret = split_vma(mm, vma, start, 1);
		if (ret)
			goto out;
	}

	if (end != vma->vm_end) {
		ret = split_vma(mm, vma, end, 0);
		if (ret)
			goto out;
	}

success:
588 589 590 591 592 593 594 595
	/*
	 * Keep track of amount of locked VM.
	 */
	nr_pages = (end - start) >> PAGE_SHIFT;
	if (!lock)
		nr_pages = -nr_pages;
	mm->locked_vm += nr_pages;

Linus Torvalds's avatar
Linus Torvalds committed
596 597 598
	/*
	 * vm_flags is protected by the mmap_sem held in write mode.
	 * It's okay if try_to_unmap_one unmaps a page just after we
599
	 * set VM_LOCKED, populate_vma_page_range will bring it back.
Linus Torvalds's avatar
Linus Torvalds committed
600 601
	 */

602
	if (lock)
603
		vma->vm_flags = newflags;
604
	else
605
		munlock_vma_pages_range(vma, start, end);
Linus Torvalds's avatar
Linus Torvalds committed
606 607

out:
608
	*prev = vma;
Linus Torvalds's avatar
Linus Torvalds committed
609 610 611 612 613 614 615 616 617
	return ret;
}

static int do_mlock(unsigned long start, size_t len, int on)
{
	unsigned long nstart, end, tmp;
	struct vm_area_struct * vma, * prev;
	int error;

618 619
	VM_BUG_ON(start & ~PAGE_MASK);
	VM_BUG_ON(len != PAGE_ALIGN(len));
Linus Torvalds's avatar
Linus Torvalds committed
620 621 622 623 624
	end = start + len;
	if (end < start)
		return -EINVAL;
	if (end == start)
		return 0;
625
	vma = find_vma(current->mm, start);
Linus Torvalds's avatar
Linus Torvalds committed
626 627 628
	if (!vma || vma->vm_start > start)
		return -ENOMEM;

629
	prev = vma->vm_prev;
Linus Torvalds's avatar
Linus Torvalds committed
630 631 632 633
	if (start > vma->vm_start)
		prev = vma;

	for (nstart = start ; ; ) {
634
		vm_flags_t newflags;
Linus Torvalds's avatar
Linus Torvalds committed
635 636 637

		/* Here we know that  vma->vm_start <= nstart < vma->vm_end. */

638 639
		newflags = vma->vm_flags & ~VM_LOCKED;
		if (on)
640
			newflags |= VM_LOCKED;
Linus Torvalds's avatar
Linus Torvalds committed
641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662

		tmp = vma->vm_end;
		if (tmp > end)
			tmp = end;
		error = mlock_fixup(vma, &prev, nstart, tmp, newflags);
		if (error)
			break;
		nstart = tmp;
		if (nstart < prev->vm_end)
			nstart = prev->vm_end;
		if (nstart >= end)
			break;

		vma = prev->vm_next;
		if (!vma || vma->vm_start != nstart) {
			error = -ENOMEM;
			break;
		}
	}
	return error;
}

663 664 665 666 667 668 669 670
/*
 * __mm_populate - populate and/or mlock pages within a range of address space.
 *
 * This is used to implement mlock() and the MAP_POPULATE / MAP_LOCKED mmap
 * flags. VMAs must be already marked with the desired vm_flags, and
 * mmap_sem must not be held.
 */
int __mm_populate(unsigned long start, unsigned long len, int ignore_errors)
671 672 673 674
{
	struct mm_struct *mm = current->mm;
	unsigned long end, nstart, nend;
	struct vm_area_struct *vma = NULL;
675
	int locked = 0;
676
	long ret = 0;
677 678 679 680 681 682 683 684 685 686

	VM_BUG_ON(start & ~PAGE_MASK);
	VM_BUG_ON(len != PAGE_ALIGN(len));
	end = start + len;

	for (nstart = start; nstart < end; nstart = nend) {
		/*
		 * We want to fault in pages for [nstart; end) address range.
		 * Find first corresponding VMA.
		 */
687 688 689
		if (!locked) {
			locked = 1;
			down_read(&mm->mmap_sem);
690
			vma = find_vma(mm, nstart);
691
		} else if (nstart >= vma->vm_end)
692 693 694 695 696 697 698 699
			vma = vma->vm_next;
		if (!vma || vma->vm_start >= end)
			break;
		/*
		 * Set [nstart; nend) to intersection of desired address
		 * range with the first VMA. Also, skip undesirable VMA types.
		 */
		nend = min(end, vma->vm_end);
700
		if (vma->vm_flags & (VM_IO | VM_PFNMAP))
701 702 703 704
			continue;
		if (nstart < vma->vm_start)
			nstart = vma->vm_start;
		/*
705
		 * Now fault in a range of pages. populate_vma_page_range()
706 707
		 * double checks the vma flags, so that it won't mlock pages
		 * if the vma was already munlocked.
708
		 */
709
		ret = populate_vma_page_range(vma, nstart, nend, &locked);
710 711 712 713 714
		if (ret < 0) {
			if (ignore_errors) {
				ret = 0;
				continue;	/* continue at next VMA */
			}
715 716 717
			ret = __mlock_posix_error_return(ret);
			break;
		}
718 719
		nend = nstart + ret * PAGE_SIZE;
		ret = 0;
720
	}
721 722
	if (locked)
		up_read(&mm->mmap_sem);
723 724 725
	return ret;	/* 0 or negative error code */
}

726
SYSCALL_DEFINE2(mlock, unsigned long, start, size_t, len)
Linus Torvalds's avatar
Linus Torvalds committed
727 728 729 730 731 732 733 734
{
	unsigned long locked;
	unsigned long lock_limit;
	int error = -ENOMEM;

	if (!can_do_mlock())
		return -EPERM;

735 736
	lru_add_drain_all();	/* flush pagevec */

Linus Torvalds's avatar
Linus Torvalds committed
737 738 739
	len = PAGE_ALIGN(len + (start & ~PAGE_MASK));
	start &= PAGE_MASK;

Jiri Slaby's avatar
Jiri Slaby committed
740
	lock_limit = rlimit(RLIMIT_MEMLOCK);
Linus Torvalds's avatar
Linus Torvalds committed
741
	lock_limit >>= PAGE_SHIFT;
742 743 744 745 746
	locked = len >> PAGE_SHIFT;

	down_write(&current->mm->mmap_sem);

	locked += current->mm->locked_vm;
Linus Torvalds's avatar
Linus Torvalds committed
747 748 749 750

	/* check against resource limits */
	if ((locked <= lock_limit) || capable(CAP_IPC_LOCK))
		error = do_mlock(start, len, 1);
751

Linus Torvalds's avatar
Linus Torvalds committed
752
	up_write(&current->mm->mmap_sem);
753
	if (!error)
754
		error = __mm_populate(start, len, 0);
Linus Torvalds's avatar
Linus Torvalds committed
755 756 757
	return error;
}

758
SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len)
Linus Torvalds's avatar
Linus Torvalds committed
759 760 761 762 763
{
	int ret;

	len = PAGE_ALIGN(len + (start & ~PAGE_MASK));
	start &= PAGE_MASK;
764 765

	down_write(&current->mm->mmap_sem);
Linus Torvalds's avatar
Linus Torvalds committed
766 767
	ret = do_mlock(start, len, 0);
	up_write(&current->mm->mmap_sem);
768

Linus Torvalds's avatar
Linus Torvalds committed
769 770 771 772 773 774 775 776
	return ret;
}

static int do_mlockall(int flags)
{
	struct vm_area_struct * vma, * prev = NULL;

	if (flags & MCL_FUTURE)
777
		current->mm->def_flags |= VM_LOCKED;
778
	else
779
		current->mm->def_flags &= ~VM_LOCKED;
Linus Torvalds's avatar
Linus Torvalds committed
780 781 782 783
	if (flags == MCL_FUTURE)
		goto out;

	for (vma = current->mm->mmap; vma ; vma = prev->vm_next) {
784
		vm_flags_t newflags;
Linus Torvalds's avatar
Linus Torvalds committed
785

786 787
		newflags = vma->vm_flags & ~VM_LOCKED;
		if (flags & MCL_CURRENT)
788
			newflags |= VM_LOCKED;
Linus Torvalds's avatar
Linus Torvalds committed
789 790 791

		/* Ignore errors */
		mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags);
792
		cond_resched_rcu_qs();
Linus Torvalds's avatar
Linus Torvalds committed
793 794 795 796 797
	}
out:
	return 0;
}

798
SYSCALL_DEFINE1(mlockall, int, flags)
Linus Torvalds's avatar
Linus Torvalds committed
799 800 801 802 803 804 805 806 807 808 809
{
	unsigned long lock_limit;
	int ret = -EINVAL;

	if (!flags || (flags & ~(MCL_CURRENT | MCL_FUTURE)))
		goto out;

	ret = -EPERM;
	if (!can_do_mlock())
		goto out;

810 811
	if (flags & MCL_CURRENT)
		lru_add_drain_all();	/* flush pagevec */
812

Jiri Slaby's avatar
Jiri Slaby committed
813
	lock_limit = rlimit(RLIMIT_MEMLOCK);
Linus Torvalds's avatar
Linus Torvalds committed
814 815 816
	lock_limit >>= PAGE_SHIFT;

	ret = -ENOMEM;
817 818
	down_write(&current->mm->mmap_sem);

Linus Torvalds's avatar
Linus Torvalds committed
819 820 821 822
	if (!(flags & MCL_CURRENT) || (current->mm->total_vm <= lock_limit) ||
	    capable(CAP_IPC_LOCK))
		ret = do_mlockall(flags);
	up_write(&current->mm->mmap_sem);
823 824
	if (!ret && (flags & MCL_CURRENT))
		mm_populate(0, TASK_SIZE);
Linus Torvalds's avatar
Linus Torvalds committed
825 826 827 828
out:
	return ret;
}

829
SYSCALL_DEFINE0(munlockall)
Linus Torvalds's avatar
Linus Torvalds committed
830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850
{
	int ret;

	down_write(&current->mm->mmap_sem);
	ret = do_mlockall(0);
	up_write(&current->mm->mmap_sem);
	return ret;
}

/*
 * Objects with different lifetime than processes (SHM_LOCK and SHM_HUGETLB
 * shm segments) get accounted against the user_struct instead.
 */
static DEFINE_SPINLOCK(shmlock_user_lock);

int user_shm_lock(size_t size, struct user_struct *user)
{
	unsigned long lock_limit, locked;
	int allowed = 0;

	locked = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
Jiri Slaby's avatar
Jiri Slaby committed
851
	lock_limit = rlimit(RLIMIT_MEMLOCK);
852 853
	if (lock_limit == RLIM_INFINITY)
		allowed = 1;
Linus Torvalds's avatar
Linus Torvalds committed
854 855
	lock_limit >>= PAGE_SHIFT;
	spin_lock(&shmlock_user_lock);
856 857
	if (!allowed &&
	    locked + user->locked_shm > lock_limit && !capable(CAP_IPC_LOCK))
Linus Torvalds's avatar
Linus Torvalds committed
858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873
		goto out;
	get_uid(user);
	user->locked_shm += locked;
	allowed = 1;
out:
	spin_unlock(&shmlock_user_lock);
	return allowed;
}

void user_shm_unlock(size_t size, struct user_struct *user)
{
	spin_lock(&shmlock_user_lock);
	user->locked_shm -= (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
	spin_unlock(&shmlock_user_lock);
	free_uid(user);
}