shm.c 42.4 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
Linus Torvalds's avatar
Linus Torvalds committed
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 * linux/ipc/shm.c
 * Copyright (C) 1992, 1993 Krishna Balasubramanian
 *	 Many improvements/fixes by Bruno Haible.
 * Replaced `struct shm_desc' by `struct vm_area_struct', July 1994.
 * Fixed the shm swap deallocation (shm_unuse()), August 1998 Andrea Arcangeli.
 *
 * /proc/sysvipc/shm support (c) 1999 Dragos Acostachioaie <dragos@iname.com>
 * BIGMEM support, Andrea Arcangeli <andrea@suse.de>
 * SMP thread shm, Jean-Luc Boyard <jean-luc.boyard@siemens.fr>
 * HIGHMEM support, Ingo Molnar <mingo@redhat.com>
 * Make shmmax, shmall, shmmni sysctl'able, Christoph Rohland <cr@sap.com>
 * Shared /dev/zero support, Kanoj Sarcar <kanoj@sgi.com>
 * Move the mm functionality over to mm/shmem.c, Christoph Rohland <cr@sap.com>
 *
17 18
 * support for audit of ipc object properties and permission changes
 * Dustin Kirkland <dustin.kirkland@us.ibm.com>
19 20 21 22
 *
 * namespaces support
 * OpenVZ, SWsoft Inc.
 * Pavel Emelianov <xemul@openvz.org>
23 24 25
 *
 * Better ipc lock (kern_ipc_perm.lock) handling
 * Davidlohr Bueso <davidlohr.bueso@hp.com>, June 2013.
Linus Torvalds's avatar
Linus Torvalds committed
26 27 28 29 30 31 32 33 34 35 36 37 38
 */

#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/shm.h>
#include <linux/init.h>
#include <linux/file.h>
#include <linux/mman.h>
#include <linux/shmem_fs.h>
#include <linux/security.h>
#include <linux/syscalls.h>
#include <linux/audit.h>
39
#include <linux/capability.h>
40
#include <linux/ptrace.h>
41
#include <linux/seq_file.h>
Nadia Derbey's avatar
Nadia Derbey committed
42
#include <linux/rwsem.h>
43
#include <linux/nsproxy.h>
44
#include <linux/mount.h>
45
#include <linux/ipc_namespace.h>
46
#include <linux/rhashtable.h>
47

48
#include <linux/uaccess.h>
Linus Torvalds's avatar
Linus Torvalds committed
49 50 51

#include "util.h"

52 53 54 55 56 57 58 59 60
struct shmid_kernel /* private to the kernel */
{
	struct kern_ipc_perm	shm_perm;
	struct file		*shm_file;
	unsigned long		shm_nattch;
	unsigned long		shm_segsz;
	time64_t		shm_atim;
	time64_t		shm_dtim;
	time64_t		shm_ctim;
61 62
	struct pid		*shm_cprid;
	struct pid		*shm_lprid;
63 64 65 66 67 68 69 70 71 72 73
	struct user_struct	*mlock_user;

	/* The task created the shm object.  NULL if the task is dead. */
	struct task_struct	*shm_creator;
	struct list_head	shm_clist;	/* list by creator */
} __randomize_layout;

/* shm_mode upper byte flags */
#define SHM_DEST	01000	/* segment will be destroyed on last detach */
#define SHM_LOCKED	02000   /* segment will not be swapped */

74 75 76 77 78 79 80 81 82
struct shm_file_data {
	int id;
	struct ipc_namespace *ns;
	struct file *file;
	const struct vm_operations_struct *vm_ops;
};

#define shm_file_data(file) (*((struct shm_file_data **)&(file)->private_data))

83
static const struct file_operations shm_file_operations;
84
static const struct vm_operations_struct shm_vm_ops;
Linus Torvalds's avatar
Linus Torvalds committed
85

86
#define shm_ids(ns)	((ns)->ids[IPC_SHM_IDS])
Linus Torvalds's avatar
Linus Torvalds committed
87

88 89
#define shm_unlock(shp)			\
	ipc_unlock(&(shp)->shm_perm)
Linus Torvalds's avatar
Linus Torvalds committed
90

91
static int newseg(struct ipc_namespace *, struct ipc_params *);
92 93
static void shm_open(struct vm_area_struct *vma);
static void shm_close(struct vm_area_struct *vma);
Manfred Spraul's avatar
Manfred Spraul committed
94
static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp);
Linus Torvalds's avatar
Linus Torvalds committed
95
#ifdef CONFIG_PROC_FS
96
static int sysvipc_shm_proc_show(struct seq_file *s, void *it);
Linus Torvalds's avatar
Linus Torvalds committed
97 98
#endif

99
void shm_init_ns(struct ipc_namespace *ns)
100 101 102 103
{
	ns->shm_ctlmax = SHMMAX;
	ns->shm_ctlall = SHMALL;
	ns->shm_ctlmni = SHMMNI;
104
	ns->shm_rmid_forced = 0;
105
	ns->shm_tot = 0;
106
	ipc_init_ids(&shm_ids(ns));
107 108
}

Nadia Derbey's avatar
Nadia Derbey committed
109
/*
110 111
 * Called with shm_ids.rwsem (writer) and the shp structure locked.
 * Only shm_ids.rwsem remains locked on exit.
Nadia Derbey's avatar
Nadia Derbey committed
112
 */
113
static void do_shm_rmid(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
114
{
115
	struct shmid_kernel *shp;
116

117 118
	shp = container_of(ipcp, struct shmid_kernel, shm_perm);

Manfred Spraul's avatar
Manfred Spraul committed
119
	if (shp->shm_nattch) {
120 121
		shp->shm_perm.mode |= SHM_DEST;
		/* Do not find it any more */
122
		ipc_set_key_private(&shm_ids(ns), &shp->shm_perm);
123 124 125 126 127
		shm_unlock(shp);
	} else
		shm_destroy(ns, shp);
}

128
#ifdef CONFIG_IPC_NS
129 130
void shm_exit_ns(struct ipc_namespace *ns)
{
131
	free_ipcs(ns, &shm_ids(ns), do_shm_rmid);
132
	idr_destroy(&ns->ids[IPC_SHM_IDS].ipcs_idr);
133
	rhashtable_destroy(&ns->ids[IPC_SHM_IDS].key_ht);
134
}
135
#endif
Linus Torvalds's avatar
Linus Torvalds committed
136

137
static int __init ipc_ns_init(void)
Linus Torvalds's avatar
Linus Torvalds committed
138
{
139 140
	shm_init_ns(&init_ipc_ns);
	return 0;
141 142 143 144
}

pure_initcall(ipc_ns_init);

Manfred Spraul's avatar
Manfred Spraul committed
145
void __init shm_init(void)
146
{
147
	ipc_init_proc_interface("sysvipc/shm",
148 149 150 151 152
#if BITS_PER_LONG <= 32
				"       key      shmid perms       size  cpid  lpid nattch   uid   gid  cuid  cgid      atime      dtime      ctime        rss       swap\n",
#else
				"       key      shmid perms                  size  cpid  lpid nattch   uid   gid  cuid  cgid      atime      dtime      ctime                   rss                  swap\n",
#endif
153
				IPC_SHM_IDS, sysvipc_shm_proc_show);
Linus Torvalds's avatar
Linus Torvalds committed
154 155
}

156 157
static inline struct shmid_kernel *shm_obtain_object(struct ipc_namespace *ns, int id)
{
158
	struct kern_ipc_perm *ipcp = ipc_obtain_object_idr(&shm_ids(ns), id);
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175

	if (IS_ERR(ipcp))
		return ERR_CAST(ipcp);

	return container_of(ipcp, struct shmid_kernel, shm_perm);
}

static inline struct shmid_kernel *shm_obtain_object_check(struct ipc_namespace *ns, int id)
{
	struct kern_ipc_perm *ipcp = ipc_obtain_object_check(&shm_ids(ns), id);

	if (IS_ERR(ipcp))
		return ERR_CAST(ipcp);

	return container_of(ipcp, struct shmid_kernel, shm_perm);
}

Nadia Derbey's avatar
Nadia Derbey committed
176
/*
177
 * shm_lock_(check_) routines are called in the paths where the rwsem
178
 * is not necessarily held.
Nadia Derbey's avatar
Nadia Derbey committed
179
 */
180
static inline struct shmid_kernel *shm_lock(struct ipc_namespace *ns, int id)
Linus Torvalds's avatar
Linus Torvalds committed
181
{
Davidlohr Bueso's avatar
Davidlohr Bueso committed
182
	struct kern_ipc_perm *ipcp;
Nadia Derbey's avatar
Nadia Derbey committed
183

Davidlohr Bueso's avatar
Davidlohr Bueso committed
184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201
	rcu_read_lock();
	ipcp = ipc_obtain_object_idr(&shm_ids(ns), id);
	if (IS_ERR(ipcp))
		goto err;

	ipc_lock_object(ipcp);
	/*
	 * ipc_rmid() may have already freed the ID while ipc_lock_object()
	 * was spinning: here verify that the structure is still valid.
	 * Upon races with RMID, return -EIDRM, thus indicating that
	 * the ID points to a removed identifier.
	 */
	if (ipc_valid_object(ipcp)) {
		/* return a locked ipc object upon success */
		return container_of(ipcp, struct shmid_kernel, shm_perm);
	}

	ipc_unlock_object(ipcp);
202
	ipcp = ERR_PTR(-EIDRM);
Davidlohr Bueso's avatar
Davidlohr Bueso committed
203 204
err:
	rcu_read_unlock();
205
	/*
206
	 * Callers of shm_lock() must validate the status of the returned ipc
Davidlohr Bueso's avatar
Davidlohr Bueso committed
207
	 * object pointer and error out as appropriate.
208
	 */
209
	return ERR_CAST(ipcp);
210 211
}

212 213 214
static inline void shm_lock_by_ptr(struct shmid_kernel *ipcp)
{
	rcu_read_lock();
215
	ipc_lock_object(&ipcp->shm_perm);
216 217
}

218 219
static void shm_rcu_free(struct rcu_head *head)
{
220 221 222 223
	struct kern_ipc_perm *ptr = container_of(head, struct kern_ipc_perm,
							rcu);
	struct shmid_kernel *shp = container_of(ptr, struct shmid_kernel,
							shm_perm);
224
	security_shm_free(&shp->shm_perm);
225
	kvfree(shp);
226 227
}

Nadia Derbey's avatar
Nadia Derbey committed
228
static inline void shm_rmid(struct ipc_namespace *ns, struct shmid_kernel *s)
Linus Torvalds's avatar
Linus Torvalds committed
229
{
230
	list_del(&s->shm_clist);
Nadia Derbey's avatar
Nadia Derbey committed
231
	ipc_rmid(&shm_ids(ns), &s->shm_perm);
Linus Torvalds's avatar
Linus Torvalds committed
232 233 234
}


235
static int __shm_open(struct vm_area_struct *vma)
236
{
237 238
	struct file *file = vma->vm_file;
	struct shm_file_data *sfd = shm_file_data(file);
Linus Torvalds's avatar
Linus Torvalds committed
239 240
	struct shmid_kernel *shp;

241
	shp = shm_lock(sfd->ns, sfd->id);
242 243 244 245

	if (IS_ERR(shp))
		return PTR_ERR(shp);

246 247 248 249 250 251
	if (shp->shm_file != sfd->file) {
		/* ID was reused */
		shm_unlock(shp);
		return -EINVAL;
	}

252
	shp->shm_atim = ktime_get_real_seconds();
253
	ipc_update_pid(&shp->shm_lprid, task_tgid(current));
Linus Torvalds's avatar
Linus Torvalds committed
254 255
	shp->shm_nattch++;
	shm_unlock(shp);
256 257 258 259 260 261 262 263 264 265 266 267
	return 0;
}

/* This is called by fork, once for every shm attach. */
static void shm_open(struct vm_area_struct *vma)
{
	int err = __shm_open(vma);
	/*
	 * We raced in the idr lookup or with shm_destroy().
	 * Either way, the ID is busted.
	 */
	WARN_ON_ONCE(err);
Linus Torvalds's avatar
Linus Torvalds committed
268 269 270 271 272
}

/*
 * shm_destroy - free the struct shmid_kernel
 *
Nadia Derbey's avatar
Nadia Derbey committed
273
 * @ns: namespace
Linus Torvalds's avatar
Linus Torvalds committed
274 275
 * @shp: struct to free
 *
276
 * It has to be called with shp and shm_ids.rwsem (writer) locked,
Linus Torvalds's avatar
Linus Torvalds committed
277 278
 * but returns with shp unlocked and freed.
 */
279
static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
Linus Torvalds's avatar
Linus Torvalds committed
280
{
281 282 283 284
	struct file *shm_file;

	shm_file = shp->shm_file;
	shp->shm_file = NULL;
285
	ns->shm_tot -= (shp->shm_segsz + PAGE_SIZE - 1) >> PAGE_SHIFT;
Nadia Derbey's avatar
Nadia Derbey committed
286
	shm_rmid(ns, shp);
Linus Torvalds's avatar
Linus Torvalds committed
287
	shm_unlock(shp);
288 289
	if (!is_file_hugepages(shm_file))
		shmem_lock(shm_file, 0, shp->mlock_user);
290
	else if (shp->mlock_user)
291 292
		user_shm_unlock(i_size_read(file_inode(shm_file)),
				shp->mlock_user);
293
	fput(shm_file);
294 295
	ipc_update_pid(&shp->shm_cprid, NULL);
	ipc_update_pid(&shp->shm_lprid, NULL);
296
	ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);
Linus Torvalds's avatar
Linus Torvalds committed
297 298
}

299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315
/*
 * shm_may_destroy - identifies whether shm segment should be destroyed now
 *
 * Returns true if and only if there are no active users of the segment and
 * one of the following is true:
 *
 * 1) shmctl(id, IPC_RMID, NULL) was called for this shp
 *
 * 2) sysctl kernel.shm_rmid_forced is set to 1.
 */
static bool shm_may_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
{
	return (shp->shm_nattch == 0) &&
	       (ns->shm_rmid_forced ||
		(shp->shm_perm.mode & SHM_DEST));
}

Linus Torvalds's avatar
Linus Torvalds committed
316
/*
317
 * remove the attach descriptor vma.
Linus Torvalds's avatar
Linus Torvalds committed
318 319 320 321
 * free memory for segment if it is marked destroyed.
 * The descriptor has already been removed from the current->mm->mmap list
 * and will later be kfree()d.
 */
322
static void shm_close(struct vm_area_struct *vma)
Linus Torvalds's avatar
Linus Torvalds committed
323
{
Manfred Spraul's avatar
Manfred Spraul committed
324
	struct file *file = vma->vm_file;
325
	struct shm_file_data *sfd = shm_file_data(file);
Linus Torvalds's avatar
Linus Torvalds committed
326
	struct shmid_kernel *shp;
327
	struct ipc_namespace *ns = sfd->ns;
328

329
	down_write(&shm_ids(ns).rwsem);
Linus Torvalds's avatar
Linus Torvalds committed
330
	/* remove from the list of attaches of the shm segment */
331
	shp = shm_lock(ns, sfd->id);
332 333 334 335 336 337 338 339

	/*
	 * We raced in the idr lookup or with shm_destroy().
	 * Either way, the ID is busted.
	 */
	if (WARN_ON_ONCE(IS_ERR(shp)))
		goto done; /* no-op */

340
	ipc_update_pid(&shp->shm_lprid, task_tgid(current));
341
	shp->shm_dtim = ktime_get_real_seconds();
Linus Torvalds's avatar
Linus Torvalds committed
342
	shp->shm_nattch--;
343 344 345 346
	if (shm_may_destroy(ns, shp))
		shm_destroy(ns, shp);
	else
		shm_unlock(shp);
347
done:
348
	up_write(&shm_ids(ns).rwsem);
349 350
}

351
/* Called with ns->shm_ids(ns).rwsem locked */
352 353 354
static int shm_try_destroy_orphaned(int id, void *p, void *data)
{
	struct ipc_namespace *ns = data;
355 356
	struct kern_ipc_perm *ipcp = p;
	struct shmid_kernel *shp = container_of(ipcp, struct shmid_kernel, shm_perm);
357 358 359 360

	/*
	 * We want to destroy segments without users and with already
	 * exit'ed originating process.
361
	 *
362
	 * As shp->* are changed under rwsem, it's safe to skip shp locking.
363
	 */
364
	if (shp->shm_creator != NULL)
365 366
		return 0;

367 368
	if (shm_may_destroy(ns, shp)) {
		shm_lock_by_ptr(shp);
369
		shm_destroy(ns, shp);
370
	}
371 372 373 374 375
	return 0;
}

void shm_destroy_orphaned(struct ipc_namespace *ns)
{
376
	down_write(&shm_ids(ns).rwsem);
Vasiliy Kulikov's avatar
Vasiliy Kulikov committed
377
	if (shm_ids(ns).in_use)
378
		idr_for_each(&shm_ids(ns).ipcs_idr, &shm_try_destroy_orphaned, ns);
379
	up_write(&shm_ids(ns).rwsem);
380 381
}

382
/* Locking assumes this will only be called with task == current */
383 384
void exit_shm(struct task_struct *task)
{
385
	struct ipc_namespace *ns = task->nsproxy->ipc_ns;
386
	struct shmid_kernel *shp, *n;
387

388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405
	if (list_empty(&task->sysvshm.shm_clist))
		return;

	/*
	 * If kernel.shm_rmid_forced is not set then only keep track of
	 * which shmids are orphaned, so that a later set of the sysctl
	 * can clean them up.
	 */
	if (!ns->shm_rmid_forced) {
		down_read(&shm_ids(ns).rwsem);
		list_for_each_entry(shp, &task->sysvshm.shm_clist, shm_clist)
			shp->shm_creator = NULL;
		/*
		 * Only under read lock but we are only called on current
		 * so no entry on the list will be shared.
		 */
		list_del(&task->sysvshm.shm_clist);
		up_read(&shm_ids(ns).rwsem);
406
		return;
407
	}
408

409 410 411 412 413
	/*
	 * Destroy all already created segments, that were not yet mapped,
	 * and mark any mapped as orphan to cover the sysctl toggling.
	 * Destroy is skipped if shm_may_destroy() returns false.
	 */
414
	down_write(&shm_ids(ns).rwsem);
415 416 417 418 419 420 421 422 423 424
	list_for_each_entry_safe(shp, n, &task->sysvshm.shm_clist, shm_clist) {
		shp->shm_creator = NULL;

		if (shm_may_destroy(ns, shp)) {
			shm_lock_by_ptr(shp);
			shm_destroy(ns, shp);
		}
	}

	/* Remove the list head from any segments still attached. */
425
	list_del(&task->sysvshm.shm_clist);
426
	up_write(&shm_ids(ns).rwsem);
Linus Torvalds's avatar
Linus Torvalds committed
427 428
}

429
static vm_fault_t shm_fault(struct vm_fault *vmf)
430
{
431
	struct file *file = vmf->vma->vm_file;
432 433
	struct shm_file_data *sfd = shm_file_data(file);

434
	return sfd->vm_ops->fault(vmf);
435 436
}

437 438 439 440 441
static int shm_split(struct vm_area_struct *vma, unsigned long addr)
{
	struct file *file = vma->vm_file;
	struct shm_file_data *sfd = shm_file_data(file);

442
	if (sfd->vm_ops->split)
443 444 445 446 447
		return sfd->vm_ops->split(vma, addr);

	return 0;
}

448 449 450 451 452 453 454 455 456 457 458
static unsigned long shm_pagesize(struct vm_area_struct *vma)
{
	struct file *file = vma->vm_file;
	struct shm_file_data *sfd = shm_file_data(file);

	if (sfd->vm_ops->pagesize)
		return sfd->vm_ops->pagesize(vma);

	return PAGE_SIZE;
}

459
#ifdef CONFIG_NUMA
460
static int shm_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
461 462 463 464
{
	struct file *file = vma->vm_file;
	struct shm_file_data *sfd = shm_file_data(file);
	int err = 0;
465

466 467 468 469 470
	if (sfd->vm_ops->set_policy)
		err = sfd->vm_ops->set_policy(vma, new);
	return err;
}

471 472
static struct mempolicy *shm_get_policy(struct vm_area_struct *vma,
					unsigned long addr)
473 474 475 476 477 478 479
{
	struct file *file = vma->vm_file;
	struct shm_file_data *sfd = shm_file_data(file);
	struct mempolicy *pol = NULL;

	if (sfd->vm_ops->get_policy)
		pol = sfd->vm_ops->get_policy(vma, addr);
480
	else if (vma->vm_policy)
481
		pol = vma->vm_policy;
482

483 484 485 486
	return pol;
}
#endif

Manfred Spraul's avatar
Manfred Spraul committed
487
static int shm_mmap(struct file *file, struct vm_area_struct *vma)
Linus Torvalds's avatar
Linus Torvalds committed
488
{
489
	struct shm_file_data *sfd = shm_file_data(file);
490 491
	int ret;

492
	/*
493 494 495
	 * In case of remap_file_pages() emulation, the file can represent an
	 * IPC ID that was removed, and possibly even reused by another shm
	 * segment already.  Propagate this case as an error to caller.
496
	 */
497
	ret = __shm_open(vma);
498 499 500
	if (ret)
		return ret;

501
	ret = call_mmap(sfd->file, vma);
502 503
	if (ret) {
		shm_close(vma);
504
		return ret;
505
	}
506
	sfd->vm_ops = vma->vm_ops;
David Howells's avatar
David Howells committed
507
#ifdef CONFIG_MMU
508
	WARN_ON(!sfd->vm_ops->fault);
David Howells's avatar
David Howells committed
509
#endif
510
	vma->vm_ops = &shm_vm_ops;
511
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
512 513
}

514 515
static int shm_release(struct inode *ino, struct file *file)
{
516
	struct shm_file_data *sfd = shm_file_data(file);
517

518
	put_ipc_ns(sfd->ns);
519
	fput(sfd->file);
520 521
	shm_file_data(file) = NULL;
	kfree(sfd);
522 523 524
	return 0;
}

525
static int shm_fsync(struct file *file, loff_t start, loff_t end, int datasync)
526 527 528
{
	struct shm_file_data *sfd = shm_file_data(file);

529 530
	if (!sfd->file->f_op->fsync)
		return -EINVAL;
531
	return sfd->file->f_op->fsync(sfd->file, start, end, datasync);
532 533
}

534 535 536 537 538 539 540 541 542 543
static long shm_fallocate(struct file *file, int mode, loff_t offset,
			  loff_t len)
{
	struct shm_file_data *sfd = shm_file_data(file);

	if (!sfd->file->f_op->fallocate)
		return -EOPNOTSUPP;
	return sfd->file->f_op->fallocate(file, mode, offset, len);
}

544 545 546 547 548
static unsigned long shm_get_unmapped_area(struct file *file,
	unsigned long addr, unsigned long len, unsigned long pgoff,
	unsigned long flags)
{
	struct shm_file_data *sfd = shm_file_data(file);
549

550 551
	return sfd->file->f_op->get_unmapped_area(sfd->file, addr, len,
						pgoff, flags);
552 553
}

554
static const struct file_operations shm_file_operations = {
555
	.mmap		= shm_mmap,
556
	.fsync		= shm_fsync,
557
	.release	= shm_release,
558
	.get_unmapped_area	= shm_get_unmapped_area,
559
	.llseek		= noop_llseek,
560
	.fallocate	= shm_fallocate,
561 562
};

563 564 565 566
/*
 * shm_file_operations_huge is now identical to shm_file_operations,
 * but we keep it distinct for the sake of is_file_shm_hugepages().
 */
567 568 569 570
static const struct file_operations shm_file_operations_huge = {
	.mmap		= shm_mmap,
	.fsync		= shm_fsync,
	.release	= shm_release,
571
	.get_unmapped_area	= shm_get_unmapped_area,
572
	.llseek		= noop_llseek,
573
	.fallocate	= shm_fallocate,
Linus Torvalds's avatar
Linus Torvalds committed
574 575
};

576
bool is_file_shm_hugepages(struct file *file)
577 578 579 580
{
	return file->f_op == &shm_file_operations_huge;
}

581
static const struct vm_operations_struct shm_vm_ops = {
Linus Torvalds's avatar
Linus Torvalds committed
582 583
	.open	= shm_open,	/* callback for a new vm-area open */
	.close	= shm_close,	/* callback for when the vm-area is released */
584
	.fault	= shm_fault,
585
	.split	= shm_split,
586
	.pagesize = shm_pagesize,
587 588 589
#if defined(CONFIG_NUMA)
	.set_policy = shm_set_policy,
	.get_policy = shm_get_policy,
Linus Torvalds's avatar
Linus Torvalds committed
590 591 592
#endif
};

Nadia Derbey's avatar
Nadia Derbey committed
593 594 595 596 597
/**
 * newseg - Create a new shared memory segment
 * @ns: namespace
 * @params: ptr to the structure that contains key, size and shmflg
 *
598
 * Called with shm_ids.rwsem held as a writer.
Nadia Derbey's avatar
Nadia Derbey committed
599
 */
600
static int newseg(struct ipc_namespace *ns, struct ipc_params *params)
Linus Torvalds's avatar
Linus Torvalds committed
601
{
602 603 604
	key_t key = params->key;
	int shmflg = params->flg;
	size_t size = params->u.size;
Linus Torvalds's avatar
Linus Torvalds committed
605 606
	int error;
	struct shmid_kernel *shp;
607
	size_t numpages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
Manfred Spraul's avatar
Manfred Spraul committed
608
	struct file *file;
Linus Torvalds's avatar
Linus Torvalds committed
609
	char name[13];
610
	vm_flags_t acctflag = 0;
Linus Torvalds's avatar
Linus Torvalds committed
611

612
	if (size < SHMMIN || size > ns->shm_ctlmax)
Linus Torvalds's avatar
Linus Torvalds committed
613 614
		return -EINVAL;

615 616 617
	if (numpages << PAGE_SHIFT < size)
		return -ENOSPC;

618 619
	if (ns->shm_tot + numpages < ns->shm_tot ||
			ns->shm_tot + numpages > ns->shm_ctlall)
Linus Torvalds's avatar
Linus Torvalds committed
620 621
		return -ENOSPC;

622 623
	shp = kvmalloc(sizeof(*shp), GFP_KERNEL);
	if (unlikely(!shp))
Linus Torvalds's avatar
Linus Torvalds committed
624 625 626
		return -ENOMEM;

	shp->shm_perm.key = key;
627
	shp->shm_perm.mode = (shmflg & S_IRWXUGO);
Linus Torvalds's avatar
Linus Torvalds committed
628 629 630
	shp->mlock_user = NULL;

	shp->shm_perm.security = NULL;
631
	error = security_shm_alloc(&shp->shm_perm);
Linus Torvalds's avatar
Linus Torvalds committed
632
	if (error) {
633
		kvfree(shp);
Linus Torvalds's avatar
Linus Torvalds committed
634 635 636
		return error;
	}

Manfred Spraul's avatar
Manfred Spraul committed
637
	sprintf(name, "SYSV%08x", key);
Linus Torvalds's avatar
Linus Torvalds committed
638
	if (shmflg & SHM_HUGETLB) {
639
		struct hstate *hs;
640 641
		size_t hugesize;

642
		hs = hstate_sizelog((shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
643 644 645 646 647
		if (!hs) {
			error = -EINVAL;
			goto no_file;
		}
		hugesize = ALIGN(size, huge_page_size(hs));
648

649 650 651
		/* hugetlb_file_setup applies strict accounting */
		if (shmflg & SHM_NORESERVE)
			acctflag = VM_NORESERVE;
652
		file = hugetlb_file_setup(name, hugesize, acctflag,
653 654
				  &shp->mlock_user, HUGETLB_SHMFS_INODE,
				(shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
Linus Torvalds's avatar
Linus Torvalds committed
655
	} else {
656 657
		/*
		 * Do not allow no accounting for OVERCOMMIT_NEVER, even
Manfred Spraul's avatar
Manfred Spraul committed
658
		 * if it's asked for.
659 660 661
		 */
		if  ((shmflg & SHM_NORESERVE) &&
				sysctl_overcommit_memory != OVERCOMMIT_NEVER)
662
			acctflag = VM_NORESERVE;
663
		file = shmem_kernel_file_setup(name, size, acctflag);
Linus Torvalds's avatar
Linus Torvalds committed
664 665 666 667 668
	}
	error = PTR_ERR(file);
	if (IS_ERR(file))
		goto no_file;

669 670
	shp->shm_cprid = get_pid(task_tgid(current));
	shp->shm_lprid = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
671
	shp->shm_atim = shp->shm_dtim = 0;
672
	shp->shm_ctim = ktime_get_real_seconds();
Linus Torvalds's avatar
Linus Torvalds committed
673 674 675
	shp->shm_segsz = size;
	shp->shm_nattch = 0;
	shp->shm_file = file;
676
	shp->shm_creator = current;
677

678
	/* ipc_addid() locks shp upon success. */
679 680
	error = ipc_addid(&shm_ids(ns), &shp->shm_perm, ns->shm_ctlmni);
	if (error < 0)
681 682
		goto no_id;

683
	list_add(&shp->shm_clist, &current->sysvshm.shm_clist);
684

685 686 687 688
	/*
	 * shmid gets reported as "inode#" in /proc/pid/maps.
	 * proc-ps tools use this. Changing this will break them.
	 */
Al Viro's avatar
Al Viro committed
689
	file_inode(file)->i_ino = shp->shm_perm.id;
690

691
	ns->shm_tot += numpages;
Nadia Derbey's avatar
Nadia Derbey committed
692
	error = shp->shm_perm.id;
693

694
	ipc_unlock_object(&shp->shm_perm);
695
	rcu_read_unlock();
Nadia Derbey's avatar
Nadia Derbey committed
696
	return error;
Linus Torvalds's avatar
Linus Torvalds committed
697 698

no_id:
699 700
	ipc_update_pid(&shp->shm_cprid, NULL);
	ipc_update_pid(&shp->shm_lprid, NULL);
701
	if (is_file_hugepages(file) && shp->mlock_user)
702
		user_shm_unlock(size, shp->mlock_user);
Linus Torvalds's avatar
Linus Torvalds committed
703
	fput(file);
704 705
	ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);
	return error;
Linus Torvalds's avatar
Linus Torvalds committed
706
no_file:
707
	call_rcu(&shp->shm_perm.rcu, shm_rcu_free);
Linus Torvalds's avatar
Linus Torvalds committed
708 709 710
	return error;
}

Nadia Derbey's avatar
Nadia Derbey committed
711
/*
712
 * Called with shm_ids.rwsem and ipcp locked.
Nadia Derbey's avatar
Nadia Derbey committed
713
 */
Nadia Derbey's avatar
Nadia Derbey committed
714 715
static inline int shm_more_checks(struct kern_ipc_perm *ipcp,
				struct ipc_params *params)
716
{
Nadia Derbey's avatar
Nadia Derbey committed
717 718 719 720
	struct shmid_kernel *shp;

	shp = container_of(ipcp, struct shmid_kernel, shm_perm);
	if (shp->shm_segsz < params->u.size)
721 722 723 724 725
		return -EINVAL;

	return 0;
}

726
long ksys_shmget(key_t key, size_t size, int shmflg)
Linus Torvalds's avatar
Linus Torvalds committed
727
{
728
	struct ipc_namespace *ns;
Mathias Krause's avatar
Mathias Krause committed
729 730
	static const struct ipc_ops shm_ops = {
		.getnew = newseg,
731
		.associate = security_shm_associate,
Mathias Krause's avatar
Mathias Krause committed
732 733
		.more_checks = shm_more_checks,
	};
734
	struct ipc_params shm_params;
735 736

	ns = current->nsproxy->ipc_ns;
Linus Torvalds's avatar
Linus Torvalds committed
737

738 739 740
	shm_params.key = key;
	shm_params.flg = shmflg;
	shm_params.u.size = size;
Linus Torvalds's avatar
Linus Torvalds committed
741

742
	return ipcget(ns, &shm_ids(ns), &shm_ops, &shm_params);
Linus Torvalds's avatar
Linus Torvalds committed
743 744
}

745 746 747 748 749
SYSCALL_DEFINE3(shmget, key_t, key, size_t, size, int, shmflg)
{
	return ksys_shmget(key, size, shmflg);
}

Linus Torvalds's avatar
Linus Torvalds committed
750 751
static inline unsigned long copy_shmid_to_user(void __user *buf, struct shmid64_ds *in, int version)
{
Manfred Spraul's avatar
Manfred Spraul committed
752
	switch (version) {
Linus Torvalds's avatar
Linus Torvalds committed
753 754 755 756 757 758
	case IPC_64:
		return copy_to_user(buf, in, sizeof(*in));
	case IPC_OLD:
	    {
		struct shmid_ds out;

759
		memset(&out, 0, sizeof(out));
Linus Torvalds's avatar
Linus Torvalds committed
760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775
		ipc64_perm_to_ipc_perm(&in->shm_perm, &out.shm_perm);
		out.shm_segsz	= in->shm_segsz;
		out.shm_atime	= in->shm_atime;
		out.shm_dtime	= in->shm_dtime;
		out.shm_ctime	= in->shm_ctime;
		out.shm_cpid	= in->shm_cpid;
		out.shm_lpid	= in->shm_lpid;
		out.shm_nattch	= in->shm_nattch;

		return copy_to_user(buf, &out, sizeof(out));
	    }
	default:
		return -EINVAL;
	}
}

776 777
static inline unsigned long
copy_shmid_from_user(struct shmid64_ds *out, void __user *buf, int version)
Linus Torvalds's avatar
Linus Torvalds committed
778
{
Manfred Spraul's avatar
Manfred Spraul committed
779
	switch (version) {
Linus Torvalds's avatar
Linus Torvalds committed
780
	case IPC_64:
781
		if (copy_from_user(out, buf, sizeof(*out)))
Linus Torvalds's avatar
Linus Torvalds committed
782 783 784 785 786 787 788 789 790
			return -EFAULT;
		return 0;
	case IPC_OLD:
	    {
		struct shmid_ds tbuf_old;

		if (copy_from_user(&tbuf_old, buf, sizeof(tbuf_old)))
			return -EFAULT;

791 792 793
		out->shm_perm.uid	= tbuf_old.shm_perm.uid;
		out->shm_perm.gid	= tbuf_old.shm_perm.gid;
		out->shm_perm.mode	= tbuf_old.shm_perm.mode;
Linus Torvalds's avatar
Linus Torvalds committed
794 795 796 797 798 799 800 801 802 803

		return 0;
	    }
	default:
		return -EINVAL;
	}
}

static inline unsigned long copy_shminfo_to_user(void __user *buf, struct shminfo64 *in, int version)
{
Manfred Spraul's avatar
Manfred Spraul committed
804
	switch (version) {
Linus Torvalds's avatar
Linus Torvalds committed
805 806 807 808 809 810
	case IPC_64:
		return copy_to_user(buf, in, sizeof(*in));
	case IPC_OLD:
	    {
		struct shminfo out;

Manfred Spraul's avatar
Manfred Spraul committed
811
		if (in->shmmax > INT_MAX)
Linus Torvalds's avatar
Linus Torvalds committed
812 813 814 815 816 817 818
			out.shmmax = INT_MAX;
		else
			out.shmmax = (int)in->shmmax;

		out.shmmin	= in->shmmin;
		out.shmmni	= in->shmmni;
		out.shmseg	= in->shmseg;
819
		out.shmall	= in->shmall;
Linus Torvalds's avatar
Linus Torvalds committed
820 821 822 823 824 825 826 827

		return copy_to_user(buf, &out, sizeof(out));
	    }
	default:
		return -EINVAL;
	}
}

828 829
/*
 * Calculate and add used RSS and swap pages of a shm.
830
 * Called with shm_ids.rwsem held as a reader
831 832 833 834 835 836
 */
static void shm_add_rss_swap(struct shmid_kernel *shp,
	unsigned long *rss_add, unsigned long *swp_add)
{
	struct inode *inode;

Al Viro's avatar
Al Viro committed
837
	inode = file_inode(shp->shm_file);
838 839 840 841 842 843 844 845

	if (is_file_hugepages(shp->shm_file)) {
		struct address_space *mapping = inode->i_mapping;
		struct hstate *h = hstate_file(shp->shm_file);
		*rss_add += pages_per_huge_page(h) * mapping->nrpages;
	} else {
#ifdef CONFIG_SHMEM
		struct shmem_inode_info *info = SHMEM_I(inode);
846

847
		spin_lock_irq(&info->lock);
848 849
		*rss_add += inode->i_mapping->nrpages;
		*swp_add += info->swapped;
850
		spin_unlock_irq(&info->lock);
851 852 853 854 855 856
#else
		*rss_add += inode->i_mapping->nrpages;
#endif
	}
}

Nadia Derbey's avatar
Nadia Derbey committed
857
/*
858
 * Called with shm_ids.rwsem held as a reader
Nadia Derbey's avatar
Nadia Derbey committed
859
 */
860 861
static void shm_get_stat(struct ipc_namespace *ns, unsigned long *rss,
		unsigned long *swp)
Linus Torvalds's avatar
Linus Torvalds committed
862
{
Nadia Derbey's avatar
Nadia Derbey committed
863 864
	int next_id;
	int total, in_use;
Linus Torvalds's avatar
Linus Torvalds committed
865 866 867 868

	*rss = 0;
	*swp = 0;

Nadia Derbey's avatar
Nadia Derbey committed
869 870 871
	in_use = shm_ids(ns).in_use;

	for (total = 0, next_id = 0; total < in_use; next_id++) {
872
		struct kern_ipc_perm *ipc;
Linus Torvalds's avatar
Linus Torvalds committed
873 874
		struct shmid_kernel *shp;

875 876
		ipc = idr_find(&shm_ids(ns).ipcs_idr, next_id);
		if (ipc == NULL)
Linus Torvalds's avatar
Linus Torvalds committed
877
			continue;
878
		shp = container_of(ipc, struct shmid_kernel, shm_perm);
Linus Torvalds's avatar
Linus Torvalds committed
879

880
		shm_add_rss_swap(shp, rss, swp);
Nadia Derbey's avatar
Nadia Derbey committed
881 882

		total++;
Linus Torvalds's avatar
Linus Torvalds committed
883 884 885
	}
}

886
/*
887
 * This function handles some shmctl commands which require the rwsem
888
 * to be held in write mode.
889
 * NOTE: no locks must be held, the rwsem is taken inside this function.
890 891
 */
static int shmctl_down(struct ipc_namespace *ns, int shmid, int cmd,
892
		       struct shmid64_ds *shmid64)
Linus Torvalds's avatar
Linus Torvalds committed
893
{
894 895 896 897
	struct kern_ipc_perm *ipcp;
	struct shmid_kernel *shp;
	int err;

898
	down_write(&shm_ids(ns).rwsem);
899 900
	rcu_read_lock();

901
	ipcp = ipcctl_obtain_check(ns, &shm_ids(ns), shmid, cmd,
902
				      &shmid64->shm_perm, 0);
903 904 905 906
	if (IS_ERR(ipcp)) {
		err = PTR_ERR(ipcp);
		goto out_unlock1;
	}
907

908
	shp = container_of(ipcp, struct shmid_kernel, shm_perm);
909

910
	err = security_shm_shmctl(&shp->shm_perm, cmd);
911
	if (err)
912
		goto out_unlock1;
913

914 915
	switch (cmd) {
	case IPC_RMID:
916
		ipc_lock_object(&shp->shm_perm);
917
		/* do_shm_rmid unlocks the ipc object and rcu */
918 919 920
		do_shm_rmid(ns, ipcp);
		goto out_up;
	case IPC_SET:
921
		ipc_lock_object(&shp->shm_perm);
922
		err = ipc_update_perm(&shmid64->shm_perm, ipcp);
923
		if (err)
924
			goto out_unlock0;
925
		shp->shm_ctim = ktime_get_real_seconds();
926 927 928
		break;
	default:
		err = -EINVAL;
929
		goto out_unlock1;
930
	}
931 932 933 934 935

out_unlock0:
	ipc_unlock_object(&shp->shm_perm);
out_unlock1:
	rcu_read_unlock();
936
out_up:
937
	up_write(&shm_ids(ns).rwsem);
938 939 940
	return err;
}

941 942
static int shmctl_ipc_info(struct ipc_namespace *ns,
			   struct shminfo64 *shminfo)
943
{
944 945 946 947 948 949 950
	int err = security_shm_shmctl(NULL, IPC_INFO);
	if (!err) {
		memset(shminfo, 0, sizeof(*shminfo));
		shminfo->shmmni = shminfo->shmseg = ns->shm_ctlmni;
		shminfo->shmmax = ns->shm_ctlmax;
		shminfo->shmall = ns->shm_ctlall;
		shminfo->shmmin = SHMMIN;
951
		down_read(&shm_ids(ns).rwsem);
952
		err = ipc_get_maxidx(&shm_ids(ns));
953
		up_read(&shm_ids(ns).rwsem);
Manfred Spraul's avatar
Manfred Spraul committed
954
		if (err < 0)
Linus Torvalds's avatar
Linus Torvalds committed
955 956
			err = 0;
	}
957 958
	return err;
}
Linus Torvalds's avatar
Linus Torvalds committed
959

960 961 962 963 964 965
static int shmctl_shm_info(struct ipc_namespace *ns,
			   struct shm_info *shm_info)
{
	int err = security_shm_shmctl(NULL, SHM_INFO);
	if (!err) {
		memset(shm_info, 0, sizeof(*shm_info));
966
		down_read(&shm_ids(ns).rwsem);
967 968 969 970 971
		shm_info->used_ids = shm_ids(ns).in_use;
		shm_get_stat(ns, &shm_info->shm_rss, &shm_info->shm_swp);
		shm_info->shm_tot = ns->shm_tot;
		shm_info->swap_attempts = 0;
		shm_info->swap_successes = 0;
972
		err = ipc_get_maxidx(&shm_ids(ns));
973
		up_read(&shm_ids(ns).rwsem);
974 975
		if (err < 0)
			err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
976
	}
977 978
	return err;
}
979

980 981 982 983 984
static int shmctl_stat(struct ipc_namespace *ns, int shmid,
			int cmd, struct shmid64_ds *tbuf)
{
	struct shmid_kernel *shp;
	int err;
985

986 987
	memset(tbuf, 0, sizeof(*tbuf));

988
	rcu_read_lock();
989
	if (cmd == SHM_STAT || cmd == SHM_STAT_ANY) {
990 991 992 993 994
		shp = shm_obtain_object(ns, shmid);
		if (IS_ERR(shp)) {
			err = PTR_ERR(shp);
			goto out_unlock;
		}
995
	} else { /* IPC_STAT */
996 997 998
		shp = shm_obtain_object_check(ns, shmid);
		if (IS_ERR(shp)) {
			err = PTR_ERR(shp);
Linus Torvalds's avatar
Linus Torvalds committed
999 1000
			goto out_unlock;
		}
1001
	}
Linus Torvalds's avatar
Linus Torvalds committed
1002

1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016
	/*
	 * Semantically SHM_STAT_ANY ought to be identical to
	 * that functionality provided by the /proc/sysvipc/
	 * interface. As such, only audit these calls and
	 * do not do traditional S_IRUGO permission checks on
	 * the ipc object.
	 */
	if (cmd == SHM_STAT_ANY)
		audit_ipc_obj(&shp->shm_perm);
	else {
		err = -EACCES;
		if (ipcperms(ns, &shp->shm_perm, S_IRUGO))
			goto out_unlock;
	}
1017

1018
	err = security_shm_shmctl(&shp->shm_perm, cmd);
1019 1020 1021
	if (err)
		goto out_unlock;

1022 1023 1024 1025 1026 1027 1028 1029
	ipc_lock_object(&shp->shm_perm);

	if (!ipc_valid_object(&shp->shm_perm)) {
		ipc_unlock_object(&shp->shm_perm);
		err = -EIDRM;
		goto out_unlock;
	}

1030 1031 1032 1033 1034
	kernel_to_ipc64_perm(&shp->shm_perm, &tbuf->shm_perm);
	tbuf->shm_segsz	= shp->shm_segsz;
	tbuf->shm_atime	= shp->shm_atim;
	tbuf->shm_dtime	= shp->shm_dtim;
	tbuf->shm_ctime	= shp->shm_ctim;
1035 1036 1037 1038 1039
#ifndef CONFIG_64BIT
	tbuf->shm_atime_high = shp->shm_atim >> 32;
	tbuf->shm_dtime_high = shp->shm_dtim >> 32;
	tbuf->shm_ctime_high = shp->shm_ctim >> 32;
#endif
1040 1041
	tbuf->shm_cpid	= pid_vnr(shp->shm_cprid);
	tbuf->shm_lpid	= pid_vnr(shp->shm_lprid);
1042
	tbuf->shm_nattch = shp->shm_nattch;
1043

1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056
	if (cmd == IPC_STAT) {
		/*
		 * As defined in SUS:
		 * Return 0 on success
		 */
		err = 0;
	} else {
		/*
		 * SHM_STAT and SHM_STAT_ANY (both Linux specific)
		 * Return the full id, including the sequence number
		 */
		err = shp->shm_perm.id;
	}
1057

1058
	ipc_unlock_object(&shp->shm_perm);
1059
out_unlock:
1060
	rcu_read_unlock();
1061 1062 1063
	return err;
}

1064
static int shmctl_do_lock(struct ipc_namespace *ns, int shmid, int cmd)
1065 1066
{
	struct shmid_kernel *shp;
1067 1068
	struct file *shm_file;
	int err;
1069

1070 1071 1072 1073 1074
	rcu_read_lock();
	shp = shm_obtain_object_check(ns, shmid);
	if (IS_ERR(shp)) {
		err = PTR_ERR(shp);
		goto out_unlock1;
Linus Torvalds's avatar
Linus Torvalds committed
1075
	}
1076

1077
	audit_ipc_obj(&(shp->shm_perm));
1078
	err = security_shm_shmctl(&shp->shm_perm, cmd);
1079 1080
	if (err)
		goto out_unlock1;
1081

1082
	ipc_lock_object(&shp->shm_perm);
1083

1084 1085 1086 1087
	/* check if shm_destroy() is tearing down shp */
	if (!ipc_valid_object(&shp->shm_perm)) {
		err = -EIDRM;
		goto out_unlock0;
Linus Torvalds's avatar
Linus Torvalds committed
1088
	}
1089

1090 1091
	if (!ns_capable(ns->user_ns, CAP_IPC_LOCK)) {
		kuid_t euid = current_euid();
1092

1093 1094 1095
		if (!uid_eq(euid, shp->shm_perm.uid) &&
		    !uid_eq(euid, shp->shm_perm.cuid)) {
			err = -EPERM;
1096 1097
			goto out_unlock0;
		}
1098 1099 1100
		if (cmd == SHM_LOCK && !rlimit(RLIMIT_MEMLOCK)) {
			err = -EPERM;
			goto out_unlock0;
Linus Torvalds's avatar
Linus Torvalds committed
1101
		}
1102 1103
	}

1104 1105 1106
	shm_file = shp->shm_file;
	if (is_file_hugepages(shm_file))
		goto out_unlock0;
1107

1108 1109
	if (cmd == SHM_LOCK) {
		struct user_struct *user = current_user();
1110

1111 1112 1113 1114
		err = shmem_lock(shm_file, 1, user);
		if (!err && !(shp->shm_perm.mode & SHM_LOCKED)) {
			shp->shm_perm.mode |= SHM_LOCKED;
			shp->mlock_user = user;
Linus Torvalds's avatar
Linus Torvalds committed
1115
		}
1116
		goto out_unlock0;
Linus Torvalds's avatar
Linus Torvalds committed
1117 1118
	}

1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132
	/* SHM_UNLOCK */
	if (!(shp->shm_perm.mode & SHM_LOCKED))
		goto out_unlock0;
	shmem_lock(shm_file, 0, shp->mlock_user);
	shp->shm_perm.mode &= ~SHM_LOCKED;
	shp->mlock_user = NULL;
	get_file(shm_file);
	ipc_unlock_object(&shp->shm_perm);
	rcu_read_unlock();
	shmem_unlock_mapping(shm_file->f_mapping);

	fput(shm_file);
	return err;

1133 1134 1135
out_unlock0:
	ipc_unlock_object(&shp->shm_perm);
out_unlock1:
1136
	rcu_read_unlock();
1137 1138 1139
	return err;
}

1140
long ksys_shmctl(int shmid, int cmd, struct shmid_ds __user *buf)
1141 1142 1143
{
	int err, version;
	struct ipc_namespace *ns;
1144
	struct shmid64_ds sem64;
1145

1146 1147
	if (cmd < 0 || shmid < 0)
		return -EINVAL;
1148 1149 1150 1151 1152

	version = ipc_parse_version(&cmd);
	ns = current->nsproxy->ipc_ns;

	switch (cmd) {
1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170
	case IPC_INFO: {
		struct shminfo64 shminfo;
		err = shmctl_ipc_info(ns, &shminfo);
		if (err < 0)
			return err;
		if (copy_shminfo_to_user(buf, &shminfo, version))
			err = -EFAULT;
		return err;
	}
	case SHM_INFO: {
		struct shm_info shm_info;
		err = shmctl_shm_info(ns, &shm_info);
		if (err < 0)
			return err;
		if (copy_to_user(buf, &shm_info, sizeof(shm_info)))
			err = -EFAULT;
		return err;
	}
1171
	case SHM_STAT:
1172
	case SHM_STAT_ANY:
1173
	case IPC_STAT: {
1174
		err = shmctl_stat(ns, shmid, cmd, &sem64);
1175 1176
		if (err < 0)
			return err;
1177
		if (copy_shmid_to_user(buf, &sem64, version))
1178 1179 1180
			err = -EFAULT;
		return err;
	}
1181
	case IPC_SET:
1182
		if (copy_shmid_from_user(&sem64, buf, version))
1183
			return -EFAULT;
1184
		/* fallthru */
1185
	case IPC_RMID:
1186
		return shmctl_down(ns, shmid, cmd, &sem64);
Linus Torvalds's avatar
Linus Torvalds committed
1187 1188
	case SHM_LOCK:
	case SHM_UNLOCK:
1189 1190 1191 1192 1193
		return shmctl_do_lock(ns, shmid, cmd);
	default:
		return -EINVAL;
	}
}
1194

1195 1196 1197 1198 1199
SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
{
	return ksys_shmctl(shmid, cmd, buf);
}

1200 1201 1202 1203 1204
#ifdef CONFIG_COMPAT

struct compat_shmid_ds {
	struct compat_ipc_perm shm_perm;
	int shm_segsz;
1205 1206 1207
	old_time32_t shm_atime;
	old_time32_t shm_dtime;
	old_time32_t shm_ctime;
1208 1209 1210 1211 1212