eventpoll.c 61.8 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
/*
Davide Libenzi's avatar
Davide Libenzi committed
2 3
 *  fs/eventpoll.c (Efficient event retrieval implementation)
 *  Copyright (C) 2001,...,2009	 Davide Libenzi
Linus Torvalds's avatar
Linus Torvalds committed
4 5 6 7 8 9 10 11 12 13 14 15
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  Davide Libenzi <davidel@xmailserver.org>
 *
 */

#include <linux/init.h>
#include <linux/kernel.h>
16
#include <linux/sched/signal.h>
Linus Torvalds's avatar
Linus Torvalds committed
17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/signal.h>
#include <linux/errno.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/poll.h>
#include <linux/string.h>
#include <linux/list.h>
#include <linux/hash.h>
#include <linux/spinlock.h>
#include <linux/syscalls.h>
#include <linux/rbtree.h>
#include <linux/wait.h>
#include <linux/eventpoll.h>
#include <linux/mount.h>
#include <linux/bitops.h>
34
#include <linux/mutex.h>
Davide Libenzi's avatar
Davide Libenzi committed
35
#include <linux/anon_inodes.h>
36
#include <linux/device.h>
37
#include <linux/uaccess.h>
Linus Torvalds's avatar
Linus Torvalds committed
38 39
#include <asm/io.h>
#include <asm/mman.h>
Arun Sharma's avatar
Arun Sharma committed
40
#include <linux/atomic.h>
41 42
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
43
#include <linux/compat.h>
44
#include <linux/rculist.h>
45
#include <net/busy_poll.h>
Linus Torvalds's avatar
Linus Torvalds committed
46 47 48 49 50

/*
 * LOCKING:
 * There are three level of locking required by epoll :
 *
51
 * 1) epmutex (mutex)
52
 * 2) ep->mtx (mutex)
53
 * 3) ep->wq.lock (spinlock)
Linus Torvalds's avatar
Linus Torvalds committed
54 55
 *
 * The acquire order is the one listed above, from 1 to 3.
56
 * We need a spinlock (ep->wq.lock) because we manipulate objects
Linus Torvalds's avatar
Linus Torvalds committed
57 58 59 60 61 62
 * from inside the poll callback, that might be triggered from
 * a wake_up() that in turn might be called from IRQ context.
 * So we can't sleep inside the poll callback and hence we need
 * a spinlock. During the event transfer loop (from kernel to
 * user space) we could end up sleeping due a copy_to_user(), so
 * we need a lock that will allow us to sleep. This lock is a
63 64 65 66 67
 * mutex (ep->mtx). It is acquired during the event transfer loop,
 * during epoll_ctl(EPOLL_CTL_DEL) and during eventpoll_release_file().
 * Then we also need a global mutex to serialize eventpoll_release_file()
 * and ep_free().
 * This mutex is acquired by ep_free() during the epoll file
Linus Torvalds's avatar
Linus Torvalds committed
68 69
 * cleanup path and it is also acquired by eventpoll_release_file()
 * if a file has been pushed inside an epoll set and it is then
Daniel Baluta's avatar
Daniel Baluta committed
70
 * close()d without a previous call to epoll_ctl(EPOLL_CTL_DEL).
71 72 73 74 75 76 77
 * It is also acquired when inserting an epoll fd onto another epoll
 * fd. We do this so that we walk the epoll tree and ensure that this
 * insertion does not create a cycle of epoll file descriptors, which
 * could lead to deadlock. We need a global mutex to prevent two
 * simultaneous inserts (A into B and B into A) from racing and
 * constructing a cycle without either insert observing that it is
 * going to.
78 79 80 81 82 83 84 85 86
 * It is necessary to acquire multiple "ep->mtx"es at once in the
 * case when one epoll fd is added to another. In this case, we
 * always acquire the locks in the order of nesting (i.e. after
 * epoll_ctl(e1, EPOLL_CTL_ADD, e2), e1->mtx will always be acquired
 * before e2->mtx). Since we disallow cycles of epoll file
 * descriptors, this ensures that the mutexes are well-ordered. In
 * order to communicate this nesting to lockdep, when walking a tree
 * of epoll file descriptors, we use the current recursion depth as
 * the lockdep subkey.
87
 * It is possible to drop the "ep->mtx" and to use the global
88
 * mutex "epmutex" (together with "ep->wq.lock") to have it working,
89
 * but having "ep->mtx" will make the interface more scalable.
90
 * Events that require holding "epmutex" are very rare, while for
91 92
 * normal operations the epoll private "ep->mtx" will guarantee
 * a better scalability.
Linus Torvalds's avatar
Linus Torvalds committed
93 94 95
 */

/* Epoll private bits inside the event mask */
Jason Baron's avatar
Jason Baron committed
96
#define EP_PRIVATE_BITS (EPOLLWAKEUP | EPOLLONESHOT | EPOLLET | EPOLLEXCLUSIVE)
Linus Torvalds's avatar
Linus Torvalds committed
97

98
#define EPOLLINOUT_BITS (EPOLLIN | EPOLLOUT)
99

100
#define EPOLLEXCLUSIVE_OK_BITS (EPOLLINOUT_BITS | EPOLLERR | EPOLLHUP | \
101 102
				EPOLLWAKEUP | EPOLLET | EPOLLEXCLUSIVE)

Davide Libenzi's avatar
Davide Libenzi committed
103 104
/* Maximum number of nesting allowed inside epoll sets */
#define EP_MAX_NESTS 4
Linus Torvalds's avatar
Linus Torvalds committed
105

Davide Libenzi's avatar
Davide Libenzi committed
106 107
#define EP_MAX_EVENTS (INT_MAX / sizeof(struct epoll_event))

108 109
#define EP_UNACTIVE_PTR ((void *) -1L)

110 111
#define EP_ITEM_COST (sizeof(struct epitem) + sizeof(struct eppoll_entry))

Linus Torvalds's avatar
Linus Torvalds committed
112 113 114
struct epoll_filefd {
	struct file *file;
	int fd;
115
} __packed;
Linus Torvalds's avatar
Linus Torvalds committed
116 117

/*
Davide Libenzi's avatar
Davide Libenzi committed
118 119
 * Structure used to track possible nested calls, for too deep recursions
 * and loop cycles.
Linus Torvalds's avatar
Linus Torvalds committed
120
 */
Davide Libenzi's avatar
Davide Libenzi committed
121
struct nested_call_node {
Linus Torvalds's avatar
Linus Torvalds committed
122
	struct list_head llink;
Davide Libenzi's avatar
Davide Libenzi committed
123
	void *cookie;
124
	void *ctx;
Linus Torvalds's avatar
Linus Torvalds committed
125 126 127
};

/*
Davide Libenzi's avatar
Davide Libenzi committed
128 129
 * This structure is used as collector for nested calls, to check for
 * maximum recursion dept and loop cycles.
Linus Torvalds's avatar
Linus Torvalds committed
130
 */
Davide Libenzi's avatar
Davide Libenzi committed
131 132
struct nested_calls {
	struct list_head tasks_call_list;
Linus Torvalds's avatar
Linus Torvalds committed
133 134 135
	spinlock_t lock;
};

136 137 138
/*
 * Each file descriptor added to the eventpoll interface will
 * have an entry of this type linked to the "rbr" RB tree.
139 140
 * Avoid increasing the size of this struct, there can be many thousands
 * of these on a server and we do not want this to take another cache line.
141 142
 */
struct epitem {
143 144 145 146 147 148
	union {
		/* RB tree node links this structure to the eventpoll RB tree */
		struct rb_node rbn;
		/* Used to free the struct epitem */
		struct rcu_head rcu;
	};
149 150 151 152

	/* List header used to link this structure to the eventpoll ready list */
	struct list_head rdllink;

153 154 155 156 157 158
	/*
	 * Works together "struct eventpoll"->ovflist in keeping the
	 * single linked chain of items.
	 */
	struct epitem *next;

159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
	/* The file descriptor information this item refers to */
	struct epoll_filefd ffd;

	/* Number of active wait queue attached to poll operations */
	int nwait;

	/* List containing poll wait queues */
	struct list_head pwqlist;

	/* The "container" of this item */
	struct eventpoll *ep;

	/* List header used to link this item to the "struct file" items list */
	struct list_head fllink;

174
	/* wakeup_source used when EPOLLWAKEUP is set */
175
	struct wakeup_source __rcu *ws;
176

177 178
	/* The structure that describe the interested events and the source fd */
	struct epoll_event event;
179 180
};

Linus Torvalds's avatar
Linus Torvalds committed
181 182
/*
 * This structure is stored inside the "private_data" member of the file
Daniel Baluta's avatar
Daniel Baluta committed
183
 * structure and represents the main data structure for the eventpoll
Linus Torvalds's avatar
Linus Torvalds committed
184
 * interface.
185 186
 *
 * Access to it is protected by the lock inside wq.
Linus Torvalds's avatar
Linus Torvalds committed
187 188 189
 */
struct eventpoll {
	/*
190 191 192 193
	 * This mutex is used to ensure that files are not removed
	 * while epoll is using them. This is held during the event
	 * collection loop, the file cleanup path, the epoll file exit
	 * code and the ctl operations.
Linus Torvalds's avatar
Linus Torvalds committed
194
	 */
195
	struct mutex mtx;
Linus Torvalds's avatar
Linus Torvalds committed
196 197 198 199 200 201 202 203 204 205

	/* Wait queue used by sys_epoll_wait() */
	wait_queue_head_t wq;

	/* Wait queue used by file->poll() */
	wait_queue_head_t poll_wait;

	/* List of ready file descriptors */
	struct list_head rdllist;

Davide Libenzi's avatar
Davide Libenzi committed
206
	/* RB tree root used to store monitored fd structs */
207
	struct rb_root_cached rbr;
208 209 210

	/*
	 * This is a single linked list that chains all the "struct epitem" that
Lucas De Marchi's avatar
Lucas De Marchi committed
211
	 * happened while transferring ready events to userspace w/out
212
	 * holding ->wq.lock.
213 214
	 */
	struct epitem *ovflist;
215

216 217 218
	/* wakeup_source used when ep_scan_ready_list is running */
	struct wakeup_source *ws;

219 220
	/* The user that created the eventpoll descriptor */
	struct user_struct *user;
Jason Baron's avatar
Jason Baron committed
221 222 223 224 225 226

	struct file *file;

	/* used to optimize loop detection check */
	int visited;
	struct list_head visited_list_link;
227 228 229 230 231

#ifdef CONFIG_NET_RX_BUSY_POLL
	/* used to track busy poll napi_id */
	unsigned int napi_id;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
232 233 234 235 236 237 238 239
};

/* Wait structure used by the poll hooks */
struct eppoll_entry {
	/* List header used to link this structure to the "struct epitem" */
	struct list_head llink;

	/* The "base" pointer is set to the container "struct epitem" */
240
	struct epitem *base;
Linus Torvalds's avatar
Linus Torvalds committed
241 242 243 244 245

	/*
	 * Wait queue item that will be linked to the target file wait
	 * queue head.
	 */
246
	wait_queue_entry_t wait;
Linus Torvalds's avatar
Linus Torvalds committed
247 248 249 250 251 252 253 254 255 256 257

	/* The wait queue head that linked the "wait" wait queue item */
	wait_queue_head_t *whead;
};

/* Wrapper struct used by poll queueing */
struct ep_pqueue {
	poll_table pt;
	struct epitem *epi;
};

Davide Libenzi's avatar
Davide Libenzi committed
258 259 260 261
/* Used by the ep_send_events() function as callback private data */
struct ep_send_events_data {
	int maxevents;
	struct epoll_event __user *events;
262
	int res;
Davide Libenzi's avatar
Davide Libenzi committed
263 264
};

265 266 267 268
/*
 * Configuration options available inside /proc/sys/fs/epoll/
 */
/* Maximum number of epoll watched descriptors, per user */
269
static long max_user_watches __read_mostly;
270

Linus Torvalds's avatar
Linus Torvalds committed
271
/*
272
 * This mutex is used to serialize ep_free() and eventpoll_release_file().
Linus Torvalds's avatar
Linus Torvalds committed
273
 */
274
static DEFINE_MUTEX(epmutex);
Linus Torvalds's avatar
Linus Torvalds committed
275

276 277 278
/* Used to check for epoll file descriptor inclusion loops */
static struct nested_calls poll_loop_ncalls;

Linus Torvalds's avatar
Linus Torvalds committed
279
/* Slab cache used to allocate "struct epitem" */
280
static struct kmem_cache *epi_cache __read_mostly;
Linus Torvalds's avatar
Linus Torvalds committed
281 282

/* Slab cache used to allocate "struct eppoll_entry" */
283
static struct kmem_cache *pwq_cache __read_mostly;
Linus Torvalds's avatar
Linus Torvalds committed
284

Jason Baron's avatar
Jason Baron committed
285 286 287 288 289 290 291 292 293
/* Visited nodes during ep_loop_check(), so we can unset them when we finish */
static LIST_HEAD(visited_list);

/*
 * List of files with newly added links, where we may need to limit the number
 * of emanating paths. Protected by the epmutex.
 */
static LIST_HEAD(tfile_check_list);

294 295 296 297
#ifdef CONFIG_SYSCTL

#include <linux/sysctl.h>

298 299
static long zero;
static long long_max = LONG_MAX;
300

301
struct ctl_table epoll_table[] = {
302 303 304
	{
		.procname	= "max_user_watches",
		.data		= &max_user_watches,
305
		.maxlen		= sizeof(max_user_watches),
306
		.mode		= 0644,
307
		.proc_handler	= proc_doulongvec_minmax,
308
		.extra1		= &zero,
309
		.extra2		= &long_max,
310
	},
311
	{ }
312 313 314
};
#endif /* CONFIG_SYSCTL */

Jason Baron's avatar
Jason Baron committed
315 316 317 318 319 320
static const struct file_operations eventpoll_fops;

static inline int is_file_epoll(struct file *f)
{
	return f->f_op == &eventpoll_fops;
}
321

Davide Libenzi's avatar
Davide Libenzi committed
322
/* Setup the structure that is used as key for the RB tree */
323 324 325 326 327 328 329
static inline void ep_set_ffd(struct epoll_filefd *ffd,
			      struct file *file, int fd)
{
	ffd->file = file;
	ffd->fd = fd;
}

Davide Libenzi's avatar
Davide Libenzi committed
330
/* Compare RB tree keys */
331 332 333 334 335 336 337 338
static inline int ep_cmp_ffd(struct epoll_filefd *p1,
			     struct epoll_filefd *p2)
{
	return (p1->file > p2->file ? +1:
	        (p1->file < p2->file ? -1 : p1->fd - p2->fd));
}

/* Tells us if the item is currently linked */
339
static inline int ep_is_linked(struct epitem *epi)
340
{
341
	return !list_empty(&epi->rdllink);
342 343
}

344
static inline struct eppoll_entry *ep_pwq_from_wait(wait_queue_entry_t *p)
345 346 347 348
{
	return container_of(p, struct eppoll_entry, wait);
}

349
/* Get the "struct epitem" from a wait queue pointer */
350
static inline struct epitem *ep_item_from_wait(wait_queue_entry_t *p)
351 352 353 354 355
{
	return container_of(p, struct eppoll_entry, wait)->base;
}

/* Get the "struct epitem" from an epoll queue wrapper */
356
static inline struct epitem *ep_item_from_epqueue(poll_table *p)
357 358 359 360 361
{
	return container_of(p, struct ep_pqueue, pt)->epi;
}

/* Tells if the epoll_ctl(2) operation needs an event copy from userspace */
362
static inline int ep_op_has_event(int op)
363
{
364
	return op != EPOLL_CTL_DEL;
365 366
}

Linus Torvalds's avatar
Linus Torvalds committed
367
/* Initialize the poll safe wake up structure */
Davide Libenzi's avatar
Davide Libenzi committed
368
static void ep_nested_calls_init(struct nested_calls *ncalls)
Linus Torvalds's avatar
Linus Torvalds committed
369
{
Davide Libenzi's avatar
Davide Libenzi committed
370 371
	INIT_LIST_HEAD(&ncalls->tasks_call_list);
	spin_lock_init(&ncalls->lock);
Linus Torvalds's avatar
Linus Torvalds committed
372 373
}

374 375 376 377 378 379 380 381 382 383
/**
 * ep_events_available - Checks if ready events might be available.
 *
 * @ep: Pointer to the eventpoll context.
 *
 * Returns: Returns a value different than zero if ready events are available,
 *          or zero otherwise.
 */
static inline int ep_events_available(struct eventpoll *ep)
{
384 385
	return !list_empty_careful(&ep->rdllist) ||
		READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR;
386 387
}

388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451
#ifdef CONFIG_NET_RX_BUSY_POLL
static bool ep_busy_loop_end(void *p, unsigned long start_time)
{
	struct eventpoll *ep = p;

	return ep_events_available(ep) || busy_loop_timeout(start_time);
}

/*
 * Busy poll if globally on and supporting sockets found && no events,
 * busy loop will return if need_resched or ep_events_available.
 *
 * we must do our busy polling with irqs enabled
 */
static void ep_busy_loop(struct eventpoll *ep, int nonblock)
{
	unsigned int napi_id = READ_ONCE(ep->napi_id);

	if ((napi_id >= MIN_NAPI_ID) && net_busy_loop_on())
		napi_busy_loop(napi_id, nonblock ? NULL : ep_busy_loop_end, ep);
}

static inline void ep_reset_busy_poll_napi_id(struct eventpoll *ep)
{
	if (ep->napi_id)
		ep->napi_id = 0;
}

/*
 * Set epoll busy poll NAPI ID from sk.
 */
static inline void ep_set_busy_poll_napi_id(struct epitem *epi)
{
	struct eventpoll *ep;
	unsigned int napi_id;
	struct socket *sock;
	struct sock *sk;
	int err;

	if (!net_busy_loop_on())
		return;

	sock = sock_from_file(epi->ffd.file, &err);
	if (!sock)
		return;

	sk = sock->sk;
	if (!sk)
		return;

	napi_id = READ_ONCE(sk->sk_napi_id);
	ep = epi->ep;

	/* Non-NAPI IDs can be rejected
	 *	or
	 * Nothing to do if we already have this ID
	 */
	if (napi_id < MIN_NAPI_ID || napi_id == ep->napi_id)
		return;

	/* record NAPI ID for use in next busy poll */
	ep->napi_id = napi_id;
}

452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467
#else

static inline void ep_busy_loop(struct eventpoll *ep, int nonblock)
{
}

static inline void ep_reset_busy_poll_napi_id(struct eventpoll *ep)
{
}

static inline void ep_set_busy_poll_napi_id(struct epitem *epi)
{
}

#endif /* CONFIG_NET_RX_BUSY_POLL */

Davide Libenzi's avatar
Davide Libenzi committed
468 469 470 471 472 473 474 475 476 477
/**
 * ep_call_nested - Perform a bound (possibly) nested call, by checking
 *                  that the recursion limit is not exceeded, and that
 *                  the same nested call (by the meaning of same cookie) is
 *                  no re-entered.
 *
 * @ncalls: Pointer to the nested_calls structure to be used for this call.
 * @nproc: Nested call core function pointer.
 * @priv: Opaque data to be passed to the @nproc callback.
 * @cookie: Cookie to be used to identify this nested call.
478
 * @ctx: This instance context.
Davide Libenzi's avatar
Davide Libenzi committed
479 480 481
 *
 * Returns: Returns the code returned by the @nproc callback, or -1 if
 *          the maximum recursion limit has been exceeded.
Linus Torvalds's avatar
Linus Torvalds committed
482
 */
483
static int ep_call_nested(struct nested_calls *ncalls,
Davide Libenzi's avatar
Davide Libenzi committed
484
			  int (*nproc)(void *, void *, int), void *priv,
485
			  void *cookie, void *ctx)
Linus Torvalds's avatar
Linus Torvalds committed
486
{
Davide Libenzi's avatar
Davide Libenzi committed
487
	int error, call_nests = 0;
Linus Torvalds's avatar
Linus Torvalds committed
488
	unsigned long flags;
Davide Libenzi's avatar
Davide Libenzi committed
489 490 491
	struct list_head *lsthead = &ncalls->tasks_call_list;
	struct nested_call_node *tncur;
	struct nested_call_node tnode;
Linus Torvalds's avatar
Linus Torvalds committed
492

Davide Libenzi's avatar
Davide Libenzi committed
493
	spin_lock_irqsave(&ncalls->lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
494

Davide Libenzi's avatar
Davide Libenzi committed
495 496 497 498 499
	/*
	 * Try to see if the current task is already inside this wakeup call.
	 * We use a list here, since the population inside this set is always
	 * very much limited.
	 */
500
	list_for_each_entry(tncur, lsthead, llink) {
501
		if (tncur->ctx == ctx &&
502
		    (tncur->cookie == cookie || ++call_nests > EP_MAX_NESTS)) {
Linus Torvalds's avatar
Linus Torvalds committed
503 504 505 506
			/*
			 * Ops ... loop detected or maximum nest level reached.
			 * We abort this wake by breaking the cycle itself.
			 */
507 508
			error = -1;
			goto out_unlock;
Linus Torvalds's avatar
Linus Torvalds committed
509 510 511
		}
	}

Davide Libenzi's avatar
Davide Libenzi committed
512
	/* Add the current task and cookie to the list */
513
	tnode.ctx = ctx;
Davide Libenzi's avatar
Davide Libenzi committed
514
	tnode.cookie = cookie;
Linus Torvalds's avatar
Linus Torvalds committed
515 516
	list_add(&tnode.llink, lsthead);

Davide Libenzi's avatar
Davide Libenzi committed
517
	spin_unlock_irqrestore(&ncalls->lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
518

Davide Libenzi's avatar
Davide Libenzi committed
519 520
	/* Call the nested function */
	error = (*nproc)(priv, cookie, call_nests);
Linus Torvalds's avatar
Linus Torvalds committed
521 522

	/* Remove the current task from the list */
Davide Libenzi's avatar
Davide Libenzi committed
523
	spin_lock_irqsave(&ncalls->lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
524
	list_del(&tnode.llink);
525
out_unlock:
Davide Libenzi's avatar
Davide Libenzi committed
526 527 528 529 530
	spin_unlock_irqrestore(&ncalls->lock, flags);

	return error;
}

531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555
/*
 * As described in commit 0ccf831cb lockdep: annotate epoll
 * the use of wait queues used by epoll is done in a very controlled
 * manner. Wake ups can nest inside each other, but are never done
 * with the same locking. For example:
 *
 *   dfd = socket(...);
 *   efd1 = epoll_create();
 *   efd2 = epoll_create();
 *   epoll_ctl(efd1, EPOLL_CTL_ADD, dfd, ...);
 *   epoll_ctl(efd2, EPOLL_CTL_ADD, efd1, ...);
 *
 * When a packet arrives to the device underneath "dfd", the net code will
 * issue a wake_up() on its poll wake list. Epoll (efd1) has installed a
 * callback wakeup entry on that queue, and the wake_up() performed by the
 * "dfd" net code will end up in ep_poll_callback(). At this point epoll
 * (efd1) notices that it may have some event ready, so it needs to wake up
 * the waiters on its poll wait list (efd2). So it calls ep_poll_safewake()
 * that ends up in another wake_up(), after having checked about the
 * recursion constraints. That are, no more than EP_MAX_POLLWAKE_NESTS, to
 * avoid stack blasting.
 *
 * When CONFIG_DEBUG_LOCK_ALLOC is enabled, make sure lockdep can handle
 * this special case of epoll.
 */
556
#ifdef CONFIG_DEBUG_LOCK_ALLOC
557 558 559 560

static struct nested_calls poll_safewake_ncalls;

static int ep_poll_wakeup_proc(void *priv, void *cookie, int call_nests)
561 562
{
	unsigned long flags;
563
	wait_queue_head_t *wqueue = (wait_queue_head_t *)cookie;
564

565
	spin_lock_irqsave_nested(&wqueue->lock, flags, call_nests + 1);
566
	wake_up_locked_poll(wqueue, EPOLLIN);
567 568
	spin_unlock_irqrestore(&wqueue->lock, flags);

Davide Libenzi's avatar
Davide Libenzi committed
569 570 571 572 573
	return 0;
}

static void ep_poll_safewake(wait_queue_head_t *wq)
{
574 575
	int this_cpu = get_cpu();

576
	ep_call_nested(&poll_safewake_ncalls,
577 578 579
		       ep_poll_wakeup_proc, NULL, wq, (void *) (long) this_cpu);

	put_cpu();
Linus Torvalds's avatar
Linus Torvalds committed
580 581
}

582 583 584 585
#else

static void ep_poll_safewake(wait_queue_head_t *wq)
{
586
	wake_up_poll(wq, EPOLLIN);
587 588 589 590
}

#endif

591 592 593 594 595
static void ep_remove_wait_queue(struct eppoll_entry *pwq)
{
	wait_queue_head_t *whead;

	rcu_read_lock();
596 597 598 599 600 601 602
	/*
	 * If it is cleared by POLLFREE, it should be rcu-safe.
	 * If we read NULL we need a barrier paired with
	 * smp_store_release() in ep_poll_callback(), otherwise
	 * we rely on whead->lock.
	 */
	whead = smp_load_acquire(&pwq->whead);
603 604 605 606 607
	if (whead)
		remove_wait_queue(whead, &pwq->wait);
	rcu_read_unlock();
}

Linus Torvalds's avatar
Linus Torvalds committed
608
/*
609 610 611
 * This function unregisters poll callbacks from the associated file
 * descriptor.  Must be called with "mtx" held (or "epmutex" if called from
 * ep_free).
Linus Torvalds's avatar
Linus Torvalds committed
612
 */
613
static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi)
Linus Torvalds's avatar
Linus Torvalds committed
614
{
615 616
	struct list_head *lsthead = &epi->pwqlist;
	struct eppoll_entry *pwq;
Linus Torvalds's avatar
Linus Torvalds committed
617

618 619
	while (!list_empty(lsthead)) {
		pwq = list_first_entry(lsthead, struct eppoll_entry, llink);
Linus Torvalds's avatar
Linus Torvalds committed
620

621
		list_del(&pwq->llink);
622
		ep_remove_wait_queue(pwq);
623
		kmem_cache_free(pwq_cache, pwq);
Linus Torvalds's avatar
Linus Torvalds committed
624 625 626
	}
}

627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658
/* call only when ep->mtx is held */
static inline struct wakeup_source *ep_wakeup_source(struct epitem *epi)
{
	return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx));
}

/* call only when ep->mtx is held */
static inline void ep_pm_stay_awake(struct epitem *epi)
{
	struct wakeup_source *ws = ep_wakeup_source(epi);

	if (ws)
		__pm_stay_awake(ws);
}

static inline bool ep_has_wakeup_source(struct epitem *epi)
{
	return rcu_access_pointer(epi->ws) ? true : false;
}

/* call when ep->mtx cannot be held (ep_poll_callback) */
static inline void ep_pm_stay_awake_rcu(struct epitem *epi)
{
	struct wakeup_source *ws;

	rcu_read_lock();
	ws = rcu_dereference(epi->ws);
	if (ws)
		__pm_stay_awake(ws);
	rcu_read_unlock();
}

Davide Libenzi's avatar
Davide Libenzi committed
659 660 661 662 663 664 665 666
/**
 * ep_scan_ready_list - Scans the ready list in a way that makes possible for
 *                      the scan code, to call f_op->poll(). Also allows for
 *                      O(NumReady) performance.
 *
 * @ep: Pointer to the epoll private data structure.
 * @sproc: Pointer to the scan callback.
 * @priv: Private opaque data passed to the @sproc callback.
667
 * @depth: The current depth of recursive f_op->poll calls.
668
 * @ep_locked: caller already holds ep->mtx
Davide Libenzi's avatar
Davide Libenzi committed
669 670 671
 *
 * Returns: The same integer error code returned by the @sproc callback.
 */
Al Viro's avatar
Al Viro committed
672 673
static __poll_t ep_scan_ready_list(struct eventpoll *ep,
			      __poll_t (*sproc)(struct eventpoll *,
Davide Libenzi's avatar
Davide Libenzi committed
674
					   struct list_head *, void *),
675
			      void *priv, int depth, bool ep_locked)
Davide Libenzi's avatar
Davide Libenzi committed
676
{
Al Viro's avatar
Al Viro committed
677 678
	__poll_t res;
	int pwake = 0;
Davide Libenzi's avatar
Davide Libenzi committed
679
	struct epitem *epi, *nepi;
680
	LIST_HEAD(txlist);
Davide Libenzi's avatar
Davide Libenzi committed
681

682 683
	lockdep_assert_irqs_enabled();

Davide Libenzi's avatar
Davide Libenzi committed
684 685
	/*
	 * We need to lock this because we could be hit by
Tony Battersby's avatar
Tony Battersby committed
686
	 * eventpoll_release_file() and epoll_ctl().
Davide Libenzi's avatar
Davide Libenzi committed
687
	 */
688 689 690

	if (!ep_locked)
		mutex_lock_nested(&ep->mtx, depth);
Davide Libenzi's avatar
Davide Libenzi committed
691 692 693 694 695 696 697 698 699

	/*
	 * Steal the ready list, and re-init the original one to the
	 * empty list. Also, set ep->ovflist to NULL so that events
	 * happening while looping w/out locks, are not lost. We cannot
	 * have the poll callback to queue directly on ep->rdllist,
	 * because we want the "sproc" callback to be able to do it
	 * in a lockless way.
	 */
700
	spin_lock_irq(&ep->wq.lock);
701
	list_splice_init(&ep->rdllist, &txlist);
702
	WRITE_ONCE(ep->ovflist, NULL);
703
	spin_unlock_irq(&ep->wq.lock);
Davide Libenzi's avatar
Davide Libenzi committed
704 705 706 707

	/*
	 * Now call the callback function.
	 */
Al Viro's avatar
Al Viro committed
708
	res = (*sproc)(ep, &txlist, priv);
Davide Libenzi's avatar
Davide Libenzi committed
709

710
	spin_lock_irq(&ep->wq.lock);
Davide Libenzi's avatar
Davide Libenzi committed
711 712 713 714 715
	/*
	 * During the time we spent inside the "sproc" callback, some
	 * other events might have been queued by the poll callback.
	 * We re-insert them inside the main ready-list here.
	 */
716
	for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL;
Davide Libenzi's avatar
Davide Libenzi committed
717 718 719 720 721 722 723
	     nepi = epi->next, epi->next = EP_UNACTIVE_PTR) {
		/*
		 * We need to check if the item is already in the list.
		 * During the "sproc" callback execution time, items are
		 * queued into ->ovflist but the "txlist" might already
		 * contain them, and the list_splice() below takes care of them.
		 */
724
		if (!ep_is_linked(epi)) {
Davide Libenzi's avatar
Davide Libenzi committed
725
			list_add_tail(&epi->rdllink, &ep->rdllist);
726
			ep_pm_stay_awake(epi);
727
		}
Davide Libenzi's avatar
Davide Libenzi committed
728 729 730 731 732 733
	}
	/*
	 * We need to set back ep->ovflist to EP_UNACTIVE_PTR, so that after
	 * releasing the lock, events will be queued in the normal way inside
	 * ep->rdllist.
	 */
734
	WRITE_ONCE(ep->ovflist, EP_UNACTIVE_PTR);
Davide Libenzi's avatar
Davide Libenzi committed
735 736 737 738 739

	/*
	 * Quickly re-inject items left on "txlist".
	 */
	list_splice(&txlist, &ep->rdllist);
740
	__pm_relax(ep->ws);
Davide Libenzi's avatar
Davide Libenzi committed
741 742 743

	if (!list_empty(&ep->rdllist)) {
		/*
744 745
		 * Wake up (if active) both the eventpoll wait list and
		 * the ->poll() wait list (delayed after we release the lock).
Davide Libenzi's avatar
Davide Libenzi committed
746 747 748 749 750 751
		 */
		if (waitqueue_active(&ep->wq))
			wake_up_locked(&ep->wq);
		if (waitqueue_active(&ep->poll_wait))
			pwake++;
	}
752
	spin_unlock_irq(&ep->wq.lock);
Davide Libenzi's avatar
Davide Libenzi committed
753

754 755
	if (!ep_locked)
		mutex_unlock(&ep->mtx);
Davide Libenzi's avatar
Davide Libenzi committed
756 757 758 759 760

	/* We have to call this outside the lock */
	if (pwake)
		ep_poll_safewake(&ep->poll_wait);

Al Viro's avatar
Al Viro committed
761
	return res;
Davide Libenzi's avatar
Davide Libenzi committed
762 763
}

764 765 766 767 768 769
static void epi_rcu_free(struct rcu_head *head)
{
	struct epitem *epi = container_of(head, struct epitem, rcu);
	kmem_cache_free(epi_cache, epi);
}

770 771
/*
 * Removes a "struct epitem" from the eventpoll RB tree and deallocates
772
 * all the associated resources. Must be called with "mtx" held.
773 774 775 776
 */
static int ep_remove(struct eventpoll *ep, struct epitem *epi)
{
	struct file *file = epi->ffd.file;
Linus Torvalds's avatar
Linus Torvalds committed
777

778 779
	lockdep_assert_irqs_enabled();

Linus Torvalds's avatar
Linus Torvalds committed
780
	/*
781
	 * Removes poll wait queue hooks.
Linus Torvalds's avatar
Linus Torvalds committed
782
	 */
783
	ep_unregister_pollwait(ep, epi);
Linus Torvalds's avatar
Linus Torvalds committed
784

785
	/* Remove the current item from the list of epoll hooks */
786
	spin_lock(&file->f_lock);
787
	list_del_rcu(&epi->fllink);
788
	spin_unlock(&file->f_lock);
Linus Torvalds's avatar
Linus Torvalds committed
789

790
	rb_erase_cached(&epi->rbn, &ep->rbr);
Linus Torvalds's avatar
Linus Torvalds committed
791

792
	spin_lock_irq(&ep->wq.lock);
793
	if (ep_is_linked(epi))
794
		list_del_init(&epi->rdllink);
795
	spin_unlock_irq(&ep->wq.lock);
Linus Torvalds's avatar
Linus Torvalds committed
796

797
	wakeup_source_unregister(ep_wakeup_source(epi));
798 799 800 801 802 803 804 805
	/*
	 * At this point it is safe to free the eventpoll item. Use the union
	 * field epi->rcu, since we are trying to minimize the size of
	 * 'struct epitem'. The 'rbn' field is no longer in use. Protected by
	 * ep->mtx. The rcu read side, reverse_path_check_proc(), does not make
	 * use of the rbn field.
	 */
	call_rcu(&epi->rcu, epi_rcu_free);
Linus Torvalds's avatar
Linus Torvalds committed
806

807
	atomic_long_dec(&ep->user->epoll_watches);
808

809
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
810 811
}

812
static void ep_free(struct eventpoll *ep)
Linus Torvalds's avatar
Linus Torvalds committed
813
{
814 815
	struct rb_node *rbp;
	struct epitem *epi;
Linus Torvalds's avatar
Linus Torvalds committed
816

817 818
	/* We need to release all tasks waiting for these file */
	if (waitqueue_active(&ep->poll_wait))
Davide Libenzi's avatar
Davide Libenzi committed
819
		ep_poll_safewake(&ep->poll_wait);
Linus Torvalds's avatar
Linus Torvalds committed
820

821 822 823
	/*
	 * We need to lock this because we could be hit by
	 * eventpoll_release_file() while we're freeing the "struct eventpoll".
824
	 * We do not need to hold "ep->mtx" here because the epoll file
825 826
	 * is on the way to be removed and no one has references to it
	 * anymore. The only hit might come from eventpoll_release_file() but
Lucas De Marchi's avatar
Lucas De Marchi committed
827
	 * holding "epmutex" is sufficient here.
828 829
	 */
	mutex_lock(&epmutex);
Linus Torvalds's avatar
Linus Torvalds committed
830 831

	/*
832
	 * Walks through the whole tree by unregistering poll callbacks.
Linus Torvalds's avatar
Linus Torvalds committed
833
	 */
834
	for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
835 836 837
		epi = rb_entry(rbp, struct epitem, rbn);

		ep_unregister_pollwait(ep, epi);
838
		cond_resched();
839
	}
Linus Torvalds's avatar
Linus Torvalds committed
840 841

	/*
842 843
	 * Walks through the whole tree by freeing each "struct epitem". At this
	 * point we are sure no poll callbacks will be lingering around, and also by
844
	 * holding "epmutex" we can be sure that no file cleanup code will hit
845
	 * us during this operation. So we can avoid the lock on "ep->wq.lock".
846 847
	 * We do not need to lock ep->mtx, either, we only do it to prevent
	 * a lockdep warning.
Linus Torvalds's avatar
Linus Torvalds committed
848
	 */
849
	mutex_lock(&ep->mtx);
850
	while ((rbp = rb_first_cached(&ep->rbr)) != NULL) {
851 852
		epi = rb_entry(rbp, struct epitem, rbn);
		ep_remove(ep, epi);
853
		cond_resched();
854
	}
855
	mutex_unlock(&ep->mtx);
Linus Torvalds's avatar
Linus Torvalds committed
856

857
	mutex_unlock(&epmutex);
858
	mutex_destroy(&ep->mtx);
859
	free_uid(ep->user);
860
	wakeup_source_unregister(ep->ws);
861
	kfree(ep);
862
}
Linus Torvalds's avatar
Linus Torvalds committed
863

864 865 866
static int ep_eventpoll_release(struct inode *inode, struct file *file)
{
	struct eventpoll *ep = file->private_data;
Linus Torvalds's avatar
Linus Torvalds committed
867

868
	if (ep)
869 870 871
		ep_free(ep);

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
872 873
}

Al Viro's avatar
Al Viro committed
874
static __poll_t ep_read_events_proc(struct eventpoll *ep, struct list_head *head,
875 876 877 878 879 880 881 882 883
			       void *priv);
static void ep_ptable_queue_proc(struct file *file, wait_queue_head_t *whead,
				 poll_table *pt);

/*
 * Differs from ep_eventpoll_poll() in that internal callers already have
 * the ep->mtx so we need to start from depth=1, such that mutex_lock_nested()
 * is correctly annotated.
 */
Al Viro's avatar
Al Viro committed
884
static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt,
885
				 int depth)
886
{
887 888 889
	struct eventpoll *ep;
	bool locked;

890
	pt->_key = epi->event.events;
891
	if (!is_file_epoll(epi->ffd.file))
892
		return vfs_poll(epi->ffd.file, pt) & epi->event.events;
893

894 895 896
	ep = epi->ffd.file->private_data;
	poll_wait(epi->ffd.file, &ep->poll_wait, pt);
	locked = pt && (pt->_qproc == ep_ptable_queue_proc);
897

898 899 900
	return ep_scan_ready_list(epi->ffd.file->private_data,
				  ep_read_events_proc, &depth, depth,
				  locked) & epi->event.events;
901 902
}

Al Viro's avatar
Al Viro committed
903
static __poll_t ep_read_events_proc(struct eventpoll *ep, struct list_head *head,
904
			       void *priv)
Davide Libenzi's avatar
Davide Libenzi committed
905 906
{
	struct epitem *epi, *tmp;
907
	poll_table pt;
908
	int depth = *(int *)priv;
Davide Libenzi's avatar
Davide Libenzi committed
909

910
	init_poll_funcptr(&pt, NULL);
911
	depth++;
912

Davide Libenzi's avatar
Davide Libenzi committed
913
	list_for_each_entry_safe(epi, tmp, head, rdllink) {
914
		if (ep_item_poll(epi, &pt, depth)) {
915
			return EPOLLIN | EPOLLRDNORM;
916
		} else {
Davide Libenzi's avatar
Davide Libenzi committed
917 918 919 920 921
			/*
			 * Item has been dropped into the ready list by the poll
			 * callback, but it's not actually ready, as far as
			 * caller requested events goes. We can remove it here.
			 */
922
			__pm_relax(ep_wakeup_source(epi));
Davide Libenzi's avatar
Davide Libenzi committed
923
			list_del_init(&epi->rdllink);
924
		}
Davide Libenzi's avatar
Davide Libenzi committed
925 926 927 928 929
	}

	return 0;
}

930
static __poll_t ep_eventpoll_poll(struct file *file, poll_table *wait)
931 932 933
{
	struct eventpoll *ep = file->private_data;
	int depth = 0;
934

935 936 937
	/* Insert inside our poll wait queue */
	poll_wait(file, &ep->poll_wait, wait);

Davide Libenzi's avatar
Davide Libenzi committed
938 939
	/*
	 * Proceed to find out if wanted events are really available inside
940
	 * the ready list.
Davide Libenzi's avatar
Davide Libenzi committed
941
	 */
942 943
	return ep_scan_ready_list(ep, ep_read_events_proc,
				  &depth, depth, false);
944 945
}

946
#ifdef CONFIG_PROC_FS
947
static void ep_show_fdinfo(struct seq_file *m, struct file *f)
948 949 950 951 952
{
	struct eventpoll *ep = f->private_data;
	struct rb_node *rbp;

	mutex_lock(&ep->mtx);
953
	for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
954
		struct epitem *epi = rb_entry(rbp, struct epitem, rbn);
955
		struct inode *inode = file_inode(epi->ffd.file);
956

957 958
		seq_printf(m, "tfd: %8d events: %8x data: %16llx "
			   " pos:%lli ino:%lx sdev:%x\n",
959
			   epi->ffd.fd, epi->event.events,
960 961 962
			   (long long)epi->event.data,
			   (long long)epi->ffd.file->f_pos,
			   inode->i_ino, inode->i_sb->s_dev);
963
		if (seq_has_overflowed(m))
964 965 966 967 968 969
			break;
	}
	mutex_unlock(&ep->mtx);
}
#endif

970 971
/* File callbacks that implement the eventpoll file behaviour */
static const struct file_operations eventpoll_fops = {
972 973 974
#ifdef CONFIG_PROC_FS
	.show_fdinfo	= ep_show_fdinfo,
#endif
975
	.release	= ep_eventpoll_release,
976
	.poll		= ep_eventpoll_poll,
977
	.llseek		= noop_llseek,
978 979
};

Davide Libenzi's avatar
Davide Libenzi committed
980
/*
981 982 983
 * This is called from eventpoll_release() to unlink files from the eventpoll
 * interface. We need to have this facility to cleanup correctly files that are
 * closed without being removed from the eventpoll interface.
Davide Libenzi's avatar
Davide Libenzi committed
984
 */
985
void eventpoll_release_file(struct file *file)
Davide Libenzi's avatar
Davide Libenzi committed
986
{
987
	struct eventpoll *ep;
988
	struct epitem *epi, *next;
Davide Libenzi's avatar
Davide Libenzi committed
989 990

	/*
991
	 * We don't want to get "file->f_lock" because it is not
992
	 * necessary. It is not necessary because we're in the "struct file"
Lucas De Marchi's avatar
Lucas De Marchi committed
993
	 * cleanup path, and this means that no one is using this file anymore.
Davide Libenzi's avatar
Davide Libenzi committed
994
	 * So, for example, epoll_ctl() cannot hit here since if we reach this
Davide Libenzi's avatar
Davide Libenzi committed
995
	 * point, the file counter already went to zero and fget() would fail.
996
	 * The only hit might come from ep_free() but by holding the mutex
997
	 * will correctly serialize the operation. We do need to acquire
998
	 * "ep->mtx" after "epmutex" because ep_remove() requires it when called
999
	 * from anywhere but ep_free().
1000 1001
	 *
	 * Besides, ep_remove() acquires the lock, so we can't hold it here.
Davide Libenzi's avatar
Davide Libenzi committed
1002
	 */
1003
	mutex_lock(&epmutex);
1004
	list_for_each_entry_safe(epi, next, &file->f_ep_links, fllink) {
1005
		ep = epi->ep;
1006
		mutex_lock_nested(&ep->mtx, 0);
1007
		ep_remove(ep, epi);
1008
		mutex_unlock(&ep->mtx);
Davide Libenzi's avatar
Davide Libenzi committed
1009
	}
1010
	mutex_unlock(&epmutex);
Davide Libenzi's avatar
Davide Libenzi committed
1011 1012
}

1013
static int ep_alloc(struct eventpoll **pep)
Linus Torvalds's avatar
Linus Torvalds committed
1014
{
1015 1016 1017
	int error;
	struct user_struct *user;
	struct eventpoll *ep;
Linus Torvalds's avatar
Linus Torvalds committed
1018

1019 1020 1021 1022 1023
	user = get_current_user();
	error = -ENOMEM;
	ep = kzalloc(sizeof(*ep), GFP_KERNEL);
	if (unlikely(!ep))
		goto free_uid;
Linus Torvalds's avatar
Linus Torvalds committed
1024

1025
	mutex_init(&ep->mtx);
Linus Torvalds's avatar
Linus Torvalds committed
1026 1027 1028
	init_waitqueue_head(&ep->wq);
	init_waitqueue_head(&ep->poll_wait);
	INIT_LIST_HEAD(&ep->rdllist);
1029
	ep->rbr = RB_ROOT_CACHED;
1030
	ep->ovflist = EP_UNACTIVE_PTR;
1031
	ep->user = user;
Linus Torvalds's avatar
Linus Torvalds committed
1032

1033
	*pep = ep;
Linus Torvalds's avatar
Linus Torvalds committed
1034 1035

	return 0;
1036 1037 1038 1039

free_uid:
	free_uid(user);
	return error;
Linus Torvalds's avatar
Linus Torvalds committed
1040 1041 1042
}

/*
1043 1044 1045
 * Search the file inside the eventpoll tree. The RB tree operations
 * are protected by the "mtx" mutex, and ep_find() must be called with
 * "mtx" held.
Linus Torvalds's avatar
Linus Torvalds committed
1046 1047 1048 1049 1050 1051 1052 1053
 */
static struct epitem *ep_find(struct eventpoll *ep, struct file *file, int fd)
{
	int kcmp;
	struct rb_node *rbp;
	struct epitem *epi, *epir = NULL;
	struct epoll_filefd ffd;

1054
	ep_set_ffd(&ffd, file, fd);
1055
	for (rbp = ep->rbr.rb_root.rb_node; rbp; ) {
Linus Torvalds's avatar
Linus Torvalds committed
1056
		epi = rb_entry(rbp, struct epitem, rbn);
1057
		kcmp = ep_cmp_ffd(&ffd, &epi->ffd);
Linus Torvalds's avatar
Linus Torvalds committed
1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070
		if (kcmp > 0)
			rbp = rbp->rb_right;
		else if (kcmp < 0)
			rbp = rbp->rb_left;
		else {
			epir = epi;
			break;
		}
	}

	return epir;
}

1071
#ifdef CONFIG_CHECKPOINT_RESTORE
1072 1073 1074 1075 1076
static struct epitem *ep_find_tfd(struct eventpoll *ep, int tfd, unsigned long toff)
{
	struct rb_node *rbp;
	struct epitem *epi;

1077
	for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112
		epi = rb_entry(rbp, struct epitem, rbn);
		if (epi->ffd.fd == tfd) {
			if (toff == 0)
				return epi;
			else
				toff--;
		}
		cond_resched();
	}

	return NULL;
}

struct file *get_epoll_tfile_raw_ptr(struct file *file, int tfd,
				     unsigned long toff)
{
	struct file *file_raw;
	struct eventpoll *ep;
	struct epitem *epi;

	if (!is_file_epoll(file))
		return ERR_PTR(-EINVAL);

	ep = file->private_data;

	mutex_lock(&ep->mtx);
	epi = ep_find_tfd(ep, tfd, toff);
	if (epi)
		file_raw = epi->ffd.file;
	else
		file_raw = ERR_PTR(-ENOENT);
	mutex_unlock(&ep->mtx);

	return file_raw;
}
1113
#endif /* CONFIG_CHECKPOINT_RESTORE */
1114

Linus Torvalds's avatar
Linus Torvalds committed
1115
/*
1116
 * This is the callback that is passed to the wait queue wakeup
Daniel Baluta's avatar
Daniel Baluta committed
1117
 * mechanism. It is called by the stored file descriptors when they
1118
 * have events to report.
Linus Torvalds's avatar
Linus Torvalds committed
1119
 */
1120
static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
Linus Torvalds's avatar
Linus Torvalds committed
1121
{
1122 1123 1124 1125
	int pwake = 0;
	unsigned long flags;
	struct epitem *epi = ep_item_from_wait(wait);
	struct eventpoll *ep = epi->ep;
Al Viro's avatar
Al Viro committed
1126
	__poll_t pollflags = key_to_poll(key);
Jason Baron's avatar
Jason Baron committed
1127
	int ewake = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1128

1129
	spin_lock_irqsave(&ep->wq.lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
1130

1131 1132
	ep_set_busy_poll_napi_id(epi);

1133 1134 1135 1136 1137 1138 1139
	/*
	 * If the event mask does not contain any poll(2) event, we consider the
	 * descriptor to be disabled. This condition is likely the effect of the
	 * EPOLLONESHOT bit that disables the descriptor when an event is received,
	 * until the next EPOLL_CTL_MOD will be issued.
	 */
	if (!(epi->event.events & ~EP_PRIVATE_BITS))
1140 1141
		goto out_unlock;

1142 1143 1144 1145 1146 1147
	/*
	 * Check the events coming with the callback. At this stage, not
	 * every device reports the events in the "key" parameter of the
	 * callback. We need to be able to handle both cases here, hence the
	 * test for "key" != NULL before the event match test.
	 */
Al Viro's avatar
Al Viro committed
1148
	if (pollflags && !(pollflags & epi->event.events))
1149 1150
		goto out_unlock;

1151
	/*
Daniel Baluta's avatar
Daniel Baluta committed
1152
	 * If we are transferring events to userspace, we can hold no locks
1153
	 * (because we're accessing user memory, and because of linux f_op->poll()
Daniel Baluta's avatar
Daniel Baluta committed
1154
	 * semantics). All the events that happen during that period of time are
1155 1156
	 * chained in ep->ovflist and requeued later on.
	 */
1157
	if (READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR) {
1158
		if (epi->next == EP_UNACTIVE_PTR) {
1159 1160
			epi->next = READ_ONCE(ep->ovflist);
			WRITE_ONCE(ep->ovflist, epi);
1161 1162 1163 1164 1165 1166 1167 1168
			if (epi->ws) {
				/*
				 * Activate ep->ws since epi->ws may get
				 * deactivated at any time.
				 */
				__pm_stay_awake(ep->ws);
			}

Davide Libenzi's avatar