Kernel/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-01-17 02:15:57 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

io_uring: carve io_poll_check_events fast path

Browse Source 
The fast path in io_poll_check_events() is when we have only one
(i.e. master) reference. Move all verification, cancellations
checks, edge case handling and so on under a common if.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/8c21c5d5e027e32dc553705e88796dec79ff6f93.1669821213.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2022-11-30 15:21:52 +00:00 committed by Jens Axboe
parent f6f7f903e7
commit 9805fa2d94
1 changed files with 22 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
io_uring/poll.c
View File

@ -247,28 +247,31 @@ static int io_poll_check_events(struct io_kiocb *req, bool *locked)
do { do {
v = atomic_read(&req->poll_refs); v = atomic_read(&req->poll_refs);
/* tw handler should be the owner, and so have some references */ if (unlikely(v != 1)) {
/* tw should be the owner and so have some refs */
if (WARN_ON_ONCE(!(v & IO_POLL_REF_MASK))) if (WARN_ON_ONCE(!(v & IO_POLL_REF_MASK)))
return IOU_POLL_DONE; return IOU_POLL_DONE;
if (v & IO_POLL_CANCEL_FLAG) if (v & IO_POLL_CANCEL_FLAG)
return -ECANCELED; return -ECANCELED;
/* /*
* cqe.res contains only events of the first wake up * cqe.res contains only events of the first wake up
* and all others are be lost. Redo vfs_poll() to get * and all others are to be lost. Redo vfs_poll() to get
* up to date state. * up to date state.
*/ */
if ((v & IO_POLL_REF_MASK) != 1) if ((v & IO_POLL_REF_MASK) != 1)
req->cqe.res = 0; req->cqe.res = 0;
if (v & IO_POLL_RETRY_FLAG) { if (v & IO_POLL_RETRY_FLAG) {
req->cqe.res = 0; req->cqe.res = 0;
/* /*
* We won't find new events that came in between * We won't find new events that came in between
* vfs_poll and the ref put unless we clear the flag * vfs_poll and the ref put unless we clear the
* in advance. * flag in advance.
*/ */
atomic_andnot(IO_POLL_RETRY_FLAG, &req->poll_refs); atomic_andnot(IO_POLL_RETRY_FLAG, &req->poll_refs);
v &= ~IO_POLL_RETRY_FLAG; v &= ~IO_POLL_RETRY_FLAG;
} }
}
/* the mask was stashed in __io_poll_execute */ /* the mask was stashed in __io_poll_execute */
if (!req->cqe.res) { if (!req->cqe.res) {