Kernel/linux-stable
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2025-01-04 04:06:26 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

io_uring: introduce io_req_cqe_overflow()

Browse Source 
__io_fill_cqe_req() is hot and inlined, we want it to be as small as
possible. Add io_req_cqe_overflow() accepting only a request and doing
all overflow accounting, and replace with it two calls to 6 argument
io_cqring_event_overflow().

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/048b9fbcce56814d77a1a540409c98c3d383edcb.1655455613.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2022-06-17 09:48:02 +01:00 committed by Jens Axboe
parent faf88dde06
commit 68494a65d0
2 changed files with 15 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
io_uring/io_uring.c
View File

@ -643,8 +643,8 @@ static __cold void io_uring_drop_tctx_refs(struct task_struct *task)
} }
} }
bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, s32 res, static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
u32 cflags, u64 extra1, u64 extra2) s32 res, u32 cflags, u64 extra1, u64 extra2)
{ {
struct io_overflow_cqe *ocqe; struct io_overflow_cqe *ocqe;
size_t ocq_size = sizeof(struct io_overflow_cqe); size_t ocq_size = sizeof(struct io_overflow_cqe);
@ -681,6 +681,17 @@ bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, s32 res,
return true; return true;
} }
bool io_req_cqe_overflow(struct io_kiocb *req)
{
if (!(req->flags & REQ_F_CQE32_INIT)) {
req->extra1 = 0;
req->extra2 = 0;
}
return io_cqring_event_overflow(req->ctx, req->cqe.user_data,
req->cqe.res, req->cqe.flags,
req->extra1, req->extra2);
}
/* /*
* writes to the cq entry need to come after reading head; the * writes to the cq entry need to come after reading head; the
* control dependency is enough as we're using WRITE_ONCE to * control dependency is enough as we're using WRITE_ONCE to

12
io_uring/io_uring.h
View File

@ -15,8 +15,7 @@ enum {
}; };
struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx); struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx);
bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, s32 res, bool io_req_cqe_overflow(struct io_kiocb *req);
u32 cflags, u64 extra1, u64 extra2);
static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx) static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)
{ {
@ -56,10 +55,6 @@ static inline bool __io_fill_cqe_req(struct io_ring_ctx *ctx,
memcpy(cqe, &req->cqe, sizeof(*cqe)); memcpy(cqe, &req->cqe, sizeof(*cqe));
return true; return true;
} }
return io_cqring_event_overflow(ctx, req->cqe.user_data,
req->cqe.res, req->cqe.flags,
0, 0);
} else { } else {
u64 extra1 = 0, extra2 = 0; u64 extra1 = 0, extra2 = 0;
@ -83,11 +78,8 @@ static inline bool __io_fill_cqe_req(struct io_ring_ctx *ctx,
WRITE_ONCE(cqe->big_cqe[1], extra2); WRITE_ONCE(cqe->big_cqe[1], extra2);
return true; return true;
} }
return io_cqring_event_overflow(ctx, req->cqe.user_data,
req->cqe.res, req->cqe.flags,
extra1, extra2);
} }
return io_req_cqe_overflow(req);
} }
static inline void req_set_fail(struct io_kiocb *req) static inline void req_set_fail(struct io_kiocb *req)