Kernel/linux-stable
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2025-01-01 10:45:49 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

io_uring: implement our own schedule timeout handling

Browse Source 
In preparation for having two distinct timeouts and avoid waking the
task if we don't need to.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe 2024-01-04 08:46:23 -07:00
parent 45a41e74b8
commit cebf123c63
2 changed files with 33 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
io_uring/io_uring.c
View File

@ -2350,6 +2350,34 @@ static bool current_pending_io(void)
return percpu_counter_read_positive(&tctx->inflight); return percpu_counter_read_positive(&tctx->inflight);
} }
static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
{
struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
WRITE_ONCE(iowq->hit_timeout, 1);
wake_up_process(iowq->wq.private);
return HRTIMER_NORESTART;
}
static int io_cqring_schedule_timeout(struct io_wait_queue *iowq,
clockid_t clock_id)
{
iowq->hit_timeout = 0;
hrtimer_init_on_stack(&iowq->t, clock_id, HRTIMER_MODE_ABS);
iowq->t.function = io_cqring_timer_wakeup;
hrtimer_set_expires_range_ns(&iowq->t, iowq->timeout, 0);
hrtimer_start_expires(&iowq->t, HRTIMER_MODE_ABS);
if (!READ_ONCE(iowq->hit_timeout))
schedule();
hrtimer_cancel(&iowq->t);
destroy_hrtimer_on_stack(&iowq->t);
__set_current_state(TASK_RUNNING);
return READ_ONCE(iowq->hit_timeout) ? -ETIME : 0;
}
static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx, static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx,
struct io_wait_queue *iowq) struct io_wait_queue *iowq)
{ {
@ -2362,11 +2390,10 @@ static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx,
*/ */
if (current_pending_io()) if (current_pending_io())
current->in_iowait = 1; current->in_iowait = 1;
if (iowq->timeout == KTIME_MAX) if (iowq->timeout != KTIME_MAX)
ret = io_cqring_schedule_timeout(iowq, ctx->clockid);
else
schedule(); schedule();
else if (!schedule_hrtimeout_range_clock(&iowq->timeout, 0,
HRTIMER_MODE_ABS, ctx->clockid))
ret = -ETIME;
current->in_iowait = 0; current->in_iowait = 0;
return ret; return ret;
} }

2
io_uring/io_uring.h
View File

@ -40,7 +40,9 @@ struct io_wait_queue {
struct io_ring_ctx *ctx; struct io_ring_ctx *ctx;
unsigned cq_tail; unsigned cq_tail;
unsigned nr_timeouts; unsigned nr_timeouts;
int hit_timeout;
ktime_t timeout; ktime_t timeout;
struct hrtimer t;
#ifdef CONFIG_NET_RX_BUSY_POLL #ifdef CONFIG_NET_RX_BUSY_POLL
ktime_t napi_busy_poll_dt; ktime_t napi_busy_poll_dt;