Kernel/linux-stable
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2025-01-09 06:33:34 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'for-io_uring-add-napi-busy-polling-support'

Browse Source 
Merge netdev bits of io_uring busy polling support.

Jens Axboe says:

====================
io_uring: add napi busy polling support

I finally got around to testing this patchset in its current form, and
results look fine to me. It Works. Using the basic ping/pong test that's
part of the liburing addition, without enabling NAPI I get:

Stock settings, no NAPI, 100k packets:

 rtt(us) min/avg/max/mdev = 31.730/37.006/87.960/0.497

 and with -t10 -b enabled:

 rtt(us) min/avg/max/mdev = 23.250/29.795/63.511/1.203

In short, this patchset enables per io_uring NAPI enablement, rather
than need to enable that globally. This allows targeted NAPI usage with
io_uring.

Here's Stefan's v15 posting, which predates this one:

https://lore.kernel.org/io-uring/20230608163839.2891748-1-shr@devkernel.io/
====================

Link: https://lore.kernel.org/r/20240206163422.646218-1-axboe@kernel.dk
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Jakub Kicinski 2024-02-09 10:01:12 -08:00
commit f6ce9a1f6a
2 changed files with 47 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/net/busy_poll.h
View File

@ -48,6 +48,10 @@ void napi_busy_loop(unsigned int napi_id,
bool (*loop_end)(void *, unsigned long), bool (*loop_end)(void *, unsigned long),
void *loop_end_arg, bool prefer_busy_poll, u16 budget); void *loop_end_arg, bool prefer_busy_poll, u16 budget);
void napi_busy_loop_rcu(unsigned int napi_id,
bool (*loop_end)(void *, unsigned long),
void *loop_end_arg, bool prefer_busy_poll, u16 budget);
#else /* CONFIG_NET_RX_BUSY_POLL */ #else /* CONFIG_NET_RX_BUSY_POLL */
static inline unsigned long net_busy_loop_on(void) static inline unsigned long net_busy_loop_on(void)
{ {

57
net/core/dev.c
View File

@ -6185,8 +6185,13 @@ static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
clear_bit(NAPI_STATE_SCHED, &napi->state); clear_bit(NAPI_STATE_SCHED, &napi->state);
} }
static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool prefer_busy_poll, enum {
u16 budget) NAPI_F_PREFER_BUSY_POLL = 1,
NAPI_F_END_ON_RESCHED = 2,
};
static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock,
unsigned flags, u16 budget)
{ {
bool skip_schedule = false; bool skip_schedule = false;
unsigned long timeout; unsigned long timeout;
@ -6206,7 +6211,7 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool
local_bh_disable(); local_bh_disable();
if (prefer_busy_poll) { if (flags & NAPI_F_PREFER_BUSY_POLL) {
napi->defer_hard_irqs_count = READ_ONCE(napi->dev->napi_defer_hard_irqs); napi->defer_hard_irqs_count = READ_ONCE(napi->dev->napi_defer_hard_irqs);
timeout = READ_ONCE(napi->dev->gro_flush_timeout); timeout = READ_ONCE(napi->dev->gro_flush_timeout);
if (napi->defer_hard_irqs_count && timeout) { if (napi->defer_hard_irqs_count && timeout) {
@ -6230,23 +6235,23 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool
local_bh_enable(); local_bh_enable();
} }
void napi_busy_loop(unsigned int napi_id, static void __napi_busy_loop(unsigned int napi_id,
bool (*loop_end)(void *, unsigned long), bool (*loop_end)(void *, unsigned long),
void *loop_end_arg, bool prefer_busy_poll, u16 budget) void *loop_end_arg, unsigned flags, u16 budget)
{ {
unsigned long start_time = loop_end ? busy_loop_current_time() : 0; unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
int (*napi_poll)(struct napi_struct *napi, int budget); int (*napi_poll)(struct napi_struct *napi, int budget);
void *have_poll_lock = NULL; void *have_poll_lock = NULL;
struct napi_struct *napi; struct napi_struct *napi;
WARN_ON_ONCE(!rcu_read_lock_held());
restart: restart:
napi_poll = NULL; napi_poll = NULL;
rcu_read_lock();
napi = napi_by_id(napi_id); napi = napi_by_id(napi_id);
if (!napi) if (!napi)
goto out; return;
if (!IS_ENABLED(CONFIG_PREEMPT_RT)) if (!IS_ENABLED(CONFIG_PREEMPT_RT))
preempt_disable(); preempt_disable();
@ -6262,14 +6267,14 @@ void napi_busy_loop(unsigned int napi_id,
*/ */
if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED | if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
NAPIF_STATE_IN_BUSY_POLL)) { NAPIF_STATE_IN_BUSY_POLL)) {
if (prefer_busy_poll) if (flags & NAPI_F_PREFER_BUSY_POLL)
set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
goto count; goto count;
} }
if (cmpxchg(&napi->state, val, if (cmpxchg(&napi->state, val,
val | NAPIF_STATE_IN_BUSY_POLL | val | NAPIF_STATE_IN_BUSY_POLL |
NAPIF_STATE_SCHED) != val) { NAPIF_STATE_SCHED) != val) {
if (prefer_busy_poll) if (flags & NAPI_F_PREFER_BUSY_POLL)
set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
goto count; goto count;
} }
@ -6289,12 +6294,15 @@ void napi_busy_loop(unsigned int napi_id,
break; break;
if (unlikely(need_resched())) { if (unlikely(need_resched())) {
if (flags & NAPI_F_END_ON_RESCHED)
break;
if (napi_poll) if (napi_poll)
busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget); busy_poll_stop(napi, have_poll_lock, flags, budget);
if (!IS_ENABLED(CONFIG_PREEMPT_RT)) if (!IS_ENABLED(CONFIG_PREEMPT_RT))
preempt_enable(); preempt_enable();
rcu_read_unlock(); rcu_read_unlock();
cond_resched(); cond_resched();
rcu_read_lock();
if (loop_end(loop_end_arg, start_time)) if (loop_end(loop_end_arg, start_time))
return; return;
goto restart; goto restart;
@ -6302,10 +6310,31 @@ void napi_busy_loop(unsigned int napi_id,
cpu_relax(); cpu_relax();
} }
if (napi_poll) if (napi_poll)
busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget); busy_poll_stop(napi, have_poll_lock, flags, budget);
if (!IS_ENABLED(CONFIG_PREEMPT_RT)) if (!IS_ENABLED(CONFIG_PREEMPT_RT))
preempt_enable(); preempt_enable();
out: }
void napi_busy_loop_rcu(unsigned int napi_id,
bool (*loop_end)(void *, unsigned long),
void *loop_end_arg, bool prefer_busy_poll, u16 budget)
{
unsigned flags = NAPI_F_END_ON_RESCHED;
if (prefer_busy_poll)
flags |= NAPI_F_PREFER_BUSY_POLL;
__napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget);
}
void napi_busy_loop(unsigned int napi_id,
bool (*loop_end)(void *, unsigned long),
void *loop_end_arg, bool prefer_busy_poll, u16 budget)
{
unsigned flags = prefer_busy_poll ? NAPI_F_PREFER_BUSY_POLL : 0;
rcu_read_lock();
__napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget);
rcu_read_unlock(); rcu_read_unlock();
} }
EXPORT_SYMBOL(napi_busy_loop); EXPORT_SYMBOL(napi_busy_loop);