[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]
Message-Id: <20231219210357.4029713-17-dw@davidwei.uk>
Date: Tue, 19 Dec 2023 13:03:53 -0800
From: David Wei <dw@...idwei.uk>
To: io-uring@...r.kernel.org,
netdev@...r.kernel.org
Cc: Jens Axboe <axboe@...nel.dk>,
Pavel Begunkov <asml.silence@...il.com>,
Jakub Kicinski <kuba@...nel.org>,
Paolo Abeni <pabeni@...hat.com>,
"David S. Miller" <davem@...emloft.net>,
Eric Dumazet <edumazet@...gle.com>,
Jesper Dangaard Brouer <hawk@...nel.org>,
David Ahern <dsahern@...nel.org>,
Mina Almasry <almasrymina@...gle.com>
Subject: [RFC PATCH v3 16/20] net: execute custom callback from napi
From: Pavel Begunkov <asml.silence@...il.com>
Sometimes we want to access a napi protected resource from task
context like in the case of io_uring zc falling back to copy and
accessing the buffer ring. Add a helper function that allows to execute
a custom function from napi context by first stopping it similarly to
napi_busy_loop().
Experimental, needs much polishing and sharing bits with
napi_busy_loop().
Signed-off-by: Pavel Begunkov <asml.silence@...il.com>
Signed-off-by: David Wei <dw@...idwei.uk>
---
include/net/busy_poll.h | 7 +++++++
net/core/dev.c | 46 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 53 insertions(+)
diff --git a/include/net/busy_poll.h b/include/net/busy_poll.h
index 4dabeb6c76d3..64238467e00a 100644
--- a/include/net/busy_poll.h
+++ b/include/net/busy_poll.h
@@ -47,6 +47,8 @@ bool sk_busy_loop_end(void *p, unsigned long start_time);
void napi_busy_loop(unsigned int napi_id,
bool (*loop_end)(void *, unsigned long),
void *loop_end_arg, bool prefer_busy_poll, u16 budget);
+void napi_execute(struct napi_struct *napi,
+ void (*cb)(void *), void *cb_arg);
#else /* CONFIG_NET_RX_BUSY_POLL */
static inline unsigned long net_busy_loop_on(void)
@@ -59,6 +61,11 @@ static inline bool sk_can_busy_loop(struct sock *sk)
return false;
}
+static inline void napi_execute(struct napi_struct *napi,
+ void (*cb)(void *), void *cb_arg)
+{
+}
+
#endif /* CONFIG_NET_RX_BUSY_POLL */
static inline unsigned long busy_loop_current_time(void)
diff --git a/net/core/dev.c b/net/core/dev.c
index e55750c47245..2dd4f3846535 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -6537,6 +6537,52 @@ void napi_busy_loop(unsigned int napi_id,
}
EXPORT_SYMBOL(napi_busy_loop);
+void napi_execute(struct napi_struct *napi,
+ void (*cb)(void *), void *cb_arg)
+{
+ bool done = false;
+ unsigned long val;
+ void *have_poll_lock = NULL;
+
+ rcu_read_lock();
+
+ if (!IS_ENABLED(CONFIG_PREEMPT_RT))
+ preempt_disable();
+ for (;;) {
+ local_bh_disable();
+ val = READ_ONCE(napi->state);
+
+ /* If multiple threads are competing for this napi,
+ * we avoid dirtying napi->state as much as we can.
+ */
+ if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
+ NAPIF_STATE_IN_BUSY_POLL))
+ goto restart;
+
+ if (cmpxchg(&napi->state, val,
+ val | NAPIF_STATE_IN_BUSY_POLL |
+ NAPIF_STATE_SCHED) != val)
+ goto restart;
+
+ have_poll_lock = netpoll_poll_lock(napi);
+ cb(cb_arg);
+ done = true;
+ gro_normal_list(napi);
+ local_bh_enable();
+ break;
+restart:
+ local_bh_enable();
+ if (unlikely(need_resched()))
+ break;
+ cpu_relax();
+ }
+ if (done)
+ busy_poll_stop(napi, have_poll_lock, false, 1);
+ if (!IS_ENABLED(CONFIG_PREEMPT_RT))
+ preempt_enable();
+ rcu_read_unlock();
+}
+
#endif /* CONFIG_NET_RX_BUSY_POLL */
static void napi_hash_add(struct napi_struct *napi)
--
2.39.3
Powered by blists - more mailing lists