public inbox for [email protected]
 help / color / mirror / Atom feed
From: Stefan Roesch <[email protected]>
To: [email protected], [email protected]
Cc: [email protected], [email protected], [email protected],
	[email protected], [email protected], [email protected]
Subject: [PATCH v14 1/8] net: split off __napi_busy_poll from napi_busy_poll
Date: Mon,  5 Jun 2023 14:20:02 -0700	[thread overview]
Message-ID: <[email protected]> (raw)
In-Reply-To: <[email protected]>

This splits off the key part of the napi_busy_poll function into its own
function __napi_busy_poll. This is done in preparation for an additional
napi_busy_poll() function, that doesn't take the rcu_read_lock(). The
new function is introduced in the next patch.

Signed-off-by: Stefan Roesch <[email protected]>
---
 net/core/dev.c | 99 ++++++++++++++++++++++++++++----------------------
 1 file changed, 56 insertions(+), 43 deletions(-)

diff --git a/net/core/dev.c b/net/core/dev.c
index 253584777101..f4677aa20f84 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -6166,66 +6166,79 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool
 	local_bh_enable();
 }
 
+struct napi_busy_poll_ctx {
+	struct napi_struct *napi;
+	int (*napi_poll)(struct napi_struct *napi, int budget);
+	void *have_poll_lock;
+};
+
+static inline void __napi_busy_poll(struct napi_busy_poll_ctx *ctx,
+				    bool prefer_busy_poll, u16 budget)
+{
+	struct napi_struct *napi = ctx->napi;
+	int work = 0;
+
+	WARN_ON_ONCE(!rcu_read_lock_held());
+
+	local_bh_disable();
+	if (!ctx->napi_poll) {
+		unsigned long val = READ_ONCE(napi->state);
+
+		/* If multiple threads are competing for this napi,
+		 * we avoid dirtying napi->state as much as we can.
+		 */
+		if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
+			   NAPIF_STATE_IN_BUSY_POLL)) {
+			if (prefer_busy_poll)
+				set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
+			goto count;
+		}
+		if (cmpxchg(&napi->state, val,
+			    val | NAPIF_STATE_IN_BUSY_POLL |
+				  NAPIF_STATE_SCHED) != val) {
+			if (prefer_busy_poll)
+				set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
+			goto count;
+		}
+		ctx->have_poll_lock = netpoll_poll_lock(napi);
+		ctx->napi_poll = napi->poll;
+	}
+	work = ctx->napi_poll(napi, budget);
+	trace_napi_poll(napi, work, budget);
+	gro_normal_list(napi);
+
+count:
+	if (work > 0)
+		__NET_ADD_STATS(dev_net(napi->dev),
+				LINUX_MIB_BUSYPOLLRXPACKETS, work);
+	local_bh_enable();
+}
 void napi_busy_loop(unsigned int napi_id,
 		    bool (*loop_end)(void *, unsigned long),
 		    void *loop_end_arg, bool prefer_busy_poll, u16 budget)
 {
 	unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
-	int (*napi_poll)(struct napi_struct *napi, int budget);
-	void *have_poll_lock = NULL;
-	struct napi_struct *napi;
+	struct napi_busy_poll_ctx ctx = {};
 
 restart:
-	napi_poll = NULL;
+	ctx.napi_poll = NULL;
 
 	rcu_read_lock();
 
-	napi = napi_by_id(napi_id);
-	if (!napi)
+	ctx.napi = napi_by_id(napi_id);
+	if (!ctx.napi)
 		goto out;
 
 	preempt_disable();
 	for (;;) {
-		int work = 0;
-
-		local_bh_disable();
-		if (!napi_poll) {
-			unsigned long val = READ_ONCE(napi->state);
-
-			/* If multiple threads are competing for this napi,
-			 * we avoid dirtying napi->state as much as we can.
-			 */
-			if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
-				   NAPIF_STATE_IN_BUSY_POLL)) {
-				if (prefer_busy_poll)
-					set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
-				goto count;
-			}
-			if (cmpxchg(&napi->state, val,
-				    val | NAPIF_STATE_IN_BUSY_POLL |
-					  NAPIF_STATE_SCHED) != val) {
-				if (prefer_busy_poll)
-					set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
-				goto count;
-			}
-			have_poll_lock = netpoll_poll_lock(napi);
-			napi_poll = napi->poll;
-		}
-		work = napi_poll(napi, budget);
-		trace_napi_poll(napi, work, budget);
-		gro_normal_list(napi);
-count:
-		if (work > 0)
-			__NET_ADD_STATS(dev_net(napi->dev),
-					LINUX_MIB_BUSYPOLLRXPACKETS, work);
-		local_bh_enable();
+		__napi_busy_poll(&ctx, prefer_busy_poll, budget);
 
 		if (!loop_end || loop_end(loop_end_arg, start_time))
 			break;
 
 		if (unlikely(need_resched())) {
-			if (napi_poll)
-				busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
+			if (ctx.napi_poll)
+				busy_poll_stop(ctx.napi, ctx.have_poll_lock, prefer_busy_poll, budget);
 			preempt_enable();
 			rcu_read_unlock();
 			cond_resched();
@@ -6235,8 +6248,8 @@ void napi_busy_loop(unsigned int napi_id,
 		}
 		cpu_relax();
 	}
-	if (napi_poll)
-		busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
+	if (ctx.napi_poll)
+		busy_poll_stop(ctx.napi, ctx.have_poll_lock, prefer_busy_poll, budget);
 	preempt_enable();
 out:
 	rcu_read_unlock();
-- 
2.39.1


  reply	other threads:[~2023-06-05 21:20 UTC|newest]

Thread overview: 11+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2023-06-05 21:20 [PATCH v14 0/8] io_uring: add napi busy polling support Stefan Roesch
2023-06-05 21:20 ` Stefan Roesch [this message]
2023-06-05 21:20 ` [PATCH v14 2/8] net: introduce napi_busy_loop_rcu() Stefan Roesch
2023-06-06 10:10   ` Simon Horman
2023-06-05 21:20 ` [PATCH v14 3/8] net: split off _napi_busy_loop() Stefan Roesch
2023-06-05 22:21   ` Jakub Kicinski
2023-06-05 21:20 ` [PATCH v14 4/8] io-uring: move io_wait_queue definition to header file Stefan Roesch
2023-06-05 21:20 ` [PATCH v14 5/8] io-uring: add napi busy poll support Stefan Roesch
2023-06-05 21:20 ` [PATCH v14 6/8] io-uring: add sqpoll support for napi busy poll Stefan Roesch
2023-06-05 21:20 ` [PATCH v14 7/8] io_uring: add register/unregister napi function Stefan Roesch
2023-06-05 21:20 ` [PATCH v14 8/8] io_uring: add prefer busy poll to register and unregister napi api Stefan Roesch

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox