From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <io-uring-owner@kernel.org>
X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on
aws-us-west-2-korg-lkml-1.web.codeaurora.org
Received: from vger.kernel.org (vger.kernel.org [23.128.96.18])
by smtp.lore.kernel.org (Postfix) with ESMTP id 2BC5BC433EF
for <io-uring@archiver.kernel.org>; Wed, 20 Apr 2022 19:15:19 +0000 (UTC)
Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand
id S1381684AbiDTTSD (ORCPT <rfc822;io-uring@archiver.kernel.org>);
Wed, 20 Apr 2022 15:18:03 -0400
Received: from lindbergh.monkeyblade.net ([23.128.96.19]:36748 "EHLO
lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
with ESMTP id S1358692AbiDTTSB (ORCPT
<rfc822;io-uring@vger.kernel.org>); Wed, 20 Apr 2022 15:18:01 -0400
Received: from mx0a-00082601.pphosted.com (mx0b-00082601.pphosted.com [67.231.153.30])
by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 7B4F6BE05
for <io-uring@vger.kernel.org>; Wed, 20 Apr 2022 12:15:14 -0700 (PDT)
Received: from pps.filterd (m0089730.ppops.net [127.0.0.1])
by m0089730.ppops.net (8.16.1.2/8.16.1.2) with ESMTP id 23KILSNt020512
for <io-uring@vger.kernel.org>; Wed, 20 Apr 2022 12:15:13 -0700
DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=fb.com; h=from : to : cc : subject
: date : message-id : in-reply-to : references : mime-version :
content-transfer-encoding : content-type; s=facebook;
bh=0dsqbiIYUpYGsz6SX+Aevv0g3blMuu1d1Q7h+Udsm+w=;
b=YdrIR7zO+bSmv5Xmaci55dkhTYL/b7MhX8lPC1KCy6x1QqnWxfy/ID59tYEtRagse9zd
oEMrUZn894HSpoWNXR3RSFpwbgxZ7STxmBN05X5u564CIhYr35EzdN0t2tIMuK2ja4Wl
y3x4mv4rnTG3OzfELdeLqmsvbaUZzlA+eIE=
Received: from maileast.thefacebook.com ([163.114.130.16])
by m0089730.ppops.net (PPS) with ESMTPS id 3fhn4j4en4-1
(version=TLSv1.2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128 verify=NOT)
for <io-uring@vger.kernel.org>; Wed, 20 Apr 2022 12:15:13 -0700
Received: from twshared8508.05.ash9.facebook.com (2620:10d:c0a8:1b::d) by
mail.thefacebook.com (2620:10d:c0a8:82::e) with Microsoft SMTP Server
(version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id
15.1.2375.24; Wed, 20 Apr 2022 12:15:12 -0700
Received: by devvm225.atn0.facebook.com (Postfix, from userid 425415)
id AFBC5DE0C414; Wed, 20 Apr 2022 12:14:53 -0700 (PDT)
From: Stefan Roesch <shr@fb.com>
To: <io-uring@vger.kernel.org>, <kernel-team@fb.com>
CC: <shr@fb.com>, Jens Axboe <axboe@kernel.dk>
Subject: [PATCH v2 08/12] io_uring: overflow processing for CQE32
Date: Wed, 20 Apr 2022 12:14:47 -0700
Message-ID: <20220420191451.2904439-9-shr@fb.com>
X-Mailer: git-send-email 2.30.2
In-Reply-To: <20220420191451.2904439-1-shr@fb.com>
References: <20220420191451.2904439-1-shr@fb.com>
MIME-Version: 1.0
Content-Transfer-Encoding: quoted-printable
X-FB-Internal: Safe
Content-Type: text/plain
X-Proofpoint-GUID: QvNNJh8KrmzgO-EWNo2U771qPe525Yv-
X-Proofpoint-ORIG-GUID: QvNNJh8KrmzgO-EWNo2U771qPe525Yv-
X-Proofpoint-Virus-Version: vendor=baseguard
engine=ICAP:2.0.205,Aquarius:18.0.858,Hydra:6.0.486,FMLib:17.11.64.514
definitions=2022-04-20_05,2022-04-20_01,2022-02-23_01
Precedence: bulk
List-ID: <io-uring.vger.kernel.org>
X-Mailing-List: io-uring@vger.kernel.org
This adds the overflow processing for large CQE's.
This adds two parameters to the io_cqring_event_overflow function and
uses these fields to initialize the large CQE fields.
Allocate enough space for large CQE's in the overflow structue. If no
large CQE's are used, the size of the allocation is unchanged.
The cqe field can have a different size depending if its a large
CQE or not. To be able to allocate different sizes, the two fields
in the structure are re-ordered.
Co-developed-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Stefan Roesch <shr@fb.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
fs/io_uring.c | 26 +++++++++++++++++++-------
1 file changed, 19 insertions(+), 7 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index ff6229b6df16..50efced63ec9 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -220,8 +220,8 @@ struct io_mapped_ubuf {
struct io_ring_ctx;
=20
struct io_overflow_cqe {
- struct io_uring_cqe cqe;
struct list_head list;
+ struct io_uring_cqe cqe;
};
=20
struct io_fixed_file {
@@ -2016,13 +2016,17 @@ static bool __io_cqring_overflow_flush(struct io_=
ring_ctx *ctx, bool force)
while (!list_empty(&ctx->cq_overflow_list)) {
struct io_uring_cqe *cqe =3D io_get_cqe(ctx);
struct io_overflow_cqe *ocqe;
+ size_t cqe_size =3D sizeof(struct io_uring_cqe);
+
+ if (ctx->flags & IORING_SETUP_CQE32)
+ cqe_size <<=3D 1;
=20
if (!cqe && !force)
break;
ocqe =3D list_first_entry(&ctx->cq_overflow_list,
struct io_overflow_cqe, list);
if (cqe)
- memcpy(cqe, &ocqe->cqe, sizeof(*cqe));
+ memcpy(cqe, &ocqe->cqe, cqe_size);
else
io_account_cq_overflow(ctx);
=20
@@ -2111,11 +2115,15 @@ static __cold void io_uring_drop_tctx_refs(struct=
task_struct *task)
}
=20
static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_d=
ata,
- s32 res, u32 cflags)
+ s32 res, u32 cflags, u64 extra1, u64 extra2)
{
struct io_overflow_cqe *ocqe;
+ size_t ocq_size =3D sizeof(struct io_overflow_cqe);
=20
- ocqe =3D kmalloc(sizeof(*ocqe), GFP_ATOMIC | __GFP_ACCOUNT);
+ if (ctx->flags & IORING_SETUP_CQE32)
+ ocq_size +=3D sizeof(struct io_uring_cqe);
+
+ ocqe =3D kmalloc(ocq_size, GFP_ATOMIC | __GFP_ACCOUNT);
if (!ocqe) {
/*
* If we're in ring overflow flush mode, or in task cancel mode,
@@ -2134,6 +2142,10 @@ static bool io_cqring_event_overflow(struct io_rin=
g_ctx *ctx, u64 user_data,
ocqe->cqe.user_data =3D user_data;
ocqe->cqe.res =3D res;
ocqe->cqe.flags =3D cflags;
+ if (ctx->flags & IORING_SETUP_CQE32) {
+ ocqe->cqe.b[0].extra1 =3D extra1;
+ ocqe->cqe.b[0].extra2 =3D extra2;
+ }
list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
return true;
}
@@ -2155,7 +2167,7 @@ static inline bool __io_fill_cqe(struct io_ring_ctx=
*ctx, u64 user_data,
WRITE_ONCE(cqe->flags, cflags);
return true;
}
- return io_cqring_event_overflow(ctx, user_data, res, cflags);
+ return io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0);
}
=20
static inline bool __io_fill_cqe_req_filled(struct io_ring_ctx *ctx,
@@ -2177,7 +2189,7 @@ static inline bool __io_fill_cqe_req_filled(struct =
io_ring_ctx *ctx,
return true;
}
return io_cqring_event_overflow(ctx, req->cqe.user_data,
- req->cqe.res, req->cqe.flags);
+ req->cqe.res, req->cqe.flags, 0, 0);
}
=20
static inline bool __io_fill_cqe32_req_filled(struct io_ring_ctx *ctx,
@@ -2241,7 +2253,7 @@ static void __io_fill_cqe32_req(struct io_kiocb *re=
q, s32 res, u32 cflags,
return;
}
=20
- io_cqring_event_overflow(ctx, req->cqe.user_data, res, cflags);
+ io_cqring_event_overflow(ctx, req->cqe.user_data, res, cflags, extra1, =
extra2);
}
=20
static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_d=
ata,
--=20
2.30.2