* [PATCH 1/4] io_uring: add wrappers for memory accounting
2020-06-16 23:36 [PATCH 0/4] io_uring: report locked memory usage Bijan Mottahedeh
@ 2020-06-16 23:36 ` Bijan Mottahedeh
2020-06-16 23:36 ` [PATCH 2/4] io_uring: rename ctx->account_mem field Bijan Mottahedeh
` (3 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Bijan Mottahedeh @ 2020-06-16 23:36 UTC (permalink / raw)
To: axboe; +Cc: io-uring
Facilitate separation of locked memory usage reporting vs. limiting for
upcoming patches. No functional changes.
Signed-off-by: Bijan Mottahedeh <[email protected]>
---
fs/io_uring.c | 48 ++++++++++++++++++++++++++++--------------------
1 file changed, 28 insertions(+), 20 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index a07c44e..ebd3f62 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -7028,12 +7028,14 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx,
return ret;
}
-static void io_unaccount_mem(struct user_struct *user, unsigned long nr_pages)
+static inline void __io_unaccount_mem(struct user_struct *user,
+ unsigned long nr_pages)
{
atomic_long_sub(nr_pages, &user->locked_vm);
}
-static int io_account_mem(struct user_struct *user, unsigned long nr_pages)
+static inline int __io_account_mem(struct user_struct *user,
+ unsigned long nr_pages)
{
unsigned long page_limit, cur_pages, new_pages;
@@ -7051,6 +7053,20 @@ static int io_account_mem(struct user_struct *user, unsigned long nr_pages)
return 0;
}
+static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
+{
+ if (ctx->account_mem)
+ __io_unaccount_mem(ctx->user, nr_pages);
+}
+
+static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
+{
+ if (ctx->account_mem)
+ return (__io_account_mem(ctx->user, nr_pages));
+
+ return 0;
+}
+
static void io_mem_free(void *ptr)
{
struct page *page;
@@ -7125,8 +7141,7 @@ static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
for (j = 0; j < imu->nr_bvecs; j++)
unpin_user_page(imu->bvec[j].bv_page);
- if (ctx->account_mem)
- io_unaccount_mem(ctx->user, imu->nr_bvecs);
+ io_unaccount_mem(ctx, imu->nr_bvecs);
kvfree(imu->bvec);
imu->nr_bvecs = 0;
}
@@ -7209,11 +7224,9 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
start = ubuf >> PAGE_SHIFT;
nr_pages = end - start;
- if (ctx->account_mem) {
- ret = io_account_mem(ctx->user, nr_pages);
- if (ret)
- goto err;
- }
+ ret = io_account_mem(ctx, nr_pages);
+ if (ret)
+ goto err;
ret = 0;
if (!pages || nr_pages > got_pages) {
@@ -7226,8 +7239,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
GFP_KERNEL);
if (!pages || !vmas) {
ret = -ENOMEM;
- if (ctx->account_mem)
- io_unaccount_mem(ctx->user, nr_pages);
+ io_unaccount_mem(ctx, nr_pages);
goto err;
}
got_pages = nr_pages;
@@ -7237,8 +7249,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
GFP_KERNEL);
ret = -ENOMEM;
if (!imu->bvec) {
- if (ctx->account_mem)
- io_unaccount_mem(ctx->user, nr_pages);
+ io_unaccount_mem(ctx, nr_pages);
goto err;
}
@@ -7269,8 +7280,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
*/
if (pret > 0)
unpin_user_pages(pages, pret);
- if (ctx->account_mem)
- io_unaccount_mem(ctx->user, nr_pages);
+ io_unaccount_mem(ctx, nr_pages);
kvfree(imu->bvec);
goto err;
}
@@ -7375,9 +7385,7 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx)
io_mem_free(ctx->sq_sqes);
percpu_ref_exit(&ctx->refs);
- if (ctx->account_mem)
- io_unaccount_mem(ctx->user,
- ring_pages(ctx->sq_entries, ctx->cq_entries));
+ io_unaccount_mem(ctx, ring_pages(ctx->sq_entries, ctx->cq_entries));
free_uid(ctx->user);
put_cred(ctx->creds);
kfree(ctx->cancel_hash);
@@ -7916,7 +7924,7 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,
account_mem = !capable(CAP_IPC_LOCK);
if (account_mem) {
- ret = io_account_mem(user,
+ ret = __io_account_mem(user,
ring_pages(p->sq_entries, p->cq_entries));
if (ret) {
free_uid(user);
@@ -7927,7 +7935,7 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,
ctx = io_ring_ctx_alloc(p);
if (!ctx) {
if (account_mem)
- io_unaccount_mem(user, ring_pages(p->sq_entries,
+ __io_unaccount_mem(user, ring_pages(p->sq_entries,
p->cq_entries));
free_uid(user);
return -ENOMEM;
--
1.8.3.1
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 2/4] io_uring: rename ctx->account_mem field
2020-06-16 23:36 [PATCH 0/4] io_uring: report locked memory usage Bijan Mottahedeh
2020-06-16 23:36 ` [PATCH 1/4] io_uring: add wrappers for memory accounting Bijan Mottahedeh
@ 2020-06-16 23:36 ` Bijan Mottahedeh
2020-06-16 23:36 ` [PATCH 3/4] io_uring: report pinned memory usage Bijan Mottahedeh
` (2 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Bijan Mottahedeh @ 2020-06-16 23:36 UTC (permalink / raw)
To: axboe; +Cc: io-uring
Rename account_mem to limit_name to clarify its purpose.
Signed-off-by: Bijan Mottahedeh <[email protected]>
---
fs/io_uring.c | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index ebd3f62..0bcbc1a 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -226,7 +226,7 @@ struct io_ring_ctx {
struct {
unsigned int flags;
unsigned int compat: 1;
- unsigned int account_mem: 1;
+ unsigned int limit_mem: 1;
unsigned int cq_overflow_flushed: 1;
unsigned int drain_next: 1;
unsigned int eventfd_async: 1;
@@ -7055,13 +7055,13 @@ static inline int __io_account_mem(struct user_struct *user,
static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
{
- if (ctx->account_mem)
+ if (ctx->limit_mem)
__io_unaccount_mem(ctx->user, nr_pages);
}
static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
{
- if (ctx->account_mem)
+ if (ctx->limit_mem)
return (__io_account_mem(ctx->user, nr_pages));
return 0;
@@ -7882,7 +7882,7 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,
{
struct user_struct *user = NULL;
struct io_ring_ctx *ctx;
- bool account_mem;
+ bool limit_mem;
int ret;
if (!entries)
@@ -7921,9 +7921,9 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,
}
user = get_uid(current_user());
- account_mem = !capable(CAP_IPC_LOCK);
+ limit_mem = !capable(CAP_IPC_LOCK);
- if (account_mem) {
+ if (limit_mem) {
ret = __io_account_mem(user,
ring_pages(p->sq_entries, p->cq_entries));
if (ret) {
@@ -7934,14 +7934,14 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,
ctx = io_ring_ctx_alloc(p);
if (!ctx) {
- if (account_mem)
+ if (limit_mem)
__io_unaccount_mem(user, ring_pages(p->sq_entries,
p->cq_entries));
free_uid(user);
return -ENOMEM;
}
ctx->compat = in_compat_syscall();
- ctx->account_mem = account_mem;
+ ctx->limit_mem = limit_mem;
ctx->user = user;
ctx->creds = get_current_cred();
--
1.8.3.1
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 3/4] io_uring: report pinned memory usage
2020-06-16 23:36 [PATCH 0/4] io_uring: report locked memory usage Bijan Mottahedeh
2020-06-16 23:36 ` [PATCH 1/4] io_uring: add wrappers for memory accounting Bijan Mottahedeh
2020-06-16 23:36 ` [PATCH 2/4] io_uring: rename ctx->account_mem field Bijan Mottahedeh
@ 2020-06-16 23:36 ` Bijan Mottahedeh
2020-06-16 23:36 ` [PATCH 4/4] io_uring: separate reporting of ring pages from registered pages Bijan Mottahedeh
2020-06-17 15:35 ` [PATCH 0/4] io_uring: report locked memory usage Jens Axboe
4 siblings, 0 replies; 6+ messages in thread
From: Bijan Mottahedeh @ 2020-06-16 23:36 UTC (permalink / raw)
To: axboe; +Cc: io-uring
Report pinned memory usage always, regardless of whether locked memory
limit is enforced.
Signed-off-by: Bijan Mottahedeh <[email protected]>
---
fs/io_uring.c | 22 ++++++++++++++++++----
1 file changed, 18 insertions(+), 4 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 0bcbc1a..851ff21 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -7057,12 +7057,23 @@ static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
{
if (ctx->limit_mem)
__io_unaccount_mem(ctx->user, nr_pages);
+
+ if (ctx->sqo_mm)
+ atomic64_sub(nr_pages, &ctx->sqo_mm->pinned_vm);
}
static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
{
- if (ctx->limit_mem)
- return (__io_account_mem(ctx->user, nr_pages));
+ int ret;
+
+ if (ctx->limit_mem) {
+ ret = __io_account_mem(ctx->user, nr_pages);
+ if (ret)
+ return ret;
+ }
+
+ if (ctx->sqo_mm)
+ atomic64_add(nr_pages, &ctx->sqo_mm->pinned_vm);
return 0;
}
@@ -7364,8 +7375,10 @@ static void io_destroy_buffers(struct io_ring_ctx *ctx)
static void io_ring_ctx_free(struct io_ring_ctx *ctx)
{
io_finish_async(ctx);
- if (ctx->sqo_mm)
+ if (ctx->sqo_mm) {
mmdrop(ctx->sqo_mm);
+ ctx->sqo_mm = NULL;
+ }
io_iopoll_reap_events(ctx);
io_sqe_buffer_unregister(ctx);
@@ -7941,7 +7954,6 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,
return -ENOMEM;
}
ctx->compat = in_compat_syscall();
- ctx->limit_mem = limit_mem;
ctx->user = user;
ctx->creds = get_current_cred();
@@ -7988,6 +8000,8 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,
goto err;
trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
+ io_account_mem(ctx, ring_pages(p->sq_entries, p->cq_entries));
+ ctx->limit_mem = limit_mem;
return ret;
err:
io_ring_ctx_wait_and_kill(ctx);
--
1.8.3.1
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 4/4] io_uring: separate reporting of ring pages from registered pages
2020-06-16 23:36 [PATCH 0/4] io_uring: report locked memory usage Bijan Mottahedeh
` (2 preceding siblings ...)
2020-06-16 23:36 ` [PATCH 3/4] io_uring: report pinned memory usage Bijan Mottahedeh
@ 2020-06-16 23:36 ` Bijan Mottahedeh
2020-06-17 15:35 ` [PATCH 0/4] io_uring: report locked memory usage Jens Axboe
4 siblings, 0 replies; 6+ messages in thread
From: Bijan Mottahedeh @ 2020-06-16 23:36 UTC (permalink / raw)
To: axboe; +Cc: io-uring
Ring pages are not pinned so it is more appropriate to report them
as locked.
Signed-off-by: Bijan Mottahedeh <[email protected]>
---
fs/io_uring.c | 43 ++++++++++++++++++++++++++++++-------------
1 file changed, 30 insertions(+), 13 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 851ff21..c308dad 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -876,6 +876,11 @@ struct io_op_def {
},
};
+enum io_mem_account {
+ ACCT_LOCKED,
+ ACCT_PINNED,
+};
+
static void io_wq_submit_work(struct io_wq_work **workptr);
static void io_cqring_fill_event(struct io_kiocb *req, long res);
static void io_put_req(struct io_kiocb *req);
@@ -7053,16 +7058,22 @@ static inline int __io_account_mem(struct user_struct *user,
return 0;
}
-static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
+static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages,
+ enum io_mem_account acct)
{
if (ctx->limit_mem)
__io_unaccount_mem(ctx->user, nr_pages);
- if (ctx->sqo_mm)
- atomic64_sub(nr_pages, &ctx->sqo_mm->pinned_vm);
+ if (ctx->sqo_mm) {
+ if (acct == ACCT_LOCKED)
+ ctx->sqo_mm->locked_vm -= nr_pages;
+ else if (acct == ACCT_PINNED)
+ atomic64_sub(nr_pages, &ctx->sqo_mm->pinned_vm);
+ }
}
-static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
+static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages,
+ enum io_mem_account acct)
{
int ret;
@@ -7072,8 +7083,12 @@ static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
return ret;
}
- if (ctx->sqo_mm)
- atomic64_add(nr_pages, &ctx->sqo_mm->pinned_vm);
+ if (ctx->sqo_mm) {
+ if (acct == ACCT_LOCKED)
+ ctx->sqo_mm->locked_vm += nr_pages;
+ else if (acct == ACCT_PINNED)
+ atomic64_add(nr_pages, &ctx->sqo_mm->pinned_vm);
+ }
return 0;
}
@@ -7152,7 +7167,7 @@ static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
for (j = 0; j < imu->nr_bvecs; j++)
unpin_user_page(imu->bvec[j].bv_page);
- io_unaccount_mem(ctx, imu->nr_bvecs);
+ io_unaccount_mem(ctx, imu->nr_bvecs, ACCT_PINNED);
kvfree(imu->bvec);
imu->nr_bvecs = 0;
}
@@ -7235,7 +7250,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
start = ubuf >> PAGE_SHIFT;
nr_pages = end - start;
- ret = io_account_mem(ctx, nr_pages);
+ ret = io_account_mem(ctx, nr_pages, ACCT_PINNED);
if (ret)
goto err;
@@ -7250,7 +7265,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
GFP_KERNEL);
if (!pages || !vmas) {
ret = -ENOMEM;
- io_unaccount_mem(ctx, nr_pages);
+ io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
goto err;
}
got_pages = nr_pages;
@@ -7260,7 +7275,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
GFP_KERNEL);
ret = -ENOMEM;
if (!imu->bvec) {
- io_unaccount_mem(ctx, nr_pages);
+ io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
goto err;
}
@@ -7291,7 +7306,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
*/
if (pret > 0)
unpin_user_pages(pages, pret);
- io_unaccount_mem(ctx, nr_pages);
+ io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
kvfree(imu->bvec);
goto err;
}
@@ -7398,7 +7413,8 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx)
io_mem_free(ctx->sq_sqes);
percpu_ref_exit(&ctx->refs);
- io_unaccount_mem(ctx, ring_pages(ctx->sq_entries, ctx->cq_entries));
+ io_unaccount_mem(ctx, ring_pages(ctx->sq_entries, ctx->cq_entries),
+ ACCT_LOCKED);
free_uid(ctx->user);
put_cred(ctx->creds);
kfree(ctx->cancel_hash);
@@ -8000,7 +8016,8 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,
goto err;
trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
- io_account_mem(ctx, ring_pages(p->sq_entries, p->cq_entries));
+ io_account_mem(ctx, ring_pages(p->sq_entries, p->cq_entries),
+ ACCT_LOCKED);
ctx->limit_mem = limit_mem;
return ret;
err:
--
1.8.3.1
^ permalink raw reply related [flat|nested] 6+ messages in thread* Re: [PATCH 0/4] io_uring: report locked memory usage
2020-06-16 23:36 [PATCH 0/4] io_uring: report locked memory usage Bijan Mottahedeh
` (3 preceding siblings ...)
2020-06-16 23:36 ` [PATCH 4/4] io_uring: separate reporting of ring pages from registered pages Bijan Mottahedeh
@ 2020-06-17 15:35 ` Jens Axboe
4 siblings, 0 replies; 6+ messages in thread
From: Jens Axboe @ 2020-06-17 15:35 UTC (permalink / raw)
To: Bijan Mottahedeh; +Cc: io-uring
On 6/16/20 5:36 PM, Bijan Mottahedeh wrote:
> This patch set adds support for reporting of locked memory usage.
>
> Patches 1 and 2 are prep patches to facilitate the reporting.
>
> Patch 3 reports all locked memory as pinned.
>
> Patch 4 reports ring memory as locked and registered memory as pinned.
> This seems more appropriate but kept it a separate patch in case it
> should be dropped.
>
> Bijan Mottahedeh (4):
> io_uring: add wrappers for memory accounting
> io_uring: rename ctx->account_mem field
> io_uring: report pinned memory usage
> io_uring: separate reporting of ring pages from registered pages
>
> fs/io_uring.c | 93 ++++++++++++++++++++++++++++++++++++++++++-----------------
> 1 file changed, 66 insertions(+), 27 deletions(-)
Applied for 5.9, thanks.
--
Jens Axboe
^ permalink raw reply [flat|nested] 6+ messages in thread