iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

io_uring: extend task put optimisations

Browse Source 
Now with IRQ completions done via IRQ, almost all requests freeing
are done from the context of submitter task, so it makes sense to
extend task_put optimisation from io_req_free_batch_finish() to cover
all the cases including task_work by moving it into io_put_task().

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/824a7cbd745ddeee4a0f3ff85c558a24fd005872.1629302453.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2021-08-18 17:01:43 +01:00 committed by Jens Axboe
parent 316319e82f
commit e98e49b2bb
1 changed files with 9 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
fs/io_uring.c
View File

@ -1623,10 +1623,14 @@ static inline void io_put_task(struct task_struct *task, int nr)
{ {
struct io_uring_task *tctx = task->io_uring; struct io_uring_task *tctx = task->io_uring;
percpu_counter_sub(&tctx->inflight, nr); if (likely(task == current)) {
if (unlikely(atomic_read(&tctx->in_idle))) tctx->cached_refs += nr;
wake_up(&tctx->wait); } else {
put_task_struct_many(task, nr); percpu_counter_sub(&tctx->inflight, nr);
if (unlikely(atomic_read(&tctx->in_idle)))
wake_up(&tctx->wait);
put_task_struct_many(task, nr);
}
} }
static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
@ -2171,9 +2175,7 @@ static void io_req_free_batch_finish(struct io_ring_ctx *ctx,
{ {
if (rb->ctx_refs) if (rb->ctx_refs)
percpu_ref_put_many(&ctx->refs, rb->ctx_refs); percpu_ref_put_many(&ctx->refs, rb->ctx_refs);
if (rb->task == current) if (rb->task)
current->io_uring->cached_refs += rb->task_refs;
else if (rb->task)
io_put_task(rb->task, rb->task_refs); io_put_task(rb->task, rb->task_refs);
} }