iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

io_uring: shove more drain bits out of hot path

Browse Source 
Place all drain_next logic into io_drain_req(), so it's never executed
if there was no drained requests before. The only thing we need is to
set ->drain_active if we see a request with IOSQE_IO_DRAIN, do that in
io_init_req() where flags are definitely in registers.

Also, all drain-related code is encapsulated in io_drain_req(), makes it
cleaner.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/68bf4f7395ddaafbf1a26bd97b57d57d45a9f900.1623772051.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2021-06-15 16:47:57 +01:00 committed by Jens Axboe
parent 10c669040e
commit 3c19966d37
1 changed files with 22 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
fs/io_uring.c
View File

@ -5997,11 +5997,31 @@ static u32 io_get_sequence(struct io_kiocb *req)
static bool io_drain_req(struct io_kiocb *req) static bool io_drain_req(struct io_kiocb *req)
{ {
struct io_kiocb *pos;
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
struct io_defer_entry *de; struct io_defer_entry *de;
int ret; int ret;
u32 seq; u32 seq;
/*
* If we need to drain a request in the middle of a link, drain the
* head request and the next request/link after the current link.
* Considering sequential execution of links, IOSQE_IO_DRAIN will be
* maintained for every request of our link.
*/
if (ctx->drain_next) {
req->flags |= REQ_F_IO_DRAIN;
ctx->drain_next = false;
}
/* not interested in head, start from the first linked */
io_for_each_link(pos, req->link) {
if (pos->flags & REQ_F_IO_DRAIN) {
ctx->drain_next = true;
req->flags |= REQ_F_IO_DRAIN;
break;
}
}
/* Still need defer if there is pending req in defer list. */ /* Still need defer if there is pending req in defer list. */
if (likely(list_empty_careful(&ctx->defer_list) && if (likely(list_empty_careful(&ctx->defer_list) &&
!(req->flags & REQ_F_IO_DRAIN))) { !(req->flags & REQ_F_IO_DRAIN))) {
@ -6522,6 +6542,8 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
if ((sqe_flags & IOSQE_BUFFER_SELECT) && if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
!io_op_defs[req->opcode].buffer_select) !io_op_defs[req->opcode].buffer_select)
return -EOPNOTSUPP; return -EOPNOTSUPP;
if (unlikely(sqe_flags & IOSQE_IO_DRAIN))
ctx->drain_active = true;
personality = READ_ONCE(sqe->personality); personality = READ_ONCE(sqe->personality);
if (personality) { if (personality) {
@ -6573,22 +6595,6 @@ fail_req:
return ret; return ret;
} }
if (unlikely(req->flags & REQ_F_IO_DRAIN)) {
ctx->drain_active = true;
/*
* Taking sequential execution of a link, draining both sides
* of the link also fullfils IOSQE_IO_DRAIN semantics for all
* requests in the link. So, it drains the head and the
* next after the link request. The last one is done via
* drain_next flag to persist the effect across calls.
*/
if (link->head) {
link->head->flags |= REQ_F_IO_DRAIN;
ctx->drain_next = 1;
}
}
ret = io_req_prep(req, sqe); ret = io_req_prep(req, sqe);
if (unlikely(ret)) if (unlikely(ret))
goto fail_req; goto fail_req;
@ -6620,10 +6626,6 @@ fail_req:
io_queue_sqe(head); io_queue_sqe(head);
} }
} else { } else {
if (unlikely(ctx->drain_next)) {
req->flags |= REQ_F_IO_DRAIN;
ctx->drain_next = 0;
}
if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) { if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
link->head = req; link->head = req;
link->last = req; link->last = req;