iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

io_uring: remove drain_active check from hot path

Browse Source 
req->ctx->active_drain is a bit too expensive, partially because of two
dereferences. Do a trick, if we see it set in io_init_req(), set
REQ_F_FORCE_ASYNC and it automatically goes through a slower path where
we can catch it. It's nearly free to do in io_init_req() because there
is already ->restricted check and it's in the same byte of a bitmask.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/d7e7ddc63c15e8a300833132abb3eb8fd3918aef.1632516769.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2021-09-24 21:59:57 +01:00 committed by Jens Axboe
parent f15a343177
commit 2a56a9bd64
1 changed files with 29 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
fs/io_uring.c
View File

@ -6438,23 +6438,15 @@ static bool io_drain_req(struct io_kiocb *req)
int ret; int ret;
u32 seq; u32 seq;
if (req->flags & REQ_F_FAIL) {
io_req_complete_fail_submit(req);
return true;
}
/*
* If we need to drain a request in the middle of a link, drain the
* head request and the next request/link after the current link.
* Considering sequential execution of links, IOSQE_IO_DRAIN will be
* maintained for every request of our link.
*/
if (ctx->drain_next) {
req->flags |= REQ_F_IO_DRAIN;
ctx->drain_next = false;
}
/* not interested in head, start from the first linked */ /* not interested in head, start from the first linked */
io_for_each_link(pos, req->link) { io_for_each_link(pos, req->link) {
/*
* If we need to drain a request in the middle of a link, drain
* the head request and the next request/link after the current
* link. Considering sequential execution of links,
* IOSQE_IO_DRAIN will be maintained for every request of our
* link.
*/
if (pos->flags & REQ_F_IO_DRAIN) { if (pos->flags & REQ_F_IO_DRAIN) {
ctx->drain_next = true; ctx->drain_next = true;
req->flags |= REQ_F_IO_DRAIN; req->flags |= REQ_F_IO_DRAIN;
@ -6946,13 +6938,12 @@ issue_sqe:
static inline void io_queue_sqe(struct io_kiocb *req) static inline void io_queue_sqe(struct io_kiocb *req)
__must_hold(&req->ctx->uring_lock) __must_hold(&req->ctx->uring_lock)
{ {
if (unlikely(req->ctx->drain_active) && io_drain_req(req))
return;
if (likely(!(req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL)))) { if (likely(!(req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL)))) {
__io_queue_sqe(req); __io_queue_sqe(req);
} else if (req->flags & REQ_F_FAIL) { } else if (req->flags & REQ_F_FAIL) {
io_req_complete_fail_submit(req); io_req_complete_fail_submit(req);
} else if (unlikely(req->ctx->drain_active) && io_drain_req(req)) {
return;
} else { } else {
int ret = io_req_prep_async(req); int ret = io_req_prep_async(req);
@ -6972,9 +6963,6 @@ static inline bool io_check_restriction(struct io_ring_ctx *ctx,
struct io_kiocb *req, struct io_kiocb *req,
unsigned int sqe_flags) unsigned int sqe_flags)
{ {
if (likely(!ctx->restricted))
return true;
if (!test_bit(req->opcode, ctx->restrictions.sqe_op)) if (!test_bit(req->opcode, ctx->restrictions.sqe_op))
return false; return false;
@ -7015,11 +7003,28 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
if ((sqe_flags & IOSQE_BUFFER_SELECT) && if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
!io_op_defs[req->opcode].buffer_select) !io_op_defs[req->opcode].buffer_select)
return -EOPNOTSUPP; return -EOPNOTSUPP;
if (sqe_flags & IOSQE_IO_DRAIN) if (sqe_flags & IOSQE_IO_DRAIN) {
struct io_submit_link *link = &ctx->submit_state.link;
ctx->drain_active = true; ctx->drain_active = true;
req->flags |= REQ_F_FORCE_ASYNC;
if (link->head)
link->head->flags |= IOSQE_IO_DRAIN | REQ_F_FORCE_ASYNC;
} }
if (!io_check_restriction(ctx, req, sqe_flags)) }
if (unlikely(ctx->restricted || ctx->drain_active || ctx->drain_next)) {
if (ctx->restricted && !io_check_restriction(ctx, req, sqe_flags))
return -EACCES; return -EACCES;
/* knock it to the slow queue path, will be drained there */
if (ctx->drain_active)
req->flags |= REQ_F_FORCE_ASYNC;
/* if there is no link, we're at "next" request and need to drain */
if (unlikely(ctx->drain_next) && !ctx->submit_state.link.head) {
ctx->drain_next = false;
ctx->drain_active = true;
req->flags |= REQ_F_FORCE_ASYNC | IOSQE_IO_DRAIN;
}
}
personality = READ_ONCE(sqe->personality); personality = READ_ONCE(sqe->personality);
if (personality) { if (personality) {