diff --git a/fs/io_uring.c b/fs/io_uring.c index 128ffa79d9d3..6d97767c82d5 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6625,6 +6625,23 @@ static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx) spin_unlock_irq(&ctx->completion_lock); } +static int io_sq_wake_function(struct wait_queue_entry *wqe, unsigned mode, + int sync, void *key) +{ + struct io_ring_ctx *ctx = container_of(wqe, struct io_ring_ctx, sqo_wait_entry); + int ret; + + ret = autoremove_wake_function(wqe, mode, sync, key); + if (ret) { + unsigned long flags; + + spin_lock_irqsave(&ctx->completion_lock, flags); + ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP; + spin_unlock_irqrestore(&ctx->completion_lock, flags); + } + return ret; +} + static int io_sq_thread(void *data) { struct io_ring_ctx *ctx = data; @@ -6633,6 +6650,7 @@ static int io_sq_thread(void *data) int ret = 0; init_wait(&ctx->sqo_wait_entry); + ctx->sqo_wait_entry.func = io_sq_wake_function; complete(&ctx->sq_thread_comp); @@ -6715,7 +6733,6 @@ static int io_sq_thread(void *data) schedule(); finish_wait(ctx->sqo_wait, &ctx->sqo_wait_entry); - io_ring_clear_wakeup_flag(ctx); ret = 0; continue; }