2019-04-30 21:42:43 +03:00
// SPDX-License-Identifier: GPL-2.0
2017-01-17 16:03:22 +03:00
/*
* blk - mq scheduling framework
*
* Copyright ( C ) 2016 Jens Axboe
*/
# include <linux/kernel.h>
# include <linux/module.h>
# include <linux/blk-mq.h>
# include <trace/events/block.h>
# include "blk.h"
# include "blk-mq.h"
2017-05-04 17:24:40 +03:00
# include "blk-mq-debugfs.h"
2017-01-17 16:03:22 +03:00
# include "blk-mq-sched.h"
# include "blk-mq-tag.h"
# include "blk-wbt.h"
void blk_mq_sched_free_hctx_data ( struct request_queue * q ,
void ( * exit ) ( struct blk_mq_hw_ctx * ) )
{
struct blk_mq_hw_ctx * hctx ;
int i ;
queue_for_each_hw_ctx ( q , hctx , i ) {
if ( exit & & hctx - > sched_data )
exit ( hctx ) ;
kfree ( hctx - > sched_data ) ;
hctx - > sched_data = NULL ;
}
}
EXPORT_SYMBOL_GPL ( blk_mq_sched_free_hctx_data ) ;
2018-11-20 04:52:34 +03:00
void blk_mq_sched_assign_ioc ( struct request * rq )
2017-01-17 16:03:22 +03:00
{
2017-06-16 19:15:25 +03:00
struct request_queue * q = rq - > q ;
2018-11-21 05:12:46 +03:00
struct io_context * ioc ;
2017-01-17 16:03:22 +03:00
struct io_cq * icq ;
2018-11-21 05:12:46 +03:00
/*
* May not have an IO context if it ' s a passthrough request
*/
ioc = current - > io_context ;
if ( ! ioc )
return ;
2018-11-15 22:17:28 +03:00
spin_lock_irq ( & q - > queue_lock ) ;
2017-01-17 16:03:22 +03:00
icq = ioc_lookup_icq ( ioc , q ) ;
2018-11-15 22:17:28 +03:00
spin_unlock_irq ( & q - > queue_lock ) ;
2017-01-17 16:03:22 +03:00
if ( ! icq ) {
icq = ioc_create_icq ( ioc , q , GFP_ATOMIC ) ;
if ( ! icq )
return ;
}
2017-06-16 19:15:20 +03:00
get_io_context ( icq - > ioc ) ;
2017-06-16 19:15:25 +03:00
rq - > elv . icq = icq ;
2017-01-17 16:03:22 +03:00
}
2017-06-21 02:56:13 +03:00
/*
* Mark a hardware queue as needing a restart . For shared queues , maintain
* a count of how many hardware queues are marked for restart .
*/
2018-12-17 09:14:05 +03:00
void blk_mq_sched_mark_restart_hctx ( struct blk_mq_hw_ctx * hctx )
2017-06-21 02:56:13 +03:00
{
if ( test_bit ( BLK_MQ_S_SCHED_RESTART , & hctx - > state ) )
return ;
blk-mq: remove synchronize_rcu() from blk_mq_del_queue_tag_set()
We have to remove synchronize_rcu() from blk_queue_cleanup(),
otherwise long delay can be caused during lun probe. For removing
it, we have to avoid to iterate the set->tag_list in IO path, eg,
blk_mq_sched_restart().
This patch reverts 5b79413946d (Revert "blk-mq: don't handle
TAG_SHARED in restart"). Given we have fixed enough IO hang issue,
and there isn't any reason to restart all queues in one tags any more,
see the following reasons:
1) blk-mq core can deal with shared-tags case well via blk_mq_get_driver_tag(),
which can wake up queues waiting for driver tag.
2) SCSI is a bit special because it may return BLK_STS_RESOURCE if queue,
target or host is ready, but SCSI built-in restart can cover all these well,
see scsi_end_request(), queue will be rerun after any request initiated from
this host/target is completed.
In my test on scsi_debug(8 luns), this patch may improve IOPS by 20% ~ 30%
when running I/O on these 8 luns concurrently.
Fixes: 705cda97ee3a ("blk-mq: Make it safe to use RCU to iterate over blk_mq_tag_set.tag_list")
Cc: Omar Sandoval <osandov@fb.com>
Cc: Bart Van Assche <bart.vanassche@wdc.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Martin K. Petersen <martin.petersen@oracle.com>
Cc: linux-scsi@vger.kernel.org
Reported-by: Andrew Jones <drjones@redhat.com>
Tested-by: Andrew Jones <drjones@redhat.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2018-06-25 14:31:48 +03:00
set_bit ( BLK_MQ_S_SCHED_RESTART , & hctx - > state ) ;
2017-06-21 02:56:13 +03:00
}
2018-12-17 09:14:05 +03:00
EXPORT_SYMBOL_GPL ( blk_mq_sched_mark_restart_hctx ) ;
2017-06-21 02:56:13 +03:00
blk-mq: remove synchronize_rcu() from blk_mq_del_queue_tag_set()
We have to remove synchronize_rcu() from blk_queue_cleanup(),
otherwise long delay can be caused during lun probe. For removing
it, we have to avoid to iterate the set->tag_list in IO path, eg,
blk_mq_sched_restart().
This patch reverts 5b79413946d (Revert "blk-mq: don't handle
TAG_SHARED in restart"). Given we have fixed enough IO hang issue,
and there isn't any reason to restart all queues in one tags any more,
see the following reasons:
1) blk-mq core can deal with shared-tags case well via blk_mq_get_driver_tag(),
which can wake up queues waiting for driver tag.
2) SCSI is a bit special because it may return BLK_STS_RESOURCE if queue,
target or host is ready, but SCSI built-in restart can cover all these well,
see scsi_end_request(), queue will be rerun after any request initiated from
this host/target is completed.
In my test on scsi_debug(8 luns), this patch may improve IOPS by 20% ~ 30%
when running I/O on these 8 luns concurrently.
Fixes: 705cda97ee3a ("blk-mq: Make it safe to use RCU to iterate over blk_mq_tag_set.tag_list")
Cc: Omar Sandoval <osandov@fb.com>
Cc: Bart Van Assche <bart.vanassche@wdc.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Martin K. Petersen <martin.petersen@oracle.com>
Cc: linux-scsi@vger.kernel.org
Reported-by: Andrew Jones <drjones@redhat.com>
Tested-by: Andrew Jones <drjones@redhat.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2018-06-25 14:31:48 +03:00
void blk_mq_sched_restart ( struct blk_mq_hw_ctx * hctx )
2017-06-21 02:56:13 +03:00
{
if ( ! test_bit ( BLK_MQ_S_SCHED_RESTART , & hctx - > state ) )
blk-mq: remove synchronize_rcu() from blk_mq_del_queue_tag_set()
We have to remove synchronize_rcu() from blk_queue_cleanup(),
otherwise long delay can be caused during lun probe. For removing
it, we have to avoid to iterate the set->tag_list in IO path, eg,
blk_mq_sched_restart().
This patch reverts 5b79413946d (Revert "blk-mq: don't handle
TAG_SHARED in restart"). Given we have fixed enough IO hang issue,
and there isn't any reason to restart all queues in one tags any more,
see the following reasons:
1) blk-mq core can deal with shared-tags case well via blk_mq_get_driver_tag(),
which can wake up queues waiting for driver tag.
2) SCSI is a bit special because it may return BLK_STS_RESOURCE if queue,
target or host is ready, but SCSI built-in restart can cover all these well,
see scsi_end_request(), queue will be rerun after any request initiated from
this host/target is completed.
In my test on scsi_debug(8 luns), this patch may improve IOPS by 20% ~ 30%
when running I/O on these 8 luns concurrently.
Fixes: 705cda97ee3a ("blk-mq: Make it safe to use RCU to iterate over blk_mq_tag_set.tag_list")
Cc: Omar Sandoval <osandov@fb.com>
Cc: Bart Van Assche <bart.vanassche@wdc.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Martin K. Petersen <martin.petersen@oracle.com>
Cc: linux-scsi@vger.kernel.org
Reported-by: Andrew Jones <drjones@redhat.com>
Tested-by: Andrew Jones <drjones@redhat.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2018-06-25 14:31:48 +03:00
return ;
clear_bit ( BLK_MQ_S_SCHED_RESTART , & hctx - > state ) ;
2017-06-21 02:56:13 +03:00
blk-mq: remove synchronize_rcu() from blk_mq_del_queue_tag_set()
We have to remove synchronize_rcu() from blk_queue_cleanup(),
otherwise long delay can be caused during lun probe. For removing
it, we have to avoid to iterate the set->tag_list in IO path, eg,
blk_mq_sched_restart().
This patch reverts 5b79413946d (Revert "blk-mq: don't handle
TAG_SHARED in restart"). Given we have fixed enough IO hang issue,
and there isn't any reason to restart all queues in one tags any more,
see the following reasons:
1) blk-mq core can deal with shared-tags case well via blk_mq_get_driver_tag(),
which can wake up queues waiting for driver tag.
2) SCSI is a bit special because it may return BLK_STS_RESOURCE if queue,
target or host is ready, but SCSI built-in restart can cover all these well,
see scsi_end_request(), queue will be rerun after any request initiated from
this host/target is completed.
In my test on scsi_debug(8 luns), this patch may improve IOPS by 20% ~ 30%
when running I/O on these 8 luns concurrently.
Fixes: 705cda97ee3a ("blk-mq: Make it safe to use RCU to iterate over blk_mq_tag_set.tag_list")
Cc: Omar Sandoval <osandov@fb.com>
Cc: Bart Van Assche <bart.vanassche@wdc.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Martin K. Petersen <martin.petersen@oracle.com>
Cc: linux-scsi@vger.kernel.org
Reported-by: Andrew Jones <drjones@redhat.com>
Tested-by: Andrew Jones <drjones@redhat.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2018-06-25 14:31:48 +03:00
blk_mq_run_hw_queue ( hctx , true ) ;
2017-06-21 02:56:13 +03:00
}
blk-mq: Rerun dispatching in the case of budget contention
If ever a thread running blk-mq code tries to get budget and fails it
immediately stops doing work and assumes that whenever budget is freed
up that queues will be kicked and whatever work the thread was trying
to do will be tried again.
One path where budget is freed and queues are kicked in the normal
case can be seen in scsi_finish_command(). Specifically:
- scsi_finish_command()
- scsi_device_unbusy()
- # Decrement "device_busy", AKA release budget
- scsi_io_completion()
- scsi_end_request()
- blk_mq_run_hw_queues()
The above is all well and good. The problem comes up when a thread
claims the budget but then releases it without actually dispatching
any work. Since we didn't schedule any work we'll never run the path
of finishing work / kicking the queues.
This isn't often actually a problem which is why this issue has
existed for a while and nobody noticed. Specifically we only get into
this situation when we unexpectedly found that we weren't going to do
any work. Code that later receives new work kicks the queues. All
good, right?
The problem shows up, however, if timing is just wrong and we hit a
race. To see this race let's think about the case where we only have
a budget of 1 (only one thread can hold budget). Now imagine that a
thread got budget and then decided not to dispatch work. It's about
to call put_budget() but then the thread gets context switched out for
a long, long time. While in this state, any and all kicks of the
queue (like the when we received new work) will be no-ops because
nobody can get budget. Finally the thread holding budget gets to run
again and returns. All the normal kicks will have been no-ops and we
have an I/O stall.
As you can see from the above, you need just the right timing to see
the race. To start with, the only case it happens if we thought we
had work, actually managed to get the budget, but then actually didn't
have work. That's pretty rare to start with. Even then, there's
usually a very small amount of time between realizing that there's no
work and putting the budget. During this small amount of time new
work has to come in and the queue kick has to make it all the way to
trying to get the budget and fail. It's pretty unlikely.
One case where this could have failed is illustrated by an example of
threads running blk_mq_do_dispatch_sched():
* Threads A and B both run has_work() at the same time with the same
"hctx". Imagine has_work() is exact. There's no lock, so it's OK
if Thread A and B both get back true.
* Thread B gets interrupted for a long time right after it decides
that there is work. Maybe its CPU gets an interrupt and the
interrupt handler is slow.
* Thread A runs, get budget, dispatches work.
* Thread A's work finishes and budget is released.
* Thread B finally runs again and gets budget.
* Since Thread A already took care of the work and no new work has
come in, Thread B will get NULL from dispatch_request(). I believe
this is specifically why dispatch_request() is allowed to return
NULL in the first place if has_work() must be exact.
* Thread B will now be holding the budget and is about to call
put_budget(), but hasn't called it yet.
* Thread B gets interrupted for a long time (again). Dang interrupts.
* Now Thread C (maybe with a different "hctx" but the same queue)
comes along and runs blk_mq_do_dispatch_sched().
* Thread C won't do anything because it can't get budget.
* Finally Thread B will run again and put the budget without kicking
any queues.
Even though the example above is with blk_mq_do_dispatch_sched() I
believe the race is possible any time someone is holding budget but
doesn't do work.
Unfortunately, the unlikely has become more likely if you happen to be
using the BFQ I/O scheduler. BFQ, by design, sometimes returns "true"
for has_work() but then NULL for dispatch_request() and stays in this
state for a while (currently up to 9 ms). Suddenly you only need one
race to hit, not two races in a row. With my current setup this is
easy to reproduce in reboot tests and traces have actually shown that
we hit a race similar to the one described above.
Note that we only need to fix blk_mq_do_dispatch_sched() and
blk_mq_do_dispatch_ctx() and not the other places that put budget. In
other cases we know that we have work to do on at least one "hctx" and
code already exists to kick that "hctx"'s queue. When that work
finally finishes all the queues will be kicked using the normal flow.
One last note is that (at least in the SCSI case) budget is shared by
all "hctx"s that have the same queue. Thus we need to make sure to
kick the whole queue, not just re-run dispatching on a single "hctx".
Signed-off-by: Douglas Anderson <dianders@chromium.org>
Reviewed-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2020-04-20 19:24:53 +03:00
# define BLK_MQ_BUDGET_DELAY 3 /* ms units */
2017-10-27 07:43:30 +03:00
/*
* Only SCSI implements . get_budget and . put_budget , and SCSI restarts
* its queue by itself in its completion handler , so we don ' t need to
* restart queue if . get_budget ( ) returns BLK_STS_NO_RESOURCE .
*/
static void blk_mq_do_dispatch_sched ( struct blk_mq_hw_ctx * hctx )
2017-10-14 12:22:26 +03:00
{
struct request_queue * q = hctx - > queue ;
struct elevator_queue * e = q - > elevator ;
LIST_HEAD ( rq_list ) ;
do {
2017-10-14 12:22:29 +03:00
struct request * rq ;
2017-10-14 12:22:26 +03:00
2018-11-02 01:41:41 +03:00
if ( e - > type - > ops . has_work & & ! e - > type - > ops . has_work ( hctx ) )
2017-10-14 12:22:26 +03:00
break ;
2017-10-14 12:22:29 +03:00
2017-11-04 21:21:12 +03:00
if ( ! blk_mq_get_dispatch_budget ( hctx ) )
2017-10-27 07:43:30 +03:00
break ;
2017-10-14 12:22:29 +03:00
2018-11-02 01:41:41 +03:00
rq = e - > type - > ops . dispatch_request ( hctx ) ;
2017-10-14 12:22:29 +03:00
if ( ! rq ) {
blk_mq_put_dispatch_budget ( hctx ) ;
blk-mq: Rerun dispatching in the case of budget contention
If ever a thread running blk-mq code tries to get budget and fails it
immediately stops doing work and assumes that whenever budget is freed
up that queues will be kicked and whatever work the thread was trying
to do will be tried again.
One path where budget is freed and queues are kicked in the normal
case can be seen in scsi_finish_command(). Specifically:
- scsi_finish_command()
- scsi_device_unbusy()
- # Decrement "device_busy", AKA release budget
- scsi_io_completion()
- scsi_end_request()
- blk_mq_run_hw_queues()
The above is all well and good. The problem comes up when a thread
claims the budget but then releases it without actually dispatching
any work. Since we didn't schedule any work we'll never run the path
of finishing work / kicking the queues.
This isn't often actually a problem which is why this issue has
existed for a while and nobody noticed. Specifically we only get into
this situation when we unexpectedly found that we weren't going to do
any work. Code that later receives new work kicks the queues. All
good, right?
The problem shows up, however, if timing is just wrong and we hit a
race. To see this race let's think about the case where we only have
a budget of 1 (only one thread can hold budget). Now imagine that a
thread got budget and then decided not to dispatch work. It's about
to call put_budget() but then the thread gets context switched out for
a long, long time. While in this state, any and all kicks of the
queue (like the when we received new work) will be no-ops because
nobody can get budget. Finally the thread holding budget gets to run
again and returns. All the normal kicks will have been no-ops and we
have an I/O stall.
As you can see from the above, you need just the right timing to see
the race. To start with, the only case it happens if we thought we
had work, actually managed to get the budget, but then actually didn't
have work. That's pretty rare to start with. Even then, there's
usually a very small amount of time between realizing that there's no
work and putting the budget. During this small amount of time new
work has to come in and the queue kick has to make it all the way to
trying to get the budget and fail. It's pretty unlikely.
One case where this could have failed is illustrated by an example of
threads running blk_mq_do_dispatch_sched():
* Threads A and B both run has_work() at the same time with the same
"hctx". Imagine has_work() is exact. There's no lock, so it's OK
if Thread A and B both get back true.
* Thread B gets interrupted for a long time right after it decides
that there is work. Maybe its CPU gets an interrupt and the
interrupt handler is slow.
* Thread A runs, get budget, dispatches work.
* Thread A's work finishes and budget is released.
* Thread B finally runs again and gets budget.
* Since Thread A already took care of the work and no new work has
come in, Thread B will get NULL from dispatch_request(). I believe
this is specifically why dispatch_request() is allowed to return
NULL in the first place if has_work() must be exact.
* Thread B will now be holding the budget and is about to call
put_budget(), but hasn't called it yet.
* Thread B gets interrupted for a long time (again). Dang interrupts.
* Now Thread C (maybe with a different "hctx" but the same queue)
comes along and runs blk_mq_do_dispatch_sched().
* Thread C won't do anything because it can't get budget.
* Finally Thread B will run again and put the budget without kicking
any queues.
Even though the example above is with blk_mq_do_dispatch_sched() I
believe the race is possible any time someone is holding budget but
doesn't do work.
Unfortunately, the unlikely has become more likely if you happen to be
using the BFQ I/O scheduler. BFQ, by design, sometimes returns "true"
for has_work() but then NULL for dispatch_request() and stays in this
state for a while (currently up to 9 ms). Suddenly you only need one
race to hit, not two races in a row. With my current setup this is
easy to reproduce in reboot tests and traces have actually shown that
we hit a race similar to the one described above.
Note that we only need to fix blk_mq_do_dispatch_sched() and
blk_mq_do_dispatch_ctx() and not the other places that put budget. In
other cases we know that we have work to do on at least one "hctx" and
code already exists to kick that "hctx"'s queue. When that work
finally finishes all the queues will be kicked using the normal flow.
One last note is that (at least in the SCSI case) budget is shared by
all "hctx"s that have the same queue. Thus we need to make sure to
kick the whole queue, not just re-run dispatching on a single "hctx".
Signed-off-by: Douglas Anderson <dianders@chromium.org>
Reviewed-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2020-04-20 19:24:53 +03:00
/*
* We ' re releasing without dispatching . Holding the
* budget could have blocked any " hctx " s with the
* same queue and if we didn ' t dispatch then there ' s
* no guarantee anyone will kick the queue . Kick it
* ourselves .
*/
blk_mq_delay_run_hw_queues ( q , BLK_MQ_BUDGET_DELAY ) ;
2017-10-14 12:22:29 +03:00
break ;
}
/*
* Now this rq owns the budget which has to be released
* if this rq won ' t be queued to driver via . queue_rq ( )
* in blk_mq_dispatch_rq_list ( ) .
*/
2017-10-14 12:22:26 +03:00
list_add ( & rq - > queuelist , & rq_list ) ;
2017-10-14 12:22:29 +03:00
} while ( blk_mq_dispatch_rq_list ( q , & rq_list , true ) ) ;
2017-10-14 12:22:26 +03:00
}
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 12:22:30 +03:00
static struct blk_mq_ctx * blk_mq_next_ctx ( struct blk_mq_hw_ctx * hctx ,
struct blk_mq_ctx * ctx )
{
2018-10-29 22:13:29 +03:00
unsigned short idx = ctx - > index_hw [ hctx - > type ] ;
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 12:22:30 +03:00
if ( + + idx = = hctx - > nr_ctx )
idx = 0 ;
return hctx - > ctxs [ idx ] ;
}
2017-10-27 07:43:30 +03:00
/*
* Only SCSI implements . get_budget and . put_budget , and SCSI restarts
* its queue by itself in its completion handler , so we don ' t need to
* restart queue if . get_budget ( ) returns BLK_STS_NO_RESOURCE .
*/
static void blk_mq_do_dispatch_ctx ( struct blk_mq_hw_ctx * hctx )
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 12:22:30 +03:00
{
struct request_queue * q = hctx - > queue ;
LIST_HEAD ( rq_list ) ;
struct blk_mq_ctx * ctx = READ_ONCE ( hctx - > dispatch_from ) ;
do {
struct request * rq ;
if ( ! sbitmap_any_bit_set ( & hctx - > ctx_map ) )
break ;
2017-11-04 21:21:12 +03:00
if ( ! blk_mq_get_dispatch_budget ( hctx ) )
2017-10-27 07:43:30 +03:00
break ;
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 12:22:30 +03:00
rq = blk_mq_dequeue_from_ctx ( hctx , ctx ) ;
if ( ! rq ) {
blk_mq_put_dispatch_budget ( hctx ) ;
blk-mq: Rerun dispatching in the case of budget contention
If ever a thread running blk-mq code tries to get budget and fails it
immediately stops doing work and assumes that whenever budget is freed
up that queues will be kicked and whatever work the thread was trying
to do will be tried again.
One path where budget is freed and queues are kicked in the normal
case can be seen in scsi_finish_command(). Specifically:
- scsi_finish_command()
- scsi_device_unbusy()
- # Decrement "device_busy", AKA release budget
- scsi_io_completion()
- scsi_end_request()
- blk_mq_run_hw_queues()
The above is all well and good. The problem comes up when a thread
claims the budget but then releases it without actually dispatching
any work. Since we didn't schedule any work we'll never run the path
of finishing work / kicking the queues.
This isn't often actually a problem which is why this issue has
existed for a while and nobody noticed. Specifically we only get into
this situation when we unexpectedly found that we weren't going to do
any work. Code that later receives new work kicks the queues. All
good, right?
The problem shows up, however, if timing is just wrong and we hit a
race. To see this race let's think about the case where we only have
a budget of 1 (only one thread can hold budget). Now imagine that a
thread got budget and then decided not to dispatch work. It's about
to call put_budget() but then the thread gets context switched out for
a long, long time. While in this state, any and all kicks of the
queue (like the when we received new work) will be no-ops because
nobody can get budget. Finally the thread holding budget gets to run
again and returns. All the normal kicks will have been no-ops and we
have an I/O stall.
As you can see from the above, you need just the right timing to see
the race. To start with, the only case it happens if we thought we
had work, actually managed to get the budget, but then actually didn't
have work. That's pretty rare to start with. Even then, there's
usually a very small amount of time between realizing that there's no
work and putting the budget. During this small amount of time new
work has to come in and the queue kick has to make it all the way to
trying to get the budget and fail. It's pretty unlikely.
One case where this could have failed is illustrated by an example of
threads running blk_mq_do_dispatch_sched():
* Threads A and B both run has_work() at the same time with the same
"hctx". Imagine has_work() is exact. There's no lock, so it's OK
if Thread A and B both get back true.
* Thread B gets interrupted for a long time right after it decides
that there is work. Maybe its CPU gets an interrupt and the
interrupt handler is slow.
* Thread A runs, get budget, dispatches work.
* Thread A's work finishes and budget is released.
* Thread B finally runs again and gets budget.
* Since Thread A already took care of the work and no new work has
come in, Thread B will get NULL from dispatch_request(). I believe
this is specifically why dispatch_request() is allowed to return
NULL in the first place if has_work() must be exact.
* Thread B will now be holding the budget and is about to call
put_budget(), but hasn't called it yet.
* Thread B gets interrupted for a long time (again). Dang interrupts.
* Now Thread C (maybe with a different "hctx" but the same queue)
comes along and runs blk_mq_do_dispatch_sched().
* Thread C won't do anything because it can't get budget.
* Finally Thread B will run again and put the budget without kicking
any queues.
Even though the example above is with blk_mq_do_dispatch_sched() I
believe the race is possible any time someone is holding budget but
doesn't do work.
Unfortunately, the unlikely has become more likely if you happen to be
using the BFQ I/O scheduler. BFQ, by design, sometimes returns "true"
for has_work() but then NULL for dispatch_request() and stays in this
state for a while (currently up to 9 ms). Suddenly you only need one
race to hit, not two races in a row. With my current setup this is
easy to reproduce in reboot tests and traces have actually shown that
we hit a race similar to the one described above.
Note that we only need to fix blk_mq_do_dispatch_sched() and
blk_mq_do_dispatch_ctx() and not the other places that put budget. In
other cases we know that we have work to do on at least one "hctx" and
code already exists to kick that "hctx"'s queue. When that work
finally finishes all the queues will be kicked using the normal flow.
One last note is that (at least in the SCSI case) budget is shared by
all "hctx"s that have the same queue. Thus we need to make sure to
kick the whole queue, not just re-run dispatching on a single "hctx".
Signed-off-by: Douglas Anderson <dianders@chromium.org>
Reviewed-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2020-04-20 19:24:53 +03:00
/*
* We ' re releasing without dispatching . Holding the
* budget could have blocked any " hctx " s with the
* same queue and if we didn ' t dispatch then there ' s
* no guarantee anyone will kick the queue . Kick it
* ourselves .
*/
blk_mq_delay_run_hw_queues ( q , BLK_MQ_BUDGET_DELAY ) ;
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 12:22:30 +03:00
break ;
}
/*
* Now this rq owns the budget which has to be released
* if this rq won ' t be queued to driver via . queue_rq ( )
* in blk_mq_dispatch_rq_list ( ) .
*/
list_add ( & rq - > queuelist , & rq_list ) ;
/* round robin for fair dispatch */
ctx = blk_mq_next_ctx ( hctx , rq - > mq_ctx ) ;
} while ( blk_mq_dispatch_rq_list ( q , & rq_list , true ) ) ;
WRITE_ONCE ( hctx - > dispatch_from , ctx ) ;
}
2017-10-27 07:43:30 +03:00
void blk_mq_sched_dispatch_requests ( struct blk_mq_hw_ctx * hctx )
2017-01-17 16:03:22 +03:00
{
blk-mq: use the right hctx when getting a driver tag fails
While dispatching requests, if we fail to get a driver tag, we mark the
hardware queue as waiting for a tag and put the requests on a
hctx->dispatch list to be run later when a driver tag is freed. However,
blk_mq_dispatch_rq_list() may dispatch requests from multiple hardware
queues if using a single-queue scheduler with a multiqueue device. If
blk_mq_get_driver_tag() fails, it doesn't update the hardware queue we
are processing. This means we end up using the hardware queue of the
previous request, which may or may not be the same as that of the
current request. If it isn't, the wrong hardware queue will end up
waiting for a tag, and the requests will be on the wrong dispatch list,
leading to a hang.
The fix is twofold:
1. Make sure we save which hardware queue we were trying to get a
request for in blk_mq_get_driver_tag() regardless of whether it
succeeds or not.
2. Make blk_mq_dispatch_rq_list() take a request_queue instead of a
blk_mq_hw_queue to make it clear that it must handle multiple
hardware queues, since I've already messed this up on a couple of
occasions.
This didn't appear in testing with nvme and mq-deadline because nvme has
more driver tags than the default number of scheduler tags. However,
with the blk_mq_update_nr_hw_queues() fix, it showed up with nbd.
Signed-off-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2017-04-07 17:56:26 +03:00
struct request_queue * q = hctx - > queue ;
struct elevator_queue * e = q - > elevator ;
2018-11-02 01:41:41 +03:00
const bool has_sched_dispatch = e & & e - > type - > ops . dispatch_request ;
2017-01-17 16:03:22 +03:00
LIST_HEAD ( rq_list ) ;
2017-06-18 23:24:27 +03:00
/* RCU or SRCU read lock is needed before checking quiesced flag */
if ( unlikely ( blk_mq_hctx_stopped ( hctx ) | | blk_queue_quiesced ( q ) ) )
2017-10-27 07:43:30 +03:00
return ;
2017-01-17 16:03:22 +03:00
hctx - > run + + ;
/*
* If we have previous entries on our dispatch list , grab them first for
* more fair dispatch .
*/
if ( ! list_empty_careful ( & hctx - > dispatch ) ) {
spin_lock ( & hctx - > lock ) ;
if ( ! list_empty ( & hctx - > dispatch ) )
list_splice_init ( & hctx - > dispatch , & rq_list ) ;
spin_unlock ( & hctx - > lock ) ;
}
/*
* Only ask the scheduler for requests , if we didn ' t have residual
* requests from the dispatch list . This is to avoid the case where
* we only ever dispatch a fraction of the requests available because
* of low device queue depth . Once we pull requests out of the IO
* scheduler , we can no longer merge or sort them . So it ' s best to
* leave them there for as long as we can . Mark the hw queue as
* needing a restart in that case .
2017-10-14 12:22:26 +03:00
*
* We want to dispatch from the scheduler if there was nothing
* on the dispatch list or we were able to dispatch from the
* dispatch list .
2017-01-17 16:03:22 +03:00
*/
2017-01-26 22:40:07 +03:00
if ( ! list_empty ( & rq_list ) ) {
2017-02-22 21:58:30 +03:00
blk_mq_sched_mark_restart_hctx ( hctx ) ;
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 12:22:30 +03:00
if ( blk_mq_dispatch_rq_list ( q , & rq_list , false ) ) {
if ( has_sched_dispatch )
2017-10-27 07:43:30 +03:00
blk_mq_do_dispatch_sched ( hctx ) ;
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 12:22:30 +03:00
else
2017-10-27 07:43:30 +03:00
blk_mq_do_dispatch_ctx ( hctx ) ;
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 12:22:30 +03:00
}
2017-10-14 12:22:26 +03:00
} else if ( has_sched_dispatch ) {
2017-10-27 07:43:30 +03:00
blk_mq_do_dispatch_sched ( hctx ) ;
2018-07-03 18:03:16 +03:00
} else if ( hctx - > dispatch_busy ) {
/* dequeue request one by one from sw queue if queue is busy */
2017-10-27 07:43:30 +03:00
blk_mq_do_dispatch_ctx ( hctx ) ;
2017-10-14 12:22:26 +03:00
} else {
2017-01-26 22:40:07 +03:00
blk_mq_flush_busy_ctxs ( hctx , & rq_list ) ;
2017-10-14 12:22:29 +03:00
blk_mq_dispatch_rq_list ( q , & rq_list , false ) ;
2017-02-17 21:39:26 +03:00
}
2017-01-17 16:03:22 +03:00
}
2017-02-03 19:48:28 +03:00
bool blk_mq_sched_try_merge ( struct request_queue * q , struct bio * bio ,
2019-06-06 13:29:01 +03:00
unsigned int nr_segs , struct request * * merged_request )
2017-01-17 16:03:22 +03:00
{
struct request * rq ;
2017-02-08 16:46:48 +03:00
switch ( elv_merge ( q , & rq , bio ) ) {
case ELEVATOR_BACK_MERGE :
2017-01-17 16:03:22 +03:00
if ( ! blk_mq_sched_allow_merge ( q , rq , bio ) )
return false ;
2019-06-06 13:29:01 +03:00
if ( ! bio_attempt_back_merge ( rq , bio , nr_segs ) )
2017-02-08 16:46:48 +03:00
return false ;
* merged_request = attempt_back_merge ( q , rq ) ;
if ( ! * merged_request )
elv_merged_request ( q , rq , ELEVATOR_BACK_MERGE ) ;
return true ;
case ELEVATOR_FRONT_MERGE :
2017-01-17 16:03:22 +03:00
if ( ! blk_mq_sched_allow_merge ( q , rq , bio ) )
return false ;
2019-06-06 13:29:01 +03:00
if ( ! bio_attempt_front_merge ( rq , bio , nr_segs ) )
2017-02-08 16:46:48 +03:00
return false ;
* merged_request = attempt_front_merge ( q , rq ) ;
if ( ! * merged_request )
elv_merged_request ( q , rq , ELEVATOR_FRONT_MERGE ) ;
return true ;
2018-02-02 00:41:15 +03:00
case ELEVATOR_DISCARD_MERGE :
return bio_attempt_discard_merge ( q , rq , bio ) ;
2017-02-08 16:46:48 +03:00
default :
return false ;
2017-01-17 16:03:22 +03:00
}
}
EXPORT_SYMBOL_GPL ( blk_mq_sched_try_merge ) ;
2017-05-26 14:53:20 +03:00
/*
2018-05-30 10:26:07 +03:00
* Iterate list of requests and see if we can merge this bio with any
* of them .
2017-05-26 14:53:20 +03:00
*/
2018-05-30 10:26:07 +03:00
bool blk_mq_bio_list_merge ( struct request_queue * q , struct list_head * list ,
2019-06-06 13:29:01 +03:00
struct bio * bio , unsigned int nr_segs )
2017-05-26 14:53:20 +03:00
{
struct request * rq ;
int checked = 8 ;
2018-05-30 10:26:07 +03:00
list_for_each_entry_reverse ( rq , list , queuelist ) {
2017-05-26 14:53:20 +03:00
bool merged = false ;
if ( ! checked - - )
break ;
if ( ! blk_rq_merge_ok ( rq , bio ) )
continue ;
switch ( blk_try_merge ( rq , bio ) ) {
case ELEVATOR_BACK_MERGE :
if ( blk_mq_sched_allow_merge ( q , rq , bio ) )
2019-06-06 13:29:01 +03:00
merged = bio_attempt_back_merge ( rq , bio ,
nr_segs ) ;
2017-05-26 14:53:20 +03:00
break ;
case ELEVATOR_FRONT_MERGE :
if ( blk_mq_sched_allow_merge ( q , rq , bio ) )
2019-06-06 13:29:01 +03:00
merged = bio_attempt_front_merge ( rq , bio ,
nr_segs ) ;
2017-05-26 14:53:20 +03:00
break ;
case ELEVATOR_DISCARD_MERGE :
merged = bio_attempt_discard_merge ( q , rq , bio ) ;
break ;
default :
continue ;
}
return merged ;
}
return false ;
}
2018-05-30 10:26:07 +03:00
EXPORT_SYMBOL_GPL ( blk_mq_bio_list_merge ) ;
/*
* Reverse check our software queue for entries that we could potentially
* merge with . Currently includes a hand - wavy stop count of 8 , to not spend
* too much time checking for merges .
*/
static bool blk_mq_attempt_merge ( struct request_queue * q ,
2018-12-17 18:44:05 +03:00
struct blk_mq_hw_ctx * hctx ,
2019-06-06 13:29:01 +03:00
struct blk_mq_ctx * ctx , struct bio * bio ,
unsigned int nr_segs )
2018-05-30 10:26:07 +03:00
{
2018-12-17 18:44:05 +03:00
enum hctx_type type = hctx - > type ;
2018-05-30 10:26:07 +03:00
lockdep_assert_held ( & ctx - > lock ) ;
2019-06-06 13:29:01 +03:00
if ( blk_mq_bio_list_merge ( q , & ctx - > rq_lists [ type ] , bio , nr_segs ) ) {
2018-05-30 10:26:07 +03:00
ctx - > rq_merged + + ;
return true ;
}
return false ;
}
2017-05-26 14:53:20 +03:00
2019-06-06 13:29:01 +03:00
bool __blk_mq_sched_bio_merge ( struct request_queue * q , struct bio * bio ,
unsigned int nr_segs )
2017-01-17 16:03:22 +03:00
{
struct elevator_queue * e = q - > elevator ;
2017-05-26 14:53:20 +03:00
struct blk_mq_ctx * ctx = blk_mq_get_ctx ( q ) ;
2019-01-24 13:25:32 +03:00
struct blk_mq_hw_ctx * hctx = blk_mq_map_queue ( q , bio - > bi_opf , ctx ) ;
2017-05-26 14:53:20 +03:00
bool ret = false ;
2018-12-17 18:44:05 +03:00
enum hctx_type type ;
2017-01-17 16:03:22 +03:00
2019-07-01 18:47:29 +03:00
if ( e & & e - > type - > ops . bio_merge )
2019-06-06 13:29:01 +03:00
return e - > type - > ops . bio_merge ( hctx , bio , nr_segs ) ;
2017-01-17 16:03:22 +03:00
2018-12-17 18:44:05 +03:00
type = hctx - > type ;
2018-07-02 12:35:59 +03:00
if ( ( hctx - > flags & BLK_MQ_F_SHOULD_MERGE ) & &
2018-12-17 18:44:05 +03:00
! list_empty_careful ( & ctx - > rq_lists [ type ] ) ) {
2017-05-26 14:53:20 +03:00
/* default per sw-queue merge */
spin_lock ( & ctx - > lock ) ;
2019-06-06 13:29:01 +03:00
ret = blk_mq_attempt_merge ( q , hctx , ctx , bio , nr_segs ) ;
2017-05-26 14:53:20 +03:00
spin_unlock ( & ctx - > lock ) ;
}
return ret ;
2017-01-17 16:03:22 +03:00
}
bool blk_mq_sched_try_insert_merge ( struct request_queue * q , struct request * rq )
{
return rq_mergeable ( rq ) & & elv_attempt_insert_merge ( q , rq ) ;
}
EXPORT_SYMBOL_GPL ( blk_mq_sched_try_insert_merge ) ;
void blk_mq_sched_request_inserted ( struct request * rq )
{
trace_block_rq_insert ( rq - > q , rq ) ;
}
EXPORT_SYMBOL_GPL ( blk_mq_sched_request_inserted ) ;
2017-02-03 02:42:39 +03:00
static bool blk_mq_sched_bypass_insert ( struct blk_mq_hw_ctx * hctx ,
2017-11-02 18:24:36 +03:00
bool has_sched ,
2017-02-03 02:42:39 +03:00
struct request * rq )
2017-01-17 16:03:22 +03:00
{
2020-02-25 04:04:32 +03:00
/*
* dispatch flush and passthrough rq directly
*
* passthrough request has to be added to hctx - > dispatch directly .
* For some reason , device may be in one situation which can ' t
* handle FS request , so STS_RESOURCE is always returned and the
* FS request will be added to hctx - > dispatch . However passthrough
* request may be required at that time for fixing the problem . If
* passthrough request is added to scheduler queue , there isn ' t any
* chance to dispatch it given we prioritize requests in hctx - > dispatch .
*/
if ( ( rq - > rq_flags & RQF_FLUSH_SEQ ) | | blk_rq_is_passthrough ( rq ) )
2017-11-02 18:24:36 +03:00
return true ;
2017-11-02 18:24:38 +03:00
if ( has_sched )
2017-01-17 16:03:22 +03:00
rq - > rq_flags | = RQF_SORTED ;
2017-11-02 18:24:36 +03:00
return false ;
2017-01-17 16:03:22 +03:00
}
2017-01-27 11:00:47 +03:00
void blk_mq_sched_insert_request ( struct request * rq , bool at_head ,
2018-01-17 19:25:58 +03:00
bool run_queue , bool async )
2017-01-27 11:00:47 +03:00
{
struct request_queue * q = rq - > q ;
struct elevator_queue * e = q - > elevator ;
struct blk_mq_ctx * ctx = rq - > mq_ctx ;
2018-10-30 00:06:13 +03:00
struct blk_mq_hw_ctx * hctx = rq - > mq_hctx ;
2017-01-27 11:00:47 +03:00
2017-11-02 18:24:36 +03:00
/* flush rq in flush machinery need to be dispatched directly */
if ( ! ( rq - > rq_flags & RQF_FLUSH_SEQ ) & & op_is_flush ( rq - > cmd_flags ) ) {
2017-11-02 18:24:38 +03:00
blk_insert_flush ( rq ) ;
goto run ;
2017-01-27 11:00:47 +03:00
}
2017-11-02 18:24:38 +03:00
WARN_ON ( e & & ( rq - > tag ! = - 1 ) ) ;
2020-02-25 04:04:32 +03:00
if ( blk_mq_sched_bypass_insert ( hctx , ! ! e , rq ) ) {
2020-03-12 12:15:48 +03:00
/*
* Firstly normal IO request is inserted to scheduler queue or
* sw queue , meantime we add flush request to dispatch queue (
* hctx - > dispatch ) directly and there is at most one in - flight
* flush request for each hw queue , so it doesn ' t matter to add
* flush request to tail or front of the dispatch queue .
*
* Secondly in case of NCQ , flush request belongs to non - NCQ
* command , and queueing it will fail when there is any
* in - flight normal IO request ( NCQ command ) . When adding flush
* rq to the front of hctx - > dispatch , it is easier to introduce
* extra time to flush rq ' s latency because of S_SCHED_RESTART
* compared with adding to the tail of dispatch queue , then
* chance of flush merge is increased , and less flush requests
* will be issued to controller . It is observed that ~ 10 % time
* is saved in blktests block / 004 on disk attached to AHCI / NCQ
* drive when adding flush rq to the front of hctx - > dispatch .
*
* Simply queue flush rq to the front of hctx - > dispatch so that
* intensive flush workloads can benefit in case of NCQ HW .
*/
at_head = ( rq - > rq_flags & RQF_FLUSH_SEQ ) ? true : at_head ;
2020-02-25 04:04:32 +03:00
blk_mq_request_bypass_insert ( rq , at_head , false ) ;
2017-02-03 02:42:39 +03:00
goto run ;
2020-02-25 04:04:32 +03:00
}
2017-02-03 02:42:39 +03:00
2018-11-02 01:41:41 +03:00
if ( e & & e - > type - > ops . insert_requests ) {
2017-01-27 11:00:47 +03:00
LIST_HEAD ( list ) ;
list_add ( & rq - > queuelist , & list ) ;
2018-11-02 01:41:41 +03:00
e - > type - > ops . insert_requests ( hctx , & list , at_head ) ;
2017-01-27 11:00:47 +03:00
} else {
spin_lock ( & ctx - > lock ) ;
__blk_mq_insert_request ( hctx , rq , at_head ) ;
spin_unlock ( & ctx - > lock ) ;
}
2017-02-03 02:42:39 +03:00
run :
2017-01-27 11:00:47 +03:00
if ( run_queue )
blk_mq_run_hw_queue ( hctx , async ) ;
}
2018-10-30 20:31:51 +03:00
void blk_mq_sched_insert_requests ( struct blk_mq_hw_ctx * hctx ,
2017-01-27 11:00:47 +03:00
struct blk_mq_ctx * ctx ,
struct list_head * list , bool run_queue_async )
{
2018-10-29 22:11:38 +03:00
struct elevator_queue * e ;
blk-mq: grab .q_usage_counter when queuing request from plug code path
Just like aio/io_uring, we need to grab 2 refcount for queuing one
request, one is for submission, another is for completion.
If the request isn't queued from plug code path, the refcount grabbed
in generic_make_request() serves for submission. In theroy, this
refcount should have been released after the sumission(async run queue)
is done. blk_freeze_queue() works with blk_sync_queue() together
for avoiding race between cleanup queue and IO submission, given async
run queue activities are canceled because hctx->run_work is scheduled with
the refcount held, so it is fine to not hold the refcount when
running the run queue work function for dispatch IO.
However, if request is staggered into plug list, and finally queued
from plug code path, the refcount in submission side is actually missed.
And we may start to run queue after queue is removed because the queue's
kobject refcount isn't guaranteed to be grabbed in flushing plug list
context, then kernel oops is triggered, see the following race:
blk_mq_flush_plug_list():
blk_mq_sched_insert_requests()
insert requests to sw queue or scheduler queue
blk_mq_run_hw_queue
Because of concurrent run queue, all requests inserted above may be
completed before calling the above blk_mq_run_hw_queue. Then queue can
be freed during the above blk_mq_run_hw_queue().
Fixes the issue by grab .q_usage_counter before calling
blk_mq_sched_insert_requests() in blk_mq_flush_plug_list(). This way is
safe because the queue is absolutely alive before inserting request.
Cc: Dongli Zhang <dongli.zhang@oracle.com>
Cc: James Smart <james.smart@broadcom.com>
Cc: linux-scsi@vger.kernel.org,
Cc: Martin K . Petersen <martin.petersen@oracle.com>,
Cc: Christoph Hellwig <hch@lst.de>,
Cc: James E . J . Bottomley <jejb@linux.vnet.ibm.com>,
Reviewed-by: Bart Van Assche <bvanassche@acm.org>
Tested-by: James Smart <james.smart@broadcom.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2019-04-30 04:52:23 +03:00
struct request_queue * q = hctx - > queue ;
/*
* blk_mq_sched_insert_requests ( ) is called from flush plug
* context only , and hold one usage counter to prevent queue
* from being released .
*/
percpu_ref_get ( & q - > q_usage_counter ) ;
2017-01-27 11:00:47 +03:00
2018-10-29 22:11:38 +03:00
e = hctx - > queue - > elevator ;
2018-11-02 01:41:41 +03:00
if ( e & & e - > type - > ops . insert_requests )
e - > type - > ops . insert_requests ( hctx , list , false ) ;
2018-07-10 04:03:31 +03:00
else {
/*
* try to issue requests directly if the hw queue isn ' t
* busy in case of ' none ' scheduler , and this way may save
* us one extra enqueue & dequeue to sw queue .
*/
2019-04-04 20:08:43 +03:00
if ( ! hctx - > dispatch_busy & & ! e & & ! run_queue_async ) {
2018-07-10 04:03:31 +03:00
blk_mq_try_issue_list_directly ( hctx , list ) ;
2019-04-04 20:08:43 +03:00
if ( list_empty ( list ) )
blk-mq: grab .q_usage_counter when queuing request from plug code path
Just like aio/io_uring, we need to grab 2 refcount for queuing one
request, one is for submission, another is for completion.
If the request isn't queued from plug code path, the refcount grabbed
in generic_make_request() serves for submission. In theroy, this
refcount should have been released after the sumission(async run queue)
is done. blk_freeze_queue() works with blk_sync_queue() together
for avoiding race between cleanup queue and IO submission, given async
run queue activities are canceled because hctx->run_work is scheduled with
the refcount held, so it is fine to not hold the refcount when
running the run queue work function for dispatch IO.
However, if request is staggered into plug list, and finally queued
from plug code path, the refcount in submission side is actually missed.
And we may start to run queue after queue is removed because the queue's
kobject refcount isn't guaranteed to be grabbed in flushing plug list
context, then kernel oops is triggered, see the following race:
blk_mq_flush_plug_list():
blk_mq_sched_insert_requests()
insert requests to sw queue or scheduler queue
blk_mq_run_hw_queue
Because of concurrent run queue, all requests inserted above may be
completed before calling the above blk_mq_run_hw_queue. Then queue can
be freed during the above blk_mq_run_hw_queue().
Fixes the issue by grab .q_usage_counter before calling
blk_mq_sched_insert_requests() in blk_mq_flush_plug_list(). This way is
safe because the queue is absolutely alive before inserting request.
Cc: Dongli Zhang <dongli.zhang@oracle.com>
Cc: James Smart <james.smart@broadcom.com>
Cc: linux-scsi@vger.kernel.org,
Cc: Martin K . Petersen <martin.petersen@oracle.com>,
Cc: Christoph Hellwig <hch@lst.de>,
Cc: James E . J . Bottomley <jejb@linux.vnet.ibm.com>,
Reviewed-by: Bart Van Assche <bvanassche@acm.org>
Tested-by: James Smart <james.smart@broadcom.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2019-04-30 04:52:23 +03:00
goto out ;
2019-04-04 20:08:43 +03:00
}
blk_mq_insert_requests ( hctx , ctx , list ) ;
2018-07-10 04:03:31 +03:00
}
2017-01-27 11:00:47 +03:00
blk_mq_run_hw_queue ( hctx , run_queue_async ) ;
blk-mq: grab .q_usage_counter when queuing request from plug code path
Just like aio/io_uring, we need to grab 2 refcount for queuing one
request, one is for submission, another is for completion.
If the request isn't queued from plug code path, the refcount grabbed
in generic_make_request() serves for submission. In theroy, this
refcount should have been released after the sumission(async run queue)
is done. blk_freeze_queue() works with blk_sync_queue() together
for avoiding race between cleanup queue and IO submission, given async
run queue activities are canceled because hctx->run_work is scheduled with
the refcount held, so it is fine to not hold the refcount when
running the run queue work function for dispatch IO.
However, if request is staggered into plug list, and finally queued
from plug code path, the refcount in submission side is actually missed.
And we may start to run queue after queue is removed because the queue's
kobject refcount isn't guaranteed to be grabbed in flushing plug list
context, then kernel oops is triggered, see the following race:
blk_mq_flush_plug_list():
blk_mq_sched_insert_requests()
insert requests to sw queue or scheduler queue
blk_mq_run_hw_queue
Because of concurrent run queue, all requests inserted above may be
completed before calling the above blk_mq_run_hw_queue. Then queue can
be freed during the above blk_mq_run_hw_queue().
Fixes the issue by grab .q_usage_counter before calling
blk_mq_sched_insert_requests() in blk_mq_flush_plug_list(). This way is
safe because the queue is absolutely alive before inserting request.
Cc: Dongli Zhang <dongli.zhang@oracle.com>
Cc: James Smart <james.smart@broadcom.com>
Cc: linux-scsi@vger.kernel.org,
Cc: Martin K . Petersen <martin.petersen@oracle.com>,
Cc: Christoph Hellwig <hch@lst.de>,
Cc: James E . J . Bottomley <jejb@linux.vnet.ibm.com>,
Reviewed-by: Bart Van Assche <bvanassche@acm.org>
Tested-by: James Smart <james.smart@broadcom.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2019-04-30 04:52:23 +03:00
out :
percpu_ref_put ( & q - > q_usage_counter ) ;
2017-01-27 11:00:47 +03:00
}
2017-01-17 16:03:22 +03:00
static void blk_mq_sched_free_tags ( struct blk_mq_tag_set * set ,
struct blk_mq_hw_ctx * hctx ,
unsigned int hctx_idx )
{
if ( hctx - > sched_tags ) {
blk_mq_free_rqs ( set , hctx - > sched_tags , hctx_idx ) ;
blk_mq_free_rq_map ( hctx - > sched_tags ) ;
hctx - > sched_tags = NULL ;
}
}
2017-04-05 22:01:30 +03:00
static int blk_mq_sched_alloc_tags ( struct request_queue * q ,
struct blk_mq_hw_ctx * hctx ,
unsigned int hctx_idx )
{
struct blk_mq_tag_set * set = q - > tag_set ;
int ret ;
hctx - > sched_tags = blk_mq_alloc_rq_map ( set , hctx_idx , q - > nr_requests ,
set - > reserved_tags ) ;
if ( ! hctx - > sched_tags )
return - ENOMEM ;
ret = blk_mq_alloc_rqs ( set , hctx - > sched_tags , hctx_idx , q - > nr_requests ) ;
if ( ret )
blk_mq_sched_free_tags ( set , hctx , hctx_idx ) ;
return ret ;
}
block: free sched's request pool in blk_cleanup_queue
In theory, IO scheduler belongs to request queue, and the request pool
of sched tags belongs to the request queue too.
However, the current tags allocation interfaces are re-used for both
driver tags and sched tags, and driver tags is definitely host wide,
and doesn't belong to any request queue, same with its request pool.
So we need tagset instance for freeing request of sched tags.
Meantime, blk_mq_free_tag_set() often follows blk_cleanup_queue() in case
of non-BLK_MQ_F_TAG_SHARED, this way requires that request pool of sched
tags to be freed before calling blk_mq_free_tag_set().
Commit 47cdee29ef9d94e ("block: move blk_exit_queue into __blk_release_queue")
moves blk_exit_queue into __blk_release_queue for simplying the fast
path in generic_make_request(), then causes oops during freeing requests
of sched tags in __blk_release_queue().
Fix the above issue by move freeing request pool of sched tags into
blk_cleanup_queue(), this way is safe becasue queue has been frozen and no any
in-queue requests at that time. Freeing sched tags has to be kept in queue's
release handler becasue there might be un-completed dispatch activity
which might refer to sched tags.
Cc: Bart Van Assche <bvanassche@acm.org>
Cc: Christoph Hellwig <hch@lst.de>
Fixes: 47cdee29ef9d94e485eb08f962c74943023a5271 ("block: move blk_exit_queue into __blk_release_queue")
Tested-by: Yi Zhang <yi.zhang@redhat.com>
Reported-by: kernel test robot <rong.a.chen@intel.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2019-06-04 16:08:02 +03:00
/* called in queue's release handler, tagset has gone away */
2017-04-07 17:52:27 +03:00
static void blk_mq_sched_tags_teardown ( struct request_queue * q )
2017-01-17 16:03:22 +03:00
{
struct blk_mq_hw_ctx * hctx ;
2017-04-05 22:01:30 +03:00
int i ;
block: free sched's request pool in blk_cleanup_queue
In theory, IO scheduler belongs to request queue, and the request pool
of sched tags belongs to the request queue too.
However, the current tags allocation interfaces are re-used for both
driver tags and sched tags, and driver tags is definitely host wide,
and doesn't belong to any request queue, same with its request pool.
So we need tagset instance for freeing request of sched tags.
Meantime, blk_mq_free_tag_set() often follows blk_cleanup_queue() in case
of non-BLK_MQ_F_TAG_SHARED, this way requires that request pool of sched
tags to be freed before calling blk_mq_free_tag_set().
Commit 47cdee29ef9d94e ("block: move blk_exit_queue into __blk_release_queue")
moves blk_exit_queue into __blk_release_queue for simplying the fast
path in generic_make_request(), then causes oops during freeing requests
of sched tags in __blk_release_queue().
Fix the above issue by move freeing request pool of sched tags into
blk_cleanup_queue(), this way is safe becasue queue has been frozen and no any
in-queue requests at that time. Freeing sched tags has to be kept in queue's
release handler becasue there might be un-completed dispatch activity
which might refer to sched tags.
Cc: Bart Van Assche <bvanassche@acm.org>
Cc: Christoph Hellwig <hch@lst.de>
Fixes: 47cdee29ef9d94e485eb08f962c74943023a5271 ("block: move blk_exit_queue into __blk_release_queue")
Tested-by: Yi Zhang <yi.zhang@redhat.com>
Reported-by: kernel test robot <rong.a.chen@intel.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2019-06-04 16:08:02 +03:00
queue_for_each_hw_ctx ( q , hctx , i ) {
if ( hctx - > sched_tags ) {
blk_mq_free_rq_map ( hctx - > sched_tags ) ;
hctx - > sched_tags = NULL ;
}
}
2017-04-05 22:01:30 +03:00
}
int blk_mq_init_sched ( struct request_queue * q , struct elevator_type * e )
{
struct blk_mq_hw_ctx * hctx ;
2017-04-05 22:01:34 +03:00
struct elevator_queue * eq ;
2017-04-05 22:01:30 +03:00
unsigned int i ;
int ret ;
if ( ! e ) {
q - > elevator = NULL ;
2018-06-02 10:18:09 +03:00
q - > nr_requests = q - > tag_set - > queue_depth ;
2017-04-05 22:01:30 +03:00
return 0 ;
}
2017-01-17 16:03:22 +03:00
/*
2017-07-03 15:37:14 +03:00
* Default to double of smaller one between hw queue_depth and 128 ,
* since we don ' t split into sync / async like the old code did .
* Additionally , this is a per - hw queue depth .
2017-01-17 16:03:22 +03:00
*/
2017-07-03 15:37:14 +03:00
q - > nr_requests = 2 * min_t ( unsigned int , q - > tag_set - > queue_depth ,
BLKDEV_MAX_RQ ) ;
2017-01-17 16:03:22 +03:00
queue_for_each_hw_ctx ( q , hctx , i ) {
2017-04-05 22:01:30 +03:00
ret = blk_mq_sched_alloc_tags ( q , hctx , i ) ;
2017-01-17 16:03:22 +03:00
if ( ret )
2017-04-05 22:01:30 +03:00
goto err ;
2017-01-17 16:03:22 +03:00
}
2018-11-02 01:41:41 +03:00
ret = e - > ops . init_sched ( q , e ) ;
2017-04-05 22:01:30 +03:00
if ( ret )
goto err ;
2017-01-17 16:03:22 +03:00
2017-05-04 17:24:40 +03:00
blk_mq_debugfs_register_sched ( q ) ;
queue_for_each_hw_ctx ( q , hctx , i ) {
2018-11-02 01:41:41 +03:00
if ( e - > ops . init_hctx ) {
ret = e - > ops . init_hctx ( hctx , i ) ;
2017-04-05 22:01:34 +03:00
if ( ret ) {
eq = q - > elevator ;
block: free sched's request pool in blk_cleanup_queue
In theory, IO scheduler belongs to request queue, and the request pool
of sched tags belongs to the request queue too.
However, the current tags allocation interfaces are re-used for both
driver tags and sched tags, and driver tags is definitely host wide,
and doesn't belong to any request queue, same with its request pool.
So we need tagset instance for freeing request of sched tags.
Meantime, blk_mq_free_tag_set() often follows blk_cleanup_queue() in case
of non-BLK_MQ_F_TAG_SHARED, this way requires that request pool of sched
tags to be freed before calling blk_mq_free_tag_set().
Commit 47cdee29ef9d94e ("block: move blk_exit_queue into __blk_release_queue")
moves blk_exit_queue into __blk_release_queue for simplying the fast
path in generic_make_request(), then causes oops during freeing requests
of sched tags in __blk_release_queue().
Fix the above issue by move freeing request pool of sched tags into
blk_cleanup_queue(), this way is safe becasue queue has been frozen and no any
in-queue requests at that time. Freeing sched tags has to be kept in queue's
release handler becasue there might be un-completed dispatch activity
which might refer to sched tags.
Cc: Bart Van Assche <bvanassche@acm.org>
Cc: Christoph Hellwig <hch@lst.de>
Fixes: 47cdee29ef9d94e485eb08f962c74943023a5271 ("block: move blk_exit_queue into __blk_release_queue")
Tested-by: Yi Zhang <yi.zhang@redhat.com>
Reported-by: kernel test robot <rong.a.chen@intel.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2019-06-04 16:08:02 +03:00
blk_mq_sched_free_requests ( q ) ;
2017-04-05 22:01:34 +03:00
blk_mq_exit_sched ( q , eq ) ;
kobject_put ( & eq - > kobj ) ;
return ret ;
}
}
2017-05-04 17:24:40 +03:00
blk_mq_debugfs_register_sched_hctx ( q , hctx ) ;
2017-04-05 22:01:34 +03:00
}
2017-01-17 16:03:22 +03:00
return 0 ;
2017-04-05 22:01:30 +03:00
err :
block: free sched's request pool in blk_cleanup_queue
In theory, IO scheduler belongs to request queue, and the request pool
of sched tags belongs to the request queue too.
However, the current tags allocation interfaces are re-used for both
driver tags and sched tags, and driver tags is definitely host wide,
and doesn't belong to any request queue, same with its request pool.
So we need tagset instance for freeing request of sched tags.
Meantime, blk_mq_free_tag_set() often follows blk_cleanup_queue() in case
of non-BLK_MQ_F_TAG_SHARED, this way requires that request pool of sched
tags to be freed before calling blk_mq_free_tag_set().
Commit 47cdee29ef9d94e ("block: move blk_exit_queue into __blk_release_queue")
moves blk_exit_queue into __blk_release_queue for simplying the fast
path in generic_make_request(), then causes oops during freeing requests
of sched tags in __blk_release_queue().
Fix the above issue by move freeing request pool of sched tags into
blk_cleanup_queue(), this way is safe becasue queue has been frozen and no any
in-queue requests at that time. Freeing sched tags has to be kept in queue's
release handler becasue there might be un-completed dispatch activity
which might refer to sched tags.
Cc: Bart Van Assche <bvanassche@acm.org>
Cc: Christoph Hellwig <hch@lst.de>
Fixes: 47cdee29ef9d94e485eb08f962c74943023a5271 ("block: move blk_exit_queue into __blk_release_queue")
Tested-by: Yi Zhang <yi.zhang@redhat.com>
Reported-by: kernel test robot <rong.a.chen@intel.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2019-06-04 16:08:02 +03:00
blk_mq_sched_free_requests ( q ) ;
2017-04-07 17:52:27 +03:00
blk_mq_sched_tags_teardown ( q ) ;
q - > elevator = NULL ;
2017-04-05 22:01:30 +03:00
return ret ;
2017-01-17 16:03:22 +03:00
}
2017-01-14 00:43:58 +03:00
block: free sched's request pool in blk_cleanup_queue
In theory, IO scheduler belongs to request queue, and the request pool
of sched tags belongs to the request queue too.
However, the current tags allocation interfaces are re-used for both
driver tags and sched tags, and driver tags is definitely host wide,
and doesn't belong to any request queue, same with its request pool.
So we need tagset instance for freeing request of sched tags.
Meantime, blk_mq_free_tag_set() often follows blk_cleanup_queue() in case
of non-BLK_MQ_F_TAG_SHARED, this way requires that request pool of sched
tags to be freed before calling blk_mq_free_tag_set().
Commit 47cdee29ef9d94e ("block: move blk_exit_queue into __blk_release_queue")
moves blk_exit_queue into __blk_release_queue for simplying the fast
path in generic_make_request(), then causes oops during freeing requests
of sched tags in __blk_release_queue().
Fix the above issue by move freeing request pool of sched tags into
blk_cleanup_queue(), this way is safe becasue queue has been frozen and no any
in-queue requests at that time. Freeing sched tags has to be kept in queue's
release handler becasue there might be un-completed dispatch activity
which might refer to sched tags.
Cc: Bart Van Assche <bvanassche@acm.org>
Cc: Christoph Hellwig <hch@lst.de>
Fixes: 47cdee29ef9d94e485eb08f962c74943023a5271 ("block: move blk_exit_queue into __blk_release_queue")
Tested-by: Yi Zhang <yi.zhang@redhat.com>
Reported-by: kernel test robot <rong.a.chen@intel.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2019-06-04 16:08:02 +03:00
/*
* called in either blk_queue_cleanup or elevator_switch , tagset
* is required for freeing requests
*/
void blk_mq_sched_free_requests ( struct request_queue * q )
{
struct blk_mq_hw_ctx * hctx ;
int i ;
queue_for_each_hw_ctx ( q , hctx , i ) {
if ( hctx - > sched_tags )
blk_mq_free_rqs ( q - > tag_set , hctx - > sched_tags , i ) ;
}
}
2017-04-07 17:52:27 +03:00
void blk_mq_exit_sched ( struct request_queue * q , struct elevator_queue * e )
{
2017-04-05 22:01:34 +03:00
struct blk_mq_hw_ctx * hctx ;
unsigned int i ;
2017-05-04 17:24:40 +03:00
queue_for_each_hw_ctx ( q , hctx , i ) {
blk_mq_debugfs_unregister_sched_hctx ( hctx ) ;
2018-11-02 01:41:41 +03:00
if ( e - > type - > ops . exit_hctx & & hctx - > sched_data ) {
e - > type - > ops . exit_hctx ( hctx , i ) ;
2017-05-04 17:24:40 +03:00
hctx - > sched_data = NULL ;
2017-04-05 22:01:34 +03:00
}
}
2017-05-04 17:24:40 +03:00
blk_mq_debugfs_unregister_sched ( q ) ;
2018-11-02 01:41:41 +03:00
if ( e - > type - > ops . exit_sched )
e - > type - > ops . exit_sched ( e ) ;
2017-04-07 17:52:27 +03:00
blk_mq_sched_tags_teardown ( q ) ;
q - > elevator = NULL ;
}