iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

block, bfq: turn bfqq_data into an array in bfq_io_cq

Browse Source 
When a bfq_queue Q is merged with another queue, several pieces of
information are saved about Q. These pieces are stored in the
bfqq_data field in the bfq_io_cq data structure of the process
associated with Q.

Yet, with a multi-actuator drive, a process may get associated with
multiple bfq_queues: one queue for each of the N actuators. Each of
these queues may undergo a merge. So, the bfq_io_cq data structure
must be able to accommodate the above information for N queues.

This commit solves this problem by turning the bfqq_data scalar field
into an array of N elements (and by changing code so as to handle
this array).

This solution is written under the assumption that bfq_queues
associated with different actuators cannot be cross-merged. This
assumption holds naturally with basic queue merging: the latter is
triggered by spatial locality, and sectors for different actuators are
not close to each other (apart from the corner case of the last
sectors served by a given actuator and the first sectors served by the
next actuator). As for stable cross-merging, the assumption here is
that it is disabled.

Reviewed-by: Damien Le Moal <damien.lemoal@opensource.wdc.com>
Signed-off-by: Gabriele Felici <felicigb@gmail.com>
Signed-off-by: Gianmarco Lusvardi <glusvardi@posteo.net>
Signed-off-by: Giulio Barabino <giuliobarabino99@gmail.com>
Signed-off-by: Emiliano Maccaferri <inbox@emilianomaccaferri.com>
Signed-off-by: Paolo Valente <paolo.valente@linaro.org>
Link: https://lore.kernel.org/r/20230103145503.71712-5-paolo.valente@linaro.org
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Paolo Valente 2023-01-03 15:54:59 +01:00 committed by Jens Axboe
parent a61230470c
commit fd571df0ac
2 changed files with 67 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
block/bfq-iosched.c
View File

@ -408,7 +408,7 @@ void bic_set_bfqq(struct bfq_io_cq *bic,
* we cancel the stable merge if
* bic->stable_merge_bfqq == bfqq.
*/
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[actuator_idx];
/* Clear bic pointer if bfqq is detached from this bic */
if (old_bfqq && old_bfqq->bic == bic)
@ -1135,9 +1135,10 @@ static void
bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd,
struct bfq_io_cq *bic, bool bfq_already_existing)
{
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;
unsigned int old_wr_coeff = 1;
bool busy = bfq_already_existing && bfq_bfqq_busy(bfqq);
unsigned int a_idx = bfqq->actuator_idx;
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[a_idx];
if (bfqq_data->saved_has_short_ttime)
bfq_mark_bfqq_has_short_ttime(bfqq);
@ -1818,7 +1819,9 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd,
arrived_in_time = ktime_get_ns() <=
bfqq->ttime.last_end_request +
bfqd->bfq_slice_idle * 3;
unsigned int act_idx = bfq_actuator_index(bfqd, rq->bio);
bool bfqq_non_merged_or_stably_merged =
bfqq->bic || RQ_BIC(rq)->bfqq_data[act_idx].stably_merged;
/*
* bfqq deserves to be weight-raised if:
@ -1852,9 +1855,8 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd,
*/
wr_or_deserves_wr = bfqd->low_latency &&
(bfqq->wr_coeff > 1 ||
(bfq_bfqq_sync(bfqq) &&
(bfqq->bic || RQ_BIC(rq)->bfqq_data.stably_merged) &&
(*interactive || soft_rt)));
(bfq_bfqq_sync(bfqq) && bfqq_non_merged_or_stably_merged &&
(*interactive || soft_rt)));
/*
* Using the last flag, update budget and check whether bfqq
@ -2828,6 +2830,35 @@ static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq,
static bool idling_boosts_thr_without_issues(struct bfq_data *bfqd,
struct bfq_queue *bfqq);
static struct bfq_queue *
bfq_setup_stable_merge(struct bfq_data *bfqd, struct bfq_queue *bfqq,
struct bfq_queue *stable_merge_bfqq,
struct bfq_iocq_bfqq_data *bfqq_data)
{
int proc_ref = min(bfqq_process_refs(bfqq),
bfqq_process_refs(stable_merge_bfqq));
struct bfq_queue *new_bfqq;
if (idling_boosts_thr_without_issues(bfqd, bfqq) ||
proc_ref == 0)
return NULL;
/* next function will take at least one ref */
new_bfqq = bfq_setup_merge(bfqq, stable_merge_bfqq);
if (new_bfqq) {
bfqq_data->stably_merged = true;
if (new_bfqq->bic) {
unsigned int new_a_idx = new_bfqq->actuator_idx;
struct bfq_iocq_bfqq_data *new_bfqq_data =
&new_bfqq->bic->bfqq_data[new_a_idx];
new_bfqq_data->stably_merged = true;
}
}
return new_bfqq;
}
/*
* Attempt to schedule a merge of bfqq with the currently in-service
* queue or with a close queue among the scheduled queues. Return
@ -2853,7 +2884,8 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
void *io_struct, bool request, struct bfq_io_cq *bic)
{
struct bfq_queue *in_service_bfqq, *new_bfqq;
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;
unsigned int a_idx = bfqq->actuator_idx;
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[a_idx];
/* if a merge has already been setup, then proceed with that first */
if (bfqq->new_bfqq)
@ -2883,29 +2915,15 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
msecs_to_jiffies(bfq_late_stable_merging))) {
struct bfq_queue *stable_merge_bfqq =
bfqq_data->stable_merge_bfqq;
int proc_ref = min(bfqq_process_refs(bfqq),
bfqq_process_refs(stable_merge_bfqq));
/* deschedule stable merge, because done or aborted here */
bfq_put_stable_ref(stable_merge_bfqq);
bfqq_data->stable_merge_bfqq = NULL;
if (!idling_boosts_thr_without_issues(bfqd, bfqq) &&
proc_ref > 0) {
/* next function will take at least one ref */
struct bfq_queue *new_bfqq =
bfq_setup_merge(bfqq, stable_merge_bfqq);
if (new_bfqq) {
bfqq_data->stably_merged = true;
if (new_bfqq->bic)
new_bfqq->bic->bfqq_data.stably_merged =
true;
}
return new_bfqq;
} else
return NULL;
return bfq_setup_stable_merge(bfqd, bfqq,
stable_merge_bfqq,
bfqq_data);
}
}
@ -3000,7 +3018,8 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
static void bfq_bfqq_save_state(struct bfq_queue *bfqq)
{
struct bfq_io_cq *bic = bfqq->bic;
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;
unsigned int a_idx = bfqq->actuator_idx;
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[a_idx];
/*
* If !bfqq->bic, the queue is already shared or its requests
@ -3011,7 +3030,7 @@ static void bfq_bfqq_save_state(struct bfq_queue *bfqq)
return;
bfqq_data->saved_last_serv_time_ns = bfqq->last_serv_time_ns;
bfqq_data->saved_inject_limit = bfqq->inject_limit;
bfqq_data->saved_inject_limit = bfqq->inject_limit;
bfqq_data->saved_decrease_time_jif = bfqq->decrease_time_jif;
bfqq_data->saved_weight = bfqq->entity.orig_weight;
@ -5377,7 +5396,7 @@ static void bfq_exit_icq(struct io_cq *icq)
* therefore on its unused per-actuator fields being NULL.
*/
unsigned int num_actuators = BFQ_MAX_ACTUATORS;
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;
struct bfq_iocq_bfqq_data *bfqq_data = bic->bfqq_data;
/*
* bfqd is NULL if scheduler already exited, and in that case
@ -5388,10 +5407,10 @@ static void bfq_exit_icq(struct io_cq *icq)
num_actuators = bfqd->num_actuators;
}
if (bfqq_data->stable_merge_bfqq)
bfq_put_stable_ref(bfqq_data->stable_merge_bfqq);
for (act_idx = 0; act_idx < num_actuators; act_idx++) {
if (bfqq_data[act_idx].stable_merge_bfqq)
bfq_put_stable_ref(bfqq_data[act_idx].stable_merge_bfqq);
bfq_exit_icq_bfqq(bic, true, act_idx);
bfq_exit_icq_bfqq(bic, false, act_idx);
}
@ -5578,16 +5597,16 @@ bfq_do_early_stable_merge(struct bfq_data *bfqd, struct bfq_queue *bfqq,
struct bfq_io_cq *bic,
struct bfq_queue *last_bfqq_created)
{
unsigned int a_idx = last_bfqq_created->actuator_idx;
struct bfq_queue *new_bfqq =
bfq_setup_merge(bfqq, last_bfqq_created);
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;
if (!new_bfqq)
return bfqq;
if (new_bfqq->bic)
new_bfqq->bic->bfqq_data.stably_merged = true;
bfqq_data->stably_merged = true;
new_bfqq->bic->bfqq_data[a_idx].stably_merged = true;
bic->bfqq_data[a_idx].stably_merged = true;
/*
* Reusing merge functions. This implies that
@ -5656,7 +5675,6 @@ static struct bfq_queue *bfq_do_or_sched_stable_merge(struct bfq_data *bfqd,
&bfqd->last_bfqq_created;
struct bfq_queue *last_bfqq_created = *source_bfqq;
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;
/*
* If last_bfqq_created has not been set yet, then init it. If
@ -5718,7 +5736,8 @@ static struct bfq_queue *bfq_do_or_sched_stable_merge(struct bfq_data *bfqd,
/*
* Record the bfqq to merge to.
*/
bfqq_data->stable_merge_bfqq = last_bfqq_created;
bic->bfqq_data[last_bfqq_created->actuator_idx].stable_merge_bfqq =
last_bfqq_created;
}
}
@ -6639,7 +6658,7 @@ static struct bfq_queue *bfq_get_bfqq_handle_split(struct bfq_data *bfqd,
{
unsigned int act_idx = bfq_actuator_index(bfqd, bio);
struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync, act_idx);
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;
struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[act_idx];
if (likely(bfqq && bfqq != &bfqd->oom_bfqq))
return bfqq;
@ -6747,7 +6766,7 @@ static struct bfq_queue *bfq_init_rq(struct request *rq)
struct bfq_queue *bfqq;
bool new_queue = false;
bool bfqq_already_existing = false, split = false;
struct bfq_iocq_bfqq_data *bfqq_data;
unsigned int a_idx = bfq_actuator_index(bfqd, bio);
if (unlikely(!rq->elv.icq))
return NULL;
@ -6771,17 +6790,16 @@ static struct bfq_queue *bfq_init_rq(struct request *rq)
bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, false, is_sync,
&new_queue);
bfqq_data = &bic->bfqq_data;
if (likely(!new_queue)) {
/* If the queue was seeky for too long, break it apart. */
if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq) &&
!bfqq_data->stably_merged) {
!bic->bfqq_data[a_idx].stably_merged) {
struct bfq_queue *old_bfqq = bfqq;
/* Update bic before losing reference to bfqq */
if (bfq_bfqq_in_large_burst(bfqq))
bfqq_data->saved_in_large_burst = true;
bic->bfqq_data[a_idx].saved_in_large_burst =
true;
bfqq = bfq_split_bfqq(bic, bfqq);
split = true;

12
block/bfq-iosched.h
View File

@ -418,7 +418,7 @@ struct bfq_queue {
struct bfq_iocq_bfqq_data {
/*
* Snapshot of the has_short_time flag before merging; taken
* to remember its value while the queue is merged, so as to
* to remember its values while the queue is merged, so as to
* be able to restore it in case of split.
*/
bool saved_has_short_ttime;
@ -432,7 +432,7 @@ struct bfq_iocq_bfqq_data {
u64 saved_tot_idle_time;
/*
* Same purpose as the previous fields for the value of the
* Same purpose as the previous fields for the values of the
* field keeping the queue's belonging to a large burst
*/
bool saved_in_large_burst;
@ -495,8 +495,12 @@ struct bfq_io_cq {
uint64_t blkcg_serial_nr; /* the current blkcg serial */
#endif
/* persistent data for associated synchronous process queue */
struct bfq_iocq_bfqq_data bfqq_data;
/*
* Persistent data for associated synchronous process queues
* (one queue per actuator, see field bfqq above). In
* particular, each of these queues may undergo a merge.
*/
struct bfq_iocq_bfqq_data bfqq_data[BFQ_MAX_ACTUATORS];
unsigned int requests; /* Number of requests this process has in flight */
};