iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dm bufio: Support IO priority

Browse Source 
Some IO will dispatch from kworker with different io_context settings
than the submitting task, we may need to specify a priority to avoid
losing priority.

Add dm_bufio_read_with_ioprio() and dm_bufio_prefetch_with_ioprio()
for use by bufio users to pass an ioprio other than IOPRIO_DEFAULT.

Co-developed-by: Yibin Ding <yibin.ding@unisoc.com>
Signed-off-by: Yibin Ding <yibin.ding@unisoc.com>
Signed-off-by: Hongyu Jin <hongyu.jin@unisoc.com>
Reviewed-by: Eric Biggers <ebiggers@google.com>
Reviewed-by: Mikulas Patocka <mpatocka@redhat.com>
[snitzer: introduced _with_ioprio() wrappers to reduce churn]
Signed-off-by: Mike Snitzer <snitzer@kernel.org>
This commit is contained in:
Hongyu Jin 2024-01-24 13:35:54 +08:00 committed by Mike Snitzer
parent 6e5f0f6383
commit e9b2238e47
2 changed files with 56 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
drivers/md/dm-bufio.c
View File

@ -1292,7 +1292,8 @@ static void dmio_complete(unsigned long error, void *context)
}
static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector,
unsigned int n_sectors, unsigned int offset)
unsigned int n_sectors, unsigned int offset,
unsigned short ioprio)
{
int r;
struct dm_io_request io_req = {
@ -1315,7 +1316,7 @@ static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector,
io_req.mem.ptr.vma = (char *)b->data + offset;
}
r = dm_io(&io_req, 1, &region, NULL, IOPRIO_DEFAULT);
r = dm_io(&io_req, 1, &region, NULL, ioprio);
if (unlikely(r))
b->end_io(b, errno_to_blk_status(r));
}
@ -1331,7 +1332,8 @@ static void bio_complete(struct bio *bio)
}
static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector,
unsigned int n_sectors, unsigned int offset)
unsigned int n_sectors, unsigned int offset,
unsigned short ioprio)
{
struct bio *bio;
char *ptr;
@ -1339,13 +1341,14 @@ static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector,
bio = bio_kmalloc(1, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOWARN);
if (!bio) {
use_dmio(b, op, sector, n_sectors, offset);
use_dmio(b, op, sector, n_sectors, offset, ioprio);
return;
}
bio_init(bio, b->c->bdev, bio->bi_inline_vecs, 1, op);
bio->bi_iter.bi_sector = sector;
bio->bi_end_io = bio_complete;
bio->bi_private = b;
bio->bi_ioprio = ioprio;
ptr = (char *)b->data + offset;
len = n_sectors << SECTOR_SHIFT;
@ -1368,7 +1371,7 @@ static inline sector_t block_to_sector(struct dm_bufio_client *c, sector_t block
return sector;
}
static void submit_io(struct dm_buffer *b, enum req_op op,
static void submit_io(struct dm_buffer *b, enum req_op op, unsigned short ioprio,
void (*end_io)(struct dm_buffer *, blk_status_t))
{
unsigned int n_sectors;
@ -1398,9 +1401,9 @@ static void submit_io(struct dm_buffer *b, enum req_op op,
}
if (b->data_mode != DATA_MODE_VMALLOC)
use_bio(b, op, sector, n_sectors, offset);
use_bio(b, op, sector, n_sectors, offset, ioprio);
else
use_dmio(b, op, sector, n_sectors, offset);
use_dmio(b, op, sector, n_sectors, offset, ioprio);
}
/*
@ -1456,7 +1459,7 @@ static void __write_dirty_buffer(struct dm_buffer *b,
b->write_end = b->dirty_end;
if (!write_list)
submit_io(b, REQ_OP_WRITE, write_endio);
submit_io(b, REQ_OP_WRITE, IOPRIO_DEFAULT, write_endio);
else
list_add_tail(&b->write_list, write_list);
}
@ -1470,7 +1473,7 @@ static void __flush_write_list(struct list_head *write_list)
struct dm_buffer *b =
list_entry(write_list->next, struct dm_buffer, write_list);
list_del(&b->write_list);
submit_io(b, REQ_OP_WRITE, write_endio);
submit_io(b, REQ_OP_WRITE, IOPRIO_DEFAULT, write_endio);
cond_resched();
}
blk_finish_plug(&plug);
@ -1852,7 +1855,8 @@ static void read_endio(struct dm_buffer *b, blk_status_t status)
* and uses dm_bufio_mark_buffer_dirty to write new data back).
*/
static void *new_read(struct dm_bufio_client *c, sector_t block,
enum new_flag nf, struct dm_buffer **bp)
enum new_flag nf, struct dm_buffer **bp,
unsigned short ioprio)
{
int need_submit = 0;
struct dm_buffer *b;
@ -1905,7 +1909,7 @@ static void *new_read(struct dm_bufio_client *c, sector_t block,
return NULL;
if (need_submit)
submit_io(b, REQ_OP_READ, read_endio);
submit_io(b, REQ_OP_READ, ioprio, read_endio);
if (nf != NF_GET) /* we already tested this condition above */
wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE);
@ -1926,32 +1930,46 @@ static void *new_read(struct dm_bufio_client *c, sector_t block,
void *dm_bufio_get(struct dm_bufio_client *c, sector_t block,
struct dm_buffer **bp)
{
return new_read(c, block, NF_GET, bp);
return new_read(c, block, NF_GET, bp, IOPRIO_DEFAULT);
}
EXPORT_SYMBOL_GPL(dm_bufio_get);
void *dm_bufio_read(struct dm_bufio_client *c, sector_t block,
struct dm_buffer **bp)
static void *__dm_bufio_read(struct dm_bufio_client *c, sector_t block,
struct dm_buffer **bp, unsigned short ioprio)
{
if (WARN_ON_ONCE(dm_bufio_in_request()))
return ERR_PTR(-EINVAL);
return new_read(c, block, NF_READ, bp);
return new_read(c, block, NF_READ, bp, ioprio);
}
void *dm_bufio_read(struct dm_bufio_client *c, sector_t block,
struct dm_buffer **bp)
{
return __dm_bufio_read(c, block, bp, IOPRIO_DEFAULT);
}
EXPORT_SYMBOL_GPL(dm_bufio_read);
void *dm_bufio_read_with_ioprio(struct dm_bufio_client *c, sector_t block,
struct dm_buffer **bp, unsigned short ioprio)
{
return __dm_bufio_read(c, block, bp, ioprio);
}
EXPORT_SYMBOL_GPL(dm_bufio_read_with_ioprio);
void *dm_bufio_new(struct dm_bufio_client *c, sector_t block,
struct dm_buffer **bp)
{
if (WARN_ON_ONCE(dm_bufio_in_request()))
return ERR_PTR(-EINVAL);
return new_read(c, block, NF_FRESH, bp);
return new_read(c, block, NF_FRESH, bp, IOPRIO_DEFAULT);
}
EXPORT_SYMBOL_GPL(dm_bufio_new);
void dm_bufio_prefetch(struct dm_bufio_client *c,
sector_t block, unsigned int n_blocks)
static void __dm_bufio_prefetch(struct dm_bufio_client *c,
sector_t block, unsigned int n_blocks,
unsigned short ioprio)
{
struct blk_plug plug;
@ -1987,7 +2005,7 @@ void dm_bufio_prefetch(struct dm_bufio_client *c,
dm_bufio_unlock(c);
if (need_submit)
submit_io(b, REQ_OP_READ, read_endio);
submit_io(b, REQ_OP_READ, ioprio, read_endio);
dm_bufio_release(b);
cond_resched();
@ -2002,8 +2020,20 @@ void dm_bufio_prefetch(struct dm_bufio_client *c,
flush_plug:
blk_finish_plug(&plug);
}
void dm_bufio_prefetch(struct dm_bufio_client *c, sector_t block, unsigned int n_blocks)
{
return __dm_bufio_prefetch(c, block, n_blocks, IOPRIO_DEFAULT);
}
EXPORT_SYMBOL_GPL(dm_bufio_prefetch);
void dm_bufio_prefetch_with_ioprio(struct dm_bufio_client *c, sector_t block,
unsigned int n_blocks, unsigned short ioprio)
{
return __dm_bufio_prefetch(c, block, n_blocks, ioprio);
}
EXPORT_SYMBOL_GPL(dm_bufio_prefetch_with_ioprio);
void dm_bufio_release(struct dm_buffer *b)
{
struct dm_bufio_client *c = b->c;

7
include/linux/dm-bufio.h
View File

@ -64,6 +64,9 @@ void dm_bufio_set_sector_offset(struct dm_bufio_client *c, sector_t start);
void *dm_bufio_read(struct dm_bufio_client *c, sector_t block,
struct dm_buffer **bp);
void *dm_bufio_read_with_ioprio(struct dm_bufio_client *c, sector_t block,
struct dm_buffer **bp, unsigned short ioprio);
/*
* Like dm_bufio_read, but return buffer from cache, don't read
* it. If the buffer is not in the cache, return NULL.
@ -86,6 +89,10 @@ void *dm_bufio_new(struct dm_bufio_client *c, sector_t block,
void dm_bufio_prefetch(struct dm_bufio_client *c,
sector_t block, unsigned int n_blocks);
void dm_bufio_prefetch_with_ioprio(struct dm_bufio_client *c,
sector_t block, unsigned int n_blocks,
unsigned short ioprio);
/*
* Release a reference obtained with dm_bufio_{read,get,new}. The data
* pointer and dm_buffer pointer is no longer valid after this call.