dm verity: Add optional "try_verify_in_tasklet" feature
Using tasklets for disk verification can reduce IO latency. When there are accelerated hash instructions it is often better to compute the hash immediately using a tasklet rather than deferring verification to a work-queue. This reduces time spent waiting to schedule work-queue jobs, but requires spending slightly more time in interrupt context. If the dm-bufio cache does not have the required hashes we fallback to the work-queue implementation. FEC is only possible using work-queue because code to support the FEC feature may sleep. The following shows a speed comparison of random reads on a dm-verity device. The dm-verity device uses a 1G ramdisk for data and a 1G ramdisk for hashes. One test was run using tasklets and one test was run using the existing work-queue solution. Both tests were run when the dm-bufio cache was hot. The tasklet implementation performs significantly better since there is no time spent waiting for work-queue jobs to be scheduled. READ: bw=181MiB/s (190MB/s), 181MiB/s-181MiB/s (190MB/s-190MB/s), io=512MiB (537MB), run=2827-2827msec READ: bw=23.6MiB/s (24.8MB/s), 23.6MiB/s-23.6MiB/s (24.8MB/s-24.8MB/s), io=512MiB (537MB), run=21688-21688msec Signed-off-by: Nathan Huckleberry <nhuck@google.com> Signed-off-by: Mike Snitzer <snitzer@kernel.org>
This commit is contained in:
Nathan Huckleberry
Mike Snitzer
parent
b32d45824a
commit
5721d4e5a9
@ -34,6 +34,7 @@
|
||||
#define DM_VERITY_OPT_PANIC "panic_on_corruption"
|
||||
#define DM_VERITY_OPT_IGN_ZEROES "ignore_zero_blocks"
|
||||
#define DM_VERITY_OPT_AT_MOST_ONCE "check_at_most_once"
|
||||
#define DM_VERITY_OPT_TASKLET_VERIFY "try_verify_in_tasklet"
|
||||
|
||||
#define DM_VERITY_OPTS_MAX (3 + DM_VERITY_OPTS_FEC + \
|
||||
DM_VERITY_ROOT_HASH_VERIFICATION_OPTS)
|
||||
@ -220,7 +221,7 @@ static int verity_handle_err(struct dm_verity *v, enum verity_block_type type,
|
||||
struct mapped_device *md = dm_table_get_md(v->ti->table);
|
||||
|
||||
/* Corruption should be visible in device status in all modes */
|
||||
v->hash_failed = 1;
|
||||
v->hash_failed = true;
|
||||
|
||||
if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS)
|
||||
goto out;
|
||||
@ -286,7 +287,19 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
|
||||
|
||||
verity_hash_at_level(v, block, level, &hash_block, &offset);
|
||||
|
||||
data = dm_bufio_read(v->bufio, hash_block, &buf);
|
||||
if (io->in_tasklet) {
|
||||
data = dm_bufio_get(v->bufio, hash_block, &buf);
|
||||
if (data == NULL) {
|
||||
/*
|
||||
* In tasklet and the hash was not in the bufio cache.
|
||||
* Return early and resume execution from a work-queue
|
||||
* to read the hash from disk.
|
||||
*/
|
||||
return -EAGAIN;
|
||||
}
|
||||
} else
|
||||
data = dm_bufio_read(v->bufio, hash_block, &buf);
|
||||
|
||||
if (IS_ERR(data))
|
||||
return PTR_ERR(data);
|
||||
|
||||
@ -307,6 +320,14 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
|
||||
if (likely(memcmp(verity_io_real_digest(v, io), want_digest,
|
||||
v->digest_size) == 0))
|
||||
aux->hash_verified = 1;
|
||||
else if (io->in_tasklet) {
|
||||
/*
|
||||
* Error handling code (FEC included) cannot be run in a
|
||||
* tasklet since it may sleep, so fallback to work-queue.
|
||||
*/
|
||||
r = -EAGAIN;
|
||||
goto release_ret_r;
|
||||
}
|
||||
else if (verity_fec_decode(v, io,
|
||||
DM_VERITY_BLOCK_TYPE_METADATA,
|
||||
hash_block, data, NULL) == 0)
|
||||
@ -474,9 +495,14 @@ static int verity_verify_io(struct dm_verity_io *io)
|
||||
bool is_zero;
|
||||
struct dm_verity *v = io->v;
|
||||
struct bvec_iter start;
|
||||
unsigned b;
|
||||
/*
|
||||
* Copy the iterator in case we need to restart verification in a
|
||||
* work-queue.
|
||||
*/
|
||||
struct bvec_iter iter_copy = io->iter;
|
||||
struct crypto_wait wait;
|
||||
struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
|
||||
unsigned int b;
|
||||
|
||||
for (b = 0; b < io->n_blocks; b++) {
|
||||
int r;
|
||||
@ -485,7 +511,7 @@ static int verity_verify_io(struct dm_verity_io *io)
|
||||
|
||||
if (v->validated_blocks &&
|
||||
likely(test_bit(cur_block, v->validated_blocks))) {
|
||||
verity_bv_skip_block(v, io, &io->iter);
|
||||
verity_bv_skip_block(v, io, &iter_copy);
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -500,7 +526,7 @@ static int verity_verify_io(struct dm_verity_io *io)
|
||||
* If we expect a zero block, don't validate, just
|
||||
* return zeros.
|
||||
*/
|
||||
r = verity_for_bv_block(v, io, &io->iter,
|
||||
r = verity_for_bv_block(v, io, &iter_copy,
|
||||
verity_bv_zero);
|
||||
if (unlikely(r < 0))
|
||||
return r;
|
||||
@ -512,8 +538,8 @@ static int verity_verify_io(struct dm_verity_io *io)
|
||||
if (unlikely(r < 0))
|
||||
return r;
|
||||
|
||||
start = io->iter;
|
||||
r = verity_for_io_block(v, io, &io->iter, &wait);
|
||||
start = iter_copy;
|
||||
r = verity_for_io_block(v, io, &iter_copy, &wait);
|
||||
if (unlikely(r < 0))
|
||||
return r;
|
||||
|
||||
@ -527,8 +553,14 @@ static int verity_verify_io(struct dm_verity_io *io)
|
||||
if (v->validated_blocks)
|
||||
set_bit(cur_block, v->validated_blocks);
|
||||
continue;
|
||||
} else if (io->in_tasklet) {
|
||||
/*
|
||||
* Error handling code (FEC included) cannot be run in a
|
||||
* tasklet since it may sleep, so fallback to work-queue.
|
||||
*/
|
||||
return -EAGAIN;
|
||||
} else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA,
|
||||
cur_block, NULL, &start) == 0) {
|
||||
cur_block, NULL, &start) == 0) {
|
||||
continue;
|
||||
} else {
|
||||
if (bio->bi_status) {
|
||||
@ -566,7 +598,8 @@ static void verity_finish_io(struct dm_verity_io *io, blk_status_t status)
|
||||
bio->bi_end_io = io->orig_bi_end_io;
|
||||
bio->bi_status = status;
|
||||
|
||||
verity_fec_finish_io(io);
|
||||
if (!io->in_tasklet)
|
||||
verity_fec_finish_io(io);
|
||||
|
||||
bio_endio(bio);
|
||||
}
|
||||
@ -575,9 +608,29 @@ static void verity_work(struct work_struct *w)
|
||||
{
|
||||
struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);
|
||||
|
||||
io->in_tasklet = false;
|
||||
|
||||
verity_fec_init_io(io);
|
||||
verity_finish_io(io, errno_to_blk_status(verity_verify_io(io)));
|
||||
}
|
||||
|
||||
static void verity_tasklet(unsigned long data)
|
||||
{
|
||||
struct dm_verity_io *io = (struct dm_verity_io *)data;
|
||||
int err;
|
||||
|
||||
io->in_tasklet = true;
|
||||
err = verity_verify_io(io);
|
||||
if (err == -EAGAIN) {
|
||||
/* fallback to retrying with work-queue */
|
||||
INIT_WORK(&io->work, verity_work);
|
||||
queue_work(io->v->verify_wq, &io->work);
|
||||
return;
|
||||
}
|
||||
|
||||
verity_finish_io(io, errno_to_blk_status(err));
|
||||
}
|
||||
|
||||
static void verity_end_io(struct bio *bio)
|
||||
{
|
||||
struct dm_verity_io *io = bio->bi_private;
|
||||
@ -588,8 +641,13 @@ static void verity_end_io(struct bio *bio)
|
||||
return;
|
||||
}
|
||||
|
||||
INIT_WORK(&io->work, verity_work);
|
||||
queue_work(io->v->verify_wq, &io->work);
|
||||
if (io->v->use_tasklet) {
|
||||
tasklet_init(&io->tasklet, verity_tasklet, (unsigned long)io);
|
||||
tasklet_schedule(&io->tasklet);
|
||||
} else {
|
||||
INIT_WORK(&io->work, verity_work);
|
||||
queue_work(io->v->verify_wq, &io->work);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
@ -700,8 +758,6 @@ static int verity_map(struct dm_target *ti, struct bio *bio)
|
||||
bio->bi_private = io;
|
||||
io->iter = bio->bi_iter;
|
||||
|
||||
verity_fec_init_io(io);
|
||||
|
||||
verity_submit_prefetch(v, io);
|
||||
|
||||
submit_bio_noacct(bio);
|
||||
@ -751,6 +807,8 @@ static void verity_status(struct dm_target *ti, status_type_t type,
|
||||
args++;
|
||||
if (v->validated_blocks)
|
||||
args++;
|
||||
if (v->use_tasklet)
|
||||
args++;
|
||||
if (v->signature_key_desc)
|
||||
args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS;
|
||||
if (!args)
|
||||
@ -776,6 +834,8 @@ static void verity_status(struct dm_target *ti, status_type_t type,
|
||||
DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES);
|
||||
if (v->validated_blocks)
|
||||
DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE);
|
||||
if (v->use_tasklet)
|
||||
DMEMIT(" " DM_VERITY_OPT_TASKLET_VERIFY);
|
||||
sz = verity_fec_status_table(v, sz, result, maxlen);
|
||||
if (v->signature_key_desc)
|
||||
DMEMIT(" " DM_VERITY_ROOT_HASH_VERIFICATION_OPT_SIG_KEY
|
||||
@ -1011,11 +1071,16 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v,
|
||||
return r;
|
||||
continue;
|
||||
|
||||
} else if (!strcasecmp(arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) {
|
||||
v->use_tasklet = true;
|
||||
continue;
|
||||
|
||||
} else if (verity_is_fec_opt_arg(arg_name)) {
|
||||
r = verity_fec_parse_opt_args(as, v, &argc, arg_name);
|
||||
if (r)
|
||||
return r;
|
||||
continue;
|
||||
|
||||
} else if (verity_verify_is_sig_opt_arg(arg_name)) {
|
||||
r = verity_verify_sig_parse_opt_args(as, v,
|
||||
verify_args,
|
||||
@ -1023,7 +1088,6 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v,
|
||||
if (r)
|
||||
return r;
|
||||
continue;
|
||||
|
||||
}
|
||||
|
||||
ti->error = "Unrecognized verity feature request";
|
||||
@ -1155,7 +1219,11 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
|
||||
goto bad;
|
||||
}
|
||||
|
||||
v->tfm = crypto_alloc_ahash(v->alg_name, 0, 0);
|
||||
/*
|
||||
* FIXME: CRYPTO_ALG_ASYNC should be conditional on v->use_tasklet
|
||||
* but verity_parse_opt_args() happens below and has data dep on tfm.
|
||||
*/
|
||||
v->tfm = crypto_alloc_ahash(v->alg_name, 0, CRYPTO_ALG_ASYNC);
|
||||
if (IS_ERR(v->tfm)) {
|
||||
ti->error = "Cannot initialize hash function";
|
||||
r = PTR_ERR(v->tfm);
|
||||
@ -1265,7 +1333,8 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
|
||||
|
||||
v->bufio = dm_bufio_client_create(v->hash_dev->bdev,
|
||||
1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),
|
||||
dm_bufio_alloc_callback, NULL, 0);
|
||||
dm_bufio_alloc_callback, NULL,
|
||||
v->use_tasklet ? DM_BUFIO_CLIENT_NO_SLEEP : 0);
|
||||
if (IS_ERR(v->bufio)) {
|
||||
ti->error = "Cannot initialize dm-bufio";
|
||||
r = PTR_ERR(v->bufio);
|
||||
@ -1312,7 +1381,7 @@ bad:
|
||||
static struct target_type verity_target = {
|
||||
.name = "verity",
|
||||
.features = DM_TARGET_IMMUTABLE,
|
||||
.version = {1, 8, 0},
|
||||
.version = {1, 9, 0},
|
||||
.module = THIS_MODULE,
|
||||
.ctr = verity_ctr,
|
||||
.dtr = verity_dtr,
|
||||
|
||||
@ -13,6 +13,7 @@
|
||||
|
||||
#include <linux/dm-bufio.h>
|
||||
#include <linux/device-mapper.h>
|
||||
#include <linux/interrupt.h>
|
||||
#include <crypto/hash.h>
|
||||
|
||||
#define DM_VERITY_MAX_LEVELS 63
|
||||
@ -51,9 +52,10 @@ struct dm_verity {
|
||||
unsigned char hash_per_block_bits; /* log2(hashes in hash block) */
|
||||
unsigned char levels; /* the number of tree levels */
|
||||
unsigned char version;
|
||||
bool hash_failed:1; /* set if hash of any block failed */
|
||||
bool use_tasklet:1; /* try to verify in tasklet before work-queue */
|
||||
unsigned digest_size; /* digest size for the current hash algorithm */
|
||||
unsigned int ahash_reqsize;/* the size of temporary space for crypto */
|
||||
int hash_failed; /* set to 1 if hash of any block failed */
|
||||
enum verity_mode mode; /* mode for handling verification errors */
|
||||
unsigned corrupted_errs;/* Number of errors for corrupted blocks */
|
||||
|
||||
@ -76,10 +78,12 @@ struct dm_verity_io {
|
||||
|
||||
sector_t block;
|
||||
unsigned n_blocks;
|
||||
bool in_tasklet;
|
||||
|
||||
struct bvec_iter iter;
|
||||
|
||||
struct work_struct work;
|
||||
struct tasklet_struct tasklet;
|
||||
|
||||
/*
|
||||
* Three variably-size fields follow this struct:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user