iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Convert the DM verity and crypt targets from (ab)using tasklets to

Browse Source 
using BH workqueues.  These changes were coordinated with Tejun and
   are based ontop of DM's 6.9 changes and Tejun's 6.9 workqueue tree.
 -----BEGIN PGP SIGNATURE-----
 
 iQEzBAABCAAdFiEEJfWUX4UqZ4x1O2wixSPxCi2dA1oFAmXqBRAACgkQxSPxCi2d
 A1pAIAf+PKFi4KlNvJlJehKzCMBxofFUK+DMxePqbhHudN1v6z2Q0lMbsa0wpCly
 eql2SQQPRhqI4j7WhhpY6uXPpxZ9TKrDsIFiOTka6kZBwIlVafMDwtb+16/PVSHa
 le7ngtCaY4EFu0aPpVV8oW5TGJSgmGsv+KvgZ/Op+ugmbOKAraMvVaGb89xh6WQN
 ngUoLqEgF0br79UWKYJ2YtjsgK1h2Ra+Nu8eh/FV3dAOtnJtNJ5Ph0MLz35dGQvv
 0djeDy/DGmWKeNxUdqQPzgguPG84P5Pfg6xbZa7bqqv4ujQOp2YIX4dxkjM2w2Aj
 III+WJ5XHaLOE1JutwZCVyuabNApMA==
 =MSmO
 -----END PGP SIGNATURE-----

Merge tag 'for-6.9/dm-bh-wq' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm

Pull device mapper BH workqueue conversion from Mike Snitzer:
 "Convert the DM verity and crypt targets from (ab)using tasklets to
  using BH workqueues.

  These changes were coordinated with Tejun and are based ontop of DM's
  6.9 changes and Tejun's 6.9 workqueue tree"

* tag 'for-6.9/dm-bh-wq' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm:
  dm-verity: Convert from tasklet to BH workqueue
  dm-crypt: Convert from tasklet to BH workqueue
This commit is contained in:
Linus Torvalds 2024-03-13 09:46:51 -07:00
commit c0499a0812
3 changed files with 50 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
drivers/md/dm-crypt.c
View File

@ -2296,7 +2296,11 @@ static void kcryptd_queue_crypt(struct dm_crypt_io *io)
* irqs_disabled(): the kernel may run some IO completion from the idle thread, but
* it is being executed with irqs disabled.
*/
if (!(in_hardirq() || irqs_disabled())) {
if (in_hardirq() || irqs_disabled()) {
INIT_WORK(&io->work, kcryptd_crypt);
queue_work(system_bh_wq, &io->work);
return;
} else {
kcryptd_crypt(&io->work);
return;
}

64
drivers/md/dm-verity-target.c
View File

@ -46,7 +46,7 @@ static unsigned int dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE
module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, 0644);
static DEFINE_STATIC_KEY_FALSE(use_tasklet_enabled);
static DEFINE_STATIC_KEY_FALSE(use_bh_wq_enabled);
struct dm_verity_prefetch_work {
struct work_struct work;
@ -299,7 +299,7 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
verity_hash_at_level(v, block, level, &hash_block, &offset);
if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) {
if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
data = dm_bufio_get(v->bufio, hash_block, &buf);
if (data == NULL) {
/*
@ -327,15 +327,14 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
r = verity_hash(v, verity_io_hash_req(v, io),
data, 1 << v->hash_dev_block_bits,
verity_io_real_digest(v, io), !io->in_tasklet);
verity_io_real_digest(v, io), !io->in_bh);
if (unlikely(r < 0))
goto release_ret_r;
if (likely(memcmp(verity_io_real_digest(v, io), want_digest,
v->digest_size) == 0))
aux->hash_verified = 1;
else if (static_branch_unlikely(&use_tasklet_enabled) &&
io->in_tasklet) {
else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
/*
* Error handling code (FEC included) cannot be run in a
* tasklet since it may sleep, so fallback to work-queue.
@ -576,7 +575,7 @@ static int verity_verify_io(struct dm_verity_io *io)
struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
unsigned int b;
if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) {
if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
/*
* Copy the iterator in case we need to restart
* verification in a work-queue.
@ -616,7 +615,7 @@ static int verity_verify_io(struct dm_verity_io *io)
continue;
}
r = verity_hash_init(v, req, &wait, !io->in_tasklet);
r = verity_hash_init(v, req, &wait, !io->in_bh);
if (unlikely(r < 0))
return r;
@ -635,8 +634,7 @@ static int verity_verify_io(struct dm_verity_io *io)
if (v->validated_blocks)
set_bit(cur_block, v->validated_blocks);
continue;
} else if (static_branch_unlikely(&use_tasklet_enabled) &&
io->in_tasklet) {
} else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
/*
* Error handling code (FEC included) cannot be run in a
* tasklet since it may sleep, so fallback to work-queue.
@ -690,7 +688,7 @@ static void verity_finish_io(struct dm_verity_io *io, blk_status_t status)
bio->bi_end_io = io->orig_bi_end_io;
bio->bi_status = status;
if (!static_branch_unlikely(&use_tasklet_enabled) || !io->in_tasklet)
if (!static_branch_unlikely(&use_bh_wq_enabled) || !io->in_bh)
verity_fec_finish_io(io);
bio_endio(bio);
@ -700,11 +698,28 @@ static void verity_work(struct work_struct *w)
{
struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);
io->in_tasklet = false;
io->in_bh = false;
verity_finish_io(io, errno_to_blk_status(verity_verify_io(io)));
}
static void verity_bh_work(struct work_struct *w)
{
struct dm_verity_io *io = container_of(w, struct dm_verity_io, bh_work);
int err;
io->in_bh = true;
err = verity_verify_io(io);
if (err == -EAGAIN || err == -ENOMEM) {
/* fallback to retrying with work-queue */
INIT_WORK(&io->work, verity_work);
queue_work(io->v->verify_wq, &io->work);
return;
}
verity_finish_io(io, errno_to_blk_status(err));
}
static void verity_end_io(struct bio *bio)
{
struct dm_verity_io *io = bio->bi_private;
@ -717,8 +732,13 @@ static void verity_end_io(struct bio *bio)
return;
}
INIT_WORK(&io->work, verity_work);
queue_work(io->v->verify_wq, &io->work);
if (static_branch_unlikely(&use_bh_wq_enabled) && io->v->use_bh_wq) {
INIT_WORK(&io->bh_work, verity_bh_work);
queue_work(system_bh_wq, &io->bh_work);
} else {
INIT_WORK(&io->work, verity_work);
queue_work(io->v->verify_wq, &io->work);
}
}
/*
@ -885,7 +905,7 @@ static void verity_status(struct dm_target *ti, status_type_t type,
args++;
if (v->validated_blocks)
args++;
if (v->use_tasklet)
if (v->use_bh_wq)
args++;
if (v->signature_key_desc)
args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS;
@ -912,7 +932,7 @@ static void verity_status(struct dm_target *ti, status_type_t type,
DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES);
if (v->validated_blocks)
DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE);
if (v->use_tasklet)
if (v->use_bh_wq)
DMEMIT(" " DM_VERITY_OPT_TASKLET_VERIFY);
sz = verity_fec_status_table(v, sz, result, maxlen);
if (v->signature_key_desc)
@ -1031,8 +1051,8 @@ static void verity_dtr(struct dm_target *ti)
kfree(v->signature_key_desc);
if (v->use_tasklet)
static_branch_dec(&use_tasklet_enabled);
if (v->use_bh_wq)
static_branch_dec(&use_bh_wq_enabled);
kfree(v);
@ -1166,8 +1186,8 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v,
continue;
} else if (!strcasecmp(arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) {
v->use_tasklet = true;
static_branch_inc(&use_tasklet_enabled);
v->use_bh_wq = true;
static_branch_inc(&use_bh_wq_enabled);
continue;
} else if (verity_is_fec_opt_arg(arg_name)) {
@ -1338,7 +1358,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
}
v->tfm = crypto_alloc_ahash(v->alg_name, 0,
v->use_tasklet ? CRYPTO_ALG_ASYNC : 0);
v->use_bh_wq ? CRYPTO_ALG_ASYNC : 0);
if (IS_ERR(v->tfm)) {
ti->error = "Cannot initialize hash function";
r = PTR_ERR(v->tfm);
@ -1463,7 +1483,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
v->bufio = dm_bufio_client_create(v->hash_dev->bdev,
1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),
dm_bufio_alloc_callback, NULL,
v->use_tasklet ? DM_BUFIO_CLIENT_NO_SLEEP : 0);
v->use_bh_wq ? DM_BUFIO_CLIENT_NO_SLEEP : 0);
if (IS_ERR(v->bufio)) {
ti->error = "Cannot initialize dm-bufio";
r = PTR_ERR(v->bufio);
@ -1482,7 +1502,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
* reducing wait times when reading from a dm-verity device.
*
* Also as required for the "try_verify_in_tasklet" feature: WQ_HIGHPRI
* allows verify_wq to preempt softirq since verification in tasklet
* allows verify_wq to preempt softirq since verification in BH workqueue
* will fall-back to using it for error handling (or if the bufio cache
* doesn't have required hashes).
*/

5
drivers/md/dm-verity.h
View File

@ -54,7 +54,7 @@ struct dm_verity {
unsigned char levels; /* the number of tree levels */
unsigned char version;
bool hash_failed:1; /* set if hash of any block failed */
bool use_tasklet:1; /* try to verify in tasklet before work-queue */
bool use_bh_wq:1; /* try to verify in BH wq before normal work-queue */
unsigned int digest_size; /* digest size for the current hash algorithm */
unsigned int ahash_reqsize;/* the size of temporary space for crypto */
enum verity_mode mode; /* mode for handling verification errors */
@ -84,9 +84,10 @@ struct dm_verity_io {
sector_t block;
unsigned int n_blocks;
bool in_tasklet;
bool in_bh;
struct work_struct work;
struct work_struct bh_work;
char *recheck_buffer;