iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

crypto: iaa - Add irq support for the crypto async interface

Browse Source 
The existing iaa crypto async support provides an implementation that
satisfies the interface but does so in a synchronous manner - it fills
and submits the IDXD descriptor and then waits for it to complete
before returning.  This isn't a problem at the moment, since all
existing callers (e.g. zswap) wrap any asynchronous callees in a
synchronous wrapper anyway.

This change makes the iaa crypto async implementation truly
asynchronous: it fills and submits the IDXD descriptor, then returns
immediately with -EINPROGRESS.  It also sets the descriptor's 'request
completion irq' bit and sets up a callback with the IDXD driver which
is called when the operation completes and the irq fires.  The
existing callers such as zswap use synchronous wrappers to deal with
-EINPROGRESS and so work as expected without any changes.

This mode can be enabled by writing 'async_irq' to the sync_mode
iaa_crypto driver attribute:

  echo async_irq > /sys/bus/dsa/drivers/crypto/sync_mode

Async mode without interrupts (caller must poll) can be enabled by
writing 'async' to it:

  echo async > /sys/bus/dsa/drivers/crypto/sync_mode

The default sync mode can be enabled by writing 'sync' to it:

  echo sync > /sys/bus/dsa/drivers/crypto/sync_mode

The sync_mode value setting at the time the IAA algorithms are
registered is captured in each algorithm's crypto_ctx and used for all
compresses and decompresses when using a given algorithm.

Signed-off-by: Tom Zanussi <tom.zanussi@linux.intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
This commit is contained in:
Tom Zanussi 2023-12-05 15:25:28 -06:00 committed by Herbert Xu
parent 2ec6761df8
commit 09646c98d0
2 changed files with 266 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/crypto/intel/iaa/iaa_crypto.h
View File

@ -153,6 +153,8 @@ enum iaa_mode {
struct iaa_compression_ctx {
enum iaa_mode mode;
bool verify_compress;
bool async_mode;
bool use_irq;
};
#endif

266
drivers/crypto/intel/iaa/iaa_crypto_main.c
View File

@ -122,6 +122,102 @@ out:
}
static DRIVER_ATTR_RW(verify_compress);
/*
* The iaa crypto driver supports three 'sync' methods determining how
* compressions and decompressions are performed:
*
* - sync: the compression or decompression completes before
* returning. This is the mode used by the async crypto
* interface when the sync mode is set to 'sync' and by
* the sync crypto interface regardless of setting.
*
* - async: the compression or decompression is submitted and returns
* immediately. Completion interrupts are not used so
* the caller is responsible for polling the descriptor
* for completion. This mode is applicable to only the
* async crypto interface and is ignored for anything
* else.
*
* - async_irq: the compression or decompression is submitted and
* returns immediately. Completion interrupts are
* enabled so the caller can wait for the completion and
* yield to other threads. When the compression or
* decompression completes, the completion is signaled
* and the caller awakened. This mode is applicable to
* only the async crypto interface and is ignored for
* anything else.
*
* These modes can be set using the iaa_crypto sync_mode driver
* attribute.
*/
/* Use async mode */
static bool async_mode;
/* Use interrupts */
static bool use_irq;
/**
* set_iaa_sync_mode - Set IAA sync mode
* @name: The name of the sync mode
*
* Make the IAA sync mode named @name the current sync mode used by
* compression/decompression.
*/
static int set_iaa_sync_mode(const char *name)
{
int ret = 0;
if (sysfs_streq(name, "sync")) {
async_mode = false;
use_irq = false;
} else if (sysfs_streq(name, "async")) {
async_mode = true;
use_irq = false;
} else if (sysfs_streq(name, "async_irq")) {
async_mode = true;
use_irq = true;
} else {
ret = -EINVAL;
}
return ret;
}
static ssize_t sync_mode_show(struct device_driver *driver, char *buf)
{
int ret = 0;
if (!async_mode && !use_irq)
ret = sprintf(buf, "%s\n", "sync");
else if (async_mode && !use_irq)
ret = sprintf(buf, "%s\n", "async");
else if (async_mode && use_irq)
ret = sprintf(buf, "%s\n", "async_irq");
return ret;
}
static ssize_t sync_mode_store(struct device_driver *driver,
const char *buf, size_t count)
{
int ret = -EBUSY;
mutex_lock(&iaa_devices_lock);
if (iaa_crypto_enabled)
goto out;
ret = set_iaa_sync_mode(buf);
if (ret == 0)
ret = count;
out:
mutex_unlock(&iaa_devices_lock);
return ret;
}
static DRIVER_ATTR_RW(sync_mode);
static struct iaa_compression_mode *iaa_compression_modes[IAA_COMP_MODES_MAX];
static int find_empty_iaa_compression_mode(void)
@ -1001,6 +1097,111 @@ static int deflate_generic_decompress(struct acomp_req *req)
return ret;
}
static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq,
struct acomp_req *req,
dma_addr_t *src_addr, dma_addr_t *dst_addr);
static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req,
struct idxd_wq *wq,
dma_addr_t src_addr, unsigned int slen,
dma_addr_t dst_addr, unsigned int *dlen,
u32 compression_crc);
static void iaa_desc_complete(struct idxd_desc *idxd_desc,
enum idxd_complete_type comp_type,
bool free_desc, void *__ctx,
u32 *status)
{
struct iaa_device_compression_mode *active_compression_mode;
struct iaa_compression_ctx *compression_ctx;
struct crypto_ctx *ctx = __ctx;
struct iaa_device *iaa_device;
struct idxd_device *idxd;
struct iaa_wq *iaa_wq;
struct pci_dev *pdev;
struct device *dev;
int ret, err = 0;
compression_ctx = crypto_tfm_ctx(ctx->tfm);
iaa_wq = idxd_wq_get_private(idxd_desc->wq);
iaa_device = iaa_wq->iaa_device;
idxd = iaa_device->idxd;
pdev = idxd->pdev;
dev = &pdev->dev;
active_compression_mode = get_iaa_device_compression_mode(iaa_device,
compression_ctx->mode);
dev_dbg(dev, "%s: compression mode %s,"
" ctx->src_addr %llx, ctx->dst_addr %llx\n", __func__,
active_compression_mode->name,
ctx->src_addr, ctx->dst_addr);
ret = check_completion(dev, idxd_desc->iax_completion,
ctx->compress, false);
if (ret) {
dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret);
if (!ctx->compress &&
idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) {
pr_warn("%s: falling back to deflate-generic decompress, "
"analytics error code %x\n", __func__,
idxd_desc->iax_completion->error_code);
ret = deflate_generic_decompress(ctx->req);
if (ret) {
dev_dbg(dev, "%s: deflate-generic failed ret=%d\n",
__func__, ret);
err = -EIO;
goto err;
}
} else {
err = -EIO;
goto err;
}
} else {
ctx->req->dlen = idxd_desc->iax_completion->output_size;
}
if (ctx->compress && compression_ctx->verify_compress) {
dma_addr_t src_addr, dst_addr;
u32 compression_crc;
compression_crc = idxd_desc->iax_completion->crc;
ret = iaa_remap_for_verify(dev, iaa_wq, ctx->req, &src_addr, &dst_addr);
if (ret) {
dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret);
err = -EIO;
goto out;
}
ret = iaa_compress_verify(ctx->tfm, ctx->req, iaa_wq->wq, src_addr,
ctx->req->slen, dst_addr, &ctx->req->dlen,
compression_crc);
if (ret) {
dev_dbg(dev, "%s: compress verify failed ret=%d\n", __func__, ret);
err = -EIO;
}
dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_TO_DEVICE);
dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_FROM_DEVICE);
goto out;
}
err:
dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_FROM_DEVICE);
dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_TO_DEVICE);
out:
if (ret != 0)
dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret);
if (ctx->req->base.complete)
acomp_request_complete(ctx->req, err);
if (free_desc)
idxd_free_desc(idxd_desc->wq, idxd_desc);
iaa_wq_put(idxd_desc->wq);
}
static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req,
struct idxd_wq *wq,
dma_addr_t src_addr, unsigned int slen,
@ -1049,6 +1250,22 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req,
desc->src2_size = sizeof(struct aecs_comp_table_record);
desc->completion_addr = idxd_desc->compl_dma;
if (ctx->use_irq && !disable_async) {
desc->flags |= IDXD_OP_FLAG_RCI;
idxd_desc->crypto.req = req;
idxd_desc->crypto.tfm = tfm;
idxd_desc->crypto.src_addr = src_addr;
idxd_desc->crypto.dst_addr = dst_addr;
idxd_desc->crypto.compress = true;
dev_dbg(dev, "%s use_async_irq: compression mode %s,"
" src_addr %llx, dst_addr %llx\n", __func__,
active_compression_mode->name,
src_addr, dst_addr);
} else if (ctx->async_mode && !disable_async)
req->base.data = idxd_desc;
dev_dbg(dev, "%s: compression mode %s,"
" desc->src1_addr %llx, desc->src1_size %d,"
" desc->dst_addr %llx, desc->max_dst_size %d,"
@ -1063,6 +1280,12 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req,
goto err;
}
if (ctx->async_mode && !disable_async) {
ret = -EINPROGRESS;
dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__);
goto out;
}
ret = check_completion(dev, idxd_desc->iax_completion, true, false);
if (ret) {
dev_dbg(dev, "check_completion failed ret=%d\n", ret);
@ -1073,7 +1296,8 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req,
*compression_crc = idxd_desc->iax_completion->crc;
idxd_free_desc(wq, idxd_desc);
if (!ctx->async_mode)
idxd_free_desc(wq, idxd_desc);
out:
return ret;
err:
@ -1256,6 +1480,22 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req,
desc->src1_size = slen;
desc->completion_addr = idxd_desc->compl_dma;
if (ctx->use_irq && !disable_async) {
desc->flags |= IDXD_OP_FLAG_RCI;
idxd_desc->crypto.req = req;
idxd_desc->crypto.tfm = tfm;
idxd_desc->crypto.src_addr = src_addr;
idxd_desc->crypto.dst_addr = dst_addr;
idxd_desc->crypto.compress = false;
dev_dbg(dev, "%s: use_async_irq compression mode %s,"
" src_addr %llx, dst_addr %llx\n", __func__,
active_compression_mode->name,
src_addr, dst_addr);
} else if (ctx->async_mode && !disable_async)
req->base.data = idxd_desc;
dev_dbg(dev, "%s: decompression mode %s,"
" desc->src1_addr %llx, desc->src1_size %d,"
" desc->dst_addr %llx, desc->max_dst_size %d,"
@ -1270,6 +1510,12 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req,
goto err;
}
if (ctx->async_mode && !disable_async) {
ret = -EINPROGRESS;
dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__);
goto out;
}
ret = check_completion(dev, idxd_desc->iax_completion, false, false);
if (ret) {
dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret);
@ -1292,7 +1538,8 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req,
*dlen = req->dlen;
idxd_free_desc(wq, idxd_desc);
if (!ctx->async_mode)
idxd_free_desc(wq, idxd_desc);
out:
return ret;
err:
@ -1601,6 +1848,8 @@ out:
static void compression_ctx_init(struct iaa_compression_ctx *ctx)
{
ctx->verify_compress = iaa_verify_compress;
ctx->async_mode = async_mode;
ctx->use_irq = use_irq;
}
static int iaa_comp_init_fixed(struct crypto_acomp *acomp_tfm)
@ -1809,6 +2058,7 @@ static struct idxd_device_driver iaa_crypto_driver = {
.remove = iaa_crypto_remove,
.name = IDXD_SUBDRIVER_NAME,
.type = dev_types,
.desc_complete = iaa_desc_complete,
};
static int __init iaa_crypto_init_module(void)
@ -1847,10 +2097,20 @@ static int __init iaa_crypto_init_module(void)
goto err_verify_attr_create;
}
ret = driver_create_file(&iaa_crypto_driver.drv,
&driver_attr_sync_mode);
if (ret) {
pr_debug("IAA sync mode attr creation failed\n");
goto err_sync_attr_create;
}
pr_debug("initialized\n");
out:
return ret;
err_sync_attr_create:
driver_remove_file(&iaa_crypto_driver.drv,
&driver_attr_verify_compress);
err_verify_attr_create:
idxd_driver_unregister(&iaa_crypto_driver);
err_driver_reg:
@ -1866,6 +2126,8 @@ static void __exit iaa_crypto_cleanup_module(void)
if (iaa_unregister_compression_device())
pr_debug("IAA compression device unregister failed\n");
driver_remove_file(&iaa_crypto_driver.drv,
&driver_attr_sync_mode);
driver_remove_file(&iaa_crypto_driver.drv,
&driver_attr_verify_compress);
idxd_driver_unregister(&iaa_crypto_driver);