From 6d48de655917a9d782953eba65de4e3db593ddf0 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 27 Oct 2021 16:30:01 +0300 Subject: [PATCH 001/145] crypto: atmel-aes - Reestablish the correct tfm context at dequeue In case there were more requests from different tfms in the crypto queue, only the context of the last initialized tfm was considered. Fixes: ec2088b66f7a ("crypto: atmel-aes - Allocate aes dev at tfm init time") Reported-by: Wolfgang Ocker Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- drivers/crypto/atmel-aes.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index 9391ccc03382..fe0558403191 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -960,6 +960,7 @@ static int atmel_aes_handle_queue(struct atmel_aes_dev *dd, ctx = crypto_tfm_ctx(areq->tfm); dd->areq = areq; + dd->ctx = ctx; start_async = (areq != new_areq); dd->is_async = start_async; @@ -1274,7 +1275,6 @@ static int atmel_aes_init_tfm(struct crypto_skcipher *tfm) crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx)); ctx->base.dd = dd; - ctx->base.dd->ctx = &ctx->base; ctx->base.start = atmel_aes_start; return 0; @@ -1291,7 +1291,6 @@ static int atmel_aes_ctr_init_tfm(struct crypto_skcipher *tfm) crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx)); ctx->base.dd = dd; - ctx->base.dd->ctx = &ctx->base; ctx->base.start = atmel_aes_ctr_start; return 0; @@ -1783,7 +1782,6 @@ static int atmel_aes_gcm_init(struct crypto_aead *tfm) crypto_aead_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx)); ctx->base.dd = dd; - ctx->base.dd->ctx = &ctx->base; ctx->base.start = atmel_aes_gcm_start; return 0; @@ -1927,7 +1925,6 @@ static int atmel_aes_xts_init_tfm(struct crypto_skcipher *tfm) crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx) + crypto_skcipher_reqsize(ctx->fallback_tfm)); ctx->base.dd = dd; - ctx->base.dd->ctx = &ctx->base; ctx->base.start = atmel_aes_xts_start; return 0; @@ -2154,7 +2151,6 @@ static int atmel_aes_authenc_init_tfm(struct crypto_aead *tfm, crypto_aead_set_reqsize(tfm, (sizeof(struct atmel_aes_authenc_reqctx) + auth_reqsize)); ctx->base.dd = dd; - ctx->base.dd->ctx = &ctx->base; ctx->base.start = atmel_aes_authenc_start; return 0; From 680efb33546be8960ccbb2f4e0e43034d9c93b30 Mon Sep 17 00:00:00 2001 From: Sunil Goutham Date: Fri, 29 Oct 2021 22:49:59 +0530 Subject: [PATCH 002/145] hwrng: cavium - Check health status while reading random data This RNG device is present on Marvell OcteonTx2 silicons as well and also provides entropy health status. HW continuously checks health condition of entropy and reports faults. Fault is in terms of co-processor cycles since last fault detected. This doesn't get cleared and only updated when new fault is detected. Also there are chances of detecting false positives. So to detect a entropy failure SW has to check if failures are persistent ie cycles elapsed is frequently updated by HW. This patch adds support to detect health failures using below algo. 1. Consider any fault detected before 10ms as a false positive and ignore. 10ms is chosen randomly, no significance. 2. Upon first failure detection make a note of cycles elapsed and when this error happened in realtime (cntvct). 3. Upon subsequent failure, check if this is new or a old one by comparing current cycles with the ones since last failure. cycles or time since last failure is calculated using cycles and time info captured at (2). HEALTH_CHECK status register is not available to VF, hence had to map PF registers. Also since cycles are in terms of co-processor cycles, had to retrieve co-processor clock rate from RST device. Signed-off-by: Sunil Goutham Signed-off-by: Herbert Xu --- drivers/char/hw_random/Kconfig | 2 +- drivers/char/hw_random/cavium-rng-vf.c | 194 +++++++++++++++++++++++-- drivers/char/hw_random/cavium-rng.c | 11 +- 3 files changed, 190 insertions(+), 17 deletions(-) diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index 814b3d0ca7b7..1da64771ee2e 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -414,7 +414,7 @@ config HW_RANDOM_MESON config HW_RANDOM_CAVIUM tristate "Cavium ThunderX Random Number Generator support" - depends on HW_RANDOM && PCI && (ARM64 || (COMPILE_TEST && 64BIT)) + depends on HW_RANDOM && PCI && ARM64 default HW_RANDOM help This driver provides kernel-side support for the Random Number diff --git a/drivers/char/hw_random/cavium-rng-vf.c b/drivers/char/hw_random/cavium-rng-vf.c index 3de4a6a443ef..6f66919652bf 100644 --- a/drivers/char/hw_random/cavium-rng-vf.c +++ b/drivers/char/hw_random/cavium-rng-vf.c @@ -1,10 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* - * Hardware Random Number Generator support for Cavium, Inc. - * Thunder processor family. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. + * Hardware Random Number Generator support. + * Cavium Thunder, Marvell OcteonTx/Tx2 processor families. * * Copyright (C) 2016 Cavium, Inc. */ @@ -15,16 +12,146 @@ #include #include +#include + +/* PCI device IDs */ +#define PCI_DEVID_CAVIUM_RNG_PF 0xA018 +#define PCI_DEVID_CAVIUM_RNG_VF 0xA033 + +#define HEALTH_STATUS_REG 0x38 + +/* RST device info */ +#define PCI_DEVICE_ID_RST_OTX2 0xA085 +#define RST_BOOT_REG 0x1600ULL +#define CLOCK_BASE_RATE 50000000ULL +#define MSEC_TO_NSEC(x) (x * 1000000) + struct cavium_rng { struct hwrng ops; void __iomem *result; + void __iomem *pf_regbase; + struct pci_dev *pdev; + u64 clock_rate; + u64 prev_error; + u64 prev_time; }; +static inline bool is_octeontx(struct pci_dev *pdev) +{ + if (midr_is_cpu_model_range(read_cpuid_id(), MIDR_THUNDERX_83XX, + MIDR_CPU_VAR_REV(0, 0), + MIDR_CPU_VAR_REV(3, 0)) || + midr_is_cpu_model_range(read_cpuid_id(), MIDR_THUNDERX_81XX, + MIDR_CPU_VAR_REV(0, 0), + MIDR_CPU_VAR_REV(3, 0)) || + midr_is_cpu_model_range(read_cpuid_id(), MIDR_THUNDERX, + MIDR_CPU_VAR_REV(0, 0), + MIDR_CPU_VAR_REV(3, 0))) + return true; + + return false; +} + +static u64 rng_get_coprocessor_clkrate(void) +{ + u64 ret = CLOCK_BASE_RATE * 16; /* Assume 800Mhz as default */ + struct pci_dev *pdev; + void __iomem *base; + + pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, + PCI_DEVICE_ID_RST_OTX2, NULL); + if (!pdev) + goto error; + + base = pci_ioremap_bar(pdev, 0); + if (!base) + goto error_put_pdev; + + /* RST: PNR_MUL * 50Mhz gives clockrate */ + ret = CLOCK_BASE_RATE * ((readq(base + RST_BOOT_REG) >> 33) & 0x3F); + + iounmap(base); + +error_put_pdev: + pci_dev_put(pdev); + +error: + return ret; +} + +static int check_rng_health(struct cavium_rng *rng) +{ + u64 cur_err, cur_time; + u64 status, cycles; + u64 time_elapsed; + + + /* Skip checking health for OcteonTx */ + if (!rng->pf_regbase) + return 0; + + status = readq(rng->pf_regbase + HEALTH_STATUS_REG); + if (status & BIT_ULL(0)) { + dev_err(&rng->pdev->dev, "HWRNG: Startup health test failed\n"); + return -EIO; + } + + cycles = status >> 1; + if (!cycles) + return 0; + + cur_time = arch_timer_read_counter(); + + /* RNM_HEALTH_STATUS[CYCLES_SINCE_HEALTH_FAILURE] + * Number of coprocessor cycles times 2 since the last failure. + * This field doesn't get cleared/updated until another failure. + */ + cycles = cycles / 2; + cur_err = (cycles * 1000000000) / rng->clock_rate; /* In nanosec */ + + /* Ignore errors that happenned a long time ago, these + * are most likely false positive errors. + */ + if (cur_err > MSEC_TO_NSEC(10)) { + rng->prev_error = 0; + rng->prev_time = 0; + return 0; + } + + if (rng->prev_error) { + /* Calculate time elapsed since last error + * '1' tick of CNTVCT is 10ns, since it runs at 100Mhz. + */ + time_elapsed = (cur_time - rng->prev_time) * 10; + time_elapsed += rng->prev_error; + + /* Check if current error is a new one or the old one itself. + * If error is a new one then consider there is a persistent + * issue with entropy, declare hardware failure. + */ + if (cur_err < time_elapsed) { + dev_err(&rng->pdev->dev, "HWRNG failure detected\n"); + rng->prev_error = cur_err; + rng->prev_time = cur_time; + return -EIO; + } + } + + rng->prev_error = cur_err; + rng->prev_time = cur_time; + return 0; +} + /* Read data from the RNG unit */ static int cavium_rng_read(struct hwrng *rng, void *dat, size_t max, bool wait) { struct cavium_rng *p = container_of(rng, struct cavium_rng, ops); unsigned int size = max; + int err = 0; + + err = check_rng_health(p); + if (err) + return err; while (size >= 8) { *((u64 *)dat) = readq(p->result); @@ -39,6 +166,39 @@ static int cavium_rng_read(struct hwrng *rng, void *dat, size_t max, bool wait) return max; } +static int cavium_map_pf_regs(struct cavium_rng *rng) +{ + struct pci_dev *pdev; + + /* Health status is not supported on 83xx, skip mapping PF CSRs */ + if (is_octeontx(rng->pdev)) { + rng->pf_regbase = NULL; + return 0; + } + + pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, + PCI_DEVID_CAVIUM_RNG_PF, NULL); + if (!pdev) { + dev_err(&pdev->dev, "Cannot find RNG PF device\n"); + return -EIO; + } + + rng->pf_regbase = ioremap(pci_resource_start(pdev, 0), + pci_resource_len(pdev, 0)); + if (!rng->pf_regbase) { + dev_err(&pdev->dev, "Failed to map PF CSR region\n"); + pci_dev_put(pdev); + return -ENOMEM; + } + + pci_dev_put(pdev); + + /* Get co-processor clock rate */ + rng->clock_rate = rng_get_coprocessor_clkrate(); + + return 0; +} + /* Map Cavium RNG to an HWRNG object */ static int cavium_rng_probe_vf(struct pci_dev *pdev, const struct pci_device_id *id) @@ -50,6 +210,8 @@ static int cavium_rng_probe_vf(struct pci_dev *pdev, if (!rng) return -ENOMEM; + rng->pdev = pdev; + /* Map the RNG result */ rng->result = pcim_iomap(pdev, 0, 0); if (!rng->result) { @@ -67,6 +229,11 @@ static int cavium_rng_probe_vf(struct pci_dev *pdev, pci_set_drvdata(pdev, rng); + /* Health status is available only at PF, hence map PF registers. */ + ret = cavium_map_pf_regs(rng); + if (ret) + return ret; + ret = devm_hwrng_register(&pdev->dev, &rng->ops); if (ret) { dev_err(&pdev->dev, "Error registering device as HWRNG.\n"); @@ -76,10 +243,18 @@ static int cavium_rng_probe_vf(struct pci_dev *pdev, return 0; } +/* Remove the VF */ +static void cavium_rng_remove_vf(struct pci_dev *pdev) +{ + struct cavium_rng *rng; + + rng = pci_get_drvdata(pdev); + iounmap(rng->pf_regbase); +} static const struct pci_device_id cavium_rng_vf_id_table[] = { - { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xa033), 0, 0, 0}, - {0,}, + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CAVIUM_RNG_VF) }, + { 0, } }; MODULE_DEVICE_TABLE(pci, cavium_rng_vf_id_table); @@ -87,8 +262,9 @@ static struct pci_driver cavium_rng_vf_driver = { .name = "cavium_rng_vf", .id_table = cavium_rng_vf_id_table, .probe = cavium_rng_probe_vf, + .remove = cavium_rng_remove_vf, }; module_pci_driver(cavium_rng_vf_driver); MODULE_AUTHOR("Omer Khaliq "); -MODULE_LICENSE("GPL"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/char/hw_random/cavium-rng.c b/drivers/char/hw_random/cavium-rng.c index 63d6e68c24d2..b96579222408 100644 --- a/drivers/char/hw_random/cavium-rng.c +++ b/drivers/char/hw_random/cavium-rng.c @@ -1,10 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* - * Hardware Random Number Generator support for Cavium Inc. - * Thunder processor family. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. + * Hardware Random Number Generator support. + * Cavium Thunder, Marvell OcteonTx/Tx2 processor families. * * Copyright (C) 2016 Cavium, Inc. */ @@ -91,4 +88,4 @@ static struct pci_driver cavium_rng_pf_driver = { module_pci_driver(cavium_rng_pf_driver); MODULE_AUTHOR("Omer Khaliq "); -MODULE_LICENSE("GPL"); +MODULE_LICENSE("GPL v2"); From efd21e10fc3bf4c6da122470a5ae89ec4ed8d180 Mon Sep 17 00:00:00 2001 From: Meng Li Date: Mon, 1 Nov 2021 11:13:53 +0800 Subject: [PATCH 003/145] crypto: caam - replace this_cpu_ptr with raw_cpu_ptr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When enable the kernel debug config, there is below calltrace detected: BUG: using smp_processor_id() in preemptible [00000000] code: cryptomgr_test/339 caller is debug_smp_processor_id+0x20/0x30 CPU: 9 PID: 339 Comm: cryptomgr_test Not tainted 5.10.63-yocto-standard #1 Hardware name: NXP Layerscape LX2160ARDB (DT) Call trace: dump_backtrace+0x0/0x1a0 show_stack+0x24/0x30 dump_stack+0xf0/0x13c check_preemption_disabled+0x100/0x110 debug_smp_processor_id+0x20/0x30 dpaa2_caam_enqueue+0x10c/0x25c ...... cryptomgr_test+0x38/0x60 kthread+0x158/0x164 ret_from_fork+0x10/0x38 According to the comment in commit ac5d15b4519f("crypto: caam/qi2 - use affine DPIOs "), because preemption is no longer disabled while trying to enqueue an FQID, it might be possible to run the enqueue on a different CPU(due to migration, when in process context), however this wouldn't be a functionality issue. But there will be above calltrace when enable kernel debug config. So, replace this_cpu_ptr with raw_cpu_ptr to avoid above call trace. Fixes: ac5d15b4519f ("crypto: caam/qi2 - use affine DPIOs") Cc: stable@vger.kernel.org Signed-off-by: Meng Li Reviewed-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg_qi2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 8b8ed77d8715..6753f0e6e55d 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -5470,7 +5470,7 @@ int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req) dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1])); dpaa2_fd_set_flc(&fd, req->flc_dma); - ppriv = this_cpu_ptr(priv->ppriv); + ppriv = raw_cpu_ptr(priv->ppriv); for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) { err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid, &fd); From 94ad2d19a97efdb603a21fcad0625f466f1cdd0f Mon Sep 17 00:00:00 2001 From: Wei Yongjun Date: Mon, 1 Nov 2021 14:02:33 +0000 Subject: [PATCH 004/145] crypto: keembay-ocs-ecc - Fix error return code in kmb_ocs_ecc_probe() Fix to return negative error code -ENOMEM from the error handling case instead of 0, as done elsewhere in this function. Fixes: c9f608c38009 ("crypto: keembay-ocs-ecc - Add Keem Bay OCS ECC Driver") Reported-by: Hulk Robot Signed-off-by: Wei Yongjun Reviewed-by: Daniele Alessandrelli Signed-off-by: Herbert Xu --- drivers/crypto/keembay/keembay-ocs-ecc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/crypto/keembay/keembay-ocs-ecc.c b/drivers/crypto/keembay/keembay-ocs-ecc.c index 679e6ae295e0..5d0785d3f1b5 100644 --- a/drivers/crypto/keembay/keembay-ocs-ecc.c +++ b/drivers/crypto/keembay/keembay-ocs-ecc.c @@ -930,6 +930,7 @@ static int kmb_ocs_ecc_probe(struct platform_device *pdev) ecc_dev->engine = crypto_engine_alloc_init(dev, 1); if (!ecc_dev->engine) { dev_err(dev, "Could not allocate crypto engine\n"); + rc = -ENOMEM; goto list_del; } From 7875506f7a755cdafe231ca310c8fbca793d262f Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Wed, 3 Nov 2021 10:47:11 +0100 Subject: [PATCH 005/145] MAINTAINERS: rectify entry for INTEL KEEM BAY OCS ECC CRYPTO DRIVER Commit c9f608c38009 ("crypto: keembay-ocs-ecc - Add Keem Bay OCS ECC Driver") only adds drivers/crypto/keembay/keembay-ocs-ecc.c, but adds a file entry drivers/crypto/keembay/ocs-ecc-curve-defs.h in MAINTAINERS. Hence, ./scripts/get_maintainer.pl --self-test=patterns warns: warning: no file matches F: drivers/crypto/keembay/ocs-ecc-curve-defs.h Assuming that this header is obsolete and will not be included in the repository, remove the unneeded file entry from MAINTAINERS. Signed-off-by: Lukas Bulwahn Reviewed-by: Daniele Alessandrelli Signed-off-by: Herbert Xu --- MAINTAINERS | 1 - 1 file changed, 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 7a2345ce8521..00475646e3e3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9673,7 +9673,6 @@ F: Documentation/devicetree/bindings/crypto/intel,keembay-ocs-ecc.yaml F: drivers/crypto/keembay/Kconfig F: drivers/crypto/keembay/Makefile F: drivers/crypto/keembay/keembay-ocs-ecc.c -F: drivers/crypto/keembay/ocs-ecc-curve-defs.h INTEL KEEM BAY OCS HCU CRYPTO DRIVER M: Daniele Alessandrelli From 574c833ef3a60dc5dff6c9e885a30ba13e88971e Mon Sep 17 00:00:00 2001 From: Yang Guang Date: Thu, 4 Nov 2021 14:19:10 +0800 Subject: [PATCH 006/145] crypto: hisilicon/hpre - use swap() to make code cleaner Use the macro 'swap()' defined in 'include/linux/minmax.h' to avoid opencoding it. Reported-by: Zeal Robot Signed-off-by: Yang Guang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index a032c192ef1d..0f1724d355b8 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -1177,13 +1177,10 @@ static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm) static void hpre_key_to_big_end(u8 *data, int len) { int i, j; - u8 tmp; for (i = 0; i < len / 2; i++) { j = len - i - 1; - tmp = data[j]; - data[j] = data[i]; - data[i] = tmp; + swap(data[j], data[i]); } } From 4a9dbd021970ffe1b92521328377b699acba7c52 Mon Sep 17 00:00:00 2001 From: Chengfeng Ye Date: Thu, 4 Nov 2021 06:28:07 -0700 Subject: [PATCH 007/145] crypto: qce - fix uaf on qce_aead_register_one Pointer alg points to sub field of tmpl, it is dereferenced after tmpl is freed. Fix this by accessing alg before free tmpl. Fixes: 9363efb4 ("crypto: qce - Add support for AEAD algorithms") Signed-off-by: Chengfeng Ye Acked-by: Thara Gopinath Signed-off-by: Herbert Xu --- drivers/crypto/qce/aead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/qce/aead.c b/drivers/crypto/qce/aead.c index 290e2446a2f3..97a530171f07 100644 --- a/drivers/crypto/qce/aead.c +++ b/drivers/crypto/qce/aead.c @@ -802,8 +802,8 @@ static int qce_aead_register_one(const struct qce_aead_def *def, struct qce_devi ret = crypto_register_aead(alg); if (ret) { - kfree(tmpl); dev_err(qce->dev, "%s registration failed\n", alg->base.cra_name); + kfree(tmpl); return ret; } From b4cb4d31631912842eb7dce02b4350cbb7562d5e Mon Sep 17 00:00:00 2001 From: Chengfeng Ye Date: Thu, 4 Nov 2021 06:38:31 -0700 Subject: [PATCH 008/145] crypto: qce - fix uaf on qce_ahash_register_one Pointer base points to sub field of tmpl, it is dereferenced after tmpl is freed. Fix this by accessing base before free tmpl. Fixes: ec8f5d8f ("crypto: qce - Qualcomm crypto engine driver") Signed-off-by: Chengfeng Ye Acked-by: Thara Gopinath Signed-off-by: Herbert Xu --- drivers/crypto/qce/sha.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c index 8e6fcf2c21cc..59159f5e64e5 100644 --- a/drivers/crypto/qce/sha.c +++ b/drivers/crypto/qce/sha.c @@ -498,8 +498,8 @@ static int qce_ahash_register_one(const struct qce_ahash_def *def, ret = crypto_register_ahash(alg); if (ret) { - kfree(tmpl); dev_err(qce->dev, "%s registration failed\n", base->cra_name); + kfree(tmpl); return ret; } From e9c195aaeed1b45c9012adbe29dedb6031e85aa8 Mon Sep 17 00:00:00 2001 From: Chengfeng Ye Date: Thu, 4 Nov 2021 06:46:42 -0700 Subject: [PATCH 009/145] crypto: qce - fix uaf on qce_skcipher_register_one Pointer alg points to sub field of tmpl, it is dereferenced after tmpl is freed. Fix this by accessing alg before free tmpl. Fixes: ec8f5d8f ("crypto: qce - Qualcomm crypto engine driver") Signed-off-by: Chengfeng Ye Acked-by: Thara Gopinath Signed-off-by: Herbert Xu --- drivers/crypto/qce/skcipher.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/qce/skcipher.c b/drivers/crypto/qce/skcipher.c index 8ff10928f581..3d27cd5210ef 100644 --- a/drivers/crypto/qce/skcipher.c +++ b/drivers/crypto/qce/skcipher.c @@ -484,8 +484,8 @@ static int qce_skcipher_register_one(const struct qce_skcipher_def *def, ret = crypto_register_skcipher(alg); if (ret) { - kfree(tmpl); dev_err(qce->dev, "%s registration failed\n", alg->base.cra_name); + kfree(tmpl); return ret; } From a9887010ed2da3fddaff83ceec80e2b71be8a966 Mon Sep 17 00:00:00 2001 From: Lei He Date: Fri, 5 Nov 2021 20:25:31 +0800 Subject: [PATCH 010/145] crypto: testmgr - Fix wrong test case of RSA According to the BER encoding rules, integer value should be encoded as two's complement, and if the highest bit of a positive integer is 1, should add a leading zero-octet. The kernel's built-in RSA algorithm cannot recognize negative numbers when parsing keys, so it can pass this test case. Export the key to file and run the following command to verify the fix result: openssl asn1parse -inform DER -in /path/to/key/file Signed-off-by: Lei He Signed-off-by: Herbert Xu --- crypto/testmgr.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 779720bf9364..a253d66ba1c1 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -257,9 +257,9 @@ static const struct akcipher_testvec rsa_tv_template[] = { }, { #endif .key = - "\x30\x82\x02\x1F" /* sequence of 543 bytes */ + "\x30\x82\x02\x20" /* sequence of 544 bytes */ "\x02\x01\x01" /* version - integer of 1 byte */ - "\x02\x82\x01\x00" /* modulus - integer of 256 bytes */ + "\x02\x82\x01\x01\x00" /* modulus - integer of 256 bytes */ "\xDB\x10\x1A\xC2\xA3\xF1\xDC\xFF\x13\x6B\xED\x44\xDF\xF0\x02\x6D" "\x13\xC7\x88\xDA\x70\x6B\x54\xF1\xE8\x27\xDC\xC3\x0F\x99\x6A\xFA" "\xC6\x67\xFF\x1D\x1E\x3C\x1D\xC1\xB5\x5F\x6C\xC0\xB2\x07\x3A\x6D" @@ -299,7 +299,7 @@ static const struct akcipher_testvec rsa_tv_template[] = { "\x02\x01\x00" /* exponent1 - integer of 1 byte */ "\x02\x01\x00" /* exponent2 - integer of 1 byte */ "\x02\x01\x00", /* coefficient - integer of 1 byte */ - .key_len = 547, + .key_len = 548, .m = "\x54\x85\x9b\x34\x2c\x49\xea\x2a", .c = "\xb2\x97\x76\xb4\xae\x3e\x38\x3c\x7e\x64\x1f\xcc\xa2\x7f\xf6\xbe" From 3121d5d1181885d9e01436fbdac78646617fb42f Mon Sep 17 00:00:00 2001 From: chiminghao Date: Tue, 9 Nov 2021 08:35:03 +0000 Subject: [PATCH 011/145] crypto: octeontx2 - use swap() to make code cleaner Fix the following coccicheck REVIEW: ./drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c:1688:16-17 use swap() to make code cleaner Reported-by: Zeal Robot Signed-off-by: chiminghao Signed-off-by: Herbert Xu --- drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c index 877a948469bd..2748a3327e39 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c @@ -1682,11 +1682,8 @@ static void swap_func(void *lptr, void *rptr, int size) { struct cpt_device_desc *ldesc = lptr; struct cpt_device_desc *rdesc = rptr; - struct cpt_device_desc desc; - desc = *ldesc; - *ldesc = *rdesc; - *rdesc = desc; + swap(*ldesc, *rdesc); } int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod, From 882ed23e103f2cc8222eb3d280ddf1af167fa9ea Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Fri, 12 Nov 2021 17:49:23 +0100 Subject: [PATCH 012/145] crypto: ccree - remove redundant 'flush_workqueue()' calls 'destroy_workqueue()' already drains the queue before destroying it, so there is no need to flush it explicitly. Remove the redundant 'flush_workqueue()' calls. This was generated with coccinelle: @@ expression E; @@ - flush_workqueue(E); destroy_workqueue(E); Signed-off-by: Christophe JAILLET Reviewed-by: Cristian Marussi Acked-by: Gilad Ben-Yossef Signed-off-by: Herbert Xu --- drivers/crypto/ccree/cc_request_mgr.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/crypto/ccree/cc_request_mgr.c b/drivers/crypto/ccree/cc_request_mgr.c index 33fb27745d52..887162df50f9 100644 --- a/drivers/crypto/ccree/cc_request_mgr.c +++ b/drivers/crypto/ccree/cc_request_mgr.c @@ -101,7 +101,6 @@ void cc_req_mgr_fini(struct cc_drvdata *drvdata) dev_dbg(dev, "max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots); #ifdef COMP_IN_WQ - flush_workqueue(req_mgr_h->workq); destroy_workqueue(req_mgr_h->workq); #else /* Kill tasklet */ From 370a40ee228340b5141e5d89dee3186452d9f5fb Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sat, 13 Nov 2021 14:30:23 +0800 Subject: [PATCH 013/145] crypto: ccp - no need to initialise statics to 0 Static variables do not need to be initialized to 0. Signed-off-by: Jason Wang Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-dev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c index 9ce4b68e9c48..c531d13d971f 100644 --- a/drivers/crypto/ccp/ccp-dev.c +++ b/drivers/crypto/ccp/ccp-dev.c @@ -31,7 +31,7 @@ #define MAX_CCPS 32 /* Limit CCP use to a specifed number of queues per device */ -static unsigned int nqueues = 0; +static unsigned int nqueues; module_param(nqueues, uint, 0444); MODULE_PARM_DESC(nqueues, "Number of queues per CCP (minimum 1; default: all available)"); From ce8ce31b2c5c8b18667784b8c515650c65d57b4e Mon Sep 17 00:00:00 2001 From: Nicolai Stange Date: Mon, 15 Nov 2021 15:18:04 +0100 Subject: [PATCH 014/145] crypto: drbg - prepare for more fine-grained tracking of seeding state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are two different randomness sources the DRBGs are getting seeded from, namely the jitterentropy source (if enabled) and get_random_bytes(). At initial DRBG seeding time during boot, the latter might not have collected sufficient entropy for seeding itself yet and thus, the DRBG implementation schedules a reseed work from a random_ready_callback once that has happened. This is particularly important for the !->pr DRBG instances, for which (almost) no further reseeds are getting triggered during their lifetime. Because collecting data from the jitterentropy source is a rather expensive operation, the aforementioned asynchronously scheduled reseed work restricts itself to get_random_bytes() only. That is, it in some sense amends the initial DRBG seed derived from jitterentropy output at full (estimated) entropy with fresh randomness obtained from get_random_bytes() once that has been seeded with sufficient entropy itself. With the advent of rng_is_initialized(), there is no real need for doing the reseed operation from an asynchronously scheduled work anymore and a subsequent patch will make it synchronous by moving it next to related logic already present in drbg_generate(). However, for tracking whether a full reseed including the jitterentropy source is required or a "partial" reseed involving only get_random_bytes() would be sufficient already, the boolean struct drbg_state's ->seeded member must become a tristate value. Prepare for this by introducing the new enum drbg_seed_state and change struct drbg_state's ->seeded member's type from bool to that type. For facilitating review, enum drbg_seed_state is made to only contain two members corresponding to the former ->seeded values of false and true resp. at this point: DRBG_SEED_STATE_UNSEEDED and DRBG_SEED_STATE_FULL. A third one for tracking the intermediate state of "seeded from jitterentropy only" will be introduced with a subsequent patch. There is no change in behaviour at this point. Signed-off-by: Nicolai Stange Reviewed-by: Stephan Müller Signed-off-by: Herbert Xu --- crypto/drbg.c | 19 ++++++++++--------- include/crypto/drbg.h | 7 ++++++- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/crypto/drbg.c b/crypto/drbg.c index f72f340a1321..6329c70e5bbb 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -1043,7 +1043,7 @@ static inline int __drbg_seed(struct drbg_state *drbg, struct list_head *seed, if (ret) return ret; - drbg->seeded = true; + drbg->seeded = DRBG_SEED_STATE_FULL; /* 10.1.1.2 / 10.1.1.3 step 5 */ drbg->reseed_ctr = 1; @@ -1088,14 +1088,14 @@ static void drbg_async_seed(struct work_struct *work) if (ret) goto unlock; - /* Set seeded to false so that if __drbg_seed fails the - * next generate call will trigger a reseed. + /* Reset ->seeded so that if __drbg_seed fails the next + * generate call will trigger a reseed. */ - drbg->seeded = false; + drbg->seeded = DRBG_SEED_STATE_UNSEEDED; __drbg_seed(drbg, &seedlist, true); - if (drbg->seeded) + if (drbg->seeded == DRBG_SEED_STATE_FULL) drbg->reseed_threshold = drbg_max_requests(drbg); unlock: @@ -1386,13 +1386,14 @@ static int drbg_generate(struct drbg_state *drbg, * here. The spec is a bit convoluted here, we make it simpler. */ if (drbg->reseed_threshold < drbg->reseed_ctr) - drbg->seeded = false; + drbg->seeded = DRBG_SEED_STATE_UNSEEDED; - if (drbg->pr || !drbg->seeded) { + if (drbg->pr || drbg->seeded == DRBG_SEED_STATE_UNSEEDED) { pr_devel("DRBG: reseeding before generation (prediction " "resistance: %s, state %s)\n", drbg->pr ? "true" : "false", - drbg->seeded ? "seeded" : "unseeded"); + (drbg->seeded == DRBG_SEED_STATE_FULL ? + "seeded" : "unseeded")); /* 9.3.1 steps 7.1 through 7.3 */ len = drbg_seed(drbg, addtl, true); if (len) @@ -1578,7 +1579,7 @@ static int drbg_instantiate(struct drbg_state *drbg, struct drbg_string *pers, if (!drbg->core) { drbg->core = &drbg_cores[coreref]; drbg->pr = pr; - drbg->seeded = false; + drbg->seeded = DRBG_SEED_STATE_UNSEEDED; drbg->reseed_threshold = drbg_max_requests(drbg); ret = drbg_alloc_state(drbg); diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h index c4165126937e..92a87b23ad2f 100644 --- a/include/crypto/drbg.h +++ b/include/crypto/drbg.h @@ -105,6 +105,11 @@ struct drbg_test_data { struct drbg_string *testentropy; /* TEST PARAMETER: test entropy */ }; +enum drbg_seed_state { + DRBG_SEED_STATE_UNSEEDED, + DRBG_SEED_STATE_FULL, +}; + struct drbg_state { struct mutex drbg_mutex; /* lock around DRBG */ unsigned char *V; /* internal state 10.1.1.1 1a) */ @@ -127,7 +132,7 @@ struct drbg_state { struct crypto_wait ctr_wait; /* CTR mode async wait obj */ struct scatterlist sg_in, sg_out; /* CTR mode SGLs */ - bool seeded; /* DRBG fully seeded? */ + enum drbg_seed_state seeded; /* DRBG fully seeded? */ bool pr; /* Prediction resistance enabled? */ bool fips_primed; /* Continuous test primed? */ unsigned char *prev; /* FIPS 140-2 continuous test value */ From 2bcd25443868aa8863779a6ebc6c9319633025d2 Mon Sep 17 00:00:00 2001 From: Nicolai Stange Date: Mon, 15 Nov 2021 15:18:05 +0100 Subject: [PATCH 015/145] crypto: drbg - track whether DRBG was seeded with !rng_is_initialized() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, the DRBG implementation schedules asynchronous works from random_ready_callbacks for reseeding the DRBG instances with output from get_random_bytes() once the latter has sufficient entropy available. However, as the get_random_bytes() initialization state can get queried by means of rng_is_initialized() now, there is no real need for this asynchronous reseeding logic anymore and it's better to keep things simple by doing it synchronously when needed instead, i.e. from drbg_generate() once rng_is_initialized() has flipped to true. Of course, for this to work, drbg_generate() would need some means by which it can tell whether or not rng_is_initialized() has flipped to true since the last seeding from get_random_bytes(). Or equivalently, whether or not the last seed from get_random_bytes() has happened when rng_is_initialized() was still evaluating to false. As it currently stands, enum drbg_seed_state allows for the representation of two different DRBG seeding states: DRBG_SEED_STATE_UNSEEDED and DRBG_SEED_STATE_FULL. The former makes drbg_generate() to invoke a full reseeding operation involving both, the rather expensive jitterentropy as well as the get_random_bytes() randomness sources. The DRBG_SEED_STATE_FULL state on the other hand implies that no reseeding at all is required for a !->pr DRBG variant. Introduce the new DRBG_SEED_STATE_PARTIAL state to enum drbg_seed_state for representing the condition that a DRBG was being seeded when rng_is_initialized() had still been false. In particular, this new state implies that - the given DRBG instance has been fully seeded from the jitterentropy source (if enabled) - and drbg_generate() is supposed to reseed from get_random_bytes() *only* once rng_is_initialized() turns to true. Up to now, the __drbg_seed() helper used to set the given DRBG instance's ->seeded state to constant DRBG_SEED_STATE_FULL. Introduce a new argument allowing for the specification of the to be written ->seeded value instead. Make the first of its two callers, drbg_seed(), determine the appropriate value based on rng_is_initialized(). The remaining caller, drbg_async_seed(), is known to get invoked only once rng_is_initialized() is true, hence let it pass constant DRBG_SEED_STATE_FULL for the new argument to __drbg_seed(). There is no change in behaviour, except for that the pr_devel() in drbg_generate() would now report "unseeded" for ->pr DRBG instances which had last been seeded when rng_is_initialized() was still evaluating to false. Signed-off-by: Nicolai Stange Reviewed-by: Stephan Müller Signed-off-by: Herbert Xu --- crypto/drbg.c | 12 ++++++++---- include/crypto/drbg.h | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crypto/drbg.c b/crypto/drbg.c index 6329c70e5bbb..d71c704d0cd2 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -1036,14 +1036,14 @@ static const struct drbg_state_ops drbg_hash_ops = { ******************************************************************/ static inline int __drbg_seed(struct drbg_state *drbg, struct list_head *seed, - int reseed) + int reseed, enum drbg_seed_state new_seed_state) { int ret = drbg->d_ops->update(drbg, seed, reseed); if (ret) return ret; - drbg->seeded = DRBG_SEED_STATE_FULL; + drbg->seeded = new_seed_state; /* 10.1.1.2 / 10.1.1.3 step 5 */ drbg->reseed_ctr = 1; @@ -1093,7 +1093,7 @@ static void drbg_async_seed(struct work_struct *work) */ drbg->seeded = DRBG_SEED_STATE_UNSEEDED; - __drbg_seed(drbg, &seedlist, true); + __drbg_seed(drbg, &seedlist, true, DRBG_SEED_STATE_FULL); if (drbg->seeded == DRBG_SEED_STATE_FULL) drbg->reseed_threshold = drbg_max_requests(drbg); @@ -1123,6 +1123,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, unsigned int entropylen = drbg_sec_strength(drbg->core->flags); struct drbg_string data1; LIST_HEAD(seedlist); + enum drbg_seed_state new_seed_state = DRBG_SEED_STATE_FULL; /* 9.1 / 9.2 / 9.3.1 step 3 */ if (pers && pers->len > (drbg_max_addtl(drbg))) { @@ -1150,6 +1151,9 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, BUG_ON((entropylen * 2) > sizeof(entropy)); /* Get seed from in-kernel /dev/urandom */ + if (!rng_is_initialized()) + new_seed_state = DRBG_SEED_STATE_PARTIAL; + ret = drbg_get_random_bytes(drbg, entropy, entropylen); if (ret) goto out; @@ -1206,7 +1210,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, memset(drbg->C, 0, drbg_statelen(drbg)); } - ret = __drbg_seed(drbg, &seedlist, reseed); + ret = __drbg_seed(drbg, &seedlist, reseed, new_seed_state); out: memzero_explicit(entropy, entropylen * 2); diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h index 92a87b23ad2f..3ebdb1effe74 100644 --- a/include/crypto/drbg.h +++ b/include/crypto/drbg.h @@ -107,6 +107,7 @@ struct drbg_test_data { enum drbg_seed_state { DRBG_SEED_STATE_UNSEEDED, + DRBG_SEED_STATE_PARTIAL, /* Seeded with !rng_is_initialized() */ DRBG_SEED_STATE_FULL, }; From 262d83a4290c331cd4f617a457408bdb82fbb738 Mon Sep 17 00:00:00 2001 From: Nicolai Stange Date: Mon, 15 Nov 2021 15:18:06 +0100 Subject: [PATCH 016/145] crypto: drbg - move dynamic ->reseed_threshold adjustments to __drbg_seed() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit 42ea507fae1a ("crypto: drbg - reseed often if seedsource is degraded"), the maximum seed lifetime represented by ->reseed_threshold gets temporarily lowered if the get_random_bytes() source cannot provide sufficient entropy yet, as is common during boot, and restored back to the original value again once that has changed. More specifically, if the add_random_ready_callback() invoked from drbg_prepare_hrng() in the course of DRBG instantiation does not return -EALREADY, that is, if get_random_bytes() has not been fully initialized at this point yet, drbg_prepare_hrng() will lower ->reseed_threshold to a value of 50. The drbg_async_seed() scheduled from said random_ready_callback will eventually restore the original value. A future patch will replace the random_ready_callback based notification mechanism and thus, there will be no add_random_ready_callback() return value anymore which could get compared to -EALREADY. However, there's __drbg_seed() which gets invoked in the course of both, the DRBG instantiation as well as the eventual reseeding from get_random_bytes() in aforementioned drbg_async_seed(), if any. Moreover, it knows about the get_random_bytes() initialization state by the time the seed data had been obtained from it: the new_seed_state argument introduced with the previous patch would get set to DRBG_SEED_STATE_PARTIAL in case get_random_bytes() had not been fully initialized yet and to DRBG_SEED_STATE_FULL otherwise. Thus, __drbg_seed() provides a convenient alternative for managing that ->reseed_threshold lowering and restoring at a central place. Move all ->reseed_threshold adjustment code from drbg_prepare_hrng() and drbg_async_seed() respectively to __drbg_seed(). Make __drbg_seed() lower the ->reseed_threshold to 50 in case its new_seed_state argument equals DRBG_SEED_STATE_PARTIAL and let it restore the original value otherwise. There is no change in behaviour. Signed-off-by: Nicolai Stange Reviewed-by: Stephan Müller Signed-off-by: Herbert Xu --- crypto/drbg.c | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/crypto/drbg.c b/crypto/drbg.c index d71c704d0cd2..9b233e86a5f0 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -1047,6 +1047,27 @@ static inline int __drbg_seed(struct drbg_state *drbg, struct list_head *seed, /* 10.1.1.2 / 10.1.1.3 step 5 */ drbg->reseed_ctr = 1; + switch (drbg->seeded) { + case DRBG_SEED_STATE_UNSEEDED: + /* Impossible, but handle it to silence compiler warnings. */ + fallthrough; + case DRBG_SEED_STATE_PARTIAL: + /* + * Require frequent reseeds until the seed source is + * fully initialized. + */ + drbg->reseed_threshold = 50; + break; + + case DRBG_SEED_STATE_FULL: + /* + * Seed source has become fully initialized, frequent + * reseeds no longer required. + */ + drbg->reseed_threshold = drbg_max_requests(drbg); + break; + } + return ret; } @@ -1095,9 +1116,6 @@ static void drbg_async_seed(struct work_struct *work) __drbg_seed(drbg, &seedlist, true, DRBG_SEED_STATE_FULL); - if (drbg->seeded == DRBG_SEED_STATE_FULL) - drbg->reseed_threshold = drbg_max_requests(drbg); - unlock: mutex_unlock(&drbg->drbg_mutex); @@ -1534,12 +1552,6 @@ static int drbg_prepare_hrng(struct drbg_state *drbg) return err; } - /* - * Require frequent reseeds until the seed source is fully - * initialized. - */ - drbg->reseed_threshold = 50; - return err; } From 074bcd4000e0d812bc253f86fedc40f81ed59ccc Mon Sep 17 00:00:00 2001 From: Nicolai Stange Date: Mon, 15 Nov 2021 15:18:07 +0100 Subject: [PATCH 017/145] crypto: drbg - make reseeding from get_random_bytes() synchronous get_random_bytes() usually hasn't full entropy available by the time DRBG instances are first getting seeded from it during boot. Thus, the DRBG implementation registers random_ready_callbacks which would in turn schedule some work for reseeding the DRBGs once get_random_bytes() has sufficient entropy available. For reference, the relevant history around handling DRBG (re)seeding in the context of a not yet fully seeded get_random_bytes() is: commit 16b369a91d0d ("random: Blocking API for accessing nonblocking_pool") commit 4c7879907edd ("crypto: drbg - add async seeding operation") commit 205a525c3342 ("random: Add callback API for random pool readiness") commit 57225e679788 ("crypto: drbg - Use callback API for random readiness") commit c2719503f5e1 ("random: Remove kernel blocking API") However, some time later, the initialization state of get_random_bytes() has been made queryable via rng_is_initialized() introduced with commit 9a47249d444d ("random: Make crng state queryable"). This primitive now allows for streamlining the DRBG reseeding from get_random_bytes() by replacing that aforementioned asynchronous work scheduling from random_ready_callbacks with some simpler, synchronous code in drbg_generate() next to the related logic already present therein. Apart from improving overall code readability, this change will also enable DRBG users to rely on wait_for_random_bytes() for ensuring that the initial seeding has completed, if desired. The previous patches already laid the grounds by making drbg_seed() to record at each DRBG instance whether it was being seeded at a time when rng_is_initialized() still had been false as indicated by ->seeded == DRBG_SEED_STATE_PARTIAL. All that remains to be done now is to make drbg_generate() check for this condition, determine whether rng_is_initialized() has flipped to true in the meanwhile and invoke a reseed from get_random_bytes() if so. Make this move: - rename the former drbg_async_seed() work handler, i.e. the one in charge of reseeding a DRBG instance from get_random_bytes(), to "drbg_seed_from_random()", - change its signature as appropriate, i.e. make it take a struct drbg_state rather than a work_struct and change its return type from "void" to "int" in order to allow for passing error information from e.g. its __drbg_seed() invocation onwards to callers, - make drbg_generate() invoke this drbg_seed_from_random() once it encounters a DRBG instance with ->seeded == DRBG_SEED_STATE_PARTIAL by the time rng_is_initialized() has flipped to true and - prune everything related to the former, random_ready_callback based mechanism. As drbg_seed_from_random() is now getting invoked from drbg_generate() with the ->drbg_mutex being held, it must not attempt to recursively grab it once again. Remove the corresponding mutex operations from what is now drbg_seed_from_random(). Furthermore, as drbg_seed_from_random() can now report errors directly to its caller, there's no need for it to temporarily switch the DRBG's ->seeded state to DRBG_SEED_STATE_UNSEEDED so that a failure of the subsequently invoked __drbg_seed() will get signaled to drbg_generate(). Don't do it then. Signed-off-by: Nicolai Stange Signed-off-by: Herbert Xu --- crypto/drbg.c | 62 ++++++++----------------------------------- include/crypto/drbg.h | 2 -- 2 files changed, 11 insertions(+), 53 deletions(-) diff --git a/crypto/drbg.c b/crypto/drbg.c index 9b233e86a5f0..2b03a05a9e99 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -1087,12 +1087,10 @@ static inline int drbg_get_random_bytes(struct drbg_state *drbg, return 0; } -static void drbg_async_seed(struct work_struct *work) +static int drbg_seed_from_random(struct drbg_state *drbg) { struct drbg_string data; LIST_HEAD(seedlist); - struct drbg_state *drbg = container_of(work, struct drbg_state, - seed_work); unsigned int entropylen = drbg_sec_strength(drbg->core->flags); unsigned char entropy[32]; int ret; @@ -1103,23 +1101,15 @@ static void drbg_async_seed(struct work_struct *work) drbg_string_fill(&data, entropy, entropylen); list_add_tail(&data.list, &seedlist); - mutex_lock(&drbg->drbg_mutex); - ret = drbg_get_random_bytes(drbg, entropy, entropylen); if (ret) - goto unlock; + goto out; - /* Reset ->seeded so that if __drbg_seed fails the next - * generate call will trigger a reseed. - */ - drbg->seeded = DRBG_SEED_STATE_UNSEEDED; - - __drbg_seed(drbg, &seedlist, true, DRBG_SEED_STATE_FULL); - -unlock: - mutex_unlock(&drbg->drbg_mutex); + ret = __drbg_seed(drbg, &seedlist, true, DRBG_SEED_STATE_FULL); +out: memzero_explicit(entropy, entropylen); + return ret; } /* @@ -1422,6 +1412,11 @@ static int drbg_generate(struct drbg_state *drbg, goto err; /* 9.3.1 step 7.4 */ addtl = NULL; + } else if (rng_is_initialized() && + drbg->seeded == DRBG_SEED_STATE_PARTIAL) { + len = drbg_seed_from_random(drbg); + if (len) + goto err; } if (addtl && 0 < addtl->len) @@ -1514,45 +1509,15 @@ static int drbg_generate_long(struct drbg_state *drbg, return 0; } -static void drbg_schedule_async_seed(struct random_ready_callback *rdy) -{ - struct drbg_state *drbg = container_of(rdy, struct drbg_state, - random_ready); - - schedule_work(&drbg->seed_work); -} - static int drbg_prepare_hrng(struct drbg_state *drbg) { - int err; - /* We do not need an HRNG in test mode. */ if (list_empty(&drbg->test_data.list)) return 0; drbg->jent = crypto_alloc_rng("jitterentropy_rng", 0, 0); - INIT_WORK(&drbg->seed_work, drbg_async_seed); - - drbg->random_ready.owner = THIS_MODULE; - drbg->random_ready.func = drbg_schedule_async_seed; - - err = add_random_ready_callback(&drbg->random_ready); - - switch (err) { - case 0: - break; - - case -EALREADY: - err = 0; - fallthrough; - - default: - drbg->random_ready.func = NULL; - return err; - } - - return err; + return 0; } /* @@ -1646,11 +1611,6 @@ free_everything: */ static int drbg_uninstantiate(struct drbg_state *drbg) { - if (drbg->random_ready.func) { - del_random_ready_callback(&drbg->random_ready); - cancel_work_sync(&drbg->seed_work); - } - if (!IS_ERR_OR_NULL(drbg->jent)) crypto_free_rng(drbg->jent); drbg->jent = NULL; diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h index 3ebdb1effe74..a6c3b8e7deb6 100644 --- a/include/crypto/drbg.h +++ b/include/crypto/drbg.h @@ -137,12 +137,10 @@ struct drbg_state { bool pr; /* Prediction resistance enabled? */ bool fips_primed; /* Continuous test primed? */ unsigned char *prev; /* FIPS 140-2 continuous test value */ - struct work_struct seed_work; /* asynchronous seeding support */ struct crypto_rng *jent; const struct drbg_state_ops *d_ops; const struct drbg_core *core; struct drbg_string test_data; - struct random_ready_callback random_ready; }; static inline __u8 drbg_statelen(struct drbg_state *drbg) From 559edd47cce4cc407d606b4d7f376822816fd4b8 Mon Sep 17 00:00:00 2001 From: Nicolai Stange Date: Mon, 15 Nov 2021 15:18:08 +0100 Subject: [PATCH 018/145] crypto: drbg - make drbg_prepare_hrng() handle jent instantiation errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that drbg_prepare_hrng() doesn't do anything but to instantiate a jitterentropy crypto_rng instance, it looks a little odd to have the related error handling at its only caller, drbg_instantiate(). Move the handling of jitterentropy allocation failures from drbg_instantiate() close to the allocation itself in drbg_prepare_hrng(). There is no change in behaviour. Signed-off-by: Nicolai Stange Reviewed-by: Stephan Müller Signed-off-by: Herbert Xu --- crypto/drbg.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crypto/drbg.c b/crypto/drbg.c index 2b03a05a9e99..9f6485962ecc 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -1516,6 +1516,14 @@ static int drbg_prepare_hrng(struct drbg_state *drbg) return 0; drbg->jent = crypto_alloc_rng("jitterentropy_rng", 0, 0); + if (IS_ERR(drbg->jent)) { + const int err = PTR_ERR(drbg->jent); + + drbg->jent = NULL; + if (fips_enabled || err != -ENOENT) + return err; + pr_info("DRBG: Continuing without Jitter RNG\n"); + } return 0; } @@ -1571,14 +1579,6 @@ static int drbg_instantiate(struct drbg_state *drbg, struct drbg_string *pers, if (ret) goto free_everything; - if (IS_ERR(drbg->jent)) { - ret = PTR_ERR(drbg->jent); - drbg->jent = NULL; - if (fips_enabled || ret != -ENOENT) - goto free_everything; - pr_info("DRBG: Continuing without Jitter RNG\n"); - } - reseed = false; } From 8ea5ee00beb925d2aa0fed0eb3faf04715a3f2bd Mon Sep 17 00:00:00 2001 From: Nicolai Stange Date: Mon, 15 Nov 2021 15:18:09 +0100 Subject: [PATCH 019/145] crypto: drbg - reseed 'nopr' drbgs periodically from get_random_bytes() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In contrast to the fully prediction resistant 'pr' DRBGs, the 'nopr' variants get seeded once at boot and reseeded only rarely thereafter, namely only after 2^20 requests have been served each. AFAICT, this reseeding based on the number of requests served is primarily motivated by information theoretic considerations, c.f. NIST SP800-90Ar1, sec. 8.6.8 ("Reseeding"). However, given the relatively large seed lifetime of 2^20 requests, the 'nopr' DRBGs can hardly be considered to provide any prediction resistance whatsoever, i.e. to protect against threats like side channel leaks of the internal DRBG state (think e.g. leaked VM snapshots). This is expected and completely in line with the 'nopr' naming, but as e.g. the "drbg_nopr_hmac_sha512" implementation is potentially being used for providing the "stdrng" and thus, the crypto_default_rng serving the in-kernel crypto, it would certainly be desirable to achieve at least the same level of prediction resistance as get_random_bytes() does. Note that the chacha20 rngs underlying get_random_bytes() get reseeded every CRNG_RESEED_INTERVAL == 5min: the secondary, per-NUMA node rngs from the primary one and the primary rng in turn from the entropy pool, provided sufficient entropy is available. The 'nopr' DRBGs do draw randomness from get_random_bytes() for their initial seed already, so making them to reseed themselves periodically from get_random_bytes() in order to let them benefit from the latter's prediction resistance is not such a big change conceptually. In principle, it would have been also possible to make the 'nopr' DRBGs to periodically invoke a full reseeding operation, i.e. to also consider the jitterentropy source (if enabled) in addition to get_random_bytes() for the seed value. However, get_random_bytes() is relatively lightweight as compared to the jitterentropy generation process and thus, even though the 'nopr' reseeding is supposed to get invoked infrequently, it's IMO still worthwhile to avoid occasional latency spikes for drbg_generate() and stick to get_random_bytes() only. As an additional remark, note that drawing randomness from the non-SP800-90B-conforming get_random_bytes() only won't adversely affect SP800-90A conformance either: the very same is being done during boot via drbg_seed_from_random() already once rng_is_initialized() flips to true and it follows that if the DRBG implementation does conform to SP800-90A now, it will continue to do so. Make the 'nopr' DRBGs to reseed themselves periodically from get_random_bytes() every CRNG_RESEED_INTERVAL == 5min. More specifically, introduce a new member ->last_seed_time to struct drbg_state for recording in units of jiffies when the last seeding operation had taken place. Make __drbg_seed() maintain it and let drbg_generate() invoke a reseed from get_random_bytes() via drbg_seed_from_random() if more than 5min have passed by since the last seeding operation. Be careful to not to reseed if in testing mode though, or otherwise the drbg related tests in crypto/testmgr.c would fail to reproduce the expected output. In order to keep the formatting clean in drbg_generate() wrap the logic for deciding whether or not a reseed is due in a new helper, drbg_nopr_reseed_interval_elapsed(). Signed-off-by: Nicolai Stange Reviewed-by: Stephan Müller Signed-off-by: Herbert Xu --- crypto/drbg.c | 26 +++++++++++++++++++++++++- include/crypto/drbg.h | 1 + 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/crypto/drbg.c b/crypto/drbg.c index 9f6485962ecc..5977a72afb03 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -100,6 +100,7 @@ #include #include #include +#include /*************************************************************** * Backend cipher definitions available to DRBG @@ -1044,6 +1045,7 @@ static inline int __drbg_seed(struct drbg_state *drbg, struct list_head *seed, return ret; drbg->seeded = new_seed_state; + drbg->last_seed_time = jiffies; /* 10.1.1.2 / 10.1.1.3 step 5 */ drbg->reseed_ctr = 1; @@ -1112,6 +1114,26 @@ out: return ret; } +static bool drbg_nopr_reseed_interval_elapsed(struct drbg_state *drbg) +{ + unsigned long next_reseed; + + /* Don't ever reseed from get_random_bytes() in test mode. */ + if (list_empty(&drbg->test_data.list)) + return false; + + /* + * Obtain fresh entropy for the nopr DRBGs after 300s have + * elapsed in order to still achieve sort of partial + * prediction resistance over the time domain at least. Note + * that the period of 300s has been chosen to match the + * CRNG_RESEED_INTERVAL of the get_random_bytes()' chacha + * rngs. + */ + next_reseed = drbg->last_seed_time + 300 * HZ; + return time_after(jiffies, next_reseed); +} + /* * Seeding or reseeding of the DRBG * @@ -1413,7 +1435,8 @@ static int drbg_generate(struct drbg_state *drbg, /* 9.3.1 step 7.4 */ addtl = NULL; } else if (rng_is_initialized() && - drbg->seeded == DRBG_SEED_STATE_PARTIAL) { + (drbg->seeded == DRBG_SEED_STATE_PARTIAL || + drbg_nopr_reseed_interval_elapsed(drbg))) { len = drbg_seed_from_random(drbg); if (len) goto err; @@ -1569,6 +1592,7 @@ static int drbg_instantiate(struct drbg_state *drbg, struct drbg_string *pers, drbg->core = &drbg_cores[coreref]; drbg->pr = pr; drbg->seeded = DRBG_SEED_STATE_UNSEEDED; + drbg->last_seed_time = 0; drbg->reseed_threshold = drbg_max_requests(drbg); ret = drbg_alloc_state(drbg); diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h index a6c3b8e7deb6..af5ad51d3eef 100644 --- a/include/crypto/drbg.h +++ b/include/crypto/drbg.h @@ -134,6 +134,7 @@ struct drbg_state { struct scatterlist sg_in, sg_out; /* CTR mode SGLs */ enum drbg_seed_state seeded; /* DRBG fully seeded? */ + unsigned long last_seed_time; bool pr; /* Prediction resistance enabled? */ bool fips_primed; /* Continuous test primed? */ unsigned char *prev; /* FIPS 140-2 continuous test value */ From c79391c696da0151c6ec8cb8a31edfa8bbe68f24 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 17 Nov 2021 14:30:34 +0000 Subject: [PATCH 020/145] crypto: qat - do not handle PFVF sources for qat_4xxx The QAT driver does not have support for PFVF interrupts for GEN4 devices, therefore report the vf2pf sources as 0. This prevents a NULL pointer dereference in the function adf_msix_isr_ae() if the device triggers a spurious interrupt. Fixes: 993161d36ab5 ("crypto: qat - fix handling of VF to PF interrupts") Reported-by: Adam Guerin Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c index fa768f10635f..fd29861526d6 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -211,6 +211,12 @@ static u32 uof_get_ae_mask(u32 obj_num) return adf_4xxx_fw_config[obj_num].ae_mask; } +static u32 get_vf2pf_sources(void __iomem *pmisc_addr) +{ + /* For the moment do not report vf2pf sources */ + return 0; +} + void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &adf_4xxx_class; @@ -254,6 +260,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) hw_data->set_msix_rttable = set_msix_default_rttable; hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer; hw_data->enable_pfvf_comms = pfvf_comms_disabled; + hw_data->get_vf2pf_sources = get_vf2pf_sources; hw_data->disable_iov = adf_disable_sriov; hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; From 5002200b4fedd7e90e4fbc2e5c42a4b3351df814 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 17 Nov 2021 14:30:35 +0000 Subject: [PATCH 021/145] crypto: qat - fix undetected PFVF timeout in ACK loop If the remote function did not ACK the reception of a message, the function __adf_iov_putmsg() could detect it as a collision. This was due to the fact that the collision and the timeout checks after the ACK loop were in the wrong order. The timeout must be checked at the end of the loop, so fix by swapping the order of the two checks. Fixes: 9b768e8a3909 ("crypto: qat - detect PFVF collision after ACK") Signed-off-by: Giovanni Cabiddu Co-developed-by: Marco Chiappero Signed-off-by: Marco Chiappero Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index 59860bdaedb6..99ee17c3d06b 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -107,6 +107,12 @@ static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset); } while ((val & int_bit) && (count++ < ADF_PFVF_MSG_ACK_MAX_RETRY)); + if (val & int_bit) { + dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n"); + val &= ~int_bit; + ret = -EIO; + } + if (val != msg) { dev_dbg(&GET_DEV(accel_dev), "Collision - PFVF CSR overwritten by remote function\n"); @@ -114,12 +120,6 @@ static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) goto out; } - if (val & int_bit) { - dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n"); - val &= ~int_bit; - ret = -EIO; - } - /* Finished with the PFVF CSR; relinquish it and leave msg in CSR */ ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, val & ~local_in_use_mask); out: From 95b4d40ed256c5d36b1dee979b65c04c3edd2b23 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Wed, 17 Nov 2021 14:30:36 +0000 Subject: [PATCH 022/145] crypto: qat - refactor PF top half for PFVF Move logic associated to handling VF2PF interrupt to its own function. This will simplify the handling of multiple interrupt sources in the function adf_msix_isr_ae() in the future. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_isr.c | 84 +++++++++++++------------ 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c index 40593c9449a2..5dfc534f1bf0 100644 --- a/drivers/crypto/qat/qat_common/adf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_isr.c @@ -54,52 +54,56 @@ static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr) return IRQ_HANDLED; } +#ifdef CONFIG_PCI_IOV +static bool adf_handle_vf2pf_int(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + int bar_id = hw_data->get_misc_bar_id(hw_data); + struct adf_bar *pmisc = &GET_BARS(accel_dev)[bar_id]; + void __iomem *pmisc_addr = pmisc->virt_addr; + bool irq_handled = false; + unsigned long vf_mask; + + /* Get the interrupt sources triggered by VFs */ + vf_mask = hw_data->get_vf2pf_sources(pmisc_addr); + + if (vf_mask) { + struct adf_accel_vf_info *vf_info; + int i; + + /* Disable VF2PF interrupts for VFs with pending ints */ + adf_disable_vf2pf_interrupts_irq(accel_dev, vf_mask); + + /* + * Handle VF2PF interrupt unless the VF is malicious and + * is attempting to flood the host OS with VF2PF interrupts. + */ + for_each_set_bit(i, &vf_mask, ADF_MAX_NUM_VFS) { + vf_info = accel_dev->pf.vf_info + i; + + if (!__ratelimit(&vf_info->vf2pf_ratelimit)) { + dev_info(&GET_DEV(accel_dev), + "Too many ints from VF%d\n", + vf_info->vf_nr + 1); + continue; + } + + adf_schedule_vf2pf_handler(vf_info); + irq_handled = true; + } + } + return irq_handled; +} +#endif /* CONFIG_PCI_IOV */ + static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr) { struct adf_accel_dev *accel_dev = dev_ptr; #ifdef CONFIG_PCI_IOV /* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */ - if (accel_dev->pf.vf_info) { - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_bar *pmisc = - &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; - void __iomem *pmisc_addr = pmisc->virt_addr; - unsigned long vf_mask; - - /* Get the interrupt sources triggered by VFs */ - vf_mask = hw_data->get_vf2pf_sources(pmisc_addr); - - if (vf_mask) { - struct adf_accel_vf_info *vf_info; - bool irq_handled = false; - int i; - - /* Disable VF2PF interrupts for VFs with pending ints */ - adf_disable_vf2pf_interrupts_irq(accel_dev, vf_mask); - - /* - * Handle VF2PF interrupt unless the VF is malicious and - * is attempting to flood the host OS with VF2PF interrupts. - */ - for_each_set_bit(i, &vf_mask, ADF_MAX_NUM_VFS) { - vf_info = accel_dev->pf.vf_info + i; - - if (!__ratelimit(&vf_info->vf2pf_ratelimit)) { - dev_info(&GET_DEV(accel_dev), - "Too many ints from VF%d\n", - vf_info->vf_nr + 1); - continue; - } - - adf_schedule_vf2pf_handler(vf_info); - irq_handled = true; - } - - if (irq_handled) - return IRQ_HANDLED; - } - } + if (accel_dev->pf.vf_info && adf_handle_vf2pf_int(accel_dev)) + return IRQ_HANDLED; #endif /* CONFIG_PCI_IOV */ dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n", From 08ea97f48883a5d3178e09433b449bafc5f2a314 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 17 Nov 2021 14:30:37 +0000 Subject: [PATCH 023/145] crypto: qat - move vf2pf interrupt helpers Move vf2pf interrupt enable and disable functions from adf_pf2vf_msg.c to adf_isr.c This it to separate the interrupt related code from the PFVF protocol logic. With this change, the function adf_disable_vf2pf_interrupts_irq() is only called from adf_isr.c and it has been marked as static. Signed-off-by: Giovanni Cabiddu Reviewed-by: Marco Chiappero Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_common_drv.h | 2 - drivers/crypto/qat/qat_common/adf_isr.c | 39 +++++++++++++++++++ drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 39 ------------------- 3 files changed, 39 insertions(+), 41 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index de94b76a6d2c..4f9f94db16e5 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -193,8 +193,6 @@ int adf_sriov_configure(struct pci_dev *pdev, int numvfs); void adf_disable_sriov(struct adf_accel_dev *accel_dev); void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask); -void adf_disable_vf2pf_interrupts_irq(struct adf_accel_dev *accel_dev, - u32 vf_mask); void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask); int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c index 5dfc534f1bf0..2b4900c91308 100644 --- a/drivers/crypto/qat/qat_common/adf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_isr.c @@ -55,6 +55,45 @@ static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr) } #ifdef CONFIG_PCI_IOV +void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data); + struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id]; + void __iomem *pmisc_addr = pmisc->virt_addr; + unsigned long flags; + + spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); + hw_data->enable_vf2pf_interrupts(pmisc_addr, vf_mask); + spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); +} + +void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data); + struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id]; + void __iomem *pmisc_addr = pmisc->virt_addr; + unsigned long flags; + + spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); + hw_data->disable_vf2pf_interrupts(pmisc_addr, vf_mask); + spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); +} + +static void adf_disable_vf2pf_interrupts_irq(struct adf_accel_dev *accel_dev, + u32 vf_mask) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data); + struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id]; + void __iomem *pmisc_addr = pmisc->virt_addr; + + spin_lock(&accel_dev->pf.vf2pf_ints_lock); + hw_data->disable_vf2pf_interrupts(pmisc_addr, vf_mask); + spin_unlock(&accel_dev->pf.vf2pf_ints_lock); +} + static bool adf_handle_vf2pf_int(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index 99ee17c3d06b..d0492530c84a 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -14,45 +14,6 @@ ADF_PFVF_MSG_ACK_MAX_RETRY + \ ADF_PFVF_MSG_COLLISION_DETECT_DELAY) -void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data); - struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id]; - void __iomem *pmisc_addr = pmisc->virt_addr; - unsigned long flags; - - spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); - hw_data->enable_vf2pf_interrupts(pmisc_addr, vf_mask); - spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); -} - -void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data); - struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id]; - void __iomem *pmisc_addr = pmisc->virt_addr; - unsigned long flags; - - spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); - hw_data->disable_vf2pf_interrupts(pmisc_addr, vf_mask); - spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); -} - -void adf_disable_vf2pf_interrupts_irq(struct adf_accel_dev *accel_dev, - u32 vf_mask) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data); - struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id]; - void __iomem *pmisc_addr = pmisc->virt_addr; - - spin_lock(&accel_dev->pf.vf2pf_ints_lock); - hw_data->disable_vf2pf_interrupts(pmisc_addr, vf_mask); - spin_unlock(&accel_dev->pf.vf2pf_ints_lock); -} - static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) { struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; From b7c13ee46cebfd106e7065cf5fee191b324d0ca9 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Wed, 17 Nov 2021 14:30:38 +0000 Subject: [PATCH 024/145] crypto: qat - move VF message handler to adf_vf2pf_msg.c Move the reading and parsing of a PF2VF message from the bottom half function in adf_vf_isr.c, adf_pf2vf_bh_handler(), to the PFVF protocol file adf_vf2pf_msg.c, for better code organization. The receive and handle logic has been moved to a new function called adf_recv_and_handle_pf2vf_msg() which returns a boolean indicating if interrupts need to be re-enabled or not. A slight refactoring has been done to avoid calculating the PF2VF CSR offset twice and repeating the clearing of the PF2VFINT bit. The "PF restarting" logic, now defined in the function adf_pf2vf_handle_pf_restaring(), has been kept in adf_vf_isr.c due to the dependencies with the adf_vf_stop_wq workqueue. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_common_drv.h | 2 + drivers/crypto/qat/qat_common/adf_vf2pf_msg.c | 59 ++++++++++++ drivers/crypto/qat/qat_common/adf_vf_isr.c | 91 +++++-------------- 3 files changed, 86 insertions(+), 66 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 4f9f94db16e5..56ee2d5a6b37 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -195,6 +195,8 @@ void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask); void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask); +bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev); +int adf_pf2vf_handle_pf_restarting(struct adf_accel_dev *accel_dev); int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev); void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c b/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c index 8d11bb24cea0..064477fcb5fb 100644 --- a/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c @@ -46,3 +46,62 @@ void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev) "Failed to send Shutdown event to PF\n"); } EXPORT_SYMBOL_GPL(adf_vf2pf_notify_shutdown); + +bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *pmisc = + &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; + void __iomem *pmisc_bar_addr = pmisc->virt_addr; + u32 offset = hw_data->get_pf2vf_offset(0); + bool ret; + u32 msg; + + /* Read the message from PF */ + msg = ADF_CSR_RD(pmisc_bar_addr, offset); + if (!(msg & ADF_PF2VF_INT)) { + dev_info(&GET_DEV(accel_dev), + "Spurious PF2VF interrupt, msg %X. Ignored\n", msg); + return true; + } + + if (!(msg & ADF_PF2VF_MSGORIGIN_SYSTEM)) + /* Ignore legacy non-system (non-kernel) PF2VF messages */ + goto err; + + switch ((msg & ADF_PF2VF_MSGTYPE_MASK) >> ADF_PF2VF_MSGTYPE_SHIFT) { + case ADF_PF2VF_MSGTYPE_RESTARTING: + dev_dbg(&GET_DEV(accel_dev), + "Restarting msg received from PF 0x%x\n", msg); + + adf_pf2vf_handle_pf_restarting(accel_dev); + ret = false; + break; + case ADF_PF2VF_MSGTYPE_VERSION_RESP: + dev_dbg(&GET_DEV(accel_dev), + "Version resp received from PF 0x%x\n", msg); + accel_dev->vf.pf_version = + (msg & ADF_PF2VF_VERSION_RESP_VERS_MASK) >> + ADF_PF2VF_VERSION_RESP_VERS_SHIFT; + accel_dev->vf.compatible = + (msg & ADF_PF2VF_VERSION_RESP_RESULT_MASK) >> + ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + complete(&accel_dev->vf.iov_msg_completion); + ret = true; + break; + default: + goto err; + } + + /* To ack, clear the PF2VFINT bit */ + msg &= ~ADF_PF2VF_INT; + ADF_CSR_WR(pmisc_bar_addr, offset, msg); + return ret; + +err: + dev_err(&GET_DEV(accel_dev), + "Unknown message from PF (0x%x); leaving PF2VF ints disabled\n", + msg); + + return false; +} diff --git a/drivers/crypto/qat/qat_common/adf_vf_isr.c b/drivers/crypto/qat/qat_common/adf_vf_isr.c index db5e7abbe5f3..b17040b8a4b9 100644 --- a/drivers/crypto/qat/qat_common/adf_vf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_vf_isr.c @@ -85,78 +85,37 @@ static void adf_dev_stop_async(struct work_struct *work) kfree(stop_data); } +int adf_pf2vf_handle_pf_restarting(struct adf_accel_dev *accel_dev) +{ + struct adf_vf_stop_data *stop_data; + + clear_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status); + stop_data = kzalloc(sizeof(*stop_data), GFP_ATOMIC); + if (!stop_data) { + dev_err(&GET_DEV(accel_dev), + "Couldn't schedule stop for vf_%d\n", + accel_dev->accel_id); + return -ENOMEM; + } + stop_data->accel_dev = accel_dev; + INIT_WORK(&stop_data->work, adf_dev_stop_async); + queue_work(adf_vf_stop_wq, &stop_data->work); + + return 0; +} + static void adf_pf2vf_bh_handler(void *data) { struct adf_accel_dev *accel_dev = data; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_bar *pmisc = - &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; - void __iomem *pmisc_bar_addr = pmisc->virt_addr; - u32 msg; + bool ret; - /* Read the message from PF */ - msg = ADF_CSR_RD(pmisc_bar_addr, hw_data->get_pf2vf_offset(0)); - if (!(msg & ADF_PF2VF_INT)) { - dev_info(&GET_DEV(accel_dev), - "Spurious PF2VF interrupt, msg %X. Ignored\n", msg); - goto out; - } + ret = adf_recv_and_handle_pf2vf_msg(accel_dev); + if (ret) + /* Re-enable PF2VF interrupts */ + adf_enable_pf2vf_interrupts(accel_dev); - if (!(msg & ADF_PF2VF_MSGORIGIN_SYSTEM)) - /* Ignore legacy non-system (non-kernel) PF2VF messages */ - goto err; - - switch ((msg & ADF_PF2VF_MSGTYPE_MASK) >> ADF_PF2VF_MSGTYPE_SHIFT) { - case ADF_PF2VF_MSGTYPE_RESTARTING: { - struct adf_vf_stop_data *stop_data; - - dev_dbg(&GET_DEV(accel_dev), - "Restarting msg received from PF 0x%x\n", msg); - - clear_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status); - - stop_data = kzalloc(sizeof(*stop_data), GFP_ATOMIC); - if (!stop_data) { - dev_err(&GET_DEV(accel_dev), - "Couldn't schedule stop for vf_%d\n", - accel_dev->accel_id); - return; - } - stop_data->accel_dev = accel_dev; - INIT_WORK(&stop_data->work, adf_dev_stop_async); - queue_work(adf_vf_stop_wq, &stop_data->work); - /* To ack, clear the PF2VFINT bit */ - msg &= ~ADF_PF2VF_INT; - ADF_CSR_WR(pmisc_bar_addr, hw_data->get_pf2vf_offset(0), msg); - return; - } - case ADF_PF2VF_MSGTYPE_VERSION_RESP: - dev_dbg(&GET_DEV(accel_dev), - "Version resp received from PF 0x%x\n", msg); - accel_dev->vf.pf_version = - (msg & ADF_PF2VF_VERSION_RESP_VERS_MASK) >> - ADF_PF2VF_VERSION_RESP_VERS_SHIFT; - accel_dev->vf.compatible = - (msg & ADF_PF2VF_VERSION_RESP_RESULT_MASK) >> - ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; - complete(&accel_dev->vf.iov_msg_completion); - break; - default: - goto err; - } - - /* To ack, clear the PF2VFINT bit */ - msg &= ~ADF_PF2VF_INT; - ADF_CSR_WR(pmisc_bar_addr, hw_data->get_pf2vf_offset(0), msg); - -out: - /* Re-enable PF2VF interrupts */ - adf_enable_pf2vf_interrupts(accel_dev); return; -err: - dev_err(&GET_DEV(accel_dev), - "Unknown message from PF (0x%x); leaving PF2VF ints disabled\n", - msg); + } static int adf_setup_pf2vf_bh(struct adf_accel_dev *accel_dev) From 720aa72a77f402a8d33ad1ddef72a4bf6e973fdd Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Wed, 17 Nov 2021 14:30:39 +0000 Subject: [PATCH 025/145] crypto: qat - move interrupt code out of the PFVF handler Move the interrupt handling call from the PF specific protocol file, adf_pf2vf_msg.c, to adf_sriov.c to maintain the PFVF files focused on the protocol handling. The function adf_vf2pf_req_hndl() has been renamed as adf_recv_and_handle_vf2pf_msg() to reflect its actual purpose and maintain consistency with the VF side. This function now returns a boolean indicating to the caller if interrupts need to be re-enabled or not. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_common_drv.h | 2 +- drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 15 ++++++--------- drivers/crypto/qat/qat_common/adf_sriov.c | 10 +++++++++- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 56ee2d5a6b37..db9d045e6a06 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -64,7 +64,6 @@ void adf_dev_shutdown(struct adf_accel_dev *accel_dev); void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev); int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev); -void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info); void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data); void adf_clean_vf_map(bool); @@ -196,6 +195,7 @@ void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask); bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev); +bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr); int adf_pf2vf_handle_pf_restarting(struct adf_accel_dev *accel_dev); int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev); void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index d0492530c84a..796301e9fe5b 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -178,21 +178,21 @@ static int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, u32 msg) return 0; } -void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info) +bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr) { - struct adf_accel_dev *accel_dev = vf_info->accel_dev; + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; struct adf_hw_device_data *hw_data = accel_dev->hw_device; int bar_id = hw_data->get_misc_bar_id(hw_data); struct adf_bar *pmisc = &GET_BARS(accel_dev)[bar_id]; void __iomem *pmisc_addr = pmisc->virt_addr; - u32 msg, resp = 0, vf_nr = vf_info->vf_nr; + u32 msg, resp = 0; /* Read message from the VF */ msg = ADF_CSR_RD(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr)); if (!(msg & ADF_VF2PF_INT)) { dev_info(&GET_DEV(accel_dev), "Spurious VF2PF interrupt, msg %X. Ignored\n", msg); - goto out; + return true; } /* To ACK, clear the VF2PFINT bit */ @@ -277,14 +277,11 @@ void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info) if (resp && adf_send_pf2vf_msg(accel_dev, vf_nr, resp)) dev_err(&GET_DEV(accel_dev), "Failed to send response to VF\n"); -out: - /* re-enable interrupt on PF from this VF */ - adf_enable_vf2pf_interrupts(accel_dev, (1 << vf_nr)); - - return; + return true; err: dev_dbg(&GET_DEV(accel_dev), "Unknown message from VF%d (0x%x);\n", vf_nr + 1, msg); + return false; } void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev) diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index 90ec057f9183..b1a814ac1d67 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -19,8 +19,16 @@ static void adf_iov_send_resp(struct work_struct *work) { struct adf_pf2vf_resp *pf2vf_resp = container_of(work, struct adf_pf2vf_resp, pf2vf_resp_work); + struct adf_accel_vf_info *vf_info = pf2vf_resp->vf_info; + struct adf_accel_dev *accel_dev = vf_info->accel_dev; + u32 vf_nr = vf_info->vf_nr; + bool ret; + + ret = adf_recv_and_handle_vf2pf_msg(accel_dev, vf_nr); + if (ret) + /* re-enable interrupt on PF from this VF */ + adf_enable_vf2pf_interrupts(accel_dev, 1 << vf_nr); - adf_vf2pf_req_hndl(pf2vf_resp->vf_info); kfree(pf2vf_resp); } From 956125e21f460ef94660019b6ec9de3f835f93ab Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 17 Nov 2021 14:30:40 +0000 Subject: [PATCH 026/145] crypto: qat - change PFVF ACK behaviour Change the PFVF receipt flow on the VF side to read, ack and handle the message instead of read, handle and ack. This is done for (1) consistency with the PF side, see the function adf_recv_and_handle_vf2pf_msg() in adf_pf2vf_msg.c, and (2) performance reasons, to avoid keeping the CSR busy while parsing the message. In addition, do not ACK PFVF legacy messages, as this driver is not capable of handling PFVF legacy messages. If a PFVF message with MSGORIGIN not set is received, do nothing. Signed-off-by: Giovanni Cabiddu Reviewed-by: Marco Chiappero Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 8 ++++---- drivers/crypto/qat/qat_common/adf_vf2pf_msg.c | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index 796301e9fe5b..4922ee2a2a08 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -195,14 +195,14 @@ bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr) return true; } - /* To ACK, clear the VF2PFINT bit */ - msg &= ~ADF_VF2PF_INT; - ADF_CSR_WR(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr), msg); - if (!(msg & ADF_VF2PF_MSGORIGIN_SYSTEM)) /* Ignore legacy non-system (non-kernel) VF2PF messages */ goto err; + /* To ACK, clear the VF2PFINT bit */ + msg &= ~ADF_VF2PF_INT; + ADF_CSR_WR(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr), msg); + switch ((msg & ADF_VF2PF_MSGTYPE_MASK) >> ADF_VF2PF_MSGTYPE_SHIFT) { case ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ: { diff --git a/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c b/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c index 064477fcb5fb..a6eaf93d5462 100644 --- a/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c @@ -69,6 +69,10 @@ bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev) /* Ignore legacy non-system (non-kernel) PF2VF messages */ goto err; + /* To ack, clear the PF2VFINT bit */ + msg &= ~ADF_PF2VF_INT; + ADF_CSR_WR(pmisc_bar_addr, offset, msg); + switch ((msg & ADF_PF2VF_MSGTYPE_MASK) >> ADF_PF2VF_MSGTYPE_SHIFT) { case ADF_PF2VF_MSGTYPE_RESTARTING: dev_dbg(&GET_DEV(accel_dev), @@ -93,9 +97,6 @@ bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev) goto err; } - /* To ack, clear the PF2VFINT bit */ - msg &= ~ADF_PF2VF_INT; - ADF_CSR_WR(pmisc_bar_addr, offset, msg); return ret; err: From 04cf47872c7edde7a11c634aef04b1dbcf9c26fa Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 17 Nov 2021 14:30:41 +0000 Subject: [PATCH 027/145] crypto: qat - re-enable interrupts for legacy PFVF messages If a PFVF message with MSGORIGIN_SYSTEM not set is received, re-enable interrupts allowing the processing of new messages. This is to simplify the refactoring of the recv function in a subsequent patch. Signed-off-by: Giovanni Cabiddu Reviewed-by: Marco Chiappero Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 2 +- drivers/crypto/qat/qat_common/adf_vf2pf_msg.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index 4922ee2a2a08..296f54805e33 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -197,7 +197,7 @@ bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr) if (!(msg & ADF_VF2PF_MSGORIGIN_SYSTEM)) /* Ignore legacy non-system (non-kernel) VF2PF messages */ - goto err; + return true; /* To ACK, clear the VF2PFINT bit */ msg &= ~ADF_VF2PF_INT; diff --git a/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c b/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c index a6eaf93d5462..e383232b0685 100644 --- a/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c @@ -67,7 +67,7 @@ bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev) if (!(msg & ADF_PF2VF_MSGORIGIN_SYSTEM)) /* Ignore legacy non-system (non-kernel) PF2VF messages */ - goto err; + return true; /* To ack, clear the PF2VFINT bit */ msg &= ~ADF_PF2VF_INT; From bd59b769ddac0db24d2d43ca5e40b29a58d7b205 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Wed, 17 Nov 2021 14:30:42 +0000 Subject: [PATCH 028/145] crypto: qat - split PFVF message decoding from handling Refactor the receive and handle logic to separate the parsing and handling of the PFVF message from the initial retrieval and ACK. This is to allow the intoduction of the recv function in a subsequent patch. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 68 ++++++++++++------- drivers/crypto/qat/qat_common/adf_vf2pf_msg.c | 62 ++++++++--------- 2 files changed, 71 insertions(+), 59 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index 296f54805e33..201744825e23 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -178,30 +178,12 @@ static int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, u32 msg) return 0; } -bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr) +static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, + u32 msg, u32 *response) { struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; struct adf_hw_device_data *hw_data = accel_dev->hw_device; - int bar_id = hw_data->get_misc_bar_id(hw_data); - struct adf_bar *pmisc = &GET_BARS(accel_dev)[bar_id]; - void __iomem *pmisc_addr = pmisc->virt_addr; - u32 msg, resp = 0; - - /* Read message from the VF */ - msg = ADF_CSR_RD(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr)); - if (!(msg & ADF_VF2PF_INT)) { - dev_info(&GET_DEV(accel_dev), - "Spurious VF2PF interrupt, msg %X. Ignored\n", msg); - return true; - } - - if (!(msg & ADF_VF2PF_MSGORIGIN_SYSTEM)) - /* Ignore legacy non-system (non-kernel) VF2PF messages */ - return true; - - /* To ACK, clear the VF2PFINT bit */ - msg &= ~ADF_VF2PF_INT; - ADF_CSR_WR(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr), msg); + u32 resp = 0; switch ((msg & ADF_VF2PF_MSGTYPE_MASK) >> ADF_VF2PF_MSGTYPE_SHIFT) { case ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ: @@ -271,17 +253,51 @@ bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr) } break; default: - goto err; + dev_dbg(&GET_DEV(accel_dev), "Unknown message from VF%d (0x%x)\n", + vf_nr + 1, msg); + return -ENOMSG; } + *response = resp; + + return 0; +} + +bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + int bar_id = hw_data->get_misc_bar_id(hw_data); + struct adf_bar *pmisc = &GET_BARS(accel_dev)[bar_id]; + void __iomem *pmisc_addr = pmisc->virt_addr; + u32 msg, resp = 0; + + /* Read message from the VF */ + msg = ADF_CSR_RD(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr)); + if (!(msg & ADF_VF2PF_INT)) { + dev_info(&GET_DEV(accel_dev), + "Spurious VF2PF interrupt, msg %X. Ignored\n", msg); + return true; + } + + /* Ignore legacy non-system (non-kernel) VF2PF messages */ + if (!(msg & ADF_VF2PF_MSGORIGIN_SYSTEM)) { + dev_dbg(&GET_DEV(accel_dev), + "Ignored non-system message from VF%d (0x%x);\n", + vf_nr + 1, msg); + return true; + } + + /* To ACK, clear the VF2PFINT bit */ + msg &= ~ADF_VF2PF_INT; + ADF_CSR_WR(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr), msg); + + if (adf_handle_vf2pf_msg(accel_dev, vf_nr, msg, &resp)) + return false; + if (resp && adf_send_pf2vf_msg(accel_dev, vf_nr, resp)) dev_err(&GET_DEV(accel_dev), "Failed to send response to VF\n"); return true; -err: - dev_dbg(&GET_DEV(accel_dev), "Unknown message from VF%d (0x%x);\n", - vf_nr + 1, msg); - return false; } void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev) diff --git a/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c b/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c index e383232b0685..01a6e68f256b 100644 --- a/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c @@ -47,6 +47,34 @@ void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev) } EXPORT_SYMBOL_GPL(adf_vf2pf_notify_shutdown); +static bool adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev, u32 msg) +{ + switch ((msg & ADF_PF2VF_MSGTYPE_MASK) >> ADF_PF2VF_MSGTYPE_SHIFT) { + case ADF_PF2VF_MSGTYPE_RESTARTING: + dev_dbg(&GET_DEV(accel_dev), + "Restarting msg received from PF 0x%x\n", msg); + + adf_pf2vf_handle_pf_restarting(accel_dev); + return false; + case ADF_PF2VF_MSGTYPE_VERSION_RESP: + dev_dbg(&GET_DEV(accel_dev), + "Version resp received from PF 0x%x\n", msg); + accel_dev->vf.pf_version = + (msg & ADF_PF2VF_VERSION_RESP_VERS_MASK) >> + ADF_PF2VF_VERSION_RESP_VERS_SHIFT; + accel_dev->vf.compatible = + (msg & ADF_PF2VF_VERSION_RESP_RESULT_MASK) >> + ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + complete(&accel_dev->vf.iov_msg_completion); + return true; + default: + dev_err(&GET_DEV(accel_dev), + "Unknown PF2VF message(0x%x)\n", msg); + } + + return false; +} + bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; @@ -54,7 +82,6 @@ bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev) &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; void __iomem *pmisc_bar_addr = pmisc->virt_addr; u32 offset = hw_data->get_pf2vf_offset(0); - bool ret; u32 msg; /* Read the message from PF */ @@ -73,36 +100,5 @@ bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev) msg &= ~ADF_PF2VF_INT; ADF_CSR_WR(pmisc_bar_addr, offset, msg); - switch ((msg & ADF_PF2VF_MSGTYPE_MASK) >> ADF_PF2VF_MSGTYPE_SHIFT) { - case ADF_PF2VF_MSGTYPE_RESTARTING: - dev_dbg(&GET_DEV(accel_dev), - "Restarting msg received from PF 0x%x\n", msg); - - adf_pf2vf_handle_pf_restarting(accel_dev); - ret = false; - break; - case ADF_PF2VF_MSGTYPE_VERSION_RESP: - dev_dbg(&GET_DEV(accel_dev), - "Version resp received from PF 0x%x\n", msg); - accel_dev->vf.pf_version = - (msg & ADF_PF2VF_VERSION_RESP_VERS_MASK) >> - ADF_PF2VF_VERSION_RESP_VERS_SHIFT; - accel_dev->vf.compatible = - (msg & ADF_PF2VF_VERSION_RESP_RESULT_MASK) >> - ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; - complete(&accel_dev->vf.iov_msg_completion); - ret = true; - break; - default: - goto err; - } - - return ret; - -err: - dev_err(&GET_DEV(accel_dev), - "Unknown message from PF (0x%x); leaving PF2VF ints disabled\n", - msg); - - return false; + return adf_handle_pf2vf_msg(accel_dev, msg); } From 1d6133123fb2626499e0e0a9d62e39bcdc5e593b Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Wed, 17 Nov 2021 14:30:43 +0000 Subject: [PATCH 029/145] crypto: qat - handle retries due to collisions in adf_iov_putmsg() Rework __adf_iov_putmsg() to handle retries due to collisions internally, removing the need for an external retry loop. The functions __adf_iov_putmsg() and adf_iov_putmsg() have been merged together maintaining the adf_iov_putmsg() name. This will allow to use this function only for GEN2 devices, since collision are peculiar of this generation and therefore should be confined to the actual implementation of the transport/medium access. Note that now adf_iov_putmsg() will retry to send a message only in case of collisions and will now fail if an ACK is not received from the remote function. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 52 +++++++------------ 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index 201744825e23..d98e3639c9d2 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -14,7 +14,7 @@ ADF_PFVF_MSG_ACK_MAX_RETRY + \ ADF_PFVF_MSG_COLLISION_DETECT_DELAY) -static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) +static int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) { struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; @@ -24,8 +24,9 @@ static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) u32 local_in_use_mask, local_in_use_pattern; u32 remote_in_use_mask, remote_in_use_pattern; struct mutex *lock; /* lock preventing concurrent acces of CSR */ + unsigned int retries = ADF_PFVF_MSG_MAX_RETRIES; u32 int_bit; - int ret = 0; + int ret; if (accel_dev->is_vf) { pf2vf_offset = hw_data->get_pf2vf_offset(0); @@ -45,20 +46,22 @@ static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) int_bit = ADF_PF2VF_INT; } + msg &= ~local_in_use_mask; + msg |= local_in_use_pattern; + mutex_lock(lock); +start: + ret = 0; + /* Check if the PFVF CSR is in use by remote function */ val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset); if ((val & remote_in_use_mask) == remote_in_use_pattern) { dev_dbg(&GET_DEV(accel_dev), "PFVF CSR in use by remote function\n"); - ret = -EBUSY; - goto out; + goto retry; } - msg &= ~local_in_use_mask; - msg |= local_in_use_pattern; - /* Attempt to get ownership of the PFVF CSR */ ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, msg | int_bit); @@ -77,8 +80,7 @@ static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) if (val != msg) { dev_dbg(&GET_DEV(accel_dev), "Collision - PFVF CSR overwritten by remote function\n"); - ret = -EIO; - goto out; + goto retry; } /* Finished with the PFVF CSR; relinquish it and leave msg in CSR */ @@ -86,31 +88,15 @@ static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) out: mutex_unlock(lock); return ret; -} -/** - * adf_iov_putmsg() - send PFVF message - * @accel_dev: Pointer to acceleration device. - * @msg: Message to send - * @vf_nr: VF number to which the message will be sent if on PF, ignored - * otherwise - * - * Function sends a message through the PFVF channel - * - * Return: 0 on success, error code otherwise. - */ -static int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) -{ - u32 count = 0; - int ret; - - do { - ret = __adf_iov_putmsg(accel_dev, msg, vf_nr); - if (ret) - msleep(ADF_PFVF_MSG_RETRY_DELAY); - } while (ret && (count++ < ADF_PFVF_MSG_MAX_RETRIES)); - - return ret; +retry: + if (--retries) { + msleep(ADF_PFVF_MSG_RETRY_DELAY); + goto start; + } else { + ret = -EBUSY; + goto out; + } } /** From b85bd9457dc302749411a9b43f78fd493499f3ef Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Wed, 17 Nov 2021 14:30:44 +0000 Subject: [PATCH 030/145] crypto: qat - relocate PFVF PF related logic Move device specific PFVF logic related to the PF to the newly created adf_gen2_pfvf.c. This refactory is done to isolate the GEN2 PFVF code into its own file in preparation for the introduction of support for PFVF for GEN4 devices. In addition the PFVF PF logic for dh895xcc has been isolated to adf_dh895xcc_hw_data.c. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 1 + .../crypto/qat/qat_c62x/adf_c62x_hw_data.c | 1 + drivers/crypto/qat/qat_common/Makefile | 3 +- .../crypto/qat/qat_common/adf_gen2_hw_data.c | 48 ---------------- .../crypto/qat/qat_common/adf_gen2_hw_data.h | 13 ----- drivers/crypto/qat/qat_common/adf_gen2_pfvf.c | 57 +++++++++++++++++++ drivers/crypto/qat/qat_common/adf_gen2_pfvf.h | 19 +++++++ .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 28 ++++++--- .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.h | 2 + 9 files changed, 103 insertions(+), 69 deletions(-) create mode 100644 drivers/crypto/qat/qat_common/adf_gen2_pfvf.c create mode 100644 drivers/crypto/qat/qat_common/adf_gen2_pfvf.h diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 1fa690219d92..0bc528004f79 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -4,6 +4,7 @@ #include #include #include +#include #include "adf_c3xxx_hw_data.h" #include "icp_qat_hw.h" diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index 0613db077689..9303f2dbcaf9 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -4,6 +4,7 @@ #include #include #include +#include #include "adf_c62x_hw_data.h" #include "icp_qat_hw.h" diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile index 9c57abdf56b7..3874e427d1f7 100644 --- a/drivers/crypto/qat/qat_common/Makefile +++ b/drivers/crypto/qat/qat_common/Makefile @@ -16,7 +16,8 @@ intel_qat-objs := adf_cfg.o \ qat_algs.o \ qat_asym_algs.o \ qat_uclo.o \ - qat_hal.o + qat_hal.o \ + adf_gen2_pfvf.o intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_pf2vf_msg.o \ diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index 262bdc05dab4..3b48fdaaff6d 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -4,54 +4,6 @@ #include "icp_qat_hw.h" #include -#define ADF_GEN2_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) - -u32 adf_gen2_get_pf2vf_offset(u32 i) -{ - return ADF_GEN2_PF2VF_OFFSET(i); -} -EXPORT_SYMBOL_GPL(adf_gen2_get_pf2vf_offset); - -u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_addr) -{ - u32 errsou3, errmsk3, vf_int_mask; - - /* Get the interrupt sources triggered by VFs */ - errsou3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU3); - vf_int_mask = ADF_GEN2_ERR_REG_VF2PF(errsou3); - - /* To avoid adding duplicate entries to work queue, clear - * vf_int_mask_sets bits that are already masked in ERRMSK register. - */ - errmsk3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3); - vf_int_mask &= ~ADF_GEN2_ERR_REG_VF2PF(errmsk3); - - return vf_int_mask; -} -EXPORT_SYMBOL_GPL(adf_gen2_get_vf2pf_sources); - -void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) -{ - /* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */ - if (vf_mask & 0xFFFF) { - u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) - & ~ADF_GEN2_ERR_MSK_VF2PF(vf_mask); - ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); - } -} -EXPORT_SYMBOL_GPL(adf_gen2_enable_vf2pf_interrupts); - -void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) -{ - /* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */ - if (vf_mask & 0xFFFF) { - u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) - | ADF_GEN2_ERR_MSK_VF2PF(vf_mask); - ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); - } -} -EXPORT_SYMBOL_GPL(adf_gen2_disable_vf2pf_interrupts); - u32 adf_gen2_get_num_accels(struct adf_hw_device_data *self) { if (!self || !self->accel_mask) diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index c169d704097d..448c97f740e7 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -136,19 +136,6 @@ do { \ #define ADF_GEN2_CERRSSMSH(i) ((i) * 0x4000 + 0x10) #define ADF_GEN2_ERRSSMSH_EN BIT(3) - /* VF2PF interrupts */ -#define ADF_GEN2_ERRSOU3 (0x3A000 + 0x0C) -#define ADF_GEN2_ERRSOU5 (0x3A000 + 0xD8) -#define ADF_GEN2_ERRMSK3 (0x3A000 + 0x1C) -#define ADF_GEN2_ERRMSK5 (0x3A000 + 0xDC) -#define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9) -#define ADF_GEN2_ERR_MSK_VF2PF(vf_mask) (((vf_mask) & 0xFFFF) << 9) - -u32 adf_gen2_get_pf2vf_offset(u32 i); -u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_bar); -void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask); -void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask); - u32 adf_gen2_get_num_accels(struct adf_hw_device_data *self); u32 adf_gen2_get_num_aes(struct adf_hw_device_data *self); void adf_gen2_enable_error_correction(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c new file mode 100644 index 000000000000..d4d79419daaa --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2021 Intel Corporation */ +#include +#include "adf_accel_devices.h" +#include "adf_gen2_pfvf.h" + + /* VF2PF interrupts */ +#define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9) +#define ADF_GEN2_ERR_MSK_VF2PF(vf_mask) (((vf_mask) & 0xFFFF) << 9) + +#define ADF_GEN2_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) + +u32 adf_gen2_get_pf2vf_offset(u32 i) +{ + return ADF_GEN2_PF2VF_OFFSET(i); +} +EXPORT_SYMBOL_GPL(adf_gen2_get_pf2vf_offset); + +u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_addr) +{ + u32 errsou3, errmsk3, vf_int_mask; + + /* Get the interrupt sources triggered by VFs */ + errsou3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU3); + vf_int_mask = ADF_GEN2_ERR_REG_VF2PF(errsou3); + + /* To avoid adding duplicate entries to work queue, clear + * vf_int_mask_sets bits that are already masked in ERRMSK register. + */ + errmsk3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3); + vf_int_mask &= ~ADF_GEN2_ERR_REG_VF2PF(errmsk3); + + return vf_int_mask; +} +EXPORT_SYMBOL_GPL(adf_gen2_get_vf2pf_sources); + +void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) +{ + /* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */ + if (vf_mask & 0xFFFF) { + u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) + & ~ADF_GEN2_ERR_MSK_VF2PF(vf_mask); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); + } +} +EXPORT_SYMBOL_GPL(adf_gen2_enable_vf2pf_interrupts); + +void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) +{ + /* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */ + if (vf_mask & 0xFFFF) { + u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) + | ADF_GEN2_ERR_MSK_VF2PF(vf_mask); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); + } +} +EXPORT_SYMBOL_GPL(adf_gen2_disable_vf2pf_interrupts); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h new file mode 100644 index 000000000000..0987e254e86b --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2021 Intel Corporation */ +#ifndef ADF_GEN2_PFVF_H +#define ADF_GEN2_PFVF_H + +#include +#include "adf_accel_devices.h" + +#define ADF_GEN2_ERRSOU3 (0x3A000 + 0x0C) +#define ADF_GEN2_ERRSOU5 (0x3A000 + 0xD8) +#define ADF_GEN2_ERRMSK3 (0x3A000 + 0x1C) +#define ADF_GEN2_ERRMSK5 (0x3A000 + 0xDC) + +u32 adf_gen2_get_pf2vf_offset(u32 i); +u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_bar); +void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask); +void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask); + +#endif /* ADF_GEN2_PFVF_H */ diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 8e2e1554dcf6..e134385b76a8 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -4,6 +4,7 @@ #include #include #include +#include #include "adf_dh895xcc_hw_data.h" #include "icp_qat_hw.h" @@ -114,14 +115,19 @@ static void adf_enable_ints(struct adf_accel_dev *accel_dev) static u32 get_vf2pf_sources(void __iomem *pmisc_bar) { - u32 errsou5, errmsk5, vf_int_mask; + u32 errsou3, errmsk3, errsou5, errmsk5, vf_int_mask; - vf_int_mask = adf_gen2_get_vf2pf_sources(pmisc_bar); + /* Get the interrupt sources triggered by VFs */ + errsou3 = ADF_CSR_RD(pmisc_bar, ADF_GEN2_ERRSOU3); + vf_int_mask = ADF_DH895XCC_ERR_REG_VF2PF_L(errsou3); - /* Get the interrupt sources triggered by VFs, but to avoid duplicates - * in the work queue, clear vf_int_mask_sets bits that are already - * masked in ERRMSK register. + /* To avoid adding duplicate entries to work queue, clear + * vf_int_mask_sets bits that are already masked in ERRMSK register. */ + errmsk3 = ADF_CSR_RD(pmisc_bar, ADF_GEN2_ERRMSK3); + vf_int_mask &= ~ADF_DH895XCC_ERR_REG_VF2PF_L(errmsk3); + + /* Do the same for ERRSOU5 */ errsou5 = ADF_CSR_RD(pmisc_bar, ADF_GEN2_ERRSOU5); errmsk5 = ADF_CSR_RD(pmisc_bar, ADF_GEN2_ERRMSK5); vf_int_mask |= ADF_DH895XCC_ERR_REG_VF2PF_U(errsou5); @@ -133,7 +139,11 @@ static u32 get_vf2pf_sources(void __iomem *pmisc_bar) static void enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) { /* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */ - adf_gen2_enable_vf2pf_interrupts(pmisc_addr, vf_mask); + if (vf_mask & 0xFFFF) { + u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) + & ~ADF_DH895XCC_ERR_MSK_VF2PF_L(vf_mask); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); + } /* Enable VF2PF Messaging Ints - VFs 16 through 31 per vf_mask[31:16] */ if (vf_mask >> 16) { @@ -147,7 +157,11 @@ static void enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) static void disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) { /* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */ - adf_gen2_disable_vf2pf_interrupts(pmisc_addr, vf_mask); + if (vf_mask & 0xFFFF) { + u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) + | ADF_DH895XCC_ERR_MSK_VF2PF_L(vf_mask); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); + } /* Disable VF2PF interrupts for VFs 16 through 31 per vf_mask[31:16] */ if (vf_mask >> 16) { diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h index 0af34dd8708a..aa17272a1507 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h @@ -25,6 +25,8 @@ #define ADF_DH895XCC_SMIA1_MASK 0x1 /* Masks for VF2PF interrupts */ +#define ADF_DH895XCC_ERR_REG_VF2PF_L(vf_src) (((vf_src) & 0x01FFFE00) >> 9) +#define ADF_DH895XCC_ERR_MSK_VF2PF_L(vf_mask) (((vf_mask) & 0xFFFF) << 9) #define ADF_DH895XCC_ERR_REG_VF2PF_U(vf_src) (((vf_src) & 0x0000FFFF) << 16) #define ADF_DH895XCC_ERR_MSK_VF2PF_U(vf_mask) ((vf_mask) >> 16) From 7e00fb3f162cfb27d8ac72910d7132612f07ec9c Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Wed, 17 Nov 2021 14:30:45 +0000 Subject: [PATCH 031/145] crypto: qat - relocate PFVF VF related logic Move device specific PFVF logic related to the VF to the newly created adf_gen2_pfvf.c. This refactory is done to isolate the GEN2 PFVF code into its own file in preparation for the introduction of support for PFVF for GEN4 devices. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 2 +- .../crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c | 8 ++------ .../crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h | 1 - drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c | 2 +- .../crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c | 8 ++------ .../crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h | 1 - drivers/crypto/qat/qat_common/adf_gen2_pfvf.c | 15 +++++++++++---- drivers/crypto/qat/qat_common/adf_gen2_pfvf.h | 3 ++- .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 2 +- .../qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c | 8 ++------ .../qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h | 1 - 11 files changed, 22 insertions(+), 29 deletions(-) diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 0bc528004f79..aaf8e65887b8 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -136,7 +136,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->enable_ints = adf_enable_ints; hw_data->reset_device = adf_reset_flr; hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer; - hw_data->get_pf2vf_offset = adf_gen2_get_pf2vf_offset; + hw_data->get_pf2vf_offset = adf_gen2_pf_get_pf2vf_offset; hw_data->get_vf2pf_sources = adf_gen2_get_vf2pf_sources; hw_data->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts; hw_data->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts; diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c index 3e69b520e82f..ee61f69a8077 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -4,6 +4,7 @@ #include #include #include +#include #include "adf_c3xxxvf_hw_data.h" static struct adf_hw_device_class c3xxxiov_class = { @@ -47,11 +48,6 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self) return DEV_SKU_VF; } -static u32 get_pf2vf_offset(u32 i) -{ - return ADF_C3XXXIOV_PF2VF_OFFSET; -} - static int adf_vf_int_noop(struct adf_accel_dev *accel_dev) { return 0; @@ -86,7 +82,7 @@ void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) hw_data->get_num_aes = get_num_aes; hw_data->get_etr_bar_id = get_etr_bar_id; hw_data->get_misc_bar_id = get_misc_bar_id; - hw_data->get_pf2vf_offset = get_pf2vf_offset; + hw_data->get_pf2vf_offset = adf_gen2_vf_get_pf2vf_offset; hw_data->get_sku = get_sku; hw_data->enable_ints = adf_vf_void_noop; hw_data->enable_pfvf_comms = adf_enable_vf2pf_comms; diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h index f5de4ce66014..6b4bf181d15b 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h @@ -12,7 +12,6 @@ #define ADF_C3XXXIOV_TX_RINGS_MASK 0xFF #define ADF_C3XXXIOV_ETR_BAR 0 #define ADF_C3XXXIOV_ETR_MAX_BANKS 1 -#define ADF_C3XXXIOV_PF2VF_OFFSET 0x200 void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data); void adf_clean_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data); diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index 9303f2dbcaf9..0d694c713797 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -138,7 +138,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->enable_ints = adf_enable_ints; hw_data->reset_device = adf_reset_flr; hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer; - hw_data->get_pf2vf_offset = adf_gen2_get_pf2vf_offset; + hw_data->get_pf2vf_offset = adf_gen2_pf_get_pf2vf_offset; hw_data->get_vf2pf_sources = adf_gen2_get_vf2pf_sources; hw_data->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts; hw_data->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts; diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c index 3bee3e467363..407f3beee43c 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -4,6 +4,7 @@ #include #include #include +#include #include "adf_c62xvf_hw_data.h" static struct adf_hw_device_class c62xiov_class = { @@ -47,11 +48,6 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self) return DEV_SKU_VF; } -static u32 get_pf2vf_offset(u32 i) -{ - return ADF_C62XIOV_PF2VF_OFFSET; -} - static int adf_vf_int_noop(struct adf_accel_dev *accel_dev) { return 0; @@ -86,7 +82,7 @@ void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data) hw_data->get_num_aes = get_num_aes; hw_data->get_etr_bar_id = get_etr_bar_id; hw_data->get_misc_bar_id = get_misc_bar_id; - hw_data->get_pf2vf_offset = get_pf2vf_offset; + hw_data->get_pf2vf_offset = adf_gen2_vf_get_pf2vf_offset; hw_data->get_sku = get_sku; hw_data->enable_ints = adf_vf_void_noop; hw_data->enable_pfvf_comms = adf_enable_vf2pf_comms; diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h index 794778c48678..a1a62c003ebf 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h +++ b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h @@ -12,7 +12,6 @@ #define ADF_C62XIOV_TX_RINGS_MASK 0xFF #define ADF_C62XIOV_ETR_BAR 0 #define ADF_C62XIOV_ETR_MAX_BANKS 1 -#define ADF_C62XIOV_PF2VF_OFFSET 0x200 void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data); void adf_clean_hw_data_c62xiov(struct adf_hw_device_data *hw_data); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c index d4d79419daaa..ea8d34922374 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c @@ -8,13 +8,20 @@ #define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9) #define ADF_GEN2_ERR_MSK_VF2PF(vf_mask) (((vf_mask) & 0xFFFF) << 9) -#define ADF_GEN2_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) +#define ADF_GEN2_PF_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) +#define ADF_GEN2_VF_PF2VF_OFFSET 0x200 -u32 adf_gen2_get_pf2vf_offset(u32 i) +u32 adf_gen2_pf_get_pf2vf_offset(u32 i) { - return ADF_GEN2_PF2VF_OFFSET(i); + return ADF_GEN2_PF_PF2VF_OFFSET(i); } -EXPORT_SYMBOL_GPL(adf_gen2_get_pf2vf_offset); +EXPORT_SYMBOL_GPL(adf_gen2_pf_get_pf2vf_offset); + +u32 adf_gen2_vf_get_pf2vf_offset(u32 i) +{ + return ADF_GEN2_VF_PF2VF_OFFSET; +} +EXPORT_SYMBOL_GPL(adf_gen2_vf_get_pf2vf_offset); u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_addr) { diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h index 0987e254e86b..a21787e3e550 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h @@ -11,7 +11,8 @@ #define ADF_GEN2_ERRMSK3 (0x3A000 + 0x1C) #define ADF_GEN2_ERRMSK5 (0x3A000 + 0xDC) -u32 adf_gen2_get_pf2vf_offset(u32 i); +u32 adf_gen2_pf_get_pf2vf_offset(u32 i); +u32 adf_gen2_vf_get_pf2vf_offset(u32 i); u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_bar); void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask); void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask); diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index e134385b76a8..5fc2e5a7f10b 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -215,7 +215,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->get_arb_mapping = adf_get_arbiter_mapping; hw_data->enable_ints = adf_enable_ints; hw_data->reset_device = adf_reset_sbr; - hw_data->get_pf2vf_offset = adf_gen2_get_pf2vf_offset; + hw_data->get_pf2vf_offset = adf_gen2_pf_get_pf2vf_offset; hw_data->get_vf2pf_sources = get_vf2pf_sources; hw_data->enable_vf2pf_interrupts = enable_vf2pf_interrupts; hw_data->disable_vf2pf_interrupts = disable_vf2pf_interrupts; diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index 7c6ed6bc8abf..30d862226026 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -4,6 +4,7 @@ #include #include #include +#include #include "adf_dh895xccvf_hw_data.h" static struct adf_hw_device_class dh895xcciov_class = { @@ -47,11 +48,6 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self) return DEV_SKU_VF; } -static u32 get_pf2vf_offset(u32 i) -{ - return ADF_DH895XCCIOV_PF2VF_OFFSET; -} - static int adf_vf_int_noop(struct adf_accel_dev *accel_dev) { return 0; @@ -86,7 +82,7 @@ void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) hw_data->get_num_aes = get_num_aes; hw_data->get_etr_bar_id = get_etr_bar_id; hw_data->get_misc_bar_id = get_misc_bar_id; - hw_data->get_pf2vf_offset = get_pf2vf_offset; + hw_data->get_pf2vf_offset = adf_gen2_vf_get_pf2vf_offset; hw_data->get_sku = get_sku; hw_data->enable_ints = adf_vf_void_noop; hw_data->enable_pfvf_comms = adf_enable_vf2pf_comms; diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h index 306ebb71a408..6973fa967bc8 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h @@ -12,7 +12,6 @@ #define ADF_DH895XCCIOV_TX_RINGS_MASK 0xFF #define ADF_DH895XCCIOV_ETR_BAR 0 #define ADF_DH895XCCIOV_ETR_MAX_BANKS 1 -#define ADF_DH895XCCIOV_PF2VF_OFFSET 0x200 void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data); void adf_clean_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data); From 6f2e28015bacfa8793902eb39a1fb20feb19a011 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 17 Nov 2021 14:30:46 +0000 Subject: [PATCH 032/145] crypto: qat - relocate PFVF disabled function Move the function pfvf_comms_disabled() from the qat_4xxx module to intel_qat as it will be used by other components to keep the PFVF feature disabled. Signed-off-by: Giovanni Cabiddu Reviewed-by: Marco Chiappero Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 7 +------ drivers/crypto/qat/qat_common/adf_common_drv.h | 2 ++ drivers/crypto/qat/qat_common/adf_gen4_hw_data.c | 7 +++++++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c index fd29861526d6..f4521eeeebed 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -191,11 +191,6 @@ static int adf_init_device(struct adf_accel_dev *accel_dev) return ret; } -static int pfvf_comms_disabled(struct adf_accel_dev *accel_dev) -{ - return 0; -} - static u32 uof_get_num_objs(void) { return ARRAY_SIZE(adf_4xxx_fw_config); @@ -259,7 +254,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) hw_data->uof_get_ae_mask = uof_get_ae_mask; hw_data->set_msix_rttable = set_msix_default_rttable; hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer; - hw_data->enable_pfvf_comms = pfvf_comms_disabled; + hw_data->enable_pfvf_comms = adf_pfvf_comms_disabled; hw_data->get_vf2pf_sources = get_vf2pf_sources; hw_data->disable_iov = adf_disable_sriov; hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index db9d045e6a06..c0699e453502 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -130,6 +130,8 @@ void adf_isr_resource_free(struct adf_accel_dev *accel_dev); int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev); void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev); +int adf_pfvf_comms_disabled(struct adf_accel_dev *accel_dev); + int qat_hal_init(struct adf_accel_dev *accel_dev); void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle); int qat_hal_start(struct icp_qat_fw_loader_handle *handle); diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c index 000528327b29..e3157df8a653 100644 --- a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2020 Intel Corporation */ #include "adf_accel_devices.h" +#include "adf_common_drv.h" #include "adf_gen4_hw_data.h" static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size) @@ -139,3 +140,9 @@ void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev) ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKEH_OFFSET, ssm_wdt_pke_high); } EXPORT_SYMBOL_GPL(adf_gen4_set_ssm_wdtimer); + +int adf_pfvf_comms_disabled(struct adf_accel_dev *accel_dev) +{ + return 0; +} +EXPORT_SYMBOL_GPL(adf_pfvf_comms_disabled); From bc63dabe525402b91320192ff1758f0d05b159c6 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Wed, 17 Nov 2021 14:30:47 +0000 Subject: [PATCH 033/145] crypto: qat - add pfvf_ops Add pfvf_ops structure to isolate PFVF related functions inside the adf_hw_device_data structure. For GEN2, the structure is populated using one of the two helper functions, adf_gen2_init_pf_pfvf_ops() or adf_gen2_init_vf_pfvf_ops(), for the PF and VF driver respectively. For the DH895XCC PF driver, the structure is populated using adf_gen2_init_pf_pfvf_ops() but some of the functions are then overwritten. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 4 +-- .../crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 6 +--- .../qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c | 3 +- .../crypto/qat/qat_c62x/adf_c62x_hw_data.c | 6 +--- .../qat/qat_c62xvf/adf_c62xvf_hw_data.c | 3 +- drivers/crypto/qat/qat_common/Makefile | 6 ++-- .../crypto/qat/qat_common/adf_accel_devices.h | 17 +++++---- drivers/crypto/qat/qat_common/adf_gen2_pfvf.c | 35 +++++++++++++------ drivers/crypto/qat/qat_common/adf_gen2_pfvf.h | 19 +++++++--- drivers/crypto/qat/qat_common/adf_init.c | 2 +- drivers/crypto/qat/qat_common/adf_isr.c | 8 ++--- drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 8 ++--- drivers/crypto/qat/qat_common/adf_sriov.c | 6 ++-- drivers/crypto/qat/qat_common/adf_vf2pf_msg.c | 2 +- .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 9 +++-- .../qat_dh895xccvf/adf_dh895xccvf_hw_data.c | 3 +- 16 files changed, 76 insertions(+), 61 deletions(-) diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c index f4521eeeebed..ec57a2e2d1fc 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -254,8 +254,8 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) hw_data->uof_get_ae_mask = uof_get_ae_mask; hw_data->set_msix_rttable = set_msix_default_rttable; hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer; - hw_data->enable_pfvf_comms = adf_pfvf_comms_disabled; - hw_data->get_vf2pf_sources = get_vf2pf_sources; + hw_data->pfvf_ops.enable_comms = adf_pfvf_comms_disabled; + hw_data->pfvf_ops.get_vf2pf_sources = get_vf2pf_sources; hw_data->disable_iov = adf_disable_sriov; hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index aaf8e65887b8..d25f78660b8c 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -136,14 +136,10 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->enable_ints = adf_enable_ints; hw_data->reset_device = adf_reset_flr; hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer; - hw_data->get_pf2vf_offset = adf_gen2_pf_get_pf2vf_offset; - hw_data->get_vf2pf_sources = adf_gen2_get_vf2pf_sources; - hw_data->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts; - hw_data->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts; - hw_data->enable_pfvf_comms = adf_enable_pf2vf_comms; hw_data->disable_iov = adf_disable_sriov; hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; + adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops); adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c index ee61f69a8077..c39733320063 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -82,13 +82,12 @@ void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) hw_data->get_num_aes = get_num_aes; hw_data->get_etr_bar_id = get_etr_bar_id; hw_data->get_misc_bar_id = get_misc_bar_id; - hw_data->get_pf2vf_offset = adf_gen2_vf_get_pf2vf_offset; hw_data->get_sku = get_sku; hw_data->enable_ints = adf_vf_void_noop; - hw_data->enable_pfvf_comms = adf_enable_vf2pf_comms; hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; hw_data->dev_class->instances++; adf_devmgr_update_class_index(hw_data); + adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops); adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index 0d694c713797..f24a01e1bc1a 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -138,14 +138,10 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->enable_ints = adf_enable_ints; hw_data->reset_device = adf_reset_flr; hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer; - hw_data->get_pf2vf_offset = adf_gen2_pf_get_pf2vf_offset; - hw_data->get_vf2pf_sources = adf_gen2_get_vf2pf_sources; - hw_data->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts; - hw_data->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts; - hw_data->enable_pfvf_comms = adf_enable_pf2vf_comms; hw_data->disable_iov = adf_disable_sriov; hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; + adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops); adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c index 407f3beee43c..03097bbe600a 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -82,13 +82,12 @@ void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data) hw_data->get_num_aes = get_num_aes; hw_data->get_etr_bar_id = get_etr_bar_id; hw_data->get_misc_bar_id = get_misc_bar_id; - hw_data->get_pf2vf_offset = adf_gen2_vf_get_pf2vf_offset; hw_data->get_sku = get_sku; hw_data->enable_ints = adf_vf_void_noop; - hw_data->enable_pfvf_comms = adf_enable_vf2pf_comms; hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; hw_data->dev_class->instances++; adf_devmgr_update_class_index(hw_data); + adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops); adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile index 3874e427d1f7..676aef6533e0 100644 --- a/drivers/crypto/qat/qat_common/Makefile +++ b/drivers/crypto/qat/qat_common/Makefile @@ -16,9 +16,9 @@ intel_qat-objs := adf_cfg.o \ qat_algs.o \ qat_asym_algs.o \ qat_uclo.o \ - qat_hal.o \ - adf_gen2_pfvf.o + qat_hal.o intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_pf2vf_msg.o \ - adf_vf2pf_msg.o adf_vf_isr.o + adf_vf2pf_msg.o adf_vf_isr.o \ + adf_gen2_pfvf.o diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 57d9ca08e611..a1baa65bd034 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -147,6 +147,14 @@ struct adf_accel_dev; struct adf_etr_data; struct adf_etr_ring_data; +struct adf_pfvf_ops { + int (*enable_comms)(struct adf_accel_dev *accel_dev); + u32 (*get_pf2vf_offset)(u32 i); + u32 (*get_vf2pf_sources)(void __iomem *pmisc_addr); + void (*enable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask); + void (*disable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask); +}; + struct adf_hw_device_data { struct adf_hw_device_class *dev_class; u32 (*get_accel_mask)(struct adf_hw_device_data *self); @@ -157,7 +165,6 @@ struct adf_hw_device_data { u32 (*get_etr_bar_id)(struct adf_hw_device_data *self); u32 (*get_num_aes)(struct adf_hw_device_data *self); u32 (*get_num_accels)(struct adf_hw_device_data *self); - u32 (*get_pf2vf_offset)(u32 i); void (*get_arb_info)(struct arb_info *arb_csrs_info); void (*get_admin_info)(struct admin_info *admin_csrs_info); enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self); @@ -176,17 +183,12 @@ struct adf_hw_device_data { bool enable); void (*enable_ints)(struct adf_accel_dev *accel_dev); void (*set_ssm_wdtimer)(struct adf_accel_dev *accel_dev); - int (*enable_pfvf_comms)(struct adf_accel_dev *accel_dev); - u32 (*get_vf2pf_sources)(void __iomem *pmisc_addr); - void (*enable_vf2pf_interrupts)(void __iomem *pmisc_bar_addr, - u32 vf_mask); - void (*disable_vf2pf_interrupts)(void __iomem *pmisc_bar_addr, - u32 vf_mask); void (*reset_device)(struct adf_accel_dev *accel_dev); void (*set_msix_rttable)(struct adf_accel_dev *accel_dev); char *(*uof_get_name)(u32 obj_num); u32 (*uof_get_num_objs)(void); u32 (*uof_get_ae_mask)(u32 obj_num); + struct adf_pfvf_ops pfvf_ops; struct adf_hw_csr_ops csr_ops; const char *fw_name; const char *fw_mmp_name; @@ -222,6 +224,7 @@ struct adf_hw_device_data { GET_HW_DATA(accel_dev)->num_rings_per_bank #define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines) #define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_ops) +#define GET_PFVF_OPS(accel_dev) (&(accel_dev)->hw_device->pfvf_ops) #define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev struct adf_admin_comms; diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c index ea8d34922374..36c8ff009661 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c @@ -2,6 +2,7 @@ /* Copyright(c) 2021 Intel Corporation */ #include #include "adf_accel_devices.h" +#include "adf_common_drv.h" #include "adf_gen2_pfvf.h" /* VF2PF interrupts */ @@ -11,19 +12,17 @@ #define ADF_GEN2_PF_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) #define ADF_GEN2_VF_PF2VF_OFFSET 0x200 -u32 adf_gen2_pf_get_pf2vf_offset(u32 i) +static u32 adf_gen2_pf_get_pf2vf_offset(u32 i) { return ADF_GEN2_PF_PF2VF_OFFSET(i); } -EXPORT_SYMBOL_GPL(adf_gen2_pf_get_pf2vf_offset); -u32 adf_gen2_vf_get_pf2vf_offset(u32 i) +static u32 adf_gen2_vf_get_pf2vf_offset(u32 i) { return ADF_GEN2_VF_PF2VF_OFFSET; } -EXPORT_SYMBOL_GPL(adf_gen2_vf_get_pf2vf_offset); -u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_addr) +static u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_addr) { u32 errsou3, errmsk3, vf_int_mask; @@ -39,9 +38,9 @@ u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_addr) return vf_int_mask; } -EXPORT_SYMBOL_GPL(adf_gen2_get_vf2pf_sources); -void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) +static void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, + u32 vf_mask) { /* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */ if (vf_mask & 0xFFFF) { @@ -50,9 +49,9 @@ void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); } } -EXPORT_SYMBOL_GPL(adf_gen2_enable_vf2pf_interrupts); -void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) +static void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, + u32 vf_mask) { /* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */ if (vf_mask & 0xFFFF) { @@ -61,4 +60,20 @@ void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); } } -EXPORT_SYMBOL_GPL(adf_gen2_disable_vf2pf_interrupts); + +void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) +{ + pfvf_ops->enable_comms = adf_enable_pf2vf_comms; + pfvf_ops->get_pf2vf_offset = adf_gen2_pf_get_pf2vf_offset; + pfvf_ops->get_vf2pf_sources = adf_gen2_get_vf2pf_sources; + pfvf_ops->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts; + pfvf_ops->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts; +} +EXPORT_SYMBOL_GPL(adf_gen2_init_pf_pfvf_ops); + +void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) +{ + pfvf_ops->enable_comms = adf_enable_vf2pf_comms; + pfvf_ops->get_pf2vf_offset = adf_gen2_vf_get_pf2vf_offset; +} +EXPORT_SYMBOL_GPL(adf_gen2_init_vf_pfvf_ops); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h index a21787e3e550..a716545a764c 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h @@ -11,10 +11,19 @@ #define ADF_GEN2_ERRMSK3 (0x3A000 + 0x1C) #define ADF_GEN2_ERRMSK5 (0x3A000 + 0xDC) -u32 adf_gen2_pf_get_pf2vf_offset(u32 i); -u32 adf_gen2_vf_get_pf2vf_offset(u32 i); -u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_bar); -void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask); -void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask); +#if defined(CONFIG_PCI_IOV) +void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops); +void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops); +#else +static inline void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) +{ + pfvf_ops->enable_comms = adf_pfvf_comms_disabled; +} + +static inline void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) +{ + pfvf_ops->enable_comms = adf_pfvf_comms_disabled; +} +#endif #endif /* ADF_GEN2_PFVF_H */ diff --git a/drivers/crypto/qat/qat_common/adf_init.c b/drivers/crypto/qat/qat_common/adf_init.c index e3749e5817d9..391d82a64a93 100644 --- a/drivers/crypto/qat/qat_common/adf_init.c +++ b/drivers/crypto/qat/qat_common/adf_init.c @@ -117,7 +117,7 @@ int adf_dev_init(struct adf_accel_dev *accel_dev) hw_data->enable_ints(accel_dev); hw_data->enable_error_correction(accel_dev); - ret = hw_data->enable_pfvf_comms(accel_dev); + ret = hw_data->pfvf_ops.enable_comms(accel_dev); if (ret) return ret; diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c index 2b4900c91308..358200c0d598 100644 --- a/drivers/crypto/qat/qat_common/adf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_isr.c @@ -64,7 +64,7 @@ void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) unsigned long flags; spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); - hw_data->enable_vf2pf_interrupts(pmisc_addr, vf_mask); + hw_data->pfvf_ops.enable_vf2pf_interrupts(pmisc_addr, vf_mask); spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); } @@ -77,7 +77,7 @@ void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) unsigned long flags; spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); - hw_data->disable_vf2pf_interrupts(pmisc_addr, vf_mask); + hw_data->pfvf_ops.disable_vf2pf_interrupts(pmisc_addr, vf_mask); spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); } @@ -90,7 +90,7 @@ static void adf_disable_vf2pf_interrupts_irq(struct adf_accel_dev *accel_dev, void __iomem *pmisc_addr = pmisc->virt_addr; spin_lock(&accel_dev->pf.vf2pf_ints_lock); - hw_data->disable_vf2pf_interrupts(pmisc_addr, vf_mask); + hw_data->pfvf_ops.disable_vf2pf_interrupts(pmisc_addr, vf_mask); spin_unlock(&accel_dev->pf.vf2pf_ints_lock); } @@ -104,7 +104,7 @@ static bool adf_handle_vf2pf_int(struct adf_accel_dev *accel_dev) unsigned long vf_mask; /* Get the interrupt sources triggered by VFs */ - vf_mask = hw_data->get_vf2pf_sources(pmisc_addr); + vf_mask = hw_data->pfvf_ops.get_vf2pf_sources(pmisc_addr); if (vf_mask) { struct adf_accel_vf_info *vf_info; diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index d98e3639c9d2..78dc8aea4866 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -29,7 +29,7 @@ static int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) int ret; if (accel_dev->is_vf) { - pf2vf_offset = hw_data->get_pf2vf_offset(0); + pf2vf_offset = hw_data->pfvf_ops.get_pf2vf_offset(0); lock = &accel_dev->vf.vf2pf_lock; local_in_use_mask = ADF_VF2PF_IN_USE_BY_VF_MASK; local_in_use_pattern = ADF_VF2PF_IN_USE_BY_VF; @@ -37,7 +37,7 @@ static int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) remote_in_use_pattern = ADF_PF2VF_IN_USE_BY_PF; int_bit = ADF_VF2PF_INT; } else { - pf2vf_offset = hw_data->get_pf2vf_offset(vf_nr); + pf2vf_offset = hw_data->pfvf_ops.get_pf2vf_offset(vf_nr); lock = &accel_dev->pf.vf_info[vf_nr].pf2vf_lock; local_in_use_mask = ADF_PF2VF_IN_USE_BY_PF_MASK; local_in_use_pattern = ADF_PF2VF_IN_USE_BY_PF; @@ -258,7 +258,7 @@ bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr) u32 msg, resp = 0; /* Read message from the VF */ - msg = ADF_CSR_RD(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr)); + msg = ADF_CSR_RD(pmisc_addr, hw_data->pfvf_ops.get_pf2vf_offset(vf_nr)); if (!(msg & ADF_VF2PF_INT)) { dev_info(&GET_DEV(accel_dev), "Spurious VF2PF interrupt, msg %X. Ignored\n", msg); @@ -275,7 +275,7 @@ bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr) /* To ACK, clear the VF2PFINT bit */ msg &= ~ADF_VF2PF_INT; - ADF_CSR_WR(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr), msg); + ADF_CSR_WR(pmisc_addr, hw_data->pfvf_ops.get_pf2vf_offset(vf_nr), msg); if (adf_handle_vf2pf_msg(accel_dev, vf_nr, msg, &resp)) return false; diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index b1a814ac1d67..342063406c19 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -70,7 +70,7 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) hw_data->configure_iov_threads(accel_dev, true); /* Enable VF to PF interrupts for all VFs */ - if (hw_data->get_pf2vf_offset) + if (hw_data->pfvf_ops.get_pf2vf_offset) adf_enable_vf2pf_interrupts(accel_dev, BIT_ULL(totalvfs) - 1); /* @@ -100,13 +100,13 @@ void adf_disable_sriov(struct adf_accel_dev *accel_dev) if (!accel_dev->pf.vf_info) return; - if (hw_data->get_pf2vf_offset) + if (hw_data->pfvf_ops.get_pf2vf_offset) adf_pf2vf_notify_restarting(accel_dev); pci_disable_sriov(accel_to_pci_dev(accel_dev)); /* Disable VF to PF interrupts */ - if (hw_data->get_pf2vf_offset) + if (hw_data->pfvf_ops.get_pf2vf_offset) adf_disable_vf2pf_interrupts(accel_dev, GENMASK(31, 0)); /* Clear Valid bits in AE Thread to PCIe Function Mapping */ diff --git a/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c b/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c index 01a6e68f256b..d11eb60b3e86 100644 --- a/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c @@ -81,7 +81,7 @@ bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev) struct adf_bar *pmisc = &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; void __iomem *pmisc_bar_addr = pmisc->virt_addr; - u32 offset = hw_data->get_pf2vf_offset(0); + u32 offset = hw_data->pfvf_ops.get_pf2vf_offset(0); u32 msg; /* Read the message from PF */ diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 5fc2e5a7f10b..aa42373a7118 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -215,14 +215,13 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->get_arb_mapping = adf_get_arbiter_mapping; hw_data->enable_ints = adf_enable_ints; hw_data->reset_device = adf_reset_sbr; - hw_data->get_pf2vf_offset = adf_gen2_pf_get_pf2vf_offset; - hw_data->get_vf2pf_sources = get_vf2pf_sources; - hw_data->enable_vf2pf_interrupts = enable_vf2pf_interrupts; - hw_data->disable_vf2pf_interrupts = disable_vf2pf_interrupts; - hw_data->enable_pfvf_comms = adf_enable_pf2vf_comms; hw_data->disable_iov = adf_disable_sriov; hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; + adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops); + hw_data->pfvf_ops.get_vf2pf_sources = get_vf2pf_sources; + hw_data->pfvf_ops.enable_vf2pf_interrupts = enable_vf2pf_interrupts; + hw_data->pfvf_ops.disable_vf2pf_interrupts = disable_vf2pf_interrupts; adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index 30d862226026..2e2ef6b5bd2a 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -82,13 +82,12 @@ void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) hw_data->get_num_aes = get_num_aes; hw_data->get_etr_bar_id = get_etr_bar_id; hw_data->get_misc_bar_id = get_misc_bar_id; - hw_data->get_pf2vf_offset = adf_gen2_vf_get_pf2vf_offset; hw_data->get_sku = get_sku; hw_data->enable_ints = adf_vf_void_noop; - hw_data->enable_pfvf_comms = adf_enable_vf2pf_comms; hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; hw_data->dev_class->instances++; adf_devmgr_update_class_index(hw_data); + adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops); adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } From 9baf2de7ee4eeaca5cf95d1a399182ab83aa42b8 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Wed, 17 Nov 2021 14:30:48 +0000 Subject: [PATCH 034/145] crypto: qat - differentiate between pf2vf and vf2pf offset Add the function get_vf2pf_offset() to adf_pfvf_ops to differentiate the CSRs used for pf2vf and vf2pf. Offsets may or may not be direction specific depending on QAT generation. Since in QAT GEN2 the CSR is not direction specific, i.e. there is a single mailbox register shared for pf2vf and vf2pf, both get_vf2pf_offset() and get_vf2pf_offset() will return the same offset. This change is to make the direction explicit, so it is easier to understand and debug and also in preparation for the introduction of PFVF support in the qat_4xxx driver since QAT GEN4 devices have a separate CSR for pf2vf and vf2pf communications. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_devices.h | 1 + drivers/crypto/qat/qat_common/adf_gen2_pfvf.c | 10 ++++++---- drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 6 +++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index a1baa65bd034..d9b2cc935b61 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -150,6 +150,7 @@ struct adf_etr_ring_data; struct adf_pfvf_ops { int (*enable_comms)(struct adf_accel_dev *accel_dev); u32 (*get_pf2vf_offset)(u32 i); + u32 (*get_vf2pf_offset)(u32 i); u32 (*get_vf2pf_sources)(void __iomem *pmisc_addr); void (*enable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask); void (*disable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c index 36c8ff009661..2f27146bb7c6 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c @@ -12,12 +12,12 @@ #define ADF_GEN2_PF_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) #define ADF_GEN2_VF_PF2VF_OFFSET 0x200 -static u32 adf_gen2_pf_get_pf2vf_offset(u32 i) +static u32 adf_gen2_pf_get_pfvf_offset(u32 i) { return ADF_GEN2_PF_PF2VF_OFFSET(i); } -static u32 adf_gen2_vf_get_pf2vf_offset(u32 i) +static u32 adf_gen2_vf_get_pfvf_offset(u32 i) { return ADF_GEN2_VF_PF2VF_OFFSET; } @@ -64,7 +64,8 @@ static void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) { pfvf_ops->enable_comms = adf_enable_pf2vf_comms; - pfvf_ops->get_pf2vf_offset = adf_gen2_pf_get_pf2vf_offset; + pfvf_ops->get_pf2vf_offset = adf_gen2_pf_get_pfvf_offset; + pfvf_ops->get_vf2pf_offset = adf_gen2_pf_get_pfvf_offset; pfvf_ops->get_vf2pf_sources = adf_gen2_get_vf2pf_sources; pfvf_ops->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts; pfvf_ops->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts; @@ -74,6 +75,7 @@ EXPORT_SYMBOL_GPL(adf_gen2_init_pf_pfvf_ops); void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) { pfvf_ops->enable_comms = adf_enable_vf2pf_comms; - pfvf_ops->get_pf2vf_offset = adf_gen2_vf_get_pf2vf_offset; + pfvf_ops->get_pf2vf_offset = adf_gen2_vf_get_pfvf_offset; + pfvf_ops->get_vf2pf_offset = adf_gen2_vf_get_pfvf_offset; } EXPORT_SYMBOL_GPL(adf_gen2_init_vf_pfvf_ops); diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index 78dc8aea4866..c420ec03081b 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -29,7 +29,7 @@ static int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) int ret; if (accel_dev->is_vf) { - pf2vf_offset = hw_data->pfvf_ops.get_pf2vf_offset(0); + pf2vf_offset = hw_data->pfvf_ops.get_vf2pf_offset(0); lock = &accel_dev->vf.vf2pf_lock; local_in_use_mask = ADF_VF2PF_IN_USE_BY_VF_MASK; local_in_use_pattern = ADF_VF2PF_IN_USE_BY_VF; @@ -258,7 +258,7 @@ bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr) u32 msg, resp = 0; /* Read message from the VF */ - msg = ADF_CSR_RD(pmisc_addr, hw_data->pfvf_ops.get_pf2vf_offset(vf_nr)); + msg = ADF_CSR_RD(pmisc_addr, hw_data->pfvf_ops.get_vf2pf_offset(vf_nr)); if (!(msg & ADF_VF2PF_INT)) { dev_info(&GET_DEV(accel_dev), "Spurious VF2PF interrupt, msg %X. Ignored\n", msg); @@ -275,7 +275,7 @@ bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr) /* To ACK, clear the VF2PFINT bit */ msg &= ~ADF_VF2PF_INT; - ADF_CSR_WR(pmisc_addr, hw_data->pfvf_ops.get_pf2vf_offset(vf_nr), msg); + ADF_CSR_WR(pmisc_addr, hw_data->pfvf_ops.get_vf2pf_offset(vf_nr), msg); if (adf_handle_vf2pf_msg(accel_dev, vf_nr, msg, &resp)) return false; From 49c43538ce05115e3fe12752dc77aa1deed3b0d2 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Wed, 17 Nov 2021 14:30:49 +0000 Subject: [PATCH 035/145] crypto: qat - abstract PFVF send function Make the PFVF send function device specific. This is in preparation for the introduction of PFVF support in the qat_4xxx driver since the send logic differs between QAT GEN2 and QAT GEN4 devices. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_accel_devices.h | 1 + drivers/crypto/qat/qat_common/adf_gen2_pfvf.c | 97 +++++++++++++++++++ drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 92 +----------------- drivers/crypto/qat/qat_common/adf_pf2vf_msg.h | 3 +- 4 files changed, 102 insertions(+), 91 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index d9b2cc935b61..893814959083 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -154,6 +154,7 @@ struct adf_pfvf_ops { u32 (*get_vf2pf_sources)(void __iomem *pmisc_addr); void (*enable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask); void (*disable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask); + int (*send_msg)(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr); }; struct adf_hw_device_data { diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c index 2f27146bb7c6..5eba042d453b 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c @@ -1,9 +1,12 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2021 Intel Corporation */ +#include +#include #include #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "adf_gen2_pfvf.h" +#include "adf_pf2vf_msg.h" /* VF2PF interrupts */ #define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9) @@ -12,6 +15,12 @@ #define ADF_GEN2_PF_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) #define ADF_GEN2_VF_PF2VF_OFFSET 0x200 +#define ADF_PFVF_MSG_ACK_DELAY 2 +#define ADF_PFVF_MSG_ACK_MAX_RETRY 100 + +#define ADF_PFVF_MSG_RETRY_DELAY 5 +#define ADF_PFVF_MSG_MAX_RETRIES 3 + static u32 adf_gen2_pf_get_pfvf_offset(u32 i) { return ADF_GEN2_PF_PF2VF_OFFSET(i); @@ -61,6 +70,92 @@ static void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, } } +static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev, u32 msg, + u8 vf_nr) +{ + struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *pmisc_bar_addr = + pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr; + u32 val, pfvf_offset, count = 0; + u32 local_in_use_mask, local_in_use_pattern; + u32 remote_in_use_mask, remote_in_use_pattern; + struct mutex *lock; /* lock preventing concurrent acces of CSR */ + unsigned int retries = ADF_PFVF_MSG_MAX_RETRIES; + u32 int_bit; + int ret; + + if (accel_dev->is_vf) { + pfvf_offset = GET_PFVF_OPS(accel_dev)->get_vf2pf_offset(0); + lock = &accel_dev->vf.vf2pf_lock; + local_in_use_mask = ADF_VF2PF_IN_USE_BY_VF_MASK; + local_in_use_pattern = ADF_VF2PF_IN_USE_BY_VF; + remote_in_use_mask = ADF_PF2VF_IN_USE_BY_PF_MASK; + remote_in_use_pattern = ADF_PF2VF_IN_USE_BY_PF; + int_bit = ADF_VF2PF_INT; + } else { + pfvf_offset = GET_PFVF_OPS(accel_dev)->get_pf2vf_offset(vf_nr); + lock = &accel_dev->pf.vf_info[vf_nr].pf2vf_lock; + local_in_use_mask = ADF_PF2VF_IN_USE_BY_PF_MASK; + local_in_use_pattern = ADF_PF2VF_IN_USE_BY_PF; + remote_in_use_mask = ADF_VF2PF_IN_USE_BY_VF_MASK; + remote_in_use_pattern = ADF_VF2PF_IN_USE_BY_VF; + int_bit = ADF_PF2VF_INT; + } + + msg &= ~local_in_use_mask; + msg |= local_in_use_pattern; + + mutex_lock(lock); + +start: + ret = 0; + + /* Check if the PFVF CSR is in use by remote function */ + val = ADF_CSR_RD(pmisc_bar_addr, pfvf_offset); + if ((val & remote_in_use_mask) == remote_in_use_pattern) { + dev_dbg(&GET_DEV(accel_dev), + "PFVF CSR in use by remote function\n"); + goto retry; + } + + /* Attempt to get ownership of the PFVF CSR */ + ADF_CSR_WR(pmisc_bar_addr, pfvf_offset, msg | int_bit); + + /* Wait for confirmation from remote func it received the message */ + do { + msleep(ADF_PFVF_MSG_ACK_DELAY); + val = ADF_CSR_RD(pmisc_bar_addr, pfvf_offset); + } while ((val & int_bit) && (count++ < ADF_PFVF_MSG_ACK_MAX_RETRY)); + + if (val & int_bit) { + dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n"); + val &= ~int_bit; + ret = -EIO; + } + + if (val != msg) { + dev_dbg(&GET_DEV(accel_dev), + "Collision - PFVF CSR overwritten by remote function\n"); + goto retry; + } + + /* Finished with the PFVF CSR; relinquish it and leave msg in CSR */ + ADF_CSR_WR(pmisc_bar_addr, pfvf_offset, val & ~local_in_use_mask); +out: + mutex_unlock(lock); + return ret; + +retry: + if (--retries) { + msleep(ADF_PFVF_MSG_RETRY_DELAY); + goto start; + } else { + ret = -EBUSY; + goto out; + } +} + void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) { pfvf_ops->enable_comms = adf_enable_pf2vf_comms; @@ -69,6 +164,7 @@ void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) pfvf_ops->get_vf2pf_sources = adf_gen2_get_vf2pf_sources; pfvf_ops->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts; pfvf_ops->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts; + pfvf_ops->send_msg = adf_gen2_pfvf_send; } EXPORT_SYMBOL_GPL(adf_gen2_init_pf_pfvf_ops); @@ -77,5 +173,6 @@ void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) pfvf_ops->enable_comms = adf_enable_vf2pf_comms; pfvf_ops->get_pf2vf_offset = adf_gen2_vf_get_pfvf_offset; pfvf_ops->get_vf2pf_offset = adf_gen2_vf_get_pfvf_offset; + pfvf_ops->send_msg = adf_gen2_pfvf_send; } EXPORT_SYMBOL_GPL(adf_gen2_init_vf_pfvf_ops); diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index c420ec03081b..074e521ed9e8 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -1,6 +1,5 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2015 - 2020 Intel Corporation */ -#include #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "adf_pf2vf_msg.h" @@ -8,97 +7,10 @@ #define ADF_PFVF_MSG_COLLISION_DETECT_DELAY 10 #define ADF_PFVF_MSG_ACK_DELAY 2 #define ADF_PFVF_MSG_ACK_MAX_RETRY 100 -#define ADF_PFVF_MSG_RETRY_DELAY 5 -#define ADF_PFVF_MSG_MAX_RETRIES 3 #define ADF_PFVF_MSG_RESP_TIMEOUT (ADF_PFVF_MSG_ACK_DELAY * \ ADF_PFVF_MSG_ACK_MAX_RETRY + \ ADF_PFVF_MSG_COLLISION_DETECT_DELAY) -static int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) -{ - struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - void __iomem *pmisc_bar_addr = - pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr; - u32 val, pf2vf_offset, count = 0; - u32 local_in_use_mask, local_in_use_pattern; - u32 remote_in_use_mask, remote_in_use_pattern; - struct mutex *lock; /* lock preventing concurrent acces of CSR */ - unsigned int retries = ADF_PFVF_MSG_MAX_RETRIES; - u32 int_bit; - int ret; - - if (accel_dev->is_vf) { - pf2vf_offset = hw_data->pfvf_ops.get_vf2pf_offset(0); - lock = &accel_dev->vf.vf2pf_lock; - local_in_use_mask = ADF_VF2PF_IN_USE_BY_VF_MASK; - local_in_use_pattern = ADF_VF2PF_IN_USE_BY_VF; - remote_in_use_mask = ADF_PF2VF_IN_USE_BY_PF_MASK; - remote_in_use_pattern = ADF_PF2VF_IN_USE_BY_PF; - int_bit = ADF_VF2PF_INT; - } else { - pf2vf_offset = hw_data->pfvf_ops.get_pf2vf_offset(vf_nr); - lock = &accel_dev->pf.vf_info[vf_nr].pf2vf_lock; - local_in_use_mask = ADF_PF2VF_IN_USE_BY_PF_MASK; - local_in_use_pattern = ADF_PF2VF_IN_USE_BY_PF; - remote_in_use_mask = ADF_VF2PF_IN_USE_BY_VF_MASK; - remote_in_use_pattern = ADF_VF2PF_IN_USE_BY_VF; - int_bit = ADF_PF2VF_INT; - } - - msg &= ~local_in_use_mask; - msg |= local_in_use_pattern; - - mutex_lock(lock); - -start: - ret = 0; - - /* Check if the PFVF CSR is in use by remote function */ - val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset); - if ((val & remote_in_use_mask) == remote_in_use_pattern) { - dev_dbg(&GET_DEV(accel_dev), - "PFVF CSR in use by remote function\n"); - goto retry; - } - - /* Attempt to get ownership of the PFVF CSR */ - ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, msg | int_bit); - - /* Wait for confirmation from remote func it received the message */ - do { - msleep(ADF_PFVF_MSG_ACK_DELAY); - val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset); - } while ((val & int_bit) && (count++ < ADF_PFVF_MSG_ACK_MAX_RETRY)); - - if (val & int_bit) { - dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n"); - val &= ~int_bit; - ret = -EIO; - } - - if (val != msg) { - dev_dbg(&GET_DEV(accel_dev), - "Collision - PFVF CSR overwritten by remote function\n"); - goto retry; - } - - /* Finished with the PFVF CSR; relinquish it and leave msg in CSR */ - ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, val & ~local_in_use_mask); -out: - mutex_unlock(lock); - return ret; - -retry: - if (--retries) { - msleep(ADF_PFVF_MSG_RETRY_DELAY); - goto start; - } else { - ret = -EBUSY; - goto out; - } -} - /** * adf_send_pf2vf_msg() - send PF to VF message * @accel_dev: Pointer to acceleration device @@ -111,7 +23,7 @@ retry: */ static int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, u32 msg) { - return adf_iov_putmsg(accel_dev, msg, vf_nr); + return GET_PFVF_OPS(accel_dev)->send_msg(accel_dev, msg, vf_nr); } /** @@ -125,7 +37,7 @@ static int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, u32 msg */ int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg) { - return adf_iov_putmsg(accel_dev, msg, 0); + return GET_PFVF_OPS(accel_dev)->send_msg(accel_dev, msg, 0); } /** diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h index a7d8f8367345..73eb8f13ad09 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h @@ -49,7 +49,8 @@ * * When a PF or VF attempts to send a message in the lower or upper 16 bits, * respectively, the other 16 bits are written to first with a defined - * IN_USE_BY pattern as part of a collision control scheme (see adf_iov_putmsg). + * IN_USE_BY pattern as part of a collision control scheme (see function + * adf_gen2_pfvf_send() in adf_pf2vf_msg.c). */ #define ADF_PFVF_COMPAT_THIS_VERSION 0x1 /* PF<->VF compat */ From 1ea7c2beca5b200240f6480d36ebd6ca775a5fca Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 17 Nov 2021 14:30:50 +0000 Subject: [PATCH 036/145] crypto: qat - abstract PFVF receive logic Refactor the PFVF receive logic so it is common between PF and VF and make it device specific. This is in preparation for the introduction of PFVF support in the qat_4xxx driver since the receive logic differs between QAT GEN2 and QAT GEN4 devices. Signed-off-by: Giovanni Cabiddu Co-developed-by: Marco Chiappero Signed-off-by: Marco Chiappero Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_accel_devices.h | 1 + drivers/crypto/qat/qat_common/adf_gen2_pfvf.c | 45 +++++++++++++++++++ drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 41 ++++++++--------- drivers/crypto/qat/qat_common/adf_vf2pf_msg.c | 38 +++++++--------- 4 files changed, 81 insertions(+), 44 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 893814959083..35e62a73f9fa 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -155,6 +155,7 @@ struct adf_pfvf_ops { void (*enable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask); void (*disable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask); int (*send_msg)(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr); + u32 (*recv_msg)(struct adf_accel_dev *accel_dev, u8 vf_nr); }; struct adf_hw_device_data { diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c index 5eba042d453b..f79c3ca28283 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c @@ -156,6 +156,49 @@ retry: } } +static u32 adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev, u8 vf_nr) +{ + struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *pmisc_addr = + pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr; + u32 pfvf_offset; + u32 msg_origin; + u32 int_bit; + u32 msg; + + if (accel_dev->is_vf) { + pfvf_offset = GET_PFVF_OPS(accel_dev)->get_pf2vf_offset(0); + int_bit = ADF_PF2VF_INT; + msg_origin = ADF_PF2VF_MSGORIGIN_SYSTEM; + } else { + pfvf_offset = GET_PFVF_OPS(accel_dev)->get_vf2pf_offset(vf_nr); + int_bit = ADF_VF2PF_INT; + msg_origin = ADF_VF2PF_MSGORIGIN_SYSTEM; + } + + /* Read message */ + msg = ADF_CSR_RD(pmisc_addr, pfvf_offset); + if (!(msg & int_bit)) { + dev_info(&GET_DEV(accel_dev), + "Spurious PFVF interrupt, msg %X. Ignored\n", msg); + return 0; + } + + /* Ignore legacy non-system (non-kernel) VF2PF messages */ + if (!(msg & msg_origin)) { + dev_dbg(&GET_DEV(accel_dev), + "Ignored non-system message (0x%x);\n", msg); + return 0; + } + + /* To ACK, clear the INT bit */ + msg &= ~int_bit; + ADF_CSR_WR(pmisc_addr, pfvf_offset, msg); + + return msg; +} + void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) { pfvf_ops->enable_comms = adf_enable_pf2vf_comms; @@ -165,6 +208,7 @@ void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) pfvf_ops->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts; pfvf_ops->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts; pfvf_ops->send_msg = adf_gen2_pfvf_send; + pfvf_ops->recv_msg = adf_gen2_pfvf_recv; } EXPORT_SYMBOL_GPL(adf_gen2_init_pf_pfvf_ops); @@ -174,5 +218,6 @@ void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) pfvf_ops->get_pf2vf_offset = adf_gen2_vf_get_pfvf_offset; pfvf_ops->get_vf2pf_offset = adf_gen2_vf_get_pfvf_offset; pfvf_ops->send_msg = adf_gen2_pfvf_send; + pfvf_ops->recv_msg = adf_gen2_pfvf_recv; } EXPORT_SYMBOL_GPL(adf_gen2_init_vf_pfvf_ops); diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index 074e521ed9e8..c064e8bab50d 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -40,6 +40,20 @@ int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg) return GET_PFVF_OPS(accel_dev)->send_msg(accel_dev, msg, 0); } +/** + * adf_recv_vf2pf_msg() - receive a VF to PF message + * @accel_dev: Pointer to acceleration device + * @vf_nr: Number of the VF from where the message will be received + * + * This function allows the PF to receive a message from a specific VF. + * + * Return: a valid message on success, zero otherwise. + */ +static u32 adf_recv_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr) +{ + return GET_PFVF_OPS(accel_dev)->recv_msg(accel_dev, vf_nr); +} + /** * adf_send_vf2pf_req() - send VF2PF request message * @accel_dev: Pointer to acceleration device. @@ -163,31 +177,12 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr) { - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - int bar_id = hw_data->get_misc_bar_id(hw_data); - struct adf_bar *pmisc = &GET_BARS(accel_dev)[bar_id]; - void __iomem *pmisc_addr = pmisc->virt_addr; - u32 msg, resp = 0; + u32 resp = 0; + u32 msg; - /* Read message from the VF */ - msg = ADF_CSR_RD(pmisc_addr, hw_data->pfvf_ops.get_vf2pf_offset(vf_nr)); - if (!(msg & ADF_VF2PF_INT)) { - dev_info(&GET_DEV(accel_dev), - "Spurious VF2PF interrupt, msg %X. Ignored\n", msg); + msg = adf_recv_vf2pf_msg(accel_dev, vf_nr); + if (!msg) return true; - } - - /* Ignore legacy non-system (non-kernel) VF2PF messages */ - if (!(msg & ADF_VF2PF_MSGORIGIN_SYSTEM)) { - dev_dbg(&GET_DEV(accel_dev), - "Ignored non-system message from VF%d (0x%x);\n", - vf_nr + 1, msg); - return true; - } - - /* To ACK, clear the VF2PFINT bit */ - msg &= ~ADF_VF2PF_INT; - ADF_CSR_WR(pmisc_addr, hw_data->pfvf_ops.get_vf2pf_offset(vf_nr), msg); if (adf_handle_vf2pf_msg(accel_dev, vf_nr, msg, &resp)) return false; diff --git a/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c b/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c index d11eb60b3e86..f3660981ad6a 100644 --- a/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c @@ -47,6 +47,19 @@ void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev) } EXPORT_SYMBOL_GPL(adf_vf2pf_notify_shutdown); +/** + * adf_recv_pf2vf_msg() - receive a PF to VF message + * @accel_dev: Pointer to acceleration device + * + * This function allows the VF to receive a message from the PF. + * + * Return: a valid message on success, zero otherwise. + */ +static u32 adf_recv_pf2vf_msg(struct adf_accel_dev *accel_dev) +{ + return GET_PFVF_OPS(accel_dev)->recv_msg(accel_dev, 0); +} + static bool adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev, u32 msg) { switch ((msg & ADF_PF2VF_MSGTYPE_MASK) >> ADF_PF2VF_MSGTYPE_SHIFT) { @@ -77,28 +90,11 @@ static bool adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev, u32 msg) bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev) { - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_bar *pmisc = - &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; - void __iomem *pmisc_bar_addr = pmisc->virt_addr; - u32 offset = hw_data->pfvf_ops.get_pf2vf_offset(0); u32 msg; - /* Read the message from PF */ - msg = ADF_CSR_RD(pmisc_bar_addr, offset); - if (!(msg & ADF_PF2VF_INT)) { - dev_info(&GET_DEV(accel_dev), - "Spurious PF2VF interrupt, msg %X. Ignored\n", msg); - return true; - } + msg = adf_recv_pf2vf_msg(accel_dev); + if (msg) + return adf_handle_pf2vf_msg(accel_dev, msg); - if (!(msg & ADF_PF2VF_MSGORIGIN_SYSTEM)) - /* Ignore legacy non-system (non-kernel) PF2VF messages */ - return true; - - /* To ack, clear the PF2VFINT bit */ - msg &= ~ADF_PF2VF_INT; - ADF_CSR_WR(pmisc_bar_addr, offset, msg); - - return adf_handle_pf2vf_msg(accel_dev, msg); + return true; } From 09ce899a592f7a8ee19857ad0ef394e31be83c1d Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Wed, 17 Nov 2021 14:30:51 +0000 Subject: [PATCH 037/145] crypto: qat - reorganize PFVF code Reorganize the structure of the PFVF code by moving the content of adf_pf2vf_msg.c and adf_vf2pf_msg.c. The logic that handles high level messages has been moved to adf_pfvf_pf_msg.c and adf_pfvf_vf_msg.c. The implementation of low level communication primitives and the protocol is now included in adf_pfvf_pf_proto.c and adf_pfvf_vf_proto.c. In addition, the file adf_pf2vf_msg.h has been renamed in adf_pfvf_msg.h since it common to PF and VF and the copyright date for the touched files has been updated. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 4 +- .../crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 4 +- .../qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c | 5 +- .../crypto/qat/qat_c62x/adf_c62x_hw_data.c | 4 +- .../qat/qat_c62xvf/adf_c62xvf_hw_data.c | 5 +- drivers/crypto/qat/qat_common/Makefile | 5 +- .../crypto/qat/qat_common/adf_common_drv.h | 22 +-- drivers/crypto/qat/qat_common/adf_gen2_pfvf.c | 4 +- .../{adf_pf2vf_msg.h => adf_pfvf_msg.h} | 8 +- .../crypto/qat/qat_common/adf_pfvf_pf_msg.c | 21 +++ .../crypto/qat/qat_common/adf_pfvf_pf_msg.h | 10 ++ .../{adf_pf2vf_msg.c => adf_pfvf_pf_proto.c} | 137 +----------------- .../crypto/qat/qat_common/adf_pfvf_pf_proto.h | 13 ++ .../crypto/qat/qat_common/adf_pfvf_vf_msg.c | 93 ++++++++++++ .../crypto/qat/qat_common/adf_pfvf_vf_msg.h | 21 +++ .../crypto/qat/qat_common/adf_pfvf_vf_proto.c | 133 +++++++++++++++++ .../crypto/qat/qat_common/adf_pfvf_vf_proto.h | 14 ++ drivers/crypto/qat/qat_common/adf_sriov.c | 4 +- drivers/crypto/qat/qat_common/adf_vf2pf_msg.c | 100 ------------- drivers/crypto/qat/qat_common/adf_vf_isr.c | 1 - .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 4 +- .../qat_dh895xccvf/adf_dh895xccvf_hw_data.c | 5 +- 22 files changed, 341 insertions(+), 276 deletions(-) rename drivers/crypto/qat/qat_common/{adf_pf2vf_msg.h => adf_pfvf_msg.h} (96%) create mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c create mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h rename drivers/crypto/qat/qat_common/{adf_pf2vf_msg.c => adf_pfvf_pf_proto.c} (53%) create mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h create mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c create mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h create mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c create mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h delete mode 100644 drivers/crypto/qat/qat_common/adf_vf2pf_msg.c diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c index ec57a2e2d1fc..2a878d98f81a 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -1,10 +1,10 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2020 Intel Corporation */ +/* Copyright(c) 2020 - 2021 Intel Corporation */ #include #include #include -#include #include +#include #include "adf_4xxx_hw_data.h" #include "icp_qat_hw.h" diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index d25f78660b8c..94a11e72edae 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -1,10 +1,10 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ +/* Copyright(c) 2014 - 2021 Intel Corporation */ #include #include -#include #include #include +#include #include "adf_c3xxx_hw_data.h" #include "icp_qat_hw.h" diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c index c39733320063..4c43a0d93fa6 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -1,10 +1,11 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2015 - 2020 Intel Corporation */ +/* Copyright(c) 2015 - 2021 Intel Corporation */ #include -#include #include #include #include +#include +#include #include "adf_c3xxxvf_hw_data.h" static struct adf_hw_device_class c3xxxiov_class = { diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index f24a01e1bc1a..3cb1a88d97ae 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -1,10 +1,10 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ +/* Copyright(c) 2014 - 2021 Intel Corporation */ #include #include -#include #include #include +#include #include "adf_c62x_hw_data.h" #include "icp_qat_hw.h" diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c index 03097bbe600a..c4b23e2cd579 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -1,10 +1,11 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2015 - 2020 Intel Corporation */ +/* Copyright(c) 2015 - 2021 Intel Corporation */ #include -#include #include #include #include +#include +#include #include "adf_c62xvf_hw_data.h" static struct adf_hw_device_class c62xiov_class = { diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile index 676aef6533e0..1376504d16ff 100644 --- a/drivers/crypto/qat/qat_common/Makefile +++ b/drivers/crypto/qat/qat_common/Makefile @@ -19,6 +19,7 @@ intel_qat-objs := adf_cfg.o \ qat_hal.o intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o -intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_pf2vf_msg.o \ - adf_vf2pf_msg.o adf_vf_isr.o \ +intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_vf_isr.o \ + adf_pfvf_pf_msg.o adf_pfvf_pf_proto.o \ + adf_pfvf_vf_msg.o adf_pfvf_vf_proto.o \ adf_gen2_pfvf.o diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index c0699e453502..2d8b72085505 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ +/* Copyright(c) 2014 - 2021 Intel Corporation */ #ifndef ADF_DRV_H #define ADF_DRV_H @@ -62,8 +62,6 @@ int adf_dev_start(struct adf_accel_dev *accel_dev); void adf_dev_stop(struct adf_accel_dev *accel_dev); void adf_dev_shutdown(struct adf_accel_dev *accel_dev); -void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev); -int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev); void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data); void adf_clean_vf_map(bool); @@ -199,13 +197,9 @@ void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev); bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr); int adf_pf2vf_handle_pf_restarting(struct adf_accel_dev *accel_dev); -int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev); void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); void adf_schedule_vf2pf_handler(struct adf_accel_vf_info *vf_info); -int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg); -int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev); -void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev); int adf_init_pf_wq(void); void adf_exit_pf_wq(void); int adf_init_vf_wq(void); @@ -214,11 +208,6 @@ void adf_flush_vf_wq(struct adf_accel_dev *accel_dev); #else #define adf_sriov_configure NULL -static inline int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev) -{ - return 0; -} - static inline void adf_disable_sriov(struct adf_accel_dev *accel_dev) { } @@ -231,15 +220,6 @@ static inline void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) { } -static inline int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev) -{ - return 0; -} - -static inline void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev) -{ -} - static inline int adf_init_pf_wq(void) { return 0; diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c index f79c3ca28283..f3a0a9d651e0 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c @@ -6,7 +6,9 @@ #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "adf_gen2_pfvf.h" -#include "adf_pf2vf_msg.h" +#include "adf_pfvf_msg.h" +#include "adf_pfvf_pf_proto.h" +#include "adf_pfvf_vf_proto.h" /* VF2PF interrupts */ #define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9) diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h similarity index 96% rename from drivers/crypto/qat/qat_common/adf_pf2vf_msg.h rename to drivers/crypto/qat/qat_common/adf_pfvf_msg.h index 73eb8f13ad09..0520466563fd 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2015 - 2020 Intel Corporation */ -#ifndef ADF_PF2VF_MSG_H -#define ADF_PF2VF_MSG_H +/* Copyright(c) 2015 - 2021 Intel Corporation */ +#ifndef ADF_PFVF_MSG_H +#define ADF_PFVF_MSG_H /* * PF<->VF Messaging @@ -91,4 +91,4 @@ /* VF->PF Compatible Version Request */ #define ADF_VF2PF_COMPAT_VER_REQ_SHIFT 22 -#endif /* ADF_IOV_MSG_H */ +#endif /* ADF_PFVF_MSG_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c new file mode 100644 index 000000000000..647b82e6c4ba --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2015 - 2021 Intel Corporation */ +#include +#include "adf_accel_devices.h" +#include "adf_pfvf_msg.h" +#include "adf_pfvf_pf_msg.h" +#include "adf_pfvf_pf_proto.h" + +void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_vf_info *vf; + u32 msg = (ADF_PF2VF_MSGORIGIN_SYSTEM | + (ADF_PF2VF_MSGTYPE_RESTARTING << ADF_PF2VF_MSGTYPE_SHIFT)); + int i, num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev)); + + for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) { + if (vf->init && adf_send_pf2vf_msg(accel_dev, i, msg)) + dev_err(&GET_DEV(accel_dev), + "Failed to send restarting msg to VF%d\n", i); + } +} diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h new file mode 100644 index 000000000000..187807b1ff88 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2021 Intel Corporation */ +#ifndef ADF_PFVF_PF_MSG_H +#define ADF_PFVF_PF_MSG_H + +#include "adf_accel_devices.h" + +void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev); + +#endif /* ADF_PFVF_PF_MSG_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c similarity index 53% rename from drivers/crypto/qat/qat_common/adf_pf2vf_msg.c rename to drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c index c064e8bab50d..ac6a54cf17f6 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c @@ -1,15 +1,11 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2015 - 2020 Intel Corporation */ +/* Copyright(c) 2015 - 2021 Intel Corporation */ +#include +#include #include "adf_accel_devices.h" #include "adf_common_drv.h" -#include "adf_pf2vf_msg.h" - -#define ADF_PFVF_MSG_COLLISION_DETECT_DELAY 10 -#define ADF_PFVF_MSG_ACK_DELAY 2 -#define ADF_PFVF_MSG_ACK_MAX_RETRY 100 -#define ADF_PFVF_MSG_RESP_TIMEOUT (ADF_PFVF_MSG_ACK_DELAY * \ - ADF_PFVF_MSG_ACK_MAX_RETRY + \ - ADF_PFVF_MSG_COLLISION_DETECT_DELAY) +#include "adf_pfvf_msg.h" +#include "adf_pfvf_pf_proto.h" /** * adf_send_pf2vf_msg() - send PF to VF message @@ -21,25 +17,11 @@ * * Return: 0 on success, error code otherwise. */ -static int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, u32 msg) +int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, u32 msg) { return GET_PFVF_OPS(accel_dev)->send_msg(accel_dev, msg, vf_nr); } -/** - * adf_send_vf2pf_msg() - send VF to PF message - * @accel_dev: Pointer to acceleration device - * @msg: Message to send - * - * This function allows the VF to send a message to the PF. - * - * Return: 0 on success, error code otherwise. - */ -int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg) -{ - return GET_PFVF_OPS(accel_dev)->send_msg(accel_dev, msg, 0); -} - /** * adf_recv_vf2pf_msg() - receive a VF to PF message * @accel_dev: Pointer to acceleration device @@ -54,42 +36,6 @@ static u32 adf_recv_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr) return GET_PFVF_OPS(accel_dev)->recv_msg(accel_dev, vf_nr); } -/** - * adf_send_vf2pf_req() - send VF2PF request message - * @accel_dev: Pointer to acceleration device. - * @msg: Request message to send - * - * This function sends a message that requires a response from the VF to the PF - * and waits for a reply. - * - * Return: 0 on success, error code otherwise. - */ -static int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, u32 msg) -{ - unsigned long timeout = msecs_to_jiffies(ADF_PFVF_MSG_RESP_TIMEOUT); - int ret; - - reinit_completion(&accel_dev->vf.iov_msg_completion); - - /* Send request from VF to PF */ - ret = adf_send_vf2pf_msg(accel_dev, msg); - if (ret) { - dev_err(&GET_DEV(accel_dev), - "Failed to send request msg to PF\n"); - return ret; - } - - /* Wait for response */ - if (!wait_for_completion_timeout(&accel_dev->vf.iov_msg_completion, - timeout)) { - dev_err(&GET_DEV(accel_dev), - "PFVF request/response message timeout expired\n"); - return -EIO; - } - - return 0; -} - static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, u32 msg, u32 *response) { @@ -193,77 +139,6 @@ bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr) return true; } -void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev) -{ - struct adf_accel_vf_info *vf; - u32 msg = (ADF_PF2VF_MSGORIGIN_SYSTEM | - (ADF_PF2VF_MSGTYPE_RESTARTING << ADF_PF2VF_MSGTYPE_SHIFT)); - int i, num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev)); - - for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) { - if (vf->init && adf_send_pf2vf_msg(accel_dev, i, msg)) - dev_err(&GET_DEV(accel_dev), - "Failed to send restarting msg to VF%d\n", i); - } -} - -static int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u32 msg = 0; - int ret; - - msg = ADF_VF2PF_MSGORIGIN_SYSTEM; - msg |= ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ << ADF_VF2PF_MSGTYPE_SHIFT; - msg |= ADF_PFVF_COMPAT_THIS_VERSION << ADF_VF2PF_COMPAT_VER_REQ_SHIFT; - BUILD_BUG_ON(ADF_PFVF_COMPAT_THIS_VERSION > 255); - - ret = adf_send_vf2pf_req(accel_dev, msg); - if (ret) { - dev_err(&GET_DEV(accel_dev), - "Failed to send Compatibility Version Request.\n"); - return ret; - } - - /* Response from PF received, check compatibility */ - switch (accel_dev->vf.compatible) { - case ADF_PF2VF_VF_COMPATIBLE: - break; - case ADF_PF2VF_VF_COMPAT_UNKNOWN: - /* VF is newer than PF and decides whether it is compatible */ - if (accel_dev->vf.pf_version >= hw_data->min_iov_compat_ver) { - accel_dev->vf.compatible = ADF_PF2VF_VF_COMPATIBLE; - break; - } - fallthrough; - case ADF_PF2VF_VF_INCOMPATIBLE: - dev_err(&GET_DEV(accel_dev), - "PF (vers %d) and VF (vers %d) are not compatible\n", - accel_dev->vf.pf_version, - ADF_PFVF_COMPAT_THIS_VERSION); - return -EINVAL; - default: - dev_err(&GET_DEV(accel_dev), - "Invalid response from PF; assume not compatible\n"); - return -EINVAL; - } - return ret; -} - -/** - * adf_enable_vf2pf_comms() - Function enables communication from vf to pf - * - * @accel_dev: Pointer to acceleration device virtual function. - * - * Return: 0 on success, error code otherwise. - */ -int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) -{ - adf_enable_pf2vf_interrupts(accel_dev); - return adf_vf2pf_request_version(accel_dev); -} -EXPORT_SYMBOL_GPL(adf_enable_vf2pf_comms); - /** * adf_enable_pf2vf_comms() - Function enables communication from pf to vf * diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h new file mode 100644 index 000000000000..63245407bfb6 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2021 Intel Corporation */ +#ifndef ADF_PFVF_PF_PROTO_H +#define ADF_PFVF_PF_PROTO_H + +#include +#include "adf_accel_devices.h" + +int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, u32 msg); + +int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev); + +#endif /* ADF_PFVF_PF_PROTO_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c new file mode 100644 index 000000000000..7969a644e24b --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2015 - 2021 Intel Corporation */ +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_pfvf_msg.h" +#include "adf_pfvf_vf_msg.h" +#include "adf_pfvf_vf_proto.h" + +/** + * adf_vf2pf_notify_init() - send init msg to PF + * @accel_dev: Pointer to acceleration VF device. + * + * Function sends an init message from the VF to a PF + * + * Return: 0 on success, error code otherwise. + */ +int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev) +{ + u32 msg = (ADF_VF2PF_MSGORIGIN_SYSTEM | + (ADF_VF2PF_MSGTYPE_INIT << ADF_VF2PF_MSGTYPE_SHIFT)); + + if (adf_send_vf2pf_msg(accel_dev, msg)) { + dev_err(&GET_DEV(accel_dev), + "Failed to send Init event to PF\n"); + return -EFAULT; + } + set_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status); + return 0; +} +EXPORT_SYMBOL_GPL(adf_vf2pf_notify_init); + +/** + * adf_vf2pf_notify_shutdown() - send shutdown msg to PF + * @accel_dev: Pointer to acceleration VF device. + * + * Function sends a shutdown message from the VF to a PF + * + * Return: void + */ +void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev) +{ + u32 msg = (ADF_VF2PF_MSGORIGIN_SYSTEM | + (ADF_VF2PF_MSGTYPE_SHUTDOWN << ADF_VF2PF_MSGTYPE_SHIFT)); + + if (test_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status)) + if (adf_send_vf2pf_msg(accel_dev, msg)) + dev_err(&GET_DEV(accel_dev), + "Failed to send Shutdown event to PF\n"); +} +EXPORT_SYMBOL_GPL(adf_vf2pf_notify_shutdown); + +int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 msg = 0; + int ret; + + msg = ADF_VF2PF_MSGORIGIN_SYSTEM; + msg |= ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ << ADF_VF2PF_MSGTYPE_SHIFT; + msg |= ADF_PFVF_COMPAT_THIS_VERSION << ADF_VF2PF_COMPAT_VER_REQ_SHIFT; + BUILD_BUG_ON(ADF_PFVF_COMPAT_THIS_VERSION > 255); + + ret = adf_send_vf2pf_req(accel_dev, msg); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to send Compatibility Version Request.\n"); + return ret; + } + + /* Response from PF received, check compatibility */ + switch (accel_dev->vf.compatible) { + case ADF_PF2VF_VF_COMPATIBLE: + break; + case ADF_PF2VF_VF_COMPAT_UNKNOWN: + /* VF is newer than PF and decides whether it is compatible */ + if (accel_dev->vf.pf_version >= hw_data->min_iov_compat_ver) { + accel_dev->vf.compatible = ADF_PF2VF_VF_COMPATIBLE; + break; + } + fallthrough; + case ADF_PF2VF_VF_INCOMPATIBLE: + dev_err(&GET_DEV(accel_dev), + "PF (vers %d) and VF (vers %d) are not compatible\n", + accel_dev->vf.pf_version, + ADF_PFVF_COMPAT_THIS_VERSION); + return -EINVAL; + default: + dev_err(&GET_DEV(accel_dev), + "Invalid response from PF; assume not compatible\n"); + return -EINVAL; + } + return ret; +} diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h new file mode 100644 index 000000000000..5091b5b2fd8f --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2021 Intel Corporation */ +#ifndef ADF_PFVF_VF_MSG_H +#define ADF_PFVF_VF_MSG_H + +#if defined(CONFIG_PCI_IOV) +int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev); +void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev); +int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev); +#else +static inline int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static inline void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev) +{ +} +#endif + +#endif /* ADF_PFVF_VF_MSG_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c new file mode 100644 index 000000000000..62817bcec121 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2015 - 2021 Intel Corporation */ +#include +#include +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_pfvf_msg.h" +#include "adf_pfvf_vf_msg.h" +#include "adf_pfvf_vf_proto.h" + +#define ADF_PFVF_MSG_COLLISION_DETECT_DELAY 10 +#define ADF_PFVF_MSG_ACK_DELAY 2 +#define ADF_PFVF_MSG_ACK_MAX_RETRY 100 + +#define ADF_PFVF_MSG_RESP_TIMEOUT (ADF_PFVF_MSG_ACK_DELAY * \ + ADF_PFVF_MSG_ACK_MAX_RETRY + \ + ADF_PFVF_MSG_COLLISION_DETECT_DELAY) + +/** + * adf_send_vf2pf_msg() - send VF to PF message + * @accel_dev: Pointer to acceleration device + * @msg: Message to send + * + * This function allows the VF to send a message to the PF. + * + * Return: 0 on success, error code otherwise. + */ +int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg) +{ + return GET_PFVF_OPS(accel_dev)->send_msg(accel_dev, msg, 0); +} + +/** + * adf_recv_pf2vf_msg() - receive a PF to VF message + * @accel_dev: Pointer to acceleration device + * + * This function allows the VF to receive a message from the PF. + * + * Return: a valid message on success, zero otherwise. + */ +static u32 adf_recv_pf2vf_msg(struct adf_accel_dev *accel_dev) +{ + return GET_PFVF_OPS(accel_dev)->recv_msg(accel_dev, 0); +} + +/** + * adf_send_vf2pf_req() - send VF2PF request message + * @accel_dev: Pointer to acceleration device. + * @msg: Request message to send + * + * This function sends a message that requires a response from the VF to the PF + * and waits for a reply. + * + * Return: 0 on success, error code otherwise. + */ +int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, u32 msg) +{ + unsigned long timeout = msecs_to_jiffies(ADF_PFVF_MSG_RESP_TIMEOUT); + int ret; + + reinit_completion(&accel_dev->vf.iov_msg_completion); + + /* Send request from VF to PF */ + ret = adf_send_vf2pf_msg(accel_dev, msg); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to send request msg to PF\n"); + return ret; + } + + /* Wait for response */ + if (!wait_for_completion_timeout(&accel_dev->vf.iov_msg_completion, + timeout)) { + dev_err(&GET_DEV(accel_dev), + "PFVF request/response message timeout expired\n"); + return -EIO; + } + + return 0; +} + +static bool adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev, u32 msg) +{ + switch ((msg & ADF_PF2VF_MSGTYPE_MASK) >> ADF_PF2VF_MSGTYPE_SHIFT) { + case ADF_PF2VF_MSGTYPE_RESTARTING: + dev_dbg(&GET_DEV(accel_dev), + "Restarting msg received from PF 0x%x\n", msg); + + adf_pf2vf_handle_pf_restarting(accel_dev); + return false; + case ADF_PF2VF_MSGTYPE_VERSION_RESP: + dev_dbg(&GET_DEV(accel_dev), + "Version resp received from PF 0x%x\n", msg); + accel_dev->vf.pf_version = + (msg & ADF_PF2VF_VERSION_RESP_VERS_MASK) >> + ADF_PF2VF_VERSION_RESP_VERS_SHIFT; + accel_dev->vf.compatible = + (msg & ADF_PF2VF_VERSION_RESP_RESULT_MASK) >> + ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + complete(&accel_dev->vf.iov_msg_completion); + return true; + default: + dev_err(&GET_DEV(accel_dev), + "Unknown PF2VF message(0x%x)\n", msg); + } + + return false; +} + +bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev) +{ + u32 msg; + + msg = adf_recv_pf2vf_msg(accel_dev); + if (msg) + return adf_handle_pf2vf_msg(accel_dev, msg); + + return true; +} + +/** + * adf_enable_vf2pf_comms() - Function enables communication from vf to pf + * + * @accel_dev: Pointer to acceleration device virtual function. + * + * Return: 0 on success, error code otherwise. + */ +int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) +{ + adf_enable_pf2vf_interrupts(accel_dev); + return adf_vf2pf_request_version(accel_dev); +} +EXPORT_SYMBOL_GPL(adf_enable_vf2pf_comms); diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h new file mode 100644 index 000000000000..a3ab24c7d18b --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2021 Intel Corporation */ +#ifndef ADF_PFVF_VF_PROTO_H +#define ADF_PFVF_VF_PROTO_H + +#include +#include "adf_accel_devices.h" + +int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg); +int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, u32 msg); + +int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev); + +#endif /* ADF_PFVF_VF_PROTO_H */ diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index 342063406c19..429990c5e0f3 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -1,12 +1,12 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2015 - 2020 Intel Corporation */ +/* Copyright(c) 2015 - 2021 Intel Corporation */ #include #include #include #include #include "adf_common_drv.h" #include "adf_cfg.h" -#include "adf_pf2vf_msg.h" +#include "adf_pfvf_pf_msg.h" static struct workqueue_struct *pf2vf_resp_wq; diff --git a/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c b/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c deleted file mode 100644 index f3660981ad6a..000000000000 --- a/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c +++ /dev/null @@ -1,100 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2015 - 2020 Intel Corporation */ -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "adf_pf2vf_msg.h" - -/** - * adf_vf2pf_notify_init() - send init msg to PF - * @accel_dev: Pointer to acceleration VF device. - * - * Function sends an init message from the VF to a PF - * - * Return: 0 on success, error code otherwise. - */ -int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev) -{ - u32 msg = (ADF_VF2PF_MSGORIGIN_SYSTEM | - (ADF_VF2PF_MSGTYPE_INIT << ADF_VF2PF_MSGTYPE_SHIFT)); - - if (adf_send_vf2pf_msg(accel_dev, msg)) { - dev_err(&GET_DEV(accel_dev), - "Failed to send Init event to PF\n"); - return -EFAULT; - } - set_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status); - return 0; -} -EXPORT_SYMBOL_GPL(adf_vf2pf_notify_init); - -/** - * adf_vf2pf_notify_shutdown() - send shutdown msg to PF - * @accel_dev: Pointer to acceleration VF device. - * - * Function sends a shutdown message from the VF to a PF - * - * Return: void - */ -void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev) -{ - u32 msg = (ADF_VF2PF_MSGORIGIN_SYSTEM | - (ADF_VF2PF_MSGTYPE_SHUTDOWN << ADF_VF2PF_MSGTYPE_SHIFT)); - - if (test_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status)) - if (adf_send_vf2pf_msg(accel_dev, msg)) - dev_err(&GET_DEV(accel_dev), - "Failed to send Shutdown event to PF\n"); -} -EXPORT_SYMBOL_GPL(adf_vf2pf_notify_shutdown); - -/** - * adf_recv_pf2vf_msg() - receive a PF to VF message - * @accel_dev: Pointer to acceleration device - * - * This function allows the VF to receive a message from the PF. - * - * Return: a valid message on success, zero otherwise. - */ -static u32 adf_recv_pf2vf_msg(struct adf_accel_dev *accel_dev) -{ - return GET_PFVF_OPS(accel_dev)->recv_msg(accel_dev, 0); -} - -static bool adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev, u32 msg) -{ - switch ((msg & ADF_PF2VF_MSGTYPE_MASK) >> ADF_PF2VF_MSGTYPE_SHIFT) { - case ADF_PF2VF_MSGTYPE_RESTARTING: - dev_dbg(&GET_DEV(accel_dev), - "Restarting msg received from PF 0x%x\n", msg); - - adf_pf2vf_handle_pf_restarting(accel_dev); - return false; - case ADF_PF2VF_MSGTYPE_VERSION_RESP: - dev_dbg(&GET_DEV(accel_dev), - "Version resp received from PF 0x%x\n", msg); - accel_dev->vf.pf_version = - (msg & ADF_PF2VF_VERSION_RESP_VERS_MASK) >> - ADF_PF2VF_VERSION_RESP_VERS_SHIFT; - accel_dev->vf.compatible = - (msg & ADF_PF2VF_VERSION_RESP_RESULT_MASK) >> - ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; - complete(&accel_dev->vf.iov_msg_completion); - return true; - default: - dev_err(&GET_DEV(accel_dev), - "Unknown PF2VF message(0x%x)\n", msg); - } - - return false; -} - -bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev) -{ - u32 msg; - - msg = adf_recv_pf2vf_msg(accel_dev); - if (msg) - return adf_handle_pf2vf_msg(accel_dev, msg); - - return true; -} diff --git a/drivers/crypto/qat/qat_common/adf_vf_isr.c b/drivers/crypto/qat/qat_common/adf_vf_isr.c index b17040b8a4b9..fe094178f065 100644 --- a/drivers/crypto/qat/qat_common/adf_vf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_vf_isr.c @@ -15,7 +15,6 @@ #include "adf_cfg_common.h" #include "adf_transport_access_macros.h" #include "adf_transport_internal.h" -#include "adf_pf2vf_msg.h" #define ADF_VINTSOU_OFFSET 0x204 #define ADF_VINTMSK_OFFSET 0x208 diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index aa42373a7118..37f43b8c29eb 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -1,10 +1,10 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ +/* Copyright(c) 2014 - 2021 Intel Corporation */ #include -#include #include #include #include +#include #include "adf_dh895xcc_hw_data.h" #include "icp_qat_hw.h" diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index 2e2ef6b5bd2a..d3795bab3725 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -1,10 +1,11 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2015 - 2020 Intel Corporation */ +/* Copyright(c) 2015 - 2021 Intel Corporation */ #include -#include #include #include #include +#include +#include #include "adf_dh895xccvf_hw_data.h" static struct adf_hw_device_class dh895xcciov_class = { From f6aff914989e9770ed96474e24570d6cab665162 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Wed, 17 Nov 2021 14:30:52 +0000 Subject: [PATCH 038/145] crypto: qat - reorganize PFVF protocol definitions Organize PFVF protocol definitions by type rather than direction, by keeping related fields close. Also, make sure the order is consistent for both PF and VF definitions. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_pfvf_msg.h | 36 +++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h index 0520466563fd..23f4c4b35dac 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h @@ -53,34 +53,21 @@ * adf_gen2_pfvf_send() in adf_pf2vf_msg.c). */ -#define ADF_PFVF_COMPAT_THIS_VERSION 0x1 /* PF<->VF compat */ - /* PF->VF messages */ #define ADF_PF2VF_INT BIT(0) #define ADF_PF2VF_MSGORIGIN_SYSTEM BIT(1) +#define ADF_PF2VF_IN_USE_BY_PF 0x6AC20000 +#define ADF_PF2VF_IN_USE_BY_PF_MASK 0xFFFE0000 #define ADF_PF2VF_MSGTYPE_MASK 0x0000003C #define ADF_PF2VF_MSGTYPE_SHIFT 2 #define ADF_PF2VF_MSGTYPE_RESTARTING 0x01 #define ADF_PF2VF_MSGTYPE_VERSION_RESP 0x02 -#define ADF_PF2VF_IN_USE_BY_PF 0x6AC20000 -#define ADF_PF2VF_IN_USE_BY_PF_MASK 0xFFFE0000 - -/* PF->VF Version Response */ -#define ADF_PF2VF_VERSION_RESP_VERS_MASK 0x00003FC0 -#define ADF_PF2VF_VERSION_RESP_VERS_SHIFT 6 -#define ADF_PF2VF_VERSION_RESP_RESULT_MASK 0x0000C000 -#define ADF_PF2VF_VERSION_RESP_RESULT_SHIFT 14 -#define ADF_PF2VF_MINORVERSION_SHIFT 6 -#define ADF_PF2VF_MAJORVERSION_SHIFT 10 -#define ADF_PF2VF_VF_COMPATIBLE 1 -#define ADF_PF2VF_VF_INCOMPATIBLE 2 -#define ADF_PF2VF_VF_COMPAT_UNKNOWN 3 /* VF->PF messages */ -#define ADF_VF2PF_IN_USE_BY_VF 0x00006AC2 -#define ADF_VF2PF_IN_USE_BY_VF_MASK 0x0000FFFE #define ADF_VF2PF_INT BIT(16) #define ADF_VF2PF_MSGORIGIN_SYSTEM BIT(17) +#define ADF_VF2PF_IN_USE_BY_VF 0x00006AC2 +#define ADF_VF2PF_IN_USE_BY_VF_MASK 0x0000FFFE #define ADF_VF2PF_MSGTYPE_MASK 0x003C0000 #define ADF_VF2PF_MSGTYPE_SHIFT 18 #define ADF_VF2PF_MSGTYPE_INIT 0x3 @@ -88,6 +75,21 @@ #define ADF_VF2PF_MSGTYPE_VERSION_REQ 0x5 #define ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ 0x6 +/* VF/PF compatibility version. */ +/* Reference to the current version */ +#define ADF_PFVF_COMPAT_THIS_VERSION 1 /* PF<->VF compat */ + +/* PF->VF Version Response */ +#define ADF_PF2VF_MINORVERSION_SHIFT 6 +#define ADF_PF2VF_MAJORVERSION_SHIFT 10 +#define ADF_PF2VF_VERSION_RESP_VERS_MASK 0x00003FC0 +#define ADF_PF2VF_VERSION_RESP_VERS_SHIFT 6 +#define ADF_PF2VF_VERSION_RESP_RESULT_MASK 0x0000C000 +#define ADF_PF2VF_VERSION_RESP_RESULT_SHIFT 14 +#define ADF_PF2VF_VF_COMPATIBLE 1 +#define ADF_PF2VF_VF_INCOMPATIBLE 2 +#define ADF_PF2VF_VF_COMPAT_UNKNOWN 3 + /* VF->PF Compatible Version Request */ #define ADF_VF2PF_COMPAT_VER_REQ_SHIFT 22 From 1d4fde6c4e805f4cb5ecb54fb39c93686d3e5924 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Wed, 17 Nov 2021 14:30:53 +0000 Subject: [PATCH 039/145] crypto: qat - use enums for PFVF protocol codes Replace PFVF constants with enumerations for valid protocol codes. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_pfvf_msg.h | 33 +++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h index 23f4c4b35dac..8b476072df28 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h @@ -60,8 +60,11 @@ #define ADF_PF2VF_IN_USE_BY_PF_MASK 0xFFFE0000 #define ADF_PF2VF_MSGTYPE_MASK 0x0000003C #define ADF_PF2VF_MSGTYPE_SHIFT 2 -#define ADF_PF2VF_MSGTYPE_RESTARTING 0x01 -#define ADF_PF2VF_MSGTYPE_VERSION_RESP 0x02 + +enum pf2vf_msgtype { + ADF_PF2VF_MSGTYPE_RESTARTING = 0x01, + ADF_PF2VF_MSGTYPE_VERSION_RESP = 0x02, +}; /* VF->PF messages */ #define ADF_VF2PF_INT BIT(16) @@ -70,14 +73,19 @@ #define ADF_VF2PF_IN_USE_BY_VF_MASK 0x0000FFFE #define ADF_VF2PF_MSGTYPE_MASK 0x003C0000 #define ADF_VF2PF_MSGTYPE_SHIFT 18 -#define ADF_VF2PF_MSGTYPE_INIT 0x3 -#define ADF_VF2PF_MSGTYPE_SHUTDOWN 0x4 -#define ADF_VF2PF_MSGTYPE_VERSION_REQ 0x5 -#define ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ 0x6 + +enum vf2pf_msgtype { + ADF_VF2PF_MSGTYPE_INIT = 0x03, + ADF_VF2PF_MSGTYPE_SHUTDOWN = 0x04, + ADF_VF2PF_MSGTYPE_VERSION_REQ = 0x05, + ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ = 0x06, +}; /* VF/PF compatibility version. */ -/* Reference to the current version */ -#define ADF_PFVF_COMPAT_THIS_VERSION 1 /* PF<->VF compat */ +enum pfvf_compatibility_version { + /* Reference to the current version */ + ADF_PFVF_COMPAT_THIS_VERSION = 0x01, +}; /* PF->VF Version Response */ #define ADF_PF2VF_MINORVERSION_SHIFT 6 @@ -86,9 +94,12 @@ #define ADF_PF2VF_VERSION_RESP_VERS_SHIFT 6 #define ADF_PF2VF_VERSION_RESP_RESULT_MASK 0x0000C000 #define ADF_PF2VF_VERSION_RESP_RESULT_SHIFT 14 -#define ADF_PF2VF_VF_COMPATIBLE 1 -#define ADF_PF2VF_VF_INCOMPATIBLE 2 -#define ADF_PF2VF_VF_COMPAT_UNKNOWN 3 + +enum pf2vf_compat_response { + ADF_PF2VF_VF_COMPATIBLE = 0x01, + ADF_PF2VF_VF_INCOMPATIBLE = 0x02, + ADF_PF2VF_VF_COMPAT_UNKNOWN = 0x03, +}; /* VF->PF Compatible Version Request */ #define ADF_VF2PF_COMPAT_VER_REQ_SHIFT 22 From 25110fd2e346449355cb795cd0d3e050ca5bdf11 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Wed, 17 Nov 2021 14:30:54 +0000 Subject: [PATCH 040/145] crypto: qat - pass the PF2VF responses back to the callers Currently, any PF response to a VF request is fully parsed during the interrupt handling. This way the individual response values are stored into the accel_dev structure, preventing the caller to access and decode the full response message itself. Change this behavior, by letting the API return back the entire message to the caller, in order to: - keep correlated code together, that is, the (building of the) request and the (decoding of the) response; - avoid polluting the accel_dev data structure with unnecessary and at times temporary values; only the entire message is stored in a temporary buffer. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_c3xxxvf/adf_drv.c | 2 +- drivers/crypto/qat/qat_c62xvf/adf_drv.c | 2 +- .../crypto/qat/qat_common/adf_accel_devices.h | 4 ++-- .../crypto/qat/qat_common/adf_pfvf_vf_msg.c | 23 +++++++++++------- .../crypto/qat/qat_common/adf_pfvf_vf_proto.c | 24 +++++++++---------- .../crypto/qat/qat_common/adf_pfvf_vf_proto.h | 2 +- drivers/crypto/qat/qat_dh895xccvf/adf_drv.c | 2 +- 7 files changed, 33 insertions(+), 26 deletions(-) diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c index 1df1b868978d..0ba1d293bb81 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c @@ -171,7 +171,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } pci_set_master(pdev); /* Completion for VF2PF request/response message exchange */ - init_completion(&accel_dev->vf.iov_msg_completion); + init_completion(&accel_dev->vf.msg_received); ret = qat_crypto_dev_config(accel_dev); if (ret) diff --git a/drivers/crypto/qat/qat_c62xvf/adf_drv.c b/drivers/crypto/qat/qat_c62xvf/adf_drv.c index 8103bd81d617..176d8e2786f4 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_drv.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_drv.c @@ -171,7 +171,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } pci_set_master(pdev); /* Completion for VF2PF request/response message exchange */ - init_completion(&accel_dev->vf.iov_msg_completion); + init_completion(&accel_dev->vf.msg_received); ret = qat_crypto_dev_config(accel_dev); if (ret) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 35e62a73f9fa..b05b217df24c 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -271,8 +271,8 @@ struct adf_accel_dev { char irq_name[ADF_MAX_MSIX_VECTOR_NAME]; struct tasklet_struct pf2vf_bh_tasklet; struct mutex vf2pf_lock; /* protect CSR access */ - struct completion iov_msg_completion; - u8 compatible; + struct completion msg_received; + u32 response; /* temp field holding pf2vf response */ u8 pf_version; } vf; }; diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c index 7969a644e24b..d5cccec03a3b 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c @@ -52,7 +52,10 @@ EXPORT_SYMBOL_GPL(adf_vf2pf_notify_shutdown); int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u8 pf_version; u32 msg = 0; + int compat; + u32 resp; int ret; msg = ADF_VF2PF_MSGORIGIN_SYSTEM; @@ -60,34 +63,38 @@ int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) msg |= ADF_PFVF_COMPAT_THIS_VERSION << ADF_VF2PF_COMPAT_VER_REQ_SHIFT; BUILD_BUG_ON(ADF_PFVF_COMPAT_THIS_VERSION > 255); - ret = adf_send_vf2pf_req(accel_dev, msg); + ret = adf_send_vf2pf_req(accel_dev, msg, &resp); if (ret) { dev_err(&GET_DEV(accel_dev), "Failed to send Compatibility Version Request.\n"); return ret; } + pf_version = (resp & ADF_PF2VF_VERSION_RESP_VERS_MASK) + >> ADF_PF2VF_VERSION_RESP_VERS_SHIFT; + compat = (resp & ADF_PF2VF_VERSION_RESP_RESULT_MASK) + >> ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + /* Response from PF received, check compatibility */ - switch (accel_dev->vf.compatible) { + switch (compat) { case ADF_PF2VF_VF_COMPATIBLE: break; case ADF_PF2VF_VF_COMPAT_UNKNOWN: /* VF is newer than PF and decides whether it is compatible */ - if (accel_dev->vf.pf_version >= hw_data->min_iov_compat_ver) { - accel_dev->vf.compatible = ADF_PF2VF_VF_COMPATIBLE; + if (pf_version >= hw_data->min_iov_compat_ver) break; - } fallthrough; case ADF_PF2VF_VF_INCOMPATIBLE: dev_err(&GET_DEV(accel_dev), "PF (vers %d) and VF (vers %d) are not compatible\n", - accel_dev->vf.pf_version, - ADF_PFVF_COMPAT_THIS_VERSION); + pf_version, ADF_PFVF_COMPAT_THIS_VERSION); return -EINVAL; default: dev_err(&GET_DEV(accel_dev), "Invalid response from PF; assume not compatible\n"); return -EINVAL; } - return ret; + + accel_dev->vf.pf_version = pf_version; + return 0; } diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c index 62817bcec121..ea1a00e746ff 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c @@ -47,18 +47,19 @@ static u32 adf_recv_pf2vf_msg(struct adf_accel_dev *accel_dev) * adf_send_vf2pf_req() - send VF2PF request message * @accel_dev: Pointer to acceleration device. * @msg: Request message to send + * @resp: Returned PF response * * This function sends a message that requires a response from the VF to the PF * and waits for a reply. * * Return: 0 on success, error code otherwise. */ -int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, u32 msg) +int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, u32 msg, u32 *resp) { unsigned long timeout = msecs_to_jiffies(ADF_PFVF_MSG_RESP_TIMEOUT); int ret; - reinit_completion(&accel_dev->vf.iov_msg_completion); + reinit_completion(&accel_dev->vf.msg_received); /* Send request from VF to PF */ ret = adf_send_vf2pf_msg(accel_dev, msg); @@ -69,13 +70,19 @@ int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, u32 msg) } /* Wait for response */ - if (!wait_for_completion_timeout(&accel_dev->vf.iov_msg_completion, + if (!wait_for_completion_timeout(&accel_dev->vf.msg_received, timeout)) { dev_err(&GET_DEV(accel_dev), "PFVF request/response message timeout expired\n"); return -EIO; } + if (likely(resp)) + *resp = accel_dev->vf.response; + + /* Once copied, set to an invalid value */ + accel_dev->vf.response = 0; + return 0; } @@ -89,15 +96,8 @@ static bool adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev, u32 msg) adf_pf2vf_handle_pf_restarting(accel_dev); return false; case ADF_PF2VF_MSGTYPE_VERSION_RESP: - dev_dbg(&GET_DEV(accel_dev), - "Version resp received from PF 0x%x\n", msg); - accel_dev->vf.pf_version = - (msg & ADF_PF2VF_VERSION_RESP_VERS_MASK) >> - ADF_PF2VF_VERSION_RESP_VERS_SHIFT; - accel_dev->vf.compatible = - (msg & ADF_PF2VF_VERSION_RESP_RESULT_MASK) >> - ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; - complete(&accel_dev->vf.iov_msg_completion); + accel_dev->vf.response = msg; + complete(&accel_dev->vf.msg_received); return true; default: dev_err(&GET_DEV(accel_dev), diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h index a3ab24c7d18b..6226d4d9d520 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h @@ -7,7 +7,7 @@ #include "adf_accel_devices.h" int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg); -int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, u32 msg); +int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, u32 msg, u32 *resp); int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c index 99d90f3ea2b7..ee45d688b5d7 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c @@ -171,7 +171,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } pci_set_master(pdev); /* Completion for VF2PF request/response message exchange */ - init_completion(&accel_dev->vf.iov_msg_completion); + init_completion(&accel_dev->vf.msg_received); ret = qat_crypto_dev_config(accel_dev); if (ret) From c35c76c6919ebcaac590f5f3afa6b8924ad305f4 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Wed, 17 Nov 2021 14:30:55 +0000 Subject: [PATCH 041/145] crypto: qat - refactor pfvf version request messages Refactor version handling logic for ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ and ADF_VF2PF_MSGTYPE_VERSION_REQ on the PF. Response messages are now filled only after fully parsing the request, in a consisted way with the rest of the PFVF codebase. This patch also fixes a harmless double setting for VERSION in the response for ADF_VF2PF_MSGTYPE_VERSION_REQ. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_pfvf_pf_proto.c | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c index ac6a54cf17f6..c0844fbd896c 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c @@ -47,12 +47,7 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, case ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ: { u8 vf_compat_ver = msg >> ADF_VF2PF_COMPAT_VER_REQ_SHIFT; - - resp = (ADF_PF2VF_MSGORIGIN_SYSTEM | - (ADF_PF2VF_MSGTYPE_VERSION_RESP << - ADF_PF2VF_MSGTYPE_SHIFT) | - (ADF_PFVF_COMPAT_THIS_VERSION << - ADF_PF2VF_VERSION_RESP_VERS_SHIFT)); + u8 compat; dev_dbg(&GET_DEV(accel_dev), "Compatibility Version Request from VF%d vers=%u\n", @@ -62,37 +57,46 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, dev_err(&GET_DEV(accel_dev), "VF (vers %d) incompatible with PF (vers %d)\n", vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION); - resp |= ADF_PF2VF_VF_INCOMPATIBLE << - ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + compat = ADF_PF2VF_VF_INCOMPATIBLE; } else if (vf_compat_ver > ADF_PFVF_COMPAT_THIS_VERSION) { dev_err(&GET_DEV(accel_dev), "VF (vers %d) compat with PF (vers %d) unkn.\n", vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION); - resp |= ADF_PF2VF_VF_COMPAT_UNKNOWN << - ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + compat = ADF_PF2VF_VF_COMPAT_UNKNOWN; } else { dev_dbg(&GET_DEV(accel_dev), "VF (vers %d) compatible with PF (vers %d)\n", vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION); - resp |= ADF_PF2VF_VF_COMPATIBLE << - ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + compat = ADF_PF2VF_VF_COMPATIBLE; } + + resp = ADF_PF2VF_MSGORIGIN_SYSTEM; + resp |= ADF_PF2VF_MSGTYPE_VERSION_RESP << + ADF_PF2VF_MSGTYPE_SHIFT; + resp |= ADF_PFVF_COMPAT_THIS_VERSION << + ADF_PF2VF_VERSION_RESP_VERS_SHIFT; + resp |= compat << ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; } break; case ADF_VF2PF_MSGTYPE_VERSION_REQ: + { + u8 compat; + dev_dbg(&GET_DEV(accel_dev), "Legacy VersionRequest received from VF%d 0x%x\n", vf_nr + 1, msg); - resp = (ADF_PF2VF_MSGORIGIN_SYSTEM | - (ADF_PF2VF_MSGTYPE_VERSION_RESP << - ADF_PF2VF_MSGTYPE_SHIFT) | - (ADF_PFVF_COMPAT_THIS_VERSION << - ADF_PF2VF_VERSION_RESP_VERS_SHIFT)); - resp |= ADF_PF2VF_VF_COMPATIBLE << - ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + + /* PF always newer than legacy VF */ + compat = ADF_PF2VF_VF_COMPATIBLE; + + resp = ADF_PF2VF_MSGORIGIN_SYSTEM; + resp |= ADF_PF2VF_MSGTYPE_VERSION_RESP << + ADF_PF2VF_MSGTYPE_SHIFT; /* Set legacy major and minor version num */ resp |= 1 << ADF_PF2VF_MAJORVERSION_SHIFT | 1 << ADF_PF2VF_MINORVERSION_SHIFT; + resp |= compat << ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + } break; case ADF_VF2PF_MSGTYPE_INIT: { From e669b4dedd899e8e4cad5ef91721d3387a3844fc Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Wed, 17 Nov 2021 14:30:56 +0000 Subject: [PATCH 042/145] crypto: qat - do not rely on min version Remove min_iov_compat_ver field as for now all versions are compatible. Compatibility is determined by a series of rules and dynamic conditions such as specific configurations. In any case the minimum version requirement for compatibility is an inadequate and obsolete approach which should be removed. At this time compatibility can be assured across the currently available versions. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 2 -- .../crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 2 -- .../qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c | 2 -- .../crypto/qat/qat_c62x/adf_c62x_hw_data.c | 2 -- .../qat/qat_c62xvf/adf_c62xvf_hw_data.c | 2 -- .../crypto/qat/qat_common/adf_accel_devices.h | 1 - .../crypto/qat/qat_common/adf_pfvf_pf_proto.c | 20 +++++++------------ .../crypto/qat/qat_common/adf_pfvf_vf_msg.c | 7 ++----- .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 2 -- .../qat_dh895xccvf/adf_dh895xccvf_hw_data.c | 2 -- 10 files changed, 9 insertions(+), 33 deletions(-) diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c index 2a878d98f81a..4658b7bf76da 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -4,7 +4,6 @@ #include #include #include -#include #include "adf_4xxx_hw_data.h" #include "icp_qat_hw.h" @@ -257,7 +256,6 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) hw_data->pfvf_ops.enable_comms = adf_pfvf_comms_disabled; hw_data->pfvf_ops.get_vf2pf_sources = get_vf2pf_sources; hw_data->disable_iov = adf_disable_sriov; - hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; adf_gen4_init_hw_csr_ops(&hw_data->csr_ops); } diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 94a11e72edae..3987a44fa164 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -4,7 +4,6 @@ #include #include #include -#include #include "adf_c3xxx_hw_data.h" #include "icp_qat_hw.h" @@ -137,7 +136,6 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->reset_device = adf_reset_flr; hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer; hw_data->disable_iov = adf_disable_sriov; - hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops); adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c index 4c43a0d93fa6..85122013534d 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -4,7 +4,6 @@ #include #include #include -#include #include #include "adf_c3xxxvf_hw_data.h" @@ -85,7 +84,6 @@ void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) hw_data->get_misc_bar_id = get_misc_bar_id; hw_data->get_sku = get_sku; hw_data->enable_ints = adf_vf_void_noop; - hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; hw_data->dev_class->instances++; adf_devmgr_update_class_index(hw_data); adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops); diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index 3cb1a88d97ae..a76e33d7a215 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -4,7 +4,6 @@ #include #include #include -#include #include "adf_c62x_hw_data.h" #include "icp_qat_hw.h" @@ -139,7 +138,6 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->reset_device = adf_reset_flr; hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer; hw_data->disable_iov = adf_disable_sriov; - hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops); adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c index c4b23e2cd579..99c56405f88f 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -4,7 +4,6 @@ #include #include #include -#include #include #include "adf_c62xvf_hw_data.h" @@ -85,7 +84,6 @@ void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data) hw_data->get_misc_bar_id = get_misc_bar_id; hw_data->get_sku = get_sku; hw_data->enable_ints = adf_vf_void_noop; - hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; hw_data->dev_class->instances++; adf_devmgr_update_class_index(hw_data); adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops); diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index b05b217df24c..a1809a7d1c90 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -209,7 +209,6 @@ struct adf_hw_device_data { u8 num_accel; u8 num_logical_accel; u8 num_engines; - u8 min_iov_compat_ver; }; /* CSR write macro */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c index c0844fbd896c..db5bbb9db32e 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c @@ -40,7 +40,6 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, u32 msg, u32 *response) { struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; u32 resp = 0; switch ((msg & ADF_VF2PF_MSGTYPE_MASK) >> ADF_VF2PF_MSGTYPE_SHIFT) { @@ -53,21 +52,16 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, "Compatibility Version Request from VF%d vers=%u\n", vf_nr + 1, vf_compat_ver); - if (vf_compat_ver < hw_data->min_iov_compat_ver) { - dev_err(&GET_DEV(accel_dev), - "VF (vers %d) incompatible with PF (vers %d)\n", - vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION); - compat = ADF_PF2VF_VF_INCOMPATIBLE; - } else if (vf_compat_ver > ADF_PFVF_COMPAT_THIS_VERSION) { - dev_err(&GET_DEV(accel_dev), - "VF (vers %d) compat with PF (vers %d) unkn.\n", - vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION); - compat = ADF_PF2VF_VF_COMPAT_UNKNOWN; - } else { + if (vf_compat_ver <= ADF_PFVF_COMPAT_THIS_VERSION) { + compat = ADF_PF2VF_VF_COMPATIBLE; dev_dbg(&GET_DEV(accel_dev), "VF (vers %d) compatible with PF (vers %d)\n", vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION); - compat = ADF_PF2VF_VF_COMPATIBLE; + } else { + compat = ADF_PF2VF_VF_COMPAT_UNKNOWN; + dev_err(&GET_DEV(accel_dev), + "VF (vers %d) compat with PF (vers %d) unkn.\n", + vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION); } resp = ADF_PF2VF_MSGORIGIN_SYSTEM; diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c index d5cccec03a3b..763581839902 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c @@ -51,7 +51,6 @@ EXPORT_SYMBOL_GPL(adf_vf2pf_notify_shutdown); int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) { - struct adf_hw_device_data *hw_data = accel_dev->hw_device; u8 pf_version; u32 msg = 0; int compat; @@ -80,10 +79,8 @@ int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) case ADF_PF2VF_VF_COMPATIBLE: break; case ADF_PF2VF_VF_COMPAT_UNKNOWN: - /* VF is newer than PF and decides whether it is compatible */ - if (pf_version >= hw_data->min_iov_compat_ver) - break; - fallthrough; + /* VF is newer than PF - compatible for now */ + break; case ADF_PF2VF_VF_INCOMPATIBLE: dev_err(&GET_DEV(accel_dev), "PF (vers %d) and VF (vers %d) are not compatible\n", diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 37f43b8c29eb..27d4cab65dd8 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -4,7 +4,6 @@ #include #include #include -#include #include "adf_dh895xcc_hw_data.h" #include "icp_qat_hw.h" @@ -216,7 +215,6 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->enable_ints = adf_enable_ints; hw_data->reset_device = adf_reset_sbr; hw_data->disable_iov = adf_disable_sriov; - hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops); hw_data->pfvf_ops.get_vf2pf_sources = get_vf2pf_sources; diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index d3795bab3725..5489d6c02256 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -4,7 +4,6 @@ #include #include #include -#include #include #include "adf_dh895xccvf_hw_data.h" @@ -85,7 +84,6 @@ void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) hw_data->get_misc_bar_id = get_misc_bar_id; hw_data->get_sku = get_sku; hw_data->enable_ints = adf_vf_void_noop; - hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; hw_data->dev_class->instances++; adf_devmgr_update_class_index(hw_data); adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops); From 1d9a915fafabc27739ad6e19db6f86c233f676b2 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Wed, 17 Nov 2021 14:30:57 +0000 Subject: [PATCH 043/145] crypto: qat - fix VF IDs in PFVF log messages PFVF debug messages use a mix of zero and one based VF IDs. Switch to zero based VF numbers in all log messages. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_isr.c | 2 +- drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c index 358200c0d598..522e0c10d9b9 100644 --- a/drivers/crypto/qat/qat_common/adf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_isr.c @@ -123,7 +123,7 @@ static bool adf_handle_vf2pf_int(struct adf_accel_dev *accel_dev) if (!__ratelimit(&vf_info->vf2pf_ratelimit)) { dev_info(&GET_DEV(accel_dev), "Too many ints from VF%d\n", - vf_info->vf_nr + 1); + vf_info->vf_nr); continue; } diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c index db5bbb9db32e..b486b2b599c2 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c @@ -50,7 +50,7 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, dev_dbg(&GET_DEV(accel_dev), "Compatibility Version Request from VF%d vers=%u\n", - vf_nr + 1, vf_compat_ver); + vf_nr, vf_compat_ver); if (vf_compat_ver <= ADF_PFVF_COMPAT_THIS_VERSION) { compat = ADF_PF2VF_VF_COMPATIBLE; @@ -78,7 +78,7 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, dev_dbg(&GET_DEV(accel_dev), "Legacy VersionRequest received from VF%d 0x%x\n", - vf_nr + 1, msg); + vf_nr, msg); /* PF always newer than legacy VF */ compat = ADF_PF2VF_VF_COMPATIBLE; @@ -96,7 +96,7 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, { dev_dbg(&GET_DEV(accel_dev), "Init message received from VF%d 0x%x\n", - vf_nr + 1, msg); + vf_nr, msg); vf_info->init = true; } break; @@ -104,13 +104,13 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, { dev_dbg(&GET_DEV(accel_dev), "Shutdown message received from VF%d 0x%x\n", - vf_nr + 1, msg); + vf_nr, msg); vf_info->init = false; } break; default: dev_dbg(&GET_DEV(accel_dev), "Unknown message from VF%d (0x%x)\n", - vf_nr + 1, msg); + vf_nr, msg); return -ENOMSG; } From 8616b628ef697eff129b8319604058751eb3ebac Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Wed, 17 Nov 2021 14:30:58 +0000 Subject: [PATCH 044/145] crypto: qat - improve logging of PFVF messages Improve and simplify logging of PFVF messages. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_gen2_pfvf.c | 4 +-- .../crypto/qat/qat_common/adf_pfvf_pf_proto.c | 30 +++++++------------ .../crypto/qat/qat_common/adf_pfvf_vf_proto.c | 7 +++-- 3 files changed, 17 insertions(+), 24 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c index f3a0a9d651e0..099e39808d13 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c @@ -183,14 +183,14 @@ static u32 adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev, u8 vf_nr) msg = ADF_CSR_RD(pmisc_addr, pfvf_offset); if (!(msg & int_bit)) { dev_info(&GET_DEV(accel_dev), - "Spurious PFVF interrupt, msg %X. Ignored\n", msg); + "Spurious PFVF interrupt, msg 0x%.8x. Ignored\n", msg); return 0; } /* Ignore legacy non-system (non-kernel) VF2PF messages */ if (!(msg & msg_origin)) { dev_dbg(&GET_DEV(accel_dev), - "Ignored non-system message (0x%x);\n", msg); + "Ignored non-system message (0x%.8x);\n", msg); return 0; } diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c index b486b2b599c2..4f20dd35fcd4 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c @@ -49,20 +49,13 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, u8 compat; dev_dbg(&GET_DEV(accel_dev), - "Compatibility Version Request from VF%d vers=%u\n", - vf_nr, vf_compat_ver); + "VersionRequest received from VF%d (vers %d) to PF (vers %d)\n", + vf_nr, vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION); - if (vf_compat_ver <= ADF_PFVF_COMPAT_THIS_VERSION) { + if (vf_compat_ver <= ADF_PFVF_COMPAT_THIS_VERSION) compat = ADF_PF2VF_VF_COMPATIBLE; - dev_dbg(&GET_DEV(accel_dev), - "VF (vers %d) compatible with PF (vers %d)\n", - vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION); - } else { + else compat = ADF_PF2VF_VF_COMPAT_UNKNOWN; - dev_err(&GET_DEV(accel_dev), - "VF (vers %d) compat with PF (vers %d) unkn.\n", - vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION); - } resp = ADF_PF2VF_MSGORIGIN_SYSTEM; resp |= ADF_PF2VF_MSGTYPE_VERSION_RESP << @@ -77,8 +70,8 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, u8 compat; dev_dbg(&GET_DEV(accel_dev), - "Legacy VersionRequest received from VF%d 0x%x\n", - vf_nr, msg); + "Legacy VersionRequest received from VF%d to PF (vers 1.1)\n", + vf_nr); /* PF always newer than legacy VF */ compat = ADF_PF2VF_VF_COMPATIBLE; @@ -95,21 +88,19 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, case ADF_VF2PF_MSGTYPE_INIT: { dev_dbg(&GET_DEV(accel_dev), - "Init message received from VF%d 0x%x\n", - vf_nr, msg); + "Init message received from VF%d\n", vf_nr); vf_info->init = true; } break; case ADF_VF2PF_MSGTYPE_SHUTDOWN: { dev_dbg(&GET_DEV(accel_dev), - "Shutdown message received from VF%d 0x%x\n", - vf_nr, msg); + "Shutdown message received from VF%d\n", vf_nr); vf_info->init = false; } break; default: - dev_dbg(&GET_DEV(accel_dev), "Unknown message from VF%d (0x%x)\n", + dev_dbg(&GET_DEV(accel_dev), "Unknown message from VF%d (0x%.8x)\n", vf_nr, msg); return -ENOMSG; } @@ -132,7 +123,8 @@ bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr) return false; if (resp && adf_send_pf2vf_msg(accel_dev, vf_nr, resp)) - dev_err(&GET_DEV(accel_dev), "Failed to send response to VF\n"); + dev_err(&GET_DEV(accel_dev), + "Failed to send response to VF%d\n", vf_nr); return true; } diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c index ea1a00e746ff..9c7489ed122c 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c @@ -90,18 +90,19 @@ static bool adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev, u32 msg) { switch ((msg & ADF_PF2VF_MSGTYPE_MASK) >> ADF_PF2VF_MSGTYPE_SHIFT) { case ADF_PF2VF_MSGTYPE_RESTARTING: - dev_dbg(&GET_DEV(accel_dev), - "Restarting msg received from PF 0x%x\n", msg); + dev_dbg(&GET_DEV(accel_dev), "Restarting message received from PF\n"); adf_pf2vf_handle_pf_restarting(accel_dev); return false; case ADF_PF2VF_MSGTYPE_VERSION_RESP: + dev_dbg(&GET_DEV(accel_dev), + "Response message received from PF (0x%.8x)\n", msg); accel_dev->vf.response = msg; complete(&accel_dev->vf.msg_received); return true; default: dev_err(&GET_DEV(accel_dev), - "Unknown PF2VF message(0x%x)\n", msg); + "Unknown PF2VF message (0x%.8x) from PF\n", msg); } return false; From 83f50f2948baef5fb5f2f547a573f113f03f2b15 Mon Sep 17 00:00:00 2001 From: Corentin Labbe Date: Thu, 18 Nov 2021 23:10:25 -0600 Subject: [PATCH 045/145] crypto: sun8i-ce - Add support for the D1 variant The Allwinner D1 SoC has a crypto engine compatible with sun8i-ce. Add support for it. Signed-off-by: Corentin Labbe Signed-off-by: Samuel Holland Signed-off-by: Herbert Xu --- .../crypto/allwinner/sun8i-ce/sun8i-ce-core.c | 21 +++++++++++++++++++ drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h | 1 + 2 files changed, 22 insertions(+) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c index 00194d1d9ae6..d8623c7e0d1d 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c @@ -106,6 +106,24 @@ static const struct ce_variant ce_a64_variant = { .trng = CE_ID_NOTSUPP, }; +static const struct ce_variant ce_d1_variant = { + .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, + }, + .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, + CE_ALG_SHA384, CE_ALG_SHA512 + }, + .op_mode = { CE_OP_ECB, CE_OP_CBC + }, + .ce_clks = { + { "bus", 0, 200000000 }, + { "mod", 300000000, 0 }, + { "ram", 0, 400000000 }, + }, + .esr = ESR_D1, + .prng = CE_ALG_PRNG, + .trng = CE_ALG_TRNG, +}; + static const struct ce_variant ce_r40_variant = { .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, }, @@ -192,6 +210,7 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) dev_err(ce->dev, "CE ERROR: keysram access error for AES\n"); break; case ESR_A64: + case ESR_D1: case ESR_H5: case ESR_R40: v >>= (flow * 4); @@ -990,6 +1009,8 @@ static const struct of_device_id sun8i_ce_crypto_of_match_table[] = { .data = &ce_h3_variant }, { .compatible = "allwinner,sun8i-r40-crypto", .data = &ce_r40_variant }, + { .compatible = "allwinner,sun20i-d1-crypto", + .data = &ce_d1_variant }, { .compatible = "allwinner,sun50i-a64-crypto", .data = &ce_a64_variant }, { .compatible = "allwinner,sun50i-h5-crypto", diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h index cec781d5063c..624a5926f21f 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h @@ -94,6 +94,7 @@ #define ESR_R40 2 #define ESR_H5 3 #define ESR_H6 4 +#define ESR_D1 5 #define PRNG_DATA_SIZE (160 / 8) #define PRNG_SEED_SIZE DIV_ROUND_UP(175, 8) From b808f32023dd8127b0fa27f60fa69a959fd70388 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20M=C3=BCller?= Date: Fri, 19 Nov 2021 07:55:33 +0100 Subject: [PATCH 046/145] crypto: kdf - Add key derivation self-test support code As a preparation to add the key derivation implementations, the self-test data structure definition and the common test code is made available. The test framework follows the testing applied by the NIST CAVP test approach. The structure of the test code follows the implementations found in crypto/testmgr.c|h. In case the KDF implementations will be made available via a kernel crypto API templates, the test code is intended to be merged into testmgr.c|h. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- include/crypto/internal/kdf_selftest.h | 71 ++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 include/crypto/internal/kdf_selftest.h diff --git a/include/crypto/internal/kdf_selftest.h b/include/crypto/internal/kdf_selftest.h new file mode 100644 index 000000000000..4d03d2af57b7 --- /dev/null +++ b/include/crypto/internal/kdf_selftest.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * Copyright (C) 2021, Stephan Mueller + */ + +#ifndef _CRYPTO_KDF_SELFTEST_H +#define _CRYPTO_KDF_SELFTEST_H + +#include +#include + +struct kdf_testvec { + unsigned char *key; + size_t keylen; + unsigned char *ikm; + size_t ikmlen; + struct kvec info; + unsigned char *expected; + size_t expectedlen; +}; + +static inline int +kdf_test(const struct kdf_testvec *test, const char *name, + int (*crypto_kdf_setkey)(struct crypto_shash *kmd, + const u8 *key, size_t keylen, + const u8 *ikm, size_t ikmlen), + int (*crypto_kdf_generate)(struct crypto_shash *kmd, + const struct kvec *info, + unsigned int info_nvec, + u8 *dst, unsigned int dlen)) +{ + struct crypto_shash *kmd; + int ret; + u8 *buf = kzalloc(test->expectedlen, GFP_KERNEL); + + if (!buf) + return -ENOMEM; + + kmd = crypto_alloc_shash(name, 0, 0); + if (IS_ERR(kmd)) { + pr_err("alg: kdf: could not allocate hash handle for %s\n", + name); + kfree(buf); + return -ENOMEM; + } + + ret = crypto_kdf_setkey(kmd, test->key, test->keylen, + test->ikm, test->ikmlen); + if (ret) { + pr_err("alg: kdf: could not set key derivation key\n"); + goto err; + } + + ret = crypto_kdf_generate(kmd, &test->info, 1, buf, test->expectedlen); + if (ret) { + pr_err("alg: kdf: could not obtain key data\n"); + goto err; + } + + ret = memcmp(test->expected, buf, test->expectedlen); + if (ret) + ret = -EINVAL; + +err: + crypto_free_shash(kmd); + kfree(buf); + return ret; +} + +#endif /* _CRYPTO_KDF_SELFTEST_H */ From 026a733e66592e743a0905c7fd6b5d3bf89b2d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20M=C3=BCller?= Date: Fri, 19 Nov 2021 07:55:58 +0100 Subject: [PATCH 047/145] crypto: kdf - add SP800-108 counter key derivation function SP800-108 defines three KDFs - this patch provides the counter KDF implementation. The KDF is implemented as a service function where the caller has to maintain the hash / HMAC state. Apart from this hash/HMAC state, no additional state is required to be maintained by either the caller or the KDF implementation. The key for the KDF is set with the crypto_kdf108_setkey function which is intended to be invoked before the caller requests a key derivation operation via crypto_kdf108_ctr_generate. SP800-108 allows the use of either a HMAC or a hash as crypto primitive for the KDF. When a HMAC primtive is intended to be used, crypto_kdf108_setkey must be used to set the HMAC key. Otherwise, for a hash crypto primitve crypto_kdf108_ctr_generate can be used immediately after allocating the hash handle. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/Kconfig | 4 + crypto/Makefile | 5 ++ crypto/kdf_sp800108.c | 153 ++++++++++++++++++++++++++++++++++ include/crypto/kdf_sp800108.h | 61 ++++++++++++++ 4 files changed, 223 insertions(+) create mode 100644 crypto/kdf_sp800108.c create mode 100644 include/crypto/kdf_sp800108.h diff --git a/crypto/Kconfig b/crypto/Kconfig index 285f82647d2b..01b9ca0836a5 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1845,6 +1845,10 @@ config CRYPTO_JITTERENTROPY random numbers. This Jitterentropy RNG registers with the kernel crypto API and can be used by any caller. +config CRYPTO_KDF800108_CTR + tristate + select CRYPTO_HASH + config CRYPTO_USER_API tristate diff --git a/crypto/Makefile b/crypto/Makefile index 429c4d57458c..d76bff8d0ffd 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -200,3 +200,8 @@ obj-$(CONFIG_ASYMMETRIC_KEY_TYPE) += asymmetric_keys/ obj-$(CONFIG_CRYPTO_HASH_INFO) += hash_info.o crypto_simd-y := simd.o obj-$(CONFIG_CRYPTO_SIMD) += crypto_simd.o + +# +# Key derivation function +# +obj-$(CONFIG_CRYPTO_KDF800108_CTR) += kdf_sp800108.o diff --git a/crypto/kdf_sp800108.c b/crypto/kdf_sp800108.c new file mode 100644 index 000000000000..58edf7797abf --- /dev/null +++ b/crypto/kdf_sp800108.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * SP800-108 Key-derivation function + * + * Copyright (C) 2021, Stephan Mueller + */ + +#include +#include +#include +#include + +/* + * SP800-108 CTR KDF implementation + */ +int crypto_kdf108_ctr_generate(struct crypto_shash *kmd, + const struct kvec *info, unsigned int info_nvec, + u8 *dst, unsigned int dlen) +{ + SHASH_DESC_ON_STACK(desc, kmd); + __be32 counter = cpu_to_be32(1); + const unsigned int h = crypto_shash_digestsize(kmd), dlen_orig = dlen; + unsigned int i; + int err = 0; + u8 *dst_orig = dst; + + desc->tfm = kmd; + + while (dlen) { + err = crypto_shash_init(desc); + if (err) + goto out; + + err = crypto_shash_update(desc, (u8 *)&counter, sizeof(__be32)); + if (err) + goto out; + + for (i = 0; i < info_nvec; i++) { + err = crypto_shash_update(desc, info[i].iov_base, + info[i].iov_len); + if (err) + goto out; + } + + if (dlen < h) { + u8 tmpbuffer[HASH_MAX_DIGESTSIZE]; + + err = crypto_shash_final(desc, tmpbuffer); + if (err) + goto out; + memcpy(dst, tmpbuffer, dlen); + memzero_explicit(tmpbuffer, h); + goto out; + } + + err = crypto_shash_final(desc, dst); + if (err) + goto out; + + dlen -= h; + dst += h; + counter = cpu_to_be32(be32_to_cpu(counter) + 1); + } + +out: + if (err) + memzero_explicit(dst_orig, dlen_orig); + shash_desc_zero(desc); + return err; +} +EXPORT_SYMBOL(crypto_kdf108_ctr_generate); + +/* + * The seeding of the KDF + */ +int crypto_kdf108_setkey(struct crypto_shash *kmd, + const u8 *key, size_t keylen, + const u8 *ikm, size_t ikmlen) +{ + unsigned int ds = crypto_shash_digestsize(kmd); + + /* SP800-108 does not support IKM */ + if (ikm || ikmlen) + return -EINVAL; + + /* Check according to SP800-108 section 7.2 */ + if (ds > keylen) + return -EINVAL; + + /* Set the key for the MAC used for the KDF. */ + return crypto_shash_setkey(kmd, key, keylen); +} +EXPORT_SYMBOL(crypto_kdf108_setkey); + +/* + * Test vector obtained from + * http://csrc.nist.gov/groups/STM/cavp/documents/KBKDF800-108/CounterMode.zip + */ +static const struct kdf_testvec kdf_ctr_hmac_sha256_tv_template[] = { + { + .key = "\xdd\x1d\x91\xb7\xd9\x0b\x2b\xd3" + "\x13\x85\x33\xce\x92\xb2\x72\xfb" + "\xf8\xa3\x69\x31\x6a\xef\xe2\x42" + "\xe6\x59\xcc\x0a\xe2\x38\xaf\xe0", + .keylen = 32, + .ikm = NULL, + .ikmlen = 0, + .info = { + .iov_base = "\x01\x32\x2b\x96\xb3\x0a\xcd\x19" + "\x79\x79\x44\x4e\x46\x8e\x1c\x5c" + "\x68\x59\xbf\x1b\x1c\xf9\x51\xb7" + "\xe7\x25\x30\x3e\x23\x7e\x46\xb8" + "\x64\xa1\x45\xfa\xb2\x5e\x51\x7b" + "\x08\xf8\x68\x3d\x03\x15\xbb\x29" + "\x11\xd8\x0a\x0e\x8a\xba\x17\xf3" + "\xb4\x13\xfa\xac", + .iov_len = 60 + }, + .expected = "\x10\x62\x13\x42\xbf\xb0\xfd\x40" + "\x04\x6c\x0e\x29\xf2\xcf\xdb\xf0", + .expectedlen = 16 + } +}; + +static int __init crypto_kdf108_init(void) +{ + int ret = kdf_test(&kdf_ctr_hmac_sha256_tv_template[0], "hmac(sha256)", + crypto_kdf108_setkey, crypto_kdf108_ctr_generate); + + if (ret) { + if (fips_enabled) + panic("alg: self-tests for CTR-KDF (hmac(sha256)) failed (rc=%d)\n", + ret); + + WARN(1, + "alg: self-tests for CTR-KDF (hmac(sha256)) failed (rc=%d)\n", + ret); + } else { + pr_info("alg: self-tests for CTR-KDF (hmac(sha256)) passed\n"); + } + + return ret; +} + +static void __exit crypto_kdf108_exit(void) { } + +module_init(crypto_kdf108_init); +module_exit(crypto_kdf108_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Stephan Mueller "); +MODULE_DESCRIPTION("Key Derivation Function conformant to SP800-108"); diff --git a/include/crypto/kdf_sp800108.h b/include/crypto/kdf_sp800108.h new file mode 100644 index 000000000000..b7b20a778fb7 --- /dev/null +++ b/include/crypto/kdf_sp800108.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * Copyright (C) 2021, Stephan Mueller + */ + +#ifndef _CRYPTO_KDF108_H +#define _CRYPTO_KDF108_H + +#include +#include + +/** + * Counter KDF generate operation according to SP800-108 section 5.1 + * as well as SP800-56A section 5.8.1 (Single-step KDF). + * + * @kmd Keyed message digest whose key was set with crypto_kdf108_setkey or + * unkeyed message digest + * @info optional context and application specific information - this may be + * NULL + * @info_vec number of optional context/application specific information entries + * @dst destination buffer that the caller already allocated + * @dlen length of the destination buffer - the KDF derives that amount of + * bytes. + * + * To comply with SP800-108, the caller must provide Label || 0x00 || Context + * in the info parameter. + * + * @return 0 on success, < 0 on error + */ +int crypto_kdf108_ctr_generate(struct crypto_shash *kmd, + const struct kvec *info, unsigned int info_nvec, + u8 *dst, unsigned int dlen); + +/** + * Counter KDF setkey operation + * + * @kmd Keyed message digest allocated by the caller. The key should not have + * been set. + * @key Seed key to be used to initialize the keyed message digest context. + * @keylen This length of the key buffer. + * @ikm The SP800-108 KDF does not support IKM - this parameter must be NULL + * @ikmlen This parameter must be 0. + * + * According to SP800-108 section 7.2, the seed key must be at least as large as + * the message digest size of the used keyed message digest. This limitation + * is enforced by the implementation. + * + * SP800-108 allows the use of either a HMAC or a hash primitive. When + * the caller intends to use a hash primitive, the call to + * crypto_kdf108_setkey is not required and the key derivation operation can + * immediately performed using crypto_kdf108_ctr_generate after allocating + * a handle. + * + * @return 0 on success, < 0 on error + */ +int crypto_kdf108_setkey(struct crypto_shash *kmd, + const u8 *key, size_t keylen, + const u8 *ikm, size_t ikmlen); + +#endif /* _CRYPTO_KDF108_H */ From d7921344234d15ce24a151d932aa0748797fc502 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20M=C3=BCller?= Date: Fri, 19 Nov 2021 07:58:44 +0100 Subject: [PATCH 048/145] security: DH - remove dead code for zero padding Remove the specific code that adds a zero padding that was intended to be invoked when the DH operation result was smaller than the modulus. However, this cannot occur any more these days because the function mpi_write_to_sgl is used in the code path that calculates the shared secret in dh_compute_value. This MPI service function guarantees that leading zeros are introduced as needed to ensure the resulting data is exactly as long as the modulus. This implies that the specific code to add zero padding is dead code which can be safely removed. Signed-off-by: Stephan Mueller Acked-by: Mat Martineau Signed-off-by: Herbert Xu --- security/keys/dh.c | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/security/keys/dh.c b/security/keys/dh.c index 1abfa70ed6e1..56e12dae4534 100644 --- a/security/keys/dh.c +++ b/security/keys/dh.c @@ -141,7 +141,7 @@ static void kdf_dealloc(struct kdf_sdesc *sdesc) * 'dlen' must be a multiple of the digest size. */ static int kdf_ctr(struct kdf_sdesc *sdesc, const u8 *src, unsigned int slen, - u8 *dst, unsigned int dlen, unsigned int zlen) + u8 *dst, unsigned int dlen) { struct shash_desc *desc = &sdesc->shash; unsigned int h = crypto_shash_digestsize(desc->tfm); @@ -158,22 +158,6 @@ static int kdf_ctr(struct kdf_sdesc *sdesc, const u8 *src, unsigned int slen, if (err) goto err; - if (zlen && h) { - u8 tmpbuffer[32]; - size_t chunk = min_t(size_t, zlen, sizeof(tmpbuffer)); - memset(tmpbuffer, 0, chunk); - - do { - err = crypto_shash_update(desc, tmpbuffer, - chunk); - if (err) - goto err; - - zlen -= chunk; - chunk = min_t(size_t, zlen, sizeof(tmpbuffer)); - } while (zlen); - } - if (src && slen) { err = crypto_shash_update(desc, src, slen); if (err) @@ -198,7 +182,7 @@ err: static int keyctl_dh_compute_kdf(struct kdf_sdesc *sdesc, char __user *buffer, size_t buflen, - uint8_t *kbuf, size_t kbuflen, size_t lzero) + uint8_t *kbuf, size_t kbuflen) { uint8_t *outbuf = NULL; int ret; @@ -211,7 +195,7 @@ static int keyctl_dh_compute_kdf(struct kdf_sdesc *sdesc, goto err; } - ret = kdf_ctr(sdesc, kbuf, kbuflen, outbuf, outbuf_len, lzero); + ret = kdf_ctr(sdesc, kbuf, kbuflen, outbuf, outbuf_len); if (ret) goto err; @@ -384,8 +368,7 @@ long __keyctl_dh_compute(struct keyctl_dh_params __user *params, } ret = keyctl_dh_compute_kdf(sdesc, buffer, buflen, outbuf, - req->dst_len + kdfcopy->otherinfolen, - outlen - req->dst_len); + req->dst_len + kdfcopy->otherinfolen); } else if (copy_to_user(buffer, outbuf, req->dst_len) == 0) { ret = req->dst_len; } else { From d3b04a4398fe8022c9ca4b5ac6ab08059334b180 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20M=C3=BCller?= Date: Fri, 19 Nov 2021 07:59:09 +0100 Subject: [PATCH 049/145] security: DH - use KDF implementation from crypto API The kernel crypto API provides the SP800-108 counter KDF implementation. Thus, the separate implementation provided as part of the keys subsystem can be replaced with calls to the KDF offered by the kernel crypto API. The keys subsystem uses the counter KDF with a hash primitive. Thus, it only uses the call to crypto_kdf108_ctr_generate. Signed-off-by: Stephan Mueller Acked-by: Mat Martineau Signed-off-by: Herbert Xu --- security/keys/Kconfig | 2 +- security/keys/dh.c | 115 ++++++++---------------------------------- 2 files changed, 22 insertions(+), 95 deletions(-) diff --git a/security/keys/Kconfig b/security/keys/Kconfig index 64b81abd087e..969122c7b92f 100644 --- a/security/keys/Kconfig +++ b/security/keys/Kconfig @@ -109,7 +109,7 @@ config KEY_DH_OPERATIONS bool "Diffie-Hellman operations on retained keys" depends on KEYS select CRYPTO - select CRYPTO_HASH + select CRYPTO_KDF800108_CTR select CRYPTO_DH help This option provides support for calculating Diffie-Hellman diff --git a/security/keys/dh.c b/security/keys/dh.c index 56e12dae4534..4573fc15617d 100644 --- a/security/keys/dh.c +++ b/security/keys/dh.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include "internal.h" @@ -79,17 +80,9 @@ static void dh_crypto_done(struct crypto_async_request *req, int err) complete(&compl->completion); } -struct kdf_sdesc { - struct shash_desc shash; - char ctx[]; -}; - -static int kdf_alloc(struct kdf_sdesc **sdesc_ret, char *hashname) +static int kdf_alloc(struct crypto_shash **hash, char *hashname) { struct crypto_shash *tfm; - struct kdf_sdesc *sdesc; - int size; - int err; /* allocate synchronous hash */ tfm = crypto_alloc_shash(hashname, 0, 0); @@ -98,96 +91,30 @@ static int kdf_alloc(struct kdf_sdesc **sdesc_ret, char *hashname) return PTR_ERR(tfm); } - err = -EINVAL; - if (crypto_shash_digestsize(tfm) == 0) - goto out_free_tfm; - - err = -ENOMEM; - size = sizeof(struct shash_desc) + crypto_shash_descsize(tfm); - sdesc = kmalloc(size, GFP_KERNEL); - if (!sdesc) - goto out_free_tfm; - sdesc->shash.tfm = tfm; - - *sdesc_ret = sdesc; - - return 0; - -out_free_tfm: - crypto_free_shash(tfm); - return err; -} - -static void kdf_dealloc(struct kdf_sdesc *sdesc) -{ - if (!sdesc) - return; - - if (sdesc->shash.tfm) - crypto_free_shash(sdesc->shash.tfm); - - kfree_sensitive(sdesc); -} - -/* - * Implementation of the KDF in counter mode according to SP800-108 section 5.1 - * as well as SP800-56A section 5.8.1 (Single-step KDF). - * - * SP800-56A: - * The src pointer is defined as Z || other info where Z is the shared secret - * from DH and other info is an arbitrary string (see SP800-56A section - * 5.8.1.2). - * - * 'dlen' must be a multiple of the digest size. - */ -static int kdf_ctr(struct kdf_sdesc *sdesc, const u8 *src, unsigned int slen, - u8 *dst, unsigned int dlen) -{ - struct shash_desc *desc = &sdesc->shash; - unsigned int h = crypto_shash_digestsize(desc->tfm); - int err = 0; - u8 *dst_orig = dst; - __be32 counter = cpu_to_be32(1); - - while (dlen) { - err = crypto_shash_init(desc); - if (err) - goto err; - - err = crypto_shash_update(desc, (u8 *)&counter, sizeof(__be32)); - if (err) - goto err; - - if (src && slen) { - err = crypto_shash_update(desc, src, slen); - if (err) - goto err; - } - - err = crypto_shash_final(desc, dst); - if (err) - goto err; - - dlen -= h; - dst += h; - counter = cpu_to_be32(be32_to_cpu(counter) + 1); + if (crypto_shash_digestsize(tfm) == 0) { + crypto_free_shash(tfm); + return -EINVAL; } - return 0; + *hash = tfm; -err: - memzero_explicit(dst_orig, dlen); - return err; + return 0; } -static int keyctl_dh_compute_kdf(struct kdf_sdesc *sdesc, +static void kdf_dealloc(struct crypto_shash *hash) +{ + if (hash) + crypto_free_shash(hash); +} + +static int keyctl_dh_compute_kdf(struct crypto_shash *hash, char __user *buffer, size_t buflen, uint8_t *kbuf, size_t kbuflen) { + struct kvec kbuf_iov = { .iov_base = kbuf, .iov_len = kbuflen }; uint8_t *outbuf = NULL; int ret; - size_t outbuf_len = roundup(buflen, - crypto_shash_digestsize(sdesc->shash.tfm)); + size_t outbuf_len = roundup(buflen, crypto_shash_digestsize(hash)); outbuf = kmalloc(outbuf_len, GFP_KERNEL); if (!outbuf) { @@ -195,7 +122,7 @@ static int keyctl_dh_compute_kdf(struct kdf_sdesc *sdesc, goto err; } - ret = kdf_ctr(sdesc, kbuf, kbuflen, outbuf, outbuf_len); + ret = crypto_kdf108_ctr_generate(hash, &kbuf_iov, 1, outbuf, outbuf_len); if (ret) goto err; @@ -224,7 +151,7 @@ long __keyctl_dh_compute(struct keyctl_dh_params __user *params, struct kpp_request *req; uint8_t *secret; uint8_t *outbuf; - struct kdf_sdesc *sdesc = NULL; + struct crypto_shash *hash = NULL; if (!params || (!buffer && buflen)) { ret = -EINVAL; @@ -257,7 +184,7 @@ long __keyctl_dh_compute(struct keyctl_dh_params __user *params, } /* allocate KDF from the kernel crypto API */ - ret = kdf_alloc(&sdesc, hashname); + ret = kdf_alloc(&hash, hashname); kfree(hashname); if (ret) goto out1; @@ -367,7 +294,7 @@ long __keyctl_dh_compute(struct keyctl_dh_params __user *params, goto out6; } - ret = keyctl_dh_compute_kdf(sdesc, buffer, buflen, outbuf, + ret = keyctl_dh_compute_kdf(hash, buffer, buflen, outbuf, req->dst_len + kdfcopy->otherinfolen); } else if (copy_to_user(buffer, outbuf, req->dst_len) == 0) { ret = req->dst_len; @@ -386,7 +313,7 @@ out3: out2: dh_free_data(&dh_inputs); out1: - kdf_dealloc(sdesc); + kdf_dealloc(hash); return ret; } From 376a5c3cdd7cf47d2436dd466548c0fff792f4c7 Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Sat, 20 Nov 2021 12:47:36 +0800 Subject: [PATCH 050/145] crypto: hisilicon - modify the value of engine type rate Modify the value of type rate from new QM spec. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_main.c | 2 +- drivers/crypto/hisilicon/sec2/sec_main.c | 2 +- drivers/crypto/hisilicon/zip/zip_main.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 65a641396c07..ebfab3e14499 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -103,7 +103,7 @@ #define HPRE_QM_PM_FLR BIT(11) #define HPRE_QM_SRIOV_FLR BIT(12) -#define HPRE_SHAPER_TYPE_RATE 128 +#define HPRE_SHAPER_TYPE_RATE 640 #define HPRE_VIA_MSI_DSM 1 #define HPRE_SQE_MASK_OFFSET 8 #define HPRE_SQE_MASK_LEN 24 diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 90551bf38b52..26d3ab1d308b 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -105,7 +105,7 @@ #define SEC_SQE_MASK_OFFSET 64 #define SEC_SQE_MASK_LEN 48 -#define SEC_SHAPER_TYPE_RATE 128 +#define SEC_SHAPER_TYPE_RATE 400 struct sec_hw_error { u32 int_msk; diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 873971ef9aee..1a237d95d482 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -103,8 +103,8 @@ #define HZIP_PREFETCH_ENABLE (~(BIT(26) | BIT(17) | BIT(0))) #define HZIP_SVA_PREFETCH_DISABLE BIT(26) #define HZIP_SVA_DISABLE_READY (BIT(26) | BIT(30)) -#define HZIP_SHAPER_RATE_COMPRESS 252 -#define HZIP_SHAPER_RATE_DECOMPRESS 229 +#define HZIP_SHAPER_RATE_COMPRESS 750 +#define HZIP_SHAPER_RATE_DECOMPRESS 140 #define HZIP_DELAY_1_US 1 #define HZIP_POLL_TIMEOUT_US 1000 From ecc7169d4f7318512d70192ce03d899949b153b2 Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Sat, 20 Nov 2021 12:47:37 +0800 Subject: [PATCH 051/145] crypto: hisilicon/qm - modify the value of qos initialization The value of qos should be reset after flr resetting or device resetting. So set the max of qos value for every function. Then update the value of qos when user writing the alg_qos. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 52d6cca6262e..2b998afc57ea 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -1126,10 +1126,10 @@ static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, static int qm_shaper_init_vft(struct hisi_qm *qm, u32 fun_num) { + u32 qos = qm->factor[fun_num].func_qos; int ret, i; - qm->factor[fun_num].func_qos = QM_QOS_MAX_VAL; - ret = qm_get_shaper_para(QM_QOS_MAX_VAL * QM_QOS_RATE, &qm->factor[fun_num]); + ret = qm_get_shaper_para(qos * QM_QOS_RATE, &qm->factor[fun_num]); if (ret) { dev_err(&qm->pdev->dev, "failed to calculate shaper parameter!\n"); return ret; @@ -5750,13 +5750,15 @@ err_init_qp_mem: static int hisi_qm_memory_init(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; - int ret, total_vfs; + int ret, total_func, i; size_t off = 0; - total_vfs = pci_sriov_get_totalvfs(qm->pdev); - qm->factor = kcalloc(total_vfs + 1, sizeof(struct qm_shaper_factor), GFP_KERNEL); + total_func = pci_sriov_get_totalvfs(qm->pdev) + 1; + qm->factor = kcalloc(total_func, sizeof(struct qm_shaper_factor), GFP_KERNEL); if (!qm->factor) return -ENOMEM; + for (i = 0; i < total_func; i++) + qm->factor[i].func_qos = QM_QOS_MAX_VAL; #define QM_INIT_BUF(qm, type, num) do { \ (qm)->type = ((qm)->qdma.va + (off)); \ From 488f30d4b8b373cd1ed057bb689704d6fc2a9bc9 Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Sat, 20 Nov 2021 12:47:38 +0800 Subject: [PATCH 052/145] crypto: hisilicon/qm - some optimizations of ths qos write process 1. Optimize overly long functions. 2. Fix the format symbol does not match the actual type. 3. Use the PCI_DEVFN to get the function id. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 91 ++++++++++++++++++----------------- 1 file changed, 48 insertions(+), 43 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 2b998afc57ea..be4e8597b452 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -4231,66 +4231,69 @@ static ssize_t qm_qos_value_init(const char *buf, unsigned long *val) return 0; } +static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf, + unsigned long *val, + unsigned int *fun_index) +{ + char tbuf_bdf[QM_DBG_READ_LEN] = {0}; + char val_buf[QM_QOS_VAL_MAX_LEN] = {0}; + u32 tmp1, device, function; + int ret, bus; + + ret = sscanf(buf, "%s %s", tbuf_bdf, val_buf); + if (ret != QM_QOS_PARAM_NUM) + return -EINVAL; + + ret = qm_qos_value_init(val_buf, val); + if (*val == 0 || *val > QM_QOS_MAX_VAL || ret) { + pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n"); + return -EINVAL; + } + + ret = sscanf(tbuf_bdf, "%u:%x:%u.%u", &tmp1, &bus, &device, &function); + if (ret != QM_QOS_BDF_PARAM_NUM) { + pci_err(qm->pdev, "input pci bdf value is error!\n"); + return -EINVAL; + } + + *fun_index = PCI_DEVFN(device, function); + + return 0; +} + static ssize_t qm_algqos_write(struct file *filp, const char __user *buf, size_t count, loff_t *pos) { struct hisi_qm *qm = filp->private_data; char tbuf[QM_DBG_READ_LEN]; - int tmp1, bus, device, function; - char tbuf_bdf[QM_DBG_READ_LEN] = {0}; - char val_buf[QM_QOS_VAL_MAX_LEN] = {0}; unsigned int fun_index; - unsigned long val = 0; + unsigned long val; int len, ret; if (qm->fun_type == QM_HW_VF) return -EINVAL; + if (*pos != 0) + return 0; + + if (count >= QM_DBG_READ_LEN) + return -ENOSPC; + + len = simple_write_to_buffer(tbuf, QM_DBG_READ_LEN - 1, pos, buf, count); + if (len < 0) + return len; + + tbuf[len] = '\0'; + ret = qm_get_qos_value(qm, tbuf, &val, &fun_index); + if (ret) + return ret; + /* Mailbox and reset cannot be operated at the same time */ if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { pci_err(qm->pdev, "dev resetting, write alg qos failed!\n"); return -EAGAIN; } - if (*pos != 0) { - ret = 0; - goto err_get_status; - } - - if (count >= QM_DBG_READ_LEN) { - ret = -ENOSPC; - goto err_get_status; - } - - len = simple_write_to_buffer(tbuf, QM_DBG_READ_LEN - 1, pos, buf, count); - if (len < 0) { - ret = len; - goto err_get_status; - } - - tbuf[len] = '\0'; - ret = sscanf(tbuf, "%s %s", tbuf_bdf, val_buf); - if (ret != QM_QOS_PARAM_NUM) { - ret = -EINVAL; - goto err_get_status; - } - - ret = qm_qos_value_init(val_buf, &val); - if (val == 0 || val > QM_QOS_MAX_VAL || ret) { - pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n"); - ret = -EINVAL; - goto err_get_status; - } - - ret = sscanf(tbuf_bdf, "%d:%x:%d.%d", &tmp1, &bus, &device, &function); - if (ret != QM_QOS_BDF_PARAM_NUM) { - pci_err(qm->pdev, "input pci bdf value is error!\n"); - ret = -EINVAL; - goto err_get_status; - } - - fun_index = device * 8 + function; - ret = qm_pm_get_sync(qm); if (ret) { ret = -EINVAL; @@ -4304,6 +4307,8 @@ static ssize_t qm_algqos_write(struct file *filp, const char __user *buf, goto err_put_sync; } + pci_info(qm->pdev, "the qos value of function%u is set to %lu.\n", + fun_index, val); ret = count; err_put_sync: From 13389403fe8a978bfc0a866b6d598451feaca212 Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Sat, 20 Nov 2021 12:47:39 +0800 Subject: [PATCH 053/145] crypto: hisilicon/qm - simplified the calculation of qos shaper parameters Some optimize for the calculation of qos shaper parameters. and modify the comments. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 84 +++++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 29 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index be4e8597b452..ad706ccfe91a 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -501,10 +501,30 @@ static const char * const qp_s[] = { "none", "init", "start", "stop", "close", }; -static const u32 typical_qos_val[QM_QOS_TYPICAL_NUM] = {100, 250, 500, 1000, - 10000, 25000, 50000, 100000}; -static const u32 typical_qos_cbs_s[QM_QOS_TYPICAL_NUM] = {9, 10, 11, 12, 16, - 17, 18, 19}; +struct qm_typical_qos_table { + u32 start; + u32 end; + u32 val; +}; + +/* the qos step is 100 */ +static struct qm_typical_qos_table shaper_cir_s[] = { + {100, 100, 4}, + {200, 200, 3}, + {300, 500, 2}, + {600, 1000, 1}, + {1100, 100000, 0}, +}; + +static struct qm_typical_qos_table shaper_cbs_s[] = { + {100, 200, 9}, + {300, 500, 11}, + {600, 1000, 12}, + {1100, 10000, 16}, + {10100, 25000, 17}, + {25100, 50000, 18}, + {50100, 100000, 19} +}; static bool qm_avail_state(struct hisi_qm *qm, enum qm_state new) { @@ -988,12 +1008,14 @@ static void qm_init_prefetch(struct hisi_qm *qm) } /* + * acc_shaper_para_calc() Get the IR value by the qos formula, the return value + * is the expected qos calculated. * the formula: * IR = X Mbps if ir = 1 means IR = 100 Mbps, if ir = 10000 means = 10Gbps * - * IR_b * (2 ^ IR_u) * 8 - * IR(Mbps) * 10 ^ -3 = ------------------------- - * Tick * (2 ^ IR_s) + * IR_b * (2 ^ IR_u) * 8000 + * IR(Mbps) = ------------------------- + * Tick * (2 ^ IR_s) */ static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s) { @@ -1003,17 +1025,28 @@ static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s) static u32 acc_shaper_calc_cbs_s(u32 ir) { + int table_size = ARRAY_SIZE(shaper_cbs_s); int i; - if (ir < typical_qos_val[0]) - return QM_SHAPER_MIN_CBS_S; - - for (i = 1; i < QM_QOS_TYPICAL_NUM; i++) { - if (ir >= typical_qos_val[i - 1] && ir < typical_qos_val[i]) - return typical_qos_cbs_s[i - 1]; + for (i = 0; i < table_size; i++) { + if (ir >= shaper_cbs_s[i].start && ir <= shaper_cbs_s[i].end) + return shaper_cbs_s[i].val; } - return typical_qos_cbs_s[QM_QOS_TYPICAL_NUM - 1]; + return QM_SHAPER_MIN_CBS_S; +} + +static u32 acc_shaper_calc_cir_s(u32 ir) +{ + int table_size = ARRAY_SIZE(shaper_cir_s); + int i; + + for (i = 0; i < table_size; i++) { + if (ir >= shaper_cir_s[i].start && ir <= shaper_cir_s[i].end) + return shaper_cir_s[i].val; + } + + return 0; } static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor) @@ -1022,25 +1055,18 @@ static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor) u32 error_rate; factor->cbs_s = acc_shaper_calc_cbs_s(ir); + cir_s = acc_shaper_calc_cir_s(ir); for (cir_b = QM_QOS_MIN_CIR_B; cir_b <= QM_QOS_MAX_CIR_B; cir_b++) { for (cir_u = 0; cir_u <= QM_QOS_MAX_CIR_U; cir_u++) { - for (cir_s = 0; cir_s <= QM_QOS_MAX_CIR_S; cir_s++) { - /** the formula is changed to: - * IR_b * (2 ^ IR_u) * DIVISOR_CLK - * IR(Mbps) = ------------------------- - * 768 * (2 ^ IR_s) - */ - ir_calc = acc_shaper_para_calc(cir_b, cir_u, - cir_s); - error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir; - if (error_rate <= QM_QOS_MIN_ERROR_RATE) { - factor->cir_b = cir_b; - factor->cir_u = cir_u; - factor->cir_s = cir_s; + ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s); - return 0; - } + error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir; + if (error_rate <= QM_QOS_MIN_ERROR_RATE) { + factor->cir_b = cir_b; + factor->cir_u = cir_u; + factor->cir_s = cir_s; + return 0; } } } From 552d03a223eda3df84526ab2c1f4d82e15eaee7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20M=C3=BCller?= Date: Sun, 21 Nov 2021 15:14:20 +0100 Subject: [PATCH 054/145] crypto: jitter - consider 32 LSB for APT The APT compares the current time stamp with a pre-set value. The current code only considered the 4 LSB only. Yet, after reviews by mathematicians of the user space Jitter RNG version >= 3.1.0, it was concluded that the APT can be calculated on the 32 LSB of the time delta. Thi change is applied to the kernel. This fixes a bug where an AMD EPYC fails this test as its RDTSC value contains zeros in the LSB. The most appropriate fix would have been to apply a GCD calculation and divide the time stamp by the GCD. Yet, this is a significant code change that will be considered for a future update. Note, tests showed that constantly the GCD always was 32 on these systems, i.e. the 5 LSB were always zero (thus failing the APT since it only considered the 4 LSB for its calculation). Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/jitterentropy.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c index 4dc2261cdeef..788d90749715 100644 --- a/crypto/jitterentropy.c +++ b/crypto/jitterentropy.c @@ -265,7 +265,6 @@ static int jent_stuck(struct rand_data *ec, __u64 current_delta) { __u64 delta2 = jent_delta(ec->last_delta, current_delta); __u64 delta3 = jent_delta(ec->last_delta2, delta2); - unsigned int delta_masked = current_delta & JENT_APT_WORD_MASK; ec->last_delta = current_delta; ec->last_delta2 = delta2; @@ -274,7 +273,7 @@ static int jent_stuck(struct rand_data *ec, __u64 current_delta) * Insert the result of the comparison of two back-to-back time * deltas. */ - jent_apt_insert(ec, delta_masked); + jent_apt_insert(ec, current_delta); if (!current_delta || !delta2 || !delta3) { /* RCT with a stuck bit */ From 1ce1bacc480965fab4420e561916ce45d2e90c05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20M=C3=BCller?= Date: Sun, 21 Nov 2021 15:31:27 +0100 Subject: [PATCH 055/145] crypto: rsa - limit key size to 2048 in FIPS mode FIPS disallows RSA with keys < 2048 bits. Thus, the kernel should consider the enforcement of this limit. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/rsa.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crypto/rsa.c b/crypto/rsa.c index 4cdbec95d077..39e04176b04b 100644 --- a/crypto/rsa.c +++ b/crypto/rsa.c @@ -5,6 +5,7 @@ * Authors: Tadeusz Struk */ +#include #include #include #include @@ -144,6 +145,9 @@ static int rsa_check_key_length(unsigned int len) case 512: case 1024: case 1536: + if (fips_enabled) + return -EINVAL; + fallthrough; case 2048: case 3072: case 4096: From 1e146c393b152a31771b49af5d104d9ed846da9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20M=C3=BCller?= Date: Sun, 21 Nov 2021 15:51:44 +0100 Subject: [PATCH 056/145] crypto: dh - limit key size to 2048 in FIPS mode FIPS disallows DH with keys < 2048 bits. Thus, the kernel should consider the enforcement of this limit. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/dh.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crypto/dh.c b/crypto/dh.c index cd4f32092e5c..38557e64b4b3 100644 --- a/crypto/dh.c +++ b/crypto/dh.c @@ -5,6 +5,7 @@ * Authors: Salvatore Benedetto */ +#include #include #include #include @@ -47,6 +48,9 @@ static inline struct dh_ctx *dh_get_ctx(struct crypto_kpp *tfm) static int dh_check_params_length(unsigned int p_len) { + if (fips_enabled) + return (p_len < 2048) ? -EINVAL : 0; + return (p_len < 1536) ? -EINVAL : 0; } From 330507fbc9d8c3bc4525ea2ae9c3774738bc0c80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20M=C3=BCller?= Date: Sun, 21 Nov 2021 16:10:33 +0100 Subject: [PATCH 057/145] crypto: des - disallow des3 in FIPS mode On Dec 31 2023 NIST sunsets TDES for FIPS use. To prevent FIPS validations to be completed in the future to be affected by the TDES sunsetting, disallow TDES already now. Otherwise a FIPS validation would need to be "touched again" end 2023 to handle TDES accordingly. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/testmgr.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 58eee8eab4bf..5831d4bbc64f 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4193,7 +4193,6 @@ static const struct alg_test_desc alg_test_descs[] = { }, { .alg = "authenc(hmac(sha1),cbc(des3_ede))", .test = alg_test_aead, - .fips_allowed = 1, .suite = { .aead = __VECS(hmac_sha1_des3_ede_cbc_tv_temp) } @@ -4220,7 +4219,6 @@ static const struct alg_test_desc alg_test_descs[] = { }, { .alg = "authenc(hmac(sha224),cbc(des3_ede))", .test = alg_test_aead, - .fips_allowed = 1, .suite = { .aead = __VECS(hmac_sha224_des3_ede_cbc_tv_temp) } @@ -4240,7 +4238,6 @@ static const struct alg_test_desc alg_test_descs[] = { }, { .alg = "authenc(hmac(sha256),cbc(des3_ede))", .test = alg_test_aead, - .fips_allowed = 1, .suite = { .aead = __VECS(hmac_sha256_des3_ede_cbc_tv_temp) } @@ -4261,7 +4258,6 @@ static const struct alg_test_desc alg_test_descs[] = { }, { .alg = "authenc(hmac(sha384),cbc(des3_ede))", .test = alg_test_aead, - .fips_allowed = 1, .suite = { .aead = __VECS(hmac_sha384_des3_ede_cbc_tv_temp) } @@ -4289,7 +4285,6 @@ static const struct alg_test_desc alg_test_descs[] = { }, { .alg = "authenc(hmac(sha512),cbc(des3_ede))", .test = alg_test_aead, - .fips_allowed = 1, .suite = { .aead = __VECS(hmac_sha512_des3_ede_cbc_tv_temp) } @@ -4399,7 +4394,6 @@ static const struct alg_test_desc alg_test_descs[] = { }, { .alg = "cbc(des3_ede)", .test = alg_test_skcipher, - .fips_allowed = 1, .suite = { .cipher = __VECS(des3_ede_cbc_tv_template) }, @@ -4505,7 +4499,6 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "cmac(des3_ede)", - .fips_allowed = 1, .test = alg_test_hash, .suite = { .hash = __VECS(des3_ede_cmac64_tv_template) @@ -4580,7 +4573,6 @@ static const struct alg_test_desc alg_test_descs[] = { }, { .alg = "ctr(des3_ede)", .test = alg_test_skcipher, - .fips_allowed = 1, .suite = { .cipher = __VECS(des3_ede_ctr_tv_template) } @@ -4846,7 +4838,6 @@ static const struct alg_test_desc alg_test_descs[] = { }, { .alg = "ecb(des3_ede)", .test = alg_test_skcipher, - .fips_allowed = 1, .suite = { .cipher = __VECS(des3_ede_tv_template) } From 087e1d715bccf25dc0e83294576e416b0386ba20 Mon Sep 17 00:00:00 2001 From: Gaurav Jain Date: Mon, 22 Nov 2021 17:02:34 +0530 Subject: [PATCH 058/145] crypto: caam - save caam memory to support crypto engine retry mechanism. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When caam queue is full (-ENOSPC), caam frees descriptor memory. crypto-engine checks if retry support is true and h/w queue is full(-ENOSPC), then requeue the crypto request. During processing the requested descriptor again, caam gives below error. (caam_jr 30902000.jr: 40000006: DECO: desc idx 0: Invalid KEY Command). This patch adds a check to return when caam input ring is full and retry support is true. so descriptor memory is not freed and requeued request can be processed again. Fixes: 2d653936eb2cf ("crypto: caam - enable crypto-engine retry mechanism") Signed-off-by: Gaurav Jain Reviewed-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 6 ++++++ drivers/crypto/caam/caamhash.c | 3 +++ drivers/crypto/caam/caampkc.c | 3 +++ 3 files changed, 12 insertions(+) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 8697ae53b063..d3d8bb0a6990 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -1533,6 +1533,9 @@ static int aead_do_one_req(struct crypto_engine *engine, void *areq) ret = caam_jr_enqueue(ctx->jrdev, desc, aead_crypt_done, req); + if (ret == -ENOSPC && engine->retry_support) + return ret; + if (ret != -EINPROGRESS) { aead_unmap(ctx->jrdev, rctx->edesc, req); kfree(rctx->edesc); @@ -1762,6 +1765,9 @@ static int skcipher_do_one_req(struct crypto_engine *engine, void *areq) ret = caam_jr_enqueue(ctx->jrdev, desc, skcipher_crypt_done, req); + if (ret == -ENOSPC && engine->retry_support) + return ret; + if (ret != -EINPROGRESS) { skcipher_unmap(ctx->jrdev, rctx->edesc, req); kfree(rctx->edesc); diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index e8a6d8bc43b5..36ef738e4a18 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -765,6 +765,9 @@ static int ahash_do_one_req(struct crypto_engine *engine, void *areq) ret = caam_jr_enqueue(jrdev, desc, state->ahash_op_done, req); + if (ret == -ENOSPC && engine->retry_support) + return ret; + if (ret != -EINPROGRESS) { ahash_unmap(jrdev, state->edesc, req, 0); kfree(state->edesc); diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c index bf6275ffc4aa..886727576710 100644 --- a/drivers/crypto/caam/caampkc.c +++ b/drivers/crypto/caam/caampkc.c @@ -380,6 +380,9 @@ static int akcipher_do_one_req(struct crypto_engine *engine, void *areq) ret = caam_jr_enqueue(jrdev, desc, req_ctx->akcipher_op_done, req); + if (ret == -ENOSPC && engine->retry_support) + return ret; + if (ret != -EINPROGRESS) { rsa_pub_unmap(jrdev, req_ctx->edesc, req); rsa_io_unmap(jrdev, req_ctx->edesc, req); From 5876b0cb883da7a16129dadc06250a36a79a8ee1 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Mon, 22 Nov 2021 16:54:02 +0100 Subject: [PATCH 059/145] crypto: sa2ul - Use bitfield helpers Use the FIELD_PREP() helper, instead of open-coding the same operation. Signed-off-by: Geert Uytterhoeven Signed-off-by: Herbert Xu --- drivers/crypto/sa2ul.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index bcbc38dc6ae8..51b58e57153f 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -8,6 +8,7 @@ * Vitaly Andrianov * Tero Kristo */ +#include #include #include #include @@ -646,8 +647,8 @@ static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl, cmdl[upd_info->enc_offset.index] &= ~SA_CMDL_SOP_BYPASS_LEN_MASK; cmdl[upd_info->enc_offset.index] |= - ((u32)req->enc_offset << - __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK)); + FIELD_PREP(SA_CMDL_SOP_BYPASS_LEN_MASK, + req->enc_offset); if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) { __be32 *data = (__be32 *)&cmdl[upd_info->enc_iv.index]; @@ -666,8 +667,8 @@ static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl, cmdl[upd_info->auth_offset.index] &= ~SA_CMDL_SOP_BYPASS_LEN_MASK; cmdl[upd_info->auth_offset.index] |= - ((u32)req->auth_offset << - __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK)); + FIELD_PREP(SA_CMDL_SOP_BYPASS_LEN_MASK, + req->auth_offset); if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) { sa_copy_iv((void *)&cmdl[upd_info->auth_iv.index], req->auth_iv, @@ -689,16 +690,16 @@ void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys, u8 hash_size, u32 *swinfo) { swinfo[0] = sc_id; - swinfo[0] |= (flags << __ffs(SA_SW0_FLAGS_MASK)); + swinfo[0] |= FIELD_PREP(SA_SW0_FLAGS_MASK, flags); if (likely(cmdl_present)) - swinfo[0] |= ((cmdl_offset | SA_SW0_CMDL_PRESENT) << - __ffs(SA_SW0_CMDL_INFO_MASK)); - swinfo[0] |= (eng_id << __ffs(SA_SW0_ENG_ID_MASK)); + swinfo[0] |= FIELD_PREP(SA_SW0_CMDL_INFO_MASK, + cmdl_offset | SA_SW0_CMDL_PRESENT); + swinfo[0] |= FIELD_PREP(SA_SW0_ENG_ID_MASK, eng_id); swinfo[0] |= SA_SW0_DEST_INFO_PRESENT; swinfo[1] = (u32)(sc_phys & 0xFFFFFFFFULL); swinfo[2] = (u32)((sc_phys & 0xFFFFFFFF00000000ULL) >> 32); - swinfo[2] |= (hash_size << __ffs(SA_SW2_EGRESS_LENGTH)); + swinfo[2] |= FIELD_PREP(SA_SW2_EGRESS_LENGTH, hash_size); } /* Dump the security context */ From 0ea275df84c389e910a3575a9233075118c173ee Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Sat, 27 Nov 2021 17:10:27 +0300 Subject: [PATCH 060/145] crypto: octeontx2 - uninitialized variable in kvf_limits_store() If kstrtoint() fails then "lfs_num" is uninitialized and the warning doesn't make any sense. Just delete it. Fixes: 8ec8015a3168 ("crypto: octeontx2 - add support to process the crypto request") Signed-off-by: Dan Carpenter Signed-off-by: Herbert Xu --- drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c index 146a55ac4b9b..be1ad55a208f 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c @@ -494,12 +494,11 @@ static ssize_t kvf_limits_store(struct device *dev, { struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev); int lfs_num; + int ret; - if (kstrtoint(buf, 0, &lfs_num)) { - dev_err(dev, "lfs count %d must be in range [1 - %d]\n", - lfs_num, num_online_cpus()); - return -EINVAL; - } + ret = kstrtoint(buf, 0, &lfs_num); + if (ret) + return ret; if (lfs_num < 1 || lfs_num > num_online_cpus()) { dev_err(dev, "lfs count %d must be in range [1 - %d]\n", lfs_num, num_online_cpus()); From 3219c2b1bd4c4ed19f35bd83e3059c94077616e4 Mon Sep 17 00:00:00 2001 From: Guo Zhengkui Date: Mon, 29 Nov 2021 16:34:56 +0800 Subject: [PATCH 061/145] crypto: dh - remove duplicate includes Remove a duplicate #include . Signed-off-by: Guo Zhengkui Signed-off-by: Herbert Xu --- crypto/dh.c | 1 - 1 file changed, 1 deletion(-) diff --git a/crypto/dh.c b/crypto/dh.c index 38557e64b4b3..27e62a2a8027 100644 --- a/crypto/dh.c +++ b/crypto/dh.c @@ -10,7 +10,6 @@ #include #include #include -#include #include struct dh_ctx { From 0a2f9f57c6ba4730f02df532e03316535a96fef8 Mon Sep 17 00:00:00 2001 From: Etienne Carriere Date: Tue, 30 Nov 2021 08:54:53 +0100 Subject: [PATCH 062/145] crypto: stm32/cryp - defer probe for reset controller Change stm32 CRYP driver to defer its probe operation when reset controller device is registered but has not been probed yet. Signed-off-by: Etienne Carriere Signed-off-by: Nicolas Toromanoff Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32-cryp.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index 7389a0536ff0..dcdd313485de 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -1973,7 +1973,11 @@ static int stm32_cryp_probe(struct platform_device *pdev) pm_runtime_enable(dev); rst = devm_reset_control_get(dev, NULL); - if (!IS_ERR(rst)) { + if (IS_ERR(rst)) { + ret = PTR_ERR(rst); + if (ret == -EPROBE_DEFER) + goto err_rst; + } else { reset_control_assert(rst); udelay(2); reset_control_deassert(rst); @@ -2024,7 +2028,7 @@ err_engine1: spin_lock(&cryp_list.lock); list_del(&cryp->list); spin_unlock(&cryp_list.lock); - +err_rst: pm_runtime_disable(dev); pm_runtime_put_noidle(dev); pm_runtime_disable(dev); From 029812aee3a120c9ca3cd891e5ef159ec3c6c547 Mon Sep 17 00:00:00 2001 From: Etienne Carriere Date: Tue, 30 Nov 2021 08:54:54 +0100 Subject: [PATCH 063/145] crypto: stm32/cryp - don't print error on probe deferral Change driver to not print an error message when the device probe is deferred for a clock resource. Signed-off-by: Etienne Carriere Signed-off-by: Nicolas Toromanoff Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32-cryp.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index dcdd313485de..7b55ad6d2f1a 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -1955,7 +1955,8 @@ static int stm32_cryp_probe(struct platform_device *pdev) cryp->clk = devm_clk_get(dev, NULL); if (IS_ERR(cryp->clk)) { - dev_err(dev, "Could not get clock\n"); + dev_err_probe(dev, PTR_ERR(cryp->clk), "Could not get clock\n"); + return PTR_ERR(cryp->clk); } From 41c76690b0990efacd15d35cfb4e77318cd80ebb Mon Sep 17 00:00:00 2001 From: Nicolas Toromanoff Date: Tue, 30 Nov 2021 08:54:55 +0100 Subject: [PATCH 064/145] crypto: stm32/cryp - fix CTR counter carry STM32 CRYP hardware doesn't manage CTR counter bigger than max U32, as a workaround, at each block the current IV is saved, if the saved IV lower u32 is 0xFFFFFFFF, the full IV is manually incremented, and set in hardware. Fixes: bbb2832620ac ("crypto: stm32 - Fix sparse warnings") Signed-off-by: Nicolas Toromanoff Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32-cryp.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index 7b55ad6d2f1a..ab4a107a0860 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -163,7 +163,7 @@ struct stm32_cryp { struct scatter_walk in_walk; struct scatter_walk out_walk; - u32 last_ctr[4]; + __be32 last_ctr[4]; u32 gcm_ctr; }; @@ -1217,27 +1217,26 @@ static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp) { u32 cr; - if (unlikely(cryp->last_ctr[3] == 0xFFFFFFFF)) { - cryp->last_ctr[3] = 0; - cryp->last_ctr[2]++; - if (!cryp->last_ctr[2]) { - cryp->last_ctr[1]++; - if (!cryp->last_ctr[1]) - cryp->last_ctr[0]++; - } + if (unlikely(cryp->last_ctr[3] == cpu_to_be32(0xFFFFFFFF))) { + /* + * In this case, we need to increment manually the ctr counter, + * as HW doesn't handle the U32 carry. + */ + crypto_inc((u8 *)cryp->last_ctr, sizeof(cryp->last_ctr)); cr = stm32_cryp_read(cryp, CRYP_CR); stm32_cryp_write(cryp, CRYP_CR, cr & ~CR_CRYPEN); - stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->last_ctr); + stm32_cryp_hw_write_iv(cryp, cryp->last_ctr); stm32_cryp_write(cryp, CRYP_CR, cr); } - cryp->last_ctr[0] = stm32_cryp_read(cryp, CRYP_IV0LR); - cryp->last_ctr[1] = stm32_cryp_read(cryp, CRYP_IV0RR); - cryp->last_ctr[2] = stm32_cryp_read(cryp, CRYP_IV1LR); - cryp->last_ctr[3] = stm32_cryp_read(cryp, CRYP_IV1RR); + /* The IV registers are BE */ + cryp->last_ctr[0] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0LR)); + cryp->last_ctr[1] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0RR)); + cryp->last_ctr[2] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1LR)); + cryp->last_ctr[3] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1RR)); } static bool stm32_cryp_irq_read_data(struct stm32_cryp *cryp) From d703c7a994ee34b7fa89baf21631fca0aa9f17fc Mon Sep 17 00:00:00 2001 From: Nicolas Toromanoff Date: Tue, 30 Nov 2021 08:54:56 +0100 Subject: [PATCH 065/145] crypto: stm32/cryp - fix xts and race condition in crypto_engine requests Don't erase key: If key is erased before the crypto_finalize_.*_request() call, some pending process will run with a key={ 0 }. Moreover if the key is reset at end of request, it breaks xts chaining mode, as for last xts block (in case input len is not a multiple of block) a new AES request is started without calling again set_key(). Fixes: 9e054ec21ef8 ("crypto: stm32 - Support for STM32 CRYP crypto module") Signed-off-by: Nicolas Toromanoff Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32-cryp.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index ab4a107a0860..b2b553651964 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -674,8 +674,6 @@ static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err) else crypto_finalize_skcipher_request(cryp->engine, cryp->req, err); - - memset(cryp->ctx->key, 0, cryp->ctx->keylen); } static int stm32_cryp_cpu_start(struct stm32_cryp *cryp) From 39e6e699c7fb92bdb2617b596ca4a4ea35c5d2a7 Mon Sep 17 00:00:00 2001 From: Nicolas Toromanoff Date: Tue, 30 Nov 2021 08:54:57 +0100 Subject: [PATCH 066/145] crypto: stm32/cryp - check early input data Some auto tests failed because driver wasn't returning the expected error with some input size/iv value/tag size. Now: Return 0 early for empty buffer. (We don't need to start the engine for an empty input buffer). Accept any valid authsize for gcm(aes). Return -EINVAL if iv for ccm(aes) is invalid. Return -EINVAL if buffer size is a not a multiple of algorithm block size. Fixes: 9e054ec21ef8 ("crypto: stm32 - Support for STM32 CRYP crypto module") Signed-off-by: Nicolas Toromanoff Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32-cryp.c | 114 +++++++++++++++++++++++++++++- 1 file changed, 113 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index b2b553651964..06c03db0d11b 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -799,7 +799,20 @@ static int stm32_cryp_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key, static int stm32_cryp_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - return authsize == AES_BLOCK_SIZE ? 0 : -EINVAL; + switch (authsize) { + case 4: + case 8: + case 12: + case 13: + case 14: + case 15: + case 16: + break; + default: + return -EINVAL; + } + + return 0; } static int stm32_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm, @@ -823,31 +836,61 @@ static int stm32_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm, static int stm32_cryp_aes_ecb_encrypt(struct skcipher_request *req) { + if (req->cryptlen % AES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + return stm32_cryp_crypt(req, FLG_AES | FLG_ECB | FLG_ENCRYPT); } static int stm32_cryp_aes_ecb_decrypt(struct skcipher_request *req) { + if (req->cryptlen % AES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + return stm32_cryp_crypt(req, FLG_AES | FLG_ECB); } static int stm32_cryp_aes_cbc_encrypt(struct skcipher_request *req) { + if (req->cryptlen % AES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + return stm32_cryp_crypt(req, FLG_AES | FLG_CBC | FLG_ENCRYPT); } static int stm32_cryp_aes_cbc_decrypt(struct skcipher_request *req) { + if (req->cryptlen % AES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + return stm32_cryp_crypt(req, FLG_AES | FLG_CBC); } static int stm32_cryp_aes_ctr_encrypt(struct skcipher_request *req) { + if (req->cryptlen == 0) + return 0; + return stm32_cryp_crypt(req, FLG_AES | FLG_CTR | FLG_ENCRYPT); } static int stm32_cryp_aes_ctr_decrypt(struct skcipher_request *req) { + if (req->cryptlen == 0) + return 0; + return stm32_cryp_crypt(req, FLG_AES | FLG_CTR); } @@ -861,53 +904,122 @@ static int stm32_cryp_aes_gcm_decrypt(struct aead_request *req) return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM); } +static inline int crypto_ccm_check_iv(const u8 *iv) +{ + /* 2 <= L <= 8, so 1 <= L' <= 7. */ + if (iv[0] < 1 || iv[0] > 7) + return -EINVAL; + + return 0; +} + static int stm32_cryp_aes_ccm_encrypt(struct aead_request *req) { + int err; + + err = crypto_ccm_check_iv(req->iv); + if (err) + return err; + return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM | FLG_ENCRYPT); } static int stm32_cryp_aes_ccm_decrypt(struct aead_request *req) { + int err; + + err = crypto_ccm_check_iv(req->iv); + if (err) + return err; + return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM); } static int stm32_cryp_des_ecb_encrypt(struct skcipher_request *req) { + if (req->cryptlen % DES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + return stm32_cryp_crypt(req, FLG_DES | FLG_ECB | FLG_ENCRYPT); } static int stm32_cryp_des_ecb_decrypt(struct skcipher_request *req) { + if (req->cryptlen % DES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + return stm32_cryp_crypt(req, FLG_DES | FLG_ECB); } static int stm32_cryp_des_cbc_encrypt(struct skcipher_request *req) { + if (req->cryptlen % DES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + return stm32_cryp_crypt(req, FLG_DES | FLG_CBC | FLG_ENCRYPT); } static int stm32_cryp_des_cbc_decrypt(struct skcipher_request *req) { + if (req->cryptlen % DES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + return stm32_cryp_crypt(req, FLG_DES | FLG_CBC); } static int stm32_cryp_tdes_ecb_encrypt(struct skcipher_request *req) { + if (req->cryptlen % DES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB | FLG_ENCRYPT); } static int stm32_cryp_tdes_ecb_decrypt(struct skcipher_request *req) { + if (req->cryptlen % DES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB); } static int stm32_cryp_tdes_cbc_encrypt(struct skcipher_request *req) { + if (req->cryptlen % DES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC | FLG_ENCRYPT); } static int stm32_cryp_tdes_cbc_decrypt(struct skcipher_request *req) { + if (req->cryptlen % DES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC); } From 6c12e742785bf9333faf60bfb96575bdd763448e Mon Sep 17 00:00:00 2001 From: Nicolas Toromanoff Date: Tue, 30 Nov 2021 08:54:58 +0100 Subject: [PATCH 067/145] crypto: stm32/cryp - fix double pm exit Delete extraneous lines in probe error handling code: pm was disabled twice. Fixes: 65f9aa36ee47 ("crypto: stm32/cryp - Add power management support") Reported-by: Marek Vasut Signed-off-by: Nicolas Toromanoff Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32-cryp.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index 06c03db0d11b..3f71d927843c 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -2141,8 +2141,6 @@ err_engine1: err_rst: pm_runtime_disable(dev); pm_runtime_put_noidle(dev); - pm_runtime_disable(dev); - pm_runtime_put_noidle(dev); clk_disable_unprepare(cryp->clk); From fa97dc2d48b476ea98199d808d3248d285987e99 Mon Sep 17 00:00:00 2001 From: Nicolas Toromanoff Date: Tue, 30 Nov 2021 08:54:59 +0100 Subject: [PATCH 068/145] crypto: stm32/cryp - fix lrw chaining mode This fixes the lrw autotest if lrw uses the CRYP as the AES block cipher provider (as ecb(aes)). At end of request, CRYP should not update the IV in case of ECB chaining mode. Indeed the ECB chaining mode never uses the IV, but the software LRW chaining mode uses the IV field as a counter and due to the (unexpected) update done by CRYP while the AES block process, the counter get a wrong value when the IV overflow. Fixes: 5f49f18d27cd ("crypto: stm32/cryp - update to return iv_out") Signed-off-by: Nicolas Toromanoff Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32-cryp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index 3f71d927843c..d3aa0afbe365 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -644,7 +644,7 @@ static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err) /* Phase 4 : output tag */ err = stm32_cryp_read_auth_tag(cryp); - if (!err && (!(is_gcm(cryp) || is_ccm(cryp)))) + if (!err && (!(is_gcm(cryp) || is_ccm(cryp) || is_ecb(cryp)))) stm32_cryp_get_iv(cryp); if (cryp->sgs_copied) { From 4b898d5cfa4d9a0ad5bc82cb5eafdc092394c6a9 Mon Sep 17 00:00:00 2001 From: Nicolas Toromanoff Date: Tue, 30 Nov 2021 08:55:00 +0100 Subject: [PATCH 069/145] crypto: stm32/cryp - fix bugs and crash in tests Extra crypto manager auto test were crashing or failling due to 2 reasons: - block in a dead loop (dues to issues in cipher end process management) - crash due to read/write unmapped memory (this crash was also reported when using openssl afalg engine) Rework interrupt management, interrupts are masked as soon as they are no more used: if input buffer is fully consumed, "Input FIFO not full" interrupt is masked and if output buffer is full, "Output FIFO not empty" interrupt is masked. And crypto request finish when input *and* outpout buffer are fully read/write. About the crash due to unmapped memory, using scatterwalk_copychunks() that will map and copy each block fix the issue. Using this api and copying full block will also fix unaligned data access, avoid early copy of in/out buffer, and make useless the extra alignment constraint. Fixes: 9e054ec21ef8 ("crypto: stm32 - Support for STM32 CRYP crypto module") Reported-by: Marek Vasut Signed-off-by: Nicolas Toromanoff Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32-cryp.c | 790 +++++++++--------------------- 1 file changed, 243 insertions(+), 547 deletions(-) diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index d3aa0afbe365..faae7ad262ac 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -37,7 +37,6 @@ /* Mode mask = bits [15..0] */ #define FLG_MODE_MASK GENMASK(15, 0) /* Bit [31..16] status */ -#define FLG_CCM_PADDED_WA BIT(16) /* Registers */ #define CRYP_CR 0x00000000 @@ -105,8 +104,6 @@ /* Misc */ #define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32)) #define GCM_CTR_INIT 2 -#define _walked_in (cryp->in_walk.offset - cryp->in_sg->offset) -#define _walked_out (cryp->out_walk.offset - cryp->out_sg->offset) #define CRYP_AUTOSUSPEND_DELAY 50 struct stm32_cryp_caps { @@ -144,21 +141,11 @@ struct stm32_cryp { size_t authsize; size_t hw_blocksize; - size_t total_in; - size_t total_in_save; - size_t total_out; - size_t total_out_save; + size_t payload_in; + size_t header_in; + size_t payload_out; - struct scatterlist *in_sg; struct scatterlist *out_sg; - struct scatterlist *out_sg_save; - - struct scatterlist in_sgl; - struct scatterlist out_sgl; - bool sgs_copied; - - int in_sg_len; - int out_sg_len; struct scatter_walk in_walk; struct scatter_walk out_walk; @@ -262,6 +249,7 @@ static inline int stm32_cryp_wait_output(struct stm32_cryp *cryp) } static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp); +static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err); static struct stm32_cryp *stm32_cryp_find_dev(struct stm32_cryp_ctx *ctx) { @@ -283,103 +271,6 @@ static struct stm32_cryp *stm32_cryp_find_dev(struct stm32_cryp_ctx *ctx) return cryp; } -static int stm32_cryp_check_aligned(struct scatterlist *sg, size_t total, - size_t align) -{ - int len = 0; - - if (!total) - return 0; - - if (!IS_ALIGNED(total, align)) - return -EINVAL; - - while (sg) { - if (!IS_ALIGNED(sg->offset, sizeof(u32))) - return -EINVAL; - - if (!IS_ALIGNED(sg->length, align)) - return -EINVAL; - - len += sg->length; - sg = sg_next(sg); - } - - if (len != total) - return -EINVAL; - - return 0; -} - -static int stm32_cryp_check_io_aligned(struct stm32_cryp *cryp) -{ - int ret; - - ret = stm32_cryp_check_aligned(cryp->in_sg, cryp->total_in, - cryp->hw_blocksize); - if (ret) - return ret; - - ret = stm32_cryp_check_aligned(cryp->out_sg, cryp->total_out, - cryp->hw_blocksize); - - return ret; -} - -static void sg_copy_buf(void *buf, struct scatterlist *sg, - unsigned int start, unsigned int nbytes, int out) -{ - struct scatter_walk walk; - - if (!nbytes) - return; - - scatterwalk_start(&walk, sg); - scatterwalk_advance(&walk, start); - scatterwalk_copychunks(buf, &walk, nbytes, out); - scatterwalk_done(&walk, out, 0); -} - -static int stm32_cryp_copy_sgs(struct stm32_cryp *cryp) -{ - void *buf_in, *buf_out; - int pages, total_in, total_out; - - if (!stm32_cryp_check_io_aligned(cryp)) { - cryp->sgs_copied = 0; - return 0; - } - - total_in = ALIGN(cryp->total_in, cryp->hw_blocksize); - pages = total_in ? get_order(total_in) : 1; - buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages); - - total_out = ALIGN(cryp->total_out, cryp->hw_blocksize); - pages = total_out ? get_order(total_out) : 1; - buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages); - - if (!buf_in || !buf_out) { - dev_err(cryp->dev, "Can't allocate pages when unaligned\n"); - cryp->sgs_copied = 0; - return -EFAULT; - } - - sg_copy_buf(buf_in, cryp->in_sg, 0, cryp->total_in, 0); - - sg_init_one(&cryp->in_sgl, buf_in, total_in); - cryp->in_sg = &cryp->in_sgl; - cryp->in_sg_len = 1; - - sg_init_one(&cryp->out_sgl, buf_out, total_out); - cryp->out_sg_save = cryp->out_sg; - cryp->out_sg = &cryp->out_sgl; - cryp->out_sg_len = 1; - - cryp->sgs_copied = 1; - - return 0; -} - static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, __be32 *iv) { if (!iv) @@ -481,16 +372,99 @@ static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg) /* Wait for end of processing */ ret = stm32_cryp_wait_enable(cryp); - if (ret) + if (ret) { dev_err(cryp->dev, "Timeout (gcm init)\n"); + return ret; + } - return ret; + /* Prepare next phase */ + if (cryp->areq->assoclen) { + cfg |= CR_PH_HEADER; + stm32_cryp_write(cryp, CRYP_CR, cfg); + } else if (stm32_cryp_get_input_text_len(cryp)) { + cfg |= CR_PH_PAYLOAD; + stm32_cryp_write(cryp, CRYP_CR, cfg); + } + + return 0; +} + +static void stm32_crypt_gcmccm_end_header(struct stm32_cryp *cryp) +{ + u32 cfg; + int err; + + /* Check if whole header written */ + if (!cryp->header_in) { + /* Wait for completion */ + err = stm32_cryp_wait_busy(cryp); + if (err) { + dev_err(cryp->dev, "Timeout (gcm/ccm header)\n"); + stm32_cryp_write(cryp, CRYP_IMSCR, 0); + stm32_cryp_finish_req(cryp, err); + return; + } + + if (stm32_cryp_get_input_text_len(cryp)) { + /* Phase 3 : payload */ + cfg = stm32_cryp_read(cryp, CRYP_CR); + cfg &= ~CR_CRYPEN; + stm32_cryp_write(cryp, CRYP_CR, cfg); + + cfg &= ~CR_PH_MASK; + cfg |= CR_PH_PAYLOAD | CR_CRYPEN; + stm32_cryp_write(cryp, CRYP_CR, cfg); + } else { + /* + * Phase 4 : tag. + * Nothing to read, nothing to write, caller have to + * end request + */ + } + } +} + +static void stm32_cryp_write_ccm_first_header(struct stm32_cryp *cryp) +{ + unsigned int i; + size_t written; + size_t len; + u32 alen = cryp->areq->assoclen; + u32 block[AES_BLOCK_32] = {0}; + u8 *b8 = (u8 *)block; + + if (alen <= 65280) { + /* Write first u32 of B1 */ + b8[0] = (alen >> 8) & 0xFF; + b8[1] = alen & 0xFF; + len = 2; + } else { + /* Build the two first u32 of B1 */ + b8[0] = 0xFF; + b8[1] = 0xFE; + b8[2] = (alen & 0xFF000000) >> 24; + b8[3] = (alen & 0x00FF0000) >> 16; + b8[4] = (alen & 0x0000FF00) >> 8; + b8[5] = alen & 0x000000FF; + len = 6; + } + + written = min_t(size_t, AES_BLOCK_SIZE - len, alen); + + scatterwalk_copychunks((char *)block + len, &cryp->in_walk, written, 0); + for (i = 0; i < AES_BLOCK_32; i++) + stm32_cryp_write(cryp, CRYP_DIN, block[i]); + + cryp->header_in -= written; + + stm32_crypt_gcmccm_end_header(cryp); } static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg) { int ret; - u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE]; + u32 iv_32[AES_BLOCK_32], b0_32[AES_BLOCK_32]; + u8 *iv = (u8 *)iv_32, *b0 = (u8 *)b0_32; __be32 *bd; u32 *d; unsigned int i, textlen; @@ -531,17 +505,30 @@ static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg) /* Wait for end of processing */ ret = stm32_cryp_wait_enable(cryp); - if (ret) + if (ret) { dev_err(cryp->dev, "Timeout (ccm init)\n"); + return ret; + } - return ret; + /* Prepare next phase */ + if (cryp->areq->assoclen) { + cfg |= CR_PH_HEADER | CR_CRYPEN; + stm32_cryp_write(cryp, CRYP_CR, cfg); + + /* Write first (special) block (may move to next phase [payload]) */ + stm32_cryp_write_ccm_first_header(cryp); + } else if (stm32_cryp_get_input_text_len(cryp)) { + cfg |= CR_PH_PAYLOAD; + stm32_cryp_write(cryp, CRYP_CR, cfg); + } + + return 0; } static int stm32_cryp_hw_init(struct stm32_cryp *cryp) { int ret; u32 cfg, hw_mode; - pm_runtime_resume_and_get(cryp->dev); /* Disable interrupt */ @@ -605,16 +592,6 @@ static int stm32_cryp_hw_init(struct stm32_cryp *cryp) if (ret) return ret; - /* Phase 2 : header (authenticated data) */ - if (cryp->areq->assoclen) { - cfg |= CR_PH_HEADER; - } else if (stm32_cryp_get_input_text_len(cryp)) { - cfg |= CR_PH_PAYLOAD; - stm32_cryp_write(cryp, CRYP_CR, cfg); - } else { - cfg |= CR_PH_INIT; - } - break; case CR_DES_CBC: @@ -633,8 +610,6 @@ static int stm32_cryp_hw_init(struct stm32_cryp *cryp) stm32_cryp_write(cryp, CRYP_CR, cfg); - cryp->flags &= ~FLG_CCM_PADDED_WA; - return 0; } @@ -647,25 +622,6 @@ static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err) if (!err && (!(is_gcm(cryp) || is_ccm(cryp) || is_ecb(cryp)))) stm32_cryp_get_iv(cryp); - if (cryp->sgs_copied) { - void *buf_in, *buf_out; - int pages, len; - - buf_in = sg_virt(&cryp->in_sgl); - buf_out = sg_virt(&cryp->out_sgl); - - sg_copy_buf(buf_out, cryp->out_sg_save, 0, - cryp->total_out_save, 1); - - len = ALIGN(cryp->total_in_save, cryp->hw_blocksize); - pages = len ? get_order(len) : 1; - free_pages((unsigned long)buf_in, pages); - - len = ALIGN(cryp->total_out_save, cryp->hw_blocksize); - pages = len ? get_order(len) : 1; - free_pages((unsigned long)buf_out, pages); - } - pm_runtime_mark_last_busy(cryp->dev); pm_runtime_put_autosuspend(cryp->dev); @@ -1029,6 +985,7 @@ static int stm32_cryp_prepare_req(struct skcipher_request *req, struct stm32_cryp_ctx *ctx; struct stm32_cryp *cryp; struct stm32_cryp_reqctx *rctx; + struct scatterlist *in_sg; int ret; if (!req && !areq) @@ -1054,76 +1011,55 @@ static int stm32_cryp_prepare_req(struct skcipher_request *req, if (req) { cryp->req = req; cryp->areq = NULL; - cryp->total_in = req->cryptlen; - cryp->total_out = cryp->total_in; + cryp->header_in = 0; + cryp->payload_in = req->cryptlen; + cryp->payload_out = req->cryptlen; + cryp->authsize = 0; } else { /* * Length of input and output data: * Encryption case: - * INPUT = AssocData || PlainText + * INPUT = AssocData || PlainText * <- assoclen -> <- cryptlen -> - * <------- total_in -----------> * - * OUTPUT = AssocData || CipherText || AuthTag - * <- assoclen -> <- cryptlen -> <- authsize -> - * <---------------- total_out -----------------> + * OUTPUT = AssocData || CipherText || AuthTag + * <- assoclen -> <-- cryptlen --> <- authsize -> * * Decryption case: - * INPUT = AssocData || CipherText || AuthTag - * <- assoclen -> <--------- cryptlen ---------> - * <- authsize -> - * <---------------- total_in ------------------> + * INPUT = AssocData || CipherTex || AuthTag + * <- assoclen ---> <---------- cryptlen ----------> * - * OUTPUT = AssocData || PlainText - * <- assoclen -> <- crypten - authsize -> - * <---------- total_out -----------------> + * OUTPUT = AssocData || PlainText + * <- assoclen -> <- cryptlen - authsize -> */ cryp->areq = areq; cryp->req = NULL; cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq)); - cryp->total_in = areq->assoclen + areq->cryptlen; - if (is_encrypt(cryp)) - /* Append auth tag to output */ - cryp->total_out = cryp->total_in + cryp->authsize; - else - /* No auth tag in output */ - cryp->total_out = cryp->total_in - cryp->authsize; + if (is_encrypt(cryp)) { + cryp->payload_in = areq->cryptlen; + cryp->header_in = areq->assoclen; + cryp->payload_out = areq->cryptlen; + } else { + cryp->payload_in = areq->cryptlen - cryp->authsize; + cryp->header_in = areq->assoclen; + cryp->payload_out = cryp->payload_in; + } } - cryp->total_in_save = cryp->total_in; - cryp->total_out_save = cryp->total_out; + in_sg = req ? req->src : areq->src; + scatterwalk_start(&cryp->in_walk, in_sg); - cryp->in_sg = req ? req->src : areq->src; cryp->out_sg = req ? req->dst : areq->dst; - cryp->out_sg_save = cryp->out_sg; - - cryp->in_sg_len = sg_nents_for_len(cryp->in_sg, cryp->total_in); - if (cryp->in_sg_len < 0) { - dev_err(cryp->dev, "Cannot get in_sg_len\n"); - ret = cryp->in_sg_len; - return ret; - } - - cryp->out_sg_len = sg_nents_for_len(cryp->out_sg, cryp->total_out); - if (cryp->out_sg_len < 0) { - dev_err(cryp->dev, "Cannot get out_sg_len\n"); - ret = cryp->out_sg_len; - return ret; - } - - ret = stm32_cryp_copy_sgs(cryp); - if (ret) - return ret; - - scatterwalk_start(&cryp->in_walk, cryp->in_sg); scatterwalk_start(&cryp->out_walk, cryp->out_sg); if (is_gcm(cryp) || is_ccm(cryp)) { /* In output, jump after assoc data */ - scatterwalk_advance(&cryp->out_walk, cryp->areq->assoclen); - cryp->total_out -= cryp->areq->assoclen; + scatterwalk_copychunks(NULL, &cryp->out_walk, cryp->areq->assoclen, 2); } + if (is_ctr(cryp)) + memset(cryp->last_ctr, 0, sizeof(cryp->last_ctr)); + ret = stm32_cryp_hw_init(cryp); return ret; } @@ -1171,8 +1107,7 @@ static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq) if (!cryp) return -ENODEV; - if (unlikely(!cryp->areq->assoclen && - !stm32_cryp_get_input_text_len(cryp))) { + if (unlikely(!cryp->payload_in && !cryp->header_in)) { /* No input data to process: get tag and finish */ stm32_cryp_finish_req(cryp, 0); return 0; @@ -1181,43 +1116,10 @@ static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq) return stm32_cryp_cpu_start(cryp); } -static u32 *stm32_cryp_next_out(struct stm32_cryp *cryp, u32 *dst, - unsigned int n) -{ - scatterwalk_advance(&cryp->out_walk, n); - - if (unlikely(cryp->out_sg->length == _walked_out)) { - cryp->out_sg = sg_next(cryp->out_sg); - if (cryp->out_sg) { - scatterwalk_start(&cryp->out_walk, cryp->out_sg); - return (sg_virt(cryp->out_sg) + _walked_out); - } - } - - return (u32 *)((u8 *)dst + n); -} - -static u32 *stm32_cryp_next_in(struct stm32_cryp *cryp, u32 *src, - unsigned int n) -{ - scatterwalk_advance(&cryp->in_walk, n); - - if (unlikely(cryp->in_sg->length == _walked_in)) { - cryp->in_sg = sg_next(cryp->in_sg); - if (cryp->in_sg) { - scatterwalk_start(&cryp->in_walk, cryp->in_sg); - return (sg_virt(cryp->in_sg) + _walked_in); - } - } - - return (u32 *)((u8 *)src + n); -} - static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp) { - u32 cfg, size_bit, *dst, d32; - u8 *d8; - unsigned int i, j; + u32 cfg, size_bit; + unsigned int i; int ret = 0; /* Update Config */ @@ -1240,7 +1142,7 @@ static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp) stm32_cryp_write(cryp, CRYP_DIN, size_bit); size_bit = is_encrypt(cryp) ? cryp->areq->cryptlen : - cryp->areq->cryptlen - AES_BLOCK_SIZE; + cryp->areq->cryptlen - cryp->authsize; size_bit *= 8; if (cryp->caps->swap_final) size_bit = (__force u32)cpu_to_be32(size_bit); @@ -1249,11 +1151,9 @@ static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp) stm32_cryp_write(cryp, CRYP_DIN, size_bit); } else { /* CCM: write CTR0 */ - u8 iv[AES_BLOCK_SIZE]; - u32 *iv32 = (u32 *)iv; - __be32 *biv; - - biv = (void *)iv; + u32 iv32[AES_BLOCK_32]; + u8 *iv = (u8 *)iv32; + __be32 *biv = (__be32 *)iv32; memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); @@ -1275,39 +1175,18 @@ static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp) } if (is_encrypt(cryp)) { + u32 out_tag[AES_BLOCK_32]; + /* Get and write tag */ - dst = sg_virt(cryp->out_sg) + _walked_out; + for (i = 0; i < AES_BLOCK_32; i++) + out_tag[i] = stm32_cryp_read(cryp, CRYP_DOUT); - for (i = 0; i < AES_BLOCK_32; i++) { - if (cryp->total_out >= sizeof(u32)) { - /* Read a full u32 */ - *dst = stm32_cryp_read(cryp, CRYP_DOUT); - - dst = stm32_cryp_next_out(cryp, dst, - sizeof(u32)); - cryp->total_out -= sizeof(u32); - } else if (!cryp->total_out) { - /* Empty fifo out (data from input padding) */ - stm32_cryp_read(cryp, CRYP_DOUT); - } else { - /* Read less than an u32 */ - d32 = stm32_cryp_read(cryp, CRYP_DOUT); - d8 = (u8 *)&d32; - - for (j = 0; j < cryp->total_out; j++) { - *((u8 *)dst) = *(d8++); - dst = stm32_cryp_next_out(cryp, dst, 1); - } - cryp->total_out = 0; - } - } + scatterwalk_copychunks(out_tag, &cryp->out_walk, cryp->authsize, 1); } else { /* Get and check tag */ u32 in_tag[AES_BLOCK_32], out_tag[AES_BLOCK_32]; - scatterwalk_map_and_copy(in_tag, cryp->in_sg, - cryp->total_in_save - cryp->authsize, - cryp->authsize, 0); + scatterwalk_copychunks(in_tag, &cryp->in_walk, cryp->authsize, 0); for (i = 0; i < AES_BLOCK_32; i++) out_tag[i] = stm32_cryp_read(cryp, CRYP_DOUT); @@ -1349,92 +1228,37 @@ static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp) cryp->last_ctr[3] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1RR)); } -static bool stm32_cryp_irq_read_data(struct stm32_cryp *cryp) +static void stm32_cryp_irq_read_data(struct stm32_cryp *cryp) { - unsigned int i, j; - u32 d32, *dst; - u8 *d8; - size_t tag_size; + unsigned int i; + u32 block[AES_BLOCK_32]; - /* Do no read tag now (if any) */ - if (is_encrypt(cryp) && (is_gcm(cryp) || is_ccm(cryp))) - tag_size = cryp->authsize; - else - tag_size = 0; + for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) + block[i] = stm32_cryp_read(cryp, CRYP_DOUT); - dst = sg_virt(cryp->out_sg) + _walked_out; - - for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) { - if (likely(cryp->total_out - tag_size >= sizeof(u32))) { - /* Read a full u32 */ - *dst = stm32_cryp_read(cryp, CRYP_DOUT); - - dst = stm32_cryp_next_out(cryp, dst, sizeof(u32)); - cryp->total_out -= sizeof(u32); - } else if (cryp->total_out == tag_size) { - /* Empty fifo out (data from input padding) */ - d32 = stm32_cryp_read(cryp, CRYP_DOUT); - } else { - /* Read less than an u32 */ - d32 = stm32_cryp_read(cryp, CRYP_DOUT); - d8 = (u8 *)&d32; - - for (j = 0; j < cryp->total_out - tag_size; j++) { - *((u8 *)dst) = *(d8++); - dst = stm32_cryp_next_out(cryp, dst, 1); - } - cryp->total_out = tag_size; - } - } - - return !(cryp->total_out - tag_size) || !cryp->total_in; + scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize, + cryp->payload_out), 1); + cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, + cryp->payload_out); } static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp) { - unsigned int i, j; - u32 *src; - u8 d8[4]; - size_t tag_size; + unsigned int i; + u32 block[AES_BLOCK_32] = {0}; - /* Do no write tag (if any) */ - if (is_decrypt(cryp) && (is_gcm(cryp) || is_ccm(cryp))) - tag_size = cryp->authsize; - else - tag_size = 0; + scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, cryp->hw_blocksize, + cryp->payload_in), 0); + for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) + stm32_cryp_write(cryp, CRYP_DIN, block[i]); - src = sg_virt(cryp->in_sg) + _walked_in; - - for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) { - if (likely(cryp->total_in - tag_size >= sizeof(u32))) { - /* Write a full u32 */ - stm32_cryp_write(cryp, CRYP_DIN, *src); - - src = stm32_cryp_next_in(cryp, src, sizeof(u32)); - cryp->total_in -= sizeof(u32); - } else if (cryp->total_in == tag_size) { - /* Write padding data */ - stm32_cryp_write(cryp, CRYP_DIN, 0); - } else { - /* Write less than an u32 */ - memset(d8, 0, sizeof(u32)); - for (j = 0; j < cryp->total_in - tag_size; j++) { - d8[j] = *((u8 *)src); - src = stm32_cryp_next_in(cryp, src, 1); - } - - stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8); - cryp->total_in = tag_size; - } - } + cryp->payload_in -= min_t(size_t, cryp->hw_blocksize, cryp->payload_in); } static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp) { int err; - u32 cfg, tmp[AES_BLOCK_32]; - size_t total_in_ori = cryp->total_in; - struct scatterlist *out_sg_ori = cryp->out_sg; + u32 cfg, block[AES_BLOCK_32] = {0}; unsigned int i; /* 'Special workaround' procedure described in the datasheet */ @@ -1459,18 +1283,25 @@ static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp) /* b) pad and write the last block */ stm32_cryp_irq_write_block(cryp); - cryp->total_in = total_in_ori; + /* wait end of process */ err = stm32_cryp_wait_output(cryp); if (err) { - dev_err(cryp->dev, "Timeout (write gcm header)\n"); + dev_err(cryp->dev, "Timeout (write gcm last data)\n"); return stm32_cryp_finish_req(cryp, err); } /* c) get and store encrypted data */ - stm32_cryp_irq_read_data(cryp); - scatterwalk_map_and_copy(tmp, out_sg_ori, - cryp->total_in_save - total_in_ori, - total_in_ori, 0); + /* + * Same code as stm32_cryp_irq_read_data(), but we want to store + * block value + */ + for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) + block[i] = stm32_cryp_read(cryp, CRYP_DOUT); + + scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize, + cryp->payload_out), 1); + cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, + cryp->payload_out); /* d) change mode back to AES GCM */ cfg &= ~CR_ALGO_MASK; @@ -1483,19 +1314,13 @@ static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp) stm32_cryp_write(cryp, CRYP_CR, cfg); /* f) write padded data */ - for (i = 0; i < AES_BLOCK_32; i++) { - if (cryp->total_in) - stm32_cryp_write(cryp, CRYP_DIN, tmp[i]); - else - stm32_cryp_write(cryp, CRYP_DIN, 0); - - cryp->total_in -= min_t(size_t, sizeof(u32), cryp->total_in); - } + for (i = 0; i < AES_BLOCK_32; i++) + stm32_cryp_write(cryp, CRYP_DIN, block[i]); /* g) Empty fifo out */ err = stm32_cryp_wait_output(cryp); if (err) { - dev_err(cryp->dev, "Timeout (write gcm header)\n"); + dev_err(cryp->dev, "Timeout (write gcm padded data)\n"); return stm32_cryp_finish_req(cryp, err); } @@ -1508,16 +1333,14 @@ static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp) static void stm32_cryp_irq_set_npblb(struct stm32_cryp *cryp) { - u32 cfg, payload_bytes; + u32 cfg; /* disable ip, set NPBLB and reneable ip */ cfg = stm32_cryp_read(cryp, CRYP_CR); cfg &= ~CR_CRYPEN; stm32_cryp_write(cryp, CRYP_CR, cfg); - payload_bytes = is_decrypt(cryp) ? cryp->total_in - cryp->authsize : - cryp->total_in; - cfg |= (cryp->hw_blocksize - payload_bytes) << CR_NBPBL_SHIFT; + cfg |= (cryp->hw_blocksize - cryp->payload_in) << CR_NBPBL_SHIFT; cfg |= CR_CRYPEN; stm32_cryp_write(cryp, CRYP_CR, cfg); } @@ -1526,13 +1349,11 @@ static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp) { int err = 0; u32 cfg, iv1tmp; - u32 cstmp1[AES_BLOCK_32], cstmp2[AES_BLOCK_32], tmp[AES_BLOCK_32]; - size_t last_total_out, total_in_ori = cryp->total_in; - struct scatterlist *out_sg_ori = cryp->out_sg; + u32 cstmp1[AES_BLOCK_32], cstmp2[AES_BLOCK_32]; + u32 block[AES_BLOCK_32] = {0}; unsigned int i; /* 'Special workaround' procedure described in the datasheet */ - cryp->flags |= FLG_CCM_PADDED_WA; /* a) disable ip */ stm32_cryp_write(cryp, CRYP_IMSCR, 0); @@ -1562,7 +1383,7 @@ static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp) /* b) pad and write the last block */ stm32_cryp_irq_write_block(cryp); - cryp->total_in = total_in_ori; + /* wait end of process */ err = stm32_cryp_wait_output(cryp); if (err) { dev_err(cryp->dev, "Timeout (wite ccm padded data)\n"); @@ -1570,13 +1391,16 @@ static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp) } /* c) get and store decrypted data */ - last_total_out = cryp->total_out; - stm32_cryp_irq_read_data(cryp); + /* + * Same code as stm32_cryp_irq_read_data(), but we want to store + * block value + */ + for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) + block[i] = stm32_cryp_read(cryp, CRYP_DOUT); - memset(tmp, 0, sizeof(tmp)); - scatterwalk_map_and_copy(tmp, out_sg_ori, - cryp->total_out_save - last_total_out, - last_total_out, 0); + scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize, + cryp->payload_out), 1); + cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, cryp->payload_out); /* d) Load again CRYP_CSGCMCCMxR */ for (i = 0; i < ARRAY_SIZE(cstmp2); i++) @@ -1593,10 +1417,10 @@ static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp) stm32_cryp_write(cryp, CRYP_CR, cfg); /* g) XOR and write padded data */ - for (i = 0; i < ARRAY_SIZE(tmp); i++) { - tmp[i] ^= cstmp1[i]; - tmp[i] ^= cstmp2[i]; - stm32_cryp_write(cryp, CRYP_DIN, tmp[i]); + for (i = 0; i < ARRAY_SIZE(block); i++) { + block[i] ^= cstmp1[i]; + block[i] ^= cstmp2[i]; + stm32_cryp_write(cryp, CRYP_DIN, block[i]); } /* h) wait for completion */ @@ -1610,30 +1434,34 @@ static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp) static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp) { - if (unlikely(!cryp->total_in)) { + if (unlikely(!cryp->payload_in)) { dev_warn(cryp->dev, "No more data to process\n"); return; } - if (unlikely(cryp->total_in < AES_BLOCK_SIZE && + if (unlikely(cryp->payload_in < AES_BLOCK_SIZE && (stm32_cryp_get_hw_mode(cryp) == CR_AES_GCM) && is_encrypt(cryp))) { /* Padding for AES GCM encryption */ - if (cryp->caps->padding_wa) + if (cryp->caps->padding_wa) { /* Special case 1 */ - return stm32_cryp_irq_write_gcm_padded_data(cryp); + stm32_cryp_irq_write_gcm_padded_data(cryp); + return; + } /* Setting padding bytes (NBBLB) */ stm32_cryp_irq_set_npblb(cryp); } - if (unlikely((cryp->total_in - cryp->authsize < AES_BLOCK_SIZE) && + if (unlikely((cryp->payload_in < AES_BLOCK_SIZE) && (stm32_cryp_get_hw_mode(cryp) == CR_AES_CCM) && is_decrypt(cryp))) { /* Padding for AES CCM decryption */ - if (cryp->caps->padding_wa) + if (cryp->caps->padding_wa) { /* Special case 2 */ - return stm32_cryp_irq_write_ccm_padded_data(cryp); + stm32_cryp_irq_write_ccm_padded_data(cryp); + return; + } /* Setting padding bytes (NBBLB) */ stm32_cryp_irq_set_npblb(cryp); @@ -1645,192 +1473,60 @@ static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp) stm32_cryp_irq_write_block(cryp); } -static void stm32_cryp_irq_write_gcm_header(struct stm32_cryp *cryp) +static void stm32_cryp_irq_write_gcmccm_header(struct stm32_cryp *cryp) { - int err; - unsigned int i, j; - u32 cfg, *src; + unsigned int i; + u32 block[AES_BLOCK_32] = {0}; + size_t written; - src = sg_virt(cryp->in_sg) + _walked_in; + written = min_t(size_t, AES_BLOCK_SIZE, cryp->header_in); - for (i = 0; i < AES_BLOCK_32; i++) { - stm32_cryp_write(cryp, CRYP_DIN, *src); + scatterwalk_copychunks(block, &cryp->in_walk, written, 0); + for (i = 0; i < AES_BLOCK_32; i++) + stm32_cryp_write(cryp, CRYP_DIN, block[i]); - src = stm32_cryp_next_in(cryp, src, sizeof(u32)); - cryp->total_in -= min_t(size_t, sizeof(u32), cryp->total_in); + cryp->header_in -= written; - /* Check if whole header written */ - if ((cryp->total_in_save - cryp->total_in) == - cryp->areq->assoclen) { - /* Write padding if needed */ - for (j = i + 1; j < AES_BLOCK_32; j++) - stm32_cryp_write(cryp, CRYP_DIN, 0); - - /* Wait for completion */ - err = stm32_cryp_wait_busy(cryp); - if (err) { - dev_err(cryp->dev, "Timeout (gcm header)\n"); - return stm32_cryp_finish_req(cryp, err); - } - - if (stm32_cryp_get_input_text_len(cryp)) { - /* Phase 3 : payload */ - cfg = stm32_cryp_read(cryp, CRYP_CR); - cfg &= ~CR_CRYPEN; - stm32_cryp_write(cryp, CRYP_CR, cfg); - - cfg &= ~CR_PH_MASK; - cfg |= CR_PH_PAYLOAD; - cfg |= CR_CRYPEN; - stm32_cryp_write(cryp, CRYP_CR, cfg); - } else { - /* Phase 4 : tag */ - stm32_cryp_write(cryp, CRYP_IMSCR, 0); - stm32_cryp_finish_req(cryp, 0); - } - - break; - } - - if (!cryp->total_in) - break; - } -} - -static void stm32_cryp_irq_write_ccm_header(struct stm32_cryp *cryp) -{ - int err; - unsigned int i = 0, j, k; - u32 alen, cfg, *src; - u8 d8[4]; - - src = sg_virt(cryp->in_sg) + _walked_in; - alen = cryp->areq->assoclen; - - if (!_walked_in) { - if (cryp->areq->assoclen <= 65280) { - /* Write first u32 of B1 */ - d8[0] = (alen >> 8) & 0xFF; - d8[1] = alen & 0xFF; - d8[2] = *((u8 *)src); - src = stm32_cryp_next_in(cryp, src, 1); - d8[3] = *((u8 *)src); - src = stm32_cryp_next_in(cryp, src, 1); - - stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8); - i++; - - cryp->total_in -= min_t(size_t, 2, cryp->total_in); - } else { - /* Build the two first u32 of B1 */ - d8[0] = 0xFF; - d8[1] = 0xFE; - d8[2] = alen & 0xFF000000; - d8[3] = alen & 0x00FF0000; - - stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8); - i++; - - d8[0] = alen & 0x0000FF00; - d8[1] = alen & 0x000000FF; - d8[2] = *((u8 *)src); - src = stm32_cryp_next_in(cryp, src, 1); - d8[3] = *((u8 *)src); - src = stm32_cryp_next_in(cryp, src, 1); - - stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8); - i++; - - cryp->total_in -= min_t(size_t, 2, cryp->total_in); - } - } - - /* Write next u32 */ - for (; i < AES_BLOCK_32; i++) { - /* Build an u32 */ - memset(d8, 0, sizeof(u32)); - for (k = 0; k < sizeof(u32); k++) { - d8[k] = *((u8 *)src); - src = stm32_cryp_next_in(cryp, src, 1); - - cryp->total_in -= min_t(size_t, 1, cryp->total_in); - if ((cryp->total_in_save - cryp->total_in) == alen) - break; - } - - stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8); - - if ((cryp->total_in_save - cryp->total_in) == alen) { - /* Write padding if needed */ - for (j = i + 1; j < AES_BLOCK_32; j++) - stm32_cryp_write(cryp, CRYP_DIN, 0); - - /* Wait for completion */ - err = stm32_cryp_wait_busy(cryp); - if (err) { - dev_err(cryp->dev, "Timeout (ccm header)\n"); - return stm32_cryp_finish_req(cryp, err); - } - - if (stm32_cryp_get_input_text_len(cryp)) { - /* Phase 3 : payload */ - cfg = stm32_cryp_read(cryp, CRYP_CR); - cfg &= ~CR_CRYPEN; - stm32_cryp_write(cryp, CRYP_CR, cfg); - - cfg &= ~CR_PH_MASK; - cfg |= CR_PH_PAYLOAD; - cfg |= CR_CRYPEN; - stm32_cryp_write(cryp, CRYP_CR, cfg); - } else { - /* Phase 4 : tag */ - stm32_cryp_write(cryp, CRYP_IMSCR, 0); - stm32_cryp_finish_req(cryp, 0); - } - - break; - } - } + stm32_crypt_gcmccm_end_header(cryp); } static irqreturn_t stm32_cryp_irq_thread(int irq, void *arg) { struct stm32_cryp *cryp = arg; u32 ph; + u32 it_mask = stm32_cryp_read(cryp, CRYP_IMSCR); if (cryp->irq_status & MISR_OUT) /* Output FIFO IRQ: read data */ - if (unlikely(stm32_cryp_irq_read_data(cryp))) { - /* All bytes processed, finish */ - stm32_cryp_write(cryp, CRYP_IMSCR, 0); - stm32_cryp_finish_req(cryp, 0); - return IRQ_HANDLED; - } + stm32_cryp_irq_read_data(cryp); if (cryp->irq_status & MISR_IN) { - if (is_gcm(cryp)) { + if (is_gcm(cryp) || is_ccm(cryp)) { ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK; if (unlikely(ph == CR_PH_HEADER)) /* Write Header */ - stm32_cryp_irq_write_gcm_header(cryp); - else - /* Input FIFO IRQ: write data */ - stm32_cryp_irq_write_data(cryp); - cryp->gcm_ctr++; - } else if (is_ccm(cryp)) { - ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK; - if (unlikely(ph == CR_PH_HEADER)) - /* Write Header */ - stm32_cryp_irq_write_ccm_header(cryp); + stm32_cryp_irq_write_gcmccm_header(cryp); else /* Input FIFO IRQ: write data */ stm32_cryp_irq_write_data(cryp); + if (is_gcm(cryp)) + cryp->gcm_ctr++; } else { /* Input FIFO IRQ: write data */ stm32_cryp_irq_write_data(cryp); } } + /* Mask useless interrupts */ + if (!cryp->payload_in && !cryp->header_in) + it_mask &= ~IMSCR_IN; + if (!cryp->payload_out) + it_mask &= ~IMSCR_OUT; + stm32_cryp_write(cryp, CRYP_IMSCR, it_mask); + + if (!cryp->payload_in && !cryp->header_in && !cryp->payload_out) + stm32_cryp_finish_req(cryp, 0); + return IRQ_HANDLED; } @@ -1851,7 +1547,7 @@ static struct skcipher_alg crypto_algs[] = { .base.cra_flags = CRYPTO_ALG_ASYNC, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), - .base.cra_alignmask = 0xf, + .base.cra_alignmask = 0, .base.cra_module = THIS_MODULE, .init = stm32_cryp_init_tfm, @@ -1868,7 +1564,7 @@ static struct skcipher_alg crypto_algs[] = { .base.cra_flags = CRYPTO_ALG_ASYNC, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), - .base.cra_alignmask = 0xf, + .base.cra_alignmask = 0, .base.cra_module = THIS_MODULE, .init = stm32_cryp_init_tfm, @@ -1886,7 +1582,7 @@ static struct skcipher_alg crypto_algs[] = { .base.cra_flags = CRYPTO_ALG_ASYNC, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), - .base.cra_alignmask = 0xf, + .base.cra_alignmask = 0, .base.cra_module = THIS_MODULE, .init = stm32_cryp_init_tfm, @@ -1904,7 +1600,7 @@ static struct skcipher_alg crypto_algs[] = { .base.cra_flags = CRYPTO_ALG_ASYNC, .base.cra_blocksize = DES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), - .base.cra_alignmask = 0xf, + .base.cra_alignmask = 0, .base.cra_module = THIS_MODULE, .init = stm32_cryp_init_tfm, @@ -1921,7 +1617,7 @@ static struct skcipher_alg crypto_algs[] = { .base.cra_flags = CRYPTO_ALG_ASYNC, .base.cra_blocksize = DES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), - .base.cra_alignmask = 0xf, + .base.cra_alignmask = 0, .base.cra_module = THIS_MODULE, .init = stm32_cryp_init_tfm, @@ -1939,7 +1635,7 @@ static struct skcipher_alg crypto_algs[] = { .base.cra_flags = CRYPTO_ALG_ASYNC, .base.cra_blocksize = DES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), - .base.cra_alignmask = 0xf, + .base.cra_alignmask = 0, .base.cra_module = THIS_MODULE, .init = stm32_cryp_init_tfm, @@ -1956,7 +1652,7 @@ static struct skcipher_alg crypto_algs[] = { .base.cra_flags = CRYPTO_ALG_ASYNC, .base.cra_blocksize = DES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), - .base.cra_alignmask = 0xf, + .base.cra_alignmask = 0, .base.cra_module = THIS_MODULE, .init = stm32_cryp_init_tfm, @@ -1986,7 +1682,7 @@ static struct aead_alg aead_algs[] = { .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct stm32_cryp_ctx), - .cra_alignmask = 0xf, + .cra_alignmask = 0, .cra_module = THIS_MODULE, }, }, @@ -2006,7 +1702,7 @@ static struct aead_alg aead_algs[] = { .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct stm32_cryp_ctx), - .cra_alignmask = 0xf, + .cra_alignmask = 0, .cra_module = THIS_MODULE, }, }, From 95fe2253cc1ad65d7f140d1944fa1b5d3c67abc0 Mon Sep 17 00:00:00 2001 From: Nicolas Toromanoff Date: Tue, 30 Nov 2021 08:55:01 +0100 Subject: [PATCH 070/145] crypto: stm32/cryp - reorder hw initialization The CRYP IP checks the written key depending of the configuration, it's safer to write the whole configuration to hardware then the key to avoid unexpected key rejection. Signed-off-by: Nicolas Toromanoff Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32-cryp.c | 43 ++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index faae7ad262ac..c6640f616d8d 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -232,6 +232,11 @@ static inline int stm32_cryp_wait_busy(struct stm32_cryp *cryp) !(status & SR_BUSY), 10, 100000); } +static inline void stm32_cryp_enable(struct stm32_cryp *cryp) +{ + writel_relaxed(readl_relaxed(cryp->regs + CRYP_CR) | CR_CRYPEN, cryp->regs + CRYP_CR); +} + static inline int stm32_cryp_wait_enable(struct stm32_cryp *cryp) { u32 status; @@ -534,9 +539,6 @@ static int stm32_cryp_hw_init(struct stm32_cryp *cryp) /* Disable interrupt */ stm32_cryp_write(cryp, CRYP_IMSCR, 0); - /* Set key */ - stm32_cryp_hw_write_key(cryp); - /* Set configuration */ cfg = CR_DATA8 | CR_FFLUSH; @@ -562,24 +564,37 @@ static int stm32_cryp_hw_init(struct stm32_cryp *cryp) /* AES ECB/CBC decrypt: run key preparation first */ if (is_decrypt(cryp) && ((hw_mode == CR_AES_ECB) || (hw_mode == CR_AES_CBC))) { - stm32_cryp_write(cryp, CRYP_CR, cfg | CR_AES_KP | CR_CRYPEN); + /* Configure in key preparation mode */ + stm32_cryp_write(cryp, CRYP_CR, cfg | CR_AES_KP); + /* Set key only after full configuration done */ + stm32_cryp_hw_write_key(cryp); + + /* Start prepare key */ + stm32_cryp_enable(cryp); /* Wait for end of processing */ ret = stm32_cryp_wait_busy(cryp); if (ret) { dev_err(cryp->dev, "Timeout (key preparation)\n"); return ret; } + + cfg |= hw_mode | CR_DEC_NOT_ENC; + + /* Apply updated config (Decrypt + algo) and flush */ + stm32_cryp_write(cryp, CRYP_CR, cfg); + } else { + cfg |= hw_mode; + if (is_decrypt(cryp)) + cfg |= CR_DEC_NOT_ENC; + + /* Apply config and flush */ + stm32_cryp_write(cryp, CRYP_CR, cfg); + + /* Set key only after configuration done */ + stm32_cryp_hw_write_key(cryp); } - cfg |= hw_mode; - - if (is_decrypt(cryp)) - cfg |= CR_DEC_NOT_ENC; - - /* Apply config and flush (valid when CRYPEN = 0) */ - stm32_cryp_write(cryp, CRYP_CR, cfg); - switch (hw_mode) { case CR_AES_GCM: case CR_AES_CCM: @@ -606,9 +621,7 @@ static int stm32_cryp_hw_init(struct stm32_cryp *cryp) } /* Enable now */ - cfg |= CR_CRYPEN; - - stm32_cryp_write(cryp, CRYP_CR, cfg); + stm32_cryp_enable(cryp); return 0; } From 8f7977284331d0b0f210efa98a5d3fdcb2a65dd3 Mon Sep 17 00:00:00 2001 From: Nicolai Stange Date: Tue, 30 Nov 2021 15:10:07 +0100 Subject: [PATCH 071/145] crypto: drbg - ignore jitterentropy errors if not in FIPS mode A subsequent patch will make the jitterentropy RNG to unconditionally report health test errors back to callers, independent of whether fips_enabled is set or not. The DRBG needs access to a functional jitterentropy instance only in FIPS mode (because it's the only SP800-90B compliant entropy source as it currently stands). Thus, it is perfectly fine for the DRBGs to obtain entropy from the jitterentropy source only on a best effort basis if fips_enabled is off. Make the DRBGs to ignore jitterentropy failures if fips_enabled is not set. Signed-off-by: Nicolai Stange Reviewed-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/drbg.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crypto/drbg.c b/crypto/drbg.c index 5977a72afb03..177983b6ae38 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -1193,11 +1193,14 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, pr_devel("DRBG: (re)seeding with %u bytes of entropy\n", entropylen); } else { - /* Get seed from Jitter RNG */ + /* + * Get seed from Jitter RNG, failures are + * fatal only in FIPS mode. + */ ret = crypto_rng_get_bytes(drbg->jent, entropy + entropylen, entropylen); - if (ret) { + if (fips_enabled && ret) { pr_devel("DRBG: jent failed with %d\n", ret); /* From b454fb702515276041b701a4eda468b6cd6b384d Mon Sep 17 00:00:00 2001 From: Nicolai Stange Date: Tue, 30 Nov 2021 15:10:08 +0100 Subject: [PATCH 072/145] crypto: jitter - don't limit ->health_failure check to FIPS mode The jitterentropy's Repetition Count Test (RCT) as well as the Adaptive Proportion Test (APT) are run unconditionally on any collected samples. However, their result, i.e. ->health_failure, will only get checked if fips_enabled is set, c.f. the jent_health_failure() wrapper. I would argue that a RCT or APT failure indicates that something's seriously off and that this should always be reported as an error, independently of whether FIPS mode is enabled or not: it should be up to callers whether or not and how to handle jitterentropy failures. Make jent_health_failure() to unconditionally return ->health_failure, independent of whether fips_enabled is set. Note that fips_enabled isn't accessed from the jitterentropy code anymore now. Remove the linux/fips.h include as well as the jent_fips_enabled() wrapper. Signed-off-by: Nicolai Stange Reviewed-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/jitterentropy-kcapi.c | 6 ------ crypto/jitterentropy.c | 4 ---- crypto/jitterentropy.h | 1 - 3 files changed, 11 deletions(-) diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c index e8a4165a1874..2d115bec15ae 100644 --- a/crypto/jitterentropy-kcapi.c +++ b/crypto/jitterentropy-kcapi.c @@ -40,7 +40,6 @@ #include #include #include -#include #include #include @@ -60,11 +59,6 @@ void jent_zfree(void *ptr) kfree_sensitive(ptr); } -int jent_fips_enabled(void) -{ - return fips_enabled; -} - void jent_panic(char *s) { panic("%s", s); diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c index 788d90749715..24e087c3f526 100644 --- a/crypto/jitterentropy.c +++ b/crypto/jitterentropy.c @@ -298,10 +298,6 @@ static int jent_stuck(struct rand_data *ec, __u64 current_delta) */ static int jent_health_failure(struct rand_data *ec) { - /* Test is only enabled in FIPS mode */ - if (!jent_fips_enabled()) - return 0; - return ec->health_failure; } diff --git a/crypto/jitterentropy.h b/crypto/jitterentropy.h index c83fff32d130..b7397b617ef0 100644 --- a/crypto/jitterentropy.h +++ b/crypto/jitterentropy.h @@ -2,7 +2,6 @@ extern void *jent_zalloc(unsigned int len); extern void jent_zfree(void *ptr); -extern int jent_fips_enabled(void); extern void jent_panic(char *s); extern void jent_memcpy(void *dest, const void *src, unsigned int n); extern void jent_get_nstime(__u64 *out); From 710ce4b88f9a93a6b2c2267e8f27ba65af3cb6ac Mon Sep 17 00:00:00 2001 From: Nicolai Stange Date: Tue, 30 Nov 2021 15:10:09 +0100 Subject: [PATCH 073/145] crypto: jitter - quit sample collection loop upon RCT failure The jitterentropy collection loop in jent_gen_entropy() can in principle run indefinitely without making any progress if it only receives stuck measurements as determined by jent_stuck(). After 31 consecutive stuck samples, the Repetition Count Test (RCT) would fail anyway and the jitterentropy RNG instances moved into ->health_failure == 1 state. jent_gen_entropy()'s caller, jent_read_entropy() would then check for this ->health_failure condition and return an error if found set. It follows that there's absolutely no point in continuing the collection loop in jent_gen_entropy() once the RCT has failed. Make the jitterentropy collection loop more robust by terminating it upon jent_health_failure() so that it won't continue to run indefinitely without making any progress. Signed-off-by: Nicolai Stange Reviewed-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/jitterentropy.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c index 24e087c3f526..8f5283f28ed3 100644 --- a/crypto/jitterentropy.c +++ b/crypto/jitterentropy.c @@ -547,7 +547,7 @@ static void jent_gen_entropy(struct rand_data *ec) /* priming of the ->prev_time value */ jent_measure_jitter(ec); - while (1) { + while (!jent_health_failure(ec)) { /* If a stuck measurement is received, repeat measurement */ if (jent_measure_jitter(ec)) continue; From 3d6b661330a7954d8136df98160d525eb04dcd6a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 1 Dec 2021 17:30:41 +1100 Subject: [PATCH 074/145] crypto: stm32 - Revert broken pm_runtime_resume_and_get changes We should not call pm_runtime_resume_and_get where the reference count is expected to be incremented unconditionally. This patch reverts these calls to the original unconditional get_sync call. Reported-by: Heiner Kallweit Fixes: 747bf30fd944 ("crypto: stm32/cryp - Fix PM reference leak...") Fixes: 1cb3ad701970 ("crypto: stm32/hash - Fix PM reference leak...") Signed-off-by: Herbert Xu Acked-by: Rafael J. Wysocki Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32-cryp.c | 3 ++- drivers/crypto/stm32/stm32-hash.c | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index c6640f616d8d..59ef541123ae 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -534,7 +534,8 @@ static int stm32_cryp_hw_init(struct stm32_cryp *cryp) { int ret; u32 cfg, hw_mode; - pm_runtime_resume_and_get(cryp->dev); + + pm_runtime_get_sync(cryp->dev); /* Disable interrupt */ stm32_cryp_write(cryp, CRYP_IMSCR, 0); diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c index 389de9e3302d..d33006d43f76 100644 --- a/drivers/crypto/stm32/stm32-hash.c +++ b/drivers/crypto/stm32/stm32-hash.c @@ -813,7 +813,7 @@ static void stm32_hash_finish_req(struct ahash_request *req, int err) static int stm32_hash_hw_init(struct stm32_hash_dev *hdev, struct stm32_hash_request_ctx *rctx) { - pm_runtime_resume_and_get(hdev->dev); + pm_runtime_get_sync(hdev->dev); if (!(HASH_FLAGS_INIT & hdev->flags)) { stm32_hash_write(hdev, HASH_CR, HASH_CR_INIT); @@ -962,7 +962,7 @@ static int stm32_hash_export(struct ahash_request *req, void *out) u32 *preg; unsigned int i; - pm_runtime_resume_and_get(hdev->dev); + pm_runtime_get_sync(hdev->dev); while ((stm32_hash_read(hdev, HASH_SR) & HASH_SR_BUSY)) cpu_relax(); @@ -1000,7 +1000,7 @@ static int stm32_hash_import(struct ahash_request *req, const void *in) preg = rctx->hw_context; - pm_runtime_resume_and_get(hdev->dev); + pm_runtime_get_sync(hdev->dev); stm32_hash_write(hdev, HASH_IMR, *preg++); stm32_hash_write(hdev, HASH_STR, *preg++); From d9d7749773e8895aaedc11f1d2885a9eee5c73ac Mon Sep 17 00:00:00 2001 From: Srujana Challa Date: Wed, 1 Dec 2021 14:32:00 +0530 Subject: [PATCH 075/145] crypto: octeontx2 - add apis for custom engine groups Octeon TX2 CPT has three type of engines to handle symmetric, asymmetric and ipsec specific workload. For better utilization, these engines can be grouped to custom groups at runtime. This patch adds APIs to create and delete custom CPT engine groups. Signed-off-by: Srujana Challa Signed-off-by: Shijith Thotton Signed-off-by: Herbert Xu --- .../marvell/octeontx2/otx2_cptpf_ucode.c | 322 +++++++++++++++++- .../marvell/octeontx2/otx2_cptpf_ucode.h | 7 +- 2 files changed, 322 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c index dff34b3ec09e..57307eac541c 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c @@ -1110,18 +1110,19 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf, struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { {0} }; struct pci_dev *pdev = cptpf->pdev; struct fw_info_t fw_info; - int ret; + int ret = 0; + mutex_lock(&eng_grps->lock); /* * We don't create engine groups if it was already * made (when user enabled VFs for the first time) */ if (eng_grps->is_grps_created) - return 0; + goto unlock; ret = cpt_ucode_load_fw(pdev, &fw_info); if (ret) - return ret; + goto unlock; /* * Create engine group with SE engines for kernel @@ -1186,7 +1187,7 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf, cpt_ucode_release_fw(&fw_info); if (is_dev_otx2(pdev)) - return 0; + goto unlock; /* * Configure engine group mask to allow context prefetching * for the groups. @@ -1201,12 +1202,15 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf, */ otx2_cpt_write_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTX_FLUSH_TIMER, CTX_FLUSH_TIMER_CNT, BLKADDR_CPT0); + mutex_unlock(&eng_grps->lock); return 0; delete_eng_grp: delete_engine_grps(pdev, eng_grps); release_fw: cpt_ucode_release_fw(&fw_info); +unlock: + mutex_unlock(&eng_grps->lock); return ret; } @@ -1286,6 +1290,7 @@ void otx2_cpt_cleanup_eng_grps(struct pci_dev *pdev, struct otx2_cpt_eng_grp_info *grp; int i, j; + mutex_lock(&eng_grps->lock); delete_engine_grps(pdev, eng_grps); /* Release memory */ for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { @@ -1295,6 +1300,7 @@ void otx2_cpt_cleanup_eng_grps(struct pci_dev *pdev, grp->engs[j].bmap = NULL; } } + mutex_unlock(&eng_grps->lock); } int otx2_cpt_init_eng_grps(struct pci_dev *pdev, @@ -1303,6 +1309,7 @@ int otx2_cpt_init_eng_grps(struct pci_dev *pdev, struct otx2_cpt_eng_grp_info *grp; int i, j, ret; + mutex_init(&eng_grps->lock); eng_grps->obj = pci_get_drvdata(pdev); eng_grps->avail.se_cnt = eng_grps->avail.max_se_cnt; eng_grps->avail.ie_cnt = eng_grps->avail.max_ie_cnt; @@ -1349,11 +1356,14 @@ static int create_eng_caps_discovery_grps(struct pci_dev *pdev, struct fw_info_t fw_info; int ret; + mutex_lock(&eng_grps->lock); ret = cpt_ucode_load_fw(pdev, &fw_info); - if (ret) + if (ret) { + mutex_unlock(&eng_grps->lock); return ret; + } - uc_info[0] = get_ucode(&fw_info, OTX2_CPT_SE_TYPES); + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_AE_TYPES); if (uc_info[0] == NULL) { dev_err(&pdev->dev, "Unable to find firmware for AE\n"); ret = -EINVAL; @@ -1396,12 +1406,14 @@ static int create_eng_caps_discovery_grps(struct pci_dev *pdev, goto delete_eng_grp; cpt_ucode_release_fw(&fw_info); + mutex_unlock(&eng_grps->lock); return 0; delete_eng_grp: delete_engine_grps(pdev, eng_grps); release_fw: cpt_ucode_release_fw(&fw_info); + mutex_unlock(&eng_grps->lock); return ret; } @@ -1501,3 +1513,301 @@ delete_grps: return ret; } + +static void swap_engines(struct otx2_cpt_engines *engsl, + struct otx2_cpt_engines *engsr) +{ + struct otx2_cpt_engines engs; + + engs = *engsl; + *engsl = *engsr; + *engsr = engs; +} + +int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf, + struct devlink_param_gset_ctx *ctx) +{ + struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { { 0 } }; + struct otx2_cpt_uc_info_t *uc_info[OTX2_CPT_MAX_ETYPES_PER_GRP] = {}; + struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps; + char *ucode_filename[OTX2_CPT_MAX_ETYPES_PER_GRP]; + char tmp_buf[OTX2_CPT_NAME_LENGTH] = { 0 }; + struct device *dev = &cptpf->pdev->dev; + char *start, *val, *err_msg, *tmp; + int grp_idx = 0, ret = -EINVAL; + bool has_se, has_ie, has_ae; + struct fw_info_t fw_info; + int ucode_idx = 0; + + if (!eng_grps->is_grps_created) { + dev_err(dev, "Not allowed before creating the default groups\n"); + return -EINVAL; + } + err_msg = "Invalid engine group format"; + strscpy(tmp_buf, ctx->val.vstr, strlen(ctx->val.vstr) + 1); + start = tmp_buf; + + has_se = has_ie = has_ae = false; + + for (;;) { + val = strsep(&start, ";"); + if (!val) + break; + val = strim(val); + if (!*val) + continue; + + if (!strncasecmp(val, "se", 2) && strchr(val, ':')) { + if (has_se || ucode_idx) + goto err_print; + tmp = strim(strsep(&val, ":")); + if (!val) + goto err_print; + if (strlen(tmp) != 2) + goto err_print; + if (kstrtoint(strim(val), 10, &engs[grp_idx].count)) + goto err_print; + engs[grp_idx++].type = OTX2_CPT_SE_TYPES; + has_se = true; + } else if (!strncasecmp(val, "ae", 2) && strchr(val, ':')) { + if (has_ae || ucode_idx) + goto err_print; + tmp = strim(strsep(&val, ":")); + if (!val) + goto err_print; + if (strlen(tmp) != 2) + goto err_print; + if (kstrtoint(strim(val), 10, &engs[grp_idx].count)) + goto err_print; + engs[grp_idx++].type = OTX2_CPT_AE_TYPES; + has_ae = true; + } else if (!strncasecmp(val, "ie", 2) && strchr(val, ':')) { + if (has_ie || ucode_idx) + goto err_print; + tmp = strim(strsep(&val, ":")); + if (!val) + goto err_print; + if (strlen(tmp) != 2) + goto err_print; + if (kstrtoint(strim(val), 10, &engs[grp_idx].count)) + goto err_print; + engs[grp_idx++].type = OTX2_CPT_IE_TYPES; + has_ie = true; + } else { + if (ucode_idx > 1) + goto err_print; + if (!strlen(val)) + goto err_print; + if (strnstr(val, " ", strlen(val))) + goto err_print; + ucode_filename[ucode_idx++] = val; + } + } + + /* Validate input parameters */ + if (!(grp_idx && ucode_idx)) + goto err_print; + + if (ucode_idx > 1 && grp_idx < 2) + goto err_print; + + if (grp_idx > OTX2_CPT_MAX_ETYPES_PER_GRP) { + err_msg = "Error max 2 engine types can be attached"; + goto err_print; + } + + if (grp_idx > 1) { + if ((engs[0].type + engs[1].type) != + (OTX2_CPT_SE_TYPES + OTX2_CPT_IE_TYPES)) { + err_msg = "Only combination of SE+IE engines is allowed"; + goto err_print; + } + /* Keep SE engines at zero index */ + if (engs[1].type == OTX2_CPT_SE_TYPES) + swap_engines(&engs[0], &engs[1]); + } + mutex_lock(&eng_grps->lock); + + if (cptpf->enabled_vfs) { + dev_err(dev, "Disable VFs before modifying engine groups\n"); + ret = -EACCES; + goto err_unlock; + } + INIT_LIST_HEAD(&fw_info.ucodes); + ret = load_fw(dev, &fw_info, ucode_filename[0]); + if (ret) { + dev_err(dev, "Unable to load firmware %s\n", ucode_filename[0]); + goto err_unlock; + } + if (ucode_idx > 1) { + ret = load_fw(dev, &fw_info, ucode_filename[1]); + if (ret) { + dev_err(dev, "Unable to load firmware %s\n", + ucode_filename[1]); + goto release_fw; + } + } + uc_info[0] = get_ucode(&fw_info, engs[0].type); + if (uc_info[0] == NULL) { + dev_err(dev, "Unable to find firmware for %s\n", + get_eng_type_str(engs[0].type)); + ret = -EINVAL; + goto release_fw; + } + if (ucode_idx > 1) { + uc_info[1] = get_ucode(&fw_info, engs[1].type); + if (uc_info[1] == NULL) { + dev_err(dev, "Unable to find firmware for %s\n", + get_eng_type_str(engs[1].type)); + ret = -EINVAL; + goto release_fw; + } + } + ret = create_engine_group(dev, eng_grps, engs, grp_idx, + (void **)uc_info, 1); + +release_fw: + cpt_ucode_release_fw(&fw_info); +err_unlock: + mutex_unlock(&eng_grps->lock); + return ret; +err_print: + dev_err(dev, "%s\n", err_msg); + return ret; +} + +int otx2_cpt_dl_custom_egrp_delete(struct otx2_cptpf_dev *cptpf, + struct devlink_param_gset_ctx *ctx) +{ + struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps; + struct device *dev = &cptpf->pdev->dev; + char *tmp, *err_msg; + int egrp; + int ret; + + err_msg = "Invalid input string format(ex: egrp:0)"; + if (strncasecmp(ctx->val.vstr, "egrp", 4)) + goto err_print; + tmp = ctx->val.vstr; + strsep(&tmp, ":"); + if (!tmp) + goto err_print; + if (kstrtoint(tmp, 10, &egrp)) + goto err_print; + + if (egrp >= OTX2_CPT_MAX_ENGINE_GROUPS) { + dev_err(dev, "Invalid engine group %d", egrp); + return -EINVAL; + } + if (!eng_grps->grp[egrp].is_enabled) { + dev_err(dev, "Error engine_group%d is not configured", egrp); + return -EINVAL; + } + mutex_lock(&eng_grps->lock); + ret = delete_engine_group(dev, &eng_grps->grp[egrp]); + mutex_unlock(&eng_grps->lock); + + return ret; + +err_print: + dev_err(dev, "%s\n", err_msg); + return -EINVAL; +} + +static void get_engs_info(struct otx2_cpt_eng_grp_info *eng_grp, char *buf, + int size, int idx) +{ + struct otx2_cpt_engs_rsvd *mirrored_engs = NULL; + struct otx2_cpt_engs_rsvd *engs; + int len, i; + + buf[0] = '\0'; + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + engs = &eng_grp->engs[i]; + if (!engs->type) + continue; + if (idx != -1 && idx != i) + continue; + + if (eng_grp->mirror.is_ena) + mirrored_engs = find_engines_by_type( + &eng_grp->g->grp[eng_grp->mirror.idx], + engs->type); + if (i > 0 && idx == -1) { + len = strlen(buf); + scnprintf(buf + len, size - len, ", "); + } + + len = strlen(buf); + scnprintf(buf + len, size - len, "%d %s ", + mirrored_engs ? engs->count + mirrored_engs->count : + engs->count, + get_eng_type_str(engs->type)); + if (mirrored_engs) { + len = strlen(buf); + scnprintf(buf + len, size - len, + "(%d shared with engine_group%d) ", + engs->count <= 0 ? + engs->count + mirrored_engs->count : + mirrored_engs->count, + eng_grp->mirror.idx); + } + } +} + +void otx2_cpt_print_uc_dbg_info(struct otx2_cptpf_dev *cptpf) +{ + struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps; + struct otx2_cpt_eng_grp_info *mirrored_grp; + char engs_info[2 * OTX2_CPT_NAME_LENGTH]; + struct otx2_cpt_eng_grp_info *grp; + struct otx2_cpt_engs_rsvd *engs; + u32 mask[4]; + int i, j; + + pr_debug("Engine groups global info"); + pr_debug("max SE %d, max IE %d, max AE %d", eng_grps->avail.max_se_cnt, + eng_grps->avail.max_ie_cnt, eng_grps->avail.max_ae_cnt); + pr_debug("free SE %d", eng_grps->avail.se_cnt); + pr_debug("free IE %d", eng_grps->avail.ie_cnt); + pr_debug("free AE %d", eng_grps->avail.ae_cnt); + + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { + grp = &eng_grps->grp[i]; + pr_debug("engine_group%d, state %s", i, + grp->is_enabled ? "enabled" : "disabled"); + if (grp->is_enabled) { + mirrored_grp = &eng_grps->grp[grp->mirror.idx]; + pr_debug("Ucode0 filename %s, version %s", + grp->mirror.is_ena ? + mirrored_grp->ucode[0].filename : + grp->ucode[0].filename, + grp->mirror.is_ena ? + mirrored_grp->ucode[0].ver_str : + grp->ucode[0].ver_str); + if (is_2nd_ucode_used(grp)) + pr_debug("Ucode1 filename %s, version %s", + grp->ucode[1].filename, + grp->ucode[1].ver_str); + } + + for (j = 0; j < OTX2_CPT_MAX_ETYPES_PER_GRP; j++) { + engs = &grp->engs[j]; + if (engs->type) { + get_engs_info(grp, engs_info, + 2 * OTX2_CPT_NAME_LENGTH, j); + pr_debug("Slot%d: %s", j, engs_info); + bitmap_to_arr32(mask, engs->bmap, + eng_grps->engs_num); + if (is_dev_otx2(cptpf->pdev)) + pr_debug("Mask: %8.8x %8.8x %8.8x %8.8x", + mask[3], mask[2], mask[1], + mask[0]); + else + pr_debug("Mask: %8.8x %8.8x %8.8x %8.8x %8.8x", + mask[4], mask[3], mask[2], mask[1], + mask[0]); + } + } + } +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h index fe019ab730b2..8f4d4e5f531a 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h @@ -143,6 +143,7 @@ struct otx2_cpt_eng_grp_info { }; struct otx2_cpt_eng_grps { + struct mutex lock; struct otx2_cpt_eng_grp_info grp[OTX2_CPT_MAX_ENGINE_GROUPS]; struct otx2_cpt_engs_available avail; void *obj; /* device specific data */ @@ -160,5 +161,9 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf, int otx2_cpt_disable_all_cores(struct otx2_cptpf_dev *cptpf); int otx2_cpt_get_eng_grp(struct otx2_cpt_eng_grps *eng_grps, int eng_type); int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf); - +int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf, + struct devlink_param_gset_ctx *ctx); +int otx2_cpt_dl_custom_egrp_delete(struct otx2_cptpf_dev *cptpf, + struct devlink_param_gset_ctx *ctx); +void otx2_cpt_print_uc_dbg_info(struct otx2_cptpf_dev *cptpf); #endif /* __OTX2_CPTPF_UCODE_H */ From fed8f4d5f9469a4b59e501bf9b6dcab484c40047 Mon Sep 17 00:00:00 2001 From: Srujana Challa Date: Wed, 1 Dec 2021 14:32:01 +0530 Subject: [PATCH 076/145] crypto: octeontx2 - parameters for custom engine groups Added devlink parameters to create and delete custom CPT engine groups. Example: devlink dev param set pci/0002:20:00.0 name egrp_create value \ "se:32;se.out" cmode runtime devlink dev param set pci/0002:20:00.0 name egrp_delete value \ "egrp:1" cmode runtime Signed-off-by: Srujana Challa Signed-off-by: Shijith Thotton Signed-off-by: Herbert Xu --- drivers/crypto/marvell/octeontx2/Makefile | 2 +- .../marvell/octeontx2/otx2_cpt_common.h | 1 + .../marvell/octeontx2/otx2_cpt_devlink.c | 108 ++++++++++++++++++ .../marvell/octeontx2/otx2_cpt_devlink.h | 20 ++++ drivers/crypto/marvell/octeontx2/otx2_cptpf.h | 3 + .../marvell/octeontx2/otx2_cptpf_main.c | 9 ++ 6 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.h diff --git a/drivers/crypto/marvell/octeontx2/Makefile b/drivers/crypto/marvell/octeontx2/Makefile index c242d22008c3..965297e96954 100644 --- a/drivers/crypto/marvell/octeontx2/Makefile +++ b/drivers/crypto/marvell/octeontx2/Makefile @@ -3,7 +3,7 @@ obj-$(CONFIG_CRYPTO_DEV_OCTEONTX2_CPT) += rvu_cptpf.o rvu_cptvf.o rvu_cptpf-objs := otx2_cptpf_main.o otx2_cptpf_mbox.o \ otx2_cpt_mbox_common.o otx2_cptpf_ucode.o otx2_cptlf.o \ - cn10k_cpt.o + cn10k_cpt.o otx2_cpt_devlink.o rvu_cptvf-objs := otx2_cptvf_main.o otx2_cptvf_mbox.o otx2_cptlf.o \ otx2_cpt_mbox_common.o otx2_cptvf_reqmgr.o \ otx2_cptvf_algs.o cn10k_cpt.o diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h index c5445b05f53c..fb56824cb0a6 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h @@ -10,6 +10,7 @@ #include #include #include +#include #include "otx2_cpt_hw_types.h" #include "rvu.h" #include "mbox.h" diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c new file mode 100644 index 000000000000..bb02e0db3615 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2021 Marvell. */ + +#include "otx2_cpt_devlink.h" + +static int otx2_cpt_dl_egrp_create(struct devlink *dl, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl); + struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf; + + return otx2_cpt_dl_custom_egrp_create(cptpf, ctx); +} + +static int otx2_cpt_dl_egrp_delete(struct devlink *dl, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl); + struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf; + + return otx2_cpt_dl_custom_egrp_delete(cptpf, ctx); +} + +static int otx2_cpt_dl_uc_info(struct devlink *dl, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl); + struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf; + + otx2_cpt_print_uc_dbg_info(cptpf); + + return 0; +} + +enum otx2_cpt_dl_param_id { + OTX2_CPT_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX, + OTX2_CPT_DEVLINK_PARAM_ID_EGRP_CREATE, + OTX2_CPT_DEVLINK_PARAM_ID_EGRP_DELETE, +}; + +static const struct devlink_param otx2_cpt_dl_params[] = { + DEVLINK_PARAM_DRIVER(OTX2_CPT_DEVLINK_PARAM_ID_EGRP_CREATE, + "egrp_create", DEVLINK_PARAM_TYPE_STRING, + BIT(DEVLINK_PARAM_CMODE_RUNTIME), + otx2_cpt_dl_uc_info, otx2_cpt_dl_egrp_create, + NULL), + DEVLINK_PARAM_DRIVER(OTX2_CPT_DEVLINK_PARAM_ID_EGRP_DELETE, + "egrp_delete", DEVLINK_PARAM_TYPE_STRING, + BIT(DEVLINK_PARAM_CMODE_RUNTIME), + otx2_cpt_dl_uc_info, otx2_cpt_dl_egrp_delete, + NULL), +}; + +static int otx2_cpt_devlink_info_get(struct devlink *devlink, + struct devlink_info_req *req, + struct netlink_ext_ack *extack) +{ + return devlink_info_driver_name_put(req, "rvu_cptpf"); +} + +static const struct devlink_ops otx2_cpt_devlink_ops = { + .info_get = otx2_cpt_devlink_info_get, +}; + +int otx2_cpt_register_dl(struct otx2_cptpf_dev *cptpf) +{ + struct device *dev = &cptpf->pdev->dev; + struct otx2_cpt_devlink *cpt_dl; + struct devlink *dl; + int ret; + + dl = devlink_alloc(&otx2_cpt_devlink_ops, + sizeof(struct otx2_cpt_devlink), dev); + if (!dl) { + dev_warn(dev, "devlink_alloc failed\n"); + return -ENOMEM; + } + + cpt_dl = devlink_priv(dl); + cpt_dl->dl = dl; + cpt_dl->cptpf = cptpf; + cptpf->dl = dl; + ret = devlink_params_register(dl, otx2_cpt_dl_params, + ARRAY_SIZE(otx2_cpt_dl_params)); + if (ret) { + dev_err(dev, "devlink params register failed with error %d", + ret); + devlink_free(dl); + return ret; + } + + devlink_register(dl); + + return 0; +} + +void otx2_cpt_unregister_dl(struct otx2_cptpf_dev *cptpf) +{ + struct devlink *dl = cptpf->dl; + + if (!dl) + return; + + devlink_unregister(dl); + devlink_params_unregister(dl, otx2_cpt_dl_params, + ARRAY_SIZE(otx2_cpt_dl_params)); + devlink_free(dl); +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.h new file mode 100644 index 000000000000..8b7d88c5d519 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2021 Marvell. + */ + +#ifndef __OTX2_CPT_DEVLINK_H +#define __OTX2_CPT_DEVLINK_H + +#include "otx2_cpt_common.h" +#include "otx2_cptpf.h" + +struct otx2_cpt_devlink { + struct devlink *dl; + struct otx2_cptpf_dev *cptpf; +}; + +/* Devlink APIs */ +int otx2_cpt_register_dl(struct otx2_cptpf_dev *cptpf); +void otx2_cpt_unregister_dl(struct otx2_cptpf_dev *cptpf); + +#endif /* __OTX2_CPT_DEVLINK_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf.h b/drivers/crypto/marvell/octeontx2/otx2_cptpf.h index 5ebba86c65d9..05b2d9c650e1 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf.h @@ -53,6 +53,9 @@ struct otx2_cptpf_dev { u8 enabled_vfs; /* Number of enabled VFs */ u8 kvf_limits; /* Kernel crypto limits */ bool has_cpt1; + + /* Devlink */ + struct devlink *dl; }; irqreturn_t otx2_cptpf_afpf_mbox_intr(int irq, void *arg); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c index be1ad55a208f..1720a5bb7016 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c @@ -4,6 +4,7 @@ #include #include "otx2_cpt_hw_types.h" #include "otx2_cpt_common.h" +#include "otx2_cpt_devlink.h" #include "otx2_cptpf_ucode.h" #include "otx2_cptpf.h" #include "cn10k_cpt.h" @@ -766,8 +767,15 @@ static int otx2_cptpf_probe(struct pci_dev *pdev, err = sysfs_create_group(&dev->kobj, &cptpf_sysfs_group); if (err) goto cleanup_eng_grps; + + err = otx2_cpt_register_dl(cptpf); + if (err) + goto sysfs_grp_del; + return 0; +sysfs_grp_del: + sysfs_remove_group(&dev->kobj, &cptpf_sysfs_group); cleanup_eng_grps: otx2_cpt_cleanup_eng_grps(pdev, &cptpf->eng_grps); unregister_intr: @@ -787,6 +795,7 @@ static void otx2_cptpf_remove(struct pci_dev *pdev) return; cptpf_sriov_disable(pdev); + otx2_cpt_unregister_dl(cptpf); /* Delete sysfs entry created for kernel VF limits */ sysfs_remove_group(&pdev->dev.kobj, &cptpf_sysfs_group); /* Cleanup engine groups */ From 3f9dd4c802b96626e869b2d29c8e401dabadd23e Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 4 Dec 2021 18:43:01 +0800 Subject: [PATCH 077/145] crypto: hisilicon/qm - fix incorrect return value of hisi_qm_resume() When hisi_qm_resume() returns 0, it indicates that the device has started successfully. If the device fails to start, hisi_qm_resume() needs to return the actual error code to the caller instead of 0. Fixes: d7ea53395b72 ("crypto: hisilicon - add runtime PM ops") Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index ad706ccfe91a..896255aec87f 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -6071,7 +6071,7 @@ int hisi_qm_resume(struct device *dev) if (ret) pci_err(pdev, "failed to start qm(%d)\n", ret); - return 0; + return ret; } EXPORT_SYMBOL_GPL(hisi_qm_resume); From 015e42c85f1ec33894579626128b385d4780e883 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Tue, 7 Dec 2021 18:58:09 +0000 Subject: [PATCH 078/145] crypto: x86/des3 - remove redundant assignment of variable nbytes The variable nbytes is being assigned a value that is never read, it is being re-assigned in the following statement. The assignment is redundant and can be removed. Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- arch/x86/crypto/des3_ede_glue.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/crypto/des3_ede_glue.c b/arch/x86/crypto/des3_ede_glue.c index e7cb68a3db3b..787c234d2469 100644 --- a/arch/x86/crypto/des3_ede_glue.c +++ b/arch/x86/crypto/des3_ede_glue.c @@ -164,7 +164,7 @@ static int cbc_encrypt(struct skcipher_request *req) err = skcipher_walk_virt(&walk, req, false); - while ((nbytes = walk.nbytes)) { + while (walk.nbytes) { nbytes = __cbc_encrypt(ctx, &walk); err = skcipher_walk_done(&walk, nbytes); } @@ -243,7 +243,7 @@ static int cbc_decrypt(struct skcipher_request *req) err = skcipher_walk_virt(&walk, req, false); - while ((nbytes = walk.nbytes)) { + while (walk.nbytes) { nbytes = __cbc_decrypt(ctx, &walk); err = skcipher_walk_done(&walk, nbytes); } From c8341ac62bed9258746a5c7fb8a76d88809ecd1f Mon Sep 17 00:00:00 2001 From: Peter Gonda Date: Tue, 7 Dec 2021 15:33:02 -0800 Subject: [PATCH 079/145] crypto: ccp - Add SEV_INIT rc error logging on init Currently only the firmware error code is printed. This is incomplete and also incorrect as error cases exists where the firmware is never called and therefore does not set an error code. Signed-off-by: Peter Gonda Reviewed-by: Marc Orr Acked-by: David Rientjes Acked-by: Tom Lendacky Acked-by: Brijesh Singh Cc: Tom Lendacky Cc: Brijesh Singh Cc: Marc Orr Cc: Joerg Roedel Cc: Herbert Xu Cc: David Rientjes Cc: John Allen Cc: "David S. Miller" Cc: Paolo Bonzini Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index e09925d86bf3..f527e5f9ed1f 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -1104,7 +1104,8 @@ void sev_pci_init(void) } if (rc) { - dev_err(sev->dev, "SEV: failed to INIT error %#x\n", error); + dev_err(sev->dev, "SEV: failed to INIT error %#x, rc %d\n", + error, rc); return; } From e423b9d75e779d921e6adf5ac3d0b59400d6ba7e Mon Sep 17 00:00:00 2001 From: Peter Gonda Date: Tue, 7 Dec 2021 15:33:03 -0800 Subject: [PATCH 080/145] crypto: ccp - Move SEV_INIT retry for corrupted data Move the data corrupted retry of SEV_INIT into the __sev_platform_init_locked() function. This is for upcoming INIT_EX support as well as helping direct callers of __sev_platform_init_locked() which currently do not support the retry. Signed-off-by: Peter Gonda Reviewed-by: Marc Orr Acked-by: David Rientjes Acked-by: Tom Lendacky Acked-by: Brijesh Singh Cc: Tom Lendacky Cc: Brijesh Singh Cc: Marc Orr Cc: Joerg Roedel Cc: Herbert Xu Cc: David Rientjes Cc: John Allen Cc: "David S. Miller" Cc: Paolo Bonzini Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index f527e5f9ed1f..ef7e8b4c6e02 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -241,7 +241,7 @@ static int __sev_platform_init_locked(int *error) struct psp_device *psp = psp_master; struct sev_data_init data; struct sev_device *sev; - int rc = 0; + int psp_ret, rc = 0; if (!psp || !psp->sev_data) return -ENODEV; @@ -266,7 +266,21 @@ static int __sev_platform_init_locked(int *error) data.tmr_len = SEV_ES_TMR_SIZE; } - rc = __sev_do_cmd_locked(SEV_CMD_INIT, &data, error); + rc = __sev_do_cmd_locked(SEV_CMD_INIT, &data, &psp_ret); + if (rc && psp_ret == SEV_RET_SECURE_DATA_INVALID) { + /* + * Initialization command returned an integrity check failure + * status code, meaning that firmware load and validation of SEV + * related persistent data has failed. Retrying the + * initialization function should succeed by replacing the state + * with a reset state. + */ + dev_dbg(sev->dev, "SEV: retrying INIT command"); + rc = __sev_do_cmd_locked(SEV_CMD_INIT, &data, &psp_ret); + } + if (error) + *error = psp_ret; + if (rc) return rc; @@ -1091,18 +1105,6 @@ void sev_pci_init(void) /* Initialize the platform */ rc = sev_platform_init(&error); - if (rc && (error == SEV_RET_SECURE_DATA_INVALID)) { - /* - * INIT command returned an integrity check failure - * status code, meaning that firmware load and - * validation of SEV related persistent data has - * failed and persistent state has been erased. - * Retrying INIT command here should succeed. - */ - dev_dbg(sev->dev, "SEV: retrying INIT command"); - rc = sev_platform_init(&error); - } - if (rc) { dev_err(sev->dev, "SEV: failed to INIT error %#x, rc %d\n", error, rc); From cc17982d58d1e67eab831e7023ede999dda56173 Mon Sep 17 00:00:00 2001 From: Peter Gonda Date: Tue, 7 Dec 2021 15:33:04 -0800 Subject: [PATCH 081/145] crypto: ccp - Refactor out sev_fw_alloc() Create a helper function sev_fw_alloc() which can be used to allocate aligned memory regions for use by the PSP firmware. Currently only used for the SEV-ES TMR region but will be used for the SEV_INIT_EX NV memory region. Signed-off-by: Peter Gonda Reviewed-by: Marc Orr Acked-by: David Rientjes Acked-by: Brijesh Singh Cc: Tom Lendacky Cc: Brijesh Singh Cc: Marc Orr Cc: Joerg Roedel Cc: Herbert Xu Cc: David Rientjes Cc: John Allen Cc: "David S. Miller" Cc: Paolo Bonzini Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index ef7e8b4c6e02..8ea362ac014f 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -141,6 +141,17 @@ static int sev_cmd_buffer_len(int cmd) return 0; } +static void *sev_fw_alloc(unsigned long len) +{ + struct page *page; + + page = alloc_pages(GFP_KERNEL, get_order(len)); + if (!page) + return NULL; + + return page_address(page); +} + static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) { struct psp_device *psp = psp_master; @@ -1078,7 +1089,6 @@ EXPORT_SYMBOL_GPL(sev_issue_cmd_external_user); void sev_pci_init(void) { struct sev_device *sev = psp_master->sev_data; - struct page *tmr_page; int error, rc; if (!sev) @@ -1094,14 +1104,10 @@ void sev_pci_init(void) sev_get_api_version(); /* Obtain the TMR memory area for SEV-ES use */ - tmr_page = alloc_pages(GFP_KERNEL, get_order(SEV_ES_TMR_SIZE)); - if (tmr_page) { - sev_es_tmr = page_address(tmr_page); - } else { - sev_es_tmr = NULL; + sev_es_tmr = sev_fw_alloc(SEV_ES_TMR_SIZE); + if (!sev_es_tmr) dev_warn(sev->dev, "SEV: TMR allocation failed, SEV-ES support unavailable\n"); - } /* Initialize the platform */ rc = sev_platform_init(&error); From b64fa5fc9f4495e3ff189d9fb31cd53e6ced1cc3 Mon Sep 17 00:00:00 2001 From: Peter Gonda Date: Tue, 7 Dec 2021 15:33:05 -0800 Subject: [PATCH 082/145] crypto: ccp - Add psp_init_on_probe module parameter Add psp_init_on_probe module parameter that allows for skipping the PSP's SEV platform initialization during module init. User may decouple module init from PSP init due to use of the INIT_EX support in upcoming patch which allows for users to save PSP's internal state to file. The file may be unavailable at module init. Also moves the PSP ABI version log message to after successful PSP init instead of module init in case this new parameter is used. Signed-off-by: Peter Gonda Acked-by: Brijesh Singh Cc: Tom Lendacky Cc: Brijesh Singh Cc: Marc Orr Cc: Joerg Roedel Cc: Herbert Xu Cc: David Rientjes Cc: John Allen Cc: "David S. Miller" Cc: Paolo Bonzini Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 8ea362ac014f..686b16e69de7 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -43,6 +43,10 @@ static int psp_probe_timeout = 5; module_param(psp_probe_timeout, int, 0644); MODULE_PARM_DESC(psp_probe_timeout, " default timeout value, in seconds, during PSP device probe"); +static bool psp_init_on_probe = true; +module_param(psp_init_on_probe, bool, 0444); +MODULE_PARM_DESC(psp_init_on_probe, " if true, the PSP will be initialized on module init. Else the PSP will be initialized on the first command requiring it"); + MODULE_FIRMWARE("amd/amd_sev_fam17h_model0xh.sbin"); /* 1st gen EPYC */ MODULE_FIRMWARE("amd/amd_sev_fam17h_model3xh.sbin"); /* 2nd gen EPYC */ MODULE_FIRMWARE("amd/amd_sev_fam19h_model0xh.sbin"); /* 3rd gen EPYC */ @@ -305,7 +309,10 @@ static int __sev_platform_init_locked(int *error) dev_dbg(sev->dev, "SEV firmware initialized\n"); - return rc; + dev_info(sev->dev, "SEV API:%d.%d build:%d\n", sev->api_major, + sev->api_minor, sev->build); + + return 0; } int sev_platform_init(int *error) @@ -1109,16 +1116,14 @@ void sev_pci_init(void) dev_warn(sev->dev, "SEV: TMR allocation failed, SEV-ES support unavailable\n"); + if (!psp_init_on_probe) + return; + /* Initialize the platform */ rc = sev_platform_init(&error); - if (rc) { + if (rc) dev_err(sev->dev, "SEV: failed to INIT error %#x, rc %d\n", error, rc); - return; - } - - dev_info(sev->dev, "SEV API:%d.%d build:%d\n", sev->api_major, - sev->api_minor, sev->build); return; From 3d725965f836a7acbd1674e33644bec18373de53 Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Tue, 7 Dec 2021 15:33:06 -0800 Subject: [PATCH 083/145] crypto: ccp - Add SEV_INIT_EX support Add new module parameter to allow users to use SEV_INIT_EX instead of SEV_INIT. This helps users who lock their SPI bus to use the PSP for SEV functionality. The 'init_ex_path' parameter defaults to NULL which means the kernel will use SEV_INIT, if a path is specified SEV_INIT_EX will be used with the data found at the path. On certain PSP commands this file is written to as the PSP updates the NV memory region. Depending on file system initialization this file open may fail during module init but the CCP driver for SEV already has sufficient retries for platform initialization. During normal operation of PSP system and SEV commands if the PSP has not been initialized it is at run time. If the file at 'init_ex_path' does not exist the PSP will not be initialized. The user must create the file prior to use with 32Kb of 0xFFs per spec. Signed-off-by: David Rientjes Co-developed-by: Peter Gonda Signed-off-by: Peter Gonda Reviewed-by: Marc Orr Reported-by: kernel test robot Acked-by: Brijesh Singh Cc: Tom Lendacky Cc: Brijesh Singh Cc: Marc Orr Cc: Joerg Roedel Cc: Herbert Xu Cc: David Rientjes Cc: John Allen Cc: "David S. Miller" Cc: Paolo Bonzini Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Herbert Xu --- .../virt/kvm/amd-memory-encryption.rst | 6 + drivers/crypto/ccp/sev-dev.c | 201 ++++++++++++++++-- include/linux/psp-sev.h | 21 ++ 3 files changed, 209 insertions(+), 19 deletions(-) diff --git a/Documentation/virt/kvm/amd-memory-encryption.rst b/Documentation/virt/kvm/amd-memory-encryption.rst index 5c081c8c7164..1c6847fff304 100644 --- a/Documentation/virt/kvm/amd-memory-encryption.rst +++ b/Documentation/virt/kvm/amd-memory-encryption.rst @@ -85,6 +85,12 @@ guests, such as launching, running, snapshotting, migrating and decommissioning. The KVM_SEV_INIT command is used by the hypervisor to initialize the SEV platform context. In a typical workflow, this command should be the first command issued. +The firmware can be initialized either by using its own non-volatile storage or +the OS can manage the NV storage for the firmware using the module parameter +``init_ex_path``. The file specified by ``init_ex_path`` must exist. To create +a new NV storage file allocate the file with 32KB bytes of 0xFF as required by +the SEV spec. + Returns: 0 on success, -negative on error 2. KVM_SEV_LAUNCH_START diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 686b16e69de7..991cc5aaaa0f 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -22,6 +22,7 @@ #include #include #include +#include #include @@ -43,6 +44,10 @@ static int psp_probe_timeout = 5; module_param(psp_probe_timeout, int, 0644); MODULE_PARM_DESC(psp_probe_timeout, " default timeout value, in seconds, during PSP device probe"); +static char *init_ex_path; +module_param(init_ex_path, charp, 0444); +MODULE_PARM_DESC(init_ex_path, " Path for INIT_EX data; if set try INIT_EX"); + static bool psp_init_on_probe = true; module_param(psp_init_on_probe, bool, 0444); MODULE_PARM_DESC(psp_init_on_probe, " if true, the PSP will be initialized on module init. Else the PSP will be initialized on the first command requiring it"); @@ -62,6 +67,14 @@ static int psp_timeout; #define SEV_ES_TMR_SIZE (1024 * 1024) static void *sev_es_tmr; +/* INIT_EX NV Storage: + * The NV Storage is a 32Kb area and must be 4Kb page aligned. Use the page + * allocator to allocate the memory, which will return aligned memory for the + * specified allocation order. + */ +#define NV_LENGTH (32 * 1024) +static void *sev_init_ex_buffer; + static inline bool sev_version_greater_or_equal(u8 maj, u8 min) { struct sev_device *sev = psp_master->sev_data; @@ -111,6 +124,7 @@ static int sev_cmd_buffer_len(int cmd) { switch (cmd) { case SEV_CMD_INIT: return sizeof(struct sev_data_init); + case SEV_CMD_INIT_EX: return sizeof(struct sev_data_init_ex); case SEV_CMD_PLATFORM_STATUS: return sizeof(struct sev_user_data_status); case SEV_CMD_PEK_CSR: return sizeof(struct sev_data_pek_csr); case SEV_CMD_PEK_CERT_IMPORT: return sizeof(struct sev_data_pek_cert_import); @@ -156,6 +170,101 @@ static void *sev_fw_alloc(unsigned long len) return page_address(page); } +static int sev_read_init_ex_file(void) +{ + struct sev_device *sev = psp_master->sev_data; + struct file *fp; + ssize_t nread; + + lockdep_assert_held(&sev_cmd_mutex); + + if (!sev_init_ex_buffer) + return -EOPNOTSUPP; + + fp = filp_open(init_ex_path, O_RDONLY, 0); + if (IS_ERR(fp)) { + int ret = PTR_ERR(fp); + + dev_err(sev->dev, + "SEV: could not open %s for read, error %d\n", + init_ex_path, ret); + return ret; + } + + nread = kernel_read(fp, sev_init_ex_buffer, NV_LENGTH, NULL); + if (nread != NV_LENGTH) { + dev_err(sev->dev, + "SEV: failed to read %u bytes to non volatile memory area, ret %ld\n", + NV_LENGTH, nread); + return -EIO; + } + + dev_dbg(sev->dev, "SEV: read %ld bytes from NV file\n", nread); + filp_close(fp, NULL); + + return 0; +} + +static void sev_write_init_ex_file(void) +{ + struct sev_device *sev = psp_master->sev_data; + struct file *fp; + loff_t offset = 0; + ssize_t nwrite; + + lockdep_assert_held(&sev_cmd_mutex); + + if (!sev_init_ex_buffer) + return; + + fp = filp_open(init_ex_path, O_CREAT | O_WRONLY, 0600); + if (IS_ERR(fp)) { + dev_err(sev->dev, + "SEV: could not open file for write, error %ld\n", + PTR_ERR(fp)); + return; + } + + nwrite = kernel_write(fp, sev_init_ex_buffer, NV_LENGTH, &offset); + vfs_fsync(fp, 0); + filp_close(fp, NULL); + + if (nwrite != NV_LENGTH) { + dev_err(sev->dev, + "SEV: failed to write %u bytes to non volatile memory area, ret %ld\n", + NV_LENGTH, nwrite); + return; + } + + dev_dbg(sev->dev, "SEV: write successful to NV file\n"); +} + +static void sev_write_init_ex_file_if_required(int cmd_id) +{ + lockdep_assert_held(&sev_cmd_mutex); + + if (!sev_init_ex_buffer) + return; + + /* + * Only a few platform commands modify the SPI/NV area, but none of the + * non-platform commands do. Only INIT(_EX), PLATFORM_RESET, PEK_GEN, + * PEK_CERT_IMPORT, and PDH_GEN do. + */ + switch (cmd_id) { + case SEV_CMD_FACTORY_RESET: + case SEV_CMD_INIT_EX: + case SEV_CMD_PDH_GEN: + case SEV_CMD_PEK_CERT_IMPORT: + case SEV_CMD_PEK_GEN: + break; + default: + return; + }; + + sev_write_init_ex_file(); +} + static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) { struct psp_device *psp = psp_master; @@ -225,6 +334,8 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) dev_dbg(sev->dev, "sev command %#x failed (%#010x)\n", cmd, reg & PSP_CMDRESP_ERR_MASK); ret = -EIO; + } else { + sev_write_init_ex_file_if_required(cmd); } print_hex_dump_debug("(out): ", DUMP_PREFIX_OFFSET, 16, 2, data, @@ -251,12 +362,59 @@ static int sev_do_cmd(int cmd, void *data, int *psp_ret) return rc; } +static int __sev_init_locked(int *error) +{ + struct sev_data_init data; + + memset(&data, 0, sizeof(data)); + if (sev_es_tmr) { + /* + * Do not include the encryption mask on the physical + * address of the TMR (firmware should clear it anyway). + */ + data.tmr_address = __pa(sev_es_tmr); + + data.flags |= SEV_INIT_FLAGS_SEV_ES; + data.tmr_len = SEV_ES_TMR_SIZE; + } + + return __sev_do_cmd_locked(SEV_CMD_INIT, &data, error); +} + +static int __sev_init_ex_locked(int *error) +{ + struct sev_data_init_ex data; + int ret; + + memset(&data, 0, sizeof(data)); + data.length = sizeof(data); + data.nv_address = __psp_pa(sev_init_ex_buffer); + data.nv_len = NV_LENGTH; + + ret = sev_read_init_ex_file(); + if (ret) + return ret; + + if (sev_es_tmr) { + /* + * Do not include the encryption mask on the physical + * address of the TMR (firmware should clear it anyway). + */ + data.tmr_address = __pa(sev_es_tmr); + + data.flags |= SEV_INIT_FLAGS_SEV_ES; + data.tmr_len = SEV_ES_TMR_SIZE; + } + + return __sev_do_cmd_locked(SEV_CMD_INIT_EX, &data, error); +} + static int __sev_platform_init_locked(int *error) { struct psp_device *psp = psp_master; - struct sev_data_init data; struct sev_device *sev; - int psp_ret, rc = 0; + int rc, psp_ret; + int (*init_function)(int *error); if (!psp || !psp->sev_data) return -ENODEV; @@ -266,22 +424,9 @@ static int __sev_platform_init_locked(int *error) if (sev->state == SEV_STATE_INIT) return 0; - memset(&data, 0, sizeof(data)); - if (sev_es_tmr) { - u64 tmr_pa; - - /* - * Do not include the encryption mask on the physical - * address of the TMR (firmware should clear it anyway). - */ - tmr_pa = __pa(sev_es_tmr); - - data.flags |= SEV_INIT_FLAGS_SEV_ES; - data.tmr_address = tmr_pa; - data.tmr_len = SEV_ES_TMR_SIZE; - } - - rc = __sev_do_cmd_locked(SEV_CMD_INIT, &data, &psp_ret); + init_function = sev_init_ex_buffer ? __sev_init_ex_locked : + __sev_init_locked; + rc = init_function(&psp_ret); if (rc && psp_ret == SEV_RET_SECURE_DATA_INVALID) { /* * Initialization command returned an integrity check failure @@ -291,7 +436,7 @@ static int __sev_platform_init_locked(int *error) * with a reset state. */ dev_dbg(sev->dev, "SEV: retrying INIT command"); - rc = __sev_do_cmd_locked(SEV_CMD_INIT, &data, &psp_ret); + rc = init_function(&psp_ret); } if (error) *error = psp_ret; @@ -1066,6 +1211,12 @@ static void sev_firmware_shutdown(struct sev_device *sev) get_order(SEV_ES_TMR_SIZE)); sev_es_tmr = NULL; } + + if (sev_init_ex_buffer) { + free_pages((unsigned long)sev_init_ex_buffer, + get_order(NV_LENGTH)); + sev_init_ex_buffer = NULL; + } } void sev_dev_destroy(struct psp_device *psp) @@ -1110,6 +1261,18 @@ void sev_pci_init(void) sev_update_firmware(sev->dev) == 0) sev_get_api_version(); + /* If an init_ex_path is provided rely on INIT_EX for PSP initialization + * instead of INIT. + */ + if (init_ex_path) { + sev_init_ex_buffer = sev_fw_alloc(NV_LENGTH); + if (!sev_init_ex_buffer) { + dev_err(sev->dev, + "SEV: INIT_EX NV memory allocation failed\n"); + goto err; + } + } + /* Obtain the TMR memory area for SEV-ES use */ sev_es_tmr = sev_fw_alloc(SEV_ES_TMR_SIZE); if (!sev_es_tmr) diff --git a/include/linux/psp-sev.h b/include/linux/psp-sev.h index d48a7192e881..1595088c428b 100644 --- a/include/linux/psp-sev.h +++ b/include/linux/psp-sev.h @@ -52,6 +52,7 @@ enum sev_cmd { SEV_CMD_DF_FLUSH = 0x00A, SEV_CMD_DOWNLOAD_FIRMWARE = 0x00B, SEV_CMD_GET_ID = 0x00C, + SEV_CMD_INIT_EX = 0x00D, /* Guest commands */ SEV_CMD_DECOMMISSION = 0x020, @@ -102,6 +103,26 @@ struct sev_data_init { u32 tmr_len; /* In */ } __packed; +/** + * struct sev_data_init_ex - INIT_EX command parameters + * + * @length: len of the command buffer read by the PSP + * @flags: processing flags + * @tmr_address: system physical address used for SEV-ES + * @tmr_len: len of tmr_address + * @nv_address: system physical address used for PSP NV storage + * @nv_len: len of nv_address + */ +struct sev_data_init_ex { + u32 length; /* In */ + u32 flags; /* In */ + u64 tmr_address; /* In */ + u32 tmr_len; /* In */ + u32 reserved; /* In */ + u64 nv_address; /* In/Out */ + u32 nv_len; /* In */ +} __packed; + #define SEV_INIT_FLAGS_SEV_ES 0x01 /** From 61a13714a9853fb7b342ecae7ecafad1a950309b Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Tue, 7 Dec 2021 19:24:59 -0600 Subject: [PATCH 084/145] crypto: cavium - Use kcalloc() instead of kzalloc() Use 2-factor multiplication argument form kcalloc() instead of kzalloc(). Link: https://github.com/KSPP/linux/issues/162 Signed-off-by: Gustavo A. R. Silva Reviewed-by: Kees Cook Signed-off-by: Herbert Xu --- drivers/crypto/cavium/cpt/cptvf_main.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c index 112b12a32542..c246920e6f54 100644 --- a/drivers/crypto/cavium/cpt/cptvf_main.c +++ b/drivers/crypto/cavium/cpt/cptvf_main.c @@ -104,17 +104,14 @@ static int alloc_pending_queues(struct pending_qinfo *pqinfo, u32 qlen, u32 nr_queues) { u32 i; - size_t size; int ret; struct pending_queue *queue = NULL; pqinfo->nr_queues = nr_queues; pqinfo->qlen = qlen; - size = (qlen * sizeof(struct pending_entry)); - for_each_pending_queue(pqinfo, queue, i) { - queue->head = kzalloc((size), GFP_KERNEL); + queue->head = kcalloc(qlen, sizeof(*queue->head), GFP_KERNEL); if (!queue->head) { ret = -ENOMEM; goto pending_qfail; From 0b62b664d52c670eb5e0fa7dfd6a85dbc518619c Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Tue, 7 Dec 2021 22:17:21 -0600 Subject: [PATCH 085/145] crypto: marvell/octeontx - Use kcalloc() instead of kzalloc() Use 2-factor multiplication argument form kcalloc() instead of kzalloc(). Link: https://github.com/KSPP/linux/issues/162 Signed-off-by: Gustavo A. R. Silva Reviewed-by: Kees Cook Signed-off-by: Herbert Xu --- drivers/crypto/marvell/octeontx/otx_cptvf_main.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c index c076d0b3ad5f..b681bd2dc6ad 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c @@ -94,15 +94,13 @@ static int alloc_pending_queues(struct otx_cpt_pending_qinfo *pqinfo, u32 qlen, u32 num_queues) { struct otx_cpt_pending_queue *queue = NULL; - size_t size; int ret; u32 i; pqinfo->num_queues = num_queues; - size = (qlen * sizeof(struct otx_cpt_pending_entry)); for_each_pending_queue(pqinfo, queue, i) { - queue->head = kzalloc((size), GFP_KERNEL); + queue->head = kcalloc(qlen, sizeof(*queue->head), GFP_KERNEL); if (!queue->head) { ret = -ENOMEM; goto pending_qfail; From 244d22ffd656bc8e5c49a2cd2fdfeb0e52a9730f Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 10 Dec 2021 16:30:09 +0200 Subject: [PATCH 086/145] crypto: api - Replace kernel.h with the necessary inclusions When kernel.h is used in the headers it adds a lot into dependency hell, especially when there are circular dependencies are involved. Replace kernel.h inclusion with the list of what is really being used. Signed-off-by: Andy Shevchenko Signed-off-by: Herbert Xu --- include/crypto/aead.h | 4 +++- include/crypto/algapi.h | 5 ++++- include/crypto/blake2b.h | 1 - include/crypto/blake2s.h | 2 +- include/crypto/cryptd.h | 3 ++- include/crypto/engine.h | 6 +++++- include/crypto/pcrypt.h | 2 +- include/crypto/scatterwalk.h | 3 ++- include/crypto/skcipher.h | 6 +++++- 9 files changed, 23 insertions(+), 9 deletions(-) diff --git a/include/crypto/aead.h b/include/crypto/aead.h index 5af914c1ab8e..14db3bee0519 100644 --- a/include/crypto/aead.h +++ b/include/crypto/aead.h @@ -8,9 +8,10 @@ #ifndef _CRYPTO_AEAD_H #define _CRYPTO_AEAD_H +#include #include -#include #include +#include /** * DOC: Authenticated Encryption With Associated Data (AEAD) Cipher API @@ -73,6 +74,7 @@ */ struct crypto_aead; +struct scatterlist; /** * struct aead_request - AEAD request diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 5f6841c73e5a..f76ec723ceae 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -7,9 +7,11 @@ #ifndef _CRYPTO_ALGAPI_H #define _CRYPTO_ALGAPI_H +#include #include +#include #include -#include +#include /* * Maximum values for blocksize and alignmask, used to allocate @@ -24,6 +26,7 @@ struct crypto_aead; struct crypto_instance; struct module; +struct notifier_block; struct rtattr; struct seq_file; struct sk_buff; diff --git a/include/crypto/blake2b.h b/include/crypto/blake2b.h index 18875f16f8ca..0c0176285349 100644 --- a/include/crypto/blake2b.h +++ b/include/crypto/blake2b.h @@ -5,7 +5,6 @@ #include #include -#include #include enum blake2b_lengths { diff --git a/include/crypto/blake2s.h b/include/crypto/blake2s.h index bc3fb59442ce..df3c6c2f9553 100644 --- a/include/crypto/blake2s.h +++ b/include/crypto/blake2s.h @@ -7,8 +7,8 @@ #define _CRYPTO_BLAKE2S_H #include +#include #include -#include #include enum blake2s_lengths { diff --git a/include/crypto/cryptd.h b/include/crypto/cryptd.h index 23169f4d87e6..796d986e58e1 100644 --- a/include/crypto/cryptd.h +++ b/include/crypto/cryptd.h @@ -13,7 +13,8 @@ #ifndef _CRYPTO_CRYPT_H #define _CRYPTO_CRYPT_H -#include +#include + #include #include #include diff --git a/include/crypto/engine.h b/include/crypto/engine.h index fd4f2fa23f51..ae133e98d813 100644 --- a/include/crypto/engine.h +++ b/include/crypto/engine.h @@ -9,8 +9,10 @@ #include #include -#include #include +#include +#include + #include #include #include @@ -18,6 +20,8 @@ #include #include +struct device; + #define ENGINE_NAME_LEN 30 /* * struct crypto_engine - crypto hardware engine diff --git a/include/crypto/pcrypt.h b/include/crypto/pcrypt.h index b9bc3436196a..234d7cf3cf5e 100644 --- a/include/crypto/pcrypt.h +++ b/include/crypto/pcrypt.h @@ -9,8 +9,8 @@ #ifndef _CRYPTO_PCRYPT_H #define _CRYPTO_PCRYPT_H +#include #include -#include #include struct pcrypt_request { diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index 7af08174a721..6407b4b61350 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -12,8 +12,9 @@ #define _CRYPTO_SCATTERWALK_H #include + #include -#include +#include #include static inline void scatterwalk_crypto_chain(struct scatterlist *head, diff --git a/include/crypto/skcipher.h b/include/crypto/skcipher.h index ef0fc9ed4342..39f5b67c3069 100644 --- a/include/crypto/skcipher.h +++ b/include/crypto/skcipher.h @@ -8,9 +8,13 @@ #ifndef _CRYPTO_SKCIPHER_H #define _CRYPTO_SKCIPHER_H +#include #include -#include #include +#include +#include + +struct scatterlist; /** * struct skcipher_request - Symmetric key cipher request From 51fa916b81e5f406a74f14a31a3a228c3cc060ad Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 11 Dec 2021 16:17:19 +0800 Subject: [PATCH 087/145] crypto: hisilicon/hpre - fix memory leak in hpre_curve25519_src_init() hpre_curve25519_src_init() allocates memory for 'ptr' before calling memcmp(). If memcmp() returns 0, the function will return '-EINVAL' without freeing memory. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 0f1724d355b8..97d54c1465c2 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -1862,7 +1862,7 @@ static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req, */ if (memcmp(ptr, p, ctx->key_sz) == 0) { dev_err(dev, "gx is p!\n"); - return -EINVAL; + goto err; } else if (memcmp(ptr, p, ctx->key_sz) > 0) { hpre_curve25519_src_modulo_p(ptr); } From 808957baf3aa42b71453c8accc71cf5e52665982 Mon Sep 17 00:00:00 2001 From: Yang Shen Date: Sat, 11 Dec 2021 16:52:05 +0800 Subject: [PATCH 088/145] crypto: hisilicon/zip - enable ssid for sva sgl For Kunpeng 920, the bit 0 of register 'HZIP_SGL_RUSER_32_63' stand for whether the ssid is valid. So this bit should be set as valid for sva mode. Signed-off-by: Yang Shen Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/zip/zip_main.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 1a237d95d482..7fd302ae4fc5 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -364,15 +364,16 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) /* user domain configurations */ writel(AXUSER_BASE, base + HZIP_BD_RUSER_32_63); - writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63); writel(AXUSER_BASE, base + HZIP_BD_WUSER_32_63); if (qm->use_sva && qm->ver == QM_HW_V2) { writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_RUSER_32_63); writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_WUSER_32_63); + writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_SGL_RUSER_32_63); } else { writel(AXUSER_BASE, base + HZIP_DATA_RUSER_32_63); writel(AXUSER_BASE, base + HZIP_DATA_WUSER_32_63); + writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63); } /* let's open all compression/decompression cores */ From fc6c01f0cd10b89c4b01dd2940e0b0cda1bd82fb Mon Sep 17 00:00:00 2001 From: Yang Shen Date: Sat, 11 Dec 2021 16:56:55 +0800 Subject: [PATCH 089/145] crypto: hisilicon/qm - fix deadlock for remove driver When remove the driver and executing the task occur at the same time, the following deadlock will be triggered: Chain exists of: sva_lock --> uacce_mutex --> &qm->qps_lock Possible unsafe locking scenario: CPU0 CPU1 ---- ---- lock(&qm->qps_lock); lock(uacce_mutex); lock(&qm->qps_lock); lock(sva_lock); And the lock 'qps_lock' is used to protect qp. Therefore, it's reasonable cycle is to continue until the qp memory is released. So move the release lock infront of 'uacce_remove'. Signed-off-by: Yang Shen Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 896255aec87f..bea8622d80e6 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -3425,6 +3425,7 @@ void hisi_qm_uninit(struct hisi_qm *qm) dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma); } + up_write(&qm->qps_lock); qm_irq_unregister(qm); hisi_qm_pci_uninit(qm); @@ -3432,8 +3433,6 @@ void hisi_qm_uninit(struct hisi_qm *qm) uacce_remove(qm->uacce); qm->uacce = NULL; } - - up_write(&qm->qps_lock); } EXPORT_SYMBOL_GPL(hisi_qm_uninit); From f123e66df6ca0fce73309dda667d7d1d5b0aa715 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 11 Dec 2021 19:25:14 +0800 Subject: [PATCH 090/145] crypto: hisilicon/qm - remove unnecessary device memory reset The internal memory of the device needs to be reset only when the device is globally initialized. Other scenarios, such as function reset, do not need to perform reset. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index bea8622d80e6..fba6a26089aa 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -3581,10 +3581,6 @@ static int __hisi_qm_start(struct hisi_qm *qm) WARN_ON(!qm->qdma.va); if (qm->fun_type == QM_HW_PF) { - ret = qm_dev_mem_reset(qm); - if (ret) - return ret; - ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num); if (ret) return ret; @@ -5083,6 +5079,12 @@ static int qm_controller_reset_done(struct hisi_qm *qm) if (qm->err_ini->open_axi_master_ooo) qm->err_ini->open_axi_master_ooo(qm); + ret = qm_dev_mem_reset(qm); + if (ret) { + pci_err(pdev, "failed to reset device memory\n"); + return ret; + } + ret = qm_restart(qm); if (ret) { pci_err(pdev, "Failed to start QM!\n"); @@ -5857,6 +5859,14 @@ int hisi_qm_init(struct hisi_qm *qm) goto err_irq_register; } + if (qm->fun_type == QM_HW_PF) { + ret = qm_dev_mem_reset(qm); + if (ret) { + dev_err(dev, "failed to reset device memory\n"); + goto err_irq_register; + } + } + if (qm->mode == UACCE_MODE_SVA) { ret = qm_alloc_uacce(qm); if (ret < 0) @@ -6014,8 +6024,11 @@ static int qm_rebuild_for_resume(struct hisi_qm *qm) qm_cmd_init(qm); hisi_qm_dev_err_init(qm); + ret = qm_dev_mem_reset(qm); + if (ret) + pci_err(pdev, "failed to reset device memory\n"); - return 0; + return ret; } /** From 9ee401eaceddb3e96fb72ef097175fcaaa1fb37a Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 11 Dec 2021 19:25:15 +0800 Subject: [PATCH 091/145] crypto: hisilicon/qm - code movement This patch does not change any code, just code movement. Preparing for next patch. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 138 +++++++++++++++++----------------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index fba6a26089aa..d124800b67e6 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -605,6 +605,75 @@ static bool qm_qp_avail_state(struct hisi_qm *qm, struct hisi_qp *qp, return avail; } +static u32 qm_get_hw_error_status(struct hisi_qm *qm) +{ + return readl(qm->io_base + QM_ABNORMAL_INT_STATUS); +} + +static u32 qm_get_dev_err_status(struct hisi_qm *qm) +{ + return qm->err_ini->get_dev_hw_err_status(qm); +} + +/* Check if the error causes the master ooo block */ +static int qm_check_dev_error(struct hisi_qm *qm) +{ + u32 val, dev_val; + + if (qm->fun_type == QM_HW_VF) + return 0; + + val = qm_get_hw_error_status(qm); + dev_val = qm_get_dev_err_status(qm); + + if (qm->ver < QM_HW_V3) + return (val & QM_ECC_MBIT) || + (dev_val & qm->err_info.ecc_2bits_mask); + + return (val & readl(qm->io_base + QM_OOO_SHUTDOWN_SEL)) || + (dev_val & (~qm->err_info.dev_ce_mask)); +} + +static int qm_wait_reset_finish(struct hisi_qm *qm) +{ + int delay = 0; + + /* All reset requests need to be queued for processing */ + while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { + msleep(++delay); + if (delay > QM_RESET_WAIT_TIMEOUT) + return -EBUSY; + } + + return 0; +} + +static int qm_reset_prepare_ready(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); + + /* + * PF and VF on host doesnot support resetting at the + * same time on Kunpeng920. + */ + if (qm->ver < QM_HW_V3) + return qm_wait_reset_finish(pf_qm); + + return qm_wait_reset_finish(qm); +} + +static void qm_reset_bit_clear(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); + + if (qm->ver < QM_HW_V3) + clear_bit(QM_RESETTING, &pf_qm->misc_ctl); + + clear_bit(QM_RESETTING, &qm->misc_ctl); +} + static void qm_mb_pre_init(struct qm_mailbox *mailbox, u8 cmd, u64 base, u16 queue, bool op) { @@ -2108,35 +2177,6 @@ static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) return ACC_ERR_RECOVERED; } -static u32 qm_get_hw_error_status(struct hisi_qm *qm) -{ - return readl(qm->io_base + QM_ABNORMAL_INT_STATUS); -} - -static u32 qm_get_dev_err_status(struct hisi_qm *qm) -{ - return qm->err_ini->get_dev_hw_err_status(qm); -} - -/* Check if the error causes the master ooo block */ -static int qm_check_dev_error(struct hisi_qm *qm) -{ - u32 val, dev_val; - - if (qm->fun_type == QM_HW_VF) - return 0; - - val = qm_get_hw_error_status(qm); - dev_val = qm_get_dev_err_status(qm); - - if (qm->ver < QM_HW_V3) - return (val & QM_ECC_MBIT) || - (dev_val & qm->err_info.ecc_2bits_mask); - - return (val & readl(qm->io_base + QM_OOO_SHUTDOWN_SEL)) || - (dev_val & (~qm->err_info.dev_ce_mask)); -} - static int qm_get_mb_cmd(struct hisi_qm *qm, u64 *msg, u16 fun_num) { struct qm_mailbox mailbox; @@ -4754,46 +4794,6 @@ static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd, return ret; } -static int qm_wait_reset_finish(struct hisi_qm *qm) -{ - int delay = 0; - - /* All reset requests need to be queued for processing */ - while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { - msleep(++delay); - if (delay > QM_RESET_WAIT_TIMEOUT) - return -EBUSY; - } - - return 0; -} - -static int qm_reset_prepare_ready(struct hisi_qm *qm) -{ - struct pci_dev *pdev = qm->pdev; - struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); - - /* - * PF and VF on host doesnot support resetting at the - * same time on Kunpeng920. - */ - if (qm->ver < QM_HW_V3) - return qm_wait_reset_finish(pf_qm); - - return qm_wait_reset_finish(qm); -} - -static void qm_reset_bit_clear(struct hisi_qm *qm) -{ - struct pci_dev *pdev = qm->pdev; - struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); - - if (qm->ver < QM_HW_V3) - clear_bit(QM_RESETTING, &pf_qm->misc_ctl); - - clear_bit(QM_RESETTING, &qm->misc_ctl); -} - static int qm_controller_reset_prepare(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; From 145dcedd0e17f40bd8066a0234a19e40463367db Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 11 Dec 2021 19:25:16 +0800 Subject: [PATCH 092/145] crypto: hisilicon/qm - modify the handling method after abnormal interruption After processing an interrupt event and the interrupt function is enabled by writing the QM_DOORBELL_CMD_AEQ register, the hardware may generate new interrupt events due to processing other user's task when the subsequent interrupt events have not been processed. The new interrupt event will disrupt the current normal processing flow and cause other problems. Therefore, the operation of writing the QM_DOORBELL_CMD_AEQ doorbell register needs to be placed after all interrupt events processing are completed. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index d124800b67e6..beea3a0fd0a5 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -1015,10 +1015,10 @@ static irqreturn_t qm_aeq_irq(int irq, void *data) aeqe++; qm->status.aeq_head++; } - - qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0); } + qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0); + return IRQ_HANDLED; } From a0a9486bebc43679ec9ded50ac6b93330c4b930f Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 11 Dec 2021 19:25:17 +0800 Subject: [PATCH 093/145] crypto: hisilicon/qm - use request_threaded_irq instead The abnormal interrupt method needs to be changed, and the changed method needs to be locked in order to maintain atomicity. Therefore, replace request_irq() with request_threaded_irq(). Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index beea3a0fd0a5..93abe1feb0b7 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -988,16 +988,12 @@ static void qm_set_qp_disable(struct hisi_qp *qp, int offset) mb(); } -static irqreturn_t qm_aeq_irq(int irq, void *data) +static irqreturn_t qm_aeq_thread(int irq, void *data) { struct hisi_qm *qm = data; struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head; u32 type; - atomic64_inc(&qm->debug.dfx.aeq_irq_cnt); - if (!readl(qm->io_base + QM_VF_AEQ_INT_SOURCE)) - return IRQ_NONE; - while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) { type = le32_to_cpu(aeqe->dw0) >> QM_AEQE_TYPE_SHIFT; if (type < ARRAY_SIZE(qm_fifo_overflow)) @@ -1022,6 +1018,17 @@ static irqreturn_t qm_aeq_irq(int irq, void *data) return IRQ_HANDLED; } +static irqreturn_t qm_aeq_irq(int irq, void *data) +{ + struct hisi_qm *qm = data; + + atomic64_inc(&qm->debug.dfx.aeq_irq_cnt); + if (!readl(qm->io_base + QM_VF_AEQ_INT_SOURCE)) + return IRQ_NONE; + + return IRQ_WAKE_THREAD; +} + static void qm_irq_unregister(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -5299,8 +5306,10 @@ static int qm_irq_register(struct hisi_qm *qm) return ret; if (qm->ver > QM_HW_V1) { - ret = request_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), - qm_aeq_irq, 0, qm->dev_name, qm); + ret = request_threaded_irq(pci_irq_vector(pdev, + QM_AEQ_EVENT_IRQ_VECTOR), + qm_aeq_irq, qm_aeq_thread, + 0, qm->dev_name, qm); if (ret) goto err_aeq_irq; From 95f0b6d536374ea1796bd0d9024faaca9c4f3380 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 11 Dec 2021 19:25:18 +0800 Subject: [PATCH 094/145] crypto: hisilicon/qm - reset function if event queue overflows If the hardware reports the event queue overflow by the abnormal interrupt, the driver needs to reset the function and re-enable the event queue interrupt and abnormal interrupt. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 68 +++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 93abe1feb0b7..6c61f9d25f75 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -89,6 +89,7 @@ #define QM_AEQE_PHASE(aeqe) ((le32_to_cpu((aeqe)->dw0) >> 16) & 0x1) #define QM_AEQE_TYPE_SHIFT 17 +#define QM_EQ_OVERFLOW 1 #define QM_DOORBELL_CMD_SQ 0 #define QM_DOORBELL_CMD_CQ 1 @@ -988,6 +989,35 @@ static void qm_set_qp_disable(struct hisi_qp *qp, int offset) mb(); } +static void qm_reset_function(struct hisi_qm *qm) +{ + struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); + struct device *dev = &qm->pdev->dev; + int ret; + + if (qm_check_dev_error(pf_qm)) + return; + + ret = qm_reset_prepare_ready(qm); + if (ret) { + dev_err(dev, "reset function not ready\n"); + return; + } + + ret = hisi_qm_stop(qm, QM_FLR); + if (ret) { + dev_err(dev, "failed to stop qm when reset function\n"); + goto clear_bit; + } + + ret = hisi_qm_start(qm); + if (ret) + dev_err(dev, "failed to start qm when reset function\n"); + +clear_bit: + qm_reset_bit_clear(qm); +} + static irqreturn_t qm_aeq_thread(int irq, void *data) { struct hisi_qm *qm = data; @@ -996,12 +1026,17 @@ static irqreturn_t qm_aeq_thread(int irq, void *data) while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) { type = le32_to_cpu(aeqe->dw0) >> QM_AEQE_TYPE_SHIFT; - if (type < ARRAY_SIZE(qm_fifo_overflow)) - dev_err(&qm->pdev->dev, "%s overflow\n", - qm_fifo_overflow[type]); - else + + switch (type) { + case QM_EQ_OVERFLOW: + dev_err(&qm->pdev->dev, "eq overflow, reset function\n"); + qm_reset_function(qm); + return IRQ_HANDLED; + default: dev_err(&qm->pdev->dev, "unknown error type %u\n", type); + break; + } if (qm->status.aeq_head == QM_Q_DEPTH - 1) { qm->status.aeqc_phase = !qm->status.aeqc_phase; @@ -3545,6 +3580,22 @@ static void qm_init_eq_aeq_status(struct hisi_qm *qm) status->aeqc_phase = true; } +static void qm_enable_eq_aeq_interrupts(struct hisi_qm *qm) +{ + /* Clear eq/aeq interrupt source */ + qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0); + qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); + + writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK); + writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK); +} + +static void qm_disable_eq_aeq_interrupts(struct hisi_qm *qm) +{ + writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK); + writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK); +} + static int qm_eq_ctx_cfg(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; @@ -3646,9 +3697,7 @@ static int __hisi_qm_start(struct hisi_qm *qm) return ret; qm_init_prefetch(qm); - - writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK); - writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK); + qm_enable_eq_aeq_interrupts(qm); return 0; } @@ -3796,10 +3845,7 @@ int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); } - /* Mask eq and aeq irq */ - writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK); - writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK); - + qm_disable_eq_aeq_interrupts(qm); if (qm->fun_type == QM_HW_PF) { ret = hisi_qm_set_vft(qm, 0, 0, 0); if (ret < 0) { From 696645d25bafd6ba3562611c29bc8ecd47066dfe Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 11 Dec 2021 19:25:19 +0800 Subject: [PATCH 095/145] crypto: hisilicon/qm - disable queue when 'CQ' error If the hardware reports the 'CQ' overflow or 'CQE' error by the abnormal interrupt, disable the queue and stop tasks send to hardware. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 6c61f9d25f75..b1fe9c7b8cc8 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -89,7 +89,10 @@ #define QM_AEQE_PHASE(aeqe) ((le32_to_cpu((aeqe)->dw0) >> 16) & 0x1) #define QM_AEQE_TYPE_SHIFT 17 +#define QM_AEQE_CQN_MASK GENMASK(15, 0) +#define QM_CQ_OVERFLOW 0 #define QM_EQ_OVERFLOW 1 +#define QM_CQE_ERROR 2 #define QM_DOORBELL_CMD_SQ 0 #define QM_DOORBELL_CMD_CQ 1 @@ -989,6 +992,15 @@ static void qm_set_qp_disable(struct hisi_qp *qp, int offset) mb(); } +static void qm_disable_qp(struct hisi_qm *qm, u32 qp_id) +{ + struct hisi_qp *qp = &qm->qp_array[qp_id]; + + qm_set_qp_disable(qp, QM_RESET_STOP_TX_OFFSET); + hisi_qm_stop_qp(qp); + qm_set_qp_disable(qp, QM_RESET_STOP_RX_OFFSET); +} + static void qm_reset_function(struct hisi_qm *qm) { struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); @@ -1022,16 +1034,24 @@ static irqreturn_t qm_aeq_thread(int irq, void *data) { struct hisi_qm *qm = data; struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head; - u32 type; + u32 type, qp_id; while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) { type = le32_to_cpu(aeqe->dw0) >> QM_AEQE_TYPE_SHIFT; + qp_id = le32_to_cpu(aeqe->dw0) & QM_AEQE_CQN_MASK; switch (type) { case QM_EQ_OVERFLOW: dev_err(&qm->pdev->dev, "eq overflow, reset function\n"); qm_reset_function(qm); return IRQ_HANDLED; + case QM_CQ_OVERFLOW: + dev_err(&qm->pdev->dev, "cq overflow, stop qp(%u)\n", + qp_id); + fallthrough; + case QM_CQE_ERROR: + qm_disable_qp(qm, qp_id); + break; default: dev_err(&qm->pdev->dev, "unknown error type %u\n", type); From 223a41f54946a22616c2b9a0abc86e55f74efc69 Mon Sep 17 00:00:00 2001 From: Yang Shen Date: Tue, 14 Dec 2021 14:45:09 +0800 Subject: [PATCH 096/145] crypto: hisilicon/zip - add new algorithms for uacce device Enable deflate/lz77_zstd algorithm for uacce device on Kunpeng930. Signed-off-by: Yang Shen Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/zip/zip_main.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 7fd302ae4fc5..678f8b58ec42 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -830,7 +830,10 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->pdev = pdev; qm->ver = pdev->revision; - qm->algs = "zlib\ngzip"; + if (pdev->revision >= QM_HW_V3) + qm->algs = "zlib\ngzip\ndeflate\nlz77_zstd"; + else + qm->algs = "zlib\ngzip"; qm->mode = uacce_mode; qm->sqe_size = HZIP_SQE_SIZE; qm->dev_name = hisi_zip_name; From 38e9791a02090414d1e3433549001689cad71098 Mon Sep 17 00:00:00 2001 From: Sunil Goutham Date: Tue, 14 Dec 2021 16:21:08 +0530 Subject: [PATCH 097/145] hwrng: cn10k - Add random number generator support CN10K series of silicons support true random number generators. This patch adds support for the same. Also supports entropy health status checking. Signed-off-by: Sunil Goutham Signed-off-by: Bharat Bhushan Signed-off-by: Joseph Longever Signed-off-by: Herbert Xu --- drivers/char/hw_random/Kconfig | 11 ++ drivers/char/hw_random/Makefile | 1 + drivers/char/hw_random/cn10k-rng.c | 181 +++++++++++++++++++++++++++++ 3 files changed, 193 insertions(+) create mode 100644 drivers/char/hw_random/cn10k-rng.c diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index 1da64771ee2e..001b819f5298 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -538,6 +538,17 @@ config HW_RANDOM_ARM_SMCCC_TRNG To compile this driver as a module, choose M here: the module will be called arm_smccc_trng. +config HW_RANDOM_CN10K + tristate "Marvell CN10K Random Number Generator support" + depends on HW_RANDOM && PCI && ARM64 + default HW_RANDOM + help + This driver provides support for the True Random Number + generator available in Marvell CN10K SoCs. + + To compile this driver as a module, choose M here. + The module will be called cn10k_rng. If unsure, say Y. + endif # HW_RANDOM config UML_RANDOM diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile index a5a1c765a394..a2f1ce0790d1 100644 --- a/drivers/char/hw_random/Makefile +++ b/drivers/char/hw_random/Makefile @@ -46,3 +46,4 @@ obj-$(CONFIG_HW_RANDOM_NPCM) += npcm-rng.o obj-$(CONFIG_HW_RANDOM_CCTRNG) += cctrng.o obj-$(CONFIG_HW_RANDOM_XIPHERA) += xiphera-trng.o obj-$(CONFIG_HW_RANDOM_ARM_SMCCC_TRNG) += arm_smccc_trng.o +obj-$(CONFIG_HW_RANDOM_CN10K) += cn10k-rng.o diff --git a/drivers/char/hw_random/cn10k-rng.c b/drivers/char/hw_random/cn10k-rng.c new file mode 100644 index 000000000000..35001c63648b --- /dev/null +++ b/drivers/char/hw_random/cn10k-rng.c @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell CN10K RVU Hardware Random Number Generator. + * + * Copyright (C) 2021 Marvell. + * + */ + +#include +#include +#include +#include +#include +#include + +#include + +/* CSRs */ +#define RNM_CTL_STATUS 0x000 +#define RNM_ENTROPY_STATUS 0x008 +#define RNM_CONST 0x030 +#define RNM_EBG_ENT 0x048 +#define RNM_PF_EBG_HEALTH 0x050 +#define RNM_PF_RANDOM 0x400 +#define RNM_TRNG_RESULT 0x408 + +struct cn10k_rng { + void __iomem *reg_base; + struct hwrng ops; + struct pci_dev *pdev; +}; + +#define PLAT_OCTEONTX_RESET_RNG_EBG_HEALTH_STATE 0xc2000b0f + +static int reset_rng_health_state(struct cn10k_rng *rng) +{ + struct arm_smccc_res res; + + /* Send SMC service call to reset EBG health state */ + arm_smccc_smc(PLAT_OCTEONTX_RESET_RNG_EBG_HEALTH_STATE, 0, 0, 0, 0, 0, 0, 0, &res); + if (res.a0 != 0UL) + return -EIO; + + return 0; +} + +static int check_rng_health(struct cn10k_rng *rng) +{ + u64 status; + int err; + + /* Skip checking health */ + if (!rng->reg_base) + return 0; + + status = readq(rng->reg_base + RNM_PF_EBG_HEALTH); + if (status & BIT_ULL(20)) { + err = reset_rng_health_state(rng); + if (err) { + dev_err(&rng->pdev->dev, "HWRNG: Health test failed (status=%llx)\n", + status); + dev_err(&rng->pdev->dev, "HWRNG: error during reset\n"); + } + } + return 0; +} + +static void cn10k_read_trng(struct cn10k_rng *rng, u64 *value) +{ + u64 upper, lower; + + *value = readq(rng->reg_base + RNM_PF_RANDOM); + + /* HW can run out of entropy if large amount random data is read in + * quick succession. Zeros may not be real random data from HW. + */ + if (!*value) { + upper = readq(rng->reg_base + RNM_PF_RANDOM); + lower = readq(rng->reg_base + RNM_PF_RANDOM); + while (!(upper & 0x00000000FFFFFFFFULL)) + upper = readq(rng->reg_base + RNM_PF_RANDOM); + while (!(lower & 0xFFFFFFFF00000000ULL)) + lower = readq(rng->reg_base + RNM_PF_RANDOM); + + *value = (upper & 0xFFFFFFFF00000000) | (lower & 0xFFFFFFFF); + } +} + +static int cn10k_rng_read(struct hwrng *hwrng, void *data, + size_t max, bool wait) +{ + struct cn10k_rng *rng = (struct cn10k_rng *)hwrng->priv; + unsigned int size; + int err = 0; + u64 value; + + err = check_rng_health(rng); + if (err) + return err; + + size = max; + + while (size >= 8) { + cn10k_read_trng(rng, &value); + + *((u64 *)data) = (u64)value; + size -= 8; + data += 8; + } + + while (size > 0) { + cn10k_read_trng(rng, &value); + + *((u8 *)data) = (u8)value; + size--; + data++; + } + + return max - size; +} + +static int cn10k_rng_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct cn10k_rng *rng; + int err; + + rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL); + if (!rng) + return -ENOMEM; + + rng->pdev = pdev; + pci_set_drvdata(pdev, rng); + + rng->reg_base = pcim_iomap(pdev, 0, 0); + if (!rng->reg_base) { + dev_err(&pdev->dev, "Error while mapping CSRs, exiting\n"); + return -ENOMEM; + } + + rng->ops.name = devm_kasprintf(&pdev->dev, GFP_KERNEL, + "cn10k-rng-%s", dev_name(&pdev->dev)); + if (!rng->ops.name) + return -ENOMEM; + + rng->ops.read = cn10k_rng_read; + rng->ops.quality = 1000; + rng->ops.priv = (unsigned long)rng; + + reset_rng_health_state(rng); + + err = devm_hwrng_register(&pdev->dev, &rng->ops); + if (err) { + dev_err(&pdev->dev, "Could not register hwrng device.\n"); + return err; + } + + return 0; +} + +static void cn10k_rng_remove(struct pci_dev *pdev) +{ + /* Nothing to do */ +} + +static const struct pci_device_id cn10k_rng_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xA098) }, /* RNG PF */ + {0,}, +}; + +MODULE_DEVICE_TABLE(pci, cn10k_rng_id_table); + +static struct pci_driver cn10k_rng_driver = { + .name = "cn10k_rng", + .id_table = cn10k_rng_id_table, + .probe = cn10k_rng_probe, + .remove = cn10k_rng_remove, +}; + +module_pci_driver(cn10k_rng_driver); +MODULE_AUTHOR("Sunil Goutham "); +MODULE_DESCRIPTION("Marvell CN10K HW RNG Driver"); +MODULE_LICENSE("GPL v2"); From acd93f8a4ca784d8eff303c6cae49f3bf7b3a499 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Tue, 14 Dec 2021 17:01:46 +0100 Subject: [PATCH 098/145] crypto: x86/curve25519 - use in/out register constraints more precisely Rather than passing all variables as modified, pass ones that are only read into that parameter. This helps with old gcc versions when alternatives are additionally used, and lets gcc's codegen be a little bit more efficient. This also syncs up with the latest Vale/EverCrypt output. Reported-by: Mathias Krause Cc: Aymeric Fromherz Link: https://lore.kernel.org/wireguard/1554725710.1290070.1639240504281.JavaMail.zimbra@inria.fr/ Link: https://github.com/project-everest/hacl-star/pull/501 Signed-off-by: Jason A. Donenfeld Reviewed-by: Mathias Krause Signed-off-by: Herbert Xu --- arch/x86/crypto/curve25519-x86_64.c | 767 ++++++++++++++++++---------- 1 file changed, 489 insertions(+), 278 deletions(-) diff --git a/arch/x86/crypto/curve25519-x86_64.c b/arch/x86/crypto/curve25519-x86_64.c index 38caf61cd5b7..d55fa9e9b9e6 100644 --- a/arch/x86/crypto/curve25519-x86_64.c +++ b/arch/x86/crypto/curve25519-x86_64.c @@ -64,10 +64,9 @@ static inline u64 add_scalar(u64 *out, const u64 *f1, u64 f2) /* Return the carry bit in a register */ " adcx %%r11, %1;" - : "+&r" (f2), "=&r" (carry_r) - : "r" (out), "r" (f1) - : "%r8", "%r9", "%r10", "%r11", "memory", "cc" - ); + : "+&r"(f2), "=&r"(carry_r) + : "r"(out), "r"(f1) + : "%r8", "%r9", "%r10", "%r11", "memory", "cc"); return carry_r; } @@ -108,10 +107,9 @@ static inline void fadd(u64 *out, const u64 *f1, const u64 *f2) " cmovc %0, %%rax;" " add %%rax, %%r8;" " movq %%r8, 0(%1);" - : "+&r" (f2) - : "r" (out), "r" (f1) - : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc" - ); + : "+&r"(f2) + : "r"(out), "r"(f1) + : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc"); } /* Computes the field subtraction of two field elements */ @@ -151,10 +149,9 @@ static inline void fsub(u64 *out, const u64 *f1, const u64 *f2) " movq %%r9, 8(%0);" " movq %%r10, 16(%0);" " movq %%r11, 24(%0);" - : - : "r" (out), "r" (f1), "r" (f2) - : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc" - ); + : + : "r"(out), "r"(f1), "r"(f2) + : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc"); } /* Computes a field multiplication: out <- f1 * f2 @@ -162,239 +159,400 @@ static inline void fsub(u64 *out, const u64 *f1, const u64 *f2) static inline void fmul(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp) { asm volatile( + /* Compute the raw multiplication: tmp <- src1 * src2 */ /* Compute src1[0] * src2 */ - " movq 0(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 0(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 8(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" + " movq 0(%0), %%rdx;" + " mulxq 0(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " movq %%r8, 0(%2);" + " mulxq 8(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " movq %%r10, 8(%2);" + " mulxq 16(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " mulxq 24(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + /* Compute src1[1] * src2 */ - " movq 8(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 16(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" + " movq 8(%0), %%rdx;" + " mulxq 0(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " adcxq 8(%2), %%r8;" + " movq %%r8, 8(%2);" + " mulxq 8(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " adcx %%rbx, %%r10;" + " movq %%r10, 16(%2);" + " mulxq 16(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " adcx %%r14, %%rbx;" + " mov $0, %%r8;" + " mulxq 24(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " adcx %%rax, %%r14;" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + " adcx %%r8, %%rax;" + /* Compute src1[2] * src2 */ - " movq 16(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 24(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" + " movq 16(%0), %%rdx;" + " mulxq 0(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " adcxq 16(%2), %%r8;" + " movq %%r8, 16(%2);" + " mulxq 8(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " adcx %%rbx, %%r10;" + " movq %%r10, 24(%2);" + " mulxq 16(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " adcx %%r14, %%rbx;" + " mov $0, %%r8;" + " mulxq 24(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " adcx %%rax, %%r14;" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + " adcx %%r8, %%rax;" + /* Compute src1[3] * src2 */ - " movq 24(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 32(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 40(%0);" " mov $0, %%r8;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 48(%0);" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 56(%0);" + " movq 24(%0), %%rdx;" + " mulxq 0(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " adcxq 24(%2), %%r8;" + " movq %%r8, 24(%2);" + " mulxq 8(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " adcx %%rbx, %%r10;" + " movq %%r10, 32(%2);" + " mulxq 16(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " adcx %%r14, %%rbx;" + " movq %%rbx, 40(%2);" + " mov $0, %%r8;" + " mulxq 24(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " adcx %%rax, %%r14;" + " movq %%r14, 48(%2);" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + " adcx %%r8, %%rax;" + " movq %%rax, 56(%2);" + /* Line up pointers */ - " mov %0, %1;" " mov %2, %0;" + " mov %3, %2;" /* Wrap the result back into the field */ /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ " mov $38, %%rdx;" - " mulxq 32(%1), %%r8, %%r13;" - " xor %k3, %k3;" - " adoxq 0(%1), %%r8;" - " mulxq 40(%1), %%r9, %%rbx;" + " mulxq 32(%0), %%r8, %%r13;" + " xor %k1, %k1;" + " adoxq 0(%0), %%r8;" + " mulxq 40(%0), %%r9, %%rbx;" " adcx %%r13, %%r9;" - " adoxq 8(%1), %%r9;" - " mulxq 48(%1), %%r10, %%r13;" + " adoxq 8(%0), %%r9;" + " mulxq 48(%0), %%r10, %%r13;" " adcx %%rbx, %%r10;" - " adoxq 16(%1), %%r10;" - " mulxq 56(%1), %%r11, %%rax;" + " adoxq 16(%0), %%r10;" + " mulxq 56(%0), %%r11, %%rax;" " adcx %%r13, %%r11;" - " adoxq 24(%1), %%r11;" - " adcx %3, %%rax;" - " adox %3, %%rax;" + " adoxq 24(%0), %%r11;" + " adcx %1, %%rax;" + " adox %1, %%rax;" " imul %%rdx, %%rax;" /* Step 2: Fold the carry back into dst */ " add %%rax, %%r8;" - " adcx %3, %%r9;" - " movq %%r9, 8(%0);" - " adcx %3, %%r10;" - " movq %%r10, 16(%0);" - " adcx %3, %%r11;" - " movq %%r11, 24(%0);" + " adcx %1, %%r9;" + " movq %%r9, 8(%2);" + " adcx %1, %%r10;" + " movq %%r10, 16(%2);" + " adcx %1, %%r11;" + " movq %%r11, 24(%2);" /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ " mov $0, %%rax;" " cmovc %%rdx, %%rax;" " add %%rax, %%r8;" - " movq %%r8, 0(%0);" - : "+&r" (tmp), "+&r" (f1), "+&r" (out), "+&r" (f2) - : - : "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "memory", "cc" - ); + " movq %%r8, 0(%2);" + : "+&r"(f1), "+&r"(f2), "+&r"(tmp) + : "r"(out) + : "%rax", "%rbx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13", + "%r14", "memory", "cc"); } /* Computes two field multiplications: - * out[0] <- f1[0] * f2[0] - * out[1] <- f1[1] * f2[1] - * Uses the 16-element buffer tmp for intermediate results. */ + * out[0] <- f1[0] * f2[0] + * out[1] <- f1[1] * f2[1] + * Uses the 16-element buffer tmp for intermediate results: */ static inline void fmul2(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp) { asm volatile( + /* Compute the raw multiplication tmp[0] <- f1[0] * f2[0] */ /* Compute src1[0] * src2 */ - " movq 0(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 0(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 8(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" + " movq 0(%0), %%rdx;" + " mulxq 0(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " movq %%r8, 0(%2);" + " mulxq 8(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " movq %%r10, 8(%2);" + " mulxq 16(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " mulxq 24(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + /* Compute src1[1] * src2 */ - " movq 8(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 16(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" + " movq 8(%0), %%rdx;" + " mulxq 0(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " adcxq 8(%2), %%r8;" + " movq %%r8, 8(%2);" + " mulxq 8(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " adcx %%rbx, %%r10;" + " movq %%r10, 16(%2);" + " mulxq 16(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " adcx %%r14, %%rbx;" + " mov $0, %%r8;" + " mulxq 24(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " adcx %%rax, %%r14;" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + " adcx %%r8, %%rax;" + /* Compute src1[2] * src2 */ - " movq 16(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 24(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" + " movq 16(%0), %%rdx;" + " mulxq 0(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " adcxq 16(%2), %%r8;" + " movq %%r8, 16(%2);" + " mulxq 8(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " adcx %%rbx, %%r10;" + " movq %%r10, 24(%2);" + " mulxq 16(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " adcx %%r14, %%rbx;" + " mov $0, %%r8;" + " mulxq 24(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " adcx %%rax, %%r14;" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + " adcx %%r8, %%rax;" + /* Compute src1[3] * src2 */ - " movq 24(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 32(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 40(%0);" " mov $0, %%r8;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 48(%0);" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 56(%0);" + " movq 24(%0), %%rdx;" + " mulxq 0(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " adcxq 24(%2), %%r8;" + " movq %%r8, 24(%2);" + " mulxq 8(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " adcx %%rbx, %%r10;" + " movq %%r10, 32(%2);" + " mulxq 16(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " adcx %%r14, %%rbx;" + " movq %%rbx, 40(%2);" + " mov $0, %%r8;" + " mulxq 24(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " adcx %%rax, %%r14;" + " movq %%r14, 48(%2);" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + " adcx %%r8, %%rax;" + " movq %%rax, 56(%2);" /* Compute the raw multiplication tmp[1] <- f1[1] * f2[1] */ /* Compute src1[0] * src2 */ - " movq 32(%1), %%rdx;" - " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 64(%0);" - " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 72(%0);" - " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" - " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" + " movq 32(%0), %%rdx;" + " mulxq 32(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " movq %%r8, 64(%2);" + " mulxq 40(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " movq %%r10, 72(%2);" + " mulxq 48(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " mulxq 56(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + /* Compute src1[1] * src2 */ - " movq 40(%1), %%rdx;" - " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 72(%0), %%r8;" " movq %%r8, 72(%0);" - " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 80(%0);" - " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" - " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" + " movq 40(%0), %%rdx;" + " mulxq 32(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " adcxq 72(%2), %%r8;" + " movq %%r8, 72(%2);" + " mulxq 40(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " adcx %%rbx, %%r10;" + " movq %%r10, 80(%2);" + " mulxq 48(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " adcx %%r14, %%rbx;" + " mov $0, %%r8;" + " mulxq 56(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " adcx %%rax, %%r14;" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + " adcx %%r8, %%rax;" + /* Compute src1[2] * src2 */ - " movq 48(%1), %%rdx;" - " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 80(%0), %%r8;" " movq %%r8, 80(%0);" - " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 88(%0);" - " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" - " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" + " movq 48(%0), %%rdx;" + " mulxq 32(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " adcxq 80(%2), %%r8;" + " movq %%r8, 80(%2);" + " mulxq 40(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " adcx %%rbx, %%r10;" + " movq %%r10, 88(%2);" + " mulxq 48(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " adcx %%r14, %%rbx;" + " mov $0, %%r8;" + " mulxq 56(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " adcx %%rax, %%r14;" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + " adcx %%r8, %%rax;" + /* Compute src1[3] * src2 */ - " movq 56(%1), %%rdx;" - " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 88(%0), %%r8;" " movq %%r8, 88(%0);" - " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 96(%0);" - " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 104(%0);" " mov $0, %%r8;" - " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 112(%0);" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 120(%0);" + " movq 56(%0), %%rdx;" + " mulxq 32(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " adcxq 88(%2), %%r8;" + " movq %%r8, 88(%2);" + " mulxq 40(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " adcx %%rbx, %%r10;" + " movq %%r10, 96(%2);" + " mulxq 48(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " adcx %%r14, %%rbx;" + " movq %%rbx, 104(%2);" + " mov $0, %%r8;" + " mulxq 56(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " adcx %%rax, %%r14;" + " movq %%r14, 112(%2);" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + " adcx %%r8, %%rax;" + " movq %%rax, 120(%2);" + /* Line up pointers */ - " mov %0, %1;" " mov %2, %0;" + " mov %3, %2;" /* Wrap the results back into the field */ /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ " mov $38, %%rdx;" - " mulxq 32(%1), %%r8, %%r13;" - " xor %k3, %k3;" - " adoxq 0(%1), %%r8;" - " mulxq 40(%1), %%r9, %%rbx;" + " mulxq 32(%0), %%r8, %%r13;" + " xor %k1, %k1;" + " adoxq 0(%0), %%r8;" + " mulxq 40(%0), %%r9, %%rbx;" " adcx %%r13, %%r9;" - " adoxq 8(%1), %%r9;" - " mulxq 48(%1), %%r10, %%r13;" + " adoxq 8(%0), %%r9;" + " mulxq 48(%0), %%r10, %%r13;" " adcx %%rbx, %%r10;" - " adoxq 16(%1), %%r10;" - " mulxq 56(%1), %%r11, %%rax;" + " adoxq 16(%0), %%r10;" + " mulxq 56(%0), %%r11, %%rax;" " adcx %%r13, %%r11;" - " adoxq 24(%1), %%r11;" - " adcx %3, %%rax;" - " adox %3, %%rax;" + " adoxq 24(%0), %%r11;" + " adcx %1, %%rax;" + " adox %1, %%rax;" " imul %%rdx, %%rax;" /* Step 2: Fold the carry back into dst */ " add %%rax, %%r8;" - " adcx %3, %%r9;" - " movq %%r9, 8(%0);" - " adcx %3, %%r10;" - " movq %%r10, 16(%0);" - " adcx %3, %%r11;" - " movq %%r11, 24(%0);" + " adcx %1, %%r9;" + " movq %%r9, 8(%2);" + " adcx %1, %%r10;" + " movq %%r10, 16(%2);" + " adcx %1, %%r11;" + " movq %%r11, 24(%2);" /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ " mov $0, %%rax;" " cmovc %%rdx, %%rax;" " add %%rax, %%r8;" - " movq %%r8, 0(%0);" + " movq %%r8, 0(%2);" /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ " mov $38, %%rdx;" - " mulxq 96(%1), %%r8, %%r13;" - " xor %k3, %k3;" - " adoxq 64(%1), %%r8;" - " mulxq 104(%1), %%r9, %%rbx;" + " mulxq 96(%0), %%r8, %%r13;" + " xor %k1, %k1;" + " adoxq 64(%0), %%r8;" + " mulxq 104(%0), %%r9, %%rbx;" " adcx %%r13, %%r9;" - " adoxq 72(%1), %%r9;" - " mulxq 112(%1), %%r10, %%r13;" + " adoxq 72(%0), %%r9;" + " mulxq 112(%0), %%r10, %%r13;" " adcx %%rbx, %%r10;" - " adoxq 80(%1), %%r10;" - " mulxq 120(%1), %%r11, %%rax;" + " adoxq 80(%0), %%r10;" + " mulxq 120(%0), %%r11, %%rax;" " adcx %%r13, %%r11;" - " adoxq 88(%1), %%r11;" - " adcx %3, %%rax;" - " adox %3, %%rax;" + " adoxq 88(%0), %%r11;" + " adcx %1, %%rax;" + " adox %1, %%rax;" " imul %%rdx, %%rax;" /* Step 2: Fold the carry back into dst */ " add %%rax, %%r8;" - " adcx %3, %%r9;" - " movq %%r9, 40(%0);" - " adcx %3, %%r10;" - " movq %%r10, 48(%0);" - " adcx %3, %%r11;" - " movq %%r11, 56(%0);" + " adcx %1, %%r9;" + " movq %%r9, 40(%2);" + " adcx %1, %%r10;" + " movq %%r10, 48(%2);" + " adcx %1, %%r11;" + " movq %%r11, 56(%2);" /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ " mov $0, %%rax;" " cmovc %%rdx, %%rax;" " add %%rax, %%r8;" - " movq %%r8, 32(%0);" - : "+&r" (tmp), "+&r" (f1), "+&r" (out), "+&r" (f2) - : - : "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "memory", "cc" - ); + " movq %%r8, 32(%2);" + : "+&r"(f1), "+&r"(f2), "+&r"(tmp) + : "r"(out) + : "%rax", "%rbx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13", + "%r14", "memory", "cc"); } -/* Computes the field multiplication of four-element f1 with value in f2 */ +/* Computes the field multiplication of four-element f1 with value in f2 + * Requires f2 to be smaller than 2^17 */ static inline void fmul_scalar(u64 *out, const u64 *f1, u64 f2) { register u64 f2_r asm("rdx") = f2; asm volatile( /* Compute the raw multiplication of f1*f2 */ - " mulxq 0(%2), %%r8, %%rcx;" /* f1[0]*f2 */ - " mulxq 8(%2), %%r9, %%rbx;" /* f1[1]*f2 */ + " mulxq 0(%2), %%r8, %%rcx;" /* f1[0]*f2 */ + " mulxq 8(%2), %%r9, %%rbx;" /* f1[1]*f2 */ " add %%rcx, %%r9;" " mov $0, %%rcx;" - " mulxq 16(%2), %%r10, %%r13;" /* f1[2]*f2 */ + " mulxq 16(%2), %%r10, %%r13;" /* f1[2]*f2 */ " adcx %%rbx, %%r10;" - " mulxq 24(%2), %%r11, %%rax;" /* f1[3]*f2 */ + " mulxq 24(%2), %%r11, %%rax;" /* f1[3]*f2 */ " adcx %%r13, %%r11;" " adcx %%rcx, %%rax;" @@ -418,17 +576,17 @@ static inline void fmul_scalar(u64 *out, const u64 *f1, u64 f2) " cmovc %%rdx, %%rax;" " add %%rax, %%r8;" " movq %%r8, 0(%1);" - : "+&r" (f2_r) - : "r" (out), "r" (f1) - : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "memory", "cc" - ); + : "+&r"(f2_r) + : "r"(out), "r"(f1) + : "%rax", "%rbx", "%rcx", "%r8", "%r9", "%r10", "%r11", "%r13", + "memory", "cc"); } /* Computes p1 <- bit ? p2 : p1 in constant time */ static inline void cswap2(u64 bit, const u64 *p1, const u64 *p2) { asm volatile( - /* Invert the polarity of bit to match cmov expectations */ + /* Transfer bit into CF flag */ " add $18446744073709551615, %0;" /* cswap p1[0], p2[0] */ @@ -502,10 +660,9 @@ static inline void cswap2(u64 bit, const u64 *p1, const u64 *p2) " cmovc %%r10, %%r9;" " movq %%r8, 56(%1);" " movq %%r9, 56(%2);" - : "+&r" (bit) - : "r" (p1), "r" (p2) - : "%r8", "%r9", "%r10", "memory", "cc" - ); + : "+&r"(bit) + : "r"(p1), "r"(p2) + : "%r8", "%r9", "%r10", "memory", "cc"); } /* Computes the square of a field element: out <- f * f @@ -516,15 +673,22 @@ static inline void fsqr(u64 *out, const u64 *f, u64 *tmp) /* Compute the raw multiplication: tmp <- f * f */ /* Step 1: Compute all partial products */ - " movq 0(%1), %%rdx;" /* f[0] */ - " mulxq 8(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */ - " mulxq 16(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */ - " mulxq 24(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */ - " movq 24(%1), %%rdx;" /* f[3] */ - " mulxq 8(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */ - " mulxq 16(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */ - " movq 8(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */ - " mulxq 16(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */ + " movq 0(%0), %%rdx;" /* f[0] */ + " mulxq 8(%0), %%r8, %%r14;" + " xor %%r15d, %%r15d;" /* f[1]*f[0] */ + " mulxq 16(%0), %%r9, %%r10;" + " adcx %%r14, %%r9;" /* f[2]*f[0] */ + " mulxq 24(%0), %%rax, %%rcx;" + " adcx %%rax, %%r10;" /* f[3]*f[0] */ + " movq 24(%0), %%rdx;" /* f[3] */ + " mulxq 8(%0), %%r11, %%rbx;" + " adcx %%rcx, %%r11;" /* f[1]*f[3] */ + " mulxq 16(%0), %%rax, %%r13;" + " adcx %%rax, %%rbx;" /* f[2]*f[3] */ + " movq 8(%0), %%rdx;" + " adcx %%r15, %%r13;" /* f1 */ + " mulxq 16(%0), %%rax, %%rcx;" + " mov $0, %%r14;" /* f[2]*f[1] */ /* Step 2: Compute two parallel carry chains */ " xor %%r15d, %%r15d;" @@ -542,39 +706,50 @@ static inline void fsqr(u64 *out, const u64 *f, u64 *tmp) " adcx %%r14, %%r14;" /* Step 3: Compute intermediate squares */ - " movq 0(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */ - " movq %%rax, 0(%0);" - " add %%rcx, %%r8;" " movq %%r8, 8(%0);" - " movq 8(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */ - " adcx %%rax, %%r9;" " movq %%r9, 16(%0);" - " adcx %%rcx, %%r10;" " movq %%r10, 24(%0);" - " movq 16(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */ - " adcx %%rax, %%r11;" " movq %%r11, 32(%0);" - " adcx %%rcx, %%rbx;" " movq %%rbx, 40(%0);" - " movq 24(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */ - " adcx %%rax, %%r13;" " movq %%r13, 48(%0);" - " adcx %%rcx, %%r14;" " movq %%r14, 56(%0);" + " movq 0(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */ + " movq %%rax, 0(%1);" + " add %%rcx, %%r8;" + " movq %%r8, 8(%1);" + " movq 8(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */ + " adcx %%rax, %%r9;" + " movq %%r9, 16(%1);" + " adcx %%rcx, %%r10;" + " movq %%r10, 24(%1);" + " movq 16(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */ + " adcx %%rax, %%r11;" + " movq %%r11, 32(%1);" + " adcx %%rcx, %%rbx;" + " movq %%rbx, 40(%1);" + " movq 24(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */ + " adcx %%rax, %%r13;" + " movq %%r13, 48(%1);" + " adcx %%rcx, %%r14;" + " movq %%r14, 56(%1);" /* Line up pointers */ - " mov %0, %1;" - " mov %2, %0;" + " mov %1, %0;" + " mov %2, %1;" /* Wrap the result back into the field */ /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ " mov $38, %%rdx;" - " mulxq 32(%1), %%r8, %%r13;" + " mulxq 32(%0), %%r8, %%r13;" " xor %%ecx, %%ecx;" - " adoxq 0(%1), %%r8;" - " mulxq 40(%1), %%r9, %%rbx;" + " adoxq 0(%0), %%r8;" + " mulxq 40(%0), %%r9, %%rbx;" " adcx %%r13, %%r9;" - " adoxq 8(%1), %%r9;" - " mulxq 48(%1), %%r10, %%r13;" + " adoxq 8(%0), %%r9;" + " mulxq 48(%0), %%r10, %%r13;" " adcx %%rbx, %%r10;" - " adoxq 16(%1), %%r10;" - " mulxq 56(%1), %%r11, %%rax;" + " adoxq 16(%0), %%r10;" + " mulxq 56(%0), %%r11, %%rax;" " adcx %%r13, %%r11;" - " adoxq 24(%1), %%r11;" + " adoxq 24(%0), %%r11;" " adcx %%rcx, %%rax;" " adox %%rcx, %%rax;" " imul %%rdx, %%rax;" @@ -582,40 +757,47 @@ static inline void fsqr(u64 *out, const u64 *f, u64 *tmp) /* Step 2: Fold the carry back into dst */ " add %%rax, %%r8;" " adcx %%rcx, %%r9;" - " movq %%r9, 8(%0);" + " movq %%r9, 8(%1);" " adcx %%rcx, %%r10;" - " movq %%r10, 16(%0);" + " movq %%r10, 16(%1);" " adcx %%rcx, %%r11;" - " movq %%r11, 24(%0);" + " movq %%r11, 24(%1);" /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ " mov $0, %%rax;" " cmovc %%rdx, %%rax;" " add %%rax, %%r8;" - " movq %%r8, 0(%0);" - : "+&r" (tmp), "+&r" (f), "+&r" (out) - : - : "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "%r15", "memory", "cc" - ); + " movq %%r8, 0(%1);" + : "+&r"(f), "+&r"(tmp) + : "r"(out) + : "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", + "%r13", "%r14", "%r15", "memory", "cc"); } /* Computes two field squarings: - * out[0] <- f[0] * f[0] - * out[1] <- f[1] * f[1] + * out[0] <- f[0] * f[0] + * out[1] <- f[1] * f[1] * Uses the 16-element buffer tmp for intermediate results */ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp) { asm volatile( /* Step 1: Compute all partial products */ - " movq 0(%1), %%rdx;" /* f[0] */ - " mulxq 8(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */ - " mulxq 16(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */ - " mulxq 24(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */ - " movq 24(%1), %%rdx;" /* f[3] */ - " mulxq 8(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */ - " mulxq 16(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */ - " movq 8(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */ - " mulxq 16(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */ + " movq 0(%0), %%rdx;" /* f[0] */ + " mulxq 8(%0), %%r8, %%r14;" + " xor %%r15d, %%r15d;" /* f[1]*f[0] */ + " mulxq 16(%0), %%r9, %%r10;" + " adcx %%r14, %%r9;" /* f[2]*f[0] */ + " mulxq 24(%0), %%rax, %%rcx;" + " adcx %%rax, %%r10;" /* f[3]*f[0] */ + " movq 24(%0), %%rdx;" /* f[3] */ + " mulxq 8(%0), %%r11, %%rbx;" + " adcx %%rcx, %%r11;" /* f[1]*f[3] */ + " mulxq 16(%0), %%rax, %%r13;" + " adcx %%rax, %%rbx;" /* f[2]*f[3] */ + " movq 8(%0), %%rdx;" + " adcx %%r15, %%r13;" /* f1 */ + " mulxq 16(%0), %%rax, %%rcx;" + " mov $0, %%r14;" /* f[2]*f[1] */ /* Step 2: Compute two parallel carry chains */ " xor %%r15d, %%r15d;" @@ -633,29 +815,47 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp) " adcx %%r14, %%r14;" /* Step 3: Compute intermediate squares */ - " movq 0(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */ - " movq %%rax, 0(%0);" - " add %%rcx, %%r8;" " movq %%r8, 8(%0);" - " movq 8(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */ - " adcx %%rax, %%r9;" " movq %%r9, 16(%0);" - " adcx %%rcx, %%r10;" " movq %%r10, 24(%0);" - " movq 16(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */ - " adcx %%rax, %%r11;" " movq %%r11, 32(%0);" - " adcx %%rcx, %%rbx;" " movq %%rbx, 40(%0);" - " movq 24(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */ - " adcx %%rax, %%r13;" " movq %%r13, 48(%0);" - " adcx %%rcx, %%r14;" " movq %%r14, 56(%0);" + " movq 0(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */ + " movq %%rax, 0(%1);" + " add %%rcx, %%r8;" + " movq %%r8, 8(%1);" + " movq 8(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */ + " adcx %%rax, %%r9;" + " movq %%r9, 16(%1);" + " adcx %%rcx, %%r10;" + " movq %%r10, 24(%1);" + " movq 16(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */ + " adcx %%rax, %%r11;" + " movq %%r11, 32(%1);" + " adcx %%rcx, %%rbx;" + " movq %%rbx, 40(%1);" + " movq 24(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */ + " adcx %%rax, %%r13;" + " movq %%r13, 48(%1);" + " adcx %%rcx, %%r14;" + " movq %%r14, 56(%1);" /* Step 1: Compute all partial products */ - " movq 32(%1), %%rdx;" /* f[0] */ - " mulxq 40(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */ - " mulxq 48(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */ - " mulxq 56(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */ - " movq 56(%1), %%rdx;" /* f[3] */ - " mulxq 40(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */ - " mulxq 48(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */ - " movq 40(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */ - " mulxq 48(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */ + " movq 32(%0), %%rdx;" /* f[0] */ + " mulxq 40(%0), %%r8, %%r14;" + " xor %%r15d, %%r15d;" /* f[1]*f[0] */ + " mulxq 48(%0), %%r9, %%r10;" + " adcx %%r14, %%r9;" /* f[2]*f[0] */ + " mulxq 56(%0), %%rax, %%rcx;" + " adcx %%rax, %%r10;" /* f[3]*f[0] */ + " movq 56(%0), %%rdx;" /* f[3] */ + " mulxq 40(%0), %%r11, %%rbx;" + " adcx %%rcx, %%r11;" /* f[1]*f[3] */ + " mulxq 48(%0), %%rax, %%r13;" + " adcx %%rax, %%rbx;" /* f[2]*f[3] */ + " movq 40(%0), %%rdx;" + " adcx %%r15, %%r13;" /* f1 */ + " mulxq 48(%0), %%rax, %%rcx;" + " mov $0, %%r14;" /* f[2]*f[1] */ /* Step 2: Compute two parallel carry chains */ " xor %%r15d, %%r15d;" @@ -673,37 +873,48 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp) " adcx %%r14, %%r14;" /* Step 3: Compute intermediate squares */ - " movq 32(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */ - " movq %%rax, 64(%0);" - " add %%rcx, %%r8;" " movq %%r8, 72(%0);" - " movq 40(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */ - " adcx %%rax, %%r9;" " movq %%r9, 80(%0);" - " adcx %%rcx, %%r10;" " movq %%r10, 88(%0);" - " movq 48(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */ - " adcx %%rax, %%r11;" " movq %%r11, 96(%0);" - " adcx %%rcx, %%rbx;" " movq %%rbx, 104(%0);" - " movq 56(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */ - " adcx %%rax, %%r13;" " movq %%r13, 112(%0);" - " adcx %%rcx, %%r14;" " movq %%r14, 120(%0);" + " movq 32(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */ + " movq %%rax, 64(%1);" + " add %%rcx, %%r8;" + " movq %%r8, 72(%1);" + " movq 40(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */ + " adcx %%rax, %%r9;" + " movq %%r9, 80(%1);" + " adcx %%rcx, %%r10;" + " movq %%r10, 88(%1);" + " movq 48(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */ + " adcx %%rax, %%r11;" + " movq %%r11, 96(%1);" + " adcx %%rcx, %%rbx;" + " movq %%rbx, 104(%1);" + " movq 56(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */ + " adcx %%rax, %%r13;" + " movq %%r13, 112(%1);" + " adcx %%rcx, %%r14;" + " movq %%r14, 120(%1);" /* Line up pointers */ - " mov %0, %1;" - " mov %2, %0;" + " mov %1, %0;" + " mov %2, %1;" /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ " mov $38, %%rdx;" - " mulxq 32(%1), %%r8, %%r13;" + " mulxq 32(%0), %%r8, %%r13;" " xor %%ecx, %%ecx;" - " adoxq 0(%1), %%r8;" - " mulxq 40(%1), %%r9, %%rbx;" + " adoxq 0(%0), %%r8;" + " mulxq 40(%0), %%r9, %%rbx;" " adcx %%r13, %%r9;" - " adoxq 8(%1), %%r9;" - " mulxq 48(%1), %%r10, %%r13;" + " adoxq 8(%0), %%r9;" + " mulxq 48(%0), %%r10, %%r13;" " adcx %%rbx, %%r10;" - " adoxq 16(%1), %%r10;" - " mulxq 56(%1), %%r11, %%rax;" + " adoxq 16(%0), %%r10;" + " mulxq 56(%0), %%r11, %%rax;" " adcx %%r13, %%r11;" - " adoxq 24(%1), %%r11;" + " adoxq 24(%0), %%r11;" " adcx %%rcx, %%rax;" " adox %%rcx, %%rax;" " imul %%rdx, %%rax;" @@ -711,32 +922,32 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp) /* Step 2: Fold the carry back into dst */ " add %%rax, %%r8;" " adcx %%rcx, %%r9;" - " movq %%r9, 8(%0);" + " movq %%r9, 8(%1);" " adcx %%rcx, %%r10;" - " movq %%r10, 16(%0);" + " movq %%r10, 16(%1);" " adcx %%rcx, %%r11;" - " movq %%r11, 24(%0);" + " movq %%r11, 24(%1);" /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ " mov $0, %%rax;" " cmovc %%rdx, %%rax;" " add %%rax, %%r8;" - " movq %%r8, 0(%0);" + " movq %%r8, 0(%1);" /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ " mov $38, %%rdx;" - " mulxq 96(%1), %%r8, %%r13;" + " mulxq 96(%0), %%r8, %%r13;" " xor %%ecx, %%ecx;" - " adoxq 64(%1), %%r8;" - " mulxq 104(%1), %%r9, %%rbx;" + " adoxq 64(%0), %%r8;" + " mulxq 104(%0), %%r9, %%rbx;" " adcx %%r13, %%r9;" - " adoxq 72(%1), %%r9;" - " mulxq 112(%1), %%r10, %%r13;" + " adoxq 72(%0), %%r9;" + " mulxq 112(%0), %%r10, %%r13;" " adcx %%rbx, %%r10;" - " adoxq 80(%1), %%r10;" - " mulxq 120(%1), %%r11, %%rax;" + " adoxq 80(%0), %%r10;" + " mulxq 120(%0), %%r11, %%rax;" " adcx %%r13, %%r11;" - " adoxq 88(%1), %%r11;" + " adoxq 88(%0), %%r11;" " adcx %%rcx, %%rax;" " adox %%rcx, %%rax;" " imul %%rdx, %%rax;" @@ -744,21 +955,21 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp) /* Step 2: Fold the carry back into dst */ " add %%rax, %%r8;" " adcx %%rcx, %%r9;" - " movq %%r9, 40(%0);" + " movq %%r9, 40(%1);" " adcx %%rcx, %%r10;" - " movq %%r10, 48(%0);" + " movq %%r10, 48(%1);" " adcx %%rcx, %%r11;" - " movq %%r11, 56(%0);" + " movq %%r11, 56(%1);" /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ " mov $0, %%rax;" " cmovc %%rdx, %%rax;" " add %%rax, %%r8;" - " movq %%r8, 32(%0);" - : "+&r" (tmp), "+&r" (f), "+&r" (out) - : - : "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "%r15", "memory", "cc" - ); + " movq %%r8, 32(%1);" + : "+&r"(f), "+&r"(tmp) + : "r"(out) + : "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", + "%r13", "%r14", "%r15", "memory", "cc"); } static void point_add_and_double(u64 *q, u64 *p01_tmp1, u64 *tmp2) From eca568a39481728224edd5c2053ead5479dd8e07 Mon Sep 17 00:00:00 2001 From: Corentin Labbe Date: Wed, 15 Dec 2021 10:42:39 +0000 Subject: [PATCH 099/145] crypto: omap - increase priority of DES/3DES Give the same priority of OMAP DES/3DES than OMAP AES for being sure it is picked before software implementation. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/omap-des.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index be77656864e3..538aff80869f 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -735,7 +735,7 @@ static struct skcipher_alg algs_ecb_cbc[] = { { .base.cra_name = "ecb(des)", .base.cra_driver_name = "ecb-des-omap", - .base.cra_priority = 100, + .base.cra_priority = 300, .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, .base.cra_blocksize = DES_BLOCK_SIZE, @@ -752,7 +752,7 @@ static struct skcipher_alg algs_ecb_cbc[] = { { .base.cra_name = "cbc(des)", .base.cra_driver_name = "cbc-des-omap", - .base.cra_priority = 100, + .base.cra_priority = 300, .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, .base.cra_blocksize = DES_BLOCK_SIZE, @@ -770,7 +770,7 @@ static struct skcipher_alg algs_ecb_cbc[] = { { .base.cra_name = "ecb(des3_ede)", .base.cra_driver_name = "ecb-des3-omap", - .base.cra_priority = 100, + .base.cra_priority = 300, .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, @@ -787,7 +787,7 @@ static struct skcipher_alg algs_ecb_cbc[] = { { .base.cra_name = "cbc(des3_ede)", .base.cra_driver_name = "cbc-des3-omap", - .base.cra_priority = 100, + .base.cra_priority = 300, .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, From 3954ab6d9fce7df915a35a6ad4d657753340b011 Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Thu, 16 Dec 2021 15:48:14 +0800 Subject: [PATCH 100/145] crypto: octeontx2 - Use swap() instead of swap_engines() Fix the following coccicheck warning: ./drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c:1523:16-17: WARNING opportunity for swap(). Reported-by: Abaci Robot Signed-off-by: Jiapeng Chong Signed-off-by: Herbert Xu --- drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c index 57307eac541c..9aef37972246 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c @@ -1514,16 +1514,6 @@ delete_grps: return ret; } -static void swap_engines(struct otx2_cpt_engines *engsl, - struct otx2_cpt_engines *engsr) -{ - struct otx2_cpt_engines engs; - - engs = *engsl; - *engsl = *engsr; - *engsr = engs; -} - int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf, struct devlink_param_gset_ctx *ctx) { @@ -1624,7 +1614,7 @@ int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf, } /* Keep SE engines at zero index */ if (engs[1].type == OTX2_CPT_SE_TYPES) - swap_engines(&engs[0], &engs[1]); + swap(engs[0], engs[1]); } mutex_lock(&eng_grps->lock); From e0441e2be1553d34341bc5b60d279f3561d1b8b7 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Thu, 16 Dec 2021 09:13:11 +0000 Subject: [PATCH 101/145] crypto: qat - get compression extended capabilities Get compression extended capabilities mask from firmware through the init/admin channel. These capabilities are stored in the accel_dev structure and will be communicated to VF through the PFVF channel. Signed-off-by: Giovanni Cabiddu Signed-off-by: Marco Chiappero Reviewed-by: Fiona Trahe Reviewed-by: Marco Chiappero Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_accel_devices.h | 1 + drivers/crypto/qat/qat_common/adf_admin.c | 37 +++++++++++++++++++ .../qat/qat_common/icp_qat_fw_init_admin.h | 4 +- 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index a1809a7d1c90..2c380fa10a09 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -198,6 +198,7 @@ struct adf_hw_device_data { u32 fuses; u32 straps; u32 accel_capabilities_mask; + u32 extended_dc_capabilities; u32 instance_id; u16 accel_mask; u32 ae_mask; diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c index 43680e178242..c381b89d548d 100644 --- a/drivers/crypto/qat/qat_common/adf_admin.c +++ b/drivers/crypto/qat/qat_common/adf_admin.c @@ -194,6 +194,35 @@ static int adf_set_fw_constants(struct adf_accel_dev *accel_dev) return adf_send_admin(accel_dev, &req, &resp, ae_mask); } +static int adf_get_dc_capabilities(struct adf_accel_dev *accel_dev, + u32 *capabilities) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + struct icp_qat_fw_init_admin_resp resp; + struct icp_qat_fw_init_admin_req req; + unsigned long ae_mask; + unsigned long ae; + int ret; + + /* Target only service accelerator engines */ + ae_mask = hw_device->ae_mask & ~hw_device->admin_ae_mask; + + memset(&req, 0, sizeof(req)); + memset(&resp, 0, sizeof(resp)); + req.cmd_id = ICP_QAT_FW_COMP_CAPABILITY_GET; + + *capabilities = 0; + for_each_set_bit(ae, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { + ret = adf_send_admin(accel_dev, &req, &resp, 1ULL << ae); + if (ret) + return ret; + + *capabilities |= resp.extended_features; + } + + return 0; +} + /** * adf_send_admin_init() - Function sends init message to FW * @accel_dev: Pointer to acceleration device. @@ -204,8 +233,16 @@ static int adf_set_fw_constants(struct adf_accel_dev *accel_dev) */ int adf_send_admin_init(struct adf_accel_dev *accel_dev) { + u32 dc_capabilities = 0; int ret; + ret = adf_get_dc_capabilities(accel_dev, &dc_capabilities); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Cannot get dc capabilities\n"); + return ret; + } + accel_dev->hw_device->extended_dc_capabilities = dc_capabilities; + ret = adf_set_fw_constants(accel_dev); if (ret) return ret; diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h b/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h index f05ad17fbdd6..afe59a7684ac 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h @@ -14,7 +14,8 @@ enum icp_qat_fw_init_admin_cmd_id { ICP_QAT_FW_COUNTERS_GET = 5, ICP_QAT_FW_LOOPBACK = 6, ICP_QAT_FW_HEARTBEAT_SYNC = 7, - ICP_QAT_FW_HEARTBEAT_GET = 8 + ICP_QAT_FW_HEARTBEAT_GET = 8, + ICP_QAT_FW_COMP_CAPABILITY_GET = 9, }; enum icp_qat_fw_init_admin_resp_status { @@ -52,6 +53,7 @@ struct icp_qat_fw_init_admin_resp { __u16 version_minor_num; __u16 version_major_num; }; + __u32 extended_features; }; __u64 opaque_data; union { From 547bde7bd4ecd78f36f98744e6c9a0999e52da5a Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Thu, 16 Dec 2021 09:13:12 +0000 Subject: [PATCH 102/145] crypto: qat - set CIPHER capability for QAT GEN2 Set the CIPHER capability for QAT GEN2 devices if the hardware supports it. This is done if both the CIPHER and the AUTHENTICATION engines are available on the device. Signed-off-by: Giovanni Cabiddu Signed-off-by: Marco Chiappero Reviewed-by: Fiona Trahe Reviewed-by: Marco Chiappero Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_gen2_hw_data.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index 3b48fdaaff6d..3ea26f2f4a22 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -211,17 +211,23 @@ u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev) u32 legfuses; u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | - ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + ICP_ACCEL_CAPABILITIES_AUTHENTICATION | + ICP_ACCEL_CAPABILITIES_CIPHER; /* Read accelerator capabilities mask */ pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, &legfuses); - if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE) + /* A set bit in legfuses means the feature is OFF in this SKU */ + if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE) { capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } if (legfuses & ICP_ACCEL_MASK_PKE_SLICE) capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; - if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE) + if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE) { capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } if ((straps | fuses) & ADF_POWERGATE_PKE) capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; From cfe4894eccdc7fa5cd35bf34e918614d06ecce38 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Thu, 16 Dec 2021 09:13:13 +0000 Subject: [PATCH 103/145] crypto: qat - set COMPRESSION capability for QAT GEN2 Enhance the device capability detection for QAT GEN2 devices to detect if a device supports the compression service. This is done by checking both the fuse and the strap registers for c62x and c3xxx and only the fuse register for dh895xcc. Signed-off-by: Giovanni Cabiddu Signed-off-by: Marco Chiappero Reviewed-by: Fiona Trahe Reviewed-by: Marco Chiappero Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_gen2_hw_data.c | 8 +++++++- drivers/crypto/qat/qat_common/adf_gen2_hw_data.h | 1 + drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index 3ea26f2f4a22..688dd6f53b0b 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -212,7 +212,8 @@ u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev) u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | ICP_ACCEL_CAPABILITIES_AUTHENTICATION | - ICP_ACCEL_CAPABILITIES_CIPHER; + ICP_ACCEL_CAPABILITIES_CIPHER | + ICP_ACCEL_CAPABILITIES_COMPRESSION; /* Read accelerator capabilities mask */ pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, &legfuses); @@ -228,10 +229,15 @@ u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev) capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; } + if (legfuses & ICP_ACCEL_MASK_COMPRESS_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; if ((straps | fuses) & ADF_POWERGATE_PKE) capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + if ((straps | fuses) & ADF_POWERGATE_DC) + capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; + return capabilities; } EXPORT_SYMBOL_GPL(adf_gen2_get_accel_cap); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index 448c97f740e7..7c2c17366460 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -113,6 +113,7 @@ do { \ (ADF_ARB_REG_SLOT * (index)), value) /* Power gating */ +#define ADF_POWERGATE_DC BIT(23) #define ADF_POWERGATE_PKE BIT(24) /* WDT timers diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 27d4cab65dd8..2d18279191d7 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -69,6 +69,8 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE) capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + if (legfuses & ICP_ACCEL_MASK_COMPRESS_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; return capabilities; } From 4b44d28c715d3db2c8de1e4ec9ccfdf230001007 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Thu, 16 Dec 2021 09:13:14 +0000 Subject: [PATCH 104/145] crypto: qat - extend crypto capability detection for 4xxx Extended the capability detection logic for 4xxx devices. Mask out unsupported algorithms and services based on the value read in the fuse register. This includes only capabilities for the crypto service. Signed-off-by: Giovanni Cabiddu Signed-off-by: Marco Chiappero Reviewed-by: Fiona Trahe Reviewed-by: Marco Chiappero Signed-off-by: Herbert Xu --- .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 29 +++++++++++++++++-- drivers/crypto/qat/qat_common/icp_qat_hw.h | 9 +++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c index 4658b7bf76da..d320c50c4561 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -98,18 +98,41 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) u32 fusectl1; u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_CIPHER | ICP_ACCEL_CAPABILITIES_AUTHENTICATION | + ICP_ACCEL_CAPABILITIES_SHA3 | + ICP_ACCEL_CAPABILITIES_SHA3_EXT | + ICP_ACCEL_CAPABILITIES_HKDF | + ICP_ACCEL_CAPABILITIES_ECEDMONT | + ICP_ACCEL_CAPABILITIES_CHACHA_POLY | + ICP_ACCEL_CAPABILITIES_AESGCM_SPC | ICP_ACCEL_CAPABILITIES_AES_V2; /* Read accelerator capabilities mask */ pci_read_config_dword(pdev, ADF_4XXX_FUSECTL1_OFFSET, &fusectl1); - if (fusectl1 & ICP_ACCEL_4XXX_MASK_CIPHER_SLICE) + /* A set bit in fusectl1 means the feature is OFF in this SKU */ + if (fusectl1 & ICP_ACCEL_4XXX_MASK_CIPHER_SLICE) { capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; - if (fusectl1 & ICP_ACCEL_4XXX_MASK_AUTH_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_HKDF; + capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + if (fusectl1 & ICP_ACCEL_4XXX_MASK_UCS_SLICE) { + capabilities &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY; + capabilities &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC; + capabilities &= ~ICP_ACCEL_CAPABILITIES_AES_V2; + capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + if (fusectl1 & ICP_ACCEL_4XXX_MASK_AUTH_SLICE) { capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; - if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_SHA3; + capabilities &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT; + capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE) { capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + capabilities &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT; + } return capabilities; } diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw.h b/drivers/crypto/qat/qat_common/icp_qat_hw.h index e39e8a2d51a7..5770b2b2c09e 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hw.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hw.h @@ -91,7 +91,14 @@ enum icp_qat_capabilities_mask { ICP_ACCEL_CAPABILITIES_RAND = BIT(7), ICP_ACCEL_CAPABILITIES_ZUC = BIT(8), ICP_ACCEL_CAPABILITIES_SHA3 = BIT(9), - /* Bits 10-25 are currently reserved */ + /* Bits 10-11 are currently reserved */ + ICP_ACCEL_CAPABILITIES_HKDF = BIT(12), + ICP_ACCEL_CAPABILITIES_ECEDMONT = BIT(13), + /* Bit 14 is currently reserved */ + ICP_ACCEL_CAPABILITIES_SHA3_EXT = BIT(15), + ICP_ACCEL_CAPABILITIES_AESGCM_SPC = BIT(16), + ICP_ACCEL_CAPABILITIES_CHACHA_POLY = BIT(17), + /* Bits 18-25 are currently reserved */ ICP_ACCEL_CAPABILITIES_AES_V2 = BIT(26) }; From 03125541ca29fda7b98bea08dfed68ae0e39b46c Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Thu, 16 Dec 2021 09:13:15 +0000 Subject: [PATCH 105/145] crypto: qat - support the reset of ring pairs on PF Add support for triggering a HW reset of a specific ring pair. Being a device specific feature, add it to the hw_device_data struct. This feature is supported only by QAT GEN4 devices. This patch is based on earlier work done by Zelin Deng. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 2 + .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.h | 1 + .../crypto/qat/qat_common/adf_accel_devices.h | 2 + .../crypto/qat/qat_common/adf_gen4_hw_data.c | 53 +++++++++++++++++++ .../crypto/qat/qat_common/adf_gen4_hw_data.h | 9 ++++ 5 files changed, 67 insertions(+) diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c index d320c50c4561..0d1603894af4 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -239,6 +239,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) hw_data->dev_class = &adf_4xxx_class; hw_data->instance_id = adf_4xxx_class.instances++; hw_data->num_banks = ADF_4XXX_ETR_MAX_BANKS; + hw_data->num_banks_per_vf = ADF_4XXX_NUM_BANKS_PER_VF; hw_data->num_rings_per_bank = ADF_4XXX_NUM_RINGS_PER_BANK; hw_data->num_accel = ADF_4XXX_MAX_ACCELERATORS; hw_data->num_engines = ADF_4XXX_MAX_ACCELENGINES; @@ -279,6 +280,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) hw_data->pfvf_ops.enable_comms = adf_pfvf_comms_disabled; hw_data->pfvf_ops.get_vf2pf_sources = get_vf2pf_sources; hw_data->disable_iov = adf_disable_sriov; + hw_data->ring_pair_reset = adf_gen4_ring_pair_reset; adf_gen4_init_hw_csr_ops(&hw_data->csr_ops); } diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h index 924bac6feb37..a0c67752317f 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h @@ -37,6 +37,7 @@ /* Bank and ring configuration */ #define ADF_4XXX_NUM_RINGS_PER_BANK 2 +#define ADF_4XXX_NUM_BANKS_PER_VF 4 /* Error source registers */ #define ADF_4XXX_ERRSOU0 (0x41A200) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 2c380fa10a09..cc8b10b23145 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -186,6 +186,7 @@ struct adf_hw_device_data { bool enable); void (*enable_ints)(struct adf_accel_dev *accel_dev); void (*set_ssm_wdtimer)(struct adf_accel_dev *accel_dev); + int (*ring_pair_reset)(struct adf_accel_dev *accel_dev, u32 bank_nr); void (*reset_device)(struct adf_accel_dev *accel_dev); void (*set_msix_rttable)(struct adf_accel_dev *accel_dev); char *(*uof_get_name)(u32 obj_num); @@ -206,6 +207,7 @@ struct adf_hw_device_data { u16 tx_rings_mask; u8 tx_rx_gap; u8 num_banks; + u16 num_banks_per_vf; u8 num_rings_per_bank; u8 num_accel; u8 num_logical_accel; diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c index e3157df8a653..c7808ff2aba1 100644 --- a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2020 Intel Corporation */ +#include #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "adf_gen4_hw_data.h" @@ -146,3 +147,55 @@ int adf_pfvf_comms_disabled(struct adf_accel_dev *accel_dev) return 0; } EXPORT_SYMBOL_GPL(adf_pfvf_comms_disabled); + +static int reset_ring_pair(void __iomem *csr, u32 bank_number) +{ + u32 status; + int ret; + + /* Write rpresetctl register BIT(0) as 1 + * Since rpresetctl registers have no RW fields, no need to preserve + * values for other bits. Just write directly. + */ + ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETCTL(bank_number), + ADF_WQM_CSR_RPRESETCTL_RESET); + + /* Read rpresetsts register and wait for rp reset to complete */ + ret = read_poll_timeout(ADF_CSR_RD, status, + status & ADF_WQM_CSR_RPRESETSTS_STATUS, + ADF_RPRESET_POLL_DELAY_US, + ADF_RPRESET_POLL_TIMEOUT_US, true, + csr, ADF_WQM_CSR_RPRESETSTS(bank_number)); + if (!ret) { + /* When rp reset is done, clear rpresetsts */ + ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number), + ADF_WQM_CSR_RPRESETSTS_STATUS); + } + + return ret; +} + +int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 etr_bar_id = hw_data->get_etr_bar_id(hw_data); + void __iomem *csr; + int ret; + + if (bank_number >= hw_data->num_banks) + return -EINVAL; + + dev_dbg(&GET_DEV(accel_dev), + "ring pair reset for bank:%d\n", bank_number); + + csr = (&GET_BARS(accel_dev)[etr_bar_id])->virt_addr; + ret = reset_ring_pair(csr, bank_number); + if (ret) + dev_err(&GET_DEV(accel_dev), + "ring pair reset failed (timeout)\n"); + else + dev_dbg(&GET_DEV(accel_dev), "ring pair reset successful\n"); + + return ret; +} +EXPORT_SYMBOL_GPL(adf_gen4_ring_pair_reset); diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h index b8fca1ff7aab..449d6a5976a9 100644 --- a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h @@ -106,6 +106,15 @@ do { \ #define ADF_SSMWDTPKEL_OFFSET 0x58 #define ADF_SSMWDTPKEH_OFFSET 0x60 +/* Ring reset */ +#define ADF_RPRESET_POLL_TIMEOUT_US (5 * USEC_PER_SEC) +#define ADF_RPRESET_POLL_DELAY_US 20 +#define ADF_WQM_CSR_RPRESETCTL_RESET BIT(0) +#define ADF_WQM_CSR_RPRESETCTL(bank) (0x6000 + ((bank) << 3)) +#define ADF_WQM_CSR_RPRESETSTS_STATUS BIT(0) +#define ADF_WQM_CSR_RPRESETSTS(bank) (ADF_WQM_CSR_RPRESETCTL(bank) + 4) + void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev); void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); +int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number); #endif From 448588adcdf4a025b0b5517a5c9557b036b3cf79 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Thu, 16 Dec 2021 09:13:16 +0000 Subject: [PATCH 106/145] crypto: qat - add the adf_get_pmisc_base() helper function Add and use the new helper function adf_get_pmisc_base() where convenient. Also: - remove no longer shared variables - leverage other utilities, such as GET_PFVF_OPS(), as a consequence - consistently use the "pmisc_addr" name for the returned value of this new helper Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_admin.c | 10 ++--- .../crypto/qat/qat_common/adf_common_drv.h | 11 +++++ .../crypto/qat/qat_common/adf_gen2_hw_data.c | 39 ++++++------------ drivers/crypto/qat/qat_common/adf_gen2_pfvf.c | 26 +++++------- .../crypto/qat/qat_common/adf_gen4_hw_data.c | 9 +--- drivers/crypto/qat/qat_common/adf_isr.c | 28 ++++--------- drivers/crypto/qat/qat_common/adf_vf_isr.c | 14 ++----- drivers/crypto/qat/qat_common/qat_hal.c | 41 +++++-------------- 8 files changed, 62 insertions(+), 116 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c index c381b89d548d..498eb6f690e3 100644 --- a/drivers/crypto/qat/qat_common/adf_admin.c +++ b/drivers/crypto/qat/qat_common/adf_admin.c @@ -255,9 +255,7 @@ int adf_init_admin_comms(struct adf_accel_dev *accel_dev) { struct adf_admin_comms *admin; struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_bar *pmisc = - &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; - void __iomem *csr = pmisc->virt_addr; + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); struct admin_info admin_csrs_info; u32 mailbox_offset, adminmsg_u, adminmsg_l; void __iomem *mailbox; @@ -291,13 +289,13 @@ int adf_init_admin_comms(struct adf_accel_dev *accel_dev) hw_data->get_admin_info(&admin_csrs_info); mailbox_offset = admin_csrs_info.mailbox_offset; - mailbox = csr + mailbox_offset; + mailbox = pmisc_addr + mailbox_offset; adminmsg_u = admin_csrs_info.admin_msg_ur; adminmsg_l = admin_csrs_info.admin_msg_lr; reg_val = (u64)admin->phy_addr; - ADF_CSR_WR(csr, adminmsg_u, upper_32_bits(reg_val)); - ADF_CSR_WR(csr, adminmsg_l, lower_32_bits(reg_val)); + ADF_CSR_WR(pmisc_addr, adminmsg_u, upper_32_bits(reg_val)); + ADF_CSR_WR(pmisc_addr, adminmsg_l, lower_32_bits(reg_val)); mutex_init(&admin->lock); admin->mailbox_addr = mailbox; diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 2d8b72085505..5212891344a9 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -243,4 +243,15 @@ static inline void adf_flush_vf_wq(struct adf_accel_dev *accel_dev) } #endif + +static inline void __iomem *adf_get_pmisc_base(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *pmisc; + + pmisc = &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; + + return pmisc->virt_addr; +} + #endif diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index 688dd6f53b0b..57035b7dd4b2 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2020 Intel Corporation */ +#include "adf_common_drv.h" #include "adf_gen2_hw_data.h" #include "icp_qat_hw.h" #include @@ -25,31 +26,29 @@ EXPORT_SYMBOL_GPL(adf_gen2_get_num_aes); void adf_gen2_enable_error_correction(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_bar *misc_bar = &GET_BARS(accel_dev) - [hw_data->get_misc_bar_id(hw_data)]; + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); unsigned long accel_mask = hw_data->accel_mask; unsigned long ae_mask = hw_data->ae_mask; - void __iomem *csr = misc_bar->virt_addr; unsigned int val, i; /* Enable Accel Engine error detection & correction */ for_each_set_bit(i, &ae_mask, hw_data->num_engines) { - val = ADF_CSR_RD(csr, ADF_GEN2_AE_CTX_ENABLES(i)); + val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_AE_CTX_ENABLES(i)); val |= ADF_GEN2_ENABLE_AE_ECC_ERR; - ADF_CSR_WR(csr, ADF_GEN2_AE_CTX_ENABLES(i), val); - val = ADF_CSR_RD(csr, ADF_GEN2_AE_MISC_CONTROL(i)); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_AE_CTX_ENABLES(i), val); + val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_AE_MISC_CONTROL(i)); val |= ADF_GEN2_ENABLE_AE_ECC_PARITY_CORR; - ADF_CSR_WR(csr, ADF_GEN2_AE_MISC_CONTROL(i), val); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_AE_MISC_CONTROL(i), val); } /* Enable shared memory error detection & correction */ for_each_set_bit(i, &accel_mask, hw_data->num_accel) { - val = ADF_CSR_RD(csr, ADF_GEN2_UERRSSMSH(i)); + val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_UERRSSMSH(i)); val |= ADF_GEN2_ERRSSMSH_EN; - ADF_CSR_WR(csr, ADF_GEN2_UERRSSMSH(i), val); - val = ADF_CSR_RD(csr, ADF_GEN2_CERRSSMSH(i)); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_UERRSSMSH(i), val); + val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_CERRSSMSH(i)); val |= ADF_GEN2_ERRSSMSH_EN; - ADF_CSR_WR(csr, ADF_GEN2_CERRSSMSH(i), val); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_CERRSSMSH(i), val); } } EXPORT_SYMBOL_GPL(adf_gen2_enable_error_correction); @@ -57,15 +56,9 @@ EXPORT_SYMBOL_GPL(adf_gen2_enable_error_correction); void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs) { - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - void __iomem *pmisc_addr; - struct adf_bar *pmisc; - int pmisc_id, i; + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); u32 reg; - - pmisc_id = hw_data->get_misc_bar_id(hw_data); - pmisc = &GET_BARS(accel_dev)[pmisc_id]; - pmisc_addr = pmisc->virt_addr; + int i; /* Set/Unset Valid bit in AE Thread to PCIe Function Mapping Group A */ for (i = 0; i < num_a_regs; i++) { @@ -245,18 +238,12 @@ EXPORT_SYMBOL_GPL(adf_gen2_get_accel_cap); void adf_gen2_set_ssm_wdtimer(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); u32 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE; u32 timer_val = ADF_SSM_WDT_DEFAULT_VALUE; unsigned long accel_mask = hw_data->accel_mask; - void __iomem *pmisc_addr; - struct adf_bar *pmisc; - int pmisc_id; u32 i = 0; - pmisc_id = hw_data->get_misc_bar_id(hw_data); - pmisc = &GET_BARS(accel_dev)[pmisc_id]; - pmisc_addr = pmisc->virt_addr; - /* Configures WDT timers */ for_each_set_bit(i, &accel_mask, hw_data->num_accel) { /* Enable WDT for sym and dc */ diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c index 099e39808d13..5ac69ece34a8 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c @@ -75,15 +75,12 @@ static void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) { - struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - void __iomem *pmisc_bar_addr = - pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr; - u32 val, pfvf_offset, count = 0; - u32 local_in_use_mask, local_in_use_pattern; - u32 remote_in_use_mask, remote_in_use_pattern; - struct mutex *lock; /* lock preventing concurrent acces of CSR */ + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); unsigned int retries = ADF_PFVF_MSG_MAX_RETRIES; + u32 remote_in_use_mask, remote_in_use_pattern; + u32 local_in_use_mask, local_in_use_pattern; + u32 val, pfvf_offset, count = 0; + struct mutex *lock; /* lock preventing concurrent acces of CSR */ u32 int_bit; int ret; @@ -114,7 +111,7 @@ start: ret = 0; /* Check if the PFVF CSR is in use by remote function */ - val = ADF_CSR_RD(pmisc_bar_addr, pfvf_offset); + val = ADF_CSR_RD(pmisc_addr, pfvf_offset); if ((val & remote_in_use_mask) == remote_in_use_pattern) { dev_dbg(&GET_DEV(accel_dev), "PFVF CSR in use by remote function\n"); @@ -122,12 +119,12 @@ start: } /* Attempt to get ownership of the PFVF CSR */ - ADF_CSR_WR(pmisc_bar_addr, pfvf_offset, msg | int_bit); + ADF_CSR_WR(pmisc_addr, pfvf_offset, msg | int_bit); /* Wait for confirmation from remote func it received the message */ do { msleep(ADF_PFVF_MSG_ACK_DELAY); - val = ADF_CSR_RD(pmisc_bar_addr, pfvf_offset); + val = ADF_CSR_RD(pmisc_addr, pfvf_offset); } while ((val & int_bit) && (count++ < ADF_PFVF_MSG_ACK_MAX_RETRY)); if (val & int_bit) { @@ -143,7 +140,7 @@ start: } /* Finished with the PFVF CSR; relinquish it and leave msg in CSR */ - ADF_CSR_WR(pmisc_bar_addr, pfvf_offset, val & ~local_in_use_mask); + ADF_CSR_WR(pmisc_addr, pfvf_offset, val & ~local_in_use_mask); out: mutex_unlock(lock); return ret; @@ -160,10 +157,7 @@ retry: static u32 adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev, u8 vf_nr) { - struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - void __iomem *pmisc_addr = - pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr; + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); u32 pfvf_offset; u32 msg_origin; u32 int_bit; diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c index c7808ff2aba1..3148a62938fd 100644 --- a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c @@ -111,20 +111,13 @@ static inline void adf_gen4_unpack_ssm_wdtimer(u64 value, u32 *upper, void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev) { - struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); u64 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE; u64 timer_val = ADF_SSM_WDT_DEFAULT_VALUE; u32 ssm_wdt_pke_high = 0; u32 ssm_wdt_pke_low = 0; u32 ssm_wdt_high = 0; u32 ssm_wdt_low = 0; - void __iomem *pmisc_addr; - struct adf_bar *pmisc; - int pmisc_id; - - pmisc_id = hw_data->get_misc_bar_id(hw_data); - pmisc = &GET_BARS(accel_dev)[pmisc_id]; - pmisc_addr = pmisc->virt_addr; /* Convert 64bit WDT timer value into 32bit values for * mmio write to 32bit CSRs. diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c index 522e0c10d9b9..4ca482aa69f7 100644 --- a/drivers/crypto/qat/qat_common/adf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_isr.c @@ -57,54 +57,42 @@ static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr) #ifdef CONFIG_PCI_IOV void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) { - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data); - struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id]; - void __iomem *pmisc_addr = pmisc->virt_addr; + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); unsigned long flags; spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); - hw_data->pfvf_ops.enable_vf2pf_interrupts(pmisc_addr, vf_mask); + GET_PFVF_OPS(accel_dev)->enable_vf2pf_interrupts(pmisc_addr, vf_mask); spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); } void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) { - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data); - struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id]; - void __iomem *pmisc_addr = pmisc->virt_addr; + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); unsigned long flags; spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); - hw_data->pfvf_ops.disable_vf2pf_interrupts(pmisc_addr, vf_mask); + GET_PFVF_OPS(accel_dev)->disable_vf2pf_interrupts(pmisc_addr, vf_mask); spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); } static void adf_disable_vf2pf_interrupts_irq(struct adf_accel_dev *accel_dev, u32 vf_mask) { - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data); - struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id]; - void __iomem *pmisc_addr = pmisc->virt_addr; + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); spin_lock(&accel_dev->pf.vf2pf_ints_lock); - hw_data->pfvf_ops.disable_vf2pf_interrupts(pmisc_addr, vf_mask); + GET_PFVF_OPS(accel_dev)->disable_vf2pf_interrupts(pmisc_addr, vf_mask); spin_unlock(&accel_dev->pf.vf2pf_ints_lock); } static bool adf_handle_vf2pf_int(struct adf_accel_dev *accel_dev) { - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - int bar_id = hw_data->get_misc_bar_id(hw_data); - struct adf_bar *pmisc = &GET_BARS(accel_dev)[bar_id]; - void __iomem *pmisc_addr = pmisc->virt_addr; + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); bool irq_handled = false; unsigned long vf_mask; /* Get the interrupt sources triggered by VFs */ - vf_mask = hw_data->pfvf_ops.get_vf2pf_sources(pmisc_addr); + vf_mask = GET_PFVF_OPS(accel_dev)->get_vf2pf_sources(pmisc_addr); if (vf_mask) { struct adf_accel_vf_info *vf_info; diff --git a/drivers/crypto/qat/qat_common/adf_vf_isr.c b/drivers/crypto/qat/qat_common/adf_vf_isr.c index fe094178f065..86c3bd0c9c2b 100644 --- a/drivers/crypto/qat/qat_common/adf_vf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_vf_isr.c @@ -30,22 +30,16 @@ struct adf_vf_stop_data { void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) { - struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - void __iomem *pmisc_bar_addr = - pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr; + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); - ADF_CSR_WR(pmisc_bar_addr, ADF_VINTMSK_OFFSET, 0x0); + ADF_CSR_WR(pmisc_addr, ADF_VINTMSK_OFFSET, 0x0); } void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) { - struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - void __iomem *pmisc_bar_addr = - pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr; + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); - ADF_CSR_WR(pmisc_bar_addr, ADF_VINTMSK_OFFSET, 0x2); + ADF_CSR_WR(pmisc_addr, ADF_VINTMSK_OFFSET, 0x2); } EXPORT_SYMBOL_GPL(adf_disable_pf2vf_interrupts); diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 12ca6b8764aa..4bfd8f3566f7 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -684,8 +684,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, { struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_bar *misc_bar = - &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)]; + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); unsigned int max_en_ae_id = 0; struct adf_bar *sram_bar; unsigned int csr_val = 0; @@ -715,18 +714,12 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->fcu_loaded_ae_csr = FCU_AE_LOADED_4XXX; handle->chip_info->fcu_loaded_ae_pos = 0; - handle->hal_cap_g_ctl_csr_addr_v = - (void __iomem *)((uintptr_t)misc_bar->virt_addr + - ICP_QAT_CAP_OFFSET_4XXX); - handle->hal_cap_ae_xfer_csr_addr_v = - (void __iomem *)((uintptr_t)misc_bar->virt_addr + - ICP_QAT_AE_OFFSET_4XXX); - handle->hal_ep_csr_addr_v = - (void __iomem *)((uintptr_t)misc_bar->virt_addr + - ICP_QAT_EP_OFFSET_4XXX); + handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET_4XXX; + handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET_4XXX; + handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET_4XXX; handle->hal_cap_ae_local_csr_addr_v = (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v - + LOCAL_TO_XFER_REG_OFFSET); + + LOCAL_TO_XFER_REG_OFFSET); break; case PCI_DEVICE_ID_INTEL_QAT_C62X: case PCI_DEVICE_ID_INTEL_QAT_C3XXX: @@ -749,15 +742,9 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO; handle->chip_info->fcu_loaded_ae_csr = FCU_STATUS; handle->chip_info->fcu_loaded_ae_pos = FCU_LOADED_AE_POS; - handle->hal_cap_g_ctl_csr_addr_v = - (void __iomem *)((uintptr_t)misc_bar->virt_addr + - ICP_QAT_CAP_OFFSET); - handle->hal_cap_ae_xfer_csr_addr_v = - (void __iomem *)((uintptr_t)misc_bar->virt_addr + - ICP_QAT_AE_OFFSET); - handle->hal_ep_csr_addr_v = - (void __iomem *)((uintptr_t)misc_bar->virt_addr + - ICP_QAT_EP_OFFSET); + handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET; + handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET; + handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET; handle->hal_cap_ae_local_csr_addr_v = (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + LOCAL_TO_XFER_REG_OFFSET); @@ -782,15 +769,9 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->fcu_dram_addr_lo = 0; handle->chip_info->fcu_loaded_ae_csr = 0; handle->chip_info->fcu_loaded_ae_pos = 0; - handle->hal_cap_g_ctl_csr_addr_v = - (void __iomem *)((uintptr_t)misc_bar->virt_addr + - ICP_QAT_CAP_OFFSET); - handle->hal_cap_ae_xfer_csr_addr_v = - (void __iomem *)((uintptr_t)misc_bar->virt_addr + - ICP_QAT_AE_OFFSET); - handle->hal_ep_csr_addr_v = - (void __iomem *)((uintptr_t)misc_bar->virt_addr + - ICP_QAT_EP_OFFSET); + handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET; + handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET; + handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET; handle->hal_cap_ae_local_csr_addr_v = (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + LOCAL_TO_XFER_REG_OFFSET); From 6ed942ed3c47b3ebb4e8de3ff10e761cdf82ba74 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Thu, 16 Dec 2021 09:13:17 +0000 Subject: [PATCH 107/145] crypto: qat - make PFVF message construction direction agnostic Currently PFVF messages are created upfront in the CSR format, that is PF2VF messages starting from bit 0 and VF2PF from bit 16, and passed along unmodified to the PFVF send function. Refactor the code to allow the VF2PF messages to be built starting from bit 0, as for the PF2VF messages. Shift the VF to PF messages just before sending them, and refactor the send logic to handle messages properly depending on the direction. As a result all the messages are composed the same way regardless of the direction. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_gen2_pfvf.c | 129 +++++++++++++----- drivers/crypto/qat/qat_common/adf_pfvf_msg.h | 26 ++-- .../crypto/qat/qat_common/adf_pfvf_pf_msg.c | 4 +- .../crypto/qat/qat_common/adf_pfvf_pf_proto.c | 10 +- .../crypto/qat/qat_common/adf_pfvf_vf_msg.c | 20 +-- .../crypto/qat/qat_common/adf_pfvf_vf_proto.c | 2 +- 6 files changed, 120 insertions(+), 71 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c index 5ac69ece34a8..2e0b9ac27393 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c @@ -17,6 +17,14 @@ #define ADF_GEN2_PF_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) #define ADF_GEN2_VF_PF2VF_OFFSET 0x200 +#define ADF_GEN2_CSR_IN_USE 0x6AC2 +#define ADF_GEN2_CSR_IN_USE_MASK 0xFFFE + +enum gen2_csr_pos { + ADF_GEN2_CSR_PF2VF_OFFSET = 0, + ADF_GEN2_CSR_VF2PF_OFFSET = 16, +}; + #define ADF_PFVF_MSG_ACK_DELAY 2 #define ADF_PFVF_MSG_ACK_MAX_RETRY 100 @@ -72,38 +80,81 @@ static void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, } } +static u32 gen2_csr_get_int_bit(enum gen2_csr_pos offset) +{ + return ADF_PFVF_INT << offset; +} + +static u32 gen2_csr_msg_to_position(u32 csr_msg, enum gen2_csr_pos offset) +{ + return (csr_msg & 0xFFFF) << offset; +} + +static u32 gen2_csr_msg_from_position(u32 csr_val, enum gen2_csr_pos offset) +{ + return (csr_val >> offset) & 0xFFFF; +} + +static bool gen2_csr_is_in_use(u32 msg, enum gen2_csr_pos offset) +{ + return ((msg >> offset) & ADF_GEN2_CSR_IN_USE_MASK) == ADF_GEN2_CSR_IN_USE; +} + +static void gen2_csr_clear_in_use(u32 *msg, enum gen2_csr_pos offset) +{ + *msg &= ~(ADF_GEN2_CSR_IN_USE_MASK << offset); +} + +static void gen2_csr_set_in_use(u32 *msg, enum gen2_csr_pos offset) +{ + *msg |= (ADF_GEN2_CSR_IN_USE << offset); +} + +static bool is_legacy_user_pfvf_message(u32 msg) +{ + return !(msg & ADF_PFVF_MSGORIGIN_SYSTEM); +} + static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) { void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); unsigned int retries = ADF_PFVF_MSG_MAX_RETRIES; - u32 remote_in_use_mask, remote_in_use_pattern; - u32 local_in_use_mask, local_in_use_pattern; - u32 val, pfvf_offset, count = 0; + enum gen2_csr_pos remote_offset; + enum gen2_csr_pos local_offset; struct mutex *lock; /* lock preventing concurrent acces of CSR */ + u32 pfvf_offset; + u32 count = 0; u32 int_bit; + u32 csr_val; int ret; + /* Gen2 messages, both PF->VF and VF->PF, are all 16 bits long. This + * allows us to build and read messages as if they where all 0 based. + * However, send and receive are in a single shared 32 bits register, + * so we need to shift and/or mask the message half before decoding + * it and after encoding it. Which one to shift depends on the + * direction. + */ if (accel_dev->is_vf) { pfvf_offset = GET_PFVF_OPS(accel_dev)->get_vf2pf_offset(0); lock = &accel_dev->vf.vf2pf_lock; - local_in_use_mask = ADF_VF2PF_IN_USE_BY_VF_MASK; - local_in_use_pattern = ADF_VF2PF_IN_USE_BY_VF; - remote_in_use_mask = ADF_PF2VF_IN_USE_BY_PF_MASK; - remote_in_use_pattern = ADF_PF2VF_IN_USE_BY_PF; - int_bit = ADF_VF2PF_INT; + local_offset = ADF_GEN2_CSR_VF2PF_OFFSET; + remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET; } else { pfvf_offset = GET_PFVF_OPS(accel_dev)->get_pf2vf_offset(vf_nr); lock = &accel_dev->pf.vf_info[vf_nr].pf2vf_lock; - local_in_use_mask = ADF_PF2VF_IN_USE_BY_PF_MASK; - local_in_use_pattern = ADF_PF2VF_IN_USE_BY_PF; - remote_in_use_mask = ADF_VF2PF_IN_USE_BY_VF_MASK; - remote_in_use_pattern = ADF_VF2PF_IN_USE_BY_VF; - int_bit = ADF_PF2VF_INT; + local_offset = ADF_GEN2_CSR_PF2VF_OFFSET; + remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET; } - msg &= ~local_in_use_mask; - msg |= local_in_use_pattern; + int_bit = gen2_csr_get_int_bit(local_offset); + + /* Pre-calculate message, shifting it in place and setting + * the in use pattern + */ + msg = gen2_csr_msg_to_position(msg, local_offset); + gen2_csr_set_in_use(&msg, remote_offset); mutex_lock(lock); @@ -111,8 +162,8 @@ start: ret = 0; /* Check if the PFVF CSR is in use by remote function */ - val = ADF_CSR_RD(pmisc_addr, pfvf_offset); - if ((val & remote_in_use_mask) == remote_in_use_pattern) { + csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset); + if (gen2_csr_is_in_use(csr_val, local_offset)) { dev_dbg(&GET_DEV(accel_dev), "PFVF CSR in use by remote function\n"); goto retry; @@ -124,23 +175,25 @@ start: /* Wait for confirmation from remote func it received the message */ do { msleep(ADF_PFVF_MSG_ACK_DELAY); - val = ADF_CSR_RD(pmisc_addr, pfvf_offset); - } while ((val & int_bit) && (count++ < ADF_PFVF_MSG_ACK_MAX_RETRY)); + csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset); + } while ((csr_val & int_bit) && (count++ < ADF_PFVF_MSG_ACK_MAX_RETRY)); - if (val & int_bit) { + if (csr_val & int_bit) { dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n"); - val &= ~int_bit; + csr_val &= ~int_bit; ret = -EIO; } - if (val != msg) { + if (csr_val != msg) { dev_dbg(&GET_DEV(accel_dev), "Collision - PFVF CSR overwritten by remote function\n"); goto retry; } /* Finished with the PFVF CSR; relinquish it and leave msg in CSR */ - ADF_CSR_WR(pmisc_addr, pfvf_offset, val & ~local_in_use_mask); + gen2_csr_clear_in_use(&csr_val, remote_offset); + ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val); + out: mutex_unlock(lock); return ret; @@ -158,39 +211,43 @@ retry: static u32 adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev, u8 vf_nr) { void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + enum gen2_csr_pos local_offset; u32 pfvf_offset; - u32 msg_origin; u32 int_bit; + u32 csr_val; u32 msg; if (accel_dev->is_vf) { pfvf_offset = GET_PFVF_OPS(accel_dev)->get_pf2vf_offset(0); - int_bit = ADF_PF2VF_INT; - msg_origin = ADF_PF2VF_MSGORIGIN_SYSTEM; + local_offset = ADF_GEN2_CSR_PF2VF_OFFSET; } else { pfvf_offset = GET_PFVF_OPS(accel_dev)->get_vf2pf_offset(vf_nr); - int_bit = ADF_VF2PF_INT; - msg_origin = ADF_VF2PF_MSGORIGIN_SYSTEM; + local_offset = ADF_GEN2_CSR_VF2PF_OFFSET; } + int_bit = gen2_csr_get_int_bit(local_offset); + /* Read message */ - msg = ADF_CSR_RD(pmisc_addr, pfvf_offset); - if (!(msg & int_bit)) { + csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset); + if (!(csr_val & int_bit)) { dev_info(&GET_DEV(accel_dev), - "Spurious PFVF interrupt, msg 0x%.8x. Ignored\n", msg); + "Spurious PFVF interrupt, msg 0x%.8x. Ignored\n", csr_val); return 0; } - /* Ignore legacy non-system (non-kernel) VF2PF messages */ - if (!(msg & msg_origin)) { + /* Extract the message from the CSR */ + msg = gen2_csr_msg_from_position(csr_val, local_offset); + + /* Ignore legacy non-system (non-kernel) messages */ + if (unlikely(is_legacy_user_pfvf_message(msg))) { dev_dbg(&GET_DEV(accel_dev), - "Ignored non-system message (0x%.8x);\n", msg); + "Ignored non-system message (0x%.8x);\n", csr_val); return 0; } /* To ACK, clear the INT bit */ - msg &= ~int_bit; - ADF_CSR_WR(pmisc_addr, pfvf_offset, msg); + csr_val &= ~int_bit; + ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val); return msg; } diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h index 8b476072df28..3ba88bcd0726 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h @@ -53,27 +53,19 @@ * adf_gen2_pfvf_send() in adf_pf2vf_msg.c). */ -/* PF->VF messages */ -#define ADF_PF2VF_INT BIT(0) -#define ADF_PF2VF_MSGORIGIN_SYSTEM BIT(1) -#define ADF_PF2VF_IN_USE_BY_PF 0x6AC20000 -#define ADF_PF2VF_IN_USE_BY_PF_MASK 0xFFFE0000 -#define ADF_PF2VF_MSGTYPE_MASK 0x0000003C -#define ADF_PF2VF_MSGTYPE_SHIFT 2 +/* PFVF message common bits */ +#define ADF_PFVF_INT BIT(0) +#define ADF_PFVF_MSGORIGIN_SYSTEM BIT(1) +#define ADF_PFVF_MSGTYPE_SHIFT 2 +#define ADF_PFVF_MSGTYPE_MASK 0x0F +/* PF->VF messages */ enum pf2vf_msgtype { ADF_PF2VF_MSGTYPE_RESTARTING = 0x01, ADF_PF2VF_MSGTYPE_VERSION_RESP = 0x02, }; /* VF->PF messages */ -#define ADF_VF2PF_INT BIT(16) -#define ADF_VF2PF_MSGORIGIN_SYSTEM BIT(17) -#define ADF_VF2PF_IN_USE_BY_VF 0x00006AC2 -#define ADF_VF2PF_IN_USE_BY_VF_MASK 0x0000FFFE -#define ADF_VF2PF_MSGTYPE_MASK 0x003C0000 -#define ADF_VF2PF_MSGTYPE_SHIFT 18 - enum vf2pf_msgtype { ADF_VF2PF_MSGTYPE_INIT = 0x03, ADF_VF2PF_MSGTYPE_SHUTDOWN = 0x04, @@ -90,10 +82,10 @@ enum pfvf_compatibility_version { /* PF->VF Version Response */ #define ADF_PF2VF_MINORVERSION_SHIFT 6 #define ADF_PF2VF_MAJORVERSION_SHIFT 10 -#define ADF_PF2VF_VERSION_RESP_VERS_MASK 0x00003FC0 #define ADF_PF2VF_VERSION_RESP_VERS_SHIFT 6 -#define ADF_PF2VF_VERSION_RESP_RESULT_MASK 0x0000C000 +#define ADF_PF2VF_VERSION_RESP_VERS_MASK 0xFF #define ADF_PF2VF_VERSION_RESP_RESULT_SHIFT 14 +#define ADF_PF2VF_VERSION_RESP_RESULT_MASK 0x03 enum pf2vf_compat_response { ADF_PF2VF_VF_COMPATIBLE = 0x01, @@ -102,6 +94,6 @@ enum pf2vf_compat_response { }; /* VF->PF Compatible Version Request */ -#define ADF_VF2PF_COMPAT_VER_REQ_SHIFT 22 +#define ADF_VF2PF_COMPAT_VER_REQ_SHIFT 6 #endif /* ADF_PFVF_MSG_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c index 647b82e6c4ba..4057d7d74d62 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c @@ -9,8 +9,8 @@ void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev) { struct adf_accel_vf_info *vf; - u32 msg = (ADF_PF2VF_MSGORIGIN_SYSTEM | - (ADF_PF2VF_MSGTYPE_RESTARTING << ADF_PF2VF_MSGTYPE_SHIFT)); + u32 msg = (ADF_PFVF_MSGORIGIN_SYSTEM | + (ADF_PF2VF_MSGTYPE_RESTARTING << ADF_PFVF_MSGTYPE_SHIFT)); int i, num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev)); for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) { diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c index 4f20dd35fcd4..fb477eb89fef 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c @@ -42,7 +42,7 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; u32 resp = 0; - switch ((msg & ADF_VF2PF_MSGTYPE_MASK) >> ADF_VF2PF_MSGTYPE_SHIFT) { + switch ((msg >> ADF_PFVF_MSGTYPE_SHIFT) & ADF_PFVF_MSGTYPE_MASK) { case ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ: { u8 vf_compat_ver = msg >> ADF_VF2PF_COMPAT_VER_REQ_SHIFT; @@ -57,9 +57,9 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, else compat = ADF_PF2VF_VF_COMPAT_UNKNOWN; - resp = ADF_PF2VF_MSGORIGIN_SYSTEM; + resp = ADF_PFVF_MSGORIGIN_SYSTEM; resp |= ADF_PF2VF_MSGTYPE_VERSION_RESP << - ADF_PF2VF_MSGTYPE_SHIFT; + ADF_PFVF_MSGTYPE_SHIFT; resp |= ADF_PFVF_COMPAT_THIS_VERSION << ADF_PF2VF_VERSION_RESP_VERS_SHIFT; resp |= compat << ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; @@ -76,9 +76,9 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, /* PF always newer than legacy VF */ compat = ADF_PF2VF_VF_COMPATIBLE; - resp = ADF_PF2VF_MSGORIGIN_SYSTEM; + resp = ADF_PFVF_MSGORIGIN_SYSTEM; resp |= ADF_PF2VF_MSGTYPE_VERSION_RESP << - ADF_PF2VF_MSGTYPE_SHIFT; + ADF_PFVF_MSGTYPE_SHIFT; /* Set legacy major and minor version num */ resp |= 1 << ADF_PF2VF_MAJORVERSION_SHIFT | 1 << ADF_PF2VF_MINORVERSION_SHIFT; diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c index 763581839902..c9e929651a7d 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c @@ -16,8 +16,8 @@ */ int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev) { - u32 msg = (ADF_VF2PF_MSGORIGIN_SYSTEM | - (ADF_VF2PF_MSGTYPE_INIT << ADF_VF2PF_MSGTYPE_SHIFT)); + u32 msg = (ADF_PFVF_MSGORIGIN_SYSTEM | + (ADF_VF2PF_MSGTYPE_INIT << ADF_PFVF_MSGTYPE_SHIFT)); if (adf_send_vf2pf_msg(accel_dev, msg)) { dev_err(&GET_DEV(accel_dev), @@ -39,8 +39,8 @@ EXPORT_SYMBOL_GPL(adf_vf2pf_notify_init); */ void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev) { - u32 msg = (ADF_VF2PF_MSGORIGIN_SYSTEM | - (ADF_VF2PF_MSGTYPE_SHUTDOWN << ADF_VF2PF_MSGTYPE_SHIFT)); + u32 msg = (ADF_PFVF_MSGORIGIN_SYSTEM | + (ADF_VF2PF_MSGTYPE_SHUTDOWN << ADF_PFVF_MSGTYPE_SHIFT)); if (test_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status)) if (adf_send_vf2pf_msg(accel_dev, msg)) @@ -57,8 +57,8 @@ int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) u32 resp; int ret; - msg = ADF_VF2PF_MSGORIGIN_SYSTEM; - msg |= ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ << ADF_VF2PF_MSGTYPE_SHIFT; + msg = ADF_PFVF_MSGORIGIN_SYSTEM; + msg |= ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ << ADF_PFVF_MSGTYPE_SHIFT; msg |= ADF_PFVF_COMPAT_THIS_VERSION << ADF_VF2PF_COMPAT_VER_REQ_SHIFT; BUILD_BUG_ON(ADF_PFVF_COMPAT_THIS_VERSION > 255); @@ -69,10 +69,10 @@ int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) return ret; } - pf_version = (resp & ADF_PF2VF_VERSION_RESP_VERS_MASK) - >> ADF_PF2VF_VERSION_RESP_VERS_SHIFT; - compat = (resp & ADF_PF2VF_VERSION_RESP_RESULT_MASK) - >> ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + pf_version = (resp >> ADF_PF2VF_VERSION_RESP_VERS_SHIFT) + & ADF_PF2VF_VERSION_RESP_VERS_MASK; + compat = (resp >> ADF_PF2VF_VERSION_RESP_RESULT_SHIFT) + & ADF_PF2VF_VERSION_RESP_RESULT_MASK; /* Response from PF received, check compatibility */ switch (compat) { diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c index 9c7489ed122c..f8d1c7d0ec4e 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c @@ -88,7 +88,7 @@ int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, u32 msg, u32 *resp) static bool adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev, u32 msg) { - switch ((msg & ADF_PF2VF_MSGTYPE_MASK) >> ADF_PF2VF_MSGTYPE_SHIFT) { + switch ((msg >> ADF_PFVF_MSGTYPE_SHIFT) & ADF_PFVF_MSGTYPE_MASK) { case ADF_PF2VF_MSGTYPE_RESTARTING: dev_dbg(&GET_DEV(accel_dev), "Restarting message received from PF\n"); From 028042856802c4731c6afebe15d95fed4d39a614 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Thu, 16 Dec 2021 09:13:18 +0000 Subject: [PATCH 108/145] crypto: qat - make PFVF send and receive direction agnostic Currently PF and VF share the same send and receive logic for the PFVF protocol. However, the inner behaviour still depends on the specific direction, requiring a test to determine the if the sender is a PF or a VF. Moreover the vf_nr parameter is only required for PF2VF messages and ignored for the opposite direction. Make the GEN2 send and recv completely direction agnostic, by calculating and determining any direction specific input in the caller instead, and feeding the send and the receive functions with the same arguments for both PF and VF. In order to accommodate for this change, the API of the pfvf_ops send and recv has been modified to remove any reference to vf_nr. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_accel_devices.h | 5 +- drivers/crypto/qat/qat_common/adf_gen2_pfvf.c | 99 +++++++++++++------ .../crypto/qat/qat_common/adf_pfvf_pf_proto.c | 11 ++- .../crypto/qat/qat_common/adf_pfvf_vf_proto.c | 11 ++- 4 files changed, 89 insertions(+), 37 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index cc8b10b23145..d65d8dda8fda 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -154,8 +154,9 @@ struct adf_pfvf_ops { u32 (*get_vf2pf_sources)(void __iomem *pmisc_addr); void (*enable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask); void (*disable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask); - int (*send_msg)(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr); - u32 (*recv_msg)(struct adf_accel_dev *accel_dev, u8 vf_nr); + int (*send_msg)(struct adf_accel_dev *accel_dev, u32 msg, + u32 pfvf_offset, struct mutex *csr_lock); + u32 (*recv_msg)(struct adf_accel_dev *accel_dev, u32 pfvf_offset); }; struct adf_hw_device_data { diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c index 2e0b9ac27393..1e45f3230c19 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c @@ -115,15 +115,22 @@ static bool is_legacy_user_pfvf_message(u32 msg) return !(msg & ADF_PFVF_MSGORIGIN_SYSTEM); } +struct pfvf_gen2_params { + u32 pfvf_offset; + struct mutex *csr_lock; /* lock preventing concurrent access of CSR */ + enum gen2_csr_pos local_offset; + enum gen2_csr_pos remote_offset; +}; + static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev, u32 msg, - u8 vf_nr) + struct pfvf_gen2_params *params) { void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + enum gen2_csr_pos remote_offset = params->remote_offset; + enum gen2_csr_pos local_offset = params->local_offset; unsigned int retries = ADF_PFVF_MSG_MAX_RETRIES; - enum gen2_csr_pos remote_offset; - enum gen2_csr_pos local_offset; - struct mutex *lock; /* lock preventing concurrent acces of CSR */ - u32 pfvf_offset; + struct mutex *lock = params->csr_lock; + u32 pfvf_offset = params->pfvf_offset; u32 count = 0; u32 int_bit; u32 csr_val; @@ -136,17 +143,6 @@ static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev, u32 msg, * it and after encoding it. Which one to shift depends on the * direction. */ - if (accel_dev->is_vf) { - pfvf_offset = GET_PFVF_OPS(accel_dev)->get_vf2pf_offset(0); - lock = &accel_dev->vf.vf2pf_lock; - local_offset = ADF_GEN2_CSR_VF2PF_OFFSET; - remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET; - } else { - pfvf_offset = GET_PFVF_OPS(accel_dev)->get_pf2vf_offset(vf_nr); - lock = &accel_dev->pf.vf_info[vf_nr].pf2vf_lock; - local_offset = ADF_GEN2_CSR_PF2VF_OFFSET; - remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET; - } int_bit = gen2_csr_get_int_bit(local_offset); @@ -208,23 +204,16 @@ retry: } } -static u32 adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev, u8 vf_nr) +static u32 adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev, + struct pfvf_gen2_params *params) { void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); - enum gen2_csr_pos local_offset; - u32 pfvf_offset; + enum gen2_csr_pos local_offset = params->local_offset; + u32 pfvf_offset = params->pfvf_offset; u32 int_bit; u32 csr_val; u32 msg; - if (accel_dev->is_vf) { - pfvf_offset = GET_PFVF_OPS(accel_dev)->get_pf2vf_offset(0); - local_offset = ADF_GEN2_CSR_PF2VF_OFFSET; - } else { - pfvf_offset = GET_PFVF_OPS(accel_dev)->get_vf2pf_offset(vf_nr); - local_offset = ADF_GEN2_CSR_VF2PF_OFFSET; - } - int_bit = gen2_csr_get_int_bit(local_offset); /* Read message */ @@ -252,6 +241,54 @@ static u32 adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev, u8 vf_nr) return msg; } +static int adf_gen2_pf2vf_send(struct adf_accel_dev *accel_dev, u32 msg, + u32 pfvf_offset, struct mutex *csr_lock) +{ + struct pfvf_gen2_params params = { + .csr_lock = csr_lock, + .pfvf_offset = pfvf_offset, + .local_offset = ADF_GEN2_CSR_PF2VF_OFFSET, + .remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET, + }; + + return adf_gen2_pfvf_send(accel_dev, msg, ¶ms); +} + +static int adf_gen2_vf2pf_send(struct adf_accel_dev *accel_dev, u32 msg, + u32 pfvf_offset, struct mutex *csr_lock) +{ + struct pfvf_gen2_params params = { + .csr_lock = csr_lock, + .pfvf_offset = pfvf_offset, + .local_offset = ADF_GEN2_CSR_VF2PF_OFFSET, + .remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET, + }; + + return adf_gen2_pfvf_send(accel_dev, msg, ¶ms); +} + +static u32 adf_gen2_pf2vf_recv(struct adf_accel_dev *accel_dev, u32 pfvf_offset) +{ + struct pfvf_gen2_params params = { + .pfvf_offset = pfvf_offset, + .local_offset = ADF_GEN2_CSR_PF2VF_OFFSET, + .remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET, + }; + + return adf_gen2_pfvf_recv(accel_dev, ¶ms); +} + +static u32 adf_gen2_vf2pf_recv(struct adf_accel_dev *accel_dev, u32 pfvf_offset) +{ + struct pfvf_gen2_params params = { + .pfvf_offset = pfvf_offset, + .local_offset = ADF_GEN2_CSR_VF2PF_OFFSET, + .remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET, + }; + + return adf_gen2_pfvf_recv(accel_dev, ¶ms); +} + void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) { pfvf_ops->enable_comms = adf_enable_pf2vf_comms; @@ -260,8 +297,8 @@ void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) pfvf_ops->get_vf2pf_sources = adf_gen2_get_vf2pf_sources; pfvf_ops->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts; pfvf_ops->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts; - pfvf_ops->send_msg = adf_gen2_pfvf_send; - pfvf_ops->recv_msg = adf_gen2_pfvf_recv; + pfvf_ops->send_msg = adf_gen2_pf2vf_send; + pfvf_ops->recv_msg = adf_gen2_vf2pf_recv; } EXPORT_SYMBOL_GPL(adf_gen2_init_pf_pfvf_ops); @@ -270,7 +307,7 @@ void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) pfvf_ops->enable_comms = adf_enable_vf2pf_comms; pfvf_ops->get_pf2vf_offset = adf_gen2_vf_get_pfvf_offset; pfvf_ops->get_vf2pf_offset = adf_gen2_vf_get_pfvf_offset; - pfvf_ops->send_msg = adf_gen2_pfvf_send; - pfvf_ops->recv_msg = adf_gen2_pfvf_recv; + pfvf_ops->send_msg = adf_gen2_vf2pf_send; + pfvf_ops->recv_msg = adf_gen2_pf2vf_recv; } EXPORT_SYMBOL_GPL(adf_gen2_init_vf_pfvf_ops); diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c index fb477eb89fef..4e4daec2ae34 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c @@ -19,7 +19,11 @@ */ int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, u32 msg) { - return GET_PFVF_OPS(accel_dev)->send_msg(accel_dev, msg, vf_nr); + struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev); + u32 pfvf_offset = pfvf_ops->get_pf2vf_offset(vf_nr); + + return pfvf_ops->send_msg(accel_dev, msg, pfvf_offset, + &accel_dev->pf.vf_info[vf_nr].pf2vf_lock); } /** @@ -33,7 +37,10 @@ int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, u32 msg) */ static u32 adf_recv_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr) { - return GET_PFVF_OPS(accel_dev)->recv_msg(accel_dev, vf_nr); + struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev); + u32 pfvf_offset = pfvf_ops->get_vf2pf_offset(vf_nr); + + return pfvf_ops->recv_msg(accel_dev, pfvf_offset); } static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c index f8d1c7d0ec4e..56e8185a9630 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c @@ -27,7 +27,11 @@ */ int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg) { - return GET_PFVF_OPS(accel_dev)->send_msg(accel_dev, msg, 0); + struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev); + u32 pfvf_offset = pfvf_ops->get_vf2pf_offset(0); + + return pfvf_ops->send_msg(accel_dev, msg, pfvf_offset, + &accel_dev->vf.vf2pf_lock); } /** @@ -40,7 +44,10 @@ int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg) */ static u32 adf_recv_pf2vf_msg(struct adf_accel_dev *accel_dev) { - return GET_PFVF_OPS(accel_dev)->recv_msg(accel_dev, 0); + struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev); + u32 pfvf_offset = pfvf_ops->get_pf2vf_offset(0); + + return pfvf_ops->recv_msg(accel_dev, pfvf_offset); } /** From 0aeda694f1870f50900603c276423ffc05035f90 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Thu, 16 Dec 2021 09:13:19 +0000 Subject: [PATCH 109/145] crypto: qat - set PFVF_MSGORIGIN just before sending In this implementation of the PFVF protocol every egressing message must include the ADF_PFVF_MSGORIGIN_SYSTEM flag. However, this flag can be set on all the outbound messages just before sending them rather than at message build time, as currently done. Remove the unnecessary code duplication by setting the ADF_PFVF_MSGORIGIN_SYSTEM flag only once at send time in adf_gen2_pfvf_send(). Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_gen2_pfvf.c | 1 + drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c | 3 +-- drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c | 2 -- drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c | 12 +++++------- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c index 1e45f3230c19..97bef41ddc47 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c @@ -149,6 +149,7 @@ static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev, u32 msg, /* Pre-calculate message, shifting it in place and setting * the in use pattern */ + msg |= ADF_PFVF_MSGORIGIN_SYSTEM; msg = gen2_csr_msg_to_position(msg, local_offset); gen2_csr_set_in_use(&msg, remote_offset); diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c index 4057d7d74d62..f6d05cd29a82 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c @@ -9,8 +9,7 @@ void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev) { struct adf_accel_vf_info *vf; - u32 msg = (ADF_PFVF_MSGORIGIN_SYSTEM | - (ADF_PF2VF_MSGTYPE_RESTARTING << ADF_PFVF_MSGTYPE_SHIFT)); + u32 msg = ADF_PF2VF_MSGTYPE_RESTARTING << ADF_PFVF_MSGTYPE_SHIFT; int i, num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev)); for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) { diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c index 4e4daec2ae34..9f4baa9b14fc 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c @@ -64,7 +64,6 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, else compat = ADF_PF2VF_VF_COMPAT_UNKNOWN; - resp = ADF_PFVF_MSGORIGIN_SYSTEM; resp |= ADF_PF2VF_MSGTYPE_VERSION_RESP << ADF_PFVF_MSGTYPE_SHIFT; resp |= ADF_PFVF_COMPAT_THIS_VERSION << @@ -83,7 +82,6 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, /* PF always newer than legacy VF */ compat = ADF_PF2VF_VF_COMPATIBLE; - resp = ADF_PFVF_MSGORIGIN_SYSTEM; resp |= ADF_PF2VF_MSGTYPE_VERSION_RESP << ADF_PFVF_MSGTYPE_SHIFT; /* Set legacy major and minor version num */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c index c9e929651a7d..0463743a8d43 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c @@ -16,8 +16,7 @@ */ int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev) { - u32 msg = (ADF_PFVF_MSGORIGIN_SYSTEM | - (ADF_VF2PF_MSGTYPE_INIT << ADF_PFVF_MSGTYPE_SHIFT)); + u32 msg = ADF_VF2PF_MSGTYPE_INIT << ADF_PFVF_MSGTYPE_SHIFT; if (adf_send_vf2pf_msg(accel_dev, msg)) { dev_err(&GET_DEV(accel_dev), @@ -39,8 +38,7 @@ EXPORT_SYMBOL_GPL(adf_vf2pf_notify_init); */ void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev) { - u32 msg = (ADF_PFVF_MSGORIGIN_SYSTEM | - (ADF_VF2PF_MSGTYPE_SHUTDOWN << ADF_PFVF_MSGTYPE_SHIFT)); + u32 msg = ADF_VF2PF_MSGTYPE_SHUTDOWN << ADF_PFVF_MSGTYPE_SHIFT; if (test_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status)) if (adf_send_vf2pf_msg(accel_dev, msg)) @@ -57,9 +55,9 @@ int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) u32 resp; int ret; - msg = ADF_PFVF_MSGORIGIN_SYSTEM; - msg |= ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ << ADF_PFVF_MSGTYPE_SHIFT; - msg |= ADF_PFVF_COMPAT_THIS_VERSION << ADF_VF2PF_COMPAT_VER_REQ_SHIFT; + msg = ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ << ADF_PFVF_MSGTYPE_SHIFT | + ADF_PFVF_COMPAT_THIS_VERSION << ADF_VF2PF_COMPAT_VER_REQ_SHIFT; + BUILD_BUG_ON(ADF_PFVF_COMPAT_THIS_VERSION > 255); ret = adf_send_vf2pf_req(accel_dev, msg, &resp); From db1c034801c402b1f600ef7f753494479950f78e Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Thu, 16 Dec 2021 09:13:20 +0000 Subject: [PATCH 110/145] crypto: qat - abstract PFVF messages with struct pfvf_message This implementation of the PFVF protocol was designed around the GEN2 devices and its CSR format. In order to support future generations, which come with differently sized fields, change the definition of the PFVF message and make it abstract by means of a new pfvf_message struct. Also, introduce some utilities to translate to and from the new message format and the device specific CSR format. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/Makefile | 2 +- .../crypto/qat/qat_common/adf_accel_devices.h | 8 ++- drivers/crypto/qat/qat_common/adf_gen2_pfvf.c | 60 +++++++++++++------ drivers/crypto/qat/qat_common/adf_pfvf_msg.h | 19 +++--- .../crypto/qat/qat_common/adf_pfvf_pf_msg.c | 2 +- .../crypto/qat/qat_common/adf_pfvf_pf_proto.c | 51 +++++++--------- .../crypto/qat/qat_common/adf_pfvf_pf_proto.h | 2 +- .../crypto/qat/qat_common/adf_pfvf_utils.c | 50 ++++++++++++++++ .../crypto/qat/qat_common/adf_pfvf_utils.h | 24 ++++++++ .../crypto/qat/qat_common/adf_pfvf_vf_msg.c | 18 +++--- .../crypto/qat/qat_common/adf_pfvf_vf_proto.c | 26 ++++---- .../crypto/qat/qat_common/adf_pfvf_vf_proto.h | 5 +- 12 files changed, 184 insertions(+), 83 deletions(-) create mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_utils.c create mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_utils.h diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile index 1376504d16ff..80f6cb424753 100644 --- a/drivers/crypto/qat/qat_common/Makefile +++ b/drivers/crypto/qat/qat_common/Makefile @@ -19,7 +19,7 @@ intel_qat-objs := adf_cfg.o \ qat_hal.o intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o -intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_vf_isr.o \ +intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_vf_isr.o adf_pfvf_utils.o \ adf_pfvf_pf_msg.o adf_pfvf_pf_proto.o \ adf_pfvf_vf_msg.o adf_pfvf_vf_proto.o \ adf_gen2_pfvf.o diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index d65d8dda8fda..d5ccefc04153 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -8,6 +8,7 @@ #include #include #include "adf_cfg_common.h" +#include "adf_pfvf_msg.h" #define ADF_DH895XCC_DEVICE_NAME "dh895xcc" #define ADF_DH895XCCVF_DEVICE_NAME "dh895xccvf" @@ -154,9 +155,10 @@ struct adf_pfvf_ops { u32 (*get_vf2pf_sources)(void __iomem *pmisc_addr); void (*enable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask); void (*disable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask); - int (*send_msg)(struct adf_accel_dev *accel_dev, u32 msg, + int (*send_msg)(struct adf_accel_dev *accel_dev, struct pfvf_message msg, u32 pfvf_offset, struct mutex *csr_lock); - u32 (*recv_msg)(struct adf_accel_dev *accel_dev, u32 pfvf_offset); + struct pfvf_message (*recv_msg)(struct adf_accel_dev *accel_dev, + u32 pfvf_offset); }; struct adf_hw_device_data { @@ -275,7 +277,7 @@ struct adf_accel_dev { struct tasklet_struct pf2vf_bh_tasklet; struct mutex vf2pf_lock; /* protect CSR access */ struct completion msg_received; - u32 response; /* temp field holding pf2vf response */ + struct pfvf_message response; /* temp field holding pf2vf response */ u8 pf_version; } vf; }; diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c index 97bef41ddc47..7a927bea4ac6 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c @@ -9,6 +9,7 @@ #include "adf_pfvf_msg.h" #include "adf_pfvf_pf_proto.h" #include "adf_pfvf_vf_proto.h" +#include "adf_pfvf_utils.h" /* VF2PF interrupts */ #define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9) @@ -25,6 +26,16 @@ enum gen2_csr_pos { ADF_GEN2_CSR_VF2PF_OFFSET = 16, }; +#define ADF_PFVF_GEN2_MSGTYPE_SHIFT 2 +#define ADF_PFVF_GEN2_MSGTYPE_MASK 0x0F +#define ADF_PFVF_GEN2_MSGDATA_SHIFT 6 +#define ADF_PFVF_GEN2_MSGDATA_MASK 0x3FF + +static const struct pfvf_csr_format csr_gen2_fmt = { + { ADF_PFVF_GEN2_MSGTYPE_SHIFT, ADF_PFVF_GEN2_MSGTYPE_MASK }, + { ADF_PFVF_GEN2_MSGDATA_SHIFT, ADF_PFVF_GEN2_MSGDATA_MASK }, +}; + #define ADF_PFVF_MSG_ACK_DELAY 2 #define ADF_PFVF_MSG_ACK_MAX_RETRY 100 @@ -122,7 +133,8 @@ struct pfvf_gen2_params { enum gen2_csr_pos remote_offset; }; -static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev, u32 msg, +static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev, + struct pfvf_message msg, struct pfvf_gen2_params *params) { void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); @@ -134,6 +146,7 @@ static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev, u32 msg, u32 count = 0; u32 int_bit; u32 csr_val; + u32 csr_msg; int ret; /* Gen2 messages, both PF->VF and VF->PF, are all 16 bits long. This @@ -146,12 +159,15 @@ static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev, u32 msg, int_bit = gen2_csr_get_int_bit(local_offset); - /* Pre-calculate message, shifting it in place and setting - * the in use pattern + csr_msg = adf_pfvf_csr_msg_of(accel_dev, msg, &csr_gen2_fmt); + if (unlikely(!csr_msg)) + return -EINVAL; + + /* Prepare for CSR format, shifting the wire message in place and + * setting the in use pattern */ - msg |= ADF_PFVF_MSGORIGIN_SYSTEM; - msg = gen2_csr_msg_to_position(msg, local_offset); - gen2_csr_set_in_use(&msg, remote_offset); + csr_msg = gen2_csr_msg_to_position(csr_msg, local_offset); + gen2_csr_set_in_use(&csr_msg, remote_offset); mutex_lock(lock); @@ -167,7 +183,7 @@ start: } /* Attempt to get ownership of the PFVF CSR */ - ADF_CSR_WR(pmisc_addr, pfvf_offset, msg | int_bit); + ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_msg | int_bit); /* Wait for confirmation from remote func it received the message */ do { @@ -181,7 +197,7 @@ start: ret = -EIO; } - if (csr_val != msg) { + if (csr_val != csr_msg) { dev_dbg(&GET_DEV(accel_dev), "Collision - PFVF CSR overwritten by remote function\n"); goto retry; @@ -205,15 +221,16 @@ retry: } } -static u32 adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev, - struct pfvf_gen2_params *params) +static struct pfvf_message adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev, + struct pfvf_gen2_params *params) { void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); enum gen2_csr_pos local_offset = params->local_offset; u32 pfvf_offset = params->pfvf_offset; + struct pfvf_message msg = { 0 }; u32 int_bit; u32 csr_val; - u32 msg; + u16 csr_msg; int_bit = gen2_csr_get_int_bit(local_offset); @@ -222,19 +239,22 @@ static u32 adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev, if (!(csr_val & int_bit)) { dev_info(&GET_DEV(accel_dev), "Spurious PFVF interrupt, msg 0x%.8x. Ignored\n", csr_val); - return 0; + return msg; } /* Extract the message from the CSR */ - msg = gen2_csr_msg_from_position(csr_val, local_offset); + csr_msg = gen2_csr_msg_from_position(csr_val, local_offset); /* Ignore legacy non-system (non-kernel) messages */ - if (unlikely(is_legacy_user_pfvf_message(msg))) { + if (unlikely(is_legacy_user_pfvf_message(csr_msg))) { dev_dbg(&GET_DEV(accel_dev), "Ignored non-system message (0x%.8x);\n", csr_val); - return 0; + return msg; } + /* Return the pfvf_message format */ + msg = adf_pfvf_message_of(accel_dev, csr_msg, &csr_gen2_fmt); + /* To ACK, clear the INT bit */ csr_val &= ~int_bit; ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val); @@ -242,7 +262,7 @@ static u32 adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev, return msg; } -static int adf_gen2_pf2vf_send(struct adf_accel_dev *accel_dev, u32 msg, +static int adf_gen2_pf2vf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg, u32 pfvf_offset, struct mutex *csr_lock) { struct pfvf_gen2_params params = { @@ -255,7 +275,7 @@ static int adf_gen2_pf2vf_send(struct adf_accel_dev *accel_dev, u32 msg, return adf_gen2_pfvf_send(accel_dev, msg, ¶ms); } -static int adf_gen2_vf2pf_send(struct adf_accel_dev *accel_dev, u32 msg, +static int adf_gen2_vf2pf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg, u32 pfvf_offset, struct mutex *csr_lock) { struct pfvf_gen2_params params = { @@ -268,7 +288,8 @@ static int adf_gen2_vf2pf_send(struct adf_accel_dev *accel_dev, u32 msg, return adf_gen2_pfvf_send(accel_dev, msg, ¶ms); } -static u32 adf_gen2_pf2vf_recv(struct adf_accel_dev *accel_dev, u32 pfvf_offset) +static struct pfvf_message adf_gen2_pf2vf_recv(struct adf_accel_dev *accel_dev, + u32 pfvf_offset) { struct pfvf_gen2_params params = { .pfvf_offset = pfvf_offset, @@ -279,7 +300,8 @@ static u32 adf_gen2_pf2vf_recv(struct adf_accel_dev *accel_dev, u32 pfvf_offset) return adf_gen2_pfvf_recv(accel_dev, ¶ms); } -static u32 adf_gen2_vf2pf_recv(struct adf_accel_dev *accel_dev, u32 pfvf_offset) +static struct pfvf_message adf_gen2_vf2pf_recv(struct adf_accel_dev *accel_dev, + u32 pfvf_offset) { struct pfvf_gen2_params params = { .pfvf_offset = pfvf_offset, diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h index 3ba88bcd0726..26eb27853e83 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h @@ -56,8 +56,14 @@ /* PFVF message common bits */ #define ADF_PFVF_INT BIT(0) #define ADF_PFVF_MSGORIGIN_SYSTEM BIT(1) -#define ADF_PFVF_MSGTYPE_SHIFT 2 -#define ADF_PFVF_MSGTYPE_MASK 0x0F + +/* Different generations have different CSR layouts, use this struct + * to abstract these differences away + */ +struct pfvf_message { + u8 type; + u32 data; +}; /* PF->VF messages */ enum pf2vf_msgtype { @@ -80,11 +86,9 @@ enum pfvf_compatibility_version { }; /* PF->VF Version Response */ -#define ADF_PF2VF_MINORVERSION_SHIFT 6 -#define ADF_PF2VF_MAJORVERSION_SHIFT 10 -#define ADF_PF2VF_VERSION_RESP_VERS_SHIFT 6 +#define ADF_PF2VF_VERSION_RESP_VERS_SHIFT 0 #define ADF_PF2VF_VERSION_RESP_VERS_MASK 0xFF -#define ADF_PF2VF_VERSION_RESP_RESULT_SHIFT 14 +#define ADF_PF2VF_VERSION_RESP_RESULT_SHIFT 8 #define ADF_PF2VF_VERSION_RESP_RESULT_MASK 0x03 enum pf2vf_compat_response { @@ -93,7 +97,4 @@ enum pf2vf_compat_response { ADF_PF2VF_VF_COMPAT_UNKNOWN = 0x03, }; -/* VF->PF Compatible Version Request */ -#define ADF_VF2PF_COMPAT_VER_REQ_SHIFT 6 - #endif /* ADF_PFVF_MSG_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c index f6d05cd29a82..ad198b624098 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c @@ -9,7 +9,7 @@ void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev) { struct adf_accel_vf_info *vf; - u32 msg = ADF_PF2VF_MSGTYPE_RESTARTING << ADF_PFVF_MSGTYPE_SHIFT; + struct pfvf_message msg = { .type = ADF_PF2VF_MSGTYPE_RESTARTING }; int i, num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev)); for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) { diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c index 9f4baa9b14fc..bb4d7db68579 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c @@ -17,7 +17,7 @@ * * Return: 0 on success, error code otherwise. */ -int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, u32 msg) +int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, struct pfvf_message msg) { struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev); u32 pfvf_offset = pfvf_ops->get_pf2vf_offset(vf_nr); @@ -35,7 +35,7 @@ int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, u32 msg) * * Return: a valid message on success, zero otherwise. */ -static u32 adf_recv_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr) +static struct pfvf_message adf_recv_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr) { struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev); u32 pfvf_offset = pfvf_ops->get_vf2pf_offset(vf_nr); @@ -43,16 +43,15 @@ static u32 adf_recv_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr) return pfvf_ops->recv_msg(accel_dev, pfvf_offset); } -static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, - u32 msg, u32 *response) +static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, + struct pfvf_message msg, struct pfvf_message *resp) { struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; - u32 resp = 0; - switch ((msg >> ADF_PFVF_MSGTYPE_SHIFT) & ADF_PFVF_MSGTYPE_MASK) { + switch (msg.type) { case ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ: { - u8 vf_compat_ver = msg >> ADF_VF2PF_COMPAT_VER_REQ_SHIFT; + u8 vf_compat_ver = msg.data; u8 compat; dev_dbg(&GET_DEV(accel_dev), @@ -64,11 +63,10 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, else compat = ADF_PF2VF_VF_COMPAT_UNKNOWN; - resp |= ADF_PF2VF_MSGTYPE_VERSION_RESP << - ADF_PFVF_MSGTYPE_SHIFT; - resp |= ADF_PFVF_COMPAT_THIS_VERSION << - ADF_PF2VF_VERSION_RESP_VERS_SHIFT; - resp |= compat << ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + resp->type = ADF_PF2VF_MSGTYPE_VERSION_RESP; + resp->data = ADF_PFVF_COMPAT_THIS_VERSION << + ADF_PF2VF_VERSION_RESP_VERS_SHIFT; + resp->data |= compat << ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; } break; case ADF_VF2PF_MSGTYPE_VERSION_REQ: @@ -82,12 +80,10 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, /* PF always newer than legacy VF */ compat = ADF_PF2VF_VF_COMPATIBLE; - resp |= ADF_PF2VF_MSGTYPE_VERSION_RESP << - ADF_PFVF_MSGTYPE_SHIFT; - /* Set legacy major and minor version num */ - resp |= 1 << ADF_PF2VF_MAJORVERSION_SHIFT | - 1 << ADF_PF2VF_MINORVERSION_SHIFT; - resp |= compat << ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + resp->type = ADF_PF2VF_MSGTYPE_VERSION_RESP; + /* Set legacy major and minor version to the latest, 1.1 */ + resp->data |= 0x11; + resp->data |= compat << ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; } break; case ADF_VF2PF_MSGTYPE_INIT: @@ -105,29 +101,28 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr, } break; default: - dev_dbg(&GET_DEV(accel_dev), "Unknown message from VF%d (0x%.8x)\n", - vf_nr, msg); + dev_dbg(&GET_DEV(accel_dev), + "Unknown message from VF%d (type 0x%.4x, data: 0x%.4x)\n", + vf_nr, msg.type, msg.data); return -ENOMSG; } - *response = resp; - return 0; } bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr) { - u32 resp = 0; - u32 msg; + struct pfvf_message req; + struct pfvf_message resp = {0}; - msg = adf_recv_vf2pf_msg(accel_dev, vf_nr); - if (!msg) + req = adf_recv_vf2pf_msg(accel_dev, vf_nr); + if (!req.type) /* Legacy or no message */ return true; - if (adf_handle_vf2pf_msg(accel_dev, vf_nr, msg, &resp)) + if (adf_handle_vf2pf_msg(accel_dev, vf_nr, req, &resp)) return false; - if (resp && adf_send_pf2vf_msg(accel_dev, vf_nr, resp)) + if (resp.type && adf_send_pf2vf_msg(accel_dev, vf_nr, resp)) dev_err(&GET_DEV(accel_dev), "Failed to send response to VF%d\n", vf_nr); diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h index 63245407bfb6..165d266d023d 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h @@ -6,7 +6,7 @@ #include #include "adf_accel_devices.h" -int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, u32 msg); +int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, struct pfvf_message msg); int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_utils.c b/drivers/crypto/qat/qat_common/adf_pfvf_utils.c new file mode 100644 index 000000000000..494da89e2048 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_pfvf_utils.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2021 Intel Corporation */ +#include +#include +#include "adf_accel_devices.h" +#include "adf_pfvf_msg.h" +#include "adf_pfvf_utils.h" + +static bool set_value_on_csr_msg(struct adf_accel_dev *accel_dev, u32 *csr_msg, + u32 value, const struct pfvf_field_format *fmt) +{ + if (unlikely((value & fmt->mask) != value)) { + dev_err(&GET_DEV(accel_dev), + "PFVF message value 0x%X out of range, %u max allowed\n", + value, fmt->mask); + return false; + } + + *csr_msg |= value << fmt->offset; + + return true; +} + +u32 adf_pfvf_csr_msg_of(struct adf_accel_dev *accel_dev, + struct pfvf_message msg, + const struct pfvf_csr_format *fmt) +{ + u32 csr_msg = 0; + + if (!set_value_on_csr_msg(accel_dev, &csr_msg, msg.type, &fmt->type) || + !set_value_on_csr_msg(accel_dev, &csr_msg, msg.data, &fmt->data)) + return 0; + + return csr_msg | ADF_PFVF_MSGORIGIN_SYSTEM; +} + +struct pfvf_message adf_pfvf_message_of(struct adf_accel_dev *accel_dev, u32 csr_msg, + const struct pfvf_csr_format *fmt) +{ + struct pfvf_message msg = { 0 }; + + msg.type = (csr_msg >> fmt->type.offset) & fmt->type.mask; + msg.data = (csr_msg >> fmt->data.offset) & fmt->data.mask; + + if (unlikely(!msg.type)) + dev_err(&GET_DEV(accel_dev), + "Invalid PFVF msg with no type received\n"); + + return msg; +} diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_utils.h b/drivers/crypto/qat/qat_common/adf_pfvf_utils.h new file mode 100644 index 000000000000..7b73b5992d03 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_pfvf_utils.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2021 Intel Corporation */ +#ifndef ADF_PFVF_UTILS_H +#define ADF_PFVF_UTILS_H + +#include +#include "adf_pfvf_msg.h" + +struct pfvf_field_format { + u8 offset; + u32 mask; +}; + +struct pfvf_csr_format { + struct pfvf_field_format type; + struct pfvf_field_format data; +}; + +u32 adf_pfvf_csr_msg_of(struct adf_accel_dev *accel_dev, struct pfvf_message msg, + const struct pfvf_csr_format *fmt); +struct pfvf_message adf_pfvf_message_of(struct adf_accel_dev *accel_dev, u32 raw_msg, + const struct pfvf_csr_format *fmt); + +#endif /* ADF_PFVF_UTILS_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c index 0463743a8d43..5184a77598d2 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c @@ -16,7 +16,7 @@ */ int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev) { - u32 msg = ADF_VF2PF_MSGTYPE_INIT << ADF_PFVF_MSGTYPE_SHIFT; + struct pfvf_message msg = { .type = ADF_VF2PF_MSGTYPE_INIT }; if (adf_send_vf2pf_msg(accel_dev, msg)) { dev_err(&GET_DEV(accel_dev), @@ -38,7 +38,7 @@ EXPORT_SYMBOL_GPL(adf_vf2pf_notify_init); */ void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev) { - u32 msg = ADF_VF2PF_MSGTYPE_SHUTDOWN << ADF_PFVF_MSGTYPE_SHIFT; + struct pfvf_message msg = { .type = ADF_VF2PF_MSGTYPE_SHUTDOWN }; if (test_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status)) if (adf_send_vf2pf_msg(accel_dev, msg)) @@ -50,13 +50,13 @@ EXPORT_SYMBOL_GPL(adf_vf2pf_notify_shutdown); int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) { u8 pf_version; - u32 msg = 0; int compat; - u32 resp; int ret; - - msg = ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ << ADF_PFVF_MSGTYPE_SHIFT | - ADF_PFVF_COMPAT_THIS_VERSION << ADF_VF2PF_COMPAT_VER_REQ_SHIFT; + struct pfvf_message resp; + struct pfvf_message msg = { + .type = ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ, + .data = ADF_PFVF_COMPAT_THIS_VERSION, + }; BUILD_BUG_ON(ADF_PFVF_COMPAT_THIS_VERSION > 255); @@ -67,9 +67,9 @@ int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) return ret; } - pf_version = (resp >> ADF_PF2VF_VERSION_RESP_VERS_SHIFT) + pf_version = (resp.data >> ADF_PF2VF_VERSION_RESP_VERS_SHIFT) & ADF_PF2VF_VERSION_RESP_VERS_MASK; - compat = (resp >> ADF_PF2VF_VERSION_RESP_RESULT_SHIFT) + compat = (resp.data >> ADF_PF2VF_VERSION_RESP_RESULT_SHIFT) & ADF_PF2VF_VERSION_RESP_RESULT_MASK; /* Response from PF received, check compatibility */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c index 56e8185a9630..729c00c0d254 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c @@ -25,7 +25,7 @@ * * Return: 0 on success, error code otherwise. */ -int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg) +int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, struct pfvf_message msg) { struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev); u32 pfvf_offset = pfvf_ops->get_vf2pf_offset(0); @@ -42,7 +42,7 @@ int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg) * * Return: a valid message on success, zero otherwise. */ -static u32 adf_recv_pf2vf_msg(struct adf_accel_dev *accel_dev) +static struct pfvf_message adf_recv_pf2vf_msg(struct adf_accel_dev *accel_dev) { struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev); u32 pfvf_offset = pfvf_ops->get_pf2vf_offset(0); @@ -61,7 +61,8 @@ static u32 adf_recv_pf2vf_msg(struct adf_accel_dev *accel_dev) * * Return: 0 on success, error code otherwise. */ -int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, u32 msg, u32 *resp) +int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, struct pfvf_message msg, + struct pfvf_message *resp) { unsigned long timeout = msecs_to_jiffies(ADF_PFVF_MSG_RESP_TIMEOUT); int ret; @@ -88,14 +89,15 @@ int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, u32 msg, u32 *resp) *resp = accel_dev->vf.response; /* Once copied, set to an invalid value */ - accel_dev->vf.response = 0; + accel_dev->vf.response.type = 0; return 0; } -static bool adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev, u32 msg) +static bool adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev, + struct pfvf_message msg) { - switch ((msg >> ADF_PFVF_MSGTYPE_SHIFT) & ADF_PFVF_MSGTYPE_MASK) { + switch (msg.type) { case ADF_PF2VF_MSGTYPE_RESTARTING: dev_dbg(&GET_DEV(accel_dev), "Restarting message received from PF\n"); @@ -103,13 +105,15 @@ static bool adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev, u32 msg) return false; case ADF_PF2VF_MSGTYPE_VERSION_RESP: dev_dbg(&GET_DEV(accel_dev), - "Response message received from PF (0x%.8x)\n", msg); + "Response Message received from PF (type 0x%.4x, data 0x%.4x)\n", + msg.type, msg.data); accel_dev->vf.response = msg; complete(&accel_dev->vf.msg_received); return true; default: dev_err(&GET_DEV(accel_dev), - "Unknown PF2VF message (0x%.8x) from PF\n", msg); + "Unknown message from PF (type 0x%.4x, data: 0x%.4x)\n", + msg.type, msg.data); } return false; @@ -117,12 +121,14 @@ static bool adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev, u32 msg) bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev) { - u32 msg; + struct pfvf_message msg; msg = adf_recv_pf2vf_msg(accel_dev); - if (msg) + if (msg.type) /* Invalid or no message */ return adf_handle_pf2vf_msg(accel_dev, msg); + /* No replies for PF->VF messages at present */ + return true; } diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h index 6226d4d9d520..e32d1bc3a740 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h @@ -6,8 +6,9 @@ #include #include "adf_accel_devices.h" -int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg); -int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, u32 msg, u32 *resp); +int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, struct pfvf_message msg); +int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, struct pfvf_message msg, + struct pfvf_message *resp); int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev); From 952f4e81274131eda4ab0d751d6f2700b0356542 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Thu, 16 Dec 2021 09:13:21 +0000 Subject: [PATCH 111/145] crypto: qat - leverage bitfield.h utils for PFVF messages The PFVF protocol defines messages composed of a number of control bitfields. Replace all the code setting and retrieving such bits with the utilities from bitfield.h, to improve code quality and readability. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_pfvf_msg.h | 8 ++++---- drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c | 13 +++++++------ drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c | 7 +++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h index 26eb27853e83..daee3d7ceb8c 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h @@ -3,6 +3,8 @@ #ifndef ADF_PFVF_MSG_H #define ADF_PFVF_MSG_H +#include + /* * PF<->VF Messaging * The PF has an array of 32-bit PF2VF registers, one for each VF. The @@ -86,10 +88,8 @@ enum pfvf_compatibility_version { }; /* PF->VF Version Response */ -#define ADF_PF2VF_VERSION_RESP_VERS_SHIFT 0 -#define ADF_PF2VF_VERSION_RESP_VERS_MASK 0xFF -#define ADF_PF2VF_VERSION_RESP_RESULT_SHIFT 8 -#define ADF_PF2VF_VERSION_RESP_RESULT_MASK 0x03 +#define ADF_PF2VF_VERSION_RESP_VERS_MASK GENMASK(7, 0) +#define ADF_PF2VF_VERSION_RESP_RESULT_MASK GENMASK(9, 8) enum pf2vf_compat_response { ADF_PF2VF_VF_COMPATIBLE = 0x01, diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c index bb4d7db68579..8785b9d1df91 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2015 - 2021 Intel Corporation */ +#include #include #include #include "adf_accel_devices.h" @@ -64,9 +65,9 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, compat = ADF_PF2VF_VF_COMPAT_UNKNOWN; resp->type = ADF_PF2VF_MSGTYPE_VERSION_RESP; - resp->data = ADF_PFVF_COMPAT_THIS_VERSION << - ADF_PF2VF_VERSION_RESP_VERS_SHIFT; - resp->data |= compat << ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + resp->data = FIELD_PREP(ADF_PF2VF_VERSION_RESP_VERS_MASK, + ADF_PFVF_COMPAT_THIS_VERSION) | + FIELD_PREP(ADF_PF2VF_VERSION_RESP_RESULT_MASK, compat); } break; case ADF_VF2PF_MSGTYPE_VERSION_REQ: @@ -80,10 +81,10 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, /* PF always newer than legacy VF */ compat = ADF_PF2VF_VF_COMPATIBLE; - resp->type = ADF_PF2VF_MSGTYPE_VERSION_RESP; /* Set legacy major and minor version to the latest, 1.1 */ - resp->data |= 0x11; - resp->data |= compat << ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + resp->type = ADF_PF2VF_MSGTYPE_VERSION_RESP; + resp->data = FIELD_PREP(ADF_PF2VF_VERSION_RESP_VERS_MASK, 0x11) | + FIELD_PREP(ADF_PF2VF_VERSION_RESP_RESULT_MASK, compat); } break; case ADF_VF2PF_MSGTYPE_INIT: diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c index 5184a77598d2..130d7b9c12ea 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2015 - 2021 Intel Corporation */ +#include #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "adf_pfvf_msg.h" @@ -67,10 +68,8 @@ int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) return ret; } - pf_version = (resp.data >> ADF_PF2VF_VERSION_RESP_VERS_SHIFT) - & ADF_PF2VF_VERSION_RESP_VERS_MASK; - compat = (resp.data >> ADF_PF2VF_VERSION_RESP_RESULT_SHIFT) - & ADF_PF2VF_VERSION_RESP_RESULT_MASK; + pf_version = FIELD_GET(ADF_PF2VF_VERSION_RESP_VERS_MASK, resp.data); + compat = FIELD_GET(ADF_PF2VF_VERSION_RESP_RESULT_MASK, resp.data); /* Response from PF received, check compatibility */ switch (compat) { From 1c94d8035905c10f4930708a9944f2ee65d26f47 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Thu, 16 Dec 2021 09:13:22 +0000 Subject: [PATCH 112/145] crypto: qat - leverage read_poll_timeout in PFVF send Replace the polling loop, waiting for the remote end to acknowledge the reception of the message, with the equivalent and standard read_poll_timeout() in adf_gen2_pfvf_send(). Also, the use of the read_poll_timeout(): - implies the use of microseconds for the timings, so update the previous values from ms to us - allows to leverage the return value for both success and error, removing the need for the reset of the 'ret' variable soon after the 'start' label. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_gen2_pfvf.c | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c index 7a927bea4ac6..53c2e124944d 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2021 Intel Corporation */ #include +#include #include #include #include "adf_accel_devices.h" @@ -36,8 +37,8 @@ static const struct pfvf_csr_format csr_gen2_fmt = { { ADF_PFVF_GEN2_MSGDATA_SHIFT, ADF_PFVF_GEN2_MSGDATA_MASK }, }; -#define ADF_PFVF_MSG_ACK_DELAY 2 -#define ADF_PFVF_MSG_ACK_MAX_RETRY 100 +#define ADF_PFVF_MSG_ACK_DELAY_US 2000 +#define ADF_PFVF_MSG_ACK_MAX_DELAY_US (ADF_PFVF_MSG_ACK_DELAY_US * 100) #define ADF_PFVF_MSG_RETRY_DELAY 5 #define ADF_PFVF_MSG_MAX_RETRIES 3 @@ -143,7 +144,6 @@ static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev, unsigned int retries = ADF_PFVF_MSG_MAX_RETRIES; struct mutex *lock = params->csr_lock; u32 pfvf_offset = params->pfvf_offset; - u32 count = 0; u32 int_bit; u32 csr_val; u32 csr_msg; @@ -172,8 +172,6 @@ static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev, mutex_lock(lock); start: - ret = 0; - /* Check if the PFVF CSR is in use by remote function */ csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset); if (gen2_csr_is_in_use(csr_val, local_offset)) { @@ -186,15 +184,13 @@ start: ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_msg | int_bit); /* Wait for confirmation from remote func it received the message */ - do { - msleep(ADF_PFVF_MSG_ACK_DELAY); - csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset); - } while ((csr_val & int_bit) && (count++ < ADF_PFVF_MSG_ACK_MAX_RETRY)); - - if (csr_val & int_bit) { + ret = read_poll_timeout(ADF_CSR_RD, csr_val, !(csr_val & int_bit), + ADF_PFVF_MSG_ACK_DELAY_US, + ADF_PFVF_MSG_ACK_MAX_DELAY_US, + true, pmisc_addr, pfvf_offset); + if (unlikely(ret < 0)) { dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n"); csr_val &= ~int_bit; - ret = -EIO; } if (csr_val != csr_msg) { From 6f87979129d10cff5b4b0b49343720b99b5357b8 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Thu, 16 Dec 2021 09:13:23 +0000 Subject: [PATCH 113/145] crypto: qat - improve the ACK timings in PFVF send Review the ACK timings in adf_gen2_pfvf_send() to improve the latency by reducing the polling interval. Also increase the timeout, for higher tolerance in highly loaded systems, and reposition these new values to allow for inclusion by the future GEN4 devices too. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_gen2_pfvf.c | 3 --- drivers/crypto/qat/qat_common/adf_pfvf_utils.h | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c index 53c2e124944d..feab01ec4bbb 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c @@ -37,9 +37,6 @@ static const struct pfvf_csr_format csr_gen2_fmt = { { ADF_PFVF_GEN2_MSGDATA_SHIFT, ADF_PFVF_GEN2_MSGDATA_MASK }, }; -#define ADF_PFVF_MSG_ACK_DELAY_US 2000 -#define ADF_PFVF_MSG_ACK_MAX_DELAY_US (ADF_PFVF_MSG_ACK_DELAY_US * 100) - #define ADF_PFVF_MSG_RETRY_DELAY 5 #define ADF_PFVF_MSG_MAX_RETRIES 3 diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_utils.h b/drivers/crypto/qat/qat_common/adf_pfvf_utils.h index 7b73b5992d03..7676fdddbe26 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_utils.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_utils.h @@ -6,6 +6,10 @@ #include #include "adf_pfvf_msg.h" +/* How long to wait for far side to acknowledge receipt */ +#define ADF_PFVF_MSG_ACK_DELAY_US 4 +#define ADF_PFVF_MSG_ACK_MAX_DELAY_US (1 * USEC_PER_SEC) + struct pfvf_field_format { u8 offset; u32 mask; From 4d03135faa059443a2cfcbee758d9db8bada7fe1 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Thu, 16 Dec 2021 09:13:24 +0000 Subject: [PATCH 114/145] crypto: qat - store the PFVF protocol version of the endpoints This patch adds an entry for storing the PFVF protocol version for both PF and VFs. While not currently used, knowing and storing the remote protocol version enables more complex compatibility checks and/or newer features for compatible PFVF endpoints in the future. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_devices.h | 3 ++- drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c | 5 +++++ drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c | 2 +- drivers/crypto/qat/qat_common/adf_sriov.c | 1 + 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index d5ccefc04153..1fb32f3e78df 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -249,6 +249,7 @@ struct adf_accel_vf_info { struct ratelimit_state vf2pf_ratelimit; u32 vf_nr; bool init; + u8 vf_compat_ver; }; struct adf_accel_dev { @@ -278,7 +279,7 @@ struct adf_accel_dev { struct mutex vf2pf_lock; /* protect CSR access */ struct completion msg_received; struct pfvf_message response; /* temp field holding pf2vf response */ - u8 pf_version; + u8 pf_compat_ver; } vf; }; bool is_vf; diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c index 8785b9d1df91..f461aa0a95c7 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c @@ -64,6 +64,8 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, else compat = ADF_PF2VF_VF_COMPAT_UNKNOWN; + vf_info->vf_compat_ver = vf_compat_ver; + resp->type = ADF_PF2VF_MSGTYPE_VERSION_RESP; resp->data = FIELD_PREP(ADF_PF2VF_VERSION_RESP_VERS_MASK, ADF_PFVF_COMPAT_THIS_VERSION) | @@ -78,6 +80,9 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, "Legacy VersionRequest received from VF%d to PF (vers 1.1)\n", vf_nr); + /* legacy driver, VF compat_ver is 0 */ + vf_info->vf_compat_ver = 0; + /* PF always newer than legacy VF */ compat = ADF_PF2VF_VF_COMPATIBLE; diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c index 130d7b9c12ea..307d593042f3 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c @@ -89,6 +89,6 @@ int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) return -EINVAL; } - accel_dev->vf.pf_version = pf_version; + accel_dev->vf.pf_compat_ver = pf_version; return 0; } diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index 429990c5e0f3..6366622ff8fd 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -58,6 +58,7 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) /* This ptr will be populated when VFs will be created */ vf_info->accel_dev = accel_dev; vf_info->vf_nr = i; + vf_info->vf_compat_ver = 0; mutex_init(&vf_info->pf2vf_lock); ratelimit_state_init(&vf_info->vf2pf_ratelimit, From 3a5b2a0883288527e71450978c0f5c442aab1218 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Thu, 16 Dec 2021 09:13:25 +0000 Subject: [PATCH 115/145] crypto: qat - store the ring-to-service mapping This driver relies on either the FW (on the PF) or the PF (on the VF) to know how crypto services and rings map to one another. Store this information so that it can be referenced in the future at runtime for checks or extensions. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 1 + drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 1 + .../crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c | 1 + drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c | 1 + drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c | 1 + drivers/crypto/qat/qat_common/adf_accel_devices.h | 8 ++++++++ drivers/crypto/qat/qat_common/adf_cfg_common.h | 13 +++++++++++++ drivers/crypto/qat/qat_common/adf_gen2_hw_data.h | 8 ++++++++ drivers/crypto/qat/qat_common/adf_gen4_hw_data.h | 8 ++++++++ .../crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 1 + .../qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c | 1 + 11 files changed, 44 insertions(+) diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c index 0d1603894af4..67cd20f443ab 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -246,6 +246,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) hw_data->num_logical_accel = 1; hw_data->tx_rx_gap = ADF_4XXX_RX_RINGS_OFFSET; hw_data->tx_rings_mask = ADF_4XXX_TX_RINGS_MASK; + hw_data->ring_to_svc_map = ADF_GEN4_DEFAULT_RING_TO_SRV_MAP; hw_data->alloc_irq = adf_isr_resource_alloc; hw_data->free_irq = adf_isr_resource_free; hw_data->enable_error_correction = adf_enable_error_correction; diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 3987a44fa164..b941fe3713ff 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -109,6 +109,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->num_engines = ADF_C3XXX_MAX_ACCELENGINES; hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET; hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK; + hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP; hw_data->alloc_irq = adf_isr_resource_alloc; hw_data->free_irq = adf_isr_resource_free; hw_data->enable_error_correction = adf_gen2_enable_error_correction; diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c index 85122013534d..a9fbe57b32ae 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -67,6 +67,7 @@ void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) hw_data->num_engines = ADF_C3XXXIOV_MAX_ACCELENGINES; hw_data->tx_rx_gap = ADF_C3XXXIOV_RX_RINGS_OFFSET; hw_data->tx_rings_mask = ADF_C3XXXIOV_TX_RINGS_MASK; + hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP; hw_data->alloc_irq = adf_vf_isr_resource_alloc; hw_data->free_irq = adf_vf_isr_resource_free; hw_data->enable_error_correction = adf_vf_void_noop; diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index a76e33d7a215..b1eac2f81faa 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -111,6 +111,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->num_engines = ADF_C62X_MAX_ACCELENGINES; hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET; hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK; + hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP; hw_data->alloc_irq = adf_isr_resource_alloc; hw_data->free_irq = adf_isr_resource_free; hw_data->enable_error_correction = adf_gen2_enable_error_correction; diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c index 99c56405f88f..0282038fca54 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -67,6 +67,7 @@ void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data) hw_data->num_engines = ADF_C62XIOV_MAX_ACCELENGINES; hw_data->tx_rx_gap = ADF_C62XIOV_RX_RINGS_OFFSET; hw_data->tx_rings_mask = ADF_C62XIOV_TX_RINGS_MASK; + hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP; hw_data->alloc_irq = adf_vf_isr_resource_alloc; hw_data->free_irq = adf_vf_isr_resource_free; hw_data->enable_error_correction = adf_vf_void_noop; diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 1fb32f3e78df..59f06e53d316 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -208,6 +208,7 @@ struct adf_hw_device_data { u32 ae_mask; u32 admin_ae_mask; u16 tx_rings_mask; + u16 ring_to_svc_map; u8 tx_rx_gap; u8 num_banks; u16 num_banks_per_vf; @@ -224,12 +225,19 @@ struct adf_hw_device_data { /* CSR read macro */ #define ADF_CSR_RD(csr_base, csr_offset) __raw_readl(csr_base + csr_offset) +#define ADF_CFG_NUM_SERVICES 4 +#define ADF_SRV_TYPE_BIT_LEN 3 +#define ADF_SRV_TYPE_MASK 0x7 + #define GET_DEV(accel_dev) ((accel_dev)->accel_pci_dev.pci_dev->dev) #define GET_BARS(accel_dev) ((accel_dev)->accel_pci_dev.pci_bars) #define GET_HW_DATA(accel_dev) (accel_dev->hw_device) #define GET_MAX_BANKS(accel_dev) (GET_HW_DATA(accel_dev)->num_banks) #define GET_NUM_RINGS_PER_BANK(accel_dev) \ GET_HW_DATA(accel_dev)->num_rings_per_bank +#define GET_SRV_TYPE(accel_dev, idx) \ + (((GET_HW_DATA(accel_dev)->ring_to_svc_map) >> (ADF_SRV_TYPE_BIT_LEN * (idx))) \ + & ADF_SRV_TYPE_MASK) #define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines) #define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_ops) #define GET_PFVF_OPS(accel_dev) (&(accel_dev)->hw_device->pfvf_ops) diff --git a/drivers/crypto/qat/qat_common/adf_cfg_common.h b/drivers/crypto/qat/qat_common/adf_cfg_common.h index 4fabb70b1f18..6e5de1dab97b 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg_common.h +++ b/drivers/crypto/qat/qat_common/adf_cfg_common.h @@ -19,6 +19,19 @@ #define ADF_MAX_DEVICES (32 * 32) #define ADF_DEVS_ARRAY_SIZE BITS_TO_LONGS(ADF_MAX_DEVICES) +#define ADF_CFG_SERV_RING_PAIR_0_SHIFT 0 +#define ADF_CFG_SERV_RING_PAIR_1_SHIFT 3 +#define ADF_CFG_SERV_RING_PAIR_2_SHIFT 6 +#define ADF_CFG_SERV_RING_PAIR_3_SHIFT 9 +enum adf_cfg_service_type { + UNUSED = 0, + CRYPTO, + COMP, + SYM, + ASYM, + USED +}; + enum adf_cfg_val_type { ADF_DEC, ADF_HEX, diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index 7c2c17366460..f2e0451b11c0 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -4,6 +4,7 @@ #define ADF_GEN2_HW_DATA_H_ #include "adf_accel_devices.h" +#include "adf_cfg_common.h" /* Transport access */ #define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL @@ -116,6 +117,13 @@ do { \ #define ADF_POWERGATE_DC BIT(23) #define ADF_POWERGATE_PKE BIT(24) +/* Default ring mapping */ +#define ADF_GEN2_DEFAULT_RING_TO_SRV_MAP \ + (CRYPTO << ADF_CFG_SERV_RING_PAIR_0_SHIFT | \ + CRYPTO << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ + UNUSED << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ + COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT) + /* WDT timers * * Timeout is in cycles. Clock speed may vary across products but this diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h index 449d6a5976a9..f0f71ca44ca3 100644 --- a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h @@ -4,6 +4,7 @@ #define ADF_GEN4_HW_CSR_DATA_H_ #include "adf_accel_devices.h" +#include "adf_cfg_common.h" /* Transport access */ #define ADF_BANK_INT_SRC_SEL_MASK 0x44UL @@ -94,6 +95,13 @@ do { \ ADF_RING_BUNDLE_SIZE * (bank) + \ ADF_RING_CSR_RING_SRV_ARB_EN, (value)) +/* Default ring mapping */ +#define ADF_GEN4_DEFAULT_RING_TO_SRV_MAP \ + (ASYM << ADF_CFG_SERV_RING_PAIR_0_SHIFT | \ + SYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ + ASYM << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ + SYM << ADF_CFG_SERV_RING_PAIR_3_SHIFT) + /* WDT timers * * Timeout is in cycles. Clock speed may vary across products but this diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 2d18279191d7..09599fe4d2f3 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -191,6 +191,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->num_engines = ADF_DH895XCC_MAX_ACCELENGINES; hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET; hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK; + hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP; hw_data->alloc_irq = adf_isr_resource_alloc; hw_data->free_irq = adf_isr_resource_free; hw_data->enable_error_correction = adf_gen2_enable_error_correction; diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index 5489d6c02256..31c14d7e1c11 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -67,6 +67,7 @@ void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) hw_data->num_engines = ADF_DH895XCCIOV_MAX_ACCELENGINES; hw_data->tx_rx_gap = ADF_DH895XCCIOV_RX_RINGS_OFFSET; hw_data->tx_rings_mask = ADF_DH895XCCIOV_TX_RINGS_MASK; + hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP; hw_data->alloc_irq = adf_vf_isr_resource_alloc; hw_data->free_irq = adf_vf_isr_resource_free; hw_data->enable_error_correction = adf_vf_void_noop; From 673184a2a58f93f0c170d30247155b3f86ba69ee Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Thu, 16 Dec 2021 09:13:26 +0000 Subject: [PATCH 116/145] crypto: qat - introduce support for PFVF block messages GEN2 devices use a single CSR for PFVF messages, which leaves up to 10 bits of payload per single message. While such amount is sufficient for the currently defined messages, the transfer of bigger and more complex data streams from the PF to the VF requires a new mechanism that extends the protocol. This patch adds a new layer on top of the basic PFVF messaging, called Block Messages, to encapsulate up to 126 bytes of data in a single logical message across multiple PFVF messages of new types (SMALL, MEDIUM and LARGE BLOCK), including (sub)types (BLKMSG_TYPE) to carry the information about the actual Block Message. Regardless of the size, each Block Message uses a two bytes header, containing the version and size, to allow for extension while maintaining compatibility. The size and the types of Block Messages are defined as follow: - small block messages: up to 16 BLKMSG types of up to 30 bytes - medium block messages: up to 8 BLKMSG types of up to 62 bytes - large block messages: up to 4 BLKMSG types of up to 126 bytes It effectively works as reading a byte at a time from a block device and for each of these new Block Messages: - the requestor (always a VF) can either request a specific byte of the larger message, in order to retrieve the full message, or request the value of the CRC calculated for a specific message up to the provided size (to allow for messages to grow while maintaining forward compatibility) - the responder (always the PF) will either return a single data or CRC byte, along with the indication of response type (or error). This patch provides the basic infrastructure to perform the above operations, without defining any new message. As CRCs are required, this code now depends on the CRC8 module. Note: as a consequence of the Block Messages design, sending multiple PFVF messages in bursts, the interrupt rate limiting values on the PF are increased. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/Kconfig | 1 + drivers/crypto/qat/qat_common/adf_pfvf_msg.h | 74 +++++++ .../crypto/qat/qat_common/adf_pfvf_pf_msg.h | 3 + .../crypto/qat/qat_common/adf_pfvf_pf_proto.c | 137 ++++++++++++ .../crypto/qat/qat_common/adf_pfvf_utils.c | 15 ++ .../crypto/qat/qat_common/adf_pfvf_utils.h | 3 + .../crypto/qat/qat_common/adf_pfvf_vf_proto.c | 203 +++++++++++++++++- .../crypto/qat/qat_common/adf_pfvf_vf_proto.h | 2 + drivers/crypto/qat/qat_common/adf_sriov.c | 7 +- 9 files changed, 442 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/qat/Kconfig b/drivers/crypto/qat/Kconfig index 77783feb62b2..4b90c0f22b03 100644 --- a/drivers/crypto/qat/Kconfig +++ b/drivers/crypto/qat/Kconfig @@ -13,6 +13,7 @@ config CRYPTO_DEV_QAT select CRYPTO_SHA512 select CRYPTO_LIB_AES select FW_LOADER + select CRC8 config CRYPTO_DEV_QAT_DH895xCC tristate "Support for Intel(R) DH895xCC" diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h index daee3d7ceb8c..6abbb5e05809 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h @@ -71,6 +71,7 @@ struct pfvf_message { enum pf2vf_msgtype { ADF_PF2VF_MSGTYPE_RESTARTING = 0x01, ADF_PF2VF_MSGTYPE_VERSION_RESP = 0x02, + ADF_PF2VF_MSGTYPE_BLKMSG_RESP = 0x03, }; /* VF->PF messages */ @@ -79,6 +80,9 @@ enum vf2pf_msgtype { ADF_VF2PF_MSGTYPE_SHUTDOWN = 0x04, ADF_VF2PF_MSGTYPE_VERSION_REQ = 0x05, ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ = 0x06, + ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ = 0x07, + ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ = 0x08, + ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ = 0x09, }; /* VF/PF compatibility version. */ @@ -97,4 +101,74 @@ enum pf2vf_compat_response { ADF_PF2VF_VF_COMPAT_UNKNOWN = 0x03, }; +/* PF->VF Block Responses */ +#define ADF_PF2VF_BLKMSG_RESP_TYPE_MASK GENMASK(1, 0) +#define ADF_PF2VF_BLKMSG_RESP_DATA_MASK GENMASK(9, 2) + +enum pf2vf_blkmsg_resp_type { + ADF_PF2VF_BLKMSG_RESP_TYPE_DATA = 0x00, + ADF_PF2VF_BLKMSG_RESP_TYPE_CRC = 0x01, + ADF_PF2VF_BLKMSG_RESP_TYPE_ERROR = 0x02, +}; + +/* PF->VF Block Error Code */ +enum pf2vf_blkmsg_error { + ADF_PF2VF_INVALID_BLOCK_TYPE = 0x00, + ADF_PF2VF_INVALID_BYTE_NUM_REQ = 0x01, + ADF_PF2VF_PAYLOAD_TRUNCATED = 0x02, + ADF_PF2VF_UNSPECIFIED_ERROR = 0x03, +}; + +/* VF->PF Block Requests */ +#define ADF_VF2PF_LARGE_BLOCK_TYPE_MASK GENMASK(1, 0) +#define ADF_VF2PF_LARGE_BLOCK_BYTE_MASK GENMASK(8, 2) +#define ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK GENMASK(2, 0) +#define ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK GENMASK(8, 3) +#define ADF_VF2PF_SMALL_BLOCK_TYPE_MASK GENMASK(3, 0) +#define ADF_VF2PF_SMALL_BLOCK_BYTE_MASK GENMASK(8, 4) +#define ADF_VF2PF_BLOCK_CRC_REQ_MASK BIT(9) + +/* PF->VF Block Request Types + * 0..15 - 32 byte message + * 16..23 - 64 byte message + * 24..27 - 128 byte message + */ +/* No block messages as of yet */ + +#define ADF_VF2PF_SMALL_BLOCK_TYPE_MAX \ + (FIELD_MAX(ADF_VF2PF_SMALL_BLOCK_TYPE_MASK)) + +#define ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX \ + (FIELD_MAX(ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK) + \ + ADF_VF2PF_SMALL_BLOCK_TYPE_MAX + 1) + +#define ADF_VF2PF_LARGE_BLOCK_TYPE_MAX \ + (FIELD_MAX(ADF_VF2PF_LARGE_BLOCK_TYPE_MASK) + \ + ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX) + +#define ADF_VF2PF_SMALL_BLOCK_BYTE_MAX \ + FIELD_MAX(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK) + +#define ADF_VF2PF_MEDIUM_BLOCK_BYTE_MAX \ + FIELD_MAX(ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK) + +#define ADF_VF2PF_LARGE_BLOCK_BYTE_MAX \ + FIELD_MAX(ADF_VF2PF_LARGE_BLOCK_BYTE_MASK) + +struct pfvf_blkmsg_header { + u8 version; + u8 payload_size; +} __packed; + +#define ADF_PFVF_BLKMSG_HEADER_SIZE (sizeof(struct pfvf_blkmsg_header)) +#define ADF_PFVF_BLKMSG_PAYLOAD_SIZE(blkmsg) (sizeof(blkmsg) - \ + ADF_PFVF_BLKMSG_HEADER_SIZE) +#define ADF_PFVF_BLKMSG_MSG_SIZE(blkmsg) (ADF_PFVF_BLKMSG_HEADER_SIZE + \ + (blkmsg)->hdr.payload_size) +#define ADF_PFVF_BLKMSG_MSG_MAX_SIZE 128 + +/* PF->VF Block message header bytes */ +#define ADF_PFVF_BLKMSG_VER_BYTE 0 +#define ADF_PFVF_BLKMSG_LEN_BYTE 1 + #endif /* ADF_PFVF_MSG_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h index 187807b1ff88..5c669f1de6e4 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h @@ -7,4 +7,7 @@ void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev); +typedef int (*adf_pf2vf_blkmsg_provider)(struct adf_accel_dev *accel_dev, + u8 *buffer, u8 compat); + #endif /* ADF_PFVF_PF_MSG_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c index f461aa0a95c7..850b5f4414a6 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c @@ -6,7 +6,16 @@ #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "adf_pfvf_msg.h" +#include "adf_pfvf_pf_msg.h" #include "adf_pfvf_pf_proto.h" +#include "adf_pfvf_utils.h" + +typedef u8 (*pf2vf_blkmsg_data_getter_fn)(u8 const *blkmsg, u8 byte); + +static const adf_pf2vf_blkmsg_provider pf2vf_blkmsg_providers[] = { + NULL, /* no message type defined for value 0 */ + NULL, /* no message type defined for value 1 */ +}; /** * adf_send_pf2vf_msg() - send PF to VF message @@ -44,6 +53,128 @@ static struct pfvf_message adf_recv_vf2pf_msg(struct adf_accel_dev *accel_dev, u return pfvf_ops->recv_msg(accel_dev, pfvf_offset); } +static adf_pf2vf_blkmsg_provider get_blkmsg_response_provider(u8 type) +{ + if (type >= ARRAY_SIZE(pf2vf_blkmsg_providers)) + return NULL; + + return pf2vf_blkmsg_providers[type]; +} + +/* Byte pf2vf_blkmsg_data_getter_fn callback */ +static u8 adf_pf2vf_blkmsg_get_byte(u8 const *blkmsg, u8 index) +{ + return blkmsg[index]; +} + +/* CRC pf2vf_blkmsg_data_getter_fn callback */ +static u8 adf_pf2vf_blkmsg_get_crc(u8 const *blkmsg, u8 count) +{ + /* count is 0-based, turn it into a length */ + return adf_pfvf_calc_blkmsg_crc(blkmsg, count + 1); +} + +static int adf_pf2vf_blkmsg_get_data(struct adf_accel_vf_info *vf_info, + u8 type, u8 byte, u8 max_size, u8 *data, + pf2vf_blkmsg_data_getter_fn data_getter) +{ + u8 blkmsg[ADF_PFVF_BLKMSG_MSG_MAX_SIZE] = { 0 }; + struct adf_accel_dev *accel_dev = vf_info->accel_dev; + adf_pf2vf_blkmsg_provider provider; + u8 msg_size; + + provider = get_blkmsg_response_provider(type); + + if (unlikely(!provider)) { + pr_err("QAT: No registered provider for message %d\n", type); + *data = ADF_PF2VF_INVALID_BLOCK_TYPE; + return -EINVAL; + } + + if (unlikely((*provider)(accel_dev, blkmsg, vf_info->vf_compat_ver))) { + pr_err("QAT: unknown error from provider for message %d\n", type); + *data = ADF_PF2VF_UNSPECIFIED_ERROR; + return -EINVAL; + } + + msg_size = ADF_PFVF_BLKMSG_HEADER_SIZE + blkmsg[ADF_PFVF_BLKMSG_LEN_BYTE]; + + if (unlikely(msg_size >= max_size)) { + pr_err("QAT: Invalid size %d provided for message type %d\n", + msg_size, type); + *data = ADF_PF2VF_PAYLOAD_TRUNCATED; + return -EINVAL; + } + + if (unlikely(byte >= msg_size)) { + pr_err("QAT: Out-of-bound byte number %d (msg size %d)\n", + byte, msg_size); + *data = ADF_PF2VF_INVALID_BYTE_NUM_REQ; + return -EINVAL; + } + + *data = data_getter(blkmsg, byte); + return 0; +} + +static struct pfvf_message handle_blkmsg_req(struct adf_accel_vf_info *vf_info, + struct pfvf_message req) +{ + u8 resp_type = ADF_PF2VF_BLKMSG_RESP_TYPE_ERROR; + struct pfvf_message resp = { 0 }; + u8 resp_data = 0; + u8 blk_type; + u8 blk_byte; + u8 byte_max; + + switch (req.type) { + case ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ: + blk_type = FIELD_GET(ADF_VF2PF_LARGE_BLOCK_TYPE_MASK, req.data) + + ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX + 1; + blk_byte = FIELD_GET(ADF_VF2PF_LARGE_BLOCK_BYTE_MASK, req.data); + byte_max = ADF_VF2PF_LARGE_BLOCK_BYTE_MAX; + break; + case ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ: + blk_type = FIELD_GET(ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK, req.data) + + ADF_VF2PF_SMALL_BLOCK_TYPE_MAX + 1; + blk_byte = FIELD_GET(ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK, req.data); + byte_max = ADF_VF2PF_MEDIUM_BLOCK_BYTE_MAX; + break; + case ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ: + blk_type = FIELD_GET(ADF_VF2PF_SMALL_BLOCK_TYPE_MASK, req.data); + blk_byte = FIELD_GET(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK, req.data); + byte_max = ADF_VF2PF_SMALL_BLOCK_BYTE_MAX; + break; + } + + /* Is this a request for CRC or data? */ + if (FIELD_GET(ADF_VF2PF_BLOCK_CRC_REQ_MASK, req.data)) { + dev_dbg(&GET_DEV(vf_info->accel_dev), + "BlockMsg of type %d for CRC over %d bytes received from VF%d\n", + blk_type, blk_byte, vf_info->vf_nr); + + if (!adf_pf2vf_blkmsg_get_data(vf_info, blk_type, blk_byte, + byte_max, &resp_data, + adf_pf2vf_blkmsg_get_crc)) + resp_type = ADF_PF2VF_BLKMSG_RESP_TYPE_CRC; + } else { + dev_dbg(&GET_DEV(vf_info->accel_dev), + "BlockMsg of type %d for data byte %d received from VF%d\n", + blk_type, blk_byte, vf_info->vf_nr); + + if (!adf_pf2vf_blkmsg_get_data(vf_info, blk_type, blk_byte, + byte_max, &resp_data, + adf_pf2vf_blkmsg_get_byte)) + resp_type = ADF_PF2VF_BLKMSG_RESP_TYPE_DATA; + } + + resp.type = ADF_PF2VF_MSGTYPE_BLKMSG_RESP; + resp.data = FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_TYPE_MASK, resp_type) | + FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_DATA_MASK, resp_data); + + return resp; +} + static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, struct pfvf_message msg, struct pfvf_message *resp) { @@ -106,6 +237,11 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, vf_info->init = false; } break; + case ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ: + case ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ: + case ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ: + *resp = handle_blkmsg_req(vf_info, msg); + break; default: dev_dbg(&GET_DEV(accel_dev), "Unknown message from VF%d (type 0x%.4x, data: 0x%.4x)\n", @@ -147,6 +283,7 @@ bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr) */ int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev) { + adf_pfvf_crc_init(); spin_lock_init(&accel_dev->pf.vf2pf_ints_lock); return 0; diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_utils.c b/drivers/crypto/qat/qat_common/adf_pfvf_utils.c index 494da89e2048..c5f6d77d4bb8 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_utils.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_utils.c @@ -1,11 +1,26 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2021 Intel Corporation */ +#include #include #include #include "adf_accel_devices.h" #include "adf_pfvf_msg.h" #include "adf_pfvf_utils.h" +/* CRC Calculation */ +DECLARE_CRC8_TABLE(pfvf_crc8_table); +#define ADF_PFVF_CRC8_POLYNOMIAL 0x97 + +void adf_pfvf_crc_init(void) +{ + crc8_populate_msb(pfvf_crc8_table, ADF_PFVF_CRC8_POLYNOMIAL); +} + +u8 adf_pfvf_calc_blkmsg_crc(u8 const *buf, u8 buf_len) +{ + return crc8(pfvf_crc8_table, buf, buf_len, CRC8_INIT_VALUE); +} + static bool set_value_on_csr_msg(struct adf_accel_dev *accel_dev, u32 *csr_msg, u32 value, const struct pfvf_field_format *fmt) { diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_utils.h b/drivers/crypto/qat/qat_common/adf_pfvf_utils.h index 7676fdddbe26..2be048e2287b 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_utils.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_utils.h @@ -10,6 +10,9 @@ #define ADF_PFVF_MSG_ACK_DELAY_US 4 #define ADF_PFVF_MSG_ACK_MAX_DELAY_US (1 * USEC_PER_SEC) +u8 adf_pfvf_calc_blkmsg_crc(u8 const *buf, u8 buf_len); +void adf_pfvf_crc_init(void); + struct pfvf_field_format { u8 offset; u32 mask; diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c index 729c00c0d254..0fdd6b9892d3 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c @@ -1,10 +1,13 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2015 - 2021 Intel Corporation */ +#include #include +#include #include #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "adf_pfvf_msg.h" +#include "adf_pfvf_utils.h" #include "adf_pfvf_vf_msg.h" #include "adf_pfvf_vf_proto.h" @@ -94,6 +97,202 @@ int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, struct pfvf_message msg, return 0; } +static int adf_vf2pf_blkmsg_data_req(struct adf_accel_dev *accel_dev, bool crc, + u8 *type, u8 *data) +{ + struct pfvf_message req = { 0 }; + struct pfvf_message resp = { 0 }; + u8 blk_type; + u8 blk_byte; + u8 msg_type; + u8 max_data; + int err; + + /* Convert the block type to {small, medium, large} size category */ + if (*type <= ADF_VF2PF_SMALL_BLOCK_TYPE_MAX) { + msg_type = ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ; + blk_type = FIELD_PREP(ADF_VF2PF_SMALL_BLOCK_TYPE_MASK, *type); + blk_byte = FIELD_PREP(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK, *data); + max_data = ADF_VF2PF_SMALL_BLOCK_BYTE_MAX; + } else if (*type <= ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX) { + msg_type = ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ; + blk_type = FIELD_PREP(ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK, + *type - ADF_VF2PF_SMALL_BLOCK_TYPE_MAX); + blk_byte = FIELD_PREP(ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK, *data); + max_data = ADF_VF2PF_MEDIUM_BLOCK_BYTE_MAX; + } else if (*type <= ADF_VF2PF_LARGE_BLOCK_TYPE_MAX) { + msg_type = ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ; + blk_type = FIELD_PREP(ADF_VF2PF_LARGE_BLOCK_TYPE_MASK, + *type - ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX); + blk_byte = FIELD_PREP(ADF_VF2PF_LARGE_BLOCK_BYTE_MASK, *data); + max_data = ADF_VF2PF_LARGE_BLOCK_BYTE_MAX; + } else { + dev_err(&GET_DEV(accel_dev), "Invalid message type %u\n", *type); + return -EINVAL; + } + + /* Sanity check */ + if (*data > max_data) { + dev_err(&GET_DEV(accel_dev), + "Invalid byte %s %u for message type %u\n", + crc ? "count" : "index", *data, *type); + return -EINVAL; + } + + /* Build the block message */ + req.type = msg_type; + req.data = blk_type | blk_byte | FIELD_PREP(ADF_VF2PF_BLOCK_CRC_REQ_MASK, crc); + + err = adf_send_vf2pf_req(accel_dev, req, &resp); + if (err) + return err; + + *type = FIELD_GET(ADF_PF2VF_BLKMSG_RESP_TYPE_MASK, resp.data); + *data = FIELD_GET(ADF_PF2VF_BLKMSG_RESP_DATA_MASK, resp.data); + + return 0; +} + +static int adf_vf2pf_blkmsg_get_byte(struct adf_accel_dev *accel_dev, u8 type, + u8 index, u8 *data) +{ + int ret; + + ret = adf_vf2pf_blkmsg_data_req(accel_dev, false, &type, &index); + if (ret < 0) + return ret; + + if (unlikely(type != ADF_PF2VF_BLKMSG_RESP_TYPE_DATA)) { + dev_err(&GET_DEV(accel_dev), + "Unexpected BLKMSG response type %u, byte 0x%x\n", + type, index); + return -EFAULT; + } + + *data = index; + return 0; +} + +static int adf_vf2pf_blkmsg_get_crc(struct adf_accel_dev *accel_dev, u8 type, + u8 bytes, u8 *crc) +{ + int ret; + + /* The count of bytes refers to a length, however shift it to a 0-based + * count to avoid overflows. Thus, a request for 0 bytes is technically + * valid. + */ + --bytes; + + ret = adf_vf2pf_blkmsg_data_req(accel_dev, true, &type, &bytes); + if (ret < 0) + return ret; + + if (unlikely(type != ADF_PF2VF_BLKMSG_RESP_TYPE_CRC)) { + dev_err(&GET_DEV(accel_dev), + "Unexpected CRC BLKMSG response type %u, crc 0x%x\n", + type, bytes); + return -EFAULT; + } + + *crc = bytes; + return 0; +} + +/** + * adf_send_vf2pf_blkmsg_req() - retrieve block message + * @accel_dev: Pointer to acceleration VF device. + * @type: The block message type, see adf_pfvf_msg.h for allowed values + * @buffer: input buffer where to place the received data + * @buffer_len: buffer length as input, the amount of written bytes on output + * + * Request a message of type 'type' over the block message transport. + * This function will send the required amount block message requests and + * return the overall content back to the caller through the provided buffer. + * The buffer should be large enough to contain the requested message type, + * otherwise the response will be truncated. + * + * Return: 0 on success, error code otherwise. + */ +int adf_send_vf2pf_blkmsg_req(struct adf_accel_dev *accel_dev, u8 type, + u8 *buffer, unsigned int *buffer_len) +{ + unsigned int index; + unsigned int msg_len; + int ret; + u8 remote_crc; + u8 local_crc; + + if (unlikely(type > ADF_VF2PF_LARGE_BLOCK_TYPE_MAX)) { + dev_err(&GET_DEV(accel_dev), "Invalid block message type %d\n", + type); + return -EINVAL; + } + + if (unlikely(*buffer_len < ADF_PFVF_BLKMSG_HEADER_SIZE)) { + dev_err(&GET_DEV(accel_dev), + "Buffer size too small for a block message\n"); + return -EINVAL; + } + + ret = adf_vf2pf_blkmsg_get_byte(accel_dev, type, + ADF_PFVF_BLKMSG_VER_BYTE, + &buffer[ADF_PFVF_BLKMSG_VER_BYTE]); + if (unlikely(ret)) + return ret; + + if (unlikely(!buffer[ADF_PFVF_BLKMSG_VER_BYTE])) { + dev_err(&GET_DEV(accel_dev), + "Invalid version 0 received for block request %u", type); + return -EFAULT; + } + + ret = adf_vf2pf_blkmsg_get_byte(accel_dev, type, + ADF_PFVF_BLKMSG_LEN_BYTE, + &buffer[ADF_PFVF_BLKMSG_LEN_BYTE]); + if (unlikely(ret)) + return ret; + + if (unlikely(!buffer[ADF_PFVF_BLKMSG_LEN_BYTE])) { + dev_err(&GET_DEV(accel_dev), + "Invalid size 0 received for block request %u", type); + return -EFAULT; + } + + /* We need to pick the minimum since there is no way to request a + * specific version. As a consequence any scenario is possible: + * - PF has a newer (longer) version which doesn't fit in the buffer + * - VF expects a newer (longer) version, so we must not ask for + * bytes in excess + * - PF and VF share the same version, no problem + */ + msg_len = ADF_PFVF_BLKMSG_HEADER_SIZE + buffer[ADF_PFVF_BLKMSG_LEN_BYTE]; + msg_len = min(*buffer_len, msg_len); + + /* Get the payload */ + for (index = ADF_PFVF_BLKMSG_HEADER_SIZE; index < msg_len; index++) { + ret = adf_vf2pf_blkmsg_get_byte(accel_dev, type, index, + &buffer[index]); + if (unlikely(ret)) + return ret; + } + + ret = adf_vf2pf_blkmsg_get_crc(accel_dev, type, msg_len, &remote_crc); + if (unlikely(ret)) + return ret; + + local_crc = adf_pfvf_calc_blkmsg_crc(buffer, msg_len); + if (unlikely(local_crc != remote_crc)) { + dev_err(&GET_DEV(accel_dev), + "CRC error on msg type %d. Local %02X, remote %02X\n", + type, local_crc, remote_crc); + return -EIO; + } + + *buffer_len = msg_len; + return 0; +} + static bool adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev, struct pfvf_message msg) { @@ -104,6 +303,7 @@ static bool adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev, adf_pf2vf_handle_pf_restarting(accel_dev); return false; case ADF_PF2VF_MSGTYPE_VERSION_RESP: + case ADF_PF2VF_MSGTYPE_BLKMSG_RESP: dev_dbg(&GET_DEV(accel_dev), "Response Message received from PF (type 0x%.4x, data 0x%.4x)\n", msg.type, msg.data); @@ -135,12 +335,13 @@ bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev) /** * adf_enable_vf2pf_comms() - Function enables communication from vf to pf * - * @accel_dev: Pointer to acceleration device virtual function. + * @accel_dev: Pointer to acceleration device virtual function. * * Return: 0 on success, error code otherwise. */ int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) { + adf_pfvf_crc_init(); adf_enable_pf2vf_interrupts(accel_dev); return adf_vf2pf_request_version(accel_dev); } diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h index e32d1bc3a740..f6ee9b38c0e1 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h @@ -9,6 +9,8 @@ int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, struct pfvf_message msg); int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, struct pfvf_message msg, struct pfvf_message *resp); +int adf_send_vf2pf_blkmsg_req(struct adf_accel_dev *accel_dev, u8 type, + u8 *buffer, unsigned int *buffer_len); int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index 6366622ff8fd..971a05d62418 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -8,6 +8,9 @@ #include "adf_cfg.h" #include "adf_pfvf_pf_msg.h" +#define ADF_VF2PF_RATELIMIT_INTERVAL 8 +#define ADF_VF2PF_RATELIMIT_BURST 130 + static struct workqueue_struct *pf2vf_resp_wq; struct adf_pf2vf_resp { @@ -62,8 +65,8 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) mutex_init(&vf_info->pf2vf_lock); ratelimit_state_init(&vf_info->vf2pf_ratelimit, - DEFAULT_RATELIMIT_INTERVAL, - DEFAULT_RATELIMIT_BURST); + ADF_VF2PF_RATELIMIT_INTERVAL, + ADF_VF2PF_RATELIMIT_BURST); } /* Set Valid bits in AE Thread to PCIe Function Mapping */ From 851ed498dba11b96d2f696e23f9084e0add1896f Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Thu, 16 Dec 2021 09:13:27 +0000 Subject: [PATCH 117/145] crypto: qat - exchange device capabilities over PFVF Allow the VF driver to get the supported device capabilities through PFVF, by adding a new block message, the Capability Summary. This messages allows to exchange the capability through masks, which report, depending on the Capability Summary version, up to the following information: - algorithms and/or services that are supported by the device (e.g. symmetric crypto, data compression, etc.) - (extended) compression capabilities, with details about the compression service (e.g. if compress and verify is supported by this device) - the frequency of the device This patch supports the latest Capabilities Summary version 3 for VFs, but will limit support for the PF driver to version 2. This change also increases the PFVF protocol to version 2. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_accel_devices.h | 1 + drivers/crypto/qat/qat_common/adf_pfvf_msg.h | 35 ++++++++++++-- .../crypto/qat/qat_common/adf_pfvf_pf_msg.c | 18 ++++++++ .../crypto/qat/qat_common/adf_pfvf_pf_msg.h | 3 ++ .../crypto/qat/qat_common/adf_pfvf_pf_proto.c | 1 + .../crypto/qat/qat_common/adf_pfvf_vf_msg.c | 46 +++++++++++++++++++ .../crypto/qat/qat_common/adf_pfvf_vf_msg.h | 1 + .../crypto/qat/qat_common/adf_pfvf_vf_proto.c | 11 ++++- 8 files changed, 112 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 59f06e53d316..55e8948a8c5e 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -203,6 +203,7 @@ struct adf_hw_device_data { u32 straps; u32 accel_capabilities_mask; u32 extended_dc_capabilities; + u32 clock_frequency; u32 instance_id; u16 accel_mask; u32 ae_mask; diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h index 6abbb5e05809..f418dd26a742 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h @@ -87,8 +87,10 @@ enum vf2pf_msgtype { /* VF/PF compatibility version. */ enum pfvf_compatibility_version { - /* Reference to the current version */ - ADF_PFVF_COMPAT_THIS_VERSION = 0x01, + /* Support for extended capabilities */ + ADF_PFVF_COMPAT_CAPABILITIES = 0x02, + /* Reference to the latest version */ + ADF_PFVF_COMPAT_THIS_VERSION = 0x02, }; /* PF->VF Version Response */ @@ -133,7 +135,9 @@ enum pf2vf_blkmsg_error { * 16..23 - 64 byte message * 24..27 - 128 byte message */ -/* No block messages as of yet */ +enum vf2pf_blkmsg_req_type { + ADF_VF2PF_BLKMSG_REQ_CAP_SUMMARY = 0x02, +}; #define ADF_VF2PF_SMALL_BLOCK_TYPE_MAX \ (FIELD_MAX(ADF_VF2PF_SMALL_BLOCK_TYPE_MASK)) @@ -171,4 +175,29 @@ struct pfvf_blkmsg_header { #define ADF_PFVF_BLKMSG_VER_BYTE 0 #define ADF_PFVF_BLKMSG_LEN_BYTE 1 +/* PF/VF Capabilities message values */ +enum blkmsg_capabilities_versions { + ADF_PFVF_CAPABILITIES_V1_VERSION = 0x01, + ADF_PFVF_CAPABILITIES_V2_VERSION = 0x02, + ADF_PFVF_CAPABILITIES_V3_VERSION = 0x03, +}; + +struct capabilities_v1 { + struct pfvf_blkmsg_header hdr; + u32 ext_dc_caps; +} __packed; + +struct capabilities_v2 { + struct pfvf_blkmsg_header hdr; + u32 ext_dc_caps; + u32 capabilities; +} __packed; + +struct capabilities_v3 { + struct pfvf_blkmsg_header hdr; + u32 ext_dc_caps; + u32 capabilities; + u32 frequency; +} __packed; + #endif /* ADF_PFVF_MSG_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c index ad198b624098..5732cea1c7ca 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c @@ -18,3 +18,21 @@ void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev) "Failed to send restarting msg to VF%d\n", i); } } + +int adf_pf_capabilities_msg_provider(struct adf_accel_dev *accel_dev, + u8 *buffer, u8 compat) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct capabilities_v2 caps_msg; + + caps_msg.ext_dc_caps = hw_data->extended_dc_capabilities; + caps_msg.capabilities = hw_data->accel_capabilities_mask; + + caps_msg.hdr.version = ADF_PFVF_CAPABILITIES_V2_VERSION; + caps_msg.hdr.payload_size = + ADF_PFVF_BLKMSG_PAYLOAD_SIZE(struct capabilities_v2); + + memcpy(buffer, &caps_msg, sizeof(caps_msg)); + + return 0; +} diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h index 5c669f1de6e4..401450bd30b0 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h @@ -10,4 +10,7 @@ void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev); typedef int (*adf_pf2vf_blkmsg_provider)(struct adf_accel_dev *accel_dev, u8 *buffer, u8 compat); +int adf_pf_capabilities_msg_provider(struct adf_accel_dev *accel_dev, + u8 *buffer, u8 comapt); + #endif /* ADF_PFVF_PF_MSG_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c index 850b5f4414a6..1256d68c3efd 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c @@ -15,6 +15,7 @@ typedef u8 (*pf2vf_blkmsg_data_getter_fn)(u8 const *blkmsg, u8 byte); static const adf_pf2vf_blkmsg_provider pf2vf_blkmsg_providers[] = { NULL, /* no message type defined for value 0 */ NULL, /* no message type defined for value 1 */ + adf_pf_capabilities_msg_provider, /* ADF_VF2PF_BLKMSG_REQ_CAP_SUMMARY */ }; /** diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c index 307d593042f3..b08f8544991a 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c @@ -92,3 +92,49 @@ int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) accel_dev->vf.pf_compat_ver = pf_version; return 0; } + +int adf_vf2pf_get_capabilities(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct capabilities_v3 cap_msg = { { 0 }, }; + unsigned int len = sizeof(cap_msg); + + if (accel_dev->vf.pf_compat_ver < ADF_PFVF_COMPAT_CAPABILITIES) + /* The PF is too old to support the extended capabilities */ + return 0; + + if (adf_send_vf2pf_blkmsg_req(accel_dev, ADF_VF2PF_BLKMSG_REQ_CAP_SUMMARY, + (u8 *)&cap_msg, &len)) { + dev_err(&GET_DEV(accel_dev), + "QAT: Failed to get block message response\n"); + return -EFAULT; + } + + switch (cap_msg.hdr.version) { + default: + /* Newer version received, handle only the know parts */ + fallthrough; + case ADF_PFVF_CAPABILITIES_V3_VERSION: + if (likely(len >= sizeof(struct capabilities_v3))) + hw_data->clock_frequency = cap_msg.frequency; + else + dev_info(&GET_DEV(accel_dev), "Could not get frequency"); + fallthrough; + case ADF_PFVF_CAPABILITIES_V2_VERSION: + if (likely(len >= sizeof(struct capabilities_v2))) + hw_data->accel_capabilities_mask = cap_msg.capabilities; + else + dev_info(&GET_DEV(accel_dev), "Could not get capabilities"); + fallthrough; + case ADF_PFVF_CAPABILITIES_V1_VERSION: + if (likely(len >= sizeof(struct capabilities_v1))) { + hw_data->extended_dc_capabilities = cap_msg.ext_dc_caps; + } else { + dev_err(&GET_DEV(accel_dev), + "Capabilities message truncated to %d bytes\n", len); + return -EFAULT; + } + } + + return 0; +} diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h index 5091b5b2fd8f..c1f31354c138 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h @@ -7,6 +7,7 @@ int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev); void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev); int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev); +int adf_vf2pf_get_capabilities(struct adf_accel_dev *accel_dev); #else static inline int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev) { diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c index 0fdd6b9892d3..a85bd6dcb62a 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c @@ -341,8 +341,17 @@ bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev) */ int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) { + int ret; + adf_pfvf_crc_init(); adf_enable_pf2vf_interrupts(accel_dev); - return adf_vf2pf_request_version(accel_dev); + + ret = adf_vf2pf_request_version(accel_dev); + if (ret) + return ret; + + ret = adf_vf2pf_get_capabilities(accel_dev); + + return ret; } EXPORT_SYMBOL_GPL(adf_enable_vf2pf_comms); From 73ef8f3382d10dbed6fff0b606f9a3351068a0b9 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Thu, 16 Dec 2021 09:13:28 +0000 Subject: [PATCH 118/145] crypto: qat - support fast ACKs in the PFVF protocol The original design and current implementation of the PFVF protocol expects the sender to both acquire and relinquish the ownership of the shared CSR by setting and clearing the "in use" pattern on the remote half of the register when sending a message. This happens regardless of the acknowledgment of the reception, to guarantee changes, including collisions, are surely detected. However, in the case of a request that requires a response, collisions can also be detected by the lack of a reply. This can be exploited to speed up and simplify the above behaviour, letting the receiver both acknowledge the message and release the CSR in a single transaction: 1) the sender can return as soon as the message has been acknowledged 2) the receiver doesn't have to wait long before acquiring ownership of the CSR for the response message, greatly improving the overall throughput. Howerver, this improvement cannot be leveraged for fire-and-forget notifications, as it would be impossible for the sender to clearly distinguish between a collision and an ack immediately followed by a new message. This patch implements this optimization in a new version of the protocol (v3), which applies the fast-ack logic only whenever possible and guarantees backward compatibility with older versions. For requests, a new retry loop guarantees a correct behaviour. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_accel_devices.h | 2 +- drivers/crypto/qat/qat_common/adf_gen2_pfvf.c | 68 ++++++++++++++++--- drivers/crypto/qat/qat_common/adf_pfvf_msg.h | 4 +- .../crypto/qat/qat_common/adf_pfvf_pf_proto.c | 3 +- .../crypto/qat/qat_common/adf_pfvf_vf_proto.c | 44 ++++++------ 5 files changed, 91 insertions(+), 30 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 55e8948a8c5e..1b9d4ed03dd0 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -158,7 +158,7 @@ struct adf_pfvf_ops { int (*send_msg)(struct adf_accel_dev *accel_dev, struct pfvf_message msg, u32 pfvf_offset, struct mutex *csr_lock); struct pfvf_message (*recv_msg)(struct adf_accel_dev *accel_dev, - u32 pfvf_offset); + u32 pfvf_offset, u8 compat_ver); }; struct adf_hw_device_data { diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c index feab01ec4bbb..1a9072aac2ca 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c @@ -124,11 +124,34 @@ static bool is_legacy_user_pfvf_message(u32 msg) return !(msg & ADF_PFVF_MSGORIGIN_SYSTEM); } +static bool is_pf2vf_notification(u8 msg_type) +{ + switch (msg_type) { + case ADF_PF2VF_MSGTYPE_RESTARTING: + return true; + default: + return false; + } +} + +static bool is_vf2pf_notification(u8 msg_type) +{ + switch (msg_type) { + case ADF_VF2PF_MSGTYPE_INIT: + case ADF_VF2PF_MSGTYPE_SHUTDOWN: + return true; + default: + return false; + } +} + struct pfvf_gen2_params { u32 pfvf_offset; struct mutex *csr_lock; /* lock preventing concurrent access of CSR */ enum gen2_csr_pos local_offset; enum gen2_csr_pos remote_offset; + bool (*is_notification_message)(u8 msg_type); + u8 compat_ver; }; static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev, @@ -190,15 +213,27 @@ start: csr_val &= ~int_bit; } - if (csr_val != csr_msg) { - dev_dbg(&GET_DEV(accel_dev), - "Collision - PFVF CSR overwritten by remote function\n"); + /* For fire-and-forget notifications, the receiver does not clear + * the in-use pattern. This is used to detect collisions. + */ + if (params->is_notification_message(msg.type) && csr_val != csr_msg) { + /* Collision must have overwritten the message */ + dev_err(&GET_DEV(accel_dev), + "Collision on notification - PFVF CSR overwritten by remote function\n"); goto retry; } - /* Finished with the PFVF CSR; relinquish it and leave msg in CSR */ - gen2_csr_clear_in_use(&csr_val, remote_offset); - ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val); + /* If the far side did not clear the in-use pattern it is either + * 1) Notification - message left intact to detect collision + * 2) Older protocol (compatibility version < 3) on the far side + * where the sender is responsible for clearing the in-use + * pattern after the received has acknowledged receipt. + * In either case, clear the in-use pattern now. + */ + if (gen2_csr_is_in_use(csr_val, remote_offset)) { + gen2_csr_clear_in_use(&csr_val, remote_offset); + ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val); + } out: mutex_unlock(lock); @@ -218,6 +253,7 @@ static struct pfvf_message adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev, struct pfvf_gen2_params *params) { void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + enum gen2_csr_pos remote_offset = params->remote_offset; enum gen2_csr_pos local_offset = params->local_offset; u32 pfvf_offset = params->pfvf_offset; struct pfvf_message msg = { 0 }; @@ -242,12 +278,22 @@ static struct pfvf_message adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev, if (unlikely(is_legacy_user_pfvf_message(csr_msg))) { dev_dbg(&GET_DEV(accel_dev), "Ignored non-system message (0x%.8x);\n", csr_val); + /* Because this must be a legacy message, the far side + * must clear the in-use pattern, so don't do it. + */ return msg; } /* Return the pfvf_message format */ msg = adf_pfvf_message_of(accel_dev, csr_msg, &csr_gen2_fmt); + /* The in-use pattern is not cleared for notifications (so that + * it can be used for collision detection) or older implementations + */ + if (params->compat_ver >= ADF_PFVF_COMPAT_FAST_ACK && + !params->is_notification_message(msg.type)) + gen2_csr_clear_in_use(&csr_val, remote_offset); + /* To ACK, clear the INT bit */ csr_val &= ~int_bit; ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val); @@ -263,6 +309,7 @@ static int adf_gen2_pf2vf_send(struct adf_accel_dev *accel_dev, struct pfvf_mess .pfvf_offset = pfvf_offset, .local_offset = ADF_GEN2_CSR_PF2VF_OFFSET, .remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET, + .is_notification_message = is_pf2vf_notification, }; return adf_gen2_pfvf_send(accel_dev, msg, ¶ms); @@ -276,30 +323,35 @@ static int adf_gen2_vf2pf_send(struct adf_accel_dev *accel_dev, struct pfvf_mess .pfvf_offset = pfvf_offset, .local_offset = ADF_GEN2_CSR_VF2PF_OFFSET, .remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET, + .is_notification_message = is_vf2pf_notification, }; return adf_gen2_pfvf_send(accel_dev, msg, ¶ms); } static struct pfvf_message adf_gen2_pf2vf_recv(struct adf_accel_dev *accel_dev, - u32 pfvf_offset) + u32 pfvf_offset, u8 compat_ver) { struct pfvf_gen2_params params = { .pfvf_offset = pfvf_offset, .local_offset = ADF_GEN2_CSR_PF2VF_OFFSET, .remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET, + .is_notification_message = is_pf2vf_notification, + .compat_ver = compat_ver, }; return adf_gen2_pfvf_recv(accel_dev, ¶ms); } static struct pfvf_message adf_gen2_vf2pf_recv(struct adf_accel_dev *accel_dev, - u32 pfvf_offset) + u32 pfvf_offset, u8 compat_ver) { struct pfvf_gen2_params params = { .pfvf_offset = pfvf_offset, .local_offset = ADF_GEN2_CSR_VF2PF_OFFSET, .remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET, + .is_notification_message = is_vf2pf_notification, + .compat_ver = compat_ver, }; return adf_gen2_pfvf_recv(accel_dev, ¶ms); diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h index f418dd26a742..df052194ece7 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h @@ -89,8 +89,10 @@ enum vf2pf_msgtype { enum pfvf_compatibility_version { /* Support for extended capabilities */ ADF_PFVF_COMPAT_CAPABILITIES = 0x02, + /* In-use pattern cleared by receiver */ + ADF_PFVF_COMPAT_FAST_ACK = 0x03, /* Reference to the latest version */ - ADF_PFVF_COMPAT_THIS_VERSION = 0x02, + ADF_PFVF_COMPAT_THIS_VERSION = 0x03, }; /* PF->VF Version Response */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c index 1256d68c3efd..cd728d5ac9ab 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c @@ -48,10 +48,11 @@ int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, struct pfvf_me */ static struct pfvf_message adf_recv_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr) { + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev); u32 pfvf_offset = pfvf_ops->get_vf2pf_offset(vf_nr); - return pfvf_ops->recv_msg(accel_dev, pfvf_offset); + return pfvf_ops->recv_msg(accel_dev, pfvf_offset, vf_info->vf_compat_ver); } static adf_pf2vf_blkmsg_provider get_blkmsg_response_provider(u8 type) diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c index a85bd6dcb62a..b9a1cf5d58a9 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c @@ -15,6 +15,8 @@ #define ADF_PFVF_MSG_ACK_DELAY 2 #define ADF_PFVF_MSG_ACK_MAX_RETRY 100 +/* How often to retry if there is no response */ +#define ADF_PFVF_MSG_RESP_RETRIES 5 #define ADF_PFVF_MSG_RESP_TIMEOUT (ADF_PFVF_MSG_ACK_DELAY * \ ADF_PFVF_MSG_ACK_MAX_RETRY + \ ADF_PFVF_MSG_COLLISION_DETECT_DELAY) @@ -50,7 +52,7 @@ static struct pfvf_message adf_recv_pf2vf_msg(struct adf_accel_dev *accel_dev) struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev); u32 pfvf_offset = pfvf_ops->get_pf2vf_offset(0); - return pfvf_ops->recv_msg(accel_dev, pfvf_offset); + return pfvf_ops->recv_msg(accel_dev, pfvf_offset, accel_dev->vf.pf_compat_ver); } /** @@ -68,33 +70,37 @@ int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, struct pfvf_message msg, struct pfvf_message *resp) { unsigned long timeout = msecs_to_jiffies(ADF_PFVF_MSG_RESP_TIMEOUT); + unsigned int retries = ADF_PFVF_MSG_RESP_RETRIES; int ret; reinit_completion(&accel_dev->vf.msg_received); /* Send request from VF to PF */ - ret = adf_send_vf2pf_msg(accel_dev, msg); - if (ret) { - dev_err(&GET_DEV(accel_dev), - "Failed to send request msg to PF\n"); - return ret; - } + do { + ret = adf_send_vf2pf_msg(accel_dev, msg); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to send request msg to PF\n"); + return ret; + } - /* Wait for response */ - if (!wait_for_completion_timeout(&accel_dev->vf.msg_received, - timeout)) { - dev_err(&GET_DEV(accel_dev), - "PFVF request/response message timeout expired\n"); - return -EIO; - } + /* Wait for response, if it times out retry */ + ret = wait_for_completion_timeout(&accel_dev->vf.msg_received, + timeout); + if (ret) { + if (likely(resp)) + *resp = accel_dev->vf.response; - if (likely(resp)) - *resp = accel_dev->vf.response; + /* Once copied, set to an invalid value */ + accel_dev->vf.response.type = 0; - /* Once copied, set to an invalid value */ - accel_dev->vf.response.type = 0; + return 0; + } - return 0; + dev_err(&GET_DEV(accel_dev), "PFVF response message timeout\n"); + } while (--retries); + + return -EIO; } static int adf_vf2pf_blkmsg_data_req(struct adf_accel_dev *accel_dev, bool crc, From e1b176af3d7ecf9f9c0c0db374b37a40073ac960 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Thu, 16 Dec 2021 09:13:29 +0000 Subject: [PATCH 119/145] crypto: qat - exchange ring-to-service mappings over PFVF In addition to retrieving the device capabilities, a VF may also need to retrieve the mapping of its ring pairs to crypto and or compression services in order to work properly. Make the VF receive the ring-to-service mappings from the PF by means of a new REQ_RING_SVC_MAP Block Message and add the request and response logic on VF and PF respectively. This change requires to bump the PFVF protocol to version 4. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_pfvf_msg.h | 15 ++++++++++- .../crypto/qat/qat_common/adf_pfvf_pf_msg.c | 14 ++++++++++ .../crypto/qat/qat_common/adf_pfvf_pf_msg.h | 2 ++ .../crypto/qat/qat_common/adf_pfvf_pf_proto.c | 1 + .../crypto/qat/qat_common/adf_pfvf_vf_msg.c | 27 +++++++++++++++++++ .../crypto/qat/qat_common/adf_pfvf_vf_msg.h | 1 + .../crypto/qat/qat_common/adf_pfvf_vf_proto.c | 4 +++ 7 files changed, 63 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h index df052194ece7..1d3cad7d4999 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h @@ -91,8 +91,10 @@ enum pfvf_compatibility_version { ADF_PFVF_COMPAT_CAPABILITIES = 0x02, /* In-use pattern cleared by receiver */ ADF_PFVF_COMPAT_FAST_ACK = 0x03, + /* Ring to service mapping support for non-standard mappings */ + ADF_PFVF_COMPAT_RING_TO_SVC_MAP = 0x04, /* Reference to the latest version */ - ADF_PFVF_COMPAT_THIS_VERSION = 0x03, + ADF_PFVF_COMPAT_THIS_VERSION = 0x04, }; /* PF->VF Version Response */ @@ -139,6 +141,7 @@ enum pf2vf_blkmsg_error { */ enum vf2pf_blkmsg_req_type { ADF_VF2PF_BLKMSG_REQ_CAP_SUMMARY = 0x02, + ADF_VF2PF_BLKMSG_REQ_RING_SVC_MAP = 0x03, }; #define ADF_VF2PF_SMALL_BLOCK_TYPE_MAX \ @@ -202,4 +205,14 @@ struct capabilities_v3 { u32 frequency; } __packed; +/* PF/VF Ring to service mapping values */ +enum blkmsg_ring_to_svc_versions { + ADF_PFVF_RING_TO_SVC_VERSION = 0x01, +}; + +struct ring_to_svc_map_v1 { + struct pfvf_blkmsg_header hdr; + u16 map; +} __packed; + #endif /* ADF_PFVF_MSG_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c index 5732cea1c7ca..14c069f0d71a 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c @@ -36,3 +36,17 @@ int adf_pf_capabilities_msg_provider(struct adf_accel_dev *accel_dev, return 0; } + +int adf_pf_ring_to_svc_msg_provider(struct adf_accel_dev *accel_dev, + u8 *buffer, u8 compat) +{ + struct ring_to_svc_map_v1 rts_map_msg; + + rts_map_msg.map = accel_dev->hw_device->ring_to_svc_map; + rts_map_msg.hdr.version = ADF_PFVF_RING_TO_SVC_VERSION; + rts_map_msg.hdr.payload_size = ADF_PFVF_BLKMSG_PAYLOAD_SIZE(rts_map_msg); + + memcpy(buffer, &rts_map_msg, sizeof(rts_map_msg)); + + return 0; +} diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h index 401450bd30b0..e8982d1ac896 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h @@ -12,5 +12,7 @@ typedef int (*adf_pf2vf_blkmsg_provider)(struct adf_accel_dev *accel_dev, int adf_pf_capabilities_msg_provider(struct adf_accel_dev *accel_dev, u8 *buffer, u8 comapt); +int adf_pf_ring_to_svc_msg_provider(struct adf_accel_dev *accel_dev, + u8 *buffer, u8 comapt); #endif /* ADF_PFVF_PF_MSG_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c index cd728d5ac9ab..84230aac67e6 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c @@ -16,6 +16,7 @@ static const adf_pf2vf_blkmsg_provider pf2vf_blkmsg_providers[] = { NULL, /* no message type defined for value 0 */ NULL, /* no message type defined for value 1 */ adf_pf_capabilities_msg_provider, /* ADF_VF2PF_BLKMSG_REQ_CAP_SUMMARY */ + adf_pf_ring_to_svc_msg_provider, /* ADF_VF2PF_BLKMSG_REQ_RING_SVC_MAP */ }; /** diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c index b08f8544991a..14b222691c9c 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c @@ -138,3 +138,30 @@ int adf_vf2pf_get_capabilities(struct adf_accel_dev *accel_dev) return 0; } + +int adf_vf2pf_get_ring_to_svc(struct adf_accel_dev *accel_dev) +{ + struct ring_to_svc_map_v1 rts_map_msg = { { 0 }, }; + unsigned int len = sizeof(rts_map_msg); + + if (accel_dev->vf.pf_compat_ver < ADF_PFVF_COMPAT_RING_TO_SVC_MAP) + /* Use already set default mappings */ + return 0; + + if (adf_send_vf2pf_blkmsg_req(accel_dev, ADF_VF2PF_BLKMSG_REQ_RING_SVC_MAP, + (u8 *)&rts_map_msg, &len)) { + dev_err(&GET_DEV(accel_dev), + "QAT: Failed to get block message response\n"); + return -EFAULT; + } + + if (unlikely(len < sizeof(struct ring_to_svc_map_v1))) { + dev_err(&GET_DEV(accel_dev), + "RING_TO_SVC message truncated to %d bytes\n", len); + return -EFAULT; + } + + /* Only v1 at present */ + accel_dev->hw_device->ring_to_svc_map = rts_map_msg.map; + return 0; +} diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h index c1f31354c138..71bc0e3f1d93 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h @@ -8,6 +8,7 @@ int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev); void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev); int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev); int adf_vf2pf_get_capabilities(struct adf_accel_dev *accel_dev); +int adf_vf2pf_get_ring_to_svc(struct adf_accel_dev *accel_dev); #else static inline int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev) { diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c index b9a1cf5d58a9..0e4b8397cbe3 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c @@ -357,6 +357,10 @@ int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) return ret; ret = adf_vf2pf_get_capabilities(accel_dev); + if (ret) + return ret; + + ret = adf_vf2pf_get_ring_to_svc(accel_dev); return ret; } From 925b3069cf6e37a86fc752a35578ec875772bc7c Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Thu, 16 Dec 2021 09:13:30 +0000 Subject: [PATCH 120/145] crypto: qat - config VFs based on ring-to-svc mapping Change the configuration logic for the VF driver to leverage the ring-to-service mappings now received via PFVF. While the driver config logic is not yet capable of supporting configurations other than the default mapping, make sure that both VF and PF share the same default configuration in order to work properly. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_c3xxxvf/adf_drv.c | 4 --- drivers/crypto/qat/qat_c62xvf/adf_drv.c | 4 --- .../crypto/qat/qat_common/adf_common_drv.h | 1 + drivers/crypto/qat/qat_common/adf_init.c | 9 ++++++- drivers/crypto/qat/qat_common/qat_crypto.c | 25 +++++++++++++++++++ drivers/crypto/qat/qat_dh895xccvf/adf_drv.c | 4 --- 6 files changed, 34 insertions(+), 13 deletions(-) diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c index 0ba1d293bb81..fa18d8009f53 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c @@ -173,10 +173,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* Completion for VF2PF request/response message exchange */ init_completion(&accel_dev->vf.msg_received); - ret = qat_crypto_dev_config(accel_dev); - if (ret) - goto out_err_free_reg; - ret = adf_dev_init(accel_dev); if (ret) goto out_err_dev_shutdown; diff --git a/drivers/crypto/qat/qat_c62xvf/adf_drv.c b/drivers/crypto/qat/qat_c62xvf/adf_drv.c index 176d8e2786f4..686ec752d0e9 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_drv.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_drv.c @@ -173,10 +173,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* Completion for VF2PF request/response message exchange */ init_completion(&accel_dev->vf.msg_received); - ret = qat_crypto_dev_config(accel_dev); - if (ret) - goto out_err_free_reg; - ret = adf_dev_init(accel_dev); if (ret) goto out_err_dev_shutdown; diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 5212891344a9..76f4f96ec5eb 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -114,6 +114,7 @@ void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev); int qat_crypto_register(void); int qat_crypto_unregister(void); int qat_crypto_dev_config(struct adf_accel_dev *accel_dev); +int qat_crypto_vf_dev_config(struct adf_accel_dev *accel_dev); struct qat_crypto_instance *qat_crypto_get_instance_node(int node); void qat_crypto_put_instance(struct qat_crypto_instance *inst); void qat_alg_callback(void *resp); diff --git a/drivers/crypto/qat/qat_common/adf_init.c b/drivers/crypto/qat/qat_common/adf_init.c index 391d82a64a93..2edc63c6b6ca 100644 --- a/drivers/crypto/qat/qat_common/adf_init.c +++ b/drivers/crypto/qat/qat_common/adf_init.c @@ -69,7 +69,8 @@ int adf_dev_init(struct adf_accel_dev *accel_dev) return -EFAULT; } - if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status)) { + if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status) && + !accel_dev->is_vf) { dev_err(&GET_DEV(accel_dev), "Device not configured\n"); return -EFAULT; } @@ -121,6 +122,12 @@ int adf_dev_init(struct adf_accel_dev *accel_dev) if (ret) return ret; + if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status) && + accel_dev->is_vf) { + if (qat_crypto_vf_dev_config(accel_dev)) + return -EFAULT; + } + /* * Subservice initialisation is divided into two stages: init and start. * This is to facilitate any ordering dependencies between services diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c index ece6776fbd53..7234c4940fae 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.c +++ b/drivers/crypto/qat/qat_common/qat_crypto.c @@ -8,6 +8,7 @@ #include "adf_transport_access_macros.h" #include "adf_cfg.h" #include "adf_cfg_strings.h" +#include "adf_gen2_hw_data.h" #include "qat_crypto.h" #include "icp_qat_fw.h" @@ -104,6 +105,30 @@ struct qat_crypto_instance *qat_crypto_get_instance_node(int node) return inst; } +/** + * qat_crypto_vf_dev_config() + * create dev config required to create crypto inst. + * + * @accel_dev: Pointer to acceleration device. + * + * Function creates device configuration required to create + * asym, sym or, crypto instances + * + * Return: 0 on success, error code otherwise. + */ +int qat_crypto_vf_dev_config(struct adf_accel_dev *accel_dev) +{ + u16 ring_to_svc_map = GET_HW_DATA(accel_dev)->ring_to_svc_map; + + if (ring_to_svc_map != ADF_GEN2_DEFAULT_RING_TO_SRV_MAP) { + dev_err(&GET_DEV(accel_dev), + "Unsupported ring/service mapping present on PF"); + return -EFAULT; + } + + return qat_crypto_dev_config(accel_dev); +} + /** * qat_crypto_dev_config() - create dev config required to create crypto inst. * diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c index ee45d688b5d7..18756b2e1c91 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c @@ -173,10 +173,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* Completion for VF2PF request/response message exchange */ init_completion(&accel_dev->vf.msg_received); - ret = qat_crypto_dev_config(accel_dev); - if (ret) - goto out_err_free_reg; - ret = adf_dev_init(accel_dev); if (ret) goto out_err_dev_shutdown; From a9dc0d966605377e9aa512efdcaf9653f40cc2d5 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Thu, 16 Dec 2021 09:13:31 +0000 Subject: [PATCH 121/145] crypto: qat - add PFVF support to the GEN4 host driver So far PFVF support for GEN4 devices has been kept effectively disabled due to lack of support. This patch adds all the GEN4 specific logic to make PFVF fully functional on PF. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 10 +- drivers/crypto/qat/qat_common/Makefile | 2 +- drivers/crypto/qat/qat_common/adf_gen4_pfvf.c | 148 ++++++++++++++++++ drivers/crypto/qat/qat_common/adf_gen4_pfvf.h | 17 ++ drivers/crypto/qat/qat_common/adf_pfvf_msg.h | 29 +++- 5 files changed, 196 insertions(+), 10 deletions(-) create mode 100644 drivers/crypto/qat/qat_common/adf_gen4_pfvf.c create mode 100644 drivers/crypto/qat/qat_common/adf_gen4_pfvf.h diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c index 67cd20f443ab..ef71aa4efd64 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -4,6 +4,7 @@ #include #include #include +#include #include "adf_4xxx_hw_data.h" #include "icp_qat_hw.h" @@ -228,12 +229,6 @@ static u32 uof_get_ae_mask(u32 obj_num) return adf_4xxx_fw_config[obj_num].ae_mask; } -static u32 get_vf2pf_sources(void __iomem *pmisc_addr) -{ - /* For the moment do not report vf2pf sources */ - return 0; -} - void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &adf_4xxx_class; @@ -278,12 +273,11 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) hw_data->uof_get_ae_mask = uof_get_ae_mask; hw_data->set_msix_rttable = set_msix_default_rttable; hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer; - hw_data->pfvf_ops.enable_comms = adf_pfvf_comms_disabled; - hw_data->pfvf_ops.get_vf2pf_sources = get_vf2pf_sources; hw_data->disable_iov = adf_disable_sriov; hw_data->ring_pair_reset = adf_gen4_ring_pair_reset; adf_gen4_init_hw_csr_ops(&hw_data->csr_ops); + adf_gen4_init_pf_pfvf_ops(&hw_data->pfvf_ops); } void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile index 80f6cb424753..7e191a42a5c7 100644 --- a/drivers/crypto/qat/qat_common/Makefile +++ b/drivers/crypto/qat/qat_common/Makefile @@ -22,4 +22,4 @@ intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_vf_isr.o adf_pfvf_utils.o \ adf_pfvf_pf_msg.o adf_pfvf_pf_proto.o \ adf_pfvf_vf_msg.o adf_pfvf_vf_proto.o \ - adf_gen2_pfvf.o + adf_gen2_pfvf.o adf_gen4_pfvf.o diff --git a/drivers/crypto/qat/qat_common/adf_gen4_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen4_pfvf.c new file mode 100644 index 000000000000..8efbedf63bc8 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_gen4_pfvf.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2021 Intel Corporation */ +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_gen4_pfvf.h" +#include "adf_pfvf_pf_proto.h" +#include "adf_pfvf_utils.h" + +#define ADF_4XXX_MAX_NUM_VFS 16 + +#define ADF_4XXX_PF2VM_OFFSET(i) (0x40B010 + ((i) * 0x20)) +#define ADF_4XXX_VM2PF_OFFSET(i) (0x40B014 + ((i) * 0x20)) + +/* VF2PF interrupt source registers */ +#define ADF_4XXX_VM2PF_SOU(i) (0x41A180 + ((i) * 4)) +#define ADF_4XXX_VM2PF_MSK(i) (0x41A1C0 + ((i) * 4)) +#define ADF_4XXX_VM2PF_INT_EN_MSK BIT(0) + +#define ADF_PFVF_GEN4_MSGTYPE_SHIFT 2 +#define ADF_PFVF_GEN4_MSGTYPE_MASK 0x3F +#define ADF_PFVF_GEN4_MSGDATA_SHIFT 8 +#define ADF_PFVF_GEN4_MSGDATA_MASK 0xFFFFFF + +static const struct pfvf_csr_format csr_gen4_fmt = { + { ADF_PFVF_GEN4_MSGTYPE_SHIFT, ADF_PFVF_GEN4_MSGTYPE_MASK }, + { ADF_PFVF_GEN4_MSGDATA_SHIFT, ADF_PFVF_GEN4_MSGDATA_MASK }, +}; + +static u32 adf_gen4_pf_get_pf2vf_offset(u32 i) +{ + return ADF_4XXX_PF2VM_OFFSET(i); +} + +static u32 adf_gen4_pf_get_vf2pf_offset(u32 i) +{ + return ADF_4XXX_VM2PF_OFFSET(i); +} + +static u32 adf_gen4_get_vf2pf_sources(void __iomem *pmisc_addr) +{ + int i; + u32 sou, mask; + int num_csrs = ADF_4XXX_MAX_NUM_VFS; + u32 vf_mask = 0; + + for (i = 0; i < num_csrs; i++) { + sou = ADF_CSR_RD(pmisc_addr, ADF_4XXX_VM2PF_SOU(i)); + mask = ADF_CSR_RD(pmisc_addr, ADF_4XXX_VM2PF_MSK(i)); + sou &= ~mask; + vf_mask |= sou << i; + } + + return vf_mask; +} + +static void adf_gen4_enable_vf2pf_interrupts(void __iomem *pmisc_addr, + u32 vf_mask) +{ + int num_csrs = ADF_4XXX_MAX_NUM_VFS; + unsigned long mask = vf_mask; + unsigned int val; + int i; + + for_each_set_bit(i, &mask, num_csrs) { + unsigned int offset = ADF_4XXX_VM2PF_MSK(i); + + val = ADF_CSR_RD(pmisc_addr, offset) & ~ADF_4XXX_VM2PF_INT_EN_MSK; + ADF_CSR_WR(pmisc_addr, offset, val); + } +} + +static void adf_gen4_disable_vf2pf_interrupts(void __iomem *pmisc_addr, + u32 vf_mask) +{ + int num_csrs = ADF_4XXX_MAX_NUM_VFS; + unsigned long mask = vf_mask; + unsigned int val; + int i; + + for_each_set_bit(i, &mask, num_csrs) { + unsigned int offset = ADF_4XXX_VM2PF_MSK(i); + + val = ADF_CSR_RD(pmisc_addr, offset) | ADF_4XXX_VM2PF_INT_EN_MSK; + ADF_CSR_WR(pmisc_addr, offset, val); + } +} + +static int adf_gen4_pfvf_send(struct adf_accel_dev *accel_dev, + struct pfvf_message msg, u32 pfvf_offset, + struct mutex *csr_lock) +{ + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + u32 csr_val; + int ret; + + csr_val = adf_pfvf_csr_msg_of(accel_dev, msg, &csr_gen4_fmt); + if (unlikely(!csr_val)) + return -EINVAL; + + mutex_lock(csr_lock); + + ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val | ADF_PFVF_INT); + + /* Wait for confirmation from remote that it received the message */ + ret = read_poll_timeout(ADF_CSR_RD, csr_val, !(csr_val & ADF_PFVF_INT), + ADF_PFVF_MSG_ACK_DELAY_US, + ADF_PFVF_MSG_ACK_MAX_DELAY_US, + true, pmisc_addr, pfvf_offset); + if (ret < 0) + dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n"); + + mutex_unlock(csr_lock); + return ret; +} + +static struct pfvf_message adf_gen4_pfvf_recv(struct adf_accel_dev *accel_dev, + u32 pfvf_offset, u8 compat_ver) +{ + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + u32 csr_val; + + /* Read message from the CSR */ + csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset); + + /* We can now acknowledge the message reception by clearing the + * interrupt bit + */ + ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val & ~ADF_PFVF_INT); + + /* Return the pfvf_message format */ + return adf_pfvf_message_of(accel_dev, csr_val, &csr_gen4_fmt); +} + +void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) +{ + pfvf_ops->enable_comms = adf_enable_pf2vf_comms; + pfvf_ops->get_pf2vf_offset = adf_gen4_pf_get_pf2vf_offset; + pfvf_ops->get_vf2pf_offset = adf_gen4_pf_get_vf2pf_offset; + pfvf_ops->get_vf2pf_sources = adf_gen4_get_vf2pf_sources; + pfvf_ops->enable_vf2pf_interrupts = adf_gen4_enable_vf2pf_interrupts; + pfvf_ops->disable_vf2pf_interrupts = adf_gen4_disable_vf2pf_interrupts; + pfvf_ops->send_msg = adf_gen4_pfvf_send; + pfvf_ops->recv_msg = adf_gen4_pfvf_recv; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_pf_pfvf_ops); diff --git a/drivers/crypto/qat/qat_common/adf_gen4_pfvf.h b/drivers/crypto/qat/qat_common/adf_gen4_pfvf.h new file mode 100644 index 000000000000..17d1b774d4a8 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_gen4_pfvf.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2021 Intel Corporation */ +#ifndef ADF_GEN4_PFVF_H +#define ADF_GEN4_PFVF_H + +#include "adf_accel_devices.h" + +#ifdef CONFIG_PCI_IOV +void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops); +#else +static inline void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) +{ + pfvf_ops->enable_comms = adf_pfvf_comms_disabled; +} +#endif + +#endif /* ADF_GEN4_PFVF_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h index 1d3cad7d4999..f00e9e2c585b 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h @@ -6,7 +6,8 @@ #include /* - * PF<->VF Messaging + * PF<->VF Gen2 Messaging format + * * The PF has an array of 32-bit PF2VF registers, one for each VF. The * PF can access all these registers; each VF can access only the one * register associated with that particular VF. @@ -53,6 +54,32 @@ * respectively, the other 16 bits are written to first with a defined * IN_USE_BY pattern as part of a collision control scheme (see function * adf_gen2_pfvf_send() in adf_pf2vf_msg.c). + * + * + * PF<->VF Gen4 Messaging format + * + * Similarly to the gen2 messaging format, 32-bit long registers are used for + * communication between PF and VFs. However, each VF and PF share a pair of + * 32-bits register to avoid collisions: one for PV to VF messages and one + * for VF to PF messages. + * + * Both the Interrupt bit and the Message Origin bit retain the same position + * and meaning, although non-system messages are now deprecated and not + * expected. + * + * 31 30 9 8 7 6 5 4 3 2 1 0 + * _______________________________________________ + * | | | . . . | | | | | | | | | | | + * +-----------------------------------------------+ + * \_____________________/ \_______________/ ^ ^ + * ^ ^ | | + * | | | PF/VF Int + * | | Message Origin + * | Message Type + * Message-specific Data/Reserved + * + * For both formats, the message reception is acknowledged by lowering the + * interrupt bit on the register where the message was sent. */ /* PFVF message common bits */ From 0bba03ce9739be004d7e7060a1a127418fd0055c Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Thu, 16 Dec 2021 09:13:32 +0000 Subject: [PATCH 122/145] crypto: qat - add PFVF support to enable the reset of ring pairs Extend support for resetting ring pairs on the device to VFs. Such reset happens by sending a request to the PF over the PFVF protocol. This patch defines two new PFVF messages and adds the PFVF logic for handling the request on PF, triggering the reset, and VFs, accepting the 'success'/'error' response. This feature is GEN4 specific. This patch is based on earlier work done by Zelin Deng. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_pfvf_msg.h | 14 +++++ .../crypto/qat/qat_common/adf_pfvf_pf_proto.c | 52 +++++++++++++++++++ .../crypto/qat/qat_common/adf_pfvf_vf_proto.c | 1 + 3 files changed, 67 insertions(+) diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h index f00e9e2c585b..86b0e7baa4d3 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h @@ -99,6 +99,8 @@ enum pf2vf_msgtype { ADF_PF2VF_MSGTYPE_RESTARTING = 0x01, ADF_PF2VF_MSGTYPE_VERSION_RESP = 0x02, ADF_PF2VF_MSGTYPE_BLKMSG_RESP = 0x03, +/* Values from 0x10 are Gen4 specific, message type is only 4 bits in Gen2 devices. */ + ADF_PF2VF_MSGTYPE_RP_RESET_RESP = 0x10, }; /* VF->PF messages */ @@ -110,6 +112,8 @@ enum vf2pf_msgtype { ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ = 0x07, ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ = 0x08, ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ = 0x09, +/* Values from 0x10 are Gen4 specific, message type is only 4 bits in Gen2 devices. */ + ADF_VF2PF_MSGTYPE_RP_RESET = 0x10, }; /* VF/PF compatibility version. */ @@ -134,6 +138,16 @@ enum pf2vf_compat_response { ADF_PF2VF_VF_COMPAT_UNKNOWN = 0x03, }; +enum ring_reset_result { + RPRESET_SUCCESS = 0x01, + RPRESET_NOT_SUPPORTED = 0x02, + RPRESET_INVAL_BANK = 0x03, + RPRESET_TIMEOUT = 0x04, +}; + +#define ADF_VF2PF_RNG_RESET_RP_MASK GENMASK(1, 0) +#define ADF_VF2PF_RNG_RESET_RSVD_MASK GENMASK(25, 2) + /* PF->VF Block Responses */ #define ADF_PF2VF_BLKMSG_RESP_TYPE_MASK GENMASK(1, 0) #define ADF_PF2VF_BLKMSG_RESP_DATA_MASK GENMASK(9, 2) diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c index 84230aac67e6..588352de1ef0 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c @@ -178,6 +178,55 @@ static struct pfvf_message handle_blkmsg_req(struct adf_accel_vf_info *vf_info, return resp; } +static struct pfvf_message handle_rp_reset_req(struct adf_accel_dev *accel_dev, u8 vf_nr, + struct pfvf_message req) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct pfvf_message resp = { + .type = ADF_PF2VF_MSGTYPE_RP_RESET_RESP, + .data = RPRESET_SUCCESS + }; + u32 bank_number; + u32 rsvd_field; + + bank_number = FIELD_GET(ADF_VF2PF_RNG_RESET_RP_MASK, req.data); + rsvd_field = FIELD_GET(ADF_VF2PF_RNG_RESET_RSVD_MASK, req.data); + + dev_dbg(&GET_DEV(accel_dev), + "Ring Pair Reset Message received from VF%d for bank 0x%x\n", + vf_nr, bank_number); + + if (!hw_data->ring_pair_reset || rsvd_field) { + dev_dbg(&GET_DEV(accel_dev), + "Ring Pair Reset for VF%d is not supported\n", vf_nr); + resp.data = RPRESET_NOT_SUPPORTED; + goto out; + } + + if (bank_number >= hw_data->num_banks_per_vf) { + dev_err(&GET_DEV(accel_dev), + "Invalid bank number (0x%x) from VF%d for Ring Reset\n", + bank_number, vf_nr); + resp.data = RPRESET_INVAL_BANK; + goto out; + } + + /* Convert the VF provided value to PF bank number */ + bank_number = vf_nr * hw_data->num_banks_per_vf + bank_number; + if (hw_data->ring_pair_reset(accel_dev, bank_number)) { + dev_dbg(&GET_DEV(accel_dev), + "Ring pair reset for VF%d failure\n", vf_nr); + resp.data = RPRESET_TIMEOUT; + goto out; + } + + dev_dbg(&GET_DEV(accel_dev), + "Ring pair reset for VF%d successfully\n", vf_nr); + +out: + return resp; +} + static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, struct pfvf_message msg, struct pfvf_message *resp) { @@ -245,6 +294,9 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, case ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ: *resp = handle_blkmsg_req(vf_info, msg); break; + case ADF_VF2PF_MSGTYPE_RP_RESET: + *resp = handle_rp_reset_req(accel_dev, vf_nr, msg); + break; default: dev_dbg(&GET_DEV(accel_dev), "Unknown message from VF%d (type 0x%.4x, data: 0x%.4x)\n", diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c index 0e4b8397cbe3..1015155b6374 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c +++ b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c @@ -310,6 +310,7 @@ static bool adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev, return false; case ADF_PF2VF_MSGTYPE_VERSION_RESP: case ADF_PF2VF_MSGTYPE_BLKMSG_RESP: + case ADF_PF2VF_MSGTYPE_RP_RESET_RESP: dev_dbg(&GET_DEV(accel_dev), "Response Message received from PF (type 0x%.4x, data 0x%.4x)\n", msg.type, msg.data); From beb1e6d71f0ef906ff986710fdd4ad4fc1542302 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Thu, 16 Dec 2021 09:13:33 +0000 Subject: [PATCH 123/145] crypto: qat - allow detection of dc capabilities for 4xxx Add logic to allow the detection of data compression capabilities for 4xxx devices. The capability detection logic has been refactored to separate the crypto capabilities from the compression ones. This patch is not updating the returned capability mask as, up to now, 4xxx devices are configured only to handle crypto operations. Signed-off-by: Giovanni Cabiddu Signed-off-by: Marco Chiappero Reviewed-by: Fiona Trahe Reviewed-by: Marco Chiappero Signed-off-by: Herbert Xu --- .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 64 +++++++++++-------- drivers/crypto/qat/qat_common/icp_qat_hw.h | 6 +- 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c index ef71aa4efd64..40f684103b29 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -96,46 +96,60 @@ static void set_msix_default_rttable(struct adf_accel_dev *accel_dev) static u32 get_accel_cap(struct adf_accel_dev *accel_dev) { struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 capabilities_cy, capabilities_dc; u32 fusectl1; - u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | - ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | - ICP_ACCEL_CAPABILITIES_CIPHER | - ICP_ACCEL_CAPABILITIES_AUTHENTICATION | - ICP_ACCEL_CAPABILITIES_SHA3 | - ICP_ACCEL_CAPABILITIES_SHA3_EXT | - ICP_ACCEL_CAPABILITIES_HKDF | - ICP_ACCEL_CAPABILITIES_ECEDMONT | - ICP_ACCEL_CAPABILITIES_CHACHA_POLY | - ICP_ACCEL_CAPABILITIES_AESGCM_SPC | - ICP_ACCEL_CAPABILITIES_AES_V2; /* Read accelerator capabilities mask */ pci_read_config_dword(pdev, ADF_4XXX_FUSECTL1_OFFSET, &fusectl1); + capabilities_cy = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_CIPHER | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION | + ICP_ACCEL_CAPABILITIES_SHA3 | + ICP_ACCEL_CAPABILITIES_SHA3_EXT | + ICP_ACCEL_CAPABILITIES_HKDF | + ICP_ACCEL_CAPABILITIES_ECEDMONT | + ICP_ACCEL_CAPABILITIES_CHACHA_POLY | + ICP_ACCEL_CAPABILITIES_AESGCM_SPC | + ICP_ACCEL_CAPABILITIES_AES_V2; + /* A set bit in fusectl1 means the feature is OFF in this SKU */ if (fusectl1 & ICP_ACCEL_4XXX_MASK_CIPHER_SLICE) { - capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; - capabilities &= ~ICP_ACCEL_CAPABILITIES_HKDF; - capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_HKDF; + capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CIPHER; } if (fusectl1 & ICP_ACCEL_4XXX_MASK_UCS_SLICE) { - capabilities &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY; - capabilities &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC; - capabilities &= ~ICP_ACCEL_CAPABILITIES_AES_V2; - capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY; + capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC; + capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_AES_V2; + capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CIPHER; } if (fusectl1 & ICP_ACCEL_4XXX_MASK_AUTH_SLICE) { - capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; - capabilities &= ~ICP_ACCEL_CAPABILITIES_SHA3; - capabilities &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT; - capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_SHA3; + capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT; + capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CIPHER; } if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE) { - capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; - capabilities &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT; + capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT; } - return capabilities; + capabilities_dc = ICP_ACCEL_CAPABILITIES_COMPRESSION | + ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION | + ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION | + ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; + + if (fusectl1 & ICP_ACCEL_4XXX_MASK_COMPRESS_SLICE) { + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; + } + + return capabilities_cy; } static enum dev_sku_info get_sku(struct adf_hw_device_data *self) diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw.h b/drivers/crypto/qat/qat_common/icp_qat_hw.h index 5770b2b2c09e..433304cad2ed 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hw.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hw.h @@ -98,7 +98,11 @@ enum icp_qat_capabilities_mask { ICP_ACCEL_CAPABILITIES_SHA3_EXT = BIT(15), ICP_ACCEL_CAPABILITIES_AESGCM_SPC = BIT(16), ICP_ACCEL_CAPABILITIES_CHACHA_POLY = BIT(17), - /* Bits 18-25 are currently reserved */ + /* Bits 18-21 are currently reserved */ + ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY = BIT(22), + ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64 = BIT(23), + ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION = BIT(24), + ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION = BIT(25), ICP_ACCEL_CAPABILITIES_AES_V2 = BIT(26) }; From 0cec19c761e5cc8d6fa43df622791e941b7a8033 Mon Sep 17 00:00:00 2001 From: Tomasz Kowalik Date: Thu, 16 Dec 2021 09:13:34 +0000 Subject: [PATCH 124/145] crypto: qat - add support for compression for 4xxx Add the logic required to enable the compression service for 4xxx devices. This allows to load the compression firmware image and report the appropriate compression capabilities. The firmware image selection for a given device is based on the 'ServicesEnabled' key stored in the internal configuration, which is added statically at the probe of the device according to the following rule, by default: - odd numbered devices assigned to compression services - even numbered devices assigned to crypto services In addition, restore the 'ServicesEnabled' key, if present, when SRIOV is enabled on the device. Signed-off-by: Tomasz Kowalik Co-developed-by: Mateuszx Potrola Signed-off-by: Mateuszx Potrola Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Marco Chiappero Reviewed-by: Fiona Trahe Reviewed-by: Marco Chiappero Signed-off-by: Herbert Xu --- .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 79 +++++++++++++++++-- .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.h | 1 + drivers/crypto/qat/qat_4xxx/adf_drv.c | 33 ++++++++ .../crypto/qat/qat_common/adf_accel_devices.h | 4 +- .../crypto/qat/qat_common/adf_accel_engine.c | 8 +- drivers/crypto/qat/qat_common/adf_cfg.c | 1 + .../crypto/qat/qat_common/adf_cfg_strings.h | 3 + drivers/crypto/qat/qat_common/adf_sriov.c | 31 +++++++- 8 files changed, 147 insertions(+), 13 deletions(-) diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c index 40f684103b29..6d10edc40aca 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -2,6 +2,7 @@ /* Copyright(c) 2020 - 2021 Intel Corporation */ #include #include +#include #include #include #include @@ -13,12 +14,18 @@ struct adf_fw_config { char *obj_name; }; -static struct adf_fw_config adf_4xxx_fw_config[] = { +static struct adf_fw_config adf_4xxx_fw_cy_config[] = { {0xF0, ADF_4XXX_SYM_OBJ}, {0xF, ADF_4XXX_ASYM_OBJ}, {0x100, ADF_4XXX_ADMIN_OBJ}, }; +static struct adf_fw_config adf_4xxx_fw_dc_config[] = { + {0xF0, ADF_4XXX_DC_OBJ}, + {0xF, ADF_4XXX_DC_OBJ}, + {0x100, ADF_4XXX_ADMIN_OBJ}, +}; + /* Worker thread to service arbiter mappings */ static const u32 thrd_to_arb_map[ADF_4XXX_MAX_ACCELENGINES] = { 0x5555555, 0x5555555, 0x5555555, 0x5555555, @@ -32,6 +39,39 @@ static struct adf_hw_device_class adf_4xxx_class = { .instances = 0, }; +enum dev_services { + SVC_CY = 0, + SVC_DC, +}; + +static const char *const dev_cfg_services[] = { + [SVC_CY] = ADF_CFG_CY, + [SVC_DC] = ADF_CFG_DC, +}; + +static int get_service_enabled(struct adf_accel_dev *accel_dev) +{ + char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; + u32 ret; + + ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, + ADF_SERVICES_ENABLED, services); + if (ret) { + dev_err(&GET_DEV(accel_dev), + ADF_SERVICES_ENABLED " param not found\n"); + return ret; + } + + ret = match_string(dev_cfg_services, ARRAY_SIZE(dev_cfg_services), + services); + if (ret < 0) + dev_err(&GET_DEV(accel_dev), + "Invalid value of " ADF_SERVICES_ENABLED " param: %s\n", + services); + + return ret; +} + static u32 get_accel_mask(struct adf_hw_device_data *self) { return ADF_4XXX_ACCELERATORS_MASK; @@ -149,7 +189,14 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; } - return capabilities_cy; + switch (get_service_enabled(accel_dev)) { + case SVC_CY: + return capabilities_cy; + case SVC_DC: + return capabilities_dc; + } + + return 0; } static enum dev_sku_info get_sku(struct adf_hw_device_data *self) @@ -230,17 +277,35 @@ static int adf_init_device(struct adf_accel_dev *accel_dev) static u32 uof_get_num_objs(void) { - return ARRAY_SIZE(adf_4xxx_fw_config); + BUILD_BUG_ON_MSG(ARRAY_SIZE(adf_4xxx_fw_cy_config) != + ARRAY_SIZE(adf_4xxx_fw_dc_config), + "Size mismatch between adf_4xxx_fw_*_config arrays"); + + return ARRAY_SIZE(adf_4xxx_fw_cy_config); } -static char *uof_get_name(u32 obj_num) +static char *uof_get_name(struct adf_accel_dev *accel_dev, u32 obj_num) { - return adf_4xxx_fw_config[obj_num].obj_name; + switch (get_service_enabled(accel_dev)) { + case SVC_CY: + return adf_4xxx_fw_cy_config[obj_num].obj_name; + case SVC_DC: + return adf_4xxx_fw_dc_config[obj_num].obj_name; + } + + return NULL; } -static u32 uof_get_ae_mask(u32 obj_num) +static u32 uof_get_ae_mask(struct adf_accel_dev *accel_dev, u32 obj_num) { - return adf_4xxx_fw_config[obj_num].ae_mask; + switch (get_service_enabled(accel_dev)) { + case SVC_CY: + return adf_4xxx_fw_cy_config[obj_num].ae_mask; + case SVC_DC: + return adf_4xxx_fw_dc_config[obj_num].ae_mask; + } + + return 0; } void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h index a0c67752317f..12e4fb9b40ce 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h @@ -77,6 +77,7 @@ #define ADF_4XXX_FW "qat_4xxx.bin" #define ADF_4XXX_MMP "qat_4xxx_mmp.bin" #define ADF_4XXX_SYM_OBJ "qat_4xxx_sym.bin" +#define ADF_4XXX_DC_OBJ "qat_4xxx_dc.bin" #define ADF_4XXX_ASYM_OBJ "qat_4xxx_asym.bin" #define ADF_4XXX_ADMIN_OBJ "qat_4xxx_admin.bin" diff --git a/drivers/crypto/qat/qat_4xxx/adf_drv.c b/drivers/crypto/qat/qat_4xxx/adf_drv.c index 71ef065914b2..a6c78b9c730b 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_drv.c +++ b/drivers/crypto/qat/qat_4xxx/adf_drv.c @@ -29,6 +29,29 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) adf_devmgr_rm_dev(accel_dev, NULL); } +static int adf_cfg_dev_init(struct adf_accel_dev *accel_dev) +{ + const char *config; + int ret; + + config = accel_dev->accel_id % 2 ? ADF_CFG_DC : ADF_CFG_CY; + + ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC); + if (ret) + return ret; + + /* Default configuration is crypto only for even devices + * and compression for odd devices + */ + ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, + ADF_SERVICES_ENABLED, config, + ADF_STR); + if (ret) + return ret; + + return 0; +} + static int adf_crypto_dev_config(struct adf_accel_dev *accel_dev) { char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; @@ -227,8 +250,18 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err; } + ret = adf_cfg_dev_init(accel_dev); + if (ret) { + dev_err(&pdev->dev, "Failed to initialize configuration.\n"); + goto out_err; + } + /* Get accelerator capabilities mask */ hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); + if (!hw_data->accel_capabilities_mask) { + dev_err(&pdev->dev, "Failed to get capabilities mask.\n"); + goto out_err; + } /* Find and map all the device's BARS */ bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_4XXX_BAR_MASK; diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 1b9d4ed03dd0..2d4cd7c7cf33 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -192,9 +192,9 @@ struct adf_hw_device_data { int (*ring_pair_reset)(struct adf_accel_dev *accel_dev, u32 bank_nr); void (*reset_device)(struct adf_accel_dev *accel_dev); void (*set_msix_rttable)(struct adf_accel_dev *accel_dev); - char *(*uof_get_name)(u32 obj_num); + char *(*uof_get_name)(struct adf_accel_dev *accel_dev, u32 obj_num); u32 (*uof_get_num_objs)(void); - u32 (*uof_get_ae_mask)(u32 obj_num); + u32 (*uof_get_ae_mask)(struct adf_accel_dev *accel_dev, u32 obj_num); struct adf_pfvf_ops pfvf_ops; struct adf_hw_csr_ops csr_ops; const char *fw_name; diff --git a/drivers/crypto/qat/qat_common/adf_accel_engine.c b/drivers/crypto/qat/qat_common/adf_accel_engine.c index ca4eae8cdd0b..4ce2b666929e 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_engine.c +++ b/drivers/crypto/qat/qat_common/adf_accel_engine.c @@ -22,8 +22,12 @@ static int adf_ae_fw_load_images(struct adf_accel_dev *accel_dev, void *fw_addr, num_objs = hw_device->uof_get_num_objs(); for (i = 0; i < num_objs; i++) { - obj_name = hw_device->uof_get_name(i); - ae_mask = hw_device->uof_get_ae_mask(i); + obj_name = hw_device->uof_get_name(accel_dev, i); + ae_mask = hw_device->uof_get_ae_mask(accel_dev, i); + if (!obj_name || !ae_mask) { + dev_err(&GET_DEV(accel_dev), "Invalid UOF image\n"); + goto out_err; + } if (qat_uclo_set_cfg_ae_mask(loader, ae_mask)) { dev_err(&GET_DEV(accel_dev), diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c index 575b6f002303..b5b208cbe5a1 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/qat/qat_common/adf_cfg.c @@ -297,3 +297,4 @@ int adf_cfg_get_param_value(struct adf_accel_dev *accel_dev, up_read(&cfg->lock); return ret; } +EXPORT_SYMBOL_GPL(adf_cfg_get_param_value); diff --git a/drivers/crypto/qat/qat_common/adf_cfg_strings.h b/drivers/crypto/qat/qat_common/adf_cfg_strings.h index 09651e1f937a..655248dbf962 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg_strings.h +++ b/drivers/crypto/qat/qat_common/adf_cfg_strings.h @@ -22,6 +22,9 @@ #define ADF_RING_ASYM_BANK_NUM "BankAsymNumber" #define ADF_CY "Cy" #define ADF_DC "Dc" +#define ADF_CFG_DC "dc" +#define ADF_CFG_CY "sym;asym" +#define ADF_SERVICES_ENABLED "ServicesEnabled" #define ADF_ETRMGR_COALESCING_ENABLED "InterruptCoalescingEnabled" #define ADF_ETRMGR_COALESCING_ENABLED_FORMAT \ ADF_ETRMGR_BANK "%d" ADF_ETRMGR_COALESCING_ENABLED diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index 971a05d62418..b960bca1f9d2 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -126,6 +126,32 @@ void adf_disable_sriov(struct adf_accel_dev *accel_dev) } EXPORT_SYMBOL_GPL(adf_disable_sriov); +static int adf_sriov_prepare_restart(struct adf_accel_dev *accel_dev) +{ + char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; + int ret; + + ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, + ADF_SERVICES_ENABLED, services); + + adf_dev_stop(accel_dev); + adf_dev_shutdown(accel_dev); + + if (!ret) { + ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC); + if (ret) + return ret; + + ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, + ADF_SERVICES_ENABLED, + services, ADF_STR); + if (ret) + return ret; + } + + return 0; +} + /** * adf_sriov_configure() - Enable SRIOV for the device * @pdev: Pointer to PCI device. @@ -165,8 +191,9 @@ int adf_sriov_configure(struct pci_dev *pdev, int numvfs) return -EBUSY; } - adf_dev_stop(accel_dev); - adf_dev_shutdown(accel_dev); + ret = adf_sriov_prepare_restart(accel_dev); + if (ret) + return ret; } if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC)) From 3438e7220b3183b1ee80227e3a0d742b18df4623 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 17 Dec 2021 10:12:32 +0300 Subject: [PATCH 125/145] crypto: octeontx2 - out of bounds access in otx2_cpt_dl_custom_egrp_delete() If "egrp" is negative then it is causes an out of bounds access in eng_grps->grp[]. Fixes: d9d7749773e8 ("crypto: octeontx2 - add apis for custom engine groups") Signed-off-by: Dan Carpenter Signed-off-by: Herbert Xu --- drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c index 9aef37972246..27f6d0a0cd3c 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c @@ -1685,7 +1685,7 @@ int otx2_cpt_dl_custom_egrp_delete(struct otx2_cptpf_dev *cptpf, if (kstrtoint(tmp, 10, &egrp)) goto err_print; - if (egrp >= OTX2_CPT_MAX_ENGINE_GROUPS) { + if (egrp < 0 || egrp >= OTX2_CPT_MAX_ENGINE_GROUPS) { dev_err(dev, "Invalid engine group %d", egrp); return -EINVAL; } From 10371b6212bb682f13247733d6b76b91b2b80f9a Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 17 Dec 2021 10:13:32 +0300 Subject: [PATCH 126/145] crypto: octeontx2 - prevent underflow in get_cores_bmap() If we're going to cap "eng_grp->g->engs_num" upper bounds then we should cap the lower bounds as well. Fixes: 43ac0b824f1c ("crypto: octeontx2 - load microcode and create engine groups") Signed-off-by: Dan Carpenter Signed-off-by: Herbert Xu --- drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c index 27f6d0a0cd3c..4c8ebdf671ca 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c @@ -29,7 +29,8 @@ static struct otx2_cpt_bitmap get_cores_bmap(struct device *dev, bool found = false; int i; - if (eng_grp->g->engs_num > OTX2_CPT_MAX_ENGINES) { + if (eng_grp->g->engs_num < 0 || + eng_grp->g->engs_num > OTX2_CPT_MAX_ENGINES) { dev_err(dev, "unsupported number of engines %d on octeontx2\n", eng_grp->g->engs_num); return bmap; From ace7660691f8a23f1a72e065babd176aed3605b9 Mon Sep 17 00:00:00 2001 From: Pankaj Gupta Date: Fri, 17 Dec 2021 13:42:33 +0530 Subject: [PATCH 127/145] MAINTAINERS: update caam crypto driver maintainers list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adding Gaurav as caam maintainer. Signed-off-by: Pankaj Gupta Reviewed-by: Horia Geantă Signed-off-by: Herbert Xu --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 00475646e3e3..5538f96329fb 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7511,6 +7511,7 @@ F: include/video/ FREESCALE CAAM (Cryptographic Acceleration and Assurance Module) DRIVER M: Horia Geantă M: Pankaj Gupta +M: Gaurav Jain L: linux-crypto@vger.kernel.org S: Maintained F: Documentation/devicetree/bindings/crypto/fsl-sec4.txt From c2aec59be093bd44627bc4f6bc67e4614a93a7b6 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Fri, 17 Dec 2021 10:03:30 +0100 Subject: [PATCH 128/145] crypto: omap-aes - Fix broken pm_runtime_and_get() usage This fix is basically the same as 3d6b661330a7 ("crypto: stm32 - Revert broken pm_runtime_resume_and_get changes"), just for the omap driver. If the return value isn't used, then pm_runtime_get_sync() has to be used for ensuring that the usage count is balanced. Fixes: 1f34cc4a8da3 ("crypto: omap-aes - Fix PM reference leak on omap-aes.c") Cc: stable@vger.kernel.org Signed-off-by: Heiner Kallweit Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 9b968ac4ee7b..a196bb8b1701 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -1302,7 +1302,7 @@ static int omap_aes_suspend(struct device *dev) static int omap_aes_resume(struct device *dev) { - pm_runtime_resume_and_get(dev); + pm_runtime_get_sync(dev); return 0; } #endif From 4cee0700cf1d23b36f2c85507ff83b466a0e63a7 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 18 Dec 2021 18:17:20 +0800 Subject: [PATCH 129/145] crypto: hisilicon/qm - disable qm clock-gating For Kunpeng930, if qm clock-gating is enabled, rate limiter will be inaccurate. Therefore, disable clock-gating before doing task. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index b1fe9c7b8cc8..b731cb4ec294 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -126,6 +126,8 @@ #define QM_CQC_VFT 0x1 #define QM_VFT_CFG 0x100060 #define QM_VFT_CFG_OP_ENABLE 0x100054 +#define QM_PM_CTRL 0x100148 +#define QM_IDLE_DISABLE BIT(9) #define QM_VFT_CFG_DATA_L 0x100064 #define QM_VFT_CFG_DATA_H 0x100068 @@ -800,6 +802,19 @@ static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) qm->ops->qm_db(qm, qn, cmd, index, priority); } +static void qm_disable_clock_gate(struct hisi_qm *qm) +{ + u32 val; + + /* if qm enables clock gating in Kunpeng930, qos will be inaccurate. */ + if (qm->ver < QM_HW_V3) + return; + + val = readl(qm->io_base + QM_PM_CTRL); + val |= QM_IDLE_DISABLE; + writel(val, qm->io_base + QM_PM_CTRL); +} + static int qm_dev_mem_reset(struct hisi_qm *qm) { u32 val; @@ -5935,6 +5950,7 @@ int hisi_qm_init(struct hisi_qm *qm) } if (qm->fun_type == QM_HW_PF) { + qm_disable_clock_gate(qm); ret = qm_dev_mem_reset(qm); if (ret) { dev_err(dev, "failed to reset device memory\n"); @@ -6099,6 +6115,7 @@ static int qm_rebuild_for_resume(struct hisi_qm *qm) qm_cmd_init(qm); hisi_qm_dev_err_init(qm); + qm_disable_clock_gate(qm); ret = qm_dev_mem_reset(qm); if (ret) pci_err(pdev, "failed to reset device memory\n"); From d7779e22e89a78aa2d3f80584a8d1672ac39c3cf Mon Sep 17 00:00:00 2001 From: Lad Prabhakar Date: Sat, 18 Dec 2021 15:06:25 +0000 Subject: [PATCH 130/145] crypto: ux500 - Use platform_get_irq() to get the interrupt platform_get_resource(pdev, IORESOURCE_IRQ, ..) relies on static allocation of IRQ resources in DT core code, this causes an issue when using hierarchical interrupt domains using "interrupts" property in the node as this bypasses the hierarchical setup and messes up the irq chaining. In preparation for removal of static setup of IRQ resource from DT core code use platform_get_irq() so that interrupt mapping is created on demand. While at it also store the IRQ number in struct cryp_device_data so that we don't have to call platform_get_irq() frequently. Signed-off-by: Lad Prabhakar Signed-off-by: Herbert Xu --- drivers/crypto/ux500/cryp/cryp.h | 2 ++ drivers/crypto/ux500/cryp/cryp_core.c | 26 +++++++------------------- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/drivers/crypto/ux500/cryp/cryp.h b/drivers/crypto/ux500/cryp/cryp.h index db5713d7c940..59e1557a620a 100644 --- a/drivers/crypto/ux500/cryp/cryp.h +++ b/drivers/crypto/ux500/cryp/cryp.h @@ -224,6 +224,7 @@ struct cryp_dma { * @phybase: Pointer to physical memory location of the cryp device. * @dev: Pointer to the devices dev structure. * @clk: Pointer to the device's clock control. + * @irq: IRQ number * @pwr_regulator: Pointer to the device's power control. * @power_status: Current status of the power. * @ctx_lock: Lock for current_ctx. @@ -239,6 +240,7 @@ struct cryp_device_data { phys_addr_t phybase; struct device *dev; struct clk *clk; + int irq; struct regulator *pwr_regulator; int power_status; spinlock_t ctx_lock; diff --git a/drivers/crypto/ux500/cryp/cryp_core.c b/drivers/crypto/ux500/cryp/cryp_core.c index 30cdd5253929..97277b7150cb 100644 --- a/drivers/crypto/ux500/cryp/cryp_core.c +++ b/drivers/crypto/ux500/cryp/cryp_core.c @@ -1257,7 +1257,6 @@ static int ux500_cryp_probe(struct platform_device *pdev) { int ret; struct resource *res; - struct resource *res_irq; struct cryp_device_data *device_data; struct cryp_protection_config prot = { .privilege_access = CRYP_STATE_ENABLE @@ -1341,15 +1340,13 @@ static int ux500_cryp_probe(struct platform_device *pdev) goto out_power; } - res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (!res_irq) { - dev_err(dev, "[%s]: IORESOURCE_IRQ unavailable", - __func__); - ret = -ENODEV; + device_data->irq = platform_get_irq(pdev, 0); + if (device_data->irq <= 0) { + ret = device_data->irq ? device_data->irq : -ENXIO; goto out_power; } - ret = devm_request_irq(&pdev->dev, res_irq->start, + ret = devm_request_irq(&pdev->dev, device_data->irq, cryp_interrupt_handler, 0, "cryp1", device_data); if (ret) { dev_err(dev, "[%s]: Unable to request IRQ", __func__); @@ -1489,7 +1486,6 @@ static int ux500_cryp_suspend(struct device *dev) int ret; struct platform_device *pdev = to_platform_device(dev); struct cryp_device_data *device_data; - struct resource *res_irq; struct cryp_ctx *temp_ctx = NULL; dev_dbg(dev, "[%s]", __func__); @@ -1501,11 +1497,7 @@ static int ux500_cryp_suspend(struct device *dev) return -ENOMEM; } - res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (!res_irq) - dev_err(dev, "[%s]: IORESOURCE_IRQ, unavailable", __func__); - else - disable_irq(res_irq->start); + disable_irq(device_data->irq); spin_lock(&device_data->ctx_lock); if (!device_data->current_ctx) @@ -1532,7 +1524,6 @@ static int ux500_cryp_resume(struct device *dev) int ret = 0; struct platform_device *pdev = to_platform_device(dev); struct cryp_device_data *device_data; - struct resource *res_irq; struct cryp_ctx *temp_ctx = NULL; dev_dbg(dev, "[%s]", __func__); @@ -1556,11 +1547,8 @@ static int ux500_cryp_resume(struct device *dev) if (ret) dev_err(dev, "[%s]: cryp_enable_power() failed!", __func__); - else { - res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (res_irq) - enable_irq(res_irq->start); - } + else + enable_irq(device_data->irq); return ret; } From 25d04a382ebb409f7512dd668d81997bf5494df1 Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Mon, 20 Dec 2021 10:14:18 +0800 Subject: [PATCH 131/145] MAINTAINERS: update SEC2 driver maintainers list Adding Kai Ye as SEC2 maintainer. Signed-off-by: Kai Ye Signed-off-by: Zaibo Xu Signed-off-by: Herbert Xu --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 5538f96329fb..4fb3a8843f29 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8629,6 +8629,7 @@ F: drivers/scsi/hisi_sas/ HISILICON SECURITY ENGINE V2 DRIVER (SEC2) M: Zaibo Xu +M: Kai Ye L: linux-crypto@vger.kernel.org S: Maintained F: Documentation/ABI/testing/debugfs-hisi-sec From 908dffaf88a248e542bdae3ca174f27b8f4ccf37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20M=C3=BCller?= Date: Mon, 20 Dec 2021 07:21:53 +0100 Subject: [PATCH 132/145] crypto: jitter - add oversampling of noise source The output n bits can receive more than n bits of min entropy, of course, but the fixed output of the conditioning function can only asymptotically approach the output size bits of min entropy, not attain that bound. Random maps will tend to have output collisions, which reduces the creditable output entropy (that is what SP 800-90B Section 3.1.5.1.2 attempts to bound). The value "64" is justified in Appendix A.4 of the current 90C draft, and aligns with NIST's in "epsilon" definition in this document, which is that a string can be considered "full entropy" if you can bound the min entropy in each bit of output to at least 1-epsilon, where epsilon is required to be <= 2^(-32). Note, this patch causes the Jitter RNG to cut its performance in half in FIPS mode because the conditioning function of the LFSR produces 64 bits of entropy in one block. The oversampling requires that additionally 64 bits of entropy are sampled from the noise source. If the conditioner is changed, such as using SHA-256, the impact of the oversampling is only one fourth, because for the 256 bit block of the conditioner, only 64 additional bits from the noise source must be sampled. This patch is derived from the user space jitterentropy-library. Signed-off-by: Stephan Mueller Reviewed-by: Simo Sorce Signed-off-by: Herbert Xu --- crypto/jitterentropy.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c index 8f5283f28ed3..93bff3213823 100644 --- a/crypto/jitterentropy.c +++ b/crypto/jitterentropy.c @@ -117,6 +117,22 @@ struct rand_data { #define JENT_EHEALTH 9 /* Health test failed during initialization */ #define JENT_ERCT 10 /* RCT failed during initialization */ +/* + * The output n bits can receive more than n bits of min entropy, of course, + * but the fixed output of the conditioning function can only asymptotically + * approach the output size bits of min entropy, not attain that bound. Random + * maps will tend to have output collisions, which reduces the creditable + * output entropy (that is what SP 800-90B Section 3.1.5.1.2 attempts to bound). + * + * The value "64" is justified in Appendix A.4 of the current 90C draft, + * and aligns with NIST's in "epsilon" definition in this document, which is + * that a string can be considered "full entropy" if you can bound the min + * entropy in each bit of output to at least 1-epsilon, where epsilon is + * required to be <= 2^(-32). + */ +#define JENT_ENTROPY_SAFETY_FACTOR 64 + +#include #include "jitterentropy.h" /*************************************************************************** @@ -542,7 +558,10 @@ static int jent_measure_jitter(struct rand_data *ec) */ static void jent_gen_entropy(struct rand_data *ec) { - unsigned int k = 0; + unsigned int k = 0, safety_factor = 0; + + if (fips_enabled) + safety_factor = JENT_ENTROPY_SAFETY_FACTOR; /* priming of the ->prev_time value */ jent_measure_jitter(ec); @@ -556,7 +575,7 @@ static void jent_gen_entropy(struct rand_data *ec) * We multiply the loop value with ->osr to obtain the * oversampling rate requested by the caller */ - if (++k >= (DATA_SIZE_BITS * ec->osr)) + if (++k >= ((DATA_SIZE_BITS + safety_factor) * ec->osr)) break; } } From 96ede30f4b172ab6c83789f6bb420b3c3c029653 Mon Sep 17 00:00:00 2001 From: Tianjia Zhang Date: Mon, 20 Dec 2021 17:23:14 +0800 Subject: [PATCH 133/145] crypto: sha256 - remove duplicate generic hash init function crypto_sha256_init() and sha256_base_init() are the same repeated implementations, remove the crypto_sha256_init() in generic implementation, sha224 is the same process. Signed-off-by: Tianjia Zhang Signed-off-by: Herbert Xu --- crypto/sha256_generic.c | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/crypto/sha256_generic.c b/crypto/sha256_generic.c index 3b377197236e..bf147b01e313 100644 --- a/crypto/sha256_generic.c +++ b/crypto/sha256_generic.c @@ -33,18 +33,6 @@ const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE] = { }; EXPORT_SYMBOL_GPL(sha256_zero_message_hash); -static int crypto_sha256_init(struct shash_desc *desc) -{ - sha256_init(shash_desc_ctx(desc)); - return 0; -} - -static int crypto_sha224_init(struct shash_desc *desc) -{ - sha224_init(shash_desc_ctx(desc)); - return 0; -} - int crypto_sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len) { @@ -72,7 +60,7 @@ EXPORT_SYMBOL(crypto_sha256_finup); static struct shash_alg sha256_algs[2] = { { .digestsize = SHA256_DIGEST_SIZE, - .init = crypto_sha256_init, + .init = sha256_base_init, .update = crypto_sha256_update, .final = crypto_sha256_final, .finup = crypto_sha256_finup, @@ -86,7 +74,7 @@ static struct shash_alg sha256_algs[2] = { { } }, { .digestsize = SHA224_DIGEST_SIZE, - .init = crypto_sha224_init, + .init = sha224_base_init, .update = crypto_sha256_update, .final = crypto_sha256_final, .finup = crypto_sha256_finup, From 63bdbfc146aec8a414f07af8208f2488ffeacf32 Mon Sep 17 00:00:00 2001 From: Tianjia Zhang Date: Mon, 20 Dec 2021 17:23:15 +0800 Subject: [PATCH 134/145] crypto: mips/sha - remove duplicate hash init function sha*_base_init() series functions has implemented the initialization of the hash context, this commit use sha*_base_init() function to replace repeated implementations. Signed-off-by: Tianjia Zhang Signed-off-by: Herbert Xu --- arch/mips/cavium-octeon/crypto/octeon-sha1.c | 17 +------- .../mips/cavium-octeon/crypto/octeon-sha256.c | 39 ++----------------- .../mips/cavium-octeon/crypto/octeon-sha512.c | 39 ++----------------- 3 files changed, 8 insertions(+), 87 deletions(-) diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha1.c b/arch/mips/cavium-octeon/crypto/octeon-sha1.c index 30f1d75208a5..37a07b3c4568 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha1.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha1.c @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -71,20 +72,6 @@ static void octeon_sha1_transform(const void *_block) octeon_sha1_start(block[7]); } -static int octeon_sha1_init(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA1_H0; - sctx->state[1] = SHA1_H1; - sctx->state[2] = SHA1_H2; - sctx->state[3] = SHA1_H3; - sctx->state[4] = SHA1_H4; - sctx->count = 0; - - return 0; -} - static void __octeon_sha1_update(struct sha1_state *sctx, const u8 *data, unsigned int len) { @@ -200,7 +187,7 @@ static int octeon_sha1_import(struct shash_desc *desc, const void *in) static struct shash_alg octeon_sha1_alg = { .digestsize = SHA1_DIGEST_SIZE, - .init = octeon_sha1_init, + .init = sha1_base_init, .update = octeon_sha1_update, .final = octeon_sha1_final, .export = octeon_sha1_export, diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha256.c b/arch/mips/cavium-octeon/crypto/octeon-sha256.c index 36cb92895d72..435e4a6e7f13 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha256.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha256.c @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -63,40 +64,6 @@ static void octeon_sha256_transform(const void *_block) octeon_sha256_start(block[7]); } -static int octeon_sha224_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA224_H0; - sctx->state[1] = SHA224_H1; - sctx->state[2] = SHA224_H2; - sctx->state[3] = SHA224_H3; - sctx->state[4] = SHA224_H4; - sctx->state[5] = SHA224_H5; - sctx->state[6] = SHA224_H6; - sctx->state[7] = SHA224_H7; - sctx->count = 0; - - return 0; -} - -static int octeon_sha256_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA256_H0; - sctx->state[1] = SHA256_H1; - sctx->state[2] = SHA256_H2; - sctx->state[3] = SHA256_H3; - sctx->state[4] = SHA256_H4; - sctx->state[5] = SHA256_H5; - sctx->state[6] = SHA256_H6; - sctx->state[7] = SHA256_H7; - sctx->count = 0; - - return 0; -} - static void __octeon_sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len) { @@ -224,7 +191,7 @@ static int octeon_sha256_import(struct shash_desc *desc, const void *in) static struct shash_alg octeon_sha256_algs[2] = { { .digestsize = SHA256_DIGEST_SIZE, - .init = octeon_sha256_init, + .init = sha256_base_init, .update = octeon_sha256_update, .final = octeon_sha256_final, .export = octeon_sha256_export, @@ -240,7 +207,7 @@ static struct shash_alg octeon_sha256_algs[2] = { { } }, { .digestsize = SHA224_DIGEST_SIZE, - .init = octeon_sha224_init, + .init = sha224_base_init, .update = octeon_sha256_update, .final = octeon_sha224_final, .descsize = sizeof(struct sha256_state), diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha512.c b/arch/mips/cavium-octeon/crypto/octeon-sha512.c index 359f039820d8..2dee9354e33f 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha512.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha512.c @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -74,40 +75,6 @@ static void octeon_sha512_transform(const void *_block) octeon_sha512_start(block[15]); } -static int octeon_sha512_init(struct shash_desc *desc) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA512_H0; - sctx->state[1] = SHA512_H1; - sctx->state[2] = SHA512_H2; - sctx->state[3] = SHA512_H3; - sctx->state[4] = SHA512_H4; - sctx->state[5] = SHA512_H5; - sctx->state[6] = SHA512_H6; - sctx->state[7] = SHA512_H7; - sctx->count[0] = sctx->count[1] = 0; - - return 0; -} - -static int octeon_sha384_init(struct shash_desc *desc) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA384_H0; - sctx->state[1] = SHA384_H1; - sctx->state[2] = SHA384_H2; - sctx->state[3] = SHA384_H3; - sctx->state[4] = SHA384_H4; - sctx->state[5] = SHA384_H5; - sctx->state[6] = SHA384_H6; - sctx->state[7] = SHA384_H7; - sctx->count[0] = sctx->count[1] = 0; - - return 0; -} - static void __octeon_sha512_update(struct sha512_state *sctx, const u8 *data, unsigned int len) { @@ -223,7 +190,7 @@ static int octeon_sha384_final(struct shash_desc *desc, u8 *hash) static struct shash_alg octeon_sha512_algs[2] = { { .digestsize = SHA512_DIGEST_SIZE, - .init = octeon_sha512_init, + .init = sha512_base_init, .update = octeon_sha512_update, .final = octeon_sha512_final, .descsize = sizeof(struct sha512_state), @@ -236,7 +203,7 @@ static struct shash_alg octeon_sha512_algs[2] = { { } }, { .digestsize = SHA384_DIGEST_SIZE, - .init = octeon_sha384_init, + .init = sha384_base_init, .update = octeon_sha512_update, .final = octeon_sha384_final, .descsize = sizeof(struct sha512_state), From 41ea0f6c19f6fa403370a9e40e4d500041eb4fc8 Mon Sep 17 00:00:00 2001 From: Tianjia Zhang Date: Mon, 20 Dec 2021 17:23:16 +0800 Subject: [PATCH 135/145] crypto: powerpc/sha - remove duplicate hash init function sha*_base_init() series functions has implemented the initialization of the hash context, this commit use sha*_base_init() function to replace repeated implementations. Signed-off-by: Tianjia Zhang Signed-off-by: Herbert Xu --- arch/powerpc/crypto/sha1-spe-glue.c | 17 ++---------- arch/powerpc/crypto/sha1.c | 14 ++-------- arch/powerpc/crypto/sha256-spe-glue.c | 39 +++------------------------ 3 files changed, 7 insertions(+), 63 deletions(-) diff --git a/arch/powerpc/crypto/sha1-spe-glue.c b/arch/powerpc/crypto/sha1-spe-glue.c index 88e8ea73bfa7..9170892a8557 100644 --- a/arch/powerpc/crypto/sha1-spe-glue.c +++ b/arch/powerpc/crypto/sha1-spe-glue.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -55,20 +56,6 @@ static inline void ppc_sha1_clear_context(struct sha1_state *sctx) do { *ptr++ = 0; } while (--count); } -static int ppc_spe_sha1_init(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA1_H0; - sctx->state[1] = SHA1_H1; - sctx->state[2] = SHA1_H2; - sctx->state[3] = SHA1_H3; - sctx->state[4] = SHA1_H4; - sctx->count = 0; - - return 0; -} - static int ppc_spe_sha1_update(struct shash_desc *desc, const u8 *data, unsigned int len) { @@ -168,7 +155,7 @@ static int ppc_spe_sha1_import(struct shash_desc *desc, const void *in) static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, - .init = ppc_spe_sha1_init, + .init = sha1_base_init, .update = ppc_spe_sha1_update, .final = ppc_spe_sha1_final, .export = ppc_spe_sha1_export, diff --git a/arch/powerpc/crypto/sha1.c b/arch/powerpc/crypto/sha1.c index 7a55d790cdb1..f283bbd3f121 100644 --- a/arch/powerpc/crypto/sha1.c +++ b/arch/powerpc/crypto/sha1.c @@ -18,21 +18,11 @@ #include #include #include +#include #include void powerpc_sha_transform(u32 *state, const u8 *src); -static int powerpc_sha1_init(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - *sctx = (struct sha1_state){ - .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, - }; - - return 0; -} - static int powerpc_sha1_update(struct shash_desc *desc, const u8 *data, unsigned int len) { @@ -114,7 +104,7 @@ static int powerpc_sha1_import(struct shash_desc *desc, const void *in) static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, - .init = powerpc_sha1_init, + .init = sha1_base_init, .update = powerpc_sha1_update, .final = powerpc_sha1_final, .export = powerpc_sha1_export, diff --git a/arch/powerpc/crypto/sha256-spe-glue.c b/arch/powerpc/crypto/sha256-spe-glue.c index ffedea7e4bef..2997d13236e0 100644 --- a/arch/powerpc/crypto/sha256-spe-glue.c +++ b/arch/powerpc/crypto/sha256-spe-glue.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -56,40 +57,6 @@ static inline void ppc_sha256_clear_context(struct sha256_state *sctx) do { *ptr++ = 0; } while (--count); } -static int ppc_spe_sha256_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA256_H0; - sctx->state[1] = SHA256_H1; - sctx->state[2] = SHA256_H2; - sctx->state[3] = SHA256_H3; - sctx->state[4] = SHA256_H4; - sctx->state[5] = SHA256_H5; - sctx->state[6] = SHA256_H6; - sctx->state[7] = SHA256_H7; - sctx->count = 0; - - return 0; -} - -static int ppc_spe_sha224_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA224_H0; - sctx->state[1] = SHA224_H1; - sctx->state[2] = SHA224_H2; - sctx->state[3] = SHA224_H3; - sctx->state[4] = SHA224_H4; - sctx->state[5] = SHA224_H5; - sctx->state[6] = SHA224_H6; - sctx->state[7] = SHA224_H7; - sctx->count = 0; - - return 0; -} - static int ppc_spe_sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len) { @@ -214,7 +181,7 @@ static int ppc_spe_sha256_import(struct shash_desc *desc, const void *in) static struct shash_alg algs[2] = { { .digestsize = SHA256_DIGEST_SIZE, - .init = ppc_spe_sha256_init, + .init = sha256_base_init, .update = ppc_spe_sha256_update, .final = ppc_spe_sha256_final, .export = ppc_spe_sha256_export, @@ -230,7 +197,7 @@ static struct shash_alg algs[2] = { { } }, { .digestsize = SHA224_DIGEST_SIZE, - .init = ppc_spe_sha224_init, + .init = sha224_base_init, .update = ppc_spe_sha256_update, .final = ppc_spe_sha224_final, .export = ppc_spe_sha256_export, From e0583b6acb92683ba192c745f12ef31997e9b8b1 Mon Sep 17 00:00:00 2001 From: Tianjia Zhang Date: Mon, 20 Dec 2021 17:23:17 +0800 Subject: [PATCH 136/145] crypto: sparc/sha - remove duplicate hash init function sha*_base_init() series functions has implemented the initialization of the hash context, this commit use sha*_base_init() function to replace repeated implementations. Signed-off-by: Tianjia Zhang Signed-off-by: Herbert Xu --- arch/sparc/crypto/sha1_glue.c | 14 ++----------- arch/sparc/crypto/sha256_glue.c | 37 +++------------------------------ arch/sparc/crypto/sha512_glue.c | 37 +++------------------------------ 3 files changed, 8 insertions(+), 80 deletions(-) diff --git a/arch/sparc/crypto/sha1_glue.c b/arch/sparc/crypto/sha1_glue.c index 86a654cce5ab..06b7becfcb21 100644 --- a/arch/sparc/crypto/sha1_glue.c +++ b/arch/sparc/crypto/sha1_glue.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -26,17 +27,6 @@ asmlinkage void sha1_sparc64_transform(u32 *digest, const char *data, unsigned int rounds); -static int sha1_sparc64_init(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - *sctx = (struct sha1_state){ - .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, - }; - - return 0; -} - static void __sha1_sparc64_update(struct sha1_state *sctx, const u8 *data, unsigned int len, unsigned int partial) { @@ -128,7 +118,7 @@ static int sha1_sparc64_import(struct shash_desc *desc, const void *in) static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_sparc64_init, + .init = sha1_base_init, .update = sha1_sparc64_update, .final = sha1_sparc64_final, .export = sha1_sparc64_export, diff --git a/arch/sparc/crypto/sha256_glue.c b/arch/sparc/crypto/sha256_glue.c index 60ec524cf9ca..285561a1cde5 100644 --- a/arch/sparc/crypto/sha256_glue.c +++ b/arch/sparc/crypto/sha256_glue.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -26,38 +27,6 @@ asmlinkage void sha256_sparc64_transform(u32 *digest, const char *data, unsigned int rounds); -static int sha224_sparc64_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - sctx->state[0] = SHA224_H0; - sctx->state[1] = SHA224_H1; - sctx->state[2] = SHA224_H2; - sctx->state[3] = SHA224_H3; - sctx->state[4] = SHA224_H4; - sctx->state[5] = SHA224_H5; - sctx->state[6] = SHA224_H6; - sctx->state[7] = SHA224_H7; - sctx->count = 0; - - return 0; -} - -static int sha256_sparc64_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - sctx->state[0] = SHA256_H0; - sctx->state[1] = SHA256_H1; - sctx->state[2] = SHA256_H2; - sctx->state[3] = SHA256_H3; - sctx->state[4] = SHA256_H4; - sctx->state[5] = SHA256_H5; - sctx->state[6] = SHA256_H6; - sctx->state[7] = SHA256_H7; - sctx->count = 0; - - return 0; -} - static void __sha256_sparc64_update(struct sha256_state *sctx, const u8 *data, unsigned int len, unsigned int partial) { @@ -158,7 +127,7 @@ static int sha256_sparc64_import(struct shash_desc *desc, const void *in) static struct shash_alg sha256_alg = { .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_sparc64_init, + .init = sha256_base_init, .update = sha256_sparc64_update, .final = sha256_sparc64_final, .export = sha256_sparc64_export, @@ -176,7 +145,7 @@ static struct shash_alg sha256_alg = { static struct shash_alg sha224_alg = { .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_sparc64_init, + .init = sha224_base_init, .update = sha256_sparc64_update, .final = sha224_sparc64_final, .descsize = sizeof(struct sha256_state), diff --git a/arch/sparc/crypto/sha512_glue.c b/arch/sparc/crypto/sha512_glue.c index 273ce21918c1..d66efa4ec59a 100644 --- a/arch/sparc/crypto/sha512_glue.c +++ b/arch/sparc/crypto/sha512_glue.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -25,38 +26,6 @@ asmlinkage void sha512_sparc64_transform(u64 *digest, const char *data, unsigned int rounds); -static int sha512_sparc64_init(struct shash_desc *desc) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - sctx->state[0] = SHA512_H0; - sctx->state[1] = SHA512_H1; - sctx->state[2] = SHA512_H2; - sctx->state[3] = SHA512_H3; - sctx->state[4] = SHA512_H4; - sctx->state[5] = SHA512_H5; - sctx->state[6] = SHA512_H6; - sctx->state[7] = SHA512_H7; - sctx->count[0] = sctx->count[1] = 0; - - return 0; -} - -static int sha384_sparc64_init(struct shash_desc *desc) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - sctx->state[0] = SHA384_H0; - sctx->state[1] = SHA384_H1; - sctx->state[2] = SHA384_H2; - sctx->state[3] = SHA384_H3; - sctx->state[4] = SHA384_H4; - sctx->state[5] = SHA384_H5; - sctx->state[6] = SHA384_H6; - sctx->state[7] = SHA384_H7; - sctx->count[0] = sctx->count[1] = 0; - - return 0; -} - static void __sha512_sparc64_update(struct sha512_state *sctx, const u8 *data, unsigned int len, unsigned int partial) { @@ -146,7 +115,7 @@ static int sha384_sparc64_final(struct shash_desc *desc, u8 *hash) static struct shash_alg sha512 = { .digestsize = SHA512_DIGEST_SIZE, - .init = sha512_sparc64_init, + .init = sha512_base_init, .update = sha512_sparc64_update, .final = sha512_sparc64_final, .descsize = sizeof(struct sha512_state), @@ -161,7 +130,7 @@ static struct shash_alg sha512 = { static struct shash_alg sha384 = { .digestsize = SHA384_DIGEST_SIZE, - .init = sha384_sparc64_init, + .init = sha384_base_init, .update = sha512_sparc64_update, .final = sha384_sparc64_final, .descsize = sizeof(struct sha512_state), From db1eafb8c512f53c4b0e8673b5a89f9ad14de4fe Mon Sep 17 00:00:00 2001 From: Tianjia Zhang Date: Mon, 20 Dec 2021 17:23:18 +0800 Subject: [PATCH 137/145] crypto: s390/sha512 - Use macros instead of direct IV numbers In the init functions of sha512 and sha384, the initial hash value use macros instead of numbers. Signed-off-by: Tianjia Zhang Signed-off-by: Herbert Xu --- arch/s390/crypto/sha512_s390.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c index 29a6bd404c59..43ce4956df73 100644 --- a/arch/s390/crypto/sha512_s390.c +++ b/arch/s390/crypto/sha512_s390.c @@ -22,14 +22,14 @@ static int sha512_init(struct shash_desc *desc) { struct s390_sha_ctx *ctx = shash_desc_ctx(desc); - *(__u64 *)&ctx->state[0] = 0x6a09e667f3bcc908ULL; - *(__u64 *)&ctx->state[2] = 0xbb67ae8584caa73bULL; - *(__u64 *)&ctx->state[4] = 0x3c6ef372fe94f82bULL; - *(__u64 *)&ctx->state[6] = 0xa54ff53a5f1d36f1ULL; - *(__u64 *)&ctx->state[8] = 0x510e527fade682d1ULL; - *(__u64 *)&ctx->state[10] = 0x9b05688c2b3e6c1fULL; - *(__u64 *)&ctx->state[12] = 0x1f83d9abfb41bd6bULL; - *(__u64 *)&ctx->state[14] = 0x5be0cd19137e2179ULL; + *(__u64 *)&ctx->state[0] = SHA512_H0; + *(__u64 *)&ctx->state[2] = SHA512_H1; + *(__u64 *)&ctx->state[4] = SHA512_H2; + *(__u64 *)&ctx->state[6] = SHA512_H3; + *(__u64 *)&ctx->state[8] = SHA512_H4; + *(__u64 *)&ctx->state[10] = SHA512_H5; + *(__u64 *)&ctx->state[12] = SHA512_H6; + *(__u64 *)&ctx->state[14] = SHA512_H7; ctx->count = 0; ctx->func = CPACF_KIMD_SHA_512; @@ -87,14 +87,14 @@ static int sha384_init(struct shash_desc *desc) { struct s390_sha_ctx *ctx = shash_desc_ctx(desc); - *(__u64 *)&ctx->state[0] = 0xcbbb9d5dc1059ed8ULL; - *(__u64 *)&ctx->state[2] = 0x629a292a367cd507ULL; - *(__u64 *)&ctx->state[4] = 0x9159015a3070dd17ULL; - *(__u64 *)&ctx->state[6] = 0x152fecd8f70e5939ULL; - *(__u64 *)&ctx->state[8] = 0x67332667ffc00b31ULL; - *(__u64 *)&ctx->state[10] = 0x8eb44a8768581511ULL; - *(__u64 *)&ctx->state[12] = 0xdb0c2e0d64f98fa7ULL; - *(__u64 *)&ctx->state[14] = 0x47b5481dbefa4fa4ULL; + *(__u64 *)&ctx->state[0] = SHA384_H0; + *(__u64 *)&ctx->state[2] = SHA384_H1; + *(__u64 *)&ctx->state[4] = SHA384_H2; + *(__u64 *)&ctx->state[6] = SHA384_H3; + *(__u64 *)&ctx->state[8] = SHA384_H4; + *(__u64 *)&ctx->state[10] = SHA384_H5; + *(__u64 *)&ctx->state[12] = SHA384_H6; + *(__u64 *)&ctx->state[14] = SHA384_H7; ctx->count = 0; ctx->func = CPACF_KIMD_SHA_512; From 29009604ad4e3ef784fd9b9fef6f23610ddf633d Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Mon, 20 Dec 2021 20:50:22 +0100 Subject: [PATCH 138/145] crypto: stm32/crc32 - Fix kernel BUG triggered in probe() The include/linux/crypto.h struct crypto_alg field cra_driver_name description states "Unique name of the transformation provider. " ... " this contains the name of the chip or provider and the name of the transformation algorithm." In case of the stm32-crc driver, field cra_driver_name is identical for all registered transformation providers and set to the name of the driver itself, which is incorrect. This patch fixes it by assigning a unique cra_driver_name to each registered transformation provider. The kernel crash is triggered when the driver calls crypto_register_shashes() which calls crypto_register_shash(), which calls crypto_register_alg(), which calls __crypto_register_alg(), which returns -EEXIST, which is propagated back through this call chain. Upon -EEXIST from crypto_register_shash(), the crypto_register_shashes() starts unregistering the providers back, and calls crypto_unregister_shash(), which calls crypto_unregister_alg(), and this is where the BUG() triggers due to incorrect cra_refcnt. Fixes: b51dbe90912a ("crypto: stm32 - Support for STM32 CRC32 crypto module") Signed-off-by: Marek Vasut Cc: # 4.12+ Cc: Alexandre Torgue Cc: Fabien Dessenne Cc: Herbert Xu Cc: Lionel Debieve Cc: Nicolas Toromanoff Cc: linux-arm-kernel@lists.infradead.org Cc: linux-stm32@st-md-mailman.stormreply.com To: linux-crypto@vger.kernel.org Acked-by: Nicolas Toromanoff Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32-crc32.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/stm32/stm32-crc32.c b/drivers/crypto/stm32/stm32-crc32.c index 75867c0b0017..be1bf39a317d 100644 --- a/drivers/crypto/stm32/stm32-crc32.c +++ b/drivers/crypto/stm32/stm32-crc32.c @@ -279,7 +279,7 @@ static struct shash_alg algs[] = { .digestsize = CHKSUM_DIGEST_SIZE, .base = { .cra_name = "crc32", - .cra_driver_name = DRIVER_NAME, + .cra_driver_name = "stm32-crc32-crc32", .cra_priority = 200, .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, .cra_blocksize = CHKSUM_BLOCK_SIZE, @@ -301,7 +301,7 @@ static struct shash_alg algs[] = { .digestsize = CHKSUM_DIGEST_SIZE, .base = { .cra_name = "crc32c", - .cra_driver_name = DRIVER_NAME, + .cra_driver_name = "stm32-crc32-crc32c", .cra_priority = 200, .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, .cra_blocksize = CHKSUM_BLOCK_SIZE, From ef4d891499442a156655b0c0df56bdf539897495 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Tue, 21 Dec 2021 08:38:28 +0800 Subject: [PATCH 139/145] crypto: ccp - remove unneeded semicolon Eliminate the following coccicheck warning: ./drivers/crypto/ccp/sev-dev.c:263:2-3: Unneeded semicolon Reported-by: Abaci Robot Signed-off-by: Yang Li Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 991cc5aaaa0f..8fd774a10edc 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -260,7 +260,7 @@ static void sev_write_init_ex_file_if_required(int cmd_id) break; default: return; - }; + } sev_write_init_ex_file(); } From d480a26bdf872529919e7c30e17f79d0d7b8c4da Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Tue, 21 Dec 2021 07:06:11 -0800 Subject: [PATCH 140/145] crypto: x86/aesni - don't require alignment of data x86 AES-NI routines can deal with unaligned data. Crypto context (key, iv etc.) have to be aligned but we take care of that separately by copying it onto the stack. We were feeding unaligned data into crypto routines up until commit 83c83e658863 ("crypto: aesni - refactor scatterlist processing") switched to use the full skcipher API which uses cra_alignmask to decide data alignment. This fixes 21% performance regression in kTLS. Tested by booting with CONFIG_CRYPTO_MANAGER_EXTRA_TESTS=y (and running thru various kTLS packets). CC: stable@vger.kernel.org # 5.15+ Fixes: 83c83e658863 ("crypto: aesni - refactor scatterlist processing") Signed-off-by: Jakub Kicinski Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_glue.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index e09f4672dd38..41901ba9d3a2 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -1107,7 +1107,7 @@ static struct aead_alg aesni_aeads[] = { { .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx), - .cra_alignmask = AESNI_ALIGN - 1, + .cra_alignmask = 0, .cra_module = THIS_MODULE, }, }, { @@ -1124,7 +1124,7 @@ static struct aead_alg aesni_aeads[] = { { .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct generic_gcmaes_ctx), - .cra_alignmask = AESNI_ALIGN - 1, + .cra_alignmask = 0, .cra_module = THIS_MODULE, }, } }; From 304b4acee2f023a67f122d15a37b40ce460244d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20M=C3=BCller?= Date: Tue, 21 Dec 2021 20:31:42 +0100 Subject: [PATCH 141/145] crypto: kdf - select SHA-256 required for self-test The self test of the KDF is based on SHA-256. Thus, this algorithm must be present as otherwise a warning is issued. Reported-by: kernel test robot Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 01b9ca0836a5..649ee1df05c6 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1847,7 +1847,7 @@ config CRYPTO_JITTERENTROPY config CRYPTO_KDF800108_CTR tristate - select CRYPTO_HASH + select CRYPTO_SHA256 config CRYPTO_USER_API tristate From c5d692a2335d64ac390aeb8ab6c4ac9f662e1be4 Mon Sep 17 00:00:00 2001 From: Tom Rix Date: Wed, 22 Dec 2021 09:29:23 -0800 Subject: [PATCH 142/145] crypto: hisilicon - cleanup warning in qm_get_qos_value() Building with clang static analysis returns this warning: qm.c:4382:11: warning: The left operand of '==' is a garbage value if (*val == 0 || *val > QM_QOS_MAX_VAL || ret) { ~~~~ ^ The call to qm_qos_value_init() can return an error without setting *val. So check ret before checking *val. Fixes: 72b010dc33b9 ("crypto: hisilicon/qm - supports writing QoS int the host") Signed-off-by: Tom Rix Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index b731cb4ec294..c5b84a5ea350 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -4394,7 +4394,7 @@ static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf, return -EINVAL; ret = qm_qos_value_init(val_buf, val); - if (*val == 0 || *val > QM_QOS_MAX_VAL || ret) { + if (ret || *val == 0 || *val > QM_QOS_MAX_VAL) { pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n"); return -EINVAL; } From 4cab5dfd15b77cde2b965bbf71a86876a42684da Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Fri, 24 Dec 2021 11:05:32 +0000 Subject: [PATCH 143/145] crypto: qat - fix definition of ring reset results The ring reset result values are defined starting from 0x1 instead of 0. This causes out-of-tree drivers that support this message to understand that a ring reset failed even if the operation was successful. Fix by starting the definition of ring reset result values from 0. Fixes: 0bba03ce9739 ("crypto: qat - add PFVF support to enable the reset of ring pairs") Signed-off-by: Giovanni Cabiddu Reported-by: Adam Guerin Reviewed-by: Marco Chiappero Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_pfvf_msg.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h index 86b0e7baa4d3..9c37a2661392 100644 --- a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h @@ -139,10 +139,10 @@ enum pf2vf_compat_response { }; enum ring_reset_result { - RPRESET_SUCCESS = 0x01, - RPRESET_NOT_SUPPORTED = 0x02, - RPRESET_INVAL_BANK = 0x03, - RPRESET_TIMEOUT = 0x04, + RPRESET_SUCCESS = 0x00, + RPRESET_NOT_SUPPORTED = 0x01, + RPRESET_INVAL_BANK = 0x02, + RPRESET_TIMEOUT = 0x03, }; #define ADF_VF2PF_RNG_RESET_RP_MASK GENMASK(1, 0) From dd827abe296fe4249b2f8c9b95f72f814ea8348c Mon Sep 17 00:00:00 2001 From: Zizhuang Deng Date: Thu, 30 Dec 2021 15:03:31 +0800 Subject: [PATCH 144/145] lib/mpi: Add the return value check of kcalloc() Add the return value check of kcalloc() to avoid potential NULL ptr dereference. Fixes: a8ea8bdd9df9 ("lib/mpi: Extend the MPI library") Signed-off-by: Zizhuang Deng Reviewed-by: Tianjia Zhang Signed-off-by: Herbert Xu --- lib/mpi/mpi-mod.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/mpi/mpi-mod.c b/lib/mpi/mpi-mod.c index 47bc59edd4ff..54fcc01564d9 100644 --- a/lib/mpi/mpi-mod.c +++ b/lib/mpi/mpi-mod.c @@ -40,6 +40,8 @@ mpi_barrett_t mpi_barrett_init(MPI m, int copy) mpi_normalize(m); ctx = kcalloc(1, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return NULL; if (copy) { ctx->m = mpi_copy(m); From 5f21d7d283dd82865bdb0123795b3accf0d42b67 Mon Sep 17 00:00:00 2001 From: Jiasheng Jiang Date: Fri, 31 Dec 2021 09:40:36 +0800 Subject: [PATCH 145/145] crypto: af_alg - rewrite NULL pointer check Because of the possible alloc failure of the alloc_page(), it could return NULL pointer. And there is a check below the sg_assign_page(). But it will be more logical to move the NULL check before the sg_assign_page(). Signed-off-by: Jiasheng Jiang Signed-off-by: Herbert Xu --- crypto/af_alg.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crypto/af_alg.c b/crypto/af_alg.c index 3dd5a773c320..e1ea18536a5f 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -931,16 +931,19 @@ int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size, sg_unmark_end(sg + sgl->cur - 1); do { + struct page *pg; unsigned int i = sgl->cur; plen = min_t(size_t, len, PAGE_SIZE); - sg_assign_page(sg + i, alloc_page(GFP_KERNEL)); - if (!sg_page(sg + i)) { + pg = alloc_page(GFP_KERNEL); + if (!pg) { err = -ENOMEM; goto unlock; } + sg_assign_page(sg + i, pg); + err = memcpy_from_msg(page_address(sg_page(sg + i)), msg, plen); if (err) {