8d195e7a8a
gcc-11 points out a mismatch between the declaration and the definition
of poly1305_core_setkey():
lib/crypto/poly1305-donna32.c:13:67: error: argument 2 of type ‘const u8[16]’ {aka ‘const unsigned char[16]’} with mismatched bound [-Werror=array-parameter=]
13 | void poly1305_core_setkey(struct poly1305_core_key *key, const u8 raw_key[16])
| ~~~~~~~~~^~~~~~~~~~~
In file included from lib/crypto/poly1305-donna32.c:11:
include/crypto/internal/poly1305.h:21:68: note: previously declared as ‘const u8 *’ {aka ‘const unsigned char *’}
21 | void poly1305_core_setkey(struct poly1305_core_key *key, const u8 *raw_key);
This is harmless in principle, as the calling conventions are the same,
but the more specific prototype allows better type checking in the
caller.
Change the declaration to match the actual function definition.
The poly1305_simd_init() is a bit suspicious here, as it previously
had a 32-byte argument type, but looks like it needs to take the
16-byte POLY1305_BLOCK_SIZE array instead.
Fixes: 1c08a10436 ("crypto: poly1305 - add new 32 and 64-bit generic versions")
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Reviewed-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
274 lines
7.1 KiB
C
274 lines
7.1 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* OpenSSL/Cryptogams accelerated Poly1305 transform for ARM
|
|
*
|
|
* Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org>
|
|
*/
|
|
|
|
#include <asm/hwcap.h>
|
|
#include <asm/neon.h>
|
|
#include <asm/simd.h>
|
|
#include <asm/unaligned.h>
|
|
#include <crypto/algapi.h>
|
|
#include <crypto/internal/hash.h>
|
|
#include <crypto/internal/poly1305.h>
|
|
#include <crypto/internal/simd.h>
|
|
#include <linux/cpufeature.h>
|
|
#include <linux/crypto.h>
|
|
#include <linux/jump_label.h>
|
|
#include <linux/module.h>
|
|
|
|
void poly1305_init_arm(void *state, const u8 *key);
|
|
void poly1305_blocks_arm(void *state, const u8 *src, u32 len, u32 hibit);
|
|
void poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit);
|
|
void poly1305_emit_arm(void *state, u8 *digest, const u32 *nonce);
|
|
|
|
void __weak poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit)
|
|
{
|
|
}
|
|
|
|
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
|
|
|
|
void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
|
|
{
|
|
poly1305_init_arm(&dctx->h, key);
|
|
dctx->s[0] = get_unaligned_le32(key + 16);
|
|
dctx->s[1] = get_unaligned_le32(key + 20);
|
|
dctx->s[2] = get_unaligned_le32(key + 24);
|
|
dctx->s[3] = get_unaligned_le32(key + 28);
|
|
dctx->buflen = 0;
|
|
}
|
|
EXPORT_SYMBOL(poly1305_init_arch);
|
|
|
|
static int arm_poly1305_init(struct shash_desc *desc)
|
|
{
|
|
struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
|
|
|
|
dctx->buflen = 0;
|
|
dctx->rset = 0;
|
|
dctx->sset = false;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void arm_poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src,
|
|
u32 len, u32 hibit, bool do_neon)
|
|
{
|
|
if (unlikely(!dctx->sset)) {
|
|
if (!dctx->rset) {
|
|
poly1305_init_arm(&dctx->h, src);
|
|
src += POLY1305_BLOCK_SIZE;
|
|
len -= POLY1305_BLOCK_SIZE;
|
|
dctx->rset = 1;
|
|
}
|
|
if (len >= POLY1305_BLOCK_SIZE) {
|
|
dctx->s[0] = get_unaligned_le32(src + 0);
|
|
dctx->s[1] = get_unaligned_le32(src + 4);
|
|
dctx->s[2] = get_unaligned_le32(src + 8);
|
|
dctx->s[3] = get_unaligned_le32(src + 12);
|
|
src += POLY1305_BLOCK_SIZE;
|
|
len -= POLY1305_BLOCK_SIZE;
|
|
dctx->sset = true;
|
|
}
|
|
if (len < POLY1305_BLOCK_SIZE)
|
|
return;
|
|
}
|
|
|
|
len &= ~(POLY1305_BLOCK_SIZE - 1);
|
|
|
|
if (static_branch_likely(&have_neon) && likely(do_neon))
|
|
poly1305_blocks_neon(&dctx->h, src, len, hibit);
|
|
else
|
|
poly1305_blocks_arm(&dctx->h, src, len, hibit);
|
|
}
|
|
|
|
static void arm_poly1305_do_update(struct poly1305_desc_ctx *dctx,
|
|
const u8 *src, u32 len, bool do_neon)
|
|
{
|
|
if (unlikely(dctx->buflen)) {
|
|
u32 bytes = min(len, POLY1305_BLOCK_SIZE - dctx->buflen);
|
|
|
|
memcpy(dctx->buf + dctx->buflen, src, bytes);
|
|
src += bytes;
|
|
len -= bytes;
|
|
dctx->buflen += bytes;
|
|
|
|
if (dctx->buflen == POLY1305_BLOCK_SIZE) {
|
|
arm_poly1305_blocks(dctx, dctx->buf,
|
|
POLY1305_BLOCK_SIZE, 1, false);
|
|
dctx->buflen = 0;
|
|
}
|
|
}
|
|
|
|
if (likely(len >= POLY1305_BLOCK_SIZE)) {
|
|
arm_poly1305_blocks(dctx, src, len, 1, do_neon);
|
|
src += round_down(len, POLY1305_BLOCK_SIZE);
|
|
len %= POLY1305_BLOCK_SIZE;
|
|
}
|
|
|
|
if (unlikely(len)) {
|
|
dctx->buflen = len;
|
|
memcpy(dctx->buf, src, len);
|
|
}
|
|
}
|
|
|
|
static int arm_poly1305_update(struct shash_desc *desc,
|
|
const u8 *src, unsigned int srclen)
|
|
{
|
|
struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
|
|
|
|
arm_poly1305_do_update(dctx, src, srclen, false);
|
|
return 0;
|
|
}
|
|
|
|
static int __maybe_unused arm_poly1305_update_neon(struct shash_desc *desc,
|
|
const u8 *src,
|
|
unsigned int srclen)
|
|
{
|
|
struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
|
|
bool do_neon = crypto_simd_usable() && srclen > 128;
|
|
|
|
if (static_branch_likely(&have_neon) && do_neon)
|
|
kernel_neon_begin();
|
|
arm_poly1305_do_update(dctx, src, srclen, do_neon);
|
|
if (static_branch_likely(&have_neon) && do_neon)
|
|
kernel_neon_end();
|
|
return 0;
|
|
}
|
|
|
|
void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
|
|
unsigned int nbytes)
|
|
{
|
|
bool do_neon = IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
|
|
crypto_simd_usable();
|
|
|
|
if (unlikely(dctx->buflen)) {
|
|
u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen);
|
|
|
|
memcpy(dctx->buf + dctx->buflen, src, bytes);
|
|
src += bytes;
|
|
nbytes -= bytes;
|
|
dctx->buflen += bytes;
|
|
|
|
if (dctx->buflen == POLY1305_BLOCK_SIZE) {
|
|
poly1305_blocks_arm(&dctx->h, dctx->buf,
|
|
POLY1305_BLOCK_SIZE, 1);
|
|
dctx->buflen = 0;
|
|
}
|
|
}
|
|
|
|
if (likely(nbytes >= POLY1305_BLOCK_SIZE)) {
|
|
unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE);
|
|
|
|
if (static_branch_likely(&have_neon) && do_neon) {
|
|
do {
|
|
unsigned int todo = min_t(unsigned int, len, SZ_4K);
|
|
|
|
kernel_neon_begin();
|
|
poly1305_blocks_neon(&dctx->h, src, todo, 1);
|
|
kernel_neon_end();
|
|
|
|
len -= todo;
|
|
src += todo;
|
|
} while (len);
|
|
} else {
|
|
poly1305_blocks_arm(&dctx->h, src, len, 1);
|
|
src += len;
|
|
}
|
|
nbytes %= POLY1305_BLOCK_SIZE;
|
|
}
|
|
|
|
if (unlikely(nbytes)) {
|
|
dctx->buflen = nbytes;
|
|
memcpy(dctx->buf, src, nbytes);
|
|
}
|
|
}
|
|
EXPORT_SYMBOL(poly1305_update_arch);
|
|
|
|
void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
|
|
{
|
|
if (unlikely(dctx->buflen)) {
|
|
dctx->buf[dctx->buflen++] = 1;
|
|
memset(dctx->buf + dctx->buflen, 0,
|
|
POLY1305_BLOCK_SIZE - dctx->buflen);
|
|
poly1305_blocks_arm(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0);
|
|
}
|
|
|
|
poly1305_emit_arm(&dctx->h, dst, dctx->s);
|
|
*dctx = (struct poly1305_desc_ctx){};
|
|
}
|
|
EXPORT_SYMBOL(poly1305_final_arch);
|
|
|
|
static int arm_poly1305_final(struct shash_desc *desc, u8 *dst)
|
|
{
|
|
struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
|
|
|
|
if (unlikely(!dctx->sset))
|
|
return -ENOKEY;
|
|
|
|
poly1305_final_arch(dctx, dst);
|
|
return 0;
|
|
}
|
|
|
|
static struct shash_alg arm_poly1305_algs[] = {{
|
|
.init = arm_poly1305_init,
|
|
.update = arm_poly1305_update,
|
|
.final = arm_poly1305_final,
|
|
.digestsize = POLY1305_DIGEST_SIZE,
|
|
.descsize = sizeof(struct poly1305_desc_ctx),
|
|
|
|
.base.cra_name = "poly1305",
|
|
.base.cra_driver_name = "poly1305-arm",
|
|
.base.cra_priority = 150,
|
|
.base.cra_blocksize = POLY1305_BLOCK_SIZE,
|
|
.base.cra_module = THIS_MODULE,
|
|
#ifdef CONFIG_KERNEL_MODE_NEON
|
|
}, {
|
|
.init = arm_poly1305_init,
|
|
.update = arm_poly1305_update_neon,
|
|
.final = arm_poly1305_final,
|
|
.digestsize = POLY1305_DIGEST_SIZE,
|
|
.descsize = sizeof(struct poly1305_desc_ctx),
|
|
|
|
.base.cra_name = "poly1305",
|
|
.base.cra_driver_name = "poly1305-neon",
|
|
.base.cra_priority = 200,
|
|
.base.cra_blocksize = POLY1305_BLOCK_SIZE,
|
|
.base.cra_module = THIS_MODULE,
|
|
#endif
|
|
}};
|
|
|
|
static int __init arm_poly1305_mod_init(void)
|
|
{
|
|
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
|
|
(elf_hwcap & HWCAP_NEON))
|
|
static_branch_enable(&have_neon);
|
|
else if (IS_REACHABLE(CONFIG_CRYPTO_HASH))
|
|
/* register only the first entry */
|
|
return crypto_register_shash(&arm_poly1305_algs[0]);
|
|
|
|
return IS_REACHABLE(CONFIG_CRYPTO_HASH) ?
|
|
crypto_register_shashes(arm_poly1305_algs,
|
|
ARRAY_SIZE(arm_poly1305_algs)) : 0;
|
|
}
|
|
|
|
static void __exit arm_poly1305_mod_exit(void)
|
|
{
|
|
if (!IS_REACHABLE(CONFIG_CRYPTO_HASH))
|
|
return;
|
|
if (!static_branch_likely(&have_neon)) {
|
|
crypto_unregister_shash(&arm_poly1305_algs[0]);
|
|
return;
|
|
}
|
|
crypto_unregister_shashes(arm_poly1305_algs,
|
|
ARRAY_SIZE(arm_poly1305_algs));
|
|
}
|
|
|
|
module_init(arm_poly1305_mod_init);
|
|
module_exit(arm_poly1305_mod_exit);
|
|
|
|
MODULE_LICENSE("GPL v2");
|
|
MODULE_ALIAS_CRYPTO("poly1305");
|
|
MODULE_ALIAS_CRYPTO("poly1305-arm");
|
|
MODULE_ALIAS_CRYPTO("poly1305-neon");
|