2009-01-18 08:28:34 +03:00
/*
* Support for Intel AES - NI instructions . This file contains glue
* code , the real AES implementation is in intel - aes_asm . S .
*
* Copyright ( C ) 2008 , Intel Corp .
* Author : Huang Ying < ying . huang @ intel . com >
*
2010-11-04 22:00:45 +03:00
* Added RFC4106 AES - GCM support for 128 - bit keys under the AEAD
* interface for 64 - bit kernels .
* Authors : Adrian Hoban < adrian . hoban @ intel . com >
* Gabriele Paoloni < gabriele . paoloni @ intel . com >
* Tadeusz Struk ( tadeusz . struk @ intel . com )
* Aidan O ' Mahony ( aidan . o . mahony @ intel . com )
* Copyright ( c ) 2010 , Intel Corporation .
*
2009-01-18 08:28:34 +03:00
* This program is free software ; you can redistribute it and / or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation ; either version 2 of the License , or
* ( at your option ) any later version .
*/
# include <linux/hardirq.h>
# include <linux/types.h>
2011-05-27 20:33:10 +04:00
# include <linux/module.h>
2009-01-18 08:28:34 +03:00
# include <linux/err.h>
# include <crypto/algapi.h>
# include <crypto/aes.h>
# include <crypto/cryptd.h>
2010-03-10 13:28:55 +03:00
# include <crypto/ctr.h>
2012-07-22 19:18:37 +04:00
# include <crypto/b128ops.h>
# include <crypto/xts.h>
2012-01-26 03:09:06 +04:00
# include <asm/cpu_device_id.h>
2015-04-24 03:46:00 +03:00
# include <asm/fpu/api.h>
2012-06-18 15:07:50 +04:00
# include <asm/crypto/aes.h>
2010-11-04 22:00:45 +03:00
# include <crypto/scatterwalk.h>
# include <crypto/internal/aead.h>
2016-11-22 15:08:33 +03:00
# include <crypto/internal/simd.h>
# include <crypto/internal/skcipher.h>
2010-11-04 22:00:45 +03:00
# include <linux/workqueue.h>
# include <linux/spinlock.h>
2013-04-08 22:51:16 +04:00
# ifdef CONFIG_X86_64
# include <asm/crypto/glue_helper.h>
# endif
2009-01-18 08:28:34 +03:00
2015-01-13 21:16:43 +03:00
2015-06-01 10:53:06 +03:00
# define AESNI_ALIGN 16
2016-11-22 15:08:33 +03:00
# define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
2015-06-01 10:53:06 +03:00
# define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1))
# define RFC4106_HASH_SUBKEY_SIZE 16
2016-11-22 15:08:33 +03:00
# define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
# define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
# define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
2015-06-01 10:53:06 +03:00
2010-11-04 22:00:45 +03:00
/* This data is stored at the end of the crypto_tfm struct.
* It ' s a type of per " session " data storage location .
* This needs to be 16 byte aligned .
*/
struct aesni_rfc4106_gcm_ctx {
2016-11-22 15:08:33 +03:00
u8 hash_subkey [ 16 ] AESNI_ALIGN_ATTR ;
struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR ;
2010-11-04 22:00:45 +03:00
u8 nonce [ 4 ] ;
} ;
2012-07-22 19:18:37 +04:00
struct aesni_xts_ctx {
2016-11-22 15:08:33 +03:00
u8 raw_tweak_ctx [ sizeof ( struct crypto_aes_ctx ) ] AESNI_ALIGN_ATTR ;
u8 raw_crypt_ctx [ sizeof ( struct crypto_aes_ctx ) ] AESNI_ALIGN_ATTR ;
2012-07-22 19:18:37 +04:00
} ;
2009-01-18 08:28:34 +03:00
asmlinkage int aesni_set_key ( struct crypto_aes_ctx * ctx , const u8 * in_key ,
unsigned int key_len ) ;
asmlinkage void aesni_enc ( struct crypto_aes_ctx * ctx , u8 * out ,
const u8 * in ) ;
asmlinkage void aesni_dec ( struct crypto_aes_ctx * ctx , u8 * out ,
const u8 * in ) ;
asmlinkage void aesni_ecb_enc ( struct crypto_aes_ctx * ctx , u8 * out ,
const u8 * in , unsigned int len ) ;
asmlinkage void aesni_ecb_dec ( struct crypto_aes_ctx * ctx , u8 * out ,
const u8 * in , unsigned int len ) ;
asmlinkage void aesni_cbc_enc ( struct crypto_aes_ctx * ctx , u8 * out ,
const u8 * in , unsigned int len , u8 * iv ) ;
asmlinkage void aesni_cbc_dec ( struct crypto_aes_ctx * ctx , u8 * out ,
const u8 * in , unsigned int len , u8 * iv ) ;
2011-05-18 03:03:34 +04:00
int crypto_fpu_init ( void ) ;
void crypto_fpu_exit ( void ) ;
2013-12-12 02:28:41 +04:00
# define AVX_GEN2_OPTSIZE 640
# define AVX_GEN4_OPTSIZE 4096
crypto: aesni-intel - Ported implementation to x86-32
The AES-NI instructions are also available in legacy mode so the 32-bit
architecture may profit from those, too.
To illustrate the performance gain here's a short summary of a dm-crypt
speed test on a Core i7 M620 running at 2.67GHz comparing both assembler
implementations:
x86: i568 aes-ni delta
ECB, 256 bit: 93.8 MB/s 123.3 MB/s +31.4%
CBC, 256 bit: 84.8 MB/s 262.3 MB/s +209.3%
LRW, 256 bit: 108.6 MB/s 222.1 MB/s +104.5%
XTS, 256 bit: 105.0 MB/s 205.5 MB/s +95.7%
Additionally, due to some minor optimizations, the 64-bit version also
got a minor performance gain as seen below:
x86-64: old impl. new impl. delta
ECB, 256 bit: 121.1 MB/s 123.0 MB/s +1.5%
CBC, 256 bit: 285.3 MB/s 290.8 MB/s +1.9%
LRW, 256 bit: 263.7 MB/s 265.3 MB/s +0.6%
XTS, 256 bit: 251.1 MB/s 255.3 MB/s +1.7%
Signed-off-by: Mathias Krause <minipli@googlemail.com>
Reviewed-by: Huang Ying <ying.huang@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2010-11-27 11:34:46 +03:00
# ifdef CONFIG_X86_64
crypto: aes - AES CTR x86_64 "by8" AVX optimization
This patch introduces "by8" AES CTR mode AVX optimization inspired by
Intel Optimized IPSEC Cryptograhpic library. For additional information,
please see:
http://downloadcenter.intel.com/Detail_Desc.aspx?agr=Y&DwnldID=22972
The functions aes_ctr_enc_128_avx_by8(), aes_ctr_enc_192_avx_by8() and
aes_ctr_enc_256_avx_by8() are adapted from
Intel Optimized IPSEC Cryptographic library. When both AES and AVX features
are enabled in a platform, the glue code in AESNI module overrieds the
existing "by4" CTR mode en/decryption with the "by8"
AES CTR mode en/decryption.
On a Haswell desktop, with turbo disabled and all cpus running
at maximum frequency, the "by8" CTR mode optimization
shows better performance results across data & key sizes
as measured by tcrypt.
The average performance improvement of the "by8" version over the "by4"
version is as follows:
For 128 bit key and data sizes >= 256 bytes, there is a 10-16% improvement.
For 192 bit key and data sizes >= 256 bytes, there is a 20-22% improvement.
For 256 bit key and data sizes >= 256 bytes, there is a 20-25% improvement.
A typical run of tcrypt with AES CTR mode encryption of the "by4" and "by8"
optimization shows the following results:
tcrypt with "by4" AES CTR mode encryption optimization on a Haswell Desktop:
---------------------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 343 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 336 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 491 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1130 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7309 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 346 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 361 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 543 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1321 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9649 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 369 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 366 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 595 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1531 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 10522 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 336 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 350 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 487 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1129 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7287 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 350 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 359 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 635 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1324 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9595 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 364 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 377 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 604 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1527 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 10549 cycles (8192 bytes)
tcrypt with "by8" AES CTR mode encryption optimization on a Haswell Desktop:
---------------------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 340 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 330 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 450 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1043 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 6597 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 339 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 352 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 539 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1153 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 8458 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 353 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 360 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 512 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1277 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 8745 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 348 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 335 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 451 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1030 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 6611 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 354 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 346 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 488 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1154 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 8390 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 357 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 362 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 515 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1284 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 8681 cycles (8192 bytes)
crypto: Incorporate feed back to AES CTR mode optimization patch
Specifically, the following:
a) alignment around main loop in aes_ctrby8_avx_x86_64.S
b) .rodata around data constants used in the assembely code.
c) the use of CONFIG_AVX in the glue code.
d) fix up white space.
e) informational message for "by8" AES CTR mode optimization
f) "by8" AES CTR mode optimization can be simply enabled
if the platform supports both AES and AVX features. The
optimization works superbly on Sandybridge as well.
Testing on Haswell shows no performance change since the last.
Testing on Sandybridge shows that the "by8" AES CTR mode optimization
greatly improves performance.
tcrypt log with "by4" AES CTR mode optimization on Sandybridge
--------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 383 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 408 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 707 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1864 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 12813 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 395 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 432 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 780 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 2132 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 15765 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 416 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 438 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 842 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 2383 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 16945 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 389 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 409 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 704 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1865 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 12783 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 409 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 434 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 792 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 2151 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 15804 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 421 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 444 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 840 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 2394 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 16928 cycles (8192 bytes)
tcrypt log with "by8" AES CTR mode optimization on Sandybridge
--------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 383 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 401 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 522 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1136 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7046 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 394 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 418 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 559 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1263 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9072 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 408 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 428 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 595 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1385 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 9224 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 390 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 402 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 530 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1135 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7079 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 414 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 417 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 572 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1312 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9073 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 415 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 454 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 598 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1407 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 9288 cycles (8192 bytes)
crypto: Fix redundant checks
a) Fix the redundant check for cpu_has_aes
b) Fix the key length check when invoking the CTR mode "by8"
encryptor/decryptor.
crypto: fix typo in AES ctr mode transform
Signed-off-by: Chandramouli Narayanan <mouli@linux.intel.com>
Reviewed-by: Mathias Krause <minipli@googlemail.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2014-06-10 20:22:47 +04:00
static void ( * aesni_ctr_enc_tfm ) ( struct crypto_aes_ctx * ctx , u8 * out ,
const u8 * in , unsigned int len , u8 * iv ) ;
2010-03-10 13:28:55 +03:00
asmlinkage void aesni_ctr_enc ( struct crypto_aes_ctx * ctx , u8 * out ,
const u8 * in , unsigned int len , u8 * iv ) ;
2009-01-18 08:28:34 +03:00
2013-04-08 22:51:16 +04:00
asmlinkage void aesni_xts_crypt8 ( struct crypto_aes_ctx * ctx , u8 * out ,
const u8 * in , bool enc , u8 * iv ) ;
2010-11-04 22:00:45 +03:00
/* asmlinkage void aesni_gcm_enc()
* void * ctx , AES Key schedule . Starts on a 16 byte boundary .
* u8 * out , Ciphertext output . Encrypt in - place is allowed .
* const u8 * in , Plaintext input
* unsigned long plaintext_len , Length of data in bytes for encryption .
* u8 * iv , Pre - counter block j0 : 4 byte salt ( from Security Association )
* concatenated with 8 byte Initialisation Vector ( from IPSec ESP
* Payload ) concatenated with 0x00000001 . 16 - byte aligned pointer .
* u8 * hash_subkey , the Hash sub key input . Data starts on a 16 - byte boundary .
* const u8 * aad , Additional Authentication Data ( AAD )
* unsigned long aad_len , Length of AAD in bytes . With RFC4106 this
* is going to be 8 or 12 bytes
* u8 * auth_tag , Authenticated Tag output .
* unsigned long auth_tag_len ) , Authenticated Tag Length in bytes .
* Valid values are 16 ( most likely ) , 12 or 8.
*/
asmlinkage void aesni_gcm_enc ( void * ctx , u8 * out ,
const u8 * in , unsigned long plaintext_len , u8 * iv ,
u8 * hash_subkey , const u8 * aad , unsigned long aad_len ,
u8 * auth_tag , unsigned long auth_tag_len ) ;
/* asmlinkage void aesni_gcm_dec()
* void * ctx , AES Key schedule . Starts on a 16 byte boundary .
* u8 * out , Plaintext output . Decrypt in - place is allowed .
* const u8 * in , Ciphertext input
* unsigned long ciphertext_len , Length of data in bytes for decryption .
* u8 * iv , Pre - counter block j0 : 4 byte salt ( from Security Association )
* concatenated with 8 byte Initialisation Vector ( from IPSec ESP
* Payload ) concatenated with 0x00000001 . 16 - byte aligned pointer .
* u8 * hash_subkey , the Hash sub key input . Data starts on a 16 - byte boundary .
* const u8 * aad , Additional Authentication Data ( AAD )
* unsigned long aad_len , Length of AAD in bytes . With RFC4106 this is going
* to be 8 or 12 bytes
* u8 * auth_tag , Authenticated Tag output .
* unsigned long auth_tag_len ) Authenticated Tag Length in bytes .
* Valid values are 16 ( most likely ) , 12 or 8.
*/
asmlinkage void aesni_gcm_dec ( void * ctx , u8 * out ,
const u8 * in , unsigned long ciphertext_len , u8 * iv ,
u8 * hash_subkey , const u8 * aad , unsigned long aad_len ,
u8 * auth_tag , unsigned long auth_tag_len ) ;
2013-12-12 02:28:41 +04:00
# ifdef CONFIG_AS_AVX
crypto: aes - AES CTR x86_64 "by8" AVX optimization
This patch introduces "by8" AES CTR mode AVX optimization inspired by
Intel Optimized IPSEC Cryptograhpic library. For additional information,
please see:
http://downloadcenter.intel.com/Detail_Desc.aspx?agr=Y&DwnldID=22972
The functions aes_ctr_enc_128_avx_by8(), aes_ctr_enc_192_avx_by8() and
aes_ctr_enc_256_avx_by8() are adapted from
Intel Optimized IPSEC Cryptographic library. When both AES and AVX features
are enabled in a platform, the glue code in AESNI module overrieds the
existing "by4" CTR mode en/decryption with the "by8"
AES CTR mode en/decryption.
On a Haswell desktop, with turbo disabled and all cpus running
at maximum frequency, the "by8" CTR mode optimization
shows better performance results across data & key sizes
as measured by tcrypt.
The average performance improvement of the "by8" version over the "by4"
version is as follows:
For 128 bit key and data sizes >= 256 bytes, there is a 10-16% improvement.
For 192 bit key and data sizes >= 256 bytes, there is a 20-22% improvement.
For 256 bit key and data sizes >= 256 bytes, there is a 20-25% improvement.
A typical run of tcrypt with AES CTR mode encryption of the "by4" and "by8"
optimization shows the following results:
tcrypt with "by4" AES CTR mode encryption optimization on a Haswell Desktop:
---------------------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 343 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 336 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 491 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1130 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7309 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 346 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 361 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 543 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1321 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9649 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 369 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 366 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 595 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1531 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 10522 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 336 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 350 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 487 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1129 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7287 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 350 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 359 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 635 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1324 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9595 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 364 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 377 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 604 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1527 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 10549 cycles (8192 bytes)
tcrypt with "by8" AES CTR mode encryption optimization on a Haswell Desktop:
---------------------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 340 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 330 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 450 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1043 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 6597 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 339 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 352 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 539 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1153 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 8458 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 353 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 360 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 512 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1277 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 8745 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 348 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 335 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 451 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1030 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 6611 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 354 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 346 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 488 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1154 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 8390 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 357 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 362 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 515 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1284 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 8681 cycles (8192 bytes)
crypto: Incorporate feed back to AES CTR mode optimization patch
Specifically, the following:
a) alignment around main loop in aes_ctrby8_avx_x86_64.S
b) .rodata around data constants used in the assembely code.
c) the use of CONFIG_AVX in the glue code.
d) fix up white space.
e) informational message for "by8" AES CTR mode optimization
f) "by8" AES CTR mode optimization can be simply enabled
if the platform supports both AES and AVX features. The
optimization works superbly on Sandybridge as well.
Testing on Haswell shows no performance change since the last.
Testing on Sandybridge shows that the "by8" AES CTR mode optimization
greatly improves performance.
tcrypt log with "by4" AES CTR mode optimization on Sandybridge
--------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 383 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 408 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 707 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1864 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 12813 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 395 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 432 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 780 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 2132 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 15765 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 416 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 438 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 842 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 2383 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 16945 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 389 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 409 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 704 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1865 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 12783 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 409 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 434 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 792 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 2151 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 15804 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 421 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 444 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 840 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 2394 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 16928 cycles (8192 bytes)
tcrypt log with "by8" AES CTR mode optimization on Sandybridge
--------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 383 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 401 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 522 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1136 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7046 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 394 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 418 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 559 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1263 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9072 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 408 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 428 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 595 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1385 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 9224 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 390 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 402 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 530 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1135 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7079 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 414 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 417 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 572 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1312 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9073 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 415 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 454 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 598 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1407 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 9288 cycles (8192 bytes)
crypto: Fix redundant checks
a) Fix the redundant check for cpu_has_aes
b) Fix the key length check when invoking the CTR mode "by8"
encryptor/decryptor.
crypto: fix typo in AES ctr mode transform
Signed-off-by: Chandramouli Narayanan <mouli@linux.intel.com>
Reviewed-by: Mathias Krause <minipli@googlemail.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2014-06-10 20:22:47 +04:00
asmlinkage void aes_ctr_enc_128_avx_by8 ( const u8 * in , u8 * iv ,
void * keys , u8 * out , unsigned int num_bytes ) ;
asmlinkage void aes_ctr_enc_192_avx_by8 ( const u8 * in , u8 * iv ,
void * keys , u8 * out , unsigned int num_bytes ) ;
asmlinkage void aes_ctr_enc_256_avx_by8 ( const u8 * in , u8 * iv ,
void * keys , u8 * out , unsigned int num_bytes ) ;
2013-12-12 02:28:41 +04:00
/*
* asmlinkage void aesni_gcm_precomp_avx_gen2 ( )
* gcm_data * my_ctx_data , context data
* u8 * hash_subkey , the Hash sub key input . Data starts on a 16 - byte boundary .
*/
asmlinkage void aesni_gcm_precomp_avx_gen2 ( void * my_ctx_data , u8 * hash_subkey ) ;
asmlinkage void aesni_gcm_enc_avx_gen2 ( void * ctx , u8 * out ,
const u8 * in , unsigned long plaintext_len , u8 * iv ,
const u8 * aad , unsigned long aad_len ,
u8 * auth_tag , unsigned long auth_tag_len ) ;
asmlinkage void aesni_gcm_dec_avx_gen2 ( void * ctx , u8 * out ,
const u8 * in , unsigned long ciphertext_len , u8 * iv ,
const u8 * aad , unsigned long aad_len ,
u8 * auth_tag , unsigned long auth_tag_len ) ;
static void aesni_gcm_enc_avx ( void * ctx , u8 * out ,
const u8 * in , unsigned long plaintext_len , u8 * iv ,
u8 * hash_subkey , const u8 * aad , unsigned long aad_len ,
u8 * auth_tag , unsigned long auth_tag_len )
{
2015-01-13 21:16:43 +03:00
struct crypto_aes_ctx * aes_ctx = ( struct crypto_aes_ctx * ) ctx ;
if ( ( plaintext_len < AVX_GEN2_OPTSIZE ) | | ( aes_ctx - > key_length ! = AES_KEYSIZE_128 ) ) {
2013-12-12 02:28:41 +04:00
aesni_gcm_enc ( ctx , out , in , plaintext_len , iv , hash_subkey , aad ,
aad_len , auth_tag , auth_tag_len ) ;
} else {
aesni_gcm_precomp_avx_gen2 ( ctx , hash_subkey ) ;
aesni_gcm_enc_avx_gen2 ( ctx , out , in , plaintext_len , iv , aad ,
aad_len , auth_tag , auth_tag_len ) ;
}
}
static void aesni_gcm_dec_avx ( void * ctx , u8 * out ,
const u8 * in , unsigned long ciphertext_len , u8 * iv ,
u8 * hash_subkey , const u8 * aad , unsigned long aad_len ,
u8 * auth_tag , unsigned long auth_tag_len )
{
2015-01-13 21:16:43 +03:00
struct crypto_aes_ctx * aes_ctx = ( struct crypto_aes_ctx * ) ctx ;
if ( ( ciphertext_len < AVX_GEN2_OPTSIZE ) | | ( aes_ctx - > key_length ! = AES_KEYSIZE_128 ) ) {
2013-12-12 02:28:41 +04:00
aesni_gcm_dec ( ctx , out , in , ciphertext_len , iv , hash_subkey , aad ,
aad_len , auth_tag , auth_tag_len ) ;
} else {
aesni_gcm_precomp_avx_gen2 ( ctx , hash_subkey ) ;
aesni_gcm_dec_avx_gen2 ( ctx , out , in , ciphertext_len , iv , aad ,
aad_len , auth_tag , auth_tag_len ) ;
}
}
# endif
# ifdef CONFIG_AS_AVX2
/*
* asmlinkage void aesni_gcm_precomp_avx_gen4 ( )
* gcm_data * my_ctx_data , context data
* u8 * hash_subkey , the Hash sub key input . Data starts on a 16 - byte boundary .
*/
asmlinkage void aesni_gcm_precomp_avx_gen4 ( void * my_ctx_data , u8 * hash_subkey ) ;
asmlinkage void aesni_gcm_enc_avx_gen4 ( void * ctx , u8 * out ,
const u8 * in , unsigned long plaintext_len , u8 * iv ,
const u8 * aad , unsigned long aad_len ,
u8 * auth_tag , unsigned long auth_tag_len ) ;
asmlinkage void aesni_gcm_dec_avx_gen4 ( void * ctx , u8 * out ,
const u8 * in , unsigned long ciphertext_len , u8 * iv ,
const u8 * aad , unsigned long aad_len ,
u8 * auth_tag , unsigned long auth_tag_len ) ;
static void aesni_gcm_enc_avx2 ( void * ctx , u8 * out ,
const u8 * in , unsigned long plaintext_len , u8 * iv ,
u8 * hash_subkey , const u8 * aad , unsigned long aad_len ,
u8 * auth_tag , unsigned long auth_tag_len )
{
2015-01-13 21:16:43 +03:00
struct crypto_aes_ctx * aes_ctx = ( struct crypto_aes_ctx * ) ctx ;
if ( ( plaintext_len < AVX_GEN2_OPTSIZE ) | | ( aes_ctx - > key_length ! = AES_KEYSIZE_128 ) ) {
2013-12-12 02:28:41 +04:00
aesni_gcm_enc ( ctx , out , in , plaintext_len , iv , hash_subkey , aad ,
aad_len , auth_tag , auth_tag_len ) ;
} else if ( plaintext_len < AVX_GEN4_OPTSIZE ) {
aesni_gcm_precomp_avx_gen2 ( ctx , hash_subkey ) ;
aesni_gcm_enc_avx_gen2 ( ctx , out , in , plaintext_len , iv , aad ,
aad_len , auth_tag , auth_tag_len ) ;
} else {
aesni_gcm_precomp_avx_gen4 ( ctx , hash_subkey ) ;
aesni_gcm_enc_avx_gen4 ( ctx , out , in , plaintext_len , iv , aad ,
aad_len , auth_tag , auth_tag_len ) ;
}
}
static void aesni_gcm_dec_avx2 ( void * ctx , u8 * out ,
const u8 * in , unsigned long ciphertext_len , u8 * iv ,
u8 * hash_subkey , const u8 * aad , unsigned long aad_len ,
u8 * auth_tag , unsigned long auth_tag_len )
{
2015-01-13 21:16:43 +03:00
struct crypto_aes_ctx * aes_ctx = ( struct crypto_aes_ctx * ) ctx ;
if ( ( ciphertext_len < AVX_GEN2_OPTSIZE ) | | ( aes_ctx - > key_length ! = AES_KEYSIZE_128 ) ) {
2013-12-12 02:28:41 +04:00
aesni_gcm_dec ( ctx , out , in , ciphertext_len , iv , hash_subkey ,
aad , aad_len , auth_tag , auth_tag_len ) ;
} else if ( ciphertext_len < AVX_GEN4_OPTSIZE ) {
aesni_gcm_precomp_avx_gen2 ( ctx , hash_subkey ) ;
aesni_gcm_dec_avx_gen2 ( ctx , out , in , ciphertext_len , iv , aad ,
aad_len , auth_tag , auth_tag_len ) ;
} else {
aesni_gcm_precomp_avx_gen4 ( ctx , hash_subkey ) ;
aesni_gcm_dec_avx_gen4 ( ctx , out , in , ciphertext_len , iv , aad ,
aad_len , auth_tag , auth_tag_len ) ;
}
}
# endif
static void ( * aesni_gcm_enc_tfm ) ( void * ctx , u8 * out ,
const u8 * in , unsigned long plaintext_len , u8 * iv ,
u8 * hash_subkey , const u8 * aad , unsigned long aad_len ,
u8 * auth_tag , unsigned long auth_tag_len ) ;
static void ( * aesni_gcm_dec_tfm ) ( void * ctx , u8 * out ,
const u8 * in , unsigned long ciphertext_len , u8 * iv ,
u8 * hash_subkey , const u8 * aad , unsigned long aad_len ,
u8 * auth_tag , unsigned long auth_tag_len ) ;
2010-11-04 22:00:45 +03:00
static inline struct
aesni_rfc4106_gcm_ctx * aesni_rfc4106_gcm_ctx_get ( struct crypto_aead * tfm )
{
2015-06-01 10:53:06 +03:00
unsigned long align = AESNI_ALIGN ;
if ( align < = crypto_tfm_ctx_alignment ( ) )
align = 1 ;
return PTR_ALIGN ( crypto_aead_ctx ( tfm ) , align ) ;
2010-11-04 22:00:45 +03:00
}
2010-11-29 03:35:39 +03:00
# endif
2010-11-04 22:00:45 +03:00
2009-01-18 08:28:34 +03:00
static inline struct crypto_aes_ctx * aes_ctx ( void * raw_ctx )
{
unsigned long addr = ( unsigned long ) raw_ctx ;
unsigned long align = AESNI_ALIGN ;
if ( align < = crypto_tfm_ctx_alignment ( ) )
align = 1 ;
return ( struct crypto_aes_ctx * ) ALIGN ( addr , align ) ;
}
static int aes_set_key_common ( struct crypto_tfm * tfm , void * raw_ctx ,
const u8 * in_key , unsigned int key_len )
{
struct crypto_aes_ctx * ctx = aes_ctx ( raw_ctx ) ;
u32 * flags = & tfm - > crt_flags ;
int err ;
if ( key_len ! = AES_KEYSIZE_128 & & key_len ! = AES_KEYSIZE_192 & &
key_len ! = AES_KEYSIZE_256 ) {
* flags | = CRYPTO_TFM_RES_BAD_KEY_LEN ;
return - EINVAL ;
}
2009-10-20 11:20:47 +04:00
if ( ! irq_fpu_usable ( ) )
2009-01-18 08:28:34 +03:00
err = crypto_aes_expand_key ( ctx , in_key , key_len ) ;
else {
kernel_fpu_begin ( ) ;
err = aesni_set_key ( ctx , in_key , key_len ) ;
kernel_fpu_end ( ) ;
}
return err ;
}
static int aes_set_key ( struct crypto_tfm * tfm , const u8 * in_key ,
unsigned int key_len )
{
return aes_set_key_common ( tfm , crypto_tfm_ctx ( tfm ) , in_key , key_len ) ;
}
static void aes_encrypt ( struct crypto_tfm * tfm , u8 * dst , const u8 * src )
{
struct crypto_aes_ctx * ctx = aes_ctx ( crypto_tfm_ctx ( tfm ) ) ;
2009-10-20 11:20:47 +04:00
if ( ! irq_fpu_usable ( ) )
2009-01-18 08:28:34 +03:00
crypto_aes_encrypt_x86 ( ctx , dst , src ) ;
else {
kernel_fpu_begin ( ) ;
aesni_enc ( ctx , dst , src ) ;
kernel_fpu_end ( ) ;
}
}
static void aes_decrypt ( struct crypto_tfm * tfm , u8 * dst , const u8 * src )
{
struct crypto_aes_ctx * ctx = aes_ctx ( crypto_tfm_ctx ( tfm ) ) ;
2009-10-20 11:20:47 +04:00
if ( ! irq_fpu_usable ( ) )
2009-01-18 08:28:34 +03:00
crypto_aes_decrypt_x86 ( ctx , dst , src ) ;
else {
kernel_fpu_begin ( ) ;
aesni_dec ( ctx , dst , src ) ;
kernel_fpu_end ( ) ;
}
}
2009-03-29 11:41:20 +04:00
static void __aes_encrypt ( struct crypto_tfm * tfm , u8 * dst , const u8 * src )
{
struct crypto_aes_ctx * ctx = aes_ctx ( crypto_tfm_ctx ( tfm ) ) ;
aesni_enc ( ctx , dst , src ) ;
}
static void __aes_decrypt ( struct crypto_tfm * tfm , u8 * dst , const u8 * src )
{
struct crypto_aes_ctx * ctx = aes_ctx ( crypto_tfm_ctx ( tfm ) ) ;
aesni_dec ( ctx , dst , src ) ;
}
2016-11-22 15:08:33 +03:00
static int aesni_skcipher_setkey ( struct crypto_skcipher * tfm , const u8 * key ,
unsigned int len )
{
return aes_set_key_common ( crypto_skcipher_tfm ( tfm ) ,
crypto_skcipher_ctx ( tfm ) , key , len ) ;
}
static int ecb_encrypt ( struct skcipher_request * req )
2009-01-18 08:28:34 +03:00
{
2016-11-22 15:08:33 +03:00
struct crypto_skcipher * tfm = crypto_skcipher_reqtfm ( req ) ;
struct crypto_aes_ctx * ctx = aes_ctx ( crypto_skcipher_ctx ( tfm ) ) ;
struct skcipher_walk walk ;
unsigned int nbytes ;
2009-01-18 08:28:34 +03:00
int err ;
2016-11-22 15:08:33 +03:00
err = skcipher_walk_virt ( & walk , req , true ) ;
2009-01-18 08:28:34 +03:00
kernel_fpu_begin ( ) ;
while ( ( nbytes = walk . nbytes ) ) {
aesni_ecb_enc ( ctx , walk . dst . virt . addr , walk . src . virt . addr ,
nbytes & AES_BLOCK_MASK ) ;
nbytes & = AES_BLOCK_SIZE - 1 ;
2016-11-22 15:08:33 +03:00
err = skcipher_walk_done ( & walk , nbytes ) ;
2009-01-18 08:28:34 +03:00
}
kernel_fpu_end ( ) ;
return err ;
}
2016-11-22 15:08:33 +03:00
static int ecb_decrypt ( struct skcipher_request * req )
2009-01-18 08:28:34 +03:00
{
2016-11-22 15:08:33 +03:00
struct crypto_skcipher * tfm = crypto_skcipher_reqtfm ( req ) ;
struct crypto_aes_ctx * ctx = aes_ctx ( crypto_skcipher_ctx ( tfm ) ) ;
struct skcipher_walk walk ;
unsigned int nbytes ;
2009-01-18 08:28:34 +03:00
int err ;
2016-11-22 15:08:33 +03:00
err = skcipher_walk_virt ( & walk , req , true ) ;
2009-01-18 08:28:34 +03:00
kernel_fpu_begin ( ) ;
while ( ( nbytes = walk . nbytes ) ) {
aesni_ecb_dec ( ctx , walk . dst . virt . addr , walk . src . virt . addr ,
nbytes & AES_BLOCK_MASK ) ;
nbytes & = AES_BLOCK_SIZE - 1 ;
2016-11-22 15:08:33 +03:00
err = skcipher_walk_done ( & walk , nbytes ) ;
2009-01-18 08:28:34 +03:00
}
kernel_fpu_end ( ) ;
return err ;
}
2016-11-22 15:08:33 +03:00
static int cbc_encrypt ( struct skcipher_request * req )
2009-01-18 08:28:34 +03:00
{
2016-11-22 15:08:33 +03:00
struct crypto_skcipher * tfm = crypto_skcipher_reqtfm ( req ) ;
struct crypto_aes_ctx * ctx = aes_ctx ( crypto_skcipher_ctx ( tfm ) ) ;
struct skcipher_walk walk ;
unsigned int nbytes ;
2009-01-18 08:28:34 +03:00
int err ;
2016-11-22 15:08:33 +03:00
err = skcipher_walk_virt ( & walk , req , true ) ;
2009-01-18 08:28:34 +03:00
kernel_fpu_begin ( ) ;
while ( ( nbytes = walk . nbytes ) ) {
aesni_cbc_enc ( ctx , walk . dst . virt . addr , walk . src . virt . addr ,
nbytes & AES_BLOCK_MASK , walk . iv ) ;
nbytes & = AES_BLOCK_SIZE - 1 ;
2016-11-22 15:08:33 +03:00
err = skcipher_walk_done ( & walk , nbytes ) ;
2009-01-18 08:28:34 +03:00
}
kernel_fpu_end ( ) ;
return err ;
}
2016-11-22 15:08:33 +03:00
static int cbc_decrypt ( struct skcipher_request * req )
2009-01-18 08:28:34 +03:00
{
2016-11-22 15:08:33 +03:00
struct crypto_skcipher * tfm = crypto_skcipher_reqtfm ( req ) ;
struct crypto_aes_ctx * ctx = aes_ctx ( crypto_skcipher_ctx ( tfm ) ) ;
struct skcipher_walk walk ;
unsigned int nbytes ;
2009-01-18 08:28:34 +03:00
int err ;
2016-11-22 15:08:33 +03:00
err = skcipher_walk_virt ( & walk , req , true ) ;
2009-01-18 08:28:34 +03:00
kernel_fpu_begin ( ) ;
while ( ( nbytes = walk . nbytes ) ) {
aesni_cbc_dec ( ctx , walk . dst . virt . addr , walk . src . virt . addr ,
nbytes & AES_BLOCK_MASK , walk . iv ) ;
nbytes & = AES_BLOCK_SIZE - 1 ;
2016-11-22 15:08:33 +03:00
err = skcipher_walk_done ( & walk , nbytes ) ;
2009-01-18 08:28:34 +03:00
}
kernel_fpu_end ( ) ;
return err ;
}
crypto: aesni-intel - Ported implementation to x86-32
The AES-NI instructions are also available in legacy mode so the 32-bit
architecture may profit from those, too.
To illustrate the performance gain here's a short summary of a dm-crypt
speed test on a Core i7 M620 running at 2.67GHz comparing both assembler
implementations:
x86: i568 aes-ni delta
ECB, 256 bit: 93.8 MB/s 123.3 MB/s +31.4%
CBC, 256 bit: 84.8 MB/s 262.3 MB/s +209.3%
LRW, 256 bit: 108.6 MB/s 222.1 MB/s +104.5%
XTS, 256 bit: 105.0 MB/s 205.5 MB/s +95.7%
Additionally, due to some minor optimizations, the 64-bit version also
got a minor performance gain as seen below:
x86-64: old impl. new impl. delta
ECB, 256 bit: 121.1 MB/s 123.0 MB/s +1.5%
CBC, 256 bit: 285.3 MB/s 290.8 MB/s +1.9%
LRW, 256 bit: 263.7 MB/s 265.3 MB/s +0.6%
XTS, 256 bit: 251.1 MB/s 255.3 MB/s +1.7%
Signed-off-by: Mathias Krause <minipli@googlemail.com>
Reviewed-by: Huang Ying <ying.huang@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2010-11-27 11:34:46 +03:00
# ifdef CONFIG_X86_64
2010-03-10 13:28:55 +03:00
static void ctr_crypt_final ( struct crypto_aes_ctx * ctx ,
2016-11-22 15:08:33 +03:00
struct skcipher_walk * walk )
2010-03-10 13:28:55 +03:00
{
u8 * ctrblk = walk - > iv ;
u8 keystream [ AES_BLOCK_SIZE ] ;
u8 * src = walk - > src . virt . addr ;
u8 * dst = walk - > dst . virt . addr ;
unsigned int nbytes = walk - > nbytes ;
aesni_enc ( ctx , keystream , ctrblk ) ;
crypto_xor ( keystream , src , nbytes ) ;
memcpy ( dst , keystream , nbytes ) ;
crypto_inc ( ctrblk , AES_BLOCK_SIZE ) ;
}
2014-09-29 00:24:01 +04:00
# ifdef CONFIG_AS_AVX
crypto: aes - AES CTR x86_64 "by8" AVX optimization
This patch introduces "by8" AES CTR mode AVX optimization inspired by
Intel Optimized IPSEC Cryptograhpic library. For additional information,
please see:
http://downloadcenter.intel.com/Detail_Desc.aspx?agr=Y&DwnldID=22972
The functions aes_ctr_enc_128_avx_by8(), aes_ctr_enc_192_avx_by8() and
aes_ctr_enc_256_avx_by8() are adapted from
Intel Optimized IPSEC Cryptographic library. When both AES and AVX features
are enabled in a platform, the glue code in AESNI module overrieds the
existing "by4" CTR mode en/decryption with the "by8"
AES CTR mode en/decryption.
On a Haswell desktop, with turbo disabled and all cpus running
at maximum frequency, the "by8" CTR mode optimization
shows better performance results across data & key sizes
as measured by tcrypt.
The average performance improvement of the "by8" version over the "by4"
version is as follows:
For 128 bit key and data sizes >= 256 bytes, there is a 10-16% improvement.
For 192 bit key and data sizes >= 256 bytes, there is a 20-22% improvement.
For 256 bit key and data sizes >= 256 bytes, there is a 20-25% improvement.
A typical run of tcrypt with AES CTR mode encryption of the "by4" and "by8"
optimization shows the following results:
tcrypt with "by4" AES CTR mode encryption optimization on a Haswell Desktop:
---------------------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 343 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 336 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 491 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1130 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7309 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 346 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 361 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 543 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1321 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9649 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 369 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 366 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 595 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1531 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 10522 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 336 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 350 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 487 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1129 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7287 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 350 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 359 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 635 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1324 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9595 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 364 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 377 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 604 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1527 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 10549 cycles (8192 bytes)
tcrypt with "by8" AES CTR mode encryption optimization on a Haswell Desktop:
---------------------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 340 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 330 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 450 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1043 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 6597 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 339 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 352 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 539 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1153 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 8458 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 353 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 360 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 512 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1277 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 8745 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 348 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 335 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 451 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1030 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 6611 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 354 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 346 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 488 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1154 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 8390 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 357 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 362 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 515 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1284 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 8681 cycles (8192 bytes)
crypto: Incorporate feed back to AES CTR mode optimization patch
Specifically, the following:
a) alignment around main loop in aes_ctrby8_avx_x86_64.S
b) .rodata around data constants used in the assembely code.
c) the use of CONFIG_AVX in the glue code.
d) fix up white space.
e) informational message for "by8" AES CTR mode optimization
f) "by8" AES CTR mode optimization can be simply enabled
if the platform supports both AES and AVX features. The
optimization works superbly on Sandybridge as well.
Testing on Haswell shows no performance change since the last.
Testing on Sandybridge shows that the "by8" AES CTR mode optimization
greatly improves performance.
tcrypt log with "by4" AES CTR mode optimization on Sandybridge
--------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 383 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 408 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 707 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1864 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 12813 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 395 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 432 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 780 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 2132 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 15765 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 416 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 438 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 842 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 2383 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 16945 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 389 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 409 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 704 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1865 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 12783 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 409 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 434 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 792 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 2151 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 15804 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 421 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 444 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 840 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 2394 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 16928 cycles (8192 bytes)
tcrypt log with "by8" AES CTR mode optimization on Sandybridge
--------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 383 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 401 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 522 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1136 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7046 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 394 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 418 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 559 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1263 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9072 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 408 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 428 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 595 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1385 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 9224 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 390 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 402 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 530 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1135 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7079 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 414 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 417 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 572 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1312 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9073 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 415 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 454 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 598 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1407 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 9288 cycles (8192 bytes)
crypto: Fix redundant checks
a) Fix the redundant check for cpu_has_aes
b) Fix the key length check when invoking the CTR mode "by8"
encryptor/decryptor.
crypto: fix typo in AES ctr mode transform
Signed-off-by: Chandramouli Narayanan <mouli@linux.intel.com>
Reviewed-by: Mathias Krause <minipli@googlemail.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2014-06-10 20:22:47 +04:00
static void aesni_ctr_enc_avx_tfm ( struct crypto_aes_ctx * ctx , u8 * out ,
const u8 * in , unsigned int len , u8 * iv )
{
/*
* based on key length , override with the by8 version
* of ctr mode encryption / decryption for improved performance
* aes_set_key_common ( ) ensures that key length is one of
* { 128 , 192 , 256 }
*/
if ( ctx - > key_length = = AES_KEYSIZE_128 )
aes_ctr_enc_128_avx_by8 ( in , iv , ( void * ) ctx , out , len ) ;
else if ( ctx - > key_length = = AES_KEYSIZE_192 )
aes_ctr_enc_192_avx_by8 ( in , iv , ( void * ) ctx , out , len ) ;
else
aes_ctr_enc_256_avx_by8 ( in , iv , ( void * ) ctx , out , len ) ;
}
# endif
2016-11-22 15:08:33 +03:00
static int ctr_crypt ( struct skcipher_request * req )
2010-03-10 13:28:55 +03:00
{
2016-11-22 15:08:33 +03:00
struct crypto_skcipher * tfm = crypto_skcipher_reqtfm ( req ) ;
struct crypto_aes_ctx * ctx = aes_ctx ( crypto_skcipher_ctx ( tfm ) ) ;
struct skcipher_walk walk ;
unsigned int nbytes ;
2010-03-10 13:28:55 +03:00
int err ;
2016-11-22 15:08:33 +03:00
err = skcipher_walk_virt ( & walk , req , true ) ;
2010-03-10 13:28:55 +03:00
kernel_fpu_begin ( ) ;
while ( ( nbytes = walk . nbytes ) > = AES_BLOCK_SIZE ) {
crypto: aes - AES CTR x86_64 "by8" AVX optimization
This patch introduces "by8" AES CTR mode AVX optimization inspired by
Intel Optimized IPSEC Cryptograhpic library. For additional information,
please see:
http://downloadcenter.intel.com/Detail_Desc.aspx?agr=Y&DwnldID=22972
The functions aes_ctr_enc_128_avx_by8(), aes_ctr_enc_192_avx_by8() and
aes_ctr_enc_256_avx_by8() are adapted from
Intel Optimized IPSEC Cryptographic library. When both AES and AVX features
are enabled in a platform, the glue code in AESNI module overrieds the
existing "by4" CTR mode en/decryption with the "by8"
AES CTR mode en/decryption.
On a Haswell desktop, with turbo disabled and all cpus running
at maximum frequency, the "by8" CTR mode optimization
shows better performance results across data & key sizes
as measured by tcrypt.
The average performance improvement of the "by8" version over the "by4"
version is as follows:
For 128 bit key and data sizes >= 256 bytes, there is a 10-16% improvement.
For 192 bit key and data sizes >= 256 bytes, there is a 20-22% improvement.
For 256 bit key and data sizes >= 256 bytes, there is a 20-25% improvement.
A typical run of tcrypt with AES CTR mode encryption of the "by4" and "by8"
optimization shows the following results:
tcrypt with "by4" AES CTR mode encryption optimization on a Haswell Desktop:
---------------------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 343 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 336 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 491 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1130 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7309 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 346 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 361 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 543 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1321 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9649 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 369 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 366 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 595 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1531 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 10522 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 336 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 350 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 487 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1129 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7287 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 350 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 359 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 635 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1324 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9595 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 364 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 377 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 604 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1527 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 10549 cycles (8192 bytes)
tcrypt with "by8" AES CTR mode encryption optimization on a Haswell Desktop:
---------------------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 340 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 330 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 450 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1043 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 6597 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 339 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 352 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 539 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1153 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 8458 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 353 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 360 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 512 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1277 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 8745 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 348 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 335 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 451 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1030 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 6611 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 354 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 346 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 488 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1154 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 8390 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 357 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 362 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 515 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1284 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 8681 cycles (8192 bytes)
crypto: Incorporate feed back to AES CTR mode optimization patch
Specifically, the following:
a) alignment around main loop in aes_ctrby8_avx_x86_64.S
b) .rodata around data constants used in the assembely code.
c) the use of CONFIG_AVX in the glue code.
d) fix up white space.
e) informational message for "by8" AES CTR mode optimization
f) "by8" AES CTR mode optimization can be simply enabled
if the platform supports both AES and AVX features. The
optimization works superbly on Sandybridge as well.
Testing on Haswell shows no performance change since the last.
Testing on Sandybridge shows that the "by8" AES CTR mode optimization
greatly improves performance.
tcrypt log with "by4" AES CTR mode optimization on Sandybridge
--------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 383 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 408 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 707 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1864 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 12813 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 395 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 432 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 780 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 2132 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 15765 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 416 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 438 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 842 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 2383 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 16945 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 389 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 409 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 704 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1865 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 12783 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 409 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 434 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 792 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 2151 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 15804 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 421 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 444 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 840 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 2394 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 16928 cycles (8192 bytes)
tcrypt log with "by8" AES CTR mode optimization on Sandybridge
--------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 383 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 401 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 522 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1136 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7046 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 394 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 418 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 559 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1263 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9072 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 408 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 428 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 595 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1385 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 9224 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 390 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 402 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 530 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1135 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7079 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 414 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 417 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 572 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1312 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9073 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 415 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 454 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 598 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1407 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 9288 cycles (8192 bytes)
crypto: Fix redundant checks
a) Fix the redundant check for cpu_has_aes
b) Fix the key length check when invoking the CTR mode "by8"
encryptor/decryptor.
crypto: fix typo in AES ctr mode transform
Signed-off-by: Chandramouli Narayanan <mouli@linux.intel.com>
Reviewed-by: Mathias Krause <minipli@googlemail.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2014-06-10 20:22:47 +04:00
aesni_ctr_enc_tfm ( ctx , walk . dst . virt . addr , walk . src . virt . addr ,
2015-01-13 21:16:43 +03:00
nbytes & AES_BLOCK_MASK , walk . iv ) ;
2010-03-10 13:28:55 +03:00
nbytes & = AES_BLOCK_SIZE - 1 ;
2016-11-22 15:08:33 +03:00
err = skcipher_walk_done ( & walk , nbytes ) ;
2010-03-10 13:28:55 +03:00
}
if ( walk . nbytes ) {
ctr_crypt_final ( ctx , & walk ) ;
2016-11-22 15:08:33 +03:00
err = skcipher_walk_done ( & walk , 0 ) ;
2010-03-10 13:28:55 +03:00
}
kernel_fpu_end ( ) ;
return err ;
}
2012-07-22 19:18:37 +04:00
2016-11-22 15:08:33 +03:00
static int xts_aesni_setkey ( struct crypto_skcipher * tfm , const u8 * key ,
2012-07-22 19:18:37 +04:00
unsigned int keylen )
{
2016-11-22 15:08:33 +03:00
struct aesni_xts_ctx * ctx = crypto_skcipher_ctx ( tfm ) ;
2012-07-22 19:18:37 +04:00
int err ;
2016-11-22 15:08:33 +03:00
err = xts_verify_key ( tfm , key , keylen ) ;
2012-07-22 19:18:37 +04:00
if ( err )
return err ;
2016-11-22 15:08:33 +03:00
keylen / = 2 ;
2012-07-22 19:18:37 +04:00
/* first half of xts-key is for crypt */
2016-11-22 15:08:33 +03:00
err = aes_set_key_common ( crypto_skcipher_tfm ( tfm ) , ctx - > raw_crypt_ctx ,
key , keylen ) ;
2012-07-22 19:18:37 +04:00
if ( err )
return err ;
/* second half of xts-key is for tweak */
2016-11-22 15:08:33 +03:00
return aes_set_key_common ( crypto_skcipher_tfm ( tfm ) , ctx - > raw_tweak_ctx ,
key + keylen , keylen ) ;
2012-07-22 19:18:37 +04:00
}
2012-10-19 00:24:57 +04:00
static void aesni_xts_tweak ( void * ctx , u8 * out , const u8 * in )
{
aesni_enc ( ctx , out , in ) ;
}
2013-04-08 22:51:16 +04:00
static void aesni_xts_enc ( void * ctx , u128 * dst , const u128 * src , le128 * iv )
{
glue_xts_crypt_128bit_one ( ctx , dst , src , iv , GLUE_FUNC_CAST ( aesni_enc ) ) ;
}
static void aesni_xts_dec ( void * ctx , u128 * dst , const u128 * src , le128 * iv )
{
glue_xts_crypt_128bit_one ( ctx , dst , src , iv , GLUE_FUNC_CAST ( aesni_dec ) ) ;
}
static void aesni_xts_enc8 ( void * ctx , u128 * dst , const u128 * src , le128 * iv )
{
aesni_xts_crypt8 ( ctx , ( u8 * ) dst , ( const u8 * ) src , true , ( u8 * ) iv ) ;
}
static void aesni_xts_dec8 ( void * ctx , u128 * dst , const u128 * src , le128 * iv )
{
aesni_xts_crypt8 ( ctx , ( u8 * ) dst , ( const u8 * ) src , false , ( u8 * ) iv ) ;
}
static const struct common_glue_ctx aesni_enc_xts = {
. num_funcs = 2 ,
. fpu_blocks_limit = 1 ,
. funcs = { {
. num_blocks = 8 ,
. fn_u = { . xts = GLUE_XTS_FUNC_CAST ( aesni_xts_enc8 ) }
} , {
. num_blocks = 1 ,
. fn_u = { . xts = GLUE_XTS_FUNC_CAST ( aesni_xts_enc ) }
} }
} ;
static const struct common_glue_ctx aesni_dec_xts = {
. num_funcs = 2 ,
. fpu_blocks_limit = 1 ,
. funcs = { {
. num_blocks = 8 ,
. fn_u = { . xts = GLUE_XTS_FUNC_CAST ( aesni_xts_dec8 ) }
} , {
. num_blocks = 1 ,
. fn_u = { . xts = GLUE_XTS_FUNC_CAST ( aesni_xts_dec ) }
} }
} ;
2016-11-22 15:08:33 +03:00
static int xts_encrypt ( struct skcipher_request * req )
2013-04-08 22:51:16 +04:00
{
2016-11-22 15:08:33 +03:00
struct crypto_skcipher * tfm = crypto_skcipher_reqtfm ( req ) ;
struct aesni_xts_ctx * ctx = crypto_skcipher_ctx ( tfm ) ;
2013-04-08 22:51:16 +04:00
2016-11-22 15:08:33 +03:00
return glue_xts_req_128bit ( & aesni_enc_xts , req ,
XTS_TWEAK_CAST ( aesni_xts_tweak ) ,
aes_ctx ( ctx - > raw_tweak_ctx ) ,
aes_ctx ( ctx - > raw_crypt_ctx ) ) ;
2013-04-08 22:51:16 +04:00
}
2016-11-22 15:08:33 +03:00
static int xts_decrypt ( struct skcipher_request * req )
2013-04-08 22:51:16 +04:00
{
2016-11-22 15:08:33 +03:00
struct crypto_skcipher * tfm = crypto_skcipher_reqtfm ( req ) ;
struct aesni_xts_ctx * ctx = crypto_skcipher_ctx ( tfm ) ;
2012-07-22 19:18:37 +04:00
2016-11-22 15:08:33 +03:00
return glue_xts_req_128bit ( & aesni_dec_xts , req ,
XTS_TWEAK_CAST ( aesni_xts_tweak ) ,
aes_ctx ( ctx - > raw_tweak_ctx ) ,
aes_ctx ( ctx - > raw_crypt_ctx ) ) ;
2009-03-29 11:41:20 +04:00
}
2015-05-28 17:08:03 +03:00
static int rfc4106_init ( struct crypto_aead * aead )
2010-11-04 22:00:45 +03:00
{
struct cryptd_aead * cryptd_tfm ;
2015-05-28 17:08:03 +03:00
struct cryptd_aead * * ctx = crypto_aead_ctx ( aead ) ;
2015-03-30 22:58:17 +03:00
cryptd_tfm = cryptd_alloc_aead ( " __driver-gcm-aes-aesni " ,
CRYPTO_ALG_INTERNAL ,
CRYPTO_ALG_INTERNAL ) ;
2010-11-04 22:00:45 +03:00
if ( IS_ERR ( cryptd_tfm ) )
return PTR_ERR ( cryptd_tfm ) ;
2011-03-13 11:56:17 +03:00
2015-05-28 17:08:03 +03:00
* ctx = cryptd_tfm ;
2015-07-09 02:17:28 +03:00
crypto_aead_set_reqsize ( aead , crypto_aead_reqsize ( & cryptd_tfm - > base ) ) ;
2010-11-04 22:00:45 +03:00
return 0 ;
}
2015-05-28 17:08:03 +03:00
static void rfc4106_exit ( struct crypto_aead * aead )
2010-11-04 22:00:45 +03:00
{
2015-05-28 17:08:03 +03:00
struct cryptd_aead * * ctx = crypto_aead_ctx ( aead ) ;
cryptd_free_aead ( * ctx ) ;
2010-11-04 22:00:45 +03:00
}
static int
rfc4106_set_hash_subkey ( u8 * hash_subkey , const u8 * key , unsigned int key_len )
{
2016-06-29 13:03:59 +03:00
struct crypto_cipher * tfm ;
int ret ;
2010-11-04 22:00:45 +03:00
2016-06-29 13:03:59 +03:00
tfm = crypto_alloc_cipher ( " aes " , 0 , 0 ) ;
if ( IS_ERR ( tfm ) )
return PTR_ERR ( tfm ) ;
2010-11-04 22:00:45 +03:00
2016-06-29 13:03:59 +03:00
ret = crypto_cipher_setkey ( tfm , key , key_len ) ;
2011-01-23 10:56:36 +03:00
if ( ret )
2016-06-29 13:03:59 +03:00
goto out_free_cipher ;
2010-11-04 22:00:45 +03:00
/* Clear the data in the hash sub key container to zero.*/
/* We want to cipher all zeros to create the hash sub key. */
memset ( hash_subkey , 0 , RFC4106_HASH_SUBKEY_SIZE ) ;
2016-06-29 13:03:59 +03:00
crypto_cipher_encrypt_one ( tfm , hash_subkey , hash_subkey ) ;
out_free_cipher :
crypto_free_cipher ( tfm ) ;
2010-11-04 22:00:45 +03:00
return ret ;
}
2015-02-06 21:25:20 +03:00
static int common_rfc4106_set_key ( struct crypto_aead * aead , const u8 * key ,
unsigned int key_len )
2010-11-04 22:00:45 +03:00
{
2015-02-06 21:25:20 +03:00
struct aesni_rfc4106_gcm_ctx * ctx = aesni_rfc4106_gcm_ctx_get ( aead ) ;
2010-11-04 22:00:45 +03:00
if ( key_len < 4 ) {
2015-06-01 10:53:06 +03:00
crypto_aead_set_flags ( aead , CRYPTO_TFM_RES_BAD_KEY_LEN ) ;
2010-11-04 22:00:45 +03:00
return - EINVAL ;
}
/*Account for 4 byte nonce at the end.*/
key_len - = 4 ;
memcpy ( ctx - > nonce , key + key_len , sizeof ( ctx - > nonce ) ) ;
2015-06-01 10:53:06 +03:00
return aes_set_key_common ( crypto_aead_tfm ( aead ) ,
& ctx - > aes_key_expanded , key , key_len ) ? :
rfc4106_set_hash_subkey ( ctx - > hash_subkey , key , key_len ) ;
2010-11-04 22:00:45 +03:00
}
2015-02-06 21:25:20 +03:00
static int rfc4106_set_key ( struct crypto_aead * parent , const u8 * key ,
unsigned int key_len )
2010-11-04 22:00:45 +03:00
{
2015-05-28 17:08:03 +03:00
struct cryptd_aead * * ctx = crypto_aead_ctx ( parent ) ;
struct cryptd_aead * cryptd_tfm = * ctx ;
2010-11-04 22:00:45 +03:00
2015-05-28 17:08:03 +03:00
return crypto_aead_setkey ( & cryptd_tfm - > base , key , key_len ) ;
2015-02-06 21:25:20 +03:00
}
static int common_rfc4106_set_authsize ( struct crypto_aead * aead ,
unsigned int authsize )
{
2010-11-04 22:00:45 +03:00
switch ( authsize ) {
case 8 :
case 12 :
case 16 :
break ;
default :
return - EINVAL ;
}
2015-06-01 10:53:06 +03:00
2010-11-04 22:00:45 +03:00
return 0 ;
}
2015-02-06 21:25:20 +03:00
/* This is the Integrity Check Value (aka the authentication tag length and can
* be 8 , 12 or 16 bytes long . */
static int rfc4106_set_authsize ( struct crypto_aead * parent ,
unsigned int authsize )
2010-11-04 22:00:45 +03:00
{
2015-05-28 17:08:03 +03:00
struct cryptd_aead * * ctx = crypto_aead_ctx ( parent ) ;
struct cryptd_aead * cryptd_tfm = * ctx ;
2010-11-04 22:00:45 +03:00
2015-05-28 17:08:03 +03:00
return crypto_aead_setauthsize ( & cryptd_tfm - > base , authsize ) ;
2010-11-04 22:00:45 +03:00
}
2015-06-01 10:53:06 +03:00
static int helper_rfc4106_encrypt ( struct aead_request * req )
2010-11-04 22:00:45 +03:00
{
u8 one_entry_in_sg = 0 ;
u8 * src , * dst , * assoc ;
__be32 counter = cpu_to_be32 ( 1 ) ;
struct crypto_aead * tfm = crypto_aead_reqtfm ( req ) ;
struct aesni_rfc4106_gcm_ctx * ctx = aesni_rfc4106_gcm_ctx_get ( tfm ) ;
void * aes_ctx = & ( ctx - > aes_key_expanded ) ;
unsigned long auth_tag_len = crypto_aead_authsize ( tfm ) ;
2015-06-01 10:53:06 +03:00
u8 iv [ 16 ] __attribute__ ( ( __aligned__ ( AESNI_ALIGN ) ) ) ;
2010-11-04 22:00:45 +03:00
struct scatter_walk src_sg_walk ;
2016-11-10 19:44:51 +03:00
struct scatter_walk dst_sg_walk = { } ;
2010-11-04 22:00:45 +03:00
unsigned int i ;
/* Assuming we are supporting rfc4106 64-bit extended */
/* sequence numbers We need to have the AAD length equal */
2015-07-09 02:17:28 +03:00
/* to 16 or 20 bytes */
if ( unlikely ( req - > assoclen ! = 16 & & req - > assoclen ! = 20 ) )
2010-11-04 22:00:45 +03:00
return - EINVAL ;
2015-01-13 21:16:43 +03:00
2010-11-04 22:00:45 +03:00
/* IV below built */
for ( i = 0 ; i < 4 ; i + + )
* ( iv + i ) = ctx - > nonce [ i ] ;
for ( i = 0 ; i < 8 ; i + + )
* ( iv + 4 + i ) = req - > iv [ i ] ;
* ( ( __be32 * ) ( iv + 12 ) ) = counter ;
2015-06-01 10:53:06 +03:00
if ( sg_is_last ( req - > src ) & &
2016-12-13 17:32:06 +03:00
( ! PageHighMem ( sg_page ( req - > src ) ) | |
req - > src - > offset + req - > src - > length < = PAGE_SIZE ) & &
2015-06-01 10:53:06 +03:00
sg_is_last ( req - > dst ) & &
2016-12-13 17:32:06 +03:00
( ! PageHighMem ( sg_page ( req - > dst ) ) | |
req - > dst - > offset + req - > dst - > length < = PAGE_SIZE ) ) {
2010-11-04 22:00:45 +03:00
one_entry_in_sg = 1 ;
scatterwalk_start ( & src_sg_walk , req - > src ) ;
2015-06-01 10:53:06 +03:00
assoc = scatterwalk_map ( & src_sg_walk ) ;
src = assoc + req - > assoclen ;
2010-11-04 22:00:45 +03:00
dst = src ;
if ( unlikely ( req - > src ! = req - > dst ) ) {
scatterwalk_start ( & dst_sg_walk , req - > dst ) ;
2015-06-01 10:53:06 +03:00
dst = scatterwalk_map ( & dst_sg_walk ) + req - > assoclen ;
2010-11-04 22:00:45 +03:00
}
} else {
/* Allocate memory for src, dst, assoc */
2015-06-01 10:53:06 +03:00
assoc = kmalloc ( req - > cryptlen + auth_tag_len + req - > assoclen ,
2010-11-04 22:00:45 +03:00
GFP_ATOMIC ) ;
2015-06-01 10:53:06 +03:00
if ( unlikely ( ! assoc ) )
2010-11-04 22:00:45 +03:00
return - ENOMEM ;
2015-06-01 10:53:06 +03:00
scatterwalk_map_and_copy ( assoc , req - > src , 0 ,
req - > assoclen + req - > cryptlen , 0 ) ;
src = assoc + req - > assoclen ;
2010-11-04 22:00:45 +03:00
dst = src ;
}
2015-06-01 10:53:06 +03:00
kernel_fpu_begin ( ) ;
2015-07-09 02:17:28 +03:00
aesni_gcm_enc_tfm ( aes_ctx , dst , src , req - > cryptlen , iv ,
ctx - > hash_subkey , assoc , req - > assoclen - 8 ,
dst + req - > cryptlen , auth_tag_len ) ;
2015-06-01 10:53:06 +03:00
kernel_fpu_end ( ) ;
2010-11-04 22:00:45 +03:00
/* The authTag (aka the Integrity Check Value) needs to be written
* back to the packet . */
if ( one_entry_in_sg ) {
if ( unlikely ( req - > src ! = req - > dst ) ) {
2015-06-01 10:53:06 +03:00
scatterwalk_unmap ( dst - req - > assoclen ) ;
scatterwalk_advance ( & dst_sg_walk , req - > dst - > length ) ;
scatterwalk_done ( & dst_sg_walk , 1 , 0 ) ;
2010-11-04 22:00:45 +03:00
}
2011-11-25 19:14:17 +04:00
scatterwalk_unmap ( assoc ) ;
2015-06-01 10:53:06 +03:00
scatterwalk_advance ( & src_sg_walk , req - > src - > length ) ;
scatterwalk_done ( & src_sg_walk , req - > src = = req - > dst , 0 ) ;
2010-11-04 22:00:45 +03:00
} else {
2015-06-01 10:53:06 +03:00
scatterwalk_map_and_copy ( dst , req - > dst , req - > assoclen ,
req - > cryptlen + auth_tag_len , 1 ) ;
kfree ( assoc ) ;
2010-11-04 22:00:45 +03:00
}
return 0 ;
}
2015-06-01 10:53:06 +03:00
static int helper_rfc4106_decrypt ( struct aead_request * req )
2010-11-04 22:00:45 +03:00
{
u8 one_entry_in_sg = 0 ;
u8 * src , * dst , * assoc ;
unsigned long tempCipherLen = 0 ;
__be32 counter = cpu_to_be32 ( 1 ) ;
int retval = 0 ;
struct crypto_aead * tfm = crypto_aead_reqtfm ( req ) ;
struct aesni_rfc4106_gcm_ctx * ctx = aesni_rfc4106_gcm_ctx_get ( tfm ) ;
void * aes_ctx = & ( ctx - > aes_key_expanded ) ;
unsigned long auth_tag_len = crypto_aead_authsize ( tfm ) ;
2015-06-01 10:53:06 +03:00
u8 iv [ 16 ] __attribute__ ( ( __aligned__ ( AESNI_ALIGN ) ) ) ;
u8 authTag [ 16 ] ;
2010-11-04 22:00:45 +03:00
struct scatter_walk src_sg_walk ;
2016-11-10 19:44:51 +03:00
struct scatter_walk dst_sg_walk = { } ;
2010-11-04 22:00:45 +03:00
unsigned int i ;
2015-07-09 02:17:28 +03:00
if ( unlikely ( req - > assoclen ! = 16 & & req - > assoclen ! = 20 ) )
2010-11-04 22:00:45 +03:00
return - EINVAL ;
2015-01-13 21:16:43 +03:00
2010-11-04 22:00:45 +03:00
/* Assuming we are supporting rfc4106 64-bit extended */
/* sequence numbers We need to have the AAD length */
2015-07-09 02:17:28 +03:00
/* equal to 16 or 20 bytes */
2010-11-04 22:00:45 +03:00
tempCipherLen = ( unsigned long ) ( req - > cryptlen - auth_tag_len ) ;
/* IV below built */
for ( i = 0 ; i < 4 ; i + + )
* ( iv + i ) = ctx - > nonce [ i ] ;
for ( i = 0 ; i < 8 ; i + + )
* ( iv + 4 + i ) = req - > iv [ i ] ;
* ( ( __be32 * ) ( iv + 12 ) ) = counter ;
2015-06-01 10:53:06 +03:00
if ( sg_is_last ( req - > src ) & &
2016-12-13 17:32:06 +03:00
( ! PageHighMem ( sg_page ( req - > src ) ) | |
req - > src - > offset + req - > src - > length < = PAGE_SIZE ) & &
2015-06-01 10:53:06 +03:00
sg_is_last ( req - > dst ) & &
2016-12-13 17:32:06 +03:00
( ! PageHighMem ( sg_page ( req - > dst ) ) | |
req - > dst - > offset + req - > dst - > length < = PAGE_SIZE ) ) {
2010-11-04 22:00:45 +03:00
one_entry_in_sg = 1 ;
scatterwalk_start ( & src_sg_walk , req - > src ) ;
2015-06-01 10:53:06 +03:00
assoc = scatterwalk_map ( & src_sg_walk ) ;
src = assoc + req - > assoclen ;
2010-11-04 22:00:45 +03:00
dst = src ;
if ( unlikely ( req - > src ! = req - > dst ) ) {
scatterwalk_start ( & dst_sg_walk , req - > dst ) ;
2015-06-01 10:53:06 +03:00
dst = scatterwalk_map ( & dst_sg_walk ) + req - > assoclen ;
2010-11-04 22:00:45 +03:00
}
} else {
/* Allocate memory for src, dst, assoc */
2015-06-01 10:53:06 +03:00
assoc = kmalloc ( req - > cryptlen + req - > assoclen , GFP_ATOMIC ) ;
if ( ! assoc )
2010-11-04 22:00:45 +03:00
return - ENOMEM ;
2015-06-01 10:53:06 +03:00
scatterwalk_map_and_copy ( assoc , req - > src , 0 ,
req - > assoclen + req - > cryptlen , 0 ) ;
src = assoc + req - > assoclen ;
2010-11-04 22:00:45 +03:00
dst = src ;
}
2015-06-01 10:53:06 +03:00
kernel_fpu_begin ( ) ;
2013-12-12 02:28:41 +04:00
aesni_gcm_dec_tfm ( aes_ctx , dst , src , tempCipherLen , iv ,
2015-07-09 02:17:28 +03:00
ctx - > hash_subkey , assoc , req - > assoclen - 8 ,
authTag , auth_tag_len ) ;
2015-06-01 10:53:06 +03:00
kernel_fpu_end ( ) ;
2010-11-04 22:00:45 +03:00
/* Compare generated tag with passed in tag. */
2013-12-11 14:28:59 +04:00
retval = crypto_memneq ( src + tempCipherLen , authTag , auth_tag_len ) ?
2010-11-04 22:00:45 +03:00
- EBADMSG : 0 ;
if ( one_entry_in_sg ) {
if ( unlikely ( req - > src ! = req - > dst ) ) {
2015-06-01 10:53:06 +03:00
scatterwalk_unmap ( dst - req - > assoclen ) ;
scatterwalk_advance ( & dst_sg_walk , req - > dst - > length ) ;
scatterwalk_done ( & dst_sg_walk , 1 , 0 ) ;
2010-11-04 22:00:45 +03:00
}
2011-11-25 19:14:17 +04:00
scatterwalk_unmap ( assoc ) ;
2015-06-01 10:53:06 +03:00
scatterwalk_advance ( & src_sg_walk , req - > src - > length ) ;
scatterwalk_done ( & src_sg_walk , req - > src = = req - > dst , 0 ) ;
2010-11-04 22:00:45 +03:00
} else {
2015-06-01 10:53:06 +03:00
scatterwalk_map_and_copy ( dst , req - > dst , req - > assoclen ,
tempCipherLen , 1 ) ;
kfree ( assoc ) ;
2010-11-04 22:00:45 +03:00
}
return retval ;
}
2015-02-06 21:25:20 +03:00
static int rfc4106_encrypt ( struct aead_request * req )
{
struct crypto_aead * tfm = crypto_aead_reqtfm ( req ) ;
2015-05-28 17:08:03 +03:00
struct cryptd_aead * * ctx = crypto_aead_ctx ( tfm ) ;
struct cryptd_aead * cryptd_tfm = * ctx ;
2015-02-06 21:25:20 +03:00
2016-06-21 11:55:14 +03:00
tfm = & cryptd_tfm - > base ;
if ( irq_fpu_usable ( ) & & ( ! in_atomic ( ) | |
! cryptd_aead_queued ( cryptd_tfm ) ) )
tfm = cryptd_aead_child ( cryptd_tfm ) ;
aead_request_set_tfm ( req , tfm ) ;
2015-02-06 21:25:20 +03:00
2015-07-09 02:17:28 +03:00
return crypto_aead_encrypt ( req ) ;
2015-02-06 21:25:20 +03:00
}
static int rfc4106_decrypt ( struct aead_request * req )
{
struct crypto_aead * tfm = crypto_aead_reqtfm ( req ) ;
2015-05-28 17:08:03 +03:00
struct cryptd_aead * * ctx = crypto_aead_ctx ( tfm ) ;
struct cryptd_aead * cryptd_tfm = * ctx ;
2015-02-06 21:25:20 +03:00
2016-06-21 11:55:14 +03:00
tfm = & cryptd_tfm - > base ;
if ( irq_fpu_usable ( ) & & ( ! in_atomic ( ) | |
! cryptd_aead_queued ( cryptd_tfm ) ) )
tfm = cryptd_aead_child ( cryptd_tfm ) ;
aead_request_set_tfm ( req , tfm ) ;
2015-02-06 21:25:20 +03:00
2015-07-09 02:17:28 +03:00
return crypto_aead_decrypt ( req ) ;
2015-02-06 21:25:20 +03:00
}
2012-05-11 17:00:48 +04:00
# endif
2010-11-04 22:00:45 +03:00
2012-05-11 17:00:48 +04:00
static struct crypto_alg aesni_algs [ ] = { {
. cra_name = " aes " ,
. cra_driver_name = " aes-aesni " ,
. cra_priority = 300 ,
. cra_flags = CRYPTO_ALG_TYPE_CIPHER ,
. cra_blocksize = AES_BLOCK_SIZE ,
2016-11-22 15:08:33 +03:00
. cra_ctxsize = CRYPTO_AES_CTX_SIZE ,
2012-05-11 17:00:48 +04:00
. cra_module = THIS_MODULE ,
. cra_u = {
. cipher = {
. cia_min_keysize = AES_MIN_KEY_SIZE ,
. cia_max_keysize = AES_MAX_KEY_SIZE ,
. cia_setkey = aes_set_key ,
. cia_encrypt = aes_encrypt ,
. cia_decrypt = aes_decrypt
}
}
} , {
2016-11-22 15:08:33 +03:00
. cra_name = " __aes " ,
. cra_driver_name = " __aes-aesni " ,
. cra_priority = 300 ,
2015-03-30 22:58:17 +03:00
. cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL ,
2012-05-11 17:00:48 +04:00
. cra_blocksize = AES_BLOCK_SIZE ,
2016-11-22 15:08:33 +03:00
. cra_ctxsize = CRYPTO_AES_CTX_SIZE ,
2012-05-11 17:00:48 +04:00
. cra_module = THIS_MODULE ,
. cra_u = {
. cipher = {
. cia_min_keysize = AES_MIN_KEY_SIZE ,
. cia_max_keysize = AES_MAX_KEY_SIZE ,
. cia_setkey = aes_set_key ,
. cia_encrypt = __aes_encrypt ,
. cia_decrypt = __aes_decrypt
}
}
2016-11-22 15:08:33 +03:00
} } ;
static struct skcipher_alg aesni_skciphers [ ] = {
{
. base = {
. cra_name = " __ecb(aes) " ,
. cra_driver_name = " __ecb-aes-aesni " ,
. cra_priority = 400 ,
. cra_flags = CRYPTO_ALG_INTERNAL ,
. cra_blocksize = AES_BLOCK_SIZE ,
. cra_ctxsize = CRYPTO_AES_CTX_SIZE ,
. cra_module = THIS_MODULE ,
2012-05-11 17:00:48 +04:00
} ,
2016-11-22 15:08:33 +03:00
. min_keysize = AES_MIN_KEY_SIZE ,
. max_keysize = AES_MAX_KEY_SIZE ,
. setkey = aesni_skcipher_setkey ,
. encrypt = ecb_encrypt ,
. decrypt = ecb_decrypt ,
} , {
. base = {
. cra_name = " __cbc(aes) " ,
. cra_driver_name = " __cbc-aes-aesni " ,
. cra_priority = 400 ,
. cra_flags = CRYPTO_ALG_INTERNAL ,
. cra_blocksize = AES_BLOCK_SIZE ,
. cra_ctxsize = CRYPTO_AES_CTX_SIZE ,
. cra_module = THIS_MODULE ,
2012-05-11 17:00:48 +04:00
} ,
2016-11-22 15:08:33 +03:00
. min_keysize = AES_MIN_KEY_SIZE ,
. max_keysize = AES_MAX_KEY_SIZE ,
. ivsize = AES_BLOCK_SIZE ,
. setkey = aesni_skcipher_setkey ,
. encrypt = cbc_encrypt ,
. decrypt = cbc_decrypt ,
2012-05-11 17:00:48 +04:00
# ifdef CONFIG_X86_64
2016-11-22 15:08:33 +03:00
} , {
. base = {
. cra_name = " __ctr(aes) " ,
. cra_driver_name = " __ctr-aes-aesni " ,
. cra_priority = 400 ,
. cra_flags = CRYPTO_ALG_INTERNAL ,
. cra_blocksize = 1 ,
. cra_ctxsize = CRYPTO_AES_CTX_SIZE ,
. cra_module = THIS_MODULE ,
2012-05-11 17:00:48 +04:00
} ,
2016-11-22 15:08:33 +03:00
. min_keysize = AES_MIN_KEY_SIZE ,
. max_keysize = AES_MAX_KEY_SIZE ,
. ivsize = AES_BLOCK_SIZE ,
. chunksize = AES_BLOCK_SIZE ,
. setkey = aesni_skcipher_setkey ,
. encrypt = ctr_crypt ,
. decrypt = ctr_crypt ,
} , {
. base = {
. cra_name = " __xts(aes) " ,
. cra_driver_name = " __xts-aes-aesni " ,
. cra_priority = 401 ,
. cra_flags = CRYPTO_ALG_INTERNAL ,
. cra_blocksize = AES_BLOCK_SIZE ,
. cra_ctxsize = XTS_AES_CTX_SIZE ,
. cra_module = THIS_MODULE ,
2012-05-11 17:00:48 +04:00
} ,
2016-11-22 15:08:33 +03:00
. min_keysize = 2 * AES_MIN_KEY_SIZE ,
. max_keysize = 2 * AES_MAX_KEY_SIZE ,
. ivsize = AES_BLOCK_SIZE ,
. setkey = xts_aesni_setkey ,
. encrypt = xts_encrypt ,
. decrypt = xts_decrypt ,
2012-05-11 17:00:48 +04:00
# endif
2016-11-22 15:08:33 +03:00
}
} ;
struct simd_skcipher_alg * aesni_simd_skciphers [ ARRAY_SIZE ( aesni_skciphers ) ] ;
struct {
const char * algname ;
const char * drvname ;
const char * basename ;
struct simd_skcipher_alg * simd ;
} aesni_simd_skciphers2 [ ] = {
2014-10-21 13:35:32 +04:00
# if IS_ENABLED(CONFIG_CRYPTO_PCBC)
2016-11-22 15:08:33 +03:00
{
. algname = " pcbc(aes) " ,
. drvname = " pcbc-aes-aesni " ,
. basename = " fpu(pcbc(__aes-aesni)) " ,
2012-05-11 17:00:48 +04:00
} ,
# endif
2016-11-22 15:08:33 +03:00
} ;
2010-11-04 22:00:45 +03:00
2015-05-28 17:08:03 +03:00
# ifdef CONFIG_X86_64
static struct aead_alg aesni_aead_algs [ ] = { {
2015-06-01 10:53:06 +03:00
. setkey = common_rfc4106_set_key ,
. setauthsize = common_rfc4106_set_authsize ,
. encrypt = helper_rfc4106_encrypt ,
. decrypt = helper_rfc4106_decrypt ,
. ivsize = 8 ,
. maxauthsize = 16 ,
. base = {
. cra_name = " __gcm-aes-aesni " ,
. cra_driver_name = " __driver-gcm-aes-aesni " ,
. cra_flags = CRYPTO_ALG_INTERNAL ,
. cra_blocksize = 1 ,
. cra_ctxsize = sizeof ( struct aesni_rfc4106_gcm_ctx ) ,
. cra_alignmask = AESNI_ALIGN - 1 ,
. cra_module = THIS_MODULE ,
} ,
} , {
2015-05-28 17:08:03 +03:00
. init = rfc4106_init ,
. exit = rfc4106_exit ,
. setkey = rfc4106_set_key ,
. setauthsize = rfc4106_set_authsize ,
. encrypt = rfc4106_encrypt ,
. decrypt = rfc4106_decrypt ,
. ivsize = 8 ,
. maxauthsize = 16 ,
. base = {
. cra_name = " rfc4106(gcm(aes)) " ,
. cra_driver_name = " rfc4106-gcm-aesni " ,
. cra_priority = 400 ,
2015-08-13 12:29:06 +03:00
. cra_flags = CRYPTO_ALG_ASYNC ,
2015-05-28 17:08:03 +03:00
. cra_blocksize = 1 ,
. cra_ctxsize = sizeof ( struct cryptd_aead * ) ,
. cra_module = THIS_MODULE ,
} ,
} } ;
# else
static struct aead_alg aesni_aead_algs [ 0 ] ;
# endif
2012-01-26 03:09:06 +04:00
static const struct x86_cpu_id aesni_cpu_id [ ] = {
X86_FEATURE_MATCH ( X86_FEATURE_AES ) ,
{ }
} ;
MODULE_DEVICE_TABLE ( x86cpu , aesni_cpu_id ) ;
2016-11-22 15:08:33 +03:00
static void aesni_free_simds ( void )
{
int i ;
for ( i = 0 ; i < ARRAY_SIZE ( aesni_simd_skciphers ) & &
aesni_simd_skciphers [ i ] ; i + + )
simd_skcipher_free ( aesni_simd_skciphers [ i ] ) ;
for ( i = 0 ; i < ARRAY_SIZE ( aesni_simd_skciphers2 ) & &
aesni_simd_skciphers2 [ i ] . simd ; i + + )
simd_skcipher_free ( aesni_simd_skciphers2 [ i ] . simd ) ;
}
2009-01-18 08:28:34 +03:00
static int __init aesni_init ( void )
{
2016-11-22 15:08:33 +03:00
struct simd_skcipher_alg * simd ;
const char * basename ;
const char * algname ;
const char * drvname ;
2012-07-11 15:20:51 +04:00
int err ;
2016-11-22 15:08:33 +03:00
int i ;
2009-01-18 08:28:34 +03:00
2012-01-26 03:09:06 +04:00
if ( ! x86_match_cpu ( aesni_cpu_id ) )
2009-01-18 08:28:34 +03:00
return - ENODEV ;
2013-12-30 17:52:24 +04:00
# ifdef CONFIG_X86_64
2013-12-12 02:28:41 +04:00
# ifdef CONFIG_AS_AVX2
if ( boot_cpu_has ( X86_FEATURE_AVX2 ) ) {
pr_info ( " AVX2 version of gcm_enc/dec engaged. \n " ) ;
aesni_gcm_enc_tfm = aesni_gcm_enc_avx2 ;
aesni_gcm_dec_tfm = aesni_gcm_dec_avx2 ;
} else
# endif
# ifdef CONFIG_AS_AVX
if ( boot_cpu_has ( X86_FEATURE_AVX ) ) {
pr_info ( " AVX version of gcm_enc/dec engaged. \n " ) ;
aesni_gcm_enc_tfm = aesni_gcm_enc_avx ;
aesni_gcm_dec_tfm = aesni_gcm_dec_avx ;
} else
# endif
{
pr_info ( " SSE version of gcm_enc/dec engaged. \n " ) ;
aesni_gcm_enc_tfm = aesni_gcm_enc ;
aesni_gcm_dec_tfm = aesni_gcm_dec ;
}
crypto: aes - AES CTR x86_64 "by8" AVX optimization
This patch introduces "by8" AES CTR mode AVX optimization inspired by
Intel Optimized IPSEC Cryptograhpic library. For additional information,
please see:
http://downloadcenter.intel.com/Detail_Desc.aspx?agr=Y&DwnldID=22972
The functions aes_ctr_enc_128_avx_by8(), aes_ctr_enc_192_avx_by8() and
aes_ctr_enc_256_avx_by8() are adapted from
Intel Optimized IPSEC Cryptographic library. When both AES and AVX features
are enabled in a platform, the glue code in AESNI module overrieds the
existing "by4" CTR mode en/decryption with the "by8"
AES CTR mode en/decryption.
On a Haswell desktop, with turbo disabled and all cpus running
at maximum frequency, the "by8" CTR mode optimization
shows better performance results across data & key sizes
as measured by tcrypt.
The average performance improvement of the "by8" version over the "by4"
version is as follows:
For 128 bit key and data sizes >= 256 bytes, there is a 10-16% improvement.
For 192 bit key and data sizes >= 256 bytes, there is a 20-22% improvement.
For 256 bit key and data sizes >= 256 bytes, there is a 20-25% improvement.
A typical run of tcrypt with AES CTR mode encryption of the "by4" and "by8"
optimization shows the following results:
tcrypt with "by4" AES CTR mode encryption optimization on a Haswell Desktop:
---------------------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 343 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 336 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 491 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1130 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7309 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 346 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 361 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 543 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1321 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9649 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 369 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 366 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 595 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1531 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 10522 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 336 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 350 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 487 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1129 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7287 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 350 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 359 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 635 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1324 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9595 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 364 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 377 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 604 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1527 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 10549 cycles (8192 bytes)
tcrypt with "by8" AES CTR mode encryption optimization on a Haswell Desktop:
---------------------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 340 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 330 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 450 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1043 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 6597 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 339 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 352 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 539 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1153 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 8458 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 353 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 360 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 512 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1277 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 8745 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 348 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 335 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 451 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1030 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 6611 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 354 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 346 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 488 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1154 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 8390 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 357 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 362 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 515 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1284 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 8681 cycles (8192 bytes)
crypto: Incorporate feed back to AES CTR mode optimization patch
Specifically, the following:
a) alignment around main loop in aes_ctrby8_avx_x86_64.S
b) .rodata around data constants used in the assembely code.
c) the use of CONFIG_AVX in the glue code.
d) fix up white space.
e) informational message for "by8" AES CTR mode optimization
f) "by8" AES CTR mode optimization can be simply enabled
if the platform supports both AES and AVX features. The
optimization works superbly on Sandybridge as well.
Testing on Haswell shows no performance change since the last.
Testing on Sandybridge shows that the "by8" AES CTR mode optimization
greatly improves performance.
tcrypt log with "by4" AES CTR mode optimization on Sandybridge
--------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 383 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 408 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 707 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1864 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 12813 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 395 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 432 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 780 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 2132 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 15765 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 416 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 438 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 842 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 2383 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 16945 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 389 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 409 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 704 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1865 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 12783 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 409 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 434 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 792 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 2151 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 15804 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 421 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 444 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 840 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 2394 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 16928 cycles (8192 bytes)
tcrypt log with "by8" AES CTR mode optimization on Sandybridge
--------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 383 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 401 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 522 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1136 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7046 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 394 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 418 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 559 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1263 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9072 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 408 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 428 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 595 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1385 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 9224 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 390 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 402 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 530 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1135 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7079 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 414 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 417 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 572 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1312 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9073 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 415 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 454 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 598 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1407 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 9288 cycles (8192 bytes)
crypto: Fix redundant checks
a) Fix the redundant check for cpu_has_aes
b) Fix the key length check when invoking the CTR mode "by8"
encryptor/decryptor.
crypto: fix typo in AES ctr mode transform
Signed-off-by: Chandramouli Narayanan <mouli@linux.intel.com>
Reviewed-by: Mathias Krause <minipli@googlemail.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2014-06-10 20:22:47 +04:00
aesni_ctr_enc_tfm = aesni_ctr_enc ;
2014-09-29 00:24:01 +04:00
# ifdef CONFIG_AS_AVX
2016-04-04 23:24:56 +03:00
if ( boot_cpu_has ( X86_FEATURE_AVX ) ) {
crypto: aes - AES CTR x86_64 "by8" AVX optimization
This patch introduces "by8" AES CTR mode AVX optimization inspired by
Intel Optimized IPSEC Cryptograhpic library. For additional information,
please see:
http://downloadcenter.intel.com/Detail_Desc.aspx?agr=Y&DwnldID=22972
The functions aes_ctr_enc_128_avx_by8(), aes_ctr_enc_192_avx_by8() and
aes_ctr_enc_256_avx_by8() are adapted from
Intel Optimized IPSEC Cryptographic library. When both AES and AVX features
are enabled in a platform, the glue code in AESNI module overrieds the
existing "by4" CTR mode en/decryption with the "by8"
AES CTR mode en/decryption.
On a Haswell desktop, with turbo disabled and all cpus running
at maximum frequency, the "by8" CTR mode optimization
shows better performance results across data & key sizes
as measured by tcrypt.
The average performance improvement of the "by8" version over the "by4"
version is as follows:
For 128 bit key and data sizes >= 256 bytes, there is a 10-16% improvement.
For 192 bit key and data sizes >= 256 bytes, there is a 20-22% improvement.
For 256 bit key and data sizes >= 256 bytes, there is a 20-25% improvement.
A typical run of tcrypt with AES CTR mode encryption of the "by4" and "by8"
optimization shows the following results:
tcrypt with "by4" AES CTR mode encryption optimization on a Haswell Desktop:
---------------------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 343 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 336 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 491 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1130 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7309 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 346 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 361 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 543 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1321 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9649 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 369 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 366 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 595 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1531 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 10522 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 336 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 350 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 487 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1129 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7287 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 350 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 359 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 635 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1324 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9595 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 364 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 377 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 604 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1527 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 10549 cycles (8192 bytes)
tcrypt with "by8" AES CTR mode encryption optimization on a Haswell Desktop:
---------------------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 340 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 330 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 450 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1043 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 6597 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 339 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 352 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 539 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1153 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 8458 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 353 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 360 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 512 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1277 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 8745 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 348 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 335 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 451 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1030 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 6611 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 354 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 346 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 488 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1154 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 8390 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 357 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 362 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 515 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1284 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 8681 cycles (8192 bytes)
crypto: Incorporate feed back to AES CTR mode optimization patch
Specifically, the following:
a) alignment around main loop in aes_ctrby8_avx_x86_64.S
b) .rodata around data constants used in the assembely code.
c) the use of CONFIG_AVX in the glue code.
d) fix up white space.
e) informational message for "by8" AES CTR mode optimization
f) "by8" AES CTR mode optimization can be simply enabled
if the platform supports both AES and AVX features. The
optimization works superbly on Sandybridge as well.
Testing on Haswell shows no performance change since the last.
Testing on Sandybridge shows that the "by8" AES CTR mode optimization
greatly improves performance.
tcrypt log with "by4" AES CTR mode optimization on Sandybridge
--------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 383 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 408 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 707 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1864 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 12813 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 395 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 432 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 780 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 2132 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 15765 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 416 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 438 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 842 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 2383 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 16945 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 389 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 409 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 704 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1865 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 12783 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 409 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 434 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 792 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 2151 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 15804 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 421 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 444 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 840 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 2394 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 16928 cycles (8192 bytes)
tcrypt log with "by8" AES CTR mode optimization on Sandybridge
--------------------------------------------------------------
testing speed of __ctr-aes-aesni encryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 383 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 401 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 522 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1136 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7046 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 394 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 418 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 559 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1263 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9072 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 408 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 428 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 595 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1385 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 9224 cycles (8192 bytes)
testing speed of __ctr-aes-aesni decryption
test 0 (128 bit key, 16 byte blocks): 1 operation in 390 cycles (16 bytes)
test 1 (128 bit key, 64 byte blocks): 1 operation in 402 cycles (64 bytes)
test 2 (128 bit key, 256 byte blocks): 1 operation in 530 cycles (256 bytes)
test 3 (128 bit key, 1024 byte blocks): 1 operation in 1135 cycles (1024 bytes)
test 4 (128 bit key, 8192 byte blocks): 1 operation in 7079 cycles (8192 bytes)
test 5 (192 bit key, 16 byte blocks): 1 operation in 414 cycles (16 bytes)
test 6 (192 bit key, 64 byte blocks): 1 operation in 417 cycles (64 bytes)
test 7 (192 bit key, 256 byte blocks): 1 operation in 572 cycles (256 bytes)
test 8 (192 bit key, 1024 byte blocks): 1 operation in 1312 cycles (1024 bytes)
test 9 (192 bit key, 8192 byte blocks): 1 operation in 9073 cycles (8192 bytes)
test 10 (256 bit key, 16 byte blocks): 1 operation in 415 cycles (16 bytes)
test 11 (256 bit key, 64 byte blocks): 1 operation in 454 cycles (64 bytes)
test 12 (256 bit key, 256 byte blocks): 1 operation in 598 cycles (256 bytes)
test 13 (256 bit key, 1024 byte blocks): 1 operation in 1407 cycles (1024 bytes)
test 14 (256 bit key, 8192 byte blocks): 1 operation in 9288 cycles (8192 bytes)
crypto: Fix redundant checks
a) Fix the redundant check for cpu_has_aes
b) Fix the key length check when invoking the CTR mode "by8"
encryptor/decryptor.
crypto: fix typo in AES ctr mode transform
Signed-off-by: Chandramouli Narayanan <mouli@linux.intel.com>
Reviewed-by: Mathias Krause <minipli@googlemail.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2014-06-10 20:22:47 +04:00
/* optimize performance of ctr mode encryption transform */
aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm ;
pr_info ( " AES CTR mode by8 optimization enabled \n " ) ;
}
# endif
2013-12-30 17:52:24 +04:00
# endif
2010-11-04 22:00:45 +03:00
2012-05-11 17:00:48 +04:00
err = crypto_fpu_init ( ) ;
if ( err )
return err ;
2009-01-18 08:28:34 +03:00
2015-05-28 17:08:03 +03:00
err = crypto_register_algs ( aesni_algs , ARRAY_SIZE ( aesni_algs ) ) ;
if ( err )
goto fpu_exit ;
2016-11-22 15:08:33 +03:00
err = crypto_register_skciphers ( aesni_skciphers ,
ARRAY_SIZE ( aesni_skciphers ) ) ;
if ( err )
goto unregister_algs ;
2015-05-28 17:08:03 +03:00
err = crypto_register_aeads ( aesni_aead_algs ,
ARRAY_SIZE ( aesni_aead_algs ) ) ;
if ( err )
2016-11-22 15:08:33 +03:00
goto unregister_skciphers ;
for ( i = 0 ; i < ARRAY_SIZE ( aesni_skciphers ) ; i + + ) {
algname = aesni_skciphers [ i ] . base . cra_name + 2 ;
drvname = aesni_skciphers [ i ] . base . cra_driver_name + 2 ;
basename = aesni_skciphers [ i ] . base . cra_driver_name ;
simd = simd_skcipher_create_compat ( algname , drvname , basename ) ;
err = PTR_ERR ( simd ) ;
if ( IS_ERR ( simd ) )
goto unregister_simds ;
aesni_simd_skciphers [ i ] = simd ;
}
2015-05-28 17:08:03 +03:00
2016-11-22 15:08:33 +03:00
for ( i = 0 ; i < ARRAY_SIZE ( aesni_simd_skciphers2 ) ; i + + ) {
algname = aesni_simd_skciphers2 [ i ] . algname ;
drvname = aesni_simd_skciphers2 [ i ] . drvname ;
basename = aesni_simd_skciphers2 [ i ] . basename ;
simd = simd_skcipher_create_compat ( algname , drvname , basename ) ;
err = PTR_ERR ( simd ) ;
if ( IS_ERR ( simd ) )
goto unregister_simds ;
2015-05-28 17:08:03 +03:00
2016-11-22 15:08:33 +03:00
aesni_simd_skciphers2 [ i ] . simd = simd ;
}
return 0 ;
unregister_simds :
aesni_free_simds ( ) ;
crypto_unregister_aeads ( aesni_aead_algs , ARRAY_SIZE ( aesni_aead_algs ) ) ;
unregister_skciphers :
crypto_unregister_skciphers ( aesni_skciphers ,
ARRAY_SIZE ( aesni_skciphers ) ) ;
2015-05-28 17:08:03 +03:00
unregister_algs :
crypto_unregister_algs ( aesni_algs , ARRAY_SIZE ( aesni_algs ) ) ;
fpu_exit :
crypto_fpu_exit ( ) ;
return err ;
2009-01-18 08:28:34 +03:00
}
static void __exit aesni_exit ( void )
{
2016-11-22 15:08:33 +03:00
aesni_free_simds ( ) ;
2015-05-28 17:08:03 +03:00
crypto_unregister_aeads ( aesni_aead_algs , ARRAY_SIZE ( aesni_aead_algs ) ) ;
2016-11-22 15:08:33 +03:00
crypto_unregister_skciphers ( aesni_skciphers ,
ARRAY_SIZE ( aesni_skciphers ) ) ;
2012-05-11 17:00:48 +04:00
crypto_unregister_algs ( aesni_algs , ARRAY_SIZE ( aesni_algs ) ) ;
2011-05-16 09:12:47 +04:00
crypto_fpu_exit ( ) ;
2009-01-18 08:28:34 +03:00
}
2015-06-27 09:56:38 +03:00
late_initcall ( aesni_init ) ;
2009-01-18 08:28:34 +03:00
module_exit ( aesni_exit ) ;
MODULE_DESCRIPTION ( " Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized " ) ;
MODULE_LICENSE ( " GPL " ) ;
2014-11-21 04:05:53 +03:00
MODULE_ALIAS_CRYPTO ( " aes " ) ;
