public-mirrors/linux
1
0
Fork 0
mirror of git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-08-05 16:54:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

crypto: x86/aes - stop using the SIMD helper

Browse source 
Stop wrapping skcipher and aead algorithms with the crypto SIMD helper
(crypto/simd.c).  The only purpose of doing so was to work around x86
not always supporting kernel-mode FPU in softirqs.  Specifically, if a
hardirq interrupted a task context kernel-mode FPU section and then a
softirqs were run at the end of that hardirq, those softirqs could not
use kernel-mode FPU.  This has now been fixed.  In combination with the
fact that the skcipher and aead APIs only support task and softirq
contexts, these can now just use kernel-mode FPU unconditionally on x86.

This simplifies the code and improves performance.

Signed-off-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
This commit is contained in:
Eric Biggers 2025-04-01 17:24:03 -07:00 • committed by Herbert Xu
parent 3a7dfdbbe3
commit 0ba6ec5b29
2 changed files with 59 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
arch/x86/crypto/Kconfig
View file

@ -23,7 +23,6 @@ config CRYPTO_AES_NI_INTEL
select CRYPTO_LIB_GF128MUL select CRYPTO_LIB_GF128MUL
select CRYPTO_ALGAPI select CRYPTO_ALGAPI
select CRYPTO_SKCIPHER select CRYPTO_SKCIPHER
select CRYPTO_SIMD
help help
Block cipher: AES cipher algorithms Block cipher: AES cipher algorithms
AEAD cipher: AES with GCM AEAD cipher: AES with GCM

158
arch/x86/crypto/aesni-intel_glue.c
View file

@ -566,10 +566,9 @@ static struct crypto_alg aesni_cipher_alg = {
static struct skcipher_alg aesni_skciphers[] = { static struct skcipher_alg aesni_skciphers[] = {
{ {
.base = { .base = {
.cra_name = "__ecb(aes)", .cra_name = "ecb(aes)",
.cra_driver_name = "__ecb-aes-aesni", .cra_driver_name = "ecb-aes-aesni",
.cra_priority = 400, .cra_priority = 400,
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = AES_BLOCK_SIZE, .cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = CRYPTO_AES_CTX_SIZE, .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
.cra_module = THIS_MODULE, .cra_module = THIS_MODULE,
@ -581,10 +580,9 @@ static struct skcipher_alg aesni_skciphers[] = {
.decrypt = ecb_decrypt, .decrypt = ecb_decrypt,
}, { }, {
.base = { .base = {
.cra_name = "__cbc(aes)", .cra_name = "cbc(aes)",
.cra_driver_name = "__cbc-aes-aesni", .cra_driver_name = "cbc-aes-aesni",
.cra_priority = 400, .cra_priority = 400,
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = AES_BLOCK_SIZE, .cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = CRYPTO_AES_CTX_SIZE, .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
.cra_module = THIS_MODULE, .cra_module = THIS_MODULE,
@ -597,10 +595,9 @@ static struct skcipher_alg aesni_skciphers[] = {
.decrypt = cbc_decrypt, .decrypt = cbc_decrypt,
}, { }, {
.base = { .base = {
.cra_name = "__cts(cbc(aes))", .cra_name = "cts(cbc(aes))",
.cra_driver_name = "__cts-cbc-aes-aesni", .cra_driver_name = "cts-cbc-aes-aesni",
.cra_priority = 400, .cra_priority = 400,
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = AES_BLOCK_SIZE, .cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = CRYPTO_AES_CTX_SIZE, .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
.cra_module = THIS_MODULE, .cra_module = THIS_MODULE,
@ -615,10 +612,9 @@ static struct skcipher_alg aesni_skciphers[] = {
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
}, { }, {
.base = { .base = {
.cra_name = "__ctr(aes)", .cra_name = "ctr(aes)",
.cra_driver_name = "__ctr-aes-aesni", .cra_driver_name = "ctr-aes-aesni",
.cra_priority = 400, .cra_priority = 400,
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = 1, .cra_blocksize = 1,
.cra_ctxsize = CRYPTO_AES_CTX_SIZE, .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
.cra_module = THIS_MODULE, .cra_module = THIS_MODULE,
@ -633,10 +629,9 @@ static struct skcipher_alg aesni_skciphers[] = {
#endif #endif
}, { }, {
.base = { .base = {
.cra_name = "__xts(aes)", .cra_name = "xts(aes)",
.cra_driver_name = "__xts-aes-aesni", .cra_driver_name = "xts-aes-aesni",
.cra_priority = 401, .cra_priority = 401,
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = AES_BLOCK_SIZE, .cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = XTS_AES_CTX_SIZE, .cra_ctxsize = XTS_AES_CTX_SIZE,
.cra_module = THIS_MODULE, .cra_module = THIS_MODULE,
@ -651,9 +646,6 @@ static struct skcipher_alg aesni_skciphers[] = {
} }
}; };
static
struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
asmlinkage void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key, asmlinkage void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
u8 iv[AES_BLOCK_SIZE]); u8 iv[AES_BLOCK_SIZE]);
@ -792,10 +784,9 @@ static int xctr_crypt_##suffix(struct skcipher_request *req) \
} \ } \
\ \
static struct skcipher_alg skcipher_algs_##suffix[] = {{ \ static struct skcipher_alg skcipher_algs_##suffix[] = {{ \
.base.cra_name = "__xts(aes)", \ .base.cra_name = "xts(aes)", \
.base.cra_driver_name = "__xts-aes-" driver_name_suffix, \ .base.cra_driver_name = "xts-aes-" driver_name_suffix, \
.base.cra_priority = priority, \ .base.cra_priority = priority, \
.base.cra_flags = CRYPTO_ALG_INTERNAL, \
.base.cra_blocksize = AES_BLOCK_SIZE, \ .base.cra_blocksize = AES_BLOCK_SIZE, \
.base.cra_ctxsize = XTS_AES_CTX_SIZE, \ .base.cra_ctxsize = XTS_AES_CTX_SIZE, \
.base.cra_module = THIS_MODULE, \ .base.cra_module = THIS_MODULE, \
@ -807,10 +798,9 @@ static struct skcipher_alg skcipher_algs_##suffix[] = {{ \
.encrypt = xts_encrypt_##suffix, \ .encrypt = xts_encrypt_##suffix, \
.decrypt = xts_decrypt_##suffix, \ .decrypt = xts_decrypt_##suffix, \
}, { \ }, { \
.base.cra_name = "__ctr(aes)", \ .base.cra_name = "ctr(aes)", \
.base.cra_driver_name = "__ctr-aes-" driver_name_suffix, \ .base.cra_driver_name = "ctr-aes-" driver_name_suffix, \
.base.cra_priority = priority, \ .base.cra_priority = priority, \
.base.cra_flags = CRYPTO_ALG_INTERNAL, \
.base.cra_blocksize = 1, \ .base.cra_blocksize = 1, \
.base.cra_ctxsize = CRYPTO_AES_CTX_SIZE, \ .base.cra_ctxsize = CRYPTO_AES_CTX_SIZE, \
.base.cra_module = THIS_MODULE, \ .base.cra_module = THIS_MODULE, \
@ -822,10 +812,9 @@ static struct skcipher_alg skcipher_algs_##suffix[] = {{ \
.encrypt = ctr_crypt_##suffix, \ .encrypt = ctr_crypt_##suffix, \
.decrypt = ctr_crypt_##suffix, \ .decrypt = ctr_crypt_##suffix, \
}, { \ }, { \
.base.cra_name = "__xctr(aes)", \ .base.cra_name = "xctr(aes)", \
.base.cra_driver_name = "__xctr-aes-" driver_name_suffix, \ .base.cra_driver_name = "xctr-aes-" driver_name_suffix, \
.base.cra_priority = priority, \ .base.cra_priority = priority, \
.base.cra_flags = CRYPTO_ALG_INTERNAL, \
.base.cra_blocksize = 1, \ .base.cra_blocksize = 1, \
.base.cra_ctxsize = CRYPTO_AES_CTX_SIZE, \ .base.cra_ctxsize = CRYPTO_AES_CTX_SIZE, \
.base.cra_module = THIS_MODULE, \ .base.cra_module = THIS_MODULE, \
@ -836,10 +825,7 @@ static struct skcipher_alg skcipher_algs_##suffix[] = {{ \
.setkey = aesni_skcipher_setkey, \ .setkey = aesni_skcipher_setkey, \
.encrypt = xctr_crypt_##suffix, \ .encrypt = xctr_crypt_##suffix, \
.decrypt = xctr_crypt_##suffix, \ .decrypt = xctr_crypt_##suffix, \
}}; \ }}
\
static struct simd_skcipher_alg * \
simd_skcipher_algs_##suffix[ARRAY_SIZE(skcipher_algs_##suffix)]
DEFINE_AVX_SKCIPHER_ALGS(aesni_avx, "aesni-avx", 500); DEFINE_AVX_SKCIPHER_ALGS(aesni_avx, "aesni-avx", 500);
#if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ) #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ)
@ -1498,10 +1484,9 @@ static struct aead_alg aes_gcm_algs_##suffix[] = { { \
.chunksize = AES_BLOCK_SIZE, \ .chunksize = AES_BLOCK_SIZE, \
.maxauthsize = 16, \ .maxauthsize = 16, \
.base = { \ .base = { \
.cra_name = "__gcm(aes)", \ .cra_name = "gcm(aes)", \
.cra_driver_name = "__" generic_driver_name, \ .cra_driver_name = generic_driver_name, \
.cra_priority = (priority), \ .cra_priority = (priority), \
.cra_flags = CRYPTO_ALG_INTERNAL, \
.cra_blocksize = 1, \ .cra_blocksize = 1, \
.cra_ctxsize = (ctxsize), \ .cra_ctxsize = (ctxsize), \
.cra_module = THIS_MODULE, \ .cra_module = THIS_MODULE, \
@ -1515,17 +1500,14 @@ static struct aead_alg aes_gcm_algs_##suffix[] = { { \
.chunksize = AES_BLOCK_SIZE, \ .chunksize = AES_BLOCK_SIZE, \
.maxauthsize = 16, \ .maxauthsize = 16, \
.base = { \ .base = { \
.cra_name = "__rfc4106(gcm(aes))", \ .cra_name = "rfc4106(gcm(aes))", \
.cra_driver_name = "__" rfc_driver_name, \ .cra_driver_name = rfc_driver_name, \
.cra_priority = (priority), \ .cra_priority = (priority), \
.cra_flags = CRYPTO_ALG_INTERNAL, \
.cra_blocksize = 1, \ .cra_blocksize = 1, \
.cra_ctxsize = (ctxsize), \ .cra_ctxsize = (ctxsize), \
.cra_module = THIS_MODULE, \ .cra_module = THIS_MODULE, \
}, \ }, \
} }; \ } }
\
static struct simd_aead_alg *aes_gcm_simdalgs_##suffix[2] \
/* aes_gcm_algs_aesni */ /* aes_gcm_algs_aesni */
DEFINE_GCM_ALGS(aesni, /* no flags */ 0, DEFINE_GCM_ALGS(aesni, /* no flags */ 0,
@ -1555,14 +1537,12 @@ static int __init register_avx_algs(void)
if (!boot_cpu_has(X86_FEATURE_AVX)) if (!boot_cpu_has(X86_FEATURE_AVX))
return 0; return 0;
err = simd_register_skciphers_compat(skcipher_algs_aesni_avx, err = crypto_register_skciphers(skcipher_algs_aesni_avx,
ARRAY_SIZE(skcipher_algs_aesni_avx), ARRAY_SIZE(skcipher_algs_aesni_avx));
simd_skcipher_algs_aesni_avx);
if (err) if (err)
return err; return err;
err = simd_register_aeads_compat(aes_gcm_algs_aesni_avx, err = crypto_register_aeads(aes_gcm_algs_aesni_avx,
ARRAY_SIZE(aes_gcm_algs_aesni_avx), ARRAY_SIZE(aes_gcm_algs_aesni_avx));
aes_gcm_simdalgs_aesni_avx);
if (err) if (err)
return err; return err;
/* /*
@ -1578,9 +1558,8 @@ static int __init register_avx_algs(void)
!boot_cpu_has(X86_FEATURE_PCLMULQDQ) || !boot_cpu_has(X86_FEATURE_PCLMULQDQ) ||
!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
return 0; return 0;
err = simd_register_skciphers_compat(skcipher_algs_vaes_avx2, err = crypto_register_skciphers(skcipher_algs_vaes_avx2,
ARRAY_SIZE(skcipher_algs_vaes_avx2), ARRAY_SIZE(skcipher_algs_vaes_avx2));
simd_skcipher_algs_vaes_avx2);
if (err) if (err)
return err; return err;
@ -1591,9 +1570,8 @@ static int __init register_avx_algs(void)
XFEATURE_MASK_AVX512, NULL)) XFEATURE_MASK_AVX512, NULL))
return 0; return 0;
err = simd_register_aeads_compat(aes_gcm_algs_vaes_avx10_256, err = crypto_register_aeads(aes_gcm_algs_vaes_avx10_256,
ARRAY_SIZE(aes_gcm_algs_vaes_avx10_256), ARRAY_SIZE(aes_gcm_algs_vaes_avx10_256));
aes_gcm_simdalgs_vaes_avx10_256);
if (err) if (err)
return err; return err;
@ -1606,52 +1584,38 @@ static int __init register_avx_algs(void)
aes_gcm_algs_vaes_avx10_512[i].base.cra_priority = 1; aes_gcm_algs_vaes_avx10_512[i].base.cra_priority = 1;
} }
err = simd_register_skciphers_compat(skcipher_algs_vaes_avx512, err = crypto_register_skciphers(skcipher_algs_vaes_avx512,
ARRAY_SIZE(skcipher_algs_vaes_avx512), ARRAY_SIZE(skcipher_algs_vaes_avx512));
simd_skcipher_algs_vaes_avx512);
if (err) if (err)
return err; return err;
err = simd_register_aeads_compat(aes_gcm_algs_vaes_avx10_512, err = crypto_register_aeads(aes_gcm_algs_vaes_avx10_512,
ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512), ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512));
aes_gcm_simdalgs_vaes_avx10_512);
if (err) if (err)
return err; return err;
#endif /* CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ */ #endif /* CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ */
return 0; return 0;
} }
#define unregister_skciphers(A) \
if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \
crypto_unregister_skciphers((A), ARRAY_SIZE(A))
#define unregister_aeads(A) \
if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \
crypto_unregister_aeads((A), ARRAY_SIZE(A))
static void unregister_avx_algs(void) static void unregister_avx_algs(void)
{ {
if (simd_skcipher_algs_aesni_avx[0]) unregister_skciphers(skcipher_algs_aesni_avx);
simd_unregister_skciphers(skcipher_algs_aesni_avx, unregister_aeads(aes_gcm_algs_aesni_avx);
ARRAY_SIZE(skcipher_algs_aesni_avx),
simd_skcipher_algs_aesni_avx);
if (aes_gcm_simdalgs_aesni_avx[0])
simd_unregister_aeads(aes_gcm_algs_aesni_avx,
ARRAY_SIZE(aes_gcm_algs_aesni_avx),
aes_gcm_simdalgs_aesni_avx);
#if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ) #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ)
if (simd_skcipher_algs_vaes_avx2[0]) unregister_skciphers(skcipher_algs_vaes_avx2);
simd_unregister_skciphers(skcipher_algs_vaes_avx2, unregister_skciphers(skcipher_algs_vaes_avx512);
ARRAY_SIZE(skcipher_algs_vaes_avx2), unregister_aeads(aes_gcm_algs_vaes_avx10_256);
simd_skcipher_algs_vaes_avx2); unregister_aeads(aes_gcm_algs_vaes_avx10_512);
if (aes_gcm_simdalgs_vaes_avx10_256[0])
simd_unregister_aeads(aes_gcm_algs_vaes_avx10_256,
ARRAY_SIZE(aes_gcm_algs_vaes_avx10_256),
aes_gcm_simdalgs_vaes_avx10_256);
if (simd_skcipher_algs_vaes_avx512[0])
simd_unregister_skciphers(skcipher_algs_vaes_avx512,
ARRAY_SIZE(skcipher_algs_vaes_avx512),
simd_skcipher_algs_vaes_avx512);
if (aes_gcm_simdalgs_vaes_avx10_512[0])
simd_unregister_aeads(aes_gcm_algs_vaes_avx10_512,
ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512),
aes_gcm_simdalgs_vaes_avx10_512);
#endif #endif
} }
#else /* CONFIG_X86_64 */ #else /* CONFIG_X86_64 */
static struct aead_alg aes_gcm_algs_aesni[0]; static struct aead_alg aes_gcm_algs_aesni[0];
static struct simd_aead_alg *aes_gcm_simdalgs_aesni[0];
static int __init register_avx_algs(void) static int __init register_avx_algs(void)
{ {
@ -1680,15 +1644,13 @@ static int __init aesni_init(void)
if (err) if (err)
return err; return err;
err = simd_register_skciphers_compat(aesni_skciphers, err = crypto_register_skciphers(aesni_skciphers,
ARRAY_SIZE(aesni_skciphers), ARRAY_SIZE(aesni_skciphers));
aesni_simd_skciphers);
if (err) if (err)
goto unregister_cipher; goto unregister_cipher;
err = simd_register_aeads_compat(aes_gcm_algs_aesni, err = crypto_register_aeads(aes_gcm_algs_aesni,
ARRAY_SIZE(aes_gcm_algs_aesni), ARRAY_SIZE(aes_gcm_algs_aesni));
aes_gcm_simdalgs_aesni);
if (err) if (err)
goto unregister_skciphers; goto unregister_skciphers;
@ -1700,12 +1662,11 @@ static int __init aesni_init(void)
unregister_avx: unregister_avx:
unregister_avx_algs(); unregister_avx_algs();
simd_unregister_aeads(aes_gcm_algs_aesni, crypto_unregister_aeads(aes_gcm_algs_aesni,
ARRAY_SIZE(aes_gcm_algs_aesni), ARRAY_SIZE(aes_gcm_algs_aesni));
aes_gcm_simdalgs_aesni);
unregister_skciphers: unregister_skciphers:
simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), crypto_unregister_skciphers(aesni_skciphers,
aesni_simd_skciphers); ARRAY_SIZE(aesni_skciphers));
unregister_cipher: unregister_cipher:
crypto_unregister_alg(&aesni_cipher_alg); crypto_unregister_alg(&aesni_cipher_alg);
return err; return err;
@ -1713,11 +1674,10 @@ unregister_cipher:
static void __exit aesni_exit(void) static void __exit aesni_exit(void)
{ {
simd_unregister_aeads(aes_gcm_algs_aesni, crypto_unregister_aeads(aes_gcm_algs_aesni,
ARRAY_SIZE(aes_gcm_algs_aesni), ARRAY_SIZE(aes_gcm_algs_aesni));
aes_gcm_simdalgs_aesni); crypto_unregister_skciphers(aesni_skciphers,
simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), ARRAY_SIZE(aesni_skciphers));
aesni_simd_skciphers);
crypto_unregister_alg(&aesni_cipher_alg); crypto_unregister_alg(&aesni_cipher_alg);
unregister_avx_algs(); unregister_avx_algs();
} }