mirror of
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2025-08-05 16:54:27 +00:00
676d45aba8
Move the contents of arch/powerpc/lib/crypto/ into lib/crypto/powerpc/. The new code organization makes a lot more sense for how this code actually works and is developed. In particular, it makes it possible to build each algorithm as a single module, with better inlining and dead code elimination. For a more detailed explanation, see the patchset which did this for the CRC library code: https://lore.kernel.org/r/20250607200454.73587-1-ebiggers@kernel.org/. Also see the patchset which did this for SHA-512: https://lore.kernel.org/linux-crypto/20250616014019.415791-1-ebiggers@kernel.org/ This is just a preparatory commit, which does the move to get the files into their new location but keeps them building the same way as before. Later commits will make the actual improvements to the way the arch-optimized code is integrated for each algorithm. Acked-by: Ard Biesheuvel <ardb@kernel.org> Reviewed-by: Martin K. Petersen <martin.petersen@oracle.com> Reviewed-by: Sohil Mehta <sohil.mehta@intel.com> Link: https://lore.kernel.org/r/20250619191908.134235-5-ebiggers@kernel.org Signed-off-by: Eric Biggers <ebiggers@kernel.org>
100 lines
2.2 KiB
C
100 lines
2.2 KiB
C
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
/*
|
|
* ChaCha stream cipher (P10 accelerated)
|
|
*
|
|
* Copyright 2023- IBM Corp. All rights reserved.
|
|
*/
|
|
|
|
#include <crypto/chacha.h>
|
|
#include <crypto/internal/simd.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/module.h>
|
|
#include <linux/cpufeature.h>
|
|
#include <linux/sizes.h>
|
|
#include <asm/simd.h>
|
|
#include <asm/switch_to.h>
|
|
|
|
asmlinkage void chacha_p10le_8x(const struct chacha_state *state, u8 *dst,
|
|
const u8 *src, unsigned int len, int nrounds);
|
|
|
|
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10);
|
|
|
|
static void vsx_begin(void)
|
|
{
|
|
preempt_disable();
|
|
enable_kernel_vsx();
|
|
}
|
|
|
|
static void vsx_end(void)
|
|
{
|
|
disable_kernel_vsx();
|
|
preempt_enable();
|
|
}
|
|
|
|
static void chacha_p10_do_8x(struct chacha_state *state, u8 *dst, const u8 *src,
|
|
unsigned int bytes, int nrounds)
|
|
{
|
|
unsigned int l = bytes & ~0x0FF;
|
|
|
|
if (l > 0) {
|
|
chacha_p10le_8x(state, dst, src, l, nrounds);
|
|
bytes -= l;
|
|
src += l;
|
|
dst += l;
|
|
state->x[12] += l / CHACHA_BLOCK_SIZE;
|
|
}
|
|
|
|
if (bytes > 0)
|
|
chacha_crypt_generic(state, dst, src, bytes, nrounds);
|
|
}
|
|
|
|
void hchacha_block_arch(const struct chacha_state *state,
|
|
u32 out[HCHACHA_OUT_WORDS], int nrounds)
|
|
{
|
|
hchacha_block_generic(state, out, nrounds);
|
|
}
|
|
EXPORT_SYMBOL(hchacha_block_arch);
|
|
|
|
void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src,
|
|
unsigned int bytes, int nrounds)
|
|
{
|
|
if (!static_branch_likely(&have_p10) || bytes <= CHACHA_BLOCK_SIZE ||
|
|
!crypto_simd_usable())
|
|
return chacha_crypt_generic(state, dst, src, bytes, nrounds);
|
|
|
|
do {
|
|
unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
|
|
|
|
vsx_begin();
|
|
chacha_p10_do_8x(state, dst, src, todo, nrounds);
|
|
vsx_end();
|
|
|
|
bytes -= todo;
|
|
src += todo;
|
|
dst += todo;
|
|
} while (bytes);
|
|
}
|
|
EXPORT_SYMBOL(chacha_crypt_arch);
|
|
|
|
bool chacha_is_arch_optimized(void)
|
|
{
|
|
return static_key_enabled(&have_p10);
|
|
}
|
|
EXPORT_SYMBOL(chacha_is_arch_optimized);
|
|
|
|
static int __init chacha_p10_init(void)
|
|
{
|
|
if (cpu_has_feature(CPU_FTR_ARCH_31))
|
|
static_branch_enable(&have_p10);
|
|
return 0;
|
|
}
|
|
subsys_initcall(chacha_p10_init);
|
|
|
|
static void __exit chacha_p10_exit(void)
|
|
{
|
|
}
|
|
module_exit(chacha_p10_exit);
|
|
|
|
MODULE_DESCRIPTION("ChaCha stream cipher (P10 accelerated)");
|
|
MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com>");
|
|
MODULE_LICENSE("GPL v2");
|