mirror of
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2025-08-05 16:54:27 +00:00
fe59e0358d
Similarly to what was done with the memcpy() routines, make copy_to_user(), copy_from_user() and clear_user() also use the Armv8.8 FEAT_MOPS instructions. Both MOPS implementation options (A and B) are supported, including asymmetric systems. The exception fixup code fixes up the registers according to the option used. In case of a fault the routines return precisely how much was not copied (as required by the comment in include/linux/uaccess.h), as unprivileged versions of CPY/SET are guaranteed not to have written past the addresses reported in the GPRs. The MOPS instructions could possibly be inlined into callers (and patched to branch to the generic implementation if not detected; similarly to what x86 does), but as a first step this patch just uses them in the out-of-line routines. Signed-off-by: Kristina Martšenko <kristina.martsenko@arm.com> Acked-by: Robin Murphy <robin.murphy@arm.com> Link: https://lore.kernel.org/r/20250228170006.390100-4-kristina.martsenko@arm.com Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
191 lines
4.1 KiB
ArmAsm
191 lines
4.1 KiB
ArmAsm
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* Copyright (C) 2013 ARM Ltd.
|
|
* Copyright (C) 2013 Linaro.
|
|
*
|
|
* This code is based on glibc cortex strings work originally authored by Linaro
|
|
* be found @
|
|
*
|
|
* http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
|
|
* files/head:/src/aarch64/
|
|
*/
|
|
|
|
|
|
/*
|
|
* Copy a buffer from src to dest (alignment handled by the hardware)
|
|
*
|
|
* Parameters:
|
|
* x0 - dest
|
|
* x1 - src
|
|
* x2 - n
|
|
* Returns:
|
|
* x0 - dest
|
|
*/
|
|
dstin .req x0
|
|
src .req x1
|
|
count .req x2
|
|
tmp1 .req x3
|
|
tmp1w .req w3
|
|
tmp2 .req x4
|
|
tmp2w .req w4
|
|
dst .req x6
|
|
|
|
A_l .req x7
|
|
A_h .req x8
|
|
B_l .req x9
|
|
B_h .req x10
|
|
C_l .req x11
|
|
C_h .req x12
|
|
D_l .req x13
|
|
D_h .req x14
|
|
|
|
mov dst, dstin
|
|
|
|
#ifdef CONFIG_AS_HAS_MOPS
|
|
alternative_if_not ARM64_HAS_MOPS
|
|
b .Lno_mops
|
|
alternative_else_nop_endif
|
|
cpy1 dst, src, count
|
|
b .Lexitfunc
|
|
.Lno_mops:
|
|
#endif
|
|
|
|
cmp count, #16
|
|
/*When memory length is less than 16, the accessed are not aligned.*/
|
|
b.lo .Ltiny15
|
|
|
|
neg tmp2, src
|
|
ands tmp2, tmp2, #15/* Bytes to reach alignment. */
|
|
b.eq .LSrcAligned
|
|
sub count, count, tmp2
|
|
/*
|
|
* Copy the leading memory data from src to dst in an increasing
|
|
* address order.By this way,the risk of overwriting the source
|
|
* memory data is eliminated when the distance between src and
|
|
* dst is less than 16. The memory accesses here are alignment.
|
|
*/
|
|
tbz tmp2, #0, 1f
|
|
ldrb1 tmp1w, src, #1
|
|
strb1 tmp1w, dst, #1
|
|
1:
|
|
tbz tmp2, #1, 2f
|
|
ldrh1 tmp1w, src, #2
|
|
strh1 tmp1w, dst, #2
|
|
2:
|
|
tbz tmp2, #2, 3f
|
|
ldr1 tmp1w, src, #4
|
|
str1 tmp1w, dst, #4
|
|
3:
|
|
tbz tmp2, #3, .LSrcAligned
|
|
ldr1 tmp1, src, #8
|
|
str1 tmp1, dst, #8
|
|
|
|
.LSrcAligned:
|
|
cmp count, #64
|
|
b.ge .Lcpy_over64
|
|
/*
|
|
* Deal with small copies quickly by dropping straight into the
|
|
* exit block.
|
|
*/
|
|
.Ltail63:
|
|
/*
|
|
* Copy up to 48 bytes of data. At this point we only need the
|
|
* bottom 6 bits of count to be accurate.
|
|
*/
|
|
ands tmp1, count, #0x30
|
|
b.eq .Ltiny15
|
|
cmp tmp1w, #0x20
|
|
b.eq 1f
|
|
b.lt 2f
|
|
ldp1 A_l, A_h, src, #16
|
|
stp1 A_l, A_h, dst, #16
|
|
1:
|
|
ldp1 A_l, A_h, src, #16
|
|
stp1 A_l, A_h, dst, #16
|
|
2:
|
|
ldp1 A_l, A_h, src, #16
|
|
stp1 A_l, A_h, dst, #16
|
|
.Ltiny15:
|
|
/*
|
|
* Prefer to break one ldp/stp into several load/store to access
|
|
* memory in an increasing address order,rather than to load/store 16
|
|
* bytes from (src-16) to (dst-16) and to backward the src to aligned
|
|
* address,which way is used in original cortex memcpy. If keeping
|
|
* the original memcpy process here, memmove need to satisfy the
|
|
* precondition that src address is at least 16 bytes bigger than dst
|
|
* address,otherwise some source data will be overwritten when memove
|
|
* call memcpy directly. To make memmove simpler and decouple the
|
|
* memcpy's dependency on memmove, withdrew the original process.
|
|
*/
|
|
tbz count, #3, 1f
|
|
ldr1 tmp1, src, #8
|
|
str1 tmp1, dst, #8
|
|
1:
|
|
tbz count, #2, 2f
|
|
ldr1 tmp1w, src, #4
|
|
str1 tmp1w, dst, #4
|
|
2:
|
|
tbz count, #1, 3f
|
|
ldrh1 tmp1w, src, #2
|
|
strh1 tmp1w, dst, #2
|
|
3:
|
|
tbz count, #0, .Lexitfunc
|
|
ldrb1 tmp1w, src, #1
|
|
strb1 tmp1w, dst, #1
|
|
|
|
b .Lexitfunc
|
|
|
|
.Lcpy_over64:
|
|
subs count, count, #128
|
|
b.ge .Lcpy_body_large
|
|
/*
|
|
* Less than 128 bytes to copy, so handle 64 here and then jump
|
|
* to the tail.
|
|
*/
|
|
ldp1 A_l, A_h, src, #16
|
|
stp1 A_l, A_h, dst, #16
|
|
ldp1 B_l, B_h, src, #16
|
|
ldp1 C_l, C_h, src, #16
|
|
stp1 B_l, B_h, dst, #16
|
|
stp1 C_l, C_h, dst, #16
|
|
ldp1 D_l, D_h, src, #16
|
|
stp1 D_l, D_h, dst, #16
|
|
|
|
tst count, #0x3f
|
|
b.ne .Ltail63
|
|
b .Lexitfunc
|
|
|
|
/*
|
|
* Critical loop. Start at a new cache line boundary. Assuming
|
|
* 64 bytes per line this ensures the entire loop is in one line.
|
|
*/
|
|
.p2align L1_CACHE_SHIFT
|
|
.Lcpy_body_large:
|
|
/* pre-get 64 bytes data. */
|
|
ldp1 A_l, A_h, src, #16
|
|
ldp1 B_l, B_h, src, #16
|
|
ldp1 C_l, C_h, src, #16
|
|
ldp1 D_l, D_h, src, #16
|
|
1:
|
|
/*
|
|
* interlace the load of next 64 bytes data block with store of the last
|
|
* loaded 64 bytes data.
|
|
*/
|
|
stp1 A_l, A_h, dst, #16
|
|
ldp1 A_l, A_h, src, #16
|
|
stp1 B_l, B_h, dst, #16
|
|
ldp1 B_l, B_h, src, #16
|
|
stp1 C_l, C_h, dst, #16
|
|
ldp1 C_l, C_h, src, #16
|
|
stp1 D_l, D_h, dst, #16
|
|
ldp1 D_l, D_h, src, #16
|
|
subs count, count, #64
|
|
b.ge 1b
|
|
stp1 A_l, A_h, dst, #16
|
|
stp1 B_l, B_h, dst, #16
|
|
stp1 C_l, C_h, dst, #16
|
|
stp1 D_l, D_h, dst, #16
|
|
|
|
tst count, #0x3f
|
|
b.ne .Ltail63
|
|
.Lexitfunc:
|