mirror of
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2025-08-05 16:54:27 +00:00
fe59e0358d
Similarly to what was done with the memcpy() routines, make copy_to_user(), copy_from_user() and clear_user() also use the Armv8.8 FEAT_MOPS instructions. Both MOPS implementation options (A and B) are supported, including asymmetric systems. The exception fixup code fixes up the registers according to the option used. In case of a fault the routines return precisely how much was not copied (as required by the comment in include/linux/uaccess.h), as unprivileged versions of CPY/SET are guaranteed not to have written past the addresses reported in the GPRs. The MOPS instructions could possibly be inlined into callers (and patched to branch to the generic implementation if not detected; similarly to what x86 does), but as a first step this patch just uses them in the out-of-line routines. Signed-off-by: Kristina Martšenko <kristina.martsenko@arm.com> Acked-by: Robin Murphy <robin.murphy@arm.com> Link: https://lore.kernel.org/r/20250228170006.390100-4-kristina.martsenko@arm.com Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
83 lines
1.7 KiB
ArmAsm
83 lines
1.7 KiB
ArmAsm
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
*/
|
|
|
|
#include <linux/linkage.h>
|
|
|
|
#include <asm/asm-uaccess.h>
|
|
#include <asm/assembler.h>
|
|
#include <asm/cache.h>
|
|
|
|
/*
|
|
* Copy to user space from a kernel buffer (alignment handled by the hardware)
|
|
*
|
|
* Parameters:
|
|
* x0 - to
|
|
* x1 - from
|
|
* x2 - n
|
|
* Returns:
|
|
* x0 - bytes not copied
|
|
*/
|
|
.macro ldrb1 reg, ptr, val
|
|
ldrb \reg, [\ptr], \val
|
|
.endm
|
|
|
|
.macro strb1 reg, ptr, val
|
|
user_ldst 9998f, sttrb, \reg, \ptr, \val
|
|
.endm
|
|
|
|
.macro ldrh1 reg, ptr, val
|
|
ldrh \reg, [\ptr], \val
|
|
.endm
|
|
|
|
.macro strh1 reg, ptr, val
|
|
user_ldst 9997f, sttrh, \reg, \ptr, \val
|
|
.endm
|
|
|
|
.macro ldr1 reg, ptr, val
|
|
ldr \reg, [\ptr], \val
|
|
.endm
|
|
|
|
.macro str1 reg, ptr, val
|
|
user_ldst 9997f, sttr, \reg, \ptr, \val
|
|
.endm
|
|
|
|
.macro ldp1 reg1, reg2, ptr, val
|
|
ldp \reg1, \reg2, [\ptr], \val
|
|
.endm
|
|
|
|
.macro stp1 reg1, reg2, ptr, val
|
|
user_stp 9997f, \reg1, \reg2, \ptr, \val
|
|
.endm
|
|
|
|
.macro cpy1 dst, src, count
|
|
.arch_extension mops
|
|
USER_CPY(9997f, 1, cpyfpwt [\dst]!, [\src]!, \count!)
|
|
USER_CPY(9996f, 1, cpyfmwt [\dst]!, [\src]!, \count!)
|
|
USER_CPY(9996f, 1, cpyfewt [\dst]!, [\src]!, \count!)
|
|
.endm
|
|
|
|
end .req x5
|
|
srcin .req x15
|
|
SYM_FUNC_START(__arch_copy_to_user)
|
|
add end, x0, x2
|
|
mov srcin, x1
|
|
#include "copy_template.S"
|
|
mov x0, #0
|
|
ret
|
|
|
|
// Exception fixups
|
|
9996: b.cs 9997f
|
|
// Registers are in Option A format
|
|
add dst, dst, count
|
|
9997: cmp dst, dstin
|
|
b.ne 9998f
|
|
// Before being absolutely sure we couldn't copy anything, try harder
|
|
ldrb tmp1w, [srcin]
|
|
USER(9998f, sttrb tmp1w, [dst])
|
|
add dst, dst, #1
|
|
9998: sub x0, end, dst // bytes not copied
|
|
ret
|
|
SYM_FUNC_END(__arch_copy_to_user)
|
|
EXPORT_SYMBOL(__arch_copy_to_user)
|