2020-05-03 13:03:54 +01:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
|
|
|
|
|
|
|
|
#ifndef __ASM_UACCESS_ASM_H__
|
|
|
|
|
#define __ASM_UACCESS_ASM_H__
|
|
|
|
|
|
|
|
|
|
#include <asm/asm-offsets.h>
|
|
|
|
|
#include <asm/domain.h>
|
ARM: mm: Make virt_to_pfn() a static inline
Making virt_to_pfn() a static inline taking a strongly typed
(const void *) makes the contract of a passing a pointer of that
type to the function explicit and exposes any misuse of the
macro virt_to_pfn() acting polymorphic and accepting many types
such as (void *), (unitptr_t) or (unsigned long) as arguments
without warnings.
Doing this is a bit intrusive: virt_to_pfn() requires
PHYS_PFN_OFFSET and PAGE_SHIFT to be defined, and this is defined in
<asm/page.h>, so this must be included *before* <asm/memory.h>.
The use of macros were obscuring the unclear inclusion order here,
as the macros would eventually be resolved, but a static inline
like this cannot be compiled with unresolved macros.
The naive solution to include <asm/page.h> at the top of
<asm/memory.h> does not work, because <asm/memory.h> sometimes
includes <asm/page.h> at the end of itself, which would create a
confusing inclusion loop. So instead, take the approach to always
unconditionally include <asm/page.h> at the end of <asm/memory.h>
arch/arm uses <asm/memory.h> explicitly in a lot of places,
however it turns out that if we just unconditionally include
<asm/memory.h> into <asm/page.h> and switch all inclusions of
<asm/memory.h> to <asm/page.h> instead, we enforce the right
order and <asm/memory.h> will always have access to the
definitions.
Put an inclusion guard in place making it impossible to include
<asm/memory.h> explicitly.
Link: https://lore.kernel.org/linux-mm/20220701160004.2ffff4e5ab59a55499f4c736@linux-foundation.org/
Signed-off-by: Linus Walleij <linus.walleij@linaro.org>
2022-06-02 10:18:32 +02:00
|
|
|
#include <asm/page.h>
|
2020-05-03 13:03:54 +01:00
|
|
|
#include <asm/thread_info.h>
|
|
|
|
|
|
|
|
|
|
.macro csdb
|
|
|
|
|
#ifdef CONFIG_THUMB2_KERNEL
|
|
|
|
|
.inst.w 0xf3af8014
|
|
|
|
|
#else
|
|
|
|
|
.inst 0xe320f014
|
|
|
|
|
#endif
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro check_uaccess, addr:req, size:req, limit:req, tmp:req, bad:req
|
|
|
|
|
#ifndef CONFIG_CPU_USE_DOMAINS
|
|
|
|
|
adds \tmp, \addr, #\size - 1
|
|
|
|
|
sbcscc \tmp, \tmp, \limit
|
|
|
|
|
bcs \bad
|
|
|
|
|
#ifdef CONFIG_CPU_SPECTRE
|
|
|
|
|
movcs \addr, #0
|
|
|
|
|
csdb
|
|
|
|
|
#endif
|
|
|
|
|
#endif
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro uaccess_mask_range_ptr, addr:req, size:req, limit:req, tmp:req
|
|
|
|
|
#ifdef CONFIG_CPU_SPECTRE
|
|
|
|
|
sub \tmp, \limit, #1
|
|
|
|
|
subs \tmp, \tmp, \addr @ tmp = limit - 1 - addr
|
|
|
|
|
addhs \tmp, \tmp, #1 @ if (tmp >= 0) {
|
|
|
|
|
subshs \tmp, \tmp, \size @ tmp = limit - (addr + size) }
|
|
|
|
|
movlo \addr, #0 @ if (tmp < 0) addr = NULL
|
|
|
|
|
csdb
|
|
|
|
|
#endif
|
|
|
|
|
.endm
|
|
|
|
|
|
2024-03-25 08:31:13 +01:00
|
|
|
#if defined(CONFIG_CPU_SW_DOMAIN_PAN)
|
2024-03-25 08:30:02 +01:00
|
|
|
|
|
|
|
|
.macro uaccess_disable, tmp, isb=1
|
2020-05-03 13:03:54 +01:00
|
|
|
/*
|
|
|
|
|
* Whenever we re-enter userspace, the domains should always be
|
|
|
|
|
* set appropriately.
|
|
|
|
|
*/
|
|
|
|
|
mov \tmp, #DACR_UACCESS_DISABLE
|
|
|
|
|
mcr p15, 0, \tmp, c3, c0, 0 @ Set domain register
|
|
|
|
|
.if \isb
|
|
|
|
|
instr_sync
|
|
|
|
|
.endif
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro uaccess_enable, tmp, isb=1
|
|
|
|
|
/*
|
|
|
|
|
* Whenever we re-enter userspace, the domains should always be
|
|
|
|
|
* set appropriately.
|
|
|
|
|
*/
|
|
|
|
|
mov \tmp, #DACR_UACCESS_ENABLE
|
|
|
|
|
mcr p15, 0, \tmp, c3, c0, 0
|
|
|
|
|
.if \isb
|
|
|
|
|
instr_sync
|
|
|
|
|
.endif
|
|
|
|
|
.endm
|
|
|
|
|
|
2024-03-25 08:31:13 +01:00
|
|
|
#elif defined(CONFIG_CPU_TTBR0_PAN)
|
|
|
|
|
|
|
|
|
|
.macro uaccess_disable, tmp, isb=1
|
|
|
|
|
/*
|
|
|
|
|
* Disable TTBR0 page table walks (EDP0 = 1), use the reserved ASID
|
|
|
|
|
* from TTBR1 (A1 = 1) and enable TTBR1 page table walks for kernel
|
|
|
|
|
* addresses by reducing TTBR0 range to 32MB (T0SZ = 7).
|
|
|
|
|
*/
|
|
|
|
|
mrc p15, 0, \tmp, c2, c0, 2 @ read TTBCR
|
|
|
|
|
orr \tmp, \tmp, #TTBCR_EPD0 | TTBCR_T0SZ_MASK
|
|
|
|
|
orr \tmp, \tmp, #TTBCR_A1
|
|
|
|
|
mcr p15, 0, \tmp, c2, c0, 2 @ write TTBCR
|
|
|
|
|
.if \isb
|
|
|
|
|
instr_sync
|
|
|
|
|
.endif
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro uaccess_enable, tmp, isb=1
|
|
|
|
|
/*
|
|
|
|
|
* Enable TTBR0 page table walks (T0SZ = 0, EDP0 = 0) and ASID from
|
|
|
|
|
* TTBR0 (A1 = 0).
|
|
|
|
|
*/
|
|
|
|
|
mrc p15, 0, \tmp, c2, c0, 2 @ read TTBCR
|
|
|
|
|
bic \tmp, \tmp, #TTBCR_EPD0 | TTBCR_T0SZ_MASK
|
|
|
|
|
bic \tmp, \tmp, #TTBCR_A1
|
|
|
|
|
mcr p15, 0, \tmp, c2, c0, 2 @ write TTBCR
|
|
|
|
|
.if \isb
|
|
|
|
|
instr_sync
|
|
|
|
|
.endif
|
|
|
|
|
.endm
|
|
|
|
|
|
2024-03-25 08:30:02 +01:00
|
|
|
#else
|
|
|
|
|
|
|
|
|
|
.macro uaccess_disable, tmp, isb=1
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro uaccess_enable, tmp, isb=1
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
|
ARM: uaccess: fix DACR mismatch with nested exceptions
Tomas Paukrt reports that his SAM9X60 based system (ARM926, ARMv5TJ)
fails to fix up alignment faults, eventually resulting in a kernel
oops.
The problem occurs when using CONFIG_CPU_USE_DOMAINS with commit
e6978e4bf181 ("ARM: save and reset the address limit when entering an
exception"). This is because the address limit is set back to
TASK_SIZE on exception entry, and, although it is restored on exception
exit, the domain register is not.
Hence, this sequence can occur:
interrupt
pt_regs->addr_limit = addr_limit // USER_DS
addr_limit = USER_DS
alignment exception
__probe_kernel_read()
old_fs = get_fs() // USER_DS
set_fs(KERNEL_DS)
addr_limit = KERNEL_DS
dacr.kernel = DOMAIN_MANAGER
interrupt
pt_regs->addr_limit = addr_limit // KERNEL_DS
addr_limit = USER_DS
alignment exception
__probe_kernel_read()
old_fs = get_fs() // USER_DS
set_fs(KERNEL_DS)
addr_limit = KERNEL_DS
dacr.kernel = DOMAIN_MANAGER
...
set_fs(old_fs)
addr_limit = USER_DS
dacr.kernel = DOMAIN_CLIENT
...
addr_limit = pt_regs->addr_limit // KERNEL_DS
interrupt returns
At this point, addr_limit is correctly restored to KERNEL_DS for
__probe_kernel_read() to continue execution, but dacr.kernel is not,
it has been reset by the set_fs(old_fs) to DOMAIN_CLIENT.
This would not have happened prior to the mentioned commit, because
addr_limit would remain KERNEL_DS, so get_fs() would have returned
KERNEL_DS, and so would correctly nest.
This commit fixes the problem by also saving the DACR on exception
entry if either CONFIG_CPU_SW_DOMAIN_PAN or CONFIG_CPU_USE_DOMAINS are
enabled, and resetting the DACR appropriately on exception entry to
match addr_limit and PAN settings.
Fixes: e6978e4bf181 ("ARM: save and reset the address limit when entering an exception")
Reported-by: Tomas Paukrt <tomas.paukrt@advantech.cz>
Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk>
2020-05-03 13:24:07 +01:00
|
|
|
#if defined(CONFIG_CPU_SW_DOMAIN_PAN) || defined(CONFIG_CPU_USE_DOMAINS)
|
2020-05-03 13:14:09 +01:00
|
|
|
#define DACR(x...) x
|
|
|
|
|
#else
|
|
|
|
|
#define DACR(x...)
|
2024-03-25 08:31:13 +01:00
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_CPU_TTBR0_PAN
|
|
|
|
|
#define PAN(x...) x
|
|
|
|
|
#else
|
|
|
|
|
#define PAN(x...)
|
2020-05-03 13:03:54 +01:00
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
/*
|
ARM: uaccess: fix DACR mismatch with nested exceptions
Tomas Paukrt reports that his SAM9X60 based system (ARM926, ARMv5TJ)
fails to fix up alignment faults, eventually resulting in a kernel
oops.
The problem occurs when using CONFIG_CPU_USE_DOMAINS with commit
e6978e4bf181 ("ARM: save and reset the address limit when entering an
exception"). This is because the address limit is set back to
TASK_SIZE on exception entry, and, although it is restored on exception
exit, the domain register is not.
Hence, this sequence can occur:
interrupt
pt_regs->addr_limit = addr_limit // USER_DS
addr_limit = USER_DS
alignment exception
__probe_kernel_read()
old_fs = get_fs() // USER_DS
set_fs(KERNEL_DS)
addr_limit = KERNEL_DS
dacr.kernel = DOMAIN_MANAGER
interrupt
pt_regs->addr_limit = addr_limit // KERNEL_DS
addr_limit = USER_DS
alignment exception
__probe_kernel_read()
old_fs = get_fs() // USER_DS
set_fs(KERNEL_DS)
addr_limit = KERNEL_DS
dacr.kernel = DOMAIN_MANAGER
...
set_fs(old_fs)
addr_limit = USER_DS
dacr.kernel = DOMAIN_CLIENT
...
addr_limit = pt_regs->addr_limit // KERNEL_DS
interrupt returns
At this point, addr_limit is correctly restored to KERNEL_DS for
__probe_kernel_read() to continue execution, but dacr.kernel is not,
it has been reset by the set_fs(old_fs) to DOMAIN_CLIENT.
This would not have happened prior to the mentioned commit, because
addr_limit would remain KERNEL_DS, so get_fs() would have returned
KERNEL_DS, and so would correctly nest.
This commit fixes the problem by also saving the DACR on exception
entry if either CONFIG_CPU_SW_DOMAIN_PAN or CONFIG_CPU_USE_DOMAINS are
enabled, and resetting the DACR appropriately on exception entry to
match addr_limit and PAN settings.
Fixes: e6978e4bf181 ("ARM: save and reset the address limit when entering an exception")
Reported-by: Tomas Paukrt <tomas.paukrt@advantech.cz>
Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk>
2020-05-03 13:24:07 +01:00
|
|
|
* Save the address limit on entry to a privileged exception.
|
|
|
|
|
*
|
|
|
|
|
* If we are using the DACR for kernel access by the user accessors
|
|
|
|
|
* (CONFIG_CPU_USE_DOMAINS=y), always reset the DACR kernel domain
|
|
|
|
|
* back to client mode, whether or not \disable is set.
|
|
|
|
|
*
|
|
|
|
|
* If we are using SW PAN, set the DACR user domain to no access
|
|
|
|
|
* if \disable is set.
|
2020-05-03 13:03:54 +01:00
|
|
|
*/
|
|
|
|
|
.macro uaccess_entry, tsk, tmp0, tmp1, tmp2, disable
|
2020-05-03 13:14:09 +01:00
|
|
|
DACR( mrc p15, 0, \tmp0, c3, c0, 0)
|
|
|
|
|
DACR( str \tmp0, [sp, #SVC_DACR])
|
2024-03-25 08:31:13 +01:00
|
|
|
PAN( mrc p15, 0, \tmp0, c2, c0, 2)
|
|
|
|
|
PAN( str \tmp0, [sp, #SVC_TTBCR])
|
ARM: uaccess: fix DACR mismatch with nested exceptions
Tomas Paukrt reports that his SAM9X60 based system (ARM926, ARMv5TJ)
fails to fix up alignment faults, eventually resulting in a kernel
oops.
The problem occurs when using CONFIG_CPU_USE_DOMAINS with commit
e6978e4bf181 ("ARM: save and reset the address limit when entering an
exception"). This is because the address limit is set back to
TASK_SIZE on exception entry, and, although it is restored on exception
exit, the domain register is not.
Hence, this sequence can occur:
interrupt
pt_regs->addr_limit = addr_limit // USER_DS
addr_limit = USER_DS
alignment exception
__probe_kernel_read()
old_fs = get_fs() // USER_DS
set_fs(KERNEL_DS)
addr_limit = KERNEL_DS
dacr.kernel = DOMAIN_MANAGER
interrupt
pt_regs->addr_limit = addr_limit // KERNEL_DS
addr_limit = USER_DS
alignment exception
__probe_kernel_read()
old_fs = get_fs() // USER_DS
set_fs(KERNEL_DS)
addr_limit = KERNEL_DS
dacr.kernel = DOMAIN_MANAGER
...
set_fs(old_fs)
addr_limit = USER_DS
dacr.kernel = DOMAIN_CLIENT
...
addr_limit = pt_regs->addr_limit // KERNEL_DS
interrupt returns
At this point, addr_limit is correctly restored to KERNEL_DS for
__probe_kernel_read() to continue execution, but dacr.kernel is not,
it has been reset by the set_fs(old_fs) to DOMAIN_CLIENT.
This would not have happened prior to the mentioned commit, because
addr_limit would remain KERNEL_DS, so get_fs() would have returned
KERNEL_DS, and so would correctly nest.
This commit fixes the problem by also saving the DACR on exception
entry if either CONFIG_CPU_SW_DOMAIN_PAN or CONFIG_CPU_USE_DOMAINS are
enabled, and resetting the DACR appropriately on exception entry to
match addr_limit and PAN settings.
Fixes: e6978e4bf181 ("ARM: save and reset the address limit when entering an exception")
Reported-by: Tomas Paukrt <tomas.paukrt@advantech.cz>
Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk>
2020-05-03 13:24:07 +01:00
|
|
|
.if \disable && IS_ENABLED(CONFIG_CPU_SW_DOMAIN_PAN)
|
|
|
|
|
/* kernel=client, user=no access */
|
|
|
|
|
mov \tmp2, #DACR_UACCESS_DISABLE
|
|
|
|
|
mcr p15, 0, \tmp2, c3, c0, 0
|
|
|
|
|
instr_sync
|
|
|
|
|
.elseif IS_ENABLED(CONFIG_CPU_USE_DOMAINS)
|
|
|
|
|
/* kernel=client */
|
|
|
|
|
bic \tmp2, \tmp0, #domain_mask(DOMAIN_KERNEL)
|
|
|
|
|
orr \tmp2, \tmp2, #domain_val(DOMAIN_KERNEL, DOMAIN_CLIENT)
|
|
|
|
|
mcr p15, 0, \tmp2, c3, c0, 0
|
|
|
|
|
instr_sync
|
2020-05-03 13:03:54 +01:00
|
|
|
.endif
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
/* Restore the user access state previously saved by uaccess_entry */
|
|
|
|
|
.macro uaccess_exit, tsk, tmp0, tmp1
|
2020-05-03 13:14:09 +01:00
|
|
|
DACR( ldr \tmp0, [sp, #SVC_DACR])
|
|
|
|
|
DACR( mcr p15, 0, \tmp0, c3, c0, 0)
|
2024-03-25 08:31:13 +01:00
|
|
|
PAN( ldr \tmp0, [sp, #SVC_TTBCR])
|
|
|
|
|
PAN( mcr p15, 0, \tmp0, c2, c0, 2)
|
2020-05-03 13:03:54 +01:00
|
|
|
.endm
|
|
|
|
|
|
2020-05-03 13:14:09 +01:00
|
|
|
#undef DACR
|
2024-03-25 08:31:13 +01:00
|
|
|
#undef PAN
|
2020-05-03 13:14:09 +01:00
|
|
|
|
2020-05-03 13:03:54 +01:00
|
|
|
#endif /* __ASM_UACCESS_ASM_H__ */
|