mirror of
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2025-08-05 16:54:27 +00:00
43a43faf53
Josh Poimboeuf reports that he got a "will-it-scale.per_process_ops 1.9% improvement" report for his patch that changed __get_user() to use pointer masking instead of the explicit speculation barrier. However, that patch doesn't actually work in the general case, because some (very bad) architecture-specific code actually depends on __get_user() also working on kernel addresses. A profile showed that the offending __get_user() was the futex code, which really should be fixed up to not use that horrid legacy case. Rewrite futex_get_value_locked() to use the modern user acccess helpers, and inline it so that the compiler not only avoids the function call for a few instructions, but can do CSE on the address masking. It also turns out the x86 futex functions have unnecessary barriers in other places, so let's fix those up too. Link: https://lore.kernel.org/all/20241115230653.hfvzyf3aqqntgp63@jpoimboe/ Reported-by: Josh Poimboeuf <jpoimboe@kernel.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
107 lines
2.7 KiB
C
107 lines
2.7 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _ASM_X86_FUTEX_H
|
|
#define _ASM_X86_FUTEX_H
|
|
|
|
#ifdef __KERNEL__
|
|
|
|
#include <linux/futex.h>
|
|
#include <linux/uaccess.h>
|
|
|
|
#include <asm/asm.h>
|
|
#include <asm/errno.h>
|
|
#include <asm/processor.h>
|
|
#include <asm/smap.h>
|
|
|
|
#define unsafe_atomic_op1(insn, oval, uaddr, oparg, label) \
|
|
do { \
|
|
int oldval = 0, ret; \
|
|
asm volatile("1:\t" insn "\n" \
|
|
"2:\n" \
|
|
_ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %1) \
|
|
: "=r" (oldval), "=r" (ret), "+m" (*uaddr) \
|
|
: "0" (oparg), "1" (0)); \
|
|
if (ret) \
|
|
goto label; \
|
|
*oval = oldval; \
|
|
} while(0)
|
|
|
|
|
|
#define unsafe_atomic_op2(insn, oval, uaddr, oparg, label) \
|
|
do { \
|
|
int oldval = 0, ret, tem; \
|
|
asm volatile("1:\tmovl %2, %0\n" \
|
|
"2:\tmovl\t%0, %3\n" \
|
|
"\t" insn "\n" \
|
|
"3:\t" LOCK_PREFIX "cmpxchgl %3, %2\n" \
|
|
"\tjnz\t2b\n" \
|
|
"4:\n" \
|
|
_ASM_EXTABLE_TYPE_REG(1b, 4b, EX_TYPE_EFAULT_REG, %1) \
|
|
_ASM_EXTABLE_TYPE_REG(3b, 4b, EX_TYPE_EFAULT_REG, %1) \
|
|
: "=&a" (oldval), "=&r" (ret), \
|
|
"+m" (*uaddr), "=&r" (tem) \
|
|
: "r" (oparg), "1" (0)); \
|
|
if (ret) \
|
|
goto label; \
|
|
*oval = oldval; \
|
|
} while(0)
|
|
|
|
static __always_inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval,
|
|
u32 __user *uaddr)
|
|
{
|
|
if (can_do_masked_user_access())
|
|
uaddr = masked_user_access_begin(uaddr);
|
|
else if (!user_access_begin(uaddr, sizeof(u32)))
|
|
return -EFAULT;
|
|
|
|
switch (op) {
|
|
case FUTEX_OP_SET:
|
|
unsafe_atomic_op1("xchgl %0, %2", oval, uaddr, oparg, Efault);
|
|
break;
|
|
case FUTEX_OP_ADD:
|
|
unsafe_atomic_op1(LOCK_PREFIX "xaddl %0, %2", oval,
|
|
uaddr, oparg, Efault);
|
|
break;
|
|
case FUTEX_OP_OR:
|
|
unsafe_atomic_op2("orl %4, %3", oval, uaddr, oparg, Efault);
|
|
break;
|
|
case FUTEX_OP_ANDN:
|
|
unsafe_atomic_op2("andl %4, %3", oval, uaddr, ~oparg, Efault);
|
|
break;
|
|
case FUTEX_OP_XOR:
|
|
unsafe_atomic_op2("xorl %4, %3", oval, uaddr, oparg, Efault);
|
|
break;
|
|
default:
|
|
user_access_end();
|
|
return -ENOSYS;
|
|
}
|
|
user_access_end();
|
|
return 0;
|
|
Efault:
|
|
user_access_end();
|
|
return -EFAULT;
|
|
}
|
|
|
|
static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
|
|
u32 oldval, u32 newval)
|
|
{
|
|
int ret = 0;
|
|
|
|
if (can_do_masked_user_access())
|
|
uaddr = masked_user_access_begin(uaddr);
|
|
else if (!user_access_begin(uaddr, sizeof(u32)))
|
|
return -EFAULT;
|
|
asm volatile("\n"
|
|
"1:\t" LOCK_PREFIX "cmpxchgl %3, %2\n"
|
|
"2:\n"
|
|
_ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %0) \
|
|
: "+r" (ret), "=a" (oldval), "+m" (*uaddr)
|
|
: "r" (newval), "1" (oldval)
|
|
: "memory"
|
|
);
|
|
user_access_end();
|
|
*uval = oldval;
|
|
return ret;
|
|
}
|
|
|
|
#endif
|
|
#endif /* _ASM_X86_FUTEX_H */
|