mirror of
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2025-08-03 15:55:38 +00:00
a493a87f38
Implement a retpoline [0] for the BPF tail call JIT'ing that converts
the indirect jump via jmp %rax that is used to make the long jump into
another JITed BPF image. Since this is subject to speculative execution,
we need to control the transient instruction sequence here as well
when CONFIG_RETPOLINE is set, and direct it into a pause + lfence loop.
The latter aligns also with what gcc / clang emits (e.g. [1]).
JIT dump after patch:
# bpftool p d x i 1
0: (18) r2 = map[id:1]
2: (b7) r3 = 0
3: (85) call bpf_tail_call#12
4: (b7) r0 = 2
5: (95) exit
With CONFIG_RETPOLINE:
# bpftool p d j i 1
[...]
33: cmp %edx,0x24(%rsi)
36: jbe 0x0000000000000072 |*
38: mov 0x24(%rbp),%eax
3e: cmp $0x20,%eax
41: ja 0x0000000000000072 |
43: add $0x1,%eax
46: mov %eax,0x24(%rbp)
4c: mov 0x90(%rsi,%rdx,8),%rax
54: test %rax,%rax
57: je 0x0000000000000072 |
59: mov 0x28(%rax),%rax
5d: add $0x25,%rax
61: callq 0x000000000000006d |+
66: pause |
68: lfence |
6b: jmp 0x0000000000000066 |
6d: mov %rax,(%rsp) |
71: retq |
72: mov $0x2,%eax
[...]
* relative fall-through jumps in error case
+ retpoline for indirect jump
Without CONFIG_RETPOLINE:
# bpftool p d j i 1
[...]
33: cmp %edx,0x24(%rsi)
36: jbe 0x0000000000000063 |*
38: mov 0x24(%rbp),%eax
3e: cmp $0x20,%eax
41: ja 0x0000000000000063 |
43: add $0x1,%eax
46: mov %eax,0x24(%rbp)
4c: mov 0x90(%rsi,%rdx,8),%rax
54: test %rax,%rax
57: je 0x0000000000000063 |
59: mov 0x28(%rax),%rax
5d: add $0x25,%rax
61: jmpq *%rax |-
63: mov $0x2,%eax
[...]
* relative fall-through jumps in error case
- plain indirect jump as before
[0] https://support.google.com/faqs/answer/7625886
[1] a31e654fa1
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
217 lines
5.5 KiB
C
217 lines
5.5 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
|
|
#ifndef _ASM_X86_NOSPEC_BRANCH_H_
|
|
#define _ASM_X86_NOSPEC_BRANCH_H_
|
|
|
|
#include <asm/alternative.h>
|
|
#include <asm/alternative-asm.h>
|
|
#include <asm/cpufeatures.h>
|
|
#include <asm/msr-index.h>
|
|
|
|
#ifdef __ASSEMBLY__
|
|
|
|
/*
|
|
* This should be used immediately before a retpoline alternative. It tells
|
|
* objtool where the retpolines are so that it can make sense of the control
|
|
* flow by just reading the original instruction(s) and ignoring the
|
|
* alternatives.
|
|
*/
|
|
.macro ANNOTATE_NOSPEC_ALTERNATIVE
|
|
.Lannotate_\@:
|
|
.pushsection .discard.nospec
|
|
.long .Lannotate_\@ - .
|
|
.popsection
|
|
.endm
|
|
|
|
/*
|
|
* These are the bare retpoline primitives for indirect jmp and call.
|
|
* Do not use these directly; they only exist to make the ALTERNATIVE
|
|
* invocation below less ugly.
|
|
*/
|
|
.macro RETPOLINE_JMP reg:req
|
|
call .Ldo_rop_\@
|
|
.Lspec_trap_\@:
|
|
pause
|
|
lfence
|
|
jmp .Lspec_trap_\@
|
|
.Ldo_rop_\@:
|
|
mov \reg, (%_ASM_SP)
|
|
ret
|
|
.endm
|
|
|
|
/*
|
|
* This is a wrapper around RETPOLINE_JMP so the called function in reg
|
|
* returns to the instruction after the macro.
|
|
*/
|
|
.macro RETPOLINE_CALL reg:req
|
|
jmp .Ldo_call_\@
|
|
.Ldo_retpoline_jmp_\@:
|
|
RETPOLINE_JMP \reg
|
|
.Ldo_call_\@:
|
|
call .Ldo_retpoline_jmp_\@
|
|
.endm
|
|
|
|
/*
|
|
* JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
|
|
* indirect jmp/call which may be susceptible to the Spectre variant 2
|
|
* attack.
|
|
*/
|
|
.macro JMP_NOSPEC reg:req
|
|
#ifdef CONFIG_RETPOLINE
|
|
ANNOTATE_NOSPEC_ALTERNATIVE
|
|
ALTERNATIVE_2 __stringify(jmp *\reg), \
|
|
__stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \
|
|
__stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
|
|
#else
|
|
jmp *\reg
|
|
#endif
|
|
.endm
|
|
|
|
.macro CALL_NOSPEC reg:req
|
|
#ifdef CONFIG_RETPOLINE
|
|
ANNOTATE_NOSPEC_ALTERNATIVE
|
|
ALTERNATIVE_2 __stringify(call *\reg), \
|
|
__stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
|
|
__stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
|
|
#else
|
|
call *\reg
|
|
#endif
|
|
.endm
|
|
|
|
/* This clobbers the BX register */
|
|
.macro FILL_RETURN_BUFFER nr:req ftr:req
|
|
#ifdef CONFIG_RETPOLINE
|
|
ALTERNATIVE "", "call __clear_rsb", \ftr
|
|
#endif
|
|
.endm
|
|
|
|
#else /* __ASSEMBLY__ */
|
|
|
|
#define ANNOTATE_NOSPEC_ALTERNATIVE \
|
|
"999:\n\t" \
|
|
".pushsection .discard.nospec\n\t" \
|
|
".long 999b - .\n\t" \
|
|
".popsection\n\t"
|
|
|
|
#if defined(CONFIG_X86_64) && defined(RETPOLINE)
|
|
|
|
/*
|
|
* Since the inline asm uses the %V modifier which is only in newer GCC,
|
|
* the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
|
|
*/
|
|
# define CALL_NOSPEC \
|
|
ANNOTATE_NOSPEC_ALTERNATIVE \
|
|
ALTERNATIVE( \
|
|
"call *%[thunk_target]\n", \
|
|
"call __x86_indirect_thunk_%V[thunk_target]\n", \
|
|
X86_FEATURE_RETPOLINE)
|
|
# define THUNK_TARGET(addr) [thunk_target] "r" (addr)
|
|
|
|
#elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
|
|
/*
|
|
* For i386 we use the original ret-equivalent retpoline, because
|
|
* otherwise we'll run out of registers. We don't care about CET
|
|
* here, anyway.
|
|
*/
|
|
# define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n", \
|
|
" jmp 904f;\n" \
|
|
" .align 16\n" \
|
|
"901: call 903f;\n" \
|
|
"902: pause;\n" \
|
|
" lfence;\n" \
|
|
" jmp 902b;\n" \
|
|
" .align 16\n" \
|
|
"903: addl $4, %%esp;\n" \
|
|
" pushl %[thunk_target];\n" \
|
|
" ret;\n" \
|
|
" .align 16\n" \
|
|
"904: call 901b;\n", \
|
|
X86_FEATURE_RETPOLINE)
|
|
|
|
# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
|
|
#else /* No retpoline for C / inline asm */
|
|
# define CALL_NOSPEC "call *%[thunk_target]\n"
|
|
# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
|
|
#endif
|
|
|
|
/* The Spectre V2 mitigation variants */
|
|
enum spectre_v2_mitigation {
|
|
SPECTRE_V2_NONE,
|
|
SPECTRE_V2_RETPOLINE_MINIMAL,
|
|
SPECTRE_V2_RETPOLINE_MINIMAL_AMD,
|
|
SPECTRE_V2_RETPOLINE_GENERIC,
|
|
SPECTRE_V2_RETPOLINE_AMD,
|
|
SPECTRE_V2_IBRS,
|
|
};
|
|
|
|
extern char __indirect_thunk_start[];
|
|
extern char __indirect_thunk_end[];
|
|
|
|
/*
|
|
* On VMEXIT we must ensure that no RSB predictions learned in the guest
|
|
* can be followed in the host, by overwriting the RSB completely. Both
|
|
* retpoline and IBRS mitigations for Spectre v2 need this; only on future
|
|
* CPUs with IBRS_ALL *might* it be avoided.
|
|
*/
|
|
static inline void vmexit_fill_RSB(void)
|
|
{
|
|
#ifdef CONFIG_RETPOLINE
|
|
alternative_input("",
|
|
"call __fill_rsb",
|
|
X86_FEATURE_RETPOLINE,
|
|
ASM_NO_INPUT_CLOBBER(_ASM_BX, "memory"));
|
|
#endif
|
|
}
|
|
|
|
static inline void indirect_branch_prediction_barrier(void)
|
|
{
|
|
asm volatile(ALTERNATIVE("",
|
|
"movl %[msr], %%ecx\n\t"
|
|
"movl %[val], %%eax\n\t"
|
|
"movl $0, %%edx\n\t"
|
|
"wrmsr",
|
|
X86_FEATURE_USE_IBPB)
|
|
: : [msr] "i" (MSR_IA32_PRED_CMD),
|
|
[val] "i" (PRED_CMD_IBPB)
|
|
: "eax", "ecx", "edx", "memory");
|
|
}
|
|
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
/*
|
|
* Below is used in the eBPF JIT compiler and emits the byte sequence
|
|
* for the following assembly:
|
|
*
|
|
* With retpolines configured:
|
|
*
|
|
* callq do_rop
|
|
* spec_trap:
|
|
* pause
|
|
* lfence
|
|
* jmp spec_trap
|
|
* do_rop:
|
|
* mov %rax,(%rsp)
|
|
* retq
|
|
*
|
|
* Without retpolines configured:
|
|
*
|
|
* jmp *%rax
|
|
*/
|
|
#ifdef CONFIG_RETPOLINE
|
|
# define RETPOLINE_RAX_BPF_JIT_SIZE 17
|
|
# define RETPOLINE_RAX_BPF_JIT() \
|
|
EMIT1_off32(0xE8, 7); /* callq do_rop */ \
|
|
/* spec_trap: */ \
|
|
EMIT2(0xF3, 0x90); /* pause */ \
|
|
EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \
|
|
EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \
|
|
/* do_rop: */ \
|
|
EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */ \
|
|
EMIT1(0xC3); /* retq */
|
|
#else
|
|
# define RETPOLINE_RAX_BPF_JIT_SIZE 2
|
|
# define RETPOLINE_RAX_BPF_JIT() \
|
|
EMIT2(0xFF, 0xE0); /* jmp *%rax */
|
|
#endif
|
|
|
|
#endif /* _ASM_X86_NOSPEC_BRANCH_H_ */
|