From 672513bf583157cc088a32e75bd850dd9f81564c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 26 Oct 2021 10:23:31 +0200 Subject: [PATCH 01/62] ARM: decompressor: disable stack protector Enabling the stack protector in the decompressor is of dubious value, given that it uses a fixed value for the canary, cannot print any output unless CONFIG_DEBUG_LL is enabled (which relies on board specific build time settings), and is already disabled for a good chunk of the code (libfdt). So let's just disable it in the decompressor. This will make it easier in the future to manage the command line options that would need to be removed again in this context for the TLS register based stack protector. Signed-off-by: Ard Biesheuvel --- arch/arm/boot/compressed/Makefile | 6 +----- arch/arm/boot/compressed/misc.c | 7 ------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile index 91265e7ff672..e2bd084b1cdf 100644 --- a/arch/arm/boot/compressed/Makefile +++ b/arch/arm/boot/compressed/Makefile @@ -93,11 +93,6 @@ ifeq ($(CONFIG_USE_OF),y) OBJS += $(libfdt_objs) fdt_check_mem_start.o endif -# -fstack-protector-strong triggers protection checks in this code, -# but it is being used too early to link to meaningful stack_chk logic. -$(foreach o, $(libfdt_objs) atags_to_fdt.o fdt_check_mem_start.o, \ - $(eval CFLAGS_$(o) := -I $(srctree)/scripts/dtc/libfdt -fno-stack-protector)) - targets := vmlinux vmlinux.lds piggy_data piggy.o \ lib1funcs.o ashldi3.o bswapsdi2.o \ head.o $(OBJS) @@ -107,6 +102,7 @@ clean-files += lib1funcs.S ashldi3.S bswapsdi2.S hyp-stub.S KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING ccflags-y := -fpic $(call cc-option,-mno-single-pic-base,) -fno-builtin \ + -I$(srctree)/scripts/dtc/libfdt -fno-stack-protector \ -I$(obj) $(DISABLE_ARM_SSP_PER_TASK_PLUGIN) ccflags-remove-$(CONFIG_FUNCTION_TRACER) += -pg asflags-y := -DZIMAGE diff --git a/arch/arm/boot/compressed/misc.c b/arch/arm/boot/compressed/misc.c index e1e9a5dde853..c3c66ff2d696 100644 --- a/arch/arm/boot/compressed/misc.c +++ b/arch/arm/boot/compressed/misc.c @@ -128,13 +128,6 @@ asmlinkage void __div0(void) error("Attempting division by 0!"); } -const unsigned long __stack_chk_guard = 0x000a0dff; - -void __stack_chk_fail(void) -{ - error("stack-protector: Kernel stack is corrupted\n"); -} - extern int do_decompress(u8 *input, int len, u8 *output, void (*error)(char *x)); From f05eb1d24eb564488014d90b31824f71984d7ef5 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 21 Oct 2021 16:16:47 +0200 Subject: [PATCH 02/62] ARM: stackprotector: prefer compiler for TLS based per-task protector Currently, we implement the per-task stack protector for ARM using a GCC plugin, due to lack of native compiler support. However, work is underway to get this implemented in the compiler, which means we will be able to deprecate the GCC plugin at some point. In the meantime, we will need to support both, where the native compiler implementation is obviously preferred. So let's wire this up in Kconfig and the Makefile. Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/Kconfig | 8 ++++++-- arch/arm/Makefile | 9 +++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index f0f9e8bec83a..4e301e4d2da6 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1607,10 +1607,14 @@ config XEN help Say Y if you want to run Linux in a Virtual Machine on Xen on ARM. +config CC_HAVE_STACKPROTECTOR_TLS + def_bool $(cc-option,-mtp=cp15 -mstack-protector-guard=tls -mstack-protector-guard-offset=0) + config STACKPROTECTOR_PER_TASK bool "Use a unique stack canary value for each task" - depends on GCC_PLUGINS && STACKPROTECTOR && THREAD_INFO_IN_TASK && !XIP_DEFLATED_DATA - select GCC_PLUGIN_ARM_SSP_PER_TASK + depends on STACKPROTECTOR && THREAD_INFO_IN_TASK && !XIP_DEFLATED_DATA + depends on GCC_PLUGINS || CC_HAVE_STACKPROTECTOR_TLS + select GCC_PLUGIN_ARM_SSP_PER_TASK if !CC_HAVE_STACKPROTECTOR_TLS default y help Due to the fact that GCC uses an ordinary symbol reference from diff --git a/arch/arm/Makefile b/arch/arm/Makefile index 77172d555c7e..e943624cbf87 100644 --- a/arch/arm/Makefile +++ b/arch/arm/Makefile @@ -275,6 +275,14 @@ endif ifeq ($(CONFIG_STACKPROTECTOR_PER_TASK),y) prepare: stack_protector_prepare +ifeq ($(CONFIG_CC_HAVE_STACKPROTECTOR_TLS),y) +stack_protector_prepare: prepare0 + $(eval KBUILD_CFLAGS += \ + -mstack-protector-guard=tls \ + -mstack-protector-guard-offset=$(shell \ + awk '{if ($$2 == "TSK_STACK_CANARY") print $$3;}'\ + include/generated/asm-offsets.h)) +else stack_protector_prepare: prepare0 $(eval SSP_PLUGIN_CFLAGS := \ -fplugin-arg-arm_ssp_per_task_plugin-offset=$(shell \ @@ -283,6 +291,7 @@ stack_protector_prepare: prepare0 $(eval KBUILD_CFLAGS += $(SSP_PLUGIN_CFLAGS)) $(eval GCC_PLUGINS_CFLAGS += $(SSP_PLUGIN_CFLAGS)) endif +endif all: $(notdir $(KBUILD_IMAGE)) From 08572cd41955166e387d9b4984294d37f8f7526c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 5 Oct 2021 09:15:34 +0200 Subject: [PATCH 03/62] ARM: remove some dead code This code appears to be no longer used so let's get rid of it. Signed-off-by: Ard Biesheuvel Reviewed-by: Arnd Bergmann Acked-by: Linus Walleij Tested-by: Keith Packard Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/include/asm/entry-macro-multi.S | 24 ------------------------ arch/arm/include/asm/smp.h | 5 ----- arch/arm/kernel/smp.c | 5 ----- 3 files changed, 34 deletions(-) diff --git a/arch/arm/include/asm/entry-macro-multi.S b/arch/arm/include/asm/entry-macro-multi.S index dfc6bfa43012..24486dad9e19 100644 --- a/arch/arm/include/asm/entry-macro-multi.S +++ b/arch/arm/include/asm/entry-macro-multi.S @@ -13,28 +13,4 @@ @ badrne lr, 1b bne asm_do_IRQ - -#ifdef CONFIG_SMP - /* - * XXX - * - * this macro assumes that irqstat (r2) and base (r6) are - * preserved from get_irqnr_and_base above - */ - ALT_SMP(test_for_ipi r0, r2, r6, lr) - ALT_UP_B(9997f) - movne r1, sp - badrne lr, 1b - bne do_IPI -#endif -9997: - .endm - - .macro arch_irq_handler, symbol_name - .align 5 - .global \symbol_name -\symbol_name: - mov r8, lr - arch_irq_handler_default - ret r8 .endm diff --git a/arch/arm/include/asm/smp.h b/arch/arm/include/asm/smp.h index f16cbbd5cda4..7c1c90d9f582 100644 --- a/arch/arm/include/asm/smp.h +++ b/arch/arm/include/asm/smp.h @@ -24,11 +24,6 @@ struct seq_file; */ extern void show_ipi_list(struct seq_file *, int); -/* - * Called from assembly code, this handles an IPI. - */ -asmlinkage void do_IPI(int ipinr, struct pt_regs *regs); - /* * Called from C code, this handles an IPI. */ diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index cde5b6d8bac5..9c55ca915ba4 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -631,11 +631,6 @@ static void ipi_complete(unsigned int cpu) /* * Main handler for inter-processor interrupts */ -asmlinkage void __exception_irq_entry do_IPI(int ipinr, struct pt_regs *regs) -{ - handle_IPI(ipinr, regs); -} - static void do_handle_IPI(int ipinr) { unsigned int cpu = smp_processor_id(); From b3ab60b1794c20fe7033d2c8f7dcdef87ada1a23 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 5 Oct 2021 09:15:35 +0200 Subject: [PATCH 04/62] ARM: assembler: introduce bl_r macro Add a bl_r macro that abstract the difference between the ways indirect calls are performed on older and newer ARM architecture revisions. The main difference is to prefer blx instructions over explicit LR assignments when possible, as these tend to confuse the prediction logic in out-of-order cores when speculating across a function return. Signed-off-by: Ard Biesheuvel Reviewed-by: Arnd Bergmann Acked-by: Linus Walleij Tested-by: Keith Packard Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/include/asm/assembler.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 7d23d4bb2168..870bfaea4318 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -624,4 +624,19 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) .endif .endm + /* + * bl_r - branch and link to register + * + * @dst: target to branch to + * @c: conditional opcode suffix + */ + .macro bl_r, dst:req, c + .if __LINUX_ARM_ARCH__ < 6 + mov\c lr, pc + mov\c pc, \dst + .else + blx\c \dst + .endif + .endm + #endif /* __ASM_ASSEMBLER_H__ */ From b6506981f880de87a069167de85935f583a95fc1 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 5 Oct 2021 09:15:37 +0200 Subject: [PATCH 05/62] ARM: unwind: support unwinding across multiple stacks Implement support in the unwinder for dealing with multiple stacks. This will be needed once we add support for IRQ stacks, or for the overflow stack used by the vmap'ed stacks code. This involves tracking the unwind opcodes that either update the virtual stack pointer from another virtual register, or perform an explicit subtract on the virtual stack pointer, and updating the low and high bounds that we use to sanitize the stack pointer accordingly. Signed-off-by: Ard Biesheuvel Reviewed-by: Arnd Bergmann Acked-by: Linus Walleij Tested-by: Keith Packard Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/kernel/unwind.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c index 59fdf257bf8b..9cb9af3fc433 100644 --- a/arch/arm/kernel/unwind.c +++ b/arch/arm/kernel/unwind.c @@ -52,6 +52,7 @@ EXPORT_SYMBOL(__aeabi_unwind_cpp_pr2); struct unwind_ctrl_block { unsigned long vrs[16]; /* virtual register set */ const unsigned long *insn; /* pointer to the current instructions word */ + unsigned long sp_low; /* lowest value of sp allowed */ unsigned long sp_high; /* highest value of sp allowed */ /* * 1 : check for stack overflow for each register pop. @@ -256,8 +257,12 @@ static int unwind_exec_pop_subset_r4_to_r13(struct unwind_ctrl_block *ctrl, mask >>= 1; reg++; } - if (!load_sp) + if (!load_sp) { ctrl->vrs[SP] = (unsigned long)vsp; + } else { + ctrl->sp_low = ctrl->vrs[SP]; + ctrl->sp_high = ALIGN(ctrl->sp_low, THREAD_SIZE); + } return URC_OK; } @@ -313,9 +318,10 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) if ((insn & 0xc0) == 0x00) ctrl->vrs[SP] += ((insn & 0x3f) << 2) + 4; - else if ((insn & 0xc0) == 0x40) + else if ((insn & 0xc0) == 0x40) { ctrl->vrs[SP] -= ((insn & 0x3f) << 2) + 4; - else if ((insn & 0xf0) == 0x80) { + ctrl->sp_low = ctrl->vrs[SP]; + } else if ((insn & 0xf0) == 0x80) { unsigned long mask; insn = (insn << 8) | unwind_get_byte(ctrl); @@ -330,9 +336,11 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) if (ret) goto error; } else if ((insn & 0xf0) == 0x90 && - (insn & 0x0d) != 0x0d) + (insn & 0x0d) != 0x0d) { ctrl->vrs[SP] = ctrl->vrs[insn & 0x0f]; - else if ((insn & 0xf0) == 0xa0) { + ctrl->sp_low = ctrl->vrs[SP]; + ctrl->sp_high = ALIGN(ctrl->sp_low, THREAD_SIZE); + } else if ((insn & 0xf0) == 0xa0) { ret = unwind_exec_pop_r4_to_rN(ctrl, insn); if (ret) goto error; @@ -375,13 +383,12 @@ error: */ int unwind_frame(struct stackframe *frame) { - unsigned long low; const struct unwind_idx *idx; struct unwind_ctrl_block ctrl; /* store the highest address on the stack to avoid crossing it*/ - low = frame->sp; - ctrl.sp_high = ALIGN(low, THREAD_SIZE); + ctrl.sp_low = frame->sp; + ctrl.sp_high = ALIGN(ctrl.sp_low, THREAD_SIZE); pr_debug("%s(pc = %08lx lr = %08lx sp = %08lx)\n", __func__, frame->pc, frame->lr, frame->sp); @@ -437,7 +444,7 @@ int unwind_frame(struct stackframe *frame) urc = unwind_exec_insn(&ctrl); if (urc < 0) return urc; - if (ctrl.vrs[SP] < low || ctrl.vrs[SP] >= ctrl.sp_high) + if (ctrl.vrs[SP] < ctrl.sp_low || ctrl.vrs[SP] > ctrl.sp_high) return -URC_FAILURE; } From 8cdfdf7fe4fec5a952edfb8927ee7cc639c58184 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 5 Oct 2021 09:15:38 +0200 Subject: [PATCH 06/62] ARM: export dump_mem() to other objects The unwind info based stack unwinder will make its own call to dump_mem() to dump the exception stack, so give it external linkage. Signed-off-by: Ard Biesheuvel Reviewed-by: Arnd Bergmann Acked-by: Linus Walleij Tested-by: Keith Packard Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/include/asm/stacktrace.h | 2 ++ arch/arm/kernel/traps.c | 7 +++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/arm/include/asm/stacktrace.h b/arch/arm/include/asm/stacktrace.h index 8f54f9ad8a9b..33ee1aa4b8c0 100644 --- a/arch/arm/include/asm/stacktrace.h +++ b/arch/arm/include/asm/stacktrace.h @@ -36,5 +36,7 @@ void arm_get_current_stackframe(struct pt_regs *regs, struct stackframe *frame) extern int unwind_frame(struct stackframe *frame); extern void walk_stackframe(struct stackframe *frame, int (*fn)(struct stackframe *, void *), void *data); +extern void dump_mem(const char *lvl, const char *str, unsigned long bottom, + unsigned long top); #endif /* __ASM_STACKTRACE_H */ diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 195dff58bafc..e4698605bb72 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include @@ -60,8 +61,6 @@ static int __init user_debug_setup(char *str) __setup("user_debug=", user_debug_setup); #endif -static void dump_mem(const char *, const char *, unsigned long, unsigned long); - void dump_backtrace_entry(unsigned long where, unsigned long from, unsigned long frame, const char *loglvl) { @@ -118,8 +117,8 @@ static int verify_stack(unsigned long sp) /* * Dump out the contents of some memory nicely... */ -static void dump_mem(const char *lvl, const char *str, unsigned long bottom, - unsigned long top) +void dump_mem(const char *lvl, const char *str, unsigned long bottom, + unsigned long top) { unsigned long first; int i; From 4ab6827081c63b83011a18d8e27f621ed34b1194 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 5 Oct 2021 09:15:39 +0200 Subject: [PATCH 07/62] ARM: unwind: dump exception stack from calling frame The existing code that dumps the contents of the pt_regs structure passed to __entry routines does so while unwinding the callee frame, and dereferences the stack pointer as a struct pt_regs*. This will no longer work when we enable support for IRQ or overflow stacks, because the struct pt_regs may live on the task stack, while we are executing from another stack. The unwinder has access to this information, but only while unwinding the calling frame. So let's combine the exception stack dumping code with the handling of the calling frame as well. By printing it before dumping the caller/callee addresses, the output order is preserved. Signed-off-by: Ard Biesheuvel Reviewed-by: Arnd Bergmann Acked-by: Linus Walleij Tested-by: Keith Packard Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/include/asm/stacktrace.h | 10 ++++++++++ arch/arm/kernel/traps.c | 3 ++- arch/arm/kernel/unwind.c | 8 +++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/arch/arm/include/asm/stacktrace.h b/arch/arm/include/asm/stacktrace.h index 33ee1aa4b8c0..d87d60532b86 100644 --- a/arch/arm/include/asm/stacktrace.h +++ b/arch/arm/include/asm/stacktrace.h @@ -18,6 +18,16 @@ struct stackframe { struct llist_node *kr_cur; struct task_struct *tsk; #endif +#ifdef CONFIG_ARM_UNWIND + /* + * This field is used to track the stack pointer value when calling + * __entry routines. This is needed when IRQ stacks and overflow stacks + * are used, because in that case, the struct pt_regs passed to these + * __entry routines may be at the top of the task stack, while we are + * executing from another stack. + */ + unsigned long sp_low; +#endif }; static __always_inline diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index e4698605bb72..89be21ec3b52 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -74,7 +74,8 @@ void dump_backtrace_entry(unsigned long where, unsigned long from, loglvl, where, from); #endif - if (in_entry_text(from) && end <= ALIGN(frame, THREAD_SIZE)) + if (!IS_ENABLED(CONFIG_UNWINDER_ARM) && + in_entry_text(from) && end <= ALIGN(frame, THREAD_SIZE)) dump_mem(loglvl, "Exception stack", frame + 4, end); } diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c index 9cb9af3fc433..b7a6141c342f 100644 --- a/arch/arm/kernel/unwind.c +++ b/arch/arm/kernel/unwind.c @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -459,6 +460,7 @@ int unwind_frame(struct stackframe *frame) frame->sp = ctrl.vrs[SP]; frame->lr = ctrl.vrs[LR]; frame->pc = ctrl.vrs[PC]; + frame->sp_low = ctrl.sp_low; return URC_OK; } @@ -502,7 +504,11 @@ void unwind_backtrace(struct pt_regs *regs, struct task_struct *tsk, urc = unwind_frame(&frame); if (urc < 0) break; - dump_backtrace_entry(where, frame.pc, frame.sp - 4, loglvl); + if (in_entry_text(where)) + dump_mem(loglvl, "Exception stack", frame.sp_low, + frame.sp_low + sizeof(struct pt_regs)); + + dump_backtrace_entry(where, frame.pc, 0, loglvl); } } From eae9523fdd7a6c592e80666681962acbd913cda2 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 15 Oct 2021 16:27:09 +0200 Subject: [PATCH 08/62] ARM: backtrace-clang: avoid crash on bogus frame pointer The Clang backtrace code dereferences the link register value pulled from the stack to decide whether the caller was a branch-and-link instruction, in order to subsequently decode the offset to find the start of the calling function. Unlike other loads in this routine, this one is not protected by a fixup, and may therefore cause a crash if the address in question is bogus. So let's fix this, by treating the fault as a failure to decode the 'bl' instruction. To avoid a label renum, reuse a fixup label that guards an instruction that cannot fault to begin with. Signed-off-by: Ard Biesheuvel Reviewed-by: Nick Desaulniers Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/lib/backtrace-clang.S | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm/lib/backtrace-clang.S b/arch/arm/lib/backtrace-clang.S index 5b2cdb1003e3..5b4bca85d06d 100644 --- a/arch/arm/lib/backtrace-clang.S +++ b/arch/arm/lib/backtrace-clang.S @@ -144,7 +144,7 @@ for_each_frame: tst frame, mask @ Check for address exceptions */ 1003: ldr sv_lr, [sv_fp, #4] @ get saved lr from next frame - ldr r0, [sv_lr, #-4] @ get call instruction +1004: ldr r0, [sv_lr, #-4] @ get call instruction ldr r3, .Lopcode+4 and r2, r3, r0 @ is this a bl call teq r2, r3 @@ -164,7 +164,7 @@ finished_setup: /* * Print the function (sv_pc) and where it was called from (sv_lr). */ -1004: mov r0, sv_pc + mov r0, sv_pc mov r1, sv_lr mov r2, frame @@ -210,7 +210,7 @@ ENDPROC(c_backtrace) .long 1001b, 1006b .long 1002b, 1006b .long 1003b, 1006b - .long 1004b, 1006b + .long 1004b, finished_setup .long 1005b, 1006b .popsection From d4664b6c987f80338407889c1e3f3abe7e16be94 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 5 Oct 2021 09:15:40 +0200 Subject: [PATCH 09/62] ARM: implement IRQ stacks Now that we no longer rely on the stack pointer to access the current task struct or thread info, we can implement support for IRQ stacks cleanly as well. Define a per-CPU IRQ stack and switch to this stack when taking an IRQ, provided that we were not already using that stack in the interrupted context. This is never the case for IRQs taken from user space, but ones taken while running in the kernel could fire while one taken from user space has not completed yet. Signed-off-by: Ard Biesheuvel Acked-by: Linus Walleij Tested-by: Keith Packard Acked-by: Nick Desaulniers Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/Kconfig | 4 +++ arch/arm/include/asm/assembler.h | 4 +++ arch/arm/kernel/entry-armv.S | 54 ++++++++++++++++++++++++++++++-- arch/arm/kernel/irq.c | 23 ++++++++++++++ arch/arm/kernel/traps.c | 15 ++++++++- arch/arm/lib/backtrace-clang.S | 8 +++++ arch/arm/lib/backtrace.S | 8 +++++ 7 files changed, 112 insertions(+), 4 deletions(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 4e301e4d2da6..b11846679cc6 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1166,6 +1166,10 @@ config CURRENT_POINTER_IN_TPIDRURO def_bool y depends on SMP && CPU_32v6K && !CPU_V6 +config IRQSTACKS + def_bool y + depends on GENERIC_IRQ_MULTI_HANDLER && THREAD_INFO_IN_TASK + config ARM_CPU_TOPOLOGY bool "Support cpu topology definition" depends on SMP && CPU_V7 diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 870bfaea4318..1b9d4df331aa 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -86,6 +86,10 @@ #define IMM12_MASK 0xfff +/* the frame pointer used for stack unwinding */ +ARM( fpreg .req r11 ) +THUMB( fpreg .req r7 ) + /* * Enable and disable interrupts */ diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index deff286eb5ea..1c7590eef712 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -36,10 +36,58 @@ /* * Interrupt handling. */ - .macro irq_handler + .macro irq_handler, from_user:req #ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER mov r0, sp +#ifdef CONFIG_IRQSTACKS + mov_l r2, irq_stack_ptr @ Take base address + mrc p15, 0, r3, c13, c0, 4 @ Get CPU offset +#ifdef CONFIG_UNWINDER_ARM + mov fpreg, sp @ Preserve original SP +#else + mov r8, fp @ Preserve original FP + mov r9, sp @ Preserve original SP +#endif + ldr sp, [r2, r3] @ Load SP from per-CPU var + .if \from_user == 0 +UNWIND( .setfp fpreg, sp ) + @ + @ If we took the interrupt while running in the kernel, we may already + @ be using the IRQ stack, so revert to the original value in that case. + @ + subs r2, sp, r0 @ SP above bottom of IRQ stack? + rsbscs r2, r2, #THREAD_SIZE @ ... and below the top? + movcs sp, r0 @ If so, revert to incoming SP + +#ifndef CONFIG_UNWINDER_ARM + @ + @ Inform the frame pointer unwinder where the next frame lives + @ + movcc lr, pc @ Make LR point into .entry.text so + @ that we will get a dump of the + @ exception stack for this frame. +#ifdef CONFIG_CC_IS_GCC + movcc ip, r0 @ Store the old SP in the frame record. + stmdbcc sp!, {fp, ip, lr, pc} @ Push frame record + addcc fp, sp, #12 +#else + stmdbcc sp!, {fp, lr} @ Push frame record + movcc fp, sp +#endif // CONFIG_CC_IS_GCC +#endif // CONFIG_UNWINDER_ARM + .endif +#endif // CONFIG_IRQSTACKS + bl generic_handle_arch_irq + +#ifdef CONFIG_IRQSTACKS +#ifdef CONFIG_UNWINDER_ARM + mov sp, fpreg @ Restore original SP +#else + mov fp, r8 @ Restore original FP + mov sp, r9 @ Restore original SP +#endif // CONFIG_UNWINDER_ARM +#endif // CONFIG_IRQSTACKS #else arch_irq_handler_default #endif @@ -199,7 +247,7 @@ ENDPROC(__dabt_svc) .align 5 __irq_svc: svc_entry - irq_handler + irq_handler from_user=0 #ifdef CONFIG_PREEMPTION ldr r8, [tsk, #TI_PREEMPT] @ get preempt count @@ -426,7 +474,7 @@ ENDPROC(__dabt_usr) __irq_usr: usr_entry kuser_cmpxchg_check - irq_handler + irq_handler from_user=1 get_thread_info tsk mov why, #0 b ret_to_user_from_irq diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index b79975bd988c..abb0aa679bba 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -43,6 +43,25 @@ unsigned long irq_err_count; +#ifdef CONFIG_IRQSTACKS + +asmlinkage DEFINE_PER_CPU_READ_MOSTLY(u8 *, irq_stack_ptr); + +static void __init init_irq_stacks(void) +{ + u8 *stack; + int cpu; + + for_each_possible_cpu(cpu) { + stack = (u8 *)__get_free_pages(GFP_KERNEL, THREAD_SIZE_ORDER); + if (WARN_ON(!stack)) + break; + per_cpu(irq_stack_ptr, cpu) = &stack[THREAD_SIZE]; + } +} + +#endif + int arch_show_interrupts(struct seq_file *p, int prec) { #ifdef CONFIG_FIQ @@ -101,6 +120,10 @@ void __init init_IRQ(void) { int ret; +#ifdef CONFIG_IRQSTACKS + init_irq_stacks(); +#endif + if (IS_ENABLED(CONFIG_OF) && !machine_desc->init_irq) irqchip_init(); else diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 89be21ec3b52..b42c446cec9a 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -66,6 +66,19 @@ void dump_backtrace_entry(unsigned long where, unsigned long from, { unsigned long end = frame + 4 + sizeof(struct pt_regs); + if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER) && + IS_ENABLED(CONFIG_CC_IS_GCC) && + end > ALIGN(frame, THREAD_SIZE)) { + /* + * If we are walking past the end of the stack, it may be due + * to the fact that we are on an IRQ or overflow stack. In this + * case, we can load the address of the other stack from the + * frame record. + */ + frame = ((unsigned long *)frame)[-2] - 4; + end = frame + 4 + sizeof(struct pt_regs); + } + #ifdef CONFIG_KALLSYMS printk("%s[<%08lx>] (%ps) from [<%08lx>] (%pS)\n", loglvl, where, (void *)where, from, (void *)from); @@ -278,7 +291,7 @@ static int __die(const char *str, int err, struct pt_regs *regs) if (!user_mode(regs) || in_interrupt()) { dump_mem(KERN_EMERG, "Stack: ", regs->ARM_sp, - THREAD_SIZE + (unsigned long)task_stack_page(tsk)); + ALIGN(regs->ARM_sp, THREAD_SIZE)); dump_backtrace(regs, tsk, KERN_EMERG); dump_instr(KERN_EMERG, regs); } diff --git a/arch/arm/lib/backtrace-clang.S b/arch/arm/lib/backtrace-clang.S index 5b4bca85d06d..290c52a60fc6 100644 --- a/arch/arm/lib/backtrace-clang.S +++ b/arch/arm/lib/backtrace-clang.S @@ -197,6 +197,14 @@ finished_setup: cmp sv_fp, frame @ next frame must be mov frame, sv_fp @ above the current frame +#ifdef CONFIG_IRQSTACKS + @ + @ Kernel stacks may be discontiguous in memory. If the next + @ frame is below the previous frame, accept it as long as it + @ lives in kernel memory. + @ + cmpls sv_fp, #PAGE_OFFSET +#endif bhi for_each_frame 1006: adr r0, .Lbad diff --git a/arch/arm/lib/backtrace.S b/arch/arm/lib/backtrace.S index e8408f22d4dc..293a2716bd20 100644 --- a/arch/arm/lib/backtrace.S +++ b/arch/arm/lib/backtrace.S @@ -98,6 +98,14 @@ for_each_frame: tst frame, mask @ Check for address exceptions cmp sv_fp, frame @ next frame must be mov frame, sv_fp @ above the current frame +#ifdef CONFIG_IRQSTACKS + @ + @ Kernel stacks may be discontiguous in memory. If the next + @ frame is below the previous frame, accept it as long as it + @ lives in kernel memory. + @ + cmpls sv_fp, #PAGE_OFFSET +#endif bhi for_each_frame 1006: adr r0, .Lbad From 0b78f2e92d0cf722baa507c20948f1b1d6baf83d Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 5 Oct 2021 09:15:41 +0200 Subject: [PATCH 10/62] ARM: call_with_stack: add unwind support Restructure the code and add the unwind annotations so that both the frame pointer unwinder as well as the EHABI unwind info based unwinder will be able to follow the call stack through call_with_stack(). Since GCC and Clang use different formats for the stack frame, two methods are implemented: a GCC version that pushes fp, sp, lr and pc for compatibility with the frame pointer unwinder, and a second version that works with Clang, as well as with the EHABI unwinder both in ARM and Thumb2 modes. Signed-off-by: Ard Biesheuvel Acked-by: Linus Walleij Tested-by: Keith Packard Reviewed-by: Nick Desaulniers Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/lib/call_with_stack.S | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/arch/arm/lib/call_with_stack.S b/arch/arm/lib/call_with_stack.S index 28b0341ae786..0a268a6c513c 100644 --- a/arch/arm/lib/call_with_stack.S +++ b/arch/arm/lib/call_with_stack.S @@ -8,25 +8,42 @@ #include #include +#include /* * void call_with_stack(void (*fn)(void *), void *arg, void *sp) * * Change the stack to that pointed at by sp, then invoke fn(arg) with * the new stack. + * + * The sequence below follows the APCS frame convention for frame pointer + * unwinding, and implements the unwinder annotations needed by the EABI + * unwinder. */ -ENTRY(call_with_stack) - str sp, [r2, #-4]! - str lr, [r2, #-4]! +ENTRY(call_with_stack) +#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC) + mov ip, sp + push {fp, ip, lr, pc} + sub fp, ip, #4 +#else +UNWIND( .fnstart ) +UNWIND( .save {fpreg, lr} ) + push {fpreg, lr} +UNWIND( .setfp fpreg, sp ) + mov fpreg, sp +#endif mov sp, r2 mov r2, r0 mov r0, r1 - badr lr, 1f - ret r2 + bl_r r2 -1: ldr lr, [sp] - ldr sp, [sp, #4] - ret lr +#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC) + ldmdb fp, {fp, sp, pc} +#else + mov sp, fpreg + pop {fpreg, pc} +UNWIND( .fnend ) +#endif ENDPROC(call_with_stack) From 9974f857768e4ea4f18c4f6eec37178d546365ec Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 5 Oct 2021 09:15:42 +0200 Subject: [PATCH 11/62] ARM: run softirqs on the per-CPU IRQ stack Now that we have enabled IRQ stacks, any softIRQs that are handled over the back of a hard IRQ will run from the IRQ stack as well. However, any synchronous softirq processing that happens when re-enabling softIRQs from task context will still execute on that task's stack. Since any call to local_bh_enable() at any level in the task's call stack may trigger a softIRQ processing run, which could potentially cause a task stack overflow if the combined stack footprints exceed the stack's size, let's run these synchronous invocations of do_softirq() on the IRQ stack as well. Signed-off-by: Ard Biesheuvel Reviewed-by: Arnd Bergmann Acked-by: Linus Walleij Tested-by: Keith Packard Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/Kconfig | 2 ++ arch/arm/kernel/irq.c | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index b11846679cc6..41849d5e0bc3 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1169,6 +1169,8 @@ config CURRENT_POINTER_IN_TPIDRURO config IRQSTACKS def_bool y depends on GENERIC_IRQ_MULTI_HANDLER && THREAD_INFO_IN_TASK + select HAVE_IRQ_EXIT_ON_IRQ_STACK + select HAVE_SOFTIRQ_ON_OWN_STACK config ARM_CPU_TOPOLOGY bool "Support cpu topology definition" diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index abb0aa679bba..e05219bca218 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -36,11 +36,14 @@ #include #include #include +#include #include #include #include #include +#include "reboot.h" + unsigned long irq_err_count; #ifdef CONFIG_IRQSTACKS @@ -60,6 +63,17 @@ static void __init init_irq_stacks(void) } } +static void ____do_softirq(void *arg) +{ + __do_softirq(); +} + +void do_softirq_own_stack(void) +{ + call_with_stack(____do_softirq, NULL, + __this_cpu_read(irq_stack_ptr)); +} + #endif int arch_show_interrupts(struct seq_file *p, int prec) From ba999a0402745ae628b5142ecba690a0aaebf201 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 3 Oct 2021 19:05:53 +0200 Subject: [PATCH 12/62] ARM: memcpy: use frame pointer as unwind anchor The memcpy template is a bit unusual in the way it manages the stack pointer: depending on the execution path through the function, the SP assumes different values as different subsets of the register file are preserved and restored again. This is problematic when it comes to EHABI unwind info, as it is not instruction accurate, and does not allow tracking the SP value as it changes. Commit 279f487e0b471 ("ARM: 8225/1: Add unwinding support for memory copy functions") addressed this by carving up the function in different chunks as far as the unwinder is concerned, and keeping a set of unwind directives for each of them, each corresponding with the state of the stack pointer during execution of the chunk in question. This not only duplicates unwind info unnecessarily, but it also complicates unwinding the stack upon overflow. Instead, let's do what the compiler does when the SP is updated halfway through a function, which is to use a frame pointer and emit the appropriate unwind directives to communicate this to the unwinder. Note that Thumb-2 uses R7 for this, while ARM uses R11 aka FP. So let's avoid touching R7 in the body of the template, so that Thumb-2 can use it as the frame pointer. R11 was not modified in the first place. Signed-off-by: Ard Biesheuvel Tested-by: Keith Packard Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/lib/copy_from_user.S | 13 +++---- arch/arm/lib/copy_template.S | 67 ++++++++++++----------------------- arch/arm/lib/copy_to_user.S | 13 +++---- arch/arm/lib/memcpy.S | 13 +++---- 4 files changed, 38 insertions(+), 68 deletions(-) diff --git a/arch/arm/lib/copy_from_user.S b/arch/arm/lib/copy_from_user.S index 480a20766137..270de7debd0f 100644 --- a/arch/arm/lib/copy_from_user.S +++ b/arch/arm/lib/copy_from_user.S @@ -91,18 +91,15 @@ strb\cond \reg, [\ptr], #1 .endm - .macro enter reg1 reg2 + .macro enter regs:vararg mov r3, #0 - stmdb sp!, {r0, r2, r3, \reg1, \reg2} +UNWIND( .save {r0, r2, r3, \regs} ) + stmdb sp!, {r0, r2, r3, \regs} .endm - .macro usave reg1 reg2 - UNWIND( .save {r0, r2, r3, \reg1, \reg2} ) - .endm - - .macro exit reg1 reg2 + .macro exit regs:vararg add sp, sp, #8 - ldmfd sp!, {r0, \reg1, \reg2} + ldmfd sp!, {r0, \regs} .endm .text diff --git a/arch/arm/lib/copy_template.S b/arch/arm/lib/copy_template.S index 810a805d36dc..8fbafb074fe9 100644 --- a/arch/arm/lib/copy_template.S +++ b/arch/arm/lib/copy_template.S @@ -69,13 +69,10 @@ * than one 32bit instruction in Thumb-2) */ - UNWIND( .fnstart ) - enter r4, lr - UNWIND( .fnend ) - - UNWIND( .fnstart ) - usave r4, lr @ in first stmdb block + enter r4, UNWIND(fpreg,) lr + UNWIND( .setfp fpreg, sp ) + UNWIND( mov fpreg, sp ) subs r2, r2, #4 blt 8f @@ -86,12 +83,7 @@ bne 10f 1: subs r2, r2, #(28) - stmfd sp!, {r5 - r8} - UNWIND( .fnend ) - - UNWIND( .fnstart ) - usave r4, lr - UNWIND( .save {r5 - r8} ) @ in second stmfd block + stmfd sp!, {r5, r6, r8, r9} blt 5f CALGN( ands ip, r0, #31 ) @@ -110,9 +102,9 @@ PLD( pld [r1, #92] ) 3: PLD( pld [r1, #124] ) -4: ldr8w r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f +4: ldr8w r1, r3, r4, r5, r6, r8, r9, ip, lr, abort=20f subs r2, r2, #32 - str8w r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f + str8w r0, r3, r4, r5, r6, r8, r9, ip, lr, abort=20f bge 3b PLD( cmn r2, #96 ) PLD( bge 4b ) @@ -132,8 +124,8 @@ ldr1w r1, r4, abort=20f ldr1w r1, r5, abort=20f ldr1w r1, r6, abort=20f - ldr1w r1, r7, abort=20f ldr1w r1, r8, abort=20f + ldr1w r1, r9, abort=20f ldr1w r1, lr, abort=20f #if LDR1W_SHIFT < STR1W_SHIFT @@ -150,17 +142,14 @@ str1w r0, r4, abort=20f str1w r0, r5, abort=20f str1w r0, r6, abort=20f - str1w r0, r7, abort=20f str1w r0, r8, abort=20f + str1w r0, r9, abort=20f str1w r0, lr, abort=20f CALGN( bcs 2b ) -7: ldmfd sp!, {r5 - r8} - UNWIND( .fnend ) @ end of second stmfd block +7: ldmfd sp!, {r5, r6, r8, r9} - UNWIND( .fnstart ) - usave r4, lr @ still in first stmdb block 8: movs r2, r2, lsl #31 ldr1b r1, r3, ne, abort=21f ldr1b r1, r4, cs, abort=21f @@ -169,7 +158,7 @@ str1b r0, r4, cs, abort=21f str1b r0, ip, cs, abort=21f - exit r4, pc + exit r4, UNWIND(fpreg,) pc 9: rsb ip, ip, #4 cmp ip, #2 @@ -189,13 +178,10 @@ ldr1w r1, lr, abort=21f beq 17f bgt 18f - UNWIND( .fnend ) .macro forward_copy_shift pull push - UNWIND( .fnstart ) - usave r4, lr @ still in first stmdb block subs r2, r2, #28 blt 14f @@ -205,12 +191,8 @@ CALGN( subcc r2, r2, ip ) CALGN( bcc 15f ) -11: stmfd sp!, {r5 - r9} - UNWIND( .fnend ) +11: stmfd sp!, {r5, r6, r8 - r10} - UNWIND( .fnstart ) - usave r4, lr - UNWIND( .save {r5 - r9} ) @ in new second stmfd block PLD( pld [r1, #0] ) PLD( subs r2, r2, #96 ) PLD( pld [r1, #28] ) @@ -219,35 +201,32 @@ PLD( pld [r1, #92] ) 12: PLD( pld [r1, #124] ) -13: ldr4w r1, r4, r5, r6, r7, abort=19f +13: ldr4w r1, r4, r5, r6, r8, abort=19f mov r3, lr, lspull #\pull subs r2, r2, #32 - ldr4w r1, r8, r9, ip, lr, abort=19f + ldr4w r1, r9, r10, ip, lr, abort=19f orr r3, r3, r4, lspush #\push mov r4, r4, lspull #\pull orr r4, r4, r5, lspush #\push mov r5, r5, lspull #\pull orr r5, r5, r6, lspush #\push mov r6, r6, lspull #\pull - orr r6, r6, r7, lspush #\push - mov r7, r7, lspull #\pull - orr r7, r7, r8, lspush #\push + orr r6, r6, r8, lspush #\push mov r8, r8, lspull #\pull orr r8, r8, r9, lspush #\push mov r9, r9, lspull #\pull - orr r9, r9, ip, lspush #\push + orr r9, r9, r10, lspush #\push + mov r10, r10, lspull #\pull + orr r10, r10, ip, lspush #\push mov ip, ip, lspull #\pull orr ip, ip, lr, lspush #\push - str8w r0, r3, r4, r5, r6, r7, r8, r9, ip, abort=19f + str8w r0, r3, r4, r5, r6, r8, r9, r10, ip, abort=19f bge 12b PLD( cmn r2, #96 ) PLD( bge 13b ) - ldmfd sp!, {r5 - r9} - UNWIND( .fnend ) @ end of the second stmfd block + ldmfd sp!, {r5, r6, r8 - r10} - UNWIND( .fnstart ) - usave r4, lr @ still in first stmdb block 14: ands ip, r2, #28 beq 16f @@ -262,7 +241,6 @@ 16: sub r1, r1, #(\push / 8) b 8b - UNWIND( .fnend ) .endm @@ -273,6 +251,7 @@ 18: forward_copy_shift pull=24 push=8 + UNWIND( .fnend ) /* * Abort preamble and completion macros. @@ -282,13 +261,13 @@ */ .macro copy_abort_preamble -19: ldmfd sp!, {r5 - r9} +19: ldmfd sp!, {r5, r6, r8 - r10} b 21f -20: ldmfd sp!, {r5 - r8} +20: ldmfd sp!, {r5, r6, r8, r9} 21: .endm .macro copy_abort_end - ldmfd sp!, {r4, pc} + ldmfd sp!, {r4, UNWIND(fpreg,) pc} .endm diff --git a/arch/arm/lib/copy_to_user.S b/arch/arm/lib/copy_to_user.S index 842ea5ede485..fac49e57cc0b 100644 --- a/arch/arm/lib/copy_to_user.S +++ b/arch/arm/lib/copy_to_user.S @@ -90,18 +90,15 @@ strusr \reg, \ptr, 1, \cond, abort=\abort .endm - .macro enter reg1 reg2 + .macro enter regs:vararg mov r3, #0 - stmdb sp!, {r0, r2, r3, \reg1, \reg2} +UNWIND( .save {r0, r2, r3, \regs} ) + stmdb sp!, {r0, r2, r3, \regs} .endm - .macro usave reg1 reg2 - UNWIND( .save {r0, r2, r3, \reg1, \reg2} ) - .endm - - .macro exit reg1 reg2 + .macro exit regs:vararg add sp, sp, #8 - ldmfd sp!, {r0, \reg1, \reg2} + ldmfd sp!, {r0, \regs} .endm .text diff --git a/arch/arm/lib/memcpy.S b/arch/arm/lib/memcpy.S index e4caf48c089f..90f2b645aa0d 100644 --- a/arch/arm/lib/memcpy.S +++ b/arch/arm/lib/memcpy.S @@ -42,16 +42,13 @@ strb\cond \reg, [\ptr], #1 .endm - .macro enter reg1 reg2 - stmdb sp!, {r0, \reg1, \reg2} + .macro enter regs:vararg +UNWIND( .save {r0, \regs} ) + stmdb sp!, {r0, \regs} .endm - .macro usave reg1 reg2 - UNWIND( .save {r0, \reg1, \reg2} ) - .endm - - .macro exit reg1 reg2 - ldmfd sp!, {r0, \reg1, \reg2} + .macro exit regs:vararg + ldmfd sp!, {r0, \regs} .endm .text From ccb81601acc092711bfb75787bce467b7dbef4b2 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 4 Oct 2021 11:29:11 +0200 Subject: [PATCH 13/62] ARM: memmove: use frame pointer as unwind anchor The memmove routine is a bit unusual in the way it manages the stack pointer: depending on the execution path through the function, the SP assumes different values as different subsets of the register file are preserved and restored again. This is problematic when it comes to EHABI unwind info, as it is not instruction accurate, and does not allow tracking the SP value as it changes. Commit 207a6cb06990c ("ARM: 8224/1: Add unwinding support for memmove function") addressed this by carving up the function in different chunks as far as the unwinder is concerned, and keeping a set of unwind directives for each of them, each corresponding with the state of the stack pointer during execution of the chunk in question. This not only duplicates unwind info unnecessarily, but it also complicates unwinding the stack upon overflow. Instead, let's do what the compiler does when the SP is updated halfway through a function, which is to use a frame pointer and emit the appropriate unwind directives to communicate this to the unwinder. Note that Thumb-2 uses R7 for this, while ARM uses R11 aka FP. So let's avoid touching R7 in the body of the function, so that Thumb-2 can use it as the frame pointer. R11 was not modified in the first place. Signed-off-by: Ard Biesheuvel Tested-by: Keith Packard Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/lib/memmove.S | 60 ++++++++++++++---------------------------- 1 file changed, 20 insertions(+), 40 deletions(-) diff --git a/arch/arm/lib/memmove.S b/arch/arm/lib/memmove.S index 6fecc12a1f51..6410554039fd 100644 --- a/arch/arm/lib/memmove.S +++ b/arch/arm/lib/memmove.S @@ -31,12 +31,13 @@ WEAK(memmove) subs ip, r0, r1 cmphi r2, ip bls __memcpy - - stmfd sp!, {r0, r4, lr} UNWIND( .fnend ) UNWIND( .fnstart ) - UNWIND( .save {r0, r4, lr} ) @ in first stmfd block + UNWIND( .save {r0, r4, fpreg, lr} ) + stmfd sp!, {r0, r4, UNWIND(fpreg,) lr} + UNWIND( .setfp fpreg, sp ) + UNWIND( mov fpreg, sp ) add r1, r1, r2 add r0, r0, r2 subs r2, r2, #4 @@ -48,12 +49,7 @@ WEAK(memmove) bne 10f 1: subs r2, r2, #(28) - stmfd sp!, {r5 - r8} - UNWIND( .fnend ) - - UNWIND( .fnstart ) - UNWIND( .save {r0, r4, lr} ) - UNWIND( .save {r5 - r8} ) @ in second stmfd block + stmfd sp!, {r5, r6, r8, r9} blt 5f CALGN( ands ip, r0, #31 ) @@ -72,9 +68,9 @@ WEAK(memmove) PLD( pld [r1, #-96] ) 3: PLD( pld [r1, #-128] ) -4: ldmdb r1!, {r3, r4, r5, r6, r7, r8, ip, lr} +4: ldmdb r1!, {r3, r4, r5, r6, r8, r9, ip, lr} subs r2, r2, #32 - stmdb r0!, {r3, r4, r5, r6, r7, r8, ip, lr} + stmdb r0!, {r3, r4, r5, r6, r8, r9, ip, lr} bge 3b PLD( cmn r2, #96 ) PLD( bge 4b ) @@ -88,8 +84,8 @@ WEAK(memmove) W(ldr) r4, [r1, #-4]! W(ldr) r5, [r1, #-4]! W(ldr) r6, [r1, #-4]! - W(ldr) r7, [r1, #-4]! W(ldr) r8, [r1, #-4]! + W(ldr) r9, [r1, #-4]! W(ldr) lr, [r1, #-4]! add pc, pc, ip @@ -99,17 +95,13 @@ WEAK(memmove) W(str) r4, [r0, #-4]! W(str) r5, [r0, #-4]! W(str) r6, [r0, #-4]! - W(str) r7, [r0, #-4]! W(str) r8, [r0, #-4]! + W(str) r9, [r0, #-4]! W(str) lr, [r0, #-4]! CALGN( bcs 2b ) -7: ldmfd sp!, {r5 - r8} - UNWIND( .fnend ) @ end of second stmfd block - - UNWIND( .fnstart ) - UNWIND( .save {r0, r4, lr} ) @ still in first stmfd block +7: ldmfd sp!, {r5, r6, r8, r9} 8: movs r2, r2, lsl #31 ldrbne r3, [r1, #-1]! @@ -118,7 +110,7 @@ WEAK(memmove) strbne r3, [r0, #-1]! strbcs r4, [r0, #-1]! strbcs ip, [r0, #-1] - ldmfd sp!, {r0, r4, pc} + ldmfd sp!, {r0, r4, UNWIND(fpreg,) pc} 9: cmp ip, #2 ldrbgt r3, [r1, #-1]! @@ -137,13 +129,10 @@ WEAK(memmove) ldr r3, [r1, #0] beq 17f blt 18f - UNWIND( .fnend ) .macro backward_copy_shift push pull - UNWIND( .fnstart ) - UNWIND( .save {r0, r4, lr} ) @ still in first stmfd block subs r2, r2, #28 blt 14f @@ -152,12 +141,7 @@ WEAK(memmove) CALGN( subcc r2, r2, ip ) CALGN( bcc 15f ) -11: stmfd sp!, {r5 - r9} - UNWIND( .fnend ) - - UNWIND( .fnstart ) - UNWIND( .save {r0, r4, lr} ) - UNWIND( .save {r5 - r9} ) @ in new second stmfd block +11: stmfd sp!, {r5, r6, r8 - r10} PLD( pld [r1, #-4] ) PLD( subs r2, r2, #96 ) @@ -167,35 +151,31 @@ WEAK(memmove) PLD( pld [r1, #-96] ) 12: PLD( pld [r1, #-128] ) -13: ldmdb r1!, {r7, r8, r9, ip} +13: ldmdb r1!, {r8, r9, r10, ip} mov lr, r3, lspush #\push subs r2, r2, #32 ldmdb r1!, {r3, r4, r5, r6} orr lr, lr, ip, lspull #\pull mov ip, ip, lspush #\push - orr ip, ip, r9, lspull #\pull + orr ip, ip, r10, lspull #\pull + mov r10, r10, lspush #\push + orr r10, r10, r9, lspull #\pull mov r9, r9, lspush #\push orr r9, r9, r8, lspull #\pull mov r8, r8, lspush #\push - orr r8, r8, r7, lspull #\pull - mov r7, r7, lspush #\push - orr r7, r7, r6, lspull #\pull + orr r8, r8, r6, lspull #\pull mov r6, r6, lspush #\push orr r6, r6, r5, lspull #\pull mov r5, r5, lspush #\push orr r5, r5, r4, lspull #\pull mov r4, r4, lspush #\push orr r4, r4, r3, lspull #\pull - stmdb r0!, {r4 - r9, ip, lr} + stmdb r0!, {r4 - r6, r8 - r10, ip, lr} bge 12b PLD( cmn r2, #96 ) PLD( bge 13b ) - ldmfd sp!, {r5 - r9} - UNWIND( .fnend ) @ end of the second stmfd block - - UNWIND( .fnstart ) - UNWIND( .save {r0, r4, lr} ) @ still in first stmfd block + ldmfd sp!, {r5, r6, r8 - r10} 14: ands ip, r2, #28 beq 16f @@ -211,7 +191,6 @@ WEAK(memmove) 16: add r1, r1, #(\pull / 8) b 8b - UNWIND( .fnend ) .endm @@ -222,5 +201,6 @@ WEAK(memmove) 18: backward_copy_shift push=24 pull=8 + UNWIND( .fnend ) ENDPROC(memmove) ENDPROC(__memmove) From ad3d09b54711ba3c5b3177ecc93943265e7bb762 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 4 Oct 2021 11:34:05 +0200 Subject: [PATCH 14/62] ARM: memset: clean up unwind annotations The memset implementation carves up the code in different sections, each covered with their own unwind info. In this case, it is done in a way similar to how the compiler might do it, to disambiguate between parts where the return address is in LR and the SP is unmodified, and parts where a stack frame is live, and the unwinder needs to know the size of the stack frame and the location of the return address within it. Only the placement of the unwind directives is slightly odd: the stack pushes are placed in the wrong sections, which may confuse the unwinder when attempting to unwind with PC pointing at the stack push in question. So let's fix this up, by reordering the directives and instructions as appropriate. Signed-off-by: Ard Biesheuvel Tested-by: Keith Packard Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/lib/memset.S | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/arch/arm/lib/memset.S b/arch/arm/lib/memset.S index 9817cb258c1a..d71ab61430b2 100644 --- a/arch/arm/lib/memset.S +++ b/arch/arm/lib/memset.S @@ -28,16 +28,16 @@ UNWIND( .fnstart ) mov r3, r1 7: cmp r2, #16 blt 4f +UNWIND( .fnend ) #if ! CALGN(1)+0 /* * We need 2 extra registers for this loop - use r8 and the LR */ - stmfd sp!, {r8, lr} -UNWIND( .fnend ) UNWIND( .fnstart ) UNWIND( .save {r8, lr} ) + stmfd sp!, {r8, lr} mov r8, r1 mov lr, r3 @@ -66,10 +66,9 @@ UNWIND( .fnend ) * whole cache lines at once. */ - stmfd sp!, {r4-r8, lr} -UNWIND( .fnend ) UNWIND( .fnstart ) UNWIND( .save {r4-r8, lr} ) + stmfd sp!, {r4-r8, lr} mov r4, r1 mov r5, r3 mov r6, r1 From 532319b9c418fc2be532c76b253b9a09f5d49dd1 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 4 Oct 2021 08:46:38 +0200 Subject: [PATCH 15/62] ARM: unwind: disregard unwind info before stack frame is set up When unwinding the stack from a stack overflow, we are likely to start from a stack push instruction, given that this is the most common way to grow the stack for compiler emitted code. This push instruction rarely appears anywhere else than at offset 0x0 of the function, and if it doesn't, the compiler tends to split up the unwind annotations, given that the stack frame layout is apparently not the same throughout the function. This means that, in the general case, if the frame's PC points at the first instruction covered by a certain unwind entry, there is no way the stack frame that the unwind entry describes could have been created yet, and so we are still on the stack frame of the caller in that case. So treat this as a special case, and return with the new PC taken from the frame's LR, without applying the unwind transformations to the virtual register set. This permits us to unwind the call stack on stack overflow when the overflow was caused by a stack push on function entry. Signed-off-by: Ard Biesheuvel Tested-by: Keith Packard Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/kernel/unwind.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c index b7a6141c342f..e8d729975f12 100644 --- a/arch/arm/kernel/unwind.c +++ b/arch/arm/kernel/unwind.c @@ -411,7 +411,21 @@ int unwind_frame(struct stackframe *frame) if (idx->insn == 1) /* can't unwind */ return -URC_FAILURE; - else if ((idx->insn & 0x80000000) == 0) + else if (frame->pc == prel31_to_addr(&idx->addr_offset)) { + /* + * Unwinding is tricky when we're halfway through the prologue, + * since the stack frame that the unwinder expects may not be + * fully set up yet. However, one thing we do know for sure is + * that if we are unwinding from the very first instruction of + * a function, we are still effectively in the stack frame of + * the caller, and the unwind info has no relevance yet. + */ + if (frame->pc == frame->lr) + return -URC_FAILURE; + frame->sp_low = frame->sp; + frame->pc = frame->lr; + return URC_OK; + } else if ((idx->insn & 0x80000000) == 0) /* prel31 to the unwind table */ ctrl.insn = (unsigned long *)prel31_to_addr(&idx->insn); else if ((idx->insn & 0xff000000) == 0x80000000) From b832faec33d4e27c32490c0732098ecf79b222af Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sat, 16 Oct 2021 18:00:01 +0200 Subject: [PATCH 16/62] ARM: switch_to: clean up Thumb2 code path The load-multiple instruction that essentially performs the switch_to operation in ARM mode, by loading all callee save registers as well the stack pointer and the program counter, is split into 3 separate loads for Thumb-2, with the IP register used as a temporary to capture the value of R4 before it gets overwritten. We can clean this up a bit, by sticking with a single LDMIA instruction, but one that pops SP and PC into IP and LR, respectively, and by using ordinary move register and branch instructions to get those values into SP and PC. This also allows us to move the set_current call closer to the assignment of SP, reducing the window where those are mutually out of sync. This is especially relevant for CONFIG_VMAP_STACK, which is being introduced in a subsequent patch, where we need to issue a load that might fault from the new stack while running from the old one, to ensure that stale PMD entries in the VMALLOC space are synced up. Signed-off-by: Ard Biesheuvel Tested-by: Keith Packard Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/kernel/entry-armv.S | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 1c7590eef712..ce8ca29461de 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -823,13 +823,26 @@ ENTRY(__switch_to) #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) str r7, [r8] #endif - THUMB( mov ip, r4 ) mov r0, r5 +#if !defined(CONFIG_THUMB2_KERNEL) set_current r7 - ARM( ldmia r4, {r4 - sl, fp, sp, pc} ) @ Load all regs saved previously - THUMB( ldmia ip!, {r4 - sl, fp} ) @ Load all regs saved previously - THUMB( ldr sp, [ip], #4 ) - THUMB( ldr pc, [ip] ) + ldmia r4, {r4 - sl, fp, sp, pc} @ Load all regs saved previously +#else + mov r1, r7 + ldmia r4, {r4 - sl, fp, ip, lr} @ Load all regs saved previously + + @ When CONFIG_THREAD_INFO_IN_TASK=n, the update of SP itself is what + @ effectuates the task switch, as that is what causes the observable + @ values of current and current_thread_info to change. When + @ CONFIG_THREAD_INFO_IN_TASK=y, setting current (and therefore + @ current_thread_info) is done explicitly, and the update of SP just + @ switches us to another stack, with few other side effects. In order + @ to prevent this distinction from causing any inconsistencies, let's + @ keep the 'set_current' call as close as we can to the update of SP. + set_current r1 + mov sp, ip + ret lr +#endif UNWIND(.fnend ) ENDPROC(__switch_to) From ae5cc07da8f8e4c9802894c05f1d96802b9de5f2 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 17 Oct 2021 17:23:47 +0200 Subject: [PATCH 17/62] ARM: entry: rework stack realignment code in svc_entry The original Thumb-2 enablement patches updated the stack realignment code in svc_entry to work around the lack of a STMIB instruction in Thumb-2, by subtracting 4 from the frame size, inverting the sense of the misaligment check, and changing to a STMIA instruction and a final stack push of a 4 byte quantity that results in the stack becoming aligned at the end of the sequence. It also pushes and pops R0 to the stack in order to have a temp register that Thumb-2 allows in general purpose ALU instructions, as TST using SP is not permitted. Both are a bit problematic for vmap'ed stacks, as using the stack is only permitted after we decide that we did not overflow the stack, or have already switched to the overflow stack. As for the alignment check: the current approach creates a corner case where, if the initial SUB of SP ends up right at the start of the stack, we will end up subtracting another 8 bytes and overflowing it. This means we would need to add the overflow check *after* the SUB that deliberately misaligns the stack. However, this would require us to keep local state (i.e., whether we performed the subtract or not) across the overflow check, but without any GPRs or stack available. So let's switch to an approach where we don't use the stack, and where the alignment check of the stack pointer occurs in the usual way, as this is guaranteed not to result in overflow. This means we will be able to do the overflow check first. While at it, switch to R1 so the mode stack pointer in R0 remains accessible. Acked-by: Nicolas Pitre Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/kernel/entry-armv.S | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index ce8ca29461de..207875ac62ff 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -191,24 +191,27 @@ ENDPROC(__und_invalid) .macro svc_entry, stack_hole=0, trace=1, uaccess=1 UNWIND(.fnstart ) UNWIND(.save {r0 - pc} ) - sub sp, sp, #(SVC_REGS_SIZE + \stack_hole - 4) + sub sp, sp, #(SVC_REGS_SIZE + \stack_hole) #ifdef CONFIG_THUMB2_KERNEL - SPFIX( str r0, [sp] ) @ temporarily saved - SPFIX( mov r0, sp ) - SPFIX( tst r0, #4 ) @ test original stack alignment - SPFIX( ldr r0, [sp] ) @ restored + add sp, r1 @ get SP in a GPR without + sub r1, sp, r1 @ using a temp register + tst r1, #4 @ test stack pointer alignment + sub r1, sp, r1 @ restore original R1 + sub sp, r1 @ restore original SP #else SPFIX( tst sp, #4 ) #endif - SPFIX( subeq sp, sp, #4 ) - stmia sp, {r1 - r12} + SPFIX( subne sp, sp, #4 ) + + ARM( stmib sp, {r1 - r12} ) + THUMB( stmia sp, {r0 - r12} ) @ No STMIB in Thumb-2 ldmia r0, {r3 - r5} - add r7, sp, #S_SP - 4 @ here for interlock avoidance + add r7, sp, #S_SP @ here for interlock avoidance mov r6, #-1 @ "" "" "" "" - add r2, sp, #(SVC_REGS_SIZE + \stack_hole - 4) - SPFIX( addeq r2, r2, #4 ) - str r3, [sp, #-4]! @ save the "real" r0 copied + add r2, sp, #(SVC_REGS_SIZE + \stack_hole) + SPFIX( addne r2, r2, #4 ) + str r3, [sp] @ save the "real" r0 copied @ from the exception stack mov r3, lr From a1c510d0adc604bb143c86052bc5be48cbcfa17c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 23 Sep 2021 09:15:53 +0200 Subject: [PATCH 18/62] ARM: implement support for vmap'ed stacks Wire up the generic support for managing task stack allocations via vmalloc, and implement the entry code that detects whether we faulted because of a stack overrun (or future stack overrun caused by pushing the pt_regs array) While this adds a fair amount of tricky entry asm code, it should be noted that it only adds a TST + branch to the svc_entry path. The code implementing the non-trivial handling of the overflow stack is emitted out-of-line into the .text section. Since on ARM, we rely on do_translation_fault() to keep PMD level page table entries that cover the vmalloc region up to date, we need to ensure that we don't hit such a stale PMD entry when accessing the stack. So we do a dummy read from the new stack while still running from the old one on the context switch path, and bump the vmalloc_seq counter when PMD level entries in the vmalloc range are modified, so that the MM switch fetches the latest version of the entries. Note that we need to increase the per-mode stack by 1 word, to gain some space to stash a GPR until we know it is safe to touch the stack. However, due to the cacheline alignment of the struct, this does not actually increase the memory footprint of the struct stack array at all. Signed-off-by: Ard Biesheuvel Tested-by: Keith Packard Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/Kconfig | 1 + arch/arm/include/asm/page.h | 4 ++ arch/arm/include/asm/thread_info.h | 8 +++ arch/arm/kernel/entry-armv.S | 97 ++++++++++++++++++++++++++++-- arch/arm/kernel/entry-header.S | 37 ++++++++++++ arch/arm/kernel/irq.c | 9 ++- arch/arm/kernel/setup.c | 8 +-- arch/arm/kernel/sleep.S | 8 +++ arch/arm/kernel/traps.c | 80 +++++++++++++++++++++++- arch/arm/kernel/unwind.c | 3 +- arch/arm/kernel/vmlinux.lds.S | 4 +- 11 files changed, 244 insertions(+), 15 deletions(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 41849d5e0bc3..e2ab72f2bf4a 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -127,6 +127,7 @@ config ARM select RTC_LIB select SYS_SUPPORTS_APM_EMULATION select THREAD_INFO_IN_TASK if CURRENT_POINTER_IN_TPIDRURO + select HAVE_ARCH_VMAP_STACK if MMU && THREAD_INFO_IN_TASK && (!LD_IS_LLD || LLD_VERSION >= 140000) select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M # Above selects are sorted alphabetically; please add new ones # according to that. Thanks. diff --git a/arch/arm/include/asm/page.h b/arch/arm/include/asm/page.h index 11b058a72a5b..7b871ed99ccf 100644 --- a/arch/arm/include/asm/page.h +++ b/arch/arm/include/asm/page.h @@ -149,6 +149,10 @@ extern void copy_page(void *to, const void *from); #include #endif +#ifdef CONFIG_VMAP_STACK +#define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_PMD_MODIFIED +#endif + #endif /* CONFIG_MMU */ typedef struct page *pgtable_t; diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index 164e15f26485..004b89d86224 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -25,6 +25,14 @@ #define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER) #define THREAD_START_SP (THREAD_SIZE - 8) +#ifdef CONFIG_VMAP_STACK +#define THREAD_ALIGN (2 * THREAD_SIZE) +#else +#define THREAD_ALIGN THREAD_SIZE +#endif + +#define OVERFLOW_STACK_SIZE SZ_4K + #ifndef __ASSEMBLY__ struct task_struct; diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 207875ac62ff..5fb7465d14d9 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -57,6 +57,10 @@ UNWIND( .setfp fpreg, sp ) @ subs r2, sp, r0 @ SP above bottom of IRQ stack? rsbscs r2, r2, #THREAD_SIZE @ ... and below the top? +#ifdef CONFIG_VMAP_STACK + ldr_l r2, high_memory, cc @ End of the linear region + cmpcc r2, r0 @ Stack pointer was below it? +#endif movcs sp, r0 @ If so, revert to incoming SP #ifndef CONFIG_UNWINDER_ARM @@ -188,13 +192,18 @@ ENDPROC(__und_invalid) #define SPFIX(code...) #endif - .macro svc_entry, stack_hole=0, trace=1, uaccess=1 + .macro svc_entry, stack_hole=0, trace=1, uaccess=1, overflow_check=1 UNWIND(.fnstart ) - UNWIND(.save {r0 - pc} ) sub sp, sp, #(SVC_REGS_SIZE + \stack_hole) + THUMB( add sp, r1 ) @ get SP in a GPR without + THUMB( sub r1, sp, r1 ) @ using a temp register + + .if \overflow_check + UNWIND(.save {r0 - pc} ) + do_overflow_check (SVC_REGS_SIZE + \stack_hole) + .endif + #ifdef CONFIG_THUMB2_KERNEL - add sp, r1 @ get SP in a GPR without - sub r1, sp, r1 @ using a temp register tst r1, #4 @ test stack pointer alignment sub r1, sp, r1 @ restore original R1 sub sp, r1 @ restore original SP @@ -827,12 +836,20 @@ ENTRY(__switch_to) str r7, [r8] #endif mov r0, r5 -#if !defined(CONFIG_THUMB2_KERNEL) +#if !defined(CONFIG_THUMB2_KERNEL) && !defined(CONFIG_VMAP_STACK) set_current r7 ldmia r4, {r4 - sl, fp, sp, pc} @ Load all regs saved previously #else mov r1, r7 ldmia r4, {r4 - sl, fp, ip, lr} @ Load all regs saved previously +#ifdef CONFIG_VMAP_STACK + @ + @ Do a dummy read from the new stack while running from the old one so + @ that we can rely on do_translation_fault() to fix up any stale PMD + @ entries covering the vmalloc region. + @ + ldr r2, [ip] +#endif @ When CONFIG_THREAD_INFO_IN_TASK=n, the update of SP itself is what @ effectuates the task switch, as that is what causes the observable @@ -849,6 +866,76 @@ ENTRY(__switch_to) UNWIND(.fnend ) ENDPROC(__switch_to) +#ifdef CONFIG_VMAP_STACK + .text + .align 2 +__bad_stack: + @ + @ We've just detected an overflow. We need to load the address of this + @ CPU's overflow stack into the stack pointer register. We have only one + @ scratch register so let's use a sequence of ADDs including one + @ involving the PC, and decorate them with PC-relative group + @ relocations. As these are ARM only, switch to ARM mode first. + @ + @ We enter here with IP clobbered and its value stashed on the mode + @ stack. + @ +THUMB( bx pc ) +THUMB( nop ) +THUMB( .arm ) + mrc p15, 0, ip, c13, c0, 4 @ Get per-CPU offset + + .globl overflow_stack_ptr + .reloc 0f, R_ARM_ALU_PC_G0_NC, overflow_stack_ptr + .reloc 1f, R_ARM_ALU_PC_G1_NC, overflow_stack_ptr + .reloc 2f, R_ARM_LDR_PC_G2, overflow_stack_ptr + add ip, ip, pc +0: add ip, ip, #-4 +1: add ip, ip, #0 +2: ldr ip, [ip, #4] + + str sp, [ip, #-4]! @ Preserve original SP value + mov sp, ip @ Switch to overflow stack + pop {ip} @ Original SP in IP + +#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC) + mov ip, ip @ mov expected by unwinder + push {fp, ip, lr, pc} @ GCC flavor frame record +#else + str ip, [sp, #-8]! @ store original SP + push {fpreg, lr} @ Clang flavor frame record +#endif +UNWIND( ldr ip, [r0, #4] ) @ load exception LR +UNWIND( str ip, [sp, #12] ) @ store in the frame record + ldr ip, [r0, #12] @ reload IP + + @ Store the original GPRs to the new stack. + svc_entry uaccess=0, overflow_check=0 + +UNWIND( .save {sp, pc} ) +UNWIND( .save {fpreg, lr} ) +UNWIND( .setfp fpreg, sp ) + + ldr fpreg, [sp, #S_SP] @ Add our frame record + @ to the linked list +#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC) + ldr r1, [fp, #4] @ reload SP at entry + add fp, fp, #12 +#else + ldr r1, [fpreg, #8] +#endif + str r1, [sp, #S_SP] @ store in pt_regs + + @ Stash the regs for handle_bad_stack + mov r0, sp + + @ Time to die + bl handle_bad_stack + nop +UNWIND( .fnend ) +ENDPROC(__bad_stack) +#endif + __INIT /* diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S index ae24dd54e9ef..81df2a3561ca 100644 --- a/arch/arm/kernel/entry-header.S +++ b/arch/arm/kernel/entry-header.S @@ -423,3 +423,40 @@ scno .req r7 @ syscall number tbl .req r8 @ syscall table pointer why .req r8 @ Linux syscall (!= 0) tsk .req r9 @ current thread_info + + .macro do_overflow_check, frame_size:req +#ifdef CONFIG_VMAP_STACK + @ + @ Test whether the SP has overflowed. Task and IRQ stacks are aligned + @ so that SP & BIT(THREAD_SIZE_ORDER + PAGE_SHIFT) should always be + @ zero. + @ +ARM( tst sp, #1 << (THREAD_SIZE_ORDER + PAGE_SHIFT) ) +THUMB( tst r1, #1 << (THREAD_SIZE_ORDER + PAGE_SHIFT) ) +THUMB( it ne ) + bne .Lstack_overflow_check\@ + + .pushsection .text +.Lstack_overflow_check\@: + @ + @ The stack pointer is not pointing to a valid vmap'ed stack, but it + @ may be pointing into the linear map instead, which may happen if we + @ are already running from the overflow stack. We cannot detect overflow + @ in such cases so just carry on. + @ + str ip, [r0, #12] @ Stash IP on the mode stack + ldr_l ip, high_memory @ Start of VMALLOC space +ARM( cmp sp, ip ) @ SP in vmalloc space? +THUMB( cmp r1, ip ) +THUMB( itt lo ) + ldrlo ip, [r0, #12] @ Restore IP + blo .Lout\@ @ Carry on + +THUMB( sub r1, sp, r1 ) @ Restore original R1 +THUMB( sub sp, r1 ) @ Restore original SP + add sp, sp, #\frame_size @ Undo svc_entry's SP change + b __bad_stack @ Handle VMAP stack overflow + .popsection +.Lout\@: +#endif + .endm diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index e05219bca218..5deb40f39999 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -56,7 +56,14 @@ static void __init init_irq_stacks(void) int cpu; for_each_possible_cpu(cpu) { - stack = (u8 *)__get_free_pages(GFP_KERNEL, THREAD_SIZE_ORDER); + if (!IS_ENABLED(CONFIG_VMAP_STACK)) + stack = (u8 *)__get_free_pages(GFP_KERNEL, + THREAD_SIZE_ORDER); + else + stack = __vmalloc_node(THREAD_SIZE, THREAD_ALIGN, + THREADINFO_GFP, NUMA_NO_NODE, + __builtin_return_address(0)); + if (WARN_ON(!stack)) break; per_cpu(irq_stack_ptr, cpu) = &stack[THREAD_SIZE]; diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 284a80c0b6e1..039feb7cd590 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -141,10 +141,10 @@ EXPORT_SYMBOL(outer_cache); int __cpu_architecture __read_mostly = CPU_ARCH_UNKNOWN; struct stack { - u32 irq[3]; - u32 abt[3]; - u32 und[3]; - u32 fiq[3]; + u32 irq[4]; + u32 abt[4]; + u32 und[4]; + u32 fiq[4]; } ____cacheline_aligned; #ifndef CONFIG_CPU_V7M diff --git a/arch/arm/kernel/sleep.S b/arch/arm/kernel/sleep.S index 43077e11dafd..803b51e5cba0 100644 --- a/arch/arm/kernel/sleep.S +++ b/arch/arm/kernel/sleep.S @@ -67,6 +67,14 @@ ENTRY(__cpu_suspend) ldr r4, =cpu_suspend_size #endif mov r5, sp @ current virtual SP +#ifdef CONFIG_VMAP_STACK + @ Run the suspend code from the overflow stack so we don't have to rely + @ on vmalloc-to-phys conversions anywhere in the arch suspend code. + @ The original SP value captured in R5 will be restored on the way out. + mov_l r6, overflow_stack_ptr @ Base pointer + mrc p15, 0, r7, c13, c0, 4 @ Get per-CPU offset + ldr sp, [r6, r7] @ Address of this CPU's overflow stack +#endif add r4, r4, #12 @ Space for pgd, virt sp, phys resume fn sub sp, sp, r4 @ allocate CPU state on stack ldr r3, =sleep_save_sp diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index b42c446cec9a..b28a705c49cb 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -121,7 +121,8 @@ void dump_backtrace_stm(u32 *stack, u32 instruction, const char *loglvl) static int verify_stack(unsigned long sp) { if (sp < PAGE_OFFSET || - (sp > (unsigned long)high_memory && high_memory != NULL)) + (!IS_ENABLED(CONFIG_VMAP_STACK) && + sp > (unsigned long)high_memory && high_memory != NULL)) return -EFAULT; return 0; @@ -291,7 +292,8 @@ static int __die(const char *str, int err, struct pt_regs *regs) if (!user_mode(regs) || in_interrupt()) { dump_mem(KERN_EMERG, "Stack: ", regs->ARM_sp, - ALIGN(regs->ARM_sp, THREAD_SIZE)); + ALIGN(regs->ARM_sp - THREAD_SIZE, THREAD_ALIGN) + + THREAD_SIZE); dump_backtrace(regs, tsk, KERN_EMERG); dump_instr(KERN_EMERG, regs); } @@ -838,3 +840,77 @@ void __init early_trap_init(void *vectors_base) */ #endif } + +#ifdef CONFIG_VMAP_STACK + +DECLARE_PER_CPU(u8 *, irq_stack_ptr); + +asmlinkage DEFINE_PER_CPU(u8 *, overflow_stack_ptr); + +static int __init allocate_overflow_stacks(void) +{ + u8 *stack; + int cpu; + + for_each_possible_cpu(cpu) { + stack = (u8 *)__get_free_page(GFP_KERNEL); + if (WARN_ON(!stack)) + return -ENOMEM; + per_cpu(overflow_stack_ptr, cpu) = &stack[OVERFLOW_STACK_SIZE]; + } + return 0; +} +early_initcall(allocate_overflow_stacks); + +asmlinkage void handle_bad_stack(struct pt_regs *regs) +{ + unsigned long tsk_stk = (unsigned long)current->stack; + unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr); + unsigned long ovf_stk = (unsigned long)this_cpu_read(overflow_stack_ptr); + + console_verbose(); + pr_emerg("Insufficient stack space to handle exception!"); + + pr_emerg("Task stack: [0x%08lx..0x%08lx]\n", + tsk_stk, tsk_stk + THREAD_SIZE); + pr_emerg("IRQ stack: [0x%08lx..0x%08lx]\n", + irq_stk - THREAD_SIZE, irq_stk); + pr_emerg("Overflow stack: [0x%08lx..0x%08lx]\n", + ovf_stk - OVERFLOW_STACK_SIZE, ovf_stk); + + die("kernel stack overflow", regs, 0); +} + +/* + * Normally, we rely on the logic in do_translation_fault() to update stale PMD + * entries covering the vmalloc space in a task's page tables when it first + * accesses the region in question. Unfortunately, this is not sufficient when + * the task stack resides in the vmalloc region, as do_translation_fault() is a + * C function that needs a stack to run. + * + * So we need to ensure that these PMD entries are up to date *before* the MM + * switch. As we already have some logic in the MM switch path that takes care + * of this, let's trigger it by bumping the counter every time the core vmalloc + * code modifies a PMD entry in the vmalloc region. + */ +void arch_sync_kernel_mappings(unsigned long start, unsigned long end) +{ + if (start > VMALLOC_END || end < VMALLOC_START) + return; + + /* + * This hooks into the core vmalloc code to receive notifications of + * any PMD level changes that have been made to the kernel page tables. + * This means it should only be triggered once for every MiB worth of + * vmalloc space, given that we don't support huge vmalloc/vmap on ARM, + * and that kernel PMD level table entries are rarely (if ever) + * updated. + * + * This means that the counter is going to max out at ~250 for the + * typical case. If it overflows, something entirely unexpected has + * occurred so let's throw a warning if that happens. + */ + WARN_ON(++init_mm.context.vmalloc_seq == UINT_MAX); +} + +#endif diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c index e8d729975f12..c5ea328c428d 100644 --- a/arch/arm/kernel/unwind.c +++ b/arch/arm/kernel/unwind.c @@ -389,7 +389,8 @@ int unwind_frame(struct stackframe *frame) /* store the highest address on the stack to avoid crossing it*/ ctrl.sp_low = frame->sp; - ctrl.sp_high = ALIGN(ctrl.sp_low, THREAD_SIZE); + ctrl.sp_high = ALIGN(ctrl.sp_low - THREAD_SIZE, THREAD_ALIGN) + + THREAD_SIZE; pr_debug("%s(pc = %08lx lr = %08lx sp = %08lx)\n", __func__, frame->pc, frame->lr, frame->sp); diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S index f02d617e3359..aa12b65a7fd6 100644 --- a/arch/arm/kernel/vmlinux.lds.S +++ b/arch/arm/kernel/vmlinux.lds.S @@ -138,12 +138,12 @@ SECTIONS #ifdef CONFIG_STRICT_KERNEL_RWX . = ALIGN(1< Date: Fri, 3 Dec 2021 17:11:36 +0100 Subject: [PATCH 19/62] ARM: riscpc: drop support for IOMD_IRQREQC/IOMD_IRQREQD IRQ groups IOMD_IRQREQC nor IOMD_IRQREQD are ever defined, so any conditionally compiled code that depends on them is dead code, and can be removed. Suggested-by: Russell King Signed-off-by: Ard Biesheuvel --- .../include/asm/hardware/entry-macro-iomd.S | 47 ------------------- 1 file changed, 47 deletions(-) diff --git a/arch/arm/include/asm/hardware/entry-macro-iomd.S b/arch/arm/include/asm/hardware/entry-macro-iomd.S index f7692731e514..81441dfa5282 100644 --- a/arch/arm/include/asm/hardware/entry-macro-iomd.S +++ b/arch/arm/include/asm/hardware/entry-macro-iomd.S @@ -24,16 +24,6 @@ ldrbeq \irqstat, [\base, #IOMD_IRQREQA] @ get low priority addeq \tmp, \tmp, #256 @ irq_prio_d table size teqeq \irqstat, #0 -#ifdef IOMD_IRQREQC - ldrbeq \irqstat, [\base, #IOMD_IRQREQC] - addeq \tmp, \tmp, #256 @ irq_prio_l table size - teqeq \irqstat, #0 -#endif -#ifdef IOMD_IRQREQD - ldrbeq \irqstat, [\base, #IOMD_IRQREQD] - addeq \tmp, \tmp, #256 @ irq_prio_lc table size - teqeq \irqstat, #0 -#endif 2406: ldrbne \irqnr, [\tmp, \irqstat] @ get IRQ number .endm @@ -92,40 +82,3 @@ irq_prio_l: .byte 0, 0, 1, 0, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3 .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 -#ifdef IOMD_IRQREQC -irq_prio_lc: .byte 24,24,25,24,26,26,26,26,27,27,27,27,27,27,27,27 - .byte 28,24,25,24,26,26,26,26,27,27,27,27,27,27,27,27 - .byte 29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29 - .byte 29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29 - .byte 30,30,30,30,30,30,30,30,27,27,27,27,27,27,27,27 - .byte 30,30,30,30,30,30,30,30,27,27,27,27,27,27,27,27 - .byte 29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29 - .byte 29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29 - .byte 31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31 - .byte 31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31 - .byte 31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31 - .byte 31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31 - .byte 31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31 - .byte 31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31 - .byte 31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31 - .byte 31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31 -#endif -#ifdef IOMD_IRQREQD -irq_prio_ld: .byte 40,40,41,40,42,42,42,42,43,43,43,43,43,43,43,43 - .byte 44,40,41,40,42,42,42,42,43,43,43,43,43,43,43,43 - .byte 45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45 - .byte 45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45 - .byte 46,46,46,46,46,46,46,46,43,43,43,43,43,43,43,43 - .byte 46,46,46,46,46,46,46,46,43,43,43,43,43,43,43,43 - .byte 45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45 - .byte 45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45 - .byte 47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47 - .byte 47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47 - .byte 47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47 - .byte 47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47 - .byte 47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47 - .byte 47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47 - .byte 47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47 - .byte 47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47 -#endif - From c1fe8d054c0a8b0ff0406dcc0ddc6c55e1cfddbc Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 30 Nov 2021 08:19:13 +0100 Subject: [PATCH 20/62] ARM: riscpc: use GENERIC_IRQ_MULTI_HANDLER This is one of the last platforms using the old entry path. While this code path is spread over a few files, it is fairly straightforward to convert it into an equivalent C version, leaving the existing algorithm and all the priority handling the same. Unlike most irqchip drivers, this means reading the status register(s) in a loop and always handling the highest-priority irq first. The IOMD_IRQREQC and IOMD_IRQREQD registers are not actaully used here, but I left the code in place for the time being, to keep the conversion as direct as possible. It could be removed in a cleanup on top. Signed-off-by: Arnd Bergmann [ardb: drop obsolete IOMD_IRQREQC/IOMD_IRQREQD handling] Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/Kconfig | 1 + .../include/asm/hardware/entry-macro-iomd.S | 84 ---------------- arch/arm/mach-rpc/fiq.S | 5 +- arch/arm/mach-rpc/include/mach/entry-macro.S | 13 --- arch/arm/mach-rpc/irq.c | 95 +++++++++++++++++++ 5 files changed, 99 insertions(+), 99 deletions(-) delete mode 100644 arch/arm/include/asm/hardware/entry-macro-iomd.S delete mode 100644 arch/arm/mach-rpc/include/mach/entry-macro.S diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index e2ab72f2bf4a..25f1868e5703 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -444,6 +444,7 @@ config ARCH_RPC select ARM_HAS_SG_CHAIN select CPU_SA110 select FIQ + select GENERIC_IRQ_MULTI_HANDLER select HAVE_PATA_PLATFORM select ISA_DMA_API select LEGACY_TIMER_TICK diff --git a/arch/arm/include/asm/hardware/entry-macro-iomd.S b/arch/arm/include/asm/hardware/entry-macro-iomd.S deleted file mode 100644 index 81441dfa5282..000000000000 --- a/arch/arm/include/asm/hardware/entry-macro-iomd.S +++ /dev/null @@ -1,84 +0,0 @@ -/* - * arch/arm/include/asm/hardware/entry-macro-iomd.S - * - * Low-level IRQ helper macros for IOC/IOMD based platforms - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. - */ - -/* IOC / IOMD based hardware */ -#include - - .macro get_irqnr_and_base, irqnr, irqstat, base, tmp - ldrb \irqstat, [\base, #IOMD_IRQREQB] @ get high priority first - ldr \tmp, =irq_prio_h - teq \irqstat, #0 -#ifdef IOMD_BASE - ldrbeq \irqstat, [\base, #IOMD_DMAREQ] @ get dma - addeq \tmp, \tmp, #256 @ irq_prio_h table size - teqeq \irqstat, #0 - bne 2406f -#endif - ldrbeq \irqstat, [\base, #IOMD_IRQREQA] @ get low priority - addeq \tmp, \tmp, #256 @ irq_prio_d table size - teqeq \irqstat, #0 -2406: ldrbne \irqnr, [\tmp, \irqstat] @ get IRQ number - .endm - -/* - * Interrupt table (incorporates priority). Please note that we - * rely on the order of these tables (see above code). - */ - .align 5 -irq_prio_h: .byte 0, 8, 9, 8,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 12, 8, 9, 8,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 14,14,14,14,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 14,14,14,14,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 -#ifdef IOMD_BASE -irq_prio_d: .byte 0,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 20,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 23,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 23,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 -#endif -irq_prio_l: .byte 0, 0, 1, 0, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3 - .byte 4, 0, 1, 0, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3 - .byte 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 - .byte 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 - .byte 6, 6, 6, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3, 3, 3 - .byte 6, 6, 6, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3, 3, 3 - .byte 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 - .byte 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 diff --git a/arch/arm/mach-rpc/fiq.S b/arch/arm/mach-rpc/fiq.S index 0de83e9b0b39..087bdf4bc093 100644 --- a/arch/arm/mach-rpc/fiq.S +++ b/arch/arm/mach-rpc/fiq.S @@ -2,10 +2,11 @@ #include #include #include -#include + + .equ ioc_base_high, IOC_BASE & 0xff000000 + .equ ioc_base_low, IOC_BASE & 0x00ff0000 .text - .global rpc_default_fiq_end ENTRY(rpc_default_fiq_start) mov r12, #ioc_base_high diff --git a/arch/arm/mach-rpc/include/mach/entry-macro.S b/arch/arm/mach-rpc/include/mach/entry-macro.S deleted file mode 100644 index a6d1a9f4bb79..000000000000 --- a/arch/arm/mach-rpc/include/mach/entry-macro.S +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include -#include - - .equ ioc_base_high, IOC_BASE & 0xff000000 - .equ ioc_base_low, IOC_BASE & 0x00ff0000 - - .macro get_irqnr_preamble, base, tmp - mov \base, #ioc_base_high @ point at IOC - .if ioc_base_low - orr \base, \base, #ioc_base_low - .endif - .endm diff --git a/arch/arm/mach-rpc/irq.c b/arch/arm/mach-rpc/irq.c index 803aeb126f0e..dc29384b6ef8 100644 --- a/arch/arm/mach-rpc/irq.c +++ b/arch/arm/mach-rpc/irq.c @@ -14,6 +14,99 @@ #define CLR 0x04 #define MASK 0x08 +static const u8 irq_prio_h[256] = { + 0, 8, 9, 8,10,10,10,10,11,11,11,11,10,10,10,10, + 12, 8, 9, 8,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 14,14,14,14,10,10,10,10,11,11,11,11,10,10,10,10, + 14,14,14,14,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10, + 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10, + 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, +}; + +static const u8 irq_prio_d[256] = { + 0,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 20,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 23,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 23,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, +}; + +static const u8 irq_prio_l[256] = { + 0, 0, 1, 0, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 0, 1, 0, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 6, 6, 6, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3, 3, 3, + 6, 6, 6, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3, 3, 3, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, +}; + +static int iomd_get_irq_nr(void) +{ + int irq; + u8 reg; + + /* get highest priority first */ + reg = readb(IOC_BASE + IOMD_IRQREQB); + irq = irq_prio_h[reg]; + if (irq) + return irq; + + /* get DMA */ + reg = readb(IOC_BASE + IOMD_DMAREQ); + irq = irq_prio_d[reg]; + if (irq) + return irq; + + /* get low priority */ + reg = readb(IOC_BASE + IOMD_IRQREQA); + irq = irq_prio_l[reg]; + if (irq) + return irq; + return 0; +} + +static void iomd_handle_irq(struct pt_regs *regs) +{ + int irq; + + do { + irq = iomd_get_irq_nr(); + if (irq) + generic_handle_irq(irq); + } while (irq); +} + static void __iomem *iomd_get_base(struct irq_data *d) { void *cd = irq_data_get_irq_chip_data(d); @@ -82,6 +175,8 @@ void __init rpc_init_irq(void) set_fiq_handler(&rpc_default_fiq_start, &rpc_default_fiq_end - &rpc_default_fiq_start); + set_handle_irq(iomd_handle_irq); + for (irq = 0; irq < NR_IRQS; irq++) { clr = IRQ_NOREQUEST; set = 0; From 90890f17ccd2aa96350abd1f4d37d4667e09027f Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 29 Nov 2021 14:43:14 +0100 Subject: [PATCH 21/62] ARM: footbridge: use GENERIC_IRQ_MULTI_HANDLER Footbridge still uses the classic IRQ entry path in assembler, but this is easily converted into an equivalent C version. In this case, the correlation between IRQ numbers and bits in the status register is non-obvious, and the priorities are handled by manually checking each bit in a static order, re-reading the status register after each handled event. I moved the code into the new file and edited the syntax without changing this sequence to keep the behavior as close as possible to what it traditionally did. Signed-off-by: Arnd Bergmann Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M Reviewed-by: Linus Walleij --- arch/arm/Kconfig | 1 + arch/arm/mach-footbridge/common.c | 87 ++++++++++++++ .../include/mach/entry-macro.S | 107 ------------------ 3 files changed, 88 insertions(+), 107 deletions(-) delete mode 100644 arch/arm/mach-footbridge/include/mach/entry-macro.S diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 25f1868e5703..a0cc9ca66ae0 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -362,6 +362,7 @@ config ARCH_FOOTBRIDGE select FOOTBRIDGE select NEED_MACH_IO_H if !MMU select NEED_MACH_MEMORY_H + select GENERIC_IRQ_MULTI_HANDLER help Support for systems based on the DC21285 companion chip ("FootBridge"), such as the Simtec CATS and the Rebel NetWinder. diff --git a/arch/arm/mach-footbridge/common.c b/arch/arm/mach-footbridge/common.c index eee095f0e2f6..322495df271d 100644 --- a/arch/arm/mach-footbridge/common.c +++ b/arch/arm/mach-footbridge/common.c @@ -27,6 +27,91 @@ #include "common.h" +#include +#include +#include + +static int dc21285_get_irq(void) +{ + void __iomem *irqstatus = (void __iomem *)CSR_IRQ_STATUS; + u32 mask = readl(irqstatus); + + if (mask & IRQ_MASK_SDRAMPARITY) + return IRQ_SDRAMPARITY; + + if (mask & IRQ_MASK_UART_RX) + return IRQ_CONRX; + + if (mask & IRQ_MASK_DMA1) + return IRQ_DMA1; + + if (mask & IRQ_MASK_DMA2) + return IRQ_DMA2; + + if (mask & IRQ_MASK_IN0) + return IRQ_IN0; + + if (mask & IRQ_MASK_IN1) + return IRQ_IN1; + + if (mask & IRQ_MASK_IN2) + return IRQ_IN2; + + if (mask & IRQ_MASK_IN3) + return IRQ_IN3; + + if (mask & IRQ_MASK_PCI) + return IRQ_PCI; + + if (mask & IRQ_MASK_DOORBELLHOST) + return IRQ_DOORBELLHOST; + + if (mask & IRQ_MASK_I2OINPOST) + return IRQ_I2OINPOST; + + if (mask & IRQ_MASK_TIMER1) + return IRQ_TIMER1; + + if (mask & IRQ_MASK_TIMER2) + return IRQ_TIMER2; + + if (mask & IRQ_MASK_TIMER3) + return IRQ_TIMER3; + + if (mask & IRQ_MASK_UART_TX) + return IRQ_CONTX; + + if (mask & IRQ_MASK_PCI_ABORT) + return IRQ_PCI_ABORT; + + if (mask & IRQ_MASK_PCI_SERR) + return IRQ_PCI_SERR; + + if (mask & IRQ_MASK_DISCARD_TIMER) + return IRQ_DISCARD_TIMER; + + if (mask & IRQ_MASK_PCI_DPERR) + return IRQ_PCI_DPERR; + + if (mask & IRQ_MASK_PCI_PERR) + return IRQ_PCI_PERR; + + return 0; +} + +static void dc21285_handle_irq(struct pt_regs *regs) +{ + int irq; + do { + irq = dc21285_get_irq(); + if (!irq) + break; + + generic_handle_irq(irq); + } while (1); +} + + unsigned int mem_fclk_21285 = 50000000; EXPORT_SYMBOL(mem_fclk_21285); @@ -108,6 +193,8 @@ static void __init __fb_init_irq(void) void __init footbridge_init_irq(void) { + set_handle_irq(dc21285_handle_irq); + __fb_init_irq(); if (!footbridge_cfn_mode()) diff --git a/arch/arm/mach-footbridge/include/mach/entry-macro.S b/arch/arm/mach-footbridge/include/mach/entry-macro.S deleted file mode 100644 index dabbd5c54a78..000000000000 --- a/arch/arm/mach-footbridge/include/mach/entry-macro.S +++ /dev/null @@ -1,107 +0,0 @@ -/* - * arch/arm/mach-footbridge/include/mach/entry-macro.S - * - * Low-level IRQ helper macros for footbridge-based platforms - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. - */ -#include -#include -#include - - .equ dc21285_high, ARMCSR_BASE & 0xff000000 - .equ dc21285_low, ARMCSR_BASE & 0x00ffffff - - .macro get_irqnr_preamble, base, tmp - mov \base, #dc21285_high - .if dc21285_low - orr \base, \base, #dc21285_low - .endif - .endm - - .macro get_irqnr_and_base, irqnr, irqstat, base, tmp - ldr \irqstat, [\base, #0x180] @ get interrupts - - mov \irqnr, #IRQ_SDRAMPARITY - tst \irqstat, #IRQ_MASK_SDRAMPARITY - bne 1001f - - tst \irqstat, #IRQ_MASK_UART_RX - movne \irqnr, #IRQ_CONRX - bne 1001f - - tst \irqstat, #IRQ_MASK_DMA1 - movne \irqnr, #IRQ_DMA1 - bne 1001f - - tst \irqstat, #IRQ_MASK_DMA2 - movne \irqnr, #IRQ_DMA2 - bne 1001f - - tst \irqstat, #IRQ_MASK_IN0 - movne \irqnr, #IRQ_IN0 - bne 1001f - - tst \irqstat, #IRQ_MASK_IN1 - movne \irqnr, #IRQ_IN1 - bne 1001f - - tst \irqstat, #IRQ_MASK_IN2 - movne \irqnr, #IRQ_IN2 - bne 1001f - - tst \irqstat, #IRQ_MASK_IN3 - movne \irqnr, #IRQ_IN3 - bne 1001f - - tst \irqstat, #IRQ_MASK_PCI - movne \irqnr, #IRQ_PCI - bne 1001f - - tst \irqstat, #IRQ_MASK_DOORBELLHOST - movne \irqnr, #IRQ_DOORBELLHOST - bne 1001f - - tst \irqstat, #IRQ_MASK_I2OINPOST - movne \irqnr, #IRQ_I2OINPOST - bne 1001f - - tst \irqstat, #IRQ_MASK_TIMER1 - movne \irqnr, #IRQ_TIMER1 - bne 1001f - - tst \irqstat, #IRQ_MASK_TIMER2 - movne \irqnr, #IRQ_TIMER2 - bne 1001f - - tst \irqstat, #IRQ_MASK_TIMER3 - movne \irqnr, #IRQ_TIMER3 - bne 1001f - - tst \irqstat, #IRQ_MASK_UART_TX - movne \irqnr, #IRQ_CONTX - bne 1001f - - tst \irqstat, #IRQ_MASK_PCI_ABORT - movne \irqnr, #IRQ_PCI_ABORT - bne 1001f - - tst \irqstat, #IRQ_MASK_PCI_SERR - movne \irqnr, #IRQ_PCI_SERR - bne 1001f - - tst \irqstat, #IRQ_MASK_DISCARD_TIMER - movne \irqnr, #IRQ_DISCARD_TIMER - bne 1001f - - tst \irqstat, #IRQ_MASK_PCI_DPERR - movne \irqnr, #IRQ_PCI_DPERR - bne 1001f - - tst \irqstat, #IRQ_MASK_PCI_PERR - movne \irqnr, #IRQ_PCI_PERR -1001: - .endm - From 9d67412f24cc3a2c05f35f7c856addb07a2960ce Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 30 Nov 2021 11:16:41 +0100 Subject: [PATCH 22/62] ARM: iop32x: offset IRQ numbers by 1 iop32x is one of the last platforms to use IRQ 0, and this has apparently stopped working in a 2014 cleanup without anyone noticing. This interrupt is used for the DMA engine, so most likely this has not actually worked in the past 7 years, but it's also not essential for using this board. I'm splitting out this change from my GENERIC_IRQ_MULTI_HANDLER conversion so it can be backported if anyone cares. Fixes: a71b092a9c68 ("ARM: Convert handle_IRQ to use __handle_domain_irq") Signed-off-by: Arnd Bergmann [ardb: take +1 offset into account in mask/unmask and init as well] Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M Reviewed-by: Linus Walleij --- .../mach-iop32x/include/mach/entry-macro.S | 2 +- arch/arm/mach-iop32x/include/mach/irqs.h | 2 +- arch/arm/mach-iop32x/irq.c | 6 +- arch/arm/mach-iop32x/irqs.h | 60 ++++++++++--------- 4 files changed, 37 insertions(+), 33 deletions(-) diff --git a/arch/arm/mach-iop32x/include/mach/entry-macro.S b/arch/arm/mach-iop32x/include/mach/entry-macro.S index 8e6766d4621e..341e5d9a6616 100644 --- a/arch/arm/mach-iop32x/include/mach/entry-macro.S +++ b/arch/arm/mach-iop32x/include/mach/entry-macro.S @@ -20,7 +20,7 @@ mrc p6, 0, \irqstat, c8, c0, 0 @ Read IINTSRC cmp \irqstat, #0 clzne \irqnr, \irqstat - rsbne \irqnr, \irqnr, #31 + rsbne \irqnr, \irqnr, #32 .endm .macro arch_ret_to_user, tmp1, tmp2 diff --git a/arch/arm/mach-iop32x/include/mach/irqs.h b/arch/arm/mach-iop32x/include/mach/irqs.h index c4e78df428e8..e09ae5f48aec 100644 --- a/arch/arm/mach-iop32x/include/mach/irqs.h +++ b/arch/arm/mach-iop32x/include/mach/irqs.h @@ -9,6 +9,6 @@ #ifndef __IRQS_H #define __IRQS_H -#define NR_IRQS 32 +#define NR_IRQS 33 #endif diff --git a/arch/arm/mach-iop32x/irq.c b/arch/arm/mach-iop32x/irq.c index 2d48bf1398c1..d1e8824cbd82 100644 --- a/arch/arm/mach-iop32x/irq.c +++ b/arch/arm/mach-iop32x/irq.c @@ -32,14 +32,14 @@ static void intstr_write(u32 val) static void iop32x_irq_mask(struct irq_data *d) { - iop32x_mask &= ~(1 << d->irq); + iop32x_mask &= ~(1 << (d->irq - 1)); intctl_write(iop32x_mask); } static void iop32x_irq_unmask(struct irq_data *d) { - iop32x_mask |= 1 << d->irq; + iop32x_mask |= 1 << (d->irq - 1); intctl_write(iop32x_mask); } @@ -65,7 +65,7 @@ void __init iop32x_init_irq(void) machine_is_em7210()) *IOP3XX_PCIIRSR = 0x0f; - for (i = 0; i < NR_IRQS; i++) { + for (i = 1; i < NR_IRQS; i++) { irq_set_chip_and_handler(i, &ext_chip, handle_level_irq); irq_clear_status_flags(i, IRQ_NOREQUEST | IRQ_NOPROBE); } diff --git a/arch/arm/mach-iop32x/irqs.h b/arch/arm/mach-iop32x/irqs.h index 69858e4e905d..e1dfc8b4e7d7 100644 --- a/arch/arm/mach-iop32x/irqs.h +++ b/arch/arm/mach-iop32x/irqs.h @@ -7,36 +7,40 @@ #ifndef __IOP32X_IRQS_H #define __IOP32X_IRQS_H +/* Interrupts in Linux start at 1, hardware starts at 0 */ + +#define IOP_IRQ(x) ((x) + 1) + /* * IOP80321 chipset interrupts */ -#define IRQ_IOP32X_DMA0_EOT 0 -#define IRQ_IOP32X_DMA0_EOC 1 -#define IRQ_IOP32X_DMA1_EOT 2 -#define IRQ_IOP32X_DMA1_EOC 3 -#define IRQ_IOP32X_AA_EOT 6 -#define IRQ_IOP32X_AA_EOC 7 -#define IRQ_IOP32X_CORE_PMON 8 -#define IRQ_IOP32X_TIMER0 9 -#define IRQ_IOP32X_TIMER1 10 -#define IRQ_IOP32X_I2C_0 11 -#define IRQ_IOP32X_I2C_1 12 -#define IRQ_IOP32X_MESSAGING 13 -#define IRQ_IOP32X_ATU_BIST 14 -#define IRQ_IOP32X_PERFMON 15 -#define IRQ_IOP32X_CORE_PMU 16 -#define IRQ_IOP32X_BIU_ERR 17 -#define IRQ_IOP32X_ATU_ERR 18 -#define IRQ_IOP32X_MCU_ERR 19 -#define IRQ_IOP32X_DMA0_ERR 20 -#define IRQ_IOP32X_DMA1_ERR 21 -#define IRQ_IOP32X_AA_ERR 23 -#define IRQ_IOP32X_MSG_ERR 24 -#define IRQ_IOP32X_SSP 25 -#define IRQ_IOP32X_XINT0 27 -#define IRQ_IOP32X_XINT1 28 -#define IRQ_IOP32X_XINT2 29 -#define IRQ_IOP32X_XINT3 30 -#define IRQ_IOP32X_HPI 31 +#define IRQ_IOP32X_DMA0_EOT IOP_IRQ(0) +#define IRQ_IOP32X_DMA0_EOC IOP_IRQ(1) +#define IRQ_IOP32X_DMA1_EOT IOP_IRQ(2) +#define IRQ_IOP32X_DMA1_EOC IOP_IRQ(3) +#define IRQ_IOP32X_AA_EOT IOP_IRQ(6) +#define IRQ_IOP32X_AA_EOC IOP_IRQ(7) +#define IRQ_IOP32X_CORE_PMON IOP_IRQ(8) +#define IRQ_IOP32X_TIMER0 IOP_IRQ(9) +#define IRQ_IOP32X_TIMER1 IOP_IRQ(10) +#define IRQ_IOP32X_I2C_0 IOP_IRQ(11) +#define IRQ_IOP32X_I2C_1 IOP_IRQ(12) +#define IRQ_IOP32X_MESSAGING IOP_IRQ(13) +#define IRQ_IOP32X_ATU_BIST IOP_IRQ(14) +#define IRQ_IOP32X_PERFMON IOP_IRQ(15) +#define IRQ_IOP32X_CORE_PMU IOP_IRQ(16) +#define IRQ_IOP32X_BIU_ERR IOP_IRQ(17) +#define IRQ_IOP32X_ATU_ERR IOP_IRQ(18) +#define IRQ_IOP32X_MCU_ERR IOP_IRQ(19) +#define IRQ_IOP32X_DMA0_ERR IOP_IRQ(20) +#define IRQ_IOP32X_DMA1_ERR IOP_IRQ(21) +#define IRQ_IOP32X_AA_ERR IOP_IRQ(23) +#define IRQ_IOP32X_MSG_ERR IOP_IRQ(24) +#define IRQ_IOP32X_SSP IOP_IRQ(25) +#define IRQ_IOP32X_XINT0 IOP_IRQ(27) +#define IRQ_IOP32X_XINT1 IOP_IRQ(28) +#define IRQ_IOP32X_XINT2 IOP_IRQ(29) +#define IRQ_IOP32X_XINT3 IOP_IRQ(30) +#define IRQ_IOP32X_HPI IOP_IRQ(31) #endif From 6f5d248d05db9c4991366154f1a657a630faa583 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 30 Nov 2021 11:21:49 +0100 Subject: [PATCH 23/62] ARM: iop32x: use GENERIC_IRQ_MULTI_HANDLER iop32x uses the entry-macro.S file for both the IRQ entry and for hooking into the arch_ret_to_user code path. This is done because the cp6 registers have to be enabled before accessing any of the interrupt controller registers but have to be disabled when running in user space. There is also a lazy-enable logic in cp6.c, but during a hardirq, we know it has to be enabled. Both the cp6-enable code and the code to read the IRQ status can be lifted into the normal generic_handle_arch_irq() path, but the cp6-disable code has to remain in the user return code. As nothing other than iop32x uses this hook, just open-code it there with an ifdef for the platform that can eventually be removed when iop32x has reached the end of its life. The cp6-enable path in the IRQ entry has an extra cp_wait barrier that the trap version does not have, but it is harmless to do it in both cases to simplify the logic here at the cost of a few extra cycles for the trap. Signed-off-by: Arnd Bergmann Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/Kconfig | 5 +-- arch/arm/kernel/entry-common.S | 16 +++++----- arch/arm/mach-iop32x/cp6.c | 10 +++++- .../mach-iop32x/include/mach/entry-macro.S | 31 ------------------- arch/arm/mach-iop32x/iop3xx.h | 1 + arch/arm/mach-iop32x/irq.c | 23 ++++++++++++++ 6 files changed, 43 insertions(+), 43 deletions(-) delete mode 100644 arch/arm/mach-iop32x/include/mach/entry-macro.S diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index a0cc9ca66ae0..d9ba6961b295 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -227,9 +227,6 @@ config GENERIC_ISA_DMA config FIQ bool -config NEED_RET_TO_USER - bool - config ARCH_MTD_XIP bool @@ -371,9 +368,9 @@ config ARCH_IOP32X bool "IOP32x-based" depends on MMU select CPU_XSCALE + select GENERIC_IRQ_MULTI_HANDLER select GPIO_IOP select GPIOLIB - select NEED_RET_TO_USER select FORCE_PCI select PLAT_IOP help diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index ac86c34682bb..c928d6b04cce 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -16,12 +16,14 @@ .equ NR_syscalls, __NR_syscalls -#ifdef CONFIG_NEED_RET_TO_USER -#include -#else - .macro arch_ret_to_user, tmp1, tmp2 - .endm + .macro arch_ret_to_user, tmp +#ifdef CONFIG_ARCH_IOP32X + mrc p15, 0, \tmp, c15, c1, 0 + tst \tmp, #(1 << 6) + bicne \tmp, \tmp, #(1 << 6) + mcrne p15, 0, \tmp, c15, c1, 0 @ Disable cp6 access #endif + .endm #include "entry-header.S" @@ -55,7 +57,7 @@ __ret_fast_syscall: /* perform architecture specific actions before user return */ - arch_ret_to_user r1, lr + arch_ret_to_user r1 restore_user_regs fast = 1, offset = S_OFF UNWIND(.fnend ) @@ -128,7 +130,7 @@ no_work_pending: asm_trace_hardirqs_on save = 0 /* perform architecture specific actions before user return */ - arch_ret_to_user r1, lr + arch_ret_to_user r1 ct_user_enter save = 0 restore_user_regs fast = 0, offset = 0 diff --git a/arch/arm/mach-iop32x/cp6.c b/arch/arm/mach-iop32x/cp6.c index ec74b07fb7e3..2882674a1c39 100644 --- a/arch/arm/mach-iop32x/cp6.c +++ b/arch/arm/mach-iop32x/cp6.c @@ -7,7 +7,7 @@ #include #include -static int cp6_trap(struct pt_regs *regs, unsigned int instr) +void iop_enable_cp6(void) { u32 temp; @@ -16,7 +16,15 @@ static int cp6_trap(struct pt_regs *regs, unsigned int instr) "mrc p15, 0, %0, c15, c1, 0\n\t" "orr %0, %0, #(1 << 6)\n\t" "mcr p15, 0, %0, c15, c1, 0\n\t" + "mrc p15, 0, %0, c15, c1, 0\n\t" + "mov %0, %0\n\t" + "sub pc, pc, #4 @ cp_wait\n\t" : "=r"(temp)); +} + +static int cp6_trap(struct pt_regs *regs, unsigned int instr) +{ + iop_enable_cp6(); return 0; } diff --git a/arch/arm/mach-iop32x/include/mach/entry-macro.S b/arch/arm/mach-iop32x/include/mach/entry-macro.S deleted file mode 100644 index 341e5d9a6616..000000000000 --- a/arch/arm/mach-iop32x/include/mach/entry-macro.S +++ /dev/null @@ -1,31 +0,0 @@ -/* - * arch/arm/mach-iop32x/include/mach/entry-macro.S - * - * Low-level IRQ helper macros for IOP32x-based platforms - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. - */ - .macro get_irqnr_preamble, base, tmp - mrc p15, 0, \tmp, c15, c1, 0 - orr \tmp, \tmp, #(1 << 6) - mcr p15, 0, \tmp, c15, c1, 0 @ Enable cp6 access - mrc p15, 0, \tmp, c15, c1, 0 - mov \tmp, \tmp - sub pc, pc, #4 @ cp_wait - .endm - - .macro get_irqnr_and_base, irqnr, irqstat, base, tmp - mrc p6, 0, \irqstat, c8, c0, 0 @ Read IINTSRC - cmp \irqstat, #0 - clzne \irqnr, \irqstat - rsbne \irqnr, \irqnr, #32 - .endm - - .macro arch_ret_to_user, tmp1, tmp2 - mrc p15, 0, \tmp1, c15, c1, 0 - ands \tmp2, \tmp1, #(1 << 6) - bicne \tmp1, \tmp1, #(1 << 6) - mcrne p15, 0, \tmp1, c15, c1, 0 @ Disable cp6 access - .endm diff --git a/arch/arm/mach-iop32x/iop3xx.h b/arch/arm/mach-iop32x/iop3xx.h index 46b4b34a4ad2..a6ec7ebadb35 100644 --- a/arch/arm/mach-iop32x/iop3xx.h +++ b/arch/arm/mach-iop32x/iop3xx.h @@ -225,6 +225,7 @@ extern int iop3xx_get_init_atu(void); #include void iop3xx_map_io(void); +void iop_enable_cp6(void); void iop_init_cp6_handler(void); void iop_init_time(unsigned long tickrate); void iop3xx_restart(enum reboot_mode, const char *); diff --git a/arch/arm/mach-iop32x/irq.c b/arch/arm/mach-iop32x/irq.c index d1e8824cbd82..b820839eaae8 100644 --- a/arch/arm/mach-iop32x/irq.c +++ b/arch/arm/mach-iop32x/irq.c @@ -29,6 +29,15 @@ static void intstr_write(u32 val) asm volatile("mcr p6, 0, %0, c4, c0, 0" : : "r" (val)); } +static u32 iintsrc_read(void) +{ + int irq; + + asm volatile("mrc p6, 0, %0, c8, c0, 0" : "=r" (irq)); + + return irq; +} + static void iop32x_irq_mask(struct irq_data *d) { @@ -50,11 +59,25 @@ struct irq_chip ext_chip = { .irq_unmask = iop32x_irq_unmask, }; +void iop_handle_irq(struct pt_regs *regs) +{ + u32 mask; + + iop_enable_cp6(); + + do { + mask = iintsrc_read(); + if (mask) + generic_handle_irq(fls(mask)); + } while (mask); +} + void __init iop32x_init_irq(void) { int i; iop_init_cp6_handler(); + set_handle_irq(iop_handle_irq); intctl_write(0); intstr_write(0); From 54f481a2308efab49d2b14c3f8263b34fdb1c65e Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Fri, 26 Nov 2021 22:36:06 +0100 Subject: [PATCH 24/62] ARM: remove old-style irq entry The last user of arch_irq_handler_default is gone now, so the entry-macro-multi.S file and all references to mach/entry-macro.S can be removed, as well as the asm_do_IRQ() entrypoint into the interrupt handling routines implemented in C. Note: The ARMv7-M entry still uses its own top-level IRQ entry, calling nvic_handle_irq() from assembly. This could be changed to go through generic_handle_arch_irq() as well, but it's unclear to me if there are any benefits. Signed-off-by: Arnd Bergmann [ardb: keep irq_handler macro as it carries all the IRQ stack handling] Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M Reviewed-by: Linus Walleij --- arch/arm/Kconfig | 14 ++------------ arch/arm/include/asm/entry-macro-multi.S | 16 ---------------- arch/arm/include/asm/irq.h | 1 - arch/arm/include/asm/mach/arch.h | 2 -- arch/arm/kernel/entry-armv.S | 8 -------- arch/arm/kernel/irq.c | 17 ----------------- 6 files changed, 2 insertions(+), 56 deletions(-) delete mode 100644 arch/arm/include/asm/entry-macro-multi.S diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index d9ba6961b295..b0e403076227 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -58,6 +58,7 @@ config ARM select GENERIC_CPU_AUTOPROBE select GENERIC_EARLY_IOREMAP select GENERIC_IDLE_POLL_SETUP + select GENERIC_IRQ_MULTI_HANDLER if MMU select GENERIC_IRQ_PROBE select GENERIC_IRQ_SHOW select GENERIC_IRQ_SHOW_LEVEL @@ -320,7 +321,6 @@ config ARCH_MULTIPLATFORM select AUTO_ZRELADDR select TIMER_OF select COMMON_CLK - select GENERIC_IRQ_MULTI_HANDLER select HAVE_PCI select PCI_DOMAINS_GENERIC if PCI select SPARSE_IRQ @@ -344,7 +344,6 @@ config ARCH_EP93XX select ARM_AMBA imply ARM_PATCH_PHYS_VIRT select ARM_VIC - select GENERIC_IRQ_MULTI_HANDLER select AUTO_ZRELADDR select CLKSRC_MMIO select CPU_ARM920T @@ -359,7 +358,6 @@ config ARCH_FOOTBRIDGE select FOOTBRIDGE select NEED_MACH_IO_H if !MMU select NEED_MACH_MEMORY_H - select GENERIC_IRQ_MULTI_HANDLER help Support for systems based on the DC21285 companion chip ("FootBridge"), such as the Simtec CATS and the Rebel NetWinder. @@ -368,7 +366,6 @@ config ARCH_IOP32X bool "IOP32x-based" depends on MMU select CPU_XSCALE - select GENERIC_IRQ_MULTI_HANDLER select GPIO_IOP select GPIOLIB select FORCE_PCI @@ -384,7 +381,6 @@ config ARCH_IXP4XX select ARCH_SUPPORTS_BIG_ENDIAN select CPU_XSCALE select DMABOUNCE if PCI - select GENERIC_IRQ_MULTI_HANDLER select GPIO_IXP4XX select GPIOLIB select HAVE_PCI @@ -400,7 +396,6 @@ config ARCH_IXP4XX config ARCH_DOVE bool "Marvell Dove" select CPU_PJ4 - select GENERIC_IRQ_MULTI_HANDLER select GPIOLIB select HAVE_PCI select MVEBU_MBUS @@ -423,7 +418,6 @@ config ARCH_PXA select CLKSRC_MMIO select TIMER_OF select CPU_XSCALE if !CPU_XSC3 - select GENERIC_IRQ_MULTI_HANDLER select GPIO_PXA select GPIOLIB select IRQ_DOMAIN @@ -442,7 +436,6 @@ config ARCH_RPC select ARM_HAS_SG_CHAIN select CPU_SA110 select FIQ - select GENERIC_IRQ_MULTI_HANDLER select HAVE_PATA_PLATFORM select ISA_DMA_API select LEGACY_TIMER_TICK @@ -463,7 +456,6 @@ config ARCH_SA1100 select COMMON_CLK select CPU_FREQ select CPU_SA1100 - select GENERIC_IRQ_MULTI_HANDLER select GPIOLIB select IRQ_DOMAIN select ISA @@ -478,7 +470,6 @@ config ARCH_S3C24XX select CLKSRC_SAMSUNG_PWM select GPIO_SAMSUNG select GPIOLIB - select GENERIC_IRQ_MULTI_HANDLER select HAVE_S3C2410_I2C if I2C select NEED_MACH_IO_H select S3C2410_WATCHDOG @@ -497,7 +488,6 @@ config ARCH_OMAP1 select ARCH_OMAP select CLKSRC_MMIO select GENERIC_IRQ_CHIP - select GENERIC_IRQ_MULTI_HANDLER select GPIOLIB select HAVE_LEGACY_CLK select IRQ_DOMAIN @@ -1168,7 +1158,7 @@ config CURRENT_POINTER_IN_TPIDRURO config IRQSTACKS def_bool y - depends on GENERIC_IRQ_MULTI_HANDLER && THREAD_INFO_IN_TASK + depends on THREAD_INFO_IN_TASK select HAVE_IRQ_EXIT_ON_IRQ_STACK select HAVE_SOFTIRQ_ON_OWN_STACK diff --git a/arch/arm/include/asm/entry-macro-multi.S b/arch/arm/include/asm/entry-macro-multi.S deleted file mode 100644 index 24486dad9e19..000000000000 --- a/arch/arm/include/asm/entry-macro-multi.S +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include - -/* - * Interrupt handling. Preserves r7, r8, r9 - */ - .macro arch_irq_handler_default - get_irqnr_preamble r6, lr -1: get_irqnr_and_base r0, r2, r6, lr - movne r1, sp - @ - @ routine called with r0 = irq number, r1 = struct pt_regs * - @ - badrne lr, 1b - bne asm_do_IRQ - .endm diff --git a/arch/arm/include/asm/irq.h b/arch/arm/include/asm/irq.h index 1cbcc462b07e..a7c2337b0c7d 100644 --- a/arch/arm/include/asm/irq.h +++ b/arch/arm/include/asm/irq.h @@ -26,7 +26,6 @@ struct irqaction; struct pt_regs; -extern void asm_do_IRQ(unsigned int, struct pt_regs *); void handle_IRQ(unsigned int, struct pt_regs *); void init_IRQ(void); diff --git a/arch/arm/include/asm/mach/arch.h b/arch/arm/include/asm/mach/arch.h index eec0c0bda766..9349e7a82c9c 100644 --- a/arch/arm/include/asm/mach/arch.h +++ b/arch/arm/include/asm/mach/arch.h @@ -56,9 +56,7 @@ struct machine_desc { void (*init_time)(void); void (*init_machine)(void); void (*init_late)(void); -#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER void (*handle_irq)(struct pt_regs *); -#endif void (*restart)(enum reboot_mode, const char *); }; diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 5fb7465d14d9..9744d087ee9f 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -19,9 +19,6 @@ #include #include #include -#ifndef CONFIG_GENERIC_IRQ_MULTI_HANDLER -#include -#endif #include #include #include @@ -30,14 +27,12 @@ #include #include "entry-header.S" -#include #include /* * Interrupt handling. */ .macro irq_handler, from_user:req -#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER mov r0, sp #ifdef CONFIG_IRQSTACKS mov_l r2, irq_stack_ptr @ Take base address @@ -92,9 +87,6 @@ UNWIND( .setfp fpreg, sp ) mov sp, r9 @ Restore original SP #endif // CONFIG_UNWINDER_ARM #endif // CONFIG_IRQSTACKS -#else - arch_irq_handler_default -#endif .endm .macro pabt_helper diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index 5deb40f39999..5c6f8d11a3ce 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -120,23 +120,6 @@ void handle_IRQ(unsigned int irq, struct pt_regs *regs) ack_bad_irq(irq); } -/* - * asm_do_IRQ is the interface to be used from assembly code. - */ -asmlinkage void __exception_irq_entry -asm_do_IRQ(unsigned int irq, struct pt_regs *regs) -{ - struct pt_regs *old_regs; - - irq_enter(); - old_regs = set_irq_regs(regs); - - handle_IRQ(irq, regs); - - set_irq_regs(old_regs); - irq_exit(); -} - void __init init_IRQ(void) { int ret; From 52d24087176055d5994ac98378426421b2d6d653 Mon Sep 17 00:00:00 2001 From: Vladimir Murzin Date: Wed, 1 Dec 2021 11:02:59 +0000 Subject: [PATCH 25/62] irqchip: nvic: Use GENERIC_IRQ_MULTI_HANDLER Rather then restructuring the ARMv7M entrly logic per TODO, just move NVIC to GENERIC_IRQ_MULTI_HANDLER. Signed-off-by: Vladimir Murzin Acked-by: Mark Rutland Acked-by: Arnd Bergmann Acked-by: Marc Zyngier Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/include/asm/v7m.h | 3 ++- arch/arm/kernel/entry-v7m.S | 10 +++------- drivers/irqchip/Kconfig | 1 + drivers/irqchip/irq-nvic.c | 22 +++++----------------- 4 files changed, 11 insertions(+), 25 deletions(-) diff --git a/arch/arm/include/asm/v7m.h b/arch/arm/include/asm/v7m.h index 2cb00d15831b..4512f7e1918f 100644 --- a/arch/arm/include/asm/v7m.h +++ b/arch/arm/include/asm/v7m.h @@ -13,6 +13,7 @@ #define V7M_SCB_ICSR_PENDSVSET (1 << 28) #define V7M_SCB_ICSR_PENDSVCLR (1 << 27) #define V7M_SCB_ICSR_RETTOBASE (1 << 11) +#define V7M_SCB_ICSR_VECTACTIVE 0x000001ff #define V7M_SCB_VTOR 0x08 @@ -38,7 +39,7 @@ #define V7M_SCB_SHCSR_MEMFAULTENA (1 << 16) #define V7M_xPSR_FRAMEPTRALIGN 0x00000200 -#define V7M_xPSR_EXCEPTIONNO 0x000001ff +#define V7M_xPSR_EXCEPTIONNO V7M_SCB_ICSR_VECTACTIVE /* * When branching to an address that has bits [31:28] == 0xf an exception return diff --git a/arch/arm/kernel/entry-v7m.S b/arch/arm/kernel/entry-v7m.S index 7bde93c10962..520dd43e7e08 100644 --- a/arch/arm/kernel/entry-v7m.S +++ b/arch/arm/kernel/entry-v7m.S @@ -39,14 +39,10 @@ __irq_entry: @ @ Invoke the IRQ handler @ - mrs r0, ipsr - ldr r1, =V7M_xPSR_EXCEPTIONNO - and r0, r1 - sub r0, #16 - mov r1, sp + mov r0, sp stmdb sp!, {lr} - @ routine called with r0 = irq number, r1 = struct pt_regs * - bl nvic_handle_irq + @ routine called with r0 = struct pt_regs * + bl generic_handle_arch_irq pop {lr} @ diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 7038957f4a77..488eaa14d3a7 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -58,6 +58,7 @@ config ARM_NVIC bool select IRQ_DOMAIN_HIERARCHY select GENERIC_IRQ_CHIP + select GENERIC_IRQ_MULTI_HANDLER config ARM_VIC bool diff --git a/drivers/irqchip/irq-nvic.c b/drivers/irqchip/irq-nvic.c index 63bac3f78863..c0de696b3823 100644 --- a/drivers/irqchip/irq-nvic.c +++ b/drivers/irqchip/irq-nvic.c @@ -37,27 +37,14 @@ static struct irq_domain *nvic_irq_domain; -static void __nvic_handle_irq(irq_hw_number_t hwirq) +static void __irq_entry nvic_handle_irq(struct pt_regs *regs) { + unsigned long icsr = readl_relaxed(BASEADDR_V7M_SCB + V7M_SCB_ICSR); + irq_hw_number_t hwirq = (icsr & V7M_SCB_ICSR_VECTACTIVE) - 16; + generic_handle_domain_irq(nvic_irq_domain, hwirq); } -/* - * TODO: restructure the ARMv7M entry logic so that this entry logic can live - * in arch code. - */ -asmlinkage void __exception_irq_entry -nvic_handle_irq(irq_hw_number_t hwirq, struct pt_regs *regs) -{ - struct pt_regs *old_regs; - - irq_enter(); - old_regs = set_irq_regs(regs); - __nvic_handle_irq(hwirq); - set_irq_regs(old_regs); - irq_exit(); -} - static int nvic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs, void *arg) { @@ -141,6 +128,7 @@ static int __init nvic_of_init(struct device_node *node, for (i = 0; i < irqs; i += 4) writel_relaxed(0, nvic_base + NVIC_IPR + i); + set_handle_irq(nvic_handle_irq); return 0; } IRQCHIP_DECLARE(armv7m_nvic, "arm,armv7m-nvic", nvic_of_init); From 831a469bc1674b7898984ce96f2902534445c0cf Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 24 Nov 2021 12:09:48 +0100 Subject: [PATCH 26/62] ARM: entry: preserve thread_info pointer in switch_to Tweak the UP stack protector handling code so that the thread info pointer is preserved in R7 until set_current is called. This is needed for a subsequent patch that implements THREAD_INFO_IN_TASK and set_current for UP as well. This also means we will prefer the per-task protector on UP systems that implement the thread ID registers, so tweak the preprocessor conditionals to reflect this. Acked-by: Linus Walleij Acked-by: Nicolas Pitre Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/kernel/entry-armv.S | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 9744d087ee9f..1a6cf711a3b4 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -806,16 +806,16 @@ ENTRY(__switch_to) ldr r6, [r2, #TI_CPU_DOMAIN] #endif switch_tls r1, r4, r5, r3, r7 -#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) - ldr r7, [r2, #TI_TASK] +#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) && \ + !defined(CONFIG_STACKPROTECTOR_PER_TASK) + ldr r9, [r2, #TI_TASK] ldr r8, =__stack_chk_guard .if (TSK_STACK_CANARY > IMM12_MASK) - add r7, r7, #TSK_STACK_CANARY & ~IMM12_MASK + add r9, r9, #TSK_STACK_CANARY & ~IMM12_MASK .endif - ldr r7, [r7, #TSK_STACK_CANARY & IMM12_MASK] -#elif defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) - mov r7, r2 @ Preserve 'next' + ldr r9, [r9, #TSK_STACK_CANARY & IMM12_MASK] #endif + mov r7, r2 @ Preserve 'next' #ifdef CONFIG_CPU_USE_DOMAINS mcr p15, 0, r6, c3, c0, 0 @ Set domain register #endif @@ -824,8 +824,9 @@ ENTRY(__switch_to) ldr r0, =thread_notify_head mov r1, #THREAD_NOTIFY_SWITCH bl atomic_notifier_call_chain -#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) - str r7, [r8] +#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) && \ + !defined(CONFIG_STACKPROTECTOR_PER_TASK) + str r9, [r8] #endif mov r0, r5 #if !defined(CONFIG_THUMB2_KERNEL) && !defined(CONFIG_VMAP_STACK) From 1fa8c4b19543ae8c8894ec92a18696c9f9b03fc8 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 24 Nov 2021 18:56:22 +0100 Subject: [PATCH 27/62] ARM: module: implement support for PC-relative group relocations Add support for the R_ARM_ALU_PC_Gn_NC and R_ARM_LDR_PC_G2 group relocations [0] so we can use them in modules. These will be used to load the current task pointer from a global variable without having to rely on a literal pool entry to carry the address of this variable, which may have a significant negative impact on cache utilization for variables that are used often and in many different places, as each occurrence will result in a literal pool entry and therefore a line in the D-cache. [0] 'ELF for the ARM architecture' https://github.com/ARM-software/abi-aa/releases Acked-by: Linus Walleij Acked-by: Nicolas Pitre Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/include/asm/elf.h | 3 ++ arch/arm/kernel/module.c | 85 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/arch/arm/include/asm/elf.h b/arch/arm/include/asm/elf.h index b8102a6ddf16..d68101655b74 100644 --- a/arch/arm/include/asm/elf.h +++ b/arch/arm/include/asm/elf.h @@ -61,6 +61,9 @@ typedef struct user_fp elf_fpregset_t; #define R_ARM_MOVT_ABS 44 #define R_ARM_MOVW_PREL_NC 45 #define R_ARM_MOVT_PREL 46 +#define R_ARM_ALU_PC_G0_NC 57 +#define R_ARM_ALU_PC_G1_NC 59 +#define R_ARM_LDR_PC_G2 63 #define R_ARM_THM_CALL 10 #define R_ARM_THM_JUMP24 30 diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c index beac45e89ba6..4d33a7acf617 100644 --- a/arch/arm/kernel/module.c +++ b/arch/arm/kernel/module.c @@ -68,6 +68,42 @@ bool module_exit_section(const char *name) strstarts(name, ".ARM.exidx.exit"); } +/* + * This implements the partitioning algorithm for group relocations as + * documented in the ARM AArch32 ELF psABI (IHI 0044). + * + * A single PC-relative symbol reference is divided in up to 3 add or subtract + * operations, where the final one could be incorporated into a load/store + * instruction with immediate offset. E.g., + * + * ADD Rd, PC, #... or ADD Rd, PC, #... + * ADD Rd, Rd, #... ADD Rd, Rd, #... + * LDR Rd, [Rd, #...] ADD Rd, Rd, #... + * + * The latter has a guaranteed range of only 16 MiB (3x8 == 24 bits), so it is + * of limited use in the kernel. However, the ADD/ADD/LDR combo has a range of + * -/+ 256 MiB, (2x8 + 12 == 28 bits), which means it has sufficient range for + * any in-kernel symbol reference (unless module PLTs are being used). + * + * The main advantage of this approach over the typical pattern using a literal + * load is that literal loads may miss in the D-cache, and generally lead to + * lower cache efficiency for variables that are referenced often from many + * different places in the code. + */ +static u32 get_group_rem(u32 group, u32 *offset) +{ + u32 val = *offset; + u32 shift; + do { + shift = val ? (31 - __fls(val)) & ~1 : 32; + *offset = val; + if (!val) + break; + val &= 0xffffff >> shift; + } while (group--); + return shift; +} + int apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, unsigned int relindex, struct module *module) @@ -82,6 +118,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, unsigned long loc; Elf32_Sym *sym; const char *symname; + u32 shift, group = 1; s32 offset; u32 tmp; #ifdef CONFIG_THUMB2_KERNEL @@ -212,6 +249,54 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, *(u32 *)loc = __opcode_to_mem_arm(tmp); break; + case R_ARM_ALU_PC_G0_NC: + group = 0; + fallthrough; + case R_ARM_ALU_PC_G1_NC: + tmp = __mem_to_opcode_arm(*(u32 *)loc); + offset = ror32(tmp & 0xff, (tmp & 0xf00) >> 7); + if (tmp & BIT(22)) + offset = -offset; + offset += sym->st_value - loc; + if (offset < 0) { + offset = -offset; + tmp = (tmp & ~BIT(23)) | BIT(22); // SUB opcode + } else { + tmp = (tmp & ~BIT(22)) | BIT(23); // ADD opcode + } + + shift = get_group_rem(group, &offset); + if (shift < 24) { + offset >>= 24 - shift; + offset |= (shift + 8) << 7; + } + *(u32 *)loc = __opcode_to_mem_arm((tmp & ~0xfff) | offset); + break; + + case R_ARM_LDR_PC_G2: + tmp = __mem_to_opcode_arm(*(u32 *)loc); + offset = tmp & 0xfff; + if (~tmp & BIT(23)) // U bit cleared? + offset = -offset; + offset += sym->st_value - loc; + if (offset < 0) { + offset = -offset; + tmp &= ~BIT(23); // clear U bit + } else { + tmp |= BIT(23); // set U bit + } + get_group_rem(2, &offset); + + if (offset > 0xfff) { + pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n", + module->name, relindex, i, symname, + ELF32_R_TYPE(rel->r_info), loc, + sym->st_value); + return -ENOEXEC; + } + *(u32 *)loc = __opcode_to_mem_arm((tmp & ~0xfff) | offset); + break; + #ifdef CONFIG_THUMB2_KERNEL case R_ARM_THM_CALL: case R_ARM_THM_JUMP24: From 4e918ab13eaf40f19938659cb5a22c93172778a8 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 26 Nov 2021 19:37:27 +0100 Subject: [PATCH 28/62] ARM: assembler: add optimized ldr/str macros to load variables from memory We will be adding variable loads to various hot paths, so it makes sense to add a helper macro that can load variables from asm code without the use of literal pool entries. On v7 or later, we can simply use MOVW/MOVT pairs, but on earlier cores, this requires a bit of hackery to emit a instruction sequence that implements this using a sequence of ADD/LDR instructions. Acked-by: Linus Walleij Acked-by: Nicolas Pitre Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/include/asm/assembler.h | 45 +++++++++++++++++++++++++++++--- arch/arm/kernel/entry-armv.S | 2 +- arch/arm/kernel/entry-header.S | 2 +- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 1b9d4df331aa..2095638b7140 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -568,12 +568,12 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) /* * mov_l - move a constant value or [relocated] address into a register */ - .macro mov_l, dst:req, imm:req + .macro mov_l, dst:req, imm:req, cond .if __LINUX_ARM_ARCH__ < 7 - ldr \dst, =\imm + ldr\cond \dst, =\imm .else - movw \dst, #:lower16:\imm - movt \dst, #:upper16:\imm + movw\cond \dst, #:lower16:\imm + movt\cond \dst, #:upper16:\imm .endif .endm @@ -611,6 +611,43 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) __adldst_l str, \src, \sym, \tmp, \cond .endm + .macro __ldst_va, op, reg, tmp, sym, cond +#if __LINUX_ARM_ARCH__ >= 7 || \ + (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) || \ + (defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) + mov_l \tmp, \sym, \cond + \op\cond \reg, [\tmp] +#else + /* + * Avoid a literal load, by emitting a sequence of ADD/LDR instructions + * with the appropriate relocations. The combined sequence has a range + * of -/+ 256 MiB, which should be sufficient for the core kernel and + * for modules loaded into the module region. + */ + .globl \sym + .reloc .L0_\@, R_ARM_ALU_PC_G0_NC, \sym + .reloc .L1_\@, R_ARM_ALU_PC_G1_NC, \sym + .reloc .L2_\@, R_ARM_LDR_PC_G2, \sym +.L0_\@: sub\cond \tmp, pc, #8 +.L1_\@: sub\cond \tmp, \tmp, #4 +.L2_\@: \op\cond \reg, [\tmp, #0] +#endif + .endm + + /* + * ldr_va - load a 32-bit word from the virtual address of \sym + */ + .macro ldr_va, rd:req, sym:req, cond + __ldst_va ldr, \rd, \rd, \sym, \cond + .endm + + /* + * str_va - store a 32-bit word to the virtual address of \sym + */ + .macro str_va, rn:req, sym:req, tmp:req, cond + __ldst_va str, \rn, \tmp, \sym, \cond + .endm + /* * rev_l - byte-swap a 32-bit value * diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 1a6cf711a3b4..7f7ac963445c 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -53,7 +53,7 @@ UNWIND( .setfp fpreg, sp ) subs r2, sp, r0 @ SP above bottom of IRQ stack? rsbscs r2, r2, #THREAD_SIZE @ ... and below the top? #ifdef CONFIG_VMAP_STACK - ldr_l r2, high_memory, cc @ End of the linear region + ldr_va r2, high_memory, cc @ End of the linear region cmpcc r2, r0 @ Stack pointer was below it? #endif movcs sp, r0 @ If so, revert to incoming SP diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S index 81df2a3561ca..268f7f4c5c05 100644 --- a/arch/arm/kernel/entry-header.S +++ b/arch/arm/kernel/entry-header.S @@ -445,7 +445,7 @@ THUMB( it ne ) @ in such cases so just carry on. @ str ip, [r0, #12] @ Stash IP on the mode stack - ldr_l ip, high_memory @ Start of VMALLOC space + ldr_va ip, high_memory @ Start of VMALLOC space ARM( cmp sp, ip ) @ SP in vmalloc space? THUMB( cmp r1, ip ) THUMB( itt lo ) From 7b9896c352073156a325c3bb0dc4c46e06e2a468 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 25 Nov 2021 10:26:44 +0100 Subject: [PATCH 29/62] ARM: percpu: add SMP_ON_UP support Permit the use of the TPIDRPRW system register for carrying the per-CPU offset in generic SMP configurations that also target non-SMP capable ARMv6 cores. This uses the SMP_ON_UP code patching framework to turn all TPIDRPRW accesses into reads/writes of entry #0 in the __per_cpu_offset array. While at it, switch over some existing direct TPIDRPRW accesses in asm code to invocations of a new helper that is patched in the same way when necessary. Note that CPU_V6+SMP without SMP_ON_UP results in a kernel that does not boot on v6 CPUs without SMP extensions, so add this dependency to Kconfig as well. Acked-by: Linus Walleij Acked-by: Nicolas Pitre Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/include/asm/assembler.h | 59 ++++++++++++++++++++++++++++++-- arch/arm/include/asm/insn.h | 24 +++++++++++++ arch/arm/include/asm/percpu.h | 25 ++++++++++++-- arch/arm/kernel/entry-armv.S | 16 ++------- arch/arm/kernel/sleep.S | 4 +-- arch/arm/mm/Kconfig | 1 + 6 files changed, 107 insertions(+), 22 deletions(-) diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 2095638b7140..f9b3dd0e9ef5 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -220,9 +220,7 @@ THUMB( fpreg .req r7 ) .macro reload_current, t1:req, t2:req #ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO - adr_l \t1, __entry_task @ get __entry_task base address - mrc p15, 0, \t2, c13, c0, 4 @ get per-CPU offset - ldr \t1, [\t1, \t2] @ load variable + ldr_this_cpu \t1, __entry_task, \t1, \t2 mcr p15, 0, \t1, c13, c0, 3 @ store in TPIDRURO #endif .endm @@ -312,6 +310,26 @@ THUMB( fpreg .req r7 ) #define ALT_UP_B(label) b label #endif + /* + * this_cpu_offset - load the per-CPU offset of this CPU into + * register 'rd' + */ + .macro this_cpu_offset, rd:req +#ifdef CONFIG_SMP +ALT_SMP(mrc p15, 0, \rd, c13, c0, 4) +#ifdef CONFIG_CPU_V6 +ALT_UP_B(.L1_\@) +.L0_\@: + .subsection 1 +.L1_\@: ldr_va \rd, __per_cpu_offset + b .L0_\@ + .previous +#endif +#else + mov \rd, #0 +#endif + .endm + /* * Instruction barrier */ @@ -648,6 +666,41 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) __ldst_va str, \rn, \tmp, \sym, \cond .endm + /* + * ldr_this_cpu_armv6 - Load a 32-bit word from the per-CPU variable 'sym', + * without using a temp register. Supported in ARM mode + * only. + */ + .macro ldr_this_cpu_armv6, rd:req, sym:req + this_cpu_offset \rd + .globl \sym + .reloc .L0_\@, R_ARM_ALU_PC_G0_NC, \sym + .reloc .L1_\@, R_ARM_ALU_PC_G1_NC, \sym + .reloc .L2_\@, R_ARM_LDR_PC_G2, \sym + add \rd, \rd, pc +.L0_\@: sub \rd, \rd, #4 +.L1_\@: sub \rd, \rd, #0 +.L2_\@: ldr \rd, [\rd, #4] + .endm + + /* + * ldr_this_cpu - Load a 32-bit word from the per-CPU variable 'sym' + * into register 'rd', which may be the stack pointer, + * using 't1' and 't2' as general temp registers. These + * are permitted to overlap with 'rd' if != sp + */ + .macro ldr_this_cpu, rd:req, sym:req, t1:req, t2:req +#if __LINUX_ARM_ARCH__ >= 7 || \ + (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) || \ + (defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) + this_cpu_offset \t1 + mov_l \t2, \sym + ldr \rd, [\t1, \t2] +#else + ldr_this_cpu_armv6 \rd, \sym +#endif + .endm + /* * rev_l - byte-swap a 32-bit value * diff --git a/arch/arm/include/asm/insn.h b/arch/arm/include/asm/insn.h index 5475cbf9fb6b..a160ed3ea427 100644 --- a/arch/arm/include/asm/insn.h +++ b/arch/arm/include/asm/insn.h @@ -2,6 +2,30 @@ #ifndef __ASM_ARM_INSN_H #define __ASM_ARM_INSN_H +#include + +/* + * Avoid a literal load by emitting a sequence of ADD/LDR instructions with the + * appropriate relocations. The combined sequence has a range of -/+ 256 MiB, + * which should be sufficient for the core kernel as well as modules loaded + * into the module region. (Not supported by LLD before release 14) + */ +#if !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) && \ + !(defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) +#define LOAD_SYM_ARMV6(reg, sym) \ + " .globl " #sym " \n\t" \ + " .reloc 10f, R_ARM_ALU_PC_G0_NC, " #sym " \n\t" \ + " .reloc 11f, R_ARM_ALU_PC_G1_NC, " #sym " \n\t" \ + " .reloc 12f, R_ARM_LDR_PC_G2, " #sym " \n\t" \ + "10: sub " #reg ", pc, #8 \n\t" \ + "11: sub " #reg ", " #reg ", #4 \n\t" \ + "12: ldr " #reg ", [" #reg ", #0] \n\t" +#else +#define LOAD_SYM_ARMV6(reg, sym) \ + " ldr " #reg ", =" #sym " \n\t" \ + " ldr " #reg ", [" #reg "] \n\t" +#endif + static inline unsigned long arm_gen_nop(void) { diff --git a/arch/arm/include/asm/percpu.h b/arch/arm/include/asm/percpu.h index e2fcb3cfd3de..a4a0d38d016a 100644 --- a/arch/arm/include/asm/percpu.h +++ b/arch/arm/include/asm/percpu.h @@ -5,15 +5,22 @@ #ifndef _ASM_ARM_PERCPU_H_ #define _ASM_ARM_PERCPU_H_ +#include + register unsigned long current_stack_pointer asm ("sp"); /* * Same as asm-generic/percpu.h, except that we store the per cpu offset * in the TPIDRPRW. TPIDRPRW only exists on V6K and V7 */ -#if defined(CONFIG_SMP) && !defined(CONFIG_CPU_V6) +#ifdef CONFIG_SMP static inline void set_my_cpu_offset(unsigned long off) { + extern unsigned int smp_on_up; + + if (IS_ENABLED(CONFIG_CPU_V6) && !smp_on_up) + return; + /* Set TPIDRPRW */ asm volatile("mcr p15, 0, %0, c13, c0, 4" : : "r" (off) : "memory"); } @@ -27,8 +34,20 @@ static inline unsigned long __my_cpu_offset(void) * We want to allow caching the value, so avoid using volatile and * instead use a fake stack read to hazard against barrier(). */ - asm("mrc p15, 0, %0, c13, c0, 4" : "=r" (off) - : "Q" (*(const unsigned long *)current_stack_pointer)); + asm("0: mrc p15, 0, %0, c13, c0, 4 \n\t" +#ifdef CONFIG_CPU_V6 + "1: \n\t" + " .subsection 1 \n\t" + "2: " LOAD_SYM_ARMV6(%0, __per_cpu_offset) " \n\t" + " b 1b \n\t" + " .previous \n\t" + " .pushsection \".alt.smp.init\", \"a\" \n\t" + " .long 0b - . \n\t" + " b . + (2b - 0b) \n\t" + " .popsection \n\t" +#endif + : "=r" (off) + : "Q" (*(const unsigned long *)current_stack_pointer)); return off; } diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 7f7ac963445c..43d917f0d9a9 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -35,15 +35,14 @@ .macro irq_handler, from_user:req mov r0, sp #ifdef CONFIG_IRQSTACKS - mov_l r2, irq_stack_ptr @ Take base address - mrc p15, 0, r3, c13, c0, 4 @ Get CPU offset #ifdef CONFIG_UNWINDER_ARM mov fpreg, sp @ Preserve original SP #else mov r8, fp @ Preserve original FP mov r9, sp @ Preserve original SP #endif - ldr sp, [r2, r3] @ Load SP from per-CPU var + ldr_this_cpu sp, irq_stack_ptr, r2, r3 + .if \from_user == 0 UNWIND( .setfp fpreg, sp ) @ @@ -876,16 +875,7 @@ __bad_stack: THUMB( bx pc ) THUMB( nop ) THUMB( .arm ) - mrc p15, 0, ip, c13, c0, 4 @ Get per-CPU offset - - .globl overflow_stack_ptr - .reloc 0f, R_ARM_ALU_PC_G0_NC, overflow_stack_ptr - .reloc 1f, R_ARM_ALU_PC_G1_NC, overflow_stack_ptr - .reloc 2f, R_ARM_LDR_PC_G2, overflow_stack_ptr - add ip, ip, pc -0: add ip, ip, #-4 -1: add ip, ip, #0 -2: ldr ip, [ip, #4] + ldr_this_cpu_armv6 ip, overflow_stack_ptr str sp, [ip, #-4]! @ Preserve original SP value mov sp, ip @ Switch to overflow stack diff --git a/arch/arm/kernel/sleep.S b/arch/arm/kernel/sleep.S index 803b51e5cba0..f909baf17912 100644 --- a/arch/arm/kernel/sleep.S +++ b/arch/arm/kernel/sleep.S @@ -71,9 +71,7 @@ ENTRY(__cpu_suspend) @ Run the suspend code from the overflow stack so we don't have to rely @ on vmalloc-to-phys conversions anywhere in the arch suspend code. @ The original SP value captured in R5 will be restored on the way out. - mov_l r6, overflow_stack_ptr @ Base pointer - mrc p15, 0, r7, c13, c0, 4 @ Get per-CPU offset - ldr sp, [r6, r7] @ Address of this CPU's overflow stack + ldr_this_cpu sp, overflow_stack_ptr, r6, r7 #endif add r4, r4, #12 @ Space for pgd, virt sp, phys resume fn sub sp, sp, r4 @ allocate CPU state on stack diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 58afba346729..a91ff22c6c2e 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -386,6 +386,7 @@ config CPU_V6 select CPU_PABRT_V6 select CPU_THUMB_CAPABLE select CPU_TLB_V6 if MMU + select SMP_ON_UP if SMP # ARMv6k config CPU_V6K From b87cf9118e03d8a83039c88fa348165148f5d487 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 26 Nov 2021 10:13:06 +0100 Subject: [PATCH 30/62] ARM: use TLS register for 'current' on !SMP as well Enable the use of the TLS register to hold the 'current' pointer also on non-SMP configurations that target v6k or later CPUs. This will permit the use of THREAD_INFO_IN_TASK as well as IRQ stacks and vmap'ed stacks for such configurations. Acked-by: Linus Walleij Acked-by: Nicolas Pitre Acked-by: Arnd Bergmann Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index b0e403076227..5ad2151c43dd 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1154,7 +1154,7 @@ config SMP_ON_UP config CURRENT_POINTER_IN_TPIDRURO def_bool y - depends on SMP && CPU_32v6K && !CPU_V6 + depends on CPU_32v6K && !CPU_V6 config IRQSTACKS def_bool y From c2755910373bb5dfb9aa68ba2924036686815c9e Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 25 Nov 2021 23:21:45 +0100 Subject: [PATCH 31/62] ARM: smp: defer TPIDRURO update for SMP v6 configurations too Defer TPIDURO updates for user space until exit also for CPU_V6+SMP configurations so that we can decide at runtime whether to use it to carry the current pointer, provided that we are running on a CPU that actually implements this register. This is needed for THREAD_INFO_IN_TASK support for UP systems, which requires that all SMP capable systems use the TPIDRURO based access to 'current' as the only remaining alternative will be a global variable which only works on UP. Acked-by: Linus Walleij Acked-by: Nicolas Pitre Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/include/asm/tls.h | 13 +++++++------ arch/arm/kernel/entry-header.S | 11 ++++++++++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/arch/arm/include/asm/tls.h b/arch/arm/include/asm/tls.h index c3296499176c..d712c170c095 100644 --- a/arch/arm/include/asm/tls.h +++ b/arch/arm/include/asm/tls.h @@ -18,13 +18,14 @@ .endm .macro switch_tls_v6, base, tp, tpuser, tmp1, tmp2 - ldr \tmp1, =elf_hwcap - ldr \tmp1, [\tmp1, #0] + ldr_va \tmp1, elf_hwcap mov \tmp2, #0xffff0fff tst \tmp1, #HWCAP_TLS @ hardware TLS available? streq \tp, [\tmp2, #-15] @ set TLS value at 0xffff0ff0 mrcne p15, 0, \tmp2, c13, c0, 2 @ get the user r/w register +#ifndef CONFIG_SMP mcrne p15, 0, \tp, c13, c0, 3 @ yes, set TLS register +#endif mcrne p15, 0, \tpuser, c13, c0, 2 @ set user r/w register strne \tmp2, [\base, #TI_TP_VALUE + 4] @ save it .endm @@ -43,7 +44,7 @@ #elif defined(CONFIG_CPU_V6) #define tls_emu 0 #define has_tls_reg (elf_hwcap & HWCAP_TLS) -#define defer_tls_reg_update 0 +#define defer_tls_reg_update IS_ENABLED(CONFIG_SMP) #define switch_tls switch_tls_v6 #elif defined(CONFIG_CPU_32v6K) #define tls_emu 0 @@ -81,11 +82,11 @@ static inline void set_tls(unsigned long val) */ barrier(); - if (!tls_emu && !defer_tls_reg_update) { - if (has_tls_reg) { + if (!tls_emu) { + if (has_tls_reg && !defer_tls_reg_update) { asm("mcr p15, 0, %0, c13, c0, 3" : : "r" (val)); - } else { + } else if (!has_tls_reg) { #ifdef CONFIG_KUSER_HELPERS /* * User space must never try to access this diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S index 268f7f4c5c05..cb82ff5adec1 100644 --- a/arch/arm/kernel/entry-header.S +++ b/arch/arm/kernel/entry-header.S @@ -292,12 +292,21 @@ .macro restore_user_regs, fast = 0, offset = 0 -#if defined(CONFIG_CPU_32v6K) && !defined(CONFIG_CPU_V6) +#if defined(CONFIG_CPU_32v6K) || defined(CONFIG_SMP) +#if defined(CONFIG_CPU_V6) && defined(CONFIG_SMP) +ALT_SMP(b .L1_\@ ) +ALT_UP( nop ) + ldr_va r1, elf_hwcap + tst r1, #HWCAP_TLS @ hardware TLS available? + beq .L2_\@ +.L1_\@: +#endif @ The TLS register update is deferred until return to user space so we @ can use it for other things while running in the kernel get_thread_info r1 ldr r1, [r1, #TI_TP_VALUE] mcr p15, 0, r1, c13, c0, 3 @ set TLS register +.L2_\@: #endif uaccess_enable r1, isb=0 From 9c46929e7989efacc1dd0a1dd662a839897ea2b6 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 24 Nov 2021 14:08:11 +0100 Subject: [PATCH 32/62] ARM: implement THREAD_INFO_IN_TASK for uniprocessor systems On UP systems, only a single task can be 'current' at the same time, which means we can use a global variable to track it. This means we can also enable THREAD_INFO_IN_TASK for those systems, as in that case, thread_info is accessed via current rather than the other way around, removing the need to store thread_info at the base of the task stack. This, in turn, permits us to enable IRQ stacks and vmap'ed stacks on UP systems as well. To partially mitigate the performance overhead of this arrangement, use a ADD/ADD/LDR sequence with the appropriate PC-relative group relocations to load the value of current when needed. This means that accessing current will still only require a single load as before, avoiding the need for a literal to carry the address of the global variable in each function. However, accessing thread_info will now require this load as well. Acked-by: Linus Walleij Acked-by: Nicolas Pitre Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/Kconfig | 8 +-- arch/arm/include/asm/assembler.h | 83 +++++++++++++++++++----------- arch/arm/include/asm/current.h | 37 +++++++------ arch/arm/include/asm/switch_to.h | 3 +- arch/arm/include/asm/thread_info.h | 27 ---------- arch/arm/kernel/asm-offsets.c | 3 -- arch/arm/kernel/entry-armv.S | 11 ++-- arch/arm/kernel/entry-v7m.S | 10 ++-- arch/arm/kernel/head-common.S | 4 +- arch/arm/kernel/process.c | 7 ++- arch/arm/kernel/smp.c | 11 ++++ arch/arm/kernel/traps.c | 4 ++ 12 files changed, 116 insertions(+), 92 deletions(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 5ad2151c43dd..50ae5286f59b 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -127,8 +127,8 @@ config ARM select PERF_USE_VMALLOC select RTC_LIB select SYS_SUPPORTS_APM_EMULATION - select THREAD_INFO_IN_TASK if CURRENT_POINTER_IN_TPIDRURO - select HAVE_ARCH_VMAP_STACK if MMU && THREAD_INFO_IN_TASK && (!LD_IS_LLD || LLD_VERSION >= 140000) + select THREAD_INFO_IN_TASK + select HAVE_ARCH_VMAP_STACK if MMU && (!LD_IS_LLD || LLD_VERSION >= 140000) select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M # Above selects are sorted alphabetically; please add new ones # according to that. Thanks. @@ -1158,7 +1158,7 @@ config CURRENT_POINTER_IN_TPIDRURO config IRQSTACKS def_bool y - depends on THREAD_INFO_IN_TASK + depends on MMU select HAVE_IRQ_EXIT_ON_IRQ_STACK select HAVE_SOFTIRQ_ON_OWN_STACK @@ -1608,7 +1608,7 @@ config CC_HAVE_STACKPROTECTOR_TLS config STACKPROTECTOR_PER_TASK bool "Use a unique stack canary value for each task" - depends on STACKPROTECTOR && THREAD_INFO_IN_TASK && !XIP_DEFLATED_DATA + depends on STACKPROTECTOR && CURRENT_POINTER_IN_TPIDRURO && !XIP_DEFLATED_DATA depends on GCC_PLUGINS || CC_HAVE_STACKPROTECTOR_TLS select GCC_PLUGIN_ARM_SSP_PER_TASK if !CC_HAVE_STACKPROTECTOR_TLS default y diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index f9b3dd0e9ef5..59d7b9e81934 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -203,41 +203,12 @@ THUMB( fpreg .req r7 ) .endm .endr - .macro get_current, rd -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO - mrc p15, 0, \rd, c13, c0, 3 @ get TPIDRURO register -#else - get_thread_info \rd - ldr \rd, [\rd, #TI_TASK] -#endif - .endm - - .macro set_current, rn -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO - mcr p15, 0, \rn, c13, c0, 3 @ set TPIDRURO register -#endif - .endm - - .macro reload_current, t1:req, t2:req -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO - ldr_this_cpu \t1, __entry_task, \t1, \t2 - mcr p15, 0, \t1, c13, c0, 3 @ store in TPIDRURO -#endif - .endm - /* * Get current thread_info. */ .macro get_thread_info, rd -#ifdef CONFIG_THREAD_INFO_IN_TASK /* thread_info is the first member of struct task_struct */ get_current \rd -#else - ARM( mov \rd, sp, lsr #THREAD_SIZE_ORDER + PAGE_SHIFT ) - THUMB( mov \rd, sp ) - THUMB( lsr \rd, \rd, #THREAD_SIZE_ORDER + PAGE_SHIFT ) - mov \rd, \rd, lsl #THREAD_SIZE_ORDER + PAGE_SHIFT -#endif .endm /* @@ -330,6 +301,60 @@ ALT_UP_B(.L1_\@) #endif .endm + /* + * set_current - store the task pointer of this CPU's current task + */ + .macro set_current, rn:req, tmp:req +#if defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP) +9998: mcr p15, 0, \rn, c13, c0, 3 @ set TPIDRURO register +#ifdef CONFIG_CPU_V6 +ALT_UP_B(.L0_\@) + .subsection 1 +.L0_\@: str_va \rn, __current, \tmp + b .L1_\@ + .previous +.L1_\@: +#endif +#else + str_va \rn, __current, \tmp +#endif + .endm + + /* + * get_current - load the task pointer of this CPU's current task + */ + .macro get_current, rd:req +#if defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP) +9998: mrc p15, 0, \rd, c13, c0, 3 @ get TPIDRURO register +#ifdef CONFIG_CPU_V6 +ALT_UP_B(.L0_\@) + .subsection 1 +.L0_\@: ldr_va \rd, __current + b .L1_\@ + .previous +.L1_\@: +#endif +#else + ldr_va \rd, __current +#endif + .endm + + /* + * reload_current - reload the task pointer of this CPU's current task + * into the TLS register + */ + .macro reload_current, t1:req, t2:req +#if defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP) +#ifdef CONFIG_CPU_V6 +ALT_SMP(nop) +ALT_UP_B(.L0_\@) +#endif + ldr_this_cpu \t1, __entry_task, \t1, \t2 + mcr p15, 0, \t1, c13, c0, 3 @ store in TPIDRURO +.L0_\@: +#endif + .endm + /* * Instruction barrier */ diff --git a/arch/arm/include/asm/current.h b/arch/arm/include/asm/current.h index 6bf0aad672c3..69ecf4c6c725 100644 --- a/arch/arm/include/asm/current.h +++ b/arch/arm/include/asm/current.h @@ -8,25 +8,18 @@ #define _ASM_ARM_CURRENT_H #ifndef __ASSEMBLY__ +#include struct task_struct; -static inline void set_current(struct task_struct *cur) -{ - if (!IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO)) - return; +extern struct task_struct *__current; - /* Set TPIDRURO */ - asm("mcr p15, 0, %0, c13, c0, 3" :: "r"(cur) : "memory"); -} - -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO - -static inline struct task_struct *get_current(void) +static inline __attribute_const__ struct task_struct *get_current(void) { struct task_struct *cur; #if __has_builtin(__builtin_thread_pointer) && \ + defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) && \ !(defined(CONFIG_THUMB2_KERNEL) && \ defined(CONFIG_CC_IS_CLANG) && CONFIG_CLANG_VERSION < 130001) /* @@ -39,16 +32,30 @@ static inline struct task_struct *get_current(void) * https://github.com/ClangBuiltLinux/linux/issues/1485 */ cur = __builtin_thread_pointer(); +#elif defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP) + asm("0: mrc p15, 0, %0, c13, c0, 3 \n\t" +#ifdef CONFIG_CPU_V6 + "1: \n\t" + " .subsection 1 \n\t" + "2: " LOAD_SYM_ARMV6(%0, __current) " \n\t" + " b 1b \n\t" + " .previous \n\t" + " .pushsection \".alt.smp.init\", \"a\" \n\t" + " .long 0b - . \n\t" + " b . + (2b - 0b) \n\t" + " .popsection \n\t" +#endif + : "=r"(cur)); +#elif __LINUX_ARM_ARCH__>=7 || \ + (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) + cur = __current; #else - asm("mrc p15, 0, %0, c13, c0, 3" : "=r"(cur)); + asm(LOAD_SYM_ARMV6(%0, __current) : "=r"(cur)); #endif return cur; } #define current get_current() -#else -#include -#endif /* CONFIG_CURRENT_POINTER_IN_TPIDRURO */ #endif /* __ASSEMBLY__ */ diff --git a/arch/arm/include/asm/switch_to.h b/arch/arm/include/asm/switch_to.h index b55c7b2755e4..a482c99934ff 100644 --- a/arch/arm/include/asm/switch_to.h +++ b/arch/arm/include/asm/switch_to.h @@ -40,7 +40,8 @@ static inline void set_ti_cpu(struct task_struct *p) do { \ __complete_pending_tlbi(); \ set_ti_cpu(next); \ - if (IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO)) \ + if (IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || \ + IS_ENABLED(CONFIG_SMP)) \ __this_cpu_write(__entry_task, next); \ last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \ } while (0) diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index 004b89d86224..aecc403b2880 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -62,9 +62,6 @@ struct cpu_context_save { struct thread_info { unsigned long flags; /* low level flags */ int preempt_count; /* 0 => preemptable, <0 => bug */ -#ifndef CONFIG_THREAD_INFO_IN_TASK - struct task_struct *task; /* main task structure */ -#endif __u32 cpu; /* cpu */ __u32 cpu_domain; /* cpu domain */ struct cpu_context_save cpu_context; /* cpu context */ @@ -80,39 +77,15 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ - INIT_THREAD_INFO_TASK(tsk) \ .flags = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ } -#ifdef CONFIG_THREAD_INFO_IN_TASK -#define INIT_THREAD_INFO_TASK(tsk) - static inline struct task_struct *thread_task(struct thread_info* ti) { return (struct task_struct *)ti; } -#else -#define INIT_THREAD_INFO_TASK(tsk) .task = &(tsk), - -static inline struct task_struct *thread_task(struct thread_info* ti) -{ - return ti->task; -} - -/* - * how to get the thread information struct from C - */ -static inline struct thread_info *current_thread_info(void) __attribute_const__; - -static inline struct thread_info *current_thread_info(void) -{ - return (struct thread_info *) - (current_stack_pointer & ~(THREAD_SIZE - 1)); -} -#endif - #define thread_saved_pc(tsk) \ ((unsigned long)(task_thread_info(tsk)->cpu_context.pc)) #define thread_saved_sp(tsk) \ diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c index 645845e4982a..2c8d76fd7c66 100644 --- a/arch/arm/kernel/asm-offsets.c +++ b/arch/arm/kernel/asm-offsets.c @@ -43,9 +43,6 @@ int main(void) BLANK(); DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count)); -#ifndef CONFIG_THREAD_INFO_IN_TASK - DEFINE(TI_TASK, offsetof(struct thread_info, task)); -#endif DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); DEFINE(TI_CPU_DOMAIN, offsetof(struct thread_info, cpu_domain)); DEFINE(TI_CPU_SAVE, offsetof(struct thread_info, cpu_context)); diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 43d917f0d9a9..b58bda51e4b8 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -807,12 +807,13 @@ ENTRY(__switch_to) switch_tls r1, r4, r5, r3, r7 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) && \ !defined(CONFIG_STACKPROTECTOR_PER_TASK) - ldr r9, [r2, #TI_TASK] ldr r8, =__stack_chk_guard .if (TSK_STACK_CANARY > IMM12_MASK) - add r9, r9, #TSK_STACK_CANARY & ~IMM12_MASK - .endif + add r9, r2, #TSK_STACK_CANARY & ~IMM12_MASK ldr r9, [r9, #TSK_STACK_CANARY & IMM12_MASK] + .else + ldr r9, [r2, #TSK_STACK_CANARY & IMM12_MASK] + .endif #endif mov r7, r2 @ Preserve 'next' #ifdef CONFIG_CPU_USE_DOMAINS @@ -829,7 +830,7 @@ ENTRY(__switch_to) #endif mov r0, r5 #if !defined(CONFIG_THUMB2_KERNEL) && !defined(CONFIG_VMAP_STACK) - set_current r7 + set_current r7, r8 ldmia r4, {r4 - sl, fp, sp, pc} @ Load all regs saved previously #else mov r1, r7 @@ -851,7 +852,7 @@ ENTRY(__switch_to) @ switches us to another stack, with few other side effects. In order @ to prevent this distinction from causing any inconsistencies, let's @ keep the 'set_current' call as close as we can to the update of SP. - set_current r1 + set_current r1, r2 mov sp, ip ret lr #endif diff --git a/arch/arm/kernel/entry-v7m.S b/arch/arm/kernel/entry-v7m.S index 520dd43e7e08..4e0d318b67c6 100644 --- a/arch/arm/kernel/entry-v7m.S +++ b/arch/arm/kernel/entry-v7m.S @@ -97,15 +97,17 @@ ENTRY(__switch_to) str sp, [ip], #4 str lr, [ip], #4 mov r5, r0 + mov r6, r2 @ Preserve 'next' add r4, r2, #TI_CPU_SAVE ldr r0, =thread_notify_head mov r1, #THREAD_NOTIFY_SWITCH bl atomic_notifier_call_chain - mov ip, r4 mov r0, r5 - ldmia ip!, {r4 - r11} @ Load all regs saved previously - ldr sp, [ip] - ldr pc, [ip, #4]! + mov r1, r6 + ldmia r4, {r4 - r12, lr} @ Load all regs saved previously + set_current r1, r2 + mov sp, ip + bx lr .fnend ENDPROC(__switch_to) diff --git a/arch/arm/kernel/head-common.S b/arch/arm/kernel/head-common.S index da18e0a17dc2..42cae73fcc19 100644 --- a/arch/arm/kernel/head-common.S +++ b/arch/arm/kernel/head-common.S @@ -105,10 +105,8 @@ __mmap_switched: mov r1, #0 bl __memset @ clear .bss -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO adr_l r0, init_task @ get swapper task_struct - set_current r0 -#endif + set_current r0, r1 ldmia r4, {r0, r1, r2, r3} str r9, [r0] @ Save processor ID diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index d47159f3791c..0617af11377f 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -36,7 +36,7 @@ #include "signal.h" -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO +#if defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP) DEFINE_PER_CPU(struct task_struct *, __entry_task); #endif @@ -46,6 +46,11 @@ unsigned long __stack_chk_guard __read_mostly; EXPORT_SYMBOL(__stack_chk_guard); #endif +#ifndef CONFIG_CURRENT_POINTER_IN_TPIDRURO +asmlinkage struct task_struct *__current; +EXPORT_SYMBOL(__current); +#endif + static const char *processor_modes[] __maybe_unused = { "USER_26", "FIQ_26" , "IRQ_26" , "SVC_26" , "UK4_26" , "UK5_26" , "UK6_26" , "UK7_26" , "UK8_26" , "UK9_26" , "UK10_26", "UK11_26", "UK12_26", "UK13_26", "UK14_26", "UK15_26", diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 9c55ca915ba4..951559e5bea3 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -403,6 +403,17 @@ static void smp_store_cpu_info(unsigned int cpuid) check_cpu_icache_size(cpuid); } +static void set_current(struct task_struct *cur) +{ + if (!IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO) && !is_smp()) { + __current = cur; + return; + } + + /* Set TPIDRURO */ + asm("mcr p15, 0, %0, c13, c0, 3" :: "r"(cur) : "memory"); +} + /* * This is the secondary CPU boot entry. We're using this CPUs * idle thread stack, but a set of temporary page tables. diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index b28a705c49cb..3f38357efc46 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -865,7 +865,9 @@ early_initcall(allocate_overflow_stacks); asmlinkage void handle_bad_stack(struct pt_regs *regs) { unsigned long tsk_stk = (unsigned long)current->stack; +#ifdef CONFIG_IRQSTACKS unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr); +#endif unsigned long ovf_stk = (unsigned long)this_cpu_read(overflow_stack_ptr); console_verbose(); @@ -873,8 +875,10 @@ asmlinkage void handle_bad_stack(struct pt_regs *regs) pr_emerg("Task stack: [0x%08lx..0x%08lx]\n", tsk_stk, tsk_stk + THREAD_SIZE); +#ifdef CONFIG_IRQSTACKS pr_emerg("IRQ stack: [0x%08lx..0x%08lx]\n", irq_stk - THREAD_SIZE, irq_stk); +#endif pr_emerg("Overflow stack: [0x%08lx..0x%08lx]\n", ovf_stk - OVERFLOW_STACK_SIZE, ovf_stk); From cafc0eab168917ec9c0cd47d530a40cd40eb2928 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 2 Dec 2021 09:36:59 +0100 Subject: [PATCH 33/62] ARM: v7m: enable support for IRQ stacks Enable support for IRQ stacks on !MMU, and add the code to the IRQ entry path to switch to the IRQ stack if not running from it already. Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/Kconfig | 1 - arch/arm/kernel/entry-v7m.S | 17 +++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 50ae5286f59b..359a3b85c8b3 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1158,7 +1158,6 @@ config CURRENT_POINTER_IN_TPIDRURO config IRQSTACKS def_bool y - depends on MMU select HAVE_IRQ_EXIT_ON_IRQ_STACK select HAVE_SOFTIRQ_ON_OWN_STACK diff --git a/arch/arm/kernel/entry-v7m.S b/arch/arm/kernel/entry-v7m.S index 4e0d318b67c6..de8a60363c85 100644 --- a/arch/arm/kernel/entry-v7m.S +++ b/arch/arm/kernel/entry-v7m.S @@ -40,11 +40,24 @@ __irq_entry: @ Invoke the IRQ handler @ mov r0, sp - stmdb sp!, {lr} + ldr_this_cpu sp, irq_stack_ptr, r1, r2 + + @ + @ If we took the interrupt while running in the kernel, we may already + @ be using the IRQ stack, so revert to the original value in that case. + @ + subs r2, sp, r0 @ SP above bottom of IRQ stack? + rsbscs r2, r2, #THREAD_SIZE @ ... and below the top? + movcs sp, r0 + + push {r0, lr} @ preserve LR and original SP + @ routine called with r0 = struct pt_regs * bl generic_handle_arch_irq - pop {lr} + pop {r0, lr} + mov sp, r0 + @ @ Check for any pending work if returning to user @ From 23d9a9280efea105852de358f21d69231992ae73 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 5 Jan 2022 15:45:26 +0100 Subject: [PATCH 34/62] ARM: 9177/1: disable vmap'ed stacks on suspend-capable SMP configs There are several reports about the new vmap'ed stacks code breaking suspend/resume on Exynos, Renesas and Tegra SMP platforms. While this is under investigation, let's disable the vmap'ed stacks feature for the time being for SMP configurations that have suspend/resume enabled. [0] https://lore.kernel.org/linux-arm-kernel/20211122092816.2865873-8-ardb@kernel.org/ Cc: Marek Szyprowski Cc: Geert Uytterhoeven Cc: Jon Hunter Signed-off-by: Ard Biesheuvel Signed-off-by: Russell King (Oracle) --- arch/arm/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 359a3b85c8b3..c32b79453ddf 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -128,7 +128,7 @@ config ARM select RTC_LIB select SYS_SUPPORTS_APM_EMULATION select THREAD_INFO_IN_TASK - select HAVE_ARCH_VMAP_STACK if MMU && (!LD_IS_LLD || LLD_VERSION >= 140000) + select HAVE_ARCH_VMAP_STACK if MMU && (!LD_IS_LLD || LLD_VERSION >= 140000) && !PM_SLEEP_SMP select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M # Above selects are sorted alphabetically; please add new ones # according to that. Thanks. From 5fe41793bc78d9bb47fea37d1a16984ad6cf294b Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 24 Dec 2021 10:31:00 +0100 Subject: [PATCH 35/62] ARM: 9176/1: avoid literal references in inline assembly Nathan reports that the new get_current() and per-CPU offset accessors may cause problems at build time due to the use of a literal to hold the address of the respective variables. This is due to the fact that LLD before v14 does not support the PC-relative group relocations that are normally used for this, and the fallback relies on literals but does not emit the literal pools explictly using the .ltorg directive. ./arch/arm/include/asm/current.h:53:6: error: out of range pc-relative fixup value asm(LOAD_SYM_ARMV6(%0, __current) : "=r"(cur)); ^ ./arch/arm/include/asm/insn.h:25:2: note: expanded from macro 'LOAD_SYM_ARMV6' " ldr " #reg ", =" #sym " nt" ^ :1:3: note: instantiated into assembly here ldr r0, =__current ^ Since emitting a literal pool in this particular case is not possible, let's avoid the LOAD_SYM_ARMV6() entirely, and use the ordinary C assigment instead. As it turns out, there are other such cases, and here, using .ltorg to emit the literal pool within range of the LDR instruction would be possible due to the presence of an unconditional branch right after it. Unfortunately, putting .ltorg directives in subsections appears to confuse the Clang inline assembler, resulting in similar errors even though the .ltorg is most definitely within range. So let's fix this by emitting the literal explicitly, and not rely on the assembler to figure this out. This means we have move the fallback out of the LOAD_SYM_ARMV6() macro and into the callers. Link: https://github.com/ClangBuiltLinux/linux/issues/1551 Fixes: 9c46929e7989 ("ARM: implement THREAD_INFO_IN_TASK for uniprocessor systems") Reported-by: Nathan Chancellor Tested-by: Nathan Chancellor Signed-off-by: Ard Biesheuvel Signed-off-by: Russell King (Oracle) --- arch/arm/include/asm/current.h | 13 +++++++++++-- arch/arm/include/asm/insn.h | 7 ------- arch/arm/include/asm/percpu.h | 8 ++++++++ 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/arch/arm/include/asm/current.h b/arch/arm/include/asm/current.h index 69ecf4c6c725..2f9d79214b25 100644 --- a/arch/arm/include/asm/current.h +++ b/arch/arm/include/asm/current.h @@ -37,8 +37,16 @@ static inline __attribute_const__ struct task_struct *get_current(void) #ifdef CONFIG_CPU_V6 "1: \n\t" " .subsection 1 \n\t" +#if !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) && \ + !(defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) "2: " LOAD_SYM_ARMV6(%0, __current) " \n\t" " b 1b \n\t" +#else + "2: ldr %0, 3f \n\t" + " ldr %0, [%0] \n\t" + " b 1b \n\t" + "3: .long __current \n\t" +#endif " .previous \n\t" " .pushsection \".alt.smp.init\", \"a\" \n\t" " .long 0b - . \n\t" @@ -46,8 +54,9 @@ static inline __attribute_const__ struct task_struct *get_current(void) " .popsection \n\t" #endif : "=r"(cur)); -#elif __LINUX_ARM_ARCH__>=7 || \ - (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) +#elif __LINUX_ARM_ARCH__>= 7 || \ + (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) || \ + (defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) cur = __current; #else asm(LOAD_SYM_ARMV6(%0, __current) : "=r"(cur)); diff --git a/arch/arm/include/asm/insn.h b/arch/arm/include/asm/insn.h index a160ed3ea427..faf3d1c28368 100644 --- a/arch/arm/include/asm/insn.h +++ b/arch/arm/include/asm/insn.h @@ -10,8 +10,6 @@ * which should be sufficient for the core kernel as well as modules loaded * into the module region. (Not supported by LLD before release 14) */ -#if !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) && \ - !(defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) #define LOAD_SYM_ARMV6(reg, sym) \ " .globl " #sym " \n\t" \ " .reloc 10f, R_ARM_ALU_PC_G0_NC, " #sym " \n\t" \ @@ -20,11 +18,6 @@ "10: sub " #reg ", pc, #8 \n\t" \ "11: sub " #reg ", " #reg ", #4 \n\t" \ "12: ldr " #reg ", [" #reg ", #0] \n\t" -#else -#define LOAD_SYM_ARMV6(reg, sym) \ - " ldr " #reg ", =" #sym " \n\t" \ - " ldr " #reg ", [" #reg "] \n\t" -#endif static inline unsigned long arm_gen_nop(void) diff --git a/arch/arm/include/asm/percpu.h b/arch/arm/include/asm/percpu.h index a4a0d38d016a..28961d60877d 100644 --- a/arch/arm/include/asm/percpu.h +++ b/arch/arm/include/asm/percpu.h @@ -38,8 +38,16 @@ static inline unsigned long __my_cpu_offset(void) #ifdef CONFIG_CPU_V6 "1: \n\t" " .subsection 1 \n\t" +#if !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) && \ + !(defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) "2: " LOAD_SYM_ARMV6(%0, __per_cpu_offset) " \n\t" " b 1b \n\t" +#else + "2: ldr %0, 3f \n\t" + " ldr %0, [%0] \n\t" + " b 1b \n\t" + "3: .long __per_cpu_offset \n\t" +#endif " .previous \n\t" " .pushsection \".alt.smp.init\", \"a\" \n\t" " .long 0b - . \n\t" From 8b806b82bc804ce8d254ea87d48eaa390451eac4 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 24 Jan 2022 19:51:58 +0100 Subject: [PATCH 36/62] ARM: mm: switch to swapper_pg_dir early for vmap'ed stack When onlining a CPU, switch to swapper_pg_dir as soon as possible so that it is guaranteed that the vmap'ed stack is mapped before it is used. Signed-off-by: Ard Biesheuvel --- arch/arm/Kconfig | 2 +- arch/arm/kernel/head.S | 7 +++++++ arch/arm/kernel/sleep.S | 7 +++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index c32b79453ddf..359a3b85c8b3 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -128,7 +128,7 @@ config ARM select RTC_LIB select SYS_SUPPORTS_APM_EMULATION select THREAD_INFO_IN_TASK - select HAVE_ARCH_VMAP_STACK if MMU && (!LD_IS_LLD || LLD_VERSION >= 140000) && !PM_SLEEP_SMP + select HAVE_ARCH_VMAP_STACK if MMU && (!LD_IS_LLD || LLD_VERSION >= 140000) select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M # Above selects are sorted alphabetically; please add new ones # according to that. Thanks. diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S index c04dd94630c7..500612d3da2e 100644 --- a/arch/arm/kernel/head.S +++ b/arch/arm/kernel/head.S @@ -424,6 +424,13 @@ ENDPROC(secondary_startup) ENDPROC(secondary_startup_arm) ENTRY(__secondary_switched) +#if defined(CONFIG_VMAP_STACK) && !defined(CONFIG_ARM_LPAE) + @ Before using the vmap'ed stack, we have to switch to swapper_pg_dir + @ as the ID map does not cover the vmalloc region. + mrc p15, 0, ip, c2, c0, 1 @ read TTBR1 + mcr p15, 0, ip, c2, c0, 0 @ set TTBR0 + instr_sync +#endif adr_l r7, secondary_data + 12 @ get secondary_data.stack ldr sp, [r7] ldr r0, [r7, #4] @ get secondary_data.task diff --git a/arch/arm/kernel/sleep.S b/arch/arm/kernel/sleep.S index f909baf17912..a86a1d4f3461 100644 --- a/arch/arm/kernel/sleep.S +++ b/arch/arm/kernel/sleep.S @@ -119,6 +119,13 @@ ENTRY(cpu_resume_mmu) ENDPROC(cpu_resume_mmu) .popsection cpu_resume_after_mmu: +#if defined(CONFIG_VMAP_STACK) && !defined(CONFIG_ARM_LPAE) + @ Before using the vmap'ed stack, we have to switch to swapper_pg_dir + @ as the ID map does not cover the vmalloc region. + mrc p15, 0, ip, c2, c0, 1 @ read TTBR1 + mcr p15, 0, ip, c2, c0, 0 @ set TTBR0 + instr_sync +#endif bl cpu_init @ restore the und/abt/irq banked regs mov r0, #0 @ return zero on success ldmfd sp!, {r4 - r11, pc} From d6905849f87596f94c2778c8337697df486de43c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 24 Jan 2022 19:16:58 +0100 Subject: [PATCH 37/62] ARM: assembler: define a Kconfig symbol for group relocation support Nathan reports the group relocations go out of range in pathological cases such as allyesconfig kernels, which have little chance of actually booting but are still used in validation. So add a Kconfig symbol for this feature, and make it depend on !COMPILE_TEST. Signed-off-by: Ard Biesheuvel --- arch/arm/Kconfig | 13 ++++++++++++- arch/arm/include/asm/assembler.h | 8 ++++---- arch/arm/include/asm/current.h | 8 ++++---- arch/arm/include/asm/percpu.h | 4 ++-- arch/arm/kernel/module.c | 7 ++++++- 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 359a3b85c8b3..70ab8d807032 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -128,7 +128,7 @@ config ARM select RTC_LIB select SYS_SUPPORTS_APM_EMULATION select THREAD_INFO_IN_TASK - select HAVE_ARCH_VMAP_STACK if MMU && (!LD_IS_LLD || LLD_VERSION >= 140000) + select HAVE_ARCH_VMAP_STACK if MMU && ARM_HAS_GROUP_RELOCS select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M # Above selects are sorted alphabetically; please add new ones # according to that. Thanks. @@ -140,6 +140,17 @@ config ARM Europe. There is an ARM Linux project with a web page at . +config ARM_HAS_GROUP_RELOCS + def_bool y + depends on !LD_IS_LLD || LLD_VERSION >= 140000 + depends on !COMPILE_TEST + help + Whether or not to use R_ARM_ALU_PC_Gn or R_ARM_LDR_PC_Gn group + relocations, which have been around for a long time, but were not + supported in LLD until version 14. The combined range is -/+ 256 MiB, + which is usually sufficient, but not for allyesconfig, so we disable + this feature when doing compile testing. + config ARM_HAS_SG_CHAIN bool diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 59d7b9e81934..9998718a49ca 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -656,8 +656,8 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) .macro __ldst_va, op, reg, tmp, sym, cond #if __LINUX_ARM_ARCH__ >= 7 || \ - (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) || \ - (defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) + !defined(CONFIG_ARM_HAS_GROUP_RELOCS) || \ + (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) mov_l \tmp, \sym, \cond \op\cond \reg, [\tmp] #else @@ -716,8 +716,8 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) */ .macro ldr_this_cpu, rd:req, sym:req, t1:req, t2:req #if __LINUX_ARM_ARCH__ >= 7 || \ - (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) || \ - (defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) + !defined(CONFIG_ARM_HAS_GROUP_RELOCS) || \ + (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) this_cpu_offset \t1 mov_l \t2, \sym ldr \rd, [\t1, \t2] diff --git a/arch/arm/include/asm/current.h b/arch/arm/include/asm/current.h index 2f9d79214b25..131a89bbec6b 100644 --- a/arch/arm/include/asm/current.h +++ b/arch/arm/include/asm/current.h @@ -37,8 +37,8 @@ static inline __attribute_const__ struct task_struct *get_current(void) #ifdef CONFIG_CPU_V6 "1: \n\t" " .subsection 1 \n\t" -#if !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) && \ - !(defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) +#if defined(CONFIG_ARM_HAS_GROUP_RELOCS) && \ + !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) "2: " LOAD_SYM_ARMV6(%0, __current) " \n\t" " b 1b \n\t" #else @@ -55,8 +55,8 @@ static inline __attribute_const__ struct task_struct *get_current(void) #endif : "=r"(cur)); #elif __LINUX_ARM_ARCH__>= 7 || \ - (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) || \ - (defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) + !defined(CONFIG_ARM_HAS_GROUP_RELOCS) || \ + (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) cur = __current; #else asm(LOAD_SYM_ARMV6(%0, __current) : "=r"(cur)); diff --git a/arch/arm/include/asm/percpu.h b/arch/arm/include/asm/percpu.h index 28961d60877d..a09034ae45a1 100644 --- a/arch/arm/include/asm/percpu.h +++ b/arch/arm/include/asm/percpu.h @@ -38,8 +38,8 @@ static inline unsigned long __my_cpu_offset(void) #ifdef CONFIG_CPU_V6 "1: \n\t" " .subsection 1 \n\t" -#if !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) && \ - !(defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) +#if defined(CONFIG_ARM_HAS_GROUP_RELOCS) && \ + !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) "2: " LOAD_SYM_ARMV6(%0, __per_cpu_offset) " \n\t" " b 1b \n\t" #else diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c index 4d33a7acf617..549abcedf795 100644 --- a/arch/arm/kernel/module.c +++ b/arch/arm/kernel/module.c @@ -68,6 +68,7 @@ bool module_exit_section(const char *name) strstarts(name, ".ARM.exidx.exit"); } +#ifdef CONFIG_ARM_HAS_GROUP_RELOCS /* * This implements the partitioning algorithm for group relocations as * documented in the ARM AArch32 ELF psABI (IHI 0044). @@ -103,6 +104,7 @@ static u32 get_group_rem(u32 group, u32 *offset) } while (group--); return shift; } +#endif int apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, @@ -118,7 +120,9 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, unsigned long loc; Elf32_Sym *sym; const char *symname; +#ifdef CONFIG_ARM_HAS_GROUP_RELOCS u32 shift, group = 1; +#endif s32 offset; u32 tmp; #ifdef CONFIG_THUMB2_KERNEL @@ -249,6 +253,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, *(u32 *)loc = __opcode_to_mem_arm(tmp); break; +#ifdef CONFIG_ARM_HAS_GROUP_RELOCS case R_ARM_ALU_PC_G0_NC: group = 0; fallthrough; @@ -296,7 +301,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, } *(u32 *)loc = __opcode_to_mem_arm((tmp & ~0xfff) | offset); break; - +#endif #ifdef CONFIG_THUMB2_KERNEL case R_ARM_THM_CALL: case R_ARM_THM_JUMP24: From 75fa4adc4f50ee52d8cdfa3e84798176ccb4a354 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 24 Jan 2022 19:28:58 +0100 Subject: [PATCH 38/62] ARM: smp: elide HWCAP_TLS checks or __entry_task updates on SMP+v6 Use the SMP_ON_UP patching framework to elide HWCAP_TLS tests from the context switch and return to userspace code paths, as SMP systems are guaranteed to have this h/w capability. At the same time, omit the update of __entry_task if the system is detected to be UP at runtime, as in that case, the value is never used. Signed-off-by: Ard Biesheuvel --- arch/arm/include/asm/switch_to.h | 4 ++-- arch/arm/include/asm/tls.h | 22 ++++++++++++++++------ arch/arm/kernel/entry-header.S | 17 +++++++---------- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/arch/arm/include/asm/switch_to.h b/arch/arm/include/asm/switch_to.h index a482c99934ff..f67ae946a3c6 100644 --- a/arch/arm/include/asm/switch_to.h +++ b/arch/arm/include/asm/switch_to.h @@ -3,6 +3,7 @@ #define __ASM_ARM_SWITCH_TO_H #include +#include /* * For v7 SMP cores running a preemptible kernel we may be pre-empted @@ -40,8 +41,7 @@ static inline void set_ti_cpu(struct task_struct *p) do { \ __complete_pending_tlbi(); \ set_ti_cpu(next); \ - if (IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || \ - IS_ENABLED(CONFIG_SMP)) \ + if (IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || is_smp()) \ __this_cpu_write(__entry_task, next); \ last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \ } while (0) diff --git a/arch/arm/include/asm/tls.h b/arch/arm/include/asm/tls.h index d712c170c095..3dcd0f71a0da 100644 --- a/arch/arm/include/asm/tls.h +++ b/arch/arm/include/asm/tls.h @@ -18,22 +18,32 @@ .endm .macro switch_tls_v6, base, tp, tpuser, tmp1, tmp2 +#ifdef CONFIG_SMP +ALT_SMP(nop) +ALT_UP_B(.L0_\@) + .subsection 1 +#endif +.L0_\@: ldr_va \tmp1, elf_hwcap mov \tmp2, #0xffff0fff tst \tmp1, #HWCAP_TLS @ hardware TLS available? streq \tp, [\tmp2, #-15] @ set TLS value at 0xffff0ff0 - mrcne p15, 0, \tmp2, c13, c0, 2 @ get the user r/w register -#ifndef CONFIG_SMP - mcrne p15, 0, \tp, c13, c0, 3 @ yes, set TLS register + beq .L2_\@ + mcr p15, 0, \tp, c13, c0, 3 @ yes, set TLS register +#ifdef CONFIG_SMP + b .L1_\@ + .previous #endif - mcrne p15, 0, \tpuser, c13, c0, 2 @ set user r/w register - strne \tmp2, [\base, #TI_TP_VALUE + 4] @ save it +.L1_\@: switch_tls_v6k \base, \tp, \tpuser, \tmp1, \tmp2 +.L2_\@: .endm .macro switch_tls_software, base, tp, tpuser, tmp1, tmp2 mov \tmp1, #0xffff0fff str \tp, [\tmp1, #-15] @ set TLS value at 0xffff0ff0 .endm +#else +#include #endif #ifdef CONFIG_TLS_REG_EMUL @@ -44,7 +54,7 @@ #elif defined(CONFIG_CPU_V6) #define tls_emu 0 #define has_tls_reg (elf_hwcap & HWCAP_TLS) -#define defer_tls_reg_update IS_ENABLED(CONFIG_SMP) +#define defer_tls_reg_update is_smp() #define switch_tls switch_tls_v6 #elif defined(CONFIG_CPU_32v6K) #define tls_emu 0 diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S index cb82ff5adec1..9a1dc142f782 100644 --- a/arch/arm/kernel/entry-header.S +++ b/arch/arm/kernel/entry-header.S @@ -292,21 +292,18 @@ .macro restore_user_regs, fast = 0, offset = 0 -#if defined(CONFIG_CPU_32v6K) || defined(CONFIG_SMP) -#if defined(CONFIG_CPU_V6) && defined(CONFIG_SMP) -ALT_SMP(b .L1_\@ ) -ALT_UP( nop ) - ldr_va r1, elf_hwcap - tst r1, #HWCAP_TLS @ hardware TLS available? - beq .L2_\@ -.L1_\@: +#if defined(CONFIG_CPU_32v6K) && \ + (!defined(CONFIG_CPU_V6) || defined(CONFIG_SMP)) +#ifdef CONFIG_CPU_V6 +ALT_SMP(nop) +ALT_UP_B(.L1_\@) #endif @ The TLS register update is deferred until return to user space so we @ can use it for other things while running in the kernel - get_thread_info r1 + mrc p15, 0, r1, c13, c0, 3 @ get current_thread_info pointer ldr r1, [r1, #TI_TP_VALUE] mcr p15, 0, r1, c13, c0, 3 @ set TLS register -.L2_\@: +.L1_\@: #endif uaccess_enable r1, isb=0 From aa0a20f521516ba83ea29b510fcc12fb35920b48 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 24 Jan 2022 19:36:03 +0100 Subject: [PATCH 39/62] ARM: entry: avoid clobbering R9 in IRQ handler Avoid using R9 in the IRQ handler code, as the entry code uses it for tsk, and expects it to remain untouched between the IRQ entry and exit code. Signed-off-by: Ard Biesheuvel --- arch/arm/kernel/entry-armv.S | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index b58bda51e4b8..038aabb6578f 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -38,11 +38,10 @@ #ifdef CONFIG_UNWINDER_ARM mov fpreg, sp @ Preserve original SP #else - mov r8, fp @ Preserve original FP - mov r9, sp @ Preserve original SP + mov r7, fp @ Preserve original FP + mov r8, sp @ Preserve original SP #endif ldr_this_cpu sp, irq_stack_ptr, r2, r3 - .if \from_user == 0 UNWIND( .setfp fpreg, sp ) @ @@ -82,8 +81,8 @@ UNWIND( .setfp fpreg, sp ) #ifdef CONFIG_UNWINDER_ARM mov sp, fpreg @ Restore original SP #else - mov fp, r8 @ Restore original FP - mov sp, r9 @ Restore original SP + mov fp, r7 @ Restore original FP + mov sp, r8 @ Restore original SP #endif // CONFIG_UNWINDER_ARM #endif // CONFIG_IRQSTACKS .endm From d31e23aff011d96278f4dbc22f2ec5db433eabaf Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 10 Jan 2022 09:54:22 +0100 Subject: [PATCH 40/62] ARM: mm: make vmalloc_seq handling SMP safe Rework the vmalloc_seq handling so it can be used safely under SMP, as we started using it to ensure that vmap'ed stacks are guaranteed to be mapped by the active mm before switching to a task, and here we need to ensure that changes to the page tables are visible to other CPUs when they observe a change in the sequence count. Since LPAE needs none of this, fold a check against it into the vmalloc_seq counter check after breaking it out into a separate static inline helper. Given that vmap'ed stacks are now also supported on !SMP configurations, let's drop the WARN() that could potentially now fire spuriously. Signed-off-by: Ard Biesheuvel --- arch/arm/include/asm/mmu.h | 2 +- arch/arm/include/asm/mmu_context.h | 22 ++++++++++++++++++++-- arch/arm/include/asm/page.h | 3 +-- arch/arm/kernel/traps.c | 25 +++++++------------------ arch/arm/mm/context.c | 3 +-- arch/arm/mm/ioremap.c | 18 +++++++++++------- 6 files changed, 41 insertions(+), 32 deletions(-) diff --git a/arch/arm/include/asm/mmu.h b/arch/arm/include/asm/mmu.h index 1592a4264488..e049723840d3 100644 --- a/arch/arm/include/asm/mmu.h +++ b/arch/arm/include/asm/mmu.h @@ -10,7 +10,7 @@ typedef struct { #else int switch_pending; #endif - unsigned int vmalloc_seq; + atomic_t vmalloc_seq; unsigned long sigpage; #ifdef CONFIG_VDSO unsigned long vdso; diff --git a/arch/arm/include/asm/mmu_context.h b/arch/arm/include/asm/mmu_context.h index 84e58956fcab..db2cb06aa8cf 100644 --- a/arch/arm/include/asm/mmu_context.h +++ b/arch/arm/include/asm/mmu_context.h @@ -23,6 +23,16 @@ void __check_vmalloc_seq(struct mm_struct *mm); +#ifdef CONFIG_MMU +static inline void check_vmalloc_seq(struct mm_struct *mm) +{ + if (!IS_ENABLED(CONFIG_ARM_LPAE) && + unlikely(atomic_read(&mm->context.vmalloc_seq) != + atomic_read(&init_mm.context.vmalloc_seq))) + __check_vmalloc_seq(mm); +} +#endif + #ifdef CONFIG_CPU_HAS_ASID void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk); @@ -52,8 +62,7 @@ static inline void a15_erratum_get_cpumask(int this_cpu, struct mm_struct *mm, static inline void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk) { - if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq)) - __check_vmalloc_seq(mm); + check_vmalloc_seq(mm); if (irqs_disabled()) /* @@ -129,6 +138,15 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next, #endif } +#ifdef CONFIG_VMAP_STACK +static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) +{ + if (mm != &init_mm) + check_vmalloc_seq(mm); +} +#define enter_lazy_tlb enter_lazy_tlb +#endif + #include #endif diff --git a/arch/arm/include/asm/page.h b/arch/arm/include/asm/page.h index 7b871ed99ccf..5fcc8a600e36 100644 --- a/arch/arm/include/asm/page.h +++ b/arch/arm/include/asm/page.h @@ -147,11 +147,10 @@ extern void copy_page(void *to, const void *from); #include #else #include -#endif - #ifdef CONFIG_VMAP_STACK #define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_PMD_MODIFIED #endif +#endif #endif /* CONFIG_MMU */ diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 3f38357efc46..08612032aefe 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -885,6 +885,7 @@ asmlinkage void handle_bad_stack(struct pt_regs *regs) die("kernel stack overflow", regs, 0); } +#ifndef CONFIG_ARM_LPAE /* * Normally, we rely on the logic in do_translation_fault() to update stale PMD * entries covering the vmalloc space in a task's page tables when it first @@ -895,26 +896,14 @@ asmlinkage void handle_bad_stack(struct pt_regs *regs) * So we need to ensure that these PMD entries are up to date *before* the MM * switch. As we already have some logic in the MM switch path that takes care * of this, let's trigger it by bumping the counter every time the core vmalloc - * code modifies a PMD entry in the vmalloc region. + * code modifies a PMD entry in the vmalloc region. Use release semantics on + * the store so that other CPUs observing the counter's new value are + * guaranteed to see the updated page table entries as well. */ void arch_sync_kernel_mappings(unsigned long start, unsigned long end) { - if (start > VMALLOC_END || end < VMALLOC_START) - return; - - /* - * This hooks into the core vmalloc code to receive notifications of - * any PMD level changes that have been made to the kernel page tables. - * This means it should only be triggered once for every MiB worth of - * vmalloc space, given that we don't support huge vmalloc/vmap on ARM, - * and that kernel PMD level table entries are rarely (if ever) - * updated. - * - * This means that the counter is going to max out at ~250 for the - * typical case. If it overflows, something entirely unexpected has - * occurred so let's throw a warning if that happens. - */ - WARN_ON(++init_mm.context.vmalloc_seq == UINT_MAX); + if (start < VMALLOC_END && end > VMALLOC_START) + atomic_inc_return_release(&init_mm.context.vmalloc_seq); } - +#endif #endif diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c index 48091870db89..4204ffa2d104 100644 --- a/arch/arm/mm/context.c +++ b/arch/arm/mm/context.c @@ -240,8 +240,7 @@ void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk) unsigned int cpu = smp_processor_id(); u64 asid; - if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq)) - __check_vmalloc_seq(mm); + check_vmalloc_seq(mm); /* * We cannot update the pgd and the ASID atomicly with classic diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c index 6e830b9418c9..8963c8c63471 100644 --- a/arch/arm/mm/ioremap.c +++ b/arch/arm/mm/ioremap.c @@ -117,16 +117,21 @@ EXPORT_SYMBOL(ioremap_page); void __check_vmalloc_seq(struct mm_struct *mm) { - unsigned int seq; + int seq; do { - seq = init_mm.context.vmalloc_seq; + seq = atomic_read(&init_mm.context.vmalloc_seq); memcpy(pgd_offset(mm, VMALLOC_START), pgd_offset_k(VMALLOC_START), sizeof(pgd_t) * (pgd_index(VMALLOC_END) - pgd_index(VMALLOC_START))); - mm->context.vmalloc_seq = seq; - } while (seq != init_mm.context.vmalloc_seq); + /* + * Use a store-release so that other CPUs that observe the + * counter's new value are guaranteed to see the results of the + * memcpy as well. + */ + atomic_set_release(&mm->context.vmalloc_seq, seq); + } while (seq != atomic_read(&init_mm.context.vmalloc_seq)); } #if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE) @@ -157,7 +162,7 @@ static void unmap_area_sections(unsigned long virt, unsigned long size) * Note: this is still racy on SMP machines. */ pmd_clear(pmdp); - init_mm.context.vmalloc_seq++; + atomic_inc_return_release(&init_mm.context.vmalloc_seq); /* * Free the page table, if there was one. @@ -174,8 +179,7 @@ static void unmap_area_sections(unsigned long virt, unsigned long size) * Ensure that the active_mm is up to date - we want to * catch any use-after-iounmap cases. */ - if (current->active_mm->context.vmalloc_seq != init_mm.context.vmalloc_seq) - __check_vmalloc_seq(current->active_mm); + check_vmalloc_seq(current->active_mm); flush_tlb_kernel_range(virt, end); } From a14a96d7560687d328e3702682c94b549e1c3911 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 24 Jan 2022 20:55:10 +0100 Subject: [PATCH 41/62] ARM: iop: make iop_handle_irq() static The build bots complain about iop_handle_irq() not being declared so let's make it static instead. Signed-off-by: Ard Biesheuvel --- arch/arm/mach-iop32x/irq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/mach-iop32x/irq.c b/arch/arm/mach-iop32x/irq.c index b820839eaae8..6dca7e97d81f 100644 --- a/arch/arm/mach-iop32x/irq.c +++ b/arch/arm/mach-iop32x/irq.c @@ -59,7 +59,7 @@ struct irq_chip ext_chip = { .irq_unmask = iop32x_irq_unmask, }; -void iop_handle_irq(struct pt_regs *regs) +static void iop_handle_irq(struct pt_regs *regs) { u32 mask; From 57a420435edcb0b947a74171bf49ada7a5892d4f Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 24 Jan 2022 23:29:59 +0100 Subject: [PATCH 42/62] ARM: drop pointless SMP check on secondary startup path Only SMP systems use the secondary startup path by definition, so there is no need for SMP conditionals there. Signed-off-by: Ard Biesheuvel --- arch/arm/kernel/smp.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 951559e5bea3..e34efa96cea1 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -405,11 +405,6 @@ static void smp_store_cpu_info(unsigned int cpuid) static void set_current(struct task_struct *cur) { - if (!IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO) && !is_smp()) { - __current = cur; - return; - } - /* Set TPIDRURO */ asm("mcr p15, 0, %0, c13, c0, 3" :: "r"(cur) : "memory"); } From 4d5a643e738c6b6ccc1a05f6938643c3f08df29b Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 24 Jan 2022 23:32:51 +0100 Subject: [PATCH 43/62] ARM: make get_current() and __my_cpu_offset() __always_inline The get_current() and __my_cpu_offset() accessors evaluate to only a single instruction emitted inline, but due to the size of the asm string that is created for SMP+v6 configurations, the compiler assumes otherwise, and may emit the functions out of line instead. So use __always_inline to avoid this. Signed-off-by: Ard Biesheuvel Reviewed-by: Nick Desaulniers --- arch/arm/include/asm/current.h | 2 +- arch/arm/include/asm/percpu.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm/include/asm/current.h b/arch/arm/include/asm/current.h index 131a89bbec6b..1e1178bf176d 100644 --- a/arch/arm/include/asm/current.h +++ b/arch/arm/include/asm/current.h @@ -14,7 +14,7 @@ struct task_struct; extern struct task_struct *__current; -static inline __attribute_const__ struct task_struct *get_current(void) +static __always_inline __attribute_const__ struct task_struct *get_current(void) { struct task_struct *cur; diff --git a/arch/arm/include/asm/percpu.h b/arch/arm/include/asm/percpu.h index a09034ae45a1..7545c87c251f 100644 --- a/arch/arm/include/asm/percpu.h +++ b/arch/arm/include/asm/percpu.h @@ -25,7 +25,7 @@ static inline void set_my_cpu_offset(unsigned long off) asm volatile("mcr p15, 0, %0, c13, c0, 4" : : "r" (off) : "memory"); } -static inline unsigned long __my_cpu_offset(void) +static __always_inline unsigned long __my_cpu_offset(void) { unsigned long off; From dd88b03ff0c84f4bcbe1419b93a4bed429fed3be Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 24 Jan 2022 10:38:15 +0100 Subject: [PATCH 44/62] ARM: ftrace: ensure that ADR takes the Thumb bit into account Using ADR to take the address of 'ftrace_stub' via a local label produces an address that has the Thumb bit cleared, which means the subsequent comparison is guaranteed to fail. Instead, use the badr macro, which forces the Thumb bit to be set. Fixes: a3ba87a61499 ("ARM: 6316/1: ftrace: add Thumb-2 support") Signed-off-by: Ard Biesheuvel Reviewed-by: Nick Desaulniers Reviewed-by: Steven Rostedt (Google) Reviewed-by: Linus Walleij --- arch/arm/kernel/entry-ftrace.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/kernel/entry-ftrace.S b/arch/arm/kernel/entry-ftrace.S index a74289ebc803..f4886fb6e9ba 100644 --- a/arch/arm/kernel/entry-ftrace.S +++ b/arch/arm/kernel/entry-ftrace.S @@ -40,7 +40,7 @@ mcount_enter ldr r0, =ftrace_trace_function ldr r2, [r0] - adr r0, .Lftrace_stub + badr r0, .Lftrace_stub cmp r0, r2 bne 1f From ad1c2f39fda0acb51b5a93604c5e1a703b849a7d Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 8 Dec 2021 08:42:03 +0100 Subject: [PATCH 45/62] ARM: ftrace: use ADD not POP to counter PUSH at entry The compiler emitted hook used for ftrace consists of a PUSH {LR} to preserve the link register, followed by a branch-and-link (BL) to __gnu_mount_nc. Dynamic ftrace patches away the latter to turn the combined sequence into a NOP, using a POP {LR} instruction. This is not necessary, since the link register does not get clobbered in this case, and simply adding #4 to the stack pointer is sufficient, and avoids a memory access that may take a few cycles to resolve depending on the micro-architecture. Signed-off-by: Ard Biesheuvel Reviewed-by: Nick Desaulniers Reviewed-by: Linus Walleij Reviewed-by: Steven Rostedt (Google) --- arch/arm/kernel/entry-ftrace.S | 2 +- arch/arm/kernel/ftrace.c | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/arch/arm/kernel/entry-ftrace.S b/arch/arm/kernel/entry-ftrace.S index f4886fb6e9ba..dca12a09322a 100644 --- a/arch/arm/kernel/entry-ftrace.S +++ b/arch/arm/kernel/entry-ftrace.S @@ -27,7 +27,7 @@ * allows it to be clobbered in subroutines and doesn't use it to hold * parameters.) * - * When using dynamic ftrace, we patch out the mcount call by a "pop {lr}" + * When using dynamic ftrace, we patch out the mcount call by a "add sp, #4" * instead of the __gnu_mcount_nc call (see arch/arm/kernel/ftrace.c). */ diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c index a006585e1c09..811cadf7eefc 100644 --- a/arch/arm/kernel/ftrace.c +++ b/arch/arm/kernel/ftrace.c @@ -24,10 +24,21 @@ #include #include +/* + * The compiler emitted profiling hook consists of + * + * PUSH {LR} + * BL __gnu_mcount_nc + * + * To turn this combined sequence into a NOP, we need to restore the value of + * SP before the PUSH. Let's use an ADD rather than a POP into LR, as LR is not + * modified anyway, and reloading LR from memory is highly likely to be less + * efficient. + */ #ifdef CONFIG_THUMB2_KERNEL -#define NOP 0xf85deb04 /* pop.w {lr} */ +#define NOP 0xf10d0d04 /* add.w sp, sp, #4 */ #else -#define NOP 0xe8bd4000 /* pop {lr} */ +#define NOP 0xe28dd004 /* add sp, sp, #4 */ #endif #ifdef CONFIG_DYNAMIC_FTRACE From dc438db5828fd6a379648ab8735ee73a8e40865a Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 23 Jan 2022 20:28:39 +0100 Subject: [PATCH 46/62] ARM: ftrace: use trampolines to keep .init.text in branching range Kernel images that are large in comparison to the range of a direct branch may fail to work as expected with ftrace, as patching a direct branch to one of the core ftrace routines may not be possible from the .init.text section, if it is emitted too far away from the normal .text section. This is more likely to affect Thumb2 builds, given that its range is only -/+ 16 MiB (as opposed to ARM which has -/+ 32 MiB), but may occur in either ISA. To work around this, add a couple of trampolines to .init.text and swap these in when the ftrace patching code is operating on callers in .init.text. Signed-off-by: Ard Biesheuvel Reviewed-by: Nick Desaulniers Reviewed-by: Linus Walleij Reviewed-by: Steven Rostedt (Google) --- arch/arm/kernel/entry-ftrace.S | 16 ++++++++++++++++ arch/arm/kernel/ftrace.c | 23 ++++++++++++++++++++--- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/arch/arm/kernel/entry-ftrace.S b/arch/arm/kernel/entry-ftrace.S index dca12a09322a..237d435e29aa 100644 --- a/arch/arm/kernel/entry-ftrace.S +++ b/arch/arm/kernel/entry-ftrace.S @@ -270,3 +270,19 @@ ENTRY(ftrace_stub) .Lftrace_stub: ret lr ENDPROC(ftrace_stub) + +#ifdef CONFIG_DYNAMIC_FTRACE + + __INIT + + .macro init_tramp, dst:req +ENTRY(\dst\()_from_init) + ldr pc, =\dst +ENDPROC(\dst\()_from_init) + .endm + + init_tramp ftrace_caller +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS + init_tramp ftrace_regs_caller +#endif +#endif diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c index 811cadf7eefc..74d3913f5590 100644 --- a/arch/arm/kernel/ftrace.c +++ b/arch/arm/kernel/ftrace.c @@ -62,9 +62,20 @@ static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) return NOP; } -static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) +void ftrace_caller_from_init(void); +void ftrace_regs_caller_from_init(void); + +static unsigned long __ref adjust_address(struct dyn_ftrace *rec, + unsigned long addr) { - return addr; + if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE) || + system_state >= SYSTEM_FREEING_INITMEM || + likely(!is_kernel_inittext(rec->ip))) + return addr; + if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) || + addr == (unsigned long)&ftrace_caller) + return (unsigned long)&ftrace_caller_from_init; + return (unsigned long)&ftrace_regs_caller_from_init; } int ftrace_arch_code_modify_prepare(void) @@ -200,7 +211,13 @@ int ftrace_make_nop(struct module *mod, #endif new = ftrace_nop_replace(rec); - ret = ftrace_modify_code(ip, old, new, true); + /* + * Locations in .init.text may call __gnu_mcount_mc via a linker + * emitted veneer if they are too far away from its implementation, and + * so validation may fail spuriously in such cases. Let's work around + * this by omitting those from validation. + */ + ret = ftrace_modify_code(ip, old, new, !is_kernel_inittext(ip)); return ret; } From d11967870815b5ab89843980e35aab616c97c463 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 23 Jan 2022 20:18:33 +0100 Subject: [PATCH 47/62] ARM: ftrace: avoid redundant loads or clobbering IP Tweak the ftrace return paths to avoid redundant loads of SP, as well as unnecessary clobbering of IP. This also fixes the inconsistency of using MOV to perform a function return, which is sub-optimal on recent micro-architectures but more importantly, does not perform an interworking return, unlike compiler generated function returns in Thumb2 builds. Let's fix this by popping PC from the stack like most ordinary code does. Signed-off-by: Ard Biesheuvel Reviewed-by: Steven Rostedt (Google) --- arch/arm/kernel/entry-ftrace.S | 53 +++++++++++++++------------------- 1 file changed, 23 insertions(+), 30 deletions(-) diff --git a/arch/arm/kernel/entry-ftrace.S b/arch/arm/kernel/entry-ftrace.S index 237d435e29aa..67548c38a567 100644 --- a/arch/arm/kernel/entry-ftrace.S +++ b/arch/arm/kernel/entry-ftrace.S @@ -22,10 +22,7 @@ * mcount can be thought of as a function called in the middle of a subroutine * call. As such, it needs to be transparent for both the caller and the * callee: the original lr needs to be restored when leaving mcount, and no - * registers should be clobbered. (In the __gnu_mcount_nc implementation, we - * clobber the ip register. This is OK because the ARM calling convention - * allows it to be clobbered in subroutines and doesn't use it to hold - * parameters.) + * registers should be clobbered. * * When using dynamic ftrace, we patch out the mcount call by a "add sp, #4" * instead of the __gnu_mcount_nc call (see arch/arm/kernel/ftrace.c). @@ -70,26 +67,25 @@ .macro __ftrace_regs_caller - sub sp, sp, #8 @ space for PC and CPSR OLD_R0, + str lr, [sp, #-8]! @ store LR as PC and make space for CPSR/OLD_R0, @ OLD_R0 will overwrite previous LR - add ip, sp, #12 @ move in IP the value of SP as it was - @ before the push {lr} of the mcount mechanism - - str lr, [sp, #0] @ store LR instead of PC - - ldr lr, [sp, #8] @ get previous LR + ldr lr, [sp, #8] @ get previous LR str r0, [sp, #8] @ write r0 as OLD_R0 over previous LR - stmdb sp!, {ip, lr} - stmdb sp!, {r0-r11, lr} + str lr, [sp, #-4]! @ store previous LR as LR + + add lr, sp, #16 @ move in LR the value of SP as it was + @ before the push {lr} of the mcount mechanism + + push {r0-r11, ip, lr} @ stack content at this point: @ 0 4 48 52 56 60 64 68 72 - @ R0 | R1 | ... | LR | SP + 4 | previous LR | LR | PSR | OLD_R0 | + @ R0 | R1 | ... | IP | SP + 4 | previous LR | LR | PSR | OLD_R0 | - mov r3, sp @ struct pt_regs* + mov r3, sp @ struct pt_regs* ldr r2, =function_trace_op ldr r2, [r2] @ pointer to the current @@ -112,11 +108,9 @@ ftrace_graph_regs_call: #endif @ pop saved regs - ldmia sp!, {r0-r12} @ restore r0 through r12 - ldr ip, [sp, #8] @ restore PC - ldr lr, [sp, #4] @ restore LR - ldr sp, [sp, #0] @ restore SP - mov pc, ip @ return + pop {r0-r11, ip, lr} @ restore r0 through r12 + ldr lr, [sp], #4 @ restore LR + ldr pc, [sp], #12 .endm #ifdef CONFIG_FUNCTION_GRAPH_TRACER @@ -132,11 +126,9 @@ ftrace_graph_regs_call: bl prepare_ftrace_return @ pop registers saved in ftrace_regs_caller - ldmia sp!, {r0-r12} @ restore r0 through r12 - ldr ip, [sp, #8] @ restore PC - ldr lr, [sp, #4] @ restore LR - ldr sp, [sp, #0] @ restore SP - mov pc, ip @ return + pop {r0-r11, ip, lr} @ restore r0 through r12 + ldr lr, [sp], #4 @ restore LR + ldr pc, [sp], #12 .endm #endif @@ -202,16 +194,17 @@ ftrace_graph_call\suffix: .endm .macro mcount_exit - ldmia sp!, {r0-r3, ip, lr} - ret ip + ldmia sp!, {r0-r3} + ldr lr, [sp, #4] + ldr pc, [sp], #8 .endm ENTRY(__gnu_mcount_nc) UNWIND(.fnstart) #ifdef CONFIG_DYNAMIC_FTRACE - mov ip, lr - ldmia sp!, {lr} - ret ip + push {lr} + ldr lr, [sp, #4] + ldr pc, [sp], #8 #else __mcount #endif From 65aa7e342a988efee372e1e5fa8ed8b88fd4f949 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 7 Dec 2021 19:50:35 +0100 Subject: [PATCH 48/62] ARM: ftrace: avoid unnecessary literal loads Avoid explicit literal loads and instead, use accessor macros that generate the optimal sequence depending on the architecture revision being targeted. Signed-off-by: Ard Biesheuvel Reviewed-by: Nick Desaulniers Reviewed-by: Steven Rostedt (Google) --- arch/arm/kernel/entry-ftrace.S | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/arch/arm/kernel/entry-ftrace.S b/arch/arm/kernel/entry-ftrace.S index 67548c38a567..99720064a4c5 100644 --- a/arch/arm/kernel/entry-ftrace.S +++ b/arch/arm/kernel/entry-ftrace.S @@ -35,23 +35,20 @@ .macro __mcount suffix mcount_enter - ldr r0, =ftrace_trace_function - ldr r2, [r0] + ldr_va r2, ftrace_trace_function badr r0, .Lftrace_stub cmp r0, r2 bne 1f #ifdef CONFIG_FUNCTION_GRAPH_TRACER - ldr r1, =ftrace_graph_return - ldr r2, [r1] - cmp r0, r2 - bne ftrace_graph_caller\suffix + ldr_va r2, ftrace_graph_return + cmp r0, r2 + bne ftrace_graph_caller\suffix - ldr r1, =ftrace_graph_entry - ldr r2, [r1] - ldr r0, =ftrace_graph_entry_stub - cmp r0, r2 - bne ftrace_graph_caller\suffix + ldr_va r2, ftrace_graph_entry + mov_l r0, ftrace_graph_entry_stub + cmp r0, r2 + bne ftrace_graph_caller\suffix #endif mcount_exit @@ -87,8 +84,7 @@ mov r3, sp @ struct pt_regs* - ldr r2, =function_trace_op - ldr r2, [r2] @ pointer to the current + ldr_va r2, function_trace_op @ pointer to the current @ function tracing op ldr r1, [sp, #S_LR] @ lr of instrumented func @@ -141,8 +137,7 @@ ftrace_graph_regs_call: mcount_adjust_addr r0, lr @ instrumented function #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS - ldr r2, =function_trace_op - ldr r2, [r2] @ pointer to the current + ldr_va r2, function_trace_op @ pointer to the current @ function tracing op mov r3, #0 @ regs is NULL #endif From 953f534a7ed6b725d4f101d2949393acc9262880 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 25 Jan 2022 11:20:49 +0100 Subject: [PATCH 49/62] ARM: ftrace: enable HAVE_FUNCTION_GRAPH_FP_TEST Fix the frame pointer handling in the function graph tracer entry and exit code so we can enable HAVE_FUNCTION_GRAPH_FP_TEST. Instead of using FP directly (which will have different values between the entry and exit pieces of the function graph tracer), use the value of SP at entry and exit, as we can derive the former value from the frame pointer. Signed-off-by: Ard Biesheuvel Reviewed-by: Steven Rostedt (Google) --- arch/arm/include/asm/ftrace.h | 2 ++ arch/arm/kernel/entry-ftrace.S | 2 +- arch/arm/kernel/ftrace.c | 5 +++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/arch/arm/include/asm/ftrace.h b/arch/arm/include/asm/ftrace.h index a4dbac07e4ef..b4f5fab6b04e 100644 --- a/arch/arm/include/asm/ftrace.h +++ b/arch/arm/include/asm/ftrace.h @@ -2,6 +2,8 @@ #ifndef _ASM_ARM_FTRACE #define _ASM_ARM_FTRACE +#define HAVE_FUNCTION_GRAPH_FP_TEST + #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS #define ARCH_SUPPORTS_FTRACE_OPS 1 #endif diff --git a/arch/arm/kernel/entry-ftrace.S b/arch/arm/kernel/entry-ftrace.S index 99720064a4c5..bbfa0954c385 100644 --- a/arch/arm/kernel/entry-ftrace.S +++ b/arch/arm/kernel/entry-ftrace.S @@ -247,7 +247,7 @@ ENDPROC(ftrace_graph_regs_caller) .globl return_to_handler return_to_handler: stmdb sp!, {r0-r3} - mov r0, fp @ frame pointer + add r0, sp, #16 @ sp at exit of instrumented routine bl ftrace_return_to_handler mov lr, r0 @ r0 has real ret addr ldmia sp!, {r0-r3} diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c index 74d3913f5590..ea2396900c7d 100644 --- a/arch/arm/kernel/ftrace.c +++ b/arch/arm/kernel/ftrace.c @@ -233,6 +233,11 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, if (unlikely(atomic_read(¤t->tracing_graph_pause))) return; + if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER)) { + /* FP points one word below parent's top of stack */ + frame_pointer += 4; + } + old = *parent; *parent = return_hooker; From 538b9265c063f081ca6b1228d242575a1db60711 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 24 Jan 2022 16:49:09 +0100 Subject: [PATCH 50/62] ARM: unwind: track location of LR value in stack frame The ftrace graph tracer needs to override the return address of an instrumented function, in order to install a hook that gets invoked when the function returns again. Currently, we only support this when building for ARM using GCC with frame pointers, as in this case, it is guaranteed that the function will reload LR from [FP, #-4] in all cases, and we can simply pass that address to the ftrace code. In order to support this for configurations that rely on the EABI unwinder, such as Thumb2 builds, make the unwinder keep track of the address from which LR was unwound, permitting ftrace to make use of this in a subsequent patch. Drop the call to is_kernel_text_address(), which is problematic in terms of ftrace recursion, given that it may be instrumented itself. The call is redundant anyway, as no unwind directives will be found unless the PC points to memory that is known to contain executable code. Signed-off-by: Ard Biesheuvel Reviewed-by: Nick Desaulniers --- arch/arm/include/asm/stacktrace.h | 3 +++ arch/arm/kernel/Makefile | 1 + arch/arm/kernel/unwind.c | 7 ++++--- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/arch/arm/include/asm/stacktrace.h b/arch/arm/include/asm/stacktrace.h index d87d60532b86..e56503fd9447 100644 --- a/arch/arm/include/asm/stacktrace.h +++ b/arch/arm/include/asm/stacktrace.h @@ -14,6 +14,9 @@ struct stackframe { unsigned long sp; unsigned long lr; unsigned long pc; + + /* address of the LR value on the stack */ + unsigned long *lr_addr; #ifdef CONFIG_KRETPROBES struct llist_node *kr_cur; struct task_struct *tsk; diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index ae295a3bcfef..56511856ff9d 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -10,6 +10,7 @@ ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_ftrace.o = -pg CFLAGS_REMOVE_insn.o = -pg CFLAGS_REMOVE_patch.o = -pg +CFLAGS_REMOVE_unwind.o = -pg endif CFLAGS_REMOVE_return_address.o = -pg diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c index c5ea328c428d..b4e468a7674b 100644 --- a/arch/arm/kernel/unwind.c +++ b/arch/arm/kernel/unwind.c @@ -55,6 +55,7 @@ struct unwind_ctrl_block { const unsigned long *insn; /* pointer to the current instructions word */ unsigned long sp_low; /* lowest value of sp allowed */ unsigned long sp_high; /* highest value of sp allowed */ + unsigned long *lr_addr; /* address of LR value on the stack */ /* * 1 : check for stack overflow for each register pop. * 0 : save overhead if there is plenty of stack remaining. @@ -239,6 +240,8 @@ static int unwind_pop_register(struct unwind_ctrl_block *ctrl, * from being tracked by KASAN. */ ctrl->vrs[reg] = READ_ONCE_NOCHECK(*(*vsp)); + if (reg == 14) + ctrl->lr_addr = *vsp; (*vsp)++; return URC_OK; } @@ -395,9 +398,6 @@ int unwind_frame(struct stackframe *frame) pr_debug("%s(pc = %08lx lr = %08lx sp = %08lx)\n", __func__, frame->pc, frame->lr, frame->sp); - if (!kernel_text_address(frame->pc)) - return -URC_FAILURE; - idx = unwind_find_idx(frame->pc); if (!idx) { pr_warn("unwind: Index not found %08lx\n", frame->pc); @@ -476,6 +476,7 @@ int unwind_frame(struct stackframe *frame) frame->lr = ctrl.vrs[LR]; frame->pc = ctrl.vrs[PC]; frame->sp_low = ctrl.sp_low; + frame->lr_addr = ctrl.lr_addr; return URC_OK; } From 41918ec82eb6f80c8b401422f27ca76c85aa0cb7 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 25 Jan 2022 15:55:24 +0100 Subject: [PATCH 51/62] ARM: ftrace: enable the graph tracer with the EABI unwinder Enable the function graph tracer in combination with the EABI unwinder, so that Thumb2 builds or Clang ARM builds can make use of it. This involves using the unwinder to locate the return address of an instrumented function on the stack, so that it can be overridden and made to refer to the ftrace handling routines that need to be called at function return. Given that for these builds, it is not guaranteed that the value of the link register is stored on the stack, fall back to the stack slot that will be used by the ftrace exit code to restore LR in the instrumented function's execution context. Signed-off-by: Ard Biesheuvel Reviewed-by: Steven Rostedt (Google) --- arch/arm/Kconfig | 2 +- arch/arm/Kconfig.debug | 2 +- arch/arm/include/asm/ftrace.h | 18 ------------------ arch/arm/kernel/Makefile | 5 +---- arch/arm/kernel/entry-ftrace.S | 28 ++++++++++++++++++++-------- arch/arm/kernel/ftrace.c | 19 +++++++++++++++++-- 6 files changed, 40 insertions(+), 34 deletions(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 70ab8d807032..ec98387367d0 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -91,7 +91,7 @@ config ARM select HAVE_EXIT_THREAD select HAVE_FAST_GUP if ARM_LPAE select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL - select HAVE_FUNCTION_GRAPH_TRACER if !THUMB2_KERNEL && !CC_IS_CLANG + select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER if !XIP_KERNEL && !(THUMB2_KERNEL && CC_IS_CLANG) select HAVE_FUTEX_CMPXCHG if FUTEX select HAVE_GCC_PLUGINS diff --git a/arch/arm/Kconfig.debug b/arch/arm/Kconfig.debug index 98436702e0c7..b79dc7fa89bf 100644 --- a/arch/arm/Kconfig.debug +++ b/arch/arm/Kconfig.debug @@ -65,7 +65,7 @@ config UNWINDER_FRAME_POINTER config UNWINDER_ARM bool "ARM EABI stack unwinder" - depends on AEABI && !FUNCTION_GRAPH_TRACER + depends on AEABI # https://github.com/ClangBuiltLinux/linux/issues/732 depends on !LD_IS_LLD || LLD_VERSION >= 110000 select ARM_UNWIND diff --git a/arch/arm/include/asm/ftrace.h b/arch/arm/include/asm/ftrace.h index b4f5fab6b04e..5358aad67831 100644 --- a/arch/arm/include/asm/ftrace.h +++ b/arch/arm/include/asm/ftrace.h @@ -35,26 +35,8 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) #ifndef __ASSEMBLY__ -#if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND) -/* - * return_address uses walk_stackframe to do it's work. If both - * CONFIG_FRAME_POINTER=y and CONFIG_ARM_UNWIND=y walk_stackframe uses unwind - * information. For this to work in the function tracer many functions would - * have to be marked with __notrace. So for now just depend on - * !CONFIG_ARM_UNWIND. - */ - void *return_address(unsigned int); -#else - -static inline void *return_address(unsigned int level) -{ - return NULL; -} - -#endif - #define ftrace_return_address(n) return_address(n) #define ARCH_HAS_SYSCALL_MATCH_SYM_NAME diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index 56511856ff9d..5cebb8d5a1d6 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -25,10 +25,7 @@ obj-y := elf.o entry-common.o irq.o opcodes.o \ KASAN_SANITIZE_stacktrace.o := n KASAN_SANITIZE_traps.o := n -ifneq ($(CONFIG_ARM_UNWIND),y) -obj-$(CONFIG_FRAME_POINTER) += return_address.o -endif - +obj-y += return_address.o obj-$(CONFIG_ATAGS) += atags_parse.o obj-$(CONFIG_ATAGS_PROC) += atags_proc.o obj-$(CONFIG_DEPRECATED_PARAM_STRUCT) += atags_compat.o diff --git a/arch/arm/kernel/entry-ftrace.S b/arch/arm/kernel/entry-ftrace.S index bbfa0954c385..3e7bcaca5e07 100644 --- a/arch/arm/kernel/entry-ftrace.S +++ b/arch/arm/kernel/entry-ftrace.S @@ -100,7 +100,8 @@ ftrace_regs_call: #ifdef CONFIG_FUNCTION_GRAPH_TRACER .globl ftrace_graph_regs_call ftrace_graph_regs_call: - mov r0, r0 +ARM( mov r0, r0 ) +THUMB( nop.w ) #endif @ pop saved regs @@ -112,13 +113,18 @@ ftrace_graph_regs_call: #ifdef CONFIG_FUNCTION_GRAPH_TRACER .macro __ftrace_graph_regs_caller - sub r0, fp, #4 @ lr of instrumented routine (parent) +#ifdef CONFIG_UNWINDER_FRAME_POINTER + sub r0, fp, #4 @ lr of instrumented routine (parent) +#else + add r0, sp, #S_LR +#endif @ called from __ftrace_regs_caller - ldr r1, [sp, #S_PC] @ instrumented routine (func) + ldr r1, [sp, #S_PC] @ instrumented routine (func) mcount_adjust_addr r1, r1 - mov r2, fp @ frame pointer + mov r2, fpreg @ frame pointer + add r3, sp, #PT_REGS_SIZE bl prepare_ftrace_return @ pop registers saved in ftrace_regs_caller @@ -149,14 +155,19 @@ ftrace_call\suffix: #ifdef CONFIG_FUNCTION_GRAPH_TRACER .globl ftrace_graph_call\suffix ftrace_graph_call\suffix: - mov r0, r0 +ARM( mov r0, r0 ) +THUMB( nop.w ) #endif mcount_exit .endm .macro __ftrace_graph_caller +#ifdef CONFIG_UNWINDER_FRAME_POINTER sub r0, fp, #4 @ &lr of instrumented routine (&parent) +#else + add r0, sp, #20 +#endif #ifdef CONFIG_DYNAMIC_FTRACE @ called from __ftrace_caller, saved in mcount_enter ldr r1, [sp, #16] @ instrumented routine (func) @@ -165,7 +176,8 @@ ftrace_graph_call\suffix: @ called from __mcount, untouched in lr mcount_adjust_addr r1, lr @ instrumented routine (func) #endif - mov r2, fp @ frame pointer + mov r2, fpreg @ frame pointer + add r3, sp, #24 bl prepare_ftrace_return mcount_exit .endm @@ -244,14 +256,14 @@ ENDPROC(ftrace_graph_regs_caller) .purgem mcount_exit #ifdef CONFIG_FUNCTION_GRAPH_TRACER - .globl return_to_handler -return_to_handler: +ENTRY(return_to_handler) stmdb sp!, {r0-r3} add r0, sp, #16 @ sp at exit of instrumented routine bl ftrace_return_to_handler mov lr, r0 @ r0 has real ret addr ldmia sp!, {r0-r3} ret lr +ENDPROC(return_to_handler) #endif ENTRY(ftrace_stub) diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c index ea2396900c7d..83cc068586bc 100644 --- a/arch/arm/kernel/ftrace.c +++ b/arch/arm/kernel/ftrace.c @@ -22,6 +22,7 @@ #include #include #include +#include #include /* @@ -224,8 +225,10 @@ int ftrace_make_nop(struct module *mod, #endif /* CONFIG_DYNAMIC_FTRACE */ #ifdef CONFIG_FUNCTION_GRAPH_TRACER +asmlinkage void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, - unsigned long frame_pointer) + unsigned long frame_pointer, + unsigned long stack_pointer) { unsigned long return_hooker = (unsigned long) &return_to_handler; unsigned long old; @@ -236,6 +239,18 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER)) { /* FP points one word below parent's top of stack */ frame_pointer += 4; + } else { + struct stackframe frame = { + .fp = frame_pointer, + .sp = stack_pointer, + .lr = self_addr, + .pc = self_addr, + }; + if (unwind_frame(&frame) < 0) + return; + if (frame.lr != self_addr) + parent = frame.lr_addr; + frame_pointer = frame.sp; } old = *parent; @@ -258,7 +273,7 @@ static int __ftrace_modify_caller(unsigned long *callsite, unsigned long caller_fn = (unsigned long) func; unsigned long pc = (unsigned long) callsite; unsigned long branch = arm_gen_branch(pc, caller_fn); - unsigned long nop = 0xe1a00000; /* mov r0, r0 */ + unsigned long nop = arm_gen_nop(); unsigned long old = enable ? nop : branch; unsigned long new = enable ? branch : nop; From dd12e97f3c7233a65a0125e5c5c793da16e1137d Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 26 Jan 2022 11:27:21 +0100 Subject: [PATCH 52/62] ARM: kprobes: treat R7 as the frame pointer register in Thumb2 builds Thumb2 code uses R7 as the frame pointer rather than R11, because the opcodes to access it are generally shorter. This means that there are cases where we cannot simply add it to the clobber list of an asm() block, but need to preserve/restore it explicitly, or the compiler may complain in some cases (e.g., Clang builds with ftrace enabled). Since R11 is not special in that case, clobber it instead, and use it to preserve/restore the value of R7. Signed-off-by: Ard Biesheuvel Reviewed-by: Masami Hiramatsu --- arch/arm/probes/kprobes/actions-common.c | 8 +++++--- arch/arm/probes/kprobes/actions-thumb.c | 16 ++++++++++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/arch/arm/probes/kprobes/actions-common.c b/arch/arm/probes/kprobes/actions-common.c index 836aebe596cd..79171344dbeb 100644 --- a/arch/arm/probes/kprobes/actions-common.c +++ b/arch/arm/probes/kprobes/actions-common.c @@ -84,7 +84,8 @@ emulate_generic_r0_12_noflags(probes_opcode_t insn, register void *rfn asm("lr") = asi->insn_fn; __asm__ __volatile__ ( - "stmdb sp!, {%[regs], r11} \n\t" +ARM( "stmdb sp!, {%[regs], r11} \n\t" ) +THUMB( "stmdb sp!, {%[regs], r7} \n\t" ) "ldmia %[regs], {r0-r12} \n\t" #if __LINUX_ARM_ARCH__ >= 6 "blx %[fn] \n\t" @@ -96,10 +97,11 @@ emulate_generic_r0_12_noflags(probes_opcode_t insn, #endif "ldr lr, [sp], #4 \n\t" /* lr = regs */ "stmia lr, {r0-r12} \n\t" - "ldr r11, [sp], #4 \n\t" +ARM( "ldr r11, [sp], #4 \n\t" ) +THUMB( "ldr r7, [sp], #4 \n\t" ) : [regs] "=r" (rregs), [fn] "=r" (rfn) : "0" (rregs), "1" (rfn) - : "r0", "r2", "r3", "r4", "r5", "r6", "r7", + : "r0", "r2", "r3", "r4", "r5", "r6", ARM("r7") THUMB("r11"), "r8", "r9", "r10", "r12", "memory", "cc" ); } diff --git a/arch/arm/probes/kprobes/actions-thumb.c b/arch/arm/probes/kprobes/actions-thumb.c index 7884fcb81c26..51624fc263fc 100644 --- a/arch/arm/probes/kprobes/actions-thumb.c +++ b/arch/arm/probes/kprobes/actions-thumb.c @@ -447,14 +447,16 @@ t16_emulate_loregs(probes_opcode_t insn, __asm__ __volatile__ ( "msr cpsr_fs, %[oldcpsr] \n\t" + "mov r11, r7 \n\t" "ldmia %[regs], {r0-r7} \n\t" "blx %[fn] \n\t" "stmia %[regs], {r0-r7} \n\t" + "mov r7, r11 \n\t" "mrs %[newcpsr], cpsr \n\t" : [newcpsr] "=r" (newcpsr) : [oldcpsr] "r" (oldcpsr), [regs] "r" (regs), [fn] "r" (asi->insn_fn) - : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", + : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r11", "lr", "memory", "cc" ); @@ -524,14 +526,16 @@ t16_emulate_push(probes_opcode_t insn, struct arch_probes_insn *asi, struct pt_regs *regs) { __asm__ __volatile__ ( + "mov r11, r7 \n\t" "ldr r9, [%[regs], #13*4] \n\t" "ldr r8, [%[regs], #14*4] \n\t" "ldmia %[regs], {r0-r7} \n\t" "blx %[fn] \n\t" "str r9, [%[regs], #13*4] \n\t" + "mov r7, r11 \n\t" : : [regs] "r" (regs), [fn] "r" (asi->insn_fn) - : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", + : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r8", "r9", "r11", "lr", "memory", "cc" ); } @@ -558,14 +562,16 @@ t16_emulate_pop_nopc(probes_opcode_t insn, struct arch_probes_insn *asi, struct pt_regs *regs) { __asm__ __volatile__ ( + "mov r11, r7 \n\t" "ldr r9, [%[regs], #13*4] \n\t" "ldmia %[regs], {r0-r7} \n\t" "blx %[fn] \n\t" "stmia %[regs], {r0-r7} \n\t" "str r9, [%[regs], #13*4] \n\t" + "mov r7, r11 \n\t" : : [regs] "r" (regs), [fn] "r" (asi->insn_fn) - : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r9", + : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r9", "r11", "lr", "memory", "cc" ); } @@ -577,14 +583,16 @@ t16_emulate_pop_pc(probes_opcode_t insn, register unsigned long pc asm("r8"); __asm__ __volatile__ ( + "mov r11, r7 \n\t" "ldr r9, [%[regs], #13*4] \n\t" "ldmia %[regs], {r0-r7} \n\t" "blx %[fn] \n\t" "stmia %[regs], {r0-r7} \n\t" "str r9, [%[regs], #13*4] \n\t" + "mov r7, r11 \n\t" : "=r" (pc) : [regs] "r" (regs), [fn] "r" (asi->insn_fn) - : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r9", + : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r9", "r11", "lr", "memory", "cc" ); From 1f640552d9878f2dbcbd46c78078e4ea2eb2b262 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 26 Jan 2022 11:40:02 +0100 Subject: [PATCH 53/62] ARM: cacheflush: avoid clobbering the frame pointer Thumb2 uses R7 rather than R11 as the frame pointer, and even if we rarely use a frame pointer to begin with when building in Thumb2 mode, there are cases where it is required by the compiler (Clang when inserting profiling hooks via -pg) However, preserving and restoring the frame pointer is risky, as any unhandled exceptions raised in the mean time will produce a bogus backtrace, and it would be better not to touch the frame pointer at all. This is the case even when CONFIG_FRAME_POINTER is not set, as the unwind directive used by the unwinder may also use R7 or R11 as the unwind anchor, even if the frame pointer is not managed strictly according to the frame pointer ABI. So let's tweak the cacheflush asm code not to clobber R7 or R11 at all, so that we can drop R7 from the clobber lists of the inline asm blocks that call these routines, and remove the code that preserves/restores R11. Signed-off-by: Ard Biesheuvel Reviewed-by: Nick Desaulniers --- arch/arm/include/asm/cacheflush.h | 12 +++------ arch/arm/mach-exynos/mcpm-exynos.c | 6 ++--- arch/arm/mm/cache-v7.S | 40 ++++++++++++++---------------- 3 files changed, 23 insertions(+), 35 deletions(-) diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h index e68fb879e4f9..d27782331556 100644 --- a/arch/arm/include/asm/cacheflush.h +++ b/arch/arm/include/asm/cacheflush.h @@ -446,15 +446,10 @@ static inline void __sync_cache_range_r(volatile void *p, size_t size) * however some exceptions may exist. Caveat emptor. * * - The clobber list is dictated by the call to v7_flush_dcache_*. - * fp is preserved to the stack explicitly prior disabling the cache - * since adding it to the clobber list is incompatible with having - * CONFIG_FRAME_POINTER=y. ip is saved as well if ever r12-clobbering - * trampoline are inserted by the linker and to keep sp 64-bit aligned. */ #define v7_exit_coherency_flush(level) \ asm volatile( \ ".arch armv7-a \n\t" \ - "stmfd sp!, {fp, ip} \n\t" \ "mrc p15, 0, r0, c1, c0, 0 @ get SCTLR \n\t" \ "bic r0, r0, #"__stringify(CR_C)" \n\t" \ "mcr p15, 0, r0, c1, c0, 0 @ set SCTLR \n\t" \ @@ -464,10 +459,9 @@ static inline void __sync_cache_range_r(volatile void *p, size_t size) "bic r0, r0, #(1 << 6) @ disable local coherency \n\t" \ "mcr p15, 0, r0, c1, c0, 1 @ set ACTLR \n\t" \ "isb \n\t" \ - "dsb \n\t" \ - "ldmfd sp!, {fp, ip}" \ - : : : "r0","r1","r2","r3","r4","r5","r6","r7", \ - "r9","r10","lr","memory" ) + "dsb" \ + : : : "r0","r1","r2","r3","r4","r5","r6", \ + "r9","r10","ip","lr","memory" ) void flush_uprobe_xol_access(struct page *page, unsigned long uaddr, void *kaddr, unsigned long len); diff --git a/arch/arm/mach-exynos/mcpm-exynos.c b/arch/arm/mach-exynos/mcpm-exynos.c index cd861c57d5ad..fd0dbeb93357 100644 --- a/arch/arm/mach-exynos/mcpm-exynos.c +++ b/arch/arm/mach-exynos/mcpm-exynos.c @@ -35,7 +35,6 @@ static bool secure_firmware __ro_after_init; */ #define exynos_v7_exit_coherency_flush(level) \ asm volatile( \ - "stmfd sp!, {fp, ip}\n\t"\ "mrc p15, 0, r0, c1, c0, 0 @ get SCTLR\n\t" \ "bic r0, r0, #"__stringify(CR_C)"\n\t" \ "mcr p15, 0, r0, c1, c0, 0 @ set SCTLR\n\t" \ @@ -50,11 +49,10 @@ static bool secure_firmware __ro_after_init; "mcr p15, 0, r0, c1, c0, 1 @ set ACTLR\n\t" \ "isb\n\t" \ "dsb\n\t" \ - "ldmfd sp!, {fp, ip}" \ : \ : "Ir" (pmu_base_addr + S5P_INFORM0) \ - : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", \ - "r9", "r10", "lr", "memory") + : "r0", "r1", "r2", "r3", "r4", "r5", "r6", \ + "r9", "r10", "ip", "lr", "memory") static int exynos_cpu_powerup(unsigned int cpu, unsigned int cluster) { diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S index 830bbfb26ca5..7c9499b728c4 100644 --- a/arch/arm/mm/cache-v7.S +++ b/arch/arm/mm/cache-v7.S @@ -90,7 +90,7 @@ ENDPROC(v7_flush_icache_all) * * Flush the D-cache up to the Level of Unification Inner Shareable * - * Corrupted registers: r0-r7, r9-r11 (r6 only in Thumb mode) + * Corrupted registers: r0-r6, r9-r10 */ ENTRY(v7_flush_dcache_louis) @@ -117,7 +117,7 @@ ENDPROC(v7_flush_dcache_louis) * * Flush the whole D-cache. * - * Corrupted registers: r0-r7, r9-r11 (r6 only in Thumb mode) + * Corrupted registers: r0-r6, r9-r10 * * - mm - mm_struct describing address space */ @@ -149,22 +149,22 @@ flush_levels: movw r4, #0x3ff ands r4, r4, r1, lsr #3 @ find maximum number on the way size clz r5, r4 @ find bit position of way size increment - movw r7, #0x7fff - ands r7, r7, r1, lsr #13 @ extract max number of the index size + movw r6, #0x7fff + and r1, r6, r1, lsr #13 @ extract max number of the index size + mov r6, #1 + movne r4, r4, lsl r5 @ # of ways shifted into bits [31:...] + movne r6, r6, lsl r5 @ 1 shifted left by same amount loop1: - mov r9, r7 @ create working copy of max index + mov r9, r1 @ create working copy of max index loop2: - ARM( orr r11, r10, r4, lsl r5 ) @ factor way and cache number into r11 - THUMB( lsl r6, r4, r5 ) - THUMB( orr r11, r10, r6 ) @ factor way and cache number into r11 - ARM( orr r11, r11, r9, lsl r2 ) @ factor index number into r11 - THUMB( lsl r6, r9, r2 ) - THUMB( orr r11, r11, r6 ) @ factor index number into r11 - mcr p15, 0, r11, c7, c14, 2 @ clean & invalidate by set/way + mov r5, r9, lsl r2 @ factor set number into r5 + orr r5, r5, r4 @ factor way number into r5 + orr r5, r5, r10 @ factor cache level into r5 + mcr p15, 0, r5, c7, c14, 2 @ clean & invalidate by set/way subs r9, r9, #1 @ decrement the index bge loop2 - subs r4, r4, #1 @ decrement the way - bge loop1 + subs r4, r4, r6 @ decrement the way + bcs loop1 skip: add r10, r10, #2 @ increment cache number cmp r3, r10 @@ -192,14 +192,12 @@ ENDPROC(v7_flush_dcache_all) * */ ENTRY(v7_flush_kern_cache_all) - ARM( stmfd sp!, {r4-r5, r7, r9-r11, lr} ) - THUMB( stmfd sp!, {r4-r7, r9-r11, lr} ) + stmfd sp!, {r4-r6, r9-r10, lr} bl v7_flush_dcache_all mov r0, #0 ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate - ARM( ldmfd sp!, {r4-r5, r7, r9-r11, lr} ) - THUMB( ldmfd sp!, {r4-r7, r9-r11, lr} ) + ldmfd sp!, {r4-r6, r9-r10, lr} ret lr ENDPROC(v7_flush_kern_cache_all) @@ -210,14 +208,12 @@ ENDPROC(v7_flush_kern_cache_all) * Invalidate the I-cache to the point of unification. */ ENTRY(v7_flush_kern_cache_louis) - ARM( stmfd sp!, {r4-r5, r7, r9-r11, lr} ) - THUMB( stmfd sp!, {r4-r7, r9-r11, lr} ) + stmfd sp!, {r4-r6, r9-r10, lr} bl v7_flush_dcache_louis mov r0, #0 ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate - ARM( ldmfd sp!, {r4-r5, r7, r9-r11, lr} ) - THUMB( ldmfd sp!, {r4-r7, r9-r11, lr} ) + ldmfd sp!, {r4-r6, r9-r10, lr} ret lr ENDPROC(v7_flush_kern_cache_louis) From 64dff07b1c37b4840a6ef8698dbd2f701a407e48 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 1 Feb 2022 11:56:44 +0100 Subject: [PATCH 54/62] ARM: mach-bcm: disable ftrace in SMC invocation routines The SMC calling convention uses R7 as an argument register, which conflicts with its use as a frame pointer when building in Thumb2 mode. Given that Clang with ftrace does not permit frame pointers to be disabled, let's omit this compilation unit from ftrace instrumentation. Signed-off-by: Ard Biesheuvel Acked-by: Nick Desaulniers --- arch/arm/mach-bcm/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm/mach-bcm/Makefile b/arch/arm/mach-bcm/Makefile index 7baa8c9427d5..b2394ddb0558 100644 --- a/arch/arm/mach-bcm/Makefile +++ b/arch/arm/mach-bcm/Makefile @@ -40,6 +40,7 @@ obj-$(CONFIG_ARCH_BCM_MOBILE_L2_CACHE) += kona_l2_cache.o # Support for secure monitor traps obj-$(CONFIG_ARCH_BCM_MOBILE_SMC) += bcm_kona_smc.o +CFLAGS_REMOVE_bcm_kona_smc.o += $(CC_FLAGS_FTRACE) # BCM2835 ifeq ($(CONFIG_ARCH_BCM2835),y) From d6800ca73a7d325627c045c16d7cfdc7465f4333 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 26 Jan 2022 11:43:39 +0100 Subject: [PATCH 55/62] Revert "ARM: 9144/1: forbid ftrace with clang and thumb2_kernel" This reverts commit ecb108e3e3f7c692082b7c6fce41779c3835854a. Clang + Thumb2 with ftrace is now supported. Signed-off-by: Ard Biesheuvel Reviewed-by: Nick Desaulniers Reviewed-by: Steven Rostedt (Google) --- arch/arm/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index ec98387367d0..5c9a8d3362cd 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -92,7 +92,7 @@ config ARM select HAVE_FAST_GUP if ARM_LPAE select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL select HAVE_FUNCTION_GRAPH_TRACER - select HAVE_FUNCTION_TRACER if !XIP_KERNEL && !(THUMB2_KERNEL && CC_IS_CLANG) + select HAVE_FUNCTION_TRACER if !XIP_KERNEL select HAVE_FUTEX_CMPXCHG if FUTEX select HAVE_GCC_PLUGINS select HAVE_HW_BREAKPOINT if PERF_EVENTS && (CPU_V6 || CPU_V6K || CPU_V7) From 81679376470ef4e070d3ab04b578b2eef1013e46 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 2 Mar 2022 12:36:56 +0100 Subject: [PATCH 56/62] ARM: 9183/1: unwind: avoid spurious warnings on bogus code addresses Corentin reports that since commit 538b9265c063 ("ARM: unwind: track location of LR value in stack frame"), numerous spurious warnings are emitted into the kernel log: [ 0.000000] unwind: Index not found c0f0c440 [ 0.000000] unwind: Index not found 00000000 [ 0.000000] unwind: Index not found c0f0c440 [ 0.000000] unwind: Index not found 00000000 This is due to the fact that the commit in question removes a check whether the PC value in the unwound frame is actually a kernel text address, on the assumption that such an address will not be associated with valid unwind data to begin with, which is checked right after. The reason for removing this check was that unwind_frame() will be called by the ftrace graph tracer code, which means that it can no longer be safely instrumented itself, or any code that it calls, as it could cause infinite recursion. In order to prevent the spurious diagnostics, let's add back the call to kernel_text_address(), but this time, only call it if no unwind data could be found for the address in question. This is more efficient for the common successful case, and should avoid any unintended recursion, considering that kernel_text_address() will only be called if no unwind data was found. Cc: Corentin Labbe Fixes: 538b9265c063 ("ARM: unwind: track location of LR value in stack frame") Signed-off-by: Ard Biesheuvel Signed-off-by: Russell King (Oracle) --- arch/arm/kernel/unwind.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c index b4e468a7674b..04ccff9d9793 100644 --- a/arch/arm/kernel/unwind.c +++ b/arch/arm/kernel/unwind.c @@ -400,7 +400,8 @@ int unwind_frame(struct stackframe *frame) idx = unwind_find_idx(frame->pc); if (!idx) { - pr_warn("unwind: Index not found %08lx\n", frame->pc); + if (frame->pc && kernel_text_address(frame->pc)) + pr_warn("unwind: Index not found %08lx\n", frame->pc); return -URC_FAILURE; } From 6845d64d51cf69c096176e34864e161429bcb664 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 2 Mar 2022 12:38:18 +0100 Subject: [PATCH 57/62] ARM: 9184/1: return_address: disable again for CONFIG_ARM_UNWIND=y Commit 41918ec82eb6 ("ARM: ftrace: enable the graph tracer with the EABI unwinder") removed the dummy version of return_address() that was provided for the CONFIG_ARM_UNWIND=y case, on the assumption that the removal of the kernel_text_address() call from unwind_frame() in the preceding patch made it safe to do so. However, this turns out not to be the case: Corentin reports warnings about suspicious RCU usage and other strange behavior that seems to originate in the stack unwinding that occurs in return_address(). Given that the function graph tracer (which is what these changes were enabling for CONFIG_ARM_UNWIND=y builds) does not appear to care about this distinction, let's revert return_address() to the old state. Cc: Corentin Labbe Fixes: 41918ec82eb6 ("ARM: ftrace: enable the graph tracer with the EABI unwinder") Signed-off-by: Ard Biesheuvel Reported-by: Corentin Labbe Tested-by: Corentin Labbe Signed-off-by: Russell King (Oracle) --- arch/arm/include/asm/ftrace.h | 18 ++++++++++++++++++ arch/arm/kernel/Makefile | 5 ++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/arch/arm/include/asm/ftrace.h b/arch/arm/include/asm/ftrace.h index 5358aad67831..7e9251ca29fe 100644 --- a/arch/arm/include/asm/ftrace.h +++ b/arch/arm/include/asm/ftrace.h @@ -35,8 +35,26 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) #ifndef __ASSEMBLY__ +#if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND) +/* + * return_address uses walk_stackframe to do it's work. If both + * CONFIG_FRAME_POINTER=y and CONFIG_ARM_UNWIND=y walk_stackframe uses unwind + * information. For this to work in the function tracer many functions would + * have to be marked with __notrace. So for now just depend on + * !CONFIG_ARM_UNWIND. + */ + void *return_address(unsigned int); +#else + +static inline void *return_address(unsigned int level) +{ + return NULL; +} + +#endif + #define ftrace_return_address(n) return_address(n) #define ARCH_HAS_SYSCALL_MATCH_SYM_NAME diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index 5cebb8d5a1d6..56511856ff9d 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -25,7 +25,10 @@ obj-y := elf.o entry-common.o irq.o opcodes.o \ KASAN_SANITIZE_stacktrace.o := n KASAN_SANITIZE_traps.o := n -obj-y += return_address.o +ifneq ($(CONFIG_ARM_UNWIND),y) +obj-$(CONFIG_FRAME_POINTER) += return_address.o +endif + obj-$(CONFIG_ATAGS) += atags_parse.o obj-$(CONFIG_ATAGS_PROC) += atags_proc.o obj-$(CONFIG_DEPRECATED_PARAM_STRUCT) += atags_compat.o From c46c2c9b43f4f08f20dc06417fbf7091e4ca6d34 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Wed, 9 Mar 2022 12:06:02 +0000 Subject: [PATCH 58/62] ARM: unwind: set frame.pc correctly for current-thread unwinding When e.g. a WARN_ON() is encountered, we attempt to unwind the current thread. To do this, we set frame.pc to unwind_backtrace, which means it points at the beginning of the function. However, the rest of the state is initialised from within the function, which means the function prologue has already been run. This can be confusing, and with a recent patch from Ard, can result in the unwinder misbehaving if we want to be strict about the PC value. If we correctly initialise the state so it is self-consistent (in other words, set frame.pc to the location we are initialising it) then we eliminate this confusion, and avoid possible future issues. Reviewed-by: Ard Biesheuvel Signed-off-by: Russell King (Oracle) --- arch/arm/kernel/return_address.c | 3 ++- arch/arm/kernel/stacktrace.c | 3 ++- arch/arm/kernel/unwind.c | 7 ++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/arch/arm/kernel/return_address.c b/arch/arm/kernel/return_address.c index 00c11579406c..8aac1e10b117 100644 --- a/arch/arm/kernel/return_address.c +++ b/arch/arm/kernel/return_address.c @@ -41,7 +41,8 @@ void *return_address(unsigned int level) frame.fp = (unsigned long)__builtin_frame_address(0); frame.sp = current_stack_pointer; frame.lr = (unsigned long)__builtin_return_address(0); - frame.pc = (unsigned long)return_address; +here: + frame.pc = (unsigned long)&&here; #ifdef CONFIG_KRETPROBES frame.kr_cur = NULL; frame.tsk = current; diff --git a/arch/arm/kernel/stacktrace.c b/arch/arm/kernel/stacktrace.c index 75e905508f27..b5efecb3d730 100644 --- a/arch/arm/kernel/stacktrace.c +++ b/arch/arm/kernel/stacktrace.c @@ -160,7 +160,8 @@ static noinline void __save_stack_trace(struct task_struct *tsk, frame.fp = (unsigned long)__builtin_frame_address(0); frame.sp = current_stack_pointer; frame.lr = (unsigned long)__builtin_return_address(0); - frame.pc = (unsigned long)__save_stack_trace; +here: + frame.pc = (unsigned long)&&here; } #ifdef CONFIG_KRETPROBES frame.kr_cur = NULL; diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c index 04ccff9d9793..3cd8892ed52b 100644 --- a/arch/arm/kernel/unwind.c +++ b/arch/arm/kernel/unwind.c @@ -501,7 +501,12 @@ void unwind_backtrace(struct pt_regs *regs, struct task_struct *tsk, frame.fp = (unsigned long)__builtin_frame_address(0); frame.sp = current_stack_pointer; frame.lr = (unsigned long)__builtin_return_address(0); - frame.pc = (unsigned long)unwind_backtrace; + /* We are saving the stack and execution state at this + * point, so we should ensure that frame.pc is within + * this block of code. + */ +here: + frame.pc = (unsigned long)&&here; } else { /* task blocked in __switch_to */ frame.fp = thread_saved_fp(tsk); From 7a8ca84a258a79e564b77b1bbb139f3f561d149b Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 10 Mar 2022 17:24:45 +0100 Subject: [PATCH 59/62] ARM: entry: fix unwinder problems caused by IRQ stacks The IRQ stacks series made some changes to the unwinder, to permit unwinding across different stacks. This is needed because otherwise, the call stack would terminate at the point where the stack switch between the task stack and the IRQ stack occurs, which would defeat any diagnostics that rely on timer interrupts, such as RCU stall detection. Unfortunately, getting the unwind annotations correct turns out to be difficult, given that this now involves a frame pointer which needs to point into the right location in the task stack when unwinding from the IRQ stack. Getting this wrong for an exception handling routine results in the stack pointer to be unwound from the wrong location, causing any subsequent unwind attempts to cause all kinds of issues, as reported by Naresh here [0]. So let's simplify this, by deferring the stack switch to call_with_stack(), which already has the correct unwind annotations, and removing all the complicated handling of the stack frame from the IRQ exception entrypoint itself. [0] https://lore.kernel.org/all/CA+G9fYtpy8VgK+ag6OsA9TDrwi5YGU4hu7GM8xwpO7v6LrCD4Q@mail.gmail.com/ Signed-off-by: Ard Biesheuvel Signed-off-by: Russell King (Oracle) --- arch/arm/kernel/entry-armv.S | 57 +++++++++--------------------------- 1 file changed, 14 insertions(+), 43 deletions(-) diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 038aabb6578f..f53525114a2b 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -33,58 +33,29 @@ * Interrupt handling. */ .macro irq_handler, from_user:req - mov r0, sp -#ifdef CONFIG_IRQSTACKS -#ifdef CONFIG_UNWINDER_ARM - mov fpreg, sp @ Preserve original SP -#else - mov r7, fp @ Preserve original FP - mov r8, sp @ Preserve original SP -#endif - ldr_this_cpu sp, irq_stack_ptr, r2, r3 + mov r1, sp + ldr_this_cpu r2, irq_stack_ptr, r2, r3 .if \from_user == 0 -UNWIND( .setfp fpreg, sp ) @ @ If we took the interrupt while running in the kernel, we may already @ be using the IRQ stack, so revert to the original value in that case. @ - subs r2, sp, r0 @ SP above bottom of IRQ stack? - rsbscs r2, r2, #THREAD_SIZE @ ... and below the top? + subs r3, r2, r1 @ SP above bottom of IRQ stack? + rsbscs r3, r3, #THREAD_SIZE @ ... and below the top? #ifdef CONFIG_VMAP_STACK - ldr_va r2, high_memory, cc @ End of the linear region - cmpcc r2, r0 @ Stack pointer was below it? + ldr_va r3, high_memory, cc @ End of the linear region + cmpcc r3, r1 @ Stack pointer was below it? #endif - movcs sp, r0 @ If so, revert to incoming SP - -#ifndef CONFIG_UNWINDER_ARM - @ - @ Inform the frame pointer unwinder where the next frame lives - @ - movcc lr, pc @ Make LR point into .entry.text so - @ that we will get a dump of the - @ exception stack for this frame. -#ifdef CONFIG_CC_IS_GCC - movcc ip, r0 @ Store the old SP in the frame record. - stmdbcc sp!, {fp, ip, lr, pc} @ Push frame record - addcc fp, sp, #12 -#else - stmdbcc sp!, {fp, lr} @ Push frame record - movcc fp, sp -#endif // CONFIG_CC_IS_GCC -#endif // CONFIG_UNWINDER_ARM - .endif -#endif // CONFIG_IRQSTACKS - + bcc 0f @ If not, switch to the IRQ stack + mov r0, r1 bl generic_handle_arch_irq + b 1f +0: + .endif -#ifdef CONFIG_IRQSTACKS -#ifdef CONFIG_UNWINDER_ARM - mov sp, fpreg @ Restore original SP -#else - mov fp, r7 @ Restore original FP - mov sp, r8 @ Restore original SP -#endif // CONFIG_UNWINDER_ARM -#endif // CONFIG_IRQSTACKS + mov_l r0, generic_handle_arch_irq + bl call_with_stack +1: .endm .macro pabt_helper From bee4e1fdc31223f8e0569370635ced223a1dd2ce Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 10 Mar 2022 18:05:28 +0100 Subject: [PATCH 60/62] ARM: Revert "unwind: dump exception stack from calling frame" After simplifying the stack switch code in the IRQ exception handler by deferring the actual stack switch to call_with_stack(), we no longer need to special case the way we dump the exception stack, since it will always be at the top of whichever stack was active when the exception was taken. So revert this special handling for the ARM unwinder. This reverts commit 4ab6827081c63b83011a18d8e27f621ed34b1194. Signed-off-by: Ard Biesheuvel Signed-off-by: Russell King (Oracle) --- arch/arm/include/asm/stacktrace.h | 10 ---------- arch/arm/kernel/traps.c | 3 +-- arch/arm/kernel/unwind.c | 9 +-------- 3 files changed, 2 insertions(+), 20 deletions(-) diff --git a/arch/arm/include/asm/stacktrace.h b/arch/arm/include/asm/stacktrace.h index e56503fd9447..3e78f921b8b2 100644 --- a/arch/arm/include/asm/stacktrace.h +++ b/arch/arm/include/asm/stacktrace.h @@ -21,16 +21,6 @@ struct stackframe { struct llist_node *kr_cur; struct task_struct *tsk; #endif -#ifdef CONFIG_ARM_UNWIND - /* - * This field is used to track the stack pointer value when calling - * __entry routines. This is needed when IRQ stacks and overflow stacks - * are used, because in that case, the struct pt_regs passed to these - * __entry routines may be at the top of the task stack, while we are - * executing from another stack. - */ - unsigned long sp_low; -#endif }; static __always_inline diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 08612032aefe..a4938b699b49 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -87,8 +87,7 @@ void dump_backtrace_entry(unsigned long where, unsigned long from, loglvl, where, from); #endif - if (!IS_ENABLED(CONFIG_UNWINDER_ARM) && - in_entry_text(from) && end <= ALIGN(frame, THREAD_SIZE)) + if (in_entry_text(from) && end <= ALIGN(frame, THREAD_SIZE)) dump_mem(loglvl, "Exception stack", frame + 4, end); } diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c index 3cd8892ed52b..e619ec7856b7 100644 --- a/arch/arm/kernel/unwind.c +++ b/arch/arm/kernel/unwind.c @@ -29,7 +29,6 @@ #include #include -#include #include #include #include @@ -424,7 +423,6 @@ int unwind_frame(struct stackframe *frame) */ if (frame->pc == frame->lr) return -URC_FAILURE; - frame->sp_low = frame->sp; frame->pc = frame->lr; return URC_OK; } else if ((idx->insn & 0x80000000) == 0) @@ -476,7 +474,6 @@ int unwind_frame(struct stackframe *frame) frame->sp = ctrl.vrs[SP]; frame->lr = ctrl.vrs[LR]; frame->pc = ctrl.vrs[PC]; - frame->sp_low = ctrl.sp_low; frame->lr_addr = ctrl.lr_addr; return URC_OK; @@ -526,11 +523,7 @@ here: urc = unwind_frame(&frame); if (urc < 0) break; - if (in_entry_text(where)) - dump_mem(loglvl, "Exception stack", frame.sp_low, - frame.sp_low + sizeof(struct pt_regs)); - - dump_backtrace_entry(where, frame.pc, 0, loglvl); + dump_backtrace_entry(where, frame.pc, frame.sp - 4, loglvl); } } From f6b8e3526feb025d0259c18d6dc6b8c2e2cfedf0 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 11 Mar 2022 08:45:29 +0100 Subject: [PATCH 61/62] ARM: unwind: only permit stack switch when unwinding call_with_stack() Commit b6506981f880 ("ARM: unwind: support unwinding across multiple stacks") updated the logic in the ARM unwinder to widen the bounds within which SP is assumed to be valid, in order to allow the unwind to traverse from the IRQ stack to the task stack. This is necessary, as otherwise, unwinds started from the IRQ stack would terminate in the IRQ exception handler, making stacktraces substantially less useful. This turns out to be a mistake, as it breaks asynchronous unwinding across exceptions, when the exception is taken before the stack frame is consistent with the unwind info. For instance, in the following backtrace: ... generic_handle_arch_irq from call_with_stack+0x18/0x20 call_with_stack from __irq_svc+0x80/0x98 Exception stack(0xc7093e20 to 0xc7093e68) 3e20: b6a94a88 c7093ea0 00000008 00000000 c7093ea0 b7e127d0 00000051 c9220000 3e40: b6a94a88 b6a94a88 00000004 0002b000 0036b570 c7093e70 c040ca2c c0994a90 3e60: 20070013 ffffffff __irq_svc from __copy_to_user_std+0x20/0x378 ... we need to apply the following unwind directives: 0xc099720c <__copy_to_user_std+0x1c>: @0xc295d1d4 Compact model index: 1 0x9b vsp = r11 0xb1 0x0d pop {r0, r2, r3} 0x84 0x81 pop {r4, r11, r14} 0xb0 finish which tell us to switch to the frame pointer register R11 and proceed with the unwind from that. However, having been interrupted 0x20 bytes into the function: c09971f0 <__copy_to_user_std>: c09971f0: e59f3350 ldr r3, [pc, #848] c09971f4: e243c001 sub ip, r3, #1 c09971f8: e05cc000 subs ip, ip, r0 c09971fc: 228cc001 addcs ip, ip, #1 c0997200: 205cc002 subscs ip, ip, r2 c0997204: 33a00000 movcc r0, #0 c0997208: e320f014 csdb c099720c: e3a03000 mov r3, #0 c0997210: e92d481d push {r0, r2, r3, r4, fp, lr} c0997214: e1a0b00d mov fp, sp c0997218: e2522004 subs r2, r2, #4 the value for R11 recovered from the previous frame (__irq_svc) will be a snapshot of its value before the exception was taken (0x0002b000), which occurred at address __copy_to_user_std+0x20 (0xc0997210), when R11 had not been assigned its value yet. This means we can never assume that the SP values recovered from the stack or from the frame pointer are ever safe to use, given the need to do asynchronous unwinding, and the only robust approach is to revert to the previous approach, which is to derive bounds for SP based on the initial value, and never update them. We can make an exception, though: now that the IRQ stack switch is guaranteed to occur in call_with_stack(), we can implement a special case for this function, and use a different set of bounds based on the knowledge that it will always unwind from R11 rather than SP. As call_with_stack() is a hand-rolled assembly routine, this is guaranteed to remain that way. So let's do a partial revert of b6506981f880, and drop all manipulations for sp_low and sp_high based on the information collected during the unwind itself. To support call_with_stack(), set sp_low and sp_high explicitly to values derived from R11 when we unwind that function. The only downside is that, while unwinding an overflow of the vmap'ed stack will work fine as before, we will no longer be able to produce a backtrace that unwinds the overflow stack itself across the exception that was raised due to the faulting access to the guard region. However, this only affects exceptions caused by problems in the stack overflow handling code itself, in which case the remaining backtrace is not that relevant. Fixes: b6506981f880 ("ARM: unwind: support unwinding across multiple stacks") Signed-off-by: Ard Biesheuvel Signed-off-by: Russell King (Oracle) --- arch/arm/kernel/unwind.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c index e619ec7856b7..a37ea6c772cd 100644 --- a/arch/arm/kernel/unwind.c +++ b/arch/arm/kernel/unwind.c @@ -33,6 +33,8 @@ #include #include +#include "reboot.h" + /* Dummy functions to avoid linker complaints */ void __aeabi_unwind_cpp_pr0(void) { @@ -52,7 +54,6 @@ EXPORT_SYMBOL(__aeabi_unwind_cpp_pr2); struct unwind_ctrl_block { unsigned long vrs[16]; /* virtual register set */ const unsigned long *insn; /* pointer to the current instructions word */ - unsigned long sp_low; /* lowest value of sp allowed */ unsigned long sp_high; /* highest value of sp allowed */ unsigned long *lr_addr; /* address of LR value on the stack */ /* @@ -262,9 +263,6 @@ static int unwind_exec_pop_subset_r4_to_r13(struct unwind_ctrl_block *ctrl, } if (!load_sp) { ctrl->vrs[SP] = (unsigned long)vsp; - } else { - ctrl->sp_low = ctrl->vrs[SP]; - ctrl->sp_high = ALIGN(ctrl->sp_low, THREAD_SIZE); } return URC_OK; @@ -323,7 +321,6 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) ctrl->vrs[SP] += ((insn & 0x3f) << 2) + 4; else if ((insn & 0xc0) == 0x40) { ctrl->vrs[SP] -= ((insn & 0x3f) << 2) + 4; - ctrl->sp_low = ctrl->vrs[SP]; } else if ((insn & 0xf0) == 0x80) { unsigned long mask; @@ -341,8 +338,6 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) } else if ((insn & 0xf0) == 0x90 && (insn & 0x0d) != 0x0d) { ctrl->vrs[SP] = ctrl->vrs[insn & 0x0f]; - ctrl->sp_low = ctrl->vrs[SP]; - ctrl->sp_high = ALIGN(ctrl->sp_low, THREAD_SIZE); } else if ((insn & 0xf0) == 0xa0) { ret = unwind_exec_pop_r4_to_rN(ctrl, insn); if (ret) @@ -388,10 +383,11 @@ int unwind_frame(struct stackframe *frame) { const struct unwind_idx *idx; struct unwind_ctrl_block ctrl; + unsigned long sp_low; /* store the highest address on the stack to avoid crossing it*/ - ctrl.sp_low = frame->sp; - ctrl.sp_high = ALIGN(ctrl.sp_low - THREAD_SIZE, THREAD_ALIGN) + sp_low = frame->sp; + ctrl.sp_high = ALIGN(sp_low - THREAD_SIZE, THREAD_ALIGN) + THREAD_SIZE; pr_debug("%s(pc = %08lx lr = %08lx sp = %08lx)\n", __func__, @@ -452,6 +448,16 @@ int unwind_frame(struct stackframe *frame) ctrl.check_each_pop = 0; + if (prel31_to_addr(&idx->addr_offset) == (u32)&call_with_stack) { + /* + * call_with_stack() is the only place where we permit SP to + * jump from one stack to another, and since we know it is + * guaranteed to happen, set up the SP bounds accordingly. + */ + sp_low = frame->fp; + ctrl.sp_high = ALIGN(frame->fp, THREAD_SIZE); + } + while (ctrl.entries > 0) { int urc; if ((ctrl.sp_high - ctrl.vrs[SP]) < sizeof(ctrl.vrs)) @@ -459,7 +465,7 @@ int unwind_frame(struct stackframe *frame) urc = unwind_exec_insn(&ctrl); if (urc < 0) return urc; - if (ctrl.vrs[SP] < ctrl.sp_low || ctrl.vrs[SP] > ctrl.sp_high) + if (ctrl.vrs[SP] < sp_low || ctrl.vrs[SP] > ctrl.sp_high) return -URC_FAILURE; } From 234a0f202a09a6144fd3c17ac6d018bdab9780bb Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 9 Mar 2022 15:40:47 +0100 Subject: [PATCH 62/62] ARM: fix building NOMMU ARMv4/v5 kernels The removal of the old-style irq entry broke obscure NOMMU configurations on machines that have an MMU: ld.lld: error: undefined symbol: generic_handle_arch_irq referenced by kernel/entry-armv.o:(__irq_svc) in archive arch/arm/built-in.a A follow-up patch to convert nvic to the generic_handle_arch_irq() could have fixed this by removing the Kconfig conditional, but did it differently. Change the Kconfig logic so ARM machines now unconditionally enable the feature. I have also submitted a patch to remove support for the configurations that broke, but fixing the regression first is a trivial and correct change. Reported-by: kernel test robot Fixes: 54f481a2308e ("ARM: remove old-style irq entry") Fixes: 52d240871760 ("irqchip: nvic: Use GENERIC_IRQ_MULTI_HANDLER") Signed-off-by: Arnd Bergmann Signed-off-by: Russell King (Oracle) --- arch/arm/Kconfig | 2 +- drivers/irqchip/Kconfig | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 5c9a8d3362cd..4b3ac41c6f7a 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -58,7 +58,7 @@ config ARM select GENERIC_CPU_AUTOPROBE select GENERIC_EARLY_IOREMAP select GENERIC_IDLE_POLL_SETUP - select GENERIC_IRQ_MULTI_HANDLER if MMU + select GENERIC_IRQ_MULTI_HANDLER select GENERIC_IRQ_PROBE select GENERIC_IRQ_SHOW select GENERIC_IRQ_SHOW_LEVEL diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 488eaa14d3a7..7038957f4a77 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -58,7 +58,6 @@ config ARM_NVIC bool select IRQ_DOMAIN_HIERARCHY select GENERIC_IRQ_CHIP - select GENERIC_IRQ_MULTI_HANDLER config ARM_VIC bool