2019-05-27 08:55:01 +02:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
2005-09-07 15:27:09 -05:00
|
|
|
#ifndef _ASM_POWERPC_MODULE_H
|
|
|
|
|
#define _ASM_POWERPC_MODULE_H
|
2005-12-16 22:43:46 +01:00
|
|
|
#ifdef __KERNEL__
|
2005-09-01 15:51:52 -05:00
|
|
|
|
|
|
|
|
#include <linux/list.h>
|
|
|
|
|
#include <asm/bug.h>
|
2012-09-28 14:31:03 +09:30
|
|
|
#include <asm-generic/module.h>
|
2005-09-01 15:51:52 -05:00
|
|
|
|
|
|
|
|
#ifndef __powerpc64__
|
|
|
|
|
/*
|
|
|
|
|
* Thanks to Paul M for explaining this.
|
|
|
|
|
*
|
|
|
|
|
* PPC can only do rel jumps += 32MB, and often the kernel and other
|
2016-02-24 10:51:11 -08:00
|
|
|
* modules are further away than this. So, we jump to a table of
|
2005-09-01 15:51:52 -05:00
|
|
|
* trampolines attached to the module (the Procedure Linkage Table)
|
|
|
|
|
* whenever that happens.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
struct ppc_plt_entry {
|
|
|
|
|
/* 16 byte jump instruction sequence (4 instructions) */
|
|
|
|
|
unsigned int jump[4];
|
|
|
|
|
};
|
|
|
|
|
#endif /* __powerpc64__ */
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
struct mod_arch_specific {
|
|
|
|
|
#ifdef __powerpc64__
|
|
|
|
|
unsigned int stubs_section; /* Index of stubs section in module */
|
2023-04-08 12:17:52 +10:00
|
|
|
#ifdef CONFIG_PPC_KERNEL_PCREL
|
|
|
|
|
unsigned int got_section; /* What section is the GOT? */
|
|
|
|
|
unsigned int pcpu_section; /* .data..percpu section */
|
|
|
|
|
#else
|
2005-09-01 15:51:52 -05:00
|
|
|
unsigned int toc_section; /* What section is the TOC? */
|
2014-03-18 19:59:26 +10:30
|
|
|
bool toc_fixed; /* Have we fixed up .TOC.? */
|
2023-04-08 12:17:52 +10:00
|
|
|
#endif
|
2008-11-14 20:47:03 -08:00
|
|
|
|
2024-08-12 16:33:12 +10:00
|
|
|
#ifdef CONFIG_PPC64_ELF_ABI_V1
|
2017-11-10 08:48:27 +09:00
|
|
|
/* For module function descriptor dereference */
|
|
|
|
|
unsigned long start_opd;
|
|
|
|
|
unsigned long end_opd;
|
2024-08-12 16:33:12 +10:00
|
|
|
#endif
|
2008-11-15 02:39:05 -05:00
|
|
|
#else /* powerpc64 */
|
2005-09-01 15:51:52 -05:00
|
|
|
/* Indices of PLT sections within module. */
|
|
|
|
|
unsigned int core_plt_section;
|
|
|
|
|
unsigned int init_plt_section;
|
2018-05-24 22:48:34 -05:00
|
|
|
#endif /* powerpc64 */
|
|
|
|
|
|
2008-11-15 02:39:05 -05:00
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE
|
|
|
|
|
unsigned long tramp;
|
2018-05-24 22:48:34 -05:00
|
|
|
unsigned long tramp_regs;
|
powerpc64/ftrace: Move ftrace sequence out of line
Function profile sequence on powerpc includes two instructions at the
beginning of each function:
mflr r0
bl ftrace_caller
The call to ftrace_caller() gets nop'ed out during kernel boot and is
patched in when ftrace is enabled.
Given the sequence, we cannot return from ftrace_caller with 'blr' as we
need to keep LR and r0 intact. This results in link stack (return
address predictor) imbalance when ftrace is enabled. To address that, we
would like to use a three instruction sequence:
mflr r0
bl ftrace_caller
mtlr r0
Further more, to support DYNAMIC_FTRACE_WITH_CALL_OPS, we need to
reserve two instruction slots before the function. This results in a
total of five instruction slots to be reserved for ftrace use on each
function that is traced.
Move the function profile sequence out-of-line to minimize its impact.
To do this, we reserve a single nop at function entry using
-fpatchable-function-entry=1 and add a pass on vmlinux.o to determine
the total number of functions that can be traced. This is then used to
generate a .S file reserving the appropriate amount of space for use as
ftrace stubs, which is built and linked into vmlinux.
On bootup, the stub space is split into separate stubs per function and
populated with the proper instruction sequence. A pointer to the
associated stub is maintained in dyn_arch_ftrace.
For modules, space for ftrace stubs is reserved from the generic module
stub space.
This is restricted to and enabled by default only on 64-bit powerpc,
though there are some changes to accommodate 32-bit powerpc. This is
done so that 32-bit powerpc could choose to opt into this based on
further tests and benchmarks.
As an example, after this patch, kernel functions will have a single nop
at function entry:
<kernel_clone>:
addis r2,r12,467
addi r2,r2,-16028
nop
mfocrf r11,8
...
When ftrace is enabled, the nop is converted to an unconditional branch
to the stub associated with that function:
<kernel_clone>:
addis r2,r12,467
addi r2,r2,-16028
b ftrace_ool_stub_text_end+0x11b28
mfocrf r11,8
...
The associated stub:
<ftrace_ool_stub_text_end+0x11b28>:
mflr r0
bl ftrace_caller
mtlr r0
b kernel_clone+0xc
...
This change showed an improvement of ~10% in null_syscall benchmark on a
Power 10 system with ftrace enabled.
Signed-off-by: Naveen N Rao <naveen@kernel.org>
Signed-off-by: Hari Bathini <hbathini@linux.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://patch.msgid.link/20241030070850.1361304-13-hbathini@linux.ibm.com
2024-10-30 12:38:45 +05:30
|
|
|
#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE
|
|
|
|
|
struct ftrace_ool_stub *ool_stubs;
|
|
|
|
|
unsigned int ool_stub_count;
|
|
|
|
|
unsigned int ool_stub_index;
|
|
|
|
|
#endif
|
2005-09-01 15:51:52 -05:00
|
|
|
#endif
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Select ELF headers.
|
2023-04-08 12:17:52 +10:00
|
|
|
* Make empty sections for module_frob_arch_sections to expand.
|
2005-09-01 15:51:52 -05:00
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#ifdef __powerpc64__
|
|
|
|
|
# ifdef MODULE
|
|
|
|
|
asm(".section .stubs,\"ax\",@nobits; .align 3; .previous");
|
2023-04-08 12:17:52 +10:00
|
|
|
# ifdef CONFIG_PPC_KERNEL_PCREL
|
|
|
|
|
asm(".section .mygot,\"a\",@nobits; .align 3; .previous");
|
|
|
|
|
# endif
|
2005-09-01 15:51:52 -05:00
|
|
|
# endif
|
|
|
|
|
#else
|
|
|
|
|
# ifdef MODULE
|
|
|
|
|
asm(".section .plt,\"ax\",@nobits; .align 3; .previous");
|
|
|
|
|
asm(".section .init.plt,\"ax\",@nobits; .align 3; .previous");
|
|
|
|
|
# endif /* MODULE */
|
|
|
|
|
#endif
|
|
|
|
|
|
2008-11-14 20:47:03 -08:00
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE
|
powerpc/module: Mark module stubs with a magic value
When a module is loaded, calls out to the kernel go via a stub which is
generated at runtime. One of these stubs is used to call _mcount(),
which is the default target of tracing calls generated by the compiler
with -pg.
If dynamic ftrace is enabled (which it typically is), another stub is
used to call ftrace_caller(), which is the target of tracing calls when
ftrace is actually active.
ftrace then wants to disable the calls to _mcount() at module startup,
and enable/disable the calls to ftrace_caller() when enabling/disabling
tracing - all of these it does by patching the code.
As part of that code patching, the ftrace code wants to confirm that the
branch it is about to modify, is in fact a call to a module stub which
calls _mcount() or ftrace_caller().
Currently it does that by inspecting the instructions and confirming
they are what it expects. Although that works, the code to do it is
pretty intricate because it requires lots of knowledge about the exact
format of the stub.
We can make that process easier by marking the generated stubs with a
magic value, and then looking for that magic value. Altough this is not
as rigorous as the current method, I believe it is sufficient in
practice.
Reviewed-by: Balbir Singh <bsingharora@gmail.com>
Reviewed-by: Torsten Duwe <duwe@suse.de>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
2016-03-03 15:26:55 +11:00
|
|
|
int module_trampoline_target(struct module *mod, unsigned long trampoline,
|
2014-04-04 15:58:42 +11:00
|
|
|
unsigned long *target);
|
2016-03-03 15:26:54 +11:00
|
|
|
int module_finalize_ftrace(struct module *mod, const Elf_Shdr *sechdrs);
|
|
|
|
|
#else
|
|
|
|
|
static inline int module_finalize_ftrace(struct module *mod, const Elf_Shdr *sechdrs)
|
|
|
|
|
{
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
2005-12-16 22:43:46 +01:00
|
|
|
#endif /* __KERNEL__ */
|
2005-09-07 15:27:09 -05:00
|
|
|
#endif /* _ASM_POWERPC_MODULE_H */
|