| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <CAAhV-H6vv8=tax30iYR3zYJ5xun9CmQDAEqdJV4js0m1JbvsKA@mail.gmail.com>
Date: Sat, 2 Dec 2023 21:45:20 +0800
From: Huacai Chen <chenhuacai@...nel.org>
To: Tiezhu Yang <yangtiezhu@...ngson.cn>
Cc: Josh Poimboeuf <jpoimboe@...nel.org>,
Peter Zijlstra <peterz@...radead.org>,
loongarch@...ts.linux.dev, linux-kernel@...r.kernel.org
Subject: Re: [PATCH v5 8/8] LoongArch: Add ORC stack unwinder support
Hi, Tiezhu,
On Wed, Nov 29, 2023 at 9:07 PM Tiezhu Yang <yangtiezhu@...ngson.cn> wrote:
>
> The kernel CONFIG_UNWINDER_ORC option enables the ORC unwinder, which is
> similar in concept to a DWARF unwinder. The difference is that the format
> of the ORC data is much simpler than DWARF, which in turn allows the ORC
> unwinder to be much simpler and faster.
>
> The ORC data consists of unwind tables which are generated by objtool.
> After analyzing all the code paths of a .o file, it determines information
> about the stack state at each instruction address in the file and outputs
> that information to the .orc_unwind and .orc_unwind_ip sections.
>
> The per-object ORC sections are combined at link time and are sorted and
> post-processed at boot time. The unwinder uses the resulting data to
> correlate instruction addresses with their stack states at run time.
>
> Most of the logic are similar with x86, in order to get ra info before ra
> is saved into stack, add ra_reg and ra_offset into orc_entry. At the same
> time, modify some arch-specific code to silence the objtool warnings.
>
> Co-developed-by: Jinyang He <hejinyang@...ngson.cn>
> Signed-off-by: Jinyang He <hejinyang@...ngson.cn>
> Co-developed-by: Youling Tang <tangyouling@...ngson.cn>
> Signed-off-by: Youling Tang <tangyouling@...ngson.cn>
> Signed-off-by: Tiezhu Yang <yangtiezhu@...ngson.cn>
> ---
> arch/loongarch/Kconfig | 2 +
> arch/loongarch/Kconfig.debug | 11 +
> arch/loongarch/Makefile | 19 +
> arch/loongarch/include/asm/Kbuild | 2 +
> arch/loongarch/include/asm/bug.h | 1 +
> arch/loongarch/include/asm/exception.h | 2 +
> arch/loongarch/include/asm/module.h | 7 +
> arch/loongarch/include/asm/orc_header.h | 18 +
> arch/loongarch/include/asm/orc_lookup.h | 31 ++
> arch/loongarch/include/asm/orc_types.h | 58 +++
> arch/loongarch/include/asm/stackframe.h | 3 +
> arch/loongarch/include/asm/unwind.h | 19 +-
> arch/loongarch/include/asm/unwind_hints.h | 28 ++
> arch/loongarch/kernel/Makefile | 4 +
> arch/loongarch/kernel/entry.S | 6 +-
> arch/loongarch/kernel/fpu.S | 7 +
> arch/loongarch/kernel/genex.S | 6 +-
> arch/loongarch/kernel/lbt.S | 5 +
> arch/loongarch/kernel/module.c | 22 +-
> arch/loongarch/kernel/relocate_kernel.S | 2 +
> arch/loongarch/kernel/setup.c | 2 +
> arch/loongarch/kernel/stacktrace.c | 1 +
> arch/loongarch/kernel/traps.c | 42 +-
> arch/loongarch/kernel/unwind_orc.c | 516 ++++++++++++++++++++++
> arch/loongarch/kernel/vmlinux.lds.S | 3 +
> arch/loongarch/kvm/switch.S | 7 +-
> arch/loongarch/lib/Makefile | 2 +
> arch/loongarch/mm/tlb.c | 27 +-
> arch/loongarch/mm/tlbex.S | 9 +
> arch/loongarch/vdso/Makefile | 1 +
> include/linux/compiler.h | 9 +
> scripts/Makefile | 7 +-
> 32 files changed, 839 insertions(+), 40 deletions(-)
> create mode 100644 arch/loongarch/include/asm/orc_header.h
> create mode 100644 arch/loongarch/include/asm/orc_lookup.h
> create mode 100644 arch/loongarch/include/asm/orc_types.h
> create mode 100644 arch/loongarch/include/asm/unwind_hints.h
> create mode 100644 arch/loongarch/kernel/unwind_orc.c
>
> diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig
> index ee123820a476..eea57378646d 100644
> --- a/arch/loongarch/Kconfig
> +++ b/arch/loongarch/Kconfig
> @@ -132,6 +132,7 @@ config LOONGARCH
> select HAVE_KVM
> select HAVE_MOD_ARCH_SPECIFIC
> select HAVE_NMI
> + select HAVE_OBJTOOL if AS_HAS_EXPLICIT_RELOCS
> select HAVE_PCI
> select HAVE_PERF_EVENTS
> select HAVE_PERF_REGS
> @@ -143,6 +144,7 @@ config LOONGARCH
> select HAVE_SAMPLE_FTRACE_DIRECT
> select HAVE_SAMPLE_FTRACE_DIRECT_MULTI
> select HAVE_SETUP_PER_CPU_AREA if NUMA
> + select HAVE_STACK_VALIDATION if HAVE_OBJTOOL
> select HAVE_STACKPROTECTOR
> select HAVE_SYSCALL_TRACEPOINTS
> select HAVE_TIF_NOHZ
> diff --git a/arch/loongarch/Kconfig.debug b/arch/loongarch/Kconfig.debug
> index 8d36aab53008..98d60630c3d4 100644
> --- a/arch/loongarch/Kconfig.debug
> +++ b/arch/loongarch/Kconfig.debug
> @@ -26,4 +26,15 @@ config UNWINDER_PROLOGUE
> Some of the addresses it reports may be incorrect (but better than the
> Guess unwinder).
>
> +config UNWINDER_ORC
> + bool "ORC unwinder"
> + select OBJTOOL
> + help
> + This option enables the ORC (Oops Rewind Capability) unwinder for
> + unwinding kernel stack traces. It uses a custom data format which is
> + a simplified version of the DWARF Call Frame Information standard.
> +
> + Enabling this option will increase the kernel's runtime memory usage
> + by roughly 2-4MB, depending on your kernel config.
> +
> endchoice
> diff --git a/arch/loongarch/Makefile b/arch/loongarch/Makefile
> index 204b94b2e6aa..1f0e41f8ab61 100644
> --- a/arch/loongarch/Makefile
> +++ b/arch/loongarch/Makefile
> @@ -25,6 +25,18 @@ endif
> 32bit-emul = elf32loongarch
> 64bit-emul = elf64loongarch
>
> +ifdef CONFIG_UNWINDER_ORC
> +orc_hash_h := arch/$(SRCARCH)/include/generated/asm/orc_hash.h
> +orc_hash_sh := $(srctree)/scripts/orc_hash.sh
> +targets += $(orc_hash_h)
> +quiet_cmd_orc_hash = GEN $@
> + cmd_orc_hash = mkdir -p $(dir $@); \
> + $(CONFIG_SHELL) $(orc_hash_sh) < $< > $@
> +$(orc_hash_h): $(srctree)/arch/loongarch/include/asm/orc_types.h $(orc_hash_sh) FORCE
> + $(call if_changed,orc_hash)
> +archprepare: $(orc_hash_h)
> +endif
> +
> ifdef CONFIG_DYNAMIC_FTRACE
> KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY
> CC_FLAGS_FTRACE := -fpatchable-function-entry=2
> @@ -81,6 +93,13 @@ KBUILD_AFLAGS_MODULE += -Wa,-mla-global-with-abs
> KBUILD_CFLAGS_MODULE += -fplt -Wa,-mla-global-with-abs,-mla-local-with-abs
> endif
>
> +KBUILD_AFLAGS += $(call cc-option,-mthin-add-sub) $(call cc-option,-Wa$(comma)-mthin-add-sub)
> +KBUILD_CFLAGS += $(call cc-option,-mthin-add-sub) $(call cc-option,-Wa$(comma)-mthin-add-sub)
> +
> +ifdef CONFIG_OBJTOOL
> +KBUILD_CFLAGS += -fno-optimize-sibling-calls -fno-jump-tables -falign-functions=4 -falign-labels=4
> +endif
> +
> ifeq ($(CONFIG_RELOCATABLE),y)
> KBUILD_CFLAGS_KERNEL += -fPIE
> LDFLAGS_vmlinux += -static -pie --no-dynamic-linker -z notext
> diff --git a/arch/loongarch/include/asm/Kbuild b/arch/loongarch/include/asm/Kbuild
> index 93783fa24f6e..a97c0edbb866 100644
> --- a/arch/loongarch/include/asm/Kbuild
> +++ b/arch/loongarch/include/asm/Kbuild
> @@ -1,4 +1,6 @@
> # SPDX-License-Identifier: GPL-2.0
> +generated-y += orc_hash.h
> +
> generic-y += dma-contiguous.h
> generic-y += mcs_spinlock.h
> generic-y += parport.h
> diff --git a/arch/loongarch/include/asm/bug.h b/arch/loongarch/include/asm/bug.h
> index d4ca3ba25418..08388876ade4 100644
> --- a/arch/loongarch/include/asm/bug.h
> +++ b/arch/loongarch/include/asm/bug.h
> @@ -44,6 +44,7 @@
> do { \
> instrumentation_begin(); \
> __BUG_FLAGS(BUGFLAG_WARNING|(flags)); \
> + annotate_reachable(); \
> instrumentation_end(); \
> } while (0)
>
> diff --git a/arch/loongarch/include/asm/exception.h b/arch/loongarch/include/asm/exception.h
> index af74a3fdcad1..c6d20736fd92 100644
> --- a/arch/loongarch/include/asm/exception.h
> +++ b/arch/loongarch/include/asm/exception.h
> @@ -6,6 +6,8 @@
> #include <asm/ptrace.h>
> #include <linux/kprobes.h>
>
> +extern void *exception_table[];
> +
> void show_registers(struct pt_regs *regs);
>
> asmlinkage void cache_parity_error(void);
> diff --git a/arch/loongarch/include/asm/module.h b/arch/loongarch/include/asm/module.h
> index 2ecd82bb64e1..f33f3fd32ecc 100644
> --- a/arch/loongarch/include/asm/module.h
> +++ b/arch/loongarch/include/asm/module.h
> @@ -6,6 +6,7 @@
> #define _ASM_MODULE_H
>
> #include <asm/inst.h>
> +#include <asm/orc_types.h>
> #include <asm-generic/module.h>
>
> #define RELA_STACK_DEPTH 16
> @@ -21,6 +22,12 @@ struct mod_arch_specific {
> struct mod_section plt;
> struct mod_section plt_idx;
>
> +#ifdef CONFIG_UNWINDER_ORC
> + unsigned int num_orcs;
> + int *orc_unwind_ip;
> + struct orc_entry *orc_unwind;
> +#endif
> +
> /* For CONFIG_DYNAMIC_FTRACE */
> struct plt_entry *ftrace_trampolines;
> };
> diff --git a/arch/loongarch/include/asm/orc_header.h b/arch/loongarch/include/asm/orc_header.h
> new file mode 100644
> index 000000000000..f9d509c3fd70
> --- /dev/null
> +++ b/arch/loongarch/include/asm/orc_header.h
> @@ -0,0 +1,18 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +
> +#ifndef _ORC_HEADER_H
> +#define _ORC_HEADER_H
> +
> +#include <linux/types.h>
> +#include <linux/compiler.h>
> +#include <asm/orc_hash.h>
> +
> +/*
> + * The header is currently a 20-byte hash of the ORC entry definition; see
> + * scripts/orc_hash.sh.
> + */
> +#define ORC_HEADER \
> + __used __section(".orc_header") __aligned(4) \
> + static const u8 orc_header[] = { ORC_HASH }
> +
> +#endif /* _ORC_HEADER_H */
> diff --git a/arch/loongarch/include/asm/orc_lookup.h b/arch/loongarch/include/asm/orc_lookup.h
> new file mode 100644
> index 000000000000..b02e6357def4
> --- /dev/null
> +++ b/arch/loongarch/include/asm/orc_lookup.h
> @@ -0,0 +1,31 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#ifndef _ORC_LOOKUP_H
> +#define _ORC_LOOKUP_H
> +
> +/*
> + * This is a lookup table for speeding up access to the .orc_unwind table.
> + * Given an input address offset, the corresponding lookup table entry
> + * specifies a subset of the .orc_unwind table to search.
> + *
> + * Each block represents the end of the previous range and the start of the
> + * next range. An extra block is added to give the last range an end.
> + *
> + * The block size should be a power of 2 to avoid a costly 'div' instruction.
> + *
> + * A block size of 256 was chosen because it roughly doubles unwinder
> + * performance while only adding ~5% to the ORC data footprint.
> + */
> +#define LOOKUP_BLOCK_ORDER 8
> +#define LOOKUP_BLOCK_SIZE (1 << LOOKUP_BLOCK_ORDER)
> +
> +#ifndef LINKER_SCRIPT
> +
> +extern unsigned int orc_lookup[];
> +extern unsigned int orc_lookup_end[];
> +
> +#define LOOKUP_START_IP (unsigned long)_stext
> +#define LOOKUP_STOP_IP (unsigned long)_etext
> +
> +#endif /* LINKER_SCRIPT */
> +
> +#endif /* _ORC_LOOKUP_H */
> diff --git a/arch/loongarch/include/asm/orc_types.h b/arch/loongarch/include/asm/orc_types.h
> new file mode 100644
> index 000000000000..caf1f71a1057
> --- /dev/null
> +++ b/arch/loongarch/include/asm/orc_types.h
> @@ -0,0 +1,58 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#ifndef _ORC_TYPES_H
> +#define _ORC_TYPES_H
> +
> +#include <linux/types.h>
> +
> +/*
> + * The ORC_REG_* registers are base registers which are used to find other
> + * registers on the stack.
> + *
> + * ORC_REG_PREV_SP, also known as DWARF Call Frame Address (CFA), is the
> + * address of the previous frame: the caller's SP before it called the current
> + * function.
> + *
> + * ORC_REG_UNDEFINED means the corresponding register's value didn't change in
> + * the current frame.
> + *
> + * The most commonly used base registers are SP and FP -- which the previous SP
> + * is usually based on -- and PREV_SP and UNDEFINED -- which the previous FP is
> + * usually based on.
> + *
> + * The rest of the base registers are needed for special cases like entry code
> + * and GCC realigned stacks.
> + */
> +#define ORC_REG_UNDEFINED 0
> +#define ORC_REG_PREV_SP 1
> +#define ORC_REG_SP 2
> +#define ORC_REG_FP 3
> +#define ORC_REG_MAX 4
> +
> +#define ORC_TYPE_UNDEFINED 0
> +#define ORC_TYPE_END_OF_STACK 1
> +#define ORC_TYPE_CALL 2
> +#define ORC_TYPE_REGS 3
> +#define ORC_TYPE_REGS_PARTIAL 4
> +
> +#ifndef __ASSEMBLY__
> +/*
> + * This struct is more or less a vastly simplified version of the DWARF Call
> + * Frame Information standard. It contains only the necessary parts of DWARF
> + * CFI, simplified for ease of access by the in-kernel unwinder. It tells the
> + * unwinder how to find the previous SP and FP (and sometimes entry regs) on
> + * the stack for a given code address. Each instance of the struct corresponds
> + * to one or more code locations.
> + */
> +struct orc_entry {
> + s16 sp_offset;
> + s16 fp_offset;
> + s16 ra_offset;
> + unsigned int sp_reg:4;
> + unsigned int fp_reg:4;
> + unsigned int ra_reg:4;
> + unsigned int type:3;
> + unsigned int signal:1;
> +};
> +#endif /* __ASSEMBLY__ */
> +
> +#endif /* _ORC_TYPES_H */
> diff --git a/arch/loongarch/include/asm/stackframe.h b/arch/loongarch/include/asm/stackframe.h
> index 4fb1e6408b98..45b507a7b06f 100644
> --- a/arch/loongarch/include/asm/stackframe.h
> +++ b/arch/loongarch/include/asm/stackframe.h
> @@ -13,6 +13,7 @@
> #include <asm/asm-offsets.h>
> #include <asm/loongarch.h>
> #include <asm/thread_info.h>
> +#include <asm/unwind_hints.h>
>
> /* Make the addition of cfi info a little easier. */
> .macro cfi_rel_offset reg offset=0 docfi=0
> @@ -162,6 +163,7 @@
> li.w t0, CSR_CRMD_WE
> csrxchg t0, t0, LOONGARCH_CSR_CRMD
> #endif
> + UNWIND_HINT_REGS
> .endm
>
> .macro SAVE_ALL docfi=0
> @@ -219,6 +221,7 @@
>
> .macro RESTORE_SP_AND_RET docfi=0
> cfi_ld sp, PT_R3, \docfi
> + UNWIND_HINT_FUNC
> ertn
> .endm
>
> diff --git a/arch/loongarch/include/asm/unwind.h b/arch/loongarch/include/asm/unwind.h
> index b9dce87afd2e..c7f52d406cce 100644
> --- a/arch/loongarch/include/asm/unwind.h
> +++ b/arch/loongarch/include/asm/unwind.h
> @@ -16,6 +16,7 @@
> enum unwinder_type {
> UNWINDER_GUESS,
> UNWINDER_PROLOGUE,
> + UNWINDER_ORC,
> };
>
> struct unwind_state {
> @@ -24,7 +25,7 @@ struct unwind_state {
> struct task_struct *task;
> bool first, error, reset;
> int graph_idx;
> - unsigned long sp, pc, ra;
> + unsigned long sp, fp, pc, ra;
> };
>
> bool default_next_frame(struct unwind_state *state);
> @@ -34,6 +35,14 @@ void unwind_start(struct unwind_state *state,
> bool unwind_next_frame(struct unwind_state *state);
> unsigned long unwind_get_return_address(struct unwind_state *state);
>
> +#ifdef CONFIG_UNWINDER_ORC
> +void unwind_init(void);
> +void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size, void *orc, size_t orc_size);
> +#else
> +static inline void unwind_init(void) {}
> +static inline void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size, void *orc, size_t orc_size) {}
> +#endif
> +
> static inline bool unwind_done(struct unwind_state *state)
> {
> return state->stack_info.type == STACK_TYPE_UNKNOWN;
> @@ -61,14 +70,17 @@ static __always_inline void __unwind_start(struct unwind_state *state,
> state->sp = regs->regs[3];
> state->pc = regs->csr_era;
> state->ra = regs->regs[1];
> + state->fp = regs->regs[22];
> } else if (task && task != current) {
> state->sp = thread_saved_fp(task);
> state->pc = thread_saved_ra(task);
> state->ra = 0;
> + state->fp = 0;
> } else {
> state->sp = (unsigned long)__builtin_frame_address(0);
> state->pc = (unsigned long)__builtin_return_address(0);
> state->ra = 0;
> + state->fp = 0;
> }
> state->task = task;
> get_stack_info(state->sp, state->task, &state->stack_info);
> @@ -77,6 +89,9 @@ static __always_inline void __unwind_start(struct unwind_state *state,
>
> static __always_inline unsigned long __unwind_get_return_address(struct unwind_state *state)
> {
> - return unwind_done(state) ? 0 : state->pc;
> + if (unwind_done(state))
> + return 0;
> +
> + return __kernel_text_address(state->pc) ? state->pc : 0;
> }
> #endif /* _ASM_UNWIND_H */
> diff --git a/arch/loongarch/include/asm/unwind_hints.h b/arch/loongarch/include/asm/unwind_hints.h
> new file mode 100644
> index 000000000000..82443fed3167
> --- /dev/null
> +++ b/arch/loongarch/include/asm/unwind_hints.h
> @@ -0,0 +1,28 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#ifndef _ASM_LOONGARCH_UNWIND_HINTS_H
> +#define _ASM_LOONGARCH_UNWIND_HINTS_H
> +
> +#include <linux/objtool.h>
> +#include <asm/orc_types.h>
> +
> +#ifdef __ASSEMBLY__
> +
> +.macro UNWIND_HINT_UNDEFINED
> + UNWIND_HINT type=UNWIND_HINT_TYPE_UNDEFINED
> +.endm
> +
> +.macro UNWIND_HINT_EMPTY
> + UNWIND_HINT sp_reg=ORC_REG_UNDEFINED type=UNWIND_HINT_TYPE_CALL
> +.endm
> +
> +.macro UNWIND_HINT_REGS
> + UNWIND_HINT sp_reg=ORC_REG_SP type=UNWIND_HINT_TYPE_REGS
> +.endm
> +
> +.macro UNWIND_HINT_FUNC
> + UNWIND_HINT sp_reg=ORC_REG_SP type=UNWIND_HINT_TYPE_CALL
> +.endm
> +
> +#endif /* __ASSEMBLY__ */
> +
> +#endif /* _ASM_LOONGARCH_UNWIND_HINTS_H */
> diff --git a/arch/loongarch/kernel/Makefile b/arch/loongarch/kernel/Makefile
> index 4fcc168f0732..ac47e11c227e 100644
> --- a/arch/loongarch/kernel/Makefile
> +++ b/arch/loongarch/kernel/Makefile
> @@ -3,6 +3,8 @@
> # Makefile for the Linux/LoongArch kernel.
> #
>
> +OBJECT_FILES_NON_STANDARD_head.o := y
> +
> extra-y := vmlinux.lds
>
> obj-y += head.o cpu-probe.o cacheinfo.o env.o setup.o entry.o genex.o \
> @@ -21,6 +23,7 @@ obj-$(CONFIG_ARCH_STRICT_ALIGN) += unaligned.o
>
> CFLAGS_module.o += $(call cc-option,-Wno-override-init,)
> CFLAGS_syscall.o += $(call cc-option,-Wno-override-init,)
> +CFLAGS_traps.o += $(call cc-option,-Wno-override-init,)
> CFLAGS_perf_event.o += $(call cc-option,-Wno-override-init,)
>
> ifdef CONFIG_FUNCTION_TRACER
> @@ -62,6 +65,7 @@ obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
>
> obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o
> obj-$(CONFIG_UNWINDER_PROLOGUE) += unwind_prologue.o
> +obj-$(CONFIG_UNWINDER_ORC) += unwind_orc.o
>
> obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_regs.o
> obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
> diff --git a/arch/loongarch/kernel/entry.S b/arch/loongarch/kernel/entry.S
> index 1ec8e4c4cc2b..2f5c74d26d5f 100644
> --- a/arch/loongarch/kernel/entry.S
> +++ b/arch/loongarch/kernel/entry.S
> @@ -14,11 +14,12 @@
> #include <asm/regdef.h>
> #include <asm/stackframe.h>
> #include <asm/thread_info.h>
> +#include <asm/unwind_hints.h>
>
> .text
> .cfi_sections .debug_frame
> - .align 5
> SYM_CODE_START(handle_syscall)
> + UNWIND_HINT_UNDEFINED
> csrrd t0, PERCPU_BASE_KS
> la.pcrel t1, kernelsp
> add.d t1, t1, t0
> @@ -57,6 +58,7 @@ SYM_CODE_START(handle_syscall)
> cfi_st fp, PT_R22
>
> SAVE_STATIC
> + UNWIND_HINT_REGS
>
> #ifdef CONFIG_KGDB
> li.w t1, CSR_CRMD_WE
> @@ -75,6 +77,7 @@ SYM_CODE_END(handle_syscall)
> _ASM_NOKPROBE(handle_syscall)
>
> SYM_CODE_START(ret_from_fork)
> + UNWIND_HINT_REGS
> bl schedule_tail # a0 = struct task_struct *prev
> move a0, sp
> bl syscall_exit_to_user_mode
> @@ -84,6 +87,7 @@ SYM_CODE_START(ret_from_fork)
> SYM_CODE_END(ret_from_fork)
>
> SYM_CODE_START(ret_from_kernel_thread)
> + UNWIND_HINT_REGS
> bl schedule_tail # a0 = struct task_struct *prev
> move a0, s1
> jirl ra, s0, 0
> diff --git a/arch/loongarch/kernel/fpu.S b/arch/loongarch/kernel/fpu.S
> index d53ab10f4644..487933899c5d 100644
> --- a/arch/loongarch/kernel/fpu.S
> +++ b/arch/loongarch/kernel/fpu.S
> @@ -15,6 +15,7 @@
> #include <asm/fpregdef.h>
> #include <asm/loongarch.h>
> #include <asm/regdef.h>
> +#include <asm/unwind_hints.h>
>
> #define FPU_REG_WIDTH 8
> #define LSX_REG_WIDTH 16
> @@ -524,3 +525,9 @@ SYM_FUNC_END(_restore_lasx_context)
> .L_fpu_fault:
> li.w a0, -EFAULT # failure
> jr ra
> +
> +#ifdef CONFIG_CPU_HAS_LBT
> +STACK_FRAME_NON_STANDARD _restore_fp
> +STACK_FRAME_NON_STANDARD _restore_lsx
> +STACK_FRAME_NON_STANDARD _restore_lasx
> +#endif
> diff --git a/arch/loongarch/kernel/genex.S b/arch/loongarch/kernel/genex.S
> index 2bb3aa2dcfcb..6b7ec9f1134d 100644
> --- a/arch/loongarch/kernel/genex.S
> +++ b/arch/loongarch/kernel/genex.S
> @@ -16,7 +16,6 @@
> #include <asm/stackframe.h>
> #include <asm/thread_info.h>
>
> - .align 5
> SYM_FUNC_START(__arch_cpu_idle)
> /* start of rollback region */
> LONG_L t0, tp, TI_FLAGS
> @@ -32,6 +31,7 @@ SYM_FUNC_START(__arch_cpu_idle)
> SYM_FUNC_END(__arch_cpu_idle)
>
> SYM_CODE_START(handle_vint)
> + UNWIND_HINT_UNDEFINED
> BACKUP_T0T1
> SAVE_ALL
> la_abs t1, __arch_cpu_idle
> @@ -49,6 +49,7 @@ SYM_CODE_START(handle_vint)
> SYM_CODE_END(handle_vint)
>
> SYM_CODE_START(except_vec_cex)
> + UNWIND_HINT_UNDEFINED
> b cache_parity_error
> SYM_CODE_END(except_vec_cex)
>
> @@ -65,8 +66,8 @@ SYM_CODE_END(except_vec_cex)
> .endm
>
> .macro BUILD_HANDLER exception handler prep
> - .align 5
> SYM_CODE_START(handle_\exception)
> + UNWIND_HINT_UNDEFINED
> 666:
> BACKUP_T0T1
> SAVE_ALL
> @@ -94,6 +95,7 @@ SYM_CODE_END(except_vec_cex)
> BUILD_HANDLER reserved reserved none /* others */
>
> SYM_CODE_START(handle_sys)
> + UNWIND_HINT_UNDEFINED
> la_abs t0, handle_syscall
> jr t0
> SYM_CODE_END(handle_sys)
> diff --git a/arch/loongarch/kernel/lbt.S b/arch/loongarch/kernel/lbt.S
> index 9c75120a26d8..4d6914f84c7e 100644
> --- a/arch/loongarch/kernel/lbt.S
> +++ b/arch/loongarch/kernel/lbt.S
> @@ -11,6 +11,7 @@
> #include <asm/asm-offsets.h>
> #include <asm/errno.h>
> #include <asm/regdef.h>
> +#include <asm/unwind_hints.h>
>
> #define SCR_REG_WIDTH 8
>
> @@ -153,3 +154,7 @@ SYM_FUNC_END(_restore_ftop_context)
> .L_lbt_fault:
> li.w a0, -EFAULT # failure
> jr ra
> +
> +#ifdef CONFIG_CPU_HAS_LBT
> +STACK_FRAME_NON_STANDARD _restore_ftop_context
> +#endif
This file is only compiled if CONFIG_CPU_HAS_LBT is set, so #ifdef can
be removed.
Huacai
> diff --git a/arch/loongarch/kernel/module.c b/arch/loongarch/kernel/module.c
> index b13b2858fe39..c7d0338d12c1 100644
> --- a/arch/loongarch/kernel/module.c
> +++ b/arch/loongarch/kernel/module.c
> @@ -20,6 +20,7 @@
> #include <linux/kernel.h>
> #include <asm/alternative.h>
> #include <asm/inst.h>
> +#include <asm/unwind.h>
>
> static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top)
> {
> @@ -515,15 +516,28 @@ static void module_init_ftrace_plt(const Elf_Ehdr *hdr,
> int module_finalize(const Elf_Ehdr *hdr,
> const Elf_Shdr *sechdrs, struct module *mod)
> {
> - const Elf_Shdr *s, *se;
> const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
> + const Elf_Shdr *s, *alt = NULL, *orc = NULL, *orc_ip = NULL, *ftrace = NULL;
>
> - for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) {
> + for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
> if (!strcmp(".altinstructions", secstrs + s->sh_name))
> - apply_alternatives((void *)s->sh_addr, (void *)s->sh_addr + s->sh_size);
> + alt = s;
> + if (!strcmp(".orc_unwind", secstrs + s->sh_name))
> + orc = s;
> + if (!strcmp(".orc_unwind_ip", secstrs + s->sh_name))
> + orc_ip = s;
> if (!strcmp(".ftrace_trampoline", secstrs + s->sh_name))
> - module_init_ftrace_plt(hdr, s, mod);
> + ftrace = s;
> }
>
> + if (alt)
> + apply_alternatives((void *)alt->sh_addr, (void *)alt->sh_addr + alt->sh_size);
> +
> + if (orc && orc_ip)
> + unwind_module_init(mod, (void *)orc_ip->sh_addr, orc_ip->sh_size, (void *)orc->sh_addr, orc->sh_size);
> +
> + if (ftrace)
> + module_init_ftrace_plt(hdr, ftrace, mod);
> +
> return 0;
> }
> diff --git a/arch/loongarch/kernel/relocate_kernel.S b/arch/loongarch/kernel/relocate_kernel.S
> index f49f6b053763..bcc191d278c1 100644
> --- a/arch/loongarch/kernel/relocate_kernel.S
> +++ b/arch/loongarch/kernel/relocate_kernel.S
> @@ -15,6 +15,7 @@
> #include <asm/addrspace.h>
>
> SYM_CODE_START(relocate_new_kernel)
> + UNWIND_HINT_UNDEFINED
> /*
> * a0: EFI boot flag for the new kernel
> * a1: Command line pointer for the new kernel
> @@ -90,6 +91,7 @@ SYM_CODE_END(relocate_new_kernel)
> * then start at the entry point from LOONGARCH_IOCSR_MBUF0.
> */
> SYM_CODE_START(kexec_smp_wait)
> + UNWIND_HINT_UNDEFINED
> 1: li.w t0, 0x100 /* wait for init loop */
> 2: addi.w t0, t0, -1 /* limit mailbox access */
> bnez t0, 2b
> diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c
> index d183a745fb85..ec4459c61db6 100644
> --- a/arch/loongarch/kernel/setup.c
> +++ b/arch/loongarch/kernel/setup.c
> @@ -47,6 +47,7 @@
> #include <asm/sections.h>
> #include <asm/setup.h>
> #include <asm/time.h>
> +#include <asm/unwind.h>
>
> #define SMBIOS_BIOSSIZE_OFFSET 0x09
> #define SMBIOS_BIOSEXTERN_OFFSET 0x13
> @@ -605,6 +606,7 @@ static void __init prefill_possible_map(void)
> void __init setup_arch(char **cmdline_p)
> {
> cpu_probe();
> + unwind_init();
>
> init_environ();
> efi_init();
> diff --git a/arch/loongarch/kernel/stacktrace.c b/arch/loongarch/kernel/stacktrace.c
> index 92270f14db94..9848d427cbfa 100644
> --- a/arch/loongarch/kernel/stacktrace.c
> +++ b/arch/loongarch/kernel/stacktrace.c
> @@ -29,6 +29,7 @@ void arch_stack_walk(stack_trace_consume_fn consume_entry, void *cookie,
> regs->csr_era = thread_saved_ra(task);
> }
> regs->regs[1] = 0;
> + regs->regs[22] = 0;
> }
>
> for (unwind_start(&state, task, regs);
> diff --git a/arch/loongarch/kernel/traps.c b/arch/loongarch/kernel/traps.c
> index aebfc3733a76..f9f4eb00c92e 100644
> --- a/arch/loongarch/kernel/traps.c
> +++ b/arch/loongarch/kernel/traps.c
> @@ -53,6 +53,32 @@
>
> #include "access-helper.h"
>
> +void *exception_table[EXCCODE_INT_START] = {
> + [0 ... EXCCODE_INT_START - 1] = handle_reserved,
> +
> + [EXCCODE_TLBI] = handle_tlb_load,
> + [EXCCODE_TLBL] = handle_tlb_load,
> + [EXCCODE_TLBS] = handle_tlb_store,
> + [EXCCODE_TLBM] = handle_tlb_modify,
> + [EXCCODE_TLBNR] = handle_tlb_protect,
> + [EXCCODE_TLBNX] = handle_tlb_protect,
> + [EXCCODE_TLBPE] = handle_tlb_protect,
> + [EXCCODE_ADE] = handle_ade,
> + [EXCCODE_ALE] = handle_ale,
> + [EXCCODE_BCE] = handle_bce,
> + [EXCCODE_SYS] = handle_sys,
> + [EXCCODE_BP] = handle_bp,
> + [EXCCODE_INE] = handle_ri,
> + [EXCCODE_IPE] = handle_ri,
> + [EXCCODE_FPDIS] = handle_fpu,
> + [EXCCODE_LSXDIS] = handle_lsx,
> + [EXCCODE_LASXDIS] = handle_lasx,
> + [EXCCODE_FPE] = handle_fpe,
> + [EXCCODE_WATCH] = handle_watch,
> + [EXCCODE_BTDIS] = handle_lbt,
> +};
> +EXPORT_SYMBOL_GPL(exception_table);
> +
> static void show_backtrace(struct task_struct *task, const struct pt_regs *regs,
> const char *loglvl, bool user)
> {
> @@ -1150,19 +1176,9 @@ void __init trap_init(void)
> for (i = EXCCODE_INT_START; i <= EXCCODE_INT_END; i++)
> set_handler(i * VECSIZE, handle_vint, VECSIZE);
>
> - set_handler(EXCCODE_ADE * VECSIZE, handle_ade, VECSIZE);
> - set_handler(EXCCODE_ALE * VECSIZE, handle_ale, VECSIZE);
> - set_handler(EXCCODE_BCE * VECSIZE, handle_bce, VECSIZE);
> - set_handler(EXCCODE_SYS * VECSIZE, handle_sys, VECSIZE);
> - set_handler(EXCCODE_BP * VECSIZE, handle_bp, VECSIZE);
> - set_handler(EXCCODE_INE * VECSIZE, handle_ri, VECSIZE);
> - set_handler(EXCCODE_IPE * VECSIZE, handle_ri, VECSIZE);
> - set_handler(EXCCODE_FPDIS * VECSIZE, handle_fpu, VECSIZE);
> - set_handler(EXCCODE_LSXDIS * VECSIZE, handle_lsx, VECSIZE);
> - set_handler(EXCCODE_LASXDIS * VECSIZE, handle_lasx, VECSIZE);
> - set_handler(EXCCODE_FPE * VECSIZE, handle_fpe, VECSIZE);
> - set_handler(EXCCODE_BTDIS * VECSIZE, handle_lbt, VECSIZE);
> - set_handler(EXCCODE_WATCH * VECSIZE, handle_watch, VECSIZE);
> + /* Set exception vector handler */
> + for (i = EXCCODE_ADE; i <= EXCCODE_BTDIS; i++)
> + set_handler(i * VECSIZE, exception_table[i], VECSIZE);
>
> cache_error_setup();
>
> diff --git a/arch/loongarch/kernel/unwind_orc.c b/arch/loongarch/kernel/unwind_orc.c
> new file mode 100644
> index 000000000000..15f18d1e0dad
> --- /dev/null
> +++ b/arch/loongarch/kernel/unwind_orc.c
> @@ -0,0 +1,516 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +#include <linux/objtool.h>
> +#include <linux/module.h>
> +#include <linux/sort.h>
> +#include <asm/exception.h>
> +#include <asm/orc_header.h>
> +#include <asm/orc_lookup.h>
> +#include <asm/orc_types.h>
> +#include <asm/ptrace.h>
> +#include <asm/setup.h>
> +#include <asm/stacktrace.h>
> +#include <asm/tlb.h>
> +#include <asm/unwind.h>
> +
> +ORC_HEADER;
> +
> +#define orc_warn(fmt, ...) \
> + printk_deferred_once(KERN_WARNING "WARNING: " fmt, ##__VA_ARGS__)
> +
> +extern int __start_orc_unwind_ip[];
> +extern int __stop_orc_unwind_ip[];
> +extern struct orc_entry __start_orc_unwind[];
> +extern struct orc_entry __stop_orc_unwind[];
> +
> +static bool orc_init __ro_after_init;
> +static unsigned int lookup_num_blocks __ro_after_init;
> +
> +/* Fake frame pointer entry -- used as a fallback for generated code */
> +static struct orc_entry orc_fp_entry = {
> + .sp_reg = ORC_REG_FP,
> + .sp_offset = 16,
> + .fp_reg = ORC_REG_PREV_SP,
> + .fp_offset = -16,
> + .ra_reg = ORC_REG_PREV_SP,
> + .ra_offset = -8,
> + .type = ORC_TYPE_CALL
> +};
> +
> +/*
> + * If we crash with IP==0, the last successfully executed instruction
> + * was probably an indirect function call with a NULL function pointer,
> + * and we don't have unwind information for NULL.
> + * This hardcoded ORC entry for IP==0 allows us to unwind from a NULL function
> + * pointer into its parent and then continue normally from there.
> + */
> +static struct orc_entry orc_null_entry = {
> + .sp_reg = ORC_REG_SP,
> + .sp_offset = sizeof(long),
> + .fp_reg = ORC_REG_UNDEFINED,
> + .type = ORC_TYPE_CALL
> +};
> +
> +static inline unsigned long orc_ip(const int *ip)
> +{
> + return (unsigned long)ip + *ip;
> +}
> +
> +static struct orc_entry *__orc_find(int *ip_table, struct orc_entry *u_table,
> + unsigned int num_entries, unsigned long ip)
> +{
> + int *first = ip_table;
> + int *mid = first, *found = first;
> + int *last = ip_table + num_entries - 1;
> +
> + if (!num_entries)
> + return NULL;
> +
> + /*
> + * Do a binary range search to find the rightmost duplicate of a given
> + * starting address. Some entries are section terminators which are
> + * "weak" entries for ensuring there are no gaps. They should be
> + * ignored when they conflict with a real entry.
> + */
> + while (first <= last) {
> + mid = first + ((last - first) / 2);
> +
> + if (orc_ip(mid) <= ip) {
> + found = mid;
> + first = mid + 1;
> + } else
> + last = mid - 1;
> + }
> +
> + return u_table + (found - ip_table);
> +}
> +
> +#ifdef CONFIG_MODULES
> +static struct orc_entry *orc_module_find(unsigned long ip)
> +{
> + struct module *mod;
> +
> + mod = __module_address(ip);
> + if (!mod || !mod->arch.orc_unwind || !mod->arch.orc_unwind_ip)
> + return NULL;
> +
> + return __orc_find(mod->arch.orc_unwind_ip, mod->arch.orc_unwind, mod->arch.num_orcs, ip);
> +}
> +#else
> +static struct orc_entry *orc_module_find(unsigned long ip)
> +{
> + return NULL;
> +}
> +#endif
> +
> +#ifdef CONFIG_DYNAMIC_FTRACE
> +static struct orc_entry *orc_find(unsigned long ip);
> +
> +/*
> + * Ftrace dynamic trampolines do not have orc entries of their own.
> + * But they are copies of the ftrace entries that are static and
> + * defined in ftrace_*.S, which do have orc entries.
> + *
> + * If the unwinder comes across a ftrace trampoline, then find the
> + * ftrace function that was used to create it, and use that ftrace
> + * function's orc entry, as the placement of the return code in
> + * the stack will be identical.
> + */
> +static struct orc_entry *orc_ftrace_find(unsigned long ip)
> +{
> + struct ftrace_ops *ops;
> + unsigned long tramp_addr, offset;
> +
> + ops = ftrace_ops_trampoline(ip);
> + if (!ops)
> + return NULL;
> +
> + /* Set tramp_addr to the start of the code copied by the trampoline */
> + if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
> + tramp_addr = (unsigned long)ftrace_regs_caller;
> + else
> + tramp_addr = (unsigned long)ftrace_caller;
> +
> + /* Now place tramp_addr to the location within the trampoline ip is at */
> + offset = ip - ops->trampoline;
> + tramp_addr += offset;
> +
> + /* Prevent unlikely recursion */
> + if (ip == tramp_addr)
> + return NULL;
> +
> + return orc_find(tramp_addr);
> +}
> +#else
> +static struct orc_entry *orc_ftrace_find(unsigned long ip)
> +{
> + return NULL;
> +}
> +#endif
> +
> +static struct orc_entry *orc_find(unsigned long ip)
> +{
> + static struct orc_entry *orc;
> +
> + if (ip == 0)
> + return &orc_null_entry;
> +
> + /* For non-init vmlinux addresses, use the fast lookup table: */
> + if (ip >= LOOKUP_START_IP && ip < LOOKUP_STOP_IP) {
> + unsigned int idx, start, stop;
> +
> + idx = (ip - LOOKUP_START_IP) / LOOKUP_BLOCK_SIZE;
> +
> + if (unlikely((idx >= lookup_num_blocks-1))) {
> + orc_warn("WARNING: bad lookup idx: idx=%u num=%u ip=%pB\n",
> + idx, lookup_num_blocks, (void *)ip);
> + return NULL;
> + }
> +
> + start = orc_lookup[idx];
> + stop = orc_lookup[idx + 1] + 1;
> +
> + if (unlikely((__start_orc_unwind + start >= __stop_orc_unwind) ||
> + (__start_orc_unwind + stop > __stop_orc_unwind))) {
> + orc_warn("WARNING: bad lookup value: idx=%u num=%u start=%u stop=%u ip=%pB\n",
> + idx, lookup_num_blocks, start, stop, (void *)ip);
> + return NULL;
> + }
> +
> + return __orc_find(__start_orc_unwind_ip + start,
> + __start_orc_unwind + start, stop - start, ip);
> + }
> +
> + /* vmlinux .init slow lookup: */
> + if (is_kernel_inittext(ip))
> + return __orc_find(__start_orc_unwind_ip, __start_orc_unwind,
> + __stop_orc_unwind_ip - __start_orc_unwind_ip, ip);
> +
> + /* Module lookup: */
> + orc = orc_module_find(ip);
> + if (orc)
> + return orc;
> +
> + return orc_ftrace_find(ip);
> +}
> +
> +#ifdef CONFIG_MODULES
> +
> +static DEFINE_MUTEX(sort_mutex);
> +static int *cur_orc_ip_table = __start_orc_unwind_ip;
> +static struct orc_entry *cur_orc_table = __start_orc_unwind;
> +
> +static void orc_sort_swap(void *_a, void *_b, int size)
> +{
> + int delta = _b - _a;
> + int *a = _a, *b = _b, tmp;
> + struct orc_entry *orc_a, *orc_b;
> +
> + /* Swap the .orc_unwind_ip entries: */
> + tmp = *a;
> + *a = *b + delta;
> + *b = tmp - delta;
> +
> + /* Swap the corresponding .orc_unwind entries: */
> + orc_a = cur_orc_table + (a - cur_orc_ip_table);
> + orc_b = cur_orc_table + (b - cur_orc_ip_table);
> + swap(*orc_a, *orc_b);
> +}
> +
> +static int orc_sort_cmp(const void *_a, const void *_b)
> +{
> + const int *a = _a, *b = _b;
> + unsigned long a_val = orc_ip(a);
> + unsigned long b_val = orc_ip(b);
> + struct orc_entry *orc_a;
> +
> + if (a_val > b_val)
> + return 1;
> + if (a_val < b_val)
> + return -1;
> +
> + /*
> + * The "weak" section terminator entries need to always be first
> + * to ensure the lookup code skips them in favor of real entries.
> + * These terminator entries exist to handle any gaps created by
> + * whitelisted .o files which didn't get objtool generation.
> + */
> + orc_a = cur_orc_table + (a - cur_orc_ip_table);
> +
> + return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1;
> +}
> +
> +void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size,
> + void *_orc, size_t orc_size)
> +{
> + int *orc_ip = _orc_ip;
> + struct orc_entry *orc = _orc;
> + unsigned int num_entries = orc_ip_size / sizeof(int);
> +
> + WARN_ON_ONCE(orc_ip_size % sizeof(int) != 0 ||
> + orc_size % sizeof(*orc) != 0 ||
> + num_entries != orc_size / sizeof(*orc));
> +
> + /*
> + * The 'cur_orc_*' globals allow the orc_sort_swap() callback to
> + * associate an .orc_unwind_ip table entry with its corresponding
> + * .orc_unwind entry so they can both be swapped.
> + */
> + mutex_lock(&sort_mutex);
> + cur_orc_ip_table = orc_ip;
> + cur_orc_table = orc;
> + sort(orc_ip, num_entries, sizeof(int), orc_sort_cmp, orc_sort_swap);
> + mutex_unlock(&sort_mutex);
> +
> + mod->arch.orc_unwind_ip = orc_ip;
> + mod->arch.orc_unwind = orc;
> + mod->arch.num_orcs = num_entries;
> +}
> +#endif
> +
> +void __init unwind_init(void)
> +{
> + int i;
> + size_t orc_size = (void *)__stop_orc_unwind - (void *)__start_orc_unwind;
> + size_t orc_ip_size = (void *)__stop_orc_unwind_ip - (void *)__start_orc_unwind_ip;
> + size_t num_entries = orc_ip_size / sizeof(int);
> + struct orc_entry *orc;
> +
> + if (!num_entries || orc_ip_size % sizeof(int) != 0 ||
> + orc_size % sizeof(struct orc_entry) != 0 ||
> + num_entries != orc_size / sizeof(struct orc_entry)) {
> + orc_warn("WARNING: Bad or missing .orc_unwind table. Disabling unwinder.\n");
> + return;
> + }
> +
> + /*
> + * Note, the orc_unwind and orc_unwind_ip tables were already
> + * sorted at build time via the 'sorttable' tool.
> + * It's ready for binary search straight away, no need to sort it.
> + */
> +
> + /* Initialize the fast lookup table: */
> + lookup_num_blocks = orc_lookup_end - orc_lookup;
> + for (i = 0; i < lookup_num_blocks-1; i++) {
> + orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind,
> + num_entries, LOOKUP_START_IP + (LOOKUP_BLOCK_SIZE * i));
> + if (!orc) {
> + orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n");
> + return;
> + }
> +
> + orc_lookup[i] = orc - __start_orc_unwind;
> + }
> +
> + /* Initialize the ending block: */
> + orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind, num_entries, LOOKUP_STOP_IP);
> + if (!orc) {
> + orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n");
> + return;
> + }
> + orc_lookup[lookup_num_blocks-1] = orc - __start_orc_unwind;
> +
> + orc_init = true;
> +}
> +
> +static inline bool on_stack(struct stack_info *info, unsigned long addr, size_t len)
> +{
> + unsigned long begin = info->begin;
> + unsigned long end = info->end;
> +
> + return (info->type != STACK_TYPE_UNKNOWN &&
> + addr >= begin && addr < end && addr + len > begin && addr + len <= end);
> +}
> +
> +static bool stack_access_ok(struct unwind_state *state, unsigned long addr, size_t len)
> +{
> + struct stack_info *info = &state->stack_info;
> +
> + if (on_stack(info, addr, len))
> + return true;
> +
> + return !get_stack_info(addr, state->task, info) && on_stack(info, addr, len);
> +}
> +
> +unsigned long unwind_get_return_address(struct unwind_state *state)
> +{
> + return __unwind_get_return_address(state);
> +}
> +EXPORT_SYMBOL_GPL(unwind_get_return_address);
> +
> +void unwind_start(struct unwind_state *state, struct task_struct *task,
> + struct pt_regs *regs)
> +{
> + __unwind_start(state, task, regs);
> + state->type = UNWINDER_ORC;
> + if (!unwind_done(state) && !__kernel_text_address(state->pc))
> + unwind_next_frame(state);
> +}
> +EXPORT_SYMBOL_GPL(unwind_start);
> +
> +static bool is_entry_func(unsigned long addr)
> +{
> + extern u32 kernel_entry;
> + extern u32 kernel_entry_end;
> +
> + return addr >= (unsigned long)&kernel_entry && addr < (unsigned long)&kernel_entry_end;
> +}
> +
> +static inline unsigned long bt_address(unsigned long ra)
> +{
> + extern unsigned long eentry;
> +
> + if (__kernel_text_address(ra))
> + return ra;
> +
> + if (__module_text_address(ra))
> + return ra;
> +
> + if (ra >= eentry && ra < eentry + EXCCODE_INT_END * VECSIZE) {
> + unsigned long func;
> + unsigned long type = (ra - eentry) / VECSIZE;
> + unsigned long offset = (ra - eentry) % VECSIZE;
> +
> + switch (type) {
> + case 0 ... EXCCODE_INT_START - 1:
> + func = (unsigned long)exception_table[type];
> + break;
> + case EXCCODE_INT_START ... EXCCODE_INT_END:
> + func = (unsigned long)handle_vint;
> + break;
> + default:
> + func = (unsigned long)handle_reserved;
> + break;
> + }
> +
> + return func + offset;
> + }
> +
> + return ra;
> +}
> +
> +bool unwind_next_frame(struct unwind_state *state)
> +{
> + unsigned long *p, pc;
> + struct pt_regs *regs;
> + struct orc_entry *orc;
> + struct stack_info *info = &state->stack_info;
> +
> + if (unwind_done(state))
> + return false;
> +
> + /* Don't let modules unload while we're reading their ORC data. */
> + preempt_disable();
> +
> + if (is_entry_func(state->pc))
> + goto end;
> +
> + orc = orc_find(state->pc);
> + if (!orc) {
> + orc = &orc_fp_entry;
> + state->error = true;
> + }
> +
> + switch (orc->sp_reg) {
> + case ORC_REG_SP:
> + if (info->type == STACK_TYPE_IRQ && state->sp == info->end)
> + orc->type = ORC_TYPE_REGS;
> + else
> + state->sp = state->sp + orc->sp_offset;
> + break;
> + case ORC_REG_FP:
> + state->sp = state->fp;
> + break;
> + default:
> + orc_warn("unknown SP base reg %d at %pB\n", orc->sp_reg, (void *)state->pc);
> + goto err;
> + }
> +
> + switch (orc->fp_reg) {
> + case ORC_REG_PREV_SP:
> + p = (unsigned long *)(state->sp + orc->fp_offset);
> + if (!stack_access_ok(state, (unsigned long)p, sizeof(unsigned long)))
> + goto err;
> +
> + state->fp = *p;
> + break;
> + case ORC_REG_UNDEFINED:
> + /* Nothing. */
> + break;
> + default:
> + orc_warn("unknown FP base reg %d at %pB\n", orc->fp_reg, (void *)state->pc);
> + goto err;
> + }
> +
> + switch (orc->type) {
> + case ORC_TYPE_CALL:
> + if (orc->ra_reg == ORC_REG_PREV_SP) {
> + p = (unsigned long *)(state->sp + orc->ra_offset);
> + if (!stack_access_ok(state, (unsigned long)p, sizeof(unsigned long)))
> + goto err;
> +
> + pc = unwind_graph_addr(state, *p, state->sp);
> + pc -= LOONGARCH_INSN_SIZE;
> + } else if (orc->ra_reg == ORC_REG_UNDEFINED) {
> + if (!state->ra || state->ra == state->pc)
> + goto err;
> +
> + pc = unwind_graph_addr(state, state->ra, state->sp);
> + pc -= LOONGARCH_INSN_SIZE;
> + state->ra = 0;
> + } else {
> + orc_warn("unknown ra base reg %d at %pB\n", orc->ra_reg, (void *)state->pc);
> + goto err;
> + }
> + break;
> + case ORC_TYPE_REGS:
> + if (info->type == STACK_TYPE_IRQ && state->sp == info->end)
> + regs = (struct pt_regs *)info->next_sp;
> + else
> + regs = (struct pt_regs *)state->sp;
> +
> + if (!stack_access_ok(state, (unsigned long)regs, sizeof(*regs)))
> + goto err;
> +
> + if ((info->end == (unsigned long)regs + sizeof(*regs)) &&
> + !regs->regs[3] && !regs->regs[1])
> + goto end;
> +
> + if (user_mode(regs))
> + goto end;
> +
> + pc = regs->csr_era;
> + if (!__kernel_text_address(pc))
> + goto err;
> +
> + state->sp = regs->regs[3];
> + state->ra = regs->regs[1];
> + state->fp = regs->regs[22];
> + get_stack_info(state->sp, state->task, info);
> +
> + break;
> + default:
> + orc_warn("unknown .orc_unwind entry type %d at %pB\n", orc->type, (void *)state->pc);
> + goto err;
> + }
> +
> + state->pc = bt_address(pc);
> + if (!state->pc) {
> + pr_err("cannot find unwind pc at %pK\n", (void *)pc);
> + goto err;
> + }
> +
> + if (!__kernel_text_address(state->pc))
> + goto err;
> +
> + preempt_enable();
> + return true;
> +
> +err:
> + state->error = true;
> +
> +end:
> + preempt_enable();
> + state->stack_info.type = STACK_TYPE_UNKNOWN;
> + return false;
> +}
> +EXPORT_SYMBOL_GPL(unwind_next_frame);
> diff --git a/arch/loongarch/kernel/vmlinux.lds.S b/arch/loongarch/kernel/vmlinux.lds.S
> index bb2ec86f37a8..eaa7a91162e6 100644
> --- a/arch/loongarch/kernel/vmlinux.lds.S
> +++ b/arch/loongarch/kernel/vmlinux.lds.S
> @@ -2,6 +2,7 @@
> #include <linux/sizes.h>
> #include <asm/asm-offsets.h>
> #include <asm/thread_info.h>
> +#include <asm/orc_lookup.h>
>
> #define PAGE_SIZE _PAGE_SIZE
> #define RO_EXCEPTION_TABLE_ALIGN 4
> @@ -122,6 +123,8 @@ SECTIONS
> }
> #endif
>
> + ORC_UNWIND_TABLE
> +
> .sdata : {
> *(.sdata)
> }
> diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
> index 0ed9040307b7..9d49c3f6fff5 100644
> --- a/arch/loongarch/kvm/switch.S
> +++ b/arch/loongarch/kvm/switch.S
> @@ -8,7 +8,7 @@
> #include <asm/asmmacro.h>
> #include <asm/loongarch.h>
> #include <asm/regdef.h>
> -#include <asm/stackframe.h>
> +#include <asm/unwind_hints.h>
>
> #define HGPR_OFFSET(x) (PT_R0 + 8*x)
> #define GGPR_OFFSET(x) (KVM_ARCH_GGPR + 8*x)
> @@ -112,6 +112,7 @@
> .text
> .cfi_sections .debug_frame
> SYM_CODE_START(kvm_exc_entry)
> + UNWIND_HINT_UNDEFINED
> csrwr a2, KVM_TEMP_KS
> csrrd a2, KVM_VCPU_KS
> addi.d a2, a2, KVM_VCPU_ARCH
> @@ -248,3 +249,7 @@ SYM_FUNC_END(kvm_restore_fpu)
> .section ".rodata"
> SYM_DATA(kvm_exception_size, .quad kvm_exc_entry_end - kvm_exc_entry)
> SYM_DATA(kvm_enter_guest_size, .quad kvm_enter_guest_end - kvm_enter_guest)
> +
> +#ifdef CONFIG_CPU_HAS_LBT
> +STACK_FRAME_NON_STANDARD kvm_restore_fpu
> +#endif
> diff --git a/arch/loongarch/lib/Makefile b/arch/loongarch/lib/Makefile
> index a77bf160bfc4..e3023d9a508c 100644
> --- a/arch/loongarch/lib/Makefile
> +++ b/arch/loongarch/lib/Makefile
> @@ -3,6 +3,8 @@
> # Makefile for LoongArch-specific library files.
> #
>
> +OBJECT_FILES_NON_STANDARD := y
> +
> lib-y += delay.o memset.o memcpy.o memmove.o \
> clear_user.o copy_user.o csum.o dump_tlb.o unaligned.o
>
> diff --git a/arch/loongarch/mm/tlb.c b/arch/loongarch/mm/tlb.c
> index 2c0a411f23aa..f01172a8f4e9 100644
> --- a/arch/loongarch/mm/tlb.c
> +++ b/arch/loongarch/mm/tlb.c
> @@ -9,8 +9,9 @@
> #include <linux/hugetlb.h>
> #include <linux/export.h>
>
> -#include <asm/cpu.h>
> #include <asm/bootinfo.h>
> +#include <asm/cpu.h>
> +#include <asm/exception.h>
> #include <asm/mmu_context.h>
> #include <asm/pgtable.h>
> #include <asm/tlb.h>
> @@ -266,24 +267,20 @@ static void setup_tlb_handler(int cpu)
> setup_ptwalker();
> local_flush_tlb_all();
>
> + if (cpu_has_ptw) {
> + exception_table[EXCCODE_TLBI] = handle_tlb_load_ptw;
> + exception_table[EXCCODE_TLBL] = handle_tlb_load_ptw;
> + exception_table[EXCCODE_TLBS] = handle_tlb_store_ptw;
> + exception_table[EXCCODE_TLBM] = handle_tlb_modify_ptw;
> + }
> +
> /* The tlb handlers are generated only once */
> if (cpu == 0) {
> memcpy((void *)tlbrentry, handle_tlb_refill, 0x80);
> local_flush_icache_range(tlbrentry, tlbrentry + 0x80);
> - if (!cpu_has_ptw) {
> - set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE);
> - set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE);
> - set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE);
> - set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE);
> - } else {
> - set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load_ptw, VECSIZE);
> - set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load_ptw, VECSIZE);
> - set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store_ptw, VECSIZE);
> - set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify_ptw, VECSIZE);
> - }
> - set_handler(EXCCODE_TLBNR * VECSIZE, handle_tlb_protect, VECSIZE);
> - set_handler(EXCCODE_TLBNX * VECSIZE, handle_tlb_protect, VECSIZE);
> - set_handler(EXCCODE_TLBPE * VECSIZE, handle_tlb_protect, VECSIZE);
> +
> + for (int i = EXCCODE_TLBL; i <= EXCCODE_TLBPE; i++)
> + set_handler(i * VECSIZE, exception_table[i], VECSIZE);
> }
> #ifdef CONFIG_NUMA
> else {
> diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S
> index d5d682f3d29f..a44387b838af 100644
> --- a/arch/loongarch/mm/tlbex.S
> +++ b/arch/loongarch/mm/tlbex.S
> @@ -18,6 +18,7 @@
>
> .macro tlb_do_page_fault, write
> SYM_CODE_START(tlb_do_page_fault_\write)
> + UNWIND_HINT_UNDEFINED
> SAVE_ALL
> csrrd a2, LOONGARCH_CSR_BADV
> move a0, sp
> @@ -32,6 +33,7 @@
> tlb_do_page_fault 1
>
> SYM_CODE_START(handle_tlb_protect)
> + UNWIND_HINT_UNDEFINED
> BACKUP_T0T1
> SAVE_ALL
> move a0, sp
> @@ -44,6 +46,7 @@ SYM_CODE_START(handle_tlb_protect)
> SYM_CODE_END(handle_tlb_protect)
>
> SYM_CODE_START(handle_tlb_load)
> + UNWIND_HINT_UNDEFINED
> csrwr t0, EXCEPTION_KS0
> csrwr t1, EXCEPTION_KS1
> csrwr ra, EXCEPTION_KS2
> @@ -190,6 +193,7 @@ nopage_tlb_load:
> SYM_CODE_END(handle_tlb_load)
>
> SYM_CODE_START(handle_tlb_load_ptw)
> + UNWIND_HINT_UNDEFINED
> csrwr t0, LOONGARCH_CSR_KS0
> csrwr t1, LOONGARCH_CSR_KS1
> la_abs t0, tlb_do_page_fault_0
> @@ -197,6 +201,7 @@ SYM_CODE_START(handle_tlb_load_ptw)
> SYM_CODE_END(handle_tlb_load_ptw)
>
> SYM_CODE_START(handle_tlb_store)
> + UNWIND_HINT_UNDEFINED
> csrwr t0, EXCEPTION_KS0
> csrwr t1, EXCEPTION_KS1
> csrwr ra, EXCEPTION_KS2
> @@ -346,6 +351,7 @@ nopage_tlb_store:
> SYM_CODE_END(handle_tlb_store)
>
> SYM_CODE_START(handle_tlb_store_ptw)
> + UNWIND_HINT_UNDEFINED
> csrwr t0, LOONGARCH_CSR_KS0
> csrwr t1, LOONGARCH_CSR_KS1
> la_abs t0, tlb_do_page_fault_1
> @@ -353,6 +359,7 @@ SYM_CODE_START(handle_tlb_store_ptw)
> SYM_CODE_END(handle_tlb_store_ptw)
>
> SYM_CODE_START(handle_tlb_modify)
> + UNWIND_HINT_UNDEFINED
> csrwr t0, EXCEPTION_KS0
> csrwr t1, EXCEPTION_KS1
> csrwr ra, EXCEPTION_KS2
> @@ -500,6 +507,7 @@ nopage_tlb_modify:
> SYM_CODE_END(handle_tlb_modify)
>
> SYM_CODE_START(handle_tlb_modify_ptw)
> + UNWIND_HINT_UNDEFINED
> csrwr t0, LOONGARCH_CSR_KS0
> csrwr t1, LOONGARCH_CSR_KS1
> la_abs t0, tlb_do_page_fault_1
> @@ -507,6 +515,7 @@ SYM_CODE_START(handle_tlb_modify_ptw)
> SYM_CODE_END(handle_tlb_modify_ptw)
>
> SYM_CODE_START(handle_tlb_refill)
> + UNWIND_HINT_UNDEFINED
> csrwr t0, LOONGARCH_CSR_TLBRSAVE
> csrrd t0, LOONGARCH_CSR_PGD
> lddir t0, t0, 3
> diff --git a/arch/loongarch/vdso/Makefile b/arch/loongarch/vdso/Makefile
> index c74c9921304f..7a4ad96522ac 100644
> --- a/arch/loongarch/vdso/Makefile
> +++ b/arch/loongarch/vdso/Makefile
> @@ -3,6 +3,7 @@
>
> KASAN_SANITIZE := n
> KCOV_INSTRUMENT := n
> +OBJECT_FILES_NON_STANDARD := y
>
> # Include the generic Makefile to check the built vdso.
> include $(srctree)/lib/vdso/Makefile
> diff --git a/include/linux/compiler.h b/include/linux/compiler.h
> index bb1339c7057b..39f2d4a05208 100644
> --- a/include/linux/compiler.h
> +++ b/include/linux/compiler.h
> @@ -116,6 +116,14 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
> */
> #define __stringify_label(n) #n
>
> +#define __annotate_reachable(c) ({ \
> + asm volatile(__stringify_label(c) ":\n\t" \
> + ".pushsection .discard.reachable\n\t" \
> + ".long " __stringify_label(c) "b - .\n\t" \
> + ".popsection\n\t"); \
> +})
> +#define annotate_reachable() __annotate_reachable(__COUNTER__)
> +
> #define __annotate_unreachable(c) ({ \
> asm volatile(__stringify_label(c) ":\n\t" \
> ".pushsection .discard.unreachable\n\t" \
> @@ -128,6 +136,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
> #define __annotate_jump_table __section(".rodata..c_jump_table")
>
> #else /* !CONFIG_OBJTOOL */
> +#define annotate_reachable()
> #define annotate_unreachable()
> #define __annotate_jump_table
> #endif /* CONFIG_OBJTOOL */
> diff --git a/scripts/Makefile b/scripts/Makefile
> index 576cf64be667..e4cca53d2285 100644
> --- a/scripts/Makefile
> +++ b/scripts/Makefile
> @@ -31,9 +31,12 @@ HOSTLDLIBS_sign-file = $(shell $(HOSTPKG_CONFIG) --libs libcrypto 2> /dev/null |
>
> ifdef CONFIG_UNWINDER_ORC
> ifeq ($(ARCH),x86_64)
> -ARCH := x86
> +SRCARCH := x86
> endif
> -HOSTCFLAGS_sorttable.o += -I$(srctree)/tools/arch/x86/include
> +ifeq ($(ARCH),loongarch)
> +SRCARCH := loongarch
> +endif
> +HOSTCFLAGS_sorttable.o += -I$(srctree)/tools/arch/$(SRCARCH)/include
> HOSTCFLAGS_sorttable.o += -DUNWINDER_ORC_ENABLED
> endif
>
> --
> 2.42.0
>
>
Powered by blists - more mailing lists