diff options
Diffstat (limited to 'tools/objtool')
26 files changed, 2284 insertions, 744 deletions
diff --git a/tools/objtool/Build b/tools/objtool/Build index 600da051af12..59f948628098 100644 --- a/tools/objtool/Build +++ b/tools/objtool/Build @@ -6,13 +6,15 @@ objtool-y += check.o objtool-y += special.o objtool-y += builtin-check.o objtool-y += elf.o +objtool-y += klp-symid.o objtool-y += objtool.o objtool-$(BUILD_DISAS) += disas.o objtool-$(BUILD_DISAS) += trace.o objtool-$(BUILD_ORC) += orc_gen.o orc_dump.o -objtool-$(BUILD_KLP) += builtin-klp.o klp-diff.o klp-post-link.o +objtool-$(BUILD_KLP) += builtin-klp.o klp-checksum.o klp-diff.o \ + klp-post-link.o klp-sympos.o objtool-y += libstring.o objtool-y += libctype.o diff --git a/tools/objtool/Makefile b/tools/objtool/Makefile index b71d1886022e..a4484fd22a96 100644 --- a/tools/objtool/Makefile +++ b/tools/objtool/Makefile @@ -58,7 +58,7 @@ INCLUDES := -I$(srctree)/tools/include \ -I$(srctree)/tools/arch/$(SRCARCH)/include \ -I$(srctree)/tools/objtool/include \ -I$(srctree)/tools/objtool/arch/$(SRCARCH)/include \ - -I$(LIBSUBCMD_OUTPUT)/include + -I$(srctree)/tools/lib OBJTOOL_CFLAGS := -std=gnu11 -fomit-frame-pointer -O2 -g $(WARNINGS) \ $(INCLUDES) $(LIBELF_FLAGS) $(LIBXXHASH_CFLAGS) $(HOSTCFLAGS) @@ -135,7 +135,7 @@ $(LIBSUBCMD): fixdep $(LIBSUBCMD_OUTPUT) FORCE $(Q)$(MAKE) -C $(LIBSUBCMD_DIR) O=$(LIBSUBCMD_OUTPUT) \ DESTDIR=$(LIBSUBCMD_OUTPUT) prefix= subdir= \ $(HOST_OVERRIDES) EXTRA_CFLAGS="$(OBJTOOL_CFLAGS)" \ - $@ install_headers + $@ $(LIBSUBCMD)-clean: $(call QUIET_CLEAN, libsubcmd) diff --git a/tools/objtool/arch/x86/decode.c b/tools/objtool/arch/x86/decode.c index c5817829cdfa..1b387d5a195b 100644 --- a/tools/objtool/arch/x86/decode.c +++ b/tools/objtool/arch/x86/decode.c @@ -805,14 +805,27 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *sec break; } - if (ins.immediate.nbytes) + if (ins.immediate.nbytes) { insn->immediate = ins.immediate.value; - else if (ins.displacement.nbytes) + insn->immediate_len = ins.immediate.nbytes; + } else if (ins.displacement.nbytes) { insn->immediate = ins.displacement.value; + insn->immediate_len = ins.displacement.nbytes; + } return 0; } +size_t arch_jump_opcode_bytes(struct objtool_file *file, struct instruction *insn, + unsigned char *buf) +{ + size_t len; + + len = insn->len - insn->immediate_len; + memcpy(buf, insn->sec->data->d_buf + insn->offset, len); + return len; +} + void arch_initial_func_cfi_state(struct cfi_init_state *state) { int i; @@ -875,14 +888,20 @@ int arch_decode_hint_reg(u8 sp_reg, int *base) case ORC_REG_UNDEFINED: *base = CFI_UNDEFINED; break; + case ORC_REG_AX: + *base = CFI_AX; + break; + case ORC_REG_DX: + *base = CFI_DX; + break; case ORC_REG_SP: *base = CFI_SP; break; case ORC_REG_BP: *base = CFI_BP; break; - case ORC_REG_SP_INDIRECT: - *base = CFI_SP_INDIRECT; + case ORC_REG_DI: + *base = CFI_DI; break; case ORC_REG_R10: *base = CFI_R10; @@ -890,11 +909,11 @@ int arch_decode_hint_reg(u8 sp_reg, int *base) case ORC_REG_R13: *base = CFI_R13; break; - case ORC_REG_DI: - *base = CFI_DI; + case ORC_REG_SP_INDIRECT: + *base = CFI_SP_INDIRECT; break; - case ORC_REG_DX: - *base = CFI_DX; + case ORC_REG_BP_INDIRECT: + *base = CFI_BP_INDIRECT; break; default: return -1; diff --git a/tools/objtool/arch/x86/orc.c b/tools/objtool/arch/x86/orc.c index 735e150ca6b7..eff078ecc945 100644 --- a/tools/objtool/arch/x86/orc.c +++ b/tools/objtool/arch/x86/orc.c @@ -46,17 +46,20 @@ int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, struct instruct orc->signal = cfi->signal; switch (cfi->cfa.base) { + case CFI_AX: + orc->sp_reg = ORC_REG_AX; + break; + case CFI_DX: + orc->sp_reg = ORC_REG_DX; + break; case CFI_SP: orc->sp_reg = ORC_REG_SP; break; - case CFI_SP_INDIRECT: - orc->sp_reg = ORC_REG_SP_INDIRECT; - break; case CFI_BP: orc->sp_reg = ORC_REG_BP; break; - case CFI_BP_INDIRECT: - orc->sp_reg = ORC_REG_BP_INDIRECT; + case CFI_DI: + orc->sp_reg = ORC_REG_DI; break; case CFI_R10: orc->sp_reg = ORC_REG_R10; @@ -64,11 +67,11 @@ int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, struct instruct case CFI_R13: orc->sp_reg = ORC_REG_R13; break; - case CFI_DI: - orc->sp_reg = ORC_REG_DI; + case CFI_SP_INDIRECT: + orc->sp_reg = ORC_REG_SP_INDIRECT; break; - case CFI_DX: - orc->sp_reg = ORC_REG_DX; + case CFI_BP_INDIRECT: + orc->sp_reg = ORC_REG_BP_INDIRECT; break; default: ERROR_INSN(insn, "unknown CFA base reg %d", cfi->cfa.base); @@ -122,22 +125,24 @@ static const char *reg_name(unsigned int reg) switch (reg) { case ORC_REG_PREV_SP: return "prevsp"; + case ORC_REG_AX: + return "ax"; case ORC_REG_DX: return "dx"; - case ORC_REG_DI: - return "di"; case ORC_REG_BP: return "bp"; case ORC_REG_SP: return "sp"; + case ORC_REG_DI: + return "di"; case ORC_REG_R10: return "r10"; case ORC_REG_R13: return "r13"; - case ORC_REG_BP_INDIRECT: - return "bp(ind)"; case ORC_REG_SP_INDIRECT: return "sp(ind)"; + case ORC_REG_BP_INDIRECT: + return "bp(ind)"; default: return "?"; } diff --git a/tools/objtool/arch/x86/special.c b/tools/objtool/arch/x86/special.c index e817a3fff449..1e84c81bfcd8 100644 --- a/tools/objtool/arch/x86/special.c +++ b/tools/objtool/arch/x86/special.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <string.h> +#include <arch/special.h> #include <objtool/special.h> #include <objtool/builtin.h> #include <objtool/warn.h> @@ -9,6 +10,32 @@ /* cpu feature name array generated from cpufeatures.h */ #include "cpu-feature-names.c" +/* + * An alternative with an empty replacement, e.g. the second entry of + * + * ALTERNATIVE_2("orig", "repl", ft1, "", ft2) + * + * still gets a relocation for its replacement offset. But the label it points + * at is the end of the previous entry's replacement, which is also the + * beginning of the *next* entry's replacement. The value is meaningless: it's + * only ever used with a length of zero. + */ +bool arch_alt_ignore_new_reloc(struct section *sec, unsigned long offset) +{ + unsigned long entry_off; + + if (strcmp(sec->name, ".altinstructions")) + return false; + + entry_off = offset - (offset % ALT_ENTRY_SIZE); + + if (offset - entry_off != ALT_NEW_OFFSET) + return false; + + return !*(unsigned char *)(sec->data->d_buf + entry_off + + ALT_NEW_LEN_OFFSET); +} + void arch_handle_alternative(struct special_alt *alt) { static struct special_alt *group, *prev; diff --git a/tools/objtool/builtin-check.c b/tools/objtool/builtin-check.c index b780df513715..75b11dc85010 100644 --- a/tools/objtool/builtin-check.c +++ b/tools/objtool/builtin-check.c @@ -73,11 +73,10 @@ static int parse_hacks(const struct option *opt, const char *str, int unset) static const struct option check_options[] = { OPT_GROUP("Actions:"), - OPT_BOOLEAN(0, "checksum", &opts.checksum, "generate per-function checksums"), - OPT_BOOLEAN(0, "cfi", &opts.cfi, "annotate kernel control flow integrity (kCFI) function preambles"), OPT_STRING_OPTARG('d', "disas", &opts.disas, "function-pattern", "disassemble functions", "*"), OPT_CALLBACK_OPTARG('h', "hacks", NULL, NULL, "jump_label,noinstr,skylake", "patch toolchain bugs/limitations", parse_hacks), OPT_BOOLEAN('i', "ibt", &opts.ibt, "validate and annotate IBT"), + OPT_BOOLEAN(0, "klp-symids", &opts.klp_symids, "generate .klp.symids for duplicate symbol disambiguation"), OPT_BOOLEAN('m', "mcount", &opts.mcount, "annotate mcount/fentry calls for ftrace"), OPT_BOOLEAN(0, "noabs", &opts.noabs, "reject absolute references in allocatable sections"), OPT_BOOLEAN('n', "noinstr", &opts.noinstr, "validate noinstr rules"), @@ -85,7 +84,7 @@ static const struct option check_options[] = { OPT_BOOLEAN('r', "retpoline", &opts.retpoline, "validate and annotate retpoline usage"), OPT_BOOLEAN(0, "rethunk", &opts.rethunk, "validate and annotate rethunk usage"), OPT_BOOLEAN(0, "unret", &opts.unret, "validate entry unret placement"), - OPT_INTEGER(0, "prefix", &opts.prefix, "generate prefix symbols"), + OPT_INTEGER(0, "prefix", &opts.prefix, "generate or grow prefix symbols for N-byte function padding"), OPT_BOOLEAN('l', "sls", &opts.sls, "validate straight-line-speculation mitigations"), OPT_BOOLEAN('s', "stackval", &opts.stackval, "validate frame pointer rules"), OPT_BOOLEAN('t', "static-call", &opts.static_call, "annotate static calls"), @@ -93,9 +92,10 @@ static const struct option check_options[] = { OPT_CALLBACK_OPTARG(0, "dump", NULL, NULL, "orc", "dump metadata", parse_dump), OPT_GROUP("Options:"), + OPT_BOOLEAN(0, "cfi", &opts.cfi, "grow kCFI preamble symbols (use with --prefix)"), + OPT_BOOLEAN(0, "fineibt", &opts.fineibt, "create .cfi_sites section for FineIBT"), OPT_BOOLEAN(0, "backtrace", &opts.backtrace, "unwind on error"), OPT_BOOLEAN(0, "backup", &opts.backup, "create backup (.orig) file on warning/error"), - OPT_STRING(0, "debug-checksum", &opts.debug_checksum, "funcs", "enable checksum debug output"), OPT_BOOLEAN(0, "dry-run", &opts.dryrun, "don't write modifications"), OPT_BOOLEAN(0, "link", &opts.link, "object is a linked object"), OPT_BOOLEAN(0, "module", &opts.module, "object is part of a kernel module"), @@ -165,23 +165,26 @@ static bool opts_valid(void) return false; } -#ifndef BUILD_KLP - if (opts.checksum) { - ERROR("--checksum not supported; install xxhash-devel/libxxhash-dev (version >= 0.8) and recompile"); + if (opts.cfi && !opts.prefix) { + ERROR("--cfi requires --prefix"); return false; } -#endif - if (opts.debug_checksum && !opts.checksum) { - ERROR("--debug-checksum requires --checksum"); + if (opts.fineibt && !opts.cfi) { + ERROR("--fineibt requires --cfi"); return false; } - if (opts.checksum || - opts.disas || + if (opts.klp_symids && !opts.link) { + ERROR("--klp-symids requires --link"); + return false; + } + + if (opts.disas || opts.hack_jump_label || opts.hack_noinstr || opts.ibt || + opts.klp_symids || opts.mcount || opts.noabs || opts.noinstr || diff --git a/tools/objtool/builtin-klp.c b/tools/objtool/builtin-klp.c index 56d5a5b92f72..58c3b9bda3eb 100644 --- a/tools/objtool/builtin-klp.c +++ b/tools/objtool/builtin-klp.c @@ -13,6 +13,7 @@ struct subcmd { }; static struct subcmd subcmds[] = { + { "checksum", "Generate per-function checksums", cmd_klp_checksum, }, { "diff", "Generate binary diff of two object files", cmd_klp_diff, }, { "post-link", "Finalize klp symbols/relocs after module linking", cmd_klp_post_link, }, }; diff --git a/tools/objtool/check.c b/tools/objtool/check.c index b6765e876507..464f6c9d9ff0 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -15,10 +15,10 @@ #include <objtool/arch.h> #include <objtool/disas.h> #include <objtool/check.h> +#include <objtool/klp.h> #include <objtool/special.h> #include <objtool/trace.h> #include <objtool/warn.h> -#include <objtool/checksum.h> #include <objtool/util.h> #include <linux/objtool_types.h> @@ -64,8 +64,8 @@ struct instruction *next_insn_same_sec(struct objtool_file *file, return insn; } -static struct instruction *next_insn_same_func(struct objtool_file *file, - struct instruction *insn) +struct instruction *next_insn_same_func(struct objtool_file *file, + struct instruction *insn) { struct instruction *next = next_insn_same_sec(file, insn); struct symbol *func = insn_func(insn); @@ -113,10 +113,6 @@ static struct instruction *prev_insn_same_sym(struct objtool_file *file, for_each_sec(file->elf, __sec) \ sec_for_each_insn(file, __sec, insn) -#define func_for_each_insn(file, func, insn) \ - for (insn = find_insn(file, func->sec, func->offset); \ - insn; \ - insn = next_insn_same_func(file, insn)) #define sym_for_each_insn(file, sym, insn) \ for (insn = find_insn(file, sym->sec, sym->offset); \ @@ -199,6 +195,8 @@ static bool is_rust_noreturn(const struct symbol *func) */ return str_ends_with(func->name, "_4core3num20from_str_radix_panic") || str_ends_with(func->name, "_4core3num22from_ascii_radix_panic") || + str_ends_with(func->name, "_4core3num28from_ascii_bytes_radix_panic") || + str_ends_with(func->name, "_4core3str16slice_error_fail") || str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") || str_ends_with(func->name, "_4core6option13expect_failed") || str_ends_with(func->name, "_4core6option13unwrap_failed") || @@ -211,6 +209,7 @@ static bool is_rust_noreturn(const struct symbol *func) str_ends_with(func->name, "_4core9panicking18panic_nounwind_fmt") || str_ends_with(func->name, "_4core9panicking19assert_failed_inner") || str_ends_with(func->name, "_4core9panicking30panic_null_pointer_dereference") || + str_ends_with(func->name, "_4core9panicking32panic_null_reference_constructed") || str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference") || str_ends_with(func->name, "_7___rustc17rust_begin_unwind") || strstr(func->name, "_4core9panicking13assert_failed") || @@ -491,7 +490,7 @@ static int decode_instructions(struct objtool_file *file) return -1; } - if (func->embedded_insn || func->alias != func) + if (func->embedded_insn || is_alias_sym(func)) continue; if (!find_insn(file, sec, func->offset)) { @@ -500,7 +499,7 @@ static int decode_instructions(struct objtool_file *file) } sym_for_each_insn(file, func, insn) { - insn->sym = func; + insn->_sym = func; if (is_func_sym(func) && insn->type == INSN_ENDBR && list_empty(&insn->call_node)) { @@ -864,15 +863,14 @@ static int create_ibt_endbr_seal_sections(struct objtool_file *file) list_for_each_entry(insn, &file->endbr_list, call_node) { int *site = (int *)sec->data->d_buf + idx; - struct symbol *sym = insn->sym; + struct symbol *func = insn_func(insn); *site = 0; - if (opts.module && sym && is_func_sym(sym) && - insn->offset == sym->offset && - (!strcmp(sym->name, "init_module") || - !strcmp(sym->name, "cleanup_module"))) { + if (opts.module && func && insn->offset == func->offset && + (!strcmp(func->name, "init_module") || + !strcmp(func->name, "cleanup_module"))) { ERROR("%s(): Magic init_module() function name is deprecated, use module_init(fn) instead", - sym->name); + func->name); return -1; } @@ -887,6 +885,31 @@ static int create_ibt_endbr_seal_sections(struct objtool_file *file) return 0; } +/* +* Grow __cfi_ symbols to fill the NOP gap between the 'mov <hash>, %rax' and +* the start of the function. +*/ +static int grow_cfi_symbols(struct objtool_file *file) +{ + struct symbol *sym; + + for_each_sym(file->elf, sym) { + if (!is_func_sym(sym) || !strstarts(sym->name, "__cfi_") || + sym->len != 5) + continue; + + if (!find_func_by_offset(sym->sec, sym->offset + sym->len + opts.prefix)) + continue; + + sym->len += opts.prefix; + sym->sym.st_size = sym->len; + if (elf_write_symbol(file->elf, sym)) + return -1; + } + + return 0; +} + static int create_cfi_sections(struct objtool_file *file) { struct section *sec; @@ -1023,59 +1046,6 @@ static int create_direct_call_sections(struct objtool_file *file) return 0; } -#ifdef BUILD_KLP -static int create_sym_checksum_section(struct objtool_file *file) -{ - struct section *sec; - struct symbol *sym; - unsigned int idx = 0; - struct sym_checksum *checksum; - size_t entsize = sizeof(struct sym_checksum); - - sec = find_section_by_name(file->elf, ".discard.sym_checksum"); - if (sec) { - if (!opts.dryrun) - WARN("file already has .discard.sym_checksum section, skipping"); - - return 0; - } - - for_each_sym(file->elf, sym) - if (sym->csum.checksum) - idx++; - - if (!idx) - return 0; - - sec = elf_create_section_pair(file->elf, ".discard.sym_checksum", entsize, - idx, idx); - if (!sec) - return -1; - - idx = 0; - for_each_sym(file->elf, sym) { - if (!sym->csum.checksum) - continue; - - if (!elf_init_reloc(file->elf, sec->rsec, idx, idx * entsize, - sym, 0, R_TEXT64)) - return -1; - - checksum = (struct sym_checksum *)sec->data->d_buf + idx; - checksum->addr = 0; /* reloc */ - checksum->checksum = sym->csum.checksum; - - mark_sec_changed(file->elf, sec, true); - - idx++; - } - - return 0; -} -#else -static int create_sym_checksum_section(struct objtool_file *file) { return -EINVAL; } -#endif - /* * Warnings shouldn't be reported for ignored functions. */ @@ -1301,7 +1271,7 @@ static const char *uaccess_safe_builtin[] = { "copy_mc_enhanced_fast_string", "rep_stos_alternative", "rep_movs_alternative", - "__copy_user_nocache", + "copy_to_nontemporal", NULL }; @@ -1349,14 +1319,11 @@ __weak bool arch_is_embedded_insn(struct symbol *sym) return false; } -static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn) +struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn) { struct reloc *reloc; - if (insn->no_reloc) - return NULL; - - if (!file) + if (!file || insn->no_reloc || insn->fake) return NULL; reloc = find_reloc_by_dest_range(file->elf, insn->sec, @@ -1642,7 +1609,7 @@ static int add_jump_destinations(struct objtool_file *file) } if (!dest_sym || is_sec_sym(dest_sym)) { - dest_sym = dest_insn->sym; + dest_sym = insn_sym(dest_insn); if (!dest_sym) goto set_jump_dest; } @@ -1658,7 +1625,7 @@ static int add_jump_destinations(struct objtool_file *file) continue; } - if (!insn->sym || insn->sym->pfunc == dest_sym->pfunc) + if (!insn_sym(insn) || insn_sym(insn)->pfunc == dest_sym->pfunc) goto set_jump_dest; /* @@ -1831,7 +1798,6 @@ static int handle_group_alt(struct objtool_file *file, nop->offset = special_alt->new_off + special_alt->new_len; nop->len = special_alt->orig_len - special_alt->new_len; nop->type = INSN_NOP; - nop->sym = orig_insn->sym; nop->alt_group = new_alt_group; nop->fake = 1; } @@ -1850,7 +1816,6 @@ static int handle_group_alt(struct objtool_file *file, last_new_insn = insn; - insn->sym = orig_insn->sym; insn->alt_group = new_alt_group; /* @@ -2232,7 +2197,7 @@ static int add_jump_table_alts(struct objtool_file *file) return 0; for_each_sym(file->elf, func) { - if (!is_func_sym(func) || func->alias != func) + if (!is_func_sym(func) || is_alias_sym(func)) continue; mark_func_jump_tables(file, func); @@ -2493,12 +2458,12 @@ static int __annotate_late(struct objtool_file *file, int type, struct instructi break; case ANNOTYPE_NOCFI: - sym = insn->sym; + sym = insn_sym(insn); if (!sym) { ERROR_INSN(insn, "dodgy NOCFI annotation"); return -1; } - insn->sym->nocfi = 1; + sym->nocfi = 1; break; default: @@ -2566,7 +2531,6 @@ static int classify_symbols(struct objtool_file *file) static void mark_rodata(struct objtool_file *file) { struct section *sec; - bool found = false; /* * Search for the following rodata sections, each of which can @@ -2579,15 +2543,11 @@ static void mark_rodata(struct objtool_file *file) * .rodata.str1.* sections are ignored; they don't contain jump tables. */ for_each_sec(file->elf, sec) { - if ((!strncmp(sec->name, ".rodata", 7) && - !strstr(sec->name, ".str1.")) || - !strncmp(sec->name, ".data.rel.ro", 12)) { - sec->rodata = true; - found = true; + if (is_rodata_sec(sec)) { + file->rodata = true; + return; } } - - file->rodata = found; } static void mark_holes(struct objtool_file *file) @@ -2604,7 +2564,7 @@ static void mark_holes(struct objtool_file *file) * favour of a regular symbol, but leaves the code in place. */ for_each_insn(file, insn) { - if (insn->sym || !find_symbol_hole_containing(insn->sec, insn->offset)) { + if (insn_sym(insn) || !find_symbol_hole_containing(insn->sec, insn->offset)) { in_hole = false; continue; } @@ -2622,7 +2582,7 @@ static void mark_holes(struct objtool_file *file) if (insn->jump_dest) { struct symbol *dest_func = insn_func(insn->jump_dest); - if (dest_func && dest_func->cold) + if (dest_func && is_cold_func(dest_func)) dest_func->ignore = true; } } @@ -2630,14 +2590,35 @@ static void mark_holes(struct objtool_file *file) static bool validate_branch_enabled(void) { - return opts.stackval || - opts.orc || - opts.uaccess || + return opts.stackval || + opts.orc || + opts.uaccess; +} + +static bool alts_needed(void) +{ + return validate_branch_enabled() || + opts.noinstr || + opts.hack_jump_label || + opts.disas || opts.checksum; } -static int decode_sections(struct objtool_file *file) +int decode_file(struct objtool_file *file) { + arch_initial_func_cfi_state(&initial_func_cfi); + init_cfi_state(&init_cfi); + init_cfi_state(&func_cfi); + set_func_state(&func_cfi); + init_cfi_state(&force_undefined_cfi); + force_undefined_cfi.force_undefined = true; + + if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3))) + return -1; + + cfi_hash_add(&init_cfi); + cfi_hash_add(&func_cfi); + file->klp = is_livepatch_module(file); mark_rodata(file); @@ -2666,7 +2647,7 @@ static int decode_sections(struct objtool_file *file) * Must be before add_jump_destinations(), which depends on 'func' * being set for alternatives, to enable proper sibling call detection. */ - if (validate_branch_enabled() || opts.noinstr || opts.hack_jump_label || opts.disas) { + if (alts_needed()) { if (add_special_section_alts(file)) return -1; } @@ -3027,7 +3008,7 @@ static int update_cfi_state(struct instruction *insn, } if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP && - insn->sym->frame_pointer) { + insn_sym(insn)->frame_pointer) { /* addi.d fp,sp,imm on LoongArch */ if (cfa->base == CFI_SP && cfa->offset == op->src.offset) { cfa->base = CFI_BP; @@ -3039,7 +3020,7 @@ static int update_cfi_state(struct instruction *insn, if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) { /* addi.d sp,fp,imm on LoongArch */ if (cfa->base == CFI_BP && cfa->offset == 0) { - if (insn->sym->frame_pointer) { + if (insn_sym(insn)->frame_pointer) { cfa->base = CFI_SP; cfa->offset = -op->src.offset; } @@ -3662,88 +3643,6 @@ static bool skip_alt_group(struct instruction *insn) return alt_insn->type == INSN_CLAC || alt_insn->type == INSN_STAC; } -static int checksum_debug_init(struct objtool_file *file) -{ - char *dup, *s; - - if (!opts.debug_checksum) - return 0; - - dup = strdup(opts.debug_checksum); - if (!dup) { - ERROR_GLIBC("strdup"); - return -1; - } - - s = dup; - while (*s) { - struct symbol *func; - char *comma; - - comma = strchr(s, ','); - if (comma) - *comma = '\0'; - - func = find_symbol_by_name(file->elf, s); - if (!func || !is_func_sym(func)) - WARN("--debug-checksum: can't find '%s'", s); - else - func->debug_checksum = 1; - - if (!comma) - break; - - s = comma + 1; - } - - free(dup); - return 0; -} - -static void checksum_update_insn(struct objtool_file *file, struct symbol *func, - struct instruction *insn) -{ - struct reloc *reloc = insn_reloc(file, insn); - unsigned long offset; - struct symbol *sym; - - if (insn->fake) - return; - - checksum_update(func, insn, insn->sec->data->d_buf + insn->offset, insn->len); - - if (!reloc) { - struct symbol *call_dest = insn_call_dest(insn); - - if (call_dest) - checksum_update(func, insn, call_dest->demangled_name, - strlen(call_dest->demangled_name)); - return; - } - - sym = reloc->sym; - offset = arch_insn_adjusted_addend(insn, reloc); - - if (is_string_sec(sym->sec)) { - char *str; - - str = sym->sec->data->d_buf + sym->offset + offset; - checksum_update(func, insn, str, strlen(str)); - return; - } - - if (is_sec_sym(sym)) { - sym = find_symbol_containing(reloc->sym->sec, offset); - if (!sym) - return; - - offset -= sym->offset; - } - - checksum_update(func, insn, sym->demangled_name, strlen(sym->demangled_name)); - checksum_update(func, insn, &offset, sizeof(offset)); -} - static int validate_branch(struct objtool_file *file, struct symbol *func, struct instruction *insn, struct insn_state state); static int do_validate_branch(struct objtool_file *file, struct symbol *func, @@ -4025,9 +3924,6 @@ static int do_validate_branch(struct objtool_file *file, struct symbol *func, insn->trace = 0; next_insn = next_insn_to_validate(file, insn); - if (opts.checksum && func && insn->sec) - checksum_update_insn(file, func, insn); - if (func && insn_func(insn) && func != insn_func(insn)->pfunc) { /* Ignore KCFI type preambles, which always fall through */ if (is_prefix_func(func)) @@ -4093,9 +3989,6 @@ static int validate_unwind_hint(struct objtool_file *file, struct symbol *func = insn_func(insn); int ret; - if (opts.checksum) - checksum_init(func); - ret = validate_branch(file, func, insn, *state); if (ret) BT_INSN(insn, "<=== (hint)"); @@ -4304,10 +4197,10 @@ static int validate_retpoline(struct objtool_file *file) * broken. */ list_for_each_entry(insn, &file->retpoline_call_list, call_node) { - struct symbol *sym = insn->sym; + struct symbol *sym = insn_sym(insn); - if (sym && (sym->type == STT_NOTYPE || - sym->type == STT_FUNC) && !sym->nocfi) { + if (sym && (is_notype_sym(sym) || + is_func_sym(sym)) && !sym->nocfi) { struct instruction *prev = prev_insn_same_sym(file, insn); @@ -4407,17 +4300,6 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio * For FineIBT or kCFI, a certain number of bytes preceding the function may be * NOPs. Those NOPs may be rewritten at runtime and executed, so give them a * proper function name: __pfx_<func>. - * - * The NOPs may not exist for the following cases: - * - * - compiler cloned functions (*.cold, *.part0, etc) - * - asm functions created with inline asm or without SYM_FUNC_START() - * - * Also, the function may already have a prefix from a previous objtool run - * (livepatch extracted functions, or manually running objtool multiple times). - * - * So return 0 if the NOPs are missing or the function already has a prefix - * symbol. */ static int create_prefix_symbol(struct objtool_file *file, struct symbol *func) { @@ -4425,10 +4307,6 @@ static int create_prefix_symbol(struct objtool_file *file, struct symbol *func) char name[SYM_NAME_LEN]; struct cfi_state *cfi; - if (!is_func_sym(func) || is_prefix_func(func) || - func->cold || func->static_call_tramp) - return 0; - if ((strlen(func->name) + sizeof("__pfx_") > SYM_NAME_LEN)) { WARN("%s: symbol name too long, can't create __pfx_ symbol", func->name); @@ -4438,59 +4316,21 @@ static int create_prefix_symbol(struct objtool_file *file, struct symbol *func) if (snprintf_check(name, SYM_NAME_LEN, "__pfx_%s", func->name)) return -1; - if (file->klp) { - struct symbol *pfx; - - pfx = find_symbol_by_offset(func->sec, func->offset - opts.prefix); - if (pfx && is_prefix_func(pfx) && !strcmp(pfx->name, name)) - return 0; - } - - insn = find_insn(file, func->sec, func->offset); - if (!insn) { - WARN("%s: can't find starting instruction", func->name); + if (!elf_create_symbol(file->elf, name, func->sec, + GELF_ST_BIND(func->sym.st_info), + GELF_ST_TYPE(func->sym.st_info), + func->offset - opts.prefix, opts.prefix)) return -1; - } - - for (prev = prev_insn_same_sec(file, insn); - prev; - prev = prev_insn_same_sec(file, prev)) { - u64 offset; - - if (prev->type != INSN_NOP) - return 0; - - offset = func->offset - prev->offset; - - if (offset > opts.prefix) - return 0; - - if (offset < opts.prefix) - continue; - if (!elf_create_symbol(file->elf, name, func->sec, - GELF_ST_BIND(func->sym.st_info), - GELF_ST_TYPE(func->sym.st_info), - prev->offset, opts.prefix)) - return -1; - - break; - } - - if (!prev) - return 0; - - if (!insn->cfi) { - /* - * This can happen if stack validation isn't enabled or the - * function is annotated with STACK_FRAME_NON_STANDARD. - */ + /* Propagate insn->cfi to the prefix code */ + insn = find_insn(file, func->sec, func->offset); + if (!insn || !insn->cfi) return 0; - } - /* Propagate insn->cfi to the prefix code */ cfi = cfi_hash_find_or_add(insn->cfi); - for (; prev != insn; prev = next_insn_same_sec(file, prev)) + for (prev = find_insn(file, func->sec, func->offset - opts.prefix); + prev && prev != insn; + prev = next_insn_same_sec(file, prev)) prev->cfi = cfi; return 0; @@ -4498,15 +4338,20 @@ static int create_prefix_symbol(struct objtool_file *file, struct symbol *func) static int create_prefix_symbols(struct objtool_file *file) { - struct section *sec; + struct section *pfe_sec; struct symbol *func; + struct reloc *reloc; - for_each_sec(file->elf, sec) { - if (!is_text_sec(sec)) + for_each_sec(file->elf, pfe_sec) { + if (strcmp(pfe_sec->name, "__patchable_function_entries")) + continue; + if (!pfe_sec->rsec) continue; - sec_for_each_sym(sec, func) { - if (create_prefix_symbol(file, func)) + for_each_reloc(pfe_sec->rsec, reloc) { + func = find_func_by_offset(reloc->sym->sec, + reloc->sym->offset + reloc_addend(reloc) + opts.prefix); + if (func && create_prefix_symbol(file, func)) return -1; } } @@ -4526,7 +4371,7 @@ static int validate_symbol(struct objtool_file *file, struct section *sec, return 1; } - if (sym->pfunc != sym || sym->alias != sym) + if (sym->pfunc != sym || is_alias_sym(sym)) return 0; insn = find_insn(file, sec, sym->offset); @@ -4538,9 +4383,6 @@ static int validate_symbol(struct objtool_file *file, struct section *sec, func = insn_func(insn); - if (opts.checksum) - checksum_init(func); - if (opts.trace && !fnmatch(opts.trace, sym->name, 0)) { trace_enable(); TRACE("%s: validation begin\n", sym->name); @@ -4553,9 +4395,6 @@ static int validate_symbol(struct objtool_file *file, struct section *sec, TRACE("%s: validation %s\n\n", sym->name, ret ? "failed" : "end"); trace_disable(); - if (opts.checksum) - checksum_finish(func); - return ret; } @@ -4809,7 +4648,6 @@ static int validate_ibt(struct objtool_file *file) !strcmp(sec->name, ".kcfi_traps") || !strcmp(sec->name, ".orc_unwind_ip") || !strcmp(sec->name, ".retpoline_sites") || - !strcmp(sec->name, ".smp_locks") || !strcmp(sec->name, ".static_call_sites") || !strcmp(sec->name, "_error_injection_whitelist") || !strcmp(sec->name, "_kprobe_blacklist") || @@ -4948,7 +4786,7 @@ struct insn_chunk { * which can trigger more allocations for .debug_* sections whose data hasn't * been read yet. */ -static void free_insns(struct objtool_file *file) +void free_insns(struct objtool_file *file) { struct instruction *insn; struct insn_chunk *chunks = NULL, *chunk; @@ -4995,26 +4833,7 @@ int check(struct objtool_file *file) objtool_disas_ctx = disas_ctx; } - arch_initial_func_cfi_state(&initial_func_cfi); - init_cfi_state(&init_cfi); - init_cfi_state(&func_cfi); - set_func_state(&func_cfi); - init_cfi_state(&force_undefined_cfi); - force_undefined_cfi.force_undefined = true; - - if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3))) { - ret = -1; - goto out; - } - - cfi_hash_add(&init_cfi); - cfi_hash_add(&func_cfi); - - ret = checksum_debug_init(file); - if (ret) - goto out; - - ret = decode_sections(file); + ret = decode_file(file); if (ret) goto out; @@ -5064,12 +4883,6 @@ int check(struct objtool_file *file) goto out; } - if (opts.cfi) { - ret = create_cfi_sections(file); - if (ret) - goto out; - } - if (opts.rethunk) { ret = create_return_sites_sections(file); if (ret) @@ -5089,9 +4902,21 @@ int check(struct objtool_file *file) } if (opts.prefix) { - ret = create_prefix_symbols(file); - if (ret) - goto out; + if (!opts.cfi) { + ret = create_prefix_symbols(file); + if (ret) + goto out; + } else { + ret = grow_cfi_symbols(file); + if (ret) + goto out; + + if (opts.fineibt) { + ret = create_cfi_sections(file); + if (ret) + goto out; + } + } } if (opts.ibt) { @@ -5100,15 +4925,15 @@ int check(struct objtool_file *file) goto out; } - if (opts.noabs) - warnings += check_abs_references(file); - - if (opts.checksum) { - ret = create_sym_checksum_section(file); + if (opts.klp_symids) { + ret = klp_create_symid_sections(file); if (ret) goto out; } + if (opts.noabs) + warnings += check_abs_references(file); + if (opts.orc && nr_insns) { ret = orc_create(file); if (ret) diff --git a/tools/objtool/disas.c b/tools/objtool/disas.c index 26f08d41f2b1..e6a54a83605c 100644 --- a/tools/objtool/disas.c +++ b/tools/objtool/disas.c @@ -210,7 +210,7 @@ static bool disas_print_addr_alt(bfd_vma addr, struct disassemble_info *dinfo) offset = addr - alt_group->first_insn->offset; addr = orig_first_insn->offset + offset; - sym = orig_first_insn->sym; + sym = insn_sym(orig_first_insn); disas_print_addr_sym(orig_first_insn->sec, sym, addr, dinfo); @@ -222,15 +222,13 @@ static void disas_print_addr_noreloc(bfd_vma addr, { struct disas_context *dctx = dinfo->application_data; struct instruction *insn = dctx->insn; - struct symbol *sym = NULL; + struct symbol *sym = insn_sym(insn); if (disas_print_addr_alt(addr, dinfo)) return; - if (insn->sym && addr >= insn->sym->offset && - addr < insn->sym->offset + insn->sym->len) { - sym = insn->sym; - } + if (sym && (addr < sym->offset || addr >= sym->offset + sym->len)) + sym = NULL; disas_print_addr_sym(insn->sec, sym, addr, dinfo); } @@ -264,7 +262,7 @@ static void disas_print_addr_reloc(bfd_vma addr, struct disassemble_info *dinfo) * If the relocation symbol is a section name (for example ".bss") * then we try to further resolve the name. */ - if (reloc->sym->type == STT_SECTION) { + if (is_sec_sym(reloc->sym)) { str = offstr(reloc->sym->sec, reloc->sym->offset + offset); DINFO_FPRINTF(dinfo, bfd_vma_fmt, addr, str); free(str); @@ -291,9 +289,9 @@ static void disas_print_address(bfd_vma addr, struct disassemble_info *dinfo) * up. So check it first. */ jump_dest = insn->jump_dest; - if (jump_dest && jump_dest->sym && jump_dest->offset == addr) { + if (jump_dest && insn_sym(jump_dest) && jump_dest->offset == addr) { if (!disas_print_addr_alt(addr, dinfo)) - disas_print_addr_sym(jump_dest->sec, jump_dest->sym, + disas_print_addr_sym(jump_dest->sec, insn_sym(jump_dest), addr, dinfo); return; } @@ -580,7 +578,7 @@ static size_t disas_insn_common(struct disas_context *dctx, */ dinfo->buffer = insn->sec->data->d_buf; dinfo->buffer_vma = 0; - dinfo->buffer_length = insn->sec->sh.sh_size; + dinfo->buffer_length = sec_size(insn->sec); return disasm(insn->offset, &dctx->info); } @@ -768,8 +766,8 @@ static int disas_alt_jump(struct disas_alt *dalt) if (orig_insn->len == 5) suffix[0] = 'q'; str = strfmt("jmp%-3s %lx <%s+0x%lx>", suffix, - dest_insn->offset, dest_insn->sym->name, - dest_insn->offset - dest_insn->sym->offset); + dest_insn->offset, insn_sym(dest_insn)->name, + dest_insn->offset - insn_sym(dest_insn)->offset); nops = 0; } else { str = strfmt("nop%d", orig_insn->len); @@ -794,8 +792,8 @@ static int disas_alt_extable(struct disas_alt *dalt) alt_insn = dalt->alt->insn; str = strfmt("resume at 0x%lx <%s+0x%lx>", - alt_insn->offset, alt_insn->sym->name, - alt_insn->offset - alt_insn->sym->offset); + alt_insn->offset, insn_sym(alt_insn)->name, + alt_insn->offset - insn_sym(alt_insn)->offset); if (!str) return -1; @@ -1231,7 +1229,7 @@ void disas_funcs(struct disas_context *dctx) for_each_sec(dctx->file->elf, sec) { - if (!(sec->sh.sh_flags & SHF_EXECINSTR)) + if (!is_text_sec(sec)) continue; sec_for_each_sym(sec, sym) { diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c index 2ffe3ebfbe37..a791f4ea6ec1 100644 --- a/tools/objtool/elf.c +++ b/tools/objtool/elf.c @@ -23,24 +23,21 @@ #include <linux/log2.h> #include <objtool/builtin.h> #include <objtool/elf.h> +#include <objtool/klp.h> #include <objtool/warn.h> -static inline u32 str_hash(const char *str) +static ssize_t demangled_name_len(const char *name); + +u32 str_hash_demangled(const char *str) { - return jhash(str, strlen(str), 0); + return jhash(str, demangled_name_len(str), 0); } -#define __elf_table(name) (elf->name##_hash) -#define __elf_bits(name) (elf->name##_bits) - -#define __elf_table_entry(name, key) \ - __elf_table(name)[hash_min(key, __elf_bits(name))] - #define elf_hash_add(name, node, key) \ ({ \ struct elf_hash_node *__node = node; \ - __node->next = __elf_table_entry(name, key); \ - __elf_table_entry(name, key) = __node; \ + __node->next = __elf_table_entry(elf, name, key); \ + __elf_table_entry(elf, name, key) = __node; \ }) static inline void __elf_hash_del(struct elf_hash_node *node, @@ -62,30 +59,20 @@ static inline void __elf_hash_del(struct elf_hash_node *node, } #define elf_hash_del(name, node, key) \ - __elf_hash_del(node, &__elf_table_entry(name, key)) - -#define elf_list_entry(ptr, type, member) \ -({ \ - typeof(ptr) __ptr = (ptr); \ - __ptr ? container_of(__ptr, type, member) : NULL; \ -}) - -#define elf_hash_for_each_possible(name, obj, member, key) \ - for (obj = elf_list_entry(__elf_table_entry(name, key), typeof(*obj), member); \ - obj; \ - obj = elf_list_entry(obj->member.next, typeof(*(obj)), member)) + __elf_hash_del(node, &__elf_table_entry(elf, name, key)) #define elf_alloc_hash(name, size) \ ({ \ - __elf_bits(name) = max(10, ilog2(size)); \ - __elf_table(name) = mmap(NULL, sizeof(struct elf_hash_node *) << __elf_bits(name), \ + __elf_bits(elf, name) = max(10, ilog2(size)); \ + __elf_table(elf, name) = mmap(NULL, \ + sizeof(struct elf_hash_node *) << __elf_bits(elf, name), \ PROT_READ|PROT_WRITE, \ MAP_PRIVATE|MAP_ANON, -1, 0); \ - if (__elf_table(name) == (void *)-1L) { \ + if (__elf_table(elf, name) == (void *)-1L) { \ ERROR_GLIBC("mmap fail " #name); \ - __elf_table(name) = NULL; \ + __elf_table(elf, name) = NULL; \ } \ - __elf_table(name); \ + __elf_table(elf, name); \ }) static inline unsigned long __sym_start(struct symbol *s) @@ -134,7 +121,7 @@ struct section *find_section_by_name(const struct elf *elf, const char *name) { struct section *sec; - elf_hash_for_each_possible(section_name, sec, name_hash, str_hash(name)) { + elf_hash_for_each_possible(elf, section_name, sec, name_hash, str_hash(name)) { if (!strcmp(sec->name, name)) return sec; } @@ -147,7 +134,7 @@ static struct section *find_section_by_index(struct elf *elf, { struct section *sec; - elf_hash_for_each_possible(section, sec, hash, idx) { + elf_hash_for_each_possible(elf, section, sec, hash, idx) { if (sec->idx == idx) return sec; } @@ -159,7 +146,7 @@ static struct symbol *find_symbol_by_index(struct elf *elf, unsigned int idx) { struct symbol *sym; - elf_hash_for_each_possible(symbol, sym, hash, idx) { + elf_hash_for_each_possible(elf, symbol, sym, hash, idx) { if (sym->idx == idx) return sym; } @@ -222,6 +209,20 @@ struct symbol *find_symbol_containing(const struct section *sec, unsigned long o } /* + * Also match the symbol end address which can be used for a bounds comparison. + */ +struct symbol *find_symbol_containing_inclusive(const struct section *sec, + unsigned long offset) +{ + struct symbol *sym = find_symbol_containing(sec, offset); + + if (!sym && offset) + sym = find_symbol_containing(sec, offset - 1); + + return sym; +} + +/* * Returns size of hole starting at @offset. */ int find_symbol_hole_containing(const struct section *sec, unsigned long offset) @@ -278,7 +279,7 @@ struct symbol *find_symbol_by_name(const struct elf *elf, const char *name) { struct symbol *sym; - elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash(name)) { + elf_hash_for_each_possible(elf, symbol_name, sym, name_hash, str_hash(name)) { if (!strcmp(sym->name, name)) return sym; } @@ -293,7 +294,7 @@ static struct symbol *find_local_symbol_by_file_and_name(const struct elf *elf, { struct symbol *sym; - elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash(name)) { + elf_hash_for_each_possible(elf, symbol_name, sym, name_hash, str_hash_demangled(name)) { if (sym->bind == STB_LOCAL && sym->file == file && !strcmp(sym->name, name)) { return sym; @@ -307,7 +308,7 @@ struct symbol *find_global_symbol_by_name(const struct elf *elf, const char *nam { struct symbol *sym; - elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash(name)) { + elf_hash_for_each_possible(elf, symbol_name, sym, name_hash, str_hash_demangled(name)) { if (!strcmp(sym->name, name) && !is_local_sym(sym)) return sym; } @@ -315,8 +316,9 @@ struct symbol *find_global_symbol_by_name(const struct elf *elf, const char *nam return NULL; } +/* If there are multiple matches, return the first one in the range */ struct reloc *find_reloc_by_dest_range(const struct elf *elf, struct section *sec, - unsigned long offset, unsigned int len) + unsigned long offset, unsigned int len) { struct reloc *reloc, *r = NULL; struct section *rsec; @@ -327,7 +329,7 @@ struct reloc *find_reloc_by_dest_range(const struct elf *elf, struct section *se return NULL; for_offset_range(o, offset, offset + len) { - elf_hash_for_each_possible(reloc, reloc, hash, + elf_hash_for_each_possible(elf, reloc, reloc, hash, sec_offset_hash(rsec, o)) { if (reloc->sec != rsec) continue; @@ -338,11 +340,11 @@ struct reloc *find_reloc_by_dest_range(const struct elf *elf, struct section *se r = reloc; } } - if (r) + if (r && (reloc_offset(r) & OFFSET_STRIDE_MASK) == o) return r; } - return NULL; + return r; } struct reloc *find_reloc_by_dest(const struct elf *elf, struct section *sec, unsigned long offset) @@ -440,34 +442,67 @@ static int read_sections(struct elf *elf) return 0; } +/* + * Returns desired length of the demangled name. + * If name doesn't need demangling, return strlen(name). + */ +static ssize_t demangled_name_len(const char *name) +{ + ssize_t idx; + const char *p; + + p = strstr(name, ".llvm."); + if (p) + return p - name; + + if (!strstarts(name, "__UNIQUE_ID_") && !strchr(name, '.')) + return strlen(name); + + for (idx = strlen(name) - 1; idx >= 0; idx--) { + char c = name[idx]; + + if (!isdigit(c) && c != '.' && c != '_') + break; + } + if (idx <= 0) + return strlen(name); + return idx + 1; +} + +/* + * Remove number suffix of a symbol. + * + * Specifically, remove trailing numbers for "__UNIQUE_ID_" symbols and + * symbols with '.'. + * + * With CONFIG_LTO_CLANG_THIN, it is possible to have nested __UNIQUE_ID_, + * such as + * + * __UNIQUE_ID_addressable___UNIQUE_ID_pci_invalid_bar_694_695 + * + * to remove both trailing numbers, also remove trailing '_'. + * + * For symbols with llvm suffix, i.e., foo.llvm.<hash>, remove the + * .llvm.<hash> part. + */ static const char *demangle_name(struct symbol *sym) { char *str; - - if (!is_local_sym(sym)) - return sym->name; + ssize_t len; if (!is_func_sym(sym) && !is_object_sym(sym)) return sym->name; - if (!strstarts(sym->name, "__UNIQUE_ID_") && !strchr(sym->name, '.')) + len = demangled_name_len(sym->name); + if (len == strlen(sym->name)) return sym->name; - str = strdup(sym->name); + str = strndup(sym->name, len); if (!str) { ERROR_GLIBC("strdup"); return NULL; } - for (int i = strlen(str) - 1; i >= 0; i--) { - char c = str[i]; - - if (!isdigit(c) && c != '.') { - str[i + 1] = '\0'; - break; - } - } - return str; } @@ -503,9 +538,13 @@ static int elf_add_symbol(struct elf *elf, struct symbol *sym) entry = &sym->sec->symbol_list; list_add(&sym->list, entry); + sym->demangled_name = demangle_name(sym); + if (!sym->demangled_name) + return -1; + list_add_tail(&sym->global_list, &elf->symbols); elf_hash_add(symbol, &sym->hash, sym->idx); - elf_hash_add(symbol_name, &sym->name_hash, str_hash(sym->name)); + elf_hash_add(symbol_name, &sym->name_hash, str_hash(sym->demangled_name)); if (is_func_sym(sym) && (strstarts(sym->name, "__pfx_") || @@ -529,10 +568,6 @@ static int elf_add_symbol(struct elf *elf, struct symbol *sym) sym->pfunc = sym->cfunc = sym; - sym->demangled_name = demangle_name(sym); - if (!sym->demangled_name) - return -1; - return 0; } @@ -592,6 +627,18 @@ static int read_symbols(struct elf *elf) return -1; } + /* + * "klp diff" renames the placeholder symbols of KLP relocs to + * hide them from modpost. Hide the prefix from the rest of + * objtool so its many name-based heuristics (noreturns, + * uaccess safe list, ...) still see the original symbol name. + * + * st_name is left alone, so the renamed symbol is preserved in + * the output file. + */ + if (strstarts(sym->name, KLP_TOMBSTONE_PREFIX)) + sym->name += strlen(KLP_TOMBSTONE_PREFIX); + if ((sym->sym.st_shndx > SHN_UNDEF && sym->sym.st_shndx < SHN_LORESERVE) || (shndx_data && sym->sym.st_shndx == SHN_XINDEX)) { @@ -613,9 +660,9 @@ static int read_symbols(struct elf *elf) if (elf_add_symbol(elf, sym)) return -1; - if (sym->type == STT_FILE) + if (is_file_sym(sym)) file = sym; - else if (sym->bind == STB_LOCAL) + else if (sym->bind == STB_LOCAL && !is_sec_sym(sym)) sym->file = file; } @@ -963,6 +1010,26 @@ non_local: return sym; } +int elf_write_symbol(struct elf *elf, struct symbol *sym) +{ + struct section *symtab, *symtab_shndx; + + symtab = find_section_by_name(elf, ".symtab"); + if (!symtab) { + ERROR("no .symtab"); + return -1; + } + + symtab_shndx = find_section_by_name(elf, ".symtab_shndx"); + + if (elf_update_symbol(elf, symtab, symtab_shndx, sym)) + return -1; + + mark_sec_changed(elf, symtab, true); + + return 0; +} + struct symbol *elf_create_section_symbol(struct elf *elf, struct section *sec) { struct symbol *sym = calloc(1, sizeof(*sym)); @@ -1119,6 +1186,17 @@ static int read_relocs(struct elf *elf) return 0; } +static void mark_rodata(struct elf *elf) +{ + struct section *sec; + + for_each_sec(elf, sec) { + if ((strstarts(sec->name, ".rodata") && !strstr(sec->name, ".str1.")) || + strstarts(sec->name, ".data.rel.ro")) + sec->rodata = true; + } +} + struct elf *elf_open_read(const char *name, int flags) { struct elf *elf; @@ -1169,6 +1247,8 @@ struct elf *elf_open_read(const char *name, int flags) if (read_sections(elf)) goto err; + mark_rodata(elf); + if (read_symbols(elf)) goto err; @@ -1318,7 +1398,7 @@ unsigned int elf_add_string(struct elf *elf, struct section *strtab, const char return -1; } - offset = ALIGN(strtab->sh.sh_size, strtab->sh.sh_addralign); + offset = ALIGN(sec_size(strtab), strtab->sh.sh_addralign); if (!elf_add_data(elf, strtab, str, strlen(str) + 1)) return -1; @@ -1360,7 +1440,7 @@ void *elf_add_data(struct elf *elf, struct section *sec, const void *data, size_ sec->data->d_size = size; sec->data->d_align = sec->sh.sh_addralign; - offset = ALIGN(sec->sh.sh_size, sec->sh.sh_addralign); + offset = ALIGN(sec_size(sec), sec->sh.sh_addralign); sec->sh.sh_size = offset + size; mark_sec_changed(elf, sec, true); diff --git a/tools/objtool/include/objtool/arch.h b/tools/objtool/include/objtool/arch.h index 8866158975fc..96d828a8401f 100644 --- a/tools/objtool/include/objtool/arch.h +++ b/tools/objtool/include/objtool/arch.h @@ -79,6 +79,9 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *sec unsigned long offset, unsigned int maxlen, struct instruction *insn); +size_t arch_jump_opcode_bytes(struct objtool_file *file, struct instruction *insn, + unsigned char *buf); + bool arch_callee_saved_reg(unsigned char reg); unsigned long arch_jump_destination(struct instruction *insn); diff --git a/tools/objtool/include/objtool/builtin.h b/tools/objtool/include/objtool/builtin.h index b9e229ed4dc0..349690bb1c50 100644 --- a/tools/objtool/include/objtool/builtin.h +++ b/tools/objtool/include/objtool/builtin.h @@ -9,17 +9,19 @@ struct opts { /* actions: */ - bool cfi; bool checksum; + const char *disas; bool dump_orc; bool hack_jump_label; bool hack_noinstr; bool hack_skylake; bool ibt; + bool klp_symids; bool mcount; bool noabs; bool noinstr; bool orc; + int prefix; bool retpoline; bool rethunk; bool unret; @@ -27,14 +29,14 @@ struct opts { bool stackval; bool static_call; bool uaccess; - int prefix; - const char *disas; /* options: */ bool backtrace; bool backup; + bool cfi; const char *debug_checksum; bool dryrun; + bool fineibt; bool link; bool mnop; bool module; diff --git a/tools/objtool/include/objtool/check.h b/tools/objtool/include/objtool/check.h index 5f2f77bd9b41..063f5985fecd 100644 --- a/tools/objtool/include/objtool/check.h +++ b/tools/objtool/include/objtool/check.h @@ -68,6 +68,7 @@ struct instruction { s8 instr; u32 idx : INSN_CHUNK_BITS, + immediate_len : 4, dead_end : 1, ignore_alts : 1, hint : 1, @@ -81,7 +82,7 @@ struct instruction { hole : 1, fake : 1, trace : 1; - /* 9 bit hole */ + /* 4 bit hole */ struct alt_group *alt_group; struct instruction *jump_dest; @@ -94,14 +95,30 @@ struct instruction { }; }; struct alternative *alts; - struct symbol *sym; + struct symbol *_sym; struct stack_op *stack_ops; struct cfi_state *cfi; }; +/* + * Return the symbol associated with an instruction. For alternative + * replacements, return the symbol of the original code being replaced rather + * than NULL. insn->_sym reflects the actual location in the ELF file. + */ +static inline struct symbol *insn_sym(struct instruction *insn) +{ + struct symbol *sym = insn->_sym; + + if ((!sym || !is_func_sym(sym)) && + insn->alt_group && insn->alt_group->orig_group) + sym = insn->alt_group->orig_group->first_insn->_sym; + + return sym; +} + static inline struct symbol *insn_func(struct instruction *insn) { - struct symbol *sym = insn->sym; + struct symbol *sym = insn_sym(insn); if (sym && sym->type != STT_FUNC) sym = NULL; @@ -144,6 +161,12 @@ struct instruction *find_insn(struct objtool_file *file, struct section *sec, unsigned long offset); struct instruction *next_insn_same_sec(struct objtool_file *file, struct instruction *insn); +struct instruction *next_insn_same_func(struct objtool_file *file, struct instruction *insn); + +#define func_for_each_insn(file, func, insn) \ + for (insn = find_insn(file, func->sec, func->offset); \ + insn; \ + insn = next_insn_same_func(file, insn)) #define sec_for_each_insn(file, _sec, insn) \ for (insn = find_insn(file, _sec, 0); \ @@ -155,6 +178,11 @@ struct instruction *next_insn_same_sec(struct objtool_file *file, struct instruc insn && insn->offset < sym->offset + sym->len; \ insn = next_insn_same_sec(file, insn)) +struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn); + +int decode_file(struct objtool_file *file); +void free_insns(struct objtool_file *file); + const char *objtool_disas_insn(struct instruction *insn); extern size_t sym_name_max_len; diff --git a/tools/objtool/include/objtool/checksum.h b/tools/objtool/include/objtool/checksum.h index 7fe21608722a..d46293f54716 100644 --- a/tools/objtool/include/objtool/checksum.h +++ b/tools/objtool/include/objtool/checksum.h @@ -6,37 +6,54 @@ #ifdef BUILD_KLP -static inline void checksum_init(struct symbol *func) +static inline void checksum_init(struct symbol *sym) { - if (func && !func->csum.state) { - func->csum.state = XXH3_createState(); - XXH3_64bits_reset(func->csum.state); + if (sym && !sym->csum.state) { + sym->csum.state = XXH3_createState(); + XXH3_64bits_reset(sym->csum.state); } } -static inline void checksum_update(struct symbol *func, - struct instruction *insn, - const void *data, size_t size) +static inline void __checksum_update(struct symbol *sym, const void *data, + size_t size) { - XXH3_64bits_update(func->csum.state, data, size); - dbg_checksum(func, insn, XXH3_64bits_digest(func->csum.state)); + XXH3_64bits_update(sym->csum.state, data, size); } -static inline void checksum_finish(struct symbol *func) +static inline void __checksum_update_insn(struct symbol *sym, + struct instruction *insn, + const void *data, size_t size) { - if (func && func->csum.state) { - func->csum.checksum = XXH3_64bits_digest(func->csum.state); - func->csum.state = NULL; + __checksum_update(sym, data, size); + dbg_checksum_insn(sym, insn, XXH3_64bits_digest(sym->csum.state)); +} + +static inline void __checksum_update_object(struct symbol *sym, + unsigned long offset, + const char *what, const void *data, + size_t size) +{ + __checksum_update(sym, &offset, sizeof(offset)); + __checksum_update(sym, data, size); + dbg_checksum_object(sym, offset, what, XXH3_64bits_digest(sym->csum.state)); +} + +static inline void checksum_finish(struct symbol *sym) +{ + if (sym && sym->csum.state) { + sym->csum.checksum = XXH3_64bits_digest(sym->csum.state); + XXH3_freeState(sym->csum.state); + sym->csum.state = NULL; } } +int calculate_checksums(struct objtool_file *file); +int create_sym_checksum_section(struct objtool_file *file); + #else /* !BUILD_KLP */ -static inline void checksum_init(struct symbol *func) {} -static inline void checksum_update(struct symbol *func, - struct instruction *insn, - const void *data, size_t size) {} -static inline void checksum_finish(struct symbol *func) {} +static inline int calculate_checksums(struct objtool_file *file) { return -ENOSYS; } +static inline int create_sym_checksum_section(struct objtool_file *file) { return -EINVAL; } #endif /* !BUILD_KLP */ diff --git a/tools/objtool/include/objtool/elf.h b/tools/objtool/include/objtool/elf.h index e12c516bd320..a82517a76a0f 100644 --- a/tools/objtool/include/objtool/elf.h +++ b/tools/objtool/include/objtool/elf.h @@ -21,6 +21,13 @@ #define SEC_NAME_LEN 1024 #define SYM_NAME_LEN 512 +static inline u32 str_hash(const char *str) +{ + return jhash(str, strlen(str), 0); +} + +u32 str_hash_demangled(const char *str); + #define bswap_if_needed(elf, val) __bswap_if_needed(&elf->ehdr, val) #ifdef LIBELF_USE_DEPRECATED @@ -89,6 +96,8 @@ struct symbol { u8 changed : 1; u8 included : 1; u8 klp : 1; + u8 dont_correlate : 1; + u8 fake : 1; struct list_head pv_target; struct reloc *relocs; struct section *group_sec; @@ -130,6 +139,23 @@ struct elf { struct symbol *symbol_data; }; +#define __elf_table(elf, name) ((elf)->name##_hash) +#define __elf_bits(elf, name) ((elf)->name##_bits) + +#define __elf_table_entry(elf, name, key) \ + __elf_table(elf, name)[hash_min(key, __elf_bits(elf, name))] + +#define elf_list_entry(ptr, type, member) \ +({ \ + typeof(ptr) __ptr = (ptr); \ + __ptr ? container_of(__ptr, type, member) : NULL; \ +}) + +#define elf_hash_for_each_possible(elf, name, obj, member, key) \ + for (obj = elf_list_entry(__elf_table_entry(elf, name, key), typeof(*obj), member); \ + obj; \ + obj = elf_list_entry(obj->member.next, typeof(*(obj)), member)) + struct elf *elf_open_read(const char *name, int flags); struct elf *elf_create_file(GElf_Ehdr *ehdr, const char *name); @@ -175,6 +201,7 @@ struct reloc *elf_init_reloc_data_sym(struct elf *elf, struct section *sec, struct symbol *sym, s64 addend); +int elf_write_symbol(struct elf *elf, struct symbol *sym); int elf_write_insn(struct elf *elf, struct section *sec, unsigned long offset, unsigned int len, const char *insn); @@ -187,6 +214,7 @@ struct symbol *find_symbol_by_offset(struct section *sec, unsigned long offset); struct symbol *find_symbol_by_name(const struct elf *elf, const char *name); struct symbol *find_global_symbol_by_name(const struct elf *elf, const char *name); struct symbol *find_symbol_containing(const struct section *sec, unsigned long offset); +struct symbol *find_symbol_containing_inclusive(const struct section *sec, unsigned long offset); int find_symbol_hole_containing(const struct section *sec, unsigned long offset); struct reloc *find_reloc_by_dest(const struct elf *elf, struct section *sec, unsigned long offset); struct reloc *find_reloc_by_dest_range(const struct elf *elf, struct section *sec, @@ -273,11 +301,21 @@ static inline bool is_local_sym(struct symbol *sym) return sym->bind == STB_LOCAL; } +static inline bool is_alias_sym(struct symbol *sym) +{ + return sym->alias != sym; +} + static inline bool is_prefix_func(struct symbol *sym) { return sym->prefix; } +static inline bool is_cold_func(struct symbol *sym) +{ + return sym->cold; +} + static inline bool is_reloc_sec(struct section *sec) { return sec->sh.sh_type == SHT_RELA || sec->sh.sh_type == SHT_REL; @@ -293,6 +331,11 @@ static inline bool is_text_sec(struct section *sec) return sec->sh.sh_flags & SHF_EXECINSTR; } +static inline bool is_rodata_sec(struct section *sec) +{ + return sec->rodata; +} + static inline bool sec_changed(struct section *sec) { return sec->_changed; @@ -465,6 +508,16 @@ static inline void set_sym_next_reloc(struct reloc *reloc, struct reloc *next) #define for_each_sym_continue(elf, sym) \ list_for_each_entry_continue(sym, &elf->symbols, global_list) +#define for_each_sym_by_name(elf, _name, sym) \ + elf_hash_for_each_possible(elf, symbol_name, sym, name_hash, \ + str_hash_demangled(_name)) \ + if (strcmp(sym->name, _name)) {} else + +#define for_each_sym_by_demangled_name(elf, name, sym) \ + elf_hash_for_each_possible(elf, symbol_name, sym, name_hash, \ + str_hash(name)) \ + if (strcmp(sym->demangled_name, name)) {} else + #define rsec_next_reloc(rsec, reloc) \ reloc_idx(reloc) < sec_num_entries(rsec) - 1 ? reloc + 1 : NULL @@ -488,10 +541,10 @@ static inline struct symbol *get_func_prefix(struct symbol *func) { struct symbol *prev; - if (!is_func_sym(func)) + if (!is_func_sym(func) || !func->offset) return NULL; - prev = sec_prev_sym(func); + prev = find_func_containing(func->sec, func->offset - 1); if (prev && is_prefix_func(prev)) return prev; diff --git a/tools/objtool/include/objtool/klp.h b/tools/objtool/include/objtool/klp.h index e32e5e8bc631..c57775d78c71 100644 --- a/tools/objtool/include/objtool/klp.h +++ b/tools/objtool/include/objtool/klp.h @@ -14,21 +14,49 @@ #define KLP_FUNCS_SEC ".init.klp_funcs" /* - * __klp_relocs is an intermediate section which are created by klp diff and - * converted into KLP symbols/relas by "objtool klp post-link". This is needed - * to work around the linker, which doesn't preserve SHN_LIVEPATCH or + * __klp_relocs.<objname> are intermediate sections which are created by klp + * diff and converted into KLP symbols/relas by "objtool klp post-link". This + * is needed to work around the linker, which doesn't preserve SHN_LIVEPATCH or * SHF_RELA_LIVEPATCH, nor does it support having two RELA sections for a * single PROGBITS section. + * + * "objname" is the object whose loading gates the relocation: "vmlinux" for + * references to vmlinux symbols, otherwise the name of the module being + * patched. post-link uses it to name the resulting + * .klp.rela.objname.section_name sections. */ #define KLP_RELOCS_SEC "__klp_relocs" #define KLP_STRINGS_SEC ".rodata.klp.str1.1" +#define KLP_TOMBSTONE_PREFIX ".klp.tombstone." + struct klp_reloc { void *offset; void *sym; u32 type; }; +/* + * .klp.symid is used to correlate symbols between vmlinux.o and vmlinux, for + * calculating sympos to disambiguate duplicately-named symbols. + */ +#define KLP_SYMID_SEC ".klp.symid" + +struct klp_symid { + u64 id; + u64 addr; +}; + +struct objtool_file; +struct elf; +struct symbol; + +int klp_create_symid_sections(struct objtool_file *file); + +int klp_sympos_init(struct elf *orig); +unsigned long klp_find_sympos(struct elf *elf, struct symbol *sym); + +int cmd_klp_checksum(int argc, const char **argv); int cmd_klp_diff(int argc, const char **argv); int cmd_klp_post_link(int argc, const char **argv); diff --git a/tools/objtool/include/objtool/special.h b/tools/objtool/include/objtool/special.h index 121c3761899c..620dbf6cb0e5 100644 --- a/tools/objtool/include/objtool/special.h +++ b/tools/objtool/include/objtool/special.h @@ -32,6 +32,13 @@ int special_get_alts(struct elf *elf, struct list_head *alts); void arch_handle_alternative(struct special_alt *alt); +/* + * Should the reloc at @offset -- the "new" (replacement) field of a special + * section group entry -- be ignored? The meaning of a zero-length replacement + * is arch specific, so the arch decides. + */ +bool arch_alt_ignore_new_reloc(struct section *sec, unsigned long offset); + bool arch_support_alt_relocation(struct special_alt *special_alt, struct instruction *insn, struct reloc *reloc); diff --git a/tools/objtool/include/objtool/warn.h b/tools/objtool/include/objtool/warn.h index fa8b7d292e83..870e147f3a56 100644 --- a/tools/objtool/include/objtool/warn.h +++ b/tools/objtool/include/objtool/warn.h @@ -77,13 +77,13 @@ static inline char *offstr(struct section *sec, unsigned long offset) #define WARN_INSN(insn, format, ...) \ ({ \ struct instruction *_insn = (insn); \ - if (!_insn->sym || !_insn->sym->warned) { \ + if (!insn_sym(_insn) || !insn_sym(_insn)->warned) { \ WARN_FUNC(_insn->sec, _insn->offset, format, \ ##__VA_ARGS__); \ BT_INSN(_insn, ""); \ } \ - if (_insn->sym) \ - _insn->sym->warned = 1; \ + if (insn_sym(_insn)) \ + insn_sym(_insn)->warned = 1; \ }) #define BT_INSN(insn, format, ...) \ @@ -109,7 +109,7 @@ static inline char *offstr(struct section *sec, unsigned long offset) #define ERROR_FUNC(sec, offset, format, ...) __WARN_FUNC(ERROR_STR, sec, offset, format, ##__VA_ARGS__) #define ERROR_INSN(insn, format, ...) ERROR_FUNC(insn->sec, insn->offset, format, ##__VA_ARGS__) -extern bool debug; +extern bool debug, debug_correlate, debug_clone; extern int indent; static inline void unindent(int *unused) { indent--; } @@ -130,32 +130,39 @@ static inline void unindent(int *unused) { indent--; } objname ? ": " : "", \ ##__VA_ARGS__) -#define dbg(args...) \ +#define dbg_checksum_insn(func, insn, checksum) \ ({ \ - if (unlikely(debug)) \ + if (unlikely(func->debug_checksum)) { \ + char *insn_off = offstr(insn->sec, insn->offset); \ + __dbg("checksum: %s(): %s %016llx", \ + func->name, insn_off, (unsigned long long)checksum);\ + free(insn_off); \ + } \ +}) + +#define dbg_checksum_object(sym, offset, what, checksum) \ +({ \ + if (unlikely(sym->debug_checksum)) \ + __dbg("checksum: %s+0x%lx: %s %016llx", \ + sym->name, offset, what, \ + (unsigned long long)checksum); \ +}) + +#define dbg_correlate(args...) \ +({ \ + if (unlikely(debug_correlate)) \ __dbg(args); \ }) -#define __dbg_indent(format, ...) \ +#define __dbg_clone(format, ...) \ ({ \ - if (unlikely(debug)) \ + if (unlikely(debug_clone)) \ __dbg("%*s" format, indent * 8, "", ##__VA_ARGS__); \ }) -#define dbg_indent(args...) \ +#define dbg_clone(args...) \ int __cleanup(unindent) __dummy_##__COUNTER__; \ - __dbg_indent(args); \ + __dbg_clone(args); \ indent++ -#define dbg_checksum(func, insn, checksum) \ -({ \ - if (unlikely(insn->sym && insn->sym->pfunc && \ - insn->sym->pfunc->debug_checksum)) { \ - char *insn_off = offstr(insn->sec, insn->offset); \ - __dbg("checksum: %s %s %016llx", \ - func->name, insn_off, (unsigned long long)checksum);\ - free(insn_off); \ - } \ -}) - #endif /* _WARN_H */ diff --git a/tools/objtool/klp-checksum.c b/tools/objtool/klp-checksum.c new file mode 100644 index 000000000000..ebe25f9c5260 --- /dev/null +++ b/tools/objtool/klp-checksum.c @@ -0,0 +1,368 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include <string.h> +#include <subcmd/parse-options.h> + +#include <objtool/arch.h> +#include <objtool/builtin.h> +#include <objtool/check.h> +#include <objtool/elf.h> +#include <objtool/klp.h> +#include <objtool/objtool.h> +#include <objtool/warn.h> +#include <objtool/checksum.h> + +static int checksum_debug_init(struct objtool_file *file) +{ + char *dup, *s; + + if (!opts.debug_checksum) + return 0; + + dup = strdup(opts.debug_checksum); + if (!dup) { + ERROR_GLIBC("strdup"); + return -1; + } + + s = dup; + while (*s) { + bool found = false; + struct symbol *sym; + char *comma; + + comma = strchr(s, ','); + if (comma) + *comma = '\0'; + + for_each_sym_by_name(file->elf, s, sym) { + if (!is_func_sym(sym) && !is_object_sym(sym)) + continue; + sym->debug_checksum = 1; + found = true; + } + + if (!found) + WARN("--debug-checksum: can't find '%s'", s); + + if (!comma) + break; + + s = comma + 1; + } + + free(dup); + return 0; +} + +/* + * Detect a reference to anonymous constant pool data which the compiler places + * in .rodata.cst<num> and which either has an .LC<num> symbol associated with + * it or (with Clang) no symbol at all. These are typically initializers for + * local function stack data, so they're considered part of the function rather + * than data per se. + */ +static bool is_anonymous_const_data(struct symbol *sym) +{ + return strstarts(sym->sec->name, ".rodata.cst") && + (is_sec_sym(sym) || strstarts(sym->name, ".LC")); +} + +static void checksum_update_insn(struct objtool_file *file, struct symbol *func, + struct instruction *insn) +{ + struct reloc *reloc = insn_reloc(file, insn); + struct alternative *alt; + unsigned long offset; + struct symbol *sym; + static bool in_alt; + + if (insn->fake) + return; + + if (!reloc) { + struct symbol *call_dest = insn_call_dest(insn); + struct instruction *jump_dest = insn->jump_dest; + + /* + * For a jump/call non-relocated dest offset embedded in the + * instruction, the offset may vary due to changes in + * surrounding code. Just hash the opcode and a + * position-independent representation of the destination. + */ + + if (call_dest || jump_dest) { + unsigned char buf[16]; + size_t len; + + len = arch_jump_opcode_bytes(file, insn, buf); + __checksum_update_insn(func, insn, buf, len); + + if (call_dest) { + __checksum_update_insn(func, insn, call_dest->demangled_name, + strlen(call_dest->demangled_name)); + + } else if (jump_dest) { + struct symbol *dest_sym; + unsigned long offset; + + /* + * use insn->_sym instead of insn_sym() here. + * For alternative replacements, the latter + * would give the function of the code being + * replaced. + */ + dest_sym = jump_dest->_sym; + if (!dest_sym) + goto alts; + + __checksum_update_insn(func, insn, dest_sym->demangled_name, + strlen(dest_sym->demangled_name)); + + offset = jump_dest->offset - dest_sym->offset; + __checksum_update_insn(func, insn, &offset, sizeof(offset)); + } + + goto alts; + } + } + + __checksum_update_insn(func, insn, insn->sec->data->d_buf + insn->offset, insn->len); + + if (!reloc) + goto alts; + + sym = reloc->sym; + offset = arch_insn_adjusted_addend(insn, reloc); + + if (is_string_sec(sym->sec)) { + char *str; + + str = sym->sec->data->d_buf + sym->offset + offset; + __checksum_update_insn(func, insn, str, strlen(str)); + goto alts; + } + + if (is_anonymous_const_data(sym)) { + void *cst; + + cst = sym->sec->data->d_buf + sym->offset + offset; + __checksum_update_insn(func, insn, cst, sym->sec->sh.sh_entsize); + goto alts; + } + + if (is_sec_sym(sym)) { + sym = find_symbol_containing(reloc->sym->sec, offset); + if (!sym) + goto alts; + + offset -= sym->offset; + } + + __checksum_update_insn(func, insn, sym->demangled_name, + strlen(sym->demangled_name)); + __checksum_update_insn(func, insn, &offset, sizeof(offset)); + +alts: + for (alt = insn->alts; alt; alt = alt->next) { + struct alt_group *alt_group = alt->insn->alt_group; + + /* Prevent __ex_table recursion, e.g. LOAD_SEGMENT() */ + if (in_alt) + break; + in_alt = true; + + __checksum_update_insn(func, insn, &alt->type, + sizeof(alt->type)); + + if (alt_group && alt_group->orig_group) { + struct instruction *alt_insn; + + __checksum_update_insn(func, insn, &alt_group->feature,sizeof(alt_group->feature)); + + for (alt_insn = alt->insn; alt_insn; alt_insn = next_insn_same_sec(file, alt_insn)) { + checksum_update_insn(file, func, alt_insn); + if (!alt_group->last_insn || alt_insn == alt_group->last_insn) + break; + } + } else { + checksum_update_insn(file, func, alt->insn); + } + + in_alt = false; + } +} + +static void checksum_update_object(struct objtool_file *file, struct symbol *sym) +{ + struct reloc *reloc; + + __checksum_update_object(sym, 0, "len", &sym->len, sizeof(sym->len)); + + if (sym->sec->data->d_buf) + __checksum_update_object(sym, 0, "data", + sym->sec->data->d_buf + sym->offset, + sym->len); + + sym_for_each_reloc(file->elf, sym, reloc) { + unsigned long sym_offset = reloc_offset(reloc) - sym->offset; + struct symbol *target = reloc->sym; + s64 offset; + + offset = reloc_addend(reloc); + + if (is_string_sec(target->sec)) { + char *str; + + str = target->sec->data->d_buf + target->offset + offset; + __checksum_update_object(sym, sym_offset, + "reloc string", str, strlen(str)); + continue; + } + + if (is_sec_sym(target)) { + target = find_symbol_containing(reloc->sym->sec, offset); + if (!target) + continue; + + offset -= target->offset; + } + + __checksum_update_object(sym, sym_offset, "reloc name", + target->demangled_name, + strlen(target->demangled_name)); + __checksum_update_object(sym, sym_offset, "reloc addend", + &offset, sizeof(offset)); + } +} + +int calculate_checksums(struct objtool_file *file) +{ + struct instruction *insn; + struct symbol *sym; + + if (checksum_debug_init(file)) + return -1; + + for_each_sym(file->elf, sym) { + + /* + * Skip cold subfunctions and aliases: they share the + * parent's checksum via func_for_each_insn() which + * follows func->cfunc into the cold subfunction. + */ + if (is_cold_func(sym) || is_alias_sym(sym) || !sym->len || + !sym->sec || !sym->sec->data) + continue; + + if (is_func_sym(sym)) { + checksum_init(sym); + func_for_each_insn(file, sym, insn) + checksum_update_insn(file, sym, insn); + checksum_finish(sym); + + } else if (is_object_sym(sym)) { + checksum_init(sym); + checksum_update_object(file, sym); + checksum_finish(sym); + } + + } + + return 0; +} + +int create_sym_checksum_section(struct objtool_file *file) +{ + struct section *sec; + struct symbol *sym; + unsigned int idx = 0; + struct sym_checksum *checksum; + size_t entsize = sizeof(struct sym_checksum); + + sec = find_section_by_name(file->elf, ".discard.sym_checksum"); + if (sec) { + if (!opts.dryrun) + WARN("file already has .discard.sym_checksum section, skipping"); + + return 0; + } + + for_each_sym(file->elf, sym) + if (sym->csum.checksum) + idx++; + + sec = elf_create_section_pair(file->elf, ".discard.sym_checksum", entsize, + idx, idx); + if (!sec) + return -1; + + idx = 0; + for_each_sym(file->elf, sym) { + if (!sym->csum.checksum) + continue; + + if (!elf_init_reloc(file->elf, sec->rsec, idx, idx * entsize, + sym, 0, R_TEXT64)) + return -1; + + checksum = (struct sym_checksum *)sec->data->d_buf + idx; + checksum->addr = 0; /* reloc */ + checksum->checksum = sym->csum.checksum; + + mark_sec_changed(file->elf, sec, true); + + idx++; + } + + return 0; +} + +static const char * const klp_checksum_usage[] = { + "objtool klp checksum [<options>] file.o", + NULL, +}; + +int cmd_klp_checksum(int argc, const char **argv) +{ + struct objtool_file *file; + int ret; + + const struct option options[] = { + OPT_STRING(0, "debug-checksum", &opts.debug_checksum, "syms", "enable checksum debug output"), + OPT_BOOLEAN(0, "dry-run", &opts.dryrun, "don't write modifications"), + OPT_END(), + }; + + argc = parse_options(argc, argv, options, klp_checksum_usage, 0); + if (argc != 1) + usage_with_options(klp_checksum_usage, options); + + opts.checksum = true; + + objname = argv[0]; + + file = objtool_open_read(objname); + if (!file) + return 1; + + ret = decode_file(file); + if (ret) + goto out; + + ret = calculate_checksums(file); + if (ret) + goto out; + + ret = create_sym_checksum_section(file); + +out: + free_insns(file); + + if (ret) + return ret; + + if (!opts.dryrun && file->elf->changed && elf_write(file->elf)) + return 1; + + return elf_close(file->elf); +} diff --git a/tools/objtool/klp-diff.c b/tools/objtool/klp-diff.c index c2c4e4968bc2..34da55e45609 100644 --- a/tools/objtool/klp-diff.c +++ b/tools/objtool/klp-diff.c @@ -12,7 +12,7 @@ #include <objtool/arch.h> #include <objtool/klp.h> #include <objtool/util.h> -#include <arch/special.h> +#include <objtool/special.h> #include <linux/align.h> #include <linux/objtool_types.h> @@ -30,9 +30,14 @@ struct elfs { struct export { struct hlist_node hash; - char *mod, *sym; + char *mod; + char *sym; + bool mod_ns; }; +bool debug, debug_correlate, debug_clone; +int indent; + static const char * const klp_diff_usage[] = { "objtool klp diff [<options>] <in1.o> <in2.o> <out.o>", NULL, @@ -40,17 +45,14 @@ static const char * const klp_diff_usage[] = { static const struct option klp_diff_options[] = { OPT_GROUP("Options:"), - OPT_BOOLEAN('d', "debug", &debug, "enable debug output"), + OPT_BOOLEAN('d', "debug", &debug, "enable all debug output"), + OPT_BOOLEAN(0, "debug-correlate", &debug_correlate, "enable correlation debug output"), + OPT_BOOLEAN(0, "debug-clone", &debug_clone, "enable cloning debug output"), OPT_END(), }; static DEFINE_HASHTABLE(exports, 15); -static inline u32 str_hash(const char *str) -{ - return jhash(str, strlen(str), 0); -} - static char *escape_str(const char *orig) { size_t len = 0; @@ -83,11 +85,40 @@ static char *escape_str(const char *orig) return new; } +/* + * Convert a build-tree object path to a runtime module name: strip + * directory components, replace '-' with '_', and remove file + * extensions. Examples: + * + * "arch/x86/kvm/kvm" -> "kvm" + * "arch/x86/kvm/kvm-intel" -> "kvm_intel". + * + * Used by read_exports() to normalize Module.symvers entries and by + * __find_modname() as a fallback when .modinfo lacks a "name=" tag. + */ +static char *normalize_modname(char *name) +{ + char *slash = strrchr(name, '/'); + + if (slash) + name = slash + 1; + + for (char *c = name; *c; c++) { + if (*c == '-') + *c = '_'; + else if (*c == '.') { + *c = '\0'; + break; + } + } + return name; +} + static int read_exports(void) { const char *symvers = "Module.symvers"; char line[1024], *path = NULL; - unsigned int line_num = 1; + unsigned int line_num = 0; FILE *file; file = fopen(symvers, "r"); @@ -106,9 +137,11 @@ static int read_exports(void) } while (fgets(line, 1024, file)) { - char *sym, *mod, *type; + char *sym, *mod, *type, *namespace; struct export *export; + line_num++; + sym = strchr(line, '\t'); if (!sym) { ERROR("malformed Module.symvers (sym) at line %d", line_num); @@ -133,6 +166,14 @@ static int read_exports(void) *type++ = '\0'; + namespace = strchr(type, '\t'); + if (!namespace) { + ERROR("malformed Module.symvers (namespace) at line %d", line_num); + return -1; + } + + *namespace++ = '\0'; + if (*sym == '\0' || *mod == '\0') { ERROR("malformed Module.symvers at line %d", line_num); return -1; @@ -150,12 +191,18 @@ static int read_exports(void) return -1; } + if (strcmp(export->mod, "vmlinux")) + export->mod = normalize_modname(export->mod); + export->sym = strdup(sym); if (!export->sym) { ERROR_GLIBC("strdup"); return -1; } + /* EXPORT_SYMBOL_FOR_MODULES() */ + export->mod_ns = strstarts(namespace, "module:"); + hash_add(exports, &export->hash, str_hash(sym)); } @@ -171,7 +218,7 @@ static int read_sym_checksums(struct elf *elf) sec = find_section_by_name(elf, ".discard.sym_checksum"); if (!sec) { - ERROR("'%s' missing .discard.sym_checksum section, file not processed by 'objtool --checksum'?", + ERROR("'%s' missing .discard.sym_checksum section, file not processed by 'objtool klp checksum'?", elf->name); return -1; } @@ -206,7 +253,7 @@ static int read_sym_checksums(struct elf *elf) return -1; } - if (is_func_sym(sym)) + if (is_func_sym(sym) || is_object_sym(sym)) sym->csum.checksum = sym_checksum->checksum; } @@ -242,25 +289,39 @@ static struct symbol *next_file_symbol(struct elf *elf, struct symbol *sym) static bool is_uncorrelated_static_local(struct symbol *sym) { static const char * const vars[] = { - "__already_done.", - "__func__.", - "__key.", - "__warned.", - "_entry.", - "_entry_ptr.", - "_rs.", - "descriptor.", - "CSWTCH.", + "__already_done", + "__func__", + "__key", + "__warned", + "_entry", + "_entry_ptr", + "_rs", + "descriptor", + "CSWTCH", }; + const char *dot; if (!is_object_sym(sym) || !is_local_sym(sym)) return false; - if (!strcmp(sym->sec->name, ".data.once")) + /* WARN_ONCE, etc */ + if (!strcmp(sym->sec->name, ".data..once")) return true; + dot = strchr(sym->name, '.'); + if (!dot) + return false; + for (int i = 0; i < ARRAY_SIZE(vars); i++) { - if (strstarts(sym->name, vars[i])) + size_t len = strlen(vars[i]); + + /* GCC: <var>.<id> */ + if (strstarts(sym->name, vars[i]) && (sym->name[len] == '.')) + return true; + + /* Clang: <func>.<var>[.<id>] */ + if (strstarts(dot + 1, vars[i]) && + (dot[1 + len] == '.' || dot[1 + len] == '\0')) return true; } @@ -268,21 +329,21 @@ static bool is_uncorrelated_static_local(struct symbol *sym) } /* - * Clang emits several useless .Ltmp_* code labels. + * .L symbols are assembler-local labels not present in kallsyms. They must + * never become KLP relocations; instead their data is cloned into the patch + * module. This covers .Ltmp* (Clang temp labels), .L__const.* (Clang local + * constants), and any other assembler-local pattern. */ -static bool is_clang_tmp_label(struct symbol *sym) +static bool is_local_label(struct symbol *sym) { - return sym->type == STT_NOTYPE && - is_text_sec(sym->sec) && - strstarts(sym->name, ".Ltmp") && - isdigit(sym->name[5]); + return strstarts(sym->name, ".L"); } static bool is_special_section(struct section *sec) { static const char * const specials[] = { ".altinstructions", - ".smp_locks", + ".kcfi_traps", "__bug_table", "__ex_table", "__jump_table", @@ -339,6 +400,45 @@ static bool is_special_section_aux(struct section *sec) } /* + * Symbols created by ___ADDRESSABLE() are only used to convince the toolchain + * not to optimize out the referenced symbol. + */ +static bool is_addressable_sym(struct symbol *sym) +{ + return !strcmp(sym->sec->name, ".discard.addressable"); +} + +/* + * ABS symbols are typically assembly .set/.equ constants which are never + * referenced by relocations. (Exclude FILE symbols which are also SHN_ABS.) + */ +static bool is_abs_sym(struct symbol *sym) +{ + return sym->sym.st_shndx == SHN_ABS && !is_file_sym(sym); +} + +static bool is_initcall_sym(struct symbol *sym) +{ + return strstarts(sym->name, "__initcall__") || + strstarts(sym->name, "__initstub__"); +} + +/* + * Some .rodata is anonymous and can't be correlated due to there being no + * symbol names. + * + * The .rodata.cst* sections aren't technically anonymous, they're SHF_MERGE + * constant pool sections containing small fixed-size data (lookup tables, + * bitmasks) which are only read by value, so pointer equivalence isn't needed. + * They are typically referenced by UBSAN data sections. + */ +static bool is_anonymous_rodata(struct symbol *sym) +{ + return is_rodata_sec(sym->sec) && + (!is_object_sym(sym) || strstarts(sym->sec->name, ".rodata.cst")); +} + +/* * These symbols should never be correlated, so their local patched versions * are used instead of linking to the originals. */ @@ -347,56 +447,391 @@ static bool dont_correlate(struct symbol *sym) return is_file_sym(sym) || is_null_sym(sym) || is_sec_sym(sym) || + is_abs_sym(sym) || is_prefix_func(sym) || is_uncorrelated_static_local(sym) || - is_clang_tmp_label(sym) || + is_local_label(sym) || is_string_sec(sym->sec) || + is_anonymous_rodata(sym) || + is_initcall_sym(sym) || + is_addressable_sym(sym) || is_special_section(sym->sec) || - is_special_section_aux(sym->sec) || - strstarts(sym->name, "__initcall__"); + is_special_section_aux(sym->sec); +} + +static const char *llvm_suffix(const char *name) +{ + return strstr(name, ".llvm."); +} + +static bool is_llvm_sym(struct symbol *sym) +{ + return llvm_suffix(sym->name); } /* - * For each symbol in the original kernel, find its corresponding "twin" in the - * patched kernel. + * Determine if two symbols have compatible source file origins: + * + * - If both symbols are local, only return true if they belong to the same + * ELF file symbol. + * + * - If both symbols are global, always return true, as globals don't have + * file associations. + * + * - If they have different scopes, also return true, as the patch might have + * changed the symbol's scope. + * + * Works for both same-ELF (direct pointer compare) and cross-ELF + * (compare via file->twin) cases. */ -static int correlate_symbols(struct elfs *e) +static bool maybe_same_file(struct symbol *sym1, struct symbol *sym2) { - struct symbol *file1_sym, *file2_sym; - struct symbol *sym1, *sym2; + if (!sym1->file || !sym2->file) + return true; + if (sym1->file == sym2->file) + return true; + return sym1->file->twin == sym2->file; +} - file1_sym = first_file_symbol(e->orig); - file2_sym = first_file_symbol(e->patched); +/* + * Similar to maybe_same_file(), but strict: no scope changes allowed. + * + * Works for both same-ELF (direct pointer compare) and cross-ELF + * (compare via file->twin) cases. + */ +static bool same_file(struct symbol *sym1, struct symbol *sym2) +{ + if (llvm_suffix(sym1->name) && llvm_suffix(sym2->name)) + return true; + if (!sym1->file && !sym2->file) + return true; + if (!sym1->file || !sym2->file) + return false; + if (sym1->file == sym2->file) + return true; + return sym1->file->twin == sym2->file; +} - /* - * Correlate any locals before the first FILE symbol. This has been - * seen when LTO inexplicably strips the initramfs_data.o FILE symbol - * due to the file only containing data and no code. - */ - for_each_sym(e->orig, sym1) { - if (sym1 == file1_sym || !is_local_sym(sym1)) - break; +/* + * Is it a local symbol, or at least was it local in the translation unit + * before LLVM promoted it? + */ +static bool is_tu_local_sym(struct symbol *sym) +{ + return is_local_sym(sym) || is_llvm_sym(sym); +} + +/* + * Try to find sym1's twin in patched using deterministic matching. + * + * Multiple symbols can share a demangled name (e.g., static functions in + * different TUs). This function counts same-named candidates through a + * funnel of progressively tighter filters. Each level is a strict subset + * of the previous one. + * + * The widest level that yields a 1:1 match wins. Narrower levels are only + * needed when the wider level is ambiguous (count > 1). + * + * Candidates are pre-filtered by maybe_same_file(), which narrows most + * local symbols to their own TU. For example, 19 different static + * type_show() functions across vmlinux.o each see only one candidate after + * pre-filtering, so they match immediately at Level 1. + * + * Level 1 (name): Works when the demangled name is unique after + * pre-filtering. Handles most symbols: unique globals like copy_signal(), + * or per-TU locals like pcspkr_probe(). + * + * Level 2 (scope): Filters by local-vs-global (TU-local-vs-not). Example: + * parse_header() exists as both a static and a global function. Level 1 + * sees both (same demangled name), but Level 2 separates them by scope. + * + * Level 3 (file): Strict file matching via same_file(), which rejects scope + * changes. Example: LLVM-promoted foo.llvm.12345 (global, no FILE symbol) + * vs genuine local foo (has FILE symbol). Both are TU-local so Level 2 + * can't distinguish them, but same_file() rejects the pair because one has + * a file association and the other doesn't. + * + * Level 4 (checksum): Distinguishes by function checksum. Example: + * usb_devnode.llvm.AAA and usb_devnode.llvm.BBB are two LLVM-promoted + * functions from different TUs with the same demangled name. After a TU + * change, the .llvm. hashes change but the functions themselves may be + * unchanged. Level 4 matches each to the patched candidate with the + * same checksum. + */ +static struct symbol *find_twin(struct elfs *e, struct symbol *sym1) +{ + struct symbol *name_last = NULL, *scope_last = NULL, + *file_last = NULL, *csum_last = NULL; + unsigned int name_orig = 0, name_patched = 0; + unsigned int scope_orig = 0, scope_patched = 0; + unsigned int file_orig = 0, file_patched = 0; + unsigned int csum_orig = 0, csum_patched = 0; + struct symbol *sym2, *match = NULL; + + /* Count orig candidates */ + for_each_sym_by_demangled_name(e->orig, sym1->demangled_name, sym2) { + if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate || + (!maybe_same_file(sym1, sym2))) + continue; + + /* Level 1: name match (widest filter) */ + name_orig++; + + /* Level 2: scope (scope changes allowed) */ + if (is_tu_local_sym(sym1) != is_tu_local_sym(sym2)) + continue; + scope_orig++; + + /* Level 3: file (scope changes disallowed) */ + if (!same_file(sym1, sym2)) + continue; + file_orig++; + + /* Level 4: checksum (unchanged symbols) */ + if (sym1->len != sym2->len || !sym1->csum.checksum || + sym1->csum.checksum != sym2->csum.checksum) + continue; + csum_orig++; + } + + /* Count patched candidates */ + for_each_sym_by_demangled_name(e->patched, sym1->demangled_name, sym2) { + if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate || + !maybe_same_file(sym1, sym2)) + continue; + + /* Level 1 */ + name_patched++; + name_last = sym2; + + /* Level 2 */ + if (is_tu_local_sym(sym1) != is_tu_local_sym(sym2)) + continue; + scope_patched++; + scope_last = sym2; + + /* Level 3 */ + if (!same_file(sym1, sym2)) + continue; + file_patched++; + file_last = sym2; + + /* Level 4 */ + if (sym1->len != sym2->len || !sym1->csum.checksum || + sym1->csum.checksum != sym2->csum.checksum) + continue; + csum_patched++; + csum_last = sym2; + } + + /* Return the widest level that yields a unique (1:1) match */ + if (name_orig == 1 && name_patched == 1) + match = name_last; + else if (scope_orig == 1 && scope_patched == 1) + match = scope_last; + else if (file_orig == 1 && file_patched == 1) + match = file_last; + else if (csum_orig == 1 && csum_patched == 1) + match = csum_last; + + if (!match) + return NULL; - if (dont_correlate(sym1)) + if (name_orig != 1 || name_patched != 1) + dbg_correlate("find_twin(): %s%s -> %s%s", + sym1->name, is_func_sym(sym1) ? "()" : "", + match->name, is_func_sym(match) ? "()" : ""); + + return match; +} + +struct llvm_suffix_pair { + struct hlist_node hash; + const char *orig; + const char *patched; +}; + +static DECLARE_HASHTABLE(suffix_map, 7); + +/* + * Build a mapping of known orig-to-patched LLVM suffixes based on + * already-correlated symbol pairs. All promoted symbols from the same TU + * share the same .llvm.<hash> suffix, so one correlated pair seeds the map + * for the entire TU. + */ +static int update_suffix_map(struct elf *elf) +{ + struct llvm_suffix_pair *entry; + struct symbol *sym; + + for_each_sym(elf, sym) { + const char *s1, *s2; + bool found; + + if (!sym->twin) + continue; + + s1 = llvm_suffix(sym->name); + s2 = llvm_suffix(sym->twin->name); + + if (!s1 || !s2) continue; - for_each_sym(e->patched, sym2) { - if (sym2 == file2_sym || !is_local_sym(sym2)) + found = false; + hash_for_each_possible(suffix_map, entry, hash, str_hash(s1)) { + if (!strcmp(entry->orig, s1)) { + found = true; break; + } + } + if (found) + continue; - if (sym2->twin || dont_correlate(sym2)) - continue; + entry = calloc(1, sizeof(*entry)); + if (!entry) { + ERROR_GLIBC("calloc"); + return -1; + } - if (strcmp(sym1->demangled_name, sym2->demangled_name)) - continue; + entry->orig = s1; + entry->patched = s2; + hash_add(suffix_map, &entry->hash, str_hash(s1)); + } - sym1->twin = sym2; - sym2->twin = sym1; + return 0; +} + +/* + * Match by translating the symbol's .llvm.<hash> suffix through the suffix + * map to find the corresponding hash suffix for the patched object. + * + * Example: In the original kernel, TU drivers/base/core.c contains + * foo.llvm.12345 and bar.llvm.12345 (same TU, same hash). After patching, + * they become foo.llvm.67890 and bar.llvm.67890. If foo was already + * correlated by find_twin() (e.g., unique by name), the suffix map records + * .llvm.12345 -> .llvm.67890. When processing bar.llvm.12345, this + * function looks up .llvm.12345, gets .llvm.67890, constructs the name + * bar.llvm.67890, and finds the match. + */ +static struct symbol *find_twin_suffixed(struct elf *elf, struct symbol *sym1) +{ + const char *suffix, *patched_suffix = NULL; + struct symbol *sym2, *match = NULL; + char name[SYM_NAME_LEN]; + struct llvm_suffix_pair *entry; + int count = 0; + + suffix = llvm_suffix(sym1->name); + if (!suffix) + return NULL; + + hash_for_each_possible(suffix_map, entry, hash, str_hash(suffix)) { + if (!strcmp(entry->orig, suffix)) { + patched_suffix = entry->patched; break; } } + if (!patched_suffix) + return NULL; + + if (snprintf_check(name, SYM_NAME_LEN, "%s%s", + sym1->demangled_name, patched_suffix)) + return NULL; + + for_each_sym_by_name(elf, name, sym2) { + if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate) + continue; + count++; + match = sym2; + } + + if (count != 1) + return NULL; + + dbg_correlate("find_suffixed_twin(): %s%s -> %s%s", + sym1->name, is_func_sym(sym1) ? "()" : "", + match->name, is_func_sym(match) ? "()" : ""); + + return match; +} + +/* + * Last-resort positional matching. + * + * Finds a symbol with the same position in the symbol table among + * same-demangled-name candidates, similar to livepatch sympos. Note that + * LLVM-promoted symbols are globals, which come after locals in the symbol + * table, so we have to be careful not to compare different scopes. + * + * Example: arch/x86/events/intel/core.c defines many __quirk variables via + * X86_MATCH_*() macros. In the symbol table they appear as __quirk.90, + * __quirk.97, __quirk.101, etc., all with demangled name __quirk, same + * scope, and same FILE symbol. No deterministic filter can distinguish + * them, so they're matched by position: the 1st __quirk in orig matches the + * 1st in patched, the 2nd matches the 2nd, etc. + * + * This is less deterministic than the other strategies, so it's done last. + */ +static struct symbol *find_twin_positional(struct elfs *e, struct symbol *sym1) +{ + unsigned int idx_orig = 0, idx_patched = 0; + unsigned int sym1_pos = 0; + struct symbol *sym2, *match = NULL; + + for_each_sym_by_demangled_name(e->orig, sym1->demangled_name, sym2) { + if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate || + !maybe_same_file(sym1, sym2)) + continue; + if (is_tu_local_sym(sym1) != is_tu_local_sym(sym2) || + is_llvm_sym(sym1) != is_llvm_sym(sym2)) + continue; + if (sym1 == sym2) + sym1_pos = idx_orig; + idx_orig++; + } + + for_each_sym_by_demangled_name(e->patched, sym1->demangled_name, sym2) { + if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate || + !maybe_same_file(sym1, sym2)) + continue; + if (is_tu_local_sym(sym1) != is_tu_local_sym(sym2) || + is_llvm_sym(sym1) != is_llvm_sym(sym2)) + continue; + if (idx_patched == sym1_pos) + match = sym2; + idx_patched++; + } + + if (idx_orig != idx_patched) + return NULL; + + dbg_correlate("find_twin_positional(): %s%s -> %s%s", + sym1->name, is_func_sym(sym1) ? "()" : "", + match->name, is_func_sym(match) ? "()" : ""); + + return match; +} + +/* + * Correlate symbols between the orig and patched objects. This is a + * prerequisite for detecting changed functions, as well as for properly + * translating relocations so they point to the correct symbol. + */ +static int correlate_symbols(struct elfs *e) +{ + struct symbol *file1_sym, *file2_sym; + struct symbol *sym1, *sym2; + bool progress; + + for_each_sym(e->orig, sym1) + sym1->dont_correlate = dont_correlate(sym1); + for_each_sym(e->patched, sym2) + sym2->dont_correlate = dont_correlate(sym2); + + /* Correlate FILE symbols */ + file1_sym = first_file_symbol(e->orig); + file2_sym = first_file_symbol(e->patched); - /* Correlate locals after the first FILE symbol */ for (; ; file1_sym = next_file_symbol(e->orig, file1_sym), file2_sym = next_file_symbol(e->patched, file2_sym)) { @@ -420,49 +855,56 @@ static int correlate_symbols(struct elfs *e) file1_sym->twin = file2_sym; file2_sym->twin = file1_sym; + } - sym1 = file1_sym; - - for_each_sym_continue(e->orig, sym1) { - if (is_file_sym(sym1) || !is_local_sym(sym1)) - break; - if (dont_correlate(sym1)) + /* + * Correlate in two phases: loop deterministic levels until no more + * progress, then use positional fallback for the rest. This prevents + * the nondeterministic positional matching from stealing symbols that + * have deterministic matches. + */ + hash_init(suffix_map); + do { + progress = false; + for_each_sym(e->orig, sym1) { + if (sym1->twin || sym1->dont_correlate) continue; - - sym2 = file2_sym; - for_each_sym_continue(e->patched, sym2) { - if (is_file_sym(sym2) || !is_local_sym(sym2)) - break; - - if (sym2->twin || dont_correlate(sym2)) - continue; - - if (strcmp(sym1->demangled_name, sym2->demangled_name)) - continue; - - sym1->twin = sym2; - sym2->twin = sym1; - break; - } + sym2 = find_twin(e, sym1); + if (!sym2) + continue; + sym1->twin = sym2; + sym2->twin = sym1; + progress = true; } - } - - /* Correlate globals */ - for_each_sym(e->orig, sym1) { - if (sym1->bind == STB_LOCAL) - continue; - sym2 = find_global_symbol_by_name(e->patched, sym1->name); + if (update_suffix_map(e->orig)) + return -1; - if (sym2 && !sym2->twin && !strcmp(sym1->name, sym2->name)) { + for_each_sym(e->orig, sym1) { + if (sym1->twin || sym1->dont_correlate) + continue; + sym2 = find_twin_suffixed(e->patched, sym1); + if (!sym2) + continue; sym1->twin = sym2; sym2->twin = sym1; + progress = true; } + } while (progress); + + for_each_sym(e->orig, sym1) { + if (sym1->twin || sym1->dont_correlate) + continue; + sym2 = find_twin_positional(e, sym1); + if (!sym2) + continue; + sym1->twin = sym2; + sym2->twin = sym1; } for_each_sym(e->orig, sym1) { - if (sym1->twin || dont_correlate(sym1)) + if (sym1->twin || sym1->dont_correlate) continue; WARN("no correlation: %s", sym1->name); } @@ -470,65 +912,6 @@ static int correlate_symbols(struct elfs *e) return 0; } -/* "sympos" is used by livepatch to disambiguate duplicate symbol names */ -static unsigned long find_sympos(struct elf *elf, struct symbol *sym) -{ - bool vmlinux = str_ends_with(objname, "vmlinux.o"); - unsigned long sympos = 0, nr_matches = 0; - bool has_dup = false; - struct symbol *s; - - if (sym->bind != STB_LOCAL) - return 0; - - if (vmlinux && sym->type == STT_FUNC) { - /* - * HACK: Unfortunately, symbol ordering can differ between - * vmlinux.o and vmlinux due to the linker script emitting - * .text.unlikely* before .text*. Count .text.unlikely* first. - * - * TODO: Disambiguate symbols more reliably (checksums?) - */ - for_each_sym(elf, s) { - if (strstarts(s->sec->name, ".text.unlikely") && - !strcmp(s->name, sym->name)) { - nr_matches++; - if (s == sym) - sympos = nr_matches; - else - has_dup = true; - } - } - for_each_sym(elf, s) { - if (!strstarts(s->sec->name, ".text.unlikely") && - !strcmp(s->name, sym->name)) { - nr_matches++; - if (s == sym) - sympos = nr_matches; - else - has_dup = true; - } - } - } else { - for_each_sym(elf, s) { - if (!strcmp(s->name, sym->name)) { - nr_matches++; - if (s == sym) - sympos = nr_matches; - else - has_dup = true; - } - } - } - - if (!sympos) { - ERROR("can't find sympos for %s", sym->name); - return ULONG_MAX; - } - - return has_dup ? sympos : 0; -} - static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym); static struct symbol *__clone_symbol(struct elf *elf, struct symbol *patched_sym, @@ -568,7 +951,7 @@ static struct symbol *__clone_symbol(struct elf *elf, struct symbol *patched_sym size_t size; /* bss doesn't have data */ - if (patched_sym->sec->data->d_buf) + if (patched_sym->sec->data && patched_sym->sec->data->d_buf) data = patched_sym->sec->data->d_buf + patched_sym->offset; if (is_sec_sym(patched_sym)) @@ -628,7 +1011,7 @@ static struct symbol *clone_symbol(struct elfs *e, struct symbol *patched_sym, if (patched_sym->clone) return patched_sym->clone; - dbg_indent("%s%s", patched_sym->name, data_too ? " [+DATA]" : ""); + dbg_clone("%s%s", patched_sym->name, data_too ? " [+DATA]" : ""); /* Make sure the prefix gets cloned first */ if (is_func_sym(patched_sym) && data_too) { @@ -670,19 +1053,24 @@ static void mark_included_function(struct symbol *func) */ static int mark_changed_functions(struct elfs *e) { - struct symbol *sym_orig, *patched_sym; + struct symbol *orig_sym, *patched_sym; bool changed = false; /* Find changed functions */ - for_each_sym(e->orig, sym_orig) { - if (!is_func_sym(sym_orig) || is_prefix_func(sym_orig)) + for_each_sym(e->orig, orig_sym) { + if (orig_sym->dont_correlate) continue; - patched_sym = sym_orig->twin; + patched_sym = orig_sym->twin; if (!patched_sym) continue; - if (sym_orig->csum.checksum != patched_sym->csum.checksum) { + if (orig_sym->csum.checksum != patched_sym->csum.checksum) { + if (!is_func_sym(orig_sym)) { + ERROR("changed data: %s", orig_sym->name); + return -1; + } + patched_sym->changed = 1; mark_included_function(patched_sym); changed = true; @@ -691,7 +1079,7 @@ static int mark_changed_functions(struct elfs *e) /* Find added functions and print them */ for_each_sym(e->patched, patched_sym) { - if (!is_func_sym(patched_sym) || is_prefix_func(patched_sym)) + if (!is_func_sym(patched_sym) || patched_sym->dont_correlate) continue; if (!patched_sym->twin) { @@ -707,7 +1095,7 @@ static int mark_changed_functions(struct elfs *e) printf("%s: changed function: %s\n", objname, patched_sym->name); } - return !changed ? -1 : 0; + return !changed ? 1 : 0; } static int clone_included_functions(struct elfs *e) @@ -724,43 +1112,13 @@ static int clone_included_functions(struct elfs *e) return 0; } -/* - * Determine whether a relocation should reference the section rather than the - * underlying symbol. - */ -static bool section_reference_needed(struct section *sec) -{ - /* - * String symbols are zero-length and uncorrelated. It's easier to - * deal with them as section symbols. - */ - if (is_string_sec(sec)) - return true; - - /* - * .rodata has mostly anonymous data so there's no way to determine the - * length of a needed reference. just copy the whole section if needed. - */ - if (strstarts(sec->name, ".rodata")) - return true; - - /* UBSAN anonymous data */ - if (strstarts(sec->name, ".data..Lubsan") || /* GCC */ - strstarts(sec->name, ".data..L__unnamed_")) /* Clang */ - return true; - - return false; -} - -static bool is_reloc_allowed(struct reloc *reloc) -{ - return section_reference_needed(reloc->sym->sec) == is_sec_sym(reloc->sym); -} - static struct export *find_export(struct symbol *sym) { struct export *export; + if (is_local_sym(sym)) + return NULL; + hash_for_each_possible(exports, export, hash, str_hash(sym->name)) { if (!strcmp(export->sym, sym->name)) return export; @@ -790,18 +1148,7 @@ static const char *__find_modname(struct elfs *e) return NULL; } - for (char *c = name; *c; c++) { - if (*c == '/') - name = c + 1; - else if (*c == '-') - *c = '_'; - else if (*c == '.') { - *c = '\0'; - break; - } - } - - return name; + return normalize_modname(name); } /* Get the object's module name as defined by the kernel (and klp_object) */ @@ -830,7 +1177,7 @@ static bool klp_reloc_needed(struct reloc *patched_reloc) struct export *export; /* no external symbol to reference */ - if (dont_correlate(patched_sym)) + if (patched_sym->dont_correlate) return false; /* For included functions, a regular reloc will do. */ @@ -842,11 +1189,16 @@ static bool klp_reloc_needed(struct reloc *patched_reloc) * clusterfunk that is late module patching, the patch module is * allowed to be loaded before any modules it depends on. * - * If exported by vmlinux, a normal reloc will do. + * If exported by vmlinux to all modules, a normal reloc will do. */ export = find_export(patched_sym); - if (export) - return strcmp(export->mod, "vmlinux"); + if (export) { + if (strcmp(export->mod, "vmlinux")) + return true; + + /* EXPORT_SYMBOL_FOR_MODULES() gets a klp reloc */ + return export->mod_ns; + } if (!patched_sym->twin) { /* @@ -865,34 +1217,60 @@ static bool klp_reloc_needed(struct reloc *patched_reloc) return true; } +/* Return -1 error, 0 success, 1 skip */ static int convert_reloc_sym_to_secsym(struct elf *elf, struct reloc *reloc) { struct symbol *sym = reloc->sym; struct section *sec = sym->sec; + if (is_sec_sym(sym)) + return 0; + if (!sec->sym && !elf_create_section_symbol(elf, sec)) return -1; reloc->sym = sec->sym; - set_reloc_sym(elf, reloc, sym->idx); + set_reloc_sym(elf, reloc, sec->sym->idx); set_reloc_addend(elf, reloc, sym->offset + reloc_addend(reloc)); return 0; } +/* Return -1 error, 0 success, 1 skip */ static int convert_reloc_secsym_to_sym(struct elf *elf, struct reloc *reloc) { struct symbol *sym = reloc->sym; struct section *sec = sym->sec; + if (!is_sec_sym(sym)) + return 0; + /* If the symbol has a dedicated section, it's easy to find */ sym = find_symbol_by_offset(sec, 0); if (sym && sym->len == sec_size(sec)) goto found_sym; /* No dedicated section; find the symbol manually */ - sym = find_symbol_containing(sec, arch_adjusted_addend(reloc)); + sym = find_symbol_containing_inclusive(sec, arch_adjusted_addend(reloc)); if (!sym) { /* + * This is presumably an .altinstr_replacement section which is + * empty due to it only having zero-length replacement(s). + */ + if (!sec_size(sec)) + return 1; + + /* + * .rodata is a mixed bag of named objects and anonymous data. + * + * Convert section symbol references to named object symbols + * when possible, to preserve pointer identity for const + * structs like file_operations. Otherwise a section symbol is + * fine. + */ + if (is_rodata_sec(sec)) + return 0; + + /* * This can happen for special section references to weak code * whose symbol has been stripped by the linker. */ @@ -907,18 +1285,55 @@ found_sym: } /* + * Sections with anonymous or uncorrelated data (strings, UBSAN data, Clang + * anonymous constants) need section symbol references. + */ +static bool is_uncorrelated_section(struct section *sec) +{ + return is_string_sec(sec) || + strstarts(sec->name, ".data..Lubsan") || /* GCC */ + strstarts(sec->name, ".data..L__unnamed_") || /* Clang */ + strstarts(sec->name, ".data..Lanon."); /* Clang */ +} + +/* * Convert a relocation symbol reference to the needed format: either a section - * symbol or the underlying symbol itself. + * symbol or the underlying symbol itself. Return -1 error, 0 success, 1 skip. */ static int convert_reloc_sym(struct elf *elf, struct reloc *reloc) { - if (is_reloc_allowed(reloc)) - return 0; + struct section *sec = reloc->sym->sec; + + if (reloc_type(reloc) == R_NONE) + return 1; - if (section_reference_needed(reloc->sym->sec)) + if (is_uncorrelated_section(sec)) return convert_reloc_sym_to_secsym(elf, reloc); - else - return convert_reloc_secsym_to_sym(elf, reloc); + + /* Everything else: references should use named symbols. */ + return convert_reloc_secsym_to_sym(elf, reloc); +} + +/* + * Check if the original module already has a dependency on dep_mod, i.e. it + * already references at least one export from that module. + */ +static bool has_module_dep(struct elfs *e, const char *dep_mod) +{ + struct symbol *sym; + + for_each_sym(e->orig, sym) { + struct export *exp; + + if (!is_undef_sym(sym) || is_weak_sym(sym)) + continue; + + exp = find_export(sym); + if (exp && !strcmp(exp->mod, dep_mod)) + return true; + } + + return false; } /* @@ -928,33 +1343,51 @@ static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc, struct section *sec, unsigned long offset, struct export *export) { + const char *sym_modname, *sym_orig_name, *sec_objname; struct symbol *patched_sym = patched_reloc->sym; s64 addend = reloc_addend(patched_reloc); - const char *sym_modname, *sym_orig_name; - static struct section *klp_relocs; + char tombstone_name[SYM_NAME_LEN]; struct symbol *sym, *klp_sym; unsigned long klp_reloc_off; + struct section *klp_relocs; + char sec_name[SEC_NAME_LEN]; char sym_name[SYM_NAME_LEN]; struct klp_reloc klp_reloc; unsigned long sympos; if (!patched_sym->twin) { - ERROR("unexpected klp reloc for new symbol %s", patched_sym->name); - return -1; + if (!export) { + ERROR("unexpected klp reloc for new symbol %s", patched_sym->name); + return -1; + } + + if (strcmp(export->mod, "vmlinux") && + !has_module_dep(e, export->mod)) { + ERROR("%s: new reference to %s (exported by %s) would create an undeclared module dependency", + patched_sym->name, export->sym, export->mod); + return -1; + } } /* * Keep the original reloc intact for now to avoid breaking objtool run * which relies on proper relocations for many of its features. This - * will be disabled later by "objtool klp post-link". + * reloc now targets a functionally dead tombstone symbol and will be + * disabled later by "objtool klp post-link". * - * Convert it to UNDEF (and WEAK to avoid modpost warnings). + * Convert the symbol to UNDEF/WEAK and rename to + * .klp.tombstone.sym_name to prevent modpost from printing warnings or + * creating false module dependencies. The prefix is hidden from the + * objtool run itself by read_symbols(). */ sym = patched_sym->clone; if (!sym) { - /* STB_WEAK: avoid modpost undefined symbol warnings */ - sym = elf_create_symbol(e->out, patched_sym->name, NULL, + if (snprintf_check(tombstone_name, SYM_NAME_LEN, + KLP_TOMBSTONE_PREFIX "%s", patched_sym->name)) + return -1; + + sym = elf_create_symbol(e->out, tombstone_name, NULL, STB_WEAK, patched_sym->type, 0, 0); if (!sym) return -1; @@ -980,7 +1413,7 @@ static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc, return -1; sym_orig_name = patched_sym->twin->name; - sympos = find_sympos(e->orig, patched_sym->twin); + sympos = klp_find_sympos(e->orig, patched_sym->twin); if (sympos == ULONG_MAX) return -1; } @@ -992,7 +1425,7 @@ static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc, klp_sym = find_symbol_by_name(e->out, sym_name); if (!klp_sym) { - __dbg_indent("%s", sym_name); + __dbg_clone("%s", sym_name); /* STB_WEAK: avoid modpost undefined symbol warnings */ klp_sym = elf_create_symbol(e->out, sym_name, NULL, @@ -1002,16 +1435,35 @@ static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc, } /* - * Create the __klp_relocs entry. This will be converted to an actual - * KLP rela by "objtool klp post-link". + * Create the __klp_relocs.<objname> entry. This will be converted to + * an actual KLP rela by "objtool klp post-link". * * This intermediate step is necessary to prevent corruption by the * linker, which doesn't know how to properly handle two rela sections * applying to the same base section. + * + * The objname decides when the reloc gets applied. A reference to a + * vmlinux symbol goes in the vmlinux section so it gets applied when + * the patch module loads. Everything else goes in the patched + * object's section, applied when the patched module is loaded. */ + if (!strcmp(sym_modname, "vmlinux")) { + sec_objname = "vmlinux"; + } else { + sec_objname = find_modname(e); + if (!sec_objname) + return -1; + } + + /* section format: __klp_relocs.objname */ + if (snprintf_check(sec_name, SEC_NAME_LEN, + KLP_RELOCS_SEC ".%s", sec_objname)) + return -1; + + klp_relocs = find_section_by_name(e->out, sec_name); if (!klp_relocs) { - klp_relocs = elf_create_section(e->out, KLP_RELOCS_SEC, 0, + klp_relocs = elf_create_section(e->out, sec_name, 0, 0, SHT_PROGBITS, 8, SHF_ALLOC); if (!klp_relocs) return -1; @@ -1043,12 +1495,12 @@ static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc, } #define dbg_clone_reloc(sec, offset, patched_sym, addend, export, klp) \ - dbg_indent("%s+0x%lx: %s%s0x%lx [%s%s%s%s%s%s]", \ + dbg_clone("%s+0x%lx: %s%s0x%lx [%s%s%s%s%s%s]", \ sec->name, offset, patched_sym->name, \ addend >= 0 ? "+" : "-", labs(addend), \ sym_type(patched_sym), \ - patched_sym->type == STT_SECTION ? "" : " ", \ - patched_sym->type == STT_SECTION ? "" : sym_bind(patched_sym), \ + is_sec_sym(patched_sym) ? "" : " ", \ + is_sec_sym(patched_sym) ? "" : sym_bind(patched_sym), \ is_undef_sym(patched_sym) ? " UNDEF" : "", \ export ? " EXPORTED" : "", \ klp ? " KLP" : "") @@ -1063,13 +1515,6 @@ static int clone_reloc(struct elfs *e, struct reloc *patched_reloc, struct symbol *out_sym; bool klp; - if (!is_reloc_allowed(patched_reloc)) { - ERROR_FUNC(patched_reloc->sec->base, reloc_offset(patched_reloc), - "missing symbol for reference to %s+%ld", - patched_sym->name, addend); - return -1; - } - klp = klp_reloc_needed(patched_reloc); dbg_clone_reloc(sec, offset, patched_sym, addend, export, klp); @@ -1099,13 +1544,13 @@ static int clone_reloc(struct elfs *e, struct reloc *patched_reloc, /* * For strings, all references use section symbols, thanks to - * section_reference_needed(). clone_symbol() has cloned an empty + * convert_reloc_sym(). clone_symbol() has cloned an empty * version of the string section. Now copy the string itself. */ if (is_string_sec(patched_sym->sec)) { const char *str = patched_sym->sec->data->d_buf + addend; - __dbg_indent("\"%s\"", escape_str(str)); + __dbg_clone("\"%s\"", escape_str(str)); addend = elf_add_string(e->out, out_sym->sec, str); if (addend == -1) @@ -1152,6 +1597,7 @@ static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym) for_each_reloc(patched_rsec, patched_reloc) { unsigned long offset; + int ret; if (reloc_offset(patched_reloc) < start || reloc_offset(patched_reloc) >= end) @@ -1165,12 +1611,19 @@ static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym) !strcmp(patched_reloc->sym->sec->name, ".altinstr_aux")) continue; - if (convert_reloc_sym(e->patched, patched_reloc)) { + if (arch_alt_ignore_new_reloc(patched_sym->sec, + reloc_offset(patched_reloc))) + continue; + + ret = convert_reloc_sym(e->patched, patched_reloc); + if (ret < 0) { ERROR_FUNC(patched_rsec->base, reloc_offset(patched_reloc), "failed to convert reloc sym '%s' to its proper format", patched_reloc->sym->name); return -1; } + if (ret > 0) + continue; offset = out_sym->offset + (reloc_offset(patched_reloc) - patched_sym->offset); @@ -1185,6 +1638,7 @@ static int create_fake_symbol(struct elf *elf, struct section *sec, unsigned long offset, size_t size) { char name[SYM_NAME_LEN]; + struct symbol *sym; unsigned int type; static int ctr; char *c; @@ -1201,7 +1655,24 @@ static int create_fake_symbol(struct elf *elf, struct section *sec, * while still allowing objdump to disassemble it. */ type = is_text_sec(sec) ? STT_NOTYPE : STT_OBJECT; - return elf_create_symbol(elf, name, sec, STB_LOCAL, type, offset, size) ? 0 : -1; + + sym = elf_create_symbol(elf, name, sec, STB_LOCAL, type, offset, size); + if (!sym) + return -1; + + sym->fake = 1; + return 0; +} + +static bool has_fake_symbols(struct section *sec) +{ + struct symbol *sym; + + sec_for_each_sym(sec, sym) + if (sym->fake) + return true; + + return false; } /* @@ -1247,18 +1718,22 @@ static int create_fake_symbols(struct elf *elf) sec = find_section_by_name(elf, ".discard.annotate_data"); if (!sec || !sec->rsec) - return 0; + goto entsize; for_each_reloc(sec->rsec, reloc) { unsigned long offset, size; struct reloc *next_reloc; + bool last = true; if (annotype(elf, sec, reloc) != ANNOTYPE_DATA_SPECIAL) continue; offset = reloc_addend(reloc); - size = 0; + /* + * Find the start of the next entry so the fake symbol size can + * be calculated. + */ next_reloc = reloc; for_each_reloc_continue(sec->rsec, next_reloc) { if (annotype(elf, sec, next_reloc) != ANNOTYPE_DATA_SPECIAL || @@ -1266,10 +1741,15 @@ static int create_fake_symbols(struct elf *elf) continue; size = reloc_addend(next_reloc) - offset; + last = false; break; } - if (!size) + /* + * If no next entry found, this is the last entry, so its size + * is from the current offset to the end of the section. + */ + if (last) size = sec_size(reloc->sym->sec) - offset; if (create_fake_symbol(elf, reloc->sym->sec, offset, size)) @@ -1279,12 +1759,16 @@ static int create_fake_symbols(struct elf *elf) /* * 2) Make symbols for sh_entsize, and simple arrays of pointers: */ - +entsize: for_each_sec(elf, sec) { unsigned int entry_size; unsigned long offset; - if (!is_special_section(sec) || find_symbol_by_offset(sec, 0)) + if (!is_special_section(sec)) + continue; + + /* Skip sections already handled by step 1 above */ + if (has_fake_symbols(sec)) continue; if (!sec->rsec) { @@ -1313,6 +1797,7 @@ static int create_fake_symbols(struct elf *elf) /* Keep a special section entry if it references an included function */ static bool should_keep_special_sym(struct elf *elf, struct symbol *sym) { + bool annotate_insn = !strcmp(sym->sec->name, ".discard.annotate_insn"); struct reloc *reloc; if (is_sec_sym(sym) || !sym->sec->rsec) @@ -1322,7 +1807,16 @@ static bool should_keep_special_sym(struct elf *elf, struct symbol *sym) if (convert_reloc_sym(elf, reloc)) continue; - if (is_func_sym(reloc->sym) && reloc->sym->included) + if (!reloc->sym->clone || is_undef_sym(reloc->sym->clone)) + continue; + + /* + * Keep special section references to cloned functions. + * In some cases annotate_insn can also reference cloned alt + * replacement fake symbols; keep those references as well. + */ + if (is_func_sym(reloc->sym) || + (annotate_insn && is_notype_sym(reloc->sym))) return true; } @@ -1466,15 +1960,28 @@ static int clone_special_section(struct elfs *e, struct section *patched_sec) /* Extract only the needed bits from special sections */ static int clone_special_sections(struct elfs *e) { - struct section *patched_sec; + struct section *sec, *annotate_insn = NULL; - for_each_sec(e->patched, patched_sec) { - if (is_special_section(patched_sec)) { - if (clone_special_section(e, patched_sec)) + for_each_sec(e->patched, sec) { + if (is_special_section(sec)) { + if (!strcmp(sec->name, ".discard.annotate_insn")) { + annotate_insn = sec; + continue; + } + if (clone_special_section(e, sec)) return -1; } } + /* + * Do .discard.annotate_insn last, it can reference other special + * sections (alt replacements) so they need to be cloned first. + */ + if (annotate_insn) { + if (clone_special_section(e, annotate_insn)) + return -1; + } + return 0; } @@ -1551,7 +2058,8 @@ static int create_klp_sections(struct elfs *e) unsigned long sympos; void *func_data; - if (!is_func_sym(sym) || sym->cold || !sym->clone || !sym->clone->changed) + if (!is_func_sym(sym) || is_cold_func(sym) || + !sym->clone || !sym->clone->changed) continue; /* allocate klp_func_ext */ @@ -1577,7 +2085,7 @@ static int create_klp_sections(struct elfs *e) /* klp_func_ext.sympos */ BUILD_BUG_ON(sizeof(sympos) != sizeof_field(struct klp_func_ext, sympos)); - sympos = find_sympos(e->orig, sym->clone->twin); + sympos = klp_find_sympos(e->orig, sym->clone->twin); if (sympos == ULONG_MAX) return -1; memcpy(func_data + offsetof(struct klp_func_ext, sympos), &sympos, @@ -1711,11 +2219,17 @@ static int copy_import_ns(struct elfs *e) int cmd_klp_diff(int argc, const char **argv) { struct elfs e = {0}; + int ret; argc = parse_options(argc, argv, klp_diff_options, klp_diff_usage, 0); if (argc != 3) usage_with_options(klp_diff_usage, klp_diff_options); + if (debug) { + debug_correlate = true; + debug_clone = true; + } + objname = argv[0]; e.orig = elf_open_read(argv[0], O_RDONLY); @@ -1725,6 +2239,9 @@ int cmd_klp_diff(int argc, const char **argv) if (!e.orig || !e.patched) return -1; + if (klp_sympos_init(e.orig)) + return -1; + if (read_exports()) return -1; @@ -1737,7 +2254,10 @@ int cmd_klp_diff(int argc, const char **argv) if (correlate_symbols(&e)) return -1; - if (mark_changed_functions(&e)) + ret = mark_changed_functions(&e); + if (ret < 0) + return -1; + if (ret > 0) return 0; e.out = elf_create_file(&e.orig->ehdr, argv[2]); diff --git a/tools/objtool/klp-post-link.c b/tools/objtool/klp-post-link.c index c013e39957b1..350d20495897 100644 --- a/tools/objtool/klp-post-link.c +++ b/tools/objtool/klp-post-link.c @@ -19,19 +19,11 @@ #include <objtool/util.h> #include <linux/livepatch_external.h> -static int fix_klp_relocs(struct elf *elf) +static int fix_klp_reloc_sec(struct elf *elf, struct section *symtab, + struct section *klp_relocs) { - struct section *symtab, *klp_relocs; - - klp_relocs = find_section_by_name(elf, KLP_RELOCS_SEC); - if (!klp_relocs) - return 0; - - symtab = find_section_by_name(elf, ".symtab"); - if (!symtab) { - ERROR("missing .symtab"); - return -1; - } + /* section format: __klp_relocs.sec_objname */ + const char *sec_objname = klp_relocs->name + strlen(KLP_RELOCS_SEC "."); for (int i = 0; i < sec_size(klp_relocs) / sizeof(struct klp_reloc); i++) { struct klp_reloc *klp_reloc; @@ -39,7 +31,6 @@ static int fix_klp_relocs(struct elf *elf) struct section *sec, *tmp, *klp_rsec; unsigned long offset; struct reloc *reloc; - char sym_modname[64]; char rsec_name[SEC_NAME_LEN]; u64 addend; struct symbol *sym, *klp_sym; @@ -55,7 +46,7 @@ static int fix_klp_relocs(struct elf *elf) reloc = find_reloc_by_dest(elf, klp_relocs, klp_reloc_off + offsetof(struct klp_reloc, offset)); if (!reloc) { - ERROR("malformed " KLP_RELOCS_SEC " section"); + ERROR("malformed %s section", klp_relocs->name); return -1; } @@ -66,17 +57,13 @@ static int fix_klp_relocs(struct elf *elf) reloc = find_reloc_by_dest(elf, klp_relocs, klp_reloc_off + offsetof(struct klp_reloc, sym)); if (!reloc) { - ERROR("malformed " KLP_RELOCS_SEC " section"); + ERROR("malformed %s section", klp_relocs->name); return -1; } klp_sym = reloc->sym; addend = reloc_addend(reloc); - /* symbol format: .klp.sym.modname.sym_name,sympos */ - if (sscanf(klp_sym->name + strlen(KLP_SYM_PREFIX), "%55[^.]", sym_modname) != 1) - ERROR("can't find modname in klp symbol '%s'", klp_sym->name); - /* * Create the KLP rela: */ @@ -84,7 +71,7 @@ static int fix_klp_relocs(struct elf *elf) /* section format: .klp.rela.sec_objname.section_name */ if (snprintf_check(rsec_name, SEC_NAME_LEN, KLP_RELOC_SEC_PREFIX "%s.%s", - sym_modname, sec->name)) + sec_objname, sec->name)) return -1; klp_rsec = find_section_by_name(elf, rsec_name); @@ -134,10 +121,32 @@ static int fix_klp_relocs(struct elf *elf) return 0; } +static int fix_klp_relocs(struct elf *elf) +{ + struct section *symtab, *sec; + + symtab = find_section_by_name(elf, ".symtab"); + if (!symtab) { + ERROR("missing .symtab"); + return -1; + } + + for_each_sec(elf, sec) { + if (strncmp(sec->name, KLP_RELOCS_SEC ".", + strlen(KLP_RELOCS_SEC "."))) + continue; + + if (fix_klp_reloc_sec(elf, symtab, sec)) + return -1; + } + + return 0; +} + /* * This runs on the livepatch module after all other linking has been done. It - * converts the intermediate __klp_relocs section into proper KLP relocs to be - * processed by livepatch. This needs to run last to avoid linker wreckage. + * converts the intermediate __klp_relocs.* sections into proper KLP relocs to + * be processed by livepatch. This needs to run last to avoid linker wreckage. * Linkers don't tend to handle the "two rela sections for a single base * section" case very well, nor do they appreciate SHN_LIVEPATCH. */ diff --git a/tools/objtool/klp-symid.c b/tools/objtool/klp-symid.c new file mode 100644 index 000000000000..b19f76dff13e --- /dev/null +++ b/tools/objtool/klp-symid.c @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Emit the .klp.symid table which allows "objtool klp diff" to reliably + * disambiguate duplicate-named local symbols in vmlinux. + * + * Livepatch identifies a duplicate-named symbol by its position (sympos) + * among the same-named kallsyms entries, counted in ascending address order + * in the final linked vmlinux. That order can't be derived from vmlinux.o + * alone: the final link reorders sub-sections (.text.unlikely*, .data..*, + * etc). + * + * Bridge the gap with a table which survives the final link: a single + * non-alloc section containing an array of { id, addr } entries, where + * 'id' is a unique counter identifier and 'addr' has a relocation to the + * symbol. The linker copies 'id' verbatim and resolves 'addr' to the symbol's + * final address. + * + * The table is only emitted for vmlinux.o, and only when klp-build asks for it + * with KLP_SYMIDS=1, which adds --klp-symids to the vmlinux.o objtool run. + * + * It can't survive --gc-sections, which sweeps the whole section; klp-build + * rejects CONFIG_LD_DEAD_CODE_DATA_ELIMINATION. + */ +#include <linux/string.h> + +#include <objtool/objtool.h> +#include <objtool/warn.h> +#include <objtool/endianness.h> +#include <objtool/klp.h> + +static const char * const discarded_secs[] = { + ".discard", + ".exitcall.exit", + ".modinfo", + ".no_trim_symbol", + "__tracepoint_check", +}; + +static bool discarded_sec(struct section *sec) +{ + if (!(sec->sh.sh_flags & SHF_ALLOC)) + return true; + + for (int i = 0; i < ARRAY_SIZE(discarded_secs); i++) + if (strstarts(sec->name, discarded_secs[i])) + return true; + + return false; +} + +static bool symid_needed(struct elf *elf, struct symbol *sym) +{ + struct symbol *s; + + if (!is_local_sym(sym) || is_undef_sym(sym)) + return false; + + if (!is_func_sym(sym) && !is_object_sym(sym)) + return false; + + if (is_prefix_func(sym)) + return false; + + if (discarded_sec(sym->sec)) + return false; + + for_each_sym_by_name(elf, sym->name, s) { + if (s == sym || is_sec_sym(s) || is_file_sym(s) || is_undef_sym(s)) + continue; + return true; + } + + return false; +} + +int klp_create_symid_sections(struct objtool_file *file) +{ + struct elf *elf = file->elf; + struct klp_symid *symids; + struct section *sec; + struct symbol *sym; + u64 nr = 0, i = 0; + + if (!str_ends_with(objname, "vmlinux.o")) + return 0; + + for_each_sym(elf, sym) + if (symid_needed(elf, sym)) + nr++; + + if (!nr) + return 0; + + sec = elf_create_section(elf, KLP_SYMID_SEC, 0, sizeof(struct klp_symid), + SHT_PROGBITS, 8, 0); + if (!sec) + return -1; + + symids = elf_add_data(elf, sec, NULL, nr * sizeof(struct klp_symid)); + if (!symids) + return -1; + + for_each_sym(elf, sym) { + if (!symid_needed(elf, sym)) + continue; + + symids[i].id = bswap_if_needed(elf, i); + + if (!elf_create_reloc(elf, sec, + i * sizeof(struct klp_symid) + + offsetof(struct klp_symid, addr), + sym, 0, R_ABS64)) + return -1; + + i++; + } + + return 0; +} diff --git a/tools/objtool/klp-sympos.c b/tools/objtool/klp-sympos.c new file mode 100644 index 000000000000..dfca9dd74681 --- /dev/null +++ b/tools/objtool/klp-sympos.c @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Compute "sympos", the position used by livepatch to disambiguate + * duplicate symbol names in the patched object. + */ +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include <objtool/objtool.h> +#include <objtool/warn.h> +#include <objtool/endianness.h> +#include <objtool/klp.h> + +#include <linux/string.h> + +struct vmlinux_sym { + struct hlist_node hash; + const char *name; + u64 addr; +}; + +struct vmlinux_symid { + struct hlist_node hash; + u64 id; + u64 addr; +}; + +struct vmlinux_o_symid { + struct hlist_node hash; + u64 id; + unsigned int sym_idx; +}; + +static DEFINE_HASHTABLE(vmlinux_o_symids, 16); + +/* + * The original linked kernel, found next to the orig vmlinux.o. Read with raw + * libelf rather than elf_open_read(): only the symbol table and the resolved + * .klp.symid table are needed, not the (huge) instruction/reloc machinery. + * + * Both tables are built once by read_orig_vmlinux(). The Elf handle stays + * open because the hashed names point into its mmapped string table. + */ +static struct { + Elf *elf; + DECLARE_HASHTABLE(syms, 16); /* name -> address */ + DECLARE_HASHTABLE(symids, 16); /* .klp.symid id -> address */ +} vmlinux; + +/* + * Would the symbol be visible to the runtime's kallsyms-based symbol lookup? + */ +static bool vmlinux_sym_in_kallsyms(Elf *elf, GElf_Sym *sym) +{ + unsigned int type = GELF_ST_TYPE(sym->st_info); + GElf_Shdr shdr; + Elf_Scn *scn; + + if (sym->st_shndx == SHN_UNDEF || sym->st_shndx >= SHN_LORESERVE) + return false; + + if (type == STT_SECTION || type == STT_FILE) + return false; + + scn = elf_getscn(elf, sym->st_shndx); + if (!scn || !gelf_getshdr(scn, &shdr)) + return false; + + return shdr.sh_flags & SHF_ALLOC; +} + +static int read_orig_vmlinux(const char *filename) +{ + size_t shstrndx, nr_syms = 0, nr_symids = 0, strtab_idx = 0; + Elf_Data *symtab_data = NULL, *symid_data = NULL; + struct klp_symid *symids; + Elf_Scn *scn = NULL; + GElf_Ehdr ehdr; + int fd; + + fd = open(filename, O_RDONLY); + if (fd == -1) { + ERROR_GLIBC("can't open '%s'", filename); + return -1; + } + + if (elf_version(EV_CURRENT) == EV_NONE) { + ERROR_ELF("elf_version"); + return -1; + } + + vmlinux.elf = elf_begin(fd, ELF_C_READ_MMAP, NULL); + if (!vmlinux.elf) { + ERROR_ELF("elf_begin"); + return -1; + } + + if (!gelf_getehdr(vmlinux.elf, &ehdr)) { + ERROR_ELF("gelf_getehdr"); + return -1; + } + + if (elf_getshdrstrndx(vmlinux.elf, &shstrndx)) { + ERROR_ELF("elf_getshdrstrndx"); + return -1; + } + + while ((scn = elf_nextscn(vmlinux.elf, scn))) { + const char *name; + GElf_Shdr shdr; + + if (!gelf_getshdr(scn, &shdr)) { + ERROR_ELF("gelf_getshdr"); + return -1; + } + + if (shdr.sh_type == SHT_SYMTAB) { + symtab_data = elf_getdata(scn, NULL); + if (!symtab_data) { + ERROR_ELF("elf_getdata"); + return -1; + } + nr_syms = shdr.sh_size / shdr.sh_entsize; + strtab_idx = shdr.sh_link; + continue; + } + + name = elf_strptr(vmlinux.elf, shstrndx, shdr.sh_name); + if (name && !strcmp(name, KLP_SYMID_SEC)) { + if (shdr.sh_size % sizeof(struct klp_symid)) { + ERROR("%s: %s: struct klp_symid size mismatch", + filename, KLP_SYMID_SEC); + return -1; + } + symid_data = elf_getdata(scn, NULL); + if (!symid_data) { + ERROR_ELF("elf_getdata"); + return -1; + } + nr_symids = shdr.sh_size / sizeof(struct klp_symid); + } + } + + if (!symtab_data) { + ERROR("%s: missing symbol table", filename); + return -1; + } + + if (!symid_data) { + ERROR("%s: missing %s section, kernel not built with CONFIG_KLP_BUILD?", + filename, KLP_SYMID_SEC); + return -1; + } + + for (size_t i = 0; i < nr_syms; i++) { + struct vmlinux_sym *vsym; + const char *name; + GElf_Sym s; + + if (!gelf_getsym(symtab_data, i, &s)) { + ERROR_ELF("gelf_getsym"); + return -1; + } + + if (!vmlinux_sym_in_kallsyms(vmlinux.elf, &s)) + continue; + + name = elf_strptr(vmlinux.elf, strtab_idx, s.st_name); + if (!name) + continue; + + vsym = calloc(1, sizeof(*vsym)); + if (!vsym) { + ERROR_GLIBC("calloc"); + return -1; + } + + vsym->name = name; + vsym->addr = s.st_value; + hash_add(vmlinux.syms, &vsym->hash, str_hash(name)); + } + + symids = symid_data->d_buf; + + for (size_t i = 0; i < nr_symids; i++) { + struct vmlinux_symid *vsymid; + + vsymid = calloc(1, sizeof(*vsymid)); + if (!vsymid) { + ERROR_GLIBC("calloc"); + return -1; + } + + vsymid->id = __bswap_if_needed(&ehdr, symids[i].id); + vsymid->addr = __bswap_if_needed(&ehdr, symids[i].addr); + hash_add(vmlinux.symids, &vsymid->hash, vsymid->id); + } + + /* the fd and Elf handle stay open, the hashed names live in the mmap */ + return 0; +} + +/* + * Read the orig vmlinux.o's .klp.symid table, an array of entries whose 'addr' + * fields have relocs to the symbols they describe. + */ +static int read_vmlinux_o_symids(struct elf *vmlinux_o) +{ + struct section *sec; + + for_each_sec(vmlinux_o, sec) { + unsigned long nr; + + if (strcmp(sec->name, KLP_SYMID_SEC)) + continue; + + if (sec_size(sec) % sizeof(struct klp_symid)) { + ERROR("%s: %s: struct klp_symid size mismatch", + vmlinux_o->name, KLP_SYMID_SEC); + return -1; + } + + nr = sec_size(sec) / sizeof(struct klp_symid); + + for (unsigned long i = 0; i < nr; i++) { + unsigned long offset = i * sizeof(struct klp_symid); + struct vmlinux_o_symid *entry; + struct klp_symid *symid; + struct reloc *reloc; + + entry = calloc(1, sizeof(*entry)); + if (!entry) { + ERROR_GLIBC("calloc"); + return -1; + } + + symid = sec->data->d_buf + offset; + entry->id = bswap_if_needed(vmlinux_o, symid->id); + + reloc = find_reloc_by_dest(vmlinux_o, sec, + offset + offsetof(struct klp_symid, addr)); + if (!reloc) { + ERROR("%s: missing reloc for %s entry", + vmlinux_o->name, KLP_SYMID_SEC); + return -1; + } + entry->sym_idx = reloc->sym->idx; + + hash_add(vmlinux_o_symids, &entry->hash, entry->sym_idx); + } + } + + return 0; +} + +int klp_sympos_init(struct elf *orig) +{ + char *filename; + int ret; + + if (!str_ends_with(objname, "vmlinux.o")) + return 0; + + if (read_vmlinux_o_symids(orig)) + return -1; + + filename = strndup(objname, strlen(objname) - 2); + if (!filename) { + ERROR_GLIBC("strndup"); + return -1; + } + + ret = read_orig_vmlinux(filename); + free(filename); + + return ret; +} + +/* Find the symbol's id in the orig vmlinux.o's .klp.symid table */ +static int find_vmlinux_o_symid(struct symbol *sym, u64 *id) +{ + struct vmlinux_o_symid *entry; + + hash_for_each_possible(vmlinux_o_symids, entry, hash, sym->idx) { + if (entry->sym_idx == sym->idx) { + *id = entry->id; + return 0; + } + } + + ERROR("no %s entry for symbol %s in orig vmlinux.o", KLP_SYMID_SEC, + sym->name); + return -1; +} + +/* Find the symbol's final address in the orig vmlinux's .klp.symid table */ +static int find_vmlinux_symid_addr(u64 id, u64 *addr) +{ + struct vmlinux_symid *symid; + + hash_for_each_possible(vmlinux.symids, symid, hash, id) { + if (symid->id == id) { + *addr = symid->addr; + return 0; + } + } + + return -1; +} + +/* + * Find the sympos of a vmlinux-local symbol by ranking its final address + * among the duplicately named symbols in the linked orig vmlinux, replicating + * the order in which kallsyms_on_each_match_symbol() counts them. + */ +static unsigned long find_vmlinux_sympos(struct symbol *sym) +{ + unsigned long nr_matches = 0, sympos = 1; + u32 key = str_hash(sym->name); + struct vmlinux_sym *vsym; + bool found = false; + u64 id, addr; + + hash_for_each_possible(vmlinux.syms, vsym, hash, key) + if (!strcmp(vsym->name, sym->name)) + nr_matches++; + + if (!nr_matches) { + ERROR("can't find symbol %s in orig vmlinux", sym->name); + return ULONG_MAX; + } + + /* + * Unique symbols don't need disambiguating. They also have no + * .klp.symid entry, which is only emitted for names duplicated in + * vmlinux.o, so the lookups below would fail. + */ + if (nr_matches == 1) + return 0; + + if (find_vmlinux_o_symid(sym, &id)) + return ULONG_MAX; + + if (find_vmlinux_symid_addr(id, &addr)) { + ERROR("no %s entry for symbol %s in orig vmlinux", KLP_SYMID_SEC, + sym->name); + return ULONG_MAX; + } + + hash_for_each_possible(vmlinux.syms, vsym, hash, key) { + if (strcmp(vsym->name, sym->name)) + continue; + + if (vsym->addr < addr) + sympos++; + else if (vsym->addr == addr) + found = true; + } + + if (!found) { + ERROR("%s address mismatch for symbol %s, stale orig vmlinux?", + KLP_SYMID_SEC, sym->name); + return ULONG_MAX; + } + + return sympos; +} + +static bool is_init_sym(struct symbol *sym) +{ + return strstarts(sym->sec->name, ".init"); +} + +/* + * "sympos" is used by livepatch to disambiguate duplicate symbol names. + */ +unsigned long klp_find_sympos(struct elf *elf, struct symbol *sym) +{ + unsigned long sympos = 0, nr_matches = 0; + bool has_dup = false; + struct symbol *s; + + if (is_init_sym(sym)) { + ERROR("%s: can't patch or reference init code/data", sym->name); + return ULONG_MAX; + } + + if (sym->bind != STB_LOCAL) + return 0; + + /* + * vmlinux: the final link reorders symbols relative to vmlinux.o, + * so the position needs to be derived from the linked orig vmlinux via + * the .klp.symid table. + */ + if (vmlinux.elf) + return find_vmlinux_sympos(sym); + + /* + * modules: the final .ko preserves symbol table order, so a + * symtab-order count here matches the runtime count done by + * module_kallsyms_on_each_symbol(). + */ + for_each_sym(elf, s) { + if (!strcmp(s->name, sym->name)) { + nr_matches++; + if (s == sym) + sympos = nr_matches; + else + has_dup = true; + } + } + + if (!sympos) { + ERROR("can't find sympos for %s", sym->name); + return ULONG_MAX; + } + + return has_dup ? sympos : 0; +} diff --git a/tools/objtool/noreturns.h b/tools/objtool/noreturns.h index 14f8ab653449..0e9dfd0a2446 100644 --- a/tools/objtool/noreturns.h +++ b/tools/objtool/noreturns.h @@ -26,6 +26,7 @@ NORETURN(cpu_startup_entry) NORETURN(do_exit) NORETURN(do_group_exit) NORETURN(do_task_dead) +NORETURN(efi_rts_park_worker) NORETURN(ex_handler_msr_mce) NORETURN(hlt_play_dead) NORETURN(hv_ghcb_terminate) diff --git a/tools/objtool/objtool.c b/tools/objtool/objtool.c index 1c3622117c33..a4e139dee7e9 100644 --- a/tools/objtool/objtool.c +++ b/tools/objtool/objtool.c @@ -16,9 +16,6 @@ #include <objtool/objtool.h> #include <objtool/warn.h> -bool debug; -int indent; - static struct objtool_file file; struct objtool_file *objtool_open_read(const char *filename) diff --git a/tools/objtool/trace.c b/tools/objtool/trace.c index 5dec44dab781..61c6aa302bc3 100644 --- a/tools/objtool/trace.c +++ b/tools/objtool/trace.c @@ -169,8 +169,8 @@ void trace_alt_begin(struct instruction *orig_insn, struct alternative *alt, */ TRACE_ALT_INFO_NOADDR(orig_insn, "/ ", "%s for instruction at 0x%lx <%s+0x%lx>", alt_name, - orig_insn->offset, orig_insn->sym->name, - orig_insn->offset - orig_insn->sym->offset); + orig_insn->offset, insn_sym(orig_insn)->name, + orig_insn->offset - insn_sym(orig_insn)->offset); } else { TRACE_ALT_INFO_NOADDR(orig_insn, "/ ", "%s", alt_name); } @@ -185,8 +185,8 @@ void trace_alt_begin(struct instruction *orig_insn, struct alternative *alt, if (orig_insn->type == INSN_NOP) { suffix[0] = (orig_insn->len == 5) ? 'q' : '\0'; TRACE_ADDR(orig_insn, "jmp%-3s %lx <%s+0x%lx>", suffix, - alt_insn->offset, alt_insn->sym->name, - alt_insn->offset - alt_insn->sym->offset); + alt_insn->offset, insn_sym(alt_insn)->name, + alt_insn->offset - insn_sym(alt_insn)->offset); } else { TRACE_ADDR(orig_insn, "nop%d", orig_insn->len); trace_depth--; |