Add [sec_]for_each_sym() and use them. Signed-off-by: Josh Poimboeuf <jpoimboe@kernel.org> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Link: https://lkml.kernel.org/r/59023e5886ab125aa30702e633be7732b1acaa7e.1681325924.git.jpoimboe@kernel.org
It's easier to use and also gives easy access to the instruction's containing function, which is useful for printing that function's symbol. It will also be useful in the future for rate-limiting and disassembly of warned functions. Signed-off-by: Josh Poimboeuf <jpoimboe@kernel.org> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Link: https://lkml.kernel.org/r/2eaa3155c90fba683d8723599f279c46025b75f3.1681325924.git.jpoimboe@kernel.org
The ENTRY unwind hint type is serving double duty as both an empty unwind hint and an unret validation annotation. Unret validation is unrelated to unwinding. Separate it out into its own annotation. Signed-off-by: Josh Poimboeuf <jpoimboe@kernel.org> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> Link: https://lore.kernel.org/r/ff7448d492ea21b86d8a90264b105fbd0d751077.1677683419.git.jpoimboe@kernel.org
There have been some recently reported ORC unwinder warnings like:
WARNING: can't access registers at entry_SYSCALL_64_after_hwframe+0x63/0xcd
WARNING: stack going in the wrong direction? at __sys_setsockopt+0x2c6/0x5b0 net/socket.c:2271
And a KASAN warning:
BUG: KASAN: stack-out-of-bounds in unwind_next_frame (arch/x86/include/asm/ptrace.h:136 arch/x86/kernel/unwind_orc.c:455)
It turns out the 'signal' bit isn't getting propagated from the unwind
hints to the ORC entries, making the unwinder confused at times.
Fixes: ffb1b4a41016 ("x86/unwind/orc: Add 'signal' field to ORC metadata")
Reported-by: kernel test robot <oliver.sang@intel.com>
Reported-by: Dmitry Vyukov <dvyukov@google.com>
Signed-off-by: Josh Poimboeuf <jpoimboe@kernel.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/97eef9db60cd86d376a9a40d49d77bb67a8f6526.1676579666.git.jpoimboe@kernel.org
Replace the instruction::list by allocating instructions in arrays of
256 entries and stringing them together by (amortized) find_insn().
This shrinks instruction by 16 bytes and brings it down to 128.
struct instruction {
- struct list_head list; /* 0 16 */
- struct hlist_node hash; /* 16 16 */
- struct list_head call_node; /* 32 16 */
- struct section * sec; /* 48 8 */
- long unsigned int offset; /* 56 8 */
- /* --- cacheline 1 boundary (64 bytes) --- */
- long unsigned int immediate; /* 64 8 */
- unsigned int len; /* 72 4 */
- u8 type; /* 76 1 */
-
- /* Bitfield combined with previous fields */
+ struct hlist_node hash; /* 0 16 */
+ struct list_head call_node; /* 16 16 */
+ struct section * sec; /* 32 8 */
+ long unsigned int offset; /* 40 8 */
+ long unsigned int immediate; /* 48 8 */
+ u8 len; /* 56 1 */
+ u8 prev_len; /* 57 1 */
+ u8 type; /* 58 1 */
+ s8 instr; /* 59 1 */
+ u32 idx:8; /* 60: 0 4 */
+ u32 dead_end:1; /* 60: 8 4 */
+ u32 ignore:1; /* 60: 9 4 */
+ u32 ignore_alts:1; /* 60:10 4 */
+ u32 hint:1; /* 60:11 4 */
+ u32 save:1; /* 60:12 4 */
+ u32 restore:1; /* 60:13 4 */
+ u32 retpoline_safe:1; /* 60:14 4 */
+ u32 noendbr:1; /* 60:15 4 */
+ u32 entry:1; /* 60:16 4 */
+ u32 visited:4; /* 60:17 4 */
+ u32 no_reloc:1; /* 60:21 4 */
- u16 dead_end:1; /* 76: 8 2 */
- u16 ignore:1; /* 76: 9 2 */
- u16 ignore_alts:1; /* 76:10 2 */
- u16 hint:1; /* 76:11 2 */
- u16 save:1; /* 76:12 2 */
- u16 restore:1; /* 76:13 2 */
- u16 retpoline_safe:1; /* 76:14 2 */
- u16 noendbr:1; /* 76:15 2 */
- u16 entry:1; /* 78: 0 2 */
- u16 visited:4; /* 78: 1 2 */
- u16 no_reloc:1; /* 78: 5 2 */
+ /* XXX 10 bits hole, try to pack */
- /* XXX 2 bits hole, try to pack */
- /* Bitfield combined with next fields */
-
- s8 instr; /* 79 1 */
- struct alt_group * alt_group; /* 80 8 */
- struct instruction * jump_dest; /* 88 8 */
- struct instruction * first_jump_src; /* 96 8 */
+ /* --- cacheline 1 boundary (64 bytes) --- */
+ struct alt_group * alt_group; /* 64 8 */
+ struct instruction * jump_dest; /* 72 8 */
+ struct instruction * first_jump_src; /* 80 8 */
union {
- struct symbol * _call_dest; /* 104 8 */
- struct reloc * _jump_table; /* 104 8 */
- }; /* 104 8 */
- struct alternative * alts; /* 112 8 */
- struct symbol * sym; /* 120 8 */
- /* --- cacheline 2 boundary (128 bytes) --- */
- struct stack_op * stack_ops; /* 128 8 */
- struct cfi_state * cfi; /* 136 8 */
+ struct symbol * _call_dest; /* 88 8 */
+ struct reloc * _jump_table; /* 88 8 */
+ }; /* 88 8 */
+ struct alternative * alts; /* 96 8 */
+ struct symbol * sym; /* 104 8 */
+ struct stack_op * stack_ops; /* 112 8 */
+ struct cfi_state * cfi; /* 120 8 */
- /* size: 144, cachelines: 3, members: 28 */
- /* sum members: 142 */
- /* sum bitfield members: 14 bits, bit holes: 1, sum bit holes: 2 bits */
- /* last cacheline: 16 bytes */
+ /* size: 128, cachelines: 2, members: 29 */
+ /* sum members: 124 */
+ /* sum bitfield members: 22 bits, bit holes: 1, sum bit holes: 10 bits */
};
pre: 5:38.18 real, 213.25 user, 124.90 sys, 23449040 mem
post: 5:03.34 real, 210.75 user, 88.80 sys, 20241232 mem
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Josh Poimboeuf <jpoimboe@kernel.org>
Tested-by: Nathan Chancellor <nathan@kernel.org> # build only
Tested-by: Thomas Weißschuh <linux@weissschuh.net> # compile and run
Link: https://lore.kernel.org/r/20230208172245.851307606@infradead.org
The instruction call_dest and jump_table members can never be used at
the same time, their usage depends on type.
struct instruction {
struct list_head list; /* 0 16 */
struct hlist_node hash; /* 16 16 */
struct list_head call_node; /* 32 16 */
struct section * sec; /* 48 8 */
long unsigned int offset; /* 56 8 */
/* --- cacheline 1 boundary (64 bytes) --- */
long unsigned int immediate; /* 64 8 */
unsigned int len; /* 72 4 */
u8 type; /* 76 1 */
/* Bitfield combined with previous fields */
u16 dead_end:1; /* 76: 8 2 */
u16 ignore:1; /* 76: 9 2 */
u16 ignore_alts:1; /* 76:10 2 */
u16 hint:1; /* 76:11 2 */
u16 save:1; /* 76:12 2 */
u16 restore:1; /* 76:13 2 */
u16 retpoline_safe:1; /* 76:14 2 */
u16 noendbr:1; /* 76:15 2 */
u16 entry:1; /* 78: 0 2 */
u16 visited:4; /* 78: 1 2 */
u16 no_reloc:1; /* 78: 5 2 */
/* XXX 2 bits hole, try to pack */
/* Bitfield combined with next fields */
s8 instr; /* 79 1 */
struct alt_group * alt_group; /* 80 8 */
- struct symbol * call_dest; /* 88 8 */
- struct instruction * jump_dest; /* 96 8 */
- struct instruction * first_jump_src; /* 104 8 */
- struct reloc * jump_table; /* 112 8 */
- struct alternative * alts; /* 120 8 */
+ struct instruction * jump_dest; /* 88 8 */
+ struct instruction * first_jump_src; /* 96 8 */
+ union {
+ struct symbol * _call_dest; /* 104 8 */
+ struct reloc * _jump_table; /* 104 8 */
+ }; /* 104 8 */
+ struct alternative * alts; /* 112 8 */
+ struct symbol * sym; /* 120 8 */
/* --- cacheline 2 boundary (128 bytes) --- */
- struct symbol * sym; /* 128 8 */
- struct stack_op * stack_ops; /* 136 8 */
- struct cfi_state * cfi; /* 144 8 */
+ struct stack_op * stack_ops; /* 128 8 */
+ struct cfi_state * cfi; /* 136 8 */
- /* size: 152, cachelines: 3, members: 29 */
- /* sum members: 150 */
+ /* size: 144, cachelines: 3, members: 28 */
+ /* sum members: 142 */
/* sum bitfield members: 14 bits, bit holes: 1, sum bit holes: 2 bits */
- /* last cacheline: 24 bytes */
+ /* last cacheline: 16 bytes */
};
pre: 5:39.35 real, 215.58 user, 123.69 sys, 23448736 mem
post: 5:38.18 real, 213.25 user, 124.90 sys, 23449040 mem
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Josh Poimboeuf <jpoimboe@kernel.org>
Tested-by: Nathan Chancellor <nathan@kernel.org> # build only
Tested-by: Thomas Weißschuh <linux@weissschuh.net> # compile and run
Link: https://lore.kernel.org/r/20230208172245.640914454@infradead.org
Instead of caching the reloc for each instruction, only keep a
negative cache of not having a reloc (by far the most common case).
struct instruction {
struct list_head list; /* 0 16 */
struct hlist_node hash; /* 16 16 */
struct list_head call_node; /* 32 16 */
struct section * sec; /* 48 8 */
long unsigned int offset; /* 56 8 */
/* --- cacheline 1 boundary (64 bytes) --- */
long unsigned int immediate; /* 64 8 */
unsigned int len; /* 72 4 */
u8 type; /* 76 1 */
/* Bitfield combined with previous fields */
u16 dead_end:1; /* 76: 8 2 */
u16 ignore:1; /* 76: 9 2 */
u16 ignore_alts:1; /* 76:10 2 */
u16 hint:1; /* 76:11 2 */
u16 save:1; /* 76:12 2 */
u16 restore:1; /* 76:13 2 */
u16 retpoline_safe:1; /* 76:14 2 */
u16 noendbr:1; /* 76:15 2 */
u16 entry:1; /* 78: 0 2 */
u16 visited:4; /* 78: 1 2 */
+ u16 no_reloc:1; /* 78: 5 2 */
- /* XXX 3 bits hole, try to pack */
+ /* XXX 2 bits hole, try to pack */
/* Bitfield combined with next fields */
s8 instr; /* 79 1 */
struct alt_group * alt_group; /* 80 8 */
struct symbol * call_dest; /* 88 8 */
struct instruction * jump_dest; /* 96 8 */
struct instruction * first_jump_src; /* 104 8 */
struct reloc * jump_table; /* 112 8 */
- struct reloc * reloc; /* 120 8 */
+ struct alternative * alts; /* 120 8 */
/* --- cacheline 2 boundary (128 bytes) --- */
- struct alternative * alts; /* 128 8 */
- struct symbol * sym; /* 136 8 */
- struct stack_op * stack_ops; /* 144 8 */
- struct cfi_state * cfi; /* 152 8 */
+ struct symbol * sym; /* 128 8 */
+ struct stack_op * stack_ops; /* 136 8 */
+ struct cfi_state * cfi; /* 144 8 */
- /* size: 160, cachelines: 3, members: 29 */
- /* sum members: 158 */
- /* sum bitfield members: 13 bits, bit holes: 1, sum bit holes: 3 bits */
- /* last cacheline: 32 bytes */
+ /* size: 152, cachelines: 3, members: 29 */
+ /* sum members: 150 */
+ /* sum bitfield members: 14 bits, bit holes: 1, sum bit holes: 2 bits */
+ /* last cacheline: 24 bytes */
};
pre: 5:48.89 real, 220.96 user, 127.55 sys, 24834672 mem
post: 5:39.35 real, 215.58 user, 123.69 sys, 23448736 mem
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Josh Poimboeuf <jpoimboe@kernel.org>
Tested-by: Nathan Chancellor <nathan@kernel.org> # build only
Tested-by: Thomas Weißschuh <linux@weissschuh.net> # compile and run
Link: https://lore.kernel.org/r/20230208172245.572145269@infradead.org
Since we don't have that many types in enum insn_type, force it into a
u8 and re-arrange member to get rid of the holes, saves another 8
bytes.
struct instruction {
struct list_head list; /* 0 16 */
struct hlist_node hash; /* 16 16 */
struct list_head call_node; /* 32 16 */
struct section * sec; /* 48 8 */
long unsigned int offset; /* 56 8 */
/* --- cacheline 1 boundary (64 bytes) --- */
- unsigned int len; /* 64 4 */
- enum insn_type type; /* 68 4 */
- long unsigned int immediate; /* 72 8 */
- u16 dead_end:1; /* 80: 0 2 */
- u16 ignore:1; /* 80: 1 2 */
- u16 ignore_alts:1; /* 80: 2 2 */
- u16 hint:1; /* 80: 3 2 */
- u16 save:1; /* 80: 4 2 */
- u16 restore:1; /* 80: 5 2 */
- u16 retpoline_safe:1; /* 80: 6 2 */
- u16 noendbr:1; /* 80: 7 2 */
- u16 entry:1; /* 80: 8 2 */
+ long unsigned int immediate; /* 64 8 */
+ unsigned int len; /* 72 4 */
+ u8 type; /* 76 1 */
- /* XXX 7 bits hole, try to pack */
+ /* Bitfield combined with previous fields */
- s8 instr; /* 82 1 */
- u8 visited; /* 83 1 */
+ u16 dead_end:1; /* 76: 8 2 */
+ u16 ignore:1; /* 76: 9 2 */
+ u16 ignore_alts:1; /* 76:10 2 */
+ u16 hint:1; /* 76:11 2 */
+ u16 save:1; /* 76:12 2 */
+ u16 restore:1; /* 76:13 2 */
+ u16 retpoline_safe:1; /* 76:14 2 */
+ u16 noendbr:1; /* 76:15 2 */
+ u16 entry:1; /* 78: 0 2 */
+ u16 visited:4; /* 78: 1 2 */
- /* XXX 4 bytes hole, try to pack */
+ /* XXX 3 bits hole, try to pack */
+ /* Bitfield combined with next fields */
- struct alt_group * alt_group; /* 88 8 */
- struct symbol * call_dest; /* 96 8 */
- struct instruction * jump_dest; /* 104 8 */
- struct instruction * first_jump_src; /* 112 8 */
- struct reloc * jump_table; /* 120 8 */
+ s8 instr; /* 79 1 */
+ struct alt_group * alt_group; /* 80 8 */
+ struct symbol * call_dest; /* 88 8 */
+ struct instruction * jump_dest; /* 96 8 */
+ struct instruction * first_jump_src; /* 104 8 */
+ struct reloc * jump_table; /* 112 8 */
+ struct reloc * reloc; /* 120 8 */
/* --- cacheline 2 boundary (128 bytes) --- */
- struct reloc * reloc; /* 128 8 */
- struct alternative * alts; /* 136 8 */
- struct symbol * sym; /* 144 8 */
- struct stack_op * stack_ops; /* 152 8 */
- struct cfi_state * cfi; /* 160 8 */
+ struct alternative * alts; /* 128 8 */
+ struct symbol * sym; /* 136 8 */
+ struct stack_op * stack_ops; /* 144 8 */
+ struct cfi_state * cfi; /* 152 8 */
- /* size: 168, cachelines: 3, members: 29 */
- /* sum members: 162, holes: 1, sum holes: 4 */
- /* sum bitfield members: 9 bits, bit holes: 1, sum bit holes: 7 bits */
- /* last cacheline: 40 bytes */
+ /* size: 160, cachelines: 3, members: 29 */
+ /* sum members: 158 */
+ /* sum bitfield members: 13 bits, bit holes: 1, sum bit holes: 3 bits */
+ /* last cacheline: 32 bytes */
};
pre: 5:48.86 real, 220.30 user, 128.34 sys, 24834672 mem
post: 5:48.89 real, 220.96 user, 127.55 sys, 24834672 mem
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Josh Poimboeuf <jpoimboe@kernel.org>
Tested-by: Nathan Chancellor <nathan@kernel.org> # build only
Tested-by: Thomas Weißschuh <linux@weissschuh.net> # compile and run
Link: https://lore.kernel.org/r/20230208172245.501847188@infradead.org
struct instruction {
struct list_head list; /* 0 16 */
struct hlist_node hash; /* 16 16 */
struct list_head call_node; /* 32 16 */
struct section * sec; /* 48 8 */
long unsigned int offset; /* 56 8 */
/* --- cacheline 1 boundary (64 bytes) --- */
unsigned int len; /* 64 4 */
enum insn_type type; /* 68 4 */
long unsigned int immediate; /* 72 8 */
u16 dead_end:1; /* 80: 0 2 */
u16 ignore:1; /* 80: 1 2 */
u16 ignore_alts:1; /* 80: 2 2 */
u16 hint:1; /* 80: 3 2 */
u16 save:1; /* 80: 4 2 */
u16 restore:1; /* 80: 5 2 */
u16 retpoline_safe:1; /* 80: 6 2 */
u16 noendbr:1; /* 80: 7 2 */
u16 entry:1; /* 80: 8 2 */
/* XXX 7 bits hole, try to pack */
s8 instr; /* 82 1 */
u8 visited; /* 83 1 */
/* XXX 4 bytes hole, try to pack */
struct alt_group * alt_group; /* 88 8 */
struct symbol * call_dest; /* 96 8 */
struct instruction * jump_dest; /* 104 8 */
struct instruction * first_jump_src; /* 112 8 */
struct reloc * jump_table; /* 120 8 */
/* --- cacheline 2 boundary (128 bytes) --- */
struct reloc * reloc; /* 128 8 */
- struct list_head alts; /* 136 16 */
- struct symbol * sym; /* 152 8 */
- struct stack_op * stack_ops; /* 160 8 */
- struct cfi_state * cfi; /* 168 8 */
+ struct alternative * alts; /* 136 8 */
+ struct symbol * sym; /* 144 8 */
+ struct stack_op * stack_ops; /* 152 8 */
+ struct cfi_state * cfi; /* 160 8 */
- /* size: 176, cachelines: 3, members: 29 */
- /* sum members: 170, holes: 1, sum holes: 4 */
+ /* size: 168, cachelines: 3, members: 29 */
+ /* sum members: 162, holes: 1, sum holes: 4 */
/* sum bitfield members: 9 bits, bit holes: 1, sum bit holes: 7 bits */
- /* last cacheline: 48 bytes */
+ /* last cacheline: 40 bytes */
};
pre: 5:58.50 real, 229.64 user, 128.65 sys, 26221520 mem
post: 5:48.86 real, 220.30 user, 128.34 sys, 24834672 mem
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Josh Poimboeuf <jpoimboe@kernel.org>
Tested-by: Nathan Chancellor <nathan@kernel.org> # build only
Tested-by: Thomas Weißschuh <linux@weissschuh.net> # compile and run
Link: https://lore.kernel.org/r/20230208172245.430556498@infradead.org
struct instruction {
struct list_head list; /* 0 16 */
struct hlist_node hash; /* 16 16 */
struct list_head call_node; /* 32 16 */
struct section * sec; /* 48 8 */
long unsigned int offset; /* 56 8 */
/* --- cacheline 1 boundary (64 bytes) --- */
unsigned int len; /* 64 4 */
enum insn_type type; /* 68 4 */
long unsigned int immediate; /* 72 8 */
u16 dead_end:1; /* 80: 0 2 */
u16 ignore:1; /* 80: 1 2 */
u16 ignore_alts:1; /* 80: 2 2 */
u16 hint:1; /* 80: 3 2 */
u16 save:1; /* 80: 4 2 */
u16 restore:1; /* 80: 5 2 */
u16 retpoline_safe:1; /* 80: 6 2 */
u16 noendbr:1; /* 80: 7 2 */
u16 entry:1; /* 80: 8 2 */
/* XXX 7 bits hole, try to pack */
s8 instr; /* 82 1 */
u8 visited; /* 83 1 */
/* XXX 4 bytes hole, try to pack */
struct alt_group * alt_group; /* 88 8 */
struct symbol * call_dest; /* 96 8 */
struct instruction * jump_dest; /* 104 8 */
struct instruction * first_jump_src; /* 112 8 */
struct reloc * jump_table; /* 120 8 */
/* --- cacheline 2 boundary (128 bytes) --- */
struct reloc * reloc; /* 128 8 */
struct list_head alts; /* 136 16 */
struct symbol * sym; /* 152 8 */
- struct list_head stack_ops; /* 160 16 */
- struct cfi_state * cfi; /* 176 8 */
+ struct stack_op * stack_ops; /* 160 8 */
+ struct cfi_state * cfi; /* 168 8 */
- /* size: 184, cachelines: 3, members: 29 */
- /* sum members: 178, holes: 1, sum holes: 4 */
+ /* size: 176, cachelines: 3, members: 29 */
+ /* sum members: 170, holes: 1, sum holes: 4 */
/* sum bitfield members: 9 bits, bit holes: 1, sum bit holes: 7 bits */
- /* last cacheline: 56 bytes */
+ /* last cacheline: 48 bytes */
};
pre: 5:58.22 real, 226.69 user, 131.22 sys, 26221520 mem
post: 5:58.50 real, 229.64 user, 128.65 sys, 26221520 mem
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Josh Poimboeuf <jpoimboe@kernel.org>
Tested-by: Nathan Chancellor <nathan@kernel.org> # build only
Tested-by: Thomas Weißschuh <linux@weissschuh.net> # compile and run
Link: https://lore.kernel.org/r/20230208172245.362196959@infradead.org