1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
|
/* SPDX-License-Identifier: GPL-2.0 */
/*
* The actual FRED entry points.
*/
#include <linux/export.h>
#include <linux/kvm_types.h>
#include <asm/asm.h>
#include <asm/fred.h>
#include <asm/segment.h>
#include "calling.h"
.code64
.section .noinstr.text, "ax"
.macro FRED_ENTER
UNWIND_HINT_END_OF_STACK
ANNOTATE_NOENDBR
PUSH_AND_CLEAR_REGS
movq %rsp, %rdi /* %rdi -> pt_regs */
.endm
.macro FRED_EXIT
UNWIND_HINT_REGS
POP_REGS
.endm
/*
* The new RIP value that FRED event delivery establishes is
* IA32_FRED_CONFIG & ~FFFH for events that occur in ring 3.
* Thus the FRED ring 3 entry point must be 4K page aligned.
*/
.align 4096
SYM_CODE_START_NOALIGN(asm_fred_entrypoint_user)
FRED_ENTER
call fred_entry_from_user
SYM_INNER_LABEL(asm_fred_exit_user, SYM_L_GLOBAL)
FRED_EXIT
1: ERETU
_ASM_EXTABLE_TYPE(1b, asm_fred_entrypoint_user, EX_TYPE_ERETU)
SYM_CODE_END(asm_fred_entrypoint_user)
/*
* The new RIP value that FRED event delivery establishes is
* (IA32_FRED_CONFIG & ~FFFH) + 256 for events that occur in
* ring 0, i.e., asm_fred_entrypoint_user + 256.
*/
.org asm_fred_entrypoint_user + 256, 0xcc
SYM_CODE_START_NOALIGN(asm_fred_entrypoint_kernel)
FRED_ENTER
call fred_entry_from_kernel
FRED_EXIT
ERETS
SYM_CODE_END(asm_fred_entrypoint_kernel)
#if IS_ENABLED(CONFIG_KVM_INTEL)
SYM_FUNC_START(asm_fred_entry_from_kvm)
ANNOTATE_NOENDBR
push %rbp
mov %rsp, %rbp
UNWIND_HINT_SAVE
/*
* Both IRQ and NMI from VMX can be handled on current task stack
* because there is no need to protect from reentrancy and the call
* stack leading to this helper is effectively constant and shallow
* (relatively speaking). Do the same when FRED is active, i.e., no
* need to check current stack level for a stack switch.
*
* Emulate the FRED-defined redzone and stack alignment.
*/
sub $(FRED_CONFIG_REDZONE_AMOUNT << 6), %rsp
and $FRED_STACK_FRAME_RSP_MASK, %rsp
/*
* Start to push a FRED stack frame, which is always 64 bytes:
*
* +--------+-----------------+
* | Bytes | Usage |
* +--------+-----------------+
* | 63:56 | Reserved |
* | 55:48 | Event Data |
* | 47:40 | SS + Event Info |
* | 39:32 | RSP |
* | 31:24 | RFLAGS |
* | 23:16 | CS + Aux Info |
* | 15:8 | RIP |
* | 7:0 | Error Code |
* +--------+-----------------+
*/
push $0 /* Reserved, must be 0 */
push $0 /* Event data, 0 for IRQ/NMI */
push %rdi /* fred_ss handed in by the caller */
push %rbp
pushf
push $__KERNEL_CS
/*
* Unlike the IDT event delivery, FRED _always_ pushes an error code
* after pushing the return RIP, thus the CALL instruction CANNOT be
* used here to push the return RIP, otherwise there is no chance to
* push an error code before invoking the IRQ/NMI handler.
*
* Use LEA to get the return RIP and push it, then push an error code.
*/
lea 1f(%rip), %rax
push %rax /* Return RIP */
push $0 /* Error code, 0 for IRQ/NMI */
PUSH_AND_CLEAR_REGS clear_callee=0 unwind_hint=0
movq %rsp, %rdi /* %rdi -> pt_regs */
/*
* At this point: {rdi, rsi, rdx, rcx, r8, r9}, {r10, r11}, {rax, rdx}
* are clobbered, which corresponds to: arguments, extra caller-saved
* and return. All registers a C function is allowed to clobber.
*
* Notably, the callee-saved registers: {rbx, r12, r13, r14, r15}
* are untouched, with the exception of rbp, which carries the stack
* frame and will be restored before exit.
*
* Further calling another C function will not alter this state.
*/
call __fred_entry_from_kvm /* Call the C entry point */
/*
* When FRED, use ERETS to potentially clear NMIs, otherwise simply
* restore the stack pointer.
*/
ALTERNATIVE "nop; nop; mov %rbp, %rsp", \
__stringify(add $C_PTREGS_SIZE, %rsp; ERETS), \
X86_FEATURE_FRED
1: /*
* Objtool doesn't understand ERETS, and the cfi register state is
* different from initial_func_cfi due to PUSH_REGS. Tell it the state
* is similar to where UNWIND_HINT_SAVE is.
*/
UNWIND_HINT_RESTORE
pop %rbp
RET
SYM_FUNC_END(asm_fred_entry_from_kvm)
EXPORT_SYMBOL_FOR_KVM(asm_fred_entry_from_kvm);
#endif
|
