1 /*
2 * Copyright 2018, Jérôme Duval, jerome.duval@gmail.com.
3 * Copyright 2002-2009, Axel Dörfler, axeld@pinc-software.de.
4 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk.
5 * Distributed under the terms of the MIT License.
6 *
7 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
8 * Distributed under the terms of the NewOS License.
9 */
10 #ifndef _KERNEL_ARCH_x86_CPU_H
11 #define _KERNEL_ARCH_x86_CPU_H
12
13
14 #ifndef _ASSEMBLER
15
16 #include <module.h>
17
18 #include <arch_thread_types.h>
19
20 #include <arch/x86/arch_altcodepatch.h>
21 #include <arch/x86/arch_cpuasm.h>
22 #include <arch/x86/descriptors.h>
23
24 #ifdef __x86_64__
25 # include <arch/x86/64/cpu.h>
26 #endif
27
28 #endif // !_ASSEMBLER
29
30
31 #define CPU_MAX_CACHE_LEVEL 8
32
33 #define CACHE_LINE_SIZE 64
34
35
36 // MSR registers (possibly Intel specific)
37 #define IA32_MSR_TSC 0x10
38 #define IA32_MSR_PLATFORM_ID 0x17
39 #define IA32_MSR_APIC_BASE 0x1b
40 #define IA32_MSR_SPEC_CTRL 0x48
41 #define IA32_MSR_PRED_CMD 0x49
42 #define IA32_MSR_UCODE_WRITE 0x79 // IA32_BIOS_UPDT_TRIG
43 #define IA32_MSR_UCODE_REV 0x8b // IA32_BIOS_SIGN_ID
44 #define IA32_MSR_PLATFORM_INFO 0xce
45 #define IA32_MSR_MPERF 0xe7
46 #define IA32_MSR_APERF 0xe8
47 #define IA32_MSR_MTRR_CAPABILITIES 0xfe
48 #define IA32_MSR_ARCH_CAPABILITIES 0x10a
49 #define IA32_MSR_FLUSH_CMD 0x10b
50 #define IA32_MSR_SYSENTER_CS 0x174
51 #define IA32_MSR_SYSENTER_ESP 0x175
52 #define IA32_MSR_SYSENTER_EIP 0x176
53 #define IA32_MSR_PERF_STATUS 0x198
54 #define IA32_MSR_PERF_CTL 0x199
55 #define IA32_MSR_TURBO_RATIO_LIMIT 0x1ad
56 #define IA32_MSR_ENERGY_PERF_BIAS 0x1b0
57 #define IA32_MSR_MTRR_DEFAULT_TYPE 0x2ff
58 #define IA32_MSR_MTRR_PHYSICAL_BASE_0 0x200
59 #define IA32_MSR_MTRR_PHYSICAL_MASK_0 0x201
60 #define IA32_MSR_PAT 0x277
61
62 // MSR SPEC CTRL bits
63 #define IA32_MSR_SPEC_CTRL_IBRS (1 << 0)
64 #define IA32_MSR_SPEC_CTRL_STIBP (1 << 1)
65 #define IA32_MSR_SPEC_CTRL_SSBD (1 << 2)
66
67 // MSR PRED CMD bits
68 #define IA32_MSR_PRED_CMD_IBPB (1 << 0)
69
70 // MSR APIC BASE bits
71 #define IA32_MSR_APIC_BASE_BSP 0x00000100
72 #define IA32_MSR_APIC_BASE_X2APIC 0x00000400
73 #define IA32_MSR_APIC_BASE_ENABLED 0x00000800
74 #define IA32_MSR_APIC_BASE_ADDRESS 0xfffff000
75
76 // MSR EFER bits
77 // reference
78 #define IA32_MSR_EFER_SYSCALL (1 << 0)
79 #define IA32_MSR_EFER_NX (1 << 11)
80
81 // MSR ARCH CAPABILITIES bits
82 #define IA32_MSR_ARCH_CAP_RDCL_NO (1 << 0)
83 #define IA32_MSR_ARCH_CAP_IBRS_ALL (1 << 1)
84 #define IA32_MSR_ARCH_CAP_RSBA (1 << 2)
85 #define IA32_MSR_ARCH_CAP_SKIP_L1D_VMENTRY (1 << 3)
86 #define IA32_MSR_ARCH_CAP_SSB_NO (1 << 4)
87
88 // MSR FLUSH CMD bits
89 #define IA32_MSR_L1D_FLUSH (1 << 1)
90
91 // X2APIC MSRs.
92 #define IA32_MSR_APIC_ID 0x00000802
93 #define IA32_MSR_APIC_VERSION 0x00000803
94 #define IA32_MSR_APIC_TASK_PRIORITY 0x00000808
95 #define IA32_MSR_APIC_PROCESSOR_PRIORITY 0x0000080a
96 #define IA32_MSR_APIC_EOI 0x0000080b
97 #define IA32_MSR_APIC_LOGICAL_DEST 0x0000080d
98 #define IA32_MSR_APIC_SPURIOUS_INTR_VECTOR 0x0000080f
99 #define IA32_MSR_APIC_ERROR_STATUS 0x00000828
100 #define IA32_MSR_APIC_INTR_COMMAND 0x00000830
101 #define IA32_MSR_APIC_LVT_TIMER 0x00000832
102 #define IA32_MSR_APIC_LVT_THERMAL_SENSOR 0x00000833
103 #define IA32_MSR_APIC_LVT_PERFMON_COUNTERS 0x00000834
104 #define IA32_MSR_APIC_LVT_LINT0 0x00000835
105 #define IA32_MSR_APIC_LVT_LINT1 0x00000836
106 #define IA32_MSR_APIC_LVT_ERROR 0x00000837
107 #define IA32_MSR_APIC_INITIAL_TIMER_COUNT 0x00000838
108 #define IA32_MSR_APIC_CURRENT_TIMER_COUNT 0x00000839
109 #define IA32_MSR_APIC_TIMER_DIVIDE_CONFIG 0x0000083e
110 #define IA32_MSR_APIC_SELF_IPI 0x0000083f
111 #define IA32_MSR_XSS 0x00000da0
112
113 // x86_64 MSRs.
114 #define IA32_MSR_EFER 0xc0000080
115 #define IA32_MSR_STAR 0xc0000081
116 #define IA32_MSR_LSTAR 0xc0000082
117 #define IA32_MSR_CSTAR 0xc0000083
118 #define IA32_MSR_FMASK 0xc0000084
119 #define IA32_MSR_FS_BASE 0xc0000100
120 #define IA32_MSR_GS_BASE 0xc0000101
121 #define IA32_MSR_KERNEL_GS_BASE 0xc0000102
122 #define IA32_MSR_TSC_AUX 0xc0000103
123
124 // AMD MSR registers
125 #define MSR_F10H_HWCR 0xc0010015
126 #define HWCR_TSCFREQSEL (1 << 24)
127 #define MSR_K8_UCODE_UPDATE 0xc0010020
128 #define K8_MSR_IPM 0xc0010055
129 #define MSR_F10H_PSTATEDEF(x) (0xc0010064 + (x))
130 #define PSTATEDEF_EN (1ULL << 63)
131 #define MSR_F10H_DE_CFG 0xc0011029
132 #define DE_CFG_SERIALIZE_LFENCE (1 << 1)
133
134 #define MSR_AMD_CPPC_CAP1 0xc00102b0
135 #define AMD_CPPC_LOWEST_PERF(x) ((x) & 0xff)
136 #define AMD_CPPC_LOWNONLIN_PERF(x) ((x >> 8) & 0xff)
137 #define AMD_CPPC_NOMINAL_PERF(x) ((x >> 16) & 0xff)
138 #define AMD_CPPC_HIGHEST_PERF(x) ((x >> 24) & 0xff)
139 #define MSR_AMD_CPPC_ENABLE 0xc00102b1
140 #define MSR_AMD_CPPC_REQ 0xc00102b3
141 #define AMD_CPPC_MAX_PERF(x) ((x) & 0xff)
142 #define AMD_CPPC_MIN_PERF(x) (((x) & 0xff) << 8)
143 #define AMD_CPPC_DES_PERF(x) (((x) & 0xff) << 16)
144 #define AMD_CPPC_EPP_PERF(x) (((x) & 0xff) << 24)
145
146 #define AMD_CPPC_EPP_PERFORMANCE 0x00
147 #define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80
148 #define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xbf
149 #define AMD_CPPC_EPP_POWERSAVE 0xff
150 #define MSR_AMD_CPPC_STATUS 0xc00102b4
151
152
153 // Hardware P-States MSR registers §14.4.1
154 // reference https://software.intel.com/content/dam/develop/public/us/en/documents/253669-sdm-vol-3b.pdf
155 #define IA32_MSR_PM_ENABLE 0x00000770
156 #define IA32_MSR_HWP_CAPABILITIES 0x00000771
157 #define IA32_MSR_HWP_REQUEST_PKG 0x00000772
158 #define IA32_MSR_HWP_INTERRUPT 0x00000773
159 #define IA32_MSR_HWP_REQUEST 0x00000774
160 #define IA32_MSR_HWP_STATUS 0x00000777
161
162 // IA32_MSR_HWP_CAPABILITIES bits §14.4.3
163 #define IA32_HWP_CAPS_HIGHEST_PERFORMANCE(x) (((x) >> 0) & 0xff)
164 #define IA32_HWP_CAPS_GUARANTEED_PERFORMANCE(x) (((x) >> 8) & 0xff)
165 #define IA32_HWP_CAPS_EFFICIENT_PERFORMANCE(x) (((x) >> 16) & 0xff)
166 #define IA32_HWP_CAPS_LOWEST_PERFORMANCE(x) (((x) >> 24) & 0xff)
167
168 // IA32_MSR_HWP_REQUEST bits §14.4.4.1
169 #define IA32_HWP_REQUEST_MINIMUM_PERFORMANCE (0xffULL << 0)
170 #define IA32_HWP_REQUEST_MAXIMUM_PERFORMANCE (0xffULL << 8)
171 #define IA32_HWP_REQUEST_DESIRED_PERFORMANCE (0xffULL << 16)
172 #define IA32_HWP_REQUEST_ENERGY_PERFORMANCE_PREFERENCE (0xffULL << 24)
173 #define IA32_HWP_REQUEST_ACTIVITY_WINDOW (0x3ffULL << 32)
174 #define IA32_HWP_REQUEST_PACKAGE_CONTROL (1ULL << 42)
175 #define IA32_HWP_REQUEST_ACTIVITY_WINDOW_VALID (1ULL << 59)
176 #define IA32_HWP_REQUEST_EPP_VALID (1ULL << 60)
177 #define IA32_HWP_REQUEST_DESIRED_VALID (1ULL << 61)
178 #define IA32_HWP_REQUEST_MAXIMUM_VALID (1ULL << 62)
179 #define IA32_HWP_REQUEST_MINIMUM_VALID (1ULL << 63)
180
181 // IA32_MSR_PAT bits
182 #define IA32_MSR_PAT_ENTRY_MASK 0x7ULL
183 #define IA32_MSR_PAT_ENTRY_SHIFT(x) (x * 8)
184 #define IA32_MSR_PAT_TYPE_UNCACHEABLE 0x0ULL
185 #define IA32_MSR_PAT_TYPE_WRITE_COMBINING 0x1ULL
186 #define IA32_MSR_PAT_TYPE_WRITE_THROUGH 0x4ULL
187 #define IA32_MSR_PAT_TYPE_WRITE_PROTECTED 0x5ULL
188 #define IA32_MSR_PAT_TYPE_WRITE_BACK 0x6ULL
189 #define IA32_MSR_PAT_TYPE_UNCACHED 0x7ULL
190
191 // x86 features from cpuid eax 1, edx register
192 // reference http://www.intel.com/Assets/en_US/PDF/appnote/241618.pdf (Table 5-5)
193 #define IA32_FEATURE_FPU (1 << 0) // x87 fpu
194 #define IA32_FEATURE_VME (1 << 1) // virtual 8086
195 #define IA32_FEATURE_DE (1 << 2) // debugging extensions
196 #define IA32_FEATURE_PSE (1 << 3) // page size extensions
197 #define IA32_FEATURE_TSC (1 << 4) // rdtsc instruction
198 #define IA32_FEATURE_MSR (1 << 5) // rdmsr/wrmsr instruction
199 #define IA32_FEATURE_PAE (1 << 6) // extended 3 level page table addressing
200 #define IA32_FEATURE_MCE (1 << 7) // machine check exception
201 #define IA32_FEATURE_CX8 (1 << 8) // cmpxchg8b instruction
202 #define IA32_FEATURE_APIC (1 << 9) // local apic on chip
203 // (1 << 10) // Reserved
204 #define IA32_FEATURE_SEP (1 << 11) // SYSENTER/SYSEXIT
205 #define IA32_FEATURE_MTRR (1 << 12) // MTRR
206 #define IA32_FEATURE_PGE (1 << 13) // paging global bit
207 #define IA32_FEATURE_MCA (1 << 14) // machine check architecture
208 #define IA32_FEATURE_CMOV (1 << 15) // cmov instruction
209 #define IA32_FEATURE_PAT (1 << 16) // page attribute table
210 #define IA32_FEATURE_PSE36 (1 << 17) // page size extensions with 4MB pages
211 #define IA32_FEATURE_PSN (1 << 18) // processor serial number
212 #define IA32_FEATURE_CLFSH (1 << 19) // cflush instruction
213 // (1 << 20) // Reserved
214 #define IA32_FEATURE_DS (1 << 21) // debug store
215 #define IA32_FEATURE_ACPI (1 << 22) // thermal monitor and clock ctrl
216 #define IA32_FEATURE_MMX (1 << 23) // mmx instructions
217 #define IA32_FEATURE_FXSR (1 << 24) // FXSAVE/FXRSTOR instruction
218 #define IA32_FEATURE_SSE (1 << 25) // SSE
219 #define IA32_FEATURE_SSE2 (1 << 26) // SSE2
220 #define IA32_FEATURE_SS (1 << 27) // self snoop
221 #define IA32_FEATURE_HTT (1 << 28) // hyperthreading
222 #define IA32_FEATURE_TM (1 << 29) // thermal monitor
223 #define IA32_FEATURE_IA64 (1 << 30) // IA64 processor emulating x86
224 #define IA32_FEATURE_PBE (1 << 31) // pending break enable
225
226 // x86 features from cpuid eax 1, ecx register
227 // reference http://www.intel.com/Assets/en_US/PDF/appnote/241618.pdf (Table 5-4)
228 #define IA32_FEATURE_EXT_SSE3 (1 << 0) // SSE3
229 #define IA32_FEATURE_EXT_PCLMULQDQ (1 << 1) // PCLMULQDQ Instruction
230 #define IA32_FEATURE_EXT_DTES64 (1 << 2) // 64-Bit Debug Store
231 #define IA32_FEATURE_EXT_MONITOR (1 << 3) // MONITOR/MWAIT
232 #define IA32_FEATURE_EXT_DSCPL (1 << 4) // CPL qualified debug store
233 #define IA32_FEATURE_EXT_VMX (1 << 5) // Virtual Machine Extensions
234 #define IA32_FEATURE_EXT_SMX (1 << 6) // Safer Mode Extensions
235 #define IA32_FEATURE_EXT_EST (1 << 7) // Enhanced SpeedStep
236 #define IA32_FEATURE_EXT_TM2 (1 << 8) // Thermal Monitor 2
237 #define IA32_FEATURE_EXT_SSSE3 (1 << 9) // Supplemental SSE-3
238 #define IA32_FEATURE_EXT_CNXTID (1 << 10) // L1 Context ID
239 // (1 << 11) // Reserved
240 #define IA32_FEATURE_EXT_FMA (1 << 12) // Fused Multiply Add
241 #define IA32_FEATURE_EXT_CX16 (1 << 13) // CMPXCHG16B
242 #define IA32_FEATURE_EXT_XTPR (1 << 14) // xTPR Update Control
243 #define IA32_FEATURE_EXT_PDCM (1 << 15) // Perfmon and Debug Capability
244 // (1 << 16) // Reserved
245 #define IA32_FEATURE_EXT_PCID (1 << 17) // Process Context Identifiers
246 #define IA32_FEATURE_EXT_DCA (1 << 18) // Direct Cache Access
247 #define IA32_FEATURE_EXT_SSE4_1 (1 << 19) // SSE4.1
248 #define IA32_FEATURE_EXT_SSE4_2 (1 << 20) // SSE4.2
249 #define IA32_FEATURE_EXT_X2APIC (1 << 21) // Extended xAPIC Support
250 #define IA32_FEATURE_EXT_MOVBE (1 << 22) // MOVBE Instruction
251 #define IA32_FEATURE_EXT_POPCNT (1 << 23) // POPCNT Instruction
252 #define IA32_FEATURE_EXT_TSCDEADLINE (1 << 24) // Time Stamp Counter Deadline
253 #define IA32_FEATURE_EXT_AES (1 << 25) // AES Instruction Extensions
254 #define IA32_FEATURE_EXT_XSAVE (1 << 26) // XSAVE/XSTOR States
255 #define IA32_FEATURE_EXT_OSXSAVE (1 << 27) // OS-Enabled XSAVE
256 #define IA32_FEATURE_EXT_AVX (1 << 28) // Advanced Vector Extensions
257 #define IA32_FEATURE_EXT_F16C (1 << 29) // 16-bit FP conversion
258 #define IA32_FEATURE_EXT_RDRND (1 << 30) // RDRAND instruction
259 #define IA32_FEATURE_EXT_HYPERVISOR (1 << 31) // Running on a hypervisor
260
261 // x86 features from cpuid eax 0x80000001, ecx register (AMD)
262 #define IA32_FEATURE_AMD_EXT_CMPLEGACY (1 << 1) // Core MP legacy mode
263 #define IA32_FEATURE_AMD_EXT_TOPOLOGY (1 << 22) // Topology extensions
264
265 // x86 features from cpuid eax 0x80000001, edx register (AMD)
266 // only care about the ones that are unique to this register
267 #define IA32_FEATURE_AMD_EXT_SYSCALL (1 << 11) // SYSCALL/SYSRET
268 #define IA32_FEATURE_AMD_EXT_NX (1 << 20) // no execute bit
269 #define IA32_FEATURE_AMD_EXT_MMXEXT (1 << 22) // mmx extensions
270 #define IA32_FEATURE_AMD_EXT_FFXSR (1 << 25) // fast FXSAVE/FXRSTOR
271 #define IA32_FEATURE_AMD_EXT_PDPE1GB (1 << 26) // Gibibyte pages
272 #define IA32_FEATURE_AMD_EXT_RDTSCP (1 << 27) // rdtscp instruction
273 #define IA32_FEATURE_AMD_EXT_LONG (1 << 29) // long mode
274 #define IA32_FEATURE_AMD_EXT_3DNOWEXT (1 << 30) // 3DNow! extensions
275 #define IA32_FEATURE_AMD_EXT_3DNOW (1 << 31) // 3DNow!
276
277 // some of the features from cpuid eax 0x80000001, edx register (AMD) are also
278 // available on Intel processors
279 #define IA32_FEATURES_INTEL_EXT (IA32_FEATURE_AMD_EXT_SYSCALL \
280 | IA32_FEATURE_AMD_EXT_NX \
281 | IA32_FEATURE_AMD_EXT_PDPE1GB \
282 | IA32_FEATURE_AMD_EXT_RDTSCP \
283 | IA32_FEATURE_AMD_EXT_LONG)
284
285 // x86 defined features from cpuid eax 5, ecx register
286 #define IA32_FEATURE_POWER_MWAIT (1 << 0)
287 #define IA32_FEATURE_INTERRUPT_MWAIT (1 << 1)
288
289 // x86 defined features from cpuid eax 6, eax register
290 // reference https://software.intel.com/content/dam/develop/public/us/en/documents/253666-sdm-vol-2a.pdf (Table 3-8)
291 #define IA32_FEATURE_DTS (1 << 0) // Digital Thermal Sensor
292 #define IA32_FEATURE_ITB (1 << 1) // Intel Turbo Boost Technology
293 #define IA32_FEATURE_ARAT (1 << 2) // Always running APIC Timer
294 #define IA32_FEATURE_PLN (1 << 4) // Power Limit Notification
295 #define IA32_FEATURE_ECMD (1 << 5) // Extended Clock Modulation Duty
296 #define IA32_FEATURE_PTM (1 << 6) // Package Thermal Management
297 #define IA32_FEATURE_HWP (1 << 7) // Hardware P-states
298 #define IA32_FEATURE_HWP_NOTIFY (1 << 8) // HWP Notification
299 #define IA32_FEATURE_HWP_ACTWIN (1 << 9) // HWP Activity Window
300 #define IA32_FEATURE_HWP_EPP (1 << 10) // HWP Energy Performance Preference
301 #define IA32_FEATURE_HWP_PLR (1 << 11) // HWP Package Level Request
302 #define IA32_FEATURE_HDC (1 << 13) // Hardware Duty Cycling
303 #define IA32_FEATURE_TBMT3 (1 << 14) // Turbo Boost Max Technology 3.0
304 #define IA32_FEATURE_HWP_CAP (1 << 15) // HWP Capabilities
305 #define IA32_FEATURE_HWP_PECI (1 << 16) // HWP PECI override
306 #define IA32_FEATURE_HWP_FLEX (1 << 17) // Flexible HWP
307 #define IA32_FEATURE_HWP_FAST (1 << 18) // Fast access for HWP_REQUEST MSR
308 #define IA32_FEATURE_HW_FEEDBACK (1 << 19) // HW_FEEDBACK*, PACKAGE_THERM*
309 #define IA32_FEATURE_HWP_IGNIDL (1 << 20) // Ignore Idle Logical Processor HWP
310
311 // x86 defined features from cpuid eax 6, ecx register
312 // reference http://www.intel.com/Assets/en_US/PDF/appnote/241618.pdf (Table 5-11)
313 #define IA32_FEATURE_APERFMPERF (1 << 0) // IA32_APERF, IA32_MPERF
314 #define IA32_FEATURE_EPB (1 << 3) // IA32_ENERGY_PERF_BIAS
315
316 // x86 features from cpuid eax 7, ebx register
317 // reference http://www.intel.com/Assets/en_US/PDF/appnote/241618.pdf (Table 3-8)
318 #define IA32_FEATURE_TSC_ADJUST (1 << 1) // IA32_TSC_ADJUST MSR supported
319 #define IA32_FEATURE_SGX (1 << 2) // Software Guard Extensions
320 #define IA32_FEATURE_BMI1 (1 << 3) // Bit Manipulation Instruction Set 1
321 #define IA32_FEATURE_HLE (1 << 4) // Hardware Lock Elision
322 #define IA32_FEATURE_AVX2 (1 << 5) // Advanced Vector Extensions 2
323 #define IA32_FEATURE_SMEP (1 << 7) // Supervisor-Mode Execution Prevention
324 #define IA32_FEATURE_BMI2 (1 << 8) // Bit Manipulation Instruction Set 2
325 #define IA32_FEATURE_ERMS (1 << 9) // Enhanced REP MOVSB/STOSB
326 #define IA32_FEATURE_INVPCID (1 << 10) // INVPCID instruction
327 #define IA32_FEATURE_RTM (1 << 11) // Transactional Synchronization Extensions
328 #define IA32_FEATURE_CQM (1 << 12) // Platform Quality of Service Monitoring
329 #define IA32_FEATURE_MPX (1 << 14) // Memory Protection Extensions
330 #define IA32_FEATURE_RDT_A (1 << 15) // Resource Director Technology Allocation
331 #define IA32_FEATURE_AVX512F (1 << 16) // AVX-512 Foundation
332 #define IA32_FEATURE_AVX512DQ (1 << 17) // AVX-512 Doubleword and Quadword Instructions
333 #define IA32_FEATURE_RDSEED (1 << 18) // RDSEED instruction
334 #define IA32_FEATURE_ADX (1 << 19) // ADX (Multi-Precision Add-Carry Instruction Extensions)
335 #define IA32_FEATURE_SMAP (1 << 20) // Supervisor Mode Access Prevention
336 #define IA32_FEATURE_AVX512IFMA (1 << 21) // AVX-512 Integer Fused Multiply-Add Instructions
337 #define IA32_FEATURE_PCOMMIT (1 << 22) // PCOMMIT instruction
338 #define IA32_FEATURE_CLFLUSHOPT (1 << 23) // CLFLUSHOPT instruction
339 #define IA32_FEATURE_CLWB (1 << 24) // CLWB instruction
340 #define IA32_FEATURE_INTEL_PT (1 << 25) // Intel Processor Trace
341 #define IA32_FEATURE_AVX512PF (1 << 26) // AVX-512 Prefetch Instructions
342 #define IA32_FEATURE_AVX512ER (1 << 27) // AVX-512 Exponential and Reciprocal Instructions
343 #define IA32_FEATURE_AVX512CD (1 << 28) // AVX-512 Conflict Detection Instructions
344 #define IA32_FEATURE_SHA_NI (1 << 29) // SHA extensions
345 #define IA32_FEATURE_AVX512BW (1 << 30) // AVX-512 Byte and Word Instructions
346 #define IA32_FEATURE_AVX512VI (1 << 31) // AVX-512 Vector Length Extensions
347
348 // x86 features from cpuid eax 7, ecx register
349 // reference http://www.intel.com/Assets/en_US/PDF/appnote/241618.pdf (Table 3-8)
350 // https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features
351 #define IA32_FEATURE_AVX512VMBI (1 << 1) // AVX-512 Vector Bit Manipulation Instructions
352 #define IA32_FEATURE_UMIP (1 << 2) // User-mode Instruction Prevention
353 #define IA32_FEATURE_PKU (1 << 3) // Memory Protection Keys for User-mode pages
354 #define IA32_FEATURE_OSPKE (1 << 4) // PKU enabled by OS
355 #define IA32_FEATURE_AVX512VMBI2 (1 << 6) // AVX-512 Vector Bit Manipulation Instructions 2
356 #define IA32_FEATURE_GFNI (1 << 8) // Galois Field instructions
357 #define IA32_FEATURE_VAES (1 << 9) // AES instruction set (VEX-256/EVEX)
358 #define IA32_FEATURE_VPCLMULQDQ (1 << 10) // CLMUL instruction set (VEX-256/EVEX)
359 #define IA32_FEATURE_AVX512_VNNI (1 << 11) // AVX-512 Vector Neural Network Instructions
360 #define IA32_FEATURE_AVX512_BITALG (1 << 12) // AVX-512 BITALG instructions
361 #define IA32_FEATURE_AVX512_VPOPCNTDQ (1 << 14) // AVX-512 Vector Population Count D/Q
362 #define IA32_FEATURE_LA57 (1 << 16) // 5-level page tables
363 #define IA32_FEATURE_RDPID (1 << 22) // RDPID Instruction
364 #define IA32_FEATURE_SGX_LC (1 << 30) // SGX Launch Configuration
365
366 // x86 features from cpuid eax 7, edx register
367 // https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features
368 #define IA32_FEATURE_AVX512_4VNNIW (1 << 2) // AVX-512 4-register Neural Network Instructions
369 #define IA32_FEATURE_AVX512_4FMAPS (1 << 3) // AVX-512 4-register Multiply Accumulation Single precision
370 #define IA32_FEATURE_HYBRID_CPU (1 << 15) // CPUs are of several types
371 #define IA32_FEATURE_IBRS (1 << 26) // IBRS / IBPB Speculation Control
372 #define IA32_FEATURE_STIBP (1 << 27) // STIBP Speculation Control
373 #define IA32_FEATURE_L1D_FLUSH (1 << 28) // L1D_FLUSH supported
374 #define IA32_FEATURE_ARCH_CAPABILITIES (1 << 29) // IA32_ARCH_CAPABILITIES MSR
375 #define IA32_FEATURE_SSBD (1 << 31) // Speculative Store Bypass Disable
376
377 // x86 features from cpuid eax 0xd, ecx 1, eax register
378 // reference http://www.intel.com/Assets/en_US/PDF/appnote/241618.pdf (Table 3-8)
379 #define IA32_FEATURE_XSAVEOPT (1 << 0) // XSAVEOPT Instruction
380 #define IA32_FEATURE_XSAVEC (1 << 1) // XSAVEC and compacted XRSTOR
381 #define IA32_FEATURE_XGETBV1 (1 << 2) // XGETBV with ECX=1 Instruction
382 #define IA32_FEATURE_XSAVES (1 << 3) // XSAVES and XRSTORS Instruction
383
384 // x86 defined features from cpuid eax 0x80000007, edx register
385 #define IA32_FEATURE_AMD_HW_PSTATE (1 << 7)
386 #define IA32_FEATURE_INVARIANT_TSC (1 << 8)
387 #define IA32_FEATURE_CPB (1 << 9)
388 #define IA32_FEATURE_PROC_FEEDBACK (1 << 11)
389
390 // x86 defined features from cpuid eax 0x80000008, ebx register
391 #define IA32_FEATURE_CLZERO (1 << 0) // CLZERO instruction
392 #define IA32_FEATURE_IBPB (1 << 12) // IBPB Support only (no IBRS)
393 #define IA32_FEATURE_AMD_SSBD (1 << 24) // Speculative Store Bypass Disable
394 #define IA32_FEATURE_VIRT_SSBD (1 << 25) // Virtualized Speculative Store Bypass Disable
395 #define IA32_FEATURE_AMD_SSB_NO (1 << 26) // Speculative Store Bypass is fixed in hardware
396 #define IA32_FEATURE_CPPC (1 << 27) // Collaborative Processor Performance Control
397
398
399 // Memory type ranges
400 #define IA32_MTR_UNCACHED 0
401 #define IA32_MTR_WRITE_COMBINING 1
402 #define IA32_MTR_WRITE_THROUGH 4
403 #define IA32_MTR_WRITE_PROTECTED 5
404 #define IA32_MTR_WRITE_BACK 6
405
406 // EFLAGS register
407 #define X86_EFLAGS_CARRY 0x00000001
408 #define X86_EFLAGS_RESERVED1 0x00000002
409 #define X86_EFLAGS_PARITY 0x00000004
410 #define X86_EFLAGS_AUXILIARY_CARRY 0x00000010
411 #define X86_EFLAGS_ZERO 0x00000040
412 #define X86_EFLAGS_SIGN 0x00000080
413 #define X86_EFLAGS_TRAP 0x00000100
414 #define X86_EFLAGS_INTERRUPT 0x00000200
415 #define X86_EFLAGS_DIRECTION 0x00000400
416 #define X86_EFLAGS_OVERFLOW 0x00000800
417 #define X86_EFLAGS_IO_PRIVILEG_LEVEL 0x00003000
418 #define X86_EFLAGS_IO_PRIVILEG_LEVEL_SHIFT 12
419 #define X86_EFLAGS_NESTED_TASK 0x00004000
420 #define X86_EFLAGS_RESUME 0x00010000
421 #define X86_EFLAGS_V86_MODE 0x00020000
422 #define X86_EFLAGS_ALIGNMENT_CHECK 0x00040000 // also SMAP status
423 #define X86_EFLAGS_VIRTUAL_INTERRUPT 0x00080000
424 #define X86_EFLAGS_VIRTUAL_INTERRUPT_PENDING 0x00100000
425 #define X86_EFLAGS_ID 0x00200000
426
427 #define X86_EFLAGS_USER_FLAGS (X86_EFLAGS_CARRY | X86_EFLAGS_PARITY \
428 | X86_EFLAGS_AUXILIARY_CARRY | X86_EFLAGS_ZERO | X86_EFLAGS_SIGN \
429 | X86_EFLAGS_DIRECTION | X86_EFLAGS_OVERFLOW)
430
431 #define CR0_CACHE_DISABLE (1UL << 30)
432 #define CR0_NOT_WRITE_THROUGH (1UL << 29)
433 #define CR0_FPU_EMULATION (1UL << 2)
434 #define CR0_MONITOR_FPU (1UL << 1)
435
436 // Control Register CR4 flags §2.5
437 // https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-vol-3a-part-1-manual.pdf
438 #define IA32_CR4_VME (1UL << 0)
439 #define IA32_CR4_PVI (1UL << 1)
440 #define IA32_CR4_TSD (1UL << 2)
441 #define IA32_CR4_DE (1UL << 3)
442 #define IA32_CR4_PSE (1UL << 4)
443 #define IA32_CR4_PAE (1UL << 5)
444 #define IA32_CR4_MCE (1UL << 6)
445 #define IA32_CR4_GLOBAL_PAGES (1UL << 7)
446 #define IA32_CR4_PCE (1UL << 8)
447 #define CR4_OS_FXSR (1UL << 9)
448 #define CR4_OS_XMM_EXCEPTION (1UL << 10)
449 #define IA32_CR4_UMIP (1UL << 11)
450 #define IA32_CR4_LA57 (1UL << 12)
451 #define IA32_CR4_VMXE (1UL << 13)
452 #define IA32_CR4_SMXE (1UL << 14)
453 #define IA32_CR4_FSGSBASE (1UL << 16)
454 #define IA32_CR4_PCIDE (1UL << 17)
455 #define IA32_CR4_OSXSAVE (1UL << 18)
456 #define IA32_CR4_SMEP (1UL << 20)
457 #define IA32_CR4_SMAP (1UL << 21)
458 #define IA32_CR4_PKE (1UL << 22)
459
460 // Extended Control Register XCR0 flags §13.3
461 // https://software.intel.com/content/dam/develop/public/us/en/documents/253665-sdm-vol-1.pdf
462 #define IA32_XCR0_X87 (1UL << 0)
463 #define IA32_XCR0_SSE (1UL << 1)
464 #define IA32_XCR0_AVX (1UL << 2)
465 #define IA32_XCR0_BNDREG (1UL << 3)
466 #define IA32_XCR0_BNDCSR (1UL << 4)
467 #define IA32_XCR0_OPMASK (1UL << 5)
468 #define IA32_XCR0_ZMM_HI256 (1UL << 6)
469 #define IA32_XCR0_HI16_ZMM (1UL << 7)
470 #define IA32_XCR0_PT (1UL << 8)
471 #define IA32_XCR0_PKRU (1UL << 9)
472
473 // page fault error codes (http://wiki.osdev.org/Page_Fault)
474 #define PGFAULT_P 0x01 // Protection violation
475 #define PGFAULT_W 0x02 // Write
476 #define PGFAULT_U 0x04 // Usermode
477 #define PGFAULT_RSVD 0x08 // Reserved bits
478 #define PGFAULT_I 0x10 // Instruction fetch
479
480 // iframe types
481 #define IFRAME_TYPE_SYSCALL 0x1
482 #define IFRAME_TYPE_OTHER 0x2
483 #define IFRAME_TYPE_MASK 0xf
484
485
486 #ifndef _ASSEMBLER
487
488
489 struct X86PagingStructures;
490
491
492 typedef struct x86_mtrr_info {
493 uint64 base;
494 uint64 size;
495 uint8 type;
496 } x86_mtrr_info;
497
498 typedef struct x86_cpu_module_info {
499 module_info info;
500 uint32 (*count_mtrrs)(void);
501 void (*init_mtrrs)(void);
502
503 void (*set_mtrr)(uint32 index, uint64 base, uint64 length,
504 uint8 type);
505 status_t (*get_mtrr)(uint32 index, uint64* _base, uint64* _length,
506 uint8* _type);
507 void (*set_mtrrs)(uint8 defaultType, const x86_mtrr_info* infos,
508 uint32 count);
509 } x86_cpu_module_info;
510
511 // features
512 enum x86_feature_type {
513 FEATURE_COMMON = 0, // cpuid eax=1, edx register
514 FEATURE_EXT, // cpuid eax=1, ecx register
515 FEATURE_EXT_AMD_ECX, // cpuid eax=0x80000001, ecx register (AMD)
516 FEATURE_EXT_AMD, // cpuid eax=0x80000001, edx register (AMD)
517 FEATURE_5_ECX, // cpuid eax=5, ecx register
518 FEATURE_6_EAX, // cpuid eax=6, eax registers
519 FEATURE_6_ECX, // cpuid eax=6, ecx registers
520 FEATURE_7_EBX, // cpuid eax=7, ebx registers
521 FEATURE_7_ECX, // cpuid eax=7, ecx registers
522 FEATURE_7_EDX, // cpuid eax=7, edx registers
523 FEATURE_EXT_7_EDX, // cpuid eax=0x80000007, edx register
524 FEATURE_EXT_8_EBX, // cpuid eax=0x80000008, ebx register
525 FEATURE_D_1_EAX, // cpuid eax=0xd, ecx=1, eax register
526
527 FEATURE_NUM
528 };
529
530 enum x86_vendors {
531 VENDOR_INTEL = 0,
532 VENDOR_AMD,
533 VENDOR_CYRIX,
534 VENDOR_UMC,
535 VENDOR_NEXGEN,
536 VENDOR_CENTAUR,
537 VENDOR_RISE,
538 VENDOR_TRANSMETA,
539 VENDOR_NSC,
540 VENDOR_HYGON,
541
542 VENDOR_NUM,
543 VENDOR_UNKNOWN,
544 };
545
546
547 typedef struct arch_cpu_info {
548 // saved cpu info
549 enum x86_vendors vendor;
550 uint32 feature[FEATURE_NUM];
551 char model_name[49];
552 const char* vendor_name;
553 int type;
554 int family;
555 int extended_family;
556 int stepping;
557 int model;
558 int extended_model;
559 uint32 patch_level;
560 uint8 hybrid_type;
561
562 uint32 logical_apic_id;
563
564 uint64 mperf_prev;
565 uint64 aperf_prev;
566 bigtime_t perf_timestamp;
567 uint64 frequency;
568
569 struct X86PagingStructures* active_paging_structures;
570
571 size_t dr6; // temporary storage for debug registers (cf.
572 size_t dr7; // x86_exit_user_debug_at_kernel_entry())
573
574 // local TSS for this cpu
575 struct tss tss;
576 #ifndef __x86_64__
577 struct tss double_fault_tss;
578 void* kernel_tls;
579 #endif
580 } arch_cpu_info;
581
582
583 // Reference Intel SDM Volume 3 9.11 "Microcode Update Facilities"
584 // https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-vol-3a-part-1-manual.pdf
585 // 9.11.1 Table 9-7. Microcode Update Field Definitions
586 struct intel_microcode_header {
587 uint32 header_version;
588 uint32 update_revision;
589 uint32 date;
590 uint32 processor_signature;
591 uint32 checksum;
592 uint32 loader_revision;
593 uint32 processor_flags;
594 uint32 data_size;
595 uint32 total_size;
596 uint32 reserved[3];
597 };
598
599
600 struct intel_microcode_extended_signature_header {
601 uint32 extended_signature_count;
602 uint32 extended_checksum;
603 uint32 reserved[3];
604 };
605
606
607 struct intel_microcode_extended_signature {
608 uint32 processor_signature;
609 uint32 processor_flags;
610 uint32 checksum;
611 };
612
613
614 // AMD Microcode structures
615
616 struct amd_container_header {
617 uint32 magic;
618 };
619
620
621 struct amd_section_header {
622 uint32 type;
623 uint32 size;
624 };
625
626
627 struct amd_equiv_cpu_entry {
628 uint32 installed_cpu;
629 uint32 fixed_errata_mask;
630 uint32 fixed_errata_compare;
631 uint16 equiv_cpu;
632 uint16 res;
633 };
634
635
636 struct amd_microcode_header {
637 uint32 data_code;
638 uint32 patch_id;
639 uint16 mc_patch_data_id;
640 uint8 mc_patch_data_len;
641 uint8 init_flag;
642 uint32 mc_patch_data_checksum;
643 uint32 nb_dev_id;
644 uint32 sb_dev_id;
645 uint16 processor_rev_id;
646 uint8 nb_rev_id;
647 uint8 sb_rev_id;
648 uint8 bios_api_rev;
649 uint8 reserved1[3];
650 uint32 match_reg[8];
651 };
652
653
654 extern void (*gCpuIdleFunc)(void);
655
656
657 #ifdef __cplusplus
658 extern "C" {
659 #endif
660
661 struct arch_thread;
662
663 #ifdef __x86_64__
664 void __x86_setup_system_time(uint64 conversionFactor,
665 uint64 conversionFactorNsecs);
666 #else
667 void __x86_setup_system_time(uint32 conversionFactor,
668 uint32 conversionFactorNsecs, bool conversionFactorNsecsShift);
669 #endif
670
671 status_t __x86_patch_errata_percpu(int cpu);
672
673 void x86_userspace_thread_exit(void);
674 void x86_end_userspace_thread_exit(void);
675
676 addr_t x86_get_stack_frame();
677 uint32 x86_count_mtrrs(void);
678 void x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type);
679 status_t x86_get_mtrr(uint32 index, uint64* _base, uint64* _length,
680 uint8* _type);
681 void x86_set_mtrrs(uint8 defaultType, const x86_mtrr_info* infos,
682 uint32 count);
683 void x86_init_fpu();
684 bool x86_check_feature(uint32 feature, enum x86_feature_type type);
685 bool x86_use_pat();
686 void* x86_get_double_fault_stack(int32 cpu, size_t* _size);
687 int32 x86_double_fault_get_cpu(void);
688
689 void x86_invalid_exception(iframe* frame);
690 void x86_fatal_exception(iframe* frame);
691 void x86_unexpected_exception(iframe* frame);
692 void x86_hardware_interrupt(iframe* frame);
693 void x86_page_fault_exception(iframe* iframe);
694
695 #ifndef __x86_64__
696
697 void x86_swap_pgdir(addr_t newPageDir);
698
699 uint64 x86_read_msr(uint32 registerNumber);
700 void x86_write_msr(uint32 registerNumber, uint64 value);
701
702 void x86_context_switch(struct arch_thread* oldState,
703 struct arch_thread* newState);
704
705 void x86_fnsave(void* fpuState);
706 void x86_frstor(const void* fpuState);
707
708 void x86_fxsave(void* fpuState);
709 void x86_fxrstor(const void* fpuState);
710
711 void x86_noop_swap(void* oldFpuState, const void* newFpuState);
712 void x86_fnsave_swap(void* oldFpuState, const void* newFpuState);
713 void x86_fxsave_swap(void* oldFpuState, const void* newFpuState);
714
715 #endif
716
717
718 static inline void
arch_cpu_idle(void)719 arch_cpu_idle(void)
720 {
721 gCpuIdleFunc();
722 }
723
724
725 static inline void
arch_cpu_pause(void)726 arch_cpu_pause(void)
727 {
728 asm volatile("pause" : : : "memory");
729 }
730
731
732 #ifdef __cplusplus
733 } // extern "C" {
734 #endif
735
736 #endif // !_ASSEMBLER
737
738 #endif /* _KERNEL_ARCH_x86_CPU_H */
739