1 /* 2 * Copyright 2018, Jérôme Duval, jerome.duval@gmail.com. 3 * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. 4 * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. 5 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. 6 * Distributed under the terms of the MIT License. 7 * 8 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 9 * Distributed under the terms of the NewOS License. 10 */ 11 12 13 #include <cpu.h> 14 15 #include <string.h> 16 #include <stdlib.h> 17 #include <stdio.h> 18 19 #include <algorithm> 20 21 #include <ACPI.h> 22 23 #include <boot_device.h> 24 #include <commpage.h> 25 #include <debug.h> 26 #include <elf.h> 27 #include <safemode.h> 28 #include <smp.h> 29 #include <util/BitUtils.h> 30 #include <vm/vm.h> 31 #include <vm/vm_types.h> 32 #include <vm/VMAddressSpace.h> 33 34 #include <arch_system_info.h> 35 #include <arch/x86/apic.h> 36 #include <boot/kernel_args.h> 37 38 #include "paging/X86PagingStructures.h" 39 #include "paging/X86VMTranslationMap.h" 40 41 42 #define DUMP_FEATURE_STRING 1 43 #define DUMP_CPU_TOPOLOGY 1 44 #define DUMP_CPU_PATCHLEVEL 1 45 46 47 /* cpu vendor info */ 48 struct cpu_vendor_info { 49 const char *vendor; 50 const char *ident_string[2]; 51 }; 52 53 static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = { 54 { "Intel", { "GenuineIntel" } }, 55 { "AMD", { "AuthenticAMD" } }, 56 { "Cyrix", { "CyrixInstead" } }, 57 { "UMC", { "UMC UMC UMC" } }, 58 { "NexGen", { "NexGenDriven" } }, 59 { "Centaur", { "CentaurHauls" } }, 60 { "Rise", { "RiseRiseRise" } }, 61 { "Transmeta", { "GenuineTMx86", "TransmetaCPU" } }, 62 { "NSC", { "Geode by NSC" } }, 63 { "Hygon", { "HygonGenuine" } }, 64 }; 65 66 #define K8_SMIONCMPHALT (1ULL << 27) 67 #define K8_C1EONCMPHALT (1ULL << 28) 68 69 #define K8_CMPHALT (K8_SMIONCMPHALT | K8_C1EONCMPHALT) 70 71 struct set_mtrr_parameter { 72 int32 index; 73 uint64 base; 74 uint64 length; 75 uint8 type; 76 }; 77 78 struct set_mtrrs_parameter { 79 const x86_mtrr_info* infos; 80 uint32 count; 81 uint8 defaultType; 82 }; 83 84 85 #ifdef __x86_64__ 86 extern addr_t _stac; 87 extern addr_t _clac; 88 extern addr_t _xsave; 89 extern addr_t _xsavec; 90 extern addr_t _xrstor; 91 uint64 gXsaveMask; 92 uint64 gFPUSaveLength = 512; 93 bool gHasXsave = false; 94 bool gHasXsavec = false; 95 #endif 96 97 extern "C" void x86_reboot(void); 98 // from arch.S 99 100 void (*gCpuIdleFunc)(void); 101 #ifndef __x86_64__ 102 void (*gX86SwapFPUFunc)(void* oldState, const void* newState) = x86_noop_swap; 103 bool gHasSSE = false; 104 #endif 105 106 static uint32 sCpuRendezvous; 107 static uint32 sCpuRendezvous2; 108 static uint32 sCpuRendezvous3; 109 static vint32 sTSCSyncRendezvous; 110 111 /* Some specials for the double fault handler */ 112 static uint8* sDoubleFaultStacks; 113 static const size_t kDoubleFaultStackSize = 4096; // size per CPU 114 115 static x86_cpu_module_info* sCpuModule; 116 117 118 /* CPU topology information */ 119 static uint32 (*sGetCPUTopologyID)(int currentCPU); 120 static uint32 sHierarchyMask[CPU_TOPOLOGY_LEVELS]; 121 static uint32 sHierarchyShift[CPU_TOPOLOGY_LEVELS]; 122 123 /* Cache topology information */ 124 static uint32 sCacheSharingMask[CPU_MAX_CACHE_LEVEL]; 125 126 static void* sUcodeData = NULL; 127 static size_t sUcodeDataSize = 0; 128 static struct intel_microcode_header* sLoadedUcodeUpdate; 129 static spinlock sUcodeUpdateLock = B_SPINLOCK_INITIALIZER; 130 131 132 static status_t 133 acpi_shutdown(bool rebootSystem) 134 { 135 if (debug_debugger_running() || !are_interrupts_enabled()) 136 return B_ERROR; 137 138 acpi_module_info* acpi; 139 if (get_module(B_ACPI_MODULE_NAME, (module_info**)&acpi) != B_OK) 140 return B_NOT_SUPPORTED; 141 142 status_t status; 143 if (rebootSystem) { 144 status = acpi->reboot(); 145 } else { 146 status = acpi->prepare_sleep_state(ACPI_POWER_STATE_OFF, NULL, 0); 147 if (status == B_OK) { 148 //cpu_status state = disable_interrupts(); 149 status = acpi->enter_sleep_state(ACPI_POWER_STATE_OFF); 150 //restore_interrupts(state); 151 } 152 } 153 154 put_module(B_ACPI_MODULE_NAME); 155 return status; 156 } 157 158 159 /*! Disable CPU caches, and invalidate them. */ 160 static void 161 disable_caches() 162 { 163 x86_write_cr0((x86_read_cr0() | CR0_CACHE_DISABLE) 164 & ~CR0_NOT_WRITE_THROUGH); 165 wbinvd(); 166 arch_cpu_global_TLB_invalidate(); 167 } 168 169 170 /*! Invalidate CPU caches, and enable them. */ 171 static void 172 enable_caches() 173 { 174 wbinvd(); 175 arch_cpu_global_TLB_invalidate(); 176 x86_write_cr0(x86_read_cr0() 177 & ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH)); 178 } 179 180 181 static void 182 set_mtrr(void* _parameter, int cpu) 183 { 184 struct set_mtrr_parameter* parameter 185 = (struct set_mtrr_parameter*)_parameter; 186 187 // wait until all CPUs have arrived here 188 smp_cpu_rendezvous(&sCpuRendezvous); 189 190 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 191 // that initiated the call_all_cpus() from doing that again and clearing 192 // sCpuRendezvous2 before the last CPU has actually left the loop in 193 // smp_cpu_rendezvous(); 194 if (cpu == 0) 195 atomic_set((int32*)&sCpuRendezvous3, 0); 196 197 disable_caches(); 198 199 sCpuModule->set_mtrr(parameter->index, parameter->base, parameter->length, 200 parameter->type); 201 202 enable_caches(); 203 204 // wait until all CPUs have arrived here 205 smp_cpu_rendezvous(&sCpuRendezvous2); 206 smp_cpu_rendezvous(&sCpuRendezvous3); 207 } 208 209 210 static void 211 set_mtrrs(void* _parameter, int cpu) 212 { 213 set_mtrrs_parameter* parameter = (set_mtrrs_parameter*)_parameter; 214 215 // wait until all CPUs have arrived here 216 smp_cpu_rendezvous(&sCpuRendezvous); 217 218 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 219 // that initiated the call_all_cpus() from doing that again and clearing 220 // sCpuRendezvous2 before the last CPU has actually left the loop in 221 // smp_cpu_rendezvous(); 222 if (cpu == 0) 223 atomic_set((int32*)&sCpuRendezvous3, 0); 224 225 disable_caches(); 226 227 sCpuModule->set_mtrrs(parameter->defaultType, parameter->infos, 228 parameter->count); 229 230 enable_caches(); 231 232 // wait until all CPUs have arrived here 233 smp_cpu_rendezvous(&sCpuRendezvous2); 234 smp_cpu_rendezvous(&sCpuRendezvous3); 235 } 236 237 238 static void 239 init_mtrrs(void* _unused, int cpu) 240 { 241 // wait until all CPUs have arrived here 242 smp_cpu_rendezvous(&sCpuRendezvous); 243 244 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 245 // that initiated the call_all_cpus() from doing that again and clearing 246 // sCpuRendezvous2 before the last CPU has actually left the loop in 247 // smp_cpu_rendezvous(); 248 if (cpu == 0) 249 atomic_set((int32*)&sCpuRendezvous3, 0); 250 251 disable_caches(); 252 253 sCpuModule->init_mtrrs(); 254 255 enable_caches(); 256 257 // wait until all CPUs have arrived here 258 smp_cpu_rendezvous(&sCpuRendezvous2); 259 smp_cpu_rendezvous(&sCpuRendezvous3); 260 } 261 262 263 uint32 264 x86_count_mtrrs(void) 265 { 266 if (sCpuModule == NULL) 267 return 0; 268 269 return sCpuModule->count_mtrrs(); 270 } 271 272 273 void 274 x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type) 275 { 276 struct set_mtrr_parameter parameter; 277 parameter.index = index; 278 parameter.base = base; 279 parameter.length = length; 280 parameter.type = type; 281 282 sCpuRendezvous = sCpuRendezvous2 = 0; 283 call_all_cpus(&set_mtrr, ¶meter); 284 } 285 286 287 status_t 288 x86_get_mtrr(uint32 index, uint64* _base, uint64* _length, uint8* _type) 289 { 290 // the MTRRs are identical on all CPUs, so it doesn't matter 291 // on which CPU this runs 292 return sCpuModule->get_mtrr(index, _base, _length, _type); 293 } 294 295 296 void 297 x86_set_mtrrs(uint8 defaultType, const x86_mtrr_info* infos, uint32 count) 298 { 299 if (sCpuModule == NULL) 300 return; 301 302 struct set_mtrrs_parameter parameter; 303 parameter.defaultType = defaultType; 304 parameter.infos = infos; 305 parameter.count = count; 306 307 sCpuRendezvous = sCpuRendezvous2 = 0; 308 call_all_cpus(&set_mtrrs, ¶meter); 309 } 310 311 312 void 313 x86_init_fpu(void) 314 { 315 // All x86_64 CPUs support SSE, don't need to bother checking for it. 316 #ifndef __x86_64__ 317 if (!x86_check_feature(IA32_FEATURE_FPU, FEATURE_COMMON)) { 318 // No FPU... time to install one in your 386? 319 dprintf("%s: Warning: CPU has no reported FPU.\n", __func__); 320 gX86SwapFPUFunc = x86_noop_swap; 321 return; 322 } 323 324 if (!x86_check_feature(IA32_FEATURE_SSE, FEATURE_COMMON) 325 || !x86_check_feature(IA32_FEATURE_FXSR, FEATURE_COMMON)) { 326 dprintf("%s: CPU has no SSE... just enabling FPU.\n", __func__); 327 // we don't have proper SSE support, just enable FPU 328 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 329 gX86SwapFPUFunc = x86_fnsave_swap; 330 return; 331 } 332 #endif 333 334 dprintf("%s: CPU has SSE... enabling FXSR and XMM.\n", __func__); 335 #ifndef __x86_64__ 336 // enable OS support for SSE 337 x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION); 338 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 339 340 gX86SwapFPUFunc = x86_fxsave_swap; 341 gHasSSE = true; 342 #endif 343 } 344 345 346 #if DUMP_FEATURE_STRING 347 static void 348 dump_feature_string(int currentCPU, cpu_ent* cpu) 349 { 350 char features[512]; 351 features[0] = 0; 352 353 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FPU) 354 strlcat(features, "fpu ", sizeof(features)); 355 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_VME) 356 strlcat(features, "vme ", sizeof(features)); 357 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DE) 358 strlcat(features, "de ", sizeof(features)); 359 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE) 360 strlcat(features, "pse ", sizeof(features)); 361 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TSC) 362 strlcat(features, "tsc ", sizeof(features)); 363 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MSR) 364 strlcat(features, "msr ", sizeof(features)); 365 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAE) 366 strlcat(features, "pae ", sizeof(features)); 367 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCE) 368 strlcat(features, "mce ", sizeof(features)); 369 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CX8) 370 strlcat(features, "cx8 ", sizeof(features)); 371 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_APIC) 372 strlcat(features, "apic ", sizeof(features)); 373 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SEP) 374 strlcat(features, "sep ", sizeof(features)); 375 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MTRR) 376 strlcat(features, "mtrr ", sizeof(features)); 377 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PGE) 378 strlcat(features, "pge ", sizeof(features)); 379 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCA) 380 strlcat(features, "mca ", sizeof(features)); 381 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CMOV) 382 strlcat(features, "cmov ", sizeof(features)); 383 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAT) 384 strlcat(features, "pat ", sizeof(features)); 385 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE36) 386 strlcat(features, "pse36 ", sizeof(features)); 387 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSN) 388 strlcat(features, "psn ", sizeof(features)); 389 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CLFSH) 390 strlcat(features, "clfsh ", sizeof(features)); 391 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DS) 392 strlcat(features, "ds ", sizeof(features)); 393 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_ACPI) 394 strlcat(features, "acpi ", sizeof(features)); 395 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MMX) 396 strlcat(features, "mmx ", sizeof(features)); 397 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FXSR) 398 strlcat(features, "fxsr ", sizeof(features)); 399 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE) 400 strlcat(features, "sse ", sizeof(features)); 401 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE2) 402 strlcat(features, "sse2 ", sizeof(features)); 403 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SS) 404 strlcat(features, "ss ", sizeof(features)); 405 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_HTT) 406 strlcat(features, "htt ", sizeof(features)); 407 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TM) 408 strlcat(features, "tm ", sizeof(features)); 409 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PBE) 410 strlcat(features, "pbe ", sizeof(features)); 411 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE3) 412 strlcat(features, "sse3 ", sizeof(features)); 413 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCLMULQDQ) 414 strlcat(features, "pclmulqdq ", sizeof(features)); 415 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DTES64) 416 strlcat(features, "dtes64 ", sizeof(features)); 417 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MONITOR) 418 strlcat(features, "monitor ", sizeof(features)); 419 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DSCPL) 420 strlcat(features, "dscpl ", sizeof(features)); 421 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_VMX) 422 strlcat(features, "vmx ", sizeof(features)); 423 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SMX) 424 strlcat(features, "smx ", sizeof(features)); 425 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_EST) 426 strlcat(features, "est ", sizeof(features)); 427 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TM2) 428 strlcat(features, "tm2 ", sizeof(features)); 429 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSSE3) 430 strlcat(features, "ssse3 ", sizeof(features)); 431 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CNXTID) 432 strlcat(features, "cnxtid ", sizeof(features)); 433 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_FMA) 434 strlcat(features, "fma ", sizeof(features)); 435 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CX16) 436 strlcat(features, "cx16 ", sizeof(features)); 437 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XTPR) 438 strlcat(features, "xtpr ", sizeof(features)); 439 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PDCM) 440 strlcat(features, "pdcm ", sizeof(features)); 441 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCID) 442 strlcat(features, "pcid ", sizeof(features)); 443 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DCA) 444 strlcat(features, "dca ", sizeof(features)); 445 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_1) 446 strlcat(features, "sse4_1 ", sizeof(features)); 447 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_2) 448 strlcat(features, "sse4_2 ", sizeof(features)); 449 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_X2APIC) 450 strlcat(features, "x2apic ", sizeof(features)); 451 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MOVBE) 452 strlcat(features, "movbe ", sizeof(features)); 453 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_POPCNT) 454 strlcat(features, "popcnt ", sizeof(features)); 455 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TSCDEADLINE) 456 strlcat(features, "tscdeadline ", sizeof(features)); 457 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AES) 458 strlcat(features, "aes ", sizeof(features)); 459 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XSAVE) 460 strlcat(features, "xsave ", sizeof(features)); 461 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_OSXSAVE) 462 strlcat(features, "osxsave ", sizeof(features)); 463 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AVX) 464 strlcat(features, "avx ", sizeof(features)); 465 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_F16C) 466 strlcat(features, "f16c ", sizeof(features)); 467 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_RDRND) 468 strlcat(features, "rdrnd ", sizeof(features)); 469 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) 470 strlcat(features, "hypervisor ", sizeof(features)); 471 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_SYSCALL) 472 strlcat(features, "syscall ", sizeof(features)); 473 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_NX) 474 strlcat(features, "nx ", sizeof(features)); 475 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_MMXEXT) 476 strlcat(features, "mmxext ", sizeof(features)); 477 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_FFXSR) 478 strlcat(features, "ffxsr ", sizeof(features)); 479 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_PDPE1GB) 480 strlcat(features, "pdpe1gb ", sizeof(features)); 481 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_LONG) 482 strlcat(features, "long ", sizeof(features)); 483 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOWEXT) 484 strlcat(features, "3dnowext ", sizeof(features)); 485 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOW) 486 strlcat(features, "3dnow ", sizeof(features)); 487 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_DTS) 488 strlcat(features, "dts ", sizeof(features)); 489 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ITB) 490 strlcat(features, "itb ", sizeof(features)); 491 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ARAT) 492 strlcat(features, "arat ", sizeof(features)); 493 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PLN) 494 strlcat(features, "pln ", sizeof(features)); 495 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ECMD) 496 strlcat(features, "ecmd ", sizeof(features)); 497 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PTM) 498 strlcat(features, "ptm ", sizeof(features)); 499 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP) 500 strlcat(features, "hwp ", sizeof(features)); 501 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP_NOTIFY) 502 strlcat(features, "hwp_notify ", sizeof(features)); 503 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP_ACTWIN) 504 strlcat(features, "hwp_actwin ", sizeof(features)); 505 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP_EPP) 506 strlcat(features, "hwp_epp ", sizeof(features)); 507 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP_PLR) 508 strlcat(features, "hwp_plr ", sizeof(features)); 509 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HDC) 510 strlcat(features, "hdc ", sizeof(features)); 511 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_TBMT3) 512 strlcat(features, "tbmt3 ", sizeof(features)); 513 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP_CAP) 514 strlcat(features, "hwp_cap ", sizeof(features)); 515 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP_PECI) 516 strlcat(features, "hwp_peci ", sizeof(features)); 517 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP_FLEX) 518 strlcat(features, "hwp_flex ", sizeof(features)); 519 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP_FAST) 520 strlcat(features, "hwp_fast ", sizeof(features)); 521 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HW_FEEDBACK) 522 strlcat(features, "hw_feedback ", sizeof(features)); 523 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP_IGNIDL) 524 strlcat(features, "hwp_ignidl ", sizeof(features)); 525 if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_APERFMPERF) 526 strlcat(features, "aperfmperf ", sizeof(features)); 527 if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_EPB) 528 strlcat(features, "epb ", sizeof(features)); 529 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_TSC_ADJUST) 530 strlcat(features, "tsc_adjust ", sizeof(features)); 531 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SGX) 532 strlcat(features, "sgx ", sizeof(features)); 533 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_BMI1) 534 strlcat(features, "bmi1 ", sizeof(features)); 535 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_HLE) 536 strlcat(features, "hle ", sizeof(features)); 537 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX2) 538 strlcat(features, "avx2 ", sizeof(features)); 539 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SMEP) 540 strlcat(features, "smep ", sizeof(features)); 541 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_BMI2) 542 strlcat(features, "bmi2 ", sizeof(features)); 543 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_ERMS) 544 strlcat(features, "erms ", sizeof(features)); 545 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_INVPCID) 546 strlcat(features, "invpcid ", sizeof(features)); 547 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_RTM) 548 strlcat(features, "rtm ", sizeof(features)); 549 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_CQM) 550 strlcat(features, "cqm ", sizeof(features)); 551 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_MPX) 552 strlcat(features, "mpx ", sizeof(features)); 553 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_RDT_A) 554 strlcat(features, "rdt_a ", sizeof(features)); 555 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512F) 556 strlcat(features, "avx512f ", sizeof(features)); 557 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512DQ) 558 strlcat(features, "avx512dq ", sizeof(features)); 559 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_RDSEED) 560 strlcat(features, "rdseed ", sizeof(features)); 561 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_ADX) 562 strlcat(features, "adx ", sizeof(features)); 563 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SMAP) 564 strlcat(features, "smap ", sizeof(features)); 565 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512IFMA) 566 strlcat(features, "avx512ifma ", sizeof(features)); 567 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_PCOMMIT) 568 strlcat(features, "pcommit ", sizeof(features)); 569 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_CLFLUSHOPT) 570 strlcat(features, "cflushopt ", sizeof(features)); 571 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_CLWB) 572 strlcat(features, "clwb ", sizeof(features)); 573 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_INTEL_PT) 574 strlcat(features, "intel_pt ", sizeof(features)); 575 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512PF) 576 strlcat(features, "avx512pf ", sizeof(features)); 577 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512ER) 578 strlcat(features, "avx512er ", sizeof(features)); 579 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512CD) 580 strlcat(features, "avx512cd ", sizeof(features)); 581 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SHA_NI) 582 strlcat(features, "sha_ni ", sizeof(features)); 583 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512BW) 584 strlcat(features, "avx512bw ", sizeof(features)); 585 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512VI) 586 strlcat(features, "avx512vi ", sizeof(features)); 587 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_IBRS) 588 strlcat(features, "ibrs ", sizeof(features)); 589 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_STIBP) 590 strlcat(features, "stibp ", sizeof(features)); 591 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_L1D_FLUSH) 592 strlcat(features, "l1d_flush ", sizeof(features)); 593 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_ARCH_CAPABILITIES) 594 strlcat(features, "msr_arch ", sizeof(features)); 595 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_SSBD) 596 strlcat(features, "ssbd ", sizeof(features)); 597 if (cpu->arch.feature[FEATURE_D_1_EAX] & IA32_FEATURE_XSAVEOPT) 598 strlcat(features, "xsaveopt ", sizeof(features)); 599 if (cpu->arch.feature[FEATURE_D_1_EAX] & IA32_FEATURE_XSAVEC) 600 strlcat(features, "xsavec ", sizeof(features)); 601 if (cpu->arch.feature[FEATURE_D_1_EAX] & IA32_FEATURE_XGETBV1) 602 strlcat(features, "xgetbv1 ", sizeof(features)); 603 if (cpu->arch.feature[FEATURE_D_1_EAX] & IA32_FEATURE_XSAVES) 604 strlcat(features, "xsaves ", sizeof(features)); 605 if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_CLZERO) 606 strlcat(features, "clzero ", sizeof(features)); 607 if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_IBPB) 608 strlcat(features, "ibpb ", sizeof(features)); 609 if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_AMD_SSBD) 610 strlcat(features, "amd_ssbd ", sizeof(features)); 611 if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_VIRT_SSBD) 612 strlcat(features, "virt_ssbd ", sizeof(features)); 613 if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_AMD_SSB_NO) 614 strlcat(features, "amd_ssb_no ", sizeof(features)); 615 dprintf("CPU %d: features: %s\n", currentCPU, features); 616 } 617 #endif // DUMP_FEATURE_STRING 618 619 620 static void 621 compute_cpu_hierarchy_masks(int maxLogicalID, int maxCoreID) 622 { 623 ASSERT(maxLogicalID >= maxCoreID); 624 const int kMaxSMTID = maxLogicalID / maxCoreID; 625 626 sHierarchyMask[CPU_TOPOLOGY_SMT] = kMaxSMTID - 1; 627 sHierarchyShift[CPU_TOPOLOGY_SMT] = 0; 628 629 sHierarchyMask[CPU_TOPOLOGY_CORE] = (maxCoreID - 1) * kMaxSMTID; 630 sHierarchyShift[CPU_TOPOLOGY_CORE] 631 = count_set_bits(sHierarchyMask[CPU_TOPOLOGY_SMT]); 632 633 const uint32 kSinglePackageMask = sHierarchyMask[CPU_TOPOLOGY_SMT] 634 | sHierarchyMask[CPU_TOPOLOGY_CORE]; 635 sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~kSinglePackageMask; 636 sHierarchyShift[CPU_TOPOLOGY_PACKAGE] = count_set_bits(kSinglePackageMask); 637 } 638 639 640 static uint32 641 get_cpu_legacy_initial_apic_id(int /* currentCPU */) 642 { 643 cpuid_info cpuid; 644 get_current_cpuid(&cpuid, 1, 0); 645 return cpuid.regs.ebx >> 24; 646 } 647 648 649 static inline status_t 650 detect_amd_cpu_topology(uint32 maxBasicLeaf, uint32 maxExtendedLeaf) 651 { 652 sGetCPUTopologyID = get_cpu_legacy_initial_apic_id; 653 654 cpuid_info cpuid; 655 get_current_cpuid(&cpuid, 1, 0); 656 int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff); 657 658 int maxCoreID = 1; 659 if (maxExtendedLeaf >= 0x80000008) { 660 get_current_cpuid(&cpuid, 0x80000008, 0); 661 maxCoreID = (cpuid.regs.ecx >> 12) & 0xf; 662 if (maxCoreID != 0) 663 maxCoreID = 1 << maxCoreID; 664 else 665 maxCoreID = next_power_of_2((cpuid.regs.edx & 0xf) + 1); 666 } 667 668 if (maxExtendedLeaf >= 0x80000001) { 669 get_current_cpuid(&cpuid, 0x80000001, 0); 670 if (x86_check_feature(IA32_FEATURE_AMD_EXT_CMPLEGACY, 671 FEATURE_EXT_AMD_ECX)) 672 maxCoreID = maxLogicalID; 673 } 674 675 compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID); 676 677 return B_OK; 678 } 679 680 681 static void 682 detect_amd_cache_topology(uint32 maxExtendedLeaf) 683 { 684 if (!x86_check_feature(IA32_FEATURE_AMD_EXT_TOPOLOGY, FEATURE_EXT_AMD_ECX)) 685 return; 686 687 if (maxExtendedLeaf < 0x8000001d) 688 return; 689 690 uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL]; 691 int maxCacheLevel = 0; 692 693 int currentLevel = 0; 694 int cacheType; 695 do { 696 cpuid_info cpuid; 697 get_current_cpuid(&cpuid, 0x8000001d, currentLevel); 698 699 cacheType = cpuid.regs.eax & 0x1f; 700 if (cacheType == 0) 701 break; 702 703 int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; 704 int coresCount = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1); 705 hierarchyLevels[cacheLevel - 1] 706 = coresCount * (sHierarchyMask[CPU_TOPOLOGY_SMT] + 1); 707 maxCacheLevel = std::max(maxCacheLevel, cacheLevel); 708 709 currentLevel++; 710 } while (true); 711 712 for (int i = 0; i < maxCacheLevel; i++) 713 sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); 714 gCPUCacheLevelCount = maxCacheLevel; 715 } 716 717 718 static uint32 719 get_intel_cpu_initial_x2apic_id(int /* currentCPU */) 720 { 721 cpuid_info cpuid; 722 get_current_cpuid(&cpuid, 11, 0); 723 return cpuid.regs.edx; 724 } 725 726 727 static inline status_t 728 detect_intel_cpu_topology_x2apic(uint32 maxBasicLeaf) 729 { 730 if (maxBasicLeaf < 11) 731 return B_UNSUPPORTED; 732 733 uint8 hierarchyLevels[CPU_TOPOLOGY_LEVELS] = { 0 }; 734 735 int currentLevel = 0; 736 int levelType; 737 unsigned int levelsSet = 0; 738 739 do { 740 cpuid_info cpuid; 741 get_current_cpuid(&cpuid, 11, currentLevel); 742 if (currentLevel == 0 && cpuid.regs.ebx == 0) 743 return B_UNSUPPORTED; 744 745 levelType = (cpuid.regs.ecx >> 8) & 0xff; 746 int levelValue = cpuid.regs.eax & 0x1f; 747 748 switch (levelType) { 749 case 1: // SMT 750 hierarchyLevels[CPU_TOPOLOGY_SMT] = levelValue; 751 levelsSet |= 1; 752 break; 753 case 2: // core 754 hierarchyLevels[CPU_TOPOLOGY_CORE] = levelValue; 755 levelsSet |= 2; 756 break; 757 } 758 759 currentLevel++; 760 } while (levelType != 0 && levelsSet != 3); 761 762 sGetCPUTopologyID = get_intel_cpu_initial_x2apic_id; 763 764 for (int i = 1; i < CPU_TOPOLOGY_LEVELS; i++) { 765 if ((levelsSet & (1u << i)) != 0) 766 continue; 767 hierarchyLevels[i] = hierarchyLevels[i - 1]; 768 } 769 770 for (int i = 0; i < CPU_TOPOLOGY_LEVELS; i++) { 771 uint32 mask = ~uint32(0); 772 if (i < CPU_TOPOLOGY_LEVELS - 1) 773 mask = (1u << hierarchyLevels[i]) - 1; 774 if (i > 0) 775 mask &= ~sHierarchyMask[i - 1]; 776 sHierarchyMask[i] = mask; 777 sHierarchyShift[i] = i > 0 ? hierarchyLevels[i - 1] : 0; 778 } 779 780 return B_OK; 781 } 782 783 784 static inline status_t 785 detect_intel_cpu_topology_legacy(uint32 maxBasicLeaf) 786 { 787 sGetCPUTopologyID = get_cpu_legacy_initial_apic_id; 788 789 cpuid_info cpuid; 790 791 get_current_cpuid(&cpuid, 1, 0); 792 int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff); 793 794 int maxCoreID = 1; 795 if (maxBasicLeaf >= 4) { 796 get_current_cpuid(&cpuid, 4, 0); 797 maxCoreID = next_power_of_2((cpuid.regs.eax >> 26) + 1); 798 } 799 800 compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID); 801 802 return B_OK; 803 } 804 805 806 static void 807 detect_intel_cache_topology(uint32 maxBasicLeaf) 808 { 809 if (maxBasicLeaf < 4) 810 return; 811 812 uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL]; 813 int maxCacheLevel = 0; 814 815 int currentLevel = 0; 816 int cacheType; 817 do { 818 cpuid_info cpuid; 819 get_current_cpuid(&cpuid, 4, currentLevel); 820 821 cacheType = cpuid.regs.eax & 0x1f; 822 if (cacheType == 0) 823 break; 824 825 int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; 826 hierarchyLevels[cacheLevel - 1] 827 = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1); 828 maxCacheLevel = std::max(maxCacheLevel, cacheLevel); 829 830 currentLevel++; 831 } while (true); 832 833 for (int i = 0; i < maxCacheLevel; i++) 834 sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); 835 836 gCPUCacheLevelCount = maxCacheLevel; 837 } 838 839 840 static uint32 841 get_simple_cpu_topology_id(int currentCPU) 842 { 843 return currentCPU; 844 } 845 846 847 static inline int 848 get_topology_level_id(uint32 id, cpu_topology_level level) 849 { 850 ASSERT(level < CPU_TOPOLOGY_LEVELS); 851 return (id & sHierarchyMask[level]) >> sHierarchyShift[level]; 852 } 853 854 855 static void 856 detect_cpu_topology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, 857 uint32 maxExtendedLeaf) 858 { 859 if (currentCPU == 0) { 860 memset(sCacheSharingMask, 0xff, sizeof(sCacheSharingMask)); 861 862 status_t result = B_UNSUPPORTED; 863 if (x86_check_feature(IA32_FEATURE_HTT, FEATURE_COMMON)) { 864 if (cpu->arch.vendor == VENDOR_AMD 865 || cpu->arch.vendor == VENDOR_HYGON) { 866 result = detect_amd_cpu_topology(maxBasicLeaf, maxExtendedLeaf); 867 868 if (result == B_OK) 869 detect_amd_cache_topology(maxExtendedLeaf); 870 } 871 872 if (cpu->arch.vendor == VENDOR_INTEL) { 873 result = detect_intel_cpu_topology_x2apic(maxBasicLeaf); 874 if (result != B_OK) 875 result = detect_intel_cpu_topology_legacy(maxBasicLeaf); 876 877 if (result == B_OK) 878 detect_intel_cache_topology(maxBasicLeaf); 879 } 880 } 881 882 if (result != B_OK) { 883 dprintf("No CPU topology information available.\n"); 884 885 sGetCPUTopologyID = get_simple_cpu_topology_id; 886 887 sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~uint32(0); 888 } 889 } 890 891 ASSERT(sGetCPUTopologyID != NULL); 892 int topologyID = sGetCPUTopologyID(currentCPU); 893 cpu->topology_id[CPU_TOPOLOGY_SMT] 894 = get_topology_level_id(topologyID, CPU_TOPOLOGY_SMT); 895 cpu->topology_id[CPU_TOPOLOGY_CORE] 896 = get_topology_level_id(topologyID, CPU_TOPOLOGY_CORE); 897 cpu->topology_id[CPU_TOPOLOGY_PACKAGE] 898 = get_topology_level_id(topologyID, CPU_TOPOLOGY_PACKAGE); 899 900 unsigned int i; 901 for (i = 0; i < gCPUCacheLevelCount; i++) 902 cpu->cache_id[i] = topologyID & sCacheSharingMask[i]; 903 for (; i < CPU_MAX_CACHE_LEVEL; i++) 904 cpu->cache_id[i] = -1; 905 906 #if DUMP_CPU_TOPOLOGY 907 dprintf("CPU %d: apic id %d, package %d, core %d, smt %d\n", currentCPU, 908 topologyID, cpu->topology_id[CPU_TOPOLOGY_PACKAGE], 909 cpu->topology_id[CPU_TOPOLOGY_CORE], 910 cpu->topology_id[CPU_TOPOLOGY_SMT]); 911 912 if (gCPUCacheLevelCount > 0) { 913 char cacheLevels[256]; 914 unsigned int offset = 0; 915 for (i = 0; i < gCPUCacheLevelCount; i++) { 916 offset += snprintf(cacheLevels + offset, 917 sizeof(cacheLevels) - offset, 918 " L%d id %d%s", i + 1, cpu->cache_id[i], 919 i < gCPUCacheLevelCount - 1 ? "," : ""); 920 921 if (offset >= sizeof(cacheLevels)) 922 break; 923 } 924 925 dprintf("CPU %d: cache sharing:%s\n", currentCPU, cacheLevels); 926 } 927 #endif 928 } 929 930 931 static void 932 detect_intel_patch_level(cpu_ent* cpu) 933 { 934 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) { 935 cpu->arch.patch_level = 0; 936 return; 937 } 938 939 x86_write_msr(IA32_MSR_UCODE_REV, 0); 940 cpuid_info cpuid; 941 get_current_cpuid(&cpuid, 1, 0); 942 943 uint64 value = x86_read_msr(IA32_MSR_UCODE_REV); 944 cpu->arch.patch_level = value >> 32; 945 } 946 947 948 static void 949 detect_amd_patch_level(cpu_ent* cpu) 950 { 951 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) { 952 cpu->arch.patch_level = 0; 953 return; 954 } 955 956 uint64 value = x86_read_msr(IA32_MSR_UCODE_REV); 957 cpu->arch.patch_level = value >> 32; 958 } 959 960 961 static struct intel_microcode_header* 962 find_microcode_intel(addr_t data, size_t size, uint32 patchLevel) 963 { 964 // 9.11.3 Processor Identification 965 cpuid_info cpuid; 966 get_current_cpuid(&cpuid, 1, 0); 967 uint32 signature = cpuid.regs.eax; 968 // 9.11.4 Platform Identification 969 uint64 platformBits = (x86_read_msr(IA32_MSR_PLATFORM_ID) >> 50) & 0x7; 970 uint64 mask = 1 << platformBits; 971 972 while (size > 0) { 973 if (size < sizeof(struct intel_microcode_header)) { 974 dprintf("find_microcode_intel update is too small for header\n"); 975 break; 976 } 977 struct intel_microcode_header* header = 978 (struct intel_microcode_header*)data; 979 980 uint32 totalSize = header->total_size; 981 uint32 dataSize = header->data_size; 982 if (dataSize == 0) { 983 dataSize = 2000; 984 totalSize = sizeof(struct intel_microcode_header) 985 + dataSize; 986 } 987 if (totalSize > size) { 988 dprintf("find_microcode_intel update is too small for data\n"); 989 break; 990 } 991 992 uint32* dwords = (uint32*)data; 993 // prepare the next update 994 size -= totalSize; 995 data += totalSize; 996 997 if (header->loader_revision != 1) { 998 dprintf("find_microcode_intel incorrect loader version\n"); 999 continue; 1000 } 1001 // 9.11.6 The microcode update data requires a 16-byte boundary 1002 // alignment. 1003 if (((addr_t)header % 16) != 0) { 1004 dprintf("find_microcode_intel incorrect alignment\n"); 1005 continue; 1006 } 1007 uint32 sum = 0; 1008 for (uint32 i = 0; i < totalSize / 4; i++) { 1009 sum += dwords[i]; 1010 } 1011 if (sum != 0) { 1012 dprintf("find_microcode_intel incorrect checksum\n"); 1013 continue; 1014 } 1015 if (patchLevel > header->update_revision) { 1016 dprintf("find_microcode_intel update_revision is lower\n"); 1017 continue; 1018 } 1019 if (signature == header->processor_signature 1020 && (mask & header->processor_flags) != 0) { 1021 return header; 1022 } 1023 if (totalSize <= (sizeof(struct intel_microcode_header) + dataSize 1024 + sizeof(struct intel_microcode_extended_signature_header))) { 1025 continue; 1026 } 1027 struct intel_microcode_extended_signature_header* extSigHeader = 1028 (struct intel_microcode_extended_signature_header*)((addr_t)header 1029 + sizeof(struct intel_microcode_header) + dataSize); 1030 struct intel_microcode_extended_signature* extended_signature = 1031 (struct intel_microcode_extended_signature*)((addr_t)extSigHeader 1032 + sizeof(struct intel_microcode_extended_signature_header)); 1033 for (uint32 i = 0; i < extSigHeader->extended_signature_count; i++) { 1034 if (signature == extended_signature[i].processor_signature 1035 && (mask & extended_signature[i].processor_flags) != 0) 1036 return header; 1037 } 1038 } 1039 return NULL; 1040 } 1041 1042 1043 static void 1044 load_microcode_intel(int currentCPU, cpu_ent* cpu) 1045 { 1046 // serialize for HT cores 1047 if (currentCPU != 0) 1048 acquire_spinlock(&sUcodeUpdateLock); 1049 detect_intel_patch_level(cpu); 1050 uint32 revision = cpu->arch.patch_level; 1051 struct intel_microcode_header* update = sLoadedUcodeUpdate; 1052 if (update == NULL) { 1053 update = find_microcode_intel((addr_t)sUcodeData, sUcodeDataSize, 1054 revision); 1055 } 1056 if (update != NULL) { 1057 addr_t data = (addr_t)update + sizeof(struct intel_microcode_header); 1058 wbinvd(); 1059 x86_write_msr(IA32_MSR_UCODE_WRITE, data); 1060 detect_intel_patch_level(cpu); 1061 if (revision == cpu->arch.patch_level) { 1062 dprintf("CPU %d: update failed\n", currentCPU); 1063 } else { 1064 if (sLoadedUcodeUpdate == NULL) 1065 sLoadedUcodeUpdate = update; 1066 dprintf("CPU %d: updated from revision %" B_PRIu32 " to %" B_PRIu32 1067 "\n", currentCPU, revision, cpu->arch.patch_level); 1068 } 1069 } else { 1070 dprintf("CPU %d: no update found\n", currentCPU); 1071 } 1072 if (currentCPU != 0) 1073 release_spinlock(&sUcodeUpdateLock); 1074 } 1075 1076 1077 static void 1078 load_microcode_amd(int currentCPU, cpu_ent* cpu) 1079 { 1080 dprintf("CPU %d: no update found\n", currentCPU); 1081 } 1082 1083 1084 static void 1085 load_microcode(int currentCPU) 1086 { 1087 if (sUcodeData == NULL) 1088 return; 1089 cpu_ent* cpu = get_cpu_struct(); 1090 if ((cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) != 0) 1091 return; 1092 if (cpu->arch.vendor == VENDOR_INTEL) 1093 load_microcode_intel(currentCPU, cpu); 1094 else if (cpu->arch.vendor == VENDOR_AMD) 1095 load_microcode_amd(currentCPU, cpu); 1096 } 1097 1098 1099 static void 1100 detect_cpu(int currentCPU) 1101 { 1102 cpu_ent* cpu = get_cpu_struct(); 1103 char vendorString[17]; 1104 cpuid_info cpuid; 1105 1106 // clear out the cpu info data 1107 cpu->arch.vendor = VENDOR_UNKNOWN; 1108 cpu->arch.vendor_name = "UNKNOWN VENDOR"; 1109 cpu->arch.feature[FEATURE_COMMON] = 0; 1110 cpu->arch.feature[FEATURE_EXT] = 0; 1111 cpu->arch.feature[FEATURE_EXT_AMD] = 0; 1112 cpu->arch.feature[FEATURE_7_EBX] = 0; 1113 cpu->arch.feature[FEATURE_7_ECX] = 0; 1114 cpu->arch.feature[FEATURE_7_EDX] = 0; 1115 cpu->arch.feature[FEATURE_D_1_EAX] = 0; 1116 cpu->arch.model_name[0] = 0; 1117 1118 // print some fun data 1119 get_current_cpuid(&cpuid, 0, 0); 1120 uint32 maxBasicLeaf = cpuid.eax_0.max_eax; 1121 1122 // build the vendor string 1123 memset(vendorString, 0, sizeof(vendorString)); 1124 memcpy(vendorString, cpuid.eax_0.vendor_id, sizeof(cpuid.eax_0.vendor_id)); 1125 1126 // get the family, model, stepping 1127 get_current_cpuid(&cpuid, 1, 0); 1128 cpu->arch.type = cpuid.eax_1.type; 1129 cpu->arch.family = cpuid.eax_1.family; 1130 cpu->arch.extended_family = cpuid.eax_1.extended_family; 1131 cpu->arch.model = cpuid.eax_1.model; 1132 cpu->arch.extended_model = cpuid.eax_1.extended_model; 1133 cpu->arch.stepping = cpuid.eax_1.stepping; 1134 dprintf("CPU %d: type %d family %d extended_family %d model %d " 1135 "extended_model %d stepping %d, string '%s'\n", 1136 currentCPU, cpu->arch.type, cpu->arch.family, 1137 cpu->arch.extended_family, cpu->arch.model, 1138 cpu->arch.extended_model, cpu->arch.stepping, vendorString); 1139 1140 // figure out what vendor we have here 1141 1142 for (int32 i = 0; i < VENDOR_NUM; i++) { 1143 if (vendor_info[i].ident_string[0] 1144 && !strcmp(vendorString, vendor_info[i].ident_string[0])) { 1145 cpu->arch.vendor = (x86_vendors)i; 1146 cpu->arch.vendor_name = vendor_info[i].vendor; 1147 break; 1148 } 1149 if (vendor_info[i].ident_string[1] 1150 && !strcmp(vendorString, vendor_info[i].ident_string[1])) { 1151 cpu->arch.vendor = (x86_vendors)i; 1152 cpu->arch.vendor_name = vendor_info[i].vendor; 1153 break; 1154 } 1155 } 1156 1157 // see if we can get the model name 1158 get_current_cpuid(&cpuid, 0x80000000, 0); 1159 uint32 maxExtendedLeaf = cpuid.eax_0.max_eax; 1160 if (maxExtendedLeaf >= 0x80000004) { 1161 // build the model string (need to swap ecx/edx data before copying) 1162 unsigned int temp; 1163 memset(cpu->arch.model_name, 0, sizeof(cpu->arch.model_name)); 1164 1165 get_current_cpuid(&cpuid, 0x80000002, 0); 1166 temp = cpuid.regs.edx; 1167 cpuid.regs.edx = cpuid.regs.ecx; 1168 cpuid.regs.ecx = temp; 1169 memcpy(cpu->arch.model_name, cpuid.as_chars, sizeof(cpuid.as_chars)); 1170 1171 get_current_cpuid(&cpuid, 0x80000003, 0); 1172 temp = cpuid.regs.edx; 1173 cpuid.regs.edx = cpuid.regs.ecx; 1174 cpuid.regs.ecx = temp; 1175 memcpy(cpu->arch.model_name + 16, cpuid.as_chars, 1176 sizeof(cpuid.as_chars)); 1177 1178 get_current_cpuid(&cpuid, 0x80000004, 0); 1179 temp = cpuid.regs.edx; 1180 cpuid.regs.edx = cpuid.regs.ecx; 1181 cpuid.regs.ecx = temp; 1182 memcpy(cpu->arch.model_name + 32, cpuid.as_chars, 1183 sizeof(cpuid.as_chars)); 1184 1185 // some cpus return a right-justified string 1186 int32 i = 0; 1187 while (cpu->arch.model_name[i] == ' ') 1188 i++; 1189 if (i > 0) { 1190 memmove(cpu->arch.model_name, &cpu->arch.model_name[i], 1191 strlen(&cpu->arch.model_name[i]) + 1); 1192 } 1193 1194 dprintf("CPU %d: vendor '%s' model name '%s'\n", 1195 currentCPU, cpu->arch.vendor_name, cpu->arch.model_name); 1196 } else { 1197 strlcpy(cpu->arch.model_name, "unknown", sizeof(cpu->arch.model_name)); 1198 } 1199 1200 // load feature bits 1201 get_current_cpuid(&cpuid, 1, 0); 1202 cpu->arch.feature[FEATURE_COMMON] = cpuid.eax_1.features; // edx 1203 cpu->arch.feature[FEATURE_EXT] = cpuid.eax_1.extended_features; // ecx 1204 1205 if (maxExtendedLeaf >= 0x80000001) { 1206 get_current_cpuid(&cpuid, 0x80000001, 0); 1207 if (cpu->arch.vendor == VENDOR_AMD) 1208 cpu->arch.feature[FEATURE_EXT_AMD_ECX] = cpuid.regs.ecx; // ecx 1209 cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx 1210 if (cpu->arch.vendor != VENDOR_AMD) 1211 cpu->arch.feature[FEATURE_EXT_AMD] &= IA32_FEATURES_INTEL_EXT; 1212 } 1213 1214 if (maxBasicLeaf >= 5) { 1215 get_current_cpuid(&cpuid, 5, 0); 1216 cpu->arch.feature[FEATURE_5_ECX] = cpuid.regs.ecx; 1217 } 1218 1219 if (maxBasicLeaf >= 6) { 1220 get_current_cpuid(&cpuid, 6, 0); 1221 cpu->arch.feature[FEATURE_6_EAX] = cpuid.regs.eax; 1222 cpu->arch.feature[FEATURE_6_ECX] = cpuid.regs.ecx; 1223 } 1224 1225 if (maxBasicLeaf >= 7) { 1226 get_current_cpuid(&cpuid, 7, 0); 1227 cpu->arch.feature[FEATURE_7_EBX] = cpuid.regs.ebx; 1228 cpu->arch.feature[FEATURE_7_ECX] = cpuid.regs.ecx; 1229 cpu->arch.feature[FEATURE_7_EDX] = cpuid.regs.edx; 1230 } 1231 1232 if (maxBasicLeaf >= 0xd) { 1233 get_current_cpuid(&cpuid, 0xd, 1); 1234 cpu->arch.feature[FEATURE_D_1_EAX] = cpuid.regs.eax; 1235 } 1236 1237 if (maxExtendedLeaf >= 0x80000007) { 1238 get_current_cpuid(&cpuid, 0x80000007, 0); 1239 cpu->arch.feature[FEATURE_EXT_7_EDX] = cpuid.regs.edx; 1240 } 1241 1242 if (maxExtendedLeaf >= 0x80000008) { 1243 get_current_cpuid(&cpuid, 0x80000008, 0); 1244 cpu->arch.feature[FEATURE_EXT_8_EBX] = cpuid.regs.ebx; 1245 } 1246 1247 detect_cpu_topology(currentCPU, cpu, maxBasicLeaf, maxExtendedLeaf); 1248 1249 if (cpu->arch.vendor == VENDOR_INTEL) 1250 detect_intel_patch_level(cpu); 1251 else if (cpu->arch.vendor == VENDOR_AMD) 1252 detect_amd_patch_level(cpu); 1253 1254 #if DUMP_FEATURE_STRING 1255 dump_feature_string(currentCPU, cpu); 1256 #endif 1257 #if DUMP_CPU_PATCHLEVEL 1258 dprintf("CPU %d: patch_level %" B_PRIu32 "\n", currentCPU, 1259 cpu->arch.patch_level); 1260 #endif 1261 } 1262 1263 1264 bool 1265 x86_check_feature(uint32 feature, enum x86_feature_type type) 1266 { 1267 cpu_ent* cpu = get_cpu_struct(); 1268 1269 #if 0 1270 int i; 1271 dprintf("x86_check_feature: feature 0x%x, type %d\n", feature, type); 1272 for (i = 0; i < FEATURE_NUM; i++) { 1273 dprintf("features %d: 0x%x\n", i, cpu->arch.feature[i]); 1274 } 1275 #endif 1276 1277 return (cpu->arch.feature[type] & feature) != 0; 1278 } 1279 1280 1281 void* 1282 x86_get_double_fault_stack(int32 cpu, size_t* _size) 1283 { 1284 *_size = kDoubleFaultStackSize; 1285 return sDoubleFaultStacks + kDoubleFaultStackSize * cpu; 1286 } 1287 1288 1289 /*! Returns the index of the current CPU. Can only be called from the double 1290 fault handler. 1291 */ 1292 int32 1293 x86_double_fault_get_cpu(void) 1294 { 1295 addr_t stack = x86_get_stack_frame(); 1296 return (stack - (addr_t)sDoubleFaultStacks) / kDoubleFaultStackSize; 1297 } 1298 1299 1300 // #pragma mark - 1301 1302 1303 status_t 1304 arch_cpu_preboot_init_percpu(kernel_args* args, int cpu) 1305 { 1306 // On SMP system we want to synchronize the CPUs' TSCs, so system_time() 1307 // will return consistent values. 1308 if (smp_get_num_cpus() > 1) { 1309 // let the first CPU prepare the rendezvous point 1310 if (cpu == 0) 1311 sTSCSyncRendezvous = smp_get_num_cpus() - 1; 1312 1313 // One CPU after the other will drop out of this loop and be caught by 1314 // the loop below, until the last CPU (0) gets there. Save for +/- a few 1315 // cycles the CPUs should pass the second loop at the same time. 1316 while (sTSCSyncRendezvous != cpu) { 1317 } 1318 1319 sTSCSyncRendezvous = cpu - 1; 1320 1321 while (sTSCSyncRendezvous != -1) { 1322 } 1323 1324 // reset TSC to 0 1325 x86_write_msr(IA32_MSR_TSC, 0); 1326 } 1327 1328 x86_descriptors_preboot_init_percpu(args, cpu); 1329 1330 return B_OK; 1331 } 1332 1333 1334 static void 1335 halt_idle(void) 1336 { 1337 asm("hlt"); 1338 } 1339 1340 1341 static void 1342 amdc1e_noarat_idle(void) 1343 { 1344 uint64 msr = x86_read_msr(K8_MSR_IPM); 1345 if (msr & K8_CMPHALT) 1346 x86_write_msr(K8_MSR_IPM, msr & ~K8_CMPHALT); 1347 halt_idle(); 1348 } 1349 1350 1351 static bool 1352 detect_amdc1e_noarat() 1353 { 1354 cpu_ent* cpu = get_cpu_struct(); 1355 1356 if (cpu->arch.vendor != VENDOR_AMD) 1357 return false; 1358 1359 // Family 0x12 and higher processors support ARAT 1360 // Family lower than 0xf processors doesn't support C1E 1361 // Family 0xf with model <= 0x40 procssors doesn't support C1E 1362 uint32 family = cpu->arch.family + cpu->arch.extended_family; 1363 uint32 model = (cpu->arch.extended_model << 4) | cpu->arch.model; 1364 return (family < 0x12 && family > 0xf) || (family == 0xf && model > 0x40); 1365 } 1366 1367 1368 status_t 1369 arch_cpu_init_percpu(kernel_args* args, int cpu) 1370 { 1371 load_microcode(cpu); 1372 detect_cpu(cpu); 1373 1374 if (!gCpuIdleFunc) { 1375 if (detect_amdc1e_noarat()) 1376 gCpuIdleFunc = amdc1e_noarat_idle; 1377 else 1378 gCpuIdleFunc = halt_idle; 1379 } 1380 1381 if (x86_check_feature(IA32_FEATURE_MCE, FEATURE_COMMON)) 1382 x86_write_cr4(x86_read_cr4() | IA32_CR4_MCE); 1383 1384 #ifdef __x86_64__ 1385 // if RDTSCP is available write cpu number in TSC_AUX 1386 if (x86_check_feature(IA32_FEATURE_AMD_EXT_RDTSCP, FEATURE_EXT_AMD)) 1387 x86_write_msr(IA32_MSR_TSC_AUX, cpu); 1388 #endif 1389 1390 return __x86_patch_errata_percpu(cpu); 1391 } 1392 1393 1394 status_t 1395 arch_cpu_init(kernel_args* args) 1396 { 1397 if (args->ucode_data != NULL 1398 && args->ucode_data_size > 0) { 1399 sUcodeData = args->ucode_data; 1400 sUcodeDataSize = args->ucode_data_size; 1401 } else { 1402 dprintf("CPU: no microcode provided\n"); 1403 } 1404 1405 // init the TSC -> system_time() conversion factors 1406 1407 uint32 conversionFactor = args->arch_args.system_time_cv_factor; 1408 uint64 conversionFactorNsecs = (uint64)conversionFactor * 1000; 1409 1410 #ifdef __x86_64__ 1411 // The x86_64 system_time() implementation uses 64-bit multiplication and 1412 // therefore shifting is not necessary for low frequencies (it's also not 1413 // too likely that there'll be any x86_64 CPUs clocked under 1GHz). 1414 __x86_setup_system_time((uint64)conversionFactor << 32, 1415 conversionFactorNsecs); 1416 #else 1417 if (conversionFactorNsecs >> 32 != 0) { 1418 // the TSC frequency is < 1 GHz, which forces us to shift the factor 1419 __x86_setup_system_time(conversionFactor, conversionFactorNsecs >> 16, 1420 true); 1421 } else { 1422 // the TSC frequency is >= 1 GHz 1423 __x86_setup_system_time(conversionFactor, conversionFactorNsecs, false); 1424 } 1425 #endif 1426 1427 // Initialize descriptor tables. 1428 x86_descriptors_init(args); 1429 1430 return B_OK; 1431 } 1432 1433 1434 #ifdef __x86_64__ 1435 static void 1436 enable_smap(void* dummy, int cpu) 1437 { 1438 x86_write_cr4(x86_read_cr4() | IA32_CR4_SMAP); 1439 } 1440 1441 1442 static void 1443 enable_smep(void* dummy, int cpu) 1444 { 1445 x86_write_cr4(x86_read_cr4() | IA32_CR4_SMEP); 1446 } 1447 1448 1449 static void 1450 enable_osxsave(void* dummy, int cpu) 1451 { 1452 x86_write_cr4(x86_read_cr4() | IA32_CR4_OSXSAVE); 1453 } 1454 1455 1456 static void 1457 enable_xsavemask(void* dummy, int cpu) 1458 { 1459 xsetbv(0, gXsaveMask); 1460 } 1461 #endif 1462 1463 1464 status_t 1465 arch_cpu_init_post_vm(kernel_args* args) 1466 { 1467 uint32 i; 1468 1469 // allocate an area for the double fault stacks 1470 virtual_address_restrictions virtualRestrictions = {}; 1471 virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS; 1472 physical_address_restrictions physicalRestrictions = {}; 1473 create_area_etc(B_SYSTEM_TEAM, "double fault stacks", 1474 kDoubleFaultStackSize * smp_get_num_cpus(), B_FULL_LOCK, 1475 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0, 1476 &virtualRestrictions, &physicalRestrictions, 1477 (void**)&sDoubleFaultStacks); 1478 1479 X86PagingStructures* kernelPagingStructures 1480 = static_cast<X86VMTranslationMap*>( 1481 VMAddressSpace::Kernel()->TranslationMap())->PagingStructures(); 1482 1483 // Set active translation map on each CPU. 1484 for (i = 0; i < args->num_cpus; i++) { 1485 gCPU[i].arch.active_paging_structures = kernelPagingStructures; 1486 kernelPagingStructures->AddReference(); 1487 } 1488 1489 if (!apic_available()) 1490 x86_init_fpu(); 1491 // else fpu gets set up in smp code 1492 1493 #ifdef __x86_64__ 1494 // if available enable SMEP (Supervisor Memory Execution Protection) 1495 if (x86_check_feature(IA32_FEATURE_SMEP, FEATURE_7_EBX)) { 1496 if (!get_safemode_boolean(B_SAFEMODE_DISABLE_SMEP_SMAP, false)) { 1497 dprintf("enable SMEP\n"); 1498 call_all_cpus_sync(&enable_smep, NULL); 1499 } else 1500 dprintf("SMEP disabled per safemode setting\n"); 1501 } 1502 1503 // if available enable SMAP (Supervisor Memory Access Protection) 1504 if (x86_check_feature(IA32_FEATURE_SMAP, FEATURE_7_EBX)) { 1505 if (!get_safemode_boolean(B_SAFEMODE_DISABLE_SMEP_SMAP, false)) { 1506 dprintf("enable SMAP\n"); 1507 call_all_cpus_sync(&enable_smap, NULL); 1508 1509 arch_altcodepatch_replace(ALTCODEPATCH_TAG_STAC, &_stac, 3); 1510 arch_altcodepatch_replace(ALTCODEPATCH_TAG_CLAC, &_clac, 3); 1511 } else 1512 dprintf("SMAP disabled per safemode setting\n"); 1513 } 1514 1515 // if available enable XSAVE (XSAVE and extended states) 1516 gHasXsave = x86_check_feature(IA32_FEATURE_EXT_XSAVE, FEATURE_EXT); 1517 if (gHasXsave) { 1518 gHasXsavec = x86_check_feature(IA32_FEATURE_XSAVEC, 1519 FEATURE_D_1_EAX); 1520 1521 call_all_cpus_sync(&enable_osxsave, NULL); 1522 gXsaveMask = IA32_XCR0_X87 | IA32_XCR0_SSE; 1523 cpuid_info cpuid; 1524 get_current_cpuid(&cpuid, 0xd, 0); 1525 gXsaveMask |= (cpuid.regs.eax & IA32_XCR0_AVX); 1526 call_all_cpus_sync(&enable_xsavemask, NULL); 1527 get_current_cpuid(&cpuid, 0xd, 0); 1528 gFPUSaveLength = cpuid.regs.ebx; 1529 1530 arch_altcodepatch_replace(ALTCODEPATCH_TAG_XSAVE, 1531 gHasXsavec ? &_xsavec : &_xsave, 4); 1532 arch_altcodepatch_replace(ALTCODEPATCH_TAG_XRSTOR, 1533 &_xrstor, 4); 1534 1535 dprintf("enable %s 0x%" B_PRIx64 " %" B_PRId64 "\n", 1536 gHasXsavec ? "XSAVEC" : "XSAVE", gXsaveMask, gFPUSaveLength); 1537 } 1538 1539 #endif 1540 1541 return B_OK; 1542 } 1543 1544 1545 status_t 1546 arch_cpu_init_post_modules(kernel_args* args) 1547 { 1548 // initialize CPU module 1549 1550 void* cookie = open_module_list("cpu"); 1551 1552 while (true) { 1553 char name[B_FILE_NAME_LENGTH]; 1554 size_t nameLength = sizeof(name); 1555 1556 if (read_next_module_name(cookie, name, &nameLength) != B_OK 1557 || get_module(name, (module_info**)&sCpuModule) == B_OK) 1558 break; 1559 } 1560 1561 close_module_list(cookie); 1562 1563 // initialize MTRRs if available 1564 if (x86_count_mtrrs() > 0) { 1565 sCpuRendezvous = sCpuRendezvous2 = 0; 1566 call_all_cpus(&init_mtrrs, NULL); 1567 } 1568 1569 size_t threadExitLen = (addr_t)x86_end_userspace_thread_exit 1570 - (addr_t)x86_userspace_thread_exit; 1571 addr_t threadExitPosition = fill_commpage_entry( 1572 COMMPAGE_ENTRY_X86_THREAD_EXIT, (const void*)x86_userspace_thread_exit, 1573 threadExitLen); 1574 1575 // add the functions to the commpage image 1576 image_id image = get_commpage_image(); 1577 1578 elf_add_memory_image_symbol(image, "commpage_thread_exit", 1579 threadExitPosition, threadExitLen, B_SYMBOL_TYPE_TEXT); 1580 1581 return B_OK; 1582 } 1583 1584 1585 void 1586 arch_cpu_user_TLB_invalidate(void) 1587 { 1588 x86_write_cr3(x86_read_cr3()); 1589 } 1590 1591 1592 void 1593 arch_cpu_global_TLB_invalidate(void) 1594 { 1595 uint32 flags = x86_read_cr4(); 1596 1597 if (flags & IA32_CR4_GLOBAL_PAGES) { 1598 // disable and reenable the global pages to flush all TLBs regardless 1599 // of the global page bit 1600 x86_write_cr4(flags & ~IA32_CR4_GLOBAL_PAGES); 1601 x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES); 1602 } else { 1603 cpu_status state = disable_interrupts(); 1604 arch_cpu_user_TLB_invalidate(); 1605 restore_interrupts(state); 1606 } 1607 } 1608 1609 1610 void 1611 arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) 1612 { 1613 int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE; 1614 while (num_pages-- >= 0) { 1615 invalidate_TLB(start); 1616 start += B_PAGE_SIZE; 1617 } 1618 } 1619 1620 1621 void 1622 arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) 1623 { 1624 int i; 1625 for (i = 0; i < num_pages; i++) { 1626 invalidate_TLB(pages[i]); 1627 } 1628 } 1629 1630 1631 status_t 1632 arch_cpu_shutdown(bool rebootSystem) 1633 { 1634 if (acpi_shutdown(rebootSystem) == B_OK) 1635 return B_OK; 1636 1637 if (!rebootSystem) { 1638 #ifndef __x86_64__ 1639 return apm_shutdown(); 1640 #else 1641 return B_NOT_SUPPORTED; 1642 #endif 1643 } 1644 1645 cpu_status state = disable_interrupts(); 1646 1647 // try to reset the system using the keyboard controller 1648 out8(0xfe, 0x64); 1649 1650 // Give some time to the controller to do its job (0.5s) 1651 snooze(500000); 1652 1653 // if that didn't help, try it this way 1654 x86_reboot(); 1655 1656 restore_interrupts(state); 1657 return B_ERROR; 1658 } 1659 1660 1661 void 1662 arch_cpu_sync_icache(void* address, size_t length) 1663 { 1664 // instruction cache is always consistent on x86 1665 } 1666 1667