1 /* 2 * Copyright 2018, Jérôme Duval, jerome.duval@gmail.com. 3 * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. 4 * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. 5 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. 6 * Distributed under the terms of the MIT License. 7 * 8 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 9 * Distributed under the terms of the NewOS License. 10 */ 11 12 13 #include <cpu.h> 14 15 #include <string.h> 16 #include <stdlib.h> 17 #include <stdio.h> 18 19 #include <algorithm> 20 21 #include <ACPI.h> 22 23 #include <boot_device.h> 24 #include <commpage.h> 25 #include <debug.h> 26 #include <elf.h> 27 #include <safemode.h> 28 #include <smp.h> 29 #include <util/BitUtils.h> 30 #include <vm/vm.h> 31 #include <vm/vm_types.h> 32 #include <vm/VMAddressSpace.h> 33 34 #include <arch_system_info.h> 35 #include <arch/x86/apic.h> 36 #include <boot/kernel_args.h> 37 38 #include "paging/X86PagingStructures.h" 39 #include "paging/X86VMTranslationMap.h" 40 41 42 #define DUMP_FEATURE_STRING 1 43 #define DUMP_CPU_TOPOLOGY 1 44 #define DUMP_CPU_PATCHLEVEL 1 45 46 47 /* cpu vendor info */ 48 struct cpu_vendor_info { 49 const char *vendor; 50 const char *ident_string[2]; 51 }; 52 53 static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = { 54 { "Intel", { "GenuineIntel" } }, 55 { "AMD", { "AuthenticAMD" } }, 56 { "Cyrix", { "CyrixInstead" } }, 57 { "UMC", { "UMC UMC UMC" } }, 58 { "NexGen", { "NexGenDriven" } }, 59 { "Centaur", { "CentaurHauls" } }, 60 { "Rise", { "RiseRiseRise" } }, 61 { "Transmeta", { "GenuineTMx86", "TransmetaCPU" } }, 62 { "NSC", { "Geode by NSC" } }, 63 }; 64 65 #define K8_SMIONCMPHALT (1ULL << 27) 66 #define K8_C1EONCMPHALT (1ULL << 28) 67 68 #define K8_CMPHALT (K8_SMIONCMPHALT | K8_C1EONCMPHALT) 69 70 struct set_mtrr_parameter { 71 int32 index; 72 uint64 base; 73 uint64 length; 74 uint8 type; 75 }; 76 77 struct set_mtrrs_parameter { 78 const x86_mtrr_info* infos; 79 uint32 count; 80 uint8 defaultType; 81 }; 82 83 84 #ifdef __x86_64__ 85 extern addr_t _stac; 86 extern addr_t _clac; 87 extern addr_t _xsave; 88 extern addr_t _xsavec; 89 extern addr_t _xrstor; 90 uint64 gXsaveMask; 91 uint64 gFPUSaveLength = 512; 92 bool gHasXsave = false; 93 bool gHasXsavec = false; 94 #endif 95 96 extern "C" void x86_reboot(void); 97 // from arch.S 98 99 void (*gCpuIdleFunc)(void); 100 #ifndef __x86_64__ 101 void (*gX86SwapFPUFunc)(void* oldState, const void* newState) = x86_noop_swap; 102 bool gHasSSE = false; 103 #endif 104 105 static uint32 sCpuRendezvous; 106 static uint32 sCpuRendezvous2; 107 static uint32 sCpuRendezvous3; 108 static vint32 sTSCSyncRendezvous; 109 110 /* Some specials for the double fault handler */ 111 static uint8* sDoubleFaultStacks; 112 static const size_t kDoubleFaultStackSize = 4096; // size per CPU 113 114 static x86_cpu_module_info* sCpuModule; 115 116 117 /* CPU topology information */ 118 static uint32 (*sGetCPUTopologyID)(int currentCPU); 119 static uint32 sHierarchyMask[CPU_TOPOLOGY_LEVELS]; 120 static uint32 sHierarchyShift[CPU_TOPOLOGY_LEVELS]; 121 122 /* Cache topology information */ 123 static uint32 sCacheSharingMask[CPU_MAX_CACHE_LEVEL]; 124 125 static void* sUcodeData = NULL; 126 static size_t sUcodeDataSize = 0; 127 static struct intel_microcode_header* sLoadedUcodeUpdate; 128 static spinlock sUcodeUpdateLock = B_SPINLOCK_INITIALIZER; 129 130 131 static status_t 132 acpi_shutdown(bool rebootSystem) 133 { 134 if (debug_debugger_running() || !are_interrupts_enabled()) 135 return B_ERROR; 136 137 acpi_module_info* acpi; 138 if (get_module(B_ACPI_MODULE_NAME, (module_info**)&acpi) != B_OK) 139 return B_NOT_SUPPORTED; 140 141 status_t status; 142 if (rebootSystem) { 143 status = acpi->reboot(); 144 } else { 145 status = acpi->prepare_sleep_state(ACPI_POWER_STATE_OFF, NULL, 0); 146 if (status == B_OK) { 147 //cpu_status state = disable_interrupts(); 148 status = acpi->enter_sleep_state(ACPI_POWER_STATE_OFF); 149 //restore_interrupts(state); 150 } 151 } 152 153 put_module(B_ACPI_MODULE_NAME); 154 return status; 155 } 156 157 158 /*! Disable CPU caches, and invalidate them. */ 159 static void 160 disable_caches() 161 { 162 x86_write_cr0((x86_read_cr0() | CR0_CACHE_DISABLE) 163 & ~CR0_NOT_WRITE_THROUGH); 164 wbinvd(); 165 arch_cpu_global_TLB_invalidate(); 166 } 167 168 169 /*! Invalidate CPU caches, and enable them. */ 170 static void 171 enable_caches() 172 { 173 wbinvd(); 174 arch_cpu_global_TLB_invalidate(); 175 x86_write_cr0(x86_read_cr0() 176 & ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH)); 177 } 178 179 180 static void 181 set_mtrr(void* _parameter, int cpu) 182 { 183 struct set_mtrr_parameter* parameter 184 = (struct set_mtrr_parameter*)_parameter; 185 186 // wait until all CPUs have arrived here 187 smp_cpu_rendezvous(&sCpuRendezvous); 188 189 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 190 // that initiated the call_all_cpus() from doing that again and clearing 191 // sCpuRendezvous2 before the last CPU has actually left the loop in 192 // smp_cpu_rendezvous(); 193 if (cpu == 0) 194 atomic_set((int32*)&sCpuRendezvous3, 0); 195 196 disable_caches(); 197 198 sCpuModule->set_mtrr(parameter->index, parameter->base, parameter->length, 199 parameter->type); 200 201 enable_caches(); 202 203 // wait until all CPUs have arrived here 204 smp_cpu_rendezvous(&sCpuRendezvous2); 205 smp_cpu_rendezvous(&sCpuRendezvous3); 206 } 207 208 209 static void 210 set_mtrrs(void* _parameter, int cpu) 211 { 212 set_mtrrs_parameter* parameter = (set_mtrrs_parameter*)_parameter; 213 214 // wait until all CPUs have arrived here 215 smp_cpu_rendezvous(&sCpuRendezvous); 216 217 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 218 // that initiated the call_all_cpus() from doing that again and clearing 219 // sCpuRendezvous2 before the last CPU has actually left the loop in 220 // smp_cpu_rendezvous(); 221 if (cpu == 0) 222 atomic_set((int32*)&sCpuRendezvous3, 0); 223 224 disable_caches(); 225 226 sCpuModule->set_mtrrs(parameter->defaultType, parameter->infos, 227 parameter->count); 228 229 enable_caches(); 230 231 // wait until all CPUs have arrived here 232 smp_cpu_rendezvous(&sCpuRendezvous2); 233 smp_cpu_rendezvous(&sCpuRendezvous3); 234 } 235 236 237 static void 238 init_mtrrs(void* _unused, int cpu) 239 { 240 // wait until all CPUs have arrived here 241 smp_cpu_rendezvous(&sCpuRendezvous); 242 243 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 244 // that initiated the call_all_cpus() from doing that again and clearing 245 // sCpuRendezvous2 before the last CPU has actually left the loop in 246 // smp_cpu_rendezvous(); 247 if (cpu == 0) 248 atomic_set((int32*)&sCpuRendezvous3, 0); 249 250 disable_caches(); 251 252 sCpuModule->init_mtrrs(); 253 254 enable_caches(); 255 256 // wait until all CPUs have arrived here 257 smp_cpu_rendezvous(&sCpuRendezvous2); 258 smp_cpu_rendezvous(&sCpuRendezvous3); 259 } 260 261 262 uint32 263 x86_count_mtrrs(void) 264 { 265 if (sCpuModule == NULL) 266 return 0; 267 268 return sCpuModule->count_mtrrs(); 269 } 270 271 272 void 273 x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type) 274 { 275 struct set_mtrr_parameter parameter; 276 parameter.index = index; 277 parameter.base = base; 278 parameter.length = length; 279 parameter.type = type; 280 281 sCpuRendezvous = sCpuRendezvous2 = 0; 282 call_all_cpus(&set_mtrr, ¶meter); 283 } 284 285 286 status_t 287 x86_get_mtrr(uint32 index, uint64* _base, uint64* _length, uint8* _type) 288 { 289 // the MTRRs are identical on all CPUs, so it doesn't matter 290 // on which CPU this runs 291 return sCpuModule->get_mtrr(index, _base, _length, _type); 292 } 293 294 295 void 296 x86_set_mtrrs(uint8 defaultType, const x86_mtrr_info* infos, uint32 count) 297 { 298 if (sCpuModule == NULL) 299 return; 300 301 struct set_mtrrs_parameter parameter; 302 parameter.defaultType = defaultType; 303 parameter.infos = infos; 304 parameter.count = count; 305 306 sCpuRendezvous = sCpuRendezvous2 = 0; 307 call_all_cpus(&set_mtrrs, ¶meter); 308 } 309 310 311 void 312 x86_init_fpu(void) 313 { 314 // All x86_64 CPUs support SSE, don't need to bother checking for it. 315 #ifndef __x86_64__ 316 if (!x86_check_feature(IA32_FEATURE_FPU, FEATURE_COMMON)) { 317 // No FPU... time to install one in your 386? 318 dprintf("%s: Warning: CPU has no reported FPU.\n", __func__); 319 gX86SwapFPUFunc = x86_noop_swap; 320 return; 321 } 322 323 if (!x86_check_feature(IA32_FEATURE_SSE, FEATURE_COMMON) 324 || !x86_check_feature(IA32_FEATURE_FXSR, FEATURE_COMMON)) { 325 dprintf("%s: CPU has no SSE... just enabling FPU.\n", __func__); 326 // we don't have proper SSE support, just enable FPU 327 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 328 gX86SwapFPUFunc = x86_fnsave_swap; 329 return; 330 } 331 #endif 332 333 dprintf("%s: CPU has SSE... enabling FXSR and XMM.\n", __func__); 334 #ifndef __x86_64__ 335 // enable OS support for SSE 336 x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION); 337 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 338 339 gX86SwapFPUFunc = x86_fxsave_swap; 340 gHasSSE = true; 341 #endif 342 } 343 344 345 #if DUMP_FEATURE_STRING 346 static void 347 dump_feature_string(int currentCPU, cpu_ent* cpu) 348 { 349 char features[512]; 350 features[0] = 0; 351 352 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FPU) 353 strlcat(features, "fpu ", sizeof(features)); 354 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_VME) 355 strlcat(features, "vme ", sizeof(features)); 356 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DE) 357 strlcat(features, "de ", sizeof(features)); 358 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE) 359 strlcat(features, "pse ", sizeof(features)); 360 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TSC) 361 strlcat(features, "tsc ", sizeof(features)); 362 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MSR) 363 strlcat(features, "msr ", sizeof(features)); 364 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAE) 365 strlcat(features, "pae ", sizeof(features)); 366 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCE) 367 strlcat(features, "mce ", sizeof(features)); 368 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CX8) 369 strlcat(features, "cx8 ", sizeof(features)); 370 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_APIC) 371 strlcat(features, "apic ", sizeof(features)); 372 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SEP) 373 strlcat(features, "sep ", sizeof(features)); 374 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MTRR) 375 strlcat(features, "mtrr ", sizeof(features)); 376 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PGE) 377 strlcat(features, "pge ", sizeof(features)); 378 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCA) 379 strlcat(features, "mca ", sizeof(features)); 380 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CMOV) 381 strlcat(features, "cmov ", sizeof(features)); 382 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAT) 383 strlcat(features, "pat ", sizeof(features)); 384 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE36) 385 strlcat(features, "pse36 ", sizeof(features)); 386 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSN) 387 strlcat(features, "psn ", sizeof(features)); 388 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CLFSH) 389 strlcat(features, "clfsh ", sizeof(features)); 390 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DS) 391 strlcat(features, "ds ", sizeof(features)); 392 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_ACPI) 393 strlcat(features, "acpi ", sizeof(features)); 394 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MMX) 395 strlcat(features, "mmx ", sizeof(features)); 396 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FXSR) 397 strlcat(features, "fxsr ", sizeof(features)); 398 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE) 399 strlcat(features, "sse ", sizeof(features)); 400 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE2) 401 strlcat(features, "sse2 ", sizeof(features)); 402 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SS) 403 strlcat(features, "ss ", sizeof(features)); 404 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_HTT) 405 strlcat(features, "htt ", sizeof(features)); 406 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TM) 407 strlcat(features, "tm ", sizeof(features)); 408 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PBE) 409 strlcat(features, "pbe ", sizeof(features)); 410 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE3) 411 strlcat(features, "sse3 ", sizeof(features)); 412 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCLMULQDQ) 413 strlcat(features, "pclmulqdq ", sizeof(features)); 414 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DTES64) 415 strlcat(features, "dtes64 ", sizeof(features)); 416 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MONITOR) 417 strlcat(features, "monitor ", sizeof(features)); 418 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DSCPL) 419 strlcat(features, "dscpl ", sizeof(features)); 420 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_VMX) 421 strlcat(features, "vmx ", sizeof(features)); 422 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SMX) 423 strlcat(features, "smx ", sizeof(features)); 424 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_EST) 425 strlcat(features, "est ", sizeof(features)); 426 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TM2) 427 strlcat(features, "tm2 ", sizeof(features)); 428 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSSE3) 429 strlcat(features, "ssse3 ", sizeof(features)); 430 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CNXTID) 431 strlcat(features, "cnxtid ", sizeof(features)); 432 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_FMA) 433 strlcat(features, "fma ", sizeof(features)); 434 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CX16) 435 strlcat(features, "cx16 ", sizeof(features)); 436 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XTPR) 437 strlcat(features, "xtpr ", sizeof(features)); 438 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PDCM) 439 strlcat(features, "pdcm ", sizeof(features)); 440 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCID) 441 strlcat(features, "pcid ", sizeof(features)); 442 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DCA) 443 strlcat(features, "dca ", sizeof(features)); 444 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_1) 445 strlcat(features, "sse4_1 ", sizeof(features)); 446 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_2) 447 strlcat(features, "sse4_2 ", sizeof(features)); 448 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_X2APIC) 449 strlcat(features, "x2apic ", sizeof(features)); 450 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MOVBE) 451 strlcat(features, "movbe ", sizeof(features)); 452 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_POPCNT) 453 strlcat(features, "popcnt ", sizeof(features)); 454 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TSCDEADLINE) 455 strlcat(features, "tscdeadline ", sizeof(features)); 456 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AES) 457 strlcat(features, "aes ", sizeof(features)); 458 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XSAVE) 459 strlcat(features, "xsave ", sizeof(features)); 460 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_OSXSAVE) 461 strlcat(features, "osxsave ", sizeof(features)); 462 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AVX) 463 strlcat(features, "avx ", sizeof(features)); 464 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_F16C) 465 strlcat(features, "f16c ", sizeof(features)); 466 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_RDRND) 467 strlcat(features, "rdrnd ", sizeof(features)); 468 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) 469 strlcat(features, "hypervisor ", sizeof(features)); 470 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_SYSCALL) 471 strlcat(features, "syscall ", sizeof(features)); 472 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_NX) 473 strlcat(features, "nx ", sizeof(features)); 474 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_MMXEXT) 475 strlcat(features, "mmxext ", sizeof(features)); 476 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_FFXSR) 477 strlcat(features, "ffxsr ", sizeof(features)); 478 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_PDPE1GB) 479 strlcat(features, "pdpe1gb ", sizeof(features)); 480 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_LONG) 481 strlcat(features, "long ", sizeof(features)); 482 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOWEXT) 483 strlcat(features, "3dnowext ", sizeof(features)); 484 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOW) 485 strlcat(features, "3dnow ", sizeof(features)); 486 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_DTS) 487 strlcat(features, "dts ", sizeof(features)); 488 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ITB) 489 strlcat(features, "itb ", sizeof(features)); 490 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ARAT) 491 strlcat(features, "arat ", sizeof(features)); 492 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PLN) 493 strlcat(features, "pln ", sizeof(features)); 494 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ECMD) 495 strlcat(features, "ecmd ", sizeof(features)); 496 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PTM) 497 strlcat(features, "ptm ", sizeof(features)); 498 if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_APERFMPERF) 499 strlcat(features, "aperfmperf ", sizeof(features)); 500 if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_EPB) 501 strlcat(features, "epb ", sizeof(features)); 502 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_TSC_ADJUST) 503 strlcat(features, "tsc_adjust ", sizeof(features)); 504 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SGX) 505 strlcat(features, "sgx ", sizeof(features)); 506 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_BMI1) 507 strlcat(features, "bmi1 ", sizeof(features)); 508 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_HLE) 509 strlcat(features, "hle ", sizeof(features)); 510 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX2) 511 strlcat(features, "avx2 ", sizeof(features)); 512 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SMEP) 513 strlcat(features, "smep ", sizeof(features)); 514 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_BMI2) 515 strlcat(features, "bmi2 ", sizeof(features)); 516 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_ERMS) 517 strlcat(features, "erms ", sizeof(features)); 518 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_INVPCID) 519 strlcat(features, "invpcid ", sizeof(features)); 520 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_RTM) 521 strlcat(features, "rtm ", sizeof(features)); 522 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_CQM) 523 strlcat(features, "cqm ", sizeof(features)); 524 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_MPX) 525 strlcat(features, "mpx ", sizeof(features)); 526 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_RDT_A) 527 strlcat(features, "rdt_a ", sizeof(features)); 528 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512F) 529 strlcat(features, "avx512f ", sizeof(features)); 530 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512DQ) 531 strlcat(features, "avx512dq ", sizeof(features)); 532 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_RDSEED) 533 strlcat(features, "rdseed ", sizeof(features)); 534 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_ADX) 535 strlcat(features, "adx ", sizeof(features)); 536 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SMAP) 537 strlcat(features, "smap ", sizeof(features)); 538 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512IFMA) 539 strlcat(features, "avx512ifma ", sizeof(features)); 540 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_PCOMMIT) 541 strlcat(features, "pcommit ", sizeof(features)); 542 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_CLFLUSHOPT) 543 strlcat(features, "cflushopt ", sizeof(features)); 544 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_CLWB) 545 strlcat(features, "clwb ", sizeof(features)); 546 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_INTEL_PT) 547 strlcat(features, "intel_pt ", sizeof(features)); 548 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512PF) 549 strlcat(features, "avx512pf ", sizeof(features)); 550 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512ER) 551 strlcat(features, "avx512er ", sizeof(features)); 552 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512CD) 553 strlcat(features, "avx512cd ", sizeof(features)); 554 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SHA_NI) 555 strlcat(features, "sha_ni ", sizeof(features)); 556 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512BW) 557 strlcat(features, "avx512bw ", sizeof(features)); 558 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512VI) 559 strlcat(features, "avx512vi ", sizeof(features)); 560 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_IBRS) 561 strlcat(features, "ibrs ", sizeof(features)); 562 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_STIBP) 563 strlcat(features, "stibp ", sizeof(features)); 564 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_L1D_FLUSH) 565 strlcat(features, "l1d_flush ", sizeof(features)); 566 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_ARCH_CAPABILITIES) 567 strlcat(features, "msr_arch ", sizeof(features)); 568 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_SSBD) 569 strlcat(features, "ssbd ", sizeof(features)); 570 if (cpu->arch.feature[FEATURE_D_1_EAX] & IA32_FEATURE_XSAVEOPT) 571 strlcat(features, "xsaveopt ", sizeof(features)); 572 if (cpu->arch.feature[FEATURE_D_1_EAX] & IA32_FEATURE_XSAVEC) 573 strlcat(features, "xsavec ", sizeof(features)); 574 if (cpu->arch.feature[FEATURE_D_1_EAX] & IA32_FEATURE_XGETBV1) 575 strlcat(features, "xgetbv1 ", sizeof(features)); 576 if (cpu->arch.feature[FEATURE_D_1_EAX] & IA32_FEATURE_XSAVES) 577 strlcat(features, "xsaves ", sizeof(features)); 578 if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_CLZERO) 579 strlcat(features, "clzero ", sizeof(features)); 580 if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_IBPB) 581 strlcat(features, "ibpb ", sizeof(features)); 582 if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_AMD_SSBD) 583 strlcat(features, "amd_ssbd ", sizeof(features)); 584 if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_VIRT_SSBD) 585 strlcat(features, "virt_ssbd ", sizeof(features)); 586 if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_AMD_SSB_NO) 587 strlcat(features, "amd_ssb_no ", sizeof(features)); 588 dprintf("CPU %d: features: %s\n", currentCPU, features); 589 } 590 #endif // DUMP_FEATURE_STRING 591 592 593 static void 594 compute_cpu_hierarchy_masks(int maxLogicalID, int maxCoreID) 595 { 596 ASSERT(maxLogicalID >= maxCoreID); 597 const int kMaxSMTID = maxLogicalID / maxCoreID; 598 599 sHierarchyMask[CPU_TOPOLOGY_SMT] = kMaxSMTID - 1; 600 sHierarchyShift[CPU_TOPOLOGY_SMT] = 0; 601 602 sHierarchyMask[CPU_TOPOLOGY_CORE] = (maxCoreID - 1) * kMaxSMTID; 603 sHierarchyShift[CPU_TOPOLOGY_CORE] 604 = count_set_bits(sHierarchyMask[CPU_TOPOLOGY_SMT]); 605 606 const uint32 kSinglePackageMask = sHierarchyMask[CPU_TOPOLOGY_SMT] 607 | sHierarchyMask[CPU_TOPOLOGY_CORE]; 608 sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~kSinglePackageMask; 609 sHierarchyShift[CPU_TOPOLOGY_PACKAGE] = count_set_bits(kSinglePackageMask); 610 } 611 612 613 static uint32 614 get_cpu_legacy_initial_apic_id(int /* currentCPU */) 615 { 616 cpuid_info cpuid; 617 get_current_cpuid(&cpuid, 1, 0); 618 return cpuid.regs.ebx >> 24; 619 } 620 621 622 static inline status_t 623 detect_amd_cpu_topology(uint32 maxBasicLeaf, uint32 maxExtendedLeaf) 624 { 625 sGetCPUTopologyID = get_cpu_legacy_initial_apic_id; 626 627 cpuid_info cpuid; 628 get_current_cpuid(&cpuid, 1, 0); 629 int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff); 630 631 int maxCoreID = 1; 632 if (maxExtendedLeaf >= 0x80000008) { 633 get_current_cpuid(&cpuid, 0x80000008, 0); 634 maxCoreID = (cpuid.regs.ecx >> 12) & 0xf; 635 if (maxCoreID != 0) 636 maxCoreID = 1 << maxCoreID; 637 else 638 maxCoreID = next_power_of_2((cpuid.regs.edx & 0xf) + 1); 639 } 640 641 if (maxExtendedLeaf >= 0x80000001) { 642 get_current_cpuid(&cpuid, 0x80000001, 0); 643 if (x86_check_feature(IA32_FEATURE_AMD_EXT_CMPLEGACY, 644 FEATURE_EXT_AMD_ECX)) 645 maxCoreID = maxLogicalID; 646 } 647 648 compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID); 649 650 return B_OK; 651 } 652 653 654 static void 655 detect_amd_cache_topology(uint32 maxExtendedLeaf) 656 { 657 if (!x86_check_feature(IA32_FEATURE_AMD_EXT_TOPOLOGY, FEATURE_EXT_AMD_ECX)) 658 return; 659 660 if (maxExtendedLeaf < 0x8000001d) 661 return; 662 663 uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL]; 664 int maxCacheLevel = 0; 665 666 int currentLevel = 0; 667 int cacheType; 668 do { 669 cpuid_info cpuid; 670 get_current_cpuid(&cpuid, 0x8000001d, currentLevel); 671 672 cacheType = cpuid.regs.eax & 0x1f; 673 if (cacheType == 0) 674 break; 675 676 int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; 677 int coresCount = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1); 678 hierarchyLevels[cacheLevel - 1] 679 = coresCount * (sHierarchyMask[CPU_TOPOLOGY_SMT] + 1); 680 maxCacheLevel = std::max(maxCacheLevel, cacheLevel); 681 682 currentLevel++; 683 } while (true); 684 685 for (int i = 0; i < maxCacheLevel; i++) 686 sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); 687 gCPUCacheLevelCount = maxCacheLevel; 688 } 689 690 691 static uint32 692 get_intel_cpu_initial_x2apic_id(int /* currentCPU */) 693 { 694 cpuid_info cpuid; 695 get_current_cpuid(&cpuid, 11, 0); 696 return cpuid.regs.edx; 697 } 698 699 700 static inline status_t 701 detect_intel_cpu_topology_x2apic(uint32 maxBasicLeaf) 702 { 703 if (maxBasicLeaf < 11) 704 return B_UNSUPPORTED; 705 706 uint8 hierarchyLevels[CPU_TOPOLOGY_LEVELS] = { 0 }; 707 708 int currentLevel = 0; 709 int levelType; 710 unsigned int levelsSet = 0; 711 712 do { 713 cpuid_info cpuid; 714 get_current_cpuid(&cpuid, 11, currentLevel); 715 if (currentLevel == 0 && cpuid.regs.ebx == 0) 716 return B_UNSUPPORTED; 717 718 levelType = (cpuid.regs.ecx >> 8) & 0xff; 719 int levelValue = cpuid.regs.eax & 0x1f; 720 721 switch (levelType) { 722 case 1: // SMT 723 hierarchyLevels[CPU_TOPOLOGY_SMT] = levelValue; 724 levelsSet |= 1; 725 break; 726 case 2: // core 727 hierarchyLevels[CPU_TOPOLOGY_CORE] = levelValue; 728 levelsSet |= 2; 729 break; 730 } 731 732 currentLevel++; 733 } while (levelType != 0 && levelsSet != 3); 734 735 sGetCPUTopologyID = get_intel_cpu_initial_x2apic_id; 736 737 for (int i = 1; i < CPU_TOPOLOGY_LEVELS; i++) { 738 if ((levelsSet & (1u << i)) != 0) 739 continue; 740 hierarchyLevels[i] = hierarchyLevels[i - 1]; 741 } 742 743 for (int i = 0; i < CPU_TOPOLOGY_LEVELS; i++) { 744 uint32 mask = ~uint32(0); 745 if (i < CPU_TOPOLOGY_LEVELS - 1) 746 mask = (1u << hierarchyLevels[i]) - 1; 747 if (i > 0) 748 mask &= ~sHierarchyMask[i - 1]; 749 sHierarchyMask[i] = mask; 750 sHierarchyShift[i] = i > 0 ? hierarchyLevels[i - 1] : 0; 751 } 752 753 return B_OK; 754 } 755 756 757 static inline status_t 758 detect_intel_cpu_topology_legacy(uint32 maxBasicLeaf) 759 { 760 sGetCPUTopologyID = get_cpu_legacy_initial_apic_id; 761 762 cpuid_info cpuid; 763 764 get_current_cpuid(&cpuid, 1, 0); 765 int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff); 766 767 int maxCoreID = 1; 768 if (maxBasicLeaf >= 4) { 769 get_current_cpuid(&cpuid, 4, 0); 770 maxCoreID = next_power_of_2((cpuid.regs.eax >> 26) + 1); 771 } 772 773 compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID); 774 775 return B_OK; 776 } 777 778 779 static void 780 detect_intel_cache_topology(uint32 maxBasicLeaf) 781 { 782 if (maxBasicLeaf < 4) 783 return; 784 785 uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL]; 786 int maxCacheLevel = 0; 787 788 int currentLevel = 0; 789 int cacheType; 790 do { 791 cpuid_info cpuid; 792 get_current_cpuid(&cpuid, 4, currentLevel); 793 794 cacheType = cpuid.regs.eax & 0x1f; 795 if (cacheType == 0) 796 break; 797 798 int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; 799 hierarchyLevels[cacheLevel - 1] 800 = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1); 801 maxCacheLevel = std::max(maxCacheLevel, cacheLevel); 802 803 currentLevel++; 804 } while (true); 805 806 for (int i = 0; i < maxCacheLevel; i++) 807 sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); 808 809 gCPUCacheLevelCount = maxCacheLevel; 810 } 811 812 813 static uint32 814 get_simple_cpu_topology_id(int currentCPU) 815 { 816 return currentCPU; 817 } 818 819 820 static inline int 821 get_topology_level_id(uint32 id, cpu_topology_level level) 822 { 823 ASSERT(level < CPU_TOPOLOGY_LEVELS); 824 return (id & sHierarchyMask[level]) >> sHierarchyShift[level]; 825 } 826 827 828 static void 829 detect_cpu_topology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, 830 uint32 maxExtendedLeaf) 831 { 832 if (currentCPU == 0) { 833 memset(sCacheSharingMask, 0xff, sizeof(sCacheSharingMask)); 834 835 status_t result = B_UNSUPPORTED; 836 if (x86_check_feature(IA32_FEATURE_HTT, FEATURE_COMMON)) { 837 if (cpu->arch.vendor == VENDOR_AMD) { 838 result = detect_amd_cpu_topology(maxBasicLeaf, maxExtendedLeaf); 839 840 if (result == B_OK) 841 detect_amd_cache_topology(maxExtendedLeaf); 842 } 843 844 if (cpu->arch.vendor == VENDOR_INTEL) { 845 result = detect_intel_cpu_topology_x2apic(maxBasicLeaf); 846 if (result != B_OK) 847 result = detect_intel_cpu_topology_legacy(maxBasicLeaf); 848 849 if (result == B_OK) 850 detect_intel_cache_topology(maxBasicLeaf); 851 } 852 } 853 854 if (result != B_OK) { 855 dprintf("No CPU topology information available.\n"); 856 857 sGetCPUTopologyID = get_simple_cpu_topology_id; 858 859 sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~uint32(0); 860 } 861 } 862 863 ASSERT(sGetCPUTopologyID != NULL); 864 int topologyID = sGetCPUTopologyID(currentCPU); 865 cpu->topology_id[CPU_TOPOLOGY_SMT] 866 = get_topology_level_id(topologyID, CPU_TOPOLOGY_SMT); 867 cpu->topology_id[CPU_TOPOLOGY_CORE] 868 = get_topology_level_id(topologyID, CPU_TOPOLOGY_CORE); 869 cpu->topology_id[CPU_TOPOLOGY_PACKAGE] 870 = get_topology_level_id(topologyID, CPU_TOPOLOGY_PACKAGE); 871 872 unsigned int i; 873 for (i = 0; i < gCPUCacheLevelCount; i++) 874 cpu->cache_id[i] = topologyID & sCacheSharingMask[i]; 875 for (; i < CPU_MAX_CACHE_LEVEL; i++) 876 cpu->cache_id[i] = -1; 877 878 #if DUMP_CPU_TOPOLOGY 879 dprintf("CPU %d: apic id %d, package %d, core %d, smt %d\n", currentCPU, 880 topologyID, cpu->topology_id[CPU_TOPOLOGY_PACKAGE], 881 cpu->topology_id[CPU_TOPOLOGY_CORE], 882 cpu->topology_id[CPU_TOPOLOGY_SMT]); 883 884 if (gCPUCacheLevelCount > 0) { 885 char cacheLevels[256]; 886 unsigned int offset = 0; 887 for (i = 0; i < gCPUCacheLevelCount; i++) { 888 offset += snprintf(cacheLevels + offset, 889 sizeof(cacheLevels) - offset, 890 " L%d id %d%s", i + 1, cpu->cache_id[i], 891 i < gCPUCacheLevelCount - 1 ? "," : ""); 892 893 if (offset >= sizeof(cacheLevels)) 894 break; 895 } 896 897 dprintf("CPU %d: cache sharing:%s\n", currentCPU, cacheLevels); 898 } 899 #endif 900 } 901 902 903 static void 904 detect_intel_patch_level(cpu_ent* cpu) 905 { 906 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) { 907 cpu->arch.patch_level = 0; 908 return; 909 } 910 911 x86_write_msr(IA32_MSR_UCODE_REV, 0); 912 cpuid_info cpuid; 913 get_current_cpuid(&cpuid, 1, 0); 914 915 uint64 value = x86_read_msr(IA32_MSR_UCODE_REV); 916 cpu->arch.patch_level = value >> 32; 917 } 918 919 920 static void 921 detect_amd_patch_level(cpu_ent* cpu) 922 { 923 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) { 924 cpu->arch.patch_level = 0; 925 return; 926 } 927 928 uint64 value = x86_read_msr(IA32_MSR_UCODE_REV); 929 cpu->arch.patch_level = value >> 32; 930 } 931 932 933 static struct intel_microcode_header* 934 find_microcode_intel(addr_t data, size_t size, uint32 patchLevel) 935 { 936 // 9.11.3 Processor Identification 937 cpuid_info cpuid; 938 get_current_cpuid(&cpuid, 1, 0); 939 uint32 signature = cpuid.regs.eax; 940 // 9.11.4 Platform Identification 941 uint64 platformBits = (x86_read_msr(IA32_MSR_PLATFORM_ID) >> 50) & 0x7; 942 uint64 mask = 1 << platformBits; 943 944 while (size > 0) { 945 if (size < sizeof(struct intel_microcode_header)) { 946 dprintf("find_microcode_intel update is too small for header\n"); 947 break; 948 } 949 struct intel_microcode_header* header = 950 (struct intel_microcode_header*)data; 951 952 uint32 totalSize = header->total_size; 953 uint32 dataSize = header->data_size; 954 if (dataSize == 0) { 955 dataSize = 2000; 956 totalSize = sizeof(struct intel_microcode_header) 957 + dataSize; 958 } 959 if (totalSize > size) { 960 dprintf("find_microcode_intel update is too small for data\n"); 961 break; 962 } 963 964 uint32* dwords = (uint32*)data; 965 // prepare the next update 966 size -= totalSize; 967 data += totalSize; 968 969 if (header->loader_revision != 1) { 970 dprintf("find_microcode_intel incorrect loader version\n"); 971 continue; 972 } 973 // 9.11.6 The microcode update data requires a 16-byte boundary 974 // alignment. 975 if (((addr_t)header % 16) != 0) { 976 dprintf("find_microcode_intel incorrect alignment\n"); 977 continue; 978 } 979 uint32 sum = 0; 980 for (uint32 i = 0; i < totalSize / 4; i++) { 981 sum += dwords[i]; 982 } 983 if (sum != 0) { 984 dprintf("find_microcode_intel incorrect checksum\n"); 985 continue; 986 } 987 if (patchLevel > header->update_revision) { 988 dprintf("find_microcode_intel update_revision is lower\n"); 989 continue; 990 } 991 if (signature == header->processor_signature 992 && (mask & header->processor_flags) != 0) { 993 return header; 994 } 995 if (totalSize <= (sizeof(struct intel_microcode_header) + dataSize 996 + sizeof(struct intel_microcode_extended_signature_header))) { 997 continue; 998 } 999 struct intel_microcode_extended_signature_header* extSigHeader = 1000 (struct intel_microcode_extended_signature_header*)((addr_t)header 1001 + sizeof(struct intel_microcode_header) + dataSize); 1002 struct intel_microcode_extended_signature* extended_signature = 1003 (struct intel_microcode_extended_signature*)((addr_t)extSigHeader 1004 + sizeof(struct intel_microcode_extended_signature_header)); 1005 for (uint32 i = 0; i < extSigHeader->extended_signature_count; i++) { 1006 if (signature == extended_signature[i].processor_signature 1007 && (mask & extended_signature[i].processor_flags) != 0) 1008 return header; 1009 } 1010 } 1011 return NULL; 1012 } 1013 1014 1015 static void 1016 load_microcode_intel(int currentCPU, cpu_ent* cpu) 1017 { 1018 // serialize for HT cores 1019 if (currentCPU != 0) 1020 acquire_spinlock(&sUcodeUpdateLock); 1021 detect_intel_patch_level(cpu); 1022 uint32 revision = cpu->arch.patch_level; 1023 struct intel_microcode_header* update = sLoadedUcodeUpdate; 1024 if (update == NULL) { 1025 update = find_microcode_intel((addr_t)sUcodeData, sUcodeDataSize, 1026 revision); 1027 } 1028 if (update != NULL) { 1029 addr_t data = (addr_t)update + sizeof(struct intel_microcode_header); 1030 wbinvd(); 1031 x86_write_msr(IA32_MSR_UCODE_WRITE, data); 1032 detect_intel_patch_level(cpu); 1033 if (revision == cpu->arch.patch_level) { 1034 dprintf("CPU %d: update failed\n", currentCPU); 1035 } else { 1036 if (sLoadedUcodeUpdate == NULL) 1037 sLoadedUcodeUpdate = update; 1038 dprintf("CPU %d: updated from revision %" B_PRIu32 " to %" B_PRIu32 1039 "\n", currentCPU, revision, cpu->arch.patch_level); 1040 } 1041 } else { 1042 dprintf("CPU %d: no update found\n", currentCPU); 1043 } 1044 if (currentCPU != 0) 1045 release_spinlock(&sUcodeUpdateLock); 1046 } 1047 1048 1049 static void 1050 load_microcode_amd(int currentCPU, cpu_ent* cpu) 1051 { 1052 dprintf("CPU %d: no update found\n", currentCPU); 1053 } 1054 1055 1056 static void 1057 load_microcode(int currentCPU) 1058 { 1059 if (sUcodeData == NULL) 1060 return; 1061 cpu_ent* cpu = get_cpu_struct(); 1062 if ((cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) != 0) 1063 return; 1064 if (cpu->arch.vendor == VENDOR_INTEL) 1065 load_microcode_intel(currentCPU, cpu); 1066 else if (cpu->arch.vendor == VENDOR_AMD) 1067 load_microcode_amd(currentCPU, cpu); 1068 } 1069 1070 1071 static void 1072 detect_cpu(int currentCPU) 1073 { 1074 cpu_ent* cpu = get_cpu_struct(); 1075 char vendorString[17]; 1076 cpuid_info cpuid; 1077 1078 // clear out the cpu info data 1079 cpu->arch.vendor = VENDOR_UNKNOWN; 1080 cpu->arch.vendor_name = "UNKNOWN VENDOR"; 1081 cpu->arch.feature[FEATURE_COMMON] = 0; 1082 cpu->arch.feature[FEATURE_EXT] = 0; 1083 cpu->arch.feature[FEATURE_EXT_AMD] = 0; 1084 cpu->arch.feature[FEATURE_7_EBX] = 0; 1085 cpu->arch.feature[FEATURE_7_ECX] = 0; 1086 cpu->arch.feature[FEATURE_7_EDX] = 0; 1087 cpu->arch.feature[FEATURE_D_1_EAX] = 0; 1088 cpu->arch.model_name[0] = 0; 1089 1090 // print some fun data 1091 get_current_cpuid(&cpuid, 0, 0); 1092 uint32 maxBasicLeaf = cpuid.eax_0.max_eax; 1093 1094 // build the vendor string 1095 memset(vendorString, 0, sizeof(vendorString)); 1096 memcpy(vendorString, cpuid.eax_0.vendor_id, sizeof(cpuid.eax_0.vendor_id)); 1097 1098 // get the family, model, stepping 1099 get_current_cpuid(&cpuid, 1, 0); 1100 cpu->arch.type = cpuid.eax_1.type; 1101 cpu->arch.family = cpuid.eax_1.family; 1102 cpu->arch.extended_family = cpuid.eax_1.extended_family; 1103 cpu->arch.model = cpuid.eax_1.model; 1104 cpu->arch.extended_model = cpuid.eax_1.extended_model; 1105 cpu->arch.stepping = cpuid.eax_1.stepping; 1106 dprintf("CPU %d: type %d family %d extended_family %d model %d " 1107 "extended_model %d stepping %d, string '%s'\n", 1108 currentCPU, cpu->arch.type, cpu->arch.family, 1109 cpu->arch.extended_family, cpu->arch.model, 1110 cpu->arch.extended_model, cpu->arch.stepping, vendorString); 1111 1112 // figure out what vendor we have here 1113 1114 for (int32 i = 0; i < VENDOR_NUM; i++) { 1115 if (vendor_info[i].ident_string[0] 1116 && !strcmp(vendorString, vendor_info[i].ident_string[0])) { 1117 cpu->arch.vendor = (x86_vendors)i; 1118 cpu->arch.vendor_name = vendor_info[i].vendor; 1119 break; 1120 } 1121 if (vendor_info[i].ident_string[1] 1122 && !strcmp(vendorString, vendor_info[i].ident_string[1])) { 1123 cpu->arch.vendor = (x86_vendors)i; 1124 cpu->arch.vendor_name = vendor_info[i].vendor; 1125 break; 1126 } 1127 } 1128 1129 // see if we can get the model name 1130 get_current_cpuid(&cpuid, 0x80000000, 0); 1131 uint32 maxExtendedLeaf = cpuid.eax_0.max_eax; 1132 if (maxExtendedLeaf >= 0x80000004) { 1133 // build the model string (need to swap ecx/edx data before copying) 1134 unsigned int temp; 1135 memset(cpu->arch.model_name, 0, sizeof(cpu->arch.model_name)); 1136 1137 get_current_cpuid(&cpuid, 0x80000002, 0); 1138 temp = cpuid.regs.edx; 1139 cpuid.regs.edx = cpuid.regs.ecx; 1140 cpuid.regs.ecx = temp; 1141 memcpy(cpu->arch.model_name, cpuid.as_chars, sizeof(cpuid.as_chars)); 1142 1143 get_current_cpuid(&cpuid, 0x80000003, 0); 1144 temp = cpuid.regs.edx; 1145 cpuid.regs.edx = cpuid.regs.ecx; 1146 cpuid.regs.ecx = temp; 1147 memcpy(cpu->arch.model_name + 16, cpuid.as_chars, 1148 sizeof(cpuid.as_chars)); 1149 1150 get_current_cpuid(&cpuid, 0x80000004, 0); 1151 temp = cpuid.regs.edx; 1152 cpuid.regs.edx = cpuid.regs.ecx; 1153 cpuid.regs.ecx = temp; 1154 memcpy(cpu->arch.model_name + 32, cpuid.as_chars, 1155 sizeof(cpuid.as_chars)); 1156 1157 // some cpus return a right-justified string 1158 int32 i = 0; 1159 while (cpu->arch.model_name[i] == ' ') 1160 i++; 1161 if (i > 0) { 1162 memmove(cpu->arch.model_name, &cpu->arch.model_name[i], 1163 strlen(&cpu->arch.model_name[i]) + 1); 1164 } 1165 1166 dprintf("CPU %d: vendor '%s' model name '%s'\n", 1167 currentCPU, cpu->arch.vendor_name, cpu->arch.model_name); 1168 } else { 1169 strlcpy(cpu->arch.model_name, "unknown", sizeof(cpu->arch.model_name)); 1170 } 1171 1172 // load feature bits 1173 get_current_cpuid(&cpuid, 1, 0); 1174 cpu->arch.feature[FEATURE_COMMON] = cpuid.eax_1.features; // edx 1175 cpu->arch.feature[FEATURE_EXT] = cpuid.eax_1.extended_features; // ecx 1176 1177 if (maxExtendedLeaf >= 0x80000001) { 1178 get_current_cpuid(&cpuid, 0x80000001, 0); 1179 if (cpu->arch.vendor == VENDOR_AMD) 1180 cpu->arch.feature[FEATURE_EXT_AMD_ECX] = cpuid.regs.ecx; // ecx 1181 cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx 1182 if (cpu->arch.vendor != VENDOR_AMD) 1183 cpu->arch.feature[FEATURE_EXT_AMD] &= IA32_FEATURES_INTEL_EXT; 1184 } 1185 1186 if (maxBasicLeaf >= 5) { 1187 get_current_cpuid(&cpuid, 5, 0); 1188 cpu->arch.feature[FEATURE_5_ECX] = cpuid.regs.ecx; 1189 } 1190 1191 if (maxBasicLeaf >= 6) { 1192 get_current_cpuid(&cpuid, 6, 0); 1193 cpu->arch.feature[FEATURE_6_EAX] = cpuid.regs.eax; 1194 cpu->arch.feature[FEATURE_6_ECX] = cpuid.regs.ecx; 1195 } 1196 1197 if (maxBasicLeaf >= 7) { 1198 get_current_cpuid(&cpuid, 7, 0); 1199 cpu->arch.feature[FEATURE_7_EBX] = cpuid.regs.ebx; 1200 cpu->arch.feature[FEATURE_7_ECX] = cpuid.regs.ecx; 1201 cpu->arch.feature[FEATURE_7_EDX] = cpuid.regs.edx; 1202 } 1203 1204 if (maxBasicLeaf >= 0xd) { 1205 get_current_cpuid(&cpuid, 0xd, 1); 1206 cpu->arch.feature[FEATURE_D_1_EAX] = cpuid.regs.eax; 1207 } 1208 1209 if (maxExtendedLeaf >= 0x80000007) { 1210 get_current_cpuid(&cpuid, 0x80000007, 0); 1211 cpu->arch.feature[FEATURE_EXT_7_EDX] = cpuid.regs.edx; 1212 } 1213 1214 if (maxExtendedLeaf >= 0x80000008) { 1215 get_current_cpuid(&cpuid, 0x80000008, 0); 1216 cpu->arch.feature[FEATURE_EXT_8_EBX] = cpuid.regs.ebx; 1217 } 1218 1219 detect_cpu_topology(currentCPU, cpu, maxBasicLeaf, maxExtendedLeaf); 1220 1221 if (cpu->arch.vendor == VENDOR_INTEL) 1222 detect_intel_patch_level(cpu); 1223 else if (cpu->arch.vendor == VENDOR_AMD) 1224 detect_amd_patch_level(cpu); 1225 1226 #if DUMP_FEATURE_STRING 1227 dump_feature_string(currentCPU, cpu); 1228 #endif 1229 #if DUMP_CPU_PATCHLEVEL 1230 dprintf("CPU %d: patch_level %" B_PRIu32 "\n", currentCPU, 1231 cpu->arch.patch_level); 1232 #endif 1233 } 1234 1235 1236 bool 1237 x86_check_feature(uint32 feature, enum x86_feature_type type) 1238 { 1239 cpu_ent* cpu = get_cpu_struct(); 1240 1241 #if 0 1242 int i; 1243 dprintf("x86_check_feature: feature 0x%x, type %d\n", feature, type); 1244 for (i = 0; i < FEATURE_NUM; i++) { 1245 dprintf("features %d: 0x%x\n", i, cpu->arch.feature[i]); 1246 } 1247 #endif 1248 1249 return (cpu->arch.feature[type] & feature) != 0; 1250 } 1251 1252 1253 void* 1254 x86_get_double_fault_stack(int32 cpu, size_t* _size) 1255 { 1256 *_size = kDoubleFaultStackSize; 1257 return sDoubleFaultStacks + kDoubleFaultStackSize * cpu; 1258 } 1259 1260 1261 /*! Returns the index of the current CPU. Can only be called from the double 1262 fault handler. 1263 */ 1264 int32 1265 x86_double_fault_get_cpu(void) 1266 { 1267 addr_t stack = x86_get_stack_frame(); 1268 return (stack - (addr_t)sDoubleFaultStacks) / kDoubleFaultStackSize; 1269 } 1270 1271 1272 // #pragma mark - 1273 1274 1275 status_t 1276 arch_cpu_preboot_init_percpu(kernel_args* args, int cpu) 1277 { 1278 // On SMP system we want to synchronize the CPUs' TSCs, so system_time() 1279 // will return consistent values. 1280 if (smp_get_num_cpus() > 1) { 1281 // let the first CPU prepare the rendezvous point 1282 if (cpu == 0) 1283 sTSCSyncRendezvous = smp_get_num_cpus() - 1; 1284 1285 // One CPU after the other will drop out of this loop and be caught by 1286 // the loop below, until the last CPU (0) gets there. Save for +/- a few 1287 // cycles the CPUs should pass the second loop at the same time. 1288 while (sTSCSyncRendezvous != cpu) { 1289 } 1290 1291 sTSCSyncRendezvous = cpu - 1; 1292 1293 while (sTSCSyncRendezvous != -1) { 1294 } 1295 1296 // reset TSC to 0 1297 x86_write_msr(IA32_MSR_TSC, 0); 1298 } 1299 1300 x86_descriptors_preboot_init_percpu(args, cpu); 1301 1302 return B_OK; 1303 } 1304 1305 1306 static void 1307 halt_idle(void) 1308 { 1309 asm("hlt"); 1310 } 1311 1312 1313 static void 1314 amdc1e_noarat_idle(void) 1315 { 1316 uint64 msr = x86_read_msr(K8_MSR_IPM); 1317 if (msr & K8_CMPHALT) 1318 x86_write_msr(K8_MSR_IPM, msr & ~K8_CMPHALT); 1319 halt_idle(); 1320 } 1321 1322 1323 static bool 1324 detect_amdc1e_noarat() 1325 { 1326 cpu_ent* cpu = get_cpu_struct(); 1327 1328 if (cpu->arch.vendor != VENDOR_AMD) 1329 return false; 1330 1331 // Family 0x12 and higher processors support ARAT 1332 // Family lower than 0xf processors doesn't support C1E 1333 // Family 0xf with model <= 0x40 procssors doesn't support C1E 1334 uint32 family = cpu->arch.family + cpu->arch.extended_family; 1335 uint32 model = (cpu->arch.extended_model << 4) | cpu->arch.model; 1336 return (family < 0x12 && family > 0xf) || (family == 0xf && model > 0x40); 1337 } 1338 1339 1340 status_t 1341 arch_cpu_init_percpu(kernel_args* args, int cpu) 1342 { 1343 load_microcode(cpu); 1344 detect_cpu(cpu); 1345 1346 if (!gCpuIdleFunc) { 1347 if (detect_amdc1e_noarat()) 1348 gCpuIdleFunc = amdc1e_noarat_idle; 1349 else 1350 gCpuIdleFunc = halt_idle; 1351 } 1352 1353 #ifdef __x86_64__ 1354 // if RDTSCP is available write cpu number in TSC_AUX 1355 if (x86_check_feature(IA32_FEATURE_AMD_EXT_RDTSCP, FEATURE_EXT_AMD)) 1356 x86_write_msr(IA32_MSR_TSC_AUX, cpu); 1357 #endif 1358 1359 return __x86_patch_errata_percpu(cpu); 1360 } 1361 1362 1363 status_t 1364 arch_cpu_init(kernel_args* args) 1365 { 1366 if (args->ucode_data != NULL 1367 && args->ucode_data_size > 0) { 1368 sUcodeData = args->ucode_data; 1369 sUcodeDataSize = args->ucode_data_size; 1370 } else { 1371 dprintf("CPU: no microcode provided\n"); 1372 } 1373 1374 // init the TSC -> system_time() conversion factors 1375 1376 uint32 conversionFactor = args->arch_args.system_time_cv_factor; 1377 uint64 conversionFactorNsecs = (uint64)conversionFactor * 1000; 1378 1379 #ifdef __x86_64__ 1380 // The x86_64 system_time() implementation uses 64-bit multiplication and 1381 // therefore shifting is not necessary for low frequencies (it's also not 1382 // too likely that there'll be any x86_64 CPUs clocked under 1GHz). 1383 __x86_setup_system_time((uint64)conversionFactor << 32, 1384 conversionFactorNsecs); 1385 #else 1386 if (conversionFactorNsecs >> 32 != 0) { 1387 // the TSC frequency is < 1 GHz, which forces us to shift the factor 1388 __x86_setup_system_time(conversionFactor, conversionFactorNsecs >> 16, 1389 true); 1390 } else { 1391 // the TSC frequency is >= 1 GHz 1392 __x86_setup_system_time(conversionFactor, conversionFactorNsecs, false); 1393 } 1394 #endif 1395 1396 // Initialize descriptor tables. 1397 x86_descriptors_init(args); 1398 1399 return B_OK; 1400 } 1401 1402 1403 #ifdef __x86_64__ 1404 static void 1405 enable_smap(void* dummy, int cpu) 1406 { 1407 x86_write_cr4(x86_read_cr4() | IA32_CR4_SMAP); 1408 } 1409 1410 1411 static void 1412 enable_smep(void* dummy, int cpu) 1413 { 1414 x86_write_cr4(x86_read_cr4() | IA32_CR4_SMEP); 1415 } 1416 1417 1418 static void 1419 enable_osxsave(void* dummy, int cpu) 1420 { 1421 x86_write_cr4(x86_read_cr4() | IA32_CR4_OSXSAVE); 1422 } 1423 1424 1425 static void 1426 enable_xsavemask(void* dummy, int cpu) 1427 { 1428 xsetbv(0, gXsaveMask); 1429 } 1430 #endif 1431 1432 1433 status_t 1434 arch_cpu_init_post_vm(kernel_args* args) 1435 { 1436 uint32 i; 1437 1438 // allocate an area for the double fault stacks 1439 virtual_address_restrictions virtualRestrictions = {}; 1440 virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS; 1441 physical_address_restrictions physicalRestrictions = {}; 1442 create_area_etc(B_SYSTEM_TEAM, "double fault stacks", 1443 kDoubleFaultStackSize * smp_get_num_cpus(), B_FULL_LOCK, 1444 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0, 1445 &virtualRestrictions, &physicalRestrictions, 1446 (void**)&sDoubleFaultStacks); 1447 1448 X86PagingStructures* kernelPagingStructures 1449 = static_cast<X86VMTranslationMap*>( 1450 VMAddressSpace::Kernel()->TranslationMap())->PagingStructures(); 1451 1452 // Set active translation map on each CPU. 1453 for (i = 0; i < args->num_cpus; i++) { 1454 gCPU[i].arch.active_paging_structures = kernelPagingStructures; 1455 kernelPagingStructures->AddReference(); 1456 } 1457 1458 if (!apic_available()) 1459 x86_init_fpu(); 1460 // else fpu gets set up in smp code 1461 1462 #ifdef __x86_64__ 1463 // if available enable SMEP (Supervisor Memory Execution Protection) 1464 if (x86_check_feature(IA32_FEATURE_SMEP, FEATURE_7_EBX)) { 1465 if (!get_safemode_boolean(B_SAFEMODE_DISABLE_SMEP_SMAP, false)) { 1466 dprintf("enable SMEP\n"); 1467 call_all_cpus_sync(&enable_smep, NULL); 1468 } else 1469 dprintf("SMEP disabled per safemode setting\n"); 1470 } 1471 1472 // if available enable SMAP (Supervisor Memory Access Protection) 1473 if (x86_check_feature(IA32_FEATURE_SMAP, FEATURE_7_EBX)) { 1474 if (!get_safemode_boolean(B_SAFEMODE_DISABLE_SMEP_SMAP, false)) { 1475 dprintf("enable SMAP\n"); 1476 call_all_cpus_sync(&enable_smap, NULL); 1477 1478 arch_altcodepatch_replace(ALTCODEPATCH_TAG_STAC, &_stac, 3); 1479 arch_altcodepatch_replace(ALTCODEPATCH_TAG_CLAC, &_clac, 3); 1480 } else 1481 dprintf("SMAP disabled per safemode setting\n"); 1482 } 1483 1484 // if available enable XSAVE (XSAVE and extended states) 1485 gHasXsave = x86_check_feature(IA32_FEATURE_EXT_XSAVE, FEATURE_EXT); 1486 if (gHasXsave) { 1487 gHasXsavec = x86_check_feature(IA32_FEATURE_XSAVEC, 1488 FEATURE_D_1_EAX); 1489 1490 call_all_cpus_sync(&enable_osxsave, NULL); 1491 gXsaveMask = IA32_XCR0_X87 | IA32_XCR0_SSE; 1492 cpuid_info cpuid; 1493 get_current_cpuid(&cpuid, 0xd, 0); 1494 gXsaveMask |= (cpuid.regs.eax & IA32_XCR0_AVX); 1495 call_all_cpus_sync(&enable_xsavemask, NULL); 1496 get_current_cpuid(&cpuid, 0xd, 0); 1497 gFPUSaveLength = cpuid.regs.ebx; 1498 1499 arch_altcodepatch_replace(ALTCODEPATCH_TAG_XSAVE, 1500 gHasXsavec ? &_xsavec : &_xsave, 4); 1501 arch_altcodepatch_replace(ALTCODEPATCH_TAG_XRSTOR, 1502 &_xrstor, 4); 1503 1504 dprintf("enable %s 0x%" B_PRIx64 " %" B_PRId64 "\n", 1505 gHasXsavec ? "XSAVEC" : "XSAVE", gXsaveMask, gFPUSaveLength); 1506 } 1507 1508 #endif 1509 1510 return B_OK; 1511 } 1512 1513 1514 status_t 1515 arch_cpu_init_post_modules(kernel_args* args) 1516 { 1517 // initialize CPU module 1518 1519 void* cookie = open_module_list("cpu"); 1520 1521 while (true) { 1522 char name[B_FILE_NAME_LENGTH]; 1523 size_t nameLength = sizeof(name); 1524 1525 if (read_next_module_name(cookie, name, &nameLength) != B_OK 1526 || get_module(name, (module_info**)&sCpuModule) == B_OK) 1527 break; 1528 } 1529 1530 close_module_list(cookie); 1531 1532 // initialize MTRRs if available 1533 if (x86_count_mtrrs() > 0) { 1534 sCpuRendezvous = sCpuRendezvous2 = 0; 1535 call_all_cpus(&init_mtrrs, NULL); 1536 } 1537 1538 size_t threadExitLen = (addr_t)x86_end_userspace_thread_exit 1539 - (addr_t)x86_userspace_thread_exit; 1540 addr_t threadExitPosition = fill_commpage_entry( 1541 COMMPAGE_ENTRY_X86_THREAD_EXIT, (const void*)x86_userspace_thread_exit, 1542 threadExitLen); 1543 1544 // add the functions to the commpage image 1545 image_id image = get_commpage_image(); 1546 1547 elf_add_memory_image_symbol(image, "commpage_thread_exit", 1548 threadExitPosition, threadExitLen, B_SYMBOL_TYPE_TEXT); 1549 1550 return B_OK; 1551 } 1552 1553 1554 void 1555 arch_cpu_user_TLB_invalidate(void) 1556 { 1557 x86_write_cr3(x86_read_cr3()); 1558 } 1559 1560 1561 void 1562 arch_cpu_global_TLB_invalidate(void) 1563 { 1564 uint32 flags = x86_read_cr4(); 1565 1566 if (flags & IA32_CR4_GLOBAL_PAGES) { 1567 // disable and reenable the global pages to flush all TLBs regardless 1568 // of the global page bit 1569 x86_write_cr4(flags & ~IA32_CR4_GLOBAL_PAGES); 1570 x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES); 1571 } else { 1572 cpu_status state = disable_interrupts(); 1573 arch_cpu_user_TLB_invalidate(); 1574 restore_interrupts(state); 1575 } 1576 } 1577 1578 1579 void 1580 arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) 1581 { 1582 int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE; 1583 while (num_pages-- >= 0) { 1584 invalidate_TLB(start); 1585 start += B_PAGE_SIZE; 1586 } 1587 } 1588 1589 1590 void 1591 arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) 1592 { 1593 int i; 1594 for (i = 0; i < num_pages; i++) { 1595 invalidate_TLB(pages[i]); 1596 } 1597 } 1598 1599 1600 status_t 1601 arch_cpu_shutdown(bool rebootSystem) 1602 { 1603 if (acpi_shutdown(rebootSystem) == B_OK) 1604 return B_OK; 1605 1606 if (!rebootSystem) { 1607 #ifndef __x86_64__ 1608 return apm_shutdown(); 1609 #else 1610 return B_NOT_SUPPORTED; 1611 #endif 1612 } 1613 1614 cpu_status state = disable_interrupts(); 1615 1616 // try to reset the system using the keyboard controller 1617 out8(0xfe, 0x64); 1618 1619 // Give some time to the controller to do its job (0.5s) 1620 snooze(500000); 1621 1622 // if that didn't help, try it this way 1623 x86_reboot(); 1624 1625 restore_interrupts(state); 1626 return B_ERROR; 1627 } 1628 1629 1630 void 1631 arch_cpu_sync_icache(void* address, size_t length) 1632 { 1633 // instruction cache is always consistent on x86 1634 } 1635 1636