1 /* 2 * Copyright 2018, Jérôme Duval, jerome.duval@gmail.com. 3 * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. 4 * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. 5 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. 6 * Distributed under the terms of the MIT License. 7 * 8 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 9 * Distributed under the terms of the NewOS License. 10 */ 11 12 13 #include <cpu.h> 14 15 #include <string.h> 16 #include <stdlib.h> 17 #include <stdio.h> 18 19 #include <algorithm> 20 21 #include <ACPI.h> 22 23 #include <boot_device.h> 24 #include <commpage.h> 25 #include <debug.h> 26 #include <elf.h> 27 #include <safemode.h> 28 #include <smp.h> 29 #include <util/BitUtils.h> 30 #include <vm/vm.h> 31 #include <vm/vm_types.h> 32 #include <vm/VMAddressSpace.h> 33 34 #include <arch_system_info.h> 35 #include <arch/x86/apic.h> 36 #include <boot/kernel_args.h> 37 38 #include "paging/X86PagingStructures.h" 39 #include "paging/X86VMTranslationMap.h" 40 41 42 #define DUMP_FEATURE_STRING 1 43 #define DUMP_CPU_TOPOLOGY 1 44 #define DUMP_CPU_PATCHLEVEL 1 45 46 47 /* cpu vendor info */ 48 struct cpu_vendor_info { 49 const char *vendor; 50 const char *ident_string[2]; 51 }; 52 53 static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = { 54 { "Intel", { "GenuineIntel" } }, 55 { "AMD", { "AuthenticAMD" } }, 56 { "Cyrix", { "CyrixInstead" } }, 57 { "UMC", { "UMC UMC UMC" } }, 58 { "NexGen", { "NexGenDriven" } }, 59 { "Centaur", { "CentaurHauls" } }, 60 { "Rise", { "RiseRiseRise" } }, 61 { "Transmeta", { "GenuineTMx86", "TransmetaCPU" } }, 62 { "NSC", { "Geode by NSC" } }, 63 }; 64 65 #define K8_SMIONCMPHALT (1ULL << 27) 66 #define K8_C1EONCMPHALT (1ULL << 28) 67 68 #define K8_CMPHALT (K8_SMIONCMPHALT | K8_C1EONCMPHALT) 69 70 struct set_mtrr_parameter { 71 int32 index; 72 uint64 base; 73 uint64 length; 74 uint8 type; 75 }; 76 77 struct set_mtrrs_parameter { 78 const x86_mtrr_info* infos; 79 uint32 count; 80 uint8 defaultType; 81 }; 82 83 84 #ifdef __x86_64__ 85 extern addr_t _stac; 86 extern addr_t _clac; 87 #endif 88 89 extern "C" void x86_reboot(void); 90 // from arch.S 91 92 void (*gCpuIdleFunc)(void); 93 #ifndef __x86_64__ 94 void (*gX86SwapFPUFunc)(void* oldState, const void* newState) = x86_noop_swap; 95 bool gHasSSE = false; 96 #endif 97 98 static uint32 sCpuRendezvous; 99 static uint32 sCpuRendezvous2; 100 static uint32 sCpuRendezvous3; 101 static vint32 sTSCSyncRendezvous; 102 103 /* Some specials for the double fault handler */ 104 static uint8* sDoubleFaultStacks; 105 static const size_t kDoubleFaultStackSize = 4096; // size per CPU 106 107 static x86_cpu_module_info* sCpuModule; 108 109 110 /* CPU topology information */ 111 static uint32 (*sGetCPUTopologyID)(int currentCPU); 112 static uint32 sHierarchyMask[CPU_TOPOLOGY_LEVELS]; 113 static uint32 sHierarchyShift[CPU_TOPOLOGY_LEVELS]; 114 115 /* Cache topology information */ 116 static uint32 sCacheSharingMask[CPU_MAX_CACHE_LEVEL]; 117 118 static void* sUcodeData = NULL; 119 static size_t sUcodeDataSize = 0; 120 static struct intel_microcode_header* sLoadedUcodeUpdate; 121 static spinlock sUcodeUpdateLock = B_SPINLOCK_INITIALIZER; 122 123 124 static status_t 125 acpi_shutdown(bool rebootSystem) 126 { 127 if (debug_debugger_running() || !are_interrupts_enabled()) 128 return B_ERROR; 129 130 acpi_module_info* acpi; 131 if (get_module(B_ACPI_MODULE_NAME, (module_info**)&acpi) != B_OK) 132 return B_NOT_SUPPORTED; 133 134 status_t status; 135 if (rebootSystem) { 136 status = acpi->reboot(); 137 } else { 138 status = acpi->prepare_sleep_state(ACPI_POWER_STATE_OFF, NULL, 0); 139 if (status == B_OK) { 140 //cpu_status state = disable_interrupts(); 141 status = acpi->enter_sleep_state(ACPI_POWER_STATE_OFF); 142 //restore_interrupts(state); 143 } 144 } 145 146 put_module(B_ACPI_MODULE_NAME); 147 return status; 148 } 149 150 151 /*! Disable CPU caches, and invalidate them. */ 152 static void 153 disable_caches() 154 { 155 x86_write_cr0((x86_read_cr0() | CR0_CACHE_DISABLE) 156 & ~CR0_NOT_WRITE_THROUGH); 157 wbinvd(); 158 arch_cpu_global_TLB_invalidate(); 159 } 160 161 162 /*! Invalidate CPU caches, and enable them. */ 163 static void 164 enable_caches() 165 { 166 wbinvd(); 167 arch_cpu_global_TLB_invalidate(); 168 x86_write_cr0(x86_read_cr0() 169 & ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH)); 170 } 171 172 173 static void 174 set_mtrr(void* _parameter, int cpu) 175 { 176 struct set_mtrr_parameter* parameter 177 = (struct set_mtrr_parameter*)_parameter; 178 179 // wait until all CPUs have arrived here 180 smp_cpu_rendezvous(&sCpuRendezvous); 181 182 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 183 // that initiated the call_all_cpus() from doing that again and clearing 184 // sCpuRendezvous2 before the last CPU has actually left the loop in 185 // smp_cpu_rendezvous(); 186 if (cpu == 0) 187 atomic_set((int32*)&sCpuRendezvous3, 0); 188 189 disable_caches(); 190 191 sCpuModule->set_mtrr(parameter->index, parameter->base, parameter->length, 192 parameter->type); 193 194 enable_caches(); 195 196 // wait until all CPUs have arrived here 197 smp_cpu_rendezvous(&sCpuRendezvous2); 198 smp_cpu_rendezvous(&sCpuRendezvous3); 199 } 200 201 202 static void 203 set_mtrrs(void* _parameter, int cpu) 204 { 205 set_mtrrs_parameter* parameter = (set_mtrrs_parameter*)_parameter; 206 207 // wait until all CPUs have arrived here 208 smp_cpu_rendezvous(&sCpuRendezvous); 209 210 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 211 // that initiated the call_all_cpus() from doing that again and clearing 212 // sCpuRendezvous2 before the last CPU has actually left the loop in 213 // smp_cpu_rendezvous(); 214 if (cpu == 0) 215 atomic_set((int32*)&sCpuRendezvous3, 0); 216 217 disable_caches(); 218 219 sCpuModule->set_mtrrs(parameter->defaultType, parameter->infos, 220 parameter->count); 221 222 enable_caches(); 223 224 // wait until all CPUs have arrived here 225 smp_cpu_rendezvous(&sCpuRendezvous2); 226 smp_cpu_rendezvous(&sCpuRendezvous3); 227 } 228 229 230 static void 231 init_mtrrs(void* _unused, int cpu) 232 { 233 // wait until all CPUs have arrived here 234 smp_cpu_rendezvous(&sCpuRendezvous); 235 236 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 237 // that initiated the call_all_cpus() from doing that again and clearing 238 // sCpuRendezvous2 before the last CPU has actually left the loop in 239 // smp_cpu_rendezvous(); 240 if (cpu == 0) 241 atomic_set((int32*)&sCpuRendezvous3, 0); 242 243 disable_caches(); 244 245 sCpuModule->init_mtrrs(); 246 247 enable_caches(); 248 249 // wait until all CPUs have arrived here 250 smp_cpu_rendezvous(&sCpuRendezvous2); 251 smp_cpu_rendezvous(&sCpuRendezvous3); 252 } 253 254 255 uint32 256 x86_count_mtrrs(void) 257 { 258 if (sCpuModule == NULL) 259 return 0; 260 261 return sCpuModule->count_mtrrs(); 262 } 263 264 265 void 266 x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type) 267 { 268 struct set_mtrr_parameter parameter; 269 parameter.index = index; 270 parameter.base = base; 271 parameter.length = length; 272 parameter.type = type; 273 274 sCpuRendezvous = sCpuRendezvous2 = 0; 275 call_all_cpus(&set_mtrr, ¶meter); 276 } 277 278 279 status_t 280 x86_get_mtrr(uint32 index, uint64* _base, uint64* _length, uint8* _type) 281 { 282 // the MTRRs are identical on all CPUs, so it doesn't matter 283 // on which CPU this runs 284 return sCpuModule->get_mtrr(index, _base, _length, _type); 285 } 286 287 288 void 289 x86_set_mtrrs(uint8 defaultType, const x86_mtrr_info* infos, uint32 count) 290 { 291 if (sCpuModule == NULL) 292 return; 293 294 struct set_mtrrs_parameter parameter; 295 parameter.defaultType = defaultType; 296 parameter.infos = infos; 297 parameter.count = count; 298 299 sCpuRendezvous = sCpuRendezvous2 = 0; 300 call_all_cpus(&set_mtrrs, ¶meter); 301 } 302 303 304 void 305 x86_init_fpu(void) 306 { 307 // All x86_64 CPUs support SSE, don't need to bother checking for it. 308 #ifndef __x86_64__ 309 if (!x86_check_feature(IA32_FEATURE_FPU, FEATURE_COMMON)) { 310 // No FPU... time to install one in your 386? 311 dprintf("%s: Warning: CPU has no reported FPU.\n", __func__); 312 gX86SwapFPUFunc = x86_noop_swap; 313 return; 314 } 315 316 if (!x86_check_feature(IA32_FEATURE_SSE, FEATURE_COMMON) 317 || !x86_check_feature(IA32_FEATURE_FXSR, FEATURE_COMMON)) { 318 dprintf("%s: CPU has no SSE... just enabling FPU.\n", __func__); 319 // we don't have proper SSE support, just enable FPU 320 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 321 gX86SwapFPUFunc = x86_fnsave_swap; 322 return; 323 } 324 #endif 325 326 dprintf("%s: CPU has SSE... enabling FXSR and XMM.\n", __func__); 327 #ifndef __x86_64__ 328 // enable OS support for SSE 329 x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION); 330 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 331 332 gX86SwapFPUFunc = x86_fxsave_swap; 333 gHasSSE = true; 334 #endif 335 } 336 337 338 #if DUMP_FEATURE_STRING 339 static void 340 dump_feature_string(int currentCPU, cpu_ent* cpu) 341 { 342 char features[512]; 343 features[0] = 0; 344 345 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FPU) 346 strlcat(features, "fpu ", sizeof(features)); 347 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_VME) 348 strlcat(features, "vme ", sizeof(features)); 349 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DE) 350 strlcat(features, "de ", sizeof(features)); 351 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE) 352 strlcat(features, "pse ", sizeof(features)); 353 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TSC) 354 strlcat(features, "tsc ", sizeof(features)); 355 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MSR) 356 strlcat(features, "msr ", sizeof(features)); 357 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAE) 358 strlcat(features, "pae ", sizeof(features)); 359 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCE) 360 strlcat(features, "mce ", sizeof(features)); 361 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CX8) 362 strlcat(features, "cx8 ", sizeof(features)); 363 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_APIC) 364 strlcat(features, "apic ", sizeof(features)); 365 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SEP) 366 strlcat(features, "sep ", sizeof(features)); 367 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MTRR) 368 strlcat(features, "mtrr ", sizeof(features)); 369 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PGE) 370 strlcat(features, "pge ", sizeof(features)); 371 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCA) 372 strlcat(features, "mca ", sizeof(features)); 373 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CMOV) 374 strlcat(features, "cmov ", sizeof(features)); 375 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAT) 376 strlcat(features, "pat ", sizeof(features)); 377 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE36) 378 strlcat(features, "pse36 ", sizeof(features)); 379 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSN) 380 strlcat(features, "psn ", sizeof(features)); 381 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CLFSH) 382 strlcat(features, "clfsh ", sizeof(features)); 383 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DS) 384 strlcat(features, "ds ", sizeof(features)); 385 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_ACPI) 386 strlcat(features, "acpi ", sizeof(features)); 387 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MMX) 388 strlcat(features, "mmx ", sizeof(features)); 389 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FXSR) 390 strlcat(features, "fxsr ", sizeof(features)); 391 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE) 392 strlcat(features, "sse ", sizeof(features)); 393 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE2) 394 strlcat(features, "sse2 ", sizeof(features)); 395 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SS) 396 strlcat(features, "ss ", sizeof(features)); 397 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_HTT) 398 strlcat(features, "htt ", sizeof(features)); 399 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TM) 400 strlcat(features, "tm ", sizeof(features)); 401 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PBE) 402 strlcat(features, "pbe ", sizeof(features)); 403 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE3) 404 strlcat(features, "sse3 ", sizeof(features)); 405 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCLMULQDQ) 406 strlcat(features, "pclmulqdq ", sizeof(features)); 407 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DTES64) 408 strlcat(features, "dtes64 ", sizeof(features)); 409 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MONITOR) 410 strlcat(features, "monitor ", sizeof(features)); 411 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DSCPL) 412 strlcat(features, "dscpl ", sizeof(features)); 413 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_VMX) 414 strlcat(features, "vmx ", sizeof(features)); 415 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SMX) 416 strlcat(features, "smx ", sizeof(features)); 417 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_EST) 418 strlcat(features, "est ", sizeof(features)); 419 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TM2) 420 strlcat(features, "tm2 ", sizeof(features)); 421 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSSE3) 422 strlcat(features, "ssse3 ", sizeof(features)); 423 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CNXTID) 424 strlcat(features, "cnxtid ", sizeof(features)); 425 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_FMA) 426 strlcat(features, "fma ", sizeof(features)); 427 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CX16) 428 strlcat(features, "cx16 ", sizeof(features)); 429 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XTPR) 430 strlcat(features, "xtpr ", sizeof(features)); 431 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PDCM) 432 strlcat(features, "pdcm ", sizeof(features)); 433 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCID) 434 strlcat(features, "pcid ", sizeof(features)); 435 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DCA) 436 strlcat(features, "dca ", sizeof(features)); 437 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_1) 438 strlcat(features, "sse4_1 ", sizeof(features)); 439 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_2) 440 strlcat(features, "sse4_2 ", sizeof(features)); 441 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_X2APIC) 442 strlcat(features, "x2apic ", sizeof(features)); 443 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MOVBE) 444 strlcat(features, "movbe ", sizeof(features)); 445 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_POPCNT) 446 strlcat(features, "popcnt ", sizeof(features)); 447 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TSCDEADLINE) 448 strlcat(features, "tscdeadline ", sizeof(features)); 449 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AES) 450 strlcat(features, "aes ", sizeof(features)); 451 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XSAVE) 452 strlcat(features, "xsave ", sizeof(features)); 453 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_OSXSAVE) 454 strlcat(features, "osxsave ", sizeof(features)); 455 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AVX) 456 strlcat(features, "avx ", sizeof(features)); 457 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_F16C) 458 strlcat(features, "f16c ", sizeof(features)); 459 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_RDRND) 460 strlcat(features, "rdrnd ", sizeof(features)); 461 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) 462 strlcat(features, "hypervisor ", sizeof(features)); 463 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_SYSCALL) 464 strlcat(features, "syscall ", sizeof(features)); 465 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_NX) 466 strlcat(features, "nx ", sizeof(features)); 467 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_MMXEXT) 468 strlcat(features, "mmxext ", sizeof(features)); 469 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_FFXSR) 470 strlcat(features, "ffxsr ", sizeof(features)); 471 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_PDPE1GB) 472 strlcat(features, "pdpe1gb ", sizeof(features)); 473 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_LONG) 474 strlcat(features, "long ", sizeof(features)); 475 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOWEXT) 476 strlcat(features, "3dnowext ", sizeof(features)); 477 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOW) 478 strlcat(features, "3dnow ", sizeof(features)); 479 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_DTS) 480 strlcat(features, "dts ", sizeof(features)); 481 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ITB) 482 strlcat(features, "itb ", sizeof(features)); 483 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ARAT) 484 strlcat(features, "arat ", sizeof(features)); 485 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PLN) 486 strlcat(features, "pln ", sizeof(features)); 487 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ECMD) 488 strlcat(features, "ecmd ", sizeof(features)); 489 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PTM) 490 strlcat(features, "ptm ", sizeof(features)); 491 if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_APERFMPERF) 492 strlcat(features, "aperfmperf ", sizeof(features)); 493 if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_EPB) 494 strlcat(features, "epb ", sizeof(features)); 495 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_TSC_ADJUST) 496 strlcat(features, "tsc_adjust ", sizeof(features)); 497 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SGX) 498 strlcat(features, "sgx ", sizeof(features)); 499 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_BMI1) 500 strlcat(features, "bmi1 ", sizeof(features)); 501 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_HLE) 502 strlcat(features, "hle ", sizeof(features)); 503 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX2) 504 strlcat(features, "avx2 ", sizeof(features)); 505 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SMEP) 506 strlcat(features, "smep ", sizeof(features)); 507 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_BMI2) 508 strlcat(features, "bmi2 ", sizeof(features)); 509 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_ERMS) 510 strlcat(features, "erms ", sizeof(features)); 511 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_INVPCID) 512 strlcat(features, "invpcid ", sizeof(features)); 513 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_RTM) 514 strlcat(features, "rtm ", sizeof(features)); 515 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_CQM) 516 strlcat(features, "cqm ", sizeof(features)); 517 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_MPX) 518 strlcat(features, "mpx ", sizeof(features)); 519 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_RDT_A) 520 strlcat(features, "rdt_a ", sizeof(features)); 521 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512F) 522 strlcat(features, "avx512f ", sizeof(features)); 523 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512DQ) 524 strlcat(features, "avx512dq ", sizeof(features)); 525 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_RDSEED) 526 strlcat(features, "rdseed ", sizeof(features)); 527 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_ADX) 528 strlcat(features, "adx ", sizeof(features)); 529 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SMAP) 530 strlcat(features, "smap ", sizeof(features)); 531 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512IFMA) 532 strlcat(features, "avx512ifma ", sizeof(features)); 533 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_PCOMMIT) 534 strlcat(features, "pcommit ", sizeof(features)); 535 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_CLFLUSHOPT) 536 strlcat(features, "cflushopt ", sizeof(features)); 537 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_CLWB) 538 strlcat(features, "clwb ", sizeof(features)); 539 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_INTEL_PT) 540 strlcat(features, "intel_pt ", sizeof(features)); 541 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512PF) 542 strlcat(features, "avx512pf ", sizeof(features)); 543 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512ER) 544 strlcat(features, "avx512er ", sizeof(features)); 545 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512CD) 546 strlcat(features, "avx512cd ", sizeof(features)); 547 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SHA_NI) 548 strlcat(features, "sha_ni ", sizeof(features)); 549 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512BW) 550 strlcat(features, "avx512bw ", sizeof(features)); 551 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512VI) 552 strlcat(features, "avx512vi ", sizeof(features)); 553 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_IBRS) 554 strlcat(features, "ibrs ", sizeof(features)); 555 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_STIBP) 556 strlcat(features, "stibp ", sizeof(features)); 557 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_L1D_FLUSH) 558 strlcat(features, "l1d_flush ", sizeof(features)); 559 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_ARCH_CAPABILITIES) 560 strlcat(features, "msr_arch ", sizeof(features)); 561 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_SSBD) 562 strlcat(features, "ssbd ", sizeof(features)); 563 if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_CLZERO) 564 strlcat(features, "clzero ", sizeof(features)); 565 if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_IBPB) 566 strlcat(features, "ibpb ", sizeof(features)); 567 if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_AMD_SSBD) 568 strlcat(features, "amd_ssbd ", sizeof(features)); 569 if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_VIRT_SSBD) 570 strlcat(features, "virt_ssbd ", sizeof(features)); 571 if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_AMD_SSB_NO) 572 strlcat(features, "amd_ssb_no ", sizeof(features)); 573 dprintf("CPU %d: features: %s\n", currentCPU, features); 574 } 575 #endif // DUMP_FEATURE_STRING 576 577 578 static void 579 compute_cpu_hierarchy_masks(int maxLogicalID, int maxCoreID) 580 { 581 ASSERT(maxLogicalID >= maxCoreID); 582 const int kMaxSMTID = maxLogicalID / maxCoreID; 583 584 sHierarchyMask[CPU_TOPOLOGY_SMT] = kMaxSMTID - 1; 585 sHierarchyShift[CPU_TOPOLOGY_SMT] = 0; 586 587 sHierarchyMask[CPU_TOPOLOGY_CORE] = (maxCoreID - 1) * kMaxSMTID; 588 sHierarchyShift[CPU_TOPOLOGY_CORE] 589 = count_set_bits(sHierarchyMask[CPU_TOPOLOGY_SMT]); 590 591 const uint32 kSinglePackageMask = sHierarchyMask[CPU_TOPOLOGY_SMT] 592 | sHierarchyMask[CPU_TOPOLOGY_CORE]; 593 sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~kSinglePackageMask; 594 sHierarchyShift[CPU_TOPOLOGY_PACKAGE] = count_set_bits(kSinglePackageMask); 595 } 596 597 598 static uint32 599 get_cpu_legacy_initial_apic_id(int /* currentCPU */) 600 { 601 cpuid_info cpuid; 602 get_current_cpuid(&cpuid, 1, 0); 603 return cpuid.regs.ebx >> 24; 604 } 605 606 607 static inline status_t 608 detect_amd_cpu_topology(uint32 maxBasicLeaf, uint32 maxExtendedLeaf) 609 { 610 sGetCPUTopologyID = get_cpu_legacy_initial_apic_id; 611 612 cpuid_info cpuid; 613 get_current_cpuid(&cpuid, 1, 0); 614 int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff); 615 616 int maxCoreID = 1; 617 if (maxExtendedLeaf >= 0x80000008) { 618 get_current_cpuid(&cpuid, 0x80000008, 0); 619 maxCoreID = (cpuid.regs.ecx >> 12) & 0xf; 620 if (maxCoreID != 0) 621 maxCoreID = 1 << maxCoreID; 622 else 623 maxCoreID = next_power_of_2((cpuid.regs.edx & 0xf) + 1); 624 } 625 626 if (maxExtendedLeaf >= 0x80000001) { 627 get_current_cpuid(&cpuid, 0x80000001, 0); 628 if (x86_check_feature(IA32_FEATURE_AMD_EXT_CMPLEGACY, 629 FEATURE_EXT_AMD_ECX)) 630 maxCoreID = maxLogicalID; 631 } 632 633 compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID); 634 635 return B_OK; 636 } 637 638 639 static void 640 detect_amd_cache_topology(uint32 maxExtendedLeaf) 641 { 642 if (!x86_check_feature(IA32_FEATURE_AMD_EXT_TOPOLOGY, FEATURE_EXT_AMD_ECX)) 643 return; 644 645 if (maxExtendedLeaf < 0x8000001d) 646 return; 647 648 uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL]; 649 int maxCacheLevel = 0; 650 651 int currentLevel = 0; 652 int cacheType; 653 do { 654 cpuid_info cpuid; 655 get_current_cpuid(&cpuid, 0x8000001d, currentLevel); 656 657 cacheType = cpuid.regs.eax & 0x1f; 658 if (cacheType == 0) 659 break; 660 661 int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; 662 int coresCount = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1); 663 hierarchyLevels[cacheLevel - 1] 664 = coresCount * (sHierarchyMask[CPU_TOPOLOGY_SMT] + 1); 665 maxCacheLevel = std::max(maxCacheLevel, cacheLevel); 666 667 currentLevel++; 668 } while (true); 669 670 for (int i = 0; i < maxCacheLevel; i++) 671 sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); 672 gCPUCacheLevelCount = maxCacheLevel; 673 } 674 675 676 static uint32 677 get_intel_cpu_initial_x2apic_id(int /* currentCPU */) 678 { 679 cpuid_info cpuid; 680 get_current_cpuid(&cpuid, 11, 0); 681 return cpuid.regs.edx; 682 } 683 684 685 static inline status_t 686 detect_intel_cpu_topology_x2apic(uint32 maxBasicLeaf) 687 { 688 if (maxBasicLeaf < 11) 689 return B_UNSUPPORTED; 690 691 uint8 hierarchyLevels[CPU_TOPOLOGY_LEVELS] = { 0 }; 692 693 int currentLevel = 0; 694 int levelType; 695 unsigned int levelsSet = 0; 696 697 do { 698 cpuid_info cpuid; 699 get_current_cpuid(&cpuid, 11, currentLevel); 700 if (currentLevel == 0 && cpuid.regs.ebx == 0) 701 return B_UNSUPPORTED; 702 703 levelType = (cpuid.regs.ecx >> 8) & 0xff; 704 int levelValue = cpuid.regs.eax & 0x1f; 705 706 switch (levelType) { 707 case 1: // SMT 708 hierarchyLevels[CPU_TOPOLOGY_SMT] = levelValue; 709 levelsSet |= 1; 710 break; 711 case 2: // core 712 hierarchyLevels[CPU_TOPOLOGY_CORE] = levelValue; 713 levelsSet |= 2; 714 break; 715 } 716 717 currentLevel++; 718 } while (levelType != 0 && levelsSet != 3); 719 720 sGetCPUTopologyID = get_intel_cpu_initial_x2apic_id; 721 722 for (int i = 1; i < CPU_TOPOLOGY_LEVELS; i++) { 723 if ((levelsSet & (1u << i)) != 0) 724 continue; 725 hierarchyLevels[i] = hierarchyLevels[i - 1]; 726 } 727 728 for (int i = 0; i < CPU_TOPOLOGY_LEVELS; i++) { 729 uint32 mask = ~uint32(0); 730 if (i < CPU_TOPOLOGY_LEVELS - 1) 731 mask = (1u << hierarchyLevels[i]) - 1; 732 if (i > 0) 733 mask &= ~sHierarchyMask[i - 1]; 734 sHierarchyMask[i] = mask; 735 sHierarchyShift[i] = i > 0 ? hierarchyLevels[i - 1] : 0; 736 } 737 738 return B_OK; 739 } 740 741 742 static inline status_t 743 detect_intel_cpu_topology_legacy(uint32 maxBasicLeaf) 744 { 745 sGetCPUTopologyID = get_cpu_legacy_initial_apic_id; 746 747 cpuid_info cpuid; 748 749 get_current_cpuid(&cpuid, 1, 0); 750 int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff); 751 752 int maxCoreID = 1; 753 if (maxBasicLeaf >= 4) { 754 get_current_cpuid(&cpuid, 4, 0); 755 maxCoreID = next_power_of_2((cpuid.regs.eax >> 26) + 1); 756 } 757 758 compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID); 759 760 return B_OK; 761 } 762 763 764 static void 765 detect_intel_cache_topology(uint32 maxBasicLeaf) 766 { 767 if (maxBasicLeaf < 4) 768 return; 769 770 uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL]; 771 int maxCacheLevel = 0; 772 773 int currentLevel = 0; 774 int cacheType; 775 do { 776 cpuid_info cpuid; 777 get_current_cpuid(&cpuid, 4, currentLevel); 778 779 cacheType = cpuid.regs.eax & 0x1f; 780 if (cacheType == 0) 781 break; 782 783 int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; 784 hierarchyLevels[cacheLevel - 1] 785 = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1); 786 maxCacheLevel = std::max(maxCacheLevel, cacheLevel); 787 788 currentLevel++; 789 } while (true); 790 791 for (int i = 0; i < maxCacheLevel; i++) 792 sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); 793 794 gCPUCacheLevelCount = maxCacheLevel; 795 } 796 797 798 static uint32 799 get_simple_cpu_topology_id(int currentCPU) 800 { 801 return currentCPU; 802 } 803 804 805 static inline int 806 get_topology_level_id(uint32 id, cpu_topology_level level) 807 { 808 ASSERT(level < CPU_TOPOLOGY_LEVELS); 809 return (id & sHierarchyMask[level]) >> sHierarchyShift[level]; 810 } 811 812 813 static void 814 detect_cpu_topology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, 815 uint32 maxExtendedLeaf) 816 { 817 if (currentCPU == 0) { 818 memset(sCacheSharingMask, 0xff, sizeof(sCacheSharingMask)); 819 820 status_t result = B_UNSUPPORTED; 821 if (x86_check_feature(IA32_FEATURE_HTT, FEATURE_COMMON)) { 822 if (cpu->arch.vendor == VENDOR_AMD) { 823 result = detect_amd_cpu_topology(maxBasicLeaf, maxExtendedLeaf); 824 825 if (result == B_OK) 826 detect_amd_cache_topology(maxExtendedLeaf); 827 } 828 829 if (cpu->arch.vendor == VENDOR_INTEL) { 830 result = detect_intel_cpu_topology_x2apic(maxBasicLeaf); 831 if (result != B_OK) 832 result = detect_intel_cpu_topology_legacy(maxBasicLeaf); 833 834 if (result == B_OK) 835 detect_intel_cache_topology(maxBasicLeaf); 836 } 837 } 838 839 if (result != B_OK) { 840 dprintf("No CPU topology information available.\n"); 841 842 sGetCPUTopologyID = get_simple_cpu_topology_id; 843 844 sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~uint32(0); 845 } 846 } 847 848 ASSERT(sGetCPUTopologyID != NULL); 849 int topologyID = sGetCPUTopologyID(currentCPU); 850 cpu->topology_id[CPU_TOPOLOGY_SMT] 851 = get_topology_level_id(topologyID, CPU_TOPOLOGY_SMT); 852 cpu->topology_id[CPU_TOPOLOGY_CORE] 853 = get_topology_level_id(topologyID, CPU_TOPOLOGY_CORE); 854 cpu->topology_id[CPU_TOPOLOGY_PACKAGE] 855 = get_topology_level_id(topologyID, CPU_TOPOLOGY_PACKAGE); 856 857 unsigned int i; 858 for (i = 0; i < gCPUCacheLevelCount; i++) 859 cpu->cache_id[i] = topologyID & sCacheSharingMask[i]; 860 for (; i < CPU_MAX_CACHE_LEVEL; i++) 861 cpu->cache_id[i] = -1; 862 863 #if DUMP_CPU_TOPOLOGY 864 dprintf("CPU %d: apic id %d, package %d, core %d, smt %d\n", currentCPU, 865 topologyID, cpu->topology_id[CPU_TOPOLOGY_PACKAGE], 866 cpu->topology_id[CPU_TOPOLOGY_CORE], 867 cpu->topology_id[CPU_TOPOLOGY_SMT]); 868 869 if (gCPUCacheLevelCount > 0) { 870 char cacheLevels[256]; 871 unsigned int offset = 0; 872 for (i = 0; i < gCPUCacheLevelCount; i++) { 873 offset += snprintf(cacheLevels + offset, 874 sizeof(cacheLevels) - offset, 875 " L%d id %d%s", i + 1, cpu->cache_id[i], 876 i < gCPUCacheLevelCount - 1 ? "," : ""); 877 878 if (offset >= sizeof(cacheLevels)) 879 break; 880 } 881 882 dprintf("CPU %d: cache sharing:%s\n", currentCPU, cacheLevels); 883 } 884 #endif 885 } 886 887 888 static void 889 detect_intel_patch_level(cpu_ent* cpu) 890 { 891 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) { 892 cpu->arch.patch_level = 0; 893 return; 894 } 895 896 x86_write_msr(IA32_MSR_UCODE_REV, 0); 897 cpuid_info cpuid; 898 get_current_cpuid(&cpuid, 1, 0); 899 900 uint64 value = x86_read_msr(IA32_MSR_UCODE_REV); 901 cpu->arch.patch_level = value >> 32; 902 } 903 904 905 static void 906 detect_amd_patch_level(cpu_ent* cpu) 907 { 908 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) { 909 cpu->arch.patch_level = 0; 910 return; 911 } 912 913 uint64 value = x86_read_msr(IA32_MSR_UCODE_REV); 914 cpu->arch.patch_level = value >> 32; 915 } 916 917 918 static struct intel_microcode_header* 919 find_microcode_intel(addr_t data, size_t size, uint32 patchLevel) 920 { 921 // 9.11.3 Processor Identification 922 cpuid_info cpuid; 923 get_current_cpuid(&cpuid, 1, 0); 924 uint32 signature = cpuid.regs.eax; 925 // 9.11.4 Platform Identification 926 uint64 platformBits = (x86_read_msr(IA32_MSR_PLATFORM_ID) >> 50) & 0x7; 927 uint64 mask = 1 << platformBits; 928 929 while (size > 0) { 930 if (size < sizeof(struct intel_microcode_header)) { 931 dprintf("find_microcode_intel update is too small for header\n"); 932 break; 933 } 934 struct intel_microcode_header* header = 935 (struct intel_microcode_header*)data; 936 937 uint32 totalSize = header->total_size; 938 uint32 dataSize = header->data_size; 939 if (dataSize == 0) { 940 dataSize = 2000; 941 totalSize = sizeof(struct intel_microcode_header) 942 + dataSize; 943 } 944 if (totalSize > size) { 945 dprintf("find_microcode_intel update is too small for data\n"); 946 break; 947 } 948 949 uint32* dwords = (uint32*)data; 950 // prepare the next update 951 size -= totalSize; 952 data += totalSize; 953 954 if (header->loader_revision != 1) { 955 dprintf("find_microcode_intel incorrect loader version\n"); 956 continue; 957 } 958 // 9.11.6 The microcode update data requires a 16-byte boundary 959 // alignment. 960 if (((addr_t)header % 16) != 0) { 961 dprintf("find_microcode_intel incorrect alignment\n"); 962 continue; 963 } 964 uint32 sum = 0; 965 for (uint32 i = 0; i < totalSize / 4; i++) { 966 sum += dwords[i]; 967 } 968 if (sum != 0) { 969 dprintf("find_microcode_intel incorrect checksum\n"); 970 continue; 971 } 972 if (patchLevel > header->update_revision) { 973 dprintf("find_microcode_intel update_revision is lower\n"); 974 continue; 975 } 976 if (signature == header->processor_signature 977 && (mask & header->processor_flags) != 0) { 978 return header; 979 } 980 if (totalSize <= (sizeof(struct intel_microcode_header) + dataSize 981 + sizeof(struct intel_microcode_extended_signature_header))) { 982 continue; 983 } 984 struct intel_microcode_extended_signature_header* extSigHeader = 985 (struct intel_microcode_extended_signature_header*)((addr_t)header 986 + sizeof(struct intel_microcode_header) + dataSize); 987 struct intel_microcode_extended_signature* extended_signature = 988 (struct intel_microcode_extended_signature*)((addr_t)extSigHeader 989 + sizeof(struct intel_microcode_extended_signature_header)); 990 for (uint32 i = 0; i < extSigHeader->extended_signature_count; i++) { 991 if (signature == extended_signature[i].processor_signature 992 && (mask & extended_signature[i].processor_flags) != 0) 993 return header; 994 } 995 } 996 return NULL; 997 } 998 999 1000 static void 1001 load_microcode_intel(int currentCPU, cpu_ent* cpu) 1002 { 1003 // serialize for HT cores 1004 if (currentCPU != 0) 1005 acquire_spinlock(&sUcodeUpdateLock); 1006 detect_intel_patch_level(cpu); 1007 uint32 revision = cpu->arch.patch_level; 1008 struct intel_microcode_header* update = sLoadedUcodeUpdate; 1009 if (update == NULL) { 1010 update = find_microcode_intel((addr_t)sUcodeData, sUcodeDataSize, 1011 revision); 1012 } 1013 if (update != NULL) { 1014 addr_t data = (addr_t)update + sizeof(struct intel_microcode_header); 1015 wbinvd(); 1016 x86_write_msr(IA32_MSR_UCODE_WRITE, data); 1017 detect_intel_patch_level(cpu); 1018 if (revision == cpu->arch.patch_level) { 1019 dprintf("CPU %d: update failed\n", currentCPU); 1020 } else { 1021 if (sLoadedUcodeUpdate == NULL) 1022 sLoadedUcodeUpdate = update; 1023 dprintf("CPU %d: updated from revision %" B_PRIu32 " to %" B_PRIu32 1024 "\n", currentCPU, revision, cpu->arch.patch_level); 1025 } 1026 } else { 1027 dprintf("CPU %d: no update found\n", currentCPU); 1028 } 1029 if (currentCPU != 0) 1030 release_spinlock(&sUcodeUpdateLock); 1031 } 1032 1033 1034 static void 1035 load_microcode_amd(int currentCPU, cpu_ent* cpu) 1036 { 1037 dprintf("CPU %d: no update found\n", currentCPU); 1038 } 1039 1040 1041 static void 1042 load_microcode(int currentCPU) 1043 { 1044 if (sUcodeData == NULL) 1045 return; 1046 cpu_ent* cpu = get_cpu_struct(); 1047 if ((cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) != 0) 1048 return; 1049 if (cpu->arch.vendor == VENDOR_INTEL) 1050 load_microcode_intel(currentCPU, cpu); 1051 else if (cpu->arch.vendor == VENDOR_AMD) 1052 load_microcode_amd(currentCPU, cpu); 1053 } 1054 1055 1056 static void 1057 detect_cpu(int currentCPU) 1058 { 1059 cpu_ent* cpu = get_cpu_struct(); 1060 char vendorString[17]; 1061 cpuid_info cpuid; 1062 1063 // clear out the cpu info data 1064 cpu->arch.vendor = VENDOR_UNKNOWN; 1065 cpu->arch.vendor_name = "UNKNOWN VENDOR"; 1066 cpu->arch.feature[FEATURE_COMMON] = 0; 1067 cpu->arch.feature[FEATURE_EXT] = 0; 1068 cpu->arch.feature[FEATURE_EXT_AMD] = 0; 1069 cpu->arch.feature[FEATURE_7_EBX] = 0; 1070 cpu->arch.feature[FEATURE_7_ECX] = 0; 1071 cpu->arch.feature[FEATURE_7_EDX] = 0; 1072 cpu->arch.model_name[0] = 0; 1073 1074 // print some fun data 1075 get_current_cpuid(&cpuid, 0, 0); 1076 uint32 maxBasicLeaf = cpuid.eax_0.max_eax; 1077 1078 // build the vendor string 1079 memset(vendorString, 0, sizeof(vendorString)); 1080 memcpy(vendorString, cpuid.eax_0.vendor_id, sizeof(cpuid.eax_0.vendor_id)); 1081 1082 // get the family, model, stepping 1083 get_current_cpuid(&cpuid, 1, 0); 1084 cpu->arch.type = cpuid.eax_1.type; 1085 cpu->arch.family = cpuid.eax_1.family; 1086 cpu->arch.extended_family = cpuid.eax_1.extended_family; 1087 cpu->arch.model = cpuid.eax_1.model; 1088 cpu->arch.extended_model = cpuid.eax_1.extended_model; 1089 cpu->arch.stepping = cpuid.eax_1.stepping; 1090 dprintf("CPU %d: type %d family %d extended_family %d model %d " 1091 "extended_model %d stepping %d, string '%s'\n", 1092 currentCPU, cpu->arch.type, cpu->arch.family, 1093 cpu->arch.extended_family, cpu->arch.model, 1094 cpu->arch.extended_model, cpu->arch.stepping, vendorString); 1095 1096 // figure out what vendor we have here 1097 1098 for (int32 i = 0; i < VENDOR_NUM; i++) { 1099 if (vendor_info[i].ident_string[0] 1100 && !strcmp(vendorString, vendor_info[i].ident_string[0])) { 1101 cpu->arch.vendor = (x86_vendors)i; 1102 cpu->arch.vendor_name = vendor_info[i].vendor; 1103 break; 1104 } 1105 if (vendor_info[i].ident_string[1] 1106 && !strcmp(vendorString, vendor_info[i].ident_string[1])) { 1107 cpu->arch.vendor = (x86_vendors)i; 1108 cpu->arch.vendor_name = vendor_info[i].vendor; 1109 break; 1110 } 1111 } 1112 1113 // see if we can get the model name 1114 get_current_cpuid(&cpuid, 0x80000000, 0); 1115 uint32 maxExtendedLeaf = cpuid.eax_0.max_eax; 1116 if (maxExtendedLeaf >= 0x80000004) { 1117 // build the model string (need to swap ecx/edx data before copying) 1118 unsigned int temp; 1119 memset(cpu->arch.model_name, 0, sizeof(cpu->arch.model_name)); 1120 1121 get_current_cpuid(&cpuid, 0x80000002, 0); 1122 temp = cpuid.regs.edx; 1123 cpuid.regs.edx = cpuid.regs.ecx; 1124 cpuid.regs.ecx = temp; 1125 memcpy(cpu->arch.model_name, cpuid.as_chars, sizeof(cpuid.as_chars)); 1126 1127 get_current_cpuid(&cpuid, 0x80000003, 0); 1128 temp = cpuid.regs.edx; 1129 cpuid.regs.edx = cpuid.regs.ecx; 1130 cpuid.regs.ecx = temp; 1131 memcpy(cpu->arch.model_name + 16, cpuid.as_chars, 1132 sizeof(cpuid.as_chars)); 1133 1134 get_current_cpuid(&cpuid, 0x80000004, 0); 1135 temp = cpuid.regs.edx; 1136 cpuid.regs.edx = cpuid.regs.ecx; 1137 cpuid.regs.ecx = temp; 1138 memcpy(cpu->arch.model_name + 32, cpuid.as_chars, 1139 sizeof(cpuid.as_chars)); 1140 1141 // some cpus return a right-justified string 1142 int32 i = 0; 1143 while (cpu->arch.model_name[i] == ' ') 1144 i++; 1145 if (i > 0) { 1146 memmove(cpu->arch.model_name, &cpu->arch.model_name[i], 1147 strlen(&cpu->arch.model_name[i]) + 1); 1148 } 1149 1150 dprintf("CPU %d: vendor '%s' model name '%s'\n", 1151 currentCPU, cpu->arch.vendor_name, cpu->arch.model_name); 1152 } else { 1153 strlcpy(cpu->arch.model_name, "unknown", sizeof(cpu->arch.model_name)); 1154 } 1155 1156 // load feature bits 1157 get_current_cpuid(&cpuid, 1, 0); 1158 cpu->arch.feature[FEATURE_COMMON] = cpuid.eax_1.features; // edx 1159 cpu->arch.feature[FEATURE_EXT] = cpuid.eax_1.extended_features; // ecx 1160 1161 if (maxExtendedLeaf >= 0x80000001) { 1162 get_current_cpuid(&cpuid, 0x80000001, 0); 1163 if (cpu->arch.vendor == VENDOR_AMD) 1164 cpu->arch.feature[FEATURE_EXT_AMD_ECX] = cpuid.regs.ecx; // ecx 1165 cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx 1166 if (cpu->arch.vendor != VENDOR_AMD) 1167 cpu->arch.feature[FEATURE_EXT_AMD] &= IA32_FEATURES_INTEL_EXT; 1168 } 1169 1170 if (maxBasicLeaf >= 5) { 1171 get_current_cpuid(&cpuid, 5, 0); 1172 cpu->arch.feature[FEATURE_5_ECX] = cpuid.regs.ecx; 1173 } 1174 1175 if (maxBasicLeaf >= 6) { 1176 get_current_cpuid(&cpuid, 6, 0); 1177 cpu->arch.feature[FEATURE_6_EAX] = cpuid.regs.eax; 1178 cpu->arch.feature[FEATURE_6_ECX] = cpuid.regs.ecx; 1179 } 1180 1181 if (maxBasicLeaf >= 7) { 1182 get_current_cpuid(&cpuid, 7, 0); 1183 cpu->arch.feature[FEATURE_7_EBX] = cpuid.regs.ebx; 1184 cpu->arch.feature[FEATURE_7_ECX] = cpuid.regs.ecx; 1185 cpu->arch.feature[FEATURE_7_EDX] = cpuid.regs.edx; 1186 } 1187 1188 if (maxExtendedLeaf >= 0x80000007) { 1189 get_current_cpuid(&cpuid, 0x80000007, 0); 1190 cpu->arch.feature[FEATURE_EXT_7_EDX] = cpuid.regs.edx; 1191 } 1192 1193 if (maxExtendedLeaf >= 0x80000008) { 1194 get_current_cpuid(&cpuid, 0x80000008, 0); 1195 cpu->arch.feature[FEATURE_EXT_8_EBX] = cpuid.regs.ebx; 1196 } 1197 1198 detect_cpu_topology(currentCPU, cpu, maxBasicLeaf, maxExtendedLeaf); 1199 1200 if (cpu->arch.vendor == VENDOR_INTEL) 1201 detect_intel_patch_level(cpu); 1202 else if (cpu->arch.vendor == VENDOR_AMD) 1203 detect_amd_patch_level(cpu); 1204 1205 #if DUMP_FEATURE_STRING 1206 dump_feature_string(currentCPU, cpu); 1207 #endif 1208 #if DUMP_CPU_PATCHLEVEL 1209 dprintf("CPU %d: patch_level %" B_PRIu32 "\n", currentCPU, 1210 cpu->arch.patch_level); 1211 #endif 1212 } 1213 1214 1215 bool 1216 x86_check_feature(uint32 feature, enum x86_feature_type type) 1217 { 1218 cpu_ent* cpu = get_cpu_struct(); 1219 1220 #if 0 1221 int i; 1222 dprintf("x86_check_feature: feature 0x%x, type %d\n", feature, type); 1223 for (i = 0; i < FEATURE_NUM; i++) { 1224 dprintf("features %d: 0x%x\n", i, cpu->arch.feature[i]); 1225 } 1226 #endif 1227 1228 return (cpu->arch.feature[type] & feature) != 0; 1229 } 1230 1231 1232 void* 1233 x86_get_double_fault_stack(int32 cpu, size_t* _size) 1234 { 1235 *_size = kDoubleFaultStackSize; 1236 return sDoubleFaultStacks + kDoubleFaultStackSize * cpu; 1237 } 1238 1239 1240 /*! Returns the index of the current CPU. Can only be called from the double 1241 fault handler. 1242 */ 1243 int32 1244 x86_double_fault_get_cpu(void) 1245 { 1246 addr_t stack = x86_get_stack_frame(); 1247 return (stack - (addr_t)sDoubleFaultStacks) / kDoubleFaultStackSize; 1248 } 1249 1250 1251 // #pragma mark - 1252 1253 1254 status_t 1255 arch_cpu_preboot_init_percpu(kernel_args* args, int cpu) 1256 { 1257 // On SMP system we want to synchronize the CPUs' TSCs, so system_time() 1258 // will return consistent values. 1259 if (smp_get_num_cpus() > 1) { 1260 // let the first CPU prepare the rendezvous point 1261 if (cpu == 0) 1262 sTSCSyncRendezvous = smp_get_num_cpus() - 1; 1263 1264 // One CPU after the other will drop out of this loop and be caught by 1265 // the loop below, until the last CPU (0) gets there. Save for +/- a few 1266 // cycles the CPUs should pass the second loop at the same time. 1267 while (sTSCSyncRendezvous != cpu) { 1268 } 1269 1270 sTSCSyncRendezvous = cpu - 1; 1271 1272 while (sTSCSyncRendezvous != -1) { 1273 } 1274 1275 // reset TSC to 0 1276 x86_write_msr(IA32_MSR_TSC, 0); 1277 } 1278 1279 x86_descriptors_preboot_init_percpu(args, cpu); 1280 1281 return B_OK; 1282 } 1283 1284 1285 static void 1286 halt_idle(void) 1287 { 1288 asm("hlt"); 1289 } 1290 1291 1292 static void 1293 amdc1e_noarat_idle(void) 1294 { 1295 uint64 msr = x86_read_msr(K8_MSR_IPM); 1296 if (msr & K8_CMPHALT) 1297 x86_write_msr(K8_MSR_IPM, msr & ~K8_CMPHALT); 1298 halt_idle(); 1299 } 1300 1301 1302 static bool 1303 detect_amdc1e_noarat() 1304 { 1305 cpu_ent* cpu = get_cpu_struct(); 1306 1307 if (cpu->arch.vendor != VENDOR_AMD) 1308 return false; 1309 1310 // Family 0x12 and higher processors support ARAT 1311 // Family lower than 0xf processors doesn't support C1E 1312 // Family 0xf with model <= 0x40 procssors doesn't support C1E 1313 uint32 family = cpu->arch.family + cpu->arch.extended_family; 1314 uint32 model = (cpu->arch.extended_model << 4) | cpu->arch.model; 1315 return (family < 0x12 && family > 0xf) || (family == 0xf && model > 0x40); 1316 } 1317 1318 1319 status_t 1320 arch_cpu_init_percpu(kernel_args* args, int cpu) 1321 { 1322 load_microcode(cpu); 1323 detect_cpu(cpu); 1324 1325 if (!gCpuIdleFunc) { 1326 if (detect_amdc1e_noarat()) 1327 gCpuIdleFunc = amdc1e_noarat_idle; 1328 else 1329 gCpuIdleFunc = halt_idle; 1330 } 1331 1332 #ifdef __x86_64__ 1333 // if RDTSCP is available write cpu number in TSC_AUX 1334 if (x86_check_feature(IA32_FEATURE_AMD_EXT_RDTSCP, FEATURE_EXT_AMD)) 1335 x86_write_msr(IA32_MSR_TSC_AUX, cpu); 1336 #endif 1337 1338 return __x86_patch_errata_percpu(cpu); 1339 } 1340 1341 1342 status_t 1343 arch_cpu_init(kernel_args* args) 1344 { 1345 if (args->ucode_data != NULL 1346 && args->ucode_data_size > 0) { 1347 sUcodeData = args->ucode_data; 1348 sUcodeDataSize = args->ucode_data_size; 1349 } else { 1350 dprintf("CPU: no microcode provided\n"); 1351 } 1352 1353 // init the TSC -> system_time() conversion factors 1354 1355 uint32 conversionFactor = args->arch_args.system_time_cv_factor; 1356 uint64 conversionFactorNsecs = (uint64)conversionFactor * 1000; 1357 1358 #ifdef __x86_64__ 1359 // The x86_64 system_time() implementation uses 64-bit multiplication and 1360 // therefore shifting is not necessary for low frequencies (it's also not 1361 // too likely that there'll be any x86_64 CPUs clocked under 1GHz). 1362 __x86_setup_system_time((uint64)conversionFactor << 32, 1363 conversionFactorNsecs); 1364 #else 1365 if (conversionFactorNsecs >> 32 != 0) { 1366 // the TSC frequency is < 1 GHz, which forces us to shift the factor 1367 __x86_setup_system_time(conversionFactor, conversionFactorNsecs >> 16, 1368 true); 1369 } else { 1370 // the TSC frequency is >= 1 GHz 1371 __x86_setup_system_time(conversionFactor, conversionFactorNsecs, false); 1372 } 1373 #endif 1374 1375 // Initialize descriptor tables. 1376 x86_descriptors_init(args); 1377 1378 return B_OK; 1379 } 1380 1381 1382 #ifdef __x86_64__ 1383 static void 1384 enable_smap(void* dummy, int cpu) 1385 { 1386 x86_write_cr4(x86_read_cr4() | IA32_CR4_SMAP); 1387 } 1388 1389 1390 static void 1391 enable_smep(void* dummy, int cpu) 1392 { 1393 x86_write_cr4(x86_read_cr4() | IA32_CR4_SMEP); 1394 } 1395 #endif 1396 1397 1398 status_t 1399 arch_cpu_init_post_vm(kernel_args* args) 1400 { 1401 uint32 i; 1402 1403 // allocate an area for the double fault stacks 1404 virtual_address_restrictions virtualRestrictions = {}; 1405 virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS; 1406 physical_address_restrictions physicalRestrictions = {}; 1407 create_area_etc(B_SYSTEM_TEAM, "double fault stacks", 1408 kDoubleFaultStackSize * smp_get_num_cpus(), B_FULL_LOCK, 1409 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0, 1410 &virtualRestrictions, &physicalRestrictions, 1411 (void**)&sDoubleFaultStacks); 1412 1413 X86PagingStructures* kernelPagingStructures 1414 = static_cast<X86VMTranslationMap*>( 1415 VMAddressSpace::Kernel()->TranslationMap())->PagingStructures(); 1416 1417 // Set active translation map on each CPU. 1418 for (i = 0; i < args->num_cpus; i++) { 1419 gCPU[i].arch.active_paging_structures = kernelPagingStructures; 1420 kernelPagingStructures->AddReference(); 1421 } 1422 1423 if (!apic_available()) 1424 x86_init_fpu(); 1425 // else fpu gets set up in smp code 1426 1427 #ifdef __x86_64__ 1428 // if available enable SMEP (Supervisor Memory Execution Protection) 1429 if (x86_check_feature(IA32_FEATURE_SMEP, FEATURE_7_EBX)) { 1430 if (!get_safemode_boolean(B_SAFEMODE_DISABLE_SMEP_SMAP, false)) { 1431 dprintf("enable SMEP\n"); 1432 call_all_cpus_sync(&enable_smep, NULL); 1433 } else 1434 dprintf("SMEP disabled per safemode setting\n"); 1435 } 1436 1437 // if available enable SMAP (Supervisor Memory Access Protection) 1438 if (x86_check_feature(IA32_FEATURE_SMAP, FEATURE_7_EBX)) { 1439 if (!get_safemode_boolean(B_SAFEMODE_DISABLE_SMEP_SMAP, false)) { 1440 dprintf("enable SMAP\n"); 1441 call_all_cpus_sync(&enable_smap, NULL); 1442 1443 arch_altcodepatch_replace(ALTCODEPATCH_TAG_STAC, &_stac, 3); 1444 arch_altcodepatch_replace(ALTCODEPATCH_TAG_CLAC, &_clac, 3); 1445 } else 1446 dprintf("SMAP disabled per safemode setting\n"); 1447 } 1448 #endif 1449 1450 return B_OK; 1451 } 1452 1453 1454 status_t 1455 arch_cpu_init_post_modules(kernel_args* args) 1456 { 1457 // initialize CPU module 1458 1459 void* cookie = open_module_list("cpu"); 1460 1461 while (true) { 1462 char name[B_FILE_NAME_LENGTH]; 1463 size_t nameLength = sizeof(name); 1464 1465 if (read_next_module_name(cookie, name, &nameLength) != B_OK 1466 || get_module(name, (module_info**)&sCpuModule) == B_OK) 1467 break; 1468 } 1469 1470 close_module_list(cookie); 1471 1472 // initialize MTRRs if available 1473 if (x86_count_mtrrs() > 0) { 1474 sCpuRendezvous = sCpuRendezvous2 = 0; 1475 call_all_cpus(&init_mtrrs, NULL); 1476 } 1477 1478 size_t threadExitLen = (addr_t)x86_end_userspace_thread_exit 1479 - (addr_t)x86_userspace_thread_exit; 1480 addr_t threadExitPosition = fill_commpage_entry( 1481 COMMPAGE_ENTRY_X86_THREAD_EXIT, (const void*)x86_userspace_thread_exit, 1482 threadExitLen); 1483 1484 // add the functions to the commpage image 1485 image_id image = get_commpage_image(); 1486 1487 elf_add_memory_image_symbol(image, "commpage_thread_exit", 1488 threadExitPosition, threadExitLen, B_SYMBOL_TYPE_TEXT); 1489 1490 return B_OK; 1491 } 1492 1493 1494 void 1495 arch_cpu_user_TLB_invalidate(void) 1496 { 1497 x86_write_cr3(x86_read_cr3()); 1498 } 1499 1500 1501 void 1502 arch_cpu_global_TLB_invalidate(void) 1503 { 1504 uint32 flags = x86_read_cr4(); 1505 1506 if (flags & IA32_CR4_GLOBAL_PAGES) { 1507 // disable and reenable the global pages to flush all TLBs regardless 1508 // of the global page bit 1509 x86_write_cr4(flags & ~IA32_CR4_GLOBAL_PAGES); 1510 x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES); 1511 } else { 1512 cpu_status state = disable_interrupts(); 1513 arch_cpu_user_TLB_invalidate(); 1514 restore_interrupts(state); 1515 } 1516 } 1517 1518 1519 void 1520 arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) 1521 { 1522 int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE; 1523 while (num_pages-- >= 0) { 1524 invalidate_TLB(start); 1525 start += B_PAGE_SIZE; 1526 } 1527 } 1528 1529 1530 void 1531 arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) 1532 { 1533 int i; 1534 for (i = 0; i < num_pages; i++) { 1535 invalidate_TLB(pages[i]); 1536 } 1537 } 1538 1539 1540 status_t 1541 arch_cpu_shutdown(bool rebootSystem) 1542 { 1543 if (acpi_shutdown(rebootSystem) == B_OK) 1544 return B_OK; 1545 1546 if (!rebootSystem) { 1547 #ifndef __x86_64__ 1548 return apm_shutdown(); 1549 #else 1550 return B_NOT_SUPPORTED; 1551 #endif 1552 } 1553 1554 cpu_status state = disable_interrupts(); 1555 1556 // try to reset the system using the keyboard controller 1557 out8(0xfe, 0x64); 1558 1559 // Give some time to the controller to do its job (0.5s) 1560 snooze(500000); 1561 1562 // if that didn't help, try it this way 1563 x86_reboot(); 1564 1565 restore_interrupts(state); 1566 return B_ERROR; 1567 } 1568 1569 1570 void 1571 arch_cpu_sync_icache(void* address, size_t length) 1572 { 1573 // instruction cache is always consistent on x86 1574 } 1575 1576