1 /* 2 * Copyright 2018, Jérôme Duval, jerome.duval@gmail.com. 3 * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. 4 * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. 5 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. 6 * Distributed under the terms of the MIT License. 7 * 8 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 9 * Distributed under the terms of the NewOS License. 10 */ 11 12 13 #include <cpu.h> 14 15 #include <string.h> 16 #include <stdlib.h> 17 #include <stdio.h> 18 19 #include <algorithm> 20 21 #include <ACPI.h> 22 23 #include <boot_device.h> 24 #include <commpage.h> 25 #include <debug.h> 26 #include <elf.h> 27 #include <safemode.h> 28 #include <smp.h> 29 #include <util/BitUtils.h> 30 #include <vm/vm.h> 31 #include <vm/vm_types.h> 32 #include <vm/VMAddressSpace.h> 33 34 #include <arch_system_info.h> 35 #include <arch/x86/apic.h> 36 #include <boot/kernel_args.h> 37 38 #include "paging/X86PagingStructures.h" 39 #include "paging/X86VMTranslationMap.h" 40 41 42 #define DUMP_FEATURE_STRING 1 43 #define DUMP_CPU_TOPOLOGY 1 44 45 46 /* cpu vendor info */ 47 struct cpu_vendor_info { 48 const char *vendor; 49 const char *ident_string[2]; 50 }; 51 52 static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = { 53 { "Intel", { "GenuineIntel" } }, 54 { "AMD", { "AuthenticAMD" } }, 55 { "Cyrix", { "CyrixInstead" } }, 56 { "UMC", { "UMC UMC UMC" } }, 57 { "NexGen", { "NexGenDriven" } }, 58 { "Centaur", { "CentaurHauls" } }, 59 { "Rise", { "RiseRiseRise" } }, 60 { "Transmeta", { "GenuineTMx86", "TransmetaCPU" } }, 61 { "NSC", { "Geode by NSC" } }, 62 }; 63 64 #define K8_SMIONCMPHALT (1ULL << 27) 65 #define K8_C1EONCMPHALT (1ULL << 28) 66 67 #define K8_CMPHALT (K8_SMIONCMPHALT | K8_C1EONCMPHALT) 68 69 struct set_mtrr_parameter { 70 int32 index; 71 uint64 base; 72 uint64 length; 73 uint8 type; 74 }; 75 76 struct set_mtrrs_parameter { 77 const x86_mtrr_info* infos; 78 uint32 count; 79 uint8 defaultType; 80 }; 81 82 83 #ifdef __x86_64__ 84 extern addr_t _stac; 85 extern addr_t _clac; 86 #endif 87 88 extern "C" void x86_reboot(void); 89 // from arch.S 90 91 void (*gCpuIdleFunc)(void); 92 #ifndef __x86_64__ 93 void (*gX86SwapFPUFunc)(void* oldState, const void* newState) = x86_noop_swap; 94 bool gHasSSE = false; 95 #endif 96 97 static uint32 sCpuRendezvous; 98 static uint32 sCpuRendezvous2; 99 static uint32 sCpuRendezvous3; 100 static vint32 sTSCSyncRendezvous; 101 102 /* Some specials for the double fault handler */ 103 static uint8* sDoubleFaultStacks; 104 static const size_t kDoubleFaultStackSize = 4096; // size per CPU 105 106 static x86_cpu_module_info* sCpuModule; 107 108 109 /* CPU topology information */ 110 static uint32 (*sGetCPUTopologyID)(int currentCPU); 111 static uint32 sHierarchyMask[CPU_TOPOLOGY_LEVELS]; 112 static uint32 sHierarchyShift[CPU_TOPOLOGY_LEVELS]; 113 114 /* Cache topology information */ 115 static uint32 sCacheSharingMask[CPU_MAX_CACHE_LEVEL]; 116 117 118 static status_t 119 acpi_shutdown(bool rebootSystem) 120 { 121 if (debug_debugger_running() || !are_interrupts_enabled()) 122 return B_ERROR; 123 124 acpi_module_info* acpi; 125 if (get_module(B_ACPI_MODULE_NAME, (module_info**)&acpi) != B_OK) 126 return B_NOT_SUPPORTED; 127 128 status_t status; 129 if (rebootSystem) { 130 status = acpi->reboot(); 131 } else { 132 status = acpi->prepare_sleep_state(ACPI_POWER_STATE_OFF, NULL, 0); 133 if (status == B_OK) { 134 //cpu_status state = disable_interrupts(); 135 status = acpi->enter_sleep_state(ACPI_POWER_STATE_OFF); 136 //restore_interrupts(state); 137 } 138 } 139 140 put_module(B_ACPI_MODULE_NAME); 141 return status; 142 } 143 144 145 /*! Disable CPU caches, and invalidate them. */ 146 static void 147 disable_caches() 148 { 149 x86_write_cr0((x86_read_cr0() | CR0_CACHE_DISABLE) 150 & ~CR0_NOT_WRITE_THROUGH); 151 wbinvd(); 152 arch_cpu_global_TLB_invalidate(); 153 } 154 155 156 /*! Invalidate CPU caches, and enable them. */ 157 static void 158 enable_caches() 159 { 160 wbinvd(); 161 arch_cpu_global_TLB_invalidate(); 162 x86_write_cr0(x86_read_cr0() 163 & ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH)); 164 } 165 166 167 static void 168 set_mtrr(void* _parameter, int cpu) 169 { 170 struct set_mtrr_parameter* parameter 171 = (struct set_mtrr_parameter*)_parameter; 172 173 // wait until all CPUs have arrived here 174 smp_cpu_rendezvous(&sCpuRendezvous); 175 176 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 177 // that initiated the call_all_cpus() from doing that again and clearing 178 // sCpuRendezvous2 before the last CPU has actually left the loop in 179 // smp_cpu_rendezvous(); 180 if (cpu == 0) 181 atomic_set((int32*)&sCpuRendezvous3, 0); 182 183 disable_caches(); 184 185 sCpuModule->set_mtrr(parameter->index, parameter->base, parameter->length, 186 parameter->type); 187 188 enable_caches(); 189 190 // wait until all CPUs have arrived here 191 smp_cpu_rendezvous(&sCpuRendezvous2); 192 smp_cpu_rendezvous(&sCpuRendezvous3); 193 } 194 195 196 static void 197 set_mtrrs(void* _parameter, int cpu) 198 { 199 set_mtrrs_parameter* parameter = (set_mtrrs_parameter*)_parameter; 200 201 // wait until all CPUs have arrived here 202 smp_cpu_rendezvous(&sCpuRendezvous); 203 204 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 205 // that initiated the call_all_cpus() from doing that again and clearing 206 // sCpuRendezvous2 before the last CPU has actually left the loop in 207 // smp_cpu_rendezvous(); 208 if (cpu == 0) 209 atomic_set((int32*)&sCpuRendezvous3, 0); 210 211 disable_caches(); 212 213 sCpuModule->set_mtrrs(parameter->defaultType, parameter->infos, 214 parameter->count); 215 216 enable_caches(); 217 218 // wait until all CPUs have arrived here 219 smp_cpu_rendezvous(&sCpuRendezvous2); 220 smp_cpu_rendezvous(&sCpuRendezvous3); 221 } 222 223 224 static void 225 init_mtrrs(void* _unused, int cpu) 226 { 227 // wait until all CPUs have arrived here 228 smp_cpu_rendezvous(&sCpuRendezvous); 229 230 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 231 // that initiated the call_all_cpus() from doing that again and clearing 232 // sCpuRendezvous2 before the last CPU has actually left the loop in 233 // smp_cpu_rendezvous(); 234 if (cpu == 0) 235 atomic_set((int32*)&sCpuRendezvous3, 0); 236 237 disable_caches(); 238 239 sCpuModule->init_mtrrs(); 240 241 enable_caches(); 242 243 // wait until all CPUs have arrived here 244 smp_cpu_rendezvous(&sCpuRendezvous2); 245 smp_cpu_rendezvous(&sCpuRendezvous3); 246 } 247 248 249 uint32 250 x86_count_mtrrs(void) 251 { 252 if (sCpuModule == NULL) 253 return 0; 254 255 return sCpuModule->count_mtrrs(); 256 } 257 258 259 void 260 x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type) 261 { 262 struct set_mtrr_parameter parameter; 263 parameter.index = index; 264 parameter.base = base; 265 parameter.length = length; 266 parameter.type = type; 267 268 sCpuRendezvous = sCpuRendezvous2 = 0; 269 call_all_cpus(&set_mtrr, ¶meter); 270 } 271 272 273 status_t 274 x86_get_mtrr(uint32 index, uint64* _base, uint64* _length, uint8* _type) 275 { 276 // the MTRRs are identical on all CPUs, so it doesn't matter 277 // on which CPU this runs 278 return sCpuModule->get_mtrr(index, _base, _length, _type); 279 } 280 281 282 void 283 x86_set_mtrrs(uint8 defaultType, const x86_mtrr_info* infos, uint32 count) 284 { 285 if (sCpuModule == NULL) 286 return; 287 288 struct set_mtrrs_parameter parameter; 289 parameter.defaultType = defaultType; 290 parameter.infos = infos; 291 parameter.count = count; 292 293 sCpuRendezvous = sCpuRendezvous2 = 0; 294 call_all_cpus(&set_mtrrs, ¶meter); 295 } 296 297 298 void 299 x86_init_fpu(void) 300 { 301 // All x86_64 CPUs support SSE, don't need to bother checking for it. 302 #ifndef __x86_64__ 303 if (!x86_check_feature(IA32_FEATURE_FPU, FEATURE_COMMON)) { 304 // No FPU... time to install one in your 386? 305 dprintf("%s: Warning: CPU has no reported FPU.\n", __func__); 306 gX86SwapFPUFunc = x86_noop_swap; 307 return; 308 } 309 310 if (!x86_check_feature(IA32_FEATURE_SSE, FEATURE_COMMON) 311 || !x86_check_feature(IA32_FEATURE_FXSR, FEATURE_COMMON)) { 312 dprintf("%s: CPU has no SSE... just enabling FPU.\n", __func__); 313 // we don't have proper SSE support, just enable FPU 314 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 315 gX86SwapFPUFunc = x86_fnsave_swap; 316 return; 317 } 318 #endif 319 320 dprintf("%s: CPU has SSE... enabling FXSR and XMM.\n", __func__); 321 #ifndef __x86_64__ 322 // enable OS support for SSE 323 x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION); 324 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 325 326 gX86SwapFPUFunc = x86_fxsave_swap; 327 gHasSSE = true; 328 #endif 329 } 330 331 332 #if DUMP_FEATURE_STRING 333 static void 334 dump_feature_string(int currentCPU, cpu_ent* cpu) 335 { 336 char features[512]; 337 features[0] = 0; 338 339 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FPU) 340 strlcat(features, "fpu ", sizeof(features)); 341 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_VME) 342 strlcat(features, "vme ", sizeof(features)); 343 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DE) 344 strlcat(features, "de ", sizeof(features)); 345 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE) 346 strlcat(features, "pse ", sizeof(features)); 347 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TSC) 348 strlcat(features, "tsc ", sizeof(features)); 349 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MSR) 350 strlcat(features, "msr ", sizeof(features)); 351 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAE) 352 strlcat(features, "pae ", sizeof(features)); 353 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCE) 354 strlcat(features, "mce ", sizeof(features)); 355 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CX8) 356 strlcat(features, "cx8 ", sizeof(features)); 357 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_APIC) 358 strlcat(features, "apic ", sizeof(features)); 359 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SEP) 360 strlcat(features, "sep ", sizeof(features)); 361 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MTRR) 362 strlcat(features, "mtrr ", sizeof(features)); 363 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PGE) 364 strlcat(features, "pge ", sizeof(features)); 365 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCA) 366 strlcat(features, "mca ", sizeof(features)); 367 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CMOV) 368 strlcat(features, "cmov ", sizeof(features)); 369 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAT) 370 strlcat(features, "pat ", sizeof(features)); 371 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE36) 372 strlcat(features, "pse36 ", sizeof(features)); 373 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSN) 374 strlcat(features, "psn ", sizeof(features)); 375 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CLFSH) 376 strlcat(features, "clfsh ", sizeof(features)); 377 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DS) 378 strlcat(features, "ds ", sizeof(features)); 379 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_ACPI) 380 strlcat(features, "acpi ", sizeof(features)); 381 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MMX) 382 strlcat(features, "mmx ", sizeof(features)); 383 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FXSR) 384 strlcat(features, "fxsr ", sizeof(features)); 385 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE) 386 strlcat(features, "sse ", sizeof(features)); 387 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE2) 388 strlcat(features, "sse2 ", sizeof(features)); 389 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SS) 390 strlcat(features, "ss ", sizeof(features)); 391 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_HTT) 392 strlcat(features, "htt ", sizeof(features)); 393 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TM) 394 strlcat(features, "tm ", sizeof(features)); 395 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PBE) 396 strlcat(features, "pbe ", sizeof(features)); 397 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE3) 398 strlcat(features, "sse3 ", sizeof(features)); 399 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCLMULQDQ) 400 strlcat(features, "pclmulqdq ", sizeof(features)); 401 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DTES64) 402 strlcat(features, "dtes64 ", sizeof(features)); 403 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MONITOR) 404 strlcat(features, "monitor ", sizeof(features)); 405 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DSCPL) 406 strlcat(features, "dscpl ", sizeof(features)); 407 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_VMX) 408 strlcat(features, "vmx ", sizeof(features)); 409 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SMX) 410 strlcat(features, "smx ", sizeof(features)); 411 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_EST) 412 strlcat(features, "est ", sizeof(features)); 413 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TM2) 414 strlcat(features, "tm2 ", sizeof(features)); 415 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSSE3) 416 strlcat(features, "ssse3 ", sizeof(features)); 417 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CNXTID) 418 strlcat(features, "cnxtid ", sizeof(features)); 419 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_FMA) 420 strlcat(features, "fma ", sizeof(features)); 421 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CX16) 422 strlcat(features, "cx16 ", sizeof(features)); 423 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XTPR) 424 strlcat(features, "xtpr ", sizeof(features)); 425 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PDCM) 426 strlcat(features, "pdcm ", sizeof(features)); 427 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCID) 428 strlcat(features, "pcid ", sizeof(features)); 429 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DCA) 430 strlcat(features, "dca ", sizeof(features)); 431 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_1) 432 strlcat(features, "sse4_1 ", sizeof(features)); 433 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_2) 434 strlcat(features, "sse4_2 ", sizeof(features)); 435 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_X2APIC) 436 strlcat(features, "x2apic ", sizeof(features)); 437 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MOVBE) 438 strlcat(features, "movbe ", sizeof(features)); 439 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_POPCNT) 440 strlcat(features, "popcnt ", sizeof(features)); 441 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TSCDEADLINE) 442 strlcat(features, "tscdeadline ", sizeof(features)); 443 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AES) 444 strlcat(features, "aes ", sizeof(features)); 445 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XSAVE) 446 strlcat(features, "xsave ", sizeof(features)); 447 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_OSXSAVE) 448 strlcat(features, "osxsave ", sizeof(features)); 449 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AVX) 450 strlcat(features, "avx ", sizeof(features)); 451 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_F16C) 452 strlcat(features, "f16c ", sizeof(features)); 453 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_RDRND) 454 strlcat(features, "rdrnd ", sizeof(features)); 455 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) 456 strlcat(features, "hypervisor ", sizeof(features)); 457 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_SYSCALL) 458 strlcat(features, "syscall ", sizeof(features)); 459 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_NX) 460 strlcat(features, "nx ", sizeof(features)); 461 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_MMXEXT) 462 strlcat(features, "mmxext ", sizeof(features)); 463 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_FFXSR) 464 strlcat(features, "ffxsr ", sizeof(features)); 465 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_LONG) 466 strlcat(features, "long ", sizeof(features)); 467 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOWEXT) 468 strlcat(features, "3dnowext ", sizeof(features)); 469 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOW) 470 strlcat(features, "3dnow ", sizeof(features)); 471 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_DTS) 472 strlcat(features, "dts ", sizeof(features)); 473 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ITB) 474 strlcat(features, "itb ", sizeof(features)); 475 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ARAT) 476 strlcat(features, "arat ", sizeof(features)); 477 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PLN) 478 strlcat(features, "pln ", sizeof(features)); 479 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ECMD) 480 strlcat(features, "ecmd ", sizeof(features)); 481 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PTM) 482 strlcat(features, "ptm ", sizeof(features)); 483 if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_APERFMPERF) 484 strlcat(features, "aperfmperf ", sizeof(features)); 485 if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_EPB) 486 strlcat(features, "epb ", sizeof(features)); 487 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_TSC_ADJUST) 488 strlcat(features, "tsc_adjust ", sizeof(features)); 489 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SGX) 490 strlcat(features, "sgx ", sizeof(features)); 491 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_BMI1) 492 strlcat(features, "bmi1 ", sizeof(features)); 493 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_HLE) 494 strlcat(features, "hle ", sizeof(features)); 495 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX2) 496 strlcat(features, "avx2 ", sizeof(features)); 497 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SMEP) 498 strlcat(features, "smep ", sizeof(features)); 499 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_BMI2) 500 strlcat(features, "bmi2 ", sizeof(features)); 501 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_ERMS) 502 strlcat(features, "erms ", sizeof(features)); 503 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_INVPCID) 504 strlcat(features, "invpcid ", sizeof(features)); 505 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_RTM) 506 strlcat(features, "rtm ", sizeof(features)); 507 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_CQM) 508 strlcat(features, "cqm ", sizeof(features)); 509 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_MPX) 510 strlcat(features, "mpx ", sizeof(features)); 511 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_RDT_A) 512 strlcat(features, "rdt_a ", sizeof(features)); 513 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512F) 514 strlcat(features, "avx512f ", sizeof(features)); 515 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512DQ) 516 strlcat(features, "avx512dq ", sizeof(features)); 517 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_RDSEED) 518 strlcat(features, "rdseed ", sizeof(features)); 519 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_ADX) 520 strlcat(features, "adx ", sizeof(features)); 521 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SMAP) 522 strlcat(features, "smap ", sizeof(features)); 523 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512IFMA) 524 strlcat(features, "avx512ifma ", sizeof(features)); 525 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_PCOMMIT) 526 strlcat(features, "pcommit ", sizeof(features)); 527 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_CLFLUSHOPT) 528 strlcat(features, "cflushopt ", sizeof(features)); 529 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_CLWB) 530 strlcat(features, "clwb ", sizeof(features)); 531 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_INTEL_PT) 532 strlcat(features, "intel_pt ", sizeof(features)); 533 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512PF) 534 strlcat(features, "avx512pf ", sizeof(features)); 535 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512ER) 536 strlcat(features, "avx512er ", sizeof(features)); 537 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512CD) 538 strlcat(features, "avx512cd ", sizeof(features)); 539 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SHA_NI) 540 strlcat(features, "sha_ni ", sizeof(features)); 541 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512BW) 542 strlcat(features, "avx512bw ", sizeof(features)); 543 if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512VI) 544 strlcat(features, "avx512vi ", sizeof(features)); 545 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_IBRS) 546 strlcat(features, "ibrs ", sizeof(features)); 547 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_STIBP) 548 strlcat(features, "stibp ", sizeof(features)); 549 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_L1D_FLUSH) 550 strlcat(features, "l1d_flush ", sizeof(features)); 551 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_ARCH_CAPABILITIES) 552 strlcat(features, "msr_arch ", sizeof(features)); 553 if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_SSBD) 554 strlcat(features, "ssbd ", sizeof(features)); 555 if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_AMD_EXT_IBPB) 556 strlcat(features, "ibpb ", sizeof(features)); 557 dprintf("CPU %d: features: %s\n", currentCPU, features); 558 } 559 #endif // DUMP_FEATURE_STRING 560 561 562 static void 563 compute_cpu_hierarchy_masks(int maxLogicalID, int maxCoreID) 564 { 565 ASSERT(maxLogicalID >= maxCoreID); 566 const int kMaxSMTID = maxLogicalID / maxCoreID; 567 568 sHierarchyMask[CPU_TOPOLOGY_SMT] = kMaxSMTID - 1; 569 sHierarchyShift[CPU_TOPOLOGY_SMT] = 0; 570 571 sHierarchyMask[CPU_TOPOLOGY_CORE] = (maxCoreID - 1) * kMaxSMTID; 572 sHierarchyShift[CPU_TOPOLOGY_CORE] 573 = count_set_bits(sHierarchyMask[CPU_TOPOLOGY_SMT]); 574 575 const uint32 kSinglePackageMask = sHierarchyMask[CPU_TOPOLOGY_SMT] 576 | sHierarchyMask[CPU_TOPOLOGY_CORE]; 577 sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~kSinglePackageMask; 578 sHierarchyShift[CPU_TOPOLOGY_PACKAGE] = count_set_bits(kSinglePackageMask); 579 } 580 581 582 static uint32 583 get_cpu_legacy_initial_apic_id(int /* currentCPU */) 584 { 585 cpuid_info cpuid; 586 get_current_cpuid(&cpuid, 1, 0); 587 return cpuid.regs.ebx >> 24; 588 } 589 590 591 static inline status_t 592 detect_amd_cpu_topology(uint32 maxBasicLeaf, uint32 maxExtendedLeaf) 593 { 594 sGetCPUTopologyID = get_cpu_legacy_initial_apic_id; 595 596 cpuid_info cpuid; 597 get_current_cpuid(&cpuid, 1, 0); 598 int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff); 599 600 int maxCoreID = 1; 601 if (maxExtendedLeaf >= 0x80000008) { 602 get_current_cpuid(&cpuid, 0x80000008, 0); 603 maxCoreID = (cpuid.regs.ecx >> 12) & 0xf; 604 if (maxCoreID != 0) 605 maxCoreID = 1 << maxCoreID; 606 else 607 maxCoreID = next_power_of_2((cpuid.regs.edx & 0xf) + 1); 608 } 609 610 if (maxExtendedLeaf >= 0x80000001) { 611 get_current_cpuid(&cpuid, 0x80000001, 0); 612 if (x86_check_feature(IA32_FEATURE_AMD_EXT_CMPLEGACY, 613 FEATURE_EXT_AMD_ECX)) 614 maxCoreID = maxLogicalID; 615 } 616 617 compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID); 618 619 return B_OK; 620 } 621 622 623 static void 624 detect_amd_cache_topology(uint32 maxExtendedLeaf) 625 { 626 if (!x86_check_feature(IA32_FEATURE_AMD_EXT_TOPOLOGY, FEATURE_EXT_AMD_ECX)) 627 return; 628 629 if (maxExtendedLeaf < 0x8000001d) 630 return; 631 632 uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL]; 633 int maxCacheLevel = 0; 634 635 int currentLevel = 0; 636 int cacheType; 637 do { 638 cpuid_info cpuid; 639 get_current_cpuid(&cpuid, 0x8000001d, currentLevel); 640 641 cacheType = cpuid.regs.eax & 0x1f; 642 if (cacheType == 0) 643 break; 644 645 int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; 646 int coresCount = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1); 647 hierarchyLevels[cacheLevel - 1] 648 = coresCount * (sHierarchyMask[CPU_TOPOLOGY_SMT] + 1); 649 maxCacheLevel = std::max(maxCacheLevel, cacheLevel); 650 651 currentLevel++; 652 } while (true); 653 654 for (int i = 0; i < maxCacheLevel; i++) 655 sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); 656 gCPUCacheLevelCount = maxCacheLevel; 657 } 658 659 660 static uint32 661 get_intel_cpu_initial_x2apic_id(int /* currentCPU */) 662 { 663 cpuid_info cpuid; 664 get_current_cpuid(&cpuid, 11, 0); 665 return cpuid.regs.edx; 666 } 667 668 669 static inline status_t 670 detect_intel_cpu_topology_x2apic(uint32 maxBasicLeaf) 671 { 672 if (maxBasicLeaf < 11) 673 return B_UNSUPPORTED; 674 675 uint8 hierarchyLevels[CPU_TOPOLOGY_LEVELS] = { 0 }; 676 677 int currentLevel = 0; 678 int levelType; 679 unsigned int levelsSet = 0; 680 681 do { 682 cpuid_info cpuid; 683 get_current_cpuid(&cpuid, 11, currentLevel); 684 if (currentLevel == 0 && cpuid.regs.ebx == 0) 685 return B_UNSUPPORTED; 686 687 levelType = (cpuid.regs.ecx >> 8) & 0xff; 688 int levelValue = cpuid.regs.eax & 0x1f; 689 690 switch (levelType) { 691 case 1: // SMT 692 hierarchyLevels[CPU_TOPOLOGY_SMT] = levelValue; 693 levelsSet |= 1; 694 break; 695 case 2: // core 696 hierarchyLevels[CPU_TOPOLOGY_CORE] = levelValue; 697 levelsSet |= 2; 698 break; 699 } 700 701 currentLevel++; 702 } while (levelType != 0 && levelsSet != 3); 703 704 sGetCPUTopologyID = get_intel_cpu_initial_x2apic_id; 705 706 for (int i = 1; i < CPU_TOPOLOGY_LEVELS; i++) { 707 if ((levelsSet & (1u << i)) != 0) 708 continue; 709 hierarchyLevels[i] = hierarchyLevels[i - 1]; 710 } 711 712 for (int i = 0; i < CPU_TOPOLOGY_LEVELS; i++) { 713 uint32 mask = ~uint32(0); 714 if (i < CPU_TOPOLOGY_LEVELS - 1) 715 mask = (1u << hierarchyLevels[i]) - 1; 716 if (i > 0) 717 mask &= ~sHierarchyMask[i - 1]; 718 sHierarchyMask[i] = mask; 719 sHierarchyShift[i] = i > 0 ? hierarchyLevels[i - 1] : 0; 720 } 721 722 return B_OK; 723 } 724 725 726 static inline status_t 727 detect_intel_cpu_topology_legacy(uint32 maxBasicLeaf) 728 { 729 sGetCPUTopologyID = get_cpu_legacy_initial_apic_id; 730 731 cpuid_info cpuid; 732 733 get_current_cpuid(&cpuid, 1, 0); 734 int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff); 735 736 int maxCoreID = 1; 737 if (maxBasicLeaf >= 4) { 738 get_current_cpuid(&cpuid, 4, 0); 739 maxCoreID = next_power_of_2((cpuid.regs.eax >> 26) + 1); 740 } 741 742 compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID); 743 744 return B_OK; 745 } 746 747 748 static void 749 detect_intel_cache_topology(uint32 maxBasicLeaf) 750 { 751 if (maxBasicLeaf < 4) 752 return; 753 754 uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL]; 755 int maxCacheLevel = 0; 756 757 int currentLevel = 0; 758 int cacheType; 759 do { 760 cpuid_info cpuid; 761 get_current_cpuid(&cpuid, 4, currentLevel); 762 763 cacheType = cpuid.regs.eax & 0x1f; 764 if (cacheType == 0) 765 break; 766 767 int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; 768 hierarchyLevels[cacheLevel - 1] 769 = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1); 770 maxCacheLevel = std::max(maxCacheLevel, cacheLevel); 771 772 currentLevel++; 773 } while (true); 774 775 for (int i = 0; i < maxCacheLevel; i++) 776 sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); 777 778 gCPUCacheLevelCount = maxCacheLevel; 779 } 780 781 782 static uint32 783 get_simple_cpu_topology_id(int currentCPU) 784 { 785 return currentCPU; 786 } 787 788 789 static inline int 790 get_topology_level_id(uint32 id, cpu_topology_level level) 791 { 792 ASSERT(level < CPU_TOPOLOGY_LEVELS); 793 return (id & sHierarchyMask[level]) >> sHierarchyShift[level]; 794 } 795 796 797 static void 798 detect_cpu_topology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, 799 uint32 maxExtendedLeaf) 800 { 801 if (currentCPU == 0) { 802 memset(sCacheSharingMask, 0xff, sizeof(sCacheSharingMask)); 803 804 status_t result = B_UNSUPPORTED; 805 if (x86_check_feature(IA32_FEATURE_HTT, FEATURE_COMMON)) { 806 if (cpu->arch.vendor == VENDOR_AMD) { 807 result = detect_amd_cpu_topology(maxBasicLeaf, maxExtendedLeaf); 808 809 if (result == B_OK) 810 detect_amd_cache_topology(maxExtendedLeaf); 811 } 812 813 if (cpu->arch.vendor == VENDOR_INTEL) { 814 result = detect_intel_cpu_topology_x2apic(maxBasicLeaf); 815 if (result != B_OK) 816 result = detect_intel_cpu_topology_legacy(maxBasicLeaf); 817 818 if (result == B_OK) 819 detect_intel_cache_topology(maxBasicLeaf); 820 } 821 } 822 823 if (result != B_OK) { 824 dprintf("No CPU topology information available.\n"); 825 826 sGetCPUTopologyID = get_simple_cpu_topology_id; 827 828 sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~uint32(0); 829 } 830 } 831 832 ASSERT(sGetCPUTopologyID != NULL); 833 int topologyID = sGetCPUTopologyID(currentCPU); 834 cpu->topology_id[CPU_TOPOLOGY_SMT] 835 = get_topology_level_id(topologyID, CPU_TOPOLOGY_SMT); 836 cpu->topology_id[CPU_TOPOLOGY_CORE] 837 = get_topology_level_id(topologyID, CPU_TOPOLOGY_CORE); 838 cpu->topology_id[CPU_TOPOLOGY_PACKAGE] 839 = get_topology_level_id(topologyID, CPU_TOPOLOGY_PACKAGE); 840 841 unsigned int i; 842 for (i = 0; i < gCPUCacheLevelCount; i++) 843 cpu->cache_id[i] = topologyID & sCacheSharingMask[i]; 844 for (; i < CPU_MAX_CACHE_LEVEL; i++) 845 cpu->cache_id[i] = -1; 846 847 #if DUMP_CPU_TOPOLOGY 848 dprintf("CPU %d: apic id %d, package %d, core %d, smt %d\n", currentCPU, 849 topologyID, cpu->topology_id[CPU_TOPOLOGY_PACKAGE], 850 cpu->topology_id[CPU_TOPOLOGY_CORE], 851 cpu->topology_id[CPU_TOPOLOGY_SMT]); 852 853 if (gCPUCacheLevelCount > 0) { 854 char cacheLevels[256]; 855 unsigned int offset = 0; 856 for (i = 0; i < gCPUCacheLevelCount; i++) { 857 offset += snprintf(cacheLevels + offset, 858 sizeof(cacheLevels) - offset, 859 " L%d id %d%s", i + 1, cpu->cache_id[i], 860 i < gCPUCacheLevelCount - 1 ? "," : ""); 861 862 if (offset >= sizeof(cacheLevels)) 863 break; 864 } 865 866 dprintf("CPU %d: cache sharing:%s\n", currentCPU, cacheLevels); 867 } 868 #endif 869 } 870 871 872 static void 873 detect_cpu(int currentCPU) 874 { 875 cpu_ent* cpu = get_cpu_struct(); 876 char vendorString[17]; 877 cpuid_info cpuid; 878 879 // clear out the cpu info data 880 cpu->arch.vendor = VENDOR_UNKNOWN; 881 cpu->arch.vendor_name = "UNKNOWN VENDOR"; 882 cpu->arch.feature[FEATURE_COMMON] = 0; 883 cpu->arch.feature[FEATURE_EXT] = 0; 884 cpu->arch.feature[FEATURE_EXT_AMD] = 0; 885 cpu->arch.feature[FEATURE_7_EBX] = 0; 886 cpu->arch.feature[FEATURE_7_ECX] = 0; 887 cpu->arch.feature[FEATURE_7_EDX] = 0; 888 cpu->arch.model_name[0] = 0; 889 890 // print some fun data 891 get_current_cpuid(&cpuid, 0, 0); 892 uint32 maxBasicLeaf = cpuid.eax_0.max_eax; 893 894 // build the vendor string 895 memset(vendorString, 0, sizeof(vendorString)); 896 memcpy(vendorString, cpuid.eax_0.vendor_id, sizeof(cpuid.eax_0.vendor_id)); 897 898 // get the family, model, stepping 899 get_current_cpuid(&cpuid, 1, 0); 900 cpu->arch.type = cpuid.eax_1.type; 901 cpu->arch.family = cpuid.eax_1.family; 902 cpu->arch.extended_family = cpuid.eax_1.extended_family; 903 cpu->arch.model = cpuid.eax_1.model; 904 cpu->arch.extended_model = cpuid.eax_1.extended_model; 905 cpu->arch.stepping = cpuid.eax_1.stepping; 906 dprintf("CPU %d: type %d family %d extended_family %d model %d " 907 "extended_model %d stepping %d, string '%s'\n", 908 currentCPU, cpu->arch.type, cpu->arch.family, 909 cpu->arch.extended_family, cpu->arch.model, 910 cpu->arch.extended_model, cpu->arch.stepping, vendorString); 911 912 // figure out what vendor we have here 913 914 for (int32 i = 0; i < VENDOR_NUM; i++) { 915 if (vendor_info[i].ident_string[0] 916 && !strcmp(vendorString, vendor_info[i].ident_string[0])) { 917 cpu->arch.vendor = (x86_vendors)i; 918 cpu->arch.vendor_name = vendor_info[i].vendor; 919 break; 920 } 921 if (vendor_info[i].ident_string[1] 922 && !strcmp(vendorString, vendor_info[i].ident_string[1])) { 923 cpu->arch.vendor = (x86_vendors)i; 924 cpu->arch.vendor_name = vendor_info[i].vendor; 925 break; 926 } 927 } 928 929 // see if we can get the model name 930 get_current_cpuid(&cpuid, 0x80000000, 0); 931 uint32 maxExtendedLeaf = cpuid.eax_0.max_eax; 932 if (maxExtendedLeaf >= 0x80000004) { 933 // build the model string (need to swap ecx/edx data before copying) 934 unsigned int temp; 935 memset(cpu->arch.model_name, 0, sizeof(cpu->arch.model_name)); 936 937 get_current_cpuid(&cpuid, 0x80000002, 0); 938 temp = cpuid.regs.edx; 939 cpuid.regs.edx = cpuid.regs.ecx; 940 cpuid.regs.ecx = temp; 941 memcpy(cpu->arch.model_name, cpuid.as_chars, sizeof(cpuid.as_chars)); 942 943 get_current_cpuid(&cpuid, 0x80000003, 0); 944 temp = cpuid.regs.edx; 945 cpuid.regs.edx = cpuid.regs.ecx; 946 cpuid.regs.ecx = temp; 947 memcpy(cpu->arch.model_name + 16, cpuid.as_chars, 948 sizeof(cpuid.as_chars)); 949 950 get_current_cpuid(&cpuid, 0x80000004, 0); 951 temp = cpuid.regs.edx; 952 cpuid.regs.edx = cpuid.regs.ecx; 953 cpuid.regs.ecx = temp; 954 memcpy(cpu->arch.model_name + 32, cpuid.as_chars, 955 sizeof(cpuid.as_chars)); 956 957 // some cpus return a right-justified string 958 int32 i = 0; 959 while (cpu->arch.model_name[i] == ' ') 960 i++; 961 if (i > 0) { 962 memmove(cpu->arch.model_name, &cpu->arch.model_name[i], 963 strlen(&cpu->arch.model_name[i]) + 1); 964 } 965 966 dprintf("CPU %d: vendor '%s' model name '%s'\n", 967 currentCPU, cpu->arch.vendor_name, cpu->arch.model_name); 968 } else { 969 strlcpy(cpu->arch.model_name, "unknown", sizeof(cpu->arch.model_name)); 970 } 971 972 // load feature bits 973 get_current_cpuid(&cpuid, 1, 0); 974 cpu->arch.feature[FEATURE_COMMON] = cpuid.eax_1.features; // edx 975 cpu->arch.feature[FEATURE_EXT] = cpuid.eax_1.extended_features; // ecx 976 977 if (maxExtendedLeaf >= 0x80000001) { 978 get_current_cpuid(&cpuid, 0x80000001, 0); 979 if (cpu->arch.vendor == VENDOR_AMD) 980 cpu->arch.feature[FEATURE_EXT_AMD_ECX] = cpuid.regs.ecx; // ecx 981 cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx 982 if (cpu->arch.vendor != VENDOR_AMD) 983 cpu->arch.feature[FEATURE_EXT_AMD] &= IA32_FEATURES_INTEL_EXT; 984 } 985 986 if (maxBasicLeaf >= 5) { 987 get_current_cpuid(&cpuid, 5, 0); 988 cpu->arch.feature[FEATURE_5_ECX] = cpuid.regs.ecx; 989 } 990 991 if (maxBasicLeaf >= 6) { 992 get_current_cpuid(&cpuid, 6, 0); 993 cpu->arch.feature[FEATURE_6_EAX] = cpuid.regs.eax; 994 cpu->arch.feature[FEATURE_6_ECX] = cpuid.regs.ecx; 995 } 996 997 if (maxBasicLeaf >= 7) { 998 get_current_cpuid(&cpuid, 7, 0); 999 cpu->arch.feature[FEATURE_7_EBX] = cpuid.regs.ebx; 1000 cpu->arch.feature[FEATURE_7_ECX] = cpuid.regs.ecx; 1001 cpu->arch.feature[FEATURE_7_EDX] = cpuid.regs.edx; 1002 } 1003 1004 if (maxExtendedLeaf >= 0x80000007) { 1005 get_current_cpuid(&cpuid, 0x80000007, 0); 1006 cpu->arch.feature[FEATURE_EXT_7_EDX] = cpuid.regs.edx; 1007 } 1008 1009 if (maxExtendedLeaf >= 0x80000008) { 1010 get_current_cpuid(&cpuid, 0x80000008, 0); 1011 cpu->arch.feature[FEATURE_EXT_8_EBX] = cpuid.regs.ebx; 1012 } 1013 1014 detect_cpu_topology(currentCPU, cpu, maxBasicLeaf, maxExtendedLeaf); 1015 1016 #if DUMP_FEATURE_STRING 1017 dump_feature_string(currentCPU, cpu); 1018 #endif 1019 } 1020 1021 1022 bool 1023 x86_check_feature(uint32 feature, enum x86_feature_type type) 1024 { 1025 cpu_ent* cpu = get_cpu_struct(); 1026 1027 #if 0 1028 int i; 1029 dprintf("x86_check_feature: feature 0x%x, type %d\n", feature, type); 1030 for (i = 0; i < FEATURE_NUM; i++) { 1031 dprintf("features %d: 0x%x\n", i, cpu->arch.feature[i]); 1032 } 1033 #endif 1034 1035 return (cpu->arch.feature[type] & feature) != 0; 1036 } 1037 1038 1039 void* 1040 x86_get_double_fault_stack(int32 cpu, size_t* _size) 1041 { 1042 *_size = kDoubleFaultStackSize; 1043 return sDoubleFaultStacks + kDoubleFaultStackSize * cpu; 1044 } 1045 1046 1047 /*! Returns the index of the current CPU. Can only be called from the double 1048 fault handler. 1049 */ 1050 int32 1051 x86_double_fault_get_cpu(void) 1052 { 1053 addr_t stack = x86_get_stack_frame(); 1054 return (stack - (addr_t)sDoubleFaultStacks) / kDoubleFaultStackSize; 1055 } 1056 1057 1058 // #pragma mark - 1059 1060 1061 status_t 1062 arch_cpu_preboot_init_percpu(kernel_args* args, int cpu) 1063 { 1064 // On SMP system we want to synchronize the CPUs' TSCs, so system_time() 1065 // will return consistent values. 1066 if (smp_get_num_cpus() > 1) { 1067 // let the first CPU prepare the rendezvous point 1068 if (cpu == 0) 1069 sTSCSyncRendezvous = smp_get_num_cpus() - 1; 1070 1071 // One CPU after the other will drop out of this loop and be caught by 1072 // the loop below, until the last CPU (0) gets there. Save for +/- a few 1073 // cycles the CPUs should pass the second loop at the same time. 1074 while (sTSCSyncRendezvous != cpu) { 1075 } 1076 1077 sTSCSyncRendezvous = cpu - 1; 1078 1079 while (sTSCSyncRendezvous != -1) { 1080 } 1081 1082 // reset TSC to 0 1083 x86_write_msr(IA32_MSR_TSC, 0); 1084 } 1085 1086 x86_descriptors_preboot_init_percpu(args, cpu); 1087 1088 return B_OK; 1089 } 1090 1091 1092 static void 1093 halt_idle(void) 1094 { 1095 asm("hlt"); 1096 } 1097 1098 1099 static void 1100 amdc1e_noarat_idle(void) 1101 { 1102 uint64 msr = x86_read_msr(K8_MSR_IPM); 1103 if (msr & K8_CMPHALT) 1104 x86_write_msr(K8_MSR_IPM, msr & ~K8_CMPHALT); 1105 halt_idle(); 1106 } 1107 1108 1109 static bool 1110 detect_amdc1e_noarat() 1111 { 1112 cpu_ent* cpu = get_cpu_struct(); 1113 1114 if (cpu->arch.vendor != VENDOR_AMD) 1115 return false; 1116 1117 // Family 0x12 and higher processors support ARAT 1118 // Family lower than 0xf processors doesn't support C1E 1119 // Family 0xf with model <= 0x40 procssors doesn't support C1E 1120 uint32 family = cpu->arch.family + cpu->arch.extended_family; 1121 uint32 model = (cpu->arch.extended_model << 4) | cpu->arch.model; 1122 return (family < 0x12 && family > 0xf) || (family == 0xf && model > 0x40); 1123 } 1124 1125 1126 status_t 1127 arch_cpu_init_percpu(kernel_args* args, int cpu) 1128 { 1129 detect_cpu(cpu); 1130 1131 if (!gCpuIdleFunc) { 1132 if (detect_amdc1e_noarat()) 1133 gCpuIdleFunc = amdc1e_noarat_idle; 1134 else 1135 gCpuIdleFunc = halt_idle; 1136 } 1137 1138 return B_OK; 1139 } 1140 1141 1142 status_t 1143 arch_cpu_init(kernel_args* args) 1144 { 1145 // init the TSC -> system_time() conversion factors 1146 1147 uint32 conversionFactor = args->arch_args.system_time_cv_factor; 1148 uint64 conversionFactorNsecs = (uint64)conversionFactor * 1000; 1149 1150 #ifdef __x86_64__ 1151 // The x86_64 system_time() implementation uses 64-bit multiplication and 1152 // therefore shifting is not necessary for low frequencies (it's also not 1153 // too likely that there'll be any x86_64 CPUs clocked under 1GHz). 1154 __x86_setup_system_time((uint64)conversionFactor << 32, 1155 conversionFactorNsecs); 1156 #else 1157 if (conversionFactorNsecs >> 32 != 0) { 1158 // the TSC frequency is < 1 GHz, which forces us to shift the factor 1159 __x86_setup_system_time(conversionFactor, conversionFactorNsecs >> 16, 1160 true); 1161 } else { 1162 // the TSC frequency is >= 1 GHz 1163 __x86_setup_system_time(conversionFactor, conversionFactorNsecs, false); 1164 } 1165 #endif 1166 1167 // Initialize descriptor tables. 1168 x86_descriptors_init(args); 1169 1170 return B_OK; 1171 } 1172 1173 1174 #ifdef __x86_64__ 1175 static void 1176 enable_smap(void* dummy, int cpu) 1177 { 1178 x86_write_cr4(x86_read_cr4() | IA32_CR4_SMAP); 1179 } 1180 1181 1182 static void 1183 enable_smep(void* dummy, int cpu) 1184 { 1185 x86_write_cr4(x86_read_cr4() | IA32_CR4_SMEP); 1186 } 1187 #endif 1188 1189 1190 status_t 1191 arch_cpu_init_post_vm(kernel_args* args) 1192 { 1193 uint32 i; 1194 1195 // allocate an area for the double fault stacks 1196 virtual_address_restrictions virtualRestrictions = {}; 1197 virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS; 1198 physical_address_restrictions physicalRestrictions = {}; 1199 create_area_etc(B_SYSTEM_TEAM, "double fault stacks", 1200 kDoubleFaultStackSize * smp_get_num_cpus(), B_FULL_LOCK, 1201 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0, 1202 &virtualRestrictions, &physicalRestrictions, 1203 (void**)&sDoubleFaultStacks); 1204 1205 X86PagingStructures* kernelPagingStructures 1206 = static_cast<X86VMTranslationMap*>( 1207 VMAddressSpace::Kernel()->TranslationMap())->PagingStructures(); 1208 1209 // Set active translation map on each CPU. 1210 for (i = 0; i < args->num_cpus; i++) { 1211 gCPU[i].arch.active_paging_structures = kernelPagingStructures; 1212 kernelPagingStructures->AddReference(); 1213 } 1214 1215 if (!apic_available()) 1216 x86_init_fpu(); 1217 // else fpu gets set up in smp code 1218 1219 #ifdef __x86_64__ 1220 // if available enable SMEP (Supervisor Memory Execution Protection) 1221 if (x86_check_feature(IA32_FEATURE_SMEP, FEATURE_7_EBX)) { 1222 if (!get_safemode_boolean(B_SAFEMODE_DISABLE_SMEP_SMAP, false)) { 1223 dprintf("enable SMEP\n"); 1224 call_all_cpus_sync(&enable_smep, NULL); 1225 } else 1226 dprintf("SMEP disabled per safemode setting\n"); 1227 } 1228 1229 // if available enable SMAP (Supervisor Memory Access Protection) 1230 if (x86_check_feature(IA32_FEATURE_SMAP, FEATURE_7_EBX)) { 1231 if (!get_safemode_boolean(B_SAFEMODE_DISABLE_SMEP_SMAP, false)) { 1232 dprintf("enable SMAP\n"); 1233 call_all_cpus_sync(&enable_smap, NULL); 1234 1235 arch_altcodepatch_replace(ALTCODEPATCH_TAG_STAC, &_stac, 3); 1236 arch_altcodepatch_replace(ALTCODEPATCH_TAG_CLAC, &_clac, 3); 1237 } else 1238 dprintf("SMAP disabled per safemode setting\n"); 1239 } 1240 #endif 1241 1242 return B_OK; 1243 } 1244 1245 1246 status_t 1247 arch_cpu_init_post_modules(kernel_args* args) 1248 { 1249 // initialize CPU module 1250 1251 void* cookie = open_module_list("cpu"); 1252 1253 while (true) { 1254 char name[B_FILE_NAME_LENGTH]; 1255 size_t nameLength = sizeof(name); 1256 1257 if (read_next_module_name(cookie, name, &nameLength) != B_OK 1258 || get_module(name, (module_info**)&sCpuModule) == B_OK) 1259 break; 1260 } 1261 1262 close_module_list(cookie); 1263 1264 // initialize MTRRs if available 1265 if (x86_count_mtrrs() > 0) { 1266 sCpuRendezvous = sCpuRendezvous2 = 0; 1267 call_all_cpus(&init_mtrrs, NULL); 1268 } 1269 1270 size_t threadExitLen = (addr_t)x86_end_userspace_thread_exit 1271 - (addr_t)x86_userspace_thread_exit; 1272 addr_t threadExitPosition = fill_commpage_entry( 1273 COMMPAGE_ENTRY_X86_THREAD_EXIT, (const void*)x86_userspace_thread_exit, 1274 threadExitLen); 1275 1276 // add the functions to the commpage image 1277 image_id image = get_commpage_image(); 1278 1279 elf_add_memory_image_symbol(image, "commpage_thread_exit", 1280 threadExitPosition, threadExitLen, B_SYMBOL_TYPE_TEXT); 1281 1282 return B_OK; 1283 } 1284 1285 1286 void 1287 arch_cpu_user_TLB_invalidate(void) 1288 { 1289 x86_write_cr3(x86_read_cr3()); 1290 } 1291 1292 1293 void 1294 arch_cpu_global_TLB_invalidate(void) 1295 { 1296 uint32 flags = x86_read_cr4(); 1297 1298 if (flags & IA32_CR4_GLOBAL_PAGES) { 1299 // disable and reenable the global pages to flush all TLBs regardless 1300 // of the global page bit 1301 x86_write_cr4(flags & ~IA32_CR4_GLOBAL_PAGES); 1302 x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES); 1303 } else { 1304 cpu_status state = disable_interrupts(); 1305 arch_cpu_user_TLB_invalidate(); 1306 restore_interrupts(state); 1307 } 1308 } 1309 1310 1311 void 1312 arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) 1313 { 1314 int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE; 1315 while (num_pages-- >= 0) { 1316 invalidate_TLB(start); 1317 start += B_PAGE_SIZE; 1318 } 1319 } 1320 1321 1322 void 1323 arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) 1324 { 1325 int i; 1326 for (i = 0; i < num_pages; i++) { 1327 invalidate_TLB(pages[i]); 1328 } 1329 } 1330 1331 1332 status_t 1333 arch_cpu_shutdown(bool rebootSystem) 1334 { 1335 if (acpi_shutdown(rebootSystem) == B_OK) 1336 return B_OK; 1337 1338 if (!rebootSystem) { 1339 #ifndef __x86_64__ 1340 return apm_shutdown(); 1341 #else 1342 return B_NOT_SUPPORTED; 1343 #endif 1344 } 1345 1346 cpu_status state = disable_interrupts(); 1347 1348 // try to reset the system using the keyboard controller 1349 out8(0xfe, 0x64); 1350 1351 // Give some time to the controller to do its job (0.5s) 1352 snooze(500000); 1353 1354 // if that didn't help, try it this way 1355 x86_reboot(); 1356 1357 restore_interrupts(state); 1358 return B_ERROR; 1359 } 1360 1361 1362 void 1363 arch_cpu_sync_icache(void* address, size_t length) 1364 { 1365 // instruction cache is always consistent on x86 1366 } 1367 1368