1 /* 2 * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. 3 * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. 4 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. 5 * Distributed under the terms of the MIT License. 6 * 7 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 8 * Distributed under the terms of the NewOS License. 9 */ 10 11 12 #include <cpu.h> 13 14 #include <string.h> 15 #include <stdlib.h> 16 #include <stdio.h> 17 18 #include <algorithm> 19 20 #include <ACPI.h> 21 22 #include <boot_device.h> 23 #include <commpage.h> 24 #include <debug.h> 25 #include <elf.h> 26 #include <smp.h> 27 #include <util/BitUtils.h> 28 #include <vm/vm.h> 29 #include <vm/vm_types.h> 30 #include <vm/VMAddressSpace.h> 31 32 #include <arch_system_info.h> 33 #include <arch/x86/apic.h> 34 #include <boot/kernel_args.h> 35 36 #include "paging/X86PagingStructures.h" 37 #include "paging/X86VMTranslationMap.h" 38 39 40 #define DUMP_FEATURE_STRING 1 41 #define DUMP_CPU_TOPOLOGY 1 42 43 44 /* cpu vendor info */ 45 struct cpu_vendor_info { 46 const char *vendor; 47 const char *ident_string[2]; 48 }; 49 50 static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = { 51 { "Intel", { "GenuineIntel" } }, 52 { "AMD", { "AuthenticAMD" } }, 53 { "Cyrix", { "CyrixInstead" } }, 54 { "UMC", { "UMC UMC UMC" } }, 55 { "NexGen", { "NexGenDriven" } }, 56 { "Centaur", { "CentaurHauls" } }, 57 { "Rise", { "RiseRiseRise" } }, 58 { "Transmeta", { "GenuineTMx86", "TransmetaCPU" } }, 59 { "NSC", { "Geode by NSC" } }, 60 }; 61 62 #define K8_SMIONCMPHALT (1ULL << 27) 63 #define K8_C1EONCMPHALT (1ULL << 28) 64 65 #define K8_CMPHALT (K8_SMIONCMPHALT | K8_C1EONCMPHALT) 66 67 struct set_mtrr_parameter { 68 int32 index; 69 uint64 base; 70 uint64 length; 71 uint8 type; 72 }; 73 74 struct set_mtrrs_parameter { 75 const x86_mtrr_info* infos; 76 uint32 count; 77 uint8 defaultType; 78 }; 79 80 81 extern "C" void x86_reboot(void); 82 // from arch.S 83 84 void (*gCpuIdleFunc)(void); 85 #ifndef __x86_64__ 86 void (*gX86SwapFPUFunc)(void* oldState, const void* newState) = x86_noop_swap; 87 bool gHasSSE = false; 88 #endif 89 90 static uint32 sCpuRendezvous; 91 static uint32 sCpuRendezvous2; 92 static uint32 sCpuRendezvous3; 93 static vint32 sTSCSyncRendezvous; 94 95 /* Some specials for the double fault handler */ 96 static uint8* sDoubleFaultStacks; 97 static const size_t kDoubleFaultStackSize = 4096; // size per CPU 98 99 static x86_cpu_module_info* sCpuModule; 100 101 102 /* CPU topology information */ 103 static uint32 (*sGetCPUTopologyID)(int currentCPU); 104 static uint32 sHierarchyMask[CPU_TOPOLOGY_LEVELS]; 105 static uint32 sHierarchyShift[CPU_TOPOLOGY_LEVELS]; 106 107 /* Cache topology information */ 108 static uint32 sCacheSharingMask[CPU_MAX_CACHE_LEVEL]; 109 110 111 static status_t 112 acpi_shutdown(bool rebootSystem) 113 { 114 if (debug_debugger_running() || !are_interrupts_enabled()) 115 return B_ERROR; 116 117 acpi_module_info* acpi; 118 if (get_module(B_ACPI_MODULE_NAME, (module_info**)&acpi) != B_OK) 119 return B_NOT_SUPPORTED; 120 121 status_t status; 122 if (rebootSystem) { 123 status = acpi->reboot(); 124 } else { 125 status = acpi->prepare_sleep_state(ACPI_POWER_STATE_OFF, NULL, 0); 126 if (status == B_OK) { 127 //cpu_status state = disable_interrupts(); 128 status = acpi->enter_sleep_state(ACPI_POWER_STATE_OFF); 129 //restore_interrupts(state); 130 } 131 } 132 133 put_module(B_ACPI_MODULE_NAME); 134 return status; 135 } 136 137 138 /*! Disable CPU caches, and invalidate them. */ 139 static void 140 disable_caches() 141 { 142 x86_write_cr0((x86_read_cr0() | CR0_CACHE_DISABLE) 143 & ~CR0_NOT_WRITE_THROUGH); 144 wbinvd(); 145 arch_cpu_global_TLB_invalidate(); 146 } 147 148 149 /*! Invalidate CPU caches, and enable them. */ 150 static void 151 enable_caches() 152 { 153 wbinvd(); 154 arch_cpu_global_TLB_invalidate(); 155 x86_write_cr0(x86_read_cr0() 156 & ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH)); 157 } 158 159 160 static void 161 set_mtrr(void* _parameter, int cpu) 162 { 163 struct set_mtrr_parameter* parameter 164 = (struct set_mtrr_parameter*)_parameter; 165 166 // wait until all CPUs have arrived here 167 smp_cpu_rendezvous(&sCpuRendezvous); 168 169 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 170 // that initiated the call_all_cpus() from doing that again and clearing 171 // sCpuRendezvous2 before the last CPU has actually left the loop in 172 // smp_cpu_rendezvous(); 173 if (cpu == 0) 174 atomic_set((int32*)&sCpuRendezvous3, 0); 175 176 disable_caches(); 177 178 sCpuModule->set_mtrr(parameter->index, parameter->base, parameter->length, 179 parameter->type); 180 181 enable_caches(); 182 183 // wait until all CPUs have arrived here 184 smp_cpu_rendezvous(&sCpuRendezvous2); 185 smp_cpu_rendezvous(&sCpuRendezvous3); 186 } 187 188 189 static void 190 set_mtrrs(void* _parameter, int cpu) 191 { 192 set_mtrrs_parameter* parameter = (set_mtrrs_parameter*)_parameter; 193 194 // wait until all CPUs have arrived here 195 smp_cpu_rendezvous(&sCpuRendezvous); 196 197 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 198 // that initiated the call_all_cpus() from doing that again and clearing 199 // sCpuRendezvous2 before the last CPU has actually left the loop in 200 // smp_cpu_rendezvous(); 201 if (cpu == 0) 202 atomic_set((int32*)&sCpuRendezvous3, 0); 203 204 disable_caches(); 205 206 sCpuModule->set_mtrrs(parameter->defaultType, parameter->infos, 207 parameter->count); 208 209 enable_caches(); 210 211 // wait until all CPUs have arrived here 212 smp_cpu_rendezvous(&sCpuRendezvous2); 213 smp_cpu_rendezvous(&sCpuRendezvous3); 214 } 215 216 217 static void 218 init_mtrrs(void* _unused, int cpu) 219 { 220 // wait until all CPUs have arrived here 221 smp_cpu_rendezvous(&sCpuRendezvous); 222 223 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 224 // that initiated the call_all_cpus() from doing that again and clearing 225 // sCpuRendezvous2 before the last CPU has actually left the loop in 226 // smp_cpu_rendezvous(); 227 if (cpu == 0) 228 atomic_set((int32*)&sCpuRendezvous3, 0); 229 230 disable_caches(); 231 232 sCpuModule->init_mtrrs(); 233 234 enable_caches(); 235 236 // wait until all CPUs have arrived here 237 smp_cpu_rendezvous(&sCpuRendezvous2); 238 smp_cpu_rendezvous(&sCpuRendezvous3); 239 } 240 241 242 uint32 243 x86_count_mtrrs(void) 244 { 245 if (sCpuModule == NULL) 246 return 0; 247 248 return sCpuModule->count_mtrrs(); 249 } 250 251 252 void 253 x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type) 254 { 255 struct set_mtrr_parameter parameter; 256 parameter.index = index; 257 parameter.base = base; 258 parameter.length = length; 259 parameter.type = type; 260 261 sCpuRendezvous = sCpuRendezvous2 = 0; 262 call_all_cpus(&set_mtrr, ¶meter); 263 } 264 265 266 status_t 267 x86_get_mtrr(uint32 index, uint64* _base, uint64* _length, uint8* _type) 268 { 269 // the MTRRs are identical on all CPUs, so it doesn't matter 270 // on which CPU this runs 271 return sCpuModule->get_mtrr(index, _base, _length, _type); 272 } 273 274 275 void 276 x86_set_mtrrs(uint8 defaultType, const x86_mtrr_info* infos, uint32 count) 277 { 278 if (sCpuModule == NULL) 279 return; 280 281 struct set_mtrrs_parameter parameter; 282 parameter.defaultType = defaultType; 283 parameter.infos = infos; 284 parameter.count = count; 285 286 sCpuRendezvous = sCpuRendezvous2 = 0; 287 call_all_cpus(&set_mtrrs, ¶meter); 288 } 289 290 291 void 292 x86_init_fpu(void) 293 { 294 // All x86_64 CPUs support SSE, don't need to bother checking for it. 295 #ifndef __x86_64__ 296 if (!x86_check_feature(IA32_FEATURE_FPU, FEATURE_COMMON)) { 297 // No FPU... time to install one in your 386? 298 dprintf("%s: Warning: CPU has no reported FPU.\n", __func__); 299 gX86SwapFPUFunc = x86_noop_swap; 300 return; 301 } 302 303 if (!x86_check_feature(IA32_FEATURE_SSE, FEATURE_COMMON) 304 || !x86_check_feature(IA32_FEATURE_FXSR, FEATURE_COMMON)) { 305 dprintf("%s: CPU has no SSE... just enabling FPU.\n", __func__); 306 // we don't have proper SSE support, just enable FPU 307 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 308 gX86SwapFPUFunc = x86_fnsave_swap; 309 return; 310 } 311 #endif 312 313 dprintf("%s: CPU has SSE... enabling FXSR and XMM.\n", __func__); 314 #ifndef __x86_64__ 315 // enable OS support for SSE 316 x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION); 317 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 318 319 gX86SwapFPUFunc = x86_fxsave_swap; 320 gHasSSE = true; 321 #endif 322 } 323 324 325 #if DUMP_FEATURE_STRING 326 static void 327 dump_feature_string(int currentCPU, cpu_ent* cpu) 328 { 329 char features[384]; 330 features[0] = 0; 331 332 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FPU) 333 strlcat(features, "fpu ", sizeof(features)); 334 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_VME) 335 strlcat(features, "vme ", sizeof(features)); 336 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DE) 337 strlcat(features, "de ", sizeof(features)); 338 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE) 339 strlcat(features, "pse ", sizeof(features)); 340 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TSC) 341 strlcat(features, "tsc ", sizeof(features)); 342 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MSR) 343 strlcat(features, "msr ", sizeof(features)); 344 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAE) 345 strlcat(features, "pae ", sizeof(features)); 346 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCE) 347 strlcat(features, "mce ", sizeof(features)); 348 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CX8) 349 strlcat(features, "cx8 ", sizeof(features)); 350 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_APIC) 351 strlcat(features, "apic ", sizeof(features)); 352 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SEP) 353 strlcat(features, "sep ", sizeof(features)); 354 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MTRR) 355 strlcat(features, "mtrr ", sizeof(features)); 356 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PGE) 357 strlcat(features, "pge ", sizeof(features)); 358 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCA) 359 strlcat(features, "mca ", sizeof(features)); 360 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CMOV) 361 strlcat(features, "cmov ", sizeof(features)); 362 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAT) 363 strlcat(features, "pat ", sizeof(features)); 364 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE36) 365 strlcat(features, "pse36 ", sizeof(features)); 366 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSN) 367 strlcat(features, "psn ", sizeof(features)); 368 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CLFSH) 369 strlcat(features, "clfsh ", sizeof(features)); 370 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DS) 371 strlcat(features, "ds ", sizeof(features)); 372 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_ACPI) 373 strlcat(features, "acpi ", sizeof(features)); 374 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MMX) 375 strlcat(features, "mmx ", sizeof(features)); 376 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FXSR) 377 strlcat(features, "fxsr ", sizeof(features)); 378 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE) 379 strlcat(features, "sse ", sizeof(features)); 380 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE2) 381 strlcat(features, "sse2 ", sizeof(features)); 382 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SS) 383 strlcat(features, "ss ", sizeof(features)); 384 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_HTT) 385 strlcat(features, "htt ", sizeof(features)); 386 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TM) 387 strlcat(features, "tm ", sizeof(features)); 388 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PBE) 389 strlcat(features, "pbe ", sizeof(features)); 390 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE3) 391 strlcat(features, "sse3 ", sizeof(features)); 392 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCLMULQDQ) 393 strlcat(features, "pclmulqdq ", sizeof(features)); 394 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DTES64) 395 strlcat(features, "dtes64 ", sizeof(features)); 396 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MONITOR) 397 strlcat(features, "monitor ", sizeof(features)); 398 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DSCPL) 399 strlcat(features, "dscpl ", sizeof(features)); 400 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_VMX) 401 strlcat(features, "vmx ", sizeof(features)); 402 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SMX) 403 strlcat(features, "smx ", sizeof(features)); 404 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_EST) 405 strlcat(features, "est ", sizeof(features)); 406 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TM2) 407 strlcat(features, "tm2 ", sizeof(features)); 408 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSSE3) 409 strlcat(features, "ssse3 ", sizeof(features)); 410 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CNXTID) 411 strlcat(features, "cnxtid ", sizeof(features)); 412 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_FMA) 413 strlcat(features, "fma ", sizeof(features)); 414 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CX16) 415 strlcat(features, "cx16 ", sizeof(features)); 416 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XTPR) 417 strlcat(features, "xtpr ", sizeof(features)); 418 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PDCM) 419 strlcat(features, "pdcm ", sizeof(features)); 420 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCID) 421 strlcat(features, "pcid ", sizeof(features)); 422 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DCA) 423 strlcat(features, "dca ", sizeof(features)); 424 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_1) 425 strlcat(features, "sse4_1 ", sizeof(features)); 426 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_2) 427 strlcat(features, "sse4_2 ", sizeof(features)); 428 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_X2APIC) 429 strlcat(features, "x2apic ", sizeof(features)); 430 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MOVBE) 431 strlcat(features, "movbe ", sizeof(features)); 432 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_POPCNT) 433 strlcat(features, "popcnt ", sizeof(features)); 434 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TSCDEADLINE) 435 strlcat(features, "tscdeadline ", sizeof(features)); 436 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AES) 437 strlcat(features, "aes ", sizeof(features)); 438 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XSAVE) 439 strlcat(features, "xsave ", sizeof(features)); 440 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_OSXSAVE) 441 strlcat(features, "osxsave ", sizeof(features)); 442 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AVX) 443 strlcat(features, "avx ", sizeof(features)); 444 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_F16C) 445 strlcat(features, "f16c ", sizeof(features)); 446 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_RDRND) 447 strlcat(features, "rdrnd ", sizeof(features)); 448 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) 449 strlcat(features, "hypervisor ", sizeof(features)); 450 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_SYSCALL) 451 strlcat(features, "syscall ", sizeof(features)); 452 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_NX) 453 strlcat(features, "nx ", sizeof(features)); 454 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_MMXEXT) 455 strlcat(features, "mmxext ", sizeof(features)); 456 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_FFXSR) 457 strlcat(features, "ffxsr ", sizeof(features)); 458 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_LONG) 459 strlcat(features, "long ", sizeof(features)); 460 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOWEXT) 461 strlcat(features, "3dnowext ", sizeof(features)); 462 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOW) 463 strlcat(features, "3dnow ", sizeof(features)); 464 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_DTS) 465 strlcat(features, "dts ", sizeof(features)); 466 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ITB) 467 strlcat(features, "itb ", sizeof(features)); 468 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ARAT) 469 strlcat(features, "arat ", sizeof(features)); 470 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PLN) 471 strlcat(features, "pln ", sizeof(features)); 472 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ECMD) 473 strlcat(features, "ecmd ", sizeof(features)); 474 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PTM) 475 strlcat(features, "ptm ", sizeof(features)); 476 if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_APERFMPERF) 477 strlcat(features, "aperfmperf ", sizeof(features)); 478 if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_EPB) 479 strlcat(features, "epb ", sizeof(features)); 480 481 dprintf("CPU %d: features: %s\n", currentCPU, features); 482 } 483 #endif // DUMP_FEATURE_STRING 484 485 486 static void 487 compute_cpu_hierarchy_masks(int maxLogicalID, int maxCoreID) 488 { 489 ASSERT(maxLogicalID >= maxCoreID); 490 const int kMaxSMTID = maxLogicalID / maxCoreID; 491 492 sHierarchyMask[CPU_TOPOLOGY_SMT] = kMaxSMTID - 1; 493 sHierarchyShift[CPU_TOPOLOGY_SMT] = 0; 494 495 sHierarchyMask[CPU_TOPOLOGY_CORE] = (maxCoreID - 1) * kMaxSMTID; 496 sHierarchyShift[CPU_TOPOLOGY_CORE] 497 = count_set_bits(sHierarchyMask[CPU_TOPOLOGY_SMT]); 498 499 const uint32 kSinglePackageMask = sHierarchyMask[CPU_TOPOLOGY_SMT] 500 | sHierarchyMask[CPU_TOPOLOGY_CORE]; 501 sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~kSinglePackageMask; 502 sHierarchyShift[CPU_TOPOLOGY_PACKAGE] = count_set_bits(kSinglePackageMask); 503 } 504 505 506 static uint32 507 get_cpu_legacy_initial_apic_id(int /* currentCPU */) 508 { 509 cpuid_info cpuid; 510 get_current_cpuid(&cpuid, 1, 0); 511 return cpuid.regs.ebx >> 24; 512 } 513 514 515 static inline status_t 516 detect_amd_cpu_topology(uint32 maxBasicLeaf, uint32 maxExtendedLeaf) 517 { 518 sGetCPUTopologyID = get_cpu_legacy_initial_apic_id; 519 520 cpuid_info cpuid; 521 get_current_cpuid(&cpuid, 1, 0); 522 int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff); 523 524 int maxCoreID = 1; 525 if (maxExtendedLeaf >= 0x80000008) { 526 get_current_cpuid(&cpuid, 0x80000008, 0); 527 maxCoreID = (cpuid.regs.ecx >> 12) & 0xf; 528 if (maxCoreID != 0) 529 maxCoreID = 1 << maxCoreID; 530 else 531 maxCoreID = next_power_of_2((cpuid.regs.edx & 0xf) + 1); 532 } 533 534 if (maxExtendedLeaf >= 0x80000001) { 535 get_current_cpuid(&cpuid, 0x80000001, 0); 536 if (x86_check_feature(IA32_FEATURE_AMD_EXT_CMPLEGACY, 537 FEATURE_EXT_AMD_ECX)) 538 maxCoreID = maxLogicalID; 539 } 540 541 compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID); 542 543 return B_OK; 544 } 545 546 547 static void 548 detect_amd_cache_topology(uint32 maxExtendedLeaf) 549 { 550 if (!x86_check_feature(IA32_FEATURE_AMD_EXT_TOPOLOGY, FEATURE_EXT_AMD_ECX)) 551 return; 552 553 if (maxExtendedLeaf < 0x8000001d) 554 return; 555 556 uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL]; 557 int maxCacheLevel = 0; 558 559 int currentLevel = 0; 560 int cacheType; 561 do { 562 cpuid_info cpuid; 563 get_current_cpuid(&cpuid, 0x8000001d, currentLevel); 564 565 cacheType = cpuid.regs.eax & 0x1f; 566 if (cacheType == 0) 567 break; 568 569 int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; 570 int coresCount = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1); 571 hierarchyLevels[cacheLevel - 1] 572 = coresCount * (sHierarchyMask[CPU_TOPOLOGY_SMT] + 1); 573 maxCacheLevel = std::max(maxCacheLevel, cacheLevel); 574 575 currentLevel++; 576 } while (true); 577 578 for (int i = 0; i < maxCacheLevel; i++) 579 sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); 580 gCPUCacheLevelCount = maxCacheLevel; 581 } 582 583 584 static uint32 585 get_intel_cpu_initial_x2apic_id(int /* currentCPU */) 586 { 587 cpuid_info cpuid; 588 get_current_cpuid(&cpuid, 11, 0); 589 return cpuid.regs.edx; 590 } 591 592 593 static inline status_t 594 detect_intel_cpu_topology_x2apic(uint32 maxBasicLeaf) 595 { 596 if (maxBasicLeaf < 11) 597 return B_UNSUPPORTED; 598 599 uint8 hierarchyLevels[CPU_TOPOLOGY_LEVELS] = { 0 }; 600 601 int currentLevel = 0; 602 int levelType; 603 unsigned int levelsSet = 0; 604 605 do { 606 cpuid_info cpuid; 607 get_current_cpuid(&cpuid, 11, currentLevel); 608 if (currentLevel == 0 && cpuid.regs.ebx == 0) 609 return B_UNSUPPORTED; 610 611 levelType = (cpuid.regs.ecx >> 8) & 0xff; 612 int levelValue = cpuid.regs.eax & 0x1f; 613 614 switch (levelType) { 615 case 1: // SMT 616 hierarchyLevels[CPU_TOPOLOGY_SMT] = levelValue; 617 levelsSet |= 1; 618 break; 619 case 2: // core 620 hierarchyLevels[CPU_TOPOLOGY_CORE] = levelValue; 621 levelsSet |= 2; 622 break; 623 } 624 625 currentLevel++; 626 } while (levelType != 0 && levelsSet != 3); 627 628 sGetCPUTopologyID = get_intel_cpu_initial_x2apic_id; 629 630 for (int i = 1; i < CPU_TOPOLOGY_LEVELS; i++) { 631 if ((levelsSet & (1u << i)) != 0) 632 continue; 633 hierarchyLevels[i] = hierarchyLevels[i - 1]; 634 } 635 636 for (int i = 0; i < CPU_TOPOLOGY_LEVELS; i++) { 637 uint32 mask = ~uint32(0); 638 if (i < CPU_TOPOLOGY_LEVELS - 1) 639 mask = (1u << hierarchyLevels[i]) - 1; 640 if (i > 0) 641 mask &= ~sHierarchyMask[i - 1]; 642 sHierarchyMask[i] = mask; 643 sHierarchyShift[i] = i > 0 ? hierarchyLevels[i - 1] : 0; 644 } 645 646 return B_OK; 647 } 648 649 650 static inline status_t 651 detect_intel_cpu_topology_legacy(uint32 maxBasicLeaf) 652 { 653 sGetCPUTopologyID = get_cpu_legacy_initial_apic_id; 654 655 cpuid_info cpuid; 656 657 get_current_cpuid(&cpuid, 1, 0); 658 int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff); 659 660 int maxCoreID = 1; 661 if (maxBasicLeaf >= 4) { 662 get_current_cpuid(&cpuid, 4, 0); 663 maxCoreID = next_power_of_2((cpuid.regs.eax >> 26) + 1); 664 } 665 666 compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID); 667 668 return B_OK; 669 } 670 671 672 static void 673 detect_intel_cache_topology(uint32 maxBasicLeaf) 674 { 675 if (maxBasicLeaf < 4) 676 return; 677 678 uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL]; 679 int maxCacheLevel = 0; 680 681 int currentLevel = 0; 682 int cacheType; 683 do { 684 cpuid_info cpuid; 685 get_current_cpuid(&cpuid, 4, currentLevel); 686 687 cacheType = cpuid.regs.eax & 0x1f; 688 if (cacheType == 0) 689 break; 690 691 int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; 692 hierarchyLevels[cacheLevel - 1] 693 = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1); 694 maxCacheLevel = std::max(maxCacheLevel, cacheLevel); 695 696 currentLevel++; 697 } while (true); 698 699 for (int i = 0; i < maxCacheLevel; i++) 700 sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); 701 702 gCPUCacheLevelCount = maxCacheLevel; 703 } 704 705 706 static uint32 707 get_simple_cpu_topology_id(int currentCPU) 708 { 709 return currentCPU; 710 } 711 712 713 static inline int 714 get_topology_level_id(uint32 id, cpu_topology_level level) 715 { 716 ASSERT(level < CPU_TOPOLOGY_LEVELS); 717 return (id & sHierarchyMask[level]) >> sHierarchyShift[level]; 718 } 719 720 721 static void 722 detect_cpu_topology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, 723 uint32 maxExtendedLeaf) 724 { 725 if (currentCPU == 0) { 726 memset(sCacheSharingMask, 0xff, sizeof(sCacheSharingMask)); 727 728 status_t result = B_UNSUPPORTED; 729 if (x86_check_feature(IA32_FEATURE_HTT, FEATURE_COMMON)) { 730 if (cpu->arch.vendor == VENDOR_AMD) { 731 result = detect_amd_cpu_topology(maxBasicLeaf, maxExtendedLeaf); 732 733 if (result == B_OK) 734 detect_amd_cache_topology(maxExtendedLeaf); 735 } 736 737 if (cpu->arch.vendor == VENDOR_INTEL) { 738 result = detect_intel_cpu_topology_x2apic(maxBasicLeaf); 739 if (result != B_OK) 740 result = detect_intel_cpu_topology_legacy(maxBasicLeaf); 741 742 if (result == B_OK) 743 detect_intel_cache_topology(maxBasicLeaf); 744 } 745 } 746 747 if (result != B_OK) { 748 dprintf("No CPU topology information available.\n"); 749 750 sGetCPUTopologyID = get_simple_cpu_topology_id; 751 752 sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~uint32(0); 753 } 754 } 755 756 ASSERT(sGetCPUTopologyID != NULL); 757 int topologyID = sGetCPUTopologyID(currentCPU); 758 cpu->topology_id[CPU_TOPOLOGY_SMT] 759 = get_topology_level_id(topologyID, CPU_TOPOLOGY_SMT); 760 cpu->topology_id[CPU_TOPOLOGY_CORE] 761 = get_topology_level_id(topologyID, CPU_TOPOLOGY_CORE); 762 cpu->topology_id[CPU_TOPOLOGY_PACKAGE] 763 = get_topology_level_id(topologyID, CPU_TOPOLOGY_PACKAGE); 764 765 unsigned int i; 766 for (i = 0; i < gCPUCacheLevelCount; i++) 767 cpu->cache_id[i] = topologyID & sCacheSharingMask[i]; 768 for (; i < CPU_MAX_CACHE_LEVEL; i++) 769 cpu->cache_id[i] = -1; 770 771 #if DUMP_CPU_TOPOLOGY 772 dprintf("CPU %d: apic id %d, package %d, core %d, smt %d\n", currentCPU, 773 topologyID, cpu->topology_id[CPU_TOPOLOGY_PACKAGE], 774 cpu->topology_id[CPU_TOPOLOGY_CORE], 775 cpu->topology_id[CPU_TOPOLOGY_SMT]); 776 777 if (gCPUCacheLevelCount > 0) { 778 char cacheLevels[256]; 779 unsigned int offset = 0; 780 for (i = 0; i < gCPUCacheLevelCount; i++) { 781 offset += snprintf(cacheLevels + offset, 782 sizeof(cacheLevels) - offset, 783 " L%d id %d%s", i + 1, cpu->cache_id[i], 784 i < gCPUCacheLevelCount - 1 ? "," : ""); 785 786 if (offset >= sizeof(cacheLevels)) 787 break; 788 } 789 790 dprintf("CPU %d: cache sharing:%s\n", currentCPU, cacheLevels); 791 } 792 #endif 793 } 794 795 796 static void 797 detect_cpu(int currentCPU) 798 { 799 cpu_ent* cpu = get_cpu_struct(); 800 char vendorString[17]; 801 cpuid_info cpuid; 802 803 // clear out the cpu info data 804 cpu->arch.vendor = VENDOR_UNKNOWN; 805 cpu->arch.vendor_name = "UNKNOWN VENDOR"; 806 cpu->arch.feature[FEATURE_COMMON] = 0; 807 cpu->arch.feature[FEATURE_EXT] = 0; 808 cpu->arch.feature[FEATURE_EXT_AMD] = 0; 809 cpu->arch.model_name[0] = 0; 810 811 // print some fun data 812 get_current_cpuid(&cpuid, 0, 0); 813 uint32 maxBasicLeaf = cpuid.eax_0.max_eax; 814 815 // build the vendor string 816 memset(vendorString, 0, sizeof(vendorString)); 817 memcpy(vendorString, cpuid.eax_0.vendor_id, sizeof(cpuid.eax_0.vendor_id)); 818 819 // get the family, model, stepping 820 get_current_cpuid(&cpuid, 1, 0); 821 cpu->arch.type = cpuid.eax_1.type; 822 cpu->arch.family = cpuid.eax_1.family; 823 cpu->arch.extended_family = cpuid.eax_1.extended_family; 824 cpu->arch.model = cpuid.eax_1.model; 825 cpu->arch.extended_model = cpuid.eax_1.extended_model; 826 cpu->arch.stepping = cpuid.eax_1.stepping; 827 dprintf("CPU %d: type %d family %d extended_family %d model %d " 828 "extended_model %d stepping %d, string '%s'\n", 829 currentCPU, cpu->arch.type, cpu->arch.family, 830 cpu->arch.extended_family, cpu->arch.model, 831 cpu->arch.extended_model, cpu->arch.stepping, vendorString); 832 833 // figure out what vendor we have here 834 835 for (int32 i = 0; i < VENDOR_NUM; i++) { 836 if (vendor_info[i].ident_string[0] 837 && !strcmp(vendorString, vendor_info[i].ident_string[0])) { 838 cpu->arch.vendor = (x86_vendors)i; 839 cpu->arch.vendor_name = vendor_info[i].vendor; 840 break; 841 } 842 if (vendor_info[i].ident_string[1] 843 && !strcmp(vendorString, vendor_info[i].ident_string[1])) { 844 cpu->arch.vendor = (x86_vendors)i; 845 cpu->arch.vendor_name = vendor_info[i].vendor; 846 break; 847 } 848 } 849 850 // see if we can get the model name 851 get_current_cpuid(&cpuid, 0x80000000, 0); 852 uint32 maxExtendedLeaf = cpuid.eax_0.max_eax; 853 if (maxExtendedLeaf >= 0x80000004) { 854 // build the model string (need to swap ecx/edx data before copying) 855 unsigned int temp; 856 memset(cpu->arch.model_name, 0, sizeof(cpu->arch.model_name)); 857 858 get_current_cpuid(&cpuid, 0x80000002, 0); 859 temp = cpuid.regs.edx; 860 cpuid.regs.edx = cpuid.regs.ecx; 861 cpuid.regs.ecx = temp; 862 memcpy(cpu->arch.model_name, cpuid.as_chars, sizeof(cpuid.as_chars)); 863 864 get_current_cpuid(&cpuid, 0x80000003, 0); 865 temp = cpuid.regs.edx; 866 cpuid.regs.edx = cpuid.regs.ecx; 867 cpuid.regs.ecx = temp; 868 memcpy(cpu->arch.model_name + 16, cpuid.as_chars, 869 sizeof(cpuid.as_chars)); 870 871 get_current_cpuid(&cpuid, 0x80000004, 0); 872 temp = cpuid.regs.edx; 873 cpuid.regs.edx = cpuid.regs.ecx; 874 cpuid.regs.ecx = temp; 875 memcpy(cpu->arch.model_name + 32, cpuid.as_chars, 876 sizeof(cpuid.as_chars)); 877 878 // some cpus return a right-justified string 879 int32 i = 0; 880 while (cpu->arch.model_name[i] == ' ') 881 i++; 882 if (i > 0) { 883 memmove(cpu->arch.model_name, &cpu->arch.model_name[i], 884 strlen(&cpu->arch.model_name[i]) + 1); 885 } 886 887 dprintf("CPU %d: vendor '%s' model name '%s'\n", 888 currentCPU, cpu->arch.vendor_name, cpu->arch.model_name); 889 } else { 890 strlcpy(cpu->arch.model_name, "unknown", sizeof(cpu->arch.model_name)); 891 } 892 893 // load feature bits 894 get_current_cpuid(&cpuid, 1, 0); 895 cpu->arch.feature[FEATURE_COMMON] = cpuid.eax_1.features; // edx 896 cpu->arch.feature[FEATURE_EXT] = cpuid.eax_1.extended_features; // ecx 897 898 if (maxExtendedLeaf >= 0x80000001) { 899 get_current_cpuid(&cpuid, 0x80000001, 0); 900 if (cpu->arch.vendor == VENDOR_AMD) 901 cpu->arch.feature[FEATURE_EXT_AMD_ECX] = cpuid.regs.ecx; // ecx 902 cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx 903 if (cpu->arch.vendor != VENDOR_AMD) 904 cpu->arch.feature[FEATURE_EXT_AMD] &= IA32_FEATURES_INTEL_EXT; 905 } 906 907 if (maxBasicLeaf >= 5) { 908 get_current_cpuid(&cpuid, 5, 0); 909 cpu->arch.feature[FEATURE_5_ECX] = cpuid.regs.ecx; 910 } 911 912 if (maxBasicLeaf >= 6) { 913 get_current_cpuid(&cpuid, 6, 0); 914 cpu->arch.feature[FEATURE_6_EAX] = cpuid.regs.eax; 915 cpu->arch.feature[FEATURE_6_ECX] = cpuid.regs.ecx; 916 } 917 918 if (maxExtendedLeaf >= 0x80000007) { 919 get_current_cpuid(&cpuid, 0x80000007, 0); 920 cpu->arch.feature[FEATURE_EXT_7_EDX] = cpuid.regs.edx; 921 } 922 923 detect_cpu_topology(currentCPU, cpu, maxBasicLeaf, maxExtendedLeaf); 924 925 #if DUMP_FEATURE_STRING 926 dump_feature_string(currentCPU, cpu); 927 #endif 928 } 929 930 931 bool 932 x86_check_feature(uint32 feature, enum x86_feature_type type) 933 { 934 cpu_ent* cpu = get_cpu_struct(); 935 936 #if 0 937 int i; 938 dprintf("x86_check_feature: feature 0x%x, type %d\n", feature, type); 939 for (i = 0; i < FEATURE_NUM; i++) { 940 dprintf("features %d: 0x%x\n", i, cpu->arch.feature[i]); 941 } 942 #endif 943 944 return (cpu->arch.feature[type] & feature) != 0; 945 } 946 947 948 void* 949 x86_get_double_fault_stack(int32 cpu, size_t* _size) 950 { 951 *_size = kDoubleFaultStackSize; 952 return sDoubleFaultStacks + kDoubleFaultStackSize * cpu; 953 } 954 955 956 /*! Returns the index of the current CPU. Can only be called from the double 957 fault handler. 958 */ 959 int32 960 x86_double_fault_get_cpu(void) 961 { 962 addr_t stack = x86_get_stack_frame(); 963 return (stack - (addr_t)sDoubleFaultStacks) / kDoubleFaultStackSize; 964 } 965 966 967 // #pragma mark - 968 969 970 status_t 971 arch_cpu_preboot_init_percpu(kernel_args* args, int cpu) 972 { 973 // On SMP system we want to synchronize the CPUs' TSCs, so system_time() 974 // will return consistent values. 975 if (smp_get_num_cpus() > 1) { 976 // let the first CPU prepare the rendezvous point 977 if (cpu == 0) 978 sTSCSyncRendezvous = smp_get_num_cpus() - 1; 979 980 // One CPU after the other will drop out of this loop and be caught by 981 // the loop below, until the last CPU (0) gets there. Save for +/- a few 982 // cycles the CPUs should pass the second loop at the same time. 983 while (sTSCSyncRendezvous != cpu) { 984 } 985 986 sTSCSyncRendezvous = cpu - 1; 987 988 while (sTSCSyncRendezvous != -1) { 989 } 990 991 // reset TSC to 0 992 x86_write_msr(IA32_MSR_TSC, 0); 993 } 994 995 x86_descriptors_preboot_init_percpu(args, cpu); 996 997 return B_OK; 998 } 999 1000 1001 static void 1002 halt_idle(void) 1003 { 1004 asm("hlt"); 1005 } 1006 1007 1008 static void 1009 amdc1e_noarat_idle(void) 1010 { 1011 uint64 msr = x86_read_msr(K8_MSR_IPM); 1012 if (msr & K8_CMPHALT) 1013 x86_write_msr(K8_MSR_IPM, msr & ~K8_CMPHALT); 1014 halt_idle(); 1015 } 1016 1017 1018 static bool 1019 detect_amdc1e_noarat() 1020 { 1021 cpu_ent* cpu = get_cpu_struct(); 1022 1023 if (cpu->arch.vendor != VENDOR_AMD) 1024 return false; 1025 1026 // Family 0x12 and higher processors support ARAT 1027 // Family lower than 0xf processors doesn't support C1E 1028 // Family 0xf with model <= 0x40 procssors doesn't support C1E 1029 uint32 family = cpu->arch.family + cpu->arch.extended_family; 1030 uint32 model = (cpu->arch.extended_model << 4) | cpu->arch.model; 1031 return (family < 0x12 && family > 0xf) || (family == 0xf && model > 0x40); 1032 } 1033 1034 1035 status_t 1036 arch_cpu_init_percpu(kernel_args* args, int cpu) 1037 { 1038 detect_cpu(cpu); 1039 1040 if (!gCpuIdleFunc) { 1041 if (detect_amdc1e_noarat()) 1042 gCpuIdleFunc = amdc1e_noarat_idle; 1043 else 1044 gCpuIdleFunc = halt_idle; 1045 } 1046 1047 return B_OK; 1048 } 1049 1050 1051 status_t 1052 arch_cpu_init(kernel_args* args) 1053 { 1054 // init the TSC -> system_time() conversion factors 1055 1056 uint32 conversionFactor = args->arch_args.system_time_cv_factor; 1057 uint64 conversionFactorNsecs = (uint64)conversionFactor * 1000; 1058 1059 #ifdef __x86_64__ 1060 // The x86_64 system_time() implementation uses 64-bit multiplication and 1061 // therefore shifting is not necessary for low frequencies (it's also not 1062 // too likely that there'll be any x86_64 CPUs clocked under 1GHz). 1063 __x86_setup_system_time((uint64)conversionFactor << 32, 1064 conversionFactorNsecs); 1065 #else 1066 if (conversionFactorNsecs >> 32 != 0) { 1067 // the TSC frequency is < 1 GHz, which forces us to shift the factor 1068 __x86_setup_system_time(conversionFactor, conversionFactorNsecs >> 16, 1069 true); 1070 } else { 1071 // the TSC frequency is >= 1 GHz 1072 __x86_setup_system_time(conversionFactor, conversionFactorNsecs, false); 1073 } 1074 #endif 1075 1076 // Initialize descriptor tables. 1077 x86_descriptors_init(args); 1078 1079 return B_OK; 1080 } 1081 1082 1083 status_t 1084 arch_cpu_init_post_vm(kernel_args* args) 1085 { 1086 uint32 i; 1087 1088 // allocate an area for the double fault stacks 1089 virtual_address_restrictions virtualRestrictions = {}; 1090 virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS; 1091 physical_address_restrictions physicalRestrictions = {}; 1092 create_area_etc(B_SYSTEM_TEAM, "double fault stacks", 1093 kDoubleFaultStackSize * smp_get_num_cpus(), B_FULL_LOCK, 1094 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0, 1095 &virtualRestrictions, &physicalRestrictions, 1096 (void**)&sDoubleFaultStacks); 1097 1098 X86PagingStructures* kernelPagingStructures 1099 = static_cast<X86VMTranslationMap*>( 1100 VMAddressSpace::Kernel()->TranslationMap())->PagingStructures(); 1101 1102 // Set active translation map on each CPU. 1103 for (i = 0; i < args->num_cpus; i++) { 1104 gCPU[i].arch.active_paging_structures = kernelPagingStructures; 1105 kernelPagingStructures->AddReference(); 1106 } 1107 1108 if (!apic_available()) 1109 x86_init_fpu(); 1110 // else fpu gets set up in smp code 1111 1112 return B_OK; 1113 } 1114 1115 1116 status_t 1117 arch_cpu_init_post_modules(kernel_args* args) 1118 { 1119 // initialize CPU module 1120 1121 void* cookie = open_module_list("cpu"); 1122 1123 while (true) { 1124 char name[B_FILE_NAME_LENGTH]; 1125 size_t nameLength = sizeof(name); 1126 1127 if (read_next_module_name(cookie, name, &nameLength) != B_OK 1128 || get_module(name, (module_info**)&sCpuModule) == B_OK) 1129 break; 1130 } 1131 1132 close_module_list(cookie); 1133 1134 // initialize MTRRs if available 1135 if (x86_count_mtrrs() > 0) { 1136 sCpuRendezvous = sCpuRendezvous2 = 0; 1137 call_all_cpus(&init_mtrrs, NULL); 1138 } 1139 1140 size_t threadExitLen = (addr_t)x86_end_userspace_thread_exit 1141 - (addr_t)x86_userspace_thread_exit; 1142 addr_t threadExitPosition = fill_commpage_entry( 1143 COMMPAGE_ENTRY_X86_THREAD_EXIT, (const void*)x86_userspace_thread_exit, 1144 threadExitLen); 1145 1146 // add the functions to the commpage image 1147 image_id image = get_commpage_image(); 1148 1149 elf_add_memory_image_symbol(image, "commpage_thread_exit", 1150 threadExitPosition, threadExitLen, B_SYMBOL_TYPE_TEXT); 1151 1152 return B_OK; 1153 } 1154 1155 1156 void 1157 arch_cpu_user_TLB_invalidate(void) 1158 { 1159 x86_write_cr3(x86_read_cr3()); 1160 } 1161 1162 1163 void 1164 arch_cpu_global_TLB_invalidate(void) 1165 { 1166 uint32 flags = x86_read_cr4(); 1167 1168 if (flags & IA32_CR4_GLOBAL_PAGES) { 1169 // disable and reenable the global pages to flush all TLBs regardless 1170 // of the global page bit 1171 x86_write_cr4(flags & ~IA32_CR4_GLOBAL_PAGES); 1172 x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES); 1173 } else { 1174 cpu_status state = disable_interrupts(); 1175 arch_cpu_user_TLB_invalidate(); 1176 restore_interrupts(state); 1177 } 1178 } 1179 1180 1181 void 1182 arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) 1183 { 1184 int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE; 1185 while (num_pages-- >= 0) { 1186 invalidate_TLB(start); 1187 start += B_PAGE_SIZE; 1188 } 1189 } 1190 1191 1192 void 1193 arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) 1194 { 1195 int i; 1196 for (i = 0; i < num_pages; i++) { 1197 invalidate_TLB(pages[i]); 1198 } 1199 } 1200 1201 1202 status_t 1203 arch_cpu_shutdown(bool rebootSystem) 1204 { 1205 if (acpi_shutdown(rebootSystem) == B_OK) 1206 return B_OK; 1207 1208 if (!rebootSystem) { 1209 #ifndef __x86_64__ 1210 return apm_shutdown(); 1211 #else 1212 return B_NOT_SUPPORTED; 1213 #endif 1214 } 1215 1216 cpu_status state = disable_interrupts(); 1217 1218 // try to reset the system using the keyboard controller 1219 out8(0xfe, 0x64); 1220 1221 // Give some time to the controller to do its job (0.5s) 1222 snooze(500000); 1223 1224 // if that didn't help, try it this way 1225 x86_reboot(); 1226 1227 restore_interrupts(state); 1228 return B_ERROR; 1229 } 1230 1231 1232 void 1233 arch_cpu_sync_icache(void* address, size_t length) 1234 { 1235 // instruction cache is always consistent on x86 1236 } 1237 1238