1 /* 2 * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. 3 * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. 4 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. 5 * Distributed under the terms of the MIT License. 6 * 7 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 8 * Distributed under the terms of the NewOS License. 9 */ 10 11 12 #include <cpu.h> 13 14 #include <string.h> 15 #include <stdlib.h> 16 #include <stdio.h> 17 18 #include <algorithm> 19 20 #include <ACPI.h> 21 22 #include <boot_device.h> 23 #include <commpage.h> 24 #include <debug.h> 25 #include <elf.h> 26 #include <smp.h> 27 #include <util/BitUtils.h> 28 #include <vm/vm.h> 29 #include <vm/vm_types.h> 30 #include <vm/VMAddressSpace.h> 31 32 #include <arch_system_info.h> 33 #include <arch/x86/apic.h> 34 #include <boot/kernel_args.h> 35 36 #include "paging/X86PagingStructures.h" 37 #include "paging/X86VMTranslationMap.h" 38 39 40 #define DUMP_FEATURE_STRING 1 41 #define DUMP_CPU_TOPOLOGY 1 42 43 44 /* cpu vendor info */ 45 struct cpu_vendor_info { 46 const char *vendor; 47 const char *ident_string[2]; 48 }; 49 50 static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = { 51 { "Intel", { "GenuineIntel" } }, 52 { "AMD", { "AuthenticAMD" } }, 53 { "Cyrix", { "CyrixInstead" } }, 54 { "UMC", { "UMC UMC UMC" } }, 55 { "NexGen", { "NexGenDriven" } }, 56 { "Centaur", { "CentaurHauls" } }, 57 { "Rise", { "RiseRiseRise" } }, 58 { "Transmeta", { "GenuineTMx86", "TransmetaCPU" } }, 59 { "NSC", { "Geode by NSC" } }, 60 }; 61 62 #define CR0_CACHE_DISABLE (1UL << 30) 63 #define CR0_NOT_WRITE_THROUGH (1UL << 29) 64 #define CR0_FPU_EMULATION (1UL << 2) 65 #define CR0_MONITOR_FPU (1UL << 1) 66 67 #define CR4_OS_FXSR (1UL << 9) 68 #define CR4_OS_XMM_EXCEPTION (1UL << 10) 69 70 #define K8_SMIONCMPHALT (1ULL << 27) 71 #define K8_C1EONCMPHALT (1ULL << 28) 72 73 #define K8_CMPHALT (K8_SMIONCMPHALT | K8_C1EONCMPHALT) 74 75 struct set_mtrr_parameter { 76 int32 index; 77 uint64 base; 78 uint64 length; 79 uint8 type; 80 }; 81 82 struct set_mtrrs_parameter { 83 const x86_mtrr_info* infos; 84 uint32 count; 85 uint8 defaultType; 86 }; 87 88 89 extern "C" void x86_reboot(void); 90 // from arch.S 91 92 void (*gCpuIdleFunc)(void); 93 void (*gX86SwapFPUFunc)(void* oldState, const void* newState) = x86_noop_swap; 94 bool gHasSSE = false; 95 96 static uint32 sCpuRendezvous; 97 static uint32 sCpuRendezvous2; 98 static uint32 sCpuRendezvous3; 99 static vint32 sTSCSyncRendezvous; 100 101 /* Some specials for the double fault handler */ 102 static uint8* sDoubleFaultStacks; 103 static const size_t kDoubleFaultStackSize = 4096; // size per CPU 104 105 static x86_cpu_module_info* sCpuModule; 106 107 108 extern "C" void memcpy_generic(void* dest, const void* source, size_t count); 109 extern int memcpy_generic_end; 110 extern "C" void memset_generic(void* dest, int value, size_t count); 111 extern int memset_generic_end; 112 113 x86_optimized_functions gOptimizedFunctions = { 114 memcpy_generic, 115 &memcpy_generic_end, 116 memset_generic, 117 &memset_generic_end 118 }; 119 120 /* CPU topology information */ 121 static uint32 (*sGetCPUTopologyID)(int currentCPU); 122 static uint32 sHierarchyMask[CPU_TOPOLOGY_LEVELS]; 123 static uint32 sHierarchyShift[CPU_TOPOLOGY_LEVELS]; 124 125 /* Cache topology information */ 126 static uint32 sCacheSharingMask[CPU_MAX_CACHE_LEVEL]; 127 128 129 static status_t 130 acpi_shutdown(bool rebootSystem) 131 { 132 if (debug_debugger_running() || !are_interrupts_enabled()) 133 return B_ERROR; 134 135 acpi_module_info* acpi; 136 if (get_module(B_ACPI_MODULE_NAME, (module_info**)&acpi) != B_OK) 137 return B_NOT_SUPPORTED; 138 139 status_t status; 140 if (rebootSystem) { 141 status = acpi->reboot(); 142 } else { 143 // Make sure we run on the boot CPU (apparently needed for some ACPI 144 // implementations) 145 _user_set_cpu_enabled(0, true); 146 for (int32 cpu = 1; cpu < smp_get_num_cpus(); cpu++) { 147 _user_set_cpu_enabled(cpu, false); 148 } 149 // TODO: must not be called from the idle thread! 150 thread_yield(); 151 152 status = acpi->prepare_sleep_state(ACPI_POWER_STATE_OFF, NULL, 0); 153 if (status == B_OK) { 154 //cpu_status state = disable_interrupts(); 155 status = acpi->enter_sleep_state(ACPI_POWER_STATE_OFF); 156 //restore_interrupts(state); 157 } 158 } 159 160 put_module(B_ACPI_MODULE_NAME); 161 return status; 162 } 163 164 165 /*! Disable CPU caches, and invalidate them. */ 166 static void 167 disable_caches() 168 { 169 x86_write_cr0((x86_read_cr0() | CR0_CACHE_DISABLE) 170 & ~CR0_NOT_WRITE_THROUGH); 171 wbinvd(); 172 arch_cpu_global_TLB_invalidate(); 173 } 174 175 176 /*! Invalidate CPU caches, and enable them. */ 177 static void 178 enable_caches() 179 { 180 wbinvd(); 181 arch_cpu_global_TLB_invalidate(); 182 x86_write_cr0(x86_read_cr0() 183 & ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH)); 184 } 185 186 187 static void 188 set_mtrr(void* _parameter, int cpu) 189 { 190 struct set_mtrr_parameter* parameter 191 = (struct set_mtrr_parameter*)_parameter; 192 193 // wait until all CPUs have arrived here 194 smp_cpu_rendezvous(&sCpuRendezvous); 195 196 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 197 // that initiated the call_all_cpus() from doing that again and clearing 198 // sCpuRendezvous2 before the last CPU has actually left the loop in 199 // smp_cpu_rendezvous(); 200 if (cpu == 0) 201 atomic_set((int32*)&sCpuRendezvous3, 0); 202 203 disable_caches(); 204 205 sCpuModule->set_mtrr(parameter->index, parameter->base, parameter->length, 206 parameter->type); 207 208 enable_caches(); 209 210 // wait until all CPUs have arrived here 211 smp_cpu_rendezvous(&sCpuRendezvous2); 212 smp_cpu_rendezvous(&sCpuRendezvous3); 213 } 214 215 216 static void 217 set_mtrrs(void* _parameter, int cpu) 218 { 219 set_mtrrs_parameter* parameter = (set_mtrrs_parameter*)_parameter; 220 221 // wait until all CPUs have arrived here 222 smp_cpu_rendezvous(&sCpuRendezvous); 223 224 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 225 // that initiated the call_all_cpus() from doing that again and clearing 226 // sCpuRendezvous2 before the last CPU has actually left the loop in 227 // smp_cpu_rendezvous(); 228 if (cpu == 0) 229 atomic_set((int32*)&sCpuRendezvous3, 0); 230 231 disable_caches(); 232 233 sCpuModule->set_mtrrs(parameter->defaultType, parameter->infos, 234 parameter->count); 235 236 enable_caches(); 237 238 // wait until all CPUs have arrived here 239 smp_cpu_rendezvous(&sCpuRendezvous2); 240 smp_cpu_rendezvous(&sCpuRendezvous3); 241 } 242 243 244 static void 245 init_mtrrs(void* _unused, int cpu) 246 { 247 // wait until all CPUs have arrived here 248 smp_cpu_rendezvous(&sCpuRendezvous); 249 250 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 251 // that initiated the call_all_cpus() from doing that again and clearing 252 // sCpuRendezvous2 before the last CPU has actually left the loop in 253 // smp_cpu_rendezvous(); 254 if (cpu == 0) 255 atomic_set((int32*)&sCpuRendezvous3, 0); 256 257 disable_caches(); 258 259 sCpuModule->init_mtrrs(); 260 261 enable_caches(); 262 263 // wait until all CPUs have arrived here 264 smp_cpu_rendezvous(&sCpuRendezvous2); 265 smp_cpu_rendezvous(&sCpuRendezvous3); 266 } 267 268 269 uint32 270 x86_count_mtrrs(void) 271 { 272 if (sCpuModule == NULL) 273 return 0; 274 275 return sCpuModule->count_mtrrs(); 276 } 277 278 279 void 280 x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type) 281 { 282 struct set_mtrr_parameter parameter; 283 parameter.index = index; 284 parameter.base = base; 285 parameter.length = length; 286 parameter.type = type; 287 288 sCpuRendezvous = sCpuRendezvous2 = 0; 289 call_all_cpus(&set_mtrr, ¶meter); 290 } 291 292 293 status_t 294 x86_get_mtrr(uint32 index, uint64* _base, uint64* _length, uint8* _type) 295 { 296 // the MTRRs are identical on all CPUs, so it doesn't matter 297 // on which CPU this runs 298 return sCpuModule->get_mtrr(index, _base, _length, _type); 299 } 300 301 302 void 303 x86_set_mtrrs(uint8 defaultType, const x86_mtrr_info* infos, uint32 count) 304 { 305 if (sCpuModule == NULL) 306 return; 307 308 struct set_mtrrs_parameter parameter; 309 parameter.defaultType = defaultType; 310 parameter.infos = infos; 311 parameter.count = count; 312 313 sCpuRendezvous = sCpuRendezvous2 = 0; 314 call_all_cpus(&set_mtrrs, ¶meter); 315 } 316 317 318 void 319 x86_init_fpu(void) 320 { 321 // All x86_64 CPUs support SSE, don't need to bother checking for it. 322 #ifndef __x86_64__ 323 if (!x86_check_feature(IA32_FEATURE_FPU, FEATURE_COMMON)) { 324 // No FPU... time to install one in your 386? 325 dprintf("%s: Warning: CPU has no reported FPU.\n", __func__); 326 gX86SwapFPUFunc = x86_noop_swap; 327 return; 328 } 329 330 if (!x86_check_feature(IA32_FEATURE_SSE, FEATURE_COMMON) 331 || !x86_check_feature(IA32_FEATURE_FXSR, FEATURE_COMMON)) { 332 dprintf("%s: CPU has no SSE... just enabling FPU.\n", __func__); 333 // we don't have proper SSE support, just enable FPU 334 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 335 gX86SwapFPUFunc = x86_fnsave_swap; 336 return; 337 } 338 #endif 339 340 dprintf("%s: CPU has SSE... enabling FXSR and XMM.\n", __func__); 341 342 // enable OS support for SSE 343 x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION); 344 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 345 346 gX86SwapFPUFunc = x86_fxsave_swap; 347 gHasSSE = true; 348 } 349 350 351 #if DUMP_FEATURE_STRING 352 static void 353 dump_feature_string(int currentCPU, cpu_ent* cpu) 354 { 355 char features[384]; 356 features[0] = 0; 357 358 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FPU) 359 strlcat(features, "fpu ", sizeof(features)); 360 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_VME) 361 strlcat(features, "vme ", sizeof(features)); 362 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DE) 363 strlcat(features, "de ", sizeof(features)); 364 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE) 365 strlcat(features, "pse ", sizeof(features)); 366 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TSC) 367 strlcat(features, "tsc ", sizeof(features)); 368 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MSR) 369 strlcat(features, "msr ", sizeof(features)); 370 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAE) 371 strlcat(features, "pae ", sizeof(features)); 372 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCE) 373 strlcat(features, "mce ", sizeof(features)); 374 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CX8) 375 strlcat(features, "cx8 ", sizeof(features)); 376 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_APIC) 377 strlcat(features, "apic ", sizeof(features)); 378 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SEP) 379 strlcat(features, "sep ", sizeof(features)); 380 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MTRR) 381 strlcat(features, "mtrr ", sizeof(features)); 382 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PGE) 383 strlcat(features, "pge ", sizeof(features)); 384 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCA) 385 strlcat(features, "mca ", sizeof(features)); 386 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CMOV) 387 strlcat(features, "cmov ", sizeof(features)); 388 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAT) 389 strlcat(features, "pat ", sizeof(features)); 390 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE36) 391 strlcat(features, "pse36 ", sizeof(features)); 392 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSN) 393 strlcat(features, "psn ", sizeof(features)); 394 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CLFSH) 395 strlcat(features, "clfsh ", sizeof(features)); 396 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DS) 397 strlcat(features, "ds ", sizeof(features)); 398 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_ACPI) 399 strlcat(features, "acpi ", sizeof(features)); 400 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MMX) 401 strlcat(features, "mmx ", sizeof(features)); 402 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FXSR) 403 strlcat(features, "fxsr ", sizeof(features)); 404 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE) 405 strlcat(features, "sse ", sizeof(features)); 406 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE2) 407 strlcat(features, "sse2 ", sizeof(features)); 408 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SS) 409 strlcat(features, "ss ", sizeof(features)); 410 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_HTT) 411 strlcat(features, "htt ", sizeof(features)); 412 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TM) 413 strlcat(features, "tm ", sizeof(features)); 414 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PBE) 415 strlcat(features, "pbe ", sizeof(features)); 416 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE3) 417 strlcat(features, "sse3 ", sizeof(features)); 418 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCLMULQDQ) 419 strlcat(features, "pclmulqdq ", sizeof(features)); 420 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DTES64) 421 strlcat(features, "dtes64 ", sizeof(features)); 422 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MONITOR) 423 strlcat(features, "monitor ", sizeof(features)); 424 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DSCPL) 425 strlcat(features, "dscpl ", sizeof(features)); 426 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_VMX) 427 strlcat(features, "vmx ", sizeof(features)); 428 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SMX) 429 strlcat(features, "smx ", sizeof(features)); 430 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_EST) 431 strlcat(features, "est ", sizeof(features)); 432 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TM2) 433 strlcat(features, "tm2 ", sizeof(features)); 434 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSSE3) 435 strlcat(features, "ssse3 ", sizeof(features)); 436 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CNXTID) 437 strlcat(features, "cnxtid ", sizeof(features)); 438 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_FMA) 439 strlcat(features, "fma ", sizeof(features)); 440 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CX16) 441 strlcat(features, "cx16 ", sizeof(features)); 442 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XTPR) 443 strlcat(features, "xtpr ", sizeof(features)); 444 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PDCM) 445 strlcat(features, "pdcm ", sizeof(features)); 446 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCID) 447 strlcat(features, "pcid ", sizeof(features)); 448 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DCA) 449 strlcat(features, "dca ", sizeof(features)); 450 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_1) 451 strlcat(features, "sse4_1 ", sizeof(features)); 452 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_2) 453 strlcat(features, "sse4_2 ", sizeof(features)); 454 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_X2APIC) 455 strlcat(features, "x2apic ", sizeof(features)); 456 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MOVBE) 457 strlcat(features, "movbe ", sizeof(features)); 458 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_POPCNT) 459 strlcat(features, "popcnt ", sizeof(features)); 460 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TSCDEADLINE) 461 strlcat(features, "tscdeadline ", sizeof(features)); 462 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AES) 463 strlcat(features, "aes ", sizeof(features)); 464 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XSAVE) 465 strlcat(features, "xsave ", sizeof(features)); 466 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_OSXSAVE) 467 strlcat(features, "osxsave ", sizeof(features)); 468 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AVX) 469 strlcat(features, "avx ", sizeof(features)); 470 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_F16C) 471 strlcat(features, "f16c ", sizeof(features)); 472 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_RDRND) 473 strlcat(features, "rdrnd ", sizeof(features)); 474 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) 475 strlcat(features, "hypervisor ", sizeof(features)); 476 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_SYSCALL) 477 strlcat(features, "syscall ", sizeof(features)); 478 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_NX) 479 strlcat(features, "nx ", sizeof(features)); 480 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_MMXEXT) 481 strlcat(features, "mmxext ", sizeof(features)); 482 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_FFXSR) 483 strlcat(features, "ffxsr ", sizeof(features)); 484 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_LONG) 485 strlcat(features, "long ", sizeof(features)); 486 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOWEXT) 487 strlcat(features, "3dnowext ", sizeof(features)); 488 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOW) 489 strlcat(features, "3dnow ", sizeof(features)); 490 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_DTS) 491 strlcat(features, "dts ", sizeof(features)); 492 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ITB) 493 strlcat(features, "itb ", sizeof(features)); 494 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ARAT) 495 strlcat(features, "arat ", sizeof(features)); 496 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PLN) 497 strlcat(features, "pln ", sizeof(features)); 498 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ECMD) 499 strlcat(features, "ecmd ", sizeof(features)); 500 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PTM) 501 strlcat(features, "ptm ", sizeof(features)); 502 if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_APERFMPERF) 503 strlcat(features, "aperfmperf ", sizeof(features)); 504 if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_EPB) 505 strlcat(features, "epb ", sizeof(features)); 506 507 dprintf("CPU %d: features: %s\n", currentCPU, features); 508 } 509 #endif // DUMP_FEATURE_STRING 510 511 512 static void 513 compute_cpu_hierarchy_masks(int maxLogicalID, int maxCoreID) 514 { 515 ASSERT(maxLogicalID >= maxCoreID); 516 const int kMaxSMTID = maxLogicalID / maxCoreID; 517 518 sHierarchyMask[CPU_TOPOLOGY_SMT] = kMaxSMTID - 1; 519 sHierarchyShift[CPU_TOPOLOGY_SMT] = 0; 520 521 sHierarchyMask[CPU_TOPOLOGY_CORE] = (maxCoreID - 1) * kMaxSMTID; 522 sHierarchyShift[CPU_TOPOLOGY_CORE] 523 = count_set_bits(sHierarchyMask[CPU_TOPOLOGY_SMT]); 524 525 const uint32 kSinglePackageMask = sHierarchyMask[CPU_TOPOLOGY_SMT] 526 | sHierarchyMask[CPU_TOPOLOGY_CORE]; 527 sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~kSinglePackageMask; 528 sHierarchyShift[CPU_TOPOLOGY_PACKAGE] = count_set_bits(kSinglePackageMask); 529 } 530 531 532 static uint32 533 get_cpu_legacy_initial_apic_id(int /* currentCPU */) 534 { 535 cpuid_info cpuid; 536 get_current_cpuid(&cpuid, 1, 0); 537 return cpuid.regs.ebx >> 24; 538 } 539 540 541 static inline status_t 542 detect_amd_cpu_topology(uint32 maxBasicLeaf, uint32 maxExtendedLeaf) 543 { 544 sGetCPUTopologyID = get_cpu_legacy_initial_apic_id; 545 546 cpuid_info cpuid; 547 get_current_cpuid(&cpuid, 1, 0); 548 int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff); 549 550 int maxCoreID = 1; 551 if (maxExtendedLeaf >= 0x80000008) { 552 get_current_cpuid(&cpuid, 0x80000008, 0); 553 maxCoreID = (cpuid.regs.ecx >> 12) & 0xf; 554 if (maxCoreID != 0) 555 maxCoreID = 1 << maxCoreID; 556 else 557 maxCoreID = next_power_of_2((cpuid.regs.edx & 0xf) + 1); 558 } 559 560 if (maxExtendedLeaf >= 0x80000001) { 561 get_current_cpuid(&cpuid, 0x80000001, 0); 562 if (x86_check_feature(IA32_FEATURE_AMD_EXT_CMPLEGACY, 563 FEATURE_EXT_AMD_ECX)) 564 maxCoreID = maxLogicalID; 565 } 566 567 compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID); 568 569 return B_OK; 570 } 571 572 573 static void 574 detect_amd_cache_topology(uint32 maxExtendedLeaf) 575 { 576 if (!x86_check_feature(IA32_FEATURE_AMD_EXT_TOPOLOGY, FEATURE_EXT_AMD_ECX)) 577 return; 578 579 if (maxExtendedLeaf < 0x8000001d) 580 return; 581 582 uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL]; 583 int maxCacheLevel = 0; 584 585 int currentLevel = 0; 586 int cacheType; 587 do { 588 cpuid_info cpuid; 589 get_current_cpuid(&cpuid, 0x8000001d, currentLevel); 590 591 cacheType = cpuid.regs.eax & 0x1f; 592 if (cacheType == 0) 593 break; 594 595 int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; 596 int coresCount = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1); 597 hierarchyLevels[cacheLevel - 1] 598 = coresCount * (sHierarchyMask[CPU_TOPOLOGY_SMT] + 1); 599 maxCacheLevel = std::max(maxCacheLevel, cacheLevel); 600 601 currentLevel++; 602 } while (true); 603 604 for (int i = 0; i < maxCacheLevel; i++) 605 sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); 606 gCPUCacheLevelCount = maxCacheLevel; 607 } 608 609 610 static uint32 611 get_intel_cpu_initial_x2apic_id(int /* currentCPU */) 612 { 613 cpuid_info cpuid; 614 get_current_cpuid(&cpuid, 11, 0); 615 return cpuid.regs.edx; 616 } 617 618 619 static inline status_t 620 detect_intel_cpu_topology_x2apic(uint32 maxBasicLeaf) 621 { 622 if (maxBasicLeaf < 11) 623 return B_UNSUPPORTED; 624 625 uint8 hierarchyLevels[CPU_TOPOLOGY_LEVELS] = { 0 }; 626 627 int currentLevel = 0; 628 int levelType; 629 unsigned int levelsSet = 0; 630 631 do { 632 cpuid_info cpuid; 633 get_current_cpuid(&cpuid, 11, currentLevel); 634 if (currentLevel == 0 && cpuid.regs.ebx == 0) 635 return B_UNSUPPORTED; 636 637 levelType = (cpuid.regs.ecx >> 8) & 0xff; 638 int levelValue = cpuid.regs.eax & 0x1f; 639 640 switch (levelType) { 641 case 1: // SMT 642 hierarchyLevels[CPU_TOPOLOGY_SMT] = levelValue; 643 levelsSet |= 1; 644 break; 645 case 2: // core 646 hierarchyLevels[CPU_TOPOLOGY_CORE] = levelValue; 647 levelsSet |= 2; 648 break; 649 } 650 651 currentLevel++; 652 } while (levelType != 0 && levelsSet != 3); 653 654 sGetCPUTopologyID = get_intel_cpu_initial_x2apic_id; 655 656 for (int i = 1; i < CPU_TOPOLOGY_LEVELS; i++) { 657 if ((levelsSet & (1u << i)) != 0) 658 continue; 659 hierarchyLevels[i] = hierarchyLevels[i - 1]; 660 } 661 662 for (int i = 0; i < CPU_TOPOLOGY_LEVELS; i++) { 663 uint32 mask = ~uint32(0); 664 if (i < CPU_TOPOLOGY_LEVELS - 1) 665 mask = (1u << hierarchyLevels[i]) - 1; 666 if (i > 0) 667 mask &= ~sHierarchyMask[i - 1]; 668 sHierarchyMask[i] = mask; 669 sHierarchyShift[i] = i > 0 ? hierarchyLevels[i - 1] : 0; 670 } 671 672 return B_OK; 673 } 674 675 676 static inline status_t 677 detect_intel_cpu_topology_legacy(uint32 maxBasicLeaf) 678 { 679 sGetCPUTopologyID = get_cpu_legacy_initial_apic_id; 680 681 cpuid_info cpuid; 682 683 get_current_cpuid(&cpuid, 1, 0); 684 int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff); 685 686 int maxCoreID = 1; 687 if (maxBasicLeaf >= 4) { 688 get_current_cpuid(&cpuid, 4, 0); 689 maxCoreID = next_power_of_2((cpuid.regs.eax >> 26) + 1); 690 } 691 692 compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID); 693 694 return B_OK; 695 } 696 697 698 static void 699 detect_intel_cache_topology(uint32 maxBasicLeaf) 700 { 701 if (maxBasicLeaf < 4) 702 return; 703 704 uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL]; 705 int maxCacheLevel = 0; 706 707 int currentLevel = 0; 708 int cacheType; 709 do { 710 cpuid_info cpuid; 711 get_current_cpuid(&cpuid, 4, currentLevel); 712 713 cacheType = cpuid.regs.eax & 0x1f; 714 if (cacheType == 0) 715 break; 716 717 int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; 718 hierarchyLevels[cacheLevel - 1] 719 = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1); 720 maxCacheLevel = std::max(maxCacheLevel, cacheLevel); 721 722 currentLevel++; 723 } while (true); 724 725 for (int i = 0; i < maxCacheLevel; i++) 726 sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); 727 728 gCPUCacheLevelCount = maxCacheLevel; 729 } 730 731 732 static uint32 733 get_simple_cpu_topology_id(int currentCPU) 734 { 735 return currentCPU; 736 } 737 738 739 static inline int 740 get_topology_level_id(uint32 id, cpu_topology_level level) 741 { 742 ASSERT(level < CPU_TOPOLOGY_LEVELS); 743 return (id & sHierarchyMask[level]) >> sHierarchyShift[level]; 744 } 745 746 747 static void 748 detect_cpu_topology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, 749 uint32 maxExtendedLeaf) 750 { 751 if (currentCPU == 0) { 752 memset(sCacheSharingMask, 0xff, sizeof(sCacheSharingMask)); 753 754 status_t result = B_UNSUPPORTED; 755 if (x86_check_feature(IA32_FEATURE_HTT, FEATURE_COMMON)) { 756 if (cpu->arch.vendor == VENDOR_AMD) { 757 result = detect_amd_cpu_topology(maxBasicLeaf, maxExtendedLeaf); 758 759 if (result == B_OK) 760 detect_amd_cache_topology(maxExtendedLeaf); 761 } 762 763 if (cpu->arch.vendor == VENDOR_INTEL) { 764 result = detect_intel_cpu_topology_x2apic(maxBasicLeaf); 765 if (result != B_OK) 766 result = detect_intel_cpu_topology_legacy(maxBasicLeaf); 767 768 if (result == B_OK) 769 detect_intel_cache_topology(maxBasicLeaf); 770 } 771 } 772 773 if (result != B_OK) { 774 dprintf("No CPU topology information available.\n"); 775 776 sGetCPUTopologyID = get_simple_cpu_topology_id; 777 778 sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~uint32(0); 779 } 780 } 781 782 ASSERT(sGetCPUTopologyID != NULL); 783 int topologyID = sGetCPUTopologyID(currentCPU); 784 cpu->topology_id[CPU_TOPOLOGY_SMT] 785 = get_topology_level_id(topologyID, CPU_TOPOLOGY_SMT); 786 cpu->topology_id[CPU_TOPOLOGY_CORE] 787 = get_topology_level_id(topologyID, CPU_TOPOLOGY_CORE); 788 cpu->topology_id[CPU_TOPOLOGY_PACKAGE] 789 = get_topology_level_id(topologyID, CPU_TOPOLOGY_PACKAGE); 790 791 unsigned int i; 792 for (i = 0; i < gCPUCacheLevelCount; i++) 793 cpu->cache_id[i] = topologyID & sCacheSharingMask[i]; 794 for (; i < CPU_MAX_CACHE_LEVEL; i++) 795 cpu->cache_id[i] = -1; 796 797 #if DUMP_CPU_TOPOLOGY 798 dprintf("CPU %d: apic id %d, package %d, core %d, smt %d\n", currentCPU, 799 topologyID, cpu->topology_id[CPU_TOPOLOGY_PACKAGE], 800 cpu->topology_id[CPU_TOPOLOGY_CORE], 801 cpu->topology_id[CPU_TOPOLOGY_SMT]); 802 803 if (gCPUCacheLevelCount > 0) { 804 char cacheLevels[256]; 805 unsigned int offset = 0; 806 for (i = 0; i < gCPUCacheLevelCount; i++) { 807 offset += snprintf(cacheLevels + offset, 808 sizeof(cacheLevels) - offset, 809 " L%d id %d%s", i + 1, cpu->cache_id[i], 810 i < gCPUCacheLevelCount - 1 ? "," : ""); 811 812 if (offset >= sizeof(cacheLevels)) 813 break; 814 } 815 816 dprintf("CPU %d: cache sharing:%s\n", currentCPU, cacheLevels); 817 } 818 #endif 819 } 820 821 822 static void 823 detect_cpu(int currentCPU) 824 { 825 cpu_ent* cpu = get_cpu_struct(); 826 char vendorString[17]; 827 cpuid_info cpuid; 828 829 // clear out the cpu info data 830 cpu->arch.vendor = VENDOR_UNKNOWN; 831 cpu->arch.vendor_name = "UNKNOWN VENDOR"; 832 cpu->arch.feature[FEATURE_COMMON] = 0; 833 cpu->arch.feature[FEATURE_EXT] = 0; 834 cpu->arch.feature[FEATURE_EXT_AMD] = 0; 835 cpu->arch.model_name[0] = 0; 836 837 // print some fun data 838 get_current_cpuid(&cpuid, 0, 0); 839 uint32 maxBasicLeaf = cpuid.eax_0.max_eax; 840 841 // build the vendor string 842 memset(vendorString, 0, sizeof(vendorString)); 843 memcpy(vendorString, cpuid.eax_0.vendor_id, sizeof(cpuid.eax_0.vendor_id)); 844 845 // get the family, model, stepping 846 get_current_cpuid(&cpuid, 1, 0); 847 cpu->arch.type = cpuid.eax_1.type; 848 cpu->arch.family = cpuid.eax_1.family; 849 cpu->arch.extended_family = cpuid.eax_1.extended_family; 850 cpu->arch.model = cpuid.eax_1.model; 851 cpu->arch.extended_model = cpuid.eax_1.extended_model; 852 cpu->arch.stepping = cpuid.eax_1.stepping; 853 dprintf("CPU %d: type %d family %d extended_family %d model %d " 854 "extended_model %d stepping %d, string '%s'\n", 855 currentCPU, cpu->arch.type, cpu->arch.family, 856 cpu->arch.extended_family, cpu->arch.model, 857 cpu->arch.extended_model, cpu->arch.stepping, vendorString); 858 859 // figure out what vendor we have here 860 861 for (int32 i = 0; i < VENDOR_NUM; i++) { 862 if (vendor_info[i].ident_string[0] 863 && !strcmp(vendorString, vendor_info[i].ident_string[0])) { 864 cpu->arch.vendor = (x86_vendors)i; 865 cpu->arch.vendor_name = vendor_info[i].vendor; 866 break; 867 } 868 if (vendor_info[i].ident_string[1] 869 && !strcmp(vendorString, vendor_info[i].ident_string[1])) { 870 cpu->arch.vendor = (x86_vendors)i; 871 cpu->arch.vendor_name = vendor_info[i].vendor; 872 break; 873 } 874 } 875 876 // see if we can get the model name 877 get_current_cpuid(&cpuid, 0x80000000, 0); 878 uint32 maxExtendedLeaf = cpuid.eax_0.max_eax; 879 if (maxExtendedLeaf >= 0x80000004) { 880 // build the model string (need to swap ecx/edx data before copying) 881 unsigned int temp; 882 memset(cpu->arch.model_name, 0, sizeof(cpu->arch.model_name)); 883 884 get_current_cpuid(&cpuid, 0x80000002, 0); 885 temp = cpuid.regs.edx; 886 cpuid.regs.edx = cpuid.regs.ecx; 887 cpuid.regs.ecx = temp; 888 memcpy(cpu->arch.model_name, cpuid.as_chars, sizeof(cpuid.as_chars)); 889 890 get_current_cpuid(&cpuid, 0x80000003, 0); 891 temp = cpuid.regs.edx; 892 cpuid.regs.edx = cpuid.regs.ecx; 893 cpuid.regs.ecx = temp; 894 memcpy(cpu->arch.model_name + 16, cpuid.as_chars, 895 sizeof(cpuid.as_chars)); 896 897 get_current_cpuid(&cpuid, 0x80000004, 0); 898 temp = cpuid.regs.edx; 899 cpuid.regs.edx = cpuid.regs.ecx; 900 cpuid.regs.ecx = temp; 901 memcpy(cpu->arch.model_name + 32, cpuid.as_chars, 902 sizeof(cpuid.as_chars)); 903 904 // some cpus return a right-justified string 905 int32 i = 0; 906 while (cpu->arch.model_name[i] == ' ') 907 i++; 908 if (i > 0) { 909 memmove(cpu->arch.model_name, &cpu->arch.model_name[i], 910 strlen(&cpu->arch.model_name[i]) + 1); 911 } 912 913 dprintf("CPU %d: vendor '%s' model name '%s'\n", 914 currentCPU, cpu->arch.vendor_name, cpu->arch.model_name); 915 } else { 916 strlcpy(cpu->arch.model_name, "unknown", sizeof(cpu->arch.model_name)); 917 } 918 919 // load feature bits 920 get_current_cpuid(&cpuid, 1, 0); 921 cpu->arch.feature[FEATURE_COMMON] = cpuid.eax_1.features; // edx 922 cpu->arch.feature[FEATURE_EXT] = cpuid.eax_1.extended_features; // ecx 923 924 if (maxExtendedLeaf >= 0x80000001) { 925 get_current_cpuid(&cpuid, 0x80000001, 0); 926 if (cpu->arch.vendor == VENDOR_AMD) 927 cpu->arch.feature[FEATURE_EXT_AMD_ECX] = cpuid.regs.ecx; // ecx 928 cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx 929 if (cpu->arch.vendor != VENDOR_AMD) 930 cpu->arch.feature[FEATURE_EXT_AMD] &= IA32_FEATURES_INTEL_EXT; 931 } 932 933 if (maxBasicLeaf >= 5) { 934 get_current_cpuid(&cpuid, 5, 0); 935 cpu->arch.feature[FEATURE_5_ECX] = cpuid.regs.ecx; 936 } 937 938 if (maxBasicLeaf >= 6) { 939 get_current_cpuid(&cpuid, 6, 0); 940 cpu->arch.feature[FEATURE_6_EAX] = cpuid.regs.eax; 941 cpu->arch.feature[FEATURE_6_ECX] = cpuid.regs.ecx; 942 } 943 944 if (maxExtendedLeaf >= 0x80000007) { 945 get_current_cpuid(&cpuid, 0x80000007, 0); 946 cpu->arch.feature[FEATURE_EXT_7_EDX] = cpuid.regs.edx; 947 } 948 949 detect_cpu_topology(currentCPU, cpu, maxBasicLeaf, maxExtendedLeaf); 950 951 #if DUMP_FEATURE_STRING 952 dump_feature_string(currentCPU, cpu); 953 #endif 954 } 955 956 957 bool 958 x86_check_feature(uint32 feature, enum x86_feature_type type) 959 { 960 cpu_ent* cpu = get_cpu_struct(); 961 962 #if 0 963 int i; 964 dprintf("x86_check_feature: feature 0x%x, type %d\n", feature, type); 965 for (i = 0; i < FEATURE_NUM; i++) { 966 dprintf("features %d: 0x%x\n", i, cpu->arch.feature[i]); 967 } 968 #endif 969 970 return (cpu->arch.feature[type] & feature) != 0; 971 } 972 973 974 void* 975 x86_get_double_fault_stack(int32 cpu, size_t* _size) 976 { 977 *_size = kDoubleFaultStackSize; 978 return sDoubleFaultStacks + kDoubleFaultStackSize * cpu; 979 } 980 981 982 /*! Returns the index of the current CPU. Can only be called from the double 983 fault handler. 984 */ 985 int32 986 x86_double_fault_get_cpu(void) 987 { 988 addr_t stack = x86_get_stack_frame(); 989 return (stack - (addr_t)sDoubleFaultStacks) / kDoubleFaultStackSize; 990 } 991 992 993 // #pragma mark - 994 995 996 status_t 997 arch_cpu_preboot_init_percpu(kernel_args* args, int cpu) 998 { 999 // On SMP system we want to synchronize the CPUs' TSCs, so system_time() 1000 // will return consistent values. 1001 if (smp_get_num_cpus() > 1) { 1002 // let the first CPU prepare the rendezvous point 1003 if (cpu == 0) 1004 sTSCSyncRendezvous = smp_get_num_cpus() - 1; 1005 1006 // One CPU after the other will drop out of this loop and be caught by 1007 // the loop below, until the last CPU (0) gets there. Save for +/- a few 1008 // cycles the CPUs should pass the second loop at the same time. 1009 while (sTSCSyncRendezvous != cpu) { 1010 } 1011 1012 sTSCSyncRendezvous = cpu - 1; 1013 1014 while (sTSCSyncRendezvous != -1) { 1015 } 1016 1017 // reset TSC to 0 1018 x86_write_msr(IA32_MSR_TSC, 0); 1019 } 1020 1021 x86_descriptors_preboot_init_percpu(args, cpu); 1022 1023 return B_OK; 1024 } 1025 1026 1027 static void 1028 halt_idle(void) 1029 { 1030 asm("hlt"); 1031 } 1032 1033 1034 static void 1035 amdc1e_noarat_idle(void) 1036 { 1037 uint64 msr = x86_read_msr(K8_MSR_IPM); 1038 if (msr & K8_CMPHALT) 1039 x86_write_msr(K8_MSR_IPM, msr & ~K8_CMPHALT); 1040 halt_idle(); 1041 } 1042 1043 1044 static bool 1045 detect_amdc1e_noarat() 1046 { 1047 cpu_ent* cpu = get_cpu_struct(); 1048 1049 if (cpu->arch.vendor != VENDOR_AMD) 1050 return false; 1051 1052 // Family 0x12 and higher processors support ARAT 1053 // Family lower than 0xf processors doesn't support C1E 1054 // Family 0xf with model <= 0x40 procssors doesn't support C1E 1055 uint32 family = cpu->arch.family + cpu->arch.extended_family; 1056 uint32 model = (cpu->arch.extended_model << 4) | cpu->arch.model; 1057 return (family < 0x12 && family > 0xf) || (family == 0xf && model > 0x40); 1058 } 1059 1060 1061 status_t 1062 arch_cpu_init_percpu(kernel_args* args, int cpu) 1063 { 1064 detect_cpu(cpu); 1065 1066 if (!gCpuIdleFunc) { 1067 if (detect_amdc1e_noarat()) 1068 gCpuIdleFunc = amdc1e_noarat_idle; 1069 else 1070 gCpuIdleFunc = halt_idle; 1071 } 1072 1073 return B_OK; 1074 } 1075 1076 1077 status_t 1078 arch_cpu_init(kernel_args* args) 1079 { 1080 // init the TSC -> system_time() conversion factors 1081 1082 uint32 conversionFactor = args->arch_args.system_time_cv_factor; 1083 uint64 conversionFactorNsecs = (uint64)conversionFactor * 1000; 1084 1085 #ifdef __x86_64__ 1086 // The x86_64 system_time() implementation uses 64-bit multiplication and 1087 // therefore shifting is not necessary for low frequencies (it's also not 1088 // too likely that there'll be any x86_64 CPUs clocked under 1GHz). 1089 __x86_setup_system_time((uint64)conversionFactor << 32, 1090 conversionFactorNsecs); 1091 #else 1092 if (conversionFactorNsecs >> 32 != 0) { 1093 // the TSC frequency is < 1 GHz, which forces us to shift the factor 1094 __x86_setup_system_time(conversionFactor, conversionFactorNsecs >> 16, 1095 true); 1096 } else { 1097 // the TSC frequency is >= 1 GHz 1098 __x86_setup_system_time(conversionFactor, conversionFactorNsecs, false); 1099 } 1100 #endif 1101 1102 // Initialize descriptor tables. 1103 x86_descriptors_init(args); 1104 1105 return B_OK; 1106 } 1107 1108 1109 status_t 1110 arch_cpu_init_post_vm(kernel_args* args) 1111 { 1112 uint32 i; 1113 1114 // allocate an area for the double fault stacks 1115 virtual_address_restrictions virtualRestrictions = {}; 1116 virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS; 1117 physical_address_restrictions physicalRestrictions = {}; 1118 create_area_etc(B_SYSTEM_TEAM, "double fault stacks", 1119 kDoubleFaultStackSize * smp_get_num_cpus(), B_FULL_LOCK, 1120 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0, 1121 &virtualRestrictions, &physicalRestrictions, 1122 (void**)&sDoubleFaultStacks); 1123 1124 X86PagingStructures* kernelPagingStructures 1125 = static_cast<X86VMTranslationMap*>( 1126 VMAddressSpace::Kernel()->TranslationMap())->PagingStructures(); 1127 1128 // Set active translation map on each CPU. 1129 for (i = 0; i < args->num_cpus; i++) { 1130 gCPU[i].arch.active_paging_structures = kernelPagingStructures; 1131 kernelPagingStructures->AddReference(); 1132 } 1133 1134 if (!apic_available()) 1135 x86_init_fpu(); 1136 // else fpu gets set up in smp code 1137 1138 return B_OK; 1139 } 1140 1141 1142 status_t 1143 arch_cpu_init_post_modules(kernel_args* args) 1144 { 1145 // initialize CPU module 1146 1147 void* cookie = open_module_list("cpu"); 1148 1149 while (true) { 1150 char name[B_FILE_NAME_LENGTH]; 1151 size_t nameLength = sizeof(name); 1152 1153 if (read_next_module_name(cookie, name, &nameLength) != B_OK 1154 || get_module(name, (module_info**)&sCpuModule) == B_OK) 1155 break; 1156 } 1157 1158 close_module_list(cookie); 1159 1160 // initialize MTRRs if available 1161 if (x86_count_mtrrs() > 0) { 1162 sCpuRendezvous = sCpuRendezvous2 = 0; 1163 call_all_cpus(&init_mtrrs, NULL); 1164 } 1165 1166 // get optimized functions from the CPU module 1167 if (sCpuModule != NULL && sCpuModule->get_optimized_functions != NULL) { 1168 x86_optimized_functions functions; 1169 memset(&functions, 0, sizeof(functions)); 1170 1171 sCpuModule->get_optimized_functions(&functions); 1172 1173 if (functions.memcpy != NULL) { 1174 gOptimizedFunctions.memcpy = functions.memcpy; 1175 gOptimizedFunctions.memcpy_end = functions.memcpy_end; 1176 } 1177 1178 if (functions.memset != NULL) { 1179 gOptimizedFunctions.memset = functions.memset; 1180 gOptimizedFunctions.memset_end = functions.memset_end; 1181 } 1182 } 1183 1184 // put the optimized functions into the commpage 1185 size_t memcpyLen = (addr_t)gOptimizedFunctions.memcpy_end 1186 - (addr_t)gOptimizedFunctions.memcpy; 1187 addr_t memcpyPosition = fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMCPY, 1188 (const void*)gOptimizedFunctions.memcpy, memcpyLen); 1189 size_t memsetLen = (addr_t)gOptimizedFunctions.memset_end 1190 - (addr_t)gOptimizedFunctions.memset; 1191 addr_t memsetPosition = fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMSET, 1192 (const void*)gOptimizedFunctions.memset, memsetLen); 1193 size_t threadExitLen = (addr_t)x86_end_userspace_thread_exit 1194 - (addr_t)x86_userspace_thread_exit; 1195 addr_t threadExitPosition = fill_commpage_entry( 1196 COMMPAGE_ENTRY_X86_THREAD_EXIT, (const void*)x86_userspace_thread_exit, 1197 threadExitLen); 1198 1199 // add the functions to the commpage image 1200 image_id image = get_commpage_image(); 1201 elf_add_memory_image_symbol(image, "commpage_memcpy", memcpyPosition, 1202 memcpyLen, B_SYMBOL_TYPE_TEXT); 1203 elf_add_memory_image_symbol(image, "commpage_memset", memsetPosition, 1204 memsetLen, B_SYMBOL_TYPE_TEXT); 1205 elf_add_memory_image_symbol(image, "commpage_thread_exit", 1206 threadExitPosition, threadExitLen, B_SYMBOL_TYPE_TEXT); 1207 1208 return B_OK; 1209 } 1210 1211 1212 void 1213 arch_cpu_user_TLB_invalidate(void) 1214 { 1215 x86_write_cr3(x86_read_cr3()); 1216 } 1217 1218 1219 void 1220 arch_cpu_global_TLB_invalidate(void) 1221 { 1222 uint32 flags = x86_read_cr4(); 1223 1224 if (flags & IA32_CR4_GLOBAL_PAGES) { 1225 // disable and reenable the global pages to flush all TLBs regardless 1226 // of the global page bit 1227 x86_write_cr4(flags & ~IA32_CR4_GLOBAL_PAGES); 1228 x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES); 1229 } else { 1230 cpu_status state = disable_interrupts(); 1231 arch_cpu_user_TLB_invalidate(); 1232 restore_interrupts(state); 1233 } 1234 } 1235 1236 1237 void 1238 arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) 1239 { 1240 int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE; 1241 while (num_pages-- >= 0) { 1242 invalidate_TLB(start); 1243 start += B_PAGE_SIZE; 1244 } 1245 } 1246 1247 1248 void 1249 arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) 1250 { 1251 int i; 1252 for (i = 0; i < num_pages; i++) { 1253 invalidate_TLB(pages[i]); 1254 } 1255 } 1256 1257 1258 status_t 1259 arch_cpu_shutdown(bool rebootSystem) 1260 { 1261 if (acpi_shutdown(rebootSystem) == B_OK) 1262 return B_OK; 1263 1264 if (!rebootSystem) { 1265 #ifndef __x86_64__ 1266 return apm_shutdown(); 1267 #else 1268 return B_NOT_SUPPORTED; 1269 #endif 1270 } 1271 1272 cpu_status state = disable_interrupts(); 1273 1274 // try to reset the system using the keyboard controller 1275 out8(0xfe, 0x64); 1276 1277 // Give some time to the controller to do its job (0.5s) 1278 snooze(500000); 1279 1280 // if that didn't help, try it this way 1281 x86_reboot(); 1282 1283 restore_interrupts(state); 1284 return B_ERROR; 1285 } 1286 1287 1288 void 1289 arch_cpu_sync_icache(void* address, size_t length) 1290 { 1291 // instruction cache is always consistent on x86 1292 } 1293 1294