1 /* 2 * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. 3 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. 4 * Distributed under the terms of the MIT License. 5 * 6 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 7 * Distributed under the terms of the NewOS License. 8 */ 9 10 11 #include <cpu.h> 12 13 #include <string.h> 14 #include <stdlib.h> 15 #include <stdio.h> 16 17 #include <ACPI.h> 18 19 #include <boot_device.h> 20 #include <commpage.h> 21 #include <debug.h> 22 #include <elf.h> 23 #include <smp.h> 24 #include <vm/vm.h> 25 #include <vm/vm_types.h> 26 #include <vm/VMAddressSpace.h> 27 28 #include <arch_system_info.h> 29 #include <arch/x86/apic.h> 30 #include <boot/kernel_args.h> 31 32 #include "paging/X86PagingStructures.h" 33 #include "paging/X86VMTranslationMap.h" 34 35 36 #define DUMP_FEATURE_STRING 1 37 38 39 /* cpu vendor info */ 40 struct cpu_vendor_info { 41 const char *vendor; 42 const char *ident_string[2]; 43 }; 44 45 static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = { 46 { "Intel", { "GenuineIntel" } }, 47 { "AMD", { "AuthenticAMD" } }, 48 { "Cyrix", { "CyrixInstead" } }, 49 { "UMC", { "UMC UMC UMC" } }, 50 { "NexGen", { "NexGenDriven" } }, 51 { "Centaur", { "CentaurHauls" } }, 52 { "Rise", { "RiseRiseRise" } }, 53 { "Transmeta", { "GenuineTMx86", "TransmetaCPU" } }, 54 { "NSC", { "Geode by NSC" } }, 55 }; 56 57 #define CR0_CACHE_DISABLE (1UL << 30) 58 #define CR0_NOT_WRITE_THROUGH (1UL << 29) 59 #define CR0_FPU_EMULATION (1UL << 2) 60 #define CR0_MONITOR_FPU (1UL << 1) 61 62 #define CR4_OS_FXSR (1UL << 9) 63 #define CR4_OS_XMM_EXCEPTION (1UL << 10) 64 65 #define K8_SMIONCMPHALT (1ULL << 27) 66 #define K8_C1EONCMPHALT (1ULL << 28) 67 68 #define K8_CMPHALT (K8_SMIONCMPHALT | K8_C1EONCMPHALT) 69 70 /* 71 * 0 favors highest performance while 15 corresponds to the maximum energy 72 * savings. 7 means balance between performance and energy savings. 73 * Refer to Section 14.3.4 in <Intel 64 and IA-32 Architectures Software 74 * Developer's Manual Volume 3> for details 75 */ 76 #define ENERGY_PERF_BIAS_PERFORMANCE 0 77 #define ENERGY_PERF_BIAS_BALANCE 7 78 #define ENERGY_PERF_BIAS_POWERSAVE 15 79 80 struct set_mtrr_parameter { 81 int32 index; 82 uint64 base; 83 uint64 length; 84 uint8 type; 85 }; 86 87 struct set_mtrrs_parameter { 88 const x86_mtrr_info* infos; 89 uint32 count; 90 uint8 defaultType; 91 }; 92 93 94 extern "C" void x86_reboot(void); 95 // from arch.S 96 97 void (*gCpuIdleFunc)(void); 98 void (*gX86SwapFPUFunc)(void* oldState, const void* newState) = x86_noop_swap; 99 bool gHasSSE = false; 100 101 static uint32 sCpuRendezvous; 102 static uint32 sCpuRendezvous2; 103 static uint32 sCpuRendezvous3; 104 static vint32 sTSCSyncRendezvous; 105 106 /* Some specials for the double fault handler */ 107 static uint8* sDoubleFaultStacks; 108 static const size_t kDoubleFaultStackSize = 4096; // size per CPU 109 110 static x86_cpu_module_info* sCpuModule; 111 112 113 extern "C" void memcpy_generic(void* dest, const void* source, size_t count); 114 extern int memcpy_generic_end; 115 extern "C" void memset_generic(void* dest, int value, size_t count); 116 extern int memset_generic_end; 117 118 x86_optimized_functions gOptimizedFunctions = { 119 memcpy_generic, 120 &memcpy_generic_end, 121 memset_generic, 122 &memset_generic_end 123 }; 124 125 126 static status_t 127 acpi_shutdown(bool rebootSystem) 128 { 129 if (debug_debugger_running() || !are_interrupts_enabled()) 130 return B_ERROR; 131 132 acpi_module_info* acpi; 133 if (get_module(B_ACPI_MODULE_NAME, (module_info**)&acpi) != B_OK) 134 return B_NOT_SUPPORTED; 135 136 status_t status; 137 if (rebootSystem) { 138 status = acpi->reboot(); 139 } else { 140 // Make sure we run on the boot CPU (apparently needed for some ACPI 141 // implementations) 142 _user_set_cpu_enabled(0, true); 143 for (int32 cpu = 1; cpu < smp_get_num_cpus(); cpu++) { 144 _user_set_cpu_enabled(cpu, false); 145 } 146 // TODO: must not be called from the idle thread! 147 thread_yield(true); 148 149 status = acpi->prepare_sleep_state(ACPI_POWER_STATE_OFF, NULL, 0); 150 if (status == B_OK) { 151 //cpu_status state = disable_interrupts(); 152 status = acpi->enter_sleep_state(ACPI_POWER_STATE_OFF); 153 //restore_interrupts(state); 154 } 155 } 156 157 put_module(B_ACPI_MODULE_NAME); 158 return status; 159 } 160 161 162 /*! Disable CPU caches, and invalidate them. */ 163 static void 164 disable_caches() 165 { 166 x86_write_cr0((x86_read_cr0() | CR0_CACHE_DISABLE) 167 & ~CR0_NOT_WRITE_THROUGH); 168 wbinvd(); 169 arch_cpu_global_TLB_invalidate(); 170 } 171 172 173 /*! Invalidate CPU caches, and enable them. */ 174 static void 175 enable_caches() 176 { 177 wbinvd(); 178 arch_cpu_global_TLB_invalidate(); 179 x86_write_cr0(x86_read_cr0() 180 & ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH)); 181 } 182 183 184 static void 185 set_mtrr(void* _parameter, int cpu) 186 { 187 struct set_mtrr_parameter* parameter 188 = (struct set_mtrr_parameter*)_parameter; 189 190 // wait until all CPUs have arrived here 191 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 192 193 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 194 // that initiated the call_all_cpus() from doing that again and clearing 195 // sCpuRendezvous2 before the last CPU has actually left the loop in 196 // smp_cpu_rendezvous(); 197 if (cpu == 0) 198 atomic_set((vint32*)&sCpuRendezvous3, 0); 199 200 disable_caches(); 201 202 sCpuModule->set_mtrr(parameter->index, parameter->base, parameter->length, 203 parameter->type); 204 205 enable_caches(); 206 207 // wait until all CPUs have arrived here 208 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 209 smp_cpu_rendezvous(&sCpuRendezvous3, cpu); 210 } 211 212 213 static void 214 set_mtrrs(void* _parameter, int cpu) 215 { 216 set_mtrrs_parameter* parameter = (set_mtrrs_parameter*)_parameter; 217 218 // wait until all CPUs have arrived here 219 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 220 221 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 222 // that initiated the call_all_cpus() from doing that again and clearing 223 // sCpuRendezvous2 before the last CPU has actually left the loop in 224 // smp_cpu_rendezvous(); 225 if (cpu == 0) 226 atomic_set((vint32*)&sCpuRendezvous3, 0); 227 228 disable_caches(); 229 230 sCpuModule->set_mtrrs(parameter->defaultType, parameter->infos, 231 parameter->count); 232 233 enable_caches(); 234 235 // wait until all CPUs have arrived here 236 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 237 smp_cpu_rendezvous(&sCpuRendezvous3, cpu); 238 } 239 240 241 static void 242 init_mtrrs(void* _unused, int cpu) 243 { 244 // wait until all CPUs have arrived here 245 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 246 247 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 248 // that initiated the call_all_cpus() from doing that again and clearing 249 // sCpuRendezvous2 before the last CPU has actually left the loop in 250 // smp_cpu_rendezvous(); 251 if (cpu == 0) 252 atomic_set((vint32*)&sCpuRendezvous3, 0); 253 254 disable_caches(); 255 256 sCpuModule->init_mtrrs(); 257 258 enable_caches(); 259 260 // wait until all CPUs have arrived here 261 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 262 smp_cpu_rendezvous(&sCpuRendezvous3, cpu); 263 } 264 265 266 uint32 267 x86_count_mtrrs(void) 268 { 269 if (sCpuModule == NULL) 270 return 0; 271 272 return sCpuModule->count_mtrrs(); 273 } 274 275 276 void 277 x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type) 278 { 279 struct set_mtrr_parameter parameter; 280 parameter.index = index; 281 parameter.base = base; 282 parameter.length = length; 283 parameter.type = type; 284 285 sCpuRendezvous = sCpuRendezvous2 = 0; 286 call_all_cpus(&set_mtrr, ¶meter); 287 } 288 289 290 status_t 291 x86_get_mtrr(uint32 index, uint64* _base, uint64* _length, uint8* _type) 292 { 293 // the MTRRs are identical on all CPUs, so it doesn't matter 294 // on which CPU this runs 295 return sCpuModule->get_mtrr(index, _base, _length, _type); 296 } 297 298 299 void 300 x86_set_mtrrs(uint8 defaultType, const x86_mtrr_info* infos, uint32 count) 301 { 302 if (sCpuModule == NULL) 303 return; 304 305 struct set_mtrrs_parameter parameter; 306 parameter.defaultType = defaultType; 307 parameter.infos = infos; 308 parameter.count = count; 309 310 sCpuRendezvous = sCpuRendezvous2 = 0; 311 call_all_cpus(&set_mtrrs, ¶meter); 312 } 313 314 315 void 316 x86_init_fpu(void) 317 { 318 // All x86_64 CPUs support SSE, don't need to bother checking for it. 319 #ifndef __x86_64__ 320 if (!x86_check_feature(IA32_FEATURE_FPU, FEATURE_COMMON)) { 321 // No FPU... time to install one in your 386? 322 dprintf("%s: Warning: CPU has no reported FPU.\n", __func__); 323 gX86SwapFPUFunc = x86_noop_swap; 324 return; 325 } 326 327 if (!x86_check_feature(IA32_FEATURE_SSE, FEATURE_COMMON) 328 || !x86_check_feature(IA32_FEATURE_FXSR, FEATURE_COMMON)) { 329 dprintf("%s: CPU has no SSE... just enabling FPU.\n", __func__); 330 // we don't have proper SSE support, just enable FPU 331 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 332 gX86SwapFPUFunc = x86_fnsave_swap; 333 return; 334 } 335 #endif 336 337 dprintf("%s: CPU has SSE... enabling FXSR and XMM.\n", __func__); 338 339 // enable OS support for SSE 340 x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION); 341 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 342 343 gX86SwapFPUFunc = x86_fxsave_swap; 344 gHasSSE = true; 345 } 346 347 348 #if DUMP_FEATURE_STRING 349 static void 350 dump_feature_string(int currentCPU, cpu_ent* cpu) 351 { 352 char features[384]; 353 features[0] = 0; 354 355 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FPU) 356 strlcat(features, "fpu ", sizeof(features)); 357 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_VME) 358 strlcat(features, "vme ", sizeof(features)); 359 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DE) 360 strlcat(features, "de ", sizeof(features)); 361 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE) 362 strlcat(features, "pse ", sizeof(features)); 363 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TSC) 364 strlcat(features, "tsc ", sizeof(features)); 365 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MSR) 366 strlcat(features, "msr ", sizeof(features)); 367 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAE) 368 strlcat(features, "pae ", sizeof(features)); 369 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCE) 370 strlcat(features, "mce ", sizeof(features)); 371 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CX8) 372 strlcat(features, "cx8 ", sizeof(features)); 373 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_APIC) 374 strlcat(features, "apic ", sizeof(features)); 375 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SEP) 376 strlcat(features, "sep ", sizeof(features)); 377 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MTRR) 378 strlcat(features, "mtrr ", sizeof(features)); 379 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PGE) 380 strlcat(features, "pge ", sizeof(features)); 381 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCA) 382 strlcat(features, "mca ", sizeof(features)); 383 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CMOV) 384 strlcat(features, "cmov ", sizeof(features)); 385 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAT) 386 strlcat(features, "pat ", sizeof(features)); 387 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE36) 388 strlcat(features, "pse36 ", sizeof(features)); 389 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSN) 390 strlcat(features, "psn ", sizeof(features)); 391 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CLFSH) 392 strlcat(features, "clfsh ", sizeof(features)); 393 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DS) 394 strlcat(features, "ds ", sizeof(features)); 395 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_ACPI) 396 strlcat(features, "acpi ", sizeof(features)); 397 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MMX) 398 strlcat(features, "mmx ", sizeof(features)); 399 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FXSR) 400 strlcat(features, "fxsr ", sizeof(features)); 401 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE) 402 strlcat(features, "sse ", sizeof(features)); 403 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE2) 404 strlcat(features, "sse2 ", sizeof(features)); 405 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SS) 406 strlcat(features, "ss ", sizeof(features)); 407 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_HTT) 408 strlcat(features, "htt ", sizeof(features)); 409 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TM) 410 strlcat(features, "tm ", sizeof(features)); 411 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PBE) 412 strlcat(features, "pbe ", sizeof(features)); 413 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE3) 414 strlcat(features, "sse3 ", sizeof(features)); 415 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCLMULQDQ) 416 strlcat(features, "pclmulqdq ", sizeof(features)); 417 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DTES64) 418 strlcat(features, "dtes64 ", sizeof(features)); 419 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MONITOR) 420 strlcat(features, "monitor ", sizeof(features)); 421 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DSCPL) 422 strlcat(features, "dscpl ", sizeof(features)); 423 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_VMX) 424 strlcat(features, "vmx ", sizeof(features)); 425 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SMX) 426 strlcat(features, "smx ", sizeof(features)); 427 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_EST) 428 strlcat(features, "est ", sizeof(features)); 429 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TM2) 430 strlcat(features, "tm2 ", sizeof(features)); 431 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSSE3) 432 strlcat(features, "ssse3 ", sizeof(features)); 433 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CNXTID) 434 strlcat(features, "cnxtid ", sizeof(features)); 435 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_FMA) 436 strlcat(features, "fma ", sizeof(features)); 437 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CX16) 438 strlcat(features, "cx16 ", sizeof(features)); 439 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XTPR) 440 strlcat(features, "xtpr ", sizeof(features)); 441 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PDCM) 442 strlcat(features, "pdcm ", sizeof(features)); 443 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCID) 444 strlcat(features, "pcid ", sizeof(features)); 445 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DCA) 446 strlcat(features, "dca ", sizeof(features)); 447 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_1) 448 strlcat(features, "sse4_1 ", sizeof(features)); 449 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_2) 450 strlcat(features, "sse4_2 ", sizeof(features)); 451 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_X2APIC) 452 strlcat(features, "x2apic ", sizeof(features)); 453 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MOVBE) 454 strlcat(features, "movbe ", sizeof(features)); 455 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_POPCNT) 456 strlcat(features, "popcnt ", sizeof(features)); 457 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TSCDEADLINE) 458 strlcat(features, "tscdeadline ", sizeof(features)); 459 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AES) 460 strlcat(features, "aes ", sizeof(features)); 461 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XSAVE) 462 strlcat(features, "xsave ", sizeof(features)); 463 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_OSXSAVE) 464 strlcat(features, "osxsave ", sizeof(features)); 465 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AVX) 466 strlcat(features, "avx ", sizeof(features)); 467 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_F16C) 468 strlcat(features, "f16c ", sizeof(features)); 469 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_RDRND) 470 strlcat(features, "rdrnd ", sizeof(features)); 471 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) 472 strlcat(features, "hypervisor ", sizeof(features)); 473 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_SYSCALL) 474 strlcat(features, "syscall ", sizeof(features)); 475 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_NX) 476 strlcat(features, "nx ", sizeof(features)); 477 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_MMXEXT) 478 strlcat(features, "mmxext ", sizeof(features)); 479 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_FFXSR) 480 strlcat(features, "ffxsr ", sizeof(features)); 481 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_LONG) 482 strlcat(features, "long ", sizeof(features)); 483 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOWEXT) 484 strlcat(features, "3dnowext ", sizeof(features)); 485 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOW) 486 strlcat(features, "3dnow ", sizeof(features)); 487 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_DTS) 488 strlcat(features, "dts ", sizeof(features)); 489 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ITB) 490 strlcat(features, "itb ", sizeof(features)); 491 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ARAT) 492 strlcat(features, "arat ", sizeof(features)); 493 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PLN) 494 strlcat(features, "pln ", sizeof(features)); 495 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ECMD) 496 strlcat(features, "ecmd ", sizeof(features)); 497 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PTM) 498 strlcat(features, "ptm ", sizeof(features)); 499 if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_APERFMPERF) 500 strlcat(features, "aperfmperf ", sizeof(features)); 501 if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_EPB) 502 strlcat(features, "epb ", sizeof(features)); 503 504 dprintf("CPU %d: features: %s\n", currentCPU, features); 505 } 506 #endif // DUMP_FEATURE_STRING 507 508 509 static void 510 detect_cpu(int currentCPU) 511 { 512 cpu_ent* cpu = get_cpu_struct(); 513 char vendorString[17]; 514 cpuid_info cpuid; 515 516 // clear out the cpu info data 517 cpu->arch.vendor = VENDOR_UNKNOWN; 518 cpu->arch.vendor_name = "UNKNOWN VENDOR"; 519 cpu->arch.feature[FEATURE_COMMON] = 0; 520 cpu->arch.feature[FEATURE_EXT] = 0; 521 cpu->arch.feature[FEATURE_EXT_AMD] = 0; 522 cpu->arch.model_name[0] = 0; 523 524 // print some fun data 525 get_current_cpuid(&cpuid, 0); 526 uint32 maxBasicLeaf = cpuid.eax_0.max_eax; 527 528 // build the vendor string 529 memset(vendorString, 0, sizeof(vendorString)); 530 memcpy(vendorString, cpuid.eax_0.vendor_id, sizeof(cpuid.eax_0.vendor_id)); 531 532 // get the family, model, stepping 533 get_current_cpuid(&cpuid, 1); 534 cpu->arch.type = cpuid.eax_1.type; 535 cpu->arch.family = cpuid.eax_1.family; 536 cpu->arch.extended_family = cpuid.eax_1.extended_family; 537 cpu->arch.model = cpuid.eax_1.model; 538 cpu->arch.extended_model = cpuid.eax_1.extended_model; 539 cpu->arch.stepping = cpuid.eax_1.stepping; 540 dprintf("CPU %d: type %d family %d extended_family %d model %d " 541 "extended_model %d stepping %d, string '%s'\n", 542 currentCPU, cpu->arch.type, cpu->arch.family, 543 cpu->arch.extended_family, cpu->arch.model, 544 cpu->arch.extended_model, cpu->arch.stepping, vendorString); 545 546 // figure out what vendor we have here 547 548 for (int32 i = 0; i < VENDOR_NUM; i++) { 549 if (vendor_info[i].ident_string[0] 550 && !strcmp(vendorString, vendor_info[i].ident_string[0])) { 551 cpu->arch.vendor = (x86_vendors)i; 552 cpu->arch.vendor_name = vendor_info[i].vendor; 553 break; 554 } 555 if (vendor_info[i].ident_string[1] 556 && !strcmp(vendorString, vendor_info[i].ident_string[1])) { 557 cpu->arch.vendor = (x86_vendors)i; 558 cpu->arch.vendor_name = vendor_info[i].vendor; 559 break; 560 } 561 } 562 563 // see if we can get the model name 564 get_current_cpuid(&cpuid, 0x80000000); 565 uint32 maxExtendedLeaf = cpuid.eax_0.max_eax; 566 if (maxExtendedLeaf >= 0x80000004) { 567 // build the model string (need to swap ecx/edx data before copying) 568 unsigned int temp; 569 memset(cpu->arch.model_name, 0, sizeof(cpu->arch.model_name)); 570 571 get_current_cpuid(&cpuid, 0x80000002); 572 temp = cpuid.regs.edx; 573 cpuid.regs.edx = cpuid.regs.ecx; 574 cpuid.regs.ecx = temp; 575 memcpy(cpu->arch.model_name, cpuid.as_chars, sizeof(cpuid.as_chars)); 576 577 get_current_cpuid(&cpuid, 0x80000003); 578 temp = cpuid.regs.edx; 579 cpuid.regs.edx = cpuid.regs.ecx; 580 cpuid.regs.ecx = temp; 581 memcpy(cpu->arch.model_name + 16, cpuid.as_chars, 582 sizeof(cpuid.as_chars)); 583 584 get_current_cpuid(&cpuid, 0x80000004); 585 temp = cpuid.regs.edx; 586 cpuid.regs.edx = cpuid.regs.ecx; 587 cpuid.regs.ecx = temp; 588 memcpy(cpu->arch.model_name + 32, cpuid.as_chars, 589 sizeof(cpuid.as_chars)); 590 591 // some cpus return a right-justified string 592 int32 i = 0; 593 while (cpu->arch.model_name[i] == ' ') 594 i++; 595 if (i > 0) { 596 memmove(cpu->arch.model_name, &cpu->arch.model_name[i], 597 strlen(&cpu->arch.model_name[i]) + 1); 598 } 599 600 dprintf("CPU %d: vendor '%s' model name '%s'\n", 601 currentCPU, cpu->arch.vendor_name, cpu->arch.model_name); 602 } else { 603 strlcpy(cpu->arch.model_name, "unknown", sizeof(cpu->arch.model_name)); 604 } 605 606 // load feature bits 607 get_current_cpuid(&cpuid, 1); 608 cpu->arch.feature[FEATURE_COMMON] = cpuid.eax_1.features; // edx 609 cpu->arch.feature[FEATURE_EXT] = cpuid.eax_1.extended_features; // ecx 610 611 if (maxExtendedLeaf >= 0x80000001) { 612 get_current_cpuid(&cpuid, 0x80000001); 613 cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx 614 if (cpu->arch.vendor != VENDOR_AMD) 615 cpu->arch.feature[FEATURE_EXT_AMD] &= IA32_FEATURES_INTEL_EXT; 616 } 617 618 if (maxBasicLeaf >= 6) { 619 get_current_cpuid(&cpuid, 6); 620 cpu->arch.feature[FEATURE_6_EAX] = cpuid.regs.eax; 621 cpu->arch.feature[FEATURE_6_ECX] = cpuid.regs.ecx; 622 } 623 624 #if DUMP_FEATURE_STRING 625 dump_feature_string(currentCPU, cpu); 626 #endif 627 } 628 629 630 bool 631 x86_check_feature(uint32 feature, enum x86_feature_type type) 632 { 633 cpu_ent* cpu = get_cpu_struct(); 634 635 #if 0 636 int i; 637 dprintf("x86_check_feature: feature 0x%x, type %d\n", feature, type); 638 for (i = 0; i < FEATURE_NUM; i++) { 639 dprintf("features %d: 0x%x\n", i, cpu->arch.feature[i]); 640 } 641 #endif 642 643 return (cpu->arch.feature[type] & feature) != 0; 644 } 645 646 647 void* 648 x86_get_double_fault_stack(int32 cpu, size_t* _size) 649 { 650 *_size = kDoubleFaultStackSize; 651 return sDoubleFaultStacks + kDoubleFaultStackSize * cpu; 652 } 653 654 655 /*! Returns the index of the current CPU. Can only be called from the double 656 fault handler. 657 */ 658 int32 659 x86_double_fault_get_cpu(void) 660 { 661 addr_t stack = x86_get_stack_frame(); 662 return (stack - (addr_t)sDoubleFaultStacks) / kDoubleFaultStackSize; 663 } 664 665 666 // #pragma mark - 667 668 669 status_t 670 arch_cpu_preboot_init_percpu(kernel_args* args, int cpu) 671 { 672 // On SMP system we want to synchronize the CPUs' TSCs, so system_time() 673 // will return consistent values. 674 if (smp_get_num_cpus() > 1) { 675 // let the first CPU prepare the rendezvous point 676 if (cpu == 0) 677 sTSCSyncRendezvous = smp_get_num_cpus() - 1; 678 679 // One CPU after the other will drop out of this loop and be caught by 680 // the loop below, until the last CPU (0) gets there. Save for +/- a few 681 // cycles the CPUs should pass the second loop at the same time. 682 while (sTSCSyncRendezvous != cpu) { 683 } 684 685 sTSCSyncRendezvous = cpu - 1; 686 687 while (sTSCSyncRendezvous != -1) { 688 } 689 690 // reset TSC to 0 691 x86_write_msr(IA32_MSR_TSC, 0); 692 } 693 694 return B_OK; 695 } 696 697 698 static void 699 halt_idle(void) 700 { 701 asm("hlt"); 702 } 703 704 705 static void 706 amdc1e_noarat_idle(void) 707 { 708 uint64 msr = x86_read_msr(K8_MSR_IPM); 709 if (msr & K8_CMPHALT) 710 x86_write_msr(K8_MSR_IPM, msr & ~K8_CMPHALT); 711 halt_idle(); 712 } 713 714 715 static bool 716 detect_amdc1e_noarat() 717 { 718 cpu_ent* cpu = get_cpu_struct(); 719 720 if (cpu->arch.vendor != VENDOR_AMD) 721 return false; 722 723 // Family 0x12 and higher processors support ARAT 724 // Family lower than 0xf processors doesn't support C1E 725 // Family 0xf with model <= 0x40 procssors doesn't support C1E 726 uint32 family = cpu->arch.family + cpu->arch.extended_family; 727 uint32 model = (cpu->arch.extended_model << 4) | cpu->arch.model; 728 return (family < 0x12 && family > 0xf) || (family == 0xf && model > 0x40); 729 } 730 731 732 status_t 733 arch_cpu_init_percpu(kernel_args* args, int cpu) 734 { 735 // Load descriptor tables for this CPU. 736 x86_descriptors_init_percpu(args, cpu); 737 738 detect_cpu(cpu); 739 740 if (!gCpuIdleFunc) { 741 if (detect_amdc1e_noarat()) 742 gCpuIdleFunc = amdc1e_noarat_idle; 743 else 744 gCpuIdleFunc = halt_idle; 745 } 746 747 if (x86_check_feature(IA32_FEATURE_EPB, FEATURE_6_ECX)) { 748 uint64 msr = x86_read_msr(IA32_MSR_ENERGY_PERF_BIAS); 749 if ((msr & 0xf) == ENERGY_PERF_BIAS_PERFORMANCE) { 750 msr &= ~0xf; 751 msr |= ENERGY_PERF_BIAS_BALANCE; 752 x86_write_msr(IA32_MSR_ENERGY_PERF_BIAS, msr); 753 } 754 } 755 756 // If availalbe enable NX-bit (No eXecute). Boot CPU can not enable 757 // NX-bit here since PAE should be enabled first. 758 if (cpu != 0) { 759 if (x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) { 760 x86_write_msr(IA32_MSR_EFER, x86_read_msr(IA32_MSR_EFER) 761 | IA32_MSR_EFER_NX); 762 } 763 } 764 765 return B_OK; 766 } 767 768 769 status_t 770 arch_cpu_init(kernel_args* args) 771 { 772 // init the TSC -> system_time() conversion factors 773 774 uint32 conversionFactor = args->arch_args.system_time_cv_factor; 775 uint64 conversionFactorNsecs = (uint64)conversionFactor * 1000; 776 777 #ifdef __x86_64__ 778 // The x86_64 system_time() implementation uses 64-bit multiplication and 779 // therefore shifting is not necessary for low frequencies (it's also not 780 // too likely that there'll be any x86_64 CPUs clocked under 1GHz). 781 __x86_setup_system_time((uint64)conversionFactor << 32, 782 conversionFactorNsecs); 783 #else 784 if (conversionFactorNsecs >> 32 != 0) { 785 // the TSC frequency is < 1 GHz, which forces us to shift the factor 786 __x86_setup_system_time(conversionFactor, conversionFactorNsecs >> 16, 787 true); 788 } else { 789 // the TSC frequency is >= 1 GHz 790 __x86_setup_system_time(conversionFactor, conversionFactorNsecs, false); 791 } 792 #endif 793 794 // Initialize descriptor tables. 795 x86_descriptors_init(args); 796 797 return B_OK; 798 } 799 800 801 status_t 802 arch_cpu_init_post_vm(kernel_args* args) 803 { 804 uint32 i; 805 806 // allocate an area for the double fault stacks 807 virtual_address_restrictions virtualRestrictions = {}; 808 virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS; 809 physical_address_restrictions physicalRestrictions = {}; 810 create_area_etc(B_SYSTEM_TEAM, "double fault stacks", 811 kDoubleFaultStackSize * smp_get_num_cpus(), B_FULL_LOCK, 812 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0, 813 &virtualRestrictions, &physicalRestrictions, 814 (void**)&sDoubleFaultStacks); 815 816 // More descriptor table setup. 817 x86_descriptors_init_post_vm(args); 818 819 X86PagingStructures* kernelPagingStructures 820 = static_cast<X86VMTranslationMap*>( 821 VMAddressSpace::Kernel()->TranslationMap())->PagingStructures(); 822 823 // Set active translation map on each CPU. 824 for (i = 0; i < args->num_cpus; i++) { 825 gCPU[i].arch.active_paging_structures = kernelPagingStructures; 826 kernelPagingStructures->AddReference(); 827 } 828 829 if (!apic_available()) 830 x86_init_fpu(); 831 // else fpu gets set up in smp code 832 833 return B_OK; 834 } 835 836 837 status_t 838 arch_cpu_init_post_modules(kernel_args* args) 839 { 840 // initialize CPU module 841 842 void* cookie = open_module_list("cpu"); 843 844 while (true) { 845 char name[B_FILE_NAME_LENGTH]; 846 size_t nameLength = sizeof(name); 847 848 if (read_next_module_name(cookie, name, &nameLength) != B_OK 849 || get_module(name, (module_info**)&sCpuModule) == B_OK) 850 break; 851 } 852 853 close_module_list(cookie); 854 855 // initialize MTRRs if available 856 if (x86_count_mtrrs() > 0) { 857 sCpuRendezvous = sCpuRendezvous2 = 0; 858 call_all_cpus(&init_mtrrs, NULL); 859 } 860 861 // get optimized functions from the CPU module 862 if (sCpuModule != NULL && sCpuModule->get_optimized_functions != NULL) { 863 x86_optimized_functions functions; 864 memset(&functions, 0, sizeof(functions)); 865 866 sCpuModule->get_optimized_functions(&functions); 867 868 if (functions.memcpy != NULL) { 869 gOptimizedFunctions.memcpy = functions.memcpy; 870 gOptimizedFunctions.memcpy_end = functions.memcpy_end; 871 } 872 873 if (functions.memset != NULL) { 874 gOptimizedFunctions.memset = functions.memset; 875 gOptimizedFunctions.memset_end = functions.memset_end; 876 } 877 } 878 879 // put the optimized functions into the commpage 880 size_t memcpyLen = (addr_t)gOptimizedFunctions.memcpy_end 881 - (addr_t)gOptimizedFunctions.memcpy; 882 addr_t memcpyPosition = fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMCPY, 883 (const void*)gOptimizedFunctions.memcpy, memcpyLen); 884 size_t memsetLen = (addr_t)gOptimizedFunctions.memset_end 885 - (addr_t)gOptimizedFunctions.memset; 886 addr_t memsetPosition = fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMSET, 887 (const void*)gOptimizedFunctions.memset, memsetLen); 888 size_t threadExitLen = (addr_t)x86_end_userspace_thread_exit 889 - (addr_t)x86_userspace_thread_exit; 890 addr_t threadExitPosition = fill_commpage_entry( 891 COMMPAGE_ENTRY_X86_THREAD_EXIT, (const void*)x86_userspace_thread_exit, 892 threadExitLen); 893 894 // add the functions to the commpage image 895 image_id image = get_commpage_image(); 896 elf_add_memory_image_symbol(image, "commpage_memcpy", memcpyPosition, 897 memcpyLen, B_SYMBOL_TYPE_TEXT); 898 elf_add_memory_image_symbol(image, "commpage_memset", memsetPosition, 899 memsetLen, B_SYMBOL_TYPE_TEXT); 900 elf_add_memory_image_symbol(image, "commpage_thread_exit", 901 threadExitPosition, threadExitLen, B_SYMBOL_TYPE_TEXT); 902 903 return B_OK; 904 } 905 906 907 void 908 arch_cpu_user_TLB_invalidate(void) 909 { 910 x86_write_cr3(x86_read_cr3()); 911 } 912 913 914 void 915 arch_cpu_global_TLB_invalidate(void) 916 { 917 uint32 flags = x86_read_cr4(); 918 919 if (flags & IA32_CR4_GLOBAL_PAGES) { 920 // disable and reenable the global pages to flush all TLBs regardless 921 // of the global page bit 922 x86_write_cr4(flags & ~IA32_CR4_GLOBAL_PAGES); 923 x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES); 924 } else { 925 cpu_status state = disable_interrupts(); 926 arch_cpu_user_TLB_invalidate(); 927 restore_interrupts(state); 928 } 929 } 930 931 932 void 933 arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) 934 { 935 int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE; 936 while (num_pages-- >= 0) { 937 invalidate_TLB(start); 938 start += B_PAGE_SIZE; 939 } 940 } 941 942 943 void 944 arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) 945 { 946 int i; 947 for (i = 0; i < num_pages; i++) { 948 invalidate_TLB(pages[i]); 949 } 950 } 951 952 953 status_t 954 arch_cpu_shutdown(bool rebootSystem) 955 { 956 if (acpi_shutdown(rebootSystem) == B_OK) 957 return B_OK; 958 959 if (!rebootSystem) { 960 #ifndef __x86_64__ 961 return apm_shutdown(); 962 #else 963 return B_NOT_SUPPORTED; 964 #endif 965 } 966 967 cpu_status state = disable_interrupts(); 968 969 // try to reset the system using the keyboard controller 970 out8(0xfe, 0x64); 971 972 // Give some time to the controller to do its job (0.5s) 973 snooze(500000); 974 975 // if that didn't help, try it this way 976 x86_reboot(); 977 978 restore_interrupts(state); 979 return B_ERROR; 980 } 981 982 983 void 984 arch_cpu_idle(void) 985 { 986 gCpuIdleFunc(); 987 } 988 989 990 void 991 arch_cpu_sync_icache(void* address, size_t length) 992 { 993 // instruction cache is always consistent on x86 994 } 995 996 997 void 998 arch_cpu_memory_read_barrier(void) 999 { 1000 #ifdef __x86_64__ 1001 asm volatile("lfence" : : : "memory"); 1002 #else 1003 asm volatile ("lock;" : : : "memory"); 1004 asm volatile ("addl $0, 0(%%esp);" : : : "memory"); 1005 #endif 1006 } 1007 1008 1009 void 1010 arch_cpu_memory_write_barrier(void) 1011 { 1012 #ifdef __x86_64__ 1013 asm volatile("sfence" : : : "memory"); 1014 #else 1015 asm volatile ("lock;" : : : "memory"); 1016 asm volatile ("addl $0, 0(%%esp);" : : : "memory"); 1017 #endif 1018 } 1019 1020