1 /* 2 * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. 3 * Distributed under the terms of the MIT License. 4 * 5 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 6 * Distributed under the terms of the NewOS License. 7 * 8 */ 9 10 11 #include <cpu.h> 12 13 #include <string.h> 14 #include <stdlib.h> 15 #include <stdio.h> 16 17 #include <ACPI.h> 18 19 #include <boot_device.h> 20 #include <commpage.h> 21 #include <debug.h> 22 #include <elf.h> 23 #include <smp.h> 24 #include <tls.h> 25 #include <vm/vm.h> 26 #include <vm/vm_types.h> 27 #include <vm/VMAddressSpace.h> 28 29 #include <arch_system_info.h> 30 #include <arch/x86/apic.h> 31 #include <arch/x86/selector.h> 32 #include <boot/kernel_args.h> 33 34 #include "interrupts.h" 35 #include "paging/X86PagingStructures.h" 36 #include "paging/X86VMTranslationMap.h" 37 38 39 #define DUMP_FEATURE_STRING 1 40 41 42 /* cpu vendor info */ 43 struct cpu_vendor_info { 44 const char *vendor; 45 const char *ident_string[2]; 46 }; 47 48 static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = { 49 { "Intel", { "GenuineIntel" } }, 50 { "AMD", { "AuthenticAMD" } }, 51 { "Cyrix", { "CyrixInstead" } }, 52 { "UMC", { "UMC UMC UMC" } }, 53 { "NexGen", { "NexGenDriven" } }, 54 { "Centaur", { "CentaurHauls" } }, 55 { "Rise", { "RiseRiseRise" } }, 56 { "Transmeta", { "GenuineTMx86", "TransmetaCPU" } }, 57 { "NSC", { "Geode by NSC" } }, 58 }; 59 60 #define CR0_CACHE_DISABLE (1UL << 30) 61 #define CR0_NOT_WRITE_THROUGH (1UL << 29) 62 #define CR0_FPU_EMULATION (1UL << 2) 63 #define CR0_MONITOR_FPU (1UL << 1) 64 65 #define CR4_OS_FXSR (1UL << 9) 66 #define CR4_OS_XMM_EXCEPTION (1UL << 10) 67 68 #define K8_SMIONCMPHALT (1ULL << 27) 69 #define K8_C1EONCMPHALT (1ULL << 28) 70 71 #define K8_CMPHALT (K8_SMIONCMPHALT | K8_C1EONCMPHALT) 72 73 struct set_mtrr_parameter { 74 int32 index; 75 uint64 base; 76 uint64 length; 77 uint8 type; 78 }; 79 80 struct set_mtrrs_parameter { 81 const x86_mtrr_info* infos; 82 uint32 count; 83 uint8 defaultType; 84 }; 85 86 87 extern "C" void reboot(void); 88 // from arch_x86.S 89 90 void (*gX86SwapFPUFunc)(void *oldState, const void *newState); 91 void (*gCpuIdleFunc)(void); 92 bool gHasSSE = false; 93 94 static uint32 sCpuRendezvous; 95 static uint32 sCpuRendezvous2; 96 static uint32 sCpuRendezvous3; 97 static vint32 sTSCSyncRendezvous; 98 99 segment_descriptor *gGDT = NULL; 100 101 /* Some specials for the double fault handler */ 102 static uint8* sDoubleFaultStacks; 103 static const size_t kDoubleFaultStackSize = 4096; // size per CPU 104 105 static x86_cpu_module_info *sCpuModule; 106 107 108 extern "C" void memcpy_generic(void* dest, const void* source, size_t count); 109 extern int memcpy_generic_end; 110 extern "C" void memset_generic(void* dest, int value, size_t count); 111 extern int memset_generic_end; 112 113 x86_optimized_functions gOptimizedFunctions = { 114 memcpy_generic, 115 &memcpy_generic_end, 116 memset_generic, 117 &memset_generic_end 118 }; 119 120 121 static status_t 122 acpi_shutdown(bool rebootSystem) 123 { 124 if (debug_debugger_running() || !are_interrupts_enabled()) 125 return B_ERROR; 126 127 acpi_module_info* acpi; 128 if (get_module(B_ACPI_MODULE_NAME, (module_info**)&acpi) != B_OK) 129 return B_NOT_SUPPORTED; 130 131 status_t status; 132 if (rebootSystem) { 133 status = acpi->reboot(); 134 } else { 135 // Make sure we run on the boot CPU (apparently needed for some ACPI 136 // implementations) 137 _user_set_cpu_enabled(0, true); 138 for (int32 cpu = 1; cpu < smp_get_num_cpus(); cpu++) { 139 _user_set_cpu_enabled(cpu, false); 140 } 141 // TODO: must not be called from the idle thread! 142 thread_yield(true); 143 144 status = acpi->prepare_sleep_state(ACPI_POWER_STATE_OFF, NULL, 0); 145 if (status == B_OK) { 146 //cpu_status state = disable_interrupts(); 147 status = acpi->enter_sleep_state(ACPI_POWER_STATE_OFF, 0); 148 //restore_interrupts(state); 149 } 150 } 151 152 put_module(B_ACPI_MODULE_NAME); 153 return status; 154 } 155 156 157 /*! Disable CPU caches, and invalidate them. */ 158 static void 159 disable_caches() 160 { 161 x86_write_cr0((x86_read_cr0() | CR0_CACHE_DISABLE) 162 & ~CR0_NOT_WRITE_THROUGH); 163 wbinvd(); 164 arch_cpu_global_TLB_invalidate(); 165 } 166 167 168 /*! Invalidate CPU caches, and enable them. */ 169 static void 170 enable_caches() 171 { 172 wbinvd(); 173 arch_cpu_global_TLB_invalidate(); 174 x86_write_cr0(x86_read_cr0() 175 & ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH)); 176 } 177 178 179 static void 180 set_mtrr(void *_parameter, int cpu) 181 { 182 struct set_mtrr_parameter *parameter 183 = (struct set_mtrr_parameter *)_parameter; 184 185 // wait until all CPUs have arrived here 186 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 187 188 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 189 // that initiated the call_all_cpus() from doing that again and clearing 190 // sCpuRendezvous2 before the last CPU has actually left the loop in 191 // smp_cpu_rendezvous(); 192 if (cpu == 0) 193 atomic_set((vint32*)&sCpuRendezvous3, 0); 194 195 disable_caches(); 196 197 sCpuModule->set_mtrr(parameter->index, parameter->base, parameter->length, 198 parameter->type); 199 200 enable_caches(); 201 202 // wait until all CPUs have arrived here 203 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 204 smp_cpu_rendezvous(&sCpuRendezvous3, cpu); 205 } 206 207 208 static void 209 set_mtrrs(void* _parameter, int cpu) 210 { 211 set_mtrrs_parameter* parameter = (set_mtrrs_parameter*)_parameter; 212 213 // wait until all CPUs have arrived here 214 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 215 216 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 217 // that initiated the call_all_cpus() from doing that again and clearing 218 // sCpuRendezvous2 before the last CPU has actually left the loop in 219 // smp_cpu_rendezvous(); 220 if (cpu == 0) 221 atomic_set((vint32*)&sCpuRendezvous3, 0); 222 223 disable_caches(); 224 225 sCpuModule->set_mtrrs(parameter->defaultType, parameter->infos, 226 parameter->count); 227 228 enable_caches(); 229 230 // wait until all CPUs have arrived here 231 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 232 smp_cpu_rendezvous(&sCpuRendezvous3, cpu); 233 } 234 235 236 static void 237 init_mtrrs(void *_unused, int cpu) 238 { 239 // wait until all CPUs have arrived here 240 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 241 242 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 243 // that initiated the call_all_cpus() from doing that again and clearing 244 // sCpuRendezvous2 before the last CPU has actually left the loop in 245 // smp_cpu_rendezvous(); 246 if (cpu == 0) 247 atomic_set((vint32*)&sCpuRendezvous3, 0); 248 249 disable_caches(); 250 251 sCpuModule->init_mtrrs(); 252 253 enable_caches(); 254 255 // wait until all CPUs have arrived here 256 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 257 smp_cpu_rendezvous(&sCpuRendezvous3, cpu); 258 } 259 260 261 uint32 262 x86_count_mtrrs(void) 263 { 264 if (sCpuModule == NULL) 265 return 0; 266 267 return sCpuModule->count_mtrrs(); 268 } 269 270 271 void 272 x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type) 273 { 274 struct set_mtrr_parameter parameter; 275 parameter.index = index; 276 parameter.base = base; 277 parameter.length = length; 278 parameter.type = type; 279 280 sCpuRendezvous = sCpuRendezvous2 = 0; 281 call_all_cpus(&set_mtrr, ¶meter); 282 } 283 284 285 status_t 286 x86_get_mtrr(uint32 index, uint64 *_base, uint64 *_length, uint8 *_type) 287 { 288 // the MTRRs are identical on all CPUs, so it doesn't matter 289 // on which CPU this runs 290 return sCpuModule->get_mtrr(index, _base, _length, _type); 291 } 292 293 294 void 295 x86_set_mtrrs(uint8 defaultType, const x86_mtrr_info* infos, uint32 count) 296 { 297 if (sCpuModule == NULL) 298 return; 299 300 struct set_mtrrs_parameter parameter; 301 parameter.defaultType = defaultType; 302 parameter.infos = infos; 303 parameter.count = count; 304 305 sCpuRendezvous = sCpuRendezvous2 = 0; 306 call_all_cpus(&set_mtrrs, ¶meter); 307 } 308 309 310 void 311 x86_init_fpu(void) 312 { 313 if (!x86_check_feature(IA32_FEATURE_FPU, FEATURE_COMMON)) { 314 // No FPU... time to install one in your 386? 315 dprintf("%s: Warning: CPU has no reported FPU.\n", __func__); 316 gX86SwapFPUFunc = i386_noop_swap; 317 return; 318 } 319 320 if (!x86_check_feature(IA32_FEATURE_SSE, FEATURE_COMMON) 321 || !x86_check_feature(IA32_FEATURE_FXSR, FEATURE_COMMON)) { 322 dprintf("%s: CPU has no SSE... just enabling FPU.\n", __func__); 323 // we don't have proper SSE support, just enable FPU 324 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 325 gX86SwapFPUFunc = i386_fnsave_swap; 326 return; 327 } 328 dprintf("%s: CPU has SSE... enabling FXSR and XMM.\n", __func__); 329 330 // enable OS support for SSE 331 x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION); 332 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 333 334 gX86SwapFPUFunc = i386_fxsave_swap; 335 gHasSSE = true; 336 } 337 338 339 static void 340 load_tss(int cpu) 341 { 342 short seg = ((TSS_BASE_SEGMENT + cpu) << 3) | DPL_KERNEL; 343 asm("movw %0, %%ax;" 344 "ltr %%ax;" : : "r" (seg) : "eax"); 345 } 346 347 348 static void 349 init_double_fault(int cpuNum) 350 { 351 // set up the double fault TSS 352 struct tss *tss = &gCPU[cpuNum].arch.double_fault_tss; 353 354 memset(tss, 0, sizeof(struct tss)); 355 size_t stackSize; 356 tss->sp0 = (uint32)x86_get_double_fault_stack(cpuNum, &stackSize); 357 tss->sp0 += stackSize; 358 tss->ss0 = KERNEL_DATA_SEG; 359 read_cr3(tss->cr3); 360 // copy the current cr3 to the double fault cr3 361 tss->eip = (uint32)&double_fault; 362 tss->es = KERNEL_DATA_SEG; 363 tss->cs = KERNEL_CODE_SEG; 364 tss->ss = KERNEL_DATA_SEG; 365 tss->esp = tss->sp0; 366 tss->ds = KERNEL_DATA_SEG; 367 tss->fs = KERNEL_DATA_SEG; 368 tss->gs = KERNEL_DATA_SEG; 369 tss->ldt_seg_selector = 0; 370 tss->io_map_base = sizeof(struct tss); 371 372 // add TSS descriptor for this new TSS 373 uint16 tssSegmentDescriptorIndex = DOUBLE_FAULT_TSS_BASE_SEGMENT + cpuNum; 374 set_tss_descriptor(&gGDT[tssSegmentDescriptorIndex], 375 (addr_t)tss, sizeof(struct tss)); 376 377 x86_set_task_gate(cpuNum, 8, tssSegmentDescriptorIndex << 3); 378 } 379 380 381 #if DUMP_FEATURE_STRING 382 static void 383 dump_feature_string(int currentCPU, cpu_ent *cpu) 384 { 385 char features[256]; 386 features[0] = 0; 387 388 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FPU) 389 strlcat(features, "fpu ", sizeof(features)); 390 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_VME) 391 strlcat(features, "vme ", sizeof(features)); 392 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DE) 393 strlcat(features, "de ", sizeof(features)); 394 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE) 395 strlcat(features, "pse ", sizeof(features)); 396 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TSC) 397 strlcat(features, "tsc ", sizeof(features)); 398 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MSR) 399 strlcat(features, "msr ", sizeof(features)); 400 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAE) 401 strlcat(features, "pae ", sizeof(features)); 402 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCE) 403 strlcat(features, "mce ", sizeof(features)); 404 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CX8) 405 strlcat(features, "cx8 ", sizeof(features)); 406 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_APIC) 407 strlcat(features, "apic ", sizeof(features)); 408 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SEP) 409 strlcat(features, "sep ", sizeof(features)); 410 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MTRR) 411 strlcat(features, "mtrr ", sizeof(features)); 412 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PGE) 413 strlcat(features, "pge ", sizeof(features)); 414 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCA) 415 strlcat(features, "mca ", sizeof(features)); 416 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CMOV) 417 strlcat(features, "cmov ", sizeof(features)); 418 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAT) 419 strlcat(features, "pat ", sizeof(features)); 420 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE36) 421 strlcat(features, "pse36 ", sizeof(features)); 422 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSN) 423 strlcat(features, "psn ", sizeof(features)); 424 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CLFSH) 425 strlcat(features, "clfsh ", sizeof(features)); 426 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DS) 427 strlcat(features, "ds ", sizeof(features)); 428 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_ACPI) 429 strlcat(features, "acpi ", sizeof(features)); 430 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MMX) 431 strlcat(features, "mmx ", sizeof(features)); 432 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FXSR) 433 strlcat(features, "fxsr ", sizeof(features)); 434 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE) 435 strlcat(features, "sse ", sizeof(features)); 436 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE2) 437 strlcat(features, "sse2 ", sizeof(features)); 438 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SS) 439 strlcat(features, "ss ", sizeof(features)); 440 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_HTT) 441 strlcat(features, "htt ", sizeof(features)); 442 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TM) 443 strlcat(features, "tm ", sizeof(features)); 444 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PBE) 445 strlcat(features, "pbe ", sizeof(features)); 446 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE3) 447 strlcat(features, "sse3 ", sizeof(features)); 448 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCLMULQDQ) 449 strlcat(features, "pclmulqdq ", sizeof(features)); 450 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DTES64) 451 strlcat(features, "dtes64 ", sizeof(features)); 452 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MONITOR) 453 strlcat(features, "monitor ", sizeof(features)); 454 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DSCPL) 455 strlcat(features, "dscpl ", sizeof(features)); 456 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_VMX) 457 strlcat(features, "vmx ", sizeof(features)); 458 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SMX) 459 strlcat(features, "smx ", sizeof(features)); 460 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_EST) 461 strlcat(features, "est ", sizeof(features)); 462 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TM2) 463 strlcat(features, "tm2 ", sizeof(features)); 464 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSSE3) 465 strlcat(features, "ssse3 ", sizeof(features)); 466 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CNXTID) 467 strlcat(features, "cnxtid ", sizeof(features)); 468 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_FMA) 469 strlcat(features, "fma ", sizeof(features)); 470 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CX16) 471 strlcat(features, "cx16 ", sizeof(features)); 472 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XTPR) 473 strlcat(features, "xtpr ", sizeof(features)); 474 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PDCM) 475 strlcat(features, "pdcm ", sizeof(features)); 476 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCID) 477 strlcat(features, "pcid ", sizeof(features)); 478 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DCA) 479 strlcat(features, "dca ", sizeof(features)); 480 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_1) 481 strlcat(features, "sse4_1 ", sizeof(features)); 482 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_2) 483 strlcat(features, "sse4_2 ", sizeof(features)); 484 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_X2APIC) 485 strlcat(features, "x2apic ", sizeof(features)); 486 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MOVBE) 487 strlcat(features, "movbe ", sizeof(features)); 488 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_POPCNT) 489 strlcat(features, "popcnt ", sizeof(features)); 490 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TSCDEADLINE) 491 strlcat(features, "tscdeadline ", sizeof(features)); 492 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AES) 493 strlcat(features, "aes ", sizeof(features)); 494 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XSAVE) 495 strlcat(features, "xsave ", sizeof(features)); 496 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_OSXSAVE) 497 strlcat(features, "osxsave ", sizeof(features)); 498 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AVX) 499 strlcat(features, "avx ", sizeof(features)); 500 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_F16C) 501 strlcat(features, "f16c ", sizeof(features)); 502 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_RDRND) 503 strlcat(features, "rdrnd ", sizeof(features)); 504 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) 505 strlcat(features, "hypervisor ", sizeof(features)); 506 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_SYSCALL) 507 strlcat(features, "syscall ", sizeof(features)); 508 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_NX) 509 strlcat(features, "nx ", sizeof(features)); 510 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_MMXEXT) 511 strlcat(features, "mmxext ", sizeof(features)); 512 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_FFXSR) 513 strlcat(features, "ffxsr ", sizeof(features)); 514 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_LONG) 515 strlcat(features, "long ", sizeof(features)); 516 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOWEXT) 517 strlcat(features, "3dnowext ", sizeof(features)); 518 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOW) 519 strlcat(features, "3dnow ", sizeof(features)); 520 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_DTS) 521 strlcat(features, "dts ", sizeof(features)); 522 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ITB) 523 strlcat(features, "itb ", sizeof(features)); 524 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ARAT) 525 strlcat(features, "arat ", sizeof(features)); 526 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PLN) 527 strlcat(features, "pln ", sizeof(features)); 528 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ECMD) 529 strlcat(features, "ecmd ", sizeof(features)); 530 if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PTM) 531 strlcat(features, "ptm ", sizeof(features)); 532 if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_APERFMPERF) 533 strlcat(features, "aperfmperf ", sizeof(features)); 534 if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_EPB) 535 strlcat(features, "epb ", sizeof(features)); 536 537 dprintf("CPU %d: features: %s\n", currentCPU, features); 538 } 539 #endif // DUMP_FEATURE_STRING 540 541 542 static int 543 detect_cpu(int currentCPU) 544 { 545 cpu_ent *cpu = get_cpu_struct(); 546 char vendorString[17]; 547 cpuid_info cpuid; 548 549 // clear out the cpu info data 550 cpu->arch.vendor = VENDOR_UNKNOWN; 551 cpu->arch.vendor_name = "UNKNOWN VENDOR"; 552 cpu->arch.feature[FEATURE_COMMON] = 0; 553 cpu->arch.feature[FEATURE_EXT] = 0; 554 cpu->arch.feature[FEATURE_EXT_AMD] = 0; 555 cpu->arch.model_name[0] = 0; 556 557 // print some fun data 558 get_current_cpuid(&cpuid, 0); 559 560 // build the vendor string 561 memset(vendorString, 0, sizeof(vendorString)); 562 memcpy(vendorString, cpuid.eax_0.vendor_id, sizeof(cpuid.eax_0.vendor_id)); 563 564 // get the family, model, stepping 565 get_current_cpuid(&cpuid, 1); 566 cpu->arch.type = cpuid.eax_1.type; 567 cpu->arch.family = cpuid.eax_1.family; 568 cpu->arch.extended_family = cpuid.eax_1.extended_family; 569 cpu->arch.model = cpuid.eax_1.model; 570 cpu->arch.extended_model = cpuid.eax_1.extended_model; 571 cpu->arch.stepping = cpuid.eax_1.stepping; 572 dprintf("CPU %d: type %d family %d extended_family %d model %d " 573 "extended_model %d stepping %d, string '%s'\n", 574 currentCPU, cpu->arch.type, cpu->arch.family, 575 cpu->arch.extended_family, cpu->arch.model, 576 cpu->arch.extended_model, cpu->arch.stepping, vendorString); 577 578 // figure out what vendor we have here 579 580 for (int32 i = 0; i < VENDOR_NUM; i++) { 581 if (vendor_info[i].ident_string[0] 582 && !strcmp(vendorString, vendor_info[i].ident_string[0])) { 583 cpu->arch.vendor = (x86_vendors)i; 584 cpu->arch.vendor_name = vendor_info[i].vendor; 585 break; 586 } 587 if (vendor_info[i].ident_string[1] 588 && !strcmp(vendorString, vendor_info[i].ident_string[1])) { 589 cpu->arch.vendor = (x86_vendors)i; 590 cpu->arch.vendor_name = vendor_info[i].vendor; 591 break; 592 } 593 } 594 595 // see if we can get the model name 596 get_current_cpuid(&cpuid, 0x80000000); 597 if (cpuid.eax_0.max_eax >= 0x80000004) { 598 // build the model string (need to swap ecx/edx data before copying) 599 unsigned int temp; 600 memset(cpu->arch.model_name, 0, sizeof(cpu->arch.model_name)); 601 602 get_current_cpuid(&cpuid, 0x80000002); 603 temp = cpuid.regs.edx; 604 cpuid.regs.edx = cpuid.regs.ecx; 605 cpuid.regs.ecx = temp; 606 memcpy(cpu->arch.model_name, cpuid.as_chars, sizeof(cpuid.as_chars)); 607 608 get_current_cpuid(&cpuid, 0x80000003); 609 temp = cpuid.regs.edx; 610 cpuid.regs.edx = cpuid.regs.ecx; 611 cpuid.regs.ecx = temp; 612 memcpy(cpu->arch.model_name + 16, cpuid.as_chars, 613 sizeof(cpuid.as_chars)); 614 615 get_current_cpuid(&cpuid, 0x80000004); 616 temp = cpuid.regs.edx; 617 cpuid.regs.edx = cpuid.regs.ecx; 618 cpuid.regs.ecx = temp; 619 memcpy(cpu->arch.model_name + 32, cpuid.as_chars, 620 sizeof(cpuid.as_chars)); 621 622 // some cpus return a right-justified string 623 int32 i = 0; 624 while (cpu->arch.model_name[i] == ' ') 625 i++; 626 if (i > 0) { 627 memmove(cpu->arch.model_name, &cpu->arch.model_name[i], 628 strlen(&cpu->arch.model_name[i]) + 1); 629 } 630 631 dprintf("CPU %d: vendor '%s' model name '%s'\n", 632 currentCPU, cpu->arch.vendor_name, cpu->arch.model_name); 633 } else { 634 strlcpy(cpu->arch.model_name, "unknown", sizeof(cpu->arch.model_name)); 635 } 636 637 // load feature bits 638 get_current_cpuid(&cpuid, 1); 639 cpu->arch.feature[FEATURE_COMMON] = cpuid.eax_1.features; // edx 640 cpu->arch.feature[FEATURE_EXT] = cpuid.eax_1.extended_features; // ecx 641 if (cpu->arch.vendor == VENDOR_AMD) { 642 get_current_cpuid(&cpuid, 0x80000001); 643 cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx 644 } 645 get_current_cpuid(&cpuid, 6); 646 cpu->arch.feature[FEATURE_6_EAX] = cpuid.regs.eax; 647 cpu->arch.feature[FEATURE_6_ECX] = cpuid.regs.ecx; 648 649 #if DUMP_FEATURE_STRING 650 dump_feature_string(currentCPU, cpu); 651 #endif 652 653 return 0; 654 } 655 656 657 bool 658 x86_check_feature(uint32 feature, enum x86_feature_type type) 659 { 660 cpu_ent *cpu = get_cpu_struct(); 661 662 #if 0 663 int i; 664 dprintf("x86_check_feature: feature 0x%x, type %d\n", feature, type); 665 for (i = 0; i < FEATURE_NUM; i++) { 666 dprintf("features %d: 0x%x\n", i, cpu->arch.feature[i]); 667 } 668 #endif 669 670 return (cpu->arch.feature[type] & feature) != 0; 671 } 672 673 674 void* 675 x86_get_double_fault_stack(int32 cpu, size_t* _size) 676 { 677 *_size = kDoubleFaultStackSize; 678 return sDoubleFaultStacks + kDoubleFaultStackSize * cpu; 679 } 680 681 682 /*! Returns the index of the current CPU. Can only be called from the double 683 fault handler. 684 */ 685 int32 686 x86_double_fault_get_cpu(void) 687 { 688 uint32 stack = x86_read_ebp(); 689 return (stack - (uint32)sDoubleFaultStacks) / kDoubleFaultStackSize; 690 } 691 692 693 // #pragma mark - 694 695 696 status_t 697 arch_cpu_preboot_init_percpu(kernel_args *args, int cpu) 698 { 699 // A simple nop FPU call until x86_init_fpu 700 gX86SwapFPUFunc = i386_noop_swap; 701 702 // On SMP system we want to synchronize the CPUs' TSCs, so system_time() 703 // will return consistent values. 704 if (smp_get_num_cpus() > 1) { 705 // let the first CPU prepare the rendezvous point 706 if (cpu == 0) 707 sTSCSyncRendezvous = smp_get_num_cpus() - 1; 708 709 // One CPU after the other will drop out of this loop and be caught by 710 // the loop below, until the last CPU (0) gets there. Save for +/- a few 711 // cycles the CPUs should pass the second loop at the same time. 712 while (sTSCSyncRendezvous != cpu) { 713 } 714 715 sTSCSyncRendezvous = cpu - 1; 716 717 while (sTSCSyncRendezvous != -1) { 718 } 719 720 // reset TSC to 0 721 x86_write_msr(IA32_MSR_TSC, 0); 722 } 723 724 return B_OK; 725 } 726 727 728 static void 729 halt_idle(void) 730 { 731 asm("hlt"); 732 } 733 734 735 static void 736 amdc1e_noarat_idle(void) 737 { 738 uint64 msr = x86_read_msr(K8_MSR_IPM); 739 if (msr & K8_CMPHALT) 740 x86_write_msr(K8_MSR_IPM, msr & ~K8_CMPHALT); 741 halt_idle(); 742 } 743 744 745 static bool 746 detect_amdc1e_noarat() 747 { 748 cpu_ent *cpu = get_cpu_struct(); 749 750 if (cpu->arch.vendor != VENDOR_AMD) 751 return false; 752 753 // Family 0x12 and higher processors support ARAT 754 // Family lower than 0xf processors doesn't support C1E 755 // Family 0xf with model <= 0x40 procssors doesn't support C1E 756 uint32 family = cpu->arch.family + cpu->arch.extended_family; 757 uint32 model = (cpu->arch.extended_model << 4) | cpu->arch.model; 758 return (family < 0x12 && family > 0xf) || (family == 0xf && model > 0x40); 759 } 760 761 762 status_t 763 arch_cpu_init_percpu(kernel_args *args, int cpu) 764 { 765 detect_cpu(cpu); 766 767 // load the TSS for this cpu 768 // note the main cpu gets initialized in arch_cpu_init_post_vm() 769 if (cpu != 0) { 770 load_tss(cpu); 771 772 // set the IDT 773 struct { 774 uint16 limit; 775 void* address; 776 } _PACKED descriptor = { 777 256 * 8 - 1, // 256 descriptors, 8 bytes each (-1 for "limit") 778 x86_get_idt(cpu) 779 }; 780 781 asm volatile("lidt %0" : : "m"(descriptor)); 782 } 783 784 if (!gCpuIdleFunc) { 785 if (detect_amdc1e_noarat()) 786 gCpuIdleFunc = amdc1e_noarat_idle; 787 else 788 gCpuIdleFunc = halt_idle; 789 } 790 return 0; 791 } 792 793 794 status_t 795 arch_cpu_init(kernel_args *args) 796 { 797 // init the TSC -> system_time() conversion factors 798 799 uint32 conversionFactor = args->arch_args.system_time_cv_factor; 800 uint64 conversionFactorNsecs = (uint64)conversionFactor * 1000; 801 802 if (conversionFactorNsecs >> 32 != 0) { 803 // the TSC frequency is < 1 GHz, which forces us to shift the factor 804 __x86_setup_system_time(conversionFactor, conversionFactorNsecs >> 16, 805 true); 806 } else { 807 // the TSC frequency is >= 1 GHz 808 __x86_setup_system_time(conversionFactor, conversionFactorNsecs, false); 809 } 810 811 return B_OK; 812 } 813 814 815 status_t 816 arch_cpu_init_post_vm(kernel_args *args) 817 { 818 uint32 i; 819 820 // account for the segment descriptors 821 gGDT = (segment_descriptor *)args->arch_args.vir_gdt; 822 create_area("gdt", (void **)&gGDT, B_EXACT_ADDRESS, B_PAGE_SIZE, 823 B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 824 825 // currently taken out of the build, because it's not yet used (and assumes 826 // (a fixed number of used GDT entries) 827 //i386_selector_init(gGDT); // pass the new gdt 828 829 // allocate an area for the double fault stacks 830 virtual_address_restrictions virtualRestrictions = {}; 831 virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS; 832 physical_address_restrictions physicalRestrictions = {}; 833 create_area_etc(B_SYSTEM_TEAM, "double fault stacks", 834 kDoubleFaultStackSize * smp_get_num_cpus(), B_FULL_LOCK, 835 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 836 &virtualRestrictions, &physicalRestrictions, 837 (void**)&sDoubleFaultStacks); 838 839 X86PagingStructures* kernelPagingStructures 840 = static_cast<X86VMTranslationMap*>( 841 VMAddressSpace::Kernel()->TranslationMap())->PagingStructures(); 842 843 // setup task-state segments 844 for (i = 0; i < args->num_cpus; i++) { 845 // initialize the regular and double fault tss stored in the per-cpu 846 // structure 847 memset(&gCPU[i].arch.tss, 0, sizeof(struct tss)); 848 gCPU[i].arch.tss.ss0 = KERNEL_DATA_SEG; 849 gCPU[i].arch.tss.io_map_base = sizeof(struct tss); 850 851 // add TSS descriptor for this new TSS 852 set_tss_descriptor(&gGDT[TSS_BASE_SEGMENT + i], 853 (addr_t)&gCPU[i].arch.tss, sizeof(struct tss)); 854 855 // initialize the double fault tss 856 init_double_fault(i); 857 858 // init active translation map 859 gCPU[i].arch.active_paging_structures = kernelPagingStructures; 860 kernelPagingStructures->AddReference(); 861 } 862 863 // set the current hardware task on cpu 0 864 load_tss(0); 865 866 // setup TLS descriptors (one for every CPU) 867 868 for (i = 0; i < args->num_cpus; i++) { 869 set_segment_descriptor(&gGDT[TLS_BASE_SEGMENT + i], 0, TLS_SIZE, 870 DT_DATA_WRITEABLE, DPL_USER); 871 } 872 873 if (!apic_available()) 874 x86_init_fpu(); 875 // else fpu gets set up in smp code 876 877 return B_OK; 878 } 879 880 881 status_t 882 arch_cpu_init_post_modules(kernel_args *args) 883 { 884 // initialize CPU module 885 886 void *cookie = open_module_list("cpu"); 887 888 while (true) { 889 char name[B_FILE_NAME_LENGTH]; 890 size_t nameLength = sizeof(name); 891 892 if (read_next_module_name(cookie, name, &nameLength) != B_OK 893 || get_module(name, (module_info **)&sCpuModule) == B_OK) 894 break; 895 } 896 897 close_module_list(cookie); 898 899 // initialize MTRRs if available 900 if (x86_count_mtrrs() > 0) { 901 sCpuRendezvous = sCpuRendezvous2 = 0; 902 call_all_cpus(&init_mtrrs, NULL); 903 } 904 905 // get optimized functions from the CPU module 906 if (sCpuModule != NULL && sCpuModule->get_optimized_functions != NULL) { 907 x86_optimized_functions functions; 908 memset(&functions, 0, sizeof(functions)); 909 910 sCpuModule->get_optimized_functions(&functions); 911 912 if (functions.memcpy != NULL) { 913 gOptimizedFunctions.memcpy = functions.memcpy; 914 gOptimizedFunctions.memcpy_end = functions.memcpy_end; 915 } 916 917 if (functions.memset != NULL) { 918 gOptimizedFunctions.memset = functions.memset; 919 gOptimizedFunctions.memset_end = functions.memset_end; 920 } 921 } 922 923 // put the optimized functions into the commpage 924 size_t memcpyLen = (addr_t)gOptimizedFunctions.memcpy_end 925 - (addr_t)gOptimizedFunctions.memcpy; 926 fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMCPY, 927 (const void*)gOptimizedFunctions.memcpy, memcpyLen); 928 size_t memsetLen = (addr_t)gOptimizedFunctions.memset_end 929 - (addr_t)gOptimizedFunctions.memset; 930 fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMSET, 931 (const void*)gOptimizedFunctions.memset, memsetLen); 932 933 // add the functions to the commpage image 934 image_id image = get_commpage_image(); 935 elf_add_memory_image_symbol(image, "commpage_memcpy", 936 ((addr_t*)USER_COMMPAGE_ADDR)[COMMPAGE_ENTRY_X86_MEMCPY], memcpyLen, 937 B_SYMBOL_TYPE_TEXT); 938 elf_add_memory_image_symbol(image, "commpage_memset", 939 ((addr_t*)USER_COMMPAGE_ADDR)[COMMPAGE_ENTRY_X86_MEMSET], memsetLen, 940 B_SYMBOL_TYPE_TEXT); 941 942 return B_OK; 943 } 944 945 946 void 947 i386_set_tss_and_kstack(addr_t kstack) 948 { 949 get_cpu_struct()->arch.tss.sp0 = kstack; 950 } 951 952 953 void 954 arch_cpu_global_TLB_invalidate(void) 955 { 956 uint32 flags = x86_read_cr4(); 957 958 if (flags & IA32_CR4_GLOBAL_PAGES) { 959 // disable and reenable the global pages to flush all TLBs regardless 960 // of the global page bit 961 x86_write_cr4(flags & ~IA32_CR4_GLOBAL_PAGES); 962 x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES); 963 } else { 964 cpu_status state = disable_interrupts(); 965 arch_cpu_user_TLB_invalidate(); 966 restore_interrupts(state); 967 } 968 } 969 970 971 void 972 arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) 973 { 974 int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE; 975 while (num_pages-- >= 0) { 976 invalidate_TLB(start); 977 start += B_PAGE_SIZE; 978 } 979 } 980 981 982 void 983 arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) 984 { 985 int i; 986 for (i = 0; i < num_pages; i++) { 987 invalidate_TLB(pages[i]); 988 } 989 } 990 991 992 status_t 993 arch_cpu_shutdown(bool rebootSystem) 994 { 995 if (acpi_shutdown(rebootSystem) == B_OK) 996 return B_OK; 997 998 if (!rebootSystem) 999 return apm_shutdown(); 1000 1001 cpu_status state = disable_interrupts(); 1002 1003 // try to reset the system using the keyboard controller 1004 out8(0xfe, 0x64); 1005 1006 // Give some time to the controller to do its job (0.5s) 1007 snooze(500000); 1008 1009 // if that didn't help, try it this way 1010 reboot(); 1011 1012 restore_interrupts(state); 1013 return B_ERROR; 1014 } 1015 1016 1017 void 1018 arch_cpu_idle(void) 1019 { 1020 gCpuIdleFunc(); 1021 } 1022 1023 1024 void 1025 arch_cpu_sync_icache(void *address, size_t length) 1026 { 1027 // instruction cache is always consistent on x86 1028 } 1029 1030 1031 void 1032 arch_cpu_memory_read_barrier(void) 1033 { 1034 asm volatile ("lock;" : : : "memory"); 1035 asm volatile ("addl $0, 0(%%esp);" : : : "memory"); 1036 } 1037 1038 1039 void 1040 arch_cpu_memory_write_barrier(void) 1041 { 1042 asm volatile ("lock;" : : : "memory"); 1043 asm volatile ("addl $0, 0(%%esp);" : : : "memory"); 1044 } 1045 1046