1 /* 2 * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. 3 * Distributed under the terms of the MIT License. 4 * 5 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 6 * Distributed under the terms of the NewOS License. 7 */ 8 9 10 #include <cpu.h> 11 12 #include <string.h> 13 #include <stdlib.h> 14 #include <stdio.h> 15 16 #include <ACPI.h> 17 18 #include <boot_device.h> 19 #include <commpage.h> 20 #include <debug.h> 21 #include <elf.h> 22 #include <smp.h> 23 #include <tls.h> 24 #include <vm/vm.h> 25 #include <vm/vm_types.h> 26 #include <vm/VMAddressSpace.h> 27 28 #include <arch_system_info.h> 29 #include <arch/x86/apic.h> 30 #include <arch/x86/selector.h> 31 #include <boot/kernel_args.h> 32 33 #include "interrupts.h" 34 #include "paging/X86PagingStructures.h" 35 #include "paging/X86VMTranslationMap.h" 36 37 38 #define DUMP_FEATURE_STRING 1 39 40 41 /* cpu vendor info */ 42 struct cpu_vendor_info { 43 const char *vendor; 44 const char *ident_string[2]; 45 }; 46 47 static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = { 48 { "Intel", { "GenuineIntel" } }, 49 { "AMD", { "AuthenticAMD" } }, 50 { "Cyrix", { "CyrixInstead" } }, 51 { "UMC", { "UMC UMC UMC" } }, 52 { "NexGen", { "NexGenDriven" } }, 53 { "Centaur", { "CentaurHauls" } }, 54 { "Rise", { "RiseRiseRise" } }, 55 { "Transmeta", { "GenuineTMx86", "TransmetaCPU" } }, 56 { "NSC", { "Geode by NSC" } }, 57 }; 58 59 #define CR0_CACHE_DISABLE (1UL << 30) 60 #define CR0_NOT_WRITE_THROUGH (1UL << 29) 61 #define CR0_FPU_EMULATION (1UL << 2) 62 #define CR0_MONITOR_FPU (1UL << 1) 63 64 #define CR4_OS_FXSR (1UL << 9) 65 #define CR4_OS_XMM_EXCEPTION (1UL << 10) 66 67 #define K8_SMIONCMPHALT (1ULL << 27) 68 #define K8_C1EONCMPHALT (1ULL << 28) 69 70 #define K8_CMPHALT (K8_SMIONCMPHALT | K8_C1EONCMPHALT) 71 72 struct set_mtrr_parameter { 73 int32 index; 74 uint64 base; 75 uint64 length; 76 uint8 type; 77 }; 78 79 struct set_mtrrs_parameter { 80 const x86_mtrr_info* infos; 81 uint32 count; 82 uint8 defaultType; 83 }; 84 85 86 extern "C" void reboot(void); 87 // from arch_x86.S 88 89 void (*gX86SwapFPUFunc)(void *oldState, const void *newState); 90 void (*gCpuIdleFunc)(void); 91 bool gHasSSE = false; 92 93 static uint32 sCpuRendezvous; 94 static uint32 sCpuRendezvous2; 95 static uint32 sCpuRendezvous3; 96 static vint32 sTSCSyncRendezvous; 97 98 segment_descriptor *gGDT = NULL; 99 100 /* Some specials for the double fault handler */ 101 static uint8* sDoubleFaultStacks; 102 static const size_t kDoubleFaultStackSize = 4096; // size per CPU 103 104 static x86_cpu_module_info *sCpuModule; 105 106 107 extern "C" void memcpy_generic(void* dest, const void* source, size_t count); 108 extern int memcpy_generic_end; 109 extern "C" void memset_generic(void* dest, int value, size_t count); 110 extern int memset_generic_end; 111 112 x86_optimized_functions gOptimizedFunctions = { 113 memcpy_generic, 114 &memcpy_generic_end, 115 memset_generic, 116 &memset_generic_end 117 }; 118 119 120 static status_t 121 acpi_shutdown(bool rebootSystem) 122 { 123 if (debug_debugger_running() || !are_interrupts_enabled()) 124 return B_ERROR; 125 126 acpi_module_info* acpi; 127 if (get_module(B_ACPI_MODULE_NAME, (module_info**)&acpi) != B_OK) 128 return B_NOT_SUPPORTED; 129 130 status_t status; 131 if (rebootSystem) { 132 status = acpi->reboot(); 133 } else { 134 // Make sure we run on the boot CPU (apparently needed for some ACPI 135 // implementations) 136 _user_set_cpu_enabled(0, true); 137 for (int32 cpu = 1; cpu < smp_get_num_cpus(); cpu++) { 138 _user_set_cpu_enabled(cpu, false); 139 } 140 // TODO: must not be called from the idle thread! 141 thread_yield(true); 142 143 status = acpi->prepare_sleep_state(ACPI_POWER_STATE_OFF, NULL, 0); 144 if (status == B_OK) { 145 //cpu_status state = disable_interrupts(); 146 status = acpi->enter_sleep_state(ACPI_POWER_STATE_OFF); 147 //restore_interrupts(state); 148 } 149 } 150 151 put_module(B_ACPI_MODULE_NAME); 152 return status; 153 } 154 155 156 /*! Disable CPU caches, and invalidate them. */ 157 static void 158 disable_caches() 159 { 160 x86_write_cr0((x86_read_cr0() | CR0_CACHE_DISABLE) 161 & ~CR0_NOT_WRITE_THROUGH); 162 wbinvd(); 163 arch_cpu_global_TLB_invalidate(); 164 } 165 166 167 /*! Invalidate CPU caches, and enable them. */ 168 static void 169 enable_caches() 170 { 171 wbinvd(); 172 arch_cpu_global_TLB_invalidate(); 173 x86_write_cr0(x86_read_cr0() 174 & ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH)); 175 } 176 177 178 static void 179 set_mtrr(void *_parameter, int cpu) 180 { 181 struct set_mtrr_parameter *parameter 182 = (struct set_mtrr_parameter *)_parameter; 183 184 // wait until all CPUs have arrived here 185 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 186 187 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 188 // that initiated the call_all_cpus() from doing that again and clearing 189 // sCpuRendezvous2 before the last CPU has actually left the loop in 190 // smp_cpu_rendezvous(); 191 if (cpu == 0) 192 atomic_set((vint32*)&sCpuRendezvous3, 0); 193 194 disable_caches(); 195 196 sCpuModule->set_mtrr(parameter->index, parameter->base, parameter->length, 197 parameter->type); 198 199 enable_caches(); 200 201 // wait until all CPUs have arrived here 202 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 203 smp_cpu_rendezvous(&sCpuRendezvous3, cpu); 204 } 205 206 207 static void 208 set_mtrrs(void* _parameter, int cpu) 209 { 210 set_mtrrs_parameter* parameter = (set_mtrrs_parameter*)_parameter; 211 212 // wait until all CPUs have arrived here 213 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 214 215 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 216 // that initiated the call_all_cpus() from doing that again and clearing 217 // sCpuRendezvous2 before the last CPU has actually left the loop in 218 // smp_cpu_rendezvous(); 219 if (cpu == 0) 220 atomic_set((vint32*)&sCpuRendezvous3, 0); 221 222 disable_caches(); 223 224 sCpuModule->set_mtrrs(parameter->defaultType, parameter->infos, 225 parameter->count); 226 227 enable_caches(); 228 229 // wait until all CPUs have arrived here 230 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 231 smp_cpu_rendezvous(&sCpuRendezvous3, cpu); 232 } 233 234 235 static void 236 init_mtrrs(void *_unused, int cpu) 237 { 238 // wait until all CPUs have arrived here 239 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 240 241 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 242 // that initiated the call_all_cpus() from doing that again and clearing 243 // sCpuRendezvous2 before the last CPU has actually left the loop in 244 // smp_cpu_rendezvous(); 245 if (cpu == 0) 246 atomic_set((vint32*)&sCpuRendezvous3, 0); 247 248 disable_caches(); 249 250 sCpuModule->init_mtrrs(); 251 252 enable_caches(); 253 254 // wait until all CPUs have arrived here 255 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 256 smp_cpu_rendezvous(&sCpuRendezvous3, cpu); 257 } 258 259 260 uint32 261 x86_count_mtrrs(void) 262 { 263 if (sCpuModule == NULL) 264 return 0; 265 266 return sCpuModule->count_mtrrs(); 267 } 268 269 270 void 271 x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type) 272 { 273 struct set_mtrr_parameter parameter; 274 parameter.index = index; 275 parameter.base = base; 276 parameter.length = length; 277 parameter.type = type; 278 279 sCpuRendezvous = sCpuRendezvous2 = 0; 280 call_all_cpus(&set_mtrr, ¶meter); 281 } 282 283 284 status_t 285 x86_get_mtrr(uint32 index, uint64 *_base, uint64 *_length, uint8 *_type) 286 { 287 // the MTRRs are identical on all CPUs, so it doesn't matter 288 // on which CPU this runs 289 return sCpuModule->get_mtrr(index, _base, _length, _type); 290 } 291 292 293 void 294 x86_set_mtrrs(uint8 defaultType, const x86_mtrr_info* infos, uint32 count) 295 { 296 if (sCpuModule == NULL) 297 return; 298 299 struct set_mtrrs_parameter parameter; 300 parameter.defaultType = defaultType; 301 parameter.infos = infos; 302 parameter.count = count; 303 304 sCpuRendezvous = sCpuRendezvous2 = 0; 305 call_all_cpus(&set_mtrrs, ¶meter); 306 } 307 308 309 void 310 x86_init_fpu(void) 311 { 312 if (!x86_check_feature(IA32_FEATURE_FPU, FEATURE_COMMON)) { 313 // No FPU... time to install one in your 386? 314 dprintf("%s: Warning: CPU has no reported FPU.\n", __func__); 315 gX86SwapFPUFunc = i386_noop_swap; 316 return; 317 } 318 319 if (!x86_check_feature(IA32_FEATURE_SSE, FEATURE_COMMON) 320 || !x86_check_feature(IA32_FEATURE_FXSR, FEATURE_COMMON)) { 321 dprintf("%s: CPU has no SSE... just enabling FPU.\n", __func__); 322 // we don't have proper SSE support, just enable FPU 323 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 324 gX86SwapFPUFunc = i386_fnsave_swap; 325 return; 326 } 327 dprintf("%s: CPU has SSE... enabling FXSR and XMM.\n", __func__); 328 329 // enable OS support for SSE 330 x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION); 331 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 332 333 gX86SwapFPUFunc = i386_fxsave_swap; 334 gHasSSE = true; 335 } 336 337 338 static void 339 load_tss(int cpu) 340 { 341 short seg = ((TSS_BASE_SEGMENT + cpu) << 3) | DPL_KERNEL; 342 asm("movw %0, %%ax;" 343 "ltr %%ax;" : : "r" (seg) : "eax"); 344 } 345 346 347 static void 348 init_double_fault(int cpuNum) 349 { 350 // set up the double fault TSS 351 struct tss *tss = &gCPU[cpuNum].arch.double_fault_tss; 352 353 memset(tss, 0, sizeof(struct tss)); 354 size_t stackSize; 355 tss->sp0 = (uint32)x86_get_double_fault_stack(cpuNum, &stackSize); 356 tss->sp0 += stackSize; 357 tss->ss0 = KERNEL_DATA_SEG; 358 read_cr3(tss->cr3); 359 // copy the current cr3 to the double fault cr3 360 tss->eip = (uint32)&double_fault; 361 tss->es = KERNEL_DATA_SEG; 362 tss->cs = KERNEL_CODE_SEG; 363 tss->ss = KERNEL_DATA_SEG; 364 tss->esp = tss->sp0; 365 tss->ds = KERNEL_DATA_SEG; 366 tss->fs = KERNEL_DATA_SEG; 367 tss->gs = KERNEL_DATA_SEG; 368 tss->ldt_seg_selector = 0; 369 tss->io_map_base = sizeof(struct tss); 370 371 // add TSS descriptor for this new TSS 372 uint16 tssSegmentDescriptorIndex = DOUBLE_FAULT_TSS_BASE_SEGMENT + cpuNum; 373 set_tss_descriptor(&gGDT[tssSegmentDescriptorIndex], 374 (addr_t)tss, sizeof(struct tss)); 375 376 x86_set_task_gate(cpuNum, 8, tssSegmentDescriptorIndex << 3); 377 } 378 379 380 #if DUMP_FEATURE_STRING 381 static void 382 dump_feature_string(int currentCPU, cpu_ent *cpu) 383 { 384 char features[256]; 385 features[0] = 0; 386 387 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FPU) 388 strlcat(features, "fpu ", sizeof(features)); 389 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_VME) 390 strlcat(features, "vme ", sizeof(features)); 391 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DE) 392 strlcat(features, "de ", sizeof(features)); 393 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE) 394 strlcat(features, "pse ", sizeof(features)); 395 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TSC) 396 strlcat(features, "tsc ", sizeof(features)); 397 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MSR) 398 strlcat(features, "msr ", sizeof(features)); 399 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAE) 400 strlcat(features, "pae ", sizeof(features)); 401 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCE) 402 strlcat(features, "mce ", sizeof(features)); 403 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CX8) 404 strlcat(features, "cx8 ", sizeof(features)); 405 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_APIC) 406 strlcat(features, "apic ", sizeof(features)); 407 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SEP) 408 strlcat(features, "sep ", sizeof(features)); 409 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MTRR) 410 strlcat(features, "mtrr ", sizeof(features)); 411 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PGE) 412 strlcat(features, "pge ", sizeof(features)); 413 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCA) 414 strlcat(features, "mca ", sizeof(features)); 415 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CMOV) 416 strlcat(features, "cmov ", sizeof(features)); 417 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAT) 418 strlcat(features, "pat ", sizeof(features)); 419 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE36) 420 strlcat(features, "pse36 ", sizeof(features)); 421 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSN) 422 strlcat(features, "psn ", sizeof(features)); 423 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CLFSH) 424 strlcat(features, "clfsh ", sizeof(features)); 425 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DS) 426 strlcat(features, "ds ", sizeof(features)); 427 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_ACPI) 428 strlcat(features, "acpi ", sizeof(features)); 429 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MMX) 430 strlcat(features, "mmx ", sizeof(features)); 431 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FXSR) 432 strlcat(features, "fxsr ", sizeof(features)); 433 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE) 434 strlcat(features, "sse ", sizeof(features)); 435 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE2) 436 strlcat(features, "sse2 ", sizeof(features)); 437 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SS) 438 strlcat(features, "ss ", sizeof(features)); 439 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_HTT) 440 strlcat(features, "htt ", sizeof(features)); 441 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TM) 442 strlcat(features, "tm ", sizeof(features)); 443 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PBE) 444 strlcat(features, "pbe ", sizeof(features)); 445 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE3) 446 strlcat(features, "sse3 ", sizeof(features)); 447 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCLMULQDQ) 448 strlcat(features, "pclmulqdq ", sizeof(features)); 449 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DTES64) 450 strlcat(features, "dtes64 ", sizeof(features)); 451 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MONITOR) 452 strlcat(features, "monitor ", sizeof(features)); 453 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DSCPL) 454 strlcat(features, "dscpl ", sizeof(features)); 455 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_VMX) 456 strlcat(features, "vmx ", sizeof(features)); 457 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SMX) 458 strlcat(features, "smx ", sizeof(features)); 459 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_EST) 460 strlcat(features, "est ", sizeof(features)); 461 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TM2) 462 strlcat(features, "tm2 ", sizeof(features)); 463 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSSE3) 464 strlcat(features, "ssse3 ", sizeof(features)); 465 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CNXTID) 466 strlcat(features, "cnxtid ", sizeof(features)); 467 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_FMA) 468 strlcat(features, "fma ", sizeof(features)); 469 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CX16) 470 strlcat(features, "cx16 ", sizeof(features)); 471 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XTPR) 472 strlcat(features, "xtpr ", sizeof(features)); 473 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PDCM) 474 strlcat(features, "pdcm ", sizeof(features)); 475 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCID) 476 strlcat(features, "pcid ", sizeof(features)); 477 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DCA) 478 strlcat(features, "dca ", sizeof(features)); 479 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_1) 480 strlcat(features, "sse4_1 ", sizeof(features)); 481 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_2) 482 strlcat(features, "sse4_2 ", sizeof(features)); 483 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_X2APIC) 484 strlcat(features, "x2apic ", sizeof(features)); 485 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MOVBE) 486 strlcat(features, "movbe ", sizeof(features)); 487 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_POPCNT) 488 strlcat(features, "popcnt ", sizeof(features)); 489 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TSCDEADLINE) 490 strlcat(features, "tscdeadline ", sizeof(features)); 491 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AES) 492 strlcat(features, "aes ", sizeof(features)); 493 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XSAVE) 494 strlcat(features, "xsave ", sizeof(features)); 495 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_OSXSAVE) 496 strlcat(features, "osxsave ", sizeof(features)); 497 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AVX) 498 strlcat(features, "avx ", sizeof(features)); 499 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_F16C) 500 strlcat(features, "f16c ", sizeof(features)); 501 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_RDRND) 502 strlcat(features, "rdrnd ", sizeof(features)); 503 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) 504 strlcat(features, "hypervisor ", sizeof(features)); 505 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_SYSCALL) 506 strlcat(features, "syscall ", sizeof(features)); 507 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_NX) 508 strlcat(features, "nx ", sizeof(features)); 509 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_MMXEXT) 510 strlcat(features, "mmxext ", sizeof(features)); 511 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_FFXSR) 512 strlcat(features, "ffxsr ", sizeof(features)); 513 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_LONG) 514 strlcat(features, "long ", sizeof(features)); 515 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOWEXT) 516 strlcat(features, "3dnowext ", sizeof(features)); 517 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOW) 518 strlcat(features, "3dnow ", sizeof(features)); 519 520 dprintf("CPU %d: features: %s\n", currentCPU, features); 521 } 522 #endif // DUMP_FEATURE_STRING 523 524 525 static int 526 detect_cpu(int currentCPU) 527 { 528 cpu_ent *cpu = get_cpu_struct(); 529 char vendorString[17]; 530 cpuid_info cpuid; 531 532 // clear out the cpu info data 533 cpu->arch.vendor = VENDOR_UNKNOWN; 534 cpu->arch.vendor_name = "UNKNOWN VENDOR"; 535 cpu->arch.feature[FEATURE_COMMON] = 0; 536 cpu->arch.feature[FEATURE_EXT] = 0; 537 cpu->arch.feature[FEATURE_EXT_AMD] = 0; 538 cpu->arch.model_name[0] = 0; 539 540 // print some fun data 541 get_current_cpuid(&cpuid, 0); 542 543 // build the vendor string 544 memset(vendorString, 0, sizeof(vendorString)); 545 memcpy(vendorString, cpuid.eax_0.vendor_id, sizeof(cpuid.eax_0.vendor_id)); 546 547 // get the family, model, stepping 548 get_current_cpuid(&cpuid, 1); 549 cpu->arch.type = cpuid.eax_1.type; 550 cpu->arch.family = cpuid.eax_1.family; 551 cpu->arch.extended_family = cpuid.eax_1.extended_family; 552 cpu->arch.model = cpuid.eax_1.model; 553 cpu->arch.extended_model = cpuid.eax_1.extended_model; 554 cpu->arch.stepping = cpuid.eax_1.stepping; 555 dprintf("CPU %d: type %d family %d extended_family %d model %d " 556 "extended_model %d stepping %d, string '%s'\n", 557 currentCPU, cpu->arch.type, cpu->arch.family, 558 cpu->arch.extended_family, cpu->arch.model, 559 cpu->arch.extended_model, cpu->arch.stepping, vendorString); 560 561 // figure out what vendor we have here 562 563 for (int32 i = 0; i < VENDOR_NUM; i++) { 564 if (vendor_info[i].ident_string[0] 565 && !strcmp(vendorString, vendor_info[i].ident_string[0])) { 566 cpu->arch.vendor = (x86_vendors)i; 567 cpu->arch.vendor_name = vendor_info[i].vendor; 568 break; 569 } 570 if (vendor_info[i].ident_string[1] 571 && !strcmp(vendorString, vendor_info[i].ident_string[1])) { 572 cpu->arch.vendor = (x86_vendors)i; 573 cpu->arch.vendor_name = vendor_info[i].vendor; 574 break; 575 } 576 } 577 578 // see if we can get the model name 579 get_current_cpuid(&cpuid, 0x80000000); 580 if (cpuid.eax_0.max_eax >= 0x80000004) { 581 // build the model string (need to swap ecx/edx data before copying) 582 unsigned int temp; 583 memset(cpu->arch.model_name, 0, sizeof(cpu->arch.model_name)); 584 585 get_current_cpuid(&cpuid, 0x80000002); 586 temp = cpuid.regs.edx; 587 cpuid.regs.edx = cpuid.regs.ecx; 588 cpuid.regs.ecx = temp; 589 memcpy(cpu->arch.model_name, cpuid.as_chars, sizeof(cpuid.as_chars)); 590 591 get_current_cpuid(&cpuid, 0x80000003); 592 temp = cpuid.regs.edx; 593 cpuid.regs.edx = cpuid.regs.ecx; 594 cpuid.regs.ecx = temp; 595 memcpy(cpu->arch.model_name + 16, cpuid.as_chars, 596 sizeof(cpuid.as_chars)); 597 598 get_current_cpuid(&cpuid, 0x80000004); 599 temp = cpuid.regs.edx; 600 cpuid.regs.edx = cpuid.regs.ecx; 601 cpuid.regs.ecx = temp; 602 memcpy(cpu->arch.model_name + 32, cpuid.as_chars, 603 sizeof(cpuid.as_chars)); 604 605 // some cpus return a right-justified string 606 int32 i = 0; 607 while (cpu->arch.model_name[i] == ' ') 608 i++; 609 if (i > 0) { 610 memmove(cpu->arch.model_name, &cpu->arch.model_name[i], 611 strlen(&cpu->arch.model_name[i]) + 1); 612 } 613 614 dprintf("CPU %d: vendor '%s' model name '%s'\n", 615 currentCPU, cpu->arch.vendor_name, cpu->arch.model_name); 616 } else { 617 strlcpy(cpu->arch.model_name, "unknown", sizeof(cpu->arch.model_name)); 618 } 619 620 // load feature bits 621 get_current_cpuid(&cpuid, 1); 622 cpu->arch.feature[FEATURE_COMMON] = cpuid.eax_1.features; // edx 623 cpu->arch.feature[FEATURE_EXT] = cpuid.eax_1.extended_features; // ecx 624 if (cpu->arch.vendor == VENDOR_AMD) { 625 get_current_cpuid(&cpuid, 0x80000001); 626 cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx 627 } 628 629 #if DUMP_FEATURE_STRING 630 dump_feature_string(currentCPU, cpu); 631 #endif 632 633 return 0; 634 } 635 636 637 bool 638 x86_check_feature(uint32 feature, enum x86_feature_type type) 639 { 640 cpu_ent *cpu = get_cpu_struct(); 641 642 #if 0 643 int i; 644 dprintf("x86_check_feature: feature 0x%x, type %d\n", feature, type); 645 for (i = 0; i < FEATURE_NUM; i++) { 646 dprintf("features %d: 0x%x\n", i, cpu->arch.feature[i]); 647 } 648 #endif 649 650 return (cpu->arch.feature[type] & feature) != 0; 651 } 652 653 654 void* 655 x86_get_double_fault_stack(int32 cpu, size_t* _size) 656 { 657 *_size = kDoubleFaultStackSize; 658 return sDoubleFaultStacks + kDoubleFaultStackSize * cpu; 659 } 660 661 662 /*! Returns the index of the current CPU. Can only be called from the double 663 fault handler. 664 */ 665 int32 666 x86_double_fault_get_cpu(void) 667 { 668 uint32 stack = x86_read_ebp(); 669 return (stack - (uint32)sDoubleFaultStacks) / kDoubleFaultStackSize; 670 } 671 672 673 // #pragma mark - 674 675 676 status_t 677 arch_cpu_preboot_init_percpu(kernel_args *args, int cpu) 678 { 679 // A simple nop FPU call until x86_init_fpu 680 gX86SwapFPUFunc = i386_noop_swap; 681 682 // On SMP system we want to synchronize the CPUs' TSCs, so system_time() 683 // will return consistent values. 684 if (smp_get_num_cpus() > 1) { 685 // let the first CPU prepare the rendezvous point 686 if (cpu == 0) 687 sTSCSyncRendezvous = smp_get_num_cpus() - 1; 688 689 // One CPU after the other will drop out of this loop and be caught by 690 // the loop below, until the last CPU (0) gets there. Save for +/- a few 691 // cycles the CPUs should pass the second loop at the same time. 692 while (sTSCSyncRendezvous != cpu) { 693 } 694 695 sTSCSyncRendezvous = cpu - 1; 696 697 while (sTSCSyncRendezvous != -1) { 698 } 699 700 // reset TSC to 0 701 x86_write_msr(IA32_MSR_TSC, 0); 702 } 703 704 return B_OK; 705 } 706 707 708 static void 709 halt_idle(void) 710 { 711 asm("hlt"); 712 } 713 714 715 static void 716 amdc1e_noarat_idle(void) 717 { 718 uint64 msr = x86_read_msr(K8_MSR_IPM); 719 if (msr & K8_CMPHALT) 720 x86_write_msr(K8_MSR_IPM, msr & ~K8_CMPHALT); 721 halt_idle(); 722 } 723 724 725 static bool 726 detect_amdc1e_noarat() 727 { 728 cpu_ent *cpu = get_cpu_struct(); 729 730 if (cpu->arch.vendor != VENDOR_AMD) 731 return false; 732 733 // Family 0x12 and higher processors support ARAT 734 // Family lower than 0xf processors doesn't support C1E 735 // Family 0xf with model <= 0x40 procssors doesn't support C1E 736 uint32 family = cpu->arch.family + cpu->arch.extended_family; 737 uint32 model = (cpu->arch.extended_model << 4) | cpu->arch.model; 738 return (family < 0x12 && family > 0xf) || (family == 0xf && model > 0x40); 739 } 740 741 742 status_t 743 arch_cpu_init_percpu(kernel_args *args, int cpu) 744 { 745 detect_cpu(cpu); 746 747 // load the TSS for this cpu 748 // note the main cpu gets initialized in arch_cpu_init_post_vm() 749 if (cpu != 0) { 750 load_tss(cpu); 751 752 // set the IDT 753 struct { 754 uint16 limit; 755 void* address; 756 } _PACKED descriptor = { 757 256 * 8 - 1, // 256 descriptors, 8 bytes each (-1 for "limit") 758 x86_get_idt(cpu) 759 }; 760 761 asm volatile("lidt %0" : : "m"(descriptor)); 762 } 763 764 if (!gCpuIdleFunc) { 765 if (detect_amdc1e_noarat()) 766 gCpuIdleFunc = amdc1e_noarat_idle; 767 else 768 gCpuIdleFunc = halt_idle; 769 } 770 return 0; 771 } 772 773 774 status_t 775 arch_cpu_init(kernel_args *args) 776 { 777 // init the TSC -> system_time() conversion factors 778 779 uint32 conversionFactor = args->arch_args.system_time_cv_factor; 780 uint64 conversionFactorNsecs = (uint64)conversionFactor * 1000; 781 782 if (conversionFactorNsecs >> 32 != 0) { 783 // the TSC frequency is < 1 GHz, which forces us to shift the factor 784 __x86_setup_system_time(conversionFactor, conversionFactorNsecs >> 16, 785 true); 786 } else { 787 // the TSC frequency is >= 1 GHz 788 __x86_setup_system_time(conversionFactor, conversionFactorNsecs, false); 789 } 790 791 return B_OK; 792 } 793 794 795 status_t 796 arch_cpu_init_post_vm(kernel_args *args) 797 { 798 uint32 i; 799 800 // account for the segment descriptors 801 gGDT = (segment_descriptor *)args->arch_args.vir_gdt; 802 create_area("gdt", (void **)&gGDT, B_EXACT_ADDRESS, B_PAGE_SIZE, 803 B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 804 805 // currently taken out of the build, because it's not yet used (and assumes 806 // (a fixed number of used GDT entries) 807 //i386_selector_init(gGDT); // pass the new gdt 808 809 // allocate an area for the double fault stacks 810 virtual_address_restrictions virtualRestrictions = {}; 811 virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS; 812 physical_address_restrictions physicalRestrictions = {}; 813 create_area_etc(B_SYSTEM_TEAM, "double fault stacks", 814 kDoubleFaultStackSize * smp_get_num_cpus(), B_FULL_LOCK, 815 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 816 &virtualRestrictions, &physicalRestrictions, 817 (void**)&sDoubleFaultStacks); 818 819 X86PagingStructures* kernelPagingStructures 820 = static_cast<X86VMTranslationMap*>( 821 VMAddressSpace::Kernel()->TranslationMap())->PagingStructures(); 822 823 // setup task-state segments 824 for (i = 0; i < args->num_cpus; i++) { 825 // initialize the regular and double fault tss stored in the per-cpu 826 // structure 827 memset(&gCPU[i].arch.tss, 0, sizeof(struct tss)); 828 gCPU[i].arch.tss.ss0 = KERNEL_DATA_SEG; 829 gCPU[i].arch.tss.io_map_base = sizeof(struct tss); 830 831 // add TSS descriptor for this new TSS 832 set_tss_descriptor(&gGDT[TSS_BASE_SEGMENT + i], 833 (addr_t)&gCPU[i].arch.tss, sizeof(struct tss)); 834 835 // initialize the double fault tss 836 init_double_fault(i); 837 838 // init active translation map 839 gCPU[i].arch.active_paging_structures = kernelPagingStructures; 840 kernelPagingStructures->AddReference(); 841 } 842 843 // set the current hardware task on cpu 0 844 load_tss(0); 845 846 // setup TLS descriptors (one for every CPU) 847 848 for (i = 0; i < args->num_cpus; i++) { 849 set_segment_descriptor(&gGDT[TLS_BASE_SEGMENT + i], 0, TLS_SIZE, 850 DT_DATA_WRITEABLE, DPL_USER); 851 } 852 853 if (!apic_available()) 854 x86_init_fpu(); 855 // else fpu gets set up in smp code 856 857 return B_OK; 858 } 859 860 861 status_t 862 arch_cpu_init_post_modules(kernel_args *args) 863 { 864 // initialize CPU module 865 866 void *cookie = open_module_list("cpu"); 867 868 while (true) { 869 char name[B_FILE_NAME_LENGTH]; 870 size_t nameLength = sizeof(name); 871 872 if (read_next_module_name(cookie, name, &nameLength) != B_OK 873 || get_module(name, (module_info **)&sCpuModule) == B_OK) 874 break; 875 } 876 877 close_module_list(cookie); 878 879 // initialize MTRRs if available 880 if (x86_count_mtrrs() > 0) { 881 sCpuRendezvous = sCpuRendezvous2 = 0; 882 call_all_cpus(&init_mtrrs, NULL); 883 } 884 885 // get optimized functions from the CPU module 886 if (sCpuModule != NULL && sCpuModule->get_optimized_functions != NULL) { 887 x86_optimized_functions functions; 888 memset(&functions, 0, sizeof(functions)); 889 890 sCpuModule->get_optimized_functions(&functions); 891 892 if (functions.memcpy != NULL) { 893 gOptimizedFunctions.memcpy = functions.memcpy; 894 gOptimizedFunctions.memcpy_end = functions.memcpy_end; 895 } 896 897 if (functions.memset != NULL) { 898 gOptimizedFunctions.memset = functions.memset; 899 gOptimizedFunctions.memset_end = functions.memset_end; 900 } 901 } 902 903 // put the optimized functions into the commpage 904 size_t memcpyLen = (addr_t)gOptimizedFunctions.memcpy_end 905 - (addr_t)gOptimizedFunctions.memcpy; 906 fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMCPY, 907 (const void*)gOptimizedFunctions.memcpy, memcpyLen); 908 size_t memsetLen = (addr_t)gOptimizedFunctions.memset_end 909 - (addr_t)gOptimizedFunctions.memset; 910 fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMSET, 911 (const void*)gOptimizedFunctions.memset, memsetLen); 912 913 // add the functions to the commpage image 914 image_id image = get_commpage_image(); 915 elf_add_memory_image_symbol(image, "commpage_memcpy", 916 ((addr_t*)USER_COMMPAGE_ADDR)[COMMPAGE_ENTRY_X86_MEMCPY], memcpyLen, 917 B_SYMBOL_TYPE_TEXT); 918 elf_add_memory_image_symbol(image, "commpage_memset", 919 ((addr_t*)USER_COMMPAGE_ADDR)[COMMPAGE_ENTRY_X86_MEMSET], memsetLen, 920 B_SYMBOL_TYPE_TEXT); 921 922 return B_OK; 923 } 924 925 926 void 927 i386_set_tss_and_kstack(addr_t kstack) 928 { 929 get_cpu_struct()->arch.tss.sp0 = kstack; 930 } 931 932 933 void 934 arch_cpu_global_TLB_invalidate(void) 935 { 936 uint32 flags = x86_read_cr4(); 937 938 if (flags & IA32_CR4_GLOBAL_PAGES) { 939 // disable and reenable the global pages to flush all TLBs regardless 940 // of the global page bit 941 x86_write_cr4(flags & ~IA32_CR4_GLOBAL_PAGES); 942 x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES); 943 } else { 944 cpu_status state = disable_interrupts(); 945 arch_cpu_user_TLB_invalidate(); 946 restore_interrupts(state); 947 } 948 } 949 950 951 void 952 arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) 953 { 954 int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE; 955 while (num_pages-- >= 0) { 956 invalidate_TLB(start); 957 start += B_PAGE_SIZE; 958 } 959 } 960 961 962 void 963 arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) 964 { 965 int i; 966 for (i = 0; i < num_pages; i++) { 967 invalidate_TLB(pages[i]); 968 } 969 } 970 971 972 ssize_t 973 arch_cpu_user_strlcpy(char *to, const char *from, size_t size, 974 addr_t *faultHandler) 975 { 976 int fromLength = 0; 977 addr_t oldFaultHandler = *faultHandler; 978 979 // this check is to trick the gcc4 compiler and have it keep the error label 980 if (to == NULL && size > 0) 981 goto error; 982 983 *faultHandler = (addr_t)&&error; 984 985 if (size > 0) { 986 to[--size] = '\0'; 987 // copy 988 for ( ; size; size--, fromLength++, to++, from++) { 989 if ((*to = *from) == '\0') 990 break; 991 } 992 } 993 // count any leftover from chars 994 while (*from++ != '\0') { 995 fromLength++; 996 } 997 998 *faultHandler = oldFaultHandler; 999 return fromLength; 1000 1001 error: 1002 *faultHandler = oldFaultHandler; 1003 return B_BAD_ADDRESS; 1004 } 1005 1006 1007 status_t 1008 arch_cpu_user_memset(void *s, char c, size_t count, addr_t *faultHandler) 1009 { 1010 char *xs = (char *)s; 1011 addr_t oldFaultHandler = *faultHandler; 1012 1013 // this check is to trick the gcc4 compiler and have it keep the error label 1014 if (s == NULL) 1015 goto error; 1016 1017 *faultHandler = (addr_t)&&error; 1018 1019 while (count--) 1020 *xs++ = c; 1021 1022 *faultHandler = oldFaultHandler; 1023 return 0; 1024 1025 error: 1026 *faultHandler = oldFaultHandler; 1027 return B_BAD_ADDRESS; 1028 } 1029 1030 1031 status_t 1032 arch_cpu_shutdown(bool rebootSystem) 1033 { 1034 if (acpi_shutdown(rebootSystem) == B_OK) 1035 return B_OK; 1036 1037 if (!rebootSystem) 1038 return apm_shutdown(); 1039 1040 cpu_status state = disable_interrupts(); 1041 1042 // try to reset the system using the keyboard controller 1043 out8(0xfe, 0x64); 1044 1045 // Give some time to the controller to do its job (0.5s) 1046 snooze(500000); 1047 1048 // if that didn't help, try it this way 1049 reboot(); 1050 1051 restore_interrupts(state); 1052 return B_ERROR; 1053 } 1054 1055 1056 void 1057 arch_cpu_idle(void) 1058 { 1059 gCpuIdleFunc(); 1060 } 1061 1062 1063 void 1064 arch_cpu_sync_icache(void *address, size_t length) 1065 { 1066 // instruction cache is always consistent on x86 1067 } 1068 1069 1070 void 1071 arch_cpu_memory_read_barrier(void) 1072 { 1073 asm volatile ("lock;" : : : "memory"); 1074 asm volatile ("addl $0, 0(%%esp);" : : : "memory"); 1075 } 1076 1077 1078 void 1079 arch_cpu_memory_write_barrier(void) 1080 { 1081 asm volatile ("lock;" : : : "memory"); 1082 asm volatile ("addl $0, 0(%%esp);" : : : "memory"); 1083 } 1084 1085