1 /* 2 * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. 3 * Distributed under the terms of the MIT License. 4 * 5 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 6 * Distributed under the terms of the NewOS License. 7 */ 8 9 10 #include <cpu.h> 11 12 #include <string.h> 13 #include <stdlib.h> 14 #include <stdio.h> 15 16 #include <ACPI.h> 17 18 #include <boot_device.h> 19 #include <commpage.h> 20 #include <debug.h> 21 #include <elf.h> 22 #include <smp.h> 23 #include <tls.h> 24 #include <vm/vm.h> 25 #include <vm/vm_types.h> 26 #include <vm/VMAddressSpace.h> 27 28 #include <arch_system_info.h> 29 #include <arch/x86/selector.h> 30 #include <boot/kernel_args.h> 31 32 #include "interrupts.h" 33 #include "paging/X86PagingStructures.h" 34 #include "paging/X86VMTranslationMap.h" 35 36 37 #define DUMP_FEATURE_STRING 1 38 39 40 /* cpu vendor info */ 41 struct cpu_vendor_info { 42 const char *vendor; 43 const char *ident_string[2]; 44 }; 45 46 static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = { 47 { "Intel", { "GenuineIntel" } }, 48 { "AMD", { "AuthenticAMD" } }, 49 { "Cyrix", { "CyrixInstead" } }, 50 { "UMC", { "UMC UMC UMC" } }, 51 { "NexGen", { "NexGenDriven" } }, 52 { "Centaur", { "CentaurHauls" } }, 53 { "Rise", { "RiseRiseRise" } }, 54 { "Transmeta", { "GenuineTMx86", "TransmetaCPU" } }, 55 { "NSC", { "Geode by NSC" } }, 56 }; 57 58 #define CR0_CACHE_DISABLE (1UL << 30) 59 #define CR0_NOT_WRITE_THROUGH (1UL << 29) 60 #define CR0_FPU_EMULATION (1UL << 2) 61 #define CR0_MONITOR_FPU (1UL << 1) 62 63 #define CR4_OS_FXSR (1UL << 9) 64 #define CR4_OS_XMM_EXCEPTION (1UL << 10) 65 66 struct set_mtrr_parameter { 67 int32 index; 68 uint64 base; 69 uint64 length; 70 uint8 type; 71 }; 72 73 struct set_mtrrs_parameter { 74 const x86_mtrr_info* infos; 75 uint32 count; 76 uint8 defaultType; 77 }; 78 79 80 extern "C" void reboot(void); 81 // from arch_x86.S 82 83 void (*gX86SwapFPUFunc)(void *oldState, const void *newState); 84 bool gHasSSE = false; 85 86 static uint32 sCpuRendezvous; 87 static uint32 sCpuRendezvous2; 88 static uint32 sCpuRendezvous3; 89 static vint32 sTSCSyncRendezvous; 90 91 segment_descriptor *gGDT = NULL; 92 93 /* Some specials for the double fault handler */ 94 static uint8* sDoubleFaultStacks; 95 static const size_t kDoubleFaultStackSize = 4096; // size per CPU 96 97 static x86_cpu_module_info *sCpuModule; 98 99 100 extern "C" void memcpy_generic(void* dest, const void* source, size_t count); 101 extern int memcpy_generic_end; 102 extern "C" void memset_generic(void* dest, int value, size_t count); 103 extern int memset_generic_end; 104 105 x86_optimized_functions gOptimizedFunctions = { 106 memcpy_generic, 107 &memcpy_generic_end, 108 memset_generic, 109 &memset_generic_end 110 }; 111 112 113 static status_t 114 acpi_shutdown(bool rebootSystem) 115 { 116 if (debug_debugger_running() || !are_interrupts_enabled()) 117 return B_ERROR; 118 119 acpi_module_info* acpi; 120 if (get_module(B_ACPI_MODULE_NAME, (module_info**)&acpi) != B_OK) 121 return B_NOT_SUPPORTED; 122 123 status_t status; 124 if (rebootSystem) { 125 status = acpi->reboot(); 126 } else { 127 // Make sure we run on the boot CPU (apparently needed for some ACPI 128 // implementations) 129 _user_set_cpu_enabled(0, true); 130 for (int32 cpu = 1; cpu < smp_get_num_cpus(); cpu++) { 131 _user_set_cpu_enabled(cpu, false); 132 } 133 // TODO: must not be called from the idle thread! 134 thread_yield(true); 135 136 status = acpi->prepare_sleep_state(ACPI_POWER_STATE_OFF, NULL, 0); 137 if (status == B_OK) { 138 //cpu_status state = disable_interrupts(); 139 status = acpi->enter_sleep_state(ACPI_POWER_STATE_OFF); 140 //restore_interrupts(state); 141 } 142 } 143 144 put_module(B_ACPI_MODULE_NAME); 145 return status; 146 } 147 148 149 /*! Disable CPU caches, and invalidate them. */ 150 static void 151 disable_caches() 152 { 153 x86_write_cr0((x86_read_cr0() | CR0_CACHE_DISABLE) 154 & ~CR0_NOT_WRITE_THROUGH); 155 wbinvd(); 156 arch_cpu_global_TLB_invalidate(); 157 } 158 159 160 /*! Invalidate CPU caches, and enable them. */ 161 static void 162 enable_caches() 163 { 164 wbinvd(); 165 arch_cpu_global_TLB_invalidate(); 166 x86_write_cr0(x86_read_cr0() 167 & ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH)); 168 } 169 170 171 static void 172 set_mtrr(void *_parameter, int cpu) 173 { 174 struct set_mtrr_parameter *parameter 175 = (struct set_mtrr_parameter *)_parameter; 176 177 // wait until all CPUs have arrived here 178 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 179 180 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 181 // that initiated the call_all_cpus() from doing that again and clearing 182 // sCpuRendezvous2 before the last CPU has actually left the loop in 183 // smp_cpu_rendezvous(); 184 if (cpu == 0) 185 atomic_set((vint32*)&sCpuRendezvous3, 0); 186 187 disable_caches(); 188 189 sCpuModule->set_mtrr(parameter->index, parameter->base, parameter->length, 190 parameter->type); 191 192 enable_caches(); 193 194 // wait until all CPUs have arrived here 195 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 196 smp_cpu_rendezvous(&sCpuRendezvous3, cpu); 197 } 198 199 200 static void 201 set_mtrrs(void* _parameter, int cpu) 202 { 203 set_mtrrs_parameter* parameter = (set_mtrrs_parameter*)_parameter; 204 205 // wait until all CPUs have arrived here 206 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 207 208 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 209 // that initiated the call_all_cpus() from doing that again and clearing 210 // sCpuRendezvous2 before the last CPU has actually left the loop in 211 // smp_cpu_rendezvous(); 212 if (cpu == 0) 213 atomic_set((vint32*)&sCpuRendezvous3, 0); 214 215 disable_caches(); 216 217 sCpuModule->set_mtrrs(parameter->defaultType, parameter->infos, 218 parameter->count); 219 220 enable_caches(); 221 222 // wait until all CPUs have arrived here 223 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 224 smp_cpu_rendezvous(&sCpuRendezvous3, cpu); 225 } 226 227 228 static void 229 init_mtrrs(void *_unused, int cpu) 230 { 231 // wait until all CPUs have arrived here 232 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 233 234 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 235 // that initiated the call_all_cpus() from doing that again and clearing 236 // sCpuRendezvous2 before the last CPU has actually left the loop in 237 // smp_cpu_rendezvous(); 238 if (cpu == 0) 239 atomic_set((vint32*)&sCpuRendezvous3, 0); 240 241 disable_caches(); 242 243 sCpuModule->init_mtrrs(); 244 245 enable_caches(); 246 247 // wait until all CPUs have arrived here 248 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 249 smp_cpu_rendezvous(&sCpuRendezvous3, cpu); 250 } 251 252 253 uint32 254 x86_count_mtrrs(void) 255 { 256 if (sCpuModule == NULL) 257 return 0; 258 259 return sCpuModule->count_mtrrs(); 260 } 261 262 263 void 264 x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type) 265 { 266 struct set_mtrr_parameter parameter; 267 parameter.index = index; 268 parameter.base = base; 269 parameter.length = length; 270 parameter.type = type; 271 272 sCpuRendezvous = sCpuRendezvous2 = 0; 273 call_all_cpus(&set_mtrr, ¶meter); 274 } 275 276 277 status_t 278 x86_get_mtrr(uint32 index, uint64 *_base, uint64 *_length, uint8 *_type) 279 { 280 // the MTRRs are identical on all CPUs, so it doesn't matter 281 // on which CPU this runs 282 return sCpuModule->get_mtrr(index, _base, _length, _type); 283 } 284 285 286 void 287 x86_set_mtrrs(uint8 defaultType, const x86_mtrr_info* infos, uint32 count) 288 { 289 if (sCpuModule == NULL) 290 return; 291 292 struct set_mtrrs_parameter parameter; 293 parameter.defaultType = defaultType; 294 parameter.infos = infos; 295 parameter.count = count; 296 297 sCpuRendezvous = sCpuRendezvous2 = 0; 298 call_all_cpus(&set_mtrrs, ¶meter); 299 } 300 301 302 extern "C" void 303 init_sse(void) 304 { 305 if (!x86_check_feature(IA32_FEATURE_SSE, FEATURE_COMMON) 306 || !x86_check_feature(IA32_FEATURE_FXSR, FEATURE_COMMON)) { 307 // we don't have proper SSE support 308 return; 309 } 310 311 // enable OS support for SSE 312 x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION); 313 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 314 315 gX86SwapFPUFunc = i386_fxsave_swap; 316 gHasSSE = true; 317 } 318 319 320 static void 321 load_tss(int cpu) 322 { 323 short seg = ((TSS_BASE_SEGMENT + cpu) << 3) | DPL_KERNEL; 324 asm("movw %0, %%ax;" 325 "ltr %%ax;" : : "r" (seg) : "eax"); 326 } 327 328 329 static void 330 init_double_fault(int cpuNum) 331 { 332 // set up the double fault TSS 333 struct tss *tss = &gCPU[cpuNum].arch.double_fault_tss; 334 335 memset(tss, 0, sizeof(struct tss)); 336 size_t stackSize; 337 tss->sp0 = (uint32)x86_get_double_fault_stack(cpuNum, &stackSize); 338 tss->sp0 += stackSize; 339 tss->ss0 = KERNEL_DATA_SEG; 340 read_cr3(tss->cr3); 341 // copy the current cr3 to the double fault cr3 342 tss->eip = (uint32)&double_fault; 343 tss->es = KERNEL_DATA_SEG; 344 tss->cs = KERNEL_CODE_SEG; 345 tss->ss = KERNEL_DATA_SEG; 346 tss->esp = tss->sp0; 347 tss->ds = KERNEL_DATA_SEG; 348 tss->fs = KERNEL_DATA_SEG; 349 tss->gs = KERNEL_DATA_SEG; 350 tss->ldt_seg_selector = 0; 351 tss->io_map_base = sizeof(struct tss); 352 353 // add TSS descriptor for this new TSS 354 uint16 tssSegmentDescriptorIndex = DOUBLE_FAULT_TSS_BASE_SEGMENT + cpuNum; 355 set_tss_descriptor(&gGDT[tssSegmentDescriptorIndex], 356 (addr_t)tss, sizeof(struct tss)); 357 358 x86_set_task_gate(cpuNum, 8, tssSegmentDescriptorIndex << 3); 359 } 360 361 362 #if DUMP_FEATURE_STRING 363 static void 364 dump_feature_string(int currentCPU, cpu_ent *cpu) 365 { 366 char features[256]; 367 features[0] = 0; 368 369 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FPU) 370 strlcat(features, "fpu ", sizeof(features)); 371 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_VME) 372 strlcat(features, "vme ", sizeof(features)); 373 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DE) 374 strlcat(features, "de ", sizeof(features)); 375 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE) 376 strlcat(features, "pse ", sizeof(features)); 377 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TSC) 378 strlcat(features, "tsc ", sizeof(features)); 379 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MSR) 380 strlcat(features, "msr ", sizeof(features)); 381 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAE) 382 strlcat(features, "pae ", sizeof(features)); 383 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCE) 384 strlcat(features, "mce ", sizeof(features)); 385 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CX8) 386 strlcat(features, "cx8 ", sizeof(features)); 387 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_APIC) 388 strlcat(features, "apic ", sizeof(features)); 389 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SEP) 390 strlcat(features, "sep ", sizeof(features)); 391 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MTRR) 392 strlcat(features, "mtrr ", sizeof(features)); 393 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PGE) 394 strlcat(features, "pge ", sizeof(features)); 395 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCA) 396 strlcat(features, "mca ", sizeof(features)); 397 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CMOV) 398 strlcat(features, "cmov ", sizeof(features)); 399 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAT) 400 strlcat(features, "pat ", sizeof(features)); 401 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE36) 402 strlcat(features, "pse36 ", sizeof(features)); 403 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSN) 404 strlcat(features, "psn ", sizeof(features)); 405 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CLFSH) 406 strlcat(features, "clfsh ", sizeof(features)); 407 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DS) 408 strlcat(features, "ds ", sizeof(features)); 409 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_ACPI) 410 strlcat(features, "acpi ", sizeof(features)); 411 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MMX) 412 strlcat(features, "mmx ", sizeof(features)); 413 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FXSR) 414 strlcat(features, "fxsr ", sizeof(features)); 415 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE) 416 strlcat(features, "sse ", sizeof(features)); 417 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE2) 418 strlcat(features, "sse2 ", sizeof(features)); 419 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SS) 420 strlcat(features, "ss ", sizeof(features)); 421 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_HTT) 422 strlcat(features, "htt ", sizeof(features)); 423 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TM) 424 strlcat(features, "tm ", sizeof(features)); 425 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PBE) 426 strlcat(features, "pbe ", sizeof(features)); 427 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE3) 428 strlcat(features, "sse3 ", sizeof(features)); 429 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCLMULQDQ) 430 strlcat(features, "pclmulqdq ", sizeof(features)); 431 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DTES64) 432 strlcat(features, "dtes64 ", sizeof(features)); 433 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MONITOR) 434 strlcat(features, "monitor ", sizeof(features)); 435 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DSCPL) 436 strlcat(features, "dscpl ", sizeof(features)); 437 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_VMX) 438 strlcat(features, "vmx ", sizeof(features)); 439 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SMX) 440 strlcat(features, "smx ", sizeof(features)); 441 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_EST) 442 strlcat(features, "est ", sizeof(features)); 443 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TM2) 444 strlcat(features, "tm2 ", sizeof(features)); 445 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSSE3) 446 strlcat(features, "ssse3 ", sizeof(features)); 447 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CNXTID) 448 strlcat(features, "cnxtid ", sizeof(features)); 449 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_FMA) 450 strlcat(features, "fma ", sizeof(features)); 451 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CX16) 452 strlcat(features, "cx16 ", sizeof(features)); 453 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XTPR) 454 strlcat(features, "xtpr ", sizeof(features)); 455 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PDCM) 456 strlcat(features, "pdcm ", sizeof(features)); 457 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCID) 458 strlcat(features, "pcid ", sizeof(features)); 459 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DCA) 460 strlcat(features, "dca ", sizeof(features)); 461 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_1) 462 strlcat(features, "sse4_1 ", sizeof(features)); 463 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_2) 464 strlcat(features, "sse4_2 ", sizeof(features)); 465 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_X2APIC) 466 strlcat(features, "x2apic ", sizeof(features)); 467 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MOVBE) 468 strlcat(features, "movbe ", sizeof(features)); 469 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_POPCNT) 470 strlcat(features, "popcnt ", sizeof(features)); 471 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TSCDEADLINE) 472 strlcat(features, "tscdeadline ", sizeof(features)); 473 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AES) 474 strlcat(features, "aes ", sizeof(features)); 475 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XSAVE) 476 strlcat(features, "xsave ", sizeof(features)); 477 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_OSXSAVE) 478 strlcat(features, "osxsave ", sizeof(features)); 479 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AVX) 480 strlcat(features, "avx ", sizeof(features)); 481 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_F16C) 482 strlcat(features, "f16c ", sizeof(features)); 483 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_RDRND) 484 strlcat(features, "rdrnd ", sizeof(features)); 485 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) 486 strlcat(features, "hypervisor ", sizeof(features)); 487 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_SYSCALL) 488 strlcat(features, "syscall ", sizeof(features)); 489 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_NX) 490 strlcat(features, "nx ", sizeof(features)); 491 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_MMXEXT) 492 strlcat(features, "mmxext ", sizeof(features)); 493 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_FFXSR) 494 strlcat(features, "ffxsr ", sizeof(features)); 495 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_LONG) 496 strlcat(features, "long ", sizeof(features)); 497 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOWEXT) 498 strlcat(features, "3dnowext ", sizeof(features)); 499 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOW) 500 strlcat(features, "3dnow ", sizeof(features)); 501 502 dprintf("CPU %d: features: %s\n", currentCPU, features); 503 } 504 #endif // DUMP_FEATURE_STRING 505 506 507 static int 508 detect_cpu(int currentCPU) 509 { 510 cpu_ent *cpu = get_cpu_struct(); 511 char vendorString[17]; 512 cpuid_info cpuid; 513 514 // clear out the cpu info data 515 cpu->arch.vendor = VENDOR_UNKNOWN; 516 cpu->arch.vendor_name = "UNKNOWN VENDOR"; 517 cpu->arch.feature[FEATURE_COMMON] = 0; 518 cpu->arch.feature[FEATURE_EXT] = 0; 519 cpu->arch.feature[FEATURE_EXT_AMD] = 0; 520 cpu->arch.model_name[0] = 0; 521 522 // print some fun data 523 get_current_cpuid(&cpuid, 0); 524 525 // build the vendor string 526 memset(vendorString, 0, sizeof(vendorString)); 527 memcpy(vendorString, cpuid.eax_0.vendor_id, sizeof(cpuid.eax_0.vendor_id)); 528 529 // get the family, model, stepping 530 get_current_cpuid(&cpuid, 1); 531 cpu->arch.type = cpuid.eax_1.type; 532 cpu->arch.family = cpuid.eax_1.family; 533 cpu->arch.extended_family = cpuid.eax_1.extended_family; 534 cpu->arch.model = cpuid.eax_1.model; 535 cpu->arch.extended_model = cpuid.eax_1.extended_model; 536 cpu->arch.stepping = cpuid.eax_1.stepping; 537 dprintf("CPU %d: type %d family %d extended_family %d model %d " 538 "extended_model %d stepping %d, string '%s'\n", 539 currentCPU, cpu->arch.type, cpu->arch.family, 540 cpu->arch.extended_family, cpu->arch.model, 541 cpu->arch.extended_model, cpu->arch.stepping, vendorString); 542 543 // figure out what vendor we have here 544 545 for (int32 i = 0; i < VENDOR_NUM; i++) { 546 if (vendor_info[i].ident_string[0] 547 && !strcmp(vendorString, vendor_info[i].ident_string[0])) { 548 cpu->arch.vendor = (x86_vendors)i; 549 cpu->arch.vendor_name = vendor_info[i].vendor; 550 break; 551 } 552 if (vendor_info[i].ident_string[1] 553 && !strcmp(vendorString, vendor_info[i].ident_string[1])) { 554 cpu->arch.vendor = (x86_vendors)i; 555 cpu->arch.vendor_name = vendor_info[i].vendor; 556 break; 557 } 558 } 559 560 // see if we can get the model name 561 get_current_cpuid(&cpuid, 0x80000000); 562 if (cpuid.eax_0.max_eax >= 0x80000004) { 563 // build the model string (need to swap ecx/edx data before copying) 564 unsigned int temp; 565 memset(cpu->arch.model_name, 0, sizeof(cpu->arch.model_name)); 566 567 get_current_cpuid(&cpuid, 0x80000002); 568 temp = cpuid.regs.edx; 569 cpuid.regs.edx = cpuid.regs.ecx; 570 cpuid.regs.ecx = temp; 571 memcpy(cpu->arch.model_name, cpuid.as_chars, sizeof(cpuid.as_chars)); 572 573 get_current_cpuid(&cpuid, 0x80000003); 574 temp = cpuid.regs.edx; 575 cpuid.regs.edx = cpuid.regs.ecx; 576 cpuid.regs.ecx = temp; 577 memcpy(cpu->arch.model_name + 16, cpuid.as_chars, 578 sizeof(cpuid.as_chars)); 579 580 get_current_cpuid(&cpuid, 0x80000004); 581 temp = cpuid.regs.edx; 582 cpuid.regs.edx = cpuid.regs.ecx; 583 cpuid.regs.ecx = temp; 584 memcpy(cpu->arch.model_name + 32, cpuid.as_chars, 585 sizeof(cpuid.as_chars)); 586 587 // some cpus return a right-justified string 588 int32 i = 0; 589 while (cpu->arch.model_name[i] == ' ') 590 i++; 591 if (i > 0) { 592 memmove(cpu->arch.model_name, &cpu->arch.model_name[i], 593 strlen(&cpu->arch.model_name[i]) + 1); 594 } 595 596 dprintf("CPU %d: vendor '%s' model name '%s'\n", 597 currentCPU, cpu->arch.vendor_name, cpu->arch.model_name); 598 } else { 599 strlcpy(cpu->arch.model_name, "unknown", sizeof(cpu->arch.model_name)); 600 } 601 602 // load feature bits 603 get_current_cpuid(&cpuid, 1); 604 cpu->arch.feature[FEATURE_COMMON] = cpuid.eax_1.features; // edx 605 cpu->arch.feature[FEATURE_EXT] = cpuid.eax_1.extended_features; // ecx 606 if (cpu->arch.vendor == VENDOR_AMD) { 607 get_current_cpuid(&cpuid, 0x80000001); 608 cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx 609 } 610 611 #if DUMP_FEATURE_STRING 612 dump_feature_string(currentCPU, cpu); 613 #endif 614 615 return 0; 616 } 617 618 619 bool 620 x86_check_feature(uint32 feature, enum x86_feature_type type) 621 { 622 cpu_ent *cpu = get_cpu_struct(); 623 624 #if 0 625 int i; 626 dprintf("x86_check_feature: feature 0x%x, type %d\n", feature, type); 627 for (i = 0; i < FEATURE_NUM; i++) { 628 dprintf("features %d: 0x%x\n", i, cpu->arch.feature[i]); 629 } 630 #endif 631 632 return (cpu->arch.feature[type] & feature) != 0; 633 } 634 635 636 void* 637 x86_get_double_fault_stack(int32 cpu, size_t* _size) 638 { 639 *_size = kDoubleFaultStackSize; 640 return sDoubleFaultStacks + kDoubleFaultStackSize * cpu; 641 } 642 643 644 /*! Returns the index of the current CPU. Can only be called from the double 645 fault handler. 646 */ 647 int32 648 x86_double_fault_get_cpu(void) 649 { 650 uint32 stack = x86_read_ebp(); 651 return (stack - (uint32)sDoubleFaultStacks) / kDoubleFaultStackSize; 652 } 653 654 655 // #pragma mark - 656 657 658 status_t 659 arch_cpu_preboot_init_percpu(kernel_args *args, int cpu) 660 { 661 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 662 gX86SwapFPUFunc = i386_fnsave_swap; 663 664 // On SMP system we want to synchronize the CPUs' TSCs, so system_time() 665 // will return consistent values. 666 if (smp_get_num_cpus() > 1) { 667 // let the first CPU prepare the rendezvous point 668 if (cpu == 0) 669 sTSCSyncRendezvous = smp_get_num_cpus() - 1; 670 671 // One CPU after the other will drop out of this loop and be caught by 672 // the loop below, until the last CPU (0) gets there. Save for +/- a few 673 // cycles the CPUs should pass the second loop at the same time. 674 while (sTSCSyncRendezvous != cpu) { 675 } 676 677 sTSCSyncRendezvous = cpu - 1; 678 679 while (sTSCSyncRendezvous != -1) { 680 } 681 682 // reset TSC to 0 683 x86_write_msr(IA32_MSR_TSC, 0); 684 } 685 686 return B_OK; 687 } 688 689 690 status_t 691 arch_cpu_init_percpu(kernel_args *args, int cpu) 692 { 693 detect_cpu(cpu); 694 695 // load the TSS for this cpu 696 // note the main cpu gets initialized in arch_cpu_init_post_vm() 697 if (cpu != 0) { 698 load_tss(cpu); 699 700 // set the IDT 701 struct { 702 uint16 limit; 703 void* address; 704 } _PACKED descriptor = { 705 256 * 8 - 1, // 256 descriptors, 8 bytes each (-1 for "limit") 706 x86_get_idt(cpu) 707 }; 708 709 asm volatile("lidt %0" : : "m"(descriptor)); 710 } 711 712 return 0; 713 } 714 715 status_t 716 arch_cpu_init(kernel_args *args) 717 { 718 // init the TSC -> system_time() conversion factors 719 720 uint32 conversionFactor = args->arch_args.system_time_cv_factor; 721 uint64 conversionFactorNsecs = (uint64)conversionFactor * 1000; 722 723 if (conversionFactorNsecs >> 32 != 0) { 724 // the TSC frequency is < 1 GHz, which forces us to shift the factor 725 __x86_setup_system_time(conversionFactor, conversionFactorNsecs >> 16, 726 true); 727 } else { 728 // the TSC frequency is >= 1 GHz 729 __x86_setup_system_time(conversionFactor, conversionFactorNsecs, false); 730 } 731 732 return B_OK; 733 } 734 735 736 status_t 737 arch_cpu_init_post_vm(kernel_args *args) 738 { 739 uint32 i; 740 741 // account for the segment descriptors 742 gGDT = (segment_descriptor *)args->arch_args.vir_gdt; 743 create_area("gdt", (void **)&gGDT, B_EXACT_ADDRESS, B_PAGE_SIZE, 744 B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 745 746 // currently taken out of the build, because it's not yet used (and assumes 747 // (a fixed number of used GDT entries) 748 //i386_selector_init(gGDT); // pass the new gdt 749 750 // allocate an area for the double fault stacks 751 virtual_address_restrictions virtualRestrictions = {}; 752 virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS; 753 physical_address_restrictions physicalRestrictions = {}; 754 create_area_etc(B_SYSTEM_TEAM, "double fault stacks", 755 kDoubleFaultStackSize * smp_get_num_cpus(), B_FULL_LOCK, 756 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 757 &virtualRestrictions, &physicalRestrictions, 758 (void**)&sDoubleFaultStacks); 759 760 X86PagingStructures* kernelPagingStructures 761 = static_cast<X86VMTranslationMap*>( 762 VMAddressSpace::Kernel()->TranslationMap())->PagingStructures(); 763 764 // setup task-state segments 765 for (i = 0; i < args->num_cpus; i++) { 766 // initialize the regular and double fault tss stored in the per-cpu 767 // structure 768 memset(&gCPU[i].arch.tss, 0, sizeof(struct tss)); 769 gCPU[i].arch.tss.ss0 = KERNEL_DATA_SEG; 770 gCPU[i].arch.tss.io_map_base = sizeof(struct tss); 771 772 // add TSS descriptor for this new TSS 773 set_tss_descriptor(&gGDT[TSS_BASE_SEGMENT + i], 774 (addr_t)&gCPU[i].arch.tss, sizeof(struct tss)); 775 776 // initialize the double fault tss 777 init_double_fault(i); 778 779 // init active translation map 780 gCPU[i].arch.active_paging_structures = kernelPagingStructures; 781 kernelPagingStructures->AddReference(); 782 } 783 784 // set the current hardware task on cpu 0 785 load_tss(0); 786 787 // setup TLS descriptors (one for every CPU) 788 789 for (i = 0; i < args->num_cpus; i++) { 790 set_segment_descriptor(&gGDT[TLS_BASE_SEGMENT + i], 0, TLS_SIZE, 791 DT_DATA_WRITEABLE, DPL_USER); 792 } 793 794 // setup SSE2/3 support 795 init_sse(); 796 797 return B_OK; 798 } 799 800 801 status_t 802 arch_cpu_init_post_modules(kernel_args *args) 803 { 804 // initialize CPU module 805 806 void *cookie = open_module_list("cpu"); 807 808 while (true) { 809 char name[B_FILE_NAME_LENGTH]; 810 size_t nameLength = sizeof(name); 811 812 if (read_next_module_name(cookie, name, &nameLength) != B_OK 813 || get_module(name, (module_info **)&sCpuModule) == B_OK) 814 break; 815 } 816 817 close_module_list(cookie); 818 819 // initialize MTRRs if available 820 if (x86_count_mtrrs() > 0) { 821 sCpuRendezvous = sCpuRendezvous2 = 0; 822 call_all_cpus(&init_mtrrs, NULL); 823 } 824 825 // get optimized functions from the CPU module 826 if (sCpuModule != NULL && sCpuModule->get_optimized_functions != NULL) { 827 x86_optimized_functions functions; 828 memset(&functions, 0, sizeof(functions)); 829 830 sCpuModule->get_optimized_functions(&functions); 831 832 if (functions.memcpy != NULL) { 833 gOptimizedFunctions.memcpy = functions.memcpy; 834 gOptimizedFunctions.memcpy_end = functions.memcpy_end; 835 } 836 837 if (functions.memset != NULL) { 838 gOptimizedFunctions.memset = functions.memset; 839 gOptimizedFunctions.memset_end = functions.memset_end; 840 } 841 } 842 843 // put the optimized functions into the commpage 844 size_t memcpyLen = (addr_t)gOptimizedFunctions.memcpy_end 845 - (addr_t)gOptimizedFunctions.memcpy; 846 fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMCPY, 847 (const void*)gOptimizedFunctions.memcpy, memcpyLen); 848 size_t memsetLen = (addr_t)gOptimizedFunctions.memset_end 849 - (addr_t)gOptimizedFunctions.memset; 850 fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMSET, 851 (const void*)gOptimizedFunctions.memset, memsetLen); 852 853 // add the functions to the commpage image 854 image_id image = get_commpage_image(); 855 elf_add_memory_image_symbol(image, "commpage_memcpy", 856 ((addr_t*)USER_COMMPAGE_ADDR)[COMMPAGE_ENTRY_X86_MEMCPY], memcpyLen, 857 B_SYMBOL_TYPE_TEXT); 858 elf_add_memory_image_symbol(image, "commpage_memset", 859 ((addr_t*)USER_COMMPAGE_ADDR)[COMMPAGE_ENTRY_X86_MEMSET], memsetLen, 860 B_SYMBOL_TYPE_TEXT); 861 862 return B_OK; 863 } 864 865 866 void 867 i386_set_tss_and_kstack(addr_t kstack) 868 { 869 get_cpu_struct()->arch.tss.sp0 = kstack; 870 } 871 872 void 873 arch_cpu_global_TLB_invalidate(void) 874 { 875 uint32 flags = x86_read_cr4(); 876 877 if (flags & IA32_CR4_GLOBAL_PAGES) { 878 // disable and reenable the global pages to flush all TLBs regardless 879 // of the global page bit 880 x86_write_cr4(flags & ~IA32_CR4_GLOBAL_PAGES); 881 x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES); 882 } else { 883 cpu_status state = disable_interrupts(); 884 arch_cpu_user_TLB_invalidate(); 885 restore_interrupts(state); 886 } 887 } 888 889 890 void 891 arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) 892 { 893 int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE; 894 while (num_pages-- >= 0) { 895 invalidate_TLB(start); 896 start += B_PAGE_SIZE; 897 } 898 } 899 900 901 void 902 arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) 903 { 904 int i; 905 for (i = 0; i < num_pages; i++) { 906 invalidate_TLB(pages[i]); 907 } 908 } 909 910 911 ssize_t 912 arch_cpu_user_strlcpy(char *to, const char *from, size_t size, 913 addr_t *faultHandler) 914 { 915 int fromLength = 0; 916 addr_t oldFaultHandler = *faultHandler; 917 918 // this check is to trick the gcc4 compiler and have it keep the error label 919 if (to == NULL && size > 0) 920 goto error; 921 922 *faultHandler = (addr_t)&&error; 923 924 if (size > 0) { 925 to[--size] = '\0'; 926 // copy 927 for ( ; size; size--, fromLength++, to++, from++) { 928 if ((*to = *from) == '\0') 929 break; 930 } 931 } 932 // count any leftover from chars 933 while (*from++ != '\0') { 934 fromLength++; 935 } 936 937 *faultHandler = oldFaultHandler; 938 return fromLength; 939 940 error: 941 *faultHandler = oldFaultHandler; 942 return B_BAD_ADDRESS; 943 } 944 945 946 status_t 947 arch_cpu_user_memset(void *s, char c, size_t count, addr_t *faultHandler) 948 { 949 char *xs = (char *)s; 950 addr_t oldFaultHandler = *faultHandler; 951 952 // this check is to trick the gcc4 compiler and have it keep the error label 953 if (s == NULL) 954 goto error; 955 956 *faultHandler = (addr_t)&&error; 957 958 while (count--) 959 *xs++ = c; 960 961 *faultHandler = oldFaultHandler; 962 return 0; 963 964 error: 965 *faultHandler = oldFaultHandler; 966 return B_BAD_ADDRESS; 967 } 968 969 970 status_t 971 arch_cpu_shutdown(bool rebootSystem) 972 { 973 if (acpi_shutdown(rebootSystem) == B_OK) 974 return B_OK; 975 976 if (!rebootSystem) 977 return apm_shutdown(); 978 979 cpu_status state = disable_interrupts(); 980 981 // try to reset the system using the keyboard controller 982 out8(0xfe, 0x64); 983 984 // Give some time to the controller to do its job (0.5s) 985 snooze(500000); 986 987 // if that didn't help, try it this way 988 reboot(); 989 990 restore_interrupts(state); 991 return B_ERROR; 992 } 993 994 995 void 996 arch_cpu_idle(void) 997 { 998 asm("hlt"); 999 } 1000 1001 1002 void 1003 arch_cpu_sync_icache(void *address, size_t length) 1004 { 1005 // instruction cache is always consistent on x86 1006 } 1007 1008 1009 void 1010 arch_cpu_memory_read_barrier(void) 1011 { 1012 asm volatile ("lock;" : : : "memory"); 1013 asm volatile ("addl $0, 0(%%esp);" : : : "memory"); 1014 } 1015 1016 1017 void 1018 arch_cpu_memory_write_barrier(void) 1019 { 1020 asm volatile ("lock;" : : : "memory"); 1021 asm volatile ("addl $0, 0(%%esp);" : : : "memory"); 1022 } 1023 1024