1 /* 2 * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. 3 * Distributed under the terms of the MIT License. 4 * 5 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 6 * Distributed under the terms of the NewOS License. 7 */ 8 9 10 #include <cpu.h> 11 12 #include <string.h> 13 #include <stdlib.h> 14 #include <stdio.h> 15 16 #include <ACPI.h> 17 18 #include <boot_device.h> 19 #include <commpage.h> 20 #include <debug.h> 21 #include <elf.h> 22 #include <smp.h> 23 #include <tls.h> 24 #include <vm/vm.h> 25 #include <vm/vm_types.h> 26 #include <vm/VMAddressSpace.h> 27 28 #include <arch_system_info.h> 29 #include <arch/x86/selector.h> 30 #include <boot/kernel_args.h> 31 32 #include "interrupts.h" 33 #include "x86_paging.h" 34 #include "X86VMTranslationMap.h" 35 36 37 #define DUMP_FEATURE_STRING 1 38 39 40 /* cpu vendor info */ 41 struct cpu_vendor_info { 42 const char *vendor; 43 const char *ident_string[2]; 44 }; 45 46 static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = { 47 { "Intel", { "GenuineIntel" } }, 48 { "AMD", { "AuthenticAMD" } }, 49 { "Cyrix", { "CyrixInstead" } }, 50 { "UMC", { "UMC UMC UMC" } }, 51 { "NexGen", { "NexGenDriven" } }, 52 { "Centaur", { "CentaurHauls" } }, 53 { "Rise", { "RiseRiseRise" } }, 54 { "Transmeta", { "GenuineTMx86", "TransmetaCPU" } }, 55 { "NSC", { "Geode by NSC" } }, 56 }; 57 58 #define CR0_CACHE_DISABLE (1UL << 30) 59 #define CR0_NOT_WRITE_THROUGH (1UL << 29) 60 #define CR0_FPU_EMULATION (1UL << 2) 61 #define CR0_MONITOR_FPU (1UL << 1) 62 63 #define CR4_OS_FXSR (1UL << 9) 64 #define CR4_OS_XMM_EXCEPTION (1UL << 10) 65 66 struct set_mtrr_parameter { 67 int32 index; 68 uint64 base; 69 uint64 length; 70 uint8 type; 71 }; 72 73 struct set_mtrrs_parameter { 74 const x86_mtrr_info* infos; 75 uint32 count; 76 uint8 defaultType; 77 }; 78 79 80 extern "C" void reboot(void); 81 // from arch_x86.S 82 83 void (*gX86SwapFPUFunc)(void *oldState, const void *newState); 84 bool gHasSSE = false; 85 86 static uint32 sCpuRendezvous; 87 static uint32 sCpuRendezvous2; 88 static uint32 sCpuRendezvous3; 89 static vint32 sTSCSyncRendezvous; 90 91 segment_descriptor *gGDT = NULL; 92 93 /* Some specials for the double fault handler */ 94 static uint8* sDoubleFaultStacks; 95 static const size_t kDoubleFaultStackSize = 4096; // size per CPU 96 97 static x86_cpu_module_info *sCpuModule; 98 99 100 extern "C" void memcpy_generic(void* dest, const void* source, size_t count); 101 extern int memcpy_generic_end; 102 extern "C" void memset_generic(void* dest, int value, size_t count); 103 extern int memset_generic_end; 104 105 x86_optimized_functions gOptimizedFunctions = { 106 memcpy_generic, 107 &memcpy_generic_end, 108 memset_generic, 109 &memset_generic_end 110 }; 111 112 113 static status_t 114 acpi_shutdown(bool rebootSystem) 115 { 116 if (debug_debugger_running() || !are_interrupts_enabled()) 117 return B_ERROR; 118 119 acpi_module_info* acpi; 120 if (get_module(B_ACPI_MODULE_NAME, (module_info**)&acpi) != B_OK) 121 return B_NOT_SUPPORTED; 122 123 status_t status; 124 if (rebootSystem) { 125 status = acpi->reboot(); 126 } else { 127 status = acpi->prepare_sleep_state(ACPI_POWER_STATE_OFF, NULL, 0); 128 if (status == B_OK) { 129 //cpu_status state = disable_interrupts(); 130 status = acpi->enter_sleep_state(ACPI_POWER_STATE_OFF); 131 //restore_interrupts(state); 132 } 133 } 134 135 put_module(B_ACPI_MODULE_NAME); 136 return status; 137 } 138 139 140 /*! Disable CPU caches, and invalidate them. */ 141 static void 142 disable_caches() 143 { 144 x86_write_cr0((x86_read_cr0() | CR0_CACHE_DISABLE) 145 & ~CR0_NOT_WRITE_THROUGH); 146 wbinvd(); 147 arch_cpu_global_TLB_invalidate(); 148 } 149 150 151 /*! Invalidate CPU caches, and enable them. */ 152 static void 153 enable_caches() 154 { 155 wbinvd(); 156 arch_cpu_global_TLB_invalidate(); 157 x86_write_cr0(x86_read_cr0() 158 & ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH)); 159 } 160 161 162 static void 163 set_mtrr(void *_parameter, int cpu) 164 { 165 struct set_mtrr_parameter *parameter 166 = (struct set_mtrr_parameter *)_parameter; 167 168 // wait until all CPUs have arrived here 169 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 170 171 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 172 // that initiated the call_all_cpus() from doing that again and clearing 173 // sCpuRendezvous2 before the last CPU has actually left the loop in 174 // smp_cpu_rendezvous(); 175 if (cpu == 0) 176 atomic_set((vint32*)&sCpuRendezvous3, 0); 177 178 disable_caches(); 179 180 sCpuModule->set_mtrr(parameter->index, parameter->base, parameter->length, 181 parameter->type); 182 183 enable_caches(); 184 185 // wait until all CPUs have arrived here 186 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 187 smp_cpu_rendezvous(&sCpuRendezvous3, cpu); 188 } 189 190 191 static void 192 set_mtrrs(void* _parameter, int cpu) 193 { 194 set_mtrrs_parameter* parameter = (set_mtrrs_parameter*)_parameter; 195 196 // wait until all CPUs have arrived here 197 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 198 199 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 200 // that initiated the call_all_cpus() from doing that again and clearing 201 // sCpuRendezvous2 before the last CPU has actually left the loop in 202 // smp_cpu_rendezvous(); 203 if (cpu == 0) 204 atomic_set((vint32*)&sCpuRendezvous3, 0); 205 206 disable_caches(); 207 208 sCpuModule->set_mtrrs(parameter->defaultType, parameter->infos, 209 parameter->count); 210 211 enable_caches(); 212 213 // wait until all CPUs have arrived here 214 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 215 smp_cpu_rendezvous(&sCpuRendezvous3, cpu); 216 } 217 218 219 static void 220 init_mtrrs(void *_unused, int cpu) 221 { 222 // wait until all CPUs have arrived here 223 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 224 225 // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU 226 // that initiated the call_all_cpus() from doing that again and clearing 227 // sCpuRendezvous2 before the last CPU has actually left the loop in 228 // smp_cpu_rendezvous(); 229 if (cpu == 0) 230 atomic_set((vint32*)&sCpuRendezvous3, 0); 231 232 disable_caches(); 233 234 sCpuModule->init_mtrrs(); 235 236 enable_caches(); 237 238 // wait until all CPUs have arrived here 239 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 240 smp_cpu_rendezvous(&sCpuRendezvous3, cpu); 241 } 242 243 244 uint32 245 x86_count_mtrrs(void) 246 { 247 if (sCpuModule == NULL) 248 return 0; 249 250 return sCpuModule->count_mtrrs(); 251 } 252 253 254 void 255 x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type) 256 { 257 struct set_mtrr_parameter parameter; 258 parameter.index = index; 259 parameter.base = base; 260 parameter.length = length; 261 parameter.type = type; 262 263 sCpuRendezvous = sCpuRendezvous2 = 0; 264 call_all_cpus(&set_mtrr, ¶meter); 265 } 266 267 268 status_t 269 x86_get_mtrr(uint32 index, uint64 *_base, uint64 *_length, uint8 *_type) 270 { 271 // the MTRRs are identical on all CPUs, so it doesn't matter 272 // on which CPU this runs 273 return sCpuModule->get_mtrr(index, _base, _length, _type); 274 } 275 276 277 void 278 x86_set_mtrrs(uint8 defaultType, const x86_mtrr_info* infos, uint32 count) 279 { 280 if (sCpuModule == NULL) 281 return; 282 283 struct set_mtrrs_parameter parameter; 284 parameter.defaultType = defaultType; 285 parameter.infos = infos; 286 parameter.count = count; 287 288 sCpuRendezvous = sCpuRendezvous2 = 0; 289 call_all_cpus(&set_mtrrs, ¶meter); 290 } 291 292 293 extern "C" void 294 init_sse(void) 295 { 296 if (!x86_check_feature(IA32_FEATURE_SSE, FEATURE_COMMON) 297 || !x86_check_feature(IA32_FEATURE_FXSR, FEATURE_COMMON)) { 298 // we don't have proper SSE support 299 return; 300 } 301 302 // enable OS support for SSE 303 x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION); 304 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 305 306 gX86SwapFPUFunc = i386_fxsave_swap; 307 gHasSSE = true; 308 } 309 310 311 static void 312 load_tss(int cpu) 313 { 314 short seg = ((TSS_BASE_SEGMENT + cpu) << 3) | DPL_KERNEL; 315 asm("movw %0, %%ax;" 316 "ltr %%ax;" : : "r" (seg) : "eax"); 317 } 318 319 320 static void 321 init_double_fault(int cpuNum) 322 { 323 // set up the double fault TSS 324 struct tss *tss = &gCPU[cpuNum].arch.double_fault_tss; 325 326 memset(tss, 0, sizeof(struct tss)); 327 size_t stackSize; 328 tss->sp0 = (uint32)x86_get_double_fault_stack(cpuNum, &stackSize); 329 tss->sp0 += stackSize; 330 tss->ss0 = KERNEL_DATA_SEG; 331 read_cr3(tss->cr3); 332 // copy the current cr3 to the double fault cr3 333 tss->eip = (uint32)&double_fault; 334 tss->es = KERNEL_DATA_SEG; 335 tss->cs = KERNEL_CODE_SEG; 336 tss->ss = KERNEL_DATA_SEG; 337 tss->esp = tss->sp0; 338 tss->ds = KERNEL_DATA_SEG; 339 tss->fs = KERNEL_DATA_SEG; 340 tss->gs = KERNEL_DATA_SEG; 341 tss->ldt_seg_selector = 0; 342 tss->io_map_base = sizeof(struct tss); 343 344 // add TSS descriptor for this new TSS 345 uint16 tssSegmentDescriptorIndex = DOUBLE_FAULT_TSS_BASE_SEGMENT + cpuNum; 346 set_tss_descriptor(&gGDT[tssSegmentDescriptorIndex], 347 (addr_t)tss, sizeof(struct tss)); 348 349 x86_set_task_gate(cpuNum, 8, tssSegmentDescriptorIndex << 3); 350 } 351 352 353 #if DUMP_FEATURE_STRING 354 static void 355 dump_feature_string(int currentCPU, cpu_ent *cpu) 356 { 357 char features[256]; 358 features[0] = 0; 359 360 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FPU) 361 strlcat(features, "fpu ", sizeof(features)); 362 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_VME) 363 strlcat(features, "vme ", sizeof(features)); 364 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DE) 365 strlcat(features, "de ", sizeof(features)); 366 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE) 367 strlcat(features, "pse ", sizeof(features)); 368 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TSC) 369 strlcat(features, "tsc ", sizeof(features)); 370 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MSR) 371 strlcat(features, "msr ", sizeof(features)); 372 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAE) 373 strlcat(features, "pae ", sizeof(features)); 374 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCE) 375 strlcat(features, "mce ", sizeof(features)); 376 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CX8) 377 strlcat(features, "cx8 ", sizeof(features)); 378 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_APIC) 379 strlcat(features, "apic ", sizeof(features)); 380 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SEP) 381 strlcat(features, "sep ", sizeof(features)); 382 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MTRR) 383 strlcat(features, "mtrr ", sizeof(features)); 384 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PGE) 385 strlcat(features, "pge ", sizeof(features)); 386 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCA) 387 strlcat(features, "mca ", sizeof(features)); 388 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CMOV) 389 strlcat(features, "cmov ", sizeof(features)); 390 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAT) 391 strlcat(features, "pat ", sizeof(features)); 392 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE36) 393 strlcat(features, "pse36 ", sizeof(features)); 394 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSN) 395 strlcat(features, "psn ", sizeof(features)); 396 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CLFSH) 397 strlcat(features, "clfsh ", sizeof(features)); 398 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DS) 399 strlcat(features, "ds ", sizeof(features)); 400 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_ACPI) 401 strlcat(features, "acpi ", sizeof(features)); 402 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MMX) 403 strlcat(features, "mmx ", sizeof(features)); 404 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FXSR) 405 strlcat(features, "fxsr ", sizeof(features)); 406 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE) 407 strlcat(features, "sse ", sizeof(features)); 408 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE2) 409 strlcat(features, "sse2 ", sizeof(features)); 410 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SS) 411 strlcat(features, "ss ", sizeof(features)); 412 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_HTT) 413 strlcat(features, "htt ", sizeof(features)); 414 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TM) 415 strlcat(features, "tm ", sizeof(features)); 416 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PBE) 417 strlcat(features, "pbe ", sizeof(features)); 418 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE3) 419 strlcat(features, "sse3 ", sizeof(features)); 420 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MONITOR) 421 strlcat(features, "monitor ", sizeof(features)); 422 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DSCPL) 423 strlcat(features, "dscpl ", sizeof(features)); 424 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_EST) 425 strlcat(features, "est ", sizeof(features)); 426 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TM2) 427 strlcat(features, "tm2 ", sizeof(features)); 428 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CNXTID) 429 strlcat(features, "cnxtid ", sizeof(features)); 430 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_SYSCALL) 431 strlcat(features, "syscall ", sizeof(features)); 432 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_NX) 433 strlcat(features, "nx ", sizeof(features)); 434 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_MMXEXT) 435 strlcat(features, "mmxext ", sizeof(features)); 436 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_FFXSR) 437 strlcat(features, "ffxsr ", sizeof(features)); 438 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_LONG) 439 strlcat(features, "long ", sizeof(features)); 440 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOWEXT) 441 strlcat(features, "3dnowext ", sizeof(features)); 442 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOW) 443 strlcat(features, "3dnow ", sizeof(features)); 444 445 dprintf("CPU %d: features: %s\n", currentCPU, features); 446 } 447 #endif // DUMP_FEATURE_STRING 448 449 450 static int 451 detect_cpu(int currentCPU) 452 { 453 cpu_ent *cpu = get_cpu_struct(); 454 char vendorString[17]; 455 cpuid_info cpuid; 456 457 // clear out the cpu info data 458 cpu->arch.vendor = VENDOR_UNKNOWN; 459 cpu->arch.vendor_name = "UNKNOWN VENDOR"; 460 cpu->arch.feature[FEATURE_COMMON] = 0; 461 cpu->arch.feature[FEATURE_EXT] = 0; 462 cpu->arch.feature[FEATURE_EXT_AMD] = 0; 463 cpu->arch.model_name[0] = 0; 464 465 // print some fun data 466 get_current_cpuid(&cpuid, 0); 467 468 // build the vendor string 469 memset(vendorString, 0, sizeof(vendorString)); 470 memcpy(vendorString, cpuid.eax_0.vendor_id, sizeof(cpuid.eax_0.vendor_id)); 471 472 // get the family, model, stepping 473 get_current_cpuid(&cpuid, 1); 474 cpu->arch.type = cpuid.eax_1.type; 475 cpu->arch.family = cpuid.eax_1.family; 476 cpu->arch.extended_family = cpuid.eax_1.extended_family; 477 cpu->arch.model = cpuid.eax_1.model; 478 cpu->arch.extended_model = cpuid.eax_1.extended_model; 479 cpu->arch.stepping = cpuid.eax_1.stepping; 480 dprintf("CPU %d: type %d family %d extended_family %d model %d " 481 "extended_model %d stepping %d, string '%s'\n", 482 currentCPU, cpu->arch.type, cpu->arch.family, 483 cpu->arch.extended_family, cpu->arch.model, 484 cpu->arch.extended_model, cpu->arch.stepping, vendorString); 485 486 // figure out what vendor we have here 487 488 for (int32 i = 0; i < VENDOR_NUM; i++) { 489 if (vendor_info[i].ident_string[0] 490 && !strcmp(vendorString, vendor_info[i].ident_string[0])) { 491 cpu->arch.vendor = (x86_vendors)i; 492 cpu->arch.vendor_name = vendor_info[i].vendor; 493 break; 494 } 495 if (vendor_info[i].ident_string[1] 496 && !strcmp(vendorString, vendor_info[i].ident_string[1])) { 497 cpu->arch.vendor = (x86_vendors)i; 498 cpu->arch.vendor_name = vendor_info[i].vendor; 499 break; 500 } 501 } 502 503 // see if we can get the model name 504 get_current_cpuid(&cpuid, 0x80000000); 505 if (cpuid.eax_0.max_eax >= 0x80000004) { 506 // build the model string (need to swap ecx/edx data before copying) 507 unsigned int temp; 508 memset(cpu->arch.model_name, 0, sizeof(cpu->arch.model_name)); 509 510 get_current_cpuid(&cpuid, 0x80000002); 511 temp = cpuid.regs.edx; 512 cpuid.regs.edx = cpuid.regs.ecx; 513 cpuid.regs.ecx = temp; 514 memcpy(cpu->arch.model_name, cpuid.as_chars, sizeof(cpuid.as_chars)); 515 516 get_current_cpuid(&cpuid, 0x80000003); 517 temp = cpuid.regs.edx; 518 cpuid.regs.edx = cpuid.regs.ecx; 519 cpuid.regs.ecx = temp; 520 memcpy(cpu->arch.model_name + 16, cpuid.as_chars, 521 sizeof(cpuid.as_chars)); 522 523 get_current_cpuid(&cpuid, 0x80000004); 524 temp = cpuid.regs.edx; 525 cpuid.regs.edx = cpuid.regs.ecx; 526 cpuid.regs.ecx = temp; 527 memcpy(cpu->arch.model_name + 32, cpuid.as_chars, 528 sizeof(cpuid.as_chars)); 529 530 // some cpus return a right-justified string 531 int32 i = 0; 532 while (cpu->arch.model_name[i] == ' ') 533 i++; 534 if (i > 0) { 535 memmove(cpu->arch.model_name, &cpu->arch.model_name[i], 536 strlen(&cpu->arch.model_name[i]) + 1); 537 } 538 539 dprintf("CPU %d: vendor '%s' model name '%s'\n", 540 currentCPU, cpu->arch.vendor_name, cpu->arch.model_name); 541 } else { 542 strcpy(cpu->arch.model_name, "unknown"); 543 } 544 545 // load feature bits 546 get_current_cpuid(&cpuid, 1); 547 cpu->arch.feature[FEATURE_COMMON] = cpuid.eax_1.features; // edx 548 cpu->arch.feature[FEATURE_EXT] = cpuid.eax_1.extended_features; // ecx 549 if (cpu->arch.vendor == VENDOR_AMD) { 550 get_current_cpuid(&cpuid, 0x80000001); 551 cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx 552 } 553 554 #if DUMP_FEATURE_STRING 555 dump_feature_string(currentCPU, cpu); 556 #endif 557 558 return 0; 559 } 560 561 562 bool 563 x86_check_feature(uint32 feature, enum x86_feature_type type) 564 { 565 cpu_ent *cpu = get_cpu_struct(); 566 567 #if 0 568 int i; 569 dprintf("x86_check_feature: feature 0x%x, type %d\n", feature, type); 570 for (i = 0; i < FEATURE_NUM; i++) { 571 dprintf("features %d: 0x%x\n", i, cpu->arch.feature[i]); 572 } 573 #endif 574 575 return (cpu->arch.feature[type] & feature) != 0; 576 } 577 578 579 void* 580 x86_get_double_fault_stack(int32 cpu, size_t* _size) 581 { 582 *_size = kDoubleFaultStackSize; 583 return sDoubleFaultStacks + kDoubleFaultStackSize * cpu; 584 } 585 586 587 /*! Returns the index of the current CPU. Can only be called from the double 588 fault handler. 589 */ 590 int32 591 x86_double_fault_get_cpu(void) 592 { 593 uint32 stack = x86_read_ebp(); 594 return (stack - (uint32)sDoubleFaultStacks) / kDoubleFaultStackSize; 595 } 596 597 598 // #pragma mark - 599 600 601 status_t 602 arch_cpu_preboot_init_percpu(kernel_args *args, int cpu) 603 { 604 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 605 gX86SwapFPUFunc = i386_fnsave_swap; 606 607 // On SMP system we want to synchronize the CPUs' TSCs, so system_time() 608 // will return consistent values. 609 if (smp_get_num_cpus() > 1) { 610 // let the first CPU prepare the rendezvous point 611 if (cpu == 0) 612 sTSCSyncRendezvous = smp_get_num_cpus() - 1; 613 614 // One CPU after the other will drop out of this loop and be caught by 615 // the loop below, until the last CPU (0) gets there. Save for +/- a few 616 // cycles the CPUs should pass the second loop at the same time. 617 while (sTSCSyncRendezvous != cpu) { 618 } 619 620 sTSCSyncRendezvous = cpu - 1; 621 622 while (sTSCSyncRendezvous != -1) { 623 } 624 625 // reset TSC to 0 626 x86_write_msr(IA32_MSR_TSC, 0); 627 } 628 629 return B_OK; 630 } 631 632 633 status_t 634 arch_cpu_init_percpu(kernel_args *args, int cpu) 635 { 636 detect_cpu(cpu); 637 638 // load the TSS for this cpu 639 // note the main cpu gets initialized in arch_cpu_init_post_vm() 640 if (cpu != 0) { 641 load_tss(cpu); 642 643 // set the IDT 644 struct { 645 uint16 limit; 646 void* address; 647 } _PACKED descriptor = { 648 256 * 8 - 1, // 256 descriptors, 8 bytes each (-1 for "limit") 649 x86_get_idt(cpu) 650 }; 651 652 asm volatile("lidt %0" : : "m"(descriptor)); 653 } 654 655 return 0; 656 } 657 658 status_t 659 arch_cpu_init(kernel_args *args) 660 { 661 // init the TSC -> system_time() conversion factors 662 663 uint32 conversionFactor = args->arch_args.system_time_cv_factor; 664 uint64 conversionFactorNsecs = (uint64)conversionFactor * 1000; 665 666 if (conversionFactorNsecs >> 32 != 0) { 667 // the TSC frequency is < 1 GHz, which forces us to shift the factor 668 __x86_setup_system_time(conversionFactor, conversionFactorNsecs >> 16, 669 true); 670 } else { 671 // the TSC frequency is >= 1 GHz 672 __x86_setup_system_time(conversionFactor, conversionFactorNsecs, false); 673 } 674 675 return B_OK; 676 } 677 678 679 status_t 680 arch_cpu_init_post_vm(kernel_args *args) 681 { 682 uint32 i; 683 684 // account for the segment descriptors 685 gGDT = (segment_descriptor *)args->arch_args.vir_gdt; 686 create_area("gdt", (void **)&gGDT, B_EXACT_ADDRESS, B_PAGE_SIZE, 687 B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 688 689 // currently taken out of the build, because it's not yet used (and assumes 690 // (a fixed number of used GDT entries) 691 //i386_selector_init(gGDT); // pass the new gdt 692 693 // allocate an area for the double fault stacks 694 create_area_etc(B_SYSTEM_TEAM, "double fault stacks", 695 (void**)&sDoubleFaultStacks, B_ANY_KERNEL_ADDRESS, 696 kDoubleFaultStackSize * smp_get_num_cpus(), B_FULL_LOCK, 697 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0, CREATE_AREA_DONT_WAIT); 698 699 vm_translation_map_arch_info* kernelArchTranslationMap 700 = static_cast<X86VMTranslationMap*>( 701 VMAddressSpace::Kernel()->TranslationMap())->ArchData(); 702 703 // setup task-state segments 704 for (i = 0; i < args->num_cpus; i++) { 705 // initialize the regular and double fault tss stored in the per-cpu 706 // structure 707 memset(&gCPU[i].arch.tss, 0, sizeof(struct tss)); 708 gCPU[i].arch.tss.ss0 = KERNEL_DATA_SEG; 709 gCPU[i].arch.tss.io_map_base = sizeof(struct tss); 710 711 // add TSS descriptor for this new TSS 712 set_tss_descriptor(&gGDT[TSS_BASE_SEGMENT + i], 713 (addr_t)&gCPU[i].arch.tss, sizeof(struct tss)); 714 715 // initialize the double fault tss 716 init_double_fault(i); 717 718 // init active translation map 719 gCPU[i].arch.active_translation_map = kernelArchTranslationMap; 720 kernelArchTranslationMap->AddReference(); 721 } 722 723 // set the current hardware task on cpu 0 724 load_tss(0); 725 726 // setup TLS descriptors (one for every CPU) 727 728 for (i = 0; i < args->num_cpus; i++) { 729 set_segment_descriptor(&gGDT[TLS_BASE_SEGMENT + i], 0, TLS_SIZE, 730 DT_DATA_WRITEABLE, DPL_USER); 731 } 732 733 // setup SSE2/3 support 734 init_sse(); 735 736 return B_OK; 737 } 738 739 740 status_t 741 arch_cpu_init_post_modules(kernel_args *args) 742 { 743 // initialize CPU module 744 745 void *cookie = open_module_list("cpu"); 746 747 while (true) { 748 char name[B_FILE_NAME_LENGTH]; 749 size_t nameLength = sizeof(name); 750 751 if (read_next_module_name(cookie, name, &nameLength) != B_OK 752 || get_module(name, (module_info **)&sCpuModule) == B_OK) 753 break; 754 } 755 756 close_module_list(cookie); 757 758 // initialize MTRRs if available 759 if (x86_count_mtrrs() > 0) { 760 sCpuRendezvous = sCpuRendezvous2 = 0; 761 call_all_cpus(&init_mtrrs, NULL); 762 } 763 764 // get optimized functions from the CPU module 765 if (sCpuModule != NULL && sCpuModule->get_optimized_functions != NULL) { 766 x86_optimized_functions functions; 767 memset(&functions, 0, sizeof(functions)); 768 769 sCpuModule->get_optimized_functions(&functions); 770 771 if (functions.memcpy != NULL) { 772 gOptimizedFunctions.memcpy = functions.memcpy; 773 gOptimizedFunctions.memcpy_end = functions.memcpy_end; 774 } 775 776 if (functions.memset != NULL) { 777 gOptimizedFunctions.memset = functions.memset; 778 gOptimizedFunctions.memset_end = functions.memset_end; 779 } 780 } 781 782 // put the optimized functions into the commpage 783 size_t memcpyLen = (addr_t)gOptimizedFunctions.memcpy_end 784 - (addr_t)gOptimizedFunctions.memcpy; 785 fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMCPY, 786 (const void*)gOptimizedFunctions.memcpy, memcpyLen); 787 size_t memsetLen = (addr_t)gOptimizedFunctions.memset_end 788 - (addr_t)gOptimizedFunctions.memset; 789 fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMSET, 790 (const void*)gOptimizedFunctions.memset, memsetLen); 791 792 // add the functions to the commpage image 793 image_id image = get_commpage_image(); 794 elf_add_memory_image_symbol(image, "commpage_memcpy", 795 ((addr_t*)USER_COMMPAGE_ADDR)[COMMPAGE_ENTRY_X86_MEMCPY], memcpyLen, 796 B_SYMBOL_TYPE_TEXT); 797 elf_add_memory_image_symbol(image, "commpage_memset", 798 ((addr_t*)USER_COMMPAGE_ADDR)[COMMPAGE_ENTRY_X86_MEMSET], memsetLen, 799 B_SYMBOL_TYPE_TEXT); 800 801 return B_OK; 802 } 803 804 805 void 806 i386_set_tss_and_kstack(addr_t kstack) 807 { 808 get_cpu_struct()->arch.tss.sp0 = kstack; 809 } 810 811 void 812 arch_cpu_global_TLB_invalidate(void) 813 { 814 uint32 flags = x86_read_cr4(); 815 816 if (flags & IA32_CR4_GLOBAL_PAGES) { 817 // disable and reenable the global pages to flush all TLBs regardless 818 // of the global page bit 819 x86_write_cr4(flags & ~IA32_CR4_GLOBAL_PAGES); 820 x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES); 821 } else { 822 cpu_status state = disable_interrupts(); 823 arch_cpu_user_TLB_invalidate(); 824 restore_interrupts(state); 825 } 826 } 827 828 829 void 830 arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) 831 { 832 int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE; 833 while (num_pages-- >= 0) { 834 invalidate_TLB(start); 835 start += B_PAGE_SIZE; 836 } 837 } 838 839 840 void 841 arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) 842 { 843 int i; 844 for (i = 0; i < num_pages; i++) { 845 invalidate_TLB(pages[i]); 846 } 847 } 848 849 850 ssize_t 851 arch_cpu_user_strlcpy(char *to, const char *from, size_t size, 852 addr_t *faultHandler) 853 { 854 int fromLength = 0; 855 addr_t oldFaultHandler = *faultHandler; 856 857 // this check is to trick the gcc4 compiler and have it keep the error label 858 if (to == NULL && size > 0) 859 goto error; 860 861 *faultHandler = (addr_t)&&error; 862 863 if (size > 0) { 864 to[--size] = '\0'; 865 // copy 866 for ( ; size; size--, fromLength++, to++, from++) { 867 if ((*to = *from) == '\0') 868 break; 869 } 870 } 871 // count any leftover from chars 872 while (*from++ != '\0') { 873 fromLength++; 874 } 875 876 *faultHandler = oldFaultHandler; 877 return fromLength; 878 879 error: 880 *faultHandler = oldFaultHandler; 881 return B_BAD_ADDRESS; 882 } 883 884 885 status_t 886 arch_cpu_user_memset(void *s, char c, size_t count, addr_t *faultHandler) 887 { 888 char *xs = (char *)s; 889 addr_t oldFaultHandler = *faultHandler; 890 891 // this check is to trick the gcc4 compiler and have it keep the error label 892 if (s == NULL) 893 goto error; 894 895 *faultHandler = (addr_t)&&error; 896 897 while (count--) 898 *xs++ = c; 899 900 *faultHandler = oldFaultHandler; 901 return 0; 902 903 error: 904 *faultHandler = oldFaultHandler; 905 return B_BAD_ADDRESS; 906 } 907 908 909 status_t 910 arch_cpu_shutdown(bool rebootSystem) 911 { 912 if (acpi_shutdown(rebootSystem) == B_OK) 913 return B_OK; 914 915 if (!rebootSystem) 916 return apm_shutdown(); 917 918 cpu_status state = disable_interrupts(); 919 920 // try to reset the system using the keyboard controller 921 out8(0xfe, 0x64); 922 923 // Give some time to the controller to do its job (0.5s) 924 snooze(500000); 925 926 // if that didn't help, try it this way 927 reboot(); 928 929 restore_interrupts(state); 930 return B_ERROR; 931 } 932 933 934 void 935 arch_cpu_idle(void) 936 { 937 asm("hlt"); 938 } 939 940 941 void 942 arch_cpu_sync_icache(void *address, size_t length) 943 { 944 // instruction cache is always consistent on x86 945 } 946 947 948 void 949 arch_cpu_memory_read_barrier(void) 950 { 951 asm volatile ("lock;" : : : "memory"); 952 asm volatile ("addl $0, 0(%%esp);" : : : "memory"); 953 } 954 955 956 void 957 arch_cpu_memory_write_barrier(void) 958 { 959 asm volatile ("lock;" : : : "memory"); 960 asm volatile ("addl $0, 0(%%esp);" : : : "memory"); 961 } 962 963