1 /* 2 * Copyright 2002-2009, Axel Dörfler, axeld@pinc-software.de. 3 * Distributed under the terms of the MIT License. 4 * 5 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 6 * Distributed under the terms of the NewOS License. 7 */ 8 9 10 #include <cpu.h> 11 12 #include <string.h> 13 #include <stdlib.h> 14 #include <stdio.h> 15 16 #include <ACPI.h> 17 18 #include <boot_device.h> 19 #include <commpage.h> 20 #include <debug.h> 21 #include <elf.h> 22 #include <smp.h> 23 #include <tls.h> 24 #include <vm/vm.h> 25 #include <vm/vm_types.h> 26 #include <vm/VMAddressSpace.h> 27 28 #include <arch_system_info.h> 29 #include <arch/x86/selector.h> 30 #include <boot/kernel_args.h> 31 32 #include "interrupts.h" 33 #include "x86_paging.h" 34 #include "X86VMTranslationMap.h" 35 36 37 #define DUMP_FEATURE_STRING 1 38 39 40 /* cpu vendor info */ 41 struct cpu_vendor_info { 42 const char *vendor; 43 const char *ident_string[2]; 44 }; 45 46 static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = { 47 { "Intel", { "GenuineIntel" } }, 48 { "AMD", { "AuthenticAMD" } }, 49 { "Cyrix", { "CyrixInstead" } }, 50 { "UMC", { "UMC UMC UMC" } }, 51 { "NexGen", { "NexGenDriven" } }, 52 { "Centaur", { "CentaurHauls" } }, 53 { "Rise", { "RiseRiseRise" } }, 54 { "Transmeta", { "GenuineTMx86", "TransmetaCPU" } }, 55 { "NSC", { "Geode by NSC" } }, 56 }; 57 58 #define CR0_CACHE_DISABLE (1UL << 30) 59 #define CR0_NOT_WRITE_THROUGH (1UL << 29) 60 #define CR0_FPU_EMULATION (1UL << 2) 61 #define CR0_MONITOR_FPU (1UL << 1) 62 63 #define CR4_OS_FXSR (1UL << 9) 64 #define CR4_OS_XMM_EXCEPTION (1UL << 10) 65 66 struct set_mtrr_parameter { 67 int32 index; 68 uint64 base; 69 uint64 length; 70 uint8 type; 71 }; 72 73 struct set_mtrrs_parameter { 74 const x86_mtrr_info* infos; 75 uint32 count; 76 }; 77 78 79 extern "C" void reboot(void); 80 // from arch_x86.S 81 82 void (*gX86SwapFPUFunc)(void *oldState, const void *newState); 83 bool gHasSSE = false; 84 85 static uint32 sCpuRendezvous; 86 static uint32 sCpuRendezvous2; 87 static vint32 sTSCSyncRendezvous; 88 89 segment_descriptor *gGDT = NULL; 90 91 /* Some specials for the double fault handler */ 92 static uint8* sDoubleFaultStacks; 93 static const size_t kDoubleFaultStackSize = 4096; // size per CPU 94 95 static x86_cpu_module_info *sCpuModule; 96 97 98 extern "C" void memcpy_generic(void* dest, const void* source, size_t count); 99 extern int memcpy_generic_end; 100 extern "C" void memset_generic(void* dest, int value, size_t count); 101 extern int memset_generic_end; 102 103 x86_optimized_functions gOptimizedFunctions = { 104 memcpy_generic, 105 &memcpy_generic_end, 106 memset_generic, 107 &memset_generic_end 108 }; 109 110 111 static status_t 112 acpi_shutdown(bool rebootSystem) 113 { 114 if (debug_debugger_running() || !are_interrupts_enabled()) 115 return B_ERROR; 116 117 acpi_module_info* acpi; 118 if (get_module(B_ACPI_MODULE_NAME, (module_info**)&acpi) != B_OK) 119 return B_NOT_SUPPORTED; 120 121 status_t status; 122 if (rebootSystem) { 123 status = acpi->reboot(); 124 } else { 125 status = acpi->prepare_sleep_state(ACPI_POWER_STATE_OFF, NULL, 0); 126 if (status == B_OK) { 127 //cpu_status state = disable_interrupts(); 128 status = acpi->enter_sleep_state(ACPI_POWER_STATE_OFF); 129 //restore_interrupts(state); 130 } 131 } 132 133 put_module(B_ACPI_MODULE_NAME); 134 return status; 135 } 136 137 138 /*! Disable CPU caches, and invalidate them. */ 139 static void 140 disable_caches() 141 { 142 x86_write_cr0((x86_read_cr0() | CR0_CACHE_DISABLE) 143 & ~CR0_NOT_WRITE_THROUGH); 144 wbinvd(); 145 arch_cpu_global_TLB_invalidate(); 146 } 147 148 149 /*! Invalidate CPU caches, and enable them. */ 150 static void 151 enable_caches() 152 { 153 wbinvd(); 154 arch_cpu_global_TLB_invalidate(); 155 x86_write_cr0(x86_read_cr0() 156 & ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH)); 157 } 158 159 160 static void 161 set_mtrr(void *_parameter, int cpu) 162 { 163 struct set_mtrr_parameter *parameter 164 = (struct set_mtrr_parameter *)_parameter; 165 166 // wait until all CPUs have arrived here 167 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 168 169 disable_caches(); 170 171 sCpuModule->set_mtrr(parameter->index, parameter->base, parameter->length, 172 parameter->type); 173 174 enable_caches(); 175 176 // wait until all CPUs have arrived here 177 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 178 } 179 180 181 static void 182 set_mtrrs(void* _parameter, int cpu) 183 { 184 set_mtrrs_parameter* parameter = (set_mtrrs_parameter*)_parameter; 185 186 // wait until all CPUs have arrived here 187 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 188 189 disable_caches(); 190 191 sCpuModule->set_mtrrs(parameter->infos, parameter->count); 192 193 enable_caches(); 194 195 // wait until all CPUs have arrived here 196 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 197 } 198 199 200 static void 201 init_mtrrs(void *_unused, int cpu) 202 { 203 // wait until all CPUs have arrived here 204 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 205 206 disable_caches(); 207 208 sCpuModule->init_mtrrs(); 209 210 enable_caches(); 211 212 // wait until all CPUs have arrived here 213 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 214 } 215 216 217 uint32 218 x86_count_mtrrs(void) 219 { 220 if (sCpuModule == NULL) 221 return 0; 222 223 return sCpuModule->count_mtrrs(); 224 } 225 226 227 void 228 x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type) 229 { 230 struct set_mtrr_parameter parameter; 231 parameter.index = index; 232 parameter.base = base; 233 parameter.length = length; 234 parameter.type = type; 235 236 sCpuRendezvous = sCpuRendezvous2 = 0; 237 call_all_cpus(&set_mtrr, ¶meter); 238 } 239 240 241 status_t 242 x86_get_mtrr(uint32 index, uint64 *_base, uint64 *_length, uint8 *_type) 243 { 244 // the MTRRs are identical on all CPUs, so it doesn't matter 245 // on which CPU this runs 246 return sCpuModule->get_mtrr(index, _base, _length, _type); 247 } 248 249 250 void 251 x86_set_mtrrs(const x86_mtrr_info* infos, uint32 count) 252 { 253 struct set_mtrrs_parameter parameter; 254 parameter.infos = infos; 255 parameter.count = count; 256 257 sCpuRendezvous = sCpuRendezvous2 = 0; 258 call_all_cpus(&set_mtrrs, ¶meter); 259 } 260 261 262 extern "C" void 263 init_sse(void) 264 { 265 if (!x86_check_feature(IA32_FEATURE_SSE, FEATURE_COMMON) 266 || !x86_check_feature(IA32_FEATURE_FXSR, FEATURE_COMMON)) { 267 // we don't have proper SSE support 268 return; 269 } 270 271 // enable OS support for SSE 272 x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION); 273 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 274 275 gX86SwapFPUFunc = i386_fxsave_swap; 276 gHasSSE = true; 277 } 278 279 280 static void 281 load_tss(int cpu) 282 { 283 short seg = ((TSS_BASE_SEGMENT + cpu) << 3) | DPL_KERNEL; 284 asm("movw %0, %%ax;" 285 "ltr %%ax;" : : "r" (seg) : "eax"); 286 } 287 288 289 static void 290 init_double_fault(int cpuNum) 291 { 292 // set up the double fault TSS 293 struct tss *tss = &gCPU[cpuNum].arch.double_fault_tss; 294 295 memset(tss, 0, sizeof(struct tss)); 296 size_t stackSize; 297 tss->sp0 = (uint32)x86_get_double_fault_stack(cpuNum, &stackSize); 298 tss->sp0 += stackSize; 299 tss->ss0 = KERNEL_DATA_SEG; 300 read_cr3(tss->cr3); 301 // copy the current cr3 to the double fault cr3 302 tss->eip = (uint32)&double_fault; 303 tss->es = KERNEL_DATA_SEG; 304 tss->cs = KERNEL_CODE_SEG; 305 tss->ss = KERNEL_DATA_SEG; 306 tss->esp = tss->sp0; 307 tss->ds = KERNEL_DATA_SEG; 308 tss->fs = KERNEL_DATA_SEG; 309 tss->gs = KERNEL_DATA_SEG; 310 tss->ldt_seg_selector = 0; 311 tss->io_map_base = sizeof(struct tss); 312 313 // add TSS descriptor for this new TSS 314 uint16 tssSegmentDescriptorIndex = DOUBLE_FAULT_TSS_BASE_SEGMENT + cpuNum; 315 set_tss_descriptor(&gGDT[tssSegmentDescriptorIndex], 316 (addr_t)tss, sizeof(struct tss)); 317 318 x86_set_task_gate(cpuNum, 8, tssSegmentDescriptorIndex << 3); 319 } 320 321 322 #if DUMP_FEATURE_STRING 323 static void 324 dump_feature_string(int currentCPU, cpu_ent *cpu) 325 { 326 char features[256]; 327 features[0] = 0; 328 329 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FPU) 330 strlcat(features, "fpu ", sizeof(features)); 331 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_VME) 332 strlcat(features, "vme ", sizeof(features)); 333 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DE) 334 strlcat(features, "de ", sizeof(features)); 335 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE) 336 strlcat(features, "pse ", sizeof(features)); 337 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TSC) 338 strlcat(features, "tsc ", sizeof(features)); 339 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MSR) 340 strlcat(features, "msr ", sizeof(features)); 341 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAE) 342 strlcat(features, "pae ", sizeof(features)); 343 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCE) 344 strlcat(features, "mce ", sizeof(features)); 345 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CX8) 346 strlcat(features, "cx8 ", sizeof(features)); 347 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_APIC) 348 strlcat(features, "apic ", sizeof(features)); 349 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SEP) 350 strlcat(features, "sep ", sizeof(features)); 351 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MTRR) 352 strlcat(features, "mtrr ", sizeof(features)); 353 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PGE) 354 strlcat(features, "pge ", sizeof(features)); 355 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCA) 356 strlcat(features, "mca ", sizeof(features)); 357 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CMOV) 358 strlcat(features, "cmov ", sizeof(features)); 359 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAT) 360 strlcat(features, "pat ", sizeof(features)); 361 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE36) 362 strlcat(features, "pse36 ", sizeof(features)); 363 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSN) 364 strlcat(features, "psn ", sizeof(features)); 365 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CLFSH) 366 strlcat(features, "clfsh ", sizeof(features)); 367 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DS) 368 strlcat(features, "ds ", sizeof(features)); 369 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_ACPI) 370 strlcat(features, "acpi ", sizeof(features)); 371 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MMX) 372 strlcat(features, "mmx ", sizeof(features)); 373 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FXSR) 374 strlcat(features, "fxsr ", sizeof(features)); 375 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE) 376 strlcat(features, "sse ", sizeof(features)); 377 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE2) 378 strlcat(features, "sse2 ", sizeof(features)); 379 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SS) 380 strlcat(features, "ss ", sizeof(features)); 381 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_HTT) 382 strlcat(features, "htt ", sizeof(features)); 383 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TM) 384 strlcat(features, "tm ", sizeof(features)); 385 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PBE) 386 strlcat(features, "pbe ", sizeof(features)); 387 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE3) 388 strlcat(features, "sse3 ", sizeof(features)); 389 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MONITOR) 390 strlcat(features, "monitor ", sizeof(features)); 391 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DSCPL) 392 strlcat(features, "dscpl ", sizeof(features)); 393 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_EST) 394 strlcat(features, "est ", sizeof(features)); 395 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TM2) 396 strlcat(features, "tm2 ", sizeof(features)); 397 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CNXTID) 398 strlcat(features, "cnxtid ", sizeof(features)); 399 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_SYSCALL) 400 strlcat(features, "syscall ", sizeof(features)); 401 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_NX) 402 strlcat(features, "nx ", sizeof(features)); 403 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_MMXEXT) 404 strlcat(features, "mmxext ", sizeof(features)); 405 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_FFXSR) 406 strlcat(features, "ffxsr ", sizeof(features)); 407 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_LONG) 408 strlcat(features, "long ", sizeof(features)); 409 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOWEXT) 410 strlcat(features, "3dnowext ", sizeof(features)); 411 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOW) 412 strlcat(features, "3dnow ", sizeof(features)); 413 414 dprintf("CPU %d: features: %s\n", currentCPU, features); 415 } 416 #endif // DUMP_FEATURE_STRING 417 418 419 static int 420 detect_cpu(int currentCPU) 421 { 422 cpu_ent *cpu = get_cpu_struct(); 423 char vendorString[17]; 424 cpuid_info cpuid; 425 426 // clear out the cpu info data 427 cpu->arch.vendor = VENDOR_UNKNOWN; 428 cpu->arch.vendor_name = "UNKNOWN VENDOR"; 429 cpu->arch.feature[FEATURE_COMMON] = 0; 430 cpu->arch.feature[FEATURE_EXT] = 0; 431 cpu->arch.feature[FEATURE_EXT_AMD] = 0; 432 cpu->arch.model_name[0] = 0; 433 434 // print some fun data 435 get_current_cpuid(&cpuid, 0); 436 437 // build the vendor string 438 memset(vendorString, 0, sizeof(vendorString)); 439 memcpy(vendorString, cpuid.eax_0.vendor_id, sizeof(cpuid.eax_0.vendor_id)); 440 441 // get the family, model, stepping 442 get_current_cpuid(&cpuid, 1); 443 cpu->arch.type = cpuid.eax_1.type; 444 cpu->arch.family = cpuid.eax_1.family; 445 cpu->arch.extended_family = cpuid.eax_1.extended_family; 446 cpu->arch.model = cpuid.eax_1.model; 447 cpu->arch.extended_model = cpuid.eax_1.extended_model; 448 cpu->arch.stepping = cpuid.eax_1.stepping; 449 dprintf("CPU %d: type %d family %d extended_family %d model %d " 450 "extended_model %d stepping %d, string '%s'\n", 451 currentCPU, cpu->arch.type, cpu->arch.family, 452 cpu->arch.extended_family, cpu->arch.model, 453 cpu->arch.extended_model, cpu->arch.stepping, vendorString); 454 455 // figure out what vendor we have here 456 457 for (int32 i = 0; i < VENDOR_NUM; i++) { 458 if (vendor_info[i].ident_string[0] 459 && !strcmp(vendorString, vendor_info[i].ident_string[0])) { 460 cpu->arch.vendor = (x86_vendors)i; 461 cpu->arch.vendor_name = vendor_info[i].vendor; 462 break; 463 } 464 if (vendor_info[i].ident_string[1] 465 && !strcmp(vendorString, vendor_info[i].ident_string[1])) { 466 cpu->arch.vendor = (x86_vendors)i; 467 cpu->arch.vendor_name = vendor_info[i].vendor; 468 break; 469 } 470 } 471 472 // see if we can get the model name 473 get_current_cpuid(&cpuid, 0x80000000); 474 if (cpuid.eax_0.max_eax >= 0x80000004) { 475 // build the model string (need to swap ecx/edx data before copying) 476 unsigned int temp; 477 memset(cpu->arch.model_name, 0, sizeof(cpu->arch.model_name)); 478 479 get_current_cpuid(&cpuid, 0x80000002); 480 temp = cpuid.regs.edx; 481 cpuid.regs.edx = cpuid.regs.ecx; 482 cpuid.regs.ecx = temp; 483 memcpy(cpu->arch.model_name, cpuid.as_chars, sizeof(cpuid.as_chars)); 484 485 get_current_cpuid(&cpuid, 0x80000003); 486 temp = cpuid.regs.edx; 487 cpuid.regs.edx = cpuid.regs.ecx; 488 cpuid.regs.ecx = temp; 489 memcpy(cpu->arch.model_name + 16, cpuid.as_chars, 490 sizeof(cpuid.as_chars)); 491 492 get_current_cpuid(&cpuid, 0x80000004); 493 temp = cpuid.regs.edx; 494 cpuid.regs.edx = cpuid.regs.ecx; 495 cpuid.regs.ecx = temp; 496 memcpy(cpu->arch.model_name + 32, cpuid.as_chars, 497 sizeof(cpuid.as_chars)); 498 499 // some cpus return a right-justified string 500 int32 i = 0; 501 while (cpu->arch.model_name[i] == ' ') 502 i++; 503 if (i > 0) { 504 memmove(cpu->arch.model_name, &cpu->arch.model_name[i], 505 strlen(&cpu->arch.model_name[i]) + 1); 506 } 507 508 dprintf("CPU %d: vendor '%s' model name '%s'\n", 509 currentCPU, cpu->arch.vendor_name, cpu->arch.model_name); 510 } else { 511 strcpy(cpu->arch.model_name, "unknown"); 512 } 513 514 // load feature bits 515 get_current_cpuid(&cpuid, 1); 516 cpu->arch.feature[FEATURE_COMMON] = cpuid.eax_1.features; // edx 517 cpu->arch.feature[FEATURE_EXT] = cpuid.eax_1.extended_features; // ecx 518 if (cpu->arch.vendor == VENDOR_AMD) { 519 get_current_cpuid(&cpuid, 0x80000001); 520 cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx 521 } 522 523 #if DUMP_FEATURE_STRING 524 dump_feature_string(currentCPU, cpu); 525 #endif 526 527 return 0; 528 } 529 530 531 bool 532 x86_check_feature(uint32 feature, enum x86_feature_type type) 533 { 534 cpu_ent *cpu = get_cpu_struct(); 535 536 #if 0 537 int i; 538 dprintf("x86_check_feature: feature 0x%x, type %d\n", feature, type); 539 for (i = 0; i < FEATURE_NUM; i++) { 540 dprintf("features %d: 0x%x\n", i, cpu->arch.feature[i]); 541 } 542 #endif 543 544 return (cpu->arch.feature[type] & feature) != 0; 545 } 546 547 548 void* 549 x86_get_double_fault_stack(int32 cpu, size_t* _size) 550 { 551 *_size = kDoubleFaultStackSize; 552 return sDoubleFaultStacks + kDoubleFaultStackSize * cpu; 553 } 554 555 556 /*! Returns the index of the current CPU. Can only be called from the double 557 fault handler. 558 */ 559 int32 560 x86_double_fault_get_cpu(void) 561 { 562 uint32 stack = x86_read_ebp(); 563 return (stack - (uint32)sDoubleFaultStacks) / kDoubleFaultStackSize; 564 } 565 566 567 // #pragma mark - 568 569 570 status_t 571 arch_cpu_preboot_init_percpu(kernel_args *args, int cpu) 572 { 573 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 574 gX86SwapFPUFunc = i386_fnsave_swap; 575 576 // On SMP system we want to synchronize the CPUs' TSCs, so system_time() 577 // will return consistent values. 578 if (smp_get_num_cpus() > 1) { 579 // let the first CPU prepare the rendezvous point 580 if (cpu == 0) 581 sTSCSyncRendezvous = smp_get_num_cpus() - 1; 582 583 // One CPU after the other will drop out of this loop and be caught by 584 // the loop below, until the last CPU (0) gets there. Save for +/- a few 585 // cycles the CPUs should pass the second loop at the same time. 586 while (sTSCSyncRendezvous != cpu) { 587 } 588 589 sTSCSyncRendezvous = cpu - 1; 590 591 while (sTSCSyncRendezvous != -1) { 592 } 593 594 // reset TSC to 0 595 x86_write_msr(IA32_MSR_TSC, 0); 596 } 597 598 return B_OK; 599 } 600 601 602 status_t 603 arch_cpu_init_percpu(kernel_args *args, int cpu) 604 { 605 detect_cpu(cpu); 606 607 // load the TSS for this cpu 608 // note the main cpu gets initialized in arch_cpu_init_post_vm() 609 if (cpu != 0) { 610 load_tss(cpu); 611 612 // set the IDT 613 struct { 614 uint16 limit; 615 void* address; 616 } _PACKED descriptor = { 617 256 * 8 - 1, // 256 descriptors, 8 bytes each (-1 for "limit") 618 x86_get_idt(cpu) 619 }; 620 621 asm volatile("lidt %0" : : "m"(descriptor)); 622 } 623 624 return 0; 625 } 626 627 status_t 628 arch_cpu_init(kernel_args *args) 629 { 630 // init the TSC -> system_time() conversion factors 631 632 uint32 conversionFactor = args->arch_args.system_time_cv_factor; 633 uint64 conversionFactorNsecs = (uint64)conversionFactor * 1000; 634 635 if (conversionFactorNsecs >> 32 != 0) { 636 // the TSC frequency is < 1 GHz, which forces us to shift the factor 637 __x86_setup_system_time(conversionFactor, conversionFactorNsecs >> 16, 638 true); 639 } else { 640 // the TSC frequency is >= 1 GHz 641 __x86_setup_system_time(conversionFactor, conversionFactorNsecs, false); 642 } 643 644 return B_OK; 645 } 646 647 648 status_t 649 arch_cpu_init_post_vm(kernel_args *args) 650 { 651 uint32 i; 652 653 // account for the segment descriptors 654 gGDT = (segment_descriptor *)args->arch_args.vir_gdt; 655 create_area("gdt", (void **)&gGDT, B_EXACT_ADDRESS, B_PAGE_SIZE, 656 B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 657 658 // currently taken out of the build, because it's not yet used (and assumes 659 // (a fixed number of used GDT entries) 660 //i386_selector_init(gGDT); // pass the new gdt 661 662 // allocate an area for the double fault stacks 663 create_area("double fault stacks", (void**)&sDoubleFaultStacks, 664 B_ANY_KERNEL_ADDRESS, kDoubleFaultStackSize * smp_get_num_cpus(), 665 B_FULL_LOCK, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 666 667 vm_translation_map_arch_info* kernelArchTranslationMap 668 = static_cast<X86VMTranslationMap*>( 669 VMAddressSpace::Kernel()->TranslationMap())->ArchData(); 670 671 // setup task-state segments 672 for (i = 0; i < args->num_cpus; i++) { 673 // initialize the regular and double fault tss stored in the per-cpu 674 // structure 675 memset(&gCPU[i].arch.tss, 0, sizeof(struct tss)); 676 gCPU[i].arch.tss.ss0 = KERNEL_DATA_SEG; 677 gCPU[i].arch.tss.io_map_base = sizeof(struct tss); 678 679 // add TSS descriptor for this new TSS 680 set_tss_descriptor(&gGDT[TSS_BASE_SEGMENT + i], 681 (addr_t)&gCPU[i].arch.tss, sizeof(struct tss)); 682 683 // initialize the double fault tss 684 init_double_fault(i); 685 686 // init active translation map 687 gCPU[i].arch.active_translation_map = kernelArchTranslationMap; 688 kernelArchTranslationMap->AddReference(); 689 } 690 691 // set the current hardware task on cpu 0 692 load_tss(0); 693 694 // setup TLS descriptors (one for every CPU) 695 696 for (i = 0; i < args->num_cpus; i++) { 697 set_segment_descriptor(&gGDT[TLS_BASE_SEGMENT + i], 0, TLS_SIZE, 698 DT_DATA_WRITEABLE, DPL_USER); 699 } 700 701 // setup SSE2/3 support 702 init_sse(); 703 704 return B_OK; 705 } 706 707 708 status_t 709 arch_cpu_init_post_modules(kernel_args *args) 710 { 711 // initialize CPU module 712 713 void *cookie = open_module_list("cpu"); 714 715 while (true) { 716 char name[B_FILE_NAME_LENGTH]; 717 size_t nameLength = sizeof(name); 718 719 if (read_next_module_name(cookie, name, &nameLength) != B_OK 720 || get_module(name, (module_info **)&sCpuModule) == B_OK) 721 break; 722 } 723 724 close_module_list(cookie); 725 726 // initialize MTRRs if available 727 if (x86_count_mtrrs() > 0) { 728 sCpuRendezvous = sCpuRendezvous2 = 0; 729 call_all_cpus(&init_mtrrs, NULL); 730 } 731 732 // get optimized functions from the CPU module 733 if (sCpuModule != NULL && sCpuModule->get_optimized_functions != NULL) { 734 x86_optimized_functions functions; 735 memset(&functions, 0, sizeof(functions)); 736 737 sCpuModule->get_optimized_functions(&functions); 738 739 if (functions.memcpy != NULL) { 740 gOptimizedFunctions.memcpy = functions.memcpy; 741 gOptimizedFunctions.memcpy_end = functions.memcpy_end; 742 } 743 744 if (functions.memset != NULL) { 745 gOptimizedFunctions.memset = functions.memset; 746 gOptimizedFunctions.memset_end = functions.memset_end; 747 } 748 } 749 750 // put the optimized functions into the commpage 751 size_t memcpyLen = (addr_t)gOptimizedFunctions.memcpy_end 752 - (addr_t)gOptimizedFunctions.memcpy; 753 fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMCPY, 754 (const void*)gOptimizedFunctions.memcpy, memcpyLen); 755 size_t memsetLen = (addr_t)gOptimizedFunctions.memset_end 756 - (addr_t)gOptimizedFunctions.memset; 757 fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMSET, 758 (const void*)gOptimizedFunctions.memset, memsetLen); 759 760 // add the functions to the commpage image 761 image_id image = get_commpage_image(); 762 elf_add_memory_image_symbol(image, "commpage_memcpy", 763 ((addr_t*)USER_COMMPAGE_ADDR)[COMMPAGE_ENTRY_X86_MEMCPY], memcpyLen, 764 B_SYMBOL_TYPE_TEXT); 765 elf_add_memory_image_symbol(image, "commpage_memset", 766 ((addr_t*)USER_COMMPAGE_ADDR)[COMMPAGE_ENTRY_X86_MEMSET], memsetLen, 767 B_SYMBOL_TYPE_TEXT); 768 769 return B_OK; 770 } 771 772 773 void 774 i386_set_tss_and_kstack(addr_t kstack) 775 { 776 get_cpu_struct()->arch.tss.sp0 = kstack; 777 } 778 779 void 780 arch_cpu_global_TLB_invalidate(void) 781 { 782 uint32 flags = x86_read_cr4(); 783 784 if (flags & IA32_CR4_GLOBAL_PAGES) { 785 // disable and reenable the global pages to flush all TLBs regardless 786 // of the global page bit 787 x86_write_cr4(flags & ~IA32_CR4_GLOBAL_PAGES); 788 x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES); 789 } else { 790 cpu_status state = disable_interrupts(); 791 arch_cpu_user_TLB_invalidate(); 792 restore_interrupts(state); 793 } 794 } 795 796 797 void 798 arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) 799 { 800 int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE; 801 while (num_pages-- >= 0) { 802 invalidate_TLB(start); 803 start += B_PAGE_SIZE; 804 } 805 } 806 807 808 void 809 arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) 810 { 811 int i; 812 for (i = 0; i < num_pages; i++) { 813 invalidate_TLB(pages[i]); 814 } 815 } 816 817 818 ssize_t 819 arch_cpu_user_strlcpy(char *to, const char *from, size_t size, 820 addr_t *faultHandler) 821 { 822 int fromLength = 0; 823 addr_t oldFaultHandler = *faultHandler; 824 825 // this check is to trick the gcc4 compiler and have it keep the error label 826 if (to == NULL && size > 0) 827 goto error; 828 829 *faultHandler = (addr_t)&&error; 830 831 if (size > 0) { 832 to[--size] = '\0'; 833 // copy 834 for ( ; size; size--, fromLength++, to++, from++) { 835 if ((*to = *from) == '\0') 836 break; 837 } 838 } 839 // count any leftover from chars 840 while (*from++ != '\0') { 841 fromLength++; 842 } 843 844 *faultHandler = oldFaultHandler; 845 return fromLength; 846 847 error: 848 *faultHandler = oldFaultHandler; 849 return B_BAD_ADDRESS; 850 } 851 852 853 status_t 854 arch_cpu_user_memset(void *s, char c, size_t count, addr_t *faultHandler) 855 { 856 char *xs = (char *)s; 857 addr_t oldFaultHandler = *faultHandler; 858 859 // this check is to trick the gcc4 compiler and have it keep the error label 860 if (s == NULL) 861 goto error; 862 863 *faultHandler = (addr_t)&&error; 864 865 while (count--) 866 *xs++ = c; 867 868 *faultHandler = oldFaultHandler; 869 return 0; 870 871 error: 872 *faultHandler = oldFaultHandler; 873 return B_BAD_ADDRESS; 874 } 875 876 877 status_t 878 arch_cpu_shutdown(bool rebootSystem) 879 { 880 if (acpi_shutdown(rebootSystem) == B_OK) 881 return B_OK; 882 883 if (!rebootSystem) 884 return apm_shutdown(); 885 886 cpu_status state = disable_interrupts(); 887 888 // try to reset the system using the keyboard controller 889 out8(0xfe, 0x64); 890 891 // Give some time to the controller to do its job (0.5s) 892 snooze(500000); 893 894 // if that didn't help, try it this way 895 reboot(); 896 897 restore_interrupts(state); 898 return B_ERROR; 899 } 900 901 902 void 903 arch_cpu_idle(void) 904 { 905 asm("hlt"); 906 } 907 908 909 void 910 arch_cpu_sync_icache(void *address, size_t length) 911 { 912 // instruction cache is always consistent on x86 913 } 914 915 916 void 917 arch_cpu_memory_read_barrier(void) 918 { 919 asm volatile ("lock;" : : : "memory"); 920 asm volatile ("addl $0, 0(%%esp);" : : : "memory"); 921 } 922 923 924 void 925 arch_cpu_memory_write_barrier(void) 926 { 927 asm volatile ("lock;" : : : "memory"); 928 asm volatile ("addl $0, 0(%%esp);" : : : "memory"); 929 } 930 931