1 /* 2 * Copyright 2002-2009, Axel Dörfler, axeld@pinc-software.de. 3 * Distributed under the terms of the MIT License. 4 * 5 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 6 * Distributed under the terms of the NewOS License. 7 */ 8 9 10 #include <cpu.h> 11 12 #include <string.h> 13 #include <stdlib.h> 14 #include <stdio.h> 15 16 #include <ACPI.h> 17 18 #include <boot_device.h> 19 #include <commpage.h> 20 #include <debug.h> 21 #include <elf.h> 22 #include <smp.h> 23 #include <tls.h> 24 #include <vm/vm.h> 25 #include <vm/vm_types.h> 26 #include <vm/VMAddressSpace.h> 27 28 #include <arch_system_info.h> 29 #include <arch/x86/selector.h> 30 #include <boot/kernel_args.h> 31 32 #include "interrupts.h" 33 #include "x86_paging.h" 34 #include "X86VMTranslationMap.h" 35 36 37 #define DUMP_FEATURE_STRING 1 38 39 40 /* cpu vendor info */ 41 struct cpu_vendor_info { 42 const char *vendor; 43 const char *ident_string[2]; 44 }; 45 46 static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = { 47 { "Intel", { "GenuineIntel" } }, 48 { "AMD", { "AuthenticAMD" } }, 49 { "Cyrix", { "CyrixInstead" } }, 50 { "UMC", { "UMC UMC UMC" } }, 51 { "NexGen", { "NexGenDriven" } }, 52 { "Centaur", { "CentaurHauls" } }, 53 { "Rise", { "RiseRiseRise" } }, 54 { "Transmeta", { "GenuineTMx86", "TransmetaCPU" } }, 55 { "NSC", { "Geode by NSC" } }, 56 }; 57 58 #define CR0_CACHE_DISABLE (1UL << 30) 59 #define CR0_NOT_WRITE_THROUGH (1UL << 29) 60 #define CR0_FPU_EMULATION (1UL << 2) 61 #define CR0_MONITOR_FPU (1UL << 1) 62 63 #define CR4_OS_FXSR (1UL << 9) 64 #define CR4_OS_XMM_EXCEPTION (1UL << 10) 65 66 struct set_mtrr_parameter { 67 int32 index; 68 uint64 base; 69 uint64 length; 70 uint8 type; 71 }; 72 73 struct set_mtrrs_parameter { 74 const x86_mtrr_info* infos; 75 uint32 count; 76 uint8 defaultType; 77 }; 78 79 80 extern "C" void reboot(void); 81 // from arch_x86.S 82 83 void (*gX86SwapFPUFunc)(void *oldState, const void *newState); 84 bool gHasSSE = false; 85 86 static uint32 sCpuRendezvous; 87 static uint32 sCpuRendezvous2; 88 static vint32 sTSCSyncRendezvous; 89 90 segment_descriptor *gGDT = NULL; 91 92 /* Some specials for the double fault handler */ 93 static uint8* sDoubleFaultStacks; 94 static const size_t kDoubleFaultStackSize = 4096; // size per CPU 95 96 static x86_cpu_module_info *sCpuModule; 97 98 99 extern "C" void memcpy_generic(void* dest, const void* source, size_t count); 100 extern int memcpy_generic_end; 101 extern "C" void memset_generic(void* dest, int value, size_t count); 102 extern int memset_generic_end; 103 104 x86_optimized_functions gOptimizedFunctions = { 105 memcpy_generic, 106 &memcpy_generic_end, 107 memset_generic, 108 &memset_generic_end 109 }; 110 111 112 static status_t 113 acpi_shutdown(bool rebootSystem) 114 { 115 if (debug_debugger_running() || !are_interrupts_enabled()) 116 return B_ERROR; 117 118 acpi_module_info* acpi; 119 if (get_module(B_ACPI_MODULE_NAME, (module_info**)&acpi) != B_OK) 120 return B_NOT_SUPPORTED; 121 122 status_t status; 123 if (rebootSystem) { 124 status = acpi->reboot(); 125 } else { 126 status = acpi->prepare_sleep_state(ACPI_POWER_STATE_OFF, NULL, 0); 127 if (status == B_OK) { 128 //cpu_status state = disable_interrupts(); 129 status = acpi->enter_sleep_state(ACPI_POWER_STATE_OFF); 130 //restore_interrupts(state); 131 } 132 } 133 134 put_module(B_ACPI_MODULE_NAME); 135 return status; 136 } 137 138 139 /*! Disable CPU caches, and invalidate them. */ 140 static void 141 disable_caches() 142 { 143 x86_write_cr0((x86_read_cr0() | CR0_CACHE_DISABLE) 144 & ~CR0_NOT_WRITE_THROUGH); 145 wbinvd(); 146 arch_cpu_global_TLB_invalidate(); 147 } 148 149 150 /*! Invalidate CPU caches, and enable them. */ 151 static void 152 enable_caches() 153 { 154 wbinvd(); 155 arch_cpu_global_TLB_invalidate(); 156 x86_write_cr0(x86_read_cr0() 157 & ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH)); 158 } 159 160 161 static void 162 set_mtrr(void *_parameter, int cpu) 163 { 164 struct set_mtrr_parameter *parameter 165 = (struct set_mtrr_parameter *)_parameter; 166 167 // wait until all CPUs have arrived here 168 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 169 170 disable_caches(); 171 172 sCpuModule->set_mtrr(parameter->index, parameter->base, parameter->length, 173 parameter->type); 174 175 enable_caches(); 176 177 // wait until all CPUs have arrived here 178 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 179 } 180 181 182 static void 183 set_mtrrs(void* _parameter, int cpu) 184 { 185 set_mtrrs_parameter* parameter = (set_mtrrs_parameter*)_parameter; 186 187 // wait until all CPUs have arrived here 188 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 189 190 disable_caches(); 191 192 sCpuModule->set_mtrrs(parameter->defaultType, parameter->infos, 193 parameter->count); 194 195 enable_caches(); 196 197 // wait until all CPUs have arrived here 198 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 199 } 200 201 202 static void 203 init_mtrrs(void *_unused, int cpu) 204 { 205 // wait until all CPUs have arrived here 206 smp_cpu_rendezvous(&sCpuRendezvous, cpu); 207 208 disable_caches(); 209 210 sCpuModule->init_mtrrs(); 211 212 enable_caches(); 213 214 // wait until all CPUs have arrived here 215 smp_cpu_rendezvous(&sCpuRendezvous2, cpu); 216 } 217 218 219 uint32 220 x86_count_mtrrs(void) 221 { 222 if (sCpuModule == NULL) 223 return 0; 224 225 return sCpuModule->count_mtrrs(); 226 } 227 228 229 void 230 x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type) 231 { 232 struct set_mtrr_parameter parameter; 233 parameter.index = index; 234 parameter.base = base; 235 parameter.length = length; 236 parameter.type = type; 237 238 sCpuRendezvous = sCpuRendezvous2 = 0; 239 call_all_cpus(&set_mtrr, ¶meter); 240 } 241 242 243 status_t 244 x86_get_mtrr(uint32 index, uint64 *_base, uint64 *_length, uint8 *_type) 245 { 246 // the MTRRs are identical on all CPUs, so it doesn't matter 247 // on which CPU this runs 248 return sCpuModule->get_mtrr(index, _base, _length, _type); 249 } 250 251 252 void 253 x86_set_mtrrs(uint8 defaultType, const x86_mtrr_info* infos, uint32 count) 254 { 255 if (sCpuModule == NULL) 256 return; 257 258 struct set_mtrrs_parameter parameter; 259 parameter.defaultType = defaultType; 260 parameter.infos = infos; 261 parameter.count = count; 262 263 sCpuRendezvous = sCpuRendezvous2 = 0; 264 call_all_cpus(&set_mtrrs, ¶meter); 265 } 266 267 268 extern "C" void 269 init_sse(void) 270 { 271 if (!x86_check_feature(IA32_FEATURE_SSE, FEATURE_COMMON) 272 || !x86_check_feature(IA32_FEATURE_FXSR, FEATURE_COMMON)) { 273 // we don't have proper SSE support 274 return; 275 } 276 277 // enable OS support for SSE 278 x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION); 279 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 280 281 gX86SwapFPUFunc = i386_fxsave_swap; 282 gHasSSE = true; 283 } 284 285 286 static void 287 load_tss(int cpu) 288 { 289 short seg = ((TSS_BASE_SEGMENT + cpu) << 3) | DPL_KERNEL; 290 asm("movw %0, %%ax;" 291 "ltr %%ax;" : : "r" (seg) : "eax"); 292 } 293 294 295 static void 296 init_double_fault(int cpuNum) 297 { 298 // set up the double fault TSS 299 struct tss *tss = &gCPU[cpuNum].arch.double_fault_tss; 300 301 memset(tss, 0, sizeof(struct tss)); 302 size_t stackSize; 303 tss->sp0 = (uint32)x86_get_double_fault_stack(cpuNum, &stackSize); 304 tss->sp0 += stackSize; 305 tss->ss0 = KERNEL_DATA_SEG; 306 read_cr3(tss->cr3); 307 // copy the current cr3 to the double fault cr3 308 tss->eip = (uint32)&double_fault; 309 tss->es = KERNEL_DATA_SEG; 310 tss->cs = KERNEL_CODE_SEG; 311 tss->ss = KERNEL_DATA_SEG; 312 tss->esp = tss->sp0; 313 tss->ds = KERNEL_DATA_SEG; 314 tss->fs = KERNEL_DATA_SEG; 315 tss->gs = KERNEL_DATA_SEG; 316 tss->ldt_seg_selector = 0; 317 tss->io_map_base = sizeof(struct tss); 318 319 // add TSS descriptor for this new TSS 320 uint16 tssSegmentDescriptorIndex = DOUBLE_FAULT_TSS_BASE_SEGMENT + cpuNum; 321 set_tss_descriptor(&gGDT[tssSegmentDescriptorIndex], 322 (addr_t)tss, sizeof(struct tss)); 323 324 x86_set_task_gate(cpuNum, 8, tssSegmentDescriptorIndex << 3); 325 } 326 327 328 #if DUMP_FEATURE_STRING 329 static void 330 dump_feature_string(int currentCPU, cpu_ent *cpu) 331 { 332 char features[256]; 333 features[0] = 0; 334 335 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FPU) 336 strlcat(features, "fpu ", sizeof(features)); 337 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_VME) 338 strlcat(features, "vme ", sizeof(features)); 339 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DE) 340 strlcat(features, "de ", sizeof(features)); 341 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE) 342 strlcat(features, "pse ", sizeof(features)); 343 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TSC) 344 strlcat(features, "tsc ", sizeof(features)); 345 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MSR) 346 strlcat(features, "msr ", sizeof(features)); 347 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAE) 348 strlcat(features, "pae ", sizeof(features)); 349 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCE) 350 strlcat(features, "mce ", sizeof(features)); 351 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CX8) 352 strlcat(features, "cx8 ", sizeof(features)); 353 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_APIC) 354 strlcat(features, "apic ", sizeof(features)); 355 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SEP) 356 strlcat(features, "sep ", sizeof(features)); 357 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MTRR) 358 strlcat(features, "mtrr ", sizeof(features)); 359 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PGE) 360 strlcat(features, "pge ", sizeof(features)); 361 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCA) 362 strlcat(features, "mca ", sizeof(features)); 363 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CMOV) 364 strlcat(features, "cmov ", sizeof(features)); 365 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAT) 366 strlcat(features, "pat ", sizeof(features)); 367 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE36) 368 strlcat(features, "pse36 ", sizeof(features)); 369 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSN) 370 strlcat(features, "psn ", sizeof(features)); 371 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CLFSH) 372 strlcat(features, "clfsh ", sizeof(features)); 373 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DS) 374 strlcat(features, "ds ", sizeof(features)); 375 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_ACPI) 376 strlcat(features, "acpi ", sizeof(features)); 377 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MMX) 378 strlcat(features, "mmx ", sizeof(features)); 379 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FXSR) 380 strlcat(features, "fxsr ", sizeof(features)); 381 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE) 382 strlcat(features, "sse ", sizeof(features)); 383 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE2) 384 strlcat(features, "sse2 ", sizeof(features)); 385 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SS) 386 strlcat(features, "ss ", sizeof(features)); 387 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_HTT) 388 strlcat(features, "htt ", sizeof(features)); 389 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TM) 390 strlcat(features, "tm ", sizeof(features)); 391 if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PBE) 392 strlcat(features, "pbe ", sizeof(features)); 393 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE3) 394 strlcat(features, "sse3 ", sizeof(features)); 395 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MONITOR) 396 strlcat(features, "monitor ", sizeof(features)); 397 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DSCPL) 398 strlcat(features, "dscpl ", sizeof(features)); 399 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_EST) 400 strlcat(features, "est ", sizeof(features)); 401 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TM2) 402 strlcat(features, "tm2 ", sizeof(features)); 403 if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CNXTID) 404 strlcat(features, "cnxtid ", sizeof(features)); 405 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_SYSCALL) 406 strlcat(features, "syscall ", sizeof(features)); 407 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_NX) 408 strlcat(features, "nx ", sizeof(features)); 409 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_MMXEXT) 410 strlcat(features, "mmxext ", sizeof(features)); 411 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_FFXSR) 412 strlcat(features, "ffxsr ", sizeof(features)); 413 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_LONG) 414 strlcat(features, "long ", sizeof(features)); 415 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOWEXT) 416 strlcat(features, "3dnowext ", sizeof(features)); 417 if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOW) 418 strlcat(features, "3dnow ", sizeof(features)); 419 420 dprintf("CPU %d: features: %s\n", currentCPU, features); 421 } 422 #endif // DUMP_FEATURE_STRING 423 424 425 static int 426 detect_cpu(int currentCPU) 427 { 428 cpu_ent *cpu = get_cpu_struct(); 429 char vendorString[17]; 430 cpuid_info cpuid; 431 432 // clear out the cpu info data 433 cpu->arch.vendor = VENDOR_UNKNOWN; 434 cpu->arch.vendor_name = "UNKNOWN VENDOR"; 435 cpu->arch.feature[FEATURE_COMMON] = 0; 436 cpu->arch.feature[FEATURE_EXT] = 0; 437 cpu->arch.feature[FEATURE_EXT_AMD] = 0; 438 cpu->arch.model_name[0] = 0; 439 440 // print some fun data 441 get_current_cpuid(&cpuid, 0); 442 443 // build the vendor string 444 memset(vendorString, 0, sizeof(vendorString)); 445 memcpy(vendorString, cpuid.eax_0.vendor_id, sizeof(cpuid.eax_0.vendor_id)); 446 447 // get the family, model, stepping 448 get_current_cpuid(&cpuid, 1); 449 cpu->arch.type = cpuid.eax_1.type; 450 cpu->arch.family = cpuid.eax_1.family; 451 cpu->arch.extended_family = cpuid.eax_1.extended_family; 452 cpu->arch.model = cpuid.eax_1.model; 453 cpu->arch.extended_model = cpuid.eax_1.extended_model; 454 cpu->arch.stepping = cpuid.eax_1.stepping; 455 dprintf("CPU %d: type %d family %d extended_family %d model %d " 456 "extended_model %d stepping %d, string '%s'\n", 457 currentCPU, cpu->arch.type, cpu->arch.family, 458 cpu->arch.extended_family, cpu->arch.model, 459 cpu->arch.extended_model, cpu->arch.stepping, vendorString); 460 461 // figure out what vendor we have here 462 463 for (int32 i = 0; i < VENDOR_NUM; i++) { 464 if (vendor_info[i].ident_string[0] 465 && !strcmp(vendorString, vendor_info[i].ident_string[0])) { 466 cpu->arch.vendor = (x86_vendors)i; 467 cpu->arch.vendor_name = vendor_info[i].vendor; 468 break; 469 } 470 if (vendor_info[i].ident_string[1] 471 && !strcmp(vendorString, vendor_info[i].ident_string[1])) { 472 cpu->arch.vendor = (x86_vendors)i; 473 cpu->arch.vendor_name = vendor_info[i].vendor; 474 break; 475 } 476 } 477 478 // see if we can get the model name 479 get_current_cpuid(&cpuid, 0x80000000); 480 if (cpuid.eax_0.max_eax >= 0x80000004) { 481 // build the model string (need to swap ecx/edx data before copying) 482 unsigned int temp; 483 memset(cpu->arch.model_name, 0, sizeof(cpu->arch.model_name)); 484 485 get_current_cpuid(&cpuid, 0x80000002); 486 temp = cpuid.regs.edx; 487 cpuid.regs.edx = cpuid.regs.ecx; 488 cpuid.regs.ecx = temp; 489 memcpy(cpu->arch.model_name, cpuid.as_chars, sizeof(cpuid.as_chars)); 490 491 get_current_cpuid(&cpuid, 0x80000003); 492 temp = cpuid.regs.edx; 493 cpuid.regs.edx = cpuid.regs.ecx; 494 cpuid.regs.ecx = temp; 495 memcpy(cpu->arch.model_name + 16, cpuid.as_chars, 496 sizeof(cpuid.as_chars)); 497 498 get_current_cpuid(&cpuid, 0x80000004); 499 temp = cpuid.regs.edx; 500 cpuid.regs.edx = cpuid.regs.ecx; 501 cpuid.regs.ecx = temp; 502 memcpy(cpu->arch.model_name + 32, cpuid.as_chars, 503 sizeof(cpuid.as_chars)); 504 505 // some cpus return a right-justified string 506 int32 i = 0; 507 while (cpu->arch.model_name[i] == ' ') 508 i++; 509 if (i > 0) { 510 memmove(cpu->arch.model_name, &cpu->arch.model_name[i], 511 strlen(&cpu->arch.model_name[i]) + 1); 512 } 513 514 dprintf("CPU %d: vendor '%s' model name '%s'\n", 515 currentCPU, cpu->arch.vendor_name, cpu->arch.model_name); 516 } else { 517 strcpy(cpu->arch.model_name, "unknown"); 518 } 519 520 // load feature bits 521 get_current_cpuid(&cpuid, 1); 522 cpu->arch.feature[FEATURE_COMMON] = cpuid.eax_1.features; // edx 523 cpu->arch.feature[FEATURE_EXT] = cpuid.eax_1.extended_features; // ecx 524 if (cpu->arch.vendor == VENDOR_AMD) { 525 get_current_cpuid(&cpuid, 0x80000001); 526 cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx 527 } 528 529 #if DUMP_FEATURE_STRING 530 dump_feature_string(currentCPU, cpu); 531 #endif 532 533 return 0; 534 } 535 536 537 bool 538 x86_check_feature(uint32 feature, enum x86_feature_type type) 539 { 540 cpu_ent *cpu = get_cpu_struct(); 541 542 #if 0 543 int i; 544 dprintf("x86_check_feature: feature 0x%x, type %d\n", feature, type); 545 for (i = 0; i < FEATURE_NUM; i++) { 546 dprintf("features %d: 0x%x\n", i, cpu->arch.feature[i]); 547 } 548 #endif 549 550 return (cpu->arch.feature[type] & feature) != 0; 551 } 552 553 554 void* 555 x86_get_double_fault_stack(int32 cpu, size_t* _size) 556 { 557 *_size = kDoubleFaultStackSize; 558 return sDoubleFaultStacks + kDoubleFaultStackSize * cpu; 559 } 560 561 562 /*! Returns the index of the current CPU. Can only be called from the double 563 fault handler. 564 */ 565 int32 566 x86_double_fault_get_cpu(void) 567 { 568 uint32 stack = x86_read_ebp(); 569 return (stack - (uint32)sDoubleFaultStacks) / kDoubleFaultStackSize; 570 } 571 572 573 // #pragma mark - 574 575 576 status_t 577 arch_cpu_preboot_init_percpu(kernel_args *args, int cpu) 578 { 579 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 580 gX86SwapFPUFunc = i386_fnsave_swap; 581 582 // On SMP system we want to synchronize the CPUs' TSCs, so system_time() 583 // will return consistent values. 584 if (smp_get_num_cpus() > 1) { 585 // let the first CPU prepare the rendezvous point 586 if (cpu == 0) 587 sTSCSyncRendezvous = smp_get_num_cpus() - 1; 588 589 // One CPU after the other will drop out of this loop and be caught by 590 // the loop below, until the last CPU (0) gets there. Save for +/- a few 591 // cycles the CPUs should pass the second loop at the same time. 592 while (sTSCSyncRendezvous != cpu) { 593 } 594 595 sTSCSyncRendezvous = cpu - 1; 596 597 while (sTSCSyncRendezvous != -1) { 598 } 599 600 // reset TSC to 0 601 x86_write_msr(IA32_MSR_TSC, 0); 602 } 603 604 return B_OK; 605 } 606 607 608 status_t 609 arch_cpu_init_percpu(kernel_args *args, int cpu) 610 { 611 detect_cpu(cpu); 612 613 // load the TSS for this cpu 614 // note the main cpu gets initialized in arch_cpu_init_post_vm() 615 if (cpu != 0) { 616 load_tss(cpu); 617 618 // set the IDT 619 struct { 620 uint16 limit; 621 void* address; 622 } _PACKED descriptor = { 623 256 * 8 - 1, // 256 descriptors, 8 bytes each (-1 for "limit") 624 x86_get_idt(cpu) 625 }; 626 627 asm volatile("lidt %0" : : "m"(descriptor)); 628 } 629 630 return 0; 631 } 632 633 status_t 634 arch_cpu_init(kernel_args *args) 635 { 636 // init the TSC -> system_time() conversion factors 637 638 uint32 conversionFactor = args->arch_args.system_time_cv_factor; 639 uint64 conversionFactorNsecs = (uint64)conversionFactor * 1000; 640 641 if (conversionFactorNsecs >> 32 != 0) { 642 // the TSC frequency is < 1 GHz, which forces us to shift the factor 643 __x86_setup_system_time(conversionFactor, conversionFactorNsecs >> 16, 644 true); 645 } else { 646 // the TSC frequency is >= 1 GHz 647 __x86_setup_system_time(conversionFactor, conversionFactorNsecs, false); 648 } 649 650 return B_OK; 651 } 652 653 654 status_t 655 arch_cpu_init_post_vm(kernel_args *args) 656 { 657 uint32 i; 658 659 // account for the segment descriptors 660 gGDT = (segment_descriptor *)args->arch_args.vir_gdt; 661 create_area("gdt", (void **)&gGDT, B_EXACT_ADDRESS, B_PAGE_SIZE, 662 B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 663 664 // currently taken out of the build, because it's not yet used (and assumes 665 // (a fixed number of used GDT entries) 666 //i386_selector_init(gGDT); // pass the new gdt 667 668 // allocate an area for the double fault stacks 669 create_area("double fault stacks", (void**)&sDoubleFaultStacks, 670 B_ANY_KERNEL_ADDRESS, kDoubleFaultStackSize * smp_get_num_cpus(), 671 B_FULL_LOCK, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 672 673 vm_translation_map_arch_info* kernelArchTranslationMap 674 = static_cast<X86VMTranslationMap*>( 675 VMAddressSpace::Kernel()->TranslationMap())->ArchData(); 676 677 // setup task-state segments 678 for (i = 0; i < args->num_cpus; i++) { 679 // initialize the regular and double fault tss stored in the per-cpu 680 // structure 681 memset(&gCPU[i].arch.tss, 0, sizeof(struct tss)); 682 gCPU[i].arch.tss.ss0 = KERNEL_DATA_SEG; 683 gCPU[i].arch.tss.io_map_base = sizeof(struct tss); 684 685 // add TSS descriptor for this new TSS 686 set_tss_descriptor(&gGDT[TSS_BASE_SEGMENT + i], 687 (addr_t)&gCPU[i].arch.tss, sizeof(struct tss)); 688 689 // initialize the double fault tss 690 init_double_fault(i); 691 692 // init active translation map 693 gCPU[i].arch.active_translation_map = kernelArchTranslationMap; 694 kernelArchTranslationMap->AddReference(); 695 } 696 697 // set the current hardware task on cpu 0 698 load_tss(0); 699 700 // setup TLS descriptors (one for every CPU) 701 702 for (i = 0; i < args->num_cpus; i++) { 703 set_segment_descriptor(&gGDT[TLS_BASE_SEGMENT + i], 0, TLS_SIZE, 704 DT_DATA_WRITEABLE, DPL_USER); 705 } 706 707 // setup SSE2/3 support 708 init_sse(); 709 710 return B_OK; 711 } 712 713 714 status_t 715 arch_cpu_init_post_modules(kernel_args *args) 716 { 717 // initialize CPU module 718 719 void *cookie = open_module_list("cpu"); 720 721 while (true) { 722 char name[B_FILE_NAME_LENGTH]; 723 size_t nameLength = sizeof(name); 724 725 if (read_next_module_name(cookie, name, &nameLength) != B_OK 726 || get_module(name, (module_info **)&sCpuModule) == B_OK) 727 break; 728 } 729 730 close_module_list(cookie); 731 732 // initialize MTRRs if available 733 if (x86_count_mtrrs() > 0) { 734 sCpuRendezvous = sCpuRendezvous2 = 0; 735 call_all_cpus(&init_mtrrs, NULL); 736 } 737 738 // get optimized functions from the CPU module 739 if (sCpuModule != NULL && sCpuModule->get_optimized_functions != NULL) { 740 x86_optimized_functions functions; 741 memset(&functions, 0, sizeof(functions)); 742 743 sCpuModule->get_optimized_functions(&functions); 744 745 if (functions.memcpy != NULL) { 746 gOptimizedFunctions.memcpy = functions.memcpy; 747 gOptimizedFunctions.memcpy_end = functions.memcpy_end; 748 } 749 750 if (functions.memset != NULL) { 751 gOptimizedFunctions.memset = functions.memset; 752 gOptimizedFunctions.memset_end = functions.memset_end; 753 } 754 } 755 756 // put the optimized functions into the commpage 757 size_t memcpyLen = (addr_t)gOptimizedFunctions.memcpy_end 758 - (addr_t)gOptimizedFunctions.memcpy; 759 fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMCPY, 760 (const void*)gOptimizedFunctions.memcpy, memcpyLen); 761 size_t memsetLen = (addr_t)gOptimizedFunctions.memset_end 762 - (addr_t)gOptimizedFunctions.memset; 763 fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMSET, 764 (const void*)gOptimizedFunctions.memset, memsetLen); 765 766 // add the functions to the commpage image 767 image_id image = get_commpage_image(); 768 elf_add_memory_image_symbol(image, "commpage_memcpy", 769 ((addr_t*)USER_COMMPAGE_ADDR)[COMMPAGE_ENTRY_X86_MEMCPY], memcpyLen, 770 B_SYMBOL_TYPE_TEXT); 771 elf_add_memory_image_symbol(image, "commpage_memset", 772 ((addr_t*)USER_COMMPAGE_ADDR)[COMMPAGE_ENTRY_X86_MEMSET], memsetLen, 773 B_SYMBOL_TYPE_TEXT); 774 775 return B_OK; 776 } 777 778 779 void 780 i386_set_tss_and_kstack(addr_t kstack) 781 { 782 get_cpu_struct()->arch.tss.sp0 = kstack; 783 } 784 785 void 786 arch_cpu_global_TLB_invalidate(void) 787 { 788 uint32 flags = x86_read_cr4(); 789 790 if (flags & IA32_CR4_GLOBAL_PAGES) { 791 // disable and reenable the global pages to flush all TLBs regardless 792 // of the global page bit 793 x86_write_cr4(flags & ~IA32_CR4_GLOBAL_PAGES); 794 x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES); 795 } else { 796 cpu_status state = disable_interrupts(); 797 arch_cpu_user_TLB_invalidate(); 798 restore_interrupts(state); 799 } 800 } 801 802 803 void 804 arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) 805 { 806 int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE; 807 while (num_pages-- >= 0) { 808 invalidate_TLB(start); 809 start += B_PAGE_SIZE; 810 } 811 } 812 813 814 void 815 arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) 816 { 817 int i; 818 for (i = 0; i < num_pages; i++) { 819 invalidate_TLB(pages[i]); 820 } 821 } 822 823 824 ssize_t 825 arch_cpu_user_strlcpy(char *to, const char *from, size_t size, 826 addr_t *faultHandler) 827 { 828 int fromLength = 0; 829 addr_t oldFaultHandler = *faultHandler; 830 831 // this check is to trick the gcc4 compiler and have it keep the error label 832 if (to == NULL && size > 0) 833 goto error; 834 835 *faultHandler = (addr_t)&&error; 836 837 if (size > 0) { 838 to[--size] = '\0'; 839 // copy 840 for ( ; size; size--, fromLength++, to++, from++) { 841 if ((*to = *from) == '\0') 842 break; 843 } 844 } 845 // count any leftover from chars 846 while (*from++ != '\0') { 847 fromLength++; 848 } 849 850 *faultHandler = oldFaultHandler; 851 return fromLength; 852 853 error: 854 *faultHandler = oldFaultHandler; 855 return B_BAD_ADDRESS; 856 } 857 858 859 status_t 860 arch_cpu_user_memset(void *s, char c, size_t count, addr_t *faultHandler) 861 { 862 char *xs = (char *)s; 863 addr_t oldFaultHandler = *faultHandler; 864 865 // this check is to trick the gcc4 compiler and have it keep the error label 866 if (s == NULL) 867 goto error; 868 869 *faultHandler = (addr_t)&&error; 870 871 while (count--) 872 *xs++ = c; 873 874 *faultHandler = oldFaultHandler; 875 return 0; 876 877 error: 878 *faultHandler = oldFaultHandler; 879 return B_BAD_ADDRESS; 880 } 881 882 883 status_t 884 arch_cpu_shutdown(bool rebootSystem) 885 { 886 if (acpi_shutdown(rebootSystem) == B_OK) 887 return B_OK; 888 889 if (!rebootSystem) 890 return apm_shutdown(); 891 892 cpu_status state = disable_interrupts(); 893 894 // try to reset the system using the keyboard controller 895 out8(0xfe, 0x64); 896 897 // Give some time to the controller to do its job (0.5s) 898 snooze(500000); 899 900 // if that didn't help, try it this way 901 reboot(); 902 903 restore_interrupts(state); 904 return B_ERROR; 905 } 906 907 908 void 909 arch_cpu_idle(void) 910 { 911 asm("hlt"); 912 } 913 914 915 void 916 arch_cpu_sync_icache(void *address, size_t length) 917 { 918 // instruction cache is always consistent on x86 919 } 920 921 922 void 923 arch_cpu_memory_read_barrier(void) 924 { 925 asm volatile ("lock;" : : : "memory"); 926 asm volatile ("addl $0, 0(%%esp);" : : : "memory"); 927 } 928 929 930 void 931 arch_cpu_memory_write_barrier(void) 932 { 933 asm volatile ("lock;" : : : "memory"); 934 asm volatile ("addl $0, 0(%%esp);" : : : "memory"); 935 } 936 937