1 /* 2 * Copyright 2008, Dustin Howett, dustin.howett@gmail.com. All rights reserved. 3 * Copyright 2004-2010, Axel Dörfler, axeld@pinc-software.de. 4 * Distributed under the terms of the MIT License. 5 * 6 * Copyright 2001, Travis Geiselbrecht. All rights reserved. 7 * Distributed under the terms of the NewOS License. 8 */ 9 10 11 #include "smp.h" 12 13 #include <string.h> 14 15 #include <KernelExport.h> 16 17 #include <kernel.h> 18 #include <safemode.h> 19 #include <boot/platform.h> 20 #include <boot/stage2.h> 21 #include <boot/menu.h> 22 #include <arch/x86/apic.h> 23 #include <arch/x86/arch_acpi.h> 24 #include <arch/x86/arch_cpu.h> 25 #include <arch/x86/arch_smp.h> 26 #include <arch/x86/arch_system_info.h> 27 #include <arch/x86/descriptors.h> 28 29 #include "mmu.h" 30 #include "acpi.h" 31 32 33 #define NO_SMP 0 34 35 //#define TRACE_SMP 36 #ifdef TRACE_SMP 37 # define TRACE(x) dprintf x 38 #else 39 # define TRACE(x) ; 40 #endif 41 42 43 extern "C" void execute_n_instructions(int count); 44 45 extern "C" void smp_trampoline(void); 46 extern "C" void smp_trampoline_args(void); 47 extern "C" void smp_trampoline_end(void); 48 49 struct gdtr { 50 uint16 limit; 51 uint32 base; 52 unsigned char null[8]; 53 unsigned char code[8]; 54 unsigned char data[8]; 55 } __attribute__((packed)); 56 57 // Arguments passed to the SMP trampoline. 58 struct trampoline_args { 59 uint32 trampoline; // Trampoline address 60 uint32 gdt32; // 32-bit GDTR 61 uint32 pml4; // 64-bit PML4 62 uint32 gdt64; // 64-bit GDTR 63 uint64 kernel_entry; // Kernel entry point 64 uint64 kernel_args; // Kernel arguments 65 uint64 current_cpu; // CPU number 66 uint64 stack_top; // Kernel stack 67 volatile uint64 sentinel; // Sentinel, AP sets to 0 when finished 68 69 // smp_boot_other_cpus puts the GDTR here. 70 struct gdtr gdtr; 71 }; 72 73 74 static uint32 75 apic_read(uint32 offset) 76 { 77 return *(volatile uint32 *)((addr_t)gKernelArgs.arch_args.apic_phys + offset); 78 } 79 80 81 static void 82 apic_write(uint32 offset, uint32 data) 83 { 84 *(volatile uint32 *)((addr_t)gKernelArgs.arch_args.apic_phys + offset) = data; 85 } 86 87 88 static status_t 89 smp_do_acpi_config(void) 90 { 91 TRACE(("smp: using ACPI to detect MP configuration\n")); 92 93 // reset CPU count 94 gKernelArgs.num_cpus = 0; 95 96 acpi_madt *madt = (acpi_madt *)acpi_find_table(ACPI_MADT_SIGNATURE); 97 98 if (madt == NULL) { 99 TRACE(("smp: Failed to find MADT!\n")); 100 return B_ERROR; 101 } 102 103 gKernelArgs.arch_args.apic_phys = madt->local_apic_address; 104 TRACE(("smp: local apic address is 0x%" B_PRIx32 "\n", madt->local_apic_address)); 105 106 acpi_apic *apic = (acpi_apic *)((uint8 *)madt + sizeof(acpi_madt)); 107 acpi_apic *end = (acpi_apic *)((uint8 *)madt + madt->header.length); 108 while (apic < end) { 109 switch (apic->type) { 110 case ACPI_MADT_LOCAL_APIC: 111 { 112 if (gKernelArgs.num_cpus == SMP_MAX_CPUS) { 113 TRACE(("smp: already reached maximum CPUs (%d)\n", 114 SMP_MAX_CPUS)); 115 break; 116 } 117 118 acpi_local_apic *localApic = (acpi_local_apic *)apic; 119 TRACE(("smp: found local APIC with id %u\n", 120 localApic->apic_id)); 121 if ((localApic->flags & ACPI_LOCAL_APIC_ENABLED) == 0) { 122 TRACE(("smp: APIC is disabled and will not be used\n")); 123 break; 124 } 125 126 gKernelArgs.arch_args.cpu_apic_id[gKernelArgs.num_cpus] 127 = localApic->apic_id; 128 // TODO: how to find out? putting 0x10 in to indicate a local apic 129 gKernelArgs.arch_args.cpu_apic_version[gKernelArgs.num_cpus] 130 = 0x10; 131 gKernelArgs.num_cpus++; 132 break; 133 } 134 135 case ACPI_MADT_IO_APIC: { 136 acpi_io_apic *ioApic = (acpi_io_apic *)apic; 137 TRACE(("smp: found io APIC with id %" B_PRIu32 " and address 0x%" B_PRIx32 "\n", 138 ioApic->io_apic_id, ioApic->io_apic_address)); 139 if (gKernelArgs.arch_args.ioapic_phys == 0) 140 gKernelArgs.arch_args.ioapic_phys = ioApic->io_apic_address; 141 break; 142 } 143 default: 144 break; 145 } 146 147 apic = (acpi_apic *)((uint8 *)apic + apic->length); 148 } 149 150 return gKernelArgs.num_cpus > 0 ? B_OK : B_ERROR; 151 } 152 153 154 static void 155 calculate_apic_timer_conversion_factor(void) 156 { 157 int64 t1, t2; 158 uint32 config; 159 uint32 count; 160 161 TRACE(("calculating apic timer conversion factor\n")); 162 163 // setup the timer 164 config = apic_read(APIC_LVT_TIMER); 165 config = (config & APIC_LVT_TIMER_MASK) + APIC_LVT_MASKED; 166 // timer masked, vector 0 167 apic_write(APIC_LVT_TIMER, config); 168 169 config = (apic_read(APIC_TIMER_DIVIDE_CONFIG) & ~0x0000000f); 170 apic_write(APIC_TIMER_DIVIDE_CONFIG, config | APIC_TIMER_DIVIDE_CONFIG_1); 171 // divide clock by one 172 173 t1 = system_time(); 174 apic_write(APIC_INITIAL_TIMER_COUNT, 0xffffffff); // start the counter 175 176 execute_n_instructions(128 * 20000); 177 178 count = apic_read(APIC_CURRENT_TIMER_COUNT); 179 t2 = system_time(); 180 181 count = 0xffffffff - count; 182 183 gKernelArgs.arch_args.apic_time_cv_factor 184 = (uint32)((1000000.0/(t2 - t1)) * count); 185 186 TRACE(("APIC ticks/sec = %" B_PRId32 "\n", 187 gKernelArgs.arch_args.apic_time_cv_factor)); 188 } 189 190 191 // #pragma mark - 192 193 194 int 195 smp_get_current_cpu(void) 196 { 197 if (gKernelArgs.arch_args.apic == NULL) 198 return 0; 199 200 uint8 apicID = apic_read(APIC_ID) >> 24; 201 for (uint32 i = 0; i < gKernelArgs.num_cpus; i++) { 202 if (gKernelArgs.arch_args.cpu_apic_id[i] == apicID) 203 return i; 204 } 205 206 return 0; 207 } 208 209 210 void 211 smp_init_other_cpus(void) 212 { 213 if (get_safemode_boolean(B_SAFEMODE_DISABLE_SMP, false)) { 214 // SMP has been disabled! 215 TRACE(("smp disabled per safemode setting\n")); 216 gKernelArgs.num_cpus = 1; 217 } 218 219 if (get_safemode_boolean(B_SAFEMODE_DISABLE_APIC, false)) { 220 TRACE(("local apic disabled per safemode setting, disabling smp\n")); 221 gKernelArgs.arch_args.apic_phys = 0; 222 gKernelArgs.num_cpus = 1; 223 } 224 225 if (gKernelArgs.arch_args.apic_phys == 0) 226 return; 227 228 TRACE(("smp: found %" B_PRId32 " cpu%s\n", gKernelArgs.num_cpus, 229 gKernelArgs.num_cpus != 1 ? "s" : "")); 230 TRACE(("smp: apic_phys = %lx\n", (addr_t)gKernelArgs.arch_args.apic_phys)); 231 TRACE(("smp: ioapic_phys = %lx\n", 232 (addr_t)gKernelArgs.arch_args.ioapic_phys)); 233 234 // map in the apic 235 gKernelArgs.arch_args.apic = (void *)mmu_map_physical_memory( 236 gKernelArgs.arch_args.apic_phys, B_PAGE_SIZE, kDefaultPageFlags); 237 238 TRACE(("smp: apic (mapped) = %lx\n", (addr_t)gKernelArgs.arch_args.apic.Pointer())); 239 240 // calculate how fast the apic timer is 241 calculate_apic_timer_conversion_factor(); 242 243 if (gKernelArgs.num_cpus < 2) 244 return; 245 246 for (uint32 i = 1; i < gKernelArgs.num_cpus; i++) { 247 // create a final stack the trampoline code will put the ap processor on 248 void * stack = NULL; 249 const size_t size = KERNEL_STACK_SIZE + KERNEL_STACK_GUARD_PAGES * B_PAGE_SIZE; 250 if (platform_allocate_region(&stack, size, 0, false) != B_OK) { 251 panic("Unable to allocate AP stack"); 252 } 253 memset(stack, 0, size); 254 gKernelArgs.cpu_kstack[i].start = fix_address((uint64_t)stack); 255 gKernelArgs.cpu_kstack[i].size = size; 256 } 257 } 258 259 260 void 261 smp_boot_other_cpus(uint32 pml4, uint32 gdtr64, uint64 kernel_entry) 262 { 263 if (gKernelArgs.num_cpus < 2) 264 return; 265 266 TRACE(("trampolining other cpus\n")); 267 268 // allocate a stack and a code area for the smp trampoline 269 // (these have to be < 1M physical, 0xa0000-0xfffff is reserved by the BIOS) 270 uint64 trampolineCode = 0x9000; 271 uint64 trampolineStack = 0x8000; 272 273 // copy the trampoline code over 274 TRACE(("copying the trampoline code to %p from %p\n", (char*)trampolineCode, (const void*)&smp_trampoline)); 275 TRACE(("size of trampoline code = %lu bytes\n", (uint64)&smp_trampoline_end - (uint64)&smp_trampoline)); 276 memcpy((char *)trampolineCode, (const void*)&smp_trampoline, 277 (uint64)&smp_trampoline_end - (uint64)&smp_trampoline); 278 279 // boot the cpus 280 TRACE(("we have %d CPUs to boot...\n", gKernelArgs.num_cpus - 1)); 281 for (uint32 i = 1; i < gKernelArgs.num_cpus; i++) { 282 TRACE(("trampolining CPU %d\n", i)); 283 uint32 config; 284 uint64 numStartups; 285 uint32 j; 286 trampoline_args * args = (trampoline_args *)trampolineStack; 287 args->trampoline = trampolineCode; 288 args->gdt32 = (uint64) &args->gdtr; 289 args->gdtr.limit = 23; 290 args->gdtr.base = (uint32)(uint64)args->gdtr.null; 291 #define COPY_ARRAY(A, X0, X1, X2, X3, X4, X5, X6, X7) \ 292 { A[0] = X0; A[1] = X1; A[2] = X2; A[3] = X3; A[4] = X4; A[5] = X5; A[6] = X6; A[7] = X7; } 293 COPY_ARRAY(args->gdtr.null, 0, 0, 0, 0, 0, 0, 0, 0); 294 COPY_ARRAY(args->gdtr.code, 0xff, 0xff, 0, 0, 0, 0x9a, 0xcf, 0); 295 COPY_ARRAY(args->gdtr.data, 0xff, 0xff, 0, 0, 0, 0x92, 0xcf, 0); 296 #undef COPY_ARRAY 297 args->pml4 = pml4; 298 args->gdt64 = gdtr64; 299 args->kernel_entry = kernel_entry; 300 args->kernel_args = (uint64)&gKernelArgs; 301 args->current_cpu = i; 302 args->stack_top = gKernelArgs.cpu_kstack[i].start + gKernelArgs.cpu_kstack[i].size; 303 args->sentinel = 1; 304 305 // put the args in the right place 306 uint32 * args_ptr = 307 (uint32 *)(trampolineCode + (uint64)smp_trampoline_args - (uint64)smp_trampoline); 308 *args_ptr = (uint32)(uint64)args; 309 310 /* clear apic errors */ 311 if (gKernelArgs.arch_args.cpu_apic_version[i] & 0xf0) { 312 apic_write(APIC_ERROR_STATUS, 0); 313 apic_read(APIC_ERROR_STATUS); 314 } 315 316 /* send (aka assert) INIT IPI */ 317 config = (apic_read(APIC_INTR_COMMAND_2) & APIC_INTR_COMMAND_2_MASK) 318 | (gKernelArgs.arch_args.cpu_apic_id[i] << 24); 319 apic_write(APIC_INTR_COMMAND_2, config); /* set target pe */ 320 config = (apic_read(APIC_INTR_COMMAND_1) & 0xfff00000) 321 | APIC_TRIGGER_MODE_LEVEL | APIC_INTR_COMMAND_1_ASSERT 322 | APIC_DELIVERY_MODE_INIT; 323 apic_write(APIC_INTR_COMMAND_1, config); 324 325 // wait for pending to end 326 while ((apic_read(APIC_INTR_COMMAND_1) & APIC_DELIVERY_STATUS) != 0) 327 asm volatile ("pause;"); 328 329 /* deassert INIT */ 330 config = (apic_read(APIC_INTR_COMMAND_2) & APIC_INTR_COMMAND_2_MASK) 331 | (gKernelArgs.arch_args.cpu_apic_id[i] << 24); 332 apic_write(APIC_INTR_COMMAND_2, config); 333 config = (apic_read(APIC_INTR_COMMAND_1) & 0xfff00000) 334 | APIC_TRIGGER_MODE_LEVEL | APIC_DELIVERY_MODE_INIT; 335 apic_write(APIC_INTR_COMMAND_1, config); 336 337 // wait for pending to end 338 while ((apic_read(APIC_INTR_COMMAND_1) & APIC_DELIVERY_STATUS) != 0) 339 asm volatile ("pause;"); 340 341 /* wait 10ms */ 342 spin(10000); 343 /* is this a local apic or an 82489dx ? */ 344 numStartups = (gKernelArgs.arch_args.cpu_apic_version[i] & 0xf0) 345 ? 2 : 0; 346 for (j = 0; j < numStartups; j++) { 347 /* it's a local apic, so send STARTUP IPIs */ 348 apic_write(APIC_ERROR_STATUS, 0); 349 350 /* set target pe */ 351 config = (apic_read(APIC_INTR_COMMAND_2) & APIC_INTR_COMMAND_2_MASK) 352 | (gKernelArgs.arch_args.cpu_apic_id[i] << 24); 353 apic_write(APIC_INTR_COMMAND_2, config); 354 355 /* send the IPI */ 356 config = (apic_read(APIC_INTR_COMMAND_1) & 0xfff0f800) 357 | APIC_DELIVERY_MODE_STARTUP | (trampolineCode >> 12); 358 apic_write(APIC_INTR_COMMAND_1, config); 359 360 /* wait */ 361 spin(200); 362 363 while ((apic_read(APIC_INTR_COMMAND_1) & APIC_DELIVERY_STATUS) != 0) 364 asm volatile ("pause;"); 365 } 366 367 // Wait for the trampoline code to clear the final stack location. 368 // This serves as a notification for us that it has loaded the address 369 // and it is safe for us to overwrite it to trampoline the next CPU. 370 while (args->sentinel != 0) 371 spin(1000); 372 } 373 374 TRACE(("done trampolining\n")); 375 } 376 377 378 void 379 smp_add_safemode_menus(Menu *menu) 380 { 381 MenuItem *item; 382 383 if (gKernelArgs.arch_args.ioapic_phys != 0) { 384 menu->AddItem(item = new(nothrow) MenuItem("Disable IO-APIC")); 385 item->SetType(MENU_ITEM_MARKABLE); 386 item->SetData(B_SAFEMODE_DISABLE_IOAPIC); 387 item->SetHelpText("Disables using the IO APIC for interrupt routing, " 388 "forcing the use of the legacy PIC instead."); 389 } 390 391 if (gKernelArgs.arch_args.apic_phys != 0) { 392 menu->AddItem(item = new(nothrow) MenuItem("Disable local APIC")); 393 item->SetType(MENU_ITEM_MARKABLE); 394 item->SetData(B_SAFEMODE_DISABLE_APIC); 395 item->SetHelpText("Disables using the local APIC, also disables SMP."); 396 397 cpuid_info info; 398 if (get_current_cpuid(&info, 1, 0) == B_OK 399 && (info.regs.ecx & IA32_FEATURE_EXT_X2APIC) != 0) { 400 menu->AddItem(item = new(nothrow) MenuItem("Disable X2APIC")); 401 item->SetType(MENU_ITEM_MARKABLE); 402 item->SetData(B_SAFEMODE_DISABLE_X2APIC); 403 item->SetHelpText("Disables using X2APIC."); 404 } 405 406 if (get_current_cpuid(&info, 7, 0) == B_OK 407 && ((info.regs.ebx & (IA32_FEATURE_SMEP 408 | IA32_FEATURE_SMAP)) != 0)) { 409 menu->AddItem(item = new(nothrow) MenuItem( 410 "Disable SMEP and SMAP")); 411 item->SetType(MENU_ITEM_MARKABLE); 412 item->SetData(B_SAFEMODE_DISABLE_SMEP_SMAP); 413 item->SetHelpText("Disables using SMEP and SMAP."); 414 } 415 } 416 417 if (gKernelArgs.num_cpus < 2) 418 return; 419 420 item = new(nothrow) MenuItem("Disable SMP"); 421 menu->AddItem(item); 422 item->SetData(B_SAFEMODE_DISABLE_SMP); 423 item->SetType(MENU_ITEM_MARKABLE); 424 item->SetHelpText("Disables all but one CPU core."); 425 } 426 427 428 void 429 smp_init(void) 430 { 431 #if NO_SMP 432 gKernelArgs.num_cpus = 1; 433 return; 434 #endif 435 436 cpuid_info info; 437 if (get_current_cpuid(&info, 1, 0) != B_OK) 438 return; 439 440 if ((info.eax_1.features & IA32_FEATURE_APIC) == 0) { 441 // Local APICs aren't present; As they form the basis for all inter CPU 442 // communication and therefore SMP, we don't need to go any further. 443 TRACE(("no local APIC present, not attempting SMP init\n")); 444 return; 445 } 446 447 // first try to find ACPI tables to get MP configuration as it handles 448 // physical as well as logical MP configurations as in multiple cpus, 449 // multiple cores or hyper threading. 450 if (smp_do_acpi_config() == B_OK) { 451 TRACE(("smp init success\n")); 452 return; 453 } 454 455 // Everything failed or we are not running an SMP system, reset anything 456 // that might have been set through an incomplete configuration attempt. 457 gKernelArgs.arch_args.apic_phys = 0; 458 gKernelArgs.arch_args.ioapic_phys = 0; 459 gKernelArgs.num_cpus = 1; 460 } 461