1 /* 2 * Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de. 3 * Copyright 2002-2009, Axel Dörfler, axeld@pinc-software.de. 4 * Distributed under the terms of the MIT License. 5 * 6 * Copyright 2001, Travis Geiselbrecht. All rights reserved. 7 * Distributed under the terms of the NewOS License. 8 */ 9 10 11 #include <cpu.h> 12 #include <int.h> 13 #include <kscheduler.h> 14 #include <ksyscalls.h> 15 #include <smp.h> 16 #include <team.h> 17 #include <thread.h> 18 #include <vm/vm.h> 19 #include <vm/vm_priv.h> 20 21 #include <arch/cpu.h> 22 #include <arch/int.h> 23 #include <arch/smp.h> 24 #include <arch/user_debugger.h> 25 #include <arch/vm.h> 26 27 #include <arch/x86/arch_apic.h> 28 #include <arch/x86/descriptors.h> 29 #include <arch/x86/vm86.h> 30 31 #include "interrupts.h" 32 33 #include <ACPI.h> 34 #include <safemode.h> 35 #include <string.h> 36 #include <stdio.h> 37 38 39 //#define TRACE_ARCH_INT 40 #ifdef TRACE_ARCH_INT 41 # define TRACE(x) dprintf x 42 #else 43 # define TRACE(x) ; 44 #endif 45 46 47 // Definitions for the PIC 8259 controller 48 // (this is not a complete list, only what we're actually using) 49 50 #define PIC_MASTER_CONTROL 0x20 51 #define PIC_MASTER_MASK 0x21 52 #define PIC_SLAVE_CONTROL 0xa0 53 #define PIC_SLAVE_MASK 0xa1 54 #define PIC_MASTER_INIT1 PIC_MASTER_CONTROL 55 #define PIC_MASTER_INIT2 PIC_MASTER_MASK 56 #define PIC_MASTER_INIT3 PIC_MASTER_MASK 57 #define PIC_MASTER_INIT4 PIC_MASTER_MASK 58 #define PIC_SLAVE_INIT1 PIC_SLAVE_CONTROL 59 #define PIC_SLAVE_INIT2 PIC_SLAVE_MASK 60 #define PIC_SLAVE_INIT3 PIC_SLAVE_MASK 61 #define PIC_SLAVE_INIT4 PIC_SLAVE_MASK 62 63 // the edge/level trigger control registers 64 #define PIC_MASTER_TRIGGER_MODE 0x4d0 65 #define PIC_SLAVE_TRIGGER_MODE 0x4d1 66 67 #define PIC_INIT1 0x10 68 #define PIC_INIT1_SEND_INIT4 0x01 69 #define PIC_INIT3_IR2_IS_SLAVE 0x04 70 #define PIC_INIT3_SLAVE_ID2 0x02 71 #define PIC_INIT4_x86_MODE 0x01 72 73 #define PIC_CONTROL3 0x08 74 #define PIC_CONTROL3_READ_ISR 0x03 75 #define PIC_CONTROL3_READ_IRR 0x02 76 77 #define PIC_NON_SPECIFIC_EOI 0x20 78 79 #define PIC_SLAVE_INT_BASE 8 80 #define PIC_NUM_INTS 0x0f 81 82 83 // Definitions for a 82093AA IO APIC controller 84 #define IO_APIC_IDENTIFICATION 0x00 85 #define IO_APIC_VERSION 0x01 86 #define IO_APIC_ARBITRATION 0x02 87 #define IO_APIC_REDIRECTION_TABLE 0x10 // entry = base + 2 * index 88 89 // Fields for the version register 90 #define IO_APIC_VERSION_SHIFT 0 91 #define IO_APIC_VERSION_MASK 0xff 92 #define IO_APIC_MAX_REDIRECTION_ENTRY_SHIFT 16 93 #define IO_APIC_MAX_REDIRECTION_ENTRY_MASK 0xff 94 95 // Fields of each redirection table entry 96 #define IO_APIC_DESTINATION_FIELD_SHIFT 56 97 #define IO_APIC_DESTINATION_FIELD_MASK 0x0f 98 #define IO_APIC_INTERRUPT_MASK_SHIFT 16 99 #define IO_APIC_INTERRUPT_MASKED 1 100 #define IO_APIC_INTERRUPT_UNMASKED 0 101 #define IO_APIC_TRIGGER_MODE_SHIFT 15 102 #define IO_APIC_TRIGGER_MODE_EDGE 0 103 #define IO_APIC_TRIGGER_MODE_LEVEL 1 104 #define IO_APIC_REMOTE_IRR_SHIFT 14 105 #define IO_APIC_PIN_POLARITY_SHIFT 13 106 #define IO_APIC_PIN_POLARITY_HIGH_ACTIVE 0 107 #define IO_APIC_PIN_POLARITY_LOW_ACTIVE 1 108 #define IO_APIC_DELIVERY_STATUS_SHIFT 12 109 #define IO_APIC_DELIVERY_STATUS_IDLE 0 110 #define IO_APIC_DELIVERY_STATUS_PENDING 1 111 #define IO_APIC_DESTINATION_MODE_SHIFT 11 112 #define IO_APIC_DESTINATION_MODE_PHYSICAL 0 113 #define IO_APIC_DESTINATION_MODE_LOGICAL 1 114 #define IO_APIC_DELIVERY_MODE_SHIFT 8 115 #define IO_APIC_DELIVERY_MODE_MASK 0x07 116 #define IO_APIC_DELIVERY_MODE_FIXED 0 117 #define IO_APIC_DELIVERY_MODE_LOWEST_PRIO 1 118 #define IO_APIC_DELIVERY_MODE_SMI 2 119 #define IO_APIC_DELIVERY_MODE_NMI 4 120 #define IO_APIC_DELIVERY_MODE_INIT 5 121 #define IO_APIC_DELIVERY_MODE_EXT_INT 7 122 #define IO_APIC_INTERRUPT_VECTOR_SHIFT 0 123 #define IO_APIC_INTERRUPT_VECTOR_MASK 0xff 124 125 typedef struct ioapic_s { 126 volatile uint32 io_register_select; 127 uint32 reserved[3]; 128 volatile uint32 io_window_register; 129 } ioapic; 130 131 static ioapic *sIOAPIC = NULL; 132 static uint32 sIOAPICMaxRedirectionEntry = 23; 133 static void *sLocalAPIC = NULL; 134 135 static uint32 sIRQToIOAPICPin[256]; 136 137 bool gUsingIOAPIC = false; 138 139 typedef struct interrupt_controller_s { 140 const char *name; 141 void (*enable_io_interrupt)(int32 num); 142 void (*disable_io_interrupt)(int32 num); 143 void (*configure_io_interrupt)(int32 num, uint32 config); 144 bool (*is_spurious_interrupt)(int32 num); 145 void (*end_of_interrupt)(int32 num); 146 } interrupt_controller; 147 148 static const interrupt_controller *sCurrentPIC = NULL; 149 150 static const char *kInterruptNames[] = { 151 /* 0 */ "Divide Error Exception", 152 /* 1 */ "Debug Exception", 153 /* 2 */ "NMI Interrupt", 154 /* 3 */ "Breakpoint Exception", 155 /* 4 */ "Overflow Exception", 156 /* 5 */ "BOUND Range Exceeded Exception", 157 /* 6 */ "Invalid Opcode Exception", 158 /* 7 */ "Device Not Available Exception", 159 /* 8 */ "Double Fault Exception", 160 /* 9 */ "Coprocessor Segment Overrun", 161 /* 10 */ "Invalid TSS Exception", 162 /* 11 */ "Segment Not Present", 163 /* 12 */ "Stack Fault Exception", 164 /* 13 */ "General Protection Exception", 165 /* 14 */ "Page-Fault Exception", 166 /* 15 */ "-", 167 /* 16 */ "x87 FPU Floating-Point Error", 168 /* 17 */ "Alignment Check Exception", 169 /* 18 */ "Machine-Check Exception", 170 /* 19 */ "SIMD Floating-Point Exception", 171 }; 172 static const int kInterruptNameCount = 20; 173 174 #define MAX_ARGS 16 175 176 typedef struct { 177 uint32 a, b; 178 } desc_table; 179 static desc_table* sIDTs[B_MAX_CPU_COUNT]; 180 181 static uint32 sLevelTriggeredInterrupts = 0; 182 // binary mask: 1 level, 0 edge 183 184 // table with functions handling respective interrupts 185 typedef void interrupt_handler_function(struct iframe* frame); 186 #define INTERRUPT_HANDLER_TABLE_SIZE 256 187 interrupt_handler_function* gInterruptHandlerTable[ 188 INTERRUPT_HANDLER_TABLE_SIZE]; 189 190 191 /*! Initializes a descriptor in an IDT. 192 */ 193 static void 194 set_gate(desc_table *gate_addr, addr_t addr, int type, int dpl) 195 { 196 unsigned int gate1; // first byte of gate desc 197 unsigned int gate2; // second byte of gate desc 198 199 gate1 = (KERNEL_CODE_SEG << 16) | (0x0000ffff & addr); 200 gate2 = (0xffff0000 & addr) | 0x8000 | (dpl << 13) | (type << 8); 201 202 gate_addr->a = gate1; 203 gate_addr->b = gate2; 204 } 205 206 207 /*! Initializes the descriptor for interrupt vector \a n in the IDT of the 208 specified CPU to an interrupt-gate descriptor with the given procedure 209 address. 210 For CPUs other than the boot CPU it must not be called before 211 arch_int_init_post_vm(). 212 */ 213 static void 214 set_interrupt_gate(int32 cpu, int n, void (*addr)()) 215 { 216 set_gate(&sIDTs[cpu][n], (addr_t)addr, 14, DPL_KERNEL); 217 } 218 219 220 /*! Initializes the descriptor for interrupt vector \a n in the IDT of the 221 specified CPU to an trap-gate descriptor with the given procedure address. 222 For CPUs other than the boot CPU it must not be called before 223 arch_int_init_post_vm(). 224 */ 225 static void 226 set_trap_gate(int32 cpu, int n, void (*addr)()) 227 { 228 set_gate(&sIDTs[cpu][n], (unsigned int)addr, 15, DPL_USER); 229 } 230 231 232 /*! Initializes the descriptor for interrupt vector \a n in the IDT of CPU 233 \a cpu to a task-gate descripter referring to the TSS segment identified 234 by TSS segment selector \a segment. 235 For CPUs other than the boot CPU it must not be called before 236 arch_int_init_post_vm() (arch_cpu_init_post_vm() is fine). 237 */ 238 void 239 x86_set_task_gate(int32 cpu, int32 n, int32 segment) 240 { 241 sIDTs[cpu][n].a = (segment << 16); 242 sIDTs[cpu][n].b = 0x8000 | (0 << 13) | (0x5 << 8); // present, dpl 0, type 5 243 } 244 245 246 /*! Returns the virtual IDT address for CPU \a cpu. */ 247 void* 248 x86_get_idt(int32 cpu) 249 { 250 return sIDTs[cpu]; 251 } 252 253 254 // #pragma mark - PIC 255 256 257 /*! Tests if the interrupt in-service register of the responsible 258 PIC is set for interrupts 7 and 15, and if that's not the case, 259 it must assume it's a spurious interrupt. 260 */ 261 static bool 262 pic_is_spurious_interrupt(int32 num) 263 { 264 if (num != 7) 265 return false; 266 267 // Note, detecting spurious interrupts on line 15 obviously doesn't 268 // work correctly - and since those are extremely rare, anyway, we 269 // just ignore them 270 271 out8(PIC_CONTROL3 | PIC_CONTROL3_READ_ISR, PIC_MASTER_CONTROL); 272 int32 isr = in8(PIC_MASTER_CONTROL); 273 out8(PIC_CONTROL3 | PIC_CONTROL3_READ_IRR, PIC_MASTER_CONTROL); 274 275 return (isr & 0x80) == 0; 276 } 277 278 279 /*! Sends a non-specified EOI (end of interrupt) notice to the PIC in 280 question (or both of them). 281 This clears the PIC interrupt in-service bit. 282 */ 283 static void 284 pic_end_of_interrupt(int32 num) 285 { 286 if (num < 0 || num > PIC_NUM_INTS) 287 return; 288 289 // PIC 8259 controlled interrupt 290 if (num >= PIC_SLAVE_INT_BASE) 291 out8(PIC_NON_SPECIFIC_EOI, PIC_SLAVE_CONTROL); 292 293 // we always need to acknowledge the master PIC 294 out8(PIC_NON_SPECIFIC_EOI, PIC_MASTER_CONTROL); 295 } 296 297 298 static void 299 pic_enable_io_interrupt(int32 num) 300 { 301 // interrupt is specified "normalized" 302 if (num < 0 || num > PIC_NUM_INTS) 303 return; 304 305 // enable PIC 8259 controlled interrupt 306 307 TRACE(("pic_enable_io_interrupt: irq %ld\n", num)); 308 309 if (num < PIC_SLAVE_INT_BASE) 310 out8(in8(PIC_MASTER_MASK) & ~(1 << num), PIC_MASTER_MASK); 311 else 312 out8(in8(PIC_SLAVE_MASK) & ~(1 << (num - PIC_SLAVE_INT_BASE)), PIC_SLAVE_MASK); 313 } 314 315 316 static void 317 pic_disable_io_interrupt(int32 num) 318 { 319 // interrupt is specified "normalized" 320 // never disable slave pic line IRQ 2 321 if (num < 0 || num > PIC_NUM_INTS || num == 2) 322 return; 323 324 // disable PIC 8259 controlled interrupt 325 326 TRACE(("pic_disable_io_interrupt: irq %ld\n", num)); 327 328 if (num < PIC_SLAVE_INT_BASE) 329 out8(in8(PIC_MASTER_MASK) | (1 << num), PIC_MASTER_MASK); 330 else 331 out8(in8(PIC_SLAVE_MASK) | (1 << (num - PIC_SLAVE_INT_BASE)), PIC_SLAVE_MASK); 332 } 333 334 335 static void 336 pic_configure_io_interrupt(int32 num, uint32 config) 337 { 338 uint8 value; 339 int32 localBit; 340 if (num < 0 || num > PIC_NUM_INTS || num == 2) 341 return; 342 343 TRACE(("pic_configure_io_interrupt: irq %ld; config 0x%08lx\n", num, config)); 344 345 if (num < PIC_SLAVE_INT_BASE) { 346 value = in8(PIC_MASTER_TRIGGER_MODE); 347 localBit = num; 348 } else { 349 value = in8(PIC_SLAVE_TRIGGER_MODE); 350 localBit = num - PIC_SLAVE_INT_BASE; 351 } 352 353 if (config & B_LEVEL_TRIGGERED) 354 value |= 1 << localBit; 355 else 356 value &= ~(1 << localBit); 357 358 if (num < PIC_SLAVE_INT_BASE) 359 out8(value, PIC_MASTER_TRIGGER_MODE); 360 else 361 out8(value, PIC_SLAVE_TRIGGER_MODE); 362 363 sLevelTriggeredInterrupts = in8(PIC_MASTER_TRIGGER_MODE) 364 | (in8(PIC_SLAVE_TRIGGER_MODE) << 8); 365 } 366 367 368 static void 369 pic_init(void) 370 { 371 static interrupt_controller picController = { 372 "8259 PIC", 373 &pic_enable_io_interrupt, 374 &pic_disable_io_interrupt, 375 &pic_configure_io_interrupt, 376 &pic_is_spurious_interrupt, 377 &pic_end_of_interrupt 378 }; 379 380 // Start initialization sequence for the master and slave PICs 381 out8(PIC_INIT1 | PIC_INIT1_SEND_INIT4, PIC_MASTER_INIT1); 382 out8(PIC_INIT1 | PIC_INIT1_SEND_INIT4, PIC_SLAVE_INIT1); 383 384 // Set start of interrupts to 0x20 for master, 0x28 for slave 385 out8(ARCH_INTERRUPT_BASE, PIC_MASTER_INIT2); 386 out8(ARCH_INTERRUPT_BASE + PIC_SLAVE_INT_BASE, PIC_SLAVE_INIT2); 387 388 // Specify cascading through interrupt 2 389 out8(PIC_INIT3_IR2_IS_SLAVE, PIC_MASTER_INIT3); 390 out8(PIC_INIT3_SLAVE_ID2, PIC_SLAVE_INIT3); 391 392 // Set both to operate in 8086 mode 393 out8(PIC_INIT4_x86_MODE, PIC_MASTER_INIT4); 394 out8(PIC_INIT4_x86_MODE, PIC_SLAVE_INIT4); 395 396 out8(0xfb, PIC_MASTER_MASK); // Mask off all interrupts (except slave pic line IRQ 2). 397 out8(0xff, PIC_SLAVE_MASK); // Mask off interrupts on the slave. 398 399 // determine which interrupts are level or edge triggered 400 401 #if 0 402 // should set everything possible to level triggered 403 out8(0xf8, PIC_MASTER_TRIGGER_MODE); 404 out8(0xde, PIC_SLAVE_TRIGGER_MODE); 405 #endif 406 407 sLevelTriggeredInterrupts = in8(PIC_MASTER_TRIGGER_MODE) 408 | (in8(PIC_SLAVE_TRIGGER_MODE) << 8); 409 410 TRACE(("PIC level trigger mode: 0x%08lx\n", sLevelTriggeredInterrupts)); 411 412 // make the pic controller the current one 413 sCurrentPIC = &picController; 414 gUsingIOAPIC = false; 415 } 416 417 418 // #pragma mark - I/O APIC 419 420 421 static inline uint32 422 ioapic_read_32(uint8 registerSelect) 423 { 424 sIOAPIC->io_register_select = registerSelect; 425 return sIOAPIC->io_window_register; 426 } 427 428 429 static inline void 430 ioapic_write_32(uint8 registerSelect, uint32 value) 431 { 432 sIOAPIC->io_register_select = registerSelect; 433 sIOAPIC->io_window_register = value; 434 } 435 436 437 static inline uint64 438 ioapic_read_64(uint8 registerSelect) 439 { 440 sIOAPIC->io_register_select = registerSelect + 1; 441 uint64 result = sIOAPIC->io_window_register; 442 result <<= 32; 443 sIOAPIC->io_register_select = registerSelect; 444 result |= sIOAPIC->io_window_register; 445 return result; 446 } 447 448 449 static inline void 450 ioapic_write_64(uint8 registerSelect, uint64 value) 451 { 452 sIOAPIC->io_register_select = registerSelect; 453 sIOAPIC->io_window_register = (uint32)value; 454 sIOAPIC->io_register_select = registerSelect + 1; 455 sIOAPIC->io_window_register = (uint32)(value >> 32); 456 } 457 458 459 static bool 460 ioapic_is_spurious_interrupt(int32 num) 461 { 462 // the spurious interrupt vector is initialized to the max value in smp 463 return num == 0xff - ARCH_INTERRUPT_BASE; 464 } 465 466 467 static void 468 ioapic_end_of_interrupt(int32 num) 469 { 470 *(volatile uint32 *)((char *)sLocalAPIC + APIC_EOI) = 0; 471 } 472 473 474 static void 475 ioapic_enable_io_interrupt(int32 num) 476 { 477 int32 pin = sIRQToIOAPICPin[num]; 478 if (pin < 0 || pin > (int32)sIOAPICMaxRedirectionEntry) 479 return; 480 481 TRACE(("ioapic_enable_io_interrupt: IRQ %ld -> pin %ld\n", num, pin)); 482 483 uint64 entry = ioapic_read_64(IO_APIC_REDIRECTION_TABLE + pin * 2); 484 entry &= ~(1 << IO_APIC_INTERRUPT_MASK_SHIFT); 485 entry |= IO_APIC_INTERRUPT_UNMASKED << IO_APIC_INTERRUPT_MASK_SHIFT; 486 ioapic_write_64(IO_APIC_REDIRECTION_TABLE + pin * 2, entry); 487 } 488 489 490 static void 491 ioapic_disable_io_interrupt(int32 num) 492 { 493 int32 pin = sIRQToIOAPICPin[num]; 494 if (pin < 0 || pin > (int32)sIOAPICMaxRedirectionEntry) 495 return; 496 497 TRACE(("ioapic_disable_io_interrupt: IRQ %ld -> pin %ld\n", num, pin)); 498 499 uint64 entry = ioapic_read_64(IO_APIC_REDIRECTION_TABLE + pin * 2); 500 entry &= ~(1 << IO_APIC_INTERRUPT_MASK_SHIFT); 501 entry |= IO_APIC_INTERRUPT_MASKED << IO_APIC_INTERRUPT_MASK_SHIFT; 502 ioapic_write_64(IO_APIC_REDIRECTION_TABLE + pin * 2, entry); 503 } 504 505 506 static void 507 ioapic_configure_io_interrupt(int32 num, uint32 config) 508 { 509 int32 pin = sIRQToIOAPICPin[num]; 510 if (pin < 0 || pin > (int32)sIOAPICMaxRedirectionEntry) 511 return; 512 513 TRACE(("ioapic_configure_io_interrupt: IRQ %ld -> pin %ld; config 0x%08lx\n", 514 num, pin, config)); 515 516 uint64 entry = ioapic_read_64(IO_APIC_REDIRECTION_TABLE + pin * 2); 517 entry &= ~((1 << IO_APIC_TRIGGER_MODE_SHIFT) 518 | (1 << IO_APIC_PIN_POLARITY_SHIFT) 519 | (IO_APIC_INTERRUPT_VECTOR_MASK << IO_APIC_INTERRUPT_VECTOR_SHIFT)); 520 521 if (config & B_LEVEL_TRIGGERED) { 522 entry |= (IO_APIC_TRIGGER_MODE_LEVEL << IO_APIC_TRIGGER_MODE_SHIFT); 523 sLevelTriggeredInterrupts |= (1 << num); 524 } else { 525 entry |= (IO_APIC_TRIGGER_MODE_EDGE << IO_APIC_TRIGGER_MODE_SHIFT); 526 sLevelTriggeredInterrupts &= ~(1 << num); 527 } 528 529 if (config & B_LOW_ACTIVE_POLARITY) 530 entry |= (IO_APIC_PIN_POLARITY_LOW_ACTIVE << IO_APIC_PIN_POLARITY_SHIFT); 531 else 532 entry |= (IO_APIC_PIN_POLARITY_HIGH_ACTIVE << IO_APIC_PIN_POLARITY_SHIFT); 533 534 entry |= (num + ARCH_INTERRUPT_BASE) << IO_APIC_INTERRUPT_VECTOR_SHIFT; 535 ioapic_write_64(IO_APIC_REDIRECTION_TABLE + pin * 2, entry); 536 } 537 538 539 static void 540 ioapic_init(kernel_args *args) 541 { 542 static const interrupt_controller ioapicController = { 543 "82093AA IOAPIC", 544 &ioapic_enable_io_interrupt, 545 &ioapic_disable_io_interrupt, 546 &ioapic_configure_io_interrupt, 547 &ioapic_is_spurious_interrupt, 548 &ioapic_end_of_interrupt 549 }; 550 551 if (args->arch_args.apic == NULL) { 552 dprintf("no local apic available\n"); 553 return; 554 } 555 556 // always map the local apic as it can be used for timers even if we 557 // don't end up using the io apic 558 sLocalAPIC = args->arch_args.apic; 559 if (vm_map_physical_memory(B_SYSTEM_TEAM, "local apic", &sLocalAPIC, 560 B_EXACT_ADDRESS, B_PAGE_SIZE, 561 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 562 args->arch_args.apic_phys, true) < 0) { 563 panic("mapping the local apic failed"); 564 return; 565 } 566 567 if (args->arch_args.ioapic == NULL) { 568 dprintf("no ioapic available, not using ioapics for interrupt routing\n"); 569 return; 570 } 571 572 if (!get_safemode_boolean(B_SAFEMODE_DISABLE_IOAPIC, false)) { 573 dprintf("ioapic explicitly disabled, not using ioapics for interrupt " 574 "routing\n"); 575 return; 576 } 577 578 // TODO: remove when the PCI IRQ routing through ACPI is available below 579 return; 580 581 acpi_module_info *acpi; 582 if (get_module(B_ACPI_MODULE_NAME, (module_info **)&acpi) != B_OK) { 583 dprintf("acpi module not available, not configuring ioapic\n"); 584 return; 585 } 586 587 // map in the ioapic 588 sIOAPIC = (ioapic *)args->arch_args.ioapic; 589 if (vm_map_physical_memory(B_SYSTEM_TEAM, "ioapic", (void**)&sIOAPIC, 590 B_EXACT_ADDRESS, B_PAGE_SIZE, 591 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 592 args->arch_args.ioapic_phys, true) < 0) { 593 panic("mapping the ioapic failed"); 594 return; 595 } 596 597 uint32 version = ioapic_read_32(IO_APIC_VERSION); 598 if (version == 0xffffffff) { 599 dprintf("ioapic seems inaccessible, not using it\n"); 600 return; 601 } 602 603 sLevelTriggeredInterrupts = 0; 604 sIOAPICMaxRedirectionEntry 605 = ((version >> IO_APIC_MAX_REDIRECTION_ENTRY_SHIFT) 606 & IO_APIC_MAX_REDIRECTION_ENTRY_MASK); 607 608 // use the boot CPU as the target for all interrupts 609 uint64 targetAPIC = args->arch_args.cpu_apic_id[0]; 610 611 // program the interrupt vectors of the ioapic 612 for (uint32 i = 0; i <= sIOAPICMaxRedirectionEntry; i++) { 613 // initialize everything to deliver to the boot CPU in physical mode 614 // and masked until explicitly enabled through enable_io_interrupt() 615 uint64 entry = (targetAPIC << IO_APIC_DESTINATION_FIELD_SHIFT) 616 | (IO_APIC_INTERRUPT_MASKED << IO_APIC_INTERRUPT_MASK_SHIFT) 617 | (IO_APIC_DESTINATION_MODE_PHYSICAL << IO_APIC_DESTINATION_MODE_SHIFT) 618 | ((i + ARCH_INTERRUPT_BASE) << IO_APIC_INTERRUPT_VECTOR_SHIFT); 619 620 if (i == 0) { 621 // make redirection entry 0 into an external interrupt 622 entry |= (IO_APIC_TRIGGER_MODE_EDGE << IO_APIC_TRIGGER_MODE_SHIFT) 623 | (IO_APIC_PIN_POLARITY_HIGH_ACTIVE << IO_APIC_PIN_POLARITY_SHIFT) 624 | (IO_APIC_DELIVERY_MODE_EXT_INT << IO_APIC_DELIVERY_MODE_SHIFT); 625 } else if (i < 16) { 626 // make 1-15 ISA interrupts 627 entry |= (IO_APIC_TRIGGER_MODE_EDGE << IO_APIC_TRIGGER_MODE_SHIFT) 628 | (IO_APIC_PIN_POLARITY_HIGH_ACTIVE << IO_APIC_PIN_POLARITY_SHIFT) 629 | (IO_APIC_DELIVERY_MODE_FIXED << IO_APIC_DELIVERY_MODE_SHIFT); 630 } else { 631 // and the rest are PCI interrupts 632 entry |= (IO_APIC_TRIGGER_MODE_LEVEL << IO_APIC_TRIGGER_MODE_SHIFT) 633 | (IO_APIC_PIN_POLARITY_LOW_ACTIVE << IO_APIC_PIN_POLARITY_SHIFT) 634 | (IO_APIC_DELIVERY_MODE_FIXED << IO_APIC_DELIVERY_MODE_SHIFT); 635 sLevelTriggeredInterrupts |= (1 << i); 636 } 637 638 ioapic_write_64(IO_APIC_REDIRECTION_TABLE + 2 * i, entry); 639 } 640 641 // setup default 1:1 mapping 642 for (uint32 i = 0; i < 256; i++) 643 sIRQToIOAPICPin[i] = i; 644 645 // TODO: here ACPI needs to be used to properly set up the PCI IRQ 646 // routing. 647 648 // prefer the ioapic over the normal pic 649 put_module(B_ACPI_MODULE_NAME); 650 dprintf("using ioapic for interrupt routing\n"); 651 sCurrentPIC = &ioapicController; 652 gUsingIOAPIC = true; 653 } 654 655 656 // #pragma mark - 657 658 659 void 660 arch_int_enable_io_interrupt(int irq) 661 { 662 sCurrentPIC->enable_io_interrupt(irq); 663 } 664 665 666 void 667 arch_int_disable_io_interrupt(int irq) 668 { 669 sCurrentPIC->disable_io_interrupt(irq); 670 } 671 672 673 void 674 arch_int_configure_io_interrupt(int irq, uint32 config) 675 { 676 sCurrentPIC->configure_io_interrupt(irq, config); 677 } 678 679 680 #undef arch_int_enable_interrupts 681 #undef arch_int_disable_interrupts 682 #undef arch_int_restore_interrupts 683 #undef arch_int_are_interrupts_enabled 684 685 686 void 687 arch_int_enable_interrupts(void) 688 { 689 arch_int_enable_interrupts_inline(); 690 } 691 692 693 int 694 arch_int_disable_interrupts(void) 695 { 696 return arch_int_disable_interrupts_inline(); 697 } 698 699 700 void 701 arch_int_restore_interrupts(int oldState) 702 { 703 arch_int_restore_interrupts_inline(oldState); 704 } 705 706 707 bool 708 arch_int_are_interrupts_enabled(void) 709 { 710 return arch_int_are_interrupts_enabled_inline(); 711 } 712 713 714 static const char * 715 exception_name(int number, char *buffer, int32 bufferSize) 716 { 717 if (number >= 0 && number < kInterruptNameCount) 718 return kInterruptNames[number]; 719 720 snprintf(buffer, bufferSize, "exception %d", number); 721 return buffer; 722 } 723 724 725 static void 726 invalid_exception(struct iframe* frame) 727 { 728 struct thread* thread = thread_get_current_thread(); 729 char name[32]; 730 panic("unhandled trap 0x%lx (%s) at ip 0x%lx, thread %ld!\n", 731 frame->vector, exception_name(frame->vector, name, sizeof(name)), 732 frame->eip, thread ? thread->id : -1); 733 } 734 735 736 static void 737 fatal_exception(struct iframe *frame) 738 { 739 char name[32]; 740 panic("Fatal exception \"%s\" occurred! Error code: 0x%lx\n", 741 exception_name(frame->vector, name, sizeof(name)), frame->error_code); 742 } 743 744 745 static void 746 unexpected_exception(struct iframe* frame) 747 { 748 debug_exception_type type; 749 int signal; 750 751 if (IFRAME_IS_VM86(frame)) { 752 x86_vm86_return((struct vm86_iframe *)frame, (frame->vector == 13) ? 753 B_OK : B_ERROR); 754 // won't get here 755 } 756 757 switch (frame->vector) { 758 case 0: // Divide Error Exception (#DE) 759 type = B_DIVIDE_ERROR; 760 signal = SIGFPE; 761 break; 762 763 case 4: // Overflow Exception (#OF) 764 type = B_OVERFLOW_EXCEPTION; 765 signal = SIGTRAP; 766 break; 767 768 case 5: // BOUND Range Exceeded Exception (#BR) 769 type = B_BOUNDS_CHECK_EXCEPTION; 770 signal = SIGTRAP; 771 break; 772 773 case 6: // Invalid Opcode Exception (#UD) 774 type = B_INVALID_OPCODE_EXCEPTION; 775 signal = SIGILL; 776 break; 777 778 case 13: // General Protection Exception (#GP) 779 type = B_GENERAL_PROTECTION_FAULT; 780 signal = SIGILL; 781 break; 782 783 case 16: // x87 FPU Floating-Point Error (#MF) 784 type = B_FLOATING_POINT_EXCEPTION; 785 signal = SIGFPE; 786 break; 787 788 case 17: // Alignment Check Exception (#AC) 789 type = B_ALIGNMENT_EXCEPTION; 790 signal = SIGTRAP; 791 break; 792 793 case 19: // SIMD Floating-Point Exception (#XF) 794 type = B_FLOATING_POINT_EXCEPTION; 795 signal = SIGFPE; 796 break; 797 798 default: 799 invalid_exception(frame); 800 return; 801 } 802 803 if (IFRAME_IS_USER(frame)) { 804 struct sigaction action; 805 struct thread* thread = thread_get_current_thread(); 806 807 enable_interrupts(); 808 809 // If the thread has a signal handler for the signal, we simply send it 810 // the signal. Otherwise we notify the user debugger first. 811 if (sigaction(signal, NULL, &action) == 0 812 && action.sa_handler != SIG_DFL 813 && action.sa_handler != SIG_IGN) { 814 send_signal(thread->id, signal); 815 } else if (user_debug_exception_occurred(type, signal)) 816 send_signal(team_get_current_team_id(), signal); 817 } else { 818 char name[32]; 819 panic("Unexpected exception \"%s\" occurred in kernel mode! " 820 "Error code: 0x%lx\n", 821 exception_name(frame->vector, name, sizeof(name)), 822 frame->error_code); 823 } 824 } 825 826 827 void 828 x86_double_fault_exception(struct iframe* frame) 829 { 830 int cpu = x86_double_fault_get_cpu(); 831 832 // The double fault iframe contains no useful information (as 833 // per Intel's architecture spec). Thus we simply save the 834 // information from the (unhandlable) exception which caused the 835 // double in our iframe. This will result even in useful stack 836 // traces. Only problem is that we trust that at least the 837 // TSS is still accessible. 838 struct tss *tss = &gCPU[cpu].arch.tss; 839 840 frame->cs = tss->cs; 841 frame->es = tss->es; 842 frame->ds = tss->ds; 843 frame->fs = tss->fs; 844 frame->gs = tss->gs; 845 frame->eip = tss->eip; 846 frame->ebp = tss->ebp; 847 frame->esp = tss->esp; 848 frame->eax = tss->eax; 849 frame->ebx = tss->ebx; 850 frame->ecx = tss->ecx; 851 frame->edx = tss->edx; 852 frame->esi = tss->esi; 853 frame->edi = tss->edi; 854 frame->flags = tss->eflags; 855 856 // Use a special handler for page faults which avoids the triple fault 857 // pitfalls. 858 set_interrupt_gate(cpu, 14, &trap14_double_fault); 859 860 debug_double_fault(cpu); 861 } 862 863 864 void 865 x86_page_fault_exception_double_fault(struct iframe* frame) 866 { 867 uint32 cr2; 868 asm("movl %%cr2, %0" : "=r" (cr2)); 869 870 // Only if this CPU has a fault handler, we're allowed to be here. 871 cpu_ent& cpu = gCPU[x86_double_fault_get_cpu()]; 872 addr_t faultHandler = cpu.fault_handler; 873 if (faultHandler != 0) { 874 debug_set_page_fault_info(cr2, frame->eip, 875 (frame->error_code & 0x2) != 0 ? DEBUG_PAGE_FAULT_WRITE : 0); 876 frame->eip = faultHandler; 877 frame->ebp = cpu.fault_handler_stack_pointer; 878 return; 879 } 880 881 // No fault handler. This is bad. Since we originally came from a double 882 // fault, we don't try to reenter the kernel debugger. Instead we just 883 // print the info we've got and enter an infinite loop. 884 kprintf("Page fault in double fault debugger without fault handler! " 885 "Touching address %p from eip %p. Entering infinite loop...\n", 886 (void*)cr2, (void*)frame->eip); 887 888 while (true); 889 } 890 891 892 static void 893 page_fault_exception(struct iframe* frame) 894 { 895 struct thread *thread = thread_get_current_thread(); 896 uint32 cr2; 897 addr_t newip; 898 899 asm("movl %%cr2, %0" : "=r" (cr2)); 900 901 if (debug_debugger_running()) { 902 // If this CPU or this thread has a fault handler, we're allowed to be 903 // here. 904 if (thread != NULL) { 905 cpu_ent* cpu = &gCPU[smp_get_current_cpu()]; 906 if (cpu->fault_handler != 0) { 907 debug_set_page_fault_info(cr2, frame->eip, 908 (frame->error_code & 0x2) != 0 909 ? DEBUG_PAGE_FAULT_WRITE : 0); 910 frame->eip = cpu->fault_handler; 911 frame->ebp = cpu->fault_handler_stack_pointer; 912 return; 913 } 914 915 if (thread->fault_handler != 0) { 916 kprintf("ERROR: thread::fault_handler used in kernel " 917 "debugger!\n"); 918 debug_set_page_fault_info(cr2, frame->eip, 919 (frame->error_code & 0x2) != 0 920 ? DEBUG_PAGE_FAULT_WRITE : 0); 921 frame->eip = thread->fault_handler; 922 return; 923 } 924 } 925 926 // otherwise, not really 927 panic("page fault in debugger without fault handler! Touching " 928 "address %p from eip %p\n", (void *)cr2, (void *)frame->eip); 929 return; 930 } else if ((frame->flags & 0x200) == 0) { 931 // interrupts disabled 932 933 // If a page fault handler is installed, we're allowed to be here. 934 // TODO: Now we are generally allowing user_memcpy() with interrupts 935 // disabled, which in most cases is a bug. We should add some thread 936 // flag allowing to explicitly indicate that this handling is desired. 937 if (thread && thread->fault_handler != 0) { 938 if (frame->eip != thread->fault_handler) { 939 frame->eip = thread->fault_handler; 940 return; 941 } 942 943 // The fault happened at the fault handler address. This is a 944 // certain infinite loop. 945 panic("page fault, interrupts disabled, fault handler loop. " 946 "Touching address %p from eip %p\n", (void*)cr2, 947 (void*)frame->eip); 948 } 949 950 // If we are not running the kernel startup the page fault was not 951 // allowed to happen and we must panic. 952 panic("page fault, but interrupts were disabled. Touching address " 953 "%p from eip %p\n", (void *)cr2, (void *)frame->eip); 954 return; 955 } else if (thread != NULL && thread->page_faults_allowed < 1) { 956 panic("page fault not allowed at this place. Touching address " 957 "%p from eip %p\n", (void *)cr2, (void *)frame->eip); 958 return; 959 } 960 961 enable_interrupts(); 962 963 vm_page_fault(cr2, frame->eip, 964 (frame->error_code & 0x2) != 0, // write access 965 (frame->error_code & 0x4) != 0, // userland 966 &newip); 967 if (newip != 0) { 968 // the page fault handler wants us to modify the iframe to set the 969 // IP the cpu will return to to be this ip 970 frame->eip = newip; 971 } 972 } 973 974 975 static void 976 hardware_interrupt(struct iframe* frame) 977 { 978 int32 vector = frame->vector - ARCH_INTERRUPT_BASE; 979 bool levelTriggered = false; 980 struct thread* thread = thread_get_current_thread(); 981 982 if (sCurrentPIC->is_spurious_interrupt(vector)) { 983 TRACE(("got spurious interrupt at vector %ld\n", vector)); 984 return; 985 } 986 987 if (vector < 32) 988 levelTriggered = (sLevelTriggeredInterrupts & (1 << vector)) != 0; 989 990 if (!levelTriggered) 991 sCurrentPIC->end_of_interrupt(vector); 992 993 int_io_interrupt_handler(vector, levelTriggered); 994 995 if (levelTriggered) 996 sCurrentPIC->end_of_interrupt(vector); 997 998 cpu_status state = disable_interrupts(); 999 if (thread->cpu->invoke_scheduler) { 1000 GRAB_THREAD_LOCK(); 1001 scheduler_reschedule(); 1002 RELEASE_THREAD_LOCK(); 1003 restore_interrupts(state); 1004 } else if (thread->post_interrupt_callback != NULL) { 1005 restore_interrupts(state); 1006 void (*callback)(void*) = thread->post_interrupt_callback; 1007 void* data = thread->post_interrupt_data; 1008 1009 thread->post_interrupt_callback = NULL; 1010 thread->post_interrupt_data = NULL; 1011 1012 callback(data); 1013 } 1014 } 1015 1016 1017 status_t 1018 arch_int_init(struct kernel_args *args) 1019 { 1020 int i; 1021 interrupt_handler_function** table; 1022 1023 // set the global sIDT variable 1024 sIDTs[0] = (desc_table *)args->arch_args.vir_idt; 1025 1026 // setup the standard programmable interrupt controller 1027 pic_init(); 1028 1029 set_interrupt_gate(0, 0, &trap0); 1030 set_interrupt_gate(0, 1, &trap1); 1031 set_interrupt_gate(0, 2, &trap2); 1032 set_trap_gate(0, 3, &trap3); 1033 set_interrupt_gate(0, 4, &trap4); 1034 set_interrupt_gate(0, 5, &trap5); 1035 set_interrupt_gate(0, 6, &trap6); 1036 set_interrupt_gate(0, 7, &trap7); 1037 // trap8 (double fault) is set in arch_cpu.c 1038 set_interrupt_gate(0, 9, &trap9); 1039 set_interrupt_gate(0, 10, &trap10); 1040 set_interrupt_gate(0, 11, &trap11); 1041 set_interrupt_gate(0, 12, &trap12); 1042 set_interrupt_gate(0, 13, &trap13); 1043 set_interrupt_gate(0, 14, &trap14); 1044 // set_interrupt_gate(0, 15, &trap15); 1045 set_interrupt_gate(0, 16, &trap16); 1046 set_interrupt_gate(0, 17, &trap17); 1047 set_interrupt_gate(0, 18, &trap18); 1048 set_interrupt_gate(0, 19, &trap19); 1049 1050 set_interrupt_gate(0, 32, &trap32); 1051 set_interrupt_gate(0, 33, &trap33); 1052 set_interrupt_gate(0, 34, &trap34); 1053 set_interrupt_gate(0, 35, &trap35); 1054 set_interrupt_gate(0, 36, &trap36); 1055 set_interrupt_gate(0, 37, &trap37); 1056 set_interrupt_gate(0, 38, &trap38); 1057 set_interrupt_gate(0, 39, &trap39); 1058 set_interrupt_gate(0, 40, &trap40); 1059 set_interrupt_gate(0, 41, &trap41); 1060 set_interrupt_gate(0, 42, &trap42); 1061 set_interrupt_gate(0, 43, &trap43); 1062 set_interrupt_gate(0, 44, &trap44); 1063 set_interrupt_gate(0, 45, &trap45); 1064 set_interrupt_gate(0, 46, &trap46); 1065 set_interrupt_gate(0, 47, &trap47); 1066 set_interrupt_gate(0, 48, &trap48); 1067 set_interrupt_gate(0, 49, &trap49); 1068 set_interrupt_gate(0, 50, &trap50); 1069 set_interrupt_gate(0, 51, &trap51); 1070 set_interrupt_gate(0, 52, &trap52); 1071 set_interrupt_gate(0, 53, &trap53); 1072 set_interrupt_gate(0, 54, &trap54); 1073 set_interrupt_gate(0, 55, &trap55); 1074 1075 set_trap_gate(0, 98, &trap98); // for performance testing only 1076 set_trap_gate(0, 99, &trap99); 1077 1078 set_interrupt_gate(0, 251, &trap251); 1079 set_interrupt_gate(0, 252, &trap252); 1080 set_interrupt_gate(0, 253, &trap253); 1081 set_interrupt_gate(0, 254, &trap254); 1082 set_interrupt_gate(0, 255, &trap255); 1083 1084 // init interrupt handler table 1085 table = gInterruptHandlerTable; 1086 1087 // defaults 1088 for (i = 0; i < ARCH_INTERRUPT_BASE; i++) 1089 table[i] = invalid_exception; 1090 for (i = ARCH_INTERRUPT_BASE; i < INTERRUPT_HANDLER_TABLE_SIZE; i++) 1091 table[i] = hardware_interrupt; 1092 1093 table[0] = unexpected_exception; // Divide Error Exception (#DE) 1094 table[1] = x86_handle_debug_exception; // Debug Exception (#DB) 1095 table[2] = fatal_exception; // NMI Interrupt 1096 table[3] = x86_handle_breakpoint_exception; // Breakpoint Exception (#BP) 1097 table[4] = unexpected_exception; // Overflow Exception (#OF) 1098 table[5] = unexpected_exception; // BOUND Range Exceeded Exception (#BR) 1099 table[6] = unexpected_exception; // Invalid Opcode Exception (#UD) 1100 table[7] = fatal_exception; // Device Not Available Exception (#NM) 1101 table[8] = x86_double_fault_exception; // Double Fault Exception (#DF) 1102 table[9] = fatal_exception; // Coprocessor Segment Overrun 1103 table[10] = fatal_exception; // Invalid TSS Exception (#TS) 1104 table[11] = fatal_exception; // Segment Not Present (#NP) 1105 table[12] = fatal_exception; // Stack Fault Exception (#SS) 1106 table[13] = unexpected_exception; // General Protection Exception (#GP) 1107 table[14] = page_fault_exception; // Page-Fault Exception (#PF) 1108 table[16] = unexpected_exception; // x87 FPU Floating-Point Error (#MF) 1109 table[17] = unexpected_exception; // Alignment Check Exception (#AC) 1110 table[18] = fatal_exception; // Machine-Check Exception (#MC) 1111 table[19] = unexpected_exception; // SIMD Floating-Point Exception (#XF) 1112 1113 return B_OK; 1114 } 1115 1116 1117 status_t 1118 arch_int_init_post_vm(struct kernel_args *args) 1119 { 1120 ioapic_init(args); 1121 1122 // create IDT area for the boot CPU 1123 area_id area = create_area("idt", (void**)&sIDTs[0], B_EXACT_ADDRESS, 1124 B_PAGE_SIZE, B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 1125 if (area < 0) 1126 return area; 1127 1128 // create IDTs for the off-boot CPU 1129 size_t idtSize = 256 * 8; 1130 // 256 8 bytes-sized descriptors 1131 int32 cpuCount = smp_get_num_cpus(); 1132 if (cpuCount > 0) { 1133 size_t areaSize = ROUNDUP(cpuCount * idtSize, B_PAGE_SIZE); 1134 desc_table* idt; 1135 area = create_area("idt", (void**)&idt, B_ANY_KERNEL_ADDRESS, 1136 areaSize, B_CONTIGUOUS, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); 1137 if (area < 0) 1138 return area; 1139 1140 for (int32 i = 1; i < cpuCount; i++) { 1141 sIDTs[i] = idt; 1142 memcpy(idt, sIDTs[0], idtSize); 1143 idt += 256; 1144 // The CPU's IDTR will be set in arch_cpu_init_percpu(). 1145 } 1146 } 1147 1148 return area >= B_OK ? B_OK : area; 1149 } 1150 1151 1152 status_t 1153 arch_int_init_post_device_manager(struct kernel_args *args) 1154 { 1155 return B_OK; 1156 } 1157