1 /*
2 * Copyright 2018, Jérôme Duval, jerome.duval@gmail.com.
3 * Copyright 2008-2011, Michael Lotz, mmlr@mlotz.ch.
4 * Copyright 2010, Clemens Zeidler, haiku@clemens-zeidler.de.
5 * Copyright 2009-2011, Ingo Weinhold, ingo_weinhold@gmx.de.
6 * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de.
7 * Distributed under the terms of the MIT License.
8 *
9 * Copyright 2001, Travis Geiselbrecht. All rights reserved.
10 * Distributed under the terms of the NewOS License.
11 */
12
13
14 #include <cpu.h>
15 #include <int.h>
16 #include <kscheduler.h>
17 #include <team.h>
18 #include <thread.h>
19 #include <util/AutoLock.h>
20 #include <vm/vm.h>
21 #include <vm/vm_priv.h>
22
23 #include <arch/cpu.h>
24 #include <arch/int.h>
25
26 #include <arch/x86/apic.h>
27 #include <arch/x86/descriptors.h>
28 #include <arch/x86/msi.h>
29 #include <arch/x86/msi_priv.h>
30
31 #include <stdio.h>
32
33 // interrupt controllers
34 #include <arch/x86/ioapic.h>
35 #include <arch/x86/pic.h>
36
37
38 //#define TRACE_ARCH_INT
39 #ifdef TRACE_ARCH_INT
40 # define TRACE(x) dprintf x
41 #else
42 # define TRACE(x) ;
43 #endif
44
45
46 static irq_source sVectorSources[NUM_IO_VECTORS];
47
48 static const char *kInterruptNames[] = {
49 /* 0 */ "Divide Error Exception",
50 /* 1 */ "Debug Exception",
51 /* 2 */ "NMI Interrupt",
52 /* 3 */ "Breakpoint Exception",
53 /* 4 */ "Overflow Exception",
54 /* 5 */ "BOUND Range Exceeded Exception",
55 /* 6 */ "Invalid Opcode Exception",
56 /* 7 */ "Device Not Available Exception",
57 /* 8 */ "Double Fault Exception",
58 /* 9 */ "Coprocessor Segment Overrun",
59 /* 10 */ "Invalid TSS Exception",
60 /* 11 */ "Segment Not Present",
61 /* 12 */ "Stack Fault Exception",
62 /* 13 */ "General Protection Exception",
63 /* 14 */ "Page-Fault Exception",
64 /* 15 */ "-",
65 /* 16 */ "x87 FPU Floating-Point Error",
66 /* 17 */ "Alignment Check Exception",
67 /* 18 */ "Machine-Check Exception",
68 /* 19 */ "SIMD Floating-Point Exception",
69 };
70 static const int kInterruptNameCount = 20;
71
72 static const interrupt_controller* sCurrentPIC = NULL;
73
74
75 static const char*
exception_name(int number,char * buffer,int32 bufferSize)76 exception_name(int number, char* buffer, int32 bufferSize)
77 {
78 if (number >= 0 && number < kInterruptNameCount)
79 return kInterruptNames[number];
80
81 snprintf(buffer, bufferSize, "exception %d", number);
82 return buffer;
83 }
84
85
86 void
x86_invalid_exception(iframe * frame)87 x86_invalid_exception(iframe* frame)
88 {
89 Thread* thread = thread_get_current_thread();
90 char name[32];
91 panic("unhandled trap 0x%lx (%s) at ip 0x%lx, thread %" B_PRId32 "!\n",
92 (long unsigned int)frame->vector,
93 exception_name(frame->vector, name, sizeof(name)),
94 (long unsigned int)frame->ip, thread ? thread->id : -1);
95 }
96
97
98 void
x86_fatal_exception(iframe * frame)99 x86_fatal_exception(iframe* frame)
100 {
101 char name[32];
102 panic("Fatal exception \"%s\" occurred! Error code: 0x%lx\n",
103 exception_name(frame->vector, name, sizeof(name)),
104 (long unsigned int)frame->error_code);
105 }
106
107
108 void
x86_unexpected_exception(iframe * frame)109 x86_unexpected_exception(iframe* frame)
110 {
111 debug_exception_type type;
112 uint32 signalNumber;
113 int32 signalCode;
114 addr_t signalAddress = 0;
115 int32 signalError = B_ERROR;
116
117 switch (frame->vector) {
118 case 0: // Divide Error Exception (#DE)
119 type = B_DIVIDE_ERROR;
120 signalNumber = SIGFPE;
121 signalCode = FPE_INTDIV;
122 signalAddress = frame->ip;
123 break;
124
125 case 4: // Overflow Exception (#OF)
126 type = B_OVERFLOW_EXCEPTION;
127 signalNumber = SIGFPE;
128 signalCode = FPE_INTOVF;
129 signalAddress = frame->ip;
130 break;
131
132 case 5: // BOUND Range Exceeded Exception (#BR)
133 type = B_BOUNDS_CHECK_EXCEPTION;
134 signalNumber = SIGTRAP;
135 signalCode = SI_USER;
136 break;
137
138 case 6: // Invalid Opcode Exception (#UD)
139 type = B_INVALID_OPCODE_EXCEPTION;
140 signalNumber = SIGILL;
141 signalCode = ILL_ILLOPC;
142 signalAddress = frame->ip;
143 break;
144
145 case 12: // Stack Fault (#SS)
146 type = B_STACK_FAULT;
147 signalNumber = SIGBUS;
148 signalCode = BUS_ADRERR;
149 signalAddress = frame->ip;
150 break;
151
152 case 13: // General Protection Exception (#GP)
153 type = B_GENERAL_PROTECTION_FAULT;
154 signalNumber = SIGILL;
155 signalCode = ILL_PRVOPC; // or ILL_PRVREG
156 signalAddress = frame->ip;
157 break;
158
159 case 16: // x87 FPU Floating-Point Error (#MF)
160 type = B_FLOATING_POINT_EXCEPTION;
161 signalNumber = SIGFPE;
162 signalCode = FPE_FLTDIV;
163 // TODO: Determine the correct cause via the FPU status
164 // register!
165 signalAddress = frame->ip;
166 break;
167
168 case 17: // Alignment Check Exception (#AC)
169 type = B_ALIGNMENT_EXCEPTION;
170 signalNumber = SIGBUS;
171 signalCode = BUS_ADRALN;
172 // TODO: Also get the address (from where?). Since we don't enable
173 // alignment checking this exception should never happen, though.
174 signalError = EFAULT;
175 break;
176
177 case 19: // SIMD Floating-Point Exception (#XF)
178 type = B_FLOATING_POINT_EXCEPTION;
179 signalNumber = SIGFPE;
180 signalCode = FPE_FLTDIV;
181 // TODO: Determine the correct cause via the MXCSR register!
182 signalAddress = frame->ip;
183 break;
184
185 default:
186 x86_invalid_exception(frame);
187 return;
188 }
189
190 if (IFRAME_IS_USER(frame)) {
191 struct sigaction action;
192 Thread* thread = thread_get_current_thread();
193
194 enable_interrupts();
195
196 // If the thread has a signal handler for the signal, we simply send it
197 // the signal. Otherwise we notify the user debugger first.
198 if ((sigaction(signalNumber, NULL, &action) == 0
199 && action.sa_handler != SIG_DFL
200 && action.sa_handler != SIG_IGN)
201 || user_debug_exception_occurred(type, signalNumber)) {
202 Signal signal(signalNumber, signalCode, signalError,
203 thread->team->id);
204 signal.SetAddress((void*)signalAddress);
205 send_signal_to_thread(thread, signal, 0);
206 }
207 } else {
208 char name[32];
209 panic("Unexpected exception \"%s\" occurred in kernel mode! "
210 "Error code: 0x%lx\n",
211 exception_name(frame->vector, name, sizeof(name)),
212 (long unsigned int)(frame->error_code));
213 }
214 }
215
216
217 void
x86_hardware_interrupt(struct iframe * frame)218 x86_hardware_interrupt(struct iframe* frame)
219 {
220 int32 vector = frame->vector - ARCH_INTERRUPT_BASE;
221 bool levelTriggered = false;
222 Thread* thread = thread_get_current_thread();
223
224 if (sCurrentPIC->is_spurious_interrupt(vector)) {
225 TRACE(("got spurious interrupt at vector %ld\n", vector));
226 return;
227 }
228
229 levelTriggered = sCurrentPIC->is_level_triggered_interrupt(vector);
230
231 if (!levelTriggered) {
232 // if it's not handled by the current pic then it's an apic generated
233 // interrupt like local interrupts, msi or ipi.
234 if (!sCurrentPIC->end_of_interrupt(vector))
235 apic_end_of_interrupt();
236 }
237
238 int_io_interrupt_handler(vector, levelTriggered);
239
240 if (levelTriggered) {
241 if (!sCurrentPIC->end_of_interrupt(vector))
242 apic_end_of_interrupt();
243 }
244
245 cpu_status state = disable_interrupts();
246 if (thread->post_interrupt_callback != NULL) {
247 void (*callback)(void*) = thread->post_interrupt_callback;
248 void* data = thread->post_interrupt_data;
249
250 thread->post_interrupt_callback = NULL;
251 thread->post_interrupt_data = NULL;
252
253 restore_interrupts(state);
254
255 callback(data);
256 } else if (thread->cpu->invoke_scheduler) {
257 SpinLocker schedulerLocker(thread->scheduler_lock);
258 scheduler_reschedule(B_THREAD_READY);
259 schedulerLocker.Unlock();
260 restore_interrupts(state);
261 }
262 }
263
264
265 void
x86_page_fault_exception(struct iframe * frame)266 x86_page_fault_exception(struct iframe* frame)
267 {
268 Thread* thread = thread_get_current_thread();
269 addr_t cr2 = x86_read_cr2();
270 addr_t newip;
271
272 if (debug_debugger_running()) {
273 // If this CPU or this thread has a fault handler, we're allowed to be
274 // here.
275 if (thread != NULL) {
276 cpu_ent* cpu = &gCPU[smp_get_current_cpu()];
277 if (cpu->fault_handler != 0) {
278 debug_set_page_fault_info(cr2, frame->ip,
279 (frame->error_code & PGFAULT_W) != 0
280 ? DEBUG_PAGE_FAULT_WRITE : 0);
281 frame->ip = cpu->fault_handler;
282 frame->bp = cpu->fault_handler_stack_pointer;
283 return;
284 }
285
286 if (thread->fault_handler != 0) {
287 kprintf("ERROR: thread::fault_handler used in kernel "
288 "debugger!\n");
289 debug_set_page_fault_info(cr2, frame->ip,
290 (frame->error_code & PGFAULT_W) != 0
291 ? DEBUG_PAGE_FAULT_WRITE : 0);
292 frame->ip = reinterpret_cast<uintptr_t>(thread->fault_handler);
293 return;
294 }
295 }
296
297 // otherwise, not really
298 panic("page fault in debugger without fault handler! Touching "
299 "address %p from ip %p\n", (void*)cr2, (void*)frame->ip);
300 return;
301 } else if (!IFRAME_IS_USER(frame)
302 && (frame->error_code & PGFAULT_I) != 0
303 && (x86_read_cr4() & IA32_CR4_SMEP) != 0) {
304 // check that: 1. come not from userland,
305 // 2. is an instruction fetch, 3. smep is enabled
306 panic("SMEP violation user-mapped address %p touched from kernel %p\n",
307 (void*)cr2, (void*)frame->ip);
308 } else if ((frame->flags & X86_EFLAGS_ALIGNMENT_CHECK) == 0
309 && !IFRAME_IS_USER(frame)
310 && (frame->error_code & PGFAULT_P) != 0
311 && (x86_read_cr4() & IA32_CR4_SMAP) != 0) {
312 // check that: 1. AC flag is not set, 2. come not from userland,
313 // 3. is a page-protection violation, 4. smap is enabled
314 panic("SMAP violation user-mapped address %p touched from kernel %p\n",
315 (void*)cr2, (void*)frame->ip);
316 } else if ((frame->flags & X86_EFLAGS_INTERRUPT) == 0) {
317 // interrupts disabled
318
319 // If a page fault handler is installed, we're allowed to be here.
320 // TODO: Now we are generally allowing user_memcpy() with interrupts
321 // disabled, which in most cases is a bug. We should add some thread
322 // flag allowing to explicitly indicate that this handling is desired.
323 if (thread != NULL && thread->fault_handler != 0) {
324 uintptr_t handler
325 = reinterpret_cast<uintptr_t>(thread->fault_handler);
326 if (frame->ip != handler) {
327 frame->ip = handler;
328 return;
329 }
330
331 // The fault happened at the fault handler address. This is a
332 // certain infinite loop.
333 panic("page fault, interrupts disabled, fault handler loop. "
334 "Touching address %p from ip %p\n", (void*)cr2,
335 (void*)frame->ip);
336 }
337
338 // If we are not running the kernel startup the page fault was not
339 // allowed to happen and we must panic.
340 panic("page fault, but interrupts were disabled. Touching address "
341 "%p from ip %p\n", (void*)cr2, (void*)frame->ip);
342 return;
343 } else if (thread != NULL && thread->page_faults_allowed < 1) {
344 panic("page fault not allowed at this place. Touching address "
345 "%p from ip %p\n", (void*)cr2, (void*)frame->ip);
346 return;
347 }
348
349 enable_interrupts();
350
351 vm_page_fault(cr2, frame->ip,
352 (frame->error_code & PGFAULT_W) != 0, // write access
353 (frame->error_code & PGFAULT_I) != 0, // instruction fetch
354 IFRAME_IS_USER(frame), // userland
355 &newip);
356 if (newip != 0) {
357 // the page fault handler wants us to modify the iframe to set the
358 // IP the cpu will return to this ip
359 frame->ip = newip;
360 }
361 }
362
363
364 void
x86_set_irq_source(int32 irq,irq_source source)365 x86_set_irq_source(int32 irq, irq_source source)
366 {
367 sVectorSources[irq] = source;
368 }
369
370
371 // #pragma mark -
372
373
374 void
arch_int_enable_io_interrupt(int32 irq)375 arch_int_enable_io_interrupt(int32 irq)
376 {
377 sCurrentPIC->enable_io_interrupt(irq);
378 }
379
380
381 void
arch_int_disable_io_interrupt(int32 irq)382 arch_int_disable_io_interrupt(int32 irq)
383 {
384 sCurrentPIC->disable_io_interrupt(irq);
385 }
386
387
388 void
arch_int_configure_io_interrupt(int32 irq,uint32 config)389 arch_int_configure_io_interrupt(int32 irq, uint32 config)
390 {
391 sCurrentPIC->configure_io_interrupt(irq, config);
392 }
393
394
395 #undef arch_int_enable_interrupts
396 #undef arch_int_disable_interrupts
397 #undef arch_int_restore_interrupts
398 #undef arch_int_are_interrupts_enabled
399
400
401 void
arch_int_enable_interrupts(void)402 arch_int_enable_interrupts(void)
403 {
404 arch_int_enable_interrupts_inline();
405 }
406
407
408 int
arch_int_disable_interrupts(void)409 arch_int_disable_interrupts(void)
410 {
411 return arch_int_disable_interrupts_inline();
412 }
413
414
415 void
arch_int_restore_interrupts(int oldState)416 arch_int_restore_interrupts(int oldState)
417 {
418 arch_int_restore_interrupts_inline(oldState);
419 }
420
421
422 bool
arch_int_are_interrupts_enabled(void)423 arch_int_are_interrupts_enabled(void)
424 {
425 return arch_int_are_interrupts_enabled_inline();
426 }
427
428
429 int32
arch_int_assign_to_cpu(int32 irq,int32 cpu)430 arch_int_assign_to_cpu(int32 irq, int32 cpu)
431 {
432 switch (sVectorSources[irq]) {
433 case IRQ_SOURCE_IOAPIC:
434 if (sCurrentPIC->assign_interrupt_to_cpu != NULL)
435 sCurrentPIC->assign_interrupt_to_cpu(irq, cpu);
436 break;
437
438 case IRQ_SOURCE_MSI:
439 msi_assign_interrupt_to_cpu(irq, cpu);
440 break;
441
442 default:
443 break;
444 }
445 return cpu;
446 }
447
448
449 status_t
arch_int_init(kernel_args * args)450 arch_int_init(kernel_args* args)
451 {
452 // setup the standard programmable interrupt controller
453 pic_init();
454 return B_OK;
455 }
456
457
458 status_t
arch_int_init_post_vm(kernel_args * args)459 arch_int_init_post_vm(kernel_args* args)
460 {
461 // Always init the local apic as it can be used for timers even if we
462 // don't end up using the io apic
463 apic_init(args);
464 return B_OK;
465 }
466
467
468 status_t
arch_int_init_io(kernel_args * args)469 arch_int_init_io(kernel_args* args)
470 {
471 msi_init(args);
472 ioapic_preinit(args);
473 return B_OK;
474 }
475
476
477 status_t
arch_int_init_post_device_manager(kernel_args * args)478 arch_int_init_post_device_manager(kernel_args* args)
479 {
480 return B_OK;
481 }
482
483
484 void
arch_int_set_interrupt_controller(const interrupt_controller & controller)485 arch_int_set_interrupt_controller(const interrupt_controller& controller)
486 {
487 sCurrentPIC = &controller;
488 }
489